forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2023-12-11 d2ccde1c8e90d38cee87a1b0309ad2827f3fd30d 
 kernel/drivers/lightnvm/pblk-write.c
....@@ -1,3 +1,4 @@
1
+// SPDX-License-Identifier: GPL-2.0
12 /*
23  * Copyright (C) 2016 CNEX Labs
34  * Initial release: Javier Gonzalez <javier@cnexlabs.com>
....@@ -16,6 +17,7 @@
1617  */
1718 
1819 #include "pblk.h"
20
+#include "pblk-trace.h"
1921 
2022 static unsigned long pblk_end_w_bio(struct pblk *pblk, struct nvm_rq *rqd,
2123 				    struct pblk_c_ctx *c_ctx)
....@@ -81,8 +83,7 @@
8183 #ifdef CONFIG_NVM_PBLK_DEBUG
8284 	atomic_long_sub(c_ctx->nr_valid, &pblk->inflight_writes);
8385 #endif
84
-
85
-	pblk_up_rq(pblk, rqd->ppa_list, rqd->nr_ppas, c_ctx->lun_bitmap);
86
+	pblk_up_rq(pblk, c_ctx->lun_bitmap);
8687 
8788 	pos = pblk_rb_sync_init(&pblk->rwb, &flags);
8889 	if (pos == c_ctx->sentry) {
....@@ -104,16 +105,20 @@
104105 }
105106 
106107 /* Map remaining sectors in chunk, starting from ppa */
107
-static void pblk_map_remaining(struct pblk *pblk, struct ppa_addr *ppa)
108
+static void pblk_map_remaining(struct pblk *pblk, struct ppa_addr *ppa,
109
+		int rqd_ppas)
108110 {
109
-	struct nvm_tgt_dev *dev = pblk->dev;
110
-	struct nvm_geo *geo = &dev->geo;
111111 	struct pblk_line *line;
112112 	struct ppa_addr map_ppa = *ppa;
113
+	__le64 addr_empty = cpu_to_le64(ADDR_EMPTY);
114
+	__le64 *lba_list;
113115 	u64 paddr;
114116 	int done = 0;
117
+	int n = 0;
115118 
116
-	line = &pblk->lines[pblk_ppa_to_line(*ppa)];
119
+	line = pblk_ppa_to_line(pblk, *ppa);
120
+	lba_list = emeta_to_lbas(pblk, line->emeta->buf);
121
+
117122 	spin_lock(&line->lock);
118123 
119124 	while (!done)  {
....@@ -122,18 +127,17 @@
122127 		if (!test_and_set_bit(paddr, line->map_bitmap))
123128 			line->left_msecs--;
124129 
130
+		if (n < rqd_ppas && lba_list[paddr] != addr_empty)
131
+			line->nr_valid_lbas--;
132
+
133
+		lba_list[paddr] = addr_empty;
134
+
125135 		if (!test_and_set_bit(paddr, line->invalid_bitmap))
126136 			le32_add_cpu(line->vsc, -1);
127137 
128
-		if (geo->version == NVM_OCSSD_SPEC_12) {
129
-			map_ppa.ppa++;
130
-			if (map_ppa.g.pg == geo->num_pg)
131
-				done = 1;
132
-		} else {
133
-			map_ppa.m.sec++;
134
-			if (map_ppa.m.sec == geo->clba)
135
-				done = 1;
136
-		}
138
+		done = nvm_next_ppa_in_chk(pblk->dev, &map_ppa);
139
+
140
+		n++;
137141 	}
138142 
139143 	line->w_err_gc->has_write_err = 1;
....@@ -149,12 +153,11 @@
149153 	struct pblk_w_ctx *w_ctx;
150154 	struct ppa_addr ppa_l2p;
151155 	int flags;
152
-	unsigned int pos, i;
156
+	unsigned int i;
153157 
154158 	spin_lock(&pblk->trans_lock);
155
-	pos = sentry;
156159 	for (i = 0; i < nr_entries; i++) {
157
-		entry = &rb->entries[pos];
160
+		entry = &rb->entries[pblk_rb_ptr_wrap(rb, sentry, i)];
158161 		w_ctx = &entry->w_ctx;
159162 
160163 		/* Check if the lba has been overwritten */
....@@ -170,13 +173,12 @@
170173 		/* Release flags on write context. Protect from writes */
171174 		smp_store_release(&w_ctx->flags, flags);
172175 
173
-		/* Decrese the reference count to the line as we will
176
+		/* Decrease the reference count to the line as we will
174177 		 * re-map these entries
175178 		 */
176
-		line = &pblk->lines[pblk_ppa_to_line(w_ctx->ppa)];
179
+		line = pblk_ppa_to_line(pblk, w_ctx->ppa);
180
+		atomic_dec(&line->sec_to_update);
177181 		kref_put(&line->ref, pblk_line_put);
178
-
179
-		pos = (pos + 1) & (rb->nr_entries - 1);
180182 	}
181183 	spin_unlock(&pblk->trans_lock);
182184 }
....@@ -210,19 +212,14 @@
210212 	struct pblk *pblk = recovery->pblk;
211213 	struct nvm_rq *rqd = recovery->rqd;
212214 	struct pblk_c_ctx *c_ctx = nvm_rq_to_pdu(rqd);
213
-	struct ppa_addr *ppa_list;
215
+	struct ppa_addr *ppa_list = nvm_rq_to_ppa_list(rqd);
214216 
215217 	pblk_log_write_err(pblk, rqd);
216218 
217
-	if (rqd->nr_ppas == 1)
218
-		ppa_list = &rqd->ppa_addr;
219
-	else
220
-		ppa_list = rqd->ppa_list;
221
-
222
-	pblk_map_remaining(pblk, ppa_list);
219
+	pblk_map_remaining(pblk, ppa_list, rqd->nr_ppas);
223220 	pblk_queue_resubmit(pblk, c_ctx);
224221 
225
-	pblk_up_rq(pblk, rqd->ppa_list, rqd->nr_ppas, c_ctx->lun_bitmap);
222
+	pblk_up_rq(pblk, c_ctx->lun_bitmap);
226223 	if (c_ctx->nr_padded)
227224 		pblk_bio_free_pages(pblk, rqd->bio, c_ctx->nr_valid,
228225 							c_ctx->nr_padded);
....@@ -231,6 +228,7 @@
231228 	mempool_free(recovery, &pblk->rec_pool);
232229 
233230 	atomic_dec(&pblk->inflight_io);
231
+	pblk_write_kick(pblk);
234232 }
235233 
236234 
....@@ -259,11 +257,13 @@
259257 	if (rqd->error) {
260258 		pblk_end_w_fail(pblk, rqd);
261259 		return;
262
-	}
260
+	} else {
261
+		if (trace_pblk_chunk_state_enabled())
262
+			pblk_check_chunk_state_update(pblk, rqd);
263263 #ifdef CONFIG_NVM_PBLK_DEBUG
264
-	else
265264 		WARN_ONCE(rqd->bio->bi_status, "pblk: corrupted write error\n");
266265 #endif
266
+	}
267267 
268268 	pblk_complete_write(pblk, rqd, c_ctx);
269269 	atomic_dec(&pblk->inflight_io);
....@@ -275,14 +275,18 @@
275275 	struct pblk_g_ctx *m_ctx = nvm_rq_to_pdu(rqd);
276276 	struct pblk_line *line = m_ctx->private;
277277 	struct pblk_emeta *emeta = line->emeta;
278
+	struct ppa_addr *ppa_list = nvm_rq_to_ppa_list(rqd);
278279 	int sync;
279280 
280
-	pblk_up_page(pblk, rqd->ppa_list, rqd->nr_ppas);
281
+	pblk_up_chunk(pblk, ppa_list[0]);
281282 
282283 	if (rqd->error) {
283284 		pblk_log_write_err(pblk, rqd);
284285 		pblk_err(pblk, "metadata I/O failed. Line %d\n", line->id);
285286 		line->w_err_gc->has_write_err = 1;
287
+	} else {
288
+		if (trace_pblk_chunk_state_enabled())
289
+			pblk_check_chunk_state_update(pblk, rqd);
286290 	}
287291 
288292 	sync = atomic_add_return(rqd->nr_ppas, &emeta->sync);
....@@ -296,27 +300,16 @@
296300 }
297301 
298302 static int pblk_alloc_w_rq(struct pblk *pblk, struct nvm_rq *rqd,
299
-			   unsigned int nr_secs,
300
-			   nvm_end_io_fn(*end_io))
303
+			   unsigned int nr_secs, nvm_end_io_fn(*end_io))
301304 {
302
-	struct nvm_tgt_dev *dev = pblk->dev;
303
-
304305 	/* Setup write request */
305306 	rqd->opcode = NVM_OP_PWRITE;
306307 	rqd->nr_ppas = nr_secs;
307
-	rqd->flags = pblk_set_progr_mode(pblk, PBLK_WRITE);
308
+	rqd->is_seq = 1;
308309 	rqd->private = pblk;
309310 	rqd->end_io = end_io;
310311 
311
-	rqd->meta_list = nvm_dev_dma_alloc(dev->parent, GFP_KERNEL,
312
-							&rqd->dma_meta_list);
313
-	if (!rqd->meta_list)
314
-		return -ENOMEM;
315
-
316
-	rqd->ppa_list = rqd->meta_list + pblk_dma_meta_size;
317
-	rqd->dma_ppa_list = rqd->dma_meta_list + pblk_dma_meta_size;
318
-
319
-	return 0;
312
+	return pblk_alloc_rqd_meta(pblk, rqd);
320313 }
321314 
322315 static int pblk_setup_w_rq(struct pblk *pblk, struct nvm_rq *rqd,
....@@ -343,12 +336,13 @@
343336 	}
344337 
345338 	if (likely(!e_line || !atomic_read(&e_line->left_eblks)))
346
-		pblk_map_rq(pblk, rqd, c_ctx->sentry, lun_bitmap, valid, 0);
339
+		ret = pblk_map_rq(pblk, rqd, c_ctx->sentry, lun_bitmap,
340
+							valid, 0);
347341 	else
348
-		pblk_map_erase_rq(pblk, rqd, c_ctx->sentry, lun_bitmap,
342
+		ret = pblk_map_erase_rq(pblk, rqd, c_ctx->sentry, lun_bitmap,
349343 							valid, erase_ppa);
350344 
351
-	return 0;
345
+	return ret;
352346 }
353347 
354348 static int pblk_calc_secs_to_sync(struct pblk *pblk, unsigned int secs_avail,
....@@ -356,7 +350,7 @@
356350 {
357351 	int secs_to_sync;
358352 
359
-	secs_to_sync = pblk_calc_secs(pblk, secs_avail, secs_to_flush);
353
+	secs_to_sync = pblk_calc_secs(pblk, secs_avail, secs_to_flush, true);
360354 
361355 #ifdef CONFIG_NVM_PBLK_DEBUG
362356 	if ((!secs_to_sync && secs_to_flush)
....@@ -377,8 +371,8 @@
377371 	struct pblk_line_mgmt *l_mg = &pblk->l_mg;
378372 	struct pblk_line_meta *lm = &pblk->lm;
379373 	struct pblk_emeta *emeta = meta_line->emeta;
374
+	struct ppa_addr *ppa_list;
380375 	struct pblk_g_ctx *m_ctx;
381
-	struct bio *bio;
382376 	struct nvm_rq *rqd;
383377 	void *data;
384378 	u64 paddr;
....@@ -396,27 +390,17 @@
396390 	rq_len = rq_ppas * geo->csecs;
397391 	data = ((void *)emeta->buf) + emeta->mem;
398392 
399
-	bio = pblk_bio_map_addr(pblk, data, rq_ppas, rq_len,
400
-					l_mg->emeta_alloc_type, GFP_KERNEL);
401
-	if (IS_ERR(bio)) {
402
-		pblk_err(pblk, "failed to map emeta io");
403
-		ret = PTR_ERR(bio);
404
-		goto fail_free_rqd;
405
-	}
406
-	bio->bi_iter.bi_sector = 0; /* internal bio */
407
-	bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
408
-	rqd->bio = bio;
409
-
410393 	ret = pblk_alloc_w_rq(pblk, rqd, rq_ppas, pblk_end_io_write_meta);
411394 	if (ret)
412
-		goto fail_free_bio;
395
+		goto fail_free_rqd;
413396 
397
+	ppa_list = nvm_rq_to_ppa_list(rqd);
414398 	for (i = 0; i < rqd->nr_ppas; ) {
415399 		spin_lock(&meta_line->lock);
416400 		paddr = __pblk_alloc_page(pblk, meta_line, rq_ppas);
417401 		spin_unlock(&meta_line->lock);
418402 		for (j = 0; j < rq_ppas; j++, i++, paddr++)
419
-			rqd->ppa_list[i] = addr_to_gen_ppa(pblk, paddr, id);
403
+			ppa_list[i] = addr_to_gen_ppa(pblk, paddr, id);
420404 	}
421405 
422406 	spin_lock(&l_mg->close_lock);
....@@ -425,9 +409,9 @@
425409 		list_del(&meta_line->list);
426410 	spin_unlock(&l_mg->close_lock);
427411 
428
-	pblk_down_page(pblk, rqd->ppa_list, rqd->nr_ppas);
412
+	pblk_down_chunk(pblk, ppa_list[0]);
429413 
430
-	ret = pblk_submit_io(pblk, rqd);
414
+	ret = pblk_submit_io(pblk, rqd, data);
431415 	if (ret) {
432416 		pblk_err(pblk, "emeta I/O submission failed: %d\n", ret);
433417 		goto fail_rollback;
....@@ -436,13 +420,11 @@
436420 	return NVM_IO_OK;
437421 
438422 fail_rollback:
439
-	pblk_up_page(pblk, rqd->ppa_list, rqd->nr_ppas);
423
+	pblk_up_chunk(pblk, ppa_list[0]);
440424 	spin_lock(&l_mg->close_lock);
441425 	pblk_dealloc_page(pblk, meta_line, rq_ppas);
442426 	list_add(&meta_line->list, &meta_line->list);
443427 	spin_unlock(&l_mg->close_lock);
444
-fail_free_bio:
445
-	bio_put(bio);
446428 fail_free_rqd:
447429 	pblk_free_rqd(pblk, rqd, PBLK_WRITE_INT);
448430 	return ret;
....@@ -527,7 +509,7 @@
527509 	meta_line = pblk_should_submit_meta_io(pblk, rqd);
528510 
529511 	/* Submit data write for current data line */
530
-	err = pblk_submit_io(pblk, rqd);
512
+	err = pblk_submit_io(pblk, rqd, NULL);
531513 	if (err) {
532514 		pblk_err(pblk, "data I/O submission failed: %d\n", err);
533515 		return NVM_IO_ERR;
....@@ -570,14 +552,16 @@
570552 							c_ctx->nr_padded);
571553 }
572554 
573
-static int pblk_submit_write(struct pblk *pblk)
555
+static int pblk_submit_write(struct pblk *pblk, int *secs_left)
574556 {
575557 	struct bio *bio;
576558 	struct nvm_rq *rqd;
577559 	unsigned int secs_avail, secs_to_sync, secs_to_com;
578
-	unsigned int secs_to_flush;
560
+	unsigned int secs_to_flush, packed_meta_pgs;
579561 	unsigned long pos;
580562 	unsigned int resubmit;
563
+
564
+	*secs_left = 0;
581565 
582566 	spin_lock(&pblk->resubmit_lock);
583567 	resubmit = !list_empty(&pblk->resubmit_list);
....@@ -608,17 +592,17 @@
608592 		 */
609593 		secs_avail = pblk_rb_read_count(&pblk->rwb);
610594 		if (!secs_avail)
611
-			return 1;
595
+			return 0;
612596 
613597 		secs_to_flush = pblk_rb_flush_point_count(&pblk->rwb);
614
-		if (!secs_to_flush && secs_avail < pblk->min_write_pgs)
615
-			return 1;
598
+		if (!secs_to_flush && secs_avail < pblk->min_write_pgs_data)
599
+			return 0;
616600 
617601 		secs_to_sync = pblk_calc_secs_to_sync(pblk, secs_avail,
618602 					secs_to_flush);
619603 		if (secs_to_sync > pblk->max_write_pgs) {
620604 			pblk_err(pblk, "bad buffer sync calculation\n");
621
-			return 1;
605
+			return 0;
622606 		}
623607 
624608 		secs_to_com = (secs_to_sync > secs_avail) ?
....@@ -626,7 +610,8 @@
626610 		pos = pblk_rb_read_commit(&pblk->rwb, secs_to_com);
627611 	}
628612 
629
-	bio = bio_alloc(GFP_KERNEL, secs_to_sync);
613
+	packed_meta_pgs = (pblk->min_write_pgs - pblk->min_write_pgs_data);
614
+	bio = bio_alloc(GFP_KERNEL, secs_to_sync + packed_meta_pgs);
630615 
631616 	bio->bi_iter.bi_sector = 0; /* internal bio */
632617 	bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
....@@ -647,6 +632,7 @@
647632 	atomic_long_add(secs_to_sync, &pblk->sub_writes);
648633 #endif
649634 
635
+	*secs_left = 1;
650636 	return 0;
651637 
652638 fail_free_bio:
....@@ -655,16 +641,22 @@
655641 	bio_put(bio);
656642 	pblk_free_rqd(pblk, rqd, PBLK_WRITE);
657643 
658
-	return 1;
644
+	return -EINTR;
659645 }
660646 
661647 int pblk_write_ts(void *data)
662648 {
663649 	struct pblk *pblk = data;
650
+	int secs_left;
651
+	int write_failure = 0;
664652 
665653 	while (!kthread_should_stop()) {
666
-		if (!pblk_submit_write(pblk))
667
-			continue;
654
+		if (!write_failure) {
655
+			write_failure = pblk_submit_write(pblk, &secs_left);
656
+
657
+			if (secs_left)
658
+				continue;
659
+		}
668660 		set_current_state(TASK_INTERRUPTIBLE);
669661 		io_schedule();
670662 	}