forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2023-12-09 95099d4622f8cb224d94e314c7a8e0df60b13f87 
 kernel/drivers/lightnvm/pblk-gc.c
....@@ -1,3 +1,4 @@
1
+// SPDX-License-Identifier: GPL-2.0
12 /*
23  * Copyright (C) 2016 CNEX Labs
34  * Initial release: Javier Gonzalez <javier@cnexlabs.com>
....@@ -16,7 +17,9 @@
1617  */
1718 
1819 #include "pblk.h"
20
+#include "pblk-trace.h"
1921 #include <linux/delay.h>
22
+
2023 
2124 static void pblk_gc_free_gc_rq(struct pblk_gc_rq *gc_rq)
2225 {
....@@ -56,22 +59,28 @@
5659 	wake_up_process(gc->gc_writer_ts);
5760 }
5861 
59
-static void pblk_put_line_back(struct pblk *pblk, struct pblk_line *line)
62
+void pblk_put_line_back(struct pblk *pblk, struct pblk_line *line)
6063 {
6164 	struct pblk_line_mgmt *l_mg = &pblk->l_mg;
6265 	struct list_head *move_list;
6366 
67
+	spin_lock(&l_mg->gc_lock);
6468 	spin_lock(&line->lock);
6569 	WARN_ON(line->state != PBLK_LINESTATE_GC);
6670 	line->state = PBLK_LINESTATE_CLOSED;
71
+	trace_pblk_line_state(pblk_disk_name(pblk), line->id,
72
+					line->state);
73
+
74
+	/* We need to reset gc_group in order to ensure that
75
+	 * pblk_line_gc_list will return proper move_list
76
+	 * since right now current line is not on any of the
77
+	 * gc lists.
78
+	 */
79
+	line->gc_group = PBLK_LINEGC_NONE;
6780 	move_list = pblk_line_gc_list(pblk, line);
6881 	spin_unlock(&line->lock);
69
-
70
-	if (move_list) {
71
-		spin_lock(&l_mg->gc_lock);
72
-		list_add_tail(&line->list, move_list);
73
-		spin_unlock(&l_mg->gc_lock);
74
-	}
82
+	list_add_tail(&line->list, move_list);
83
+	spin_unlock(&l_mg->gc_lock);
7584 }
7685 
7786 static void pblk_gc_line_ws(struct work_struct *work)
....@@ -79,8 +88,6 @@
7988 	struct pblk_line_ws *gc_rq_ws = container_of(work,
8089 						struct pblk_line_ws, ws);
8190 	struct pblk *pblk = gc_rq_ws->pblk;
82
-	struct nvm_tgt_dev *dev = pblk->dev;
83
-	struct nvm_geo *geo = &dev->geo;
8491 	struct pblk_gc *gc = &pblk->gc;
8592 	struct pblk_line *line = gc_rq_ws->line;
8693 	struct pblk_gc_rq *gc_rq = gc_rq_ws->priv;
....@@ -88,18 +95,10 @@
8895 
8996 	up(&gc->gc_sem);
9097 
91
-	gc_rq->data = vmalloc(array_size(gc_rq->nr_secs, geo->csecs));
92
-	if (!gc_rq->data) {
93
-		pblk_err(pblk, "could not GC line:%d (%d/%d)\n",
94
-					line->id, *line->vsc, gc_rq->nr_secs);
95
-		goto out;
96
-	}
97
-
9898 	/* Read from GC victim block */
9999 	ret = pblk_submit_read_gc(pblk, gc_rq);
100100 	if (ret) {
101
-		pblk_err(pblk, "failed GC read in line:%d (err:%d)\n",
102
-								line->id, ret);
101
+		line->w_err_gc->has_gc_err = 1;
103102 		goto out;
104103 	}
105104 
....@@ -133,22 +132,20 @@
133132 				       struct pblk_line *line)
134133 {
135134 	struct line_emeta *emeta_buf;
136
-	struct pblk_line_mgmt *l_mg = &pblk->l_mg;
137135 	struct pblk_line_meta *lm = &pblk->lm;
138136 	unsigned int lba_list_size = lm->emeta_len[2];
139137 	__le64 *lba_list;
140138 	int ret;
141139 
142
-	emeta_buf = pblk_malloc(lm->emeta_len[0],
143
-				l_mg->emeta_alloc_type, GFP_KERNEL);
140
+	emeta_buf = kvmalloc(lm->emeta_len[0], GFP_KERNEL);
144141 	if (!emeta_buf)
145142 		return NULL;
146143 
147
-	ret = pblk_line_read_emeta(pblk, line, emeta_buf);
144
+	ret = pblk_line_emeta_read(pblk, line, emeta_buf);
148145 	if (ret) {
149146 		pblk_err(pblk, "line %d read emeta failed (%d)\n",
150147 				line->id, ret);
151
-		pblk_mfree(emeta_buf, l_mg->emeta_alloc_type);
148
+		kvfree(emeta_buf);
152149 		return NULL;
153150 	}
154151 
....@@ -162,16 +159,16 @@
162159 	if (ret) {
163160 		pblk_err(pblk, "inconsistent emeta (line %d)\n",
164161 				line->id);
165
-		pblk_mfree(emeta_buf, l_mg->emeta_alloc_type);
162
+		kvfree(emeta_buf);
166163 		return NULL;
167164 	}
168165 
169
-	lba_list = pblk_malloc(lba_list_size,
170
-			       l_mg->emeta_alloc_type, GFP_KERNEL);
166
+	lba_list = kvmalloc(lba_list_size, GFP_KERNEL);
167
+
171168 	if (lba_list)
172169 		memcpy(lba_list, emeta_to_lbas(pblk, emeta_buf), lba_list_size);
173170 
174
-	pblk_mfree(emeta_buf, l_mg->emeta_alloc_type);
171
+	kvfree(emeta_buf);
175172 
176173 	return lba_list;
177174 }
....@@ -182,8 +179,9 @@
182179 									ws);
183180 	struct pblk *pblk = line_ws->pblk;
184181 	struct pblk_line *line = line_ws->line;
185
-	struct pblk_line_mgmt *l_mg = &pblk->l_mg;
186182 	struct pblk_line_meta *lm = &pblk->lm;
183
+	struct nvm_tgt_dev *dev = pblk->dev;
184
+	struct nvm_geo *geo = &dev->geo;
187185 	struct pblk_gc *gc = &pblk->gc;
188186 	struct pblk_line_ws *gc_rq_ws;
189187 	struct pblk_gc_rq *gc_rq;
....@@ -242,9 +240,13 @@
242240 	gc_rq->nr_secs = nr_secs;
243241 	gc_rq->line = line;
244242 
243
+	gc_rq->data = vmalloc(array_size(gc_rq->nr_secs, geo->csecs));
244
+	if (!gc_rq->data)
245
+		goto fail_free_gc_rq;
246
+
245247 	gc_rq_ws = kmalloc(sizeof(struct pblk_line_ws), GFP_KERNEL);
246248 	if (!gc_rq_ws)
247
-		goto fail_free_gc_rq;
249
+		goto fail_free_gc_data;
248250 
249251 	gc_rq_ws->pblk = pblk;
250252 	gc_rq_ws->line = line;
....@@ -267,7 +269,7 @@
267269 		goto next_rq;
268270 
269271 out:
270
-	pblk_mfree(lba_list, l_mg->emeta_alloc_type);
272
+	kvfree(lba_list);
271273 	kfree(line_ws);
272274 	kfree(invalid_bitmap);
273275 
....@@ -276,17 +278,22 @@
276278 
277279 	return;
278280 
281
+fail_free_gc_data:
282
+	vfree(gc_rq->data);
279283 fail_free_gc_rq:
280284 	kfree(gc_rq);
281285 fail_free_lba_list:
282
-	pblk_mfree(lba_list, l_mg->emeta_alloc_type);
286
+	kvfree(lba_list);
283287 fail_free_invalid_bitmap:
284288 	kfree(invalid_bitmap);
285289 fail_free_ws:
286290 	kfree(line_ws);
287291 
292
+	/* Line goes back to closed state, so we cannot release additional
293
+	 * reference for line, since we do that only when we want to do
294
+	 * gc to free line state transition.
295
+	 */
288296 	pblk_put_line_back(pblk, line);
289
-	kref_put(&line->ref, pblk_line_put);
290297 	atomic_dec(&gc->read_inflight_gc);
291298 
292299 	pblk_err(pblk, "failed to GC line %d\n", line->id);
....@@ -350,8 +357,13 @@
350357 
351358 	pblk_gc_kick(pblk);
352359 
353
-	if (pblk_gc_line(pblk, line))
360
+	if (pblk_gc_line(pblk, line)) {
354361 		pblk_err(pblk, "failed to GC line %d\n", line->id);
362
+		/* rollback */
363
+		spin_lock(&gc->r_lock);
364
+		list_add_tail(&line->list, &gc->r_list);
365
+		spin_unlock(&gc->r_lock);
366
+	}
355367 
356368 	return 0;
357369 }
....@@ -360,15 +372,21 @@
360372 						 struct list_head *group_list)
361373 {
362374 	struct pblk_line *line, *victim;
363
-	int line_vsc, victim_vsc;
375
+	unsigned int line_vsc = ~0x0L, victim_vsc = ~0x0L;
364376 
365377 	victim = list_first_entry(group_list, struct pblk_line, list);
378
+
366379 	list_for_each_entry(line, group_list, list) {
367
-		line_vsc = le32_to_cpu(*line->vsc);
368
-		victim_vsc = le32_to_cpu(*victim->vsc);
369
-		if (line_vsc < victim_vsc)
380
+		if (!atomic_read(&line->sec_to_update))
381
+			line_vsc = le32_to_cpu(*line->vsc);
382
+		if (line_vsc < victim_vsc) {
370383 			victim = line;
384
+			victim_vsc = le32_to_cpu(*victim->vsc);
385
+		}
371386 	}
387
+
388
+	if (victim_vsc == ~0x0)
389
+		return NULL;
372390 
373391 	return victim;
374392 }
....@@ -405,6 +423,8 @@
405423 		spin_lock(&line->lock);
406424 		WARN_ON(line->state != PBLK_LINESTATE_CLOSED);
407425 		line->state = PBLK_LINESTATE_GC;
426
+		trace_pblk_line_state(pblk_disk_name(pblk), line->id,
427
+					line->state);
408428 		spin_unlock(&line->lock);
409429 
410430 		list_del(&line->list);
....@@ -441,16 +461,18 @@
441461 
442462 	do {
443463 		spin_lock(&l_mg->gc_lock);
444
-		if (list_empty(group_list)) {
464
+
465
+		line = pblk_gc_get_victim_line(pblk, group_list);
466
+		if (!line) {
445467 			spin_unlock(&l_mg->gc_lock);
446468 			break;
447469 		}
448470 
449
-		line = pblk_gc_get_victim_line(pblk, group_list);
450
-
451471 		spin_lock(&line->lock);
452472 		WARN_ON(line->state != PBLK_LINESTATE_CLOSED);
453473 		line->state = PBLK_LINESTATE_GC;
474
+		trace_pblk_line_state(pblk_disk_name(pblk), line->id,
475
+					line->state);
454476 		spin_unlock(&line->lock);
455477 
456478 		list_del(&line->list);