forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2023-12-11 d2ccde1c8e90d38cee87a1b0309ad2827f3fd30d 
 kernel/drivers/md/bcache/request.c
....@@ -62,18 +62,6 @@
6262 	struct bkey *replace_key = op->replace ? &op->replace_key : NULL;
6363 	int ret;
6464 
65
-	/*
66
-	 * If we're looping, might already be waiting on
67
-	 * another journal write - can't wait on more than one journal write at
68
-	 * a time
69
-	 *
70
-	 * XXX: this looks wrong
71
-	 */
72
-#if 0
73
-	while (atomic_read(&s->cl.remaining) & CLOSURE_WAITING)
74
-		closure_sync(&s->cl);
75
-#endif
76
-
7765 	if (!op->replace)
7866 		journal_ref = bch_journal(op->c, &op->insert_keys,
7967 					  op->flush_journal ? cl : NULL);
....@@ -111,7 +99,7 @@
11199 	 * bch_data_insert_keys() will insert the keys created so far
112100 	 * and finish the rest when the keylist is empty.
113101 	 */
114
-	if (newsize * sizeof(uint64_t) > block_bytes(c) - sizeof(struct jset))
102
+	if (newsize * sizeof(uint64_t) > block_bytes(c->cache) - sizeof(struct jset))
115103 		return -ENOMEM;
116104 
117105 	return __bch_keylist_realloc(l, u64s);
....@@ -122,7 +110,7 @@
122110 	struct data_insert_op *op = container_of(cl, struct data_insert_op, cl);
123111 	struct bio *bio = op->bio;
124112 
125
-	pr_debug("invalidating %i sectors from %llu",
113
+	pr_debug("invalidating %i sectors from %llu\n",
126114 		 bio_sectors(bio), (uint64_t) bio->bi_iter.bi_sector);
127115 
128116 	while (bio_sectors(bio)) {
....@@ -311,11 +299,11 @@
311299  * data is written it calls bch_journal, and after the keys have been added to
312300  * the next journal write they're inserted into the btree.
313301  *
314
- * It inserts the data in s->cache_bio; bi_sector is used for the key offset,
302
+ * It inserts the data in op->bio; bi_sector is used for the key offset,
315303  * and op->inode is used for the key inode.
316304  *
317
- * If s->bypass is true, instead of inserting the data it invalidates the
318
- * region of the cache represented by s->cache_bio and op->inode.
305
+ * If op->bypass is true, instead of inserting the data it invalidates the
306
+ * region of the cache represented by op->bio and op->inode.
319307  */
320308 void bch_data_insert(struct closure *cl)
321309 {
....@@ -329,12 +317,13 @@
329317 	bch_data_insert_start(cl);
330318 }
331319 
332
-/* Congested? */
333
-
334
-unsigned int bch_get_congested(struct cache_set *c)
320
+/*
321
+ * Congested?  Return 0 (not congested) or the limit (in sectors)
322
+ * beyond which we should bypass the cache due to congestion.
323
+ */
324
+unsigned int bch_get_congested(const struct cache_set *c)
335325 {
336326 	int i;
337
-	long rand;
338327 
339328 	if (!c->congested_read_threshold_us &&
340329 	    !c->congested_write_threshold_us)
....@@ -353,8 +342,7 @@
353342 	if (i > 0)
354343 		i = fract_exp_two(i, 6);
355344 
356
-	rand = get_random_int();
357
-	i -= bitmap_weight(&rand, BITS_PER_LONG);
345
+	i -= hweight32(get_random_u32());
358346 
359347 	return i > 0 ? i : 1;
360348 }
....@@ -376,7 +364,7 @@
376364 {
377365 	struct cache_set *c = dc->disk.c;
378366 	unsigned int mode = cache_mode(dc);
379
-	unsigned int sectors, congested = bch_get_congested(c);
367
+	unsigned int sectors, congested;
380368 	struct task_struct *task = current;
381369 	struct io *i;
382370 
....@@ -406,9 +394,9 @@
406394 			goto skip;
407395 	}
408396 
409
-	if (bio->bi_iter.bi_sector & (c->sb.block_size - 1) ||
410
-	    bio_sectors(bio) & (c->sb.block_size - 1)) {
411
-		pr_debug("skipping unaligned io");
397
+	if (bio->bi_iter.bi_sector & (c->cache->sb.block_size - 1) ||
398
+	    bio_sectors(bio) & (c->cache->sb.block_size - 1)) {
399
+		pr_debug("skipping unaligned io\n");
412400 		goto skip;
413401 	}
414402 
....@@ -419,6 +407,7 @@
419407 			goto rescale;
420408 	}
421409 
410
+	congested = bch_get_congested(c);
422411 	if (!congested && !dc->sequential_cutoff)
423412 		goto rescale;
424413 
....@@ -486,6 +475,7 @@
486475 	unsigned int		read_dirty_data:1;
487476 	unsigned int		cache_missed:1;
488477 
478
+	struct hd_struct	*part;
489479 	unsigned long		start_time;
490480 
491481 	struct btree_op		op;
....@@ -661,7 +651,7 @@
661651 		 */
662652 		if (unlikely(s->iop.writeback &&
663653 			     bio->bi_opf & REQ_PREFLUSH)) {
664
-			pr_err("Can't flush %s: returned bi_status %i",
654
+			pr_err("Can't flush %s: returned bi_status %i\n",
665655 				dc->backing_dev_name, bio->bi_status);
666656 		} else {
667657 			/* set to orig_bio->bi_status in bio_complete() */
....@@ -679,8 +669,8 @@
679669 static void bio_complete(struct search *s)
680670 {
681671 	if (s->orig_bio) {
682
-		generic_end_io_acct(s->d->disk->queue, bio_op(s->orig_bio),
683
-				    &s->d->disk->part0, s->start_time);
672
+		/* Count on bcache device */
673
+		part_end_io_acct(s->part, s->orig_bio, s->start_time);
684674 
685675 		trace_bcache_request_end(s->d, s->orig_bio);
686676 		s->orig_bio->bi_status = s->iop.status;
....@@ -713,14 +703,14 @@
713703 {
714704 	struct search *s = container_of(cl, struct search, cl);
715705 
716
-	atomic_dec(&s->d->c->search_inflight);
706
+	atomic_dec(&s->iop.c->search_inflight);
717707 
718708 	if (s->iop.bio)
719709 		bio_put(s->iop.bio);
720710 
721711 	bio_complete(s);
722712 	closure_debug_destroy(cl);
723
-	mempool_free(s, &s->d->c->search);
713
+	mempool_free(s, &s->iop.c->search);
724714 }
725715 
726716 static inline struct search *search_alloc(struct bio *bio,
....@@ -741,8 +731,8 @@
741731 	s->recoverable		= 1;
742732 	s->write		= op_is_write(bio_op(bio));
743733 	s->read_dirty_data	= 0;
744
-	s->start_time		= jiffies;
745
-
734
+	/* Count on the bcache device */
735
+	s->start_time		= part_start_io_acct(d->disk, &s->part, bio);
746736 	s->iop.c		= d->c;
747737 	s->iop.bio		= NULL;
748738 	s->iop.inode		= d->id;
....@@ -763,13 +753,13 @@
763753 	struct search *s = container_of(cl, struct search, cl);
764754 	struct cached_dev *dc = container_of(s->d, struct cached_dev, disk);
765755 
766
-	search_free(cl);
767756 	cached_dev_put(dc);
757
+	search_free(cl);
768758 }
769759 
770760 /* Process reads */
771761 
772
-static void cached_dev_cache_miss_done(struct closure *cl)
762
+static void cached_dev_read_error_done(struct closure *cl)
773763 {
774764 	struct search *s = container_of(cl, struct search, cl);
775765 
....@@ -807,7 +797,22 @@
807797 		closure_bio_submit(s->iop.c, bio, cl);
808798 	}
809799 
810
-	continue_at(cl, cached_dev_cache_miss_done, NULL);
800
+	continue_at(cl, cached_dev_read_error_done, NULL);
801
+}
802
+
803
+static void cached_dev_cache_miss_done(struct closure *cl)
804
+{
805
+	struct search *s = container_of(cl, struct search, cl);
806
+	struct bcache_device *d = s->d;
807
+
808
+	if (s->iop.replace_collision)
809
+		bch_mark_cache_miss_collision(s->iop.c, s->d);
810
+
811
+	if (s->iop.bio)
812
+		bio_free_pages(s->iop.bio);
813
+
814
+	cached_dev_bio_complete(cl);
815
+	closure_put(&d->cl);
811816 }
812817 
813818 static void cached_dev_read_done(struct closure *cl)
....@@ -840,6 +845,7 @@
840845 	if (verify(dc) && s->recoverable && !s->read_dirty_data)
841846 		bch_data_verify(dc, s->orig_bio);
842847 
848
+	closure_get(&dc->disk.cl);
843849 	bio_complete(s);
844850 
845851 	if (s->iop.bio &&
....@@ -1067,6 +1073,7 @@
10671073 	unsigned long		start_time;
10681074 	bio_end_io_t		*bi_end_io;
10691075 	void			*bi_private;
1076
+	struct hd_struct	*part;
10701077 };
10711078 
10721079 static void detached_dev_end_io(struct bio *bio)
....@@ -1077,8 +1084,8 @@
10771084 	bio->bi_end_io = ddip->bi_end_io;
10781085 	bio->bi_private = ddip->bi_private;
10791086 
1080
-	generic_end_io_acct(ddip->d->disk->queue, bio_op(bio),
1081
-			    &ddip->d->disk->part0, ddip->start_time);
1087
+	/* Count on the bcache device */
1088
+	part_end_io_acct(ddip->part, bio, ddip->start_time);
10821089 
10831090 	if (bio->bi_status) {
10841091 		struct cached_dev *dc = container_of(ddip->d,
....@@ -1102,8 +1109,15 @@
11021109 	 * which would call closure_get(&dc->disk.cl)
11031110 	 */
11041111 	ddip = kzalloc(sizeof(struct detached_dev_io_private), GFP_NOIO);
1112
+	if (!ddip) {
1113
+		bio->bi_status = BLK_STS_RESOURCE;
1114
+		bio->bi_end_io(bio);
1115
+		return;
1116
+	}
1117
+
11051118 	ddip->d = d;
1106
-	ddip->start_time = jiffies;
1119
+	/* Count on the bcache device */
1120
+	ddip->start_time = part_start_io_acct(d->disk, &ddip->part, bio);
11071121 	ddip->bi_end_io = bio->bi_end_io;
11081122 	ddip->bi_private = bio->bi_private;
11091123 	bio->bi_end_io = detached_dev_end_io;
....@@ -1113,7 +1127,7 @@
11131127 	    !blk_queue_discard(bdev_get_queue(dc->bdev)))
11141128 		bio->bi_end_io(bio);
11151129 	else
1116
-		generic_make_request(bio);
1130
+		submit_bio_noacct(bio);
11171131 }
11181132 
11191133 static void quit_max_writeback_rate(struct cache_set *c,
....@@ -1156,8 +1170,7 @@
11561170 
11571171 /* Cached devices - read & write stuff */
11581172 
1159
-static blk_qc_t cached_dev_make_request(struct request_queue *q,
1160
-					struct bio *bio)
1173
+blk_qc_t cached_dev_submit_bio(struct bio *bio)
11611174 {
11621175 	struct search *s;
11631176 	struct bcache_device *d = bio->bi_disk->private_data;
....@@ -1186,11 +1199,6 @@
11861199 		}
11871200 	}
11881201 
1189
-	generic_start_io_acct(q,
1190
-			      bio_op(bio),
1191
-			      bio_sectors(bio),
1192
-			      &d->disk->part0);
1193
-
11941202 	bio_set_dev(bio, dc->bdev);
11951203 	bio->bi_iter.bi_sector += dc->sb.data_offset;
11961204 
....@@ -1201,7 +1209,7 @@
12011209 		if (!bio->bi_iter.bi_size) {
12021210 			/*
12031211 			 * can't call bch_journal_meta from under
1204
-			 * generic_make_request
1212
+			 * submit_bio_noacct
12051213 			 */
12061214 			continue_at_nobarrier(&s->cl,
12071215 					      cached_dev_nodata,
....@@ -1232,37 +1240,8 @@
12321240 	return __blkdev_driver_ioctl(dc->bdev, mode, cmd, arg);
12331241 }
12341242 
1235
-static int cached_dev_congested(void *data, int bits)
1236
-{
1237
-	struct bcache_device *d = data;
1238
-	struct cached_dev *dc = container_of(d, struct cached_dev, disk);
1239
-	struct request_queue *q = bdev_get_queue(dc->bdev);
1240
-	int ret = 0;
1241
-
1242
-	if (bdi_congested(q->backing_dev_info, bits))
1243
-		return 1;
1244
-
1245
-	if (cached_dev_get(dc)) {
1246
-		unsigned int i;
1247
-		struct cache *ca;
1248
-
1249
-		for_each_cache(ca, d->c, i) {
1250
-			q = bdev_get_queue(ca->bdev);
1251
-			ret |= bdi_congested(q->backing_dev_info, bits);
1252
-		}
1253
-
1254
-		cached_dev_put(dc);
1255
-	}
1256
-
1257
-	return ret;
1258
-}
1259
-
12601243 void bch_cached_dev_request_init(struct cached_dev *dc)
12611244 {
1262
-	struct gendisk *g = dc->disk.disk;
1263
-
1264
-	g->queue->make_request_fn		= cached_dev_make_request;
1265
-	g->queue->backing_dev_info->congested_fn = cached_dev_congested;
12661245 	dc->disk.cache_miss			= cached_dev_cache_miss;
12671246 	dc->disk.ioctl				= cached_dev_ioctl;
12681247 }
....@@ -1296,8 +1275,7 @@
12961275 	continue_at(cl, search_free, NULL);
12971276 }
12981277 
1299
-static blk_qc_t flash_dev_make_request(struct request_queue *q,
1300
-					     struct bio *bio)
1278
+blk_qc_t flash_dev_submit_bio(struct bio *bio)
13011279 {
13021280 	struct search *s;
13031281 	struct closure *cl;
....@@ -1309,8 +1287,6 @@
13091287 		return BLK_QC_T_NONE;
13101288 	}
13111289 
1312
-	generic_start_io_acct(q, bio_op(bio), bio_sectors(bio), &d->disk->part0);
1313
-
13141290 	s = search_alloc(bio, d);
13151291 	cl = &s->cl;
13161292 	bio = &s->bio.bio;
....@@ -1319,8 +1295,7 @@
13191295 
13201296 	if (!bio->bi_iter.bi_size) {
13211297 		/*
1322
-		 * can't call bch_journal_meta from under
1323
-		 * generic_make_request
1298
+		 * can't call bch_journal_meta from under submit_bio_noacct
13241299 		 */
13251300 		continue_at_nobarrier(&s->cl,
13261301 				      flash_dev_nodata,
....@@ -1350,28 +1325,8 @@
13501325 	return -ENOTTY;
13511326 }
13521327 
1353
-static int flash_dev_congested(void *data, int bits)
1354
-{
1355
-	struct bcache_device *d = data;
1356
-	struct request_queue *q;
1357
-	struct cache *ca;
1358
-	unsigned int i;
1359
-	int ret = 0;
1360
-
1361
-	for_each_cache(ca, d->c, i) {
1362
-		q = bdev_get_queue(ca->bdev);
1363
-		ret |= bdi_congested(q->backing_dev_info, bits);
1364
-	}
1365
-
1366
-	return ret;
1367
-}
1368
-
13691328 void bch_flash_dev_request_init(struct bcache_device *d)
13701329 {
1371
-	struct gendisk *g = d->disk;
1372
-
1373
-	g->queue->make_request_fn		= flash_dev_make_request;
1374
-	g->queue->backing_dev_info->congested_fn = flash_dev_congested;
13751330 	d->cache_miss				= flash_dev_cache_miss;
13761331 	d->ioctl				= flash_dev_ioctl;
13771332 }