forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2024-12-19 9370bb92b2d16684ee45cf24e879c93c509162da 
 kernel/drivers/md/dm-cache-target.c
....@@ -74,22 +74,19 @@
7474 static bool iot_idle_for(struct io_tracker *iot, unsigned long jifs)
7575 {
7676 	bool r;
77
-	unsigned long flags;
7877 
79
-	spin_lock_irqsave(&iot->lock, flags);
78
+	spin_lock_irq(&iot->lock);
8079 	r = __iot_idle_for(iot, jifs);
81
-	spin_unlock_irqrestore(&iot->lock, flags);
80
+	spin_unlock_irq(&iot->lock);
8281 
8382 	return r;
8483 }
8584 
8685 static void iot_io_begin(struct io_tracker *iot, sector_t len)
8786 {
88
-	unsigned long flags;
89
-
90
-	spin_lock_irqsave(&iot->lock, flags);
87
+	spin_lock_irq(&iot->lock);
9188 	iot->in_flight += len;
92
-	spin_unlock_irqrestore(&iot->lock, flags);
89
+	spin_unlock_irq(&iot->lock);
9390 }
9491 
9592 static void __iot_io_end(struct io_tracker *iot, sector_t len)
....@@ -172,7 +169,6 @@
172169 {
173170 	struct batcher *b = container_of(_ws, struct batcher, commit_work);
174171 	blk_status_t r;
175
-	unsigned long flags;
176172 	struct list_head work_items;
177173 	struct work_struct *ws, *tmp;
178174 	struct continuation *k;
....@@ -186,12 +182,12 @@
186182 	 * We have to grab these before the commit_op to avoid a race
187183 	 * condition.
188184 	 */
189
-	spin_lock_irqsave(&b->lock, flags);
185
+	spin_lock_irq(&b->lock);
190186 	list_splice_init(&b->work_items, &work_items);
191187 	bio_list_merge(&bios, &b->bios);
192188 	bio_list_init(&b->bios);
193189 	b->commit_scheduled = false;
194
-	spin_unlock_irqrestore(&b->lock, flags);
190
+	spin_unlock_irq(&b->lock);
195191 
196192 	r = b->commit_op(b->commit_context);
197193 
....@@ -238,13 +234,12 @@
238234 
239235 static void continue_after_commit(struct batcher *b, struct continuation *k)
240236 {
241
-	unsigned long flags;
242237 	bool commit_scheduled;
243238 
244
-	spin_lock_irqsave(&b->lock, flags);
239
+	spin_lock_irq(&b->lock);
245240 	commit_scheduled = b->commit_scheduled;
246241 	list_add_tail(&k->ws.entry, &b->work_items);
247
-	spin_unlock_irqrestore(&b->lock, flags);
242
+	spin_unlock_irq(&b->lock);
248243 
249244 	if (commit_scheduled)
250245 		async_commit(b);
....@@ -255,13 +250,12 @@
255250  */
256251 static void issue_after_commit(struct batcher *b, struct bio *bio)
257252 {
258
-       unsigned long flags;
259253        bool commit_scheduled;
260254 
261
-       spin_lock_irqsave(&b->lock, flags);
255
+       spin_lock_irq(&b->lock);
262256        commit_scheduled = b->commit_scheduled;
263257        bio_list_add(&b->bios, bio);
264
-       spin_unlock_irqrestore(&b->lock, flags);
258
+       spin_unlock_irq(&b->lock);
265259 
266260        if (commit_scheduled)
267261 	       async_commit(b);
....@@ -273,12 +267,11 @@
273267 static void schedule_commit(struct batcher *b)
274268 {
275269 	bool immediate;
276
-	unsigned long flags;
277270 
278
-	spin_lock_irqsave(&b->lock, flags);
271
+	spin_lock_irq(&b->lock);
279272 	immediate = !list_empty(&b->work_items) || !bio_list_empty(&b->bios);
280273 	b->commit_scheduled = true;
281
-	spin_unlock_irqrestore(&b->lock, flags);
274
+	spin_unlock_irq(&b->lock);
282275 
283276 	if (immediate)
284277 		async_commit(b);
....@@ -353,6 +346,7 @@
353346 	enum cache_metadata_mode mode;
354347 	enum cache_io_mode io_mode;
355348 	unsigned metadata_version;
349
+	bool discard_passdown:1;
356350 };
357351 
358352 struct cache_stats {
....@@ -426,8 +420,6 @@
426420 	struct bio_list deferred_bios;
427421 
428422 	struct rw_semaphore quiesce_lock;
429
-
430
-	struct dm_target_callbacks callbacks;
431423 
432424 	/*
433425 	 * origin_blocks entries, discarded if set.
....@@ -629,23 +621,19 @@
629621 
630622 static void defer_bio(struct cache *cache, struct bio *bio)
631623 {
632
-	unsigned long flags;
633
-
634
-	spin_lock_irqsave(&cache->lock, flags);
624
+	spin_lock_irq(&cache->lock);
635625 	bio_list_add(&cache->deferred_bios, bio);
636
-	spin_unlock_irqrestore(&cache->lock, flags);
626
+	spin_unlock_irq(&cache->lock);
637627 
638628 	wake_deferred_bio_worker(cache);
639629 }
640630 
641631 static void defer_bios(struct cache *cache, struct bio_list *bios)
642632 {
643
-	unsigned long flags;
644
-
645
-	spin_lock_irqsave(&cache->lock, flags);
633
+	spin_lock_irq(&cache->lock);
646634 	bio_list_merge(&cache->deferred_bios, bios);
647635 	bio_list_init(bios);
648
-	spin_unlock_irqrestore(&cache->lock, flags);
636
+	spin_unlock_irq(&cache->lock);
649637 
650638 	wake_deferred_bio_worker(cache);
651639 }
....@@ -724,10 +712,6 @@
724712 	return cache->sectors_per_block_shift >= 0;
725713 }
726714 
727
-/* gcc on ARM generates spurious references to __udivdi3 and __umoddi3 */
728
-#if defined(CONFIG_ARM) && __GNUC__ == 4 && __GNUC_MINOR__ <= 6
729
-__always_inline
730
-#endif
731715 static dm_block_t block_div(dm_block_t b, uint32_t n)
732716 {
733717 	do_div(b, n);
....@@ -755,33 +739,27 @@
755739 
756740 static void set_discard(struct cache *cache, dm_dblock_t b)
757741 {
758
-	unsigned long flags;
759
-
760742 	BUG_ON(from_dblock(b) >= from_dblock(cache->discard_nr_blocks));
761743 	atomic_inc(&cache->stats.discard_count);
762744 
763
-	spin_lock_irqsave(&cache->lock, flags);
745
+	spin_lock_irq(&cache->lock);
764746 	set_bit(from_dblock(b), cache->discard_bitset);
765
-	spin_unlock_irqrestore(&cache->lock, flags);
747
+	spin_unlock_irq(&cache->lock);
766748 }
767749 
768750 static void clear_discard(struct cache *cache, dm_dblock_t b)
769751 {
770
-	unsigned long flags;
771
-
772
-	spin_lock_irqsave(&cache->lock, flags);
752
+	spin_lock_irq(&cache->lock);
773753 	clear_bit(from_dblock(b), cache->discard_bitset);
774
-	spin_unlock_irqrestore(&cache->lock, flags);
754
+	spin_unlock_irq(&cache->lock);
775755 }
776756 
777757 static bool is_discarded(struct cache *cache, dm_dblock_t b)
778758 {
779759 	int r;
780
-	unsigned long flags;
781
-
782
-	spin_lock_irqsave(&cache->lock, flags);
760
+	spin_lock_irq(&cache->lock);
783761 	r = test_bit(from_dblock(b), cache->discard_bitset);
784
-	spin_unlock_irqrestore(&cache->lock, flags);
762
+	spin_unlock_irq(&cache->lock);
785763 
786764 	return r;
787765 }
....@@ -789,12 +767,10 @@
789767 static bool is_discarded_oblock(struct cache *cache, dm_oblock_t b)
790768 {
791769 	int r;
792
-	unsigned long flags;
793
-
794
-	spin_lock_irqsave(&cache->lock, flags);
770
+	spin_lock_irq(&cache->lock);
795771 	r = test_bit(from_dblock(oblock_to_dblock(cache, b)),
796772 		     cache->discard_bitset);
797
-	spin_unlock_irqrestore(&cache->lock, flags);
773
+	spin_unlock_irq(&cache->lock);
798774 
799775 	return r;
800776 }
....@@ -826,17 +802,16 @@
826802 
827803 static void check_if_tick_bio_needed(struct cache *cache, struct bio *bio)
828804 {
829
-	unsigned long flags;
830805 	struct per_bio_data *pb;
831806 
832
-	spin_lock_irqsave(&cache->lock, flags);
807
+	spin_lock_irq(&cache->lock);
833808 	if (cache->need_tick_bio && !op_is_flush(bio->bi_opf) &&
834809 	    bio_op(bio) != REQ_OP_DISCARD) {
835810 		pb = get_per_bio_data(bio);
836811 		pb->tick = true;
837812 		cache->need_tick_bio = false;
838813 	}
839
-	spin_unlock_irqrestore(&cache->lock, flags);
814
+	spin_unlock_irq(&cache->lock);
840815 }
841816 
842817 static void __remap_to_origin_clear_discard(struct cache *cache, struct bio *bio,
....@@ -905,7 +880,7 @@
905880 static void accounted_request(struct cache *cache, struct bio *bio)
906881 {
907882 	accounted_begin(cache, bio);
908
-	generic_make_request(bio);
883
+	submit_bio_noacct(bio);
909884 }
910885 
911886 static void issue_op(struct bio *bio, void *context)
....@@ -946,7 +921,7 @@
946921 
947922 static const char *cache_device_name(struct cache *cache)
948923 {
949
-	return dm_device_name(dm_table_get_md(cache->ti->table));
924
+	return dm_table_device_name(cache->ti->table);
950925 }
951926 
952927 static void notify_mode_switch(struct cache *cache, enum cache_metadata_mode mode)
....@@ -1010,14 +985,14 @@
1010985 	if (get_cache_mode(cache) >= CM_READ_ONLY)
1011986 		return;
1012987 
1013
-	if (dm_cache_metadata_set_needs_check(cache->cmd)) {
1014
-		DMERR("%s: failed to set 'needs_check' flag in metadata", dev_name);
1015
-		set_cache_mode(cache, CM_FAIL);
1016
-	}
1017
-
1018988 	DMERR_LIMIT("%s: aborting current metadata transaction", dev_name);
1019989 	if (dm_cache_metadata_abort(cache->cmd)) {
1020990 		DMERR("%s: failed to abort metadata transaction", dev_name);
991
+		set_cache_mode(cache, CM_FAIL);
992
+	}
993
+
994
+	if (dm_cache_metadata_set_needs_check(cache->cmd)) {
995
+		DMERR("%s: failed to set 'needs_check' flag in metadata", dev_name);
1021996 		set_cache_mode(cache, CM_FAIL);
1022997 	}
1023998 }
....@@ -1811,7 +1786,7 @@
18111786 	bool commit_needed;
18121787 
18131788 	if (map_bio(cache, bio, get_bio_block(cache, bio), &commit_needed) == DM_MAPIO_REMAPPED)
1814
-		generic_make_request(bio);
1789
+		submit_bio_noacct(bio);
18151790 
18161791 	return commit_needed;
18171792 }
....@@ -1875,7 +1850,11 @@
18751850 		b = to_dblock(from_dblock(b) + 1);
18761851 	}
18771852 
1878
-	bio_endio(bio);
1853
+	if (cache->features.discard_passdown) {
1854
+		remap_to_origin(cache, bio);
1855
+		submit_bio_noacct(bio);
1856
+	} else
1857
+		bio_endio(bio);
18791858 
18801859 	return false;
18811860 }
....@@ -1884,17 +1863,16 @@
18841863 {
18851864 	struct cache *cache = container_of(ws, struct cache, deferred_bio_worker);
18861865 
1887
-	unsigned long flags;
18881866 	bool commit_needed = false;
18891867 	struct bio_list bios;
18901868 	struct bio *bio;
18911869 
18921870 	bio_list_init(&bios);
18931871 
1894
-	spin_lock_irqsave(&cache->lock, flags);
1872
+	spin_lock_irq(&cache->lock);
18951873 	bio_list_merge(&bios, &cache->deferred_bios);
18961874 	bio_list_init(&cache->deferred_bios);
1897
-	spin_unlock_irqrestore(&cache->lock, flags);
1875
+	spin_unlock_irq(&cache->lock);
18981876 
18991877 	while ((bio = bio_list_pop(&bios))) {
19001878 		if (bio->bi_opf & REQ_PREFLUSH)
....@@ -1905,6 +1883,7 @@
19051883 
19061884 		else
19071885 			commit_needed = process_bio(cache, bio) || commit_needed;
1886
+		cond_resched();
19081887 	}
19091888 
19101889 	if (commit_needed)
....@@ -1927,6 +1906,7 @@
19271906 	while ((bio = bio_list_pop(&bios))) {
19281907 		bio->bi_status = BLK_STS_DM_REQUEUE;
19291908 		bio_endio(bio);
1909
+		cond_resched();
19301910 	}
19311911 }
19321912 
....@@ -1967,6 +1947,8 @@
19671947 		r = mg_start(cache, op, NULL);
19681948 		if (r)
19691949 			break;
1950
+
1951
+		cond_resched();
19701952 	}
19711953 }
19721954 
....@@ -1987,6 +1969,7 @@
19871969 	if (cache->prison)
19881970 		dm_bio_prison_destroy_v2(cache->prison);
19891971 
1972
+	cancel_delayed_work_sync(&cache->waker);
19901973 	if (cache->wq)
19911974 		destroy_workqueue(cache->wq);
19921975 
....@@ -2209,13 +2192,14 @@
22092192 	cf->mode = CM_WRITE;
22102193 	cf->io_mode = CM_IO_WRITEBACK;
22112194 	cf->metadata_version = 1;
2195
+	cf->discard_passdown = true;
22122196 }
22132197 
22142198 static int parse_features(struct cache_args *ca, struct dm_arg_set *as,
22152199 			  char **error)
22162200 {
22172201 	static const struct dm_arg _args[] = {
2218
-		{0, 2, "Invalid number of cache feature arguments"},
2202
+		{0, 3, "Invalid number of cache feature arguments"},
22192203 	};
22202204 
22212205 	int r, mode_ctr = 0;
....@@ -2249,6 +2233,9 @@
22492233 
22502234 		else if (!strcasecmp(arg, "metadata2"))
22512235 			cf->metadata_version = 2;
2236
+
2237
+		else if (!strcasecmp(arg, "no_discard_passdown"))
2238
+			cf->discard_passdown = false;
22522239 
22532240 		else {
22542241 			*error = "Unrecognised cache feature requested";
....@@ -2435,20 +2422,6 @@
24352422 	cache->cache_size = size;
24362423 }
24372424 
2438
-static int is_congested(struct dm_dev *dev, int bdi_bits)
2439
-{
2440
-	struct request_queue *q = bdev_get_queue(dev->bdev);
2441
-	return bdi_congested(q->backing_dev_info, bdi_bits);
2442
-}
2443
-
2444
-static int cache_is_congested(struct dm_target_callbacks *cb, int bdi_bits)
2445
-{
2446
-	struct cache *cache = container_of(cb, struct cache, callbacks);
2447
-
2448
-	return is_congested(cache->origin_dev, bdi_bits) ||
2449
-		is_congested(cache->cache_dev, bdi_bits);
2450
-}
2451
-
24522425 #define DEFAULT_MIGRATION_THRESHOLD 2048
24532426 
24542427 static int cache_create(struct cache_args *ca, struct cache **result)
....@@ -2472,7 +2445,6 @@
24722445 
24732446 	ti->num_discard_bios = 1;
24742447 	ti->discards_supported = true;
2475
-	ti->split_discard_bios = false;
24762448 
24772449 	ti->per_io_data_size = sizeof(struct per_bio_data);
24782450 
....@@ -2483,9 +2455,6 @@
24832455 		if (r)
24842456 			goto bad;
24852457 	}
2486
-
2487
-	cache->callbacks.congested_fn = cache_is_congested;
2488
-	dm_table_add_target_callbacks(ti->table, &cache->callbacks);
24892458 
24902459 	cache->metadata_dev = ca->metadata_dev;
24912460 	cache->origin_dev = ca->origin_dev;
....@@ -3096,6 +3065,39 @@
30963065 	do_waker(&cache->waker.work);
30973066 }
30983067 
3068
+static void emit_flags(struct cache *cache, char *result,
3069
+		       unsigned maxlen, ssize_t *sz_ptr)
3070
+{
3071
+	ssize_t sz = *sz_ptr;
3072
+	struct cache_features *cf = &cache->features;
3073
+	unsigned count = (cf->metadata_version == 2) + !cf->discard_passdown + 1;
3074
+
3075
+	DMEMIT("%u ", count);
3076
+
3077
+	if (cf->metadata_version == 2)
3078
+		DMEMIT("metadata2 ");
3079
+
3080
+	if (writethrough_mode(cache))
3081
+		DMEMIT("writethrough ");
3082
+
3083
+	else if (passthrough_mode(cache))
3084
+		DMEMIT("passthrough ");
3085
+
3086
+	else if (writeback_mode(cache))
3087
+		DMEMIT("writeback ");
3088
+
3089
+	else {
3090
+		DMEMIT("unknown ");
3091
+		DMERR("%s: internal error: unknown io mode: %d",
3092
+		      cache_device_name(cache), (int) cf->io_mode);
3093
+	}
3094
+
3095
+	if (!cf->discard_passdown)
3096
+		DMEMIT("no_discard_passdown ");
3097
+
3098
+	*sz_ptr = sz;
3099
+}
3100
+
30993101 /*
31003102  * Status format:
31013103  *
....@@ -3162,25 +3164,7 @@
31623164 		       (unsigned) atomic_read(&cache->stats.promotion),
31633165 		       (unsigned long) atomic_read(&cache->nr_dirty));
31643166 
3165
-		if (cache->features.metadata_version == 2)
3166
-			DMEMIT("2 metadata2 ");
3167
-		else
3168
-			DMEMIT("1 ");
3169
-
3170
-		if (writethrough_mode(cache))
3171
-			DMEMIT("writethrough ");
3172
-
3173
-		else if (passthrough_mode(cache))
3174
-			DMEMIT("passthrough ");
3175
-
3176
-		else if (writeback_mode(cache))
3177
-			DMEMIT("writeback ");
3178
-
3179
-		else {
3180
-			DMERR("%s: internal error: unknown io mode: %d",
3181
-			      cache_device_name(cache), (int) cache->features.io_mode);
3182
-			goto err;
3183
-		}
3167
+		emit_flags(cache, result, maxlen, &sz);
31843168 
31853169 		DMEMIT("2 migration_threshold %llu ", (unsigned long long) cache->migration_threshold);
31863170 
....@@ -3409,14 +3393,62 @@
34093393 	return r;
34103394 }
34113395 
3396
+static bool origin_dev_supports_discard(struct block_device *origin_bdev)
3397
+{
3398
+	struct request_queue *q = bdev_get_queue(origin_bdev);
3399
+
3400
+	return q && blk_queue_discard(q);
3401
+}
3402
+
3403
+/*
3404
+ * If discard_passdown was enabled verify that the origin device
3405
+ * supports discards.  Disable discard_passdown if not.
3406
+ */
3407
+static void disable_passdown_if_not_supported(struct cache *cache)
3408
+{
3409
+	struct block_device *origin_bdev = cache->origin_dev->bdev;
3410
+	struct queue_limits *origin_limits = &bdev_get_queue(origin_bdev)->limits;
3411
+	const char *reason = NULL;
3412
+	char buf[BDEVNAME_SIZE];
3413
+
3414
+	if (!cache->features.discard_passdown)
3415
+		return;
3416
+
3417
+	if (!origin_dev_supports_discard(origin_bdev))
3418
+		reason = "discard unsupported";
3419
+
3420
+	else if (origin_limits->max_discard_sectors < cache->sectors_per_block)
3421
+		reason = "max discard sectors smaller than a block";
3422
+
3423
+	if (reason) {
3424
+		DMWARN("Origin device (%s) %s: Disabling discard passdown.",
3425
+		       bdevname(origin_bdev, buf), reason);
3426
+		cache->features.discard_passdown = false;
3427
+	}
3428
+}
3429
+
34123430 static void set_discard_limits(struct cache *cache, struct queue_limits *limits)
34133431 {
3432
+	struct block_device *origin_bdev = cache->origin_dev->bdev;
3433
+	struct queue_limits *origin_limits = &bdev_get_queue(origin_bdev)->limits;
3434
+
3435
+	if (!cache->features.discard_passdown) {
3436
+		/* No passdown is done so setting own virtual limits */
3437
+		limits->max_discard_sectors = min_t(sector_t, cache->discard_block_size * 1024,
3438
+						    cache->origin_sectors);
3439
+		limits->discard_granularity = cache->discard_block_size << SECTOR_SHIFT;
3440
+		return;
3441
+	}
3442
+
34143443 	/*
3415
-	 * FIXME: these limits may be incompatible with the cache device
3444
+	 * cache_iterate_devices() is stacking both origin and fast device limits
3445
+	 * but discards aren't passed to fast device, so inherit origin's limits.
34163446 	 */
3417
-	limits->max_discard_sectors = min_t(sector_t, cache->discard_block_size * 1024,
3418
-					    cache->origin_sectors);
3419
-	limits->discard_granularity = cache->discard_block_size << SECTOR_SHIFT;
3447
+	limits->max_discard_sectors = origin_limits->max_discard_sectors;
3448
+	limits->max_hw_discard_sectors = origin_limits->max_hw_discard_sectors;
3449
+	limits->discard_granularity = origin_limits->discard_granularity;
3450
+	limits->discard_alignment = origin_limits->discard_alignment;
3451
+	limits->discard_misaligned = origin_limits->discard_misaligned;
34203452 }
34213453 
34223454 static void cache_io_hints(struct dm_target *ti, struct queue_limits *limits)
....@@ -3433,6 +3465,8 @@
34333465 		blk_limits_io_min(limits, cache->sectors_per_block << SECTOR_SHIFT);
34343466 		blk_limits_io_opt(limits, cache->sectors_per_block << SECTOR_SHIFT);
34353467 	}
3468
+
3469
+	disable_passdown_if_not_supported(cache);
34363470 	set_discard_limits(cache, limits);
34373471 }
34383472 
....@@ -3440,7 +3474,7 @@
34403474 
34413475 static struct target_type cache_target = {
34423476 	.name = "cache",
3443
-	.version = {2, 0, 0},
3477
+	.version = {2, 2, 0},
34443478 	.module = THIS_MODULE,
34453479 	.ctr = cache_ctr,
34463480 	.dtr = cache_dtr,