forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2023-12-09 95099d4622f8cb224d94e314c7a8e0df60b13f87 
 kernel/drivers/block/zram/zram_drv.c
....@@ -33,6 +33,7 @@
3333 #include <linux/sysfs.h>
3434 #include <linux/debugfs.h>
3535 #include <linux/cpuhotplug.h>
36
+#include <linux/part_stat.h>
3637 
3738 #include "zram_drv.h"
3839 
....@@ -41,7 +42,7 @@
4142 static DEFINE_MUTEX(zram_index_mutex);
4243 
4344 static int zram_major;
44
-static const char *default_compressor = "lzo";
45
+static const char *default_compressor = "lzo-rle";
4546 
4647 /* Module params (documentation at end) */
4748 static unsigned int num_devices = 1;
....@@ -51,10 +52,47 @@
5152  */
5253 static size_t huge_class_size;
5354 
55
+static const struct block_device_operations zram_devops;
56
+static const struct block_device_operations zram_wb_devops;
57
+
5458 static void zram_free_page(struct zram *zram, size_t index);
5559 static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec,
5660 				u32 index, int offset, struct bio *bio);
5761 
62
+#ifdef CONFIG_PREEMPT_RT
63
+static void zram_meta_init_table_locks(struct zram *zram, size_t num_pages)
64
+{
65
+	size_t index;
66
+
67
+	for (index = 0; index < num_pages; index++)
68
+		spin_lock_init(&zram->table[index].lock);
69
+}
70
+
71
+static int zram_slot_trylock(struct zram *zram, u32 index)
72
+{
73
+	int ret;
74
+
75
+	ret = spin_trylock(&zram->table[index].lock);
76
+	if (ret)
77
+		__set_bit(ZRAM_LOCK, &zram->table[index].flags);
78
+	return ret;
79
+}
80
+
81
+static void zram_slot_lock(struct zram *zram, u32 index)
82
+{
83
+	spin_lock(&zram->table[index].lock);
84
+	__set_bit(ZRAM_LOCK, &zram->table[index].flags);
85
+}
86
+
87
+static void zram_slot_unlock(struct zram *zram, u32 index)
88
+{
89
+	__clear_bit(ZRAM_LOCK, &zram->table[index].flags);
90
+	spin_unlock(&zram->table[index].lock);
91
+}
92
+
93
+#else
94
+
95
+static void zram_meta_init_table_locks(struct zram *zram, size_t num_pages) { }
5896 
5997 static int zram_slot_trylock(struct zram *zram, u32 index)
6098 {
....@@ -70,6 +108,7 @@
70108 {
71109 	bit_spin_unlock(ZRAM_LOCK, &zram->table[index].flags);
72110 }
111
+#endif
73112 
74113 static inline bool init_done(struct zram *zram)
75114 {
....@@ -207,14 +246,17 @@
207246 
208247 static bool page_same_filled(void *ptr, unsigned long *element)
209248 {
210
-	unsigned int pos;
211249 	unsigned long *page;
212250 	unsigned long val;
251
+	unsigned int pos, last_pos = PAGE_SIZE / sizeof(*page) - 1;
213252 
214253 	page = (unsigned long *)ptr;
215254 	val = page[0];
216255 
217
-	for (pos = 1; pos < PAGE_SIZE / sizeof(*page); pos++) {
256
+	if (val != page[last_pos])
257
+		return false;
258
+
259
+	for (pos = 1; pos < last_pos; pos++) {
218260 		if (val != page[pos])
219261 			return false;
220262 	}
....@@ -290,18 +332,8 @@
290332 	struct zram *zram = dev_to_zram(dev);
291333 	unsigned long nr_pages = zram->disksize >> PAGE_SHIFT;
292334 	int index;
293
-	char mode_buf[8];
294
-	ssize_t sz;
295335 
296
-	sz = strscpy(mode_buf, buf, sizeof(mode_buf));
297
-	if (sz <= 0)
298
-		return -EINVAL;
299
-
300
-	/* ignore trailing new line */
301
-	if (mode_buf[sz - 1] == '\n')
302
-		mode_buf[sz - 1] = 0x00;
303
-
304
-	if (strcmp(mode_buf, "all"))
336
+	if (!sysfs_streq(buf, "all"))
305337 		return -EINVAL;
306338 
307339 	down_read(&zram->init_lock);
....@@ -414,8 +446,7 @@
414446 	zram->backing_dev = NULL;
415447 	zram->old_block_size = 0;
416448 	zram->bdev = NULL;
417
-	zram->disk->queue->backing_dev_info->capabilities |=
418
-				BDI_CAP_SYNCHRONOUS_IO;
449
+	zram->disk->fops = &zram_devops;
419450 	kvfree(zram->bitmap);
420451 	zram->bitmap = NULL;
421452 }
....@@ -481,7 +512,7 @@
481512 	if (sz > 0 && file_name[sz - 1] == '\n')
482513 		file_name[sz - 1] = 0x00;
483514 
484
-	backing_dev = filp_open(file_name, O_RDWR|O_LARGEFILE, 0);
515
+	backing_dev = filp_open_block(file_name, O_RDWR|O_LARGEFILE, 0);
485516 	if (IS_ERR(backing_dev)) {
486517 		err = PTR_ERR(backing_dev);
487518 		backing_dev = NULL;
....@@ -497,9 +528,10 @@
497528 		goto out;
498529 	}
499530 
500
-	bdev = bdgrab(I_BDEV(inode));
501
-	err = blkdev_get(bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL, zram);
502
-	if (err < 0) {
531
+	bdev = blkdev_get_by_dev(inode->i_rdev,
532
+			FMODE_READ | FMODE_WRITE | FMODE_EXCL, zram);
533
+	if (IS_ERR(bdev)) {
534
+		err = PTR_ERR(bdev);
503535 		bdev = NULL;
504536 		goto out;
505537 	}
....@@ -534,8 +566,7 @@
534566 	 * freely but in fact, IO is going on so finally could cause
535567 	 * use-after-free when the IO is really done.
536568 	 */
537
-	zram->disk->queue->backing_dev_info->capabilities &=
538
-			~BDI_CAP_SYNCHRONOUS_IO;
569
+	zram->disk->fops = &zram_wb_devops;
539570 	up_write(&zram->init_lock);
540571 
541572 	pr_info("setup backing device %s\n", file_name);
....@@ -624,38 +655,41 @@
624655 	return 1;
625656 }
626657 
658
+#define PAGE_WB_SIG "page_index="
659
+
660
+#define PAGE_WRITEBACK 0
627661 #define HUGE_WRITEBACK 1
628662 #define IDLE_WRITEBACK 2
663
+
629664 
630665 static ssize_t writeback_store(struct device *dev,
631666 		struct device_attribute *attr, const char *buf, size_t len)
632667 {
633668 	struct zram *zram = dev_to_zram(dev);
634669 	unsigned long nr_pages = zram->disksize >> PAGE_SHIFT;
635
-	unsigned long index;
670
+	unsigned long index = 0;
636671 	struct bio bio;
637672 	struct bio_vec bio_vec;
638673 	struct page *page;
639
-	ssize_t ret, sz;
640
-	char mode_buf[8];
641
-	int mode = -1;
674
+	ssize_t ret = len;
675
+	int mode, err;
642676 	unsigned long blk_idx = 0;
643677 
644
-	sz = strscpy(mode_buf, buf, sizeof(mode_buf));
645
-	if (sz <= 0)
646
-		return -EINVAL;
647
-
648
-	/* ignore trailing newline */
649
-	if (mode_buf[sz - 1] == '\n')
650
-		mode_buf[sz - 1] = 0x00;
651
-
652
-	if (!strcmp(mode_buf, "idle"))
678
+	if (sysfs_streq(buf, "idle"))
653679 		mode = IDLE_WRITEBACK;
654
-	else if (!strcmp(mode_buf, "huge"))
680
+	else if (sysfs_streq(buf, "huge"))
655681 		mode = HUGE_WRITEBACK;
682
+	else {
683
+		if (strncmp(buf, PAGE_WB_SIG, sizeof(PAGE_WB_SIG) - 1))
684
+			return -EINVAL;
656685 
657
-	if (mode == -1)
658
-		return -EINVAL;
686
+		if (kstrtol(buf + sizeof(PAGE_WB_SIG) - 1, 10, &index) ||
687
+				index >= nr_pages)
688
+			return -EINVAL;
689
+
690
+		nr_pages = 1;
691
+		mode = PAGE_WRITEBACK;
692
+	}
659693 
660694 	down_read(&zram->init_lock);
661695 	if (!init_done(zram)) {
....@@ -674,7 +708,7 @@
674708 		goto release_init_lock;
675709 	}
676710 
677
-	for (index = 0; index < nr_pages; index++) {
711
+	for (; nr_pages != 0; index++, nr_pages--) {
678712 		struct bio_vec bvec;
679713 
680714 		bvec.bv_page = page;
....@@ -739,12 +773,17 @@
739773 		 * XXX: A single page IO would be inefficient for write
740774 		 * but it would be not bad as starter.
741775 		 */
742
-		ret = submit_bio_wait(&bio);
743
-		if (ret) {
776
+		err = submit_bio_wait(&bio);
777
+		if (err) {
744778 			zram_slot_lock(zram, index);
745779 			zram_clear_flag(zram, index, ZRAM_UNDER_WB);
746780 			zram_clear_flag(zram, index, ZRAM_IDLE);
747781 			zram_slot_unlock(zram, index);
782
+			/*
783
+			 * Return last IO error unless every IO were
784
+			 * not suceeded.
785
+			 */
786
+			ret = err;
748787 			continue;
749788 		}
750789 
....@@ -782,7 +821,6 @@
782821 
783822 	if (blk_idx)
784823 		free_block_bdev(zram, blk_idx);
785
-	ret = len;
786824 	__free_page(page);
787825 release_init_lock:
788826 	up_read(&zram->init_lock);
....@@ -810,9 +848,9 @@
810848 }
811849 
812850 /*
813
- * Block layer want one ->make_request_fn to be active at a time
814
- * so if we use chained IO with parent IO in same context,
815
- * it's a deadlock. To avoid, it, it uses worker thread context.
851
+ * Block layer want one ->submit_bio to be active at a time, so if we use
852
+ * chained IO with parent IO in same context, it's a deadlock. To avoid that,
853
+ * use a worker thread context.
816854  */
817855 static int read_from_bdev_sync(struct zram *zram, struct bio_vec *bvec,
818856 				unsigned long entry, struct bio *bio)
....@@ -1175,6 +1213,7 @@
11751213 
11761214 	if (!huge_class_size)
11771215 		huge_class_size = zs_huge_class_size(zram->mem_pool);
1216
+	zram_meta_init_table_locks(zram, num_pages);
11781217 	return true;
11791218 }
11801219 
....@@ -1233,10 +1272,11 @@
12331272 static int __zram_bvec_read(struct zram *zram, struct page *page, u32 index,
12341273 				struct bio *bio, bool partial_io)
12351274 {
1236
-	int ret;
1275
+	struct zcomp_strm *zstrm;
12371276 	unsigned long handle;
12381277 	unsigned int size;
12391278 	void *src, *dst;
1279
+	int ret;
12401280 
12411281 	zram_slot_lock(zram, index);
12421282 	if (zram_test_flag(zram, index, ZRAM_WB)) {
....@@ -1267,6 +1307,9 @@
12671307 
12681308 	size = zram_get_obj_size(zram, index);
12691309 
1310
+	if (size != PAGE_SIZE)
1311
+		zstrm = zcomp_stream_get(zram->comp);
1312
+
12701313 	src = zs_map_object(zram->mem_pool, handle, ZS_MM_RO);
12711314 	if (size == PAGE_SIZE) {
12721315 		dst = kmap_atomic(page);
....@@ -1274,8 +1317,6 @@
12741317 		kunmap_atomic(dst);
12751318 		ret = 0;
12761319 	} else {
1277
-		struct zcomp_strm *zstrm = zcomp_stream_get(zram->comp);
1278
-
12791320 		dst = kmap_atomic(page);
12801321 		ret = zcomp_decompress(zstrm, src, size, dst);
12811322 		kunmap_atomic(dst);
....@@ -1285,7 +1326,7 @@
12851326 	zram_slot_unlock(zram, index);
12861327 
12871328 	/* Should NEVER happen. Return bio error if it does. */
1288
-	if (unlikely(ret))
1329
+	if (WARN_ON(ret))
12891330 		pr_err("Decompression failed! err=%d, page=%u\n", ret, index);
12901331 
12911332 	return ret;
....@@ -1380,13 +1421,14 @@
13801421 				__GFP_KSWAPD_RECLAIM |
13811422 				__GFP_NOWARN |
13821423 				__GFP_HIGHMEM |
1383
-				__GFP_MOVABLE);
1424
+				__GFP_MOVABLE |
1425
+				__GFP_CMA);
13841426 	if (!handle) {
13851427 		zcomp_stream_put(zram->comp);
13861428 		atomic64_inc(&zram->stats.writestall);
13871429 		handle = zs_malloc(zram->mem_pool, comp_len,
13881430 				GFP_NOIO | __GFP_HIGHMEM |
1389
-				__GFP_MOVABLE);
1431
+				__GFP_MOVABLE | __GFP_CMA);
13901432 		if (handle)
13911433 			goto compress_again;
13921434 		return -ENOMEM;
....@@ -1527,12 +1569,7 @@
15271569 static int zram_bvec_rw(struct zram *zram, struct bio_vec *bvec, u32 index,
15281570 			int offset, unsigned int op, struct bio *bio)
15291571 {
1530
-	unsigned long start_time = jiffies;
1531
-	struct request_queue *q = zram->disk->queue;
15321572 	int ret;
1533
-
1534
-	generic_start_io_acct(q, op, bvec->bv_len >> SECTOR_SHIFT,
1535
-			&zram->disk->part0);
15361573 
15371574 	if (!op_is_write(op)) {
15381575 		atomic64_inc(&zram->stats.num_reads);
....@@ -1542,8 +1579,6 @@
15421579 		atomic64_inc(&zram->stats.num_writes);
15431580 		ret = zram_bvec_write(zram, bvec, index, offset, bio);
15441581 	}
1545
-
1546
-	generic_end_io_acct(q, op, &zram->disk->part0, start_time);
15471582 
15481583 	zram_slot_lock(zram, index);
15491584 	zram_accessed(zram, index);
....@@ -1565,6 +1600,7 @@
15651600 	u32 index;
15661601 	struct bio_vec bvec;
15671602 	struct bvec_iter iter;
1603
+	unsigned long start_time;
15681604 
15691605 	index = bio->bi_iter.bi_sector >> SECTORS_PER_PAGE_SHIFT;
15701606 	offset = (bio->bi_iter.bi_sector &
....@@ -1580,6 +1616,7 @@
15801616 		break;
15811617 	}
15821618 
1619
+	start_time = bio_start_io_acct(bio);
15831620 	bio_for_each_segment(bvec, bio, iter) {
15841621 		struct bio_vec bv = bvec;
15851622 		unsigned int unwritten = bvec.bv_len;
....@@ -1588,8 +1625,10 @@
15881625 			bv.bv_len = min_t(unsigned int, PAGE_SIZE - offset,
15891626 							unwritten);
15901627 			if (zram_bvec_rw(zram, &bv, index, offset,
1591
-					 bio_op(bio), bio) < 0)
1592
-				goto out;
1628
+					 bio_op(bio), bio) < 0) {
1629
+				bio->bi_status = BLK_STS_IOERR;
1630
+				break;
1631
+			}
15931632 
15941633 			bv.bv_offset += bv.bv_len;
15951634 			unwritten -= bv.bv_len;
....@@ -1597,20 +1636,16 @@
15971636 			update_position(&index, &offset, &bv);
15981637 		} while (unwritten);
15991638 	}
1600
-
1639
+	bio_end_io_acct(bio, start_time);
16011640 	bio_endio(bio);
1602
-	return;
1603
-
1604
-out:
1605
-	bio_io_error(bio);
16061641 }
16071642 
16081643 /*
16091644  * Handler function for all zram I/O requests.
16101645  */
1611
-static blk_qc_t zram_make_request(struct request_queue *queue, struct bio *bio)
1646
+static blk_qc_t zram_submit_bio(struct bio *bio)
16121647 {
1613
-	struct zram *zram = queue->queuedata;
1648
+	struct zram *zram = bio->bi_disk->private_data;
16141649 
16151650 	if (!valid_io_request(zram, bio->bi_iter.bi_sector,
16161651 					bio->bi_iter.bi_size)) {
....@@ -1650,6 +1685,7 @@
16501685 	u32 index;
16511686 	struct zram *zram;
16521687 	struct bio_vec bv;
1688
+	unsigned long start_time;
16531689 
16541690 	if (PageTransHuge(page))
16551691 		return -ENOTSUPP;
....@@ -1668,7 +1704,9 @@
16681704 	bv.bv_len = PAGE_SIZE;
16691705 	bv.bv_offset = 0;
16701706 
1707
+	start_time = disk_start_io_acct(bdev->bd_disk, SECTORS_PER_PAGE, op);
16711708 	ret = zram_bvec_rw(zram, &bv, index, offset, op, NULL);
1709
+	disk_end_io_acct(bdev->bd_disk, op, start_time);
16721710 out:
16731711 	/*
16741712 	 * If I/O fails, just return error(ie, non-zero) without
....@@ -1760,7 +1798,7 @@
17601798 	zram->disksize = disksize;
17611799 	set_capacity(zram->disk, zram->disksize >> SECTOR_SHIFT);
17621800 
1763
-	revalidate_disk(zram->disk);
1801
+	revalidate_disk_size(zram->disk, true);
17641802 	up_write(&zram->init_lock);
17651803 
17661804 	return len;
....@@ -1807,7 +1845,7 @@
18071845 	/* Make sure all the pending I/O are finished */
18081846 	fsync_bdev(bdev);
18091847 	zram_reset_device(zram);
1810
-	revalidate_disk(zram->disk);
1848
+	revalidate_disk_size(zram->disk, true);
18111849 	bdput(bdev);
18121850 
18131851 	mutex_lock(&bdev->bd_mutex);
....@@ -1834,8 +1872,16 @@
18341872 
18351873 static const struct block_device_operations zram_devops = {
18361874 	.open = zram_open,
1875
+	.submit_bio = zram_submit_bio,
18371876 	.swap_slot_free_notify = zram_slot_free_notify,
18381877 	.rw_page = zram_rw_page,
1878
+	.owner = THIS_MODULE
1879
+};
1880
+
1881
+static const struct block_device_operations zram_wb_devops = {
1882
+	.open = zram_open,
1883
+	.submit_bio = zram_submit_bio,
1884
+	.swap_slot_free_notify = zram_slot_free_notify,
18391885 	.owner = THIS_MODULE
18401886 };
18411887 
....@@ -1912,15 +1958,13 @@
19121958 #ifdef CONFIG_ZRAM_WRITEBACK
19131959 	spin_lock_init(&zram->wb_limit_lock);
19141960 #endif
1915
-	queue = blk_alloc_queue(GFP_KERNEL);
1961
+	queue = blk_alloc_queue(NUMA_NO_NODE);
19161962 	if (!queue) {
19171963 		pr_err("Error allocating disk queue for device %d\n",
19181964 			device_id);
19191965 		ret = -ENOMEM;
19201966 		goto out_free_idr;
19211967 	}
1922
-
1923
-	blk_queue_make_request(queue, zram_make_request);
19241968 
19251969 	/* gendisk structure */
19261970 	zram->disk = alloc_disk(1);
....@@ -1935,7 +1979,6 @@
19351979 	zram->disk->first_minor = device_id;
19361980 	zram->disk->fops = &zram_devops;
19371981 	zram->disk->queue = queue;
1938
-	zram->disk->queue->queuedata = zram;
19391982 	zram->disk->private_data = zram;
19401983 	snprintf(zram->disk->disk_name, 16, "zram%d", device_id);
19411984 
....@@ -1969,10 +2012,8 @@
19692012 	if (ZRAM_LOGICAL_BLOCK_SIZE == PAGE_SIZE)
19702013 		blk_queue_max_write_zeroes_sectors(zram->disk->queue, UINT_MAX);
19712014 
1972
-	zram->disk->queue->backing_dev_info->capabilities |=
1973
-			(BDI_CAP_STABLE_WRITES | BDI_CAP_SYNCHRONOUS_IO);
1974
-	disk_to_dev(zram->disk)->groups = zram_disk_attr_groups;
1975
-	add_disk(zram->disk);
2015
+	blk_queue_flag_set(QUEUE_FLAG_STABLE_WRITES, zram->disk->queue);
2016
+	device_add_disk(NULL, zram->disk, zram_disk_attr_groups);
19762017 
19772018 	strlcpy(zram->compressor, default_compressor, sizeof(zram->compressor));
19782019 
....@@ -2008,6 +2049,7 @@
20082049 	mutex_unlock(&bdev->bd_mutex);
20092050 
20102051 	zram_debugfs_unregister(zram);
2052
+
20112053 	/* Make sure all the pending I/O are finished */
20122054 	fsync_bdev(bdev);
20132055 	zram_reset_device(zram);