From 102a0743326a03cd1a1202ceda21e175b7d3575c Mon Sep 17 00:00:00 2001
From: hc <hc@nodka.com>
Date: Tue, 20 Feb 2024 01:20:52 +0000
Subject: [PATCH] add new system file
---
kernel/drivers/block/zram/zram_drv.c | 201 +++++++++++++++++++++-----------------------------
1 files changed, 84 insertions(+), 117 deletions(-)
diff --git a/kernel/drivers/block/zram/zram_drv.c b/kernel/drivers/block/zram/zram_drv.c
index 6d17aee..a8b8538 100644
--- a/kernel/drivers/block/zram/zram_drv.c
+++ b/kernel/drivers/block/zram/zram_drv.c
@@ -33,6 +33,7 @@
#include <linux/sysfs.h>
#include <linux/debugfs.h>
#include <linux/cpuhotplug.h>
+#include <linux/part_stat.h>
#include "zram_drv.h"
@@ -41,7 +42,7 @@
static DEFINE_MUTEX(zram_index_mutex);
static int zram_major;
-static const char *default_compressor = "lzo";
+static const char *default_compressor = "lzo-rle";
/* Module params (documentation at end) */
static unsigned int num_devices = 1;
@@ -51,44 +52,13 @@
*/
static size_t huge_class_size;
+static const struct block_device_operations zram_devops;
+static const struct block_device_operations zram_wb_devops;
+
static void zram_free_page(struct zram *zram, size_t index);
static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec,
u32 index, int offset, struct bio *bio);
-
-#ifdef CONFIG_PREEMPT_RT_BASE
-static void zram_meta_init_table_locks(struct zram *zram, size_t num_pages)
-{
- size_t index;
-
- for (index = 0; index < num_pages; index++)
- spin_lock_init(&zram->table[index].lock);
-}
-
-static int zram_slot_trylock(struct zram *zram, u32 index)
-{
- int ret;
-
- ret = spin_trylock(&zram->table[index].lock);
- if (ret)
- __set_bit(ZRAM_LOCK, &zram->table[index].value);
- return ret;
-}
-
-static void zram_slot_lock(struct zram *zram, u32 index)
-{
- spin_lock(&zram->table[index].lock);
- __set_bit(ZRAM_LOCK, &zram->table[index].value);
-}
-
-static void zram_slot_unlock(struct zram *zram, u32 index)
-{
- __clear_bit(ZRAM_LOCK, &zram->table[index].value);
- spin_unlock(&zram->table[index].lock);
-}
-
-#else
-static void zram_meta_init_table_locks(struct zram *zram, size_t num_pages) { }
static int zram_slot_trylock(struct zram *zram, u32 index)
{
@@ -104,7 +74,6 @@
{
bit_spin_unlock(ZRAM_LOCK, &zram->table[index].flags);
}
-#endif
static inline bool init_done(struct zram *zram)
{
@@ -242,14 +211,17 @@
static bool page_same_filled(void *ptr, unsigned long *element)
{
- unsigned int pos;
unsigned long *page;
unsigned long val;
+ unsigned int pos, last_pos = PAGE_SIZE / sizeof(*page) - 1;
page = (unsigned long *)ptr;
val = page[0];
- for (pos = 1; pos < PAGE_SIZE / sizeof(*page); pos++) {
+ if (val != page[last_pos])
+ return false;
+
+ for (pos = 1; pos < last_pos; pos++) {
if (val != page[pos])
return false;
}
@@ -325,18 +297,8 @@
struct zram *zram = dev_to_zram(dev);
unsigned long nr_pages = zram->disksize >> PAGE_SHIFT;
int index;
- char mode_buf[8];
- ssize_t sz;
- sz = strscpy(mode_buf, buf, sizeof(mode_buf));
- if (sz <= 0)
- return -EINVAL;
-
- /* ignore trailing new line */
- if (mode_buf[sz - 1] == '\n')
- mode_buf[sz - 1] = 0x00;
-
- if (strcmp(mode_buf, "all"))
+ if (!sysfs_streq(buf, "all"))
return -EINVAL;
down_read(&zram->init_lock);
@@ -449,8 +411,7 @@
zram->backing_dev = NULL;
zram->old_block_size = 0;
zram->bdev = NULL;
- zram->disk->queue->backing_dev_info->capabilities |=
- BDI_CAP_SYNCHRONOUS_IO;
+ zram->disk->fops = &zram_devops;
kvfree(zram->bitmap);
zram->bitmap = NULL;
}
@@ -516,7 +477,7 @@
if (sz > 0 && file_name[sz - 1] == '\n')
file_name[sz - 1] = 0x00;
- backing_dev = filp_open(file_name, O_RDWR|O_LARGEFILE, 0);
+ backing_dev = filp_open_block(file_name, O_RDWR|O_LARGEFILE, 0);
if (IS_ERR(backing_dev)) {
err = PTR_ERR(backing_dev);
backing_dev = NULL;
@@ -532,9 +493,10 @@
goto out;
}
- bdev = bdgrab(I_BDEV(inode));
- err = blkdev_get(bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL, zram);
- if (err < 0) {
+ bdev = blkdev_get_by_dev(inode->i_rdev,
+ FMODE_READ | FMODE_WRITE | FMODE_EXCL, zram);
+ if (IS_ERR(bdev)) {
+ err = PTR_ERR(bdev);
bdev = NULL;
goto out;
}
@@ -569,8 +531,7 @@
* freely but in fact, IO is going on so finally could cause
* use-after-free when the IO is really done.
*/
- zram->disk->queue->backing_dev_info->capabilities &=
- ~BDI_CAP_SYNCHRONOUS_IO;
+ zram->disk->fops = &zram_wb_devops;
up_write(&zram->init_lock);
pr_info("setup backing device %s\n", file_name);
@@ -659,38 +620,41 @@
return 1;
}
+#define PAGE_WB_SIG "page_index="
+
+#define PAGE_WRITEBACK 0
#define HUGE_WRITEBACK 1
#define IDLE_WRITEBACK 2
+
static ssize_t writeback_store(struct device *dev,
struct device_attribute *attr, const char *buf, size_t len)
{
struct zram *zram = dev_to_zram(dev);
unsigned long nr_pages = zram->disksize >> PAGE_SHIFT;
- unsigned long index;
+ unsigned long index = 0;
struct bio bio;
struct bio_vec bio_vec;
struct page *page;
- ssize_t ret, sz;
- char mode_buf[8];
- int mode = -1;
+ ssize_t ret = len;
+ int mode, err;
unsigned long blk_idx = 0;
- sz = strscpy(mode_buf, buf, sizeof(mode_buf));
- if (sz <= 0)
- return -EINVAL;
-
- /* ignore trailing newline */
- if (mode_buf[sz - 1] == '\n')
- mode_buf[sz - 1] = 0x00;
-
- if (!strcmp(mode_buf, "idle"))
+ if (sysfs_streq(buf, "idle"))
mode = IDLE_WRITEBACK;
- else if (!strcmp(mode_buf, "huge"))
+ else if (sysfs_streq(buf, "huge"))
mode = HUGE_WRITEBACK;
+ else {
+ if (strncmp(buf, PAGE_WB_SIG, sizeof(PAGE_WB_SIG) - 1))
+ return -EINVAL;
- if (mode == -1)
- return -EINVAL;
+ if (kstrtol(buf + sizeof(PAGE_WB_SIG) - 1, 10, &index) ||
+ index >= nr_pages)
+ return -EINVAL;
+
+ nr_pages = 1;
+ mode = PAGE_WRITEBACK;
+ }
down_read(&zram->init_lock);
if (!init_done(zram)) {
@@ -709,7 +673,7 @@
goto release_init_lock;
}
- for (index = 0; index < nr_pages; index++) {
+ for (; nr_pages != 0; index++, nr_pages--) {
struct bio_vec bvec;
bvec.bv_page = page;
@@ -774,12 +738,17 @@
* XXX: A single page IO would be inefficient for write
* but it would be not bad as starter.
*/
- ret = submit_bio_wait(&bio);
- if (ret) {
+ err = submit_bio_wait(&bio);
+ if (err) {
zram_slot_lock(zram, index);
zram_clear_flag(zram, index, ZRAM_UNDER_WB);
zram_clear_flag(zram, index, ZRAM_IDLE);
zram_slot_unlock(zram, index);
+ /*
+ * Return last IO error unless every IO were
+ * not suceeded.
+ */
+ ret = err;
continue;
}
@@ -817,7 +786,6 @@
if (blk_idx)
free_block_bdev(zram, blk_idx);
- ret = len;
__free_page(page);
release_init_lock:
up_read(&zram->init_lock);
@@ -845,9 +813,9 @@
}
/*
- * Block layer want one ->make_request_fn to be active at a time
- * so if we use chained IO with parent IO in same context,
- * it's a deadlock. To avoid, it, it uses worker thread context.
+ * Block layer want one ->submit_bio to be active at a time, so if we use
+ * chained IO with parent IO in same context, it's a deadlock. To avoid that,
+ * use a worker thread context.
*/
static int read_from_bdev_sync(struct zram *zram, struct bio_vec *bvec,
unsigned long entry, struct bio *bio)
@@ -1180,8 +1148,6 @@
#endif
static DEVICE_ATTR_RO(debug_stat);
-
-
static void zram_meta_free(struct zram *zram, u64 disksize)
{
size_t num_pages = disksize >> PAGE_SHIFT;
@@ -1212,7 +1178,6 @@
if (!huge_class_size)
huge_class_size = zs_huge_class_size(zram->mem_pool);
- zram_meta_init_table_locks(zram, num_pages);
return true;
}
@@ -1271,11 +1236,11 @@
static int __zram_bvec_read(struct zram *zram, struct page *page, u32 index,
struct bio *bio, bool partial_io)
{
- int ret;
+ struct zcomp_strm *zstrm;
unsigned long handle;
unsigned int size;
void *src, *dst;
- struct zcomp_strm *zstrm;
+ int ret;
zram_slot_lock(zram, index);
if (zram_test_flag(zram, index, ZRAM_WB)) {
@@ -1306,7 +1271,9 @@
size = zram_get_obj_size(zram, index);
- zstrm = zcomp_stream_get(zram->comp);
+ if (size != PAGE_SIZE)
+ zstrm = zcomp_stream_get(zram->comp);
+
src = zs_map_object(zram->mem_pool, handle, ZS_MM_RO);
if (size == PAGE_SIZE) {
dst = kmap_atomic(page);
@@ -1314,17 +1281,16 @@
kunmap_atomic(dst);
ret = 0;
} else {
-
dst = kmap_atomic(page);
ret = zcomp_decompress(zstrm, src, size, dst);
kunmap_atomic(dst);
+ zcomp_stream_put(zram->comp);
}
zs_unmap_object(zram->mem_pool, handle);
- zcomp_stream_put(zram->comp);
zram_slot_unlock(zram, index);
/* Should NEVER happen. Return bio error if it does. */
- if (unlikely(ret))
+ if (WARN_ON(ret))
pr_err("Decompression failed! err=%d, page=%u\n", ret, index);
return ret;
@@ -1419,13 +1385,14 @@
__GFP_KSWAPD_RECLAIM |
__GFP_NOWARN |
__GFP_HIGHMEM |
- __GFP_MOVABLE);
+ __GFP_MOVABLE |
+ __GFP_CMA);
if (!handle) {
zcomp_stream_put(zram->comp);
atomic64_inc(&zram->stats.writestall);
handle = zs_malloc(zram->mem_pool, comp_len,
GFP_NOIO | __GFP_HIGHMEM |
- __GFP_MOVABLE);
+ __GFP_MOVABLE | __GFP_CMA);
if (handle)
goto compress_again;
return -ENOMEM;
@@ -1566,12 +1533,7 @@
static int zram_bvec_rw(struct zram *zram, struct bio_vec *bvec, u32 index,
int offset, unsigned int op, struct bio *bio)
{
- unsigned long start_time = jiffies;
- struct request_queue *q = zram->disk->queue;
int ret;
-
- generic_start_io_acct(q, op, bvec->bv_len >> SECTOR_SHIFT,
- &zram->disk->part0);
if (!op_is_write(op)) {
atomic64_inc(&zram->stats.num_reads);
@@ -1581,8 +1543,6 @@
atomic64_inc(&zram->stats.num_writes);
ret = zram_bvec_write(zram, bvec, index, offset, bio);
}
-
- generic_end_io_acct(q, op, &zram->disk->part0, start_time);
zram_slot_lock(zram, index);
zram_accessed(zram, index);
@@ -1604,6 +1564,7 @@
u32 index;
struct bio_vec bvec;
struct bvec_iter iter;
+ unsigned long start_time;
index = bio->bi_iter.bi_sector >> SECTORS_PER_PAGE_SHIFT;
offset = (bio->bi_iter.bi_sector &
@@ -1619,6 +1580,7 @@
break;
}
+ start_time = bio_start_io_acct(bio);
bio_for_each_segment(bvec, bio, iter) {
struct bio_vec bv = bvec;
unsigned int unwritten = bvec.bv_len;
@@ -1627,8 +1589,10 @@
bv.bv_len = min_t(unsigned int, PAGE_SIZE - offset,
unwritten);
if (zram_bvec_rw(zram, &bv, index, offset,
- bio_op(bio), bio) < 0)
- goto out;
+ bio_op(bio), bio) < 0) {
+ bio->bi_status = BLK_STS_IOERR;
+ break;
+ }
bv.bv_offset += bv.bv_len;
unwritten -= bv.bv_len;
@@ -1636,20 +1600,16 @@
update_position(&index, &offset, &bv);
} while (unwritten);
}
-
+ bio_end_io_acct(bio, start_time);
bio_endio(bio);
- return;
-
-out:
- bio_io_error(bio);
}
/*
* Handler function for all zram I/O requests.
*/
-static blk_qc_t zram_make_request(struct request_queue *queue, struct bio *bio)
+static blk_qc_t zram_submit_bio(struct bio *bio)
{
- struct zram *zram = queue->queuedata;
+ struct zram *zram = bio->bi_disk->private_data;
if (!valid_io_request(zram, bio->bi_iter.bi_sector,
bio->bi_iter.bi_size)) {
@@ -1689,6 +1649,7 @@
u32 index;
struct zram *zram;
struct bio_vec bv;
+ unsigned long start_time;
if (PageTransHuge(page))
return -ENOTSUPP;
@@ -1707,7 +1668,9 @@
bv.bv_len = PAGE_SIZE;
bv.bv_offset = 0;
+ start_time = disk_start_io_acct(bdev->bd_disk, SECTORS_PER_PAGE, op);
ret = zram_bvec_rw(zram, &bv, index, offset, op, NULL);
+ disk_end_io_acct(bdev->bd_disk, op, start_time);
out:
/*
* If I/O fails, just return error(ie, non-zero) without
@@ -1799,7 +1762,7 @@
zram->disksize = disksize;
set_capacity(zram->disk, zram->disksize >> SECTOR_SHIFT);
- revalidate_disk(zram->disk);
+ revalidate_disk_size(zram->disk, true);
up_write(&zram->init_lock);
return len;
@@ -1846,7 +1809,7 @@
/* Make sure all the pending I/O are finished */
fsync_bdev(bdev);
zram_reset_device(zram);
- revalidate_disk(zram->disk);
+ revalidate_disk_size(zram->disk, true);
bdput(bdev);
mutex_lock(&bdev->bd_mutex);
@@ -1873,8 +1836,16 @@
static const struct block_device_operations zram_devops = {
.open = zram_open,
+ .submit_bio = zram_submit_bio,
.swap_slot_free_notify = zram_slot_free_notify,
.rw_page = zram_rw_page,
+ .owner = THIS_MODULE
+};
+
+static const struct block_device_operations zram_wb_devops = {
+ .open = zram_open,
+ .submit_bio = zram_submit_bio,
+ .swap_slot_free_notify = zram_slot_free_notify,
.owner = THIS_MODULE
};
@@ -1951,15 +1922,13 @@
#ifdef CONFIG_ZRAM_WRITEBACK
spin_lock_init(&zram->wb_limit_lock);
#endif
- queue = blk_alloc_queue(GFP_KERNEL);
+ queue = blk_alloc_queue(NUMA_NO_NODE);
if (!queue) {
pr_err("Error allocating disk queue for device %d\n",
device_id);
ret = -ENOMEM;
goto out_free_idr;
}
-
- blk_queue_make_request(queue, zram_make_request);
/* gendisk structure */
zram->disk = alloc_disk(1);
@@ -1974,7 +1943,6 @@
zram->disk->first_minor = device_id;
zram->disk->fops = &zram_devops;
zram->disk->queue = queue;
- zram->disk->queue->queuedata = zram;
zram->disk->private_data = zram;
snprintf(zram->disk->disk_name, 16, "zram%d", device_id);
@@ -2008,10 +1976,8 @@
if (ZRAM_LOGICAL_BLOCK_SIZE == PAGE_SIZE)
blk_queue_max_write_zeroes_sectors(zram->disk->queue, UINT_MAX);
- zram->disk->queue->backing_dev_info->capabilities |=
- (BDI_CAP_STABLE_WRITES | BDI_CAP_SYNCHRONOUS_IO);
- disk_to_dev(zram->disk)->groups = zram_disk_attr_groups;
- add_disk(zram->disk);
+ blk_queue_flag_set(QUEUE_FLAG_STABLE_WRITES, zram->disk->queue);
+ device_add_disk(NULL, zram->disk, zram_disk_attr_groups);
strlcpy(zram->compressor, default_compressor, sizeof(zram->compressor));
@@ -2047,6 +2013,7 @@
mutex_unlock(&bdev->bd_mutex);
zram_debugfs_unregister(zram);
+
/* Make sure all the pending I/O are finished */
fsync_bdev(bdev);
zram_reset_device(zram);
--
Gitblit v1.6.2