From 102a0743326a03cd1a1202ceda21e175b7d3575c Mon Sep 17 00:00:00 2001
From: hc <hc@nodka.com>
Date: Tue, 20 Feb 2024 01:20:52 +0000
Subject: [PATCH] add new system file
---
kernel/drivers/lightnvm/pblk-init.c | 500 +++++++++++++++++++++++++------------------------------
1 files changed, 229 insertions(+), 271 deletions(-)
diff --git a/kernel/drivers/lightnvm/pblk-init.c b/kernel/drivers/lightnvm/pblk-init.c
index 88b63278..b6246f7 100644
--- a/kernel/drivers/lightnvm/pblk-init.c
+++ b/kernel/drivers/lightnvm/pblk-init.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-2.0
/*
* Copyright (C) 2015 IT University of Copenhagen (rrpc.c)
* Copyright (C) 2016 CNEX Labs
@@ -19,47 +20,36 @@
*/
#include "pblk.h"
+#include "pblk-trace.h"
static unsigned int write_buffer_size;
module_param(write_buffer_size, uint, 0644);
MODULE_PARM_DESC(write_buffer_size, "number of entries in a write buffer");
-static struct kmem_cache *pblk_ws_cache, *pblk_rec_cache, *pblk_g_rq_cache,
- *pblk_w_rq_cache;
-static DECLARE_RWSEM(pblk_lock);
+struct pblk_global_caches {
+ struct kmem_cache *ws;
+ struct kmem_cache *rec;
+ struct kmem_cache *g_rq;
+ struct kmem_cache *w_rq;
+
+ struct kref kref;
+
+ struct mutex mutex; /* Ensures consistency between
+ * caches and kref
+ */
+};
+
+static struct pblk_global_caches pblk_caches = {
+ .mutex = __MUTEX_INITIALIZER(pblk_caches.mutex),
+ .kref = KREF_INIT(0),
+};
+
struct bio_set pblk_bio_set;
-static int pblk_rw_io(struct request_queue *q, struct pblk *pblk,
- struct bio *bio)
+static blk_qc_t pblk_submit_bio(struct bio *bio)
{
- int ret;
-
- /* Read requests must be <= 256kb due to NVMe's 64 bit completion bitmap
- * constraint. Writes can be of arbitrary size.
- */
- if (bio_data_dir(bio) == READ) {
- blk_queue_split(q, &bio);
- ret = pblk_submit_read(pblk, bio);
- if (ret == NVM_IO_DONE && bio_flagged(bio, BIO_CLONED))
- bio_put(bio);
-
- return ret;
- }
-
- /* Prevent deadlock in the case of a modest LUN configuration and large
- * user I/Os. Unless stalled, the rate limiter leaves at least 256KB
- * available for user I/O.
- */
- if (pblk_get_secs(bio) > pblk_rl_max_io(&pblk->rl))
- blk_queue_split(q, &bio);
-
- return pblk_write_to_cache(pblk, bio, PBLK_IOTYPE_USER);
-}
-
-static blk_qc_t pblk_make_rq(struct request_queue *q, struct bio *bio)
-{
- struct pblk *pblk = q->queuedata;
+ struct pblk *pblk = bio->bi_disk->queue->queuedata;
if (bio_op(bio) == REQ_OP_DISCARD) {
pblk_discard(pblk, bio);
@@ -69,17 +59,31 @@
}
}
- switch (pblk_rw_io(q, pblk, bio)) {
- case NVM_IO_ERR:
- bio_io_error(bio);
- break;
- case NVM_IO_DONE:
- bio_endio(bio);
- break;
+ /* Read requests must be <= 256kb due to NVMe's 64 bit completion bitmap
+ * constraint. Writes can be of arbitrary size.
+ */
+ if (bio_data_dir(bio) == READ) {
+ blk_queue_split(&bio);
+ pblk_submit_read(pblk, bio);
+ } else {
+ /* Prevent deadlock in the case of a modest LUN configuration
+ * and large user I/Os. Unless stalled, the rate limiter
+ * leaves at least 256KB available for user I/O.
+ */
+ if (pblk_get_secs(bio) > pblk_rl_max_io(&pblk->rl))
+ blk_queue_split(&bio);
+
+ pblk_write_to_cache(pblk, bio, PBLK_IOTYPE_USER);
}
return BLK_QC_T_NONE;
}
+
+static const struct block_device_operations pblk_bops = {
+ .owner = THIS_MODULE,
+ .submit_bio = pblk_submit_bio,
+};
+
static size_t pblk_trans_map_size(struct pblk *pblk)
{
@@ -88,7 +92,7 @@
if (pblk->addrf_len < 32)
entry_size = 4;
- return entry_size * pblk->rl.nr_secs;
+ return entry_size * pblk->capacity;
}
#ifdef CONFIG_NVM_PBLK_DEBUG
@@ -113,7 +117,7 @@
struct pblk_line *line = NULL;
if (factory_init) {
- pblk_setup_uuid(pblk);
+ guid_gen(&pblk->instance_uuid);
} else {
line = pblk_recov_l2p(pblk);
if (IS_ERR(line)) {
@@ -147,13 +151,17 @@
int ret = 0;
map_size = pblk_trans_map_size(pblk);
- pblk->trans_map = vmalloc(map_size);
- if (!pblk->trans_map)
+ pblk->trans_map = __vmalloc(map_size, GFP_KERNEL | __GFP_NOWARN |
+ __GFP_RETRY_MAYFAIL | __GFP_HIGHMEM);
+ if (!pblk->trans_map) {
+ pblk_err(pblk, "failed to allocate L2P (need %zu of memory)\n",
+ map_size);
return -ENOMEM;
+ }
pblk_ppa_set_empty(&ppa);
- for (i = 0; i < pblk->rl.nr_secs; i++)
+ for (i = 0; i < pblk->capacity; i++)
pblk_trans_map_set(pblk, i, ppa);
ret = pblk_l2p_recover(pblk, factory_init);
@@ -168,19 +176,17 @@
if (pblk_rb_tear_down_check(&pblk->rwb))
pblk_err(pblk, "write buffer error on tear down\n");
- pblk_rb_data_free(&pblk->rwb);
- vfree(pblk_rb_entries_ref(&pblk->rwb));
+ pblk_rb_free(&pblk->rwb);
}
static int pblk_rwb_init(struct pblk *pblk)
{
struct nvm_tgt_dev *dev = pblk->dev;
struct nvm_geo *geo = &dev->geo;
- struct pblk_rb_entry *entries;
- unsigned long nr_entries, buffer_size;
- unsigned int power_size, power_seg_sz;
- int pgs_in_buffer;
+ unsigned long buffer_size;
+ int pgs_in_buffer, threshold;
+ threshold = geo->mw_cunits * geo->all_luns;
pgs_in_buffer = (max(geo->mw_cunits, geo->ws_opt) + geo->ws_opt)
* geo->all_luns;
@@ -189,20 +195,8 @@
else
buffer_size = pgs_in_buffer;
- nr_entries = pblk_rb_calculate_size(buffer_size);
-
- entries = vzalloc(array_size(nr_entries, sizeof(struct pblk_rb_entry)));
- if (!entries)
- return -ENOMEM;
-
- power_size = get_count_order(nr_entries);
- power_seg_sz = get_count_order(geo->csecs);
-
- return pblk_rb_init(&pblk->rwb, entries, power_size, power_seg_sz);
+ return pblk_rb_init(&pblk->rwb, buffer_size, threshold, geo->csecs);
}
-
-/* Minimum pages needed within a lun */
-#define ADDR_POOL_SIZE 64
static int pblk_set_addrf_12(struct pblk *pblk, struct nvm_geo *geo,
struct nvm_addrf_12 *dst)
@@ -307,53 +301,76 @@
return 0;
}
-static int pblk_init_global_caches(struct pblk *pblk)
+static int pblk_create_global_caches(void)
{
- down_write(&pblk_lock);
- pblk_ws_cache = kmem_cache_create("pblk_blk_ws",
+
+ pblk_caches.ws = kmem_cache_create("pblk_blk_ws",
sizeof(struct pblk_line_ws), 0, 0, NULL);
- if (!pblk_ws_cache) {
- up_write(&pblk_lock);
+ if (!pblk_caches.ws)
return -ENOMEM;
- }
- pblk_rec_cache = kmem_cache_create("pblk_rec",
+ pblk_caches.rec = kmem_cache_create("pblk_rec",
sizeof(struct pblk_rec_ctx), 0, 0, NULL);
- if (!pblk_rec_cache) {
- kmem_cache_destroy(pblk_ws_cache);
- up_write(&pblk_lock);
- return -ENOMEM;
- }
+ if (!pblk_caches.rec)
+ goto fail_destroy_ws;
- pblk_g_rq_cache = kmem_cache_create("pblk_g_rq", pblk_g_rq_size,
+ pblk_caches.g_rq = kmem_cache_create("pblk_g_rq", pblk_g_rq_size,
0, 0, NULL);
- if (!pblk_g_rq_cache) {
- kmem_cache_destroy(pblk_ws_cache);
- kmem_cache_destroy(pblk_rec_cache);
- up_write(&pblk_lock);
- return -ENOMEM;
- }
+ if (!pblk_caches.g_rq)
+ goto fail_destroy_rec;
- pblk_w_rq_cache = kmem_cache_create("pblk_w_rq", pblk_w_rq_size,
+ pblk_caches.w_rq = kmem_cache_create("pblk_w_rq", pblk_w_rq_size,
0, 0, NULL);
- if (!pblk_w_rq_cache) {
- kmem_cache_destroy(pblk_ws_cache);
- kmem_cache_destroy(pblk_rec_cache);
- kmem_cache_destroy(pblk_g_rq_cache);
- up_write(&pblk_lock);
- return -ENOMEM;
- }
- up_write(&pblk_lock);
+ if (!pblk_caches.w_rq)
+ goto fail_destroy_g_rq;
return 0;
+
+fail_destroy_g_rq:
+ kmem_cache_destroy(pblk_caches.g_rq);
+fail_destroy_rec:
+ kmem_cache_destroy(pblk_caches.rec);
+fail_destroy_ws:
+ kmem_cache_destroy(pblk_caches.ws);
+
+ return -ENOMEM;
}
-static void pblk_free_global_caches(struct pblk *pblk)
+static int pblk_get_global_caches(void)
{
- kmem_cache_destroy(pblk_ws_cache);
- kmem_cache_destroy(pblk_rec_cache);
- kmem_cache_destroy(pblk_g_rq_cache);
- kmem_cache_destroy(pblk_w_rq_cache);
+ int ret = 0;
+
+ mutex_lock(&pblk_caches.mutex);
+
+ if (kref_get_unless_zero(&pblk_caches.kref))
+ goto out;
+
+ ret = pblk_create_global_caches();
+ if (!ret)
+ kref_init(&pblk_caches.kref);
+
+out:
+ mutex_unlock(&pblk_caches.mutex);
+ return ret;
+}
+
+static void pblk_destroy_global_caches(struct kref *ref)
+{
+ struct pblk_global_caches *c;
+
+ c = container_of(ref, struct pblk_global_caches, kref);
+
+ kmem_cache_destroy(c->ws);
+ kmem_cache_destroy(c->rec);
+ kmem_cache_destroy(c->g_rq);
+ kmem_cache_destroy(c->w_rq);
+}
+
+static void pblk_put_global_caches(void)
+{
+ mutex_lock(&pblk_caches.mutex);
+ kref_put(&pblk_caches.kref, pblk_destroy_global_caches);
+ mutex_unlock(&pblk_caches.mutex);
}
static int pblk_core_init(struct pblk *pblk)
@@ -373,16 +390,43 @@
pblk->nr_flush_rst = 0;
pblk->min_write_pgs = geo->ws_opt;
+ pblk->min_write_pgs_data = pblk->min_write_pgs;
max_write_ppas = pblk->min_write_pgs * geo->all_luns;
pblk->max_write_pgs = min_t(int, max_write_ppas, NVM_MAX_VLBA);
pblk->max_write_pgs = min_t(int, pblk->max_write_pgs,
queue_max_hw_sectors(dev->q) / (geo->csecs >> SECTOR_SHIFT));
pblk_set_sec_per_write(pblk, pblk->min_write_pgs);
- if (pblk->max_write_pgs > PBLK_MAX_REQ_ADDRS) {
- pblk_err(pblk, "vector list too big(%u > %u)\n",
- pblk->max_write_pgs, PBLK_MAX_REQ_ADDRS);
- return -EINVAL;
+ pblk->oob_meta_size = geo->sos;
+ if (!pblk_is_oob_meta_supported(pblk)) {
+ /* For drives which does not have OOB metadata feature
+ * in order to support recovery feature we need to use
+ * so called packed metadata. Packed metada will store
+ * the same information as OOB metadata (l2p table mapping,
+ * but in the form of the single page at the end of
+ * every write request.
+ */
+ if (pblk->min_write_pgs
+ * sizeof(struct pblk_sec_meta) > PAGE_SIZE) {
+ /* We want to keep all the packed metadata on single
+ * page per write requests. So we need to ensure that
+ * it will fit.
+ *
+ * This is more like sanity check, since there is
+ * no device with such a big minimal write size
+ * (above 1 metabytes).
+ */
+ pblk_err(pblk, "Not supported min write size\n");
+ return -EINVAL;
+ }
+ /* For packed meta approach we do some simplification.
+ * On read path we always issue requests which size
+ * equal to max_write_pgs, with all pages filled with
+ * user payload except of last one page which will be
+ * filled with packed metadata.
+ */
+ pblk->max_write_pgs = pblk->min_write_pgs;
+ pblk->min_write_pgs_data = pblk->min_write_pgs - 1;
}
pblk->pad_dist = kcalloc(pblk->min_write_pgs - 1, sizeof(atomic64_t),
@@ -390,7 +434,7 @@
if (!pblk->pad_dist)
return -ENOMEM;
- if (pblk_init_global_caches(pblk))
+ if (pblk_get_global_caches())
goto fail_free_pad_dist;
/* Internal bios can be at most the sectors signaled by the device. */
@@ -399,27 +443,27 @@
goto free_global_caches;
ret = mempool_init_slab_pool(&pblk->gen_ws_pool, PBLK_GEN_WS_POOL_SIZE,
- pblk_ws_cache);
+ pblk_caches.ws);
if (ret)
goto free_page_bio_pool;
ret = mempool_init_slab_pool(&pblk->rec_pool, geo->all_luns,
- pblk_rec_cache);
+ pblk_caches.rec);
if (ret)
goto free_gen_ws_pool;
ret = mempool_init_slab_pool(&pblk->r_rq_pool, geo->all_luns,
- pblk_g_rq_cache);
+ pblk_caches.g_rq);
if (ret)
goto free_rec_pool;
ret = mempool_init_slab_pool(&pblk->e_rq_pool, geo->all_luns,
- pblk_g_rq_cache);
+ pblk_caches.g_rq);
if (ret)
goto free_r_rq_pool;
ret = mempool_init_slab_pool(&pblk->w_rq_pool, geo->all_luns,
- pblk_w_rq_cache);
+ pblk_caches.w_rq);
if (ret)
goto free_e_rq_pool;
@@ -465,7 +509,7 @@
free_page_bio_pool:
mempool_exit(&pblk->page_bio_pool);
free_global_caches:
- pblk_free_global_caches(pblk);
+ pblk_put_global_caches();
fail_free_pad_dist:
kfree(pblk->pad_dist);
return -ENOMEM;
@@ -489,7 +533,7 @@
mempool_exit(&pblk->e_rq_pool);
mempool_exit(&pblk->w_rq_pool);
- pblk_free_global_caches(pblk);
+ pblk_put_global_caches();
kfree(pblk->pad_dist);
}
@@ -504,9 +548,12 @@
for (i = 0; i < PBLK_DATA_LINES; i++) {
kfree(l_mg->sline_meta[i]);
- pblk_mfree(l_mg->eline_meta[i]->buf, l_mg->emeta_alloc_type);
+ kvfree(l_mg->eline_meta[i]->buf);
kfree(l_mg->eline_meta[i]);
}
+
+ mempool_destroy(l_mg->bitmap_pool);
+ kmem_cache_destroy(l_mg->bitmap_cache);
}
static void pblk_line_meta_free(struct pblk_line_mgmt *l_mg,
@@ -518,7 +565,7 @@
kfree(line->erase_bitmap);
kfree(line->chks);
- pblk_mfree(w_err_gc->lba_list, l_mg->emeta_alloc_type);
+ kvfree(w_err_gc->lba_list);
kfree(w_err_gc);
}
@@ -528,80 +575,17 @@
struct pblk_line *line;
int i;
- spin_lock(&l_mg->free_lock);
for (i = 0; i < l_mg->nr_lines; i++) {
line = &pblk->lines[i];
pblk_line_free(line);
pblk_line_meta_free(l_mg, line);
}
- spin_unlock(&l_mg->free_lock);
pblk_line_mg_free(pblk);
kfree(pblk->luns);
kfree(pblk->lines);
-}
-
-static int pblk_bb_get_tbl(struct nvm_tgt_dev *dev, struct pblk_lun *rlun,
- u8 *blks, int nr_blks)
-{
- struct ppa_addr ppa;
- int ret;
-
- ppa.ppa = 0;
- ppa.g.ch = rlun->bppa.g.ch;
- ppa.g.lun = rlun->bppa.g.lun;
-
- ret = nvm_get_tgt_bb_tbl(dev, ppa, blks);
- if (ret)
- return ret;
-
- nr_blks = nvm_bb_tbl_fold(dev->parent, blks, nr_blks);
- if (nr_blks < 0)
- return -EIO;
-
- return 0;
-}
-
-static void *pblk_bb_get_meta(struct pblk *pblk)
-{
- struct nvm_tgt_dev *dev = pblk->dev;
- struct nvm_geo *geo = &dev->geo;
- u8 *meta;
- int i, nr_blks, blk_per_lun;
- int ret;
-
- blk_per_lun = geo->num_chk * geo->pln_mode;
- nr_blks = blk_per_lun * geo->all_luns;
-
- meta = kmalloc(nr_blks, GFP_KERNEL);
- if (!meta)
- return ERR_PTR(-ENOMEM);
-
- for (i = 0; i < geo->all_luns; i++) {
- struct pblk_lun *rlun = &pblk->luns[i];
- u8 *meta_pos = meta + i * blk_per_lun;
-
- ret = pblk_bb_get_tbl(dev, rlun, meta_pos, blk_per_lun);
- if (ret) {
- kfree(meta);
- return ERR_PTR(-EIO);
- }
- }
-
- return meta;
-}
-
-static void *pblk_chunk_get_meta(struct pblk *pblk)
-{
- struct nvm_tgt_dev *dev = pblk->dev;
- struct nvm_geo *geo = &dev->geo;
-
- if (geo->version == NVM_OCSSD_SPEC_12)
- return pblk_bb_get_meta(pblk);
- else
- return pblk_chunk_get_info(pblk);
}
static int pblk_luns_init(struct pblk *pblk)
@@ -666,87 +650,63 @@
return (lm->emeta_len[1] + lm->emeta_len[2] + lm->emeta_len[3]);
}
-static void pblk_set_provision(struct pblk *pblk, long nr_free_blks)
+static int pblk_set_provision(struct pblk *pblk, int nr_free_chks)
{
struct nvm_tgt_dev *dev = pblk->dev;
struct pblk_line_mgmt *l_mg = &pblk->l_mg;
struct pblk_line_meta *lm = &pblk->lm;
struct nvm_geo *geo = &dev->geo;
sector_t provisioned;
- int sec_meta, blk_meta;
+ int sec_meta, blk_meta, clba;
+ int minimum;
if (geo->op == NVM_TARGET_DEFAULT_OP)
pblk->op = PBLK_DEFAULT_OP;
else
pblk->op = geo->op;
- provisioned = nr_free_blks;
+ minimum = pblk_get_min_chks(pblk);
+ provisioned = nr_free_chks;
provisioned *= (100 - pblk->op);
sector_div(provisioned, 100);
- pblk->op_blks = nr_free_blks - provisioned;
+ if ((nr_free_chks - provisioned) < minimum) {
+ if (geo->op != NVM_TARGET_DEFAULT_OP) {
+ pblk_err(pblk, "OP too small to create a sane instance\n");
+ return -EINTR;
+ }
+
+ /* If the user did not specify an OP value, and PBLK_DEFAULT_OP
+ * is not enough, calculate and set sane value
+ */
+
+ provisioned = nr_free_chks - minimum;
+ pblk->op = (100 * minimum) / nr_free_chks;
+ pblk_info(pblk, "Default OP insufficient, adjusting OP to %d\n",
+ pblk->op);
+ }
+
+ pblk->op_blks = nr_free_chks - provisioned;
/* Internally pblk manages all free blocks, but all calculations based
* on user capacity consider only provisioned blocks
*/
- pblk->rl.total_blocks = nr_free_blks;
- pblk->rl.nr_secs = nr_free_blks * geo->clba;
+ pblk->rl.total_blocks = nr_free_chks;
/* Consider sectors used for metadata */
sec_meta = (lm->smeta_sec + lm->emeta_sec[0]) * l_mg->nr_free_lines;
blk_meta = DIV_ROUND_UP(sec_meta, geo->clba);
- pblk->capacity = (provisioned - blk_meta) * geo->clba;
+ clba = (geo->clba / pblk->min_write_pgs) * pblk->min_write_pgs_data;
+ pblk->capacity = (provisioned - blk_meta) * clba;
- atomic_set(&pblk->rl.free_blocks, nr_free_blks);
- atomic_set(&pblk->rl.free_user_blocks, nr_free_blks);
+ atomic_set(&pblk->rl.free_blocks, nr_free_chks);
+ atomic_set(&pblk->rl.free_user_blocks, nr_free_chks);
+
+ return 0;
}
-static int pblk_setup_line_meta_12(struct pblk *pblk, struct pblk_line *line,
- void *chunk_meta)
-{
- struct nvm_tgt_dev *dev = pblk->dev;
- struct nvm_geo *geo = &dev->geo;
- struct pblk_line_meta *lm = &pblk->lm;
- int i, chk_per_lun, nr_bad_chks = 0;
-
- chk_per_lun = geo->num_chk * geo->pln_mode;
-
- for (i = 0; i < lm->blk_per_line; i++) {
- struct pblk_lun *rlun = &pblk->luns[i];
- struct nvm_chk_meta *chunk;
- int pos = pblk_ppa_to_pos(geo, rlun->bppa);
- u8 *lun_bb_meta = chunk_meta + pos * chk_per_lun;
-
- chunk = &line->chks[pos];
-
- /*
- * In 1.2 spec. chunk state is not persisted by the device. Thus
- * some of the values are reset each time pblk is instantiated,
- * so we have to assume that the block is closed.
- */
- if (lun_bb_meta[line->id] == NVM_BLK_T_FREE)
- chunk->state = NVM_CHK_ST_CLOSED;
- else
- chunk->state = NVM_CHK_ST_OFFLINE;
-
- chunk->type = NVM_CHK_TP_W_SEQ;
- chunk->wi = 0;
- chunk->slba = -1;
- chunk->cnlb = geo->clba;
- chunk->wp = 0;
-
- if (!(chunk->state & NVM_CHK_ST_OFFLINE))
- continue;
-
- set_bit(pos, line->blk_bitmap);
- nr_bad_chks++;
- }
-
- return nr_bad_chks;
-}
-
-static int pblk_setup_line_meta_20(struct pblk *pblk, struct pblk_line *line,
+static int pblk_setup_line_meta_chk(struct pblk *pblk, struct pblk_line *line,
struct nvm_chk_meta *meta)
{
struct nvm_tgt_dev *dev = pblk->dev;
@@ -775,6 +735,9 @@
chunk->cnlb = chunk_meta->cnlb;
chunk->wp = chunk_meta->wp;
+ trace_pblk_chunk_state(pblk_disk_name(pblk), &ppa,
+ chunk->state);
+
if (chunk->type & NVM_CHK_TP_SZ_SPEC) {
WARN_ONCE(1, "pblk: custom-sized chunks unsupported\n");
continue;
@@ -793,8 +756,6 @@
static long pblk_setup_line_meta(struct pblk *pblk, struct pblk_line *line,
void *chunk_meta, int line_id)
{
- struct nvm_tgt_dev *dev = pblk->dev;
- struct nvm_geo *geo = &dev->geo;
struct pblk_line_mgmt *l_mg = &pblk->l_mg;
struct pblk_line_meta *lm = &pblk->lm;
long nr_bad_chks, chk_in_line;
@@ -807,10 +768,7 @@
line->vsc = &l_mg->vsc_list[line_id];
spin_lock_init(&line->lock);
- if (geo->version == NVM_OCSSD_SPEC_12)
- nr_bad_chks = pblk_setup_line_meta_12(pblk, line, chunk_meta);
- else
- nr_bad_chks = pblk_setup_line_meta_20(pblk, line, chunk_meta);
+ nr_bad_chks = pblk_setup_line_meta_chk(pblk, line, chunk_meta);
chk_in_line = lm->blk_per_line - nr_bad_chks;
if (nr_bad_chks < 0 || nr_bad_chks > lm->blk_per_line ||
@@ -916,6 +874,17 @@
goto fail_free_smeta;
}
+ l_mg->bitmap_cache = kmem_cache_create("pblk_lm_bitmap",
+ lm->sec_bitmap_len, 0, 0, NULL);
+ if (!l_mg->bitmap_cache)
+ goto fail_free_smeta;
+
+ /* the bitmap pool is used for both valid and map bitmaps */
+ l_mg->bitmap_pool = mempool_create_slab_pool(PBLK_DATA_LINES * 2,
+ l_mg->bitmap_cache);
+ if (!l_mg->bitmap_pool)
+ goto fail_destroy_bitmap_cache;
+
/* emeta allocates three different buffers for managing metadata with
* in-memory and in-media layouts
*/
@@ -926,29 +895,14 @@
if (!emeta)
goto fail_free_emeta;
- if (lm->emeta_len[0] > KMALLOC_MAX_CACHE_SIZE) {
- l_mg->emeta_alloc_type = PBLK_VMALLOC_META;
-
- emeta->buf = vmalloc(lm->emeta_len[0]);
- if (!emeta->buf) {
- kfree(emeta);
- goto fail_free_emeta;
- }
-
- emeta->nr_entries = lm->emeta_sec[0];
- l_mg->eline_meta[i] = emeta;
- } else {
- l_mg->emeta_alloc_type = PBLK_KMALLOC_META;
-
- emeta->buf = kmalloc(lm->emeta_len[0], GFP_KERNEL);
- if (!emeta->buf) {
- kfree(emeta);
- goto fail_free_emeta;
- }
-
- emeta->nr_entries = lm->emeta_sec[0];
- l_mg->eline_meta[i] = emeta;
+ emeta->buf = kvmalloc(lm->emeta_len[0], GFP_KERNEL);
+ if (!emeta->buf) {
+ kfree(emeta);
+ goto fail_free_emeta;
}
+
+ emeta->nr_entries = lm->emeta_sec[0];
+ l_mg->eline_meta[i] = emeta;
}
for (i = 0; i < l_mg->nr_lines; i++)
@@ -962,12 +916,13 @@
fail_free_emeta:
while (--i >= 0) {
- if (l_mg->emeta_alloc_type == PBLK_VMALLOC_META)
- vfree(l_mg->eline_meta[i]->buf);
- else
- kfree(l_mg->eline_meta[i]->buf);
+ kvfree(l_mg->eline_meta[i]->buf);
kfree(l_mg->eline_meta[i]);
}
+
+ mempool_destroy(l_mg->bitmap_pool);
+fail_destroy_bitmap_cache:
+ kmem_cache_destroy(l_mg->bitmap_cache);
fail_free_smeta:
for (i = 0; i < PBLK_DATA_LINES; i++)
kfree(l_mg->sline_meta[i]);
@@ -1046,7 +1001,7 @@
struct pblk_line_mgmt *l_mg = &pblk->l_mg;
struct pblk_line *line;
void *chunk_meta;
- long nr_free_chks = 0;
+ int nr_free_chks = 0;
int i, ret;
ret = pblk_line_meta_init(pblk);
@@ -1061,7 +1016,7 @@
if (ret)
goto fail_free_meta;
- chunk_meta = pblk_chunk_get_meta(pblk);
+ chunk_meta = pblk_get_chunk_meta(pblk);
if (IS_ERR(chunk_meta)) {
ret = PTR_ERR(chunk_meta);
goto fail_free_luns;
@@ -1082,6 +1037,9 @@
goto fail_free_lines;
nr_free_chks += pblk_setup_line_meta(pblk, line, chunk_meta, i);
+
+ trace_pblk_line_state(pblk_disk_name(pblk), line->id,
+ line->state);
}
if (!nr_free_chks) {
@@ -1090,9 +1048,11 @@
goto fail_free_lines;
}
- pblk_set_provision(pblk, nr_free_chks);
+ ret = pblk_set_provision(pblk, nr_free_chks);
+ if (ret)
+ goto fail_free_lines;
- kfree(chunk_meta);
+ vfree(chunk_meta);
return 0;
fail_free_lines:
@@ -1100,7 +1060,7 @@
pblk_line_meta_free(l_mg, &pblk->lines[i]);
kfree(pblk->lines);
fail_free_chunk_meta:
- kfree(chunk_meta);
+ vfree(chunk_meta);
fail_free_luns:
kfree(pblk->luns);
fail_free_meta:
@@ -1169,7 +1129,6 @@
{
struct pblk *pblk = private;
- down_write(&pblk_lock);
pblk_gc_exit(pblk, graceful);
pblk_tear_down(pblk, graceful);
@@ -1178,7 +1137,6 @@
#endif
pblk_free(pblk);
- up_write(&pblk_lock);
}
static sector_t pblk_capacity(void *private)
@@ -1204,6 +1162,7 @@
pblk->dev = dev;
pblk->disk = tdisk;
pblk->state = PBLK_STATE_RUNNING;
+ trace_pblk_state(pblk_disk_name(pblk), pblk->state);
pblk->gc.gc_enabled = 0;
if (!(geo->version == NVM_OCSSD_SPEC_12 ||
@@ -1214,9 +1173,8 @@
return ERR_PTR(-EINVAL);
}
- if (geo->version == NVM_OCSSD_SPEC_12 && geo->dom & NVM_RSP_L2P) {
- pblk_err(pblk, "host-side L2P table not supported. (%x)\n",
- geo->dom);
+ if (geo->ext) {
+ pblk_err(pblk, "extended metadata not supported\n");
kfree(pblk);
return ERR_PTR(-EINVAL);
}
@@ -1298,7 +1256,7 @@
pblk_info(pblk, "luns:%u, lines:%d, secs:%llu, buf entries:%u\n",
geo->all_luns, pblk->l_mg.nr_lines,
- (unsigned long long)pblk->rl.nr_secs,
+ (unsigned long long)pblk->capacity,
pblk->rwb.nr_entries);
wake_up_process(pblk->writer_ts);
@@ -1328,7 +1286,7 @@
.name = "pblk",
.version = {1, 0, 0},
- .make_rq = pblk_make_rq,
+ .bops = &pblk_bops,
.capacity = pblk_capacity,
.init = pblk_init,
--
Gitblit v1.6.2