.. | .. |
---|
| 1 | +// SPDX-License-Identifier: GPL-2.0 |
---|
1 | 2 | /* |
---|
2 | 3 | * Copyright (C) 2016 CNEX Labs |
---|
3 | 4 | * Initial release: Javier Gonzalez <javier@cnexlabs.com> |
---|
.. | .. |
---|
21 | 22 | static int pblk_map_page_data(struct pblk *pblk, unsigned int sentry, |
---|
22 | 23 | struct ppa_addr *ppa_list, |
---|
23 | 24 | unsigned long *lun_bitmap, |
---|
24 | | - struct pblk_sec_meta *meta_list, |
---|
| 25 | + void *meta_list, |
---|
25 | 26 | unsigned int valid_secs) |
---|
26 | 27 | { |
---|
27 | 28 | struct pblk_line *line = pblk_line_get_data(pblk); |
---|
.. | .. |
---|
32 | 33 | int nr_secs = pblk->min_write_pgs; |
---|
33 | 34 | int i; |
---|
34 | 35 | |
---|
| 36 | + if (!line) |
---|
| 37 | + return -ENOSPC; |
---|
| 38 | + |
---|
35 | 39 | if (pblk_line_is_full(line)) { |
---|
36 | 40 | struct pblk_line *prev_line = line; |
---|
37 | 41 | |
---|
.. | .. |
---|
41 | 45 | line = pblk_line_replace_data(pblk); |
---|
42 | 46 | pblk_line_close_meta(pblk, prev_line); |
---|
43 | 47 | |
---|
44 | | - if (!line) |
---|
45 | | - return -EINTR; |
---|
| 48 | + if (!line) { |
---|
| 49 | + pblk_pipeline_stop(pblk); |
---|
| 50 | + return -ENOSPC; |
---|
| 51 | + } |
---|
| 52 | + |
---|
46 | 53 | } |
---|
47 | 54 | |
---|
48 | 55 | emeta = line->emeta; |
---|
.. | .. |
---|
51 | 58 | paddr = pblk_alloc_page(pblk, line, nr_secs); |
---|
52 | 59 | |
---|
53 | 60 | for (i = 0; i < nr_secs; i++, paddr++) { |
---|
| 61 | + struct pblk_sec_meta *meta = pblk_get_meta(pblk, meta_list, i); |
---|
54 | 62 | __le64 addr_empty = cpu_to_le64(ADDR_EMPTY); |
---|
55 | 63 | |
---|
56 | 64 | /* ppa to be sent to the device */ |
---|
.. | .. |
---|
65 | 73 | */ |
---|
66 | 74 | if (i < valid_secs) { |
---|
67 | 75 | kref_get(&line->ref); |
---|
| 76 | + atomic_inc(&line->sec_to_update); |
---|
68 | 77 | w_ctx = pblk_rb_w_ctx(&pblk->rwb, sentry + i); |
---|
69 | 78 | w_ctx->ppa = ppa_list[i]; |
---|
70 | | - meta_list[i].lba = cpu_to_le64(w_ctx->lba); |
---|
| 79 | + meta->lba = cpu_to_le64(w_ctx->lba); |
---|
71 | 80 | lba_list[paddr] = cpu_to_le64(w_ctx->lba); |
---|
72 | 81 | if (lba_list[paddr] != addr_empty) |
---|
73 | 82 | line->nr_valid_lbas++; |
---|
74 | 83 | else |
---|
75 | 84 | atomic64_inc(&pblk->pad_wa); |
---|
76 | 85 | } else { |
---|
77 | | - lba_list[paddr] = meta_list[i].lba = addr_empty; |
---|
| 86 | + lba_list[paddr] = addr_empty; |
---|
| 87 | + meta->lba = addr_empty; |
---|
78 | 88 | __pblk_map_invalidate(pblk, line, paddr); |
---|
79 | 89 | } |
---|
80 | 90 | } |
---|
81 | 91 | |
---|
82 | | - pblk_down_rq(pblk, ppa_list, nr_secs, lun_bitmap); |
---|
| 92 | + pblk_down_rq(pblk, ppa_list[0], lun_bitmap); |
---|
83 | 93 | return 0; |
---|
84 | 94 | } |
---|
85 | 95 | |
---|
86 | | -void pblk_map_rq(struct pblk *pblk, struct nvm_rq *rqd, unsigned int sentry, |
---|
| 96 | +int pblk_map_rq(struct pblk *pblk, struct nvm_rq *rqd, unsigned int sentry, |
---|
87 | 97 | unsigned long *lun_bitmap, unsigned int valid_secs, |
---|
88 | 98 | unsigned int off) |
---|
89 | 99 | { |
---|
90 | | - struct pblk_sec_meta *meta_list = rqd->meta_list; |
---|
| 100 | + void *meta_list = pblk_get_meta_for_writes(pblk, rqd); |
---|
| 101 | + void *meta_buffer; |
---|
| 102 | + struct ppa_addr *ppa_list = nvm_rq_to_ppa_list(rqd); |
---|
91 | 103 | unsigned int map_secs; |
---|
92 | 104 | int min = pblk->min_write_pgs; |
---|
93 | 105 | int i; |
---|
| 106 | + int ret; |
---|
94 | 107 | |
---|
95 | 108 | for (i = off; i < rqd->nr_ppas; i += min) { |
---|
96 | 109 | map_secs = (i + min > valid_secs) ? (valid_secs % min) : min; |
---|
97 | | - if (pblk_map_page_data(pblk, sentry + i, &rqd->ppa_list[i], |
---|
98 | | - lun_bitmap, &meta_list[i], map_secs)) { |
---|
99 | | - bio_put(rqd->bio); |
---|
100 | | - pblk_free_rqd(pblk, rqd, PBLK_WRITE); |
---|
101 | | - pblk_pipeline_stop(pblk); |
---|
102 | | - } |
---|
| 110 | + meta_buffer = pblk_get_meta(pblk, meta_list, i); |
---|
| 111 | + |
---|
| 112 | + ret = pblk_map_page_data(pblk, sentry + i, &ppa_list[i], |
---|
| 113 | + lun_bitmap, meta_buffer, map_secs); |
---|
| 114 | + if (ret) |
---|
| 115 | + return ret; |
---|
103 | 116 | } |
---|
| 117 | + |
---|
| 118 | + return 0; |
---|
104 | 119 | } |
---|
105 | 120 | |
---|
106 | 121 | /* only if erase_ppa is set, acquire erase semaphore */ |
---|
107 | | -void pblk_map_erase_rq(struct pblk *pblk, struct nvm_rq *rqd, |
---|
| 122 | +int pblk_map_erase_rq(struct pblk *pblk, struct nvm_rq *rqd, |
---|
108 | 123 | unsigned int sentry, unsigned long *lun_bitmap, |
---|
109 | 124 | unsigned int valid_secs, struct ppa_addr *erase_ppa) |
---|
110 | 125 | { |
---|
111 | 126 | struct nvm_tgt_dev *dev = pblk->dev; |
---|
112 | 127 | struct nvm_geo *geo = &dev->geo; |
---|
113 | 128 | struct pblk_line_meta *lm = &pblk->lm; |
---|
114 | | - struct pblk_sec_meta *meta_list = rqd->meta_list; |
---|
| 129 | + void *meta_list = pblk_get_meta_for_writes(pblk, rqd); |
---|
| 130 | + void *meta_buffer; |
---|
| 131 | + struct ppa_addr *ppa_list = nvm_rq_to_ppa_list(rqd); |
---|
115 | 132 | struct pblk_line *e_line, *d_line; |
---|
116 | 133 | unsigned int map_secs; |
---|
117 | 134 | int min = pblk->min_write_pgs; |
---|
118 | 135 | int i, erase_lun; |
---|
| 136 | + int ret; |
---|
| 137 | + |
---|
119 | 138 | |
---|
120 | 139 | for (i = 0; i < rqd->nr_ppas; i += min) { |
---|
121 | 140 | map_secs = (i + min > valid_secs) ? (valid_secs % min) : min; |
---|
122 | | - if (pblk_map_page_data(pblk, sentry + i, &rqd->ppa_list[i], |
---|
123 | | - lun_bitmap, &meta_list[i], map_secs)) { |
---|
124 | | - bio_put(rqd->bio); |
---|
125 | | - pblk_free_rqd(pblk, rqd, PBLK_WRITE); |
---|
126 | | - pblk_pipeline_stop(pblk); |
---|
127 | | - } |
---|
| 141 | + meta_buffer = pblk_get_meta(pblk, meta_list, i); |
---|
128 | 142 | |
---|
129 | | - erase_lun = pblk_ppa_to_pos(geo, rqd->ppa_list[i]); |
---|
| 143 | + ret = pblk_map_page_data(pblk, sentry + i, &ppa_list[i], |
---|
| 144 | + lun_bitmap, meta_buffer, map_secs); |
---|
| 145 | + if (ret) |
---|
| 146 | + return ret; |
---|
| 147 | + |
---|
| 148 | + erase_lun = pblk_ppa_to_pos(geo, ppa_list[i]); |
---|
130 | 149 | |
---|
131 | 150 | /* line can change after page map. We might also be writing the |
---|
132 | 151 | * last line. |
---|
.. | .. |
---|
141 | 160 | set_bit(erase_lun, e_line->erase_bitmap); |
---|
142 | 161 | atomic_dec(&e_line->left_eblks); |
---|
143 | 162 | |
---|
144 | | - *erase_ppa = rqd->ppa_list[i]; |
---|
| 163 | + *erase_ppa = ppa_list[i]; |
---|
145 | 164 | erase_ppa->a.blk = e_line->id; |
---|
| 165 | + erase_ppa->a.reserved = 0; |
---|
146 | 166 | |
---|
147 | 167 | spin_unlock(&e_line->lock); |
---|
148 | 168 | |
---|
.. | .. |
---|
160 | 180 | */ |
---|
161 | 181 | e_line = pblk_line_get_erase(pblk); |
---|
162 | 182 | if (!e_line) |
---|
163 | | - return; |
---|
| 183 | + return -ENOSPC; |
---|
164 | 184 | |
---|
165 | 185 | /* Erase blocks that are bad in this line but might not be in next */ |
---|
166 | 186 | if (unlikely(pblk_ppa_empty(*erase_ppa)) && |
---|
.. | .. |
---|
171 | 191 | bit = find_next_bit(d_line->blk_bitmap, |
---|
172 | 192 | lm->blk_per_line, bit + 1); |
---|
173 | 193 | if (bit >= lm->blk_per_line) |
---|
174 | | - return; |
---|
| 194 | + return 0; |
---|
175 | 195 | |
---|
176 | 196 | spin_lock(&e_line->lock); |
---|
177 | 197 | if (test_bit(bit, e_line->erase_bitmap)) { |
---|
.. | .. |
---|
185 | 205 | *erase_ppa = pblk->luns[bit].bppa; /* set ch and lun */ |
---|
186 | 206 | erase_ppa->a.blk = e_line->id; |
---|
187 | 207 | } |
---|
| 208 | + |
---|
| 209 | + return 0; |
---|
188 | 210 | } |
---|