.. | .. |
---|
7 | 7 | */ |
---|
8 | 8 | |
---|
9 | 9 | #include <linux/backing-dev.h> |
---|
| 10 | +#include <linux/dax.h> |
---|
10 | 11 | #include <linux/module.h> |
---|
11 | 12 | #include <linux/of_address.h> |
---|
| 13 | +#include <linux/pfn_t.h> |
---|
12 | 14 | #include <linux/platform_device.h> |
---|
| 15 | +#include <linux/uio.h> |
---|
13 | 16 | |
---|
14 | 17 | #define PAGE_SECTORS_SHIFT (PAGE_SHIFT - SECTOR_SHIFT) |
---|
15 | 18 | #define PAGE_SECTORS (1 << PAGE_SECTORS_SHIFT) |
---|
.. | .. |
---|
21 | 24 | struct device *dev; |
---|
22 | 25 | phys_addr_t mem_addr; |
---|
23 | 26 | size_t mem_size; |
---|
| 27 | + size_t mem_pages; |
---|
| 28 | + void *mem_kaddr; |
---|
| 29 | + struct dax_device *dax_dev; |
---|
24 | 30 | }; |
---|
25 | 31 | |
---|
26 | 32 | static int rd_major; |
---|
.. | .. |
---|
130 | 136 | return 0; |
---|
131 | 137 | } |
---|
132 | 138 | |
---|
133 | | -static blk_qc_t rd_make_request(struct request_queue *q, struct bio *bio) |
---|
| 139 | +static blk_qc_t rd_submit_bio(struct bio *bio) |
---|
134 | 140 | { |
---|
135 | 141 | struct rd_device *rd = bio->bi_disk->private_data; |
---|
136 | 142 | struct bio_vec bvec; |
---|
.. | .. |
---|
144 | 150 | bio_for_each_segment(bvec, bio, iter) { |
---|
145 | 151 | unsigned int len = bvec.bv_len; |
---|
146 | 152 | int err; |
---|
| 153 | + |
---|
| 154 | + /* Don't support un-aligned buffer */ |
---|
| 155 | + WARN_ON_ONCE((bvec.bv_offset & (SECTOR_SIZE - 1)) || |
---|
| 156 | + (len & (SECTOR_SIZE - 1))); |
---|
147 | 157 | |
---|
148 | 158 | err = rd_do_bvec(rd, bvec.bv_page, len, bvec.bv_offset, |
---|
149 | 159 | bio_op(bio), sector); |
---|
.. | .. |
---|
174 | 184 | |
---|
175 | 185 | static const struct block_device_operations rd_fops = { |
---|
176 | 186 | .owner = THIS_MODULE, |
---|
| 187 | + .submit_bio = rd_submit_bio, |
---|
177 | 188 | .rw_page = rd_rw_page, |
---|
| 189 | +}; |
---|
| 190 | + |
---|
| 191 | +static long rd_dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff, |
---|
| 192 | + long nr_pages, void **kaddr, pfn_t *pfn) |
---|
| 193 | +{ |
---|
| 194 | + struct rd_device *rd = dax_get_private(dax_dev); |
---|
| 195 | + |
---|
| 196 | + phys_addr_t offset = PFN_PHYS(pgoff); |
---|
| 197 | + size_t max_nr_pages = rd->mem_pages - pgoff; |
---|
| 198 | + |
---|
| 199 | + if (kaddr) |
---|
| 200 | + *kaddr = rd->mem_kaddr + offset; |
---|
| 201 | + if (pfn) |
---|
| 202 | + *pfn = phys_to_pfn_t(rd->mem_addr + offset, PFN_DEV | PFN_MAP); |
---|
| 203 | + |
---|
| 204 | + return nr_pages > max_nr_pages ? max_nr_pages : nr_pages; |
---|
| 205 | +} |
---|
| 206 | + |
---|
| 207 | +static bool rd_dax_supported(struct dax_device *dax_dev, |
---|
| 208 | + struct block_device *bdev, int blocksize, |
---|
| 209 | + sector_t start, sector_t sectors) |
---|
| 210 | +{ |
---|
| 211 | + return true; |
---|
| 212 | +} |
---|
| 213 | + |
---|
| 214 | +static size_t rd_dax_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff, |
---|
| 215 | + void *addr, size_t bytes, struct iov_iter *i) |
---|
| 216 | +{ |
---|
| 217 | + return copy_from_iter(addr, bytes, i); |
---|
| 218 | +} |
---|
| 219 | + |
---|
| 220 | +static size_t rd_dax_copy_to_iter(struct dax_device *dax_dev, pgoff_t pgoff, |
---|
| 221 | + void *addr, size_t bytes, struct iov_iter *i) |
---|
| 222 | +{ |
---|
| 223 | + return copy_to_iter(addr, bytes, i); |
---|
| 224 | +} |
---|
| 225 | + |
---|
| 226 | +static int rd_dax_zero_page_range(struct dax_device *dax_dev, pgoff_t pgoff, size_t nr_pages) |
---|
| 227 | +{ |
---|
| 228 | + long rc; |
---|
| 229 | + void *kaddr; |
---|
| 230 | + |
---|
| 231 | + rc = dax_direct_access(dax_dev, pgoff, nr_pages, &kaddr, NULL); |
---|
| 232 | + if (rc < 0) |
---|
| 233 | + return rc; |
---|
| 234 | + memset(kaddr, 0, nr_pages << PAGE_SHIFT); |
---|
| 235 | + |
---|
| 236 | + return 0; |
---|
| 237 | +} |
---|
| 238 | + |
---|
| 239 | +static const struct dax_operations rd_dax_ops = { |
---|
| 240 | + .direct_access = rd_dax_direct_access, |
---|
| 241 | + .dax_supported = rd_dax_supported, |
---|
| 242 | + .copy_from_iter = rd_dax_copy_from_iter, |
---|
| 243 | + .copy_to_iter = rd_dax_copy_to_iter, |
---|
| 244 | + .zero_page_range = rd_dax_zero_page_range, |
---|
178 | 245 | }; |
---|
179 | 246 | |
---|
180 | 247 | static int rd_init(struct rd_device *rd, int major, int minor) |
---|
181 | 248 | { |
---|
| 249 | + int ret; |
---|
182 | 250 | struct gendisk *disk; |
---|
183 | 251 | |
---|
184 | | - rd->rd_queue = blk_alloc_queue(GFP_KERNEL); |
---|
| 252 | + rd->rd_queue = blk_alloc_queue(NUMA_NO_NODE); |
---|
185 | 253 | if (!rd->rd_queue) |
---|
186 | 254 | return -ENOMEM; |
---|
187 | | - |
---|
188 | | - blk_queue_make_request(rd->rd_queue, rd_make_request); |
---|
189 | | - blk_queue_max_hw_sectors(rd->rd_queue, 1024); |
---|
190 | 255 | |
---|
191 | 256 | /* This is so fdisk will align partitions on 4k, because of |
---|
192 | 257 | * direct_access API needing 4k alignment, returning a PFN |
---|
.. | .. |
---|
196 | 261 | */ |
---|
197 | 262 | blk_queue_physical_block_size(rd->rd_queue, PAGE_SIZE); |
---|
198 | 263 | disk = alloc_disk(1); |
---|
199 | | - if (!disk) |
---|
| 264 | + if (!disk) { |
---|
| 265 | + ret = -ENOMEM; |
---|
200 | 266 | goto out_free_queue; |
---|
| 267 | + } |
---|
201 | 268 | disk->major = major; |
---|
202 | 269 | disk->first_minor = 0; |
---|
203 | 270 | disk->fops = &rd_fops; |
---|
.. | .. |
---|
206 | 273 | sprintf(disk->disk_name, "rd%d", minor); |
---|
207 | 274 | set_capacity(disk, rd->mem_size >> SECTOR_SHIFT); |
---|
208 | 275 | rd->rd_disk = disk; |
---|
209 | | - rd->rd_queue->backing_dev_info->capabilities |= BDI_CAP_SYNCHRONOUS_IO; |
---|
| 276 | + |
---|
| 277 | + rd->mem_kaddr = phys_to_virt(rd->mem_addr); |
---|
| 278 | + rd->mem_pages = PHYS_PFN(rd->mem_size); |
---|
| 279 | + rd->dax_dev = alloc_dax(rd, disk->disk_name, &rd_dax_ops, DAXDEV_F_SYNC); |
---|
| 280 | + if (IS_ERR(rd->dax_dev)) { |
---|
| 281 | + ret = PTR_ERR(rd->dax_dev); |
---|
| 282 | + dev_err(rd->dev, "alloc_dax failed %d\n", ret); |
---|
| 283 | + rd->dax_dev = NULL; |
---|
| 284 | + goto out_free_queue; |
---|
| 285 | + } |
---|
210 | 286 | |
---|
211 | 287 | /* Tell the block layer that this is not a rotational device */ |
---|
212 | 288 | blk_queue_flag_set(QUEUE_FLAG_NONROT, rd->rd_queue); |
---|
213 | 289 | blk_queue_flag_clear(QUEUE_FLAG_ADD_RANDOM, rd->rd_queue); |
---|
| 290 | + if (rd->dax_dev) |
---|
| 291 | + blk_queue_flag_set(QUEUE_FLAG_DAX, rd->rd_queue); |
---|
214 | 292 | |
---|
215 | 293 | rd->rd_disk->queue = rd->rd_queue; |
---|
216 | 294 | add_disk(rd->rd_disk); |
---|
.. | .. |
---|
219 | 297 | |
---|
220 | 298 | out_free_queue: |
---|
221 | 299 | blk_cleanup_queue(rd->rd_queue); |
---|
222 | | - return -ENOMEM; |
---|
| 300 | + return ret; |
---|
223 | 301 | } |
---|
224 | 302 | |
---|
225 | 303 | static int rd_probe(struct platform_device *pdev) |
---|
.. | .. |
---|
252 | 330 | rd->mem_size = resource_size(®); |
---|
253 | 331 | |
---|
254 | 332 | ret = rd_init(rd, rd_major, 0); |
---|
| 333 | + dev_info(dev, "0x%zx@%pa -> 0x%px dax:%d ret:%d\n", |
---|
| 334 | + rd->mem_size, &rd->mem_addr, rd->mem_kaddr, (bool)rd->dax_dev, ret); |
---|
255 | 335 | |
---|
256 | 336 | return ret; |
---|
257 | 337 | } |
---|