| .. | .. |
|---|
| 1 | +// SPDX-License-Identifier: GPL-2.0-or-later |
|---|
| 1 | 2 | /* |
|---|
| 2 | 3 | drbd_req.c |
|---|
| 3 | 4 | |
|---|
| .. | .. |
|---|
| 7 | 8 | Copyright (C) 1999-2008, Philipp Reisner <philipp.reisner@linbit.com>. |
|---|
| 8 | 9 | Copyright (C) 2002-2008, Lars Ellenberg <lars.ellenberg@linbit.com>. |
|---|
| 9 | 10 | |
|---|
| 10 | | - drbd is free software; you can redistribute it and/or modify |
|---|
| 11 | | - it under the terms of the GNU General Public License as published by |
|---|
| 12 | | - the Free Software Foundation; either version 2, or (at your option) |
|---|
| 13 | | - any later version. |
|---|
| 14 | | - |
|---|
| 15 | | - drbd is distributed in the hope that it will be useful, |
|---|
| 16 | | - but WITHOUT ANY WARRANTY; without even the implied warranty of |
|---|
| 17 | | - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
|---|
| 18 | | - GNU General Public License for more details. |
|---|
| 19 | | - |
|---|
| 20 | | - You should have received a copy of the GNU General Public License |
|---|
| 21 | | - along with drbd; see the file COPYING. If not, write to |
|---|
| 22 | | - the Free Software Foundation, 675 Mass Ave, Cambridge, MA 02139, USA. |
|---|
| 23 | 11 | |
|---|
| 24 | 12 | */ |
|---|
| 25 | 13 | |
|---|
| .. | .. |
|---|
| 33 | 21 | |
|---|
| 34 | 22 | static bool drbd_may_do_local_read(struct drbd_device *device, sector_t sector, int size); |
|---|
| 35 | 23 | |
|---|
| 36 | | -/* Update disk stats at start of I/O request */ |
|---|
| 37 | | -static void _drbd_start_io_acct(struct drbd_device *device, struct drbd_request *req) |
|---|
| 38 | | -{ |
|---|
| 39 | | - struct request_queue *q = device->rq_queue; |
|---|
| 40 | | - |
|---|
| 41 | | - generic_start_io_acct(q, bio_op(req->master_bio), |
|---|
| 42 | | - req->i.size >> 9, &device->vdisk->part0); |
|---|
| 43 | | -} |
|---|
| 44 | | - |
|---|
| 45 | | -/* Update disk stats when completing request upwards */ |
|---|
| 46 | | -static void _drbd_end_io_acct(struct drbd_device *device, struct drbd_request *req) |
|---|
| 47 | | -{ |
|---|
| 48 | | - struct request_queue *q = device->rq_queue; |
|---|
| 49 | | - |
|---|
| 50 | | - generic_end_io_acct(q, bio_op(req->master_bio), |
|---|
| 51 | | - &device->vdisk->part0, req->start_jif); |
|---|
| 52 | | -} |
|---|
| 53 | | - |
|---|
| 54 | 24 | static struct drbd_request *drbd_req_new(struct drbd_device *device, struct bio *bio_src) |
|---|
| 55 | 25 | { |
|---|
| 56 | 26 | struct drbd_request *req; |
|---|
| .. | .. |
|---|
| 63 | 33 | drbd_req_make_private_bio(req, bio_src); |
|---|
| 64 | 34 | req->rq_state = (bio_data_dir(bio_src) == WRITE ? RQ_WRITE : 0) |
|---|
| 65 | 35 | | (bio_op(bio_src) == REQ_OP_WRITE_SAME ? RQ_WSAME : 0) |
|---|
| 66 | | - | (bio_op(bio_src) == REQ_OP_WRITE_ZEROES ? RQ_UNMAP : 0) |
|---|
| 36 | + | (bio_op(bio_src) == REQ_OP_WRITE_ZEROES ? RQ_ZEROES : 0) |
|---|
| 67 | 37 | | (bio_op(bio_src) == REQ_OP_DISCARD ? RQ_UNMAP : 0); |
|---|
| 68 | 38 | req->device = device; |
|---|
| 69 | 39 | req->master_bio = bio_src; |
|---|
| .. | .. |
|---|
| 207 | 177 | void complete_master_bio(struct drbd_device *device, |
|---|
| 208 | 178 | struct bio_and_error *m) |
|---|
| 209 | 179 | { |
|---|
| 210 | | - m->bio->bi_status = errno_to_blk_status(m->error); |
|---|
| 180 | + if (unlikely(m->error)) |
|---|
| 181 | + m->bio->bi_status = errno_to_blk_status(m->error); |
|---|
| 211 | 182 | bio_endio(m->bio); |
|---|
| 212 | 183 | dec_ap_bio(device); |
|---|
| 213 | 184 | } |
|---|
| .. | .. |
|---|
| 275 | 246 | start_new_tl_epoch(first_peer_device(device)->connection); |
|---|
| 276 | 247 | |
|---|
| 277 | 248 | /* Update disk stats */ |
|---|
| 278 | | - _drbd_end_io_acct(device, req); |
|---|
| 249 | + bio_end_io_acct(req->master_bio, req->start_jif); |
|---|
| 279 | 250 | |
|---|
| 280 | 251 | /* If READ failed, |
|---|
| 281 | 252 | * have it be pushed back to the retry work queue, |
|---|
| .. | .. |
|---|
| 641 | 612 | drbd_set_out_of_sync(device, req->i.sector, req->i.size); |
|---|
| 642 | 613 | drbd_report_io_error(device, req); |
|---|
| 643 | 614 | __drbd_chk_io_error(device, DRBD_READ_ERROR); |
|---|
| 644 | | - /* fall through. */ |
|---|
| 615 | + fallthrough; |
|---|
| 645 | 616 | case READ_AHEAD_COMPLETED_WITH_ERROR: |
|---|
| 646 | 617 | /* it is legal to fail read-ahead, no __drbd_chk_io_error in that case. */ |
|---|
| 647 | 618 | mod_rq_state(req, m, RQ_LOCAL_PENDING, RQ_LOCAL_COMPLETED); |
|---|
| .. | .. |
|---|
| 650 | 621 | case DISCARD_COMPLETED_NOTSUPP: |
|---|
| 651 | 622 | case DISCARD_COMPLETED_WITH_ERROR: |
|---|
| 652 | 623 | /* I'd rather not detach from local disk just because it |
|---|
| 653 | | - * failed a REQ_DISCARD. */ |
|---|
| 624 | + * failed a REQ_OP_DISCARD. */ |
|---|
| 654 | 625 | mod_rq_state(req, m, RQ_LOCAL_PENDING, RQ_LOCAL_COMPLETED); |
|---|
| 655 | 626 | break; |
|---|
| 656 | 627 | |
|---|
| .. | .. |
|---|
| 866 | 837 | } /* else: FIXME can this happen? */ |
|---|
| 867 | 838 | break; |
|---|
| 868 | 839 | } |
|---|
| 869 | | - /* else, fall through to BARRIER_ACKED */ |
|---|
| 840 | + fallthrough; /* to BARRIER_ACKED */ |
|---|
| 870 | 841 | |
|---|
| 871 | 842 | case BARRIER_ACKED: |
|---|
| 872 | 843 | /* barrier ack for READ requests does not make sense */ |
|---|
| .. | .. |
|---|
| 896 | 867 | start_new_tl_epoch(connection); |
|---|
| 897 | 868 | mod_rq_state(req, m, 0, RQ_NET_OK|RQ_NET_DONE); |
|---|
| 898 | 869 | break; |
|---|
| 899 | | - }; |
|---|
| 870 | + } |
|---|
| 900 | 871 | |
|---|
| 901 | 872 | return rv; |
|---|
| 902 | 873 | } |
|---|
| .. | .. |
|---|
| 918 | 889 | if (device->state.disk != D_INCONSISTENT) |
|---|
| 919 | 890 | return false; |
|---|
| 920 | 891 | esector = sector + (size >> 9) - 1; |
|---|
| 921 | | - nr_sectors = drbd_get_capacity(device->this_bdev); |
|---|
| 892 | + nr_sectors = get_capacity(device->vdisk); |
|---|
| 922 | 893 | D_ASSERT(device, sector < nr_sectors); |
|---|
| 923 | 894 | D_ASSERT(device, esector < nr_sectors); |
|---|
| 924 | 895 | |
|---|
| .. | .. |
|---|
| 1155 | 1126 | return remote; |
|---|
| 1156 | 1127 | } |
|---|
| 1157 | 1128 | |
|---|
| 1158 | | -static void drbd_process_discard_req(struct drbd_request *req) |
|---|
| 1129 | +static void drbd_process_discard_or_zeroes_req(struct drbd_request *req, int flags) |
|---|
| 1159 | 1130 | { |
|---|
| 1160 | | - struct block_device *bdev = req->device->ldev->backing_bdev; |
|---|
| 1161 | | - |
|---|
| 1162 | | - if (blkdev_issue_zeroout(bdev, req->i.sector, req->i.size >> 9, |
|---|
| 1163 | | - GFP_NOIO, 0)) |
|---|
| 1131 | + int err = drbd_issue_discard_or_zero_out(req->device, |
|---|
| 1132 | + req->i.sector, req->i.size >> 9, flags); |
|---|
| 1133 | + if (err) |
|---|
| 1164 | 1134 | req->private_bio->bi_status = BLK_STS_IOERR; |
|---|
| 1165 | 1135 | bio_endio(req->private_bio); |
|---|
| 1166 | 1136 | } |
|---|
| .. | .. |
|---|
| 1189 | 1159 | if (get_ldev(device)) { |
|---|
| 1190 | 1160 | if (drbd_insert_fault(device, type)) |
|---|
| 1191 | 1161 | bio_io_error(bio); |
|---|
| 1192 | | - else if (bio_op(bio) == REQ_OP_WRITE_ZEROES || |
|---|
| 1193 | | - bio_op(bio) == REQ_OP_DISCARD) |
|---|
| 1194 | | - drbd_process_discard_req(req); |
|---|
| 1162 | + else if (bio_op(bio) == REQ_OP_WRITE_ZEROES) |
|---|
| 1163 | + drbd_process_discard_or_zeroes_req(req, EE_ZEROOUT | |
|---|
| 1164 | + ((bio->bi_opf & REQ_NOUNMAP) ? 0 : EE_TRIM)); |
|---|
| 1165 | + else if (bio_op(bio) == REQ_OP_DISCARD) |
|---|
| 1166 | + drbd_process_discard_or_zeroes_req(req, EE_TRIM); |
|---|
| 1195 | 1167 | else |
|---|
| 1196 | | - generic_make_request(bio); |
|---|
| 1168 | + submit_bio_noacct(bio); |
|---|
| 1197 | 1169 | put_ldev(device); |
|---|
| 1198 | 1170 | } else |
|---|
| 1199 | 1171 | bio_io_error(bio); |
|---|
| .. | .. |
|---|
| 1233 | 1205 | bio_endio(bio); |
|---|
| 1234 | 1206 | return ERR_PTR(-ENOMEM); |
|---|
| 1235 | 1207 | } |
|---|
| 1236 | | - req->start_jif = start_jif; |
|---|
| 1208 | + |
|---|
| 1209 | + /* Update disk stats */ |
|---|
| 1210 | + req->start_jif = bio_start_io_acct(req->master_bio); |
|---|
| 1237 | 1211 | |
|---|
| 1238 | 1212 | if (!get_ldev(device)) { |
|---|
| 1239 | 1213 | bio_put(req->private_bio); |
|---|
| 1240 | 1214 | req->private_bio = NULL; |
|---|
| 1241 | 1215 | } |
|---|
| 1242 | | - |
|---|
| 1243 | | - /* Update disk stats */ |
|---|
| 1244 | | - _drbd_start_io_acct(device, req); |
|---|
| 1245 | 1216 | |
|---|
| 1246 | 1217 | /* process discards always from our submitter thread */ |
|---|
| 1247 | 1218 | if (bio_op(bio) == REQ_OP_WRITE_ZEROES || |
|---|
| .. | .. |
|---|
| 1623 | 1594 | } |
|---|
| 1624 | 1595 | } |
|---|
| 1625 | 1596 | |
|---|
| 1626 | | -blk_qc_t drbd_make_request(struct request_queue *q, struct bio *bio) |
|---|
| 1597 | +blk_qc_t drbd_submit_bio(struct bio *bio) |
|---|
| 1627 | 1598 | { |
|---|
| 1628 | | - struct drbd_device *device = (struct drbd_device *) q->queuedata; |
|---|
| 1599 | + struct drbd_device *device = bio->bi_disk->private_data; |
|---|
| 1629 | 1600 | unsigned long start_jif; |
|---|
| 1630 | 1601 | |
|---|
| 1631 | | - blk_queue_split(q, &bio); |
|---|
| 1602 | + blk_queue_split(&bio); |
|---|
| 1632 | 1603 | |
|---|
| 1633 | 1604 | start_jif = jiffies; |
|---|
| 1634 | 1605 | |
|---|