| .. | .. |
|---|
| 24 | 24 | ELEVATOR_DISCARD_MERGE = 3, |
|---|
| 25 | 25 | }; |
|---|
| 26 | 26 | |
|---|
| 27 | | -typedef enum elv_merge (elevator_merge_fn) (struct request_queue *, struct request **, |
|---|
| 28 | | - struct bio *); |
|---|
| 29 | | - |
|---|
| 30 | | -typedef void (elevator_merge_req_fn) (struct request_queue *, struct request *, struct request *); |
|---|
| 31 | | - |
|---|
| 32 | | -typedef void (elevator_merged_fn) (struct request_queue *, struct request *, enum elv_merge); |
|---|
| 33 | | - |
|---|
| 34 | | -typedef int (elevator_allow_bio_merge_fn) (struct request_queue *, |
|---|
| 35 | | - struct request *, struct bio *); |
|---|
| 36 | | - |
|---|
| 37 | | -typedef int (elevator_allow_rq_merge_fn) (struct request_queue *, |
|---|
| 38 | | - struct request *, struct request *); |
|---|
| 39 | | - |
|---|
| 40 | | -typedef void (elevator_bio_merged_fn) (struct request_queue *, |
|---|
| 41 | | - struct request *, struct bio *); |
|---|
| 42 | | - |
|---|
| 43 | | -typedef int (elevator_dispatch_fn) (struct request_queue *, int); |
|---|
| 44 | | - |
|---|
| 45 | | -typedef void (elevator_add_req_fn) (struct request_queue *, struct request *); |
|---|
| 46 | | -typedef struct request *(elevator_request_list_fn) (struct request_queue *, struct request *); |
|---|
| 47 | | -typedef void (elevator_completed_req_fn) (struct request_queue *, struct request *); |
|---|
| 48 | | -typedef int (elevator_may_queue_fn) (struct request_queue *, unsigned int); |
|---|
| 49 | | - |
|---|
| 50 | | -typedef void (elevator_init_icq_fn) (struct io_cq *); |
|---|
| 51 | | -typedef void (elevator_exit_icq_fn) (struct io_cq *); |
|---|
| 52 | | -typedef int (elevator_set_req_fn) (struct request_queue *, struct request *, |
|---|
| 53 | | - struct bio *, gfp_t); |
|---|
| 54 | | -typedef void (elevator_put_req_fn) (struct request *); |
|---|
| 55 | | -typedef void (elevator_activate_req_fn) (struct request_queue *, struct request *); |
|---|
| 56 | | -typedef void (elevator_deactivate_req_fn) (struct request_queue *, struct request *); |
|---|
| 57 | | - |
|---|
| 58 | | -typedef int (elevator_init_fn) (struct request_queue *, |
|---|
| 59 | | - struct elevator_type *e); |
|---|
| 60 | | -typedef void (elevator_exit_fn) (struct elevator_queue *); |
|---|
| 61 | | -typedef void (elevator_registered_fn) (struct request_queue *); |
|---|
| 62 | | - |
|---|
| 63 | | -struct elevator_ops |
|---|
| 64 | | -{ |
|---|
| 65 | | - elevator_merge_fn *elevator_merge_fn; |
|---|
| 66 | | - elevator_merged_fn *elevator_merged_fn; |
|---|
| 67 | | - elevator_merge_req_fn *elevator_merge_req_fn; |
|---|
| 68 | | - elevator_allow_bio_merge_fn *elevator_allow_bio_merge_fn; |
|---|
| 69 | | - elevator_allow_rq_merge_fn *elevator_allow_rq_merge_fn; |
|---|
| 70 | | - elevator_bio_merged_fn *elevator_bio_merged_fn; |
|---|
| 71 | | - |
|---|
| 72 | | - elevator_dispatch_fn *elevator_dispatch_fn; |
|---|
| 73 | | - elevator_add_req_fn *elevator_add_req_fn; |
|---|
| 74 | | - elevator_activate_req_fn *elevator_activate_req_fn; |
|---|
| 75 | | - elevator_deactivate_req_fn *elevator_deactivate_req_fn; |
|---|
| 76 | | - |
|---|
| 77 | | - elevator_completed_req_fn *elevator_completed_req_fn; |
|---|
| 78 | | - |
|---|
| 79 | | - elevator_request_list_fn *elevator_former_req_fn; |
|---|
| 80 | | - elevator_request_list_fn *elevator_latter_req_fn; |
|---|
| 81 | | - |
|---|
| 82 | | - elevator_init_icq_fn *elevator_init_icq_fn; /* see iocontext.h */ |
|---|
| 83 | | - elevator_exit_icq_fn *elevator_exit_icq_fn; /* ditto */ |
|---|
| 84 | | - |
|---|
| 85 | | - elevator_set_req_fn *elevator_set_req_fn; |
|---|
| 86 | | - elevator_put_req_fn *elevator_put_req_fn; |
|---|
| 87 | | - |
|---|
| 88 | | - elevator_may_queue_fn *elevator_may_queue_fn; |
|---|
| 89 | | - |
|---|
| 90 | | - elevator_init_fn *elevator_init_fn; |
|---|
| 91 | | - elevator_exit_fn *elevator_exit_fn; |
|---|
| 92 | | - elevator_registered_fn *elevator_registered_fn; |
|---|
| 93 | | -}; |
|---|
| 94 | | - |
|---|
| 95 | 27 | struct blk_mq_alloc_data; |
|---|
| 96 | 28 | struct blk_mq_hw_ctx; |
|---|
| 97 | 29 | |
|---|
| .. | .. |
|---|
| 103 | 35 | void (*depth_updated)(struct blk_mq_hw_ctx *); |
|---|
| 104 | 36 | |
|---|
| 105 | 37 | bool (*allow_merge)(struct request_queue *, struct request *, struct bio *); |
|---|
| 106 | | - bool (*bio_merge)(struct blk_mq_hw_ctx *, struct bio *); |
|---|
| 38 | + bool (*bio_merge)(struct request_queue *, struct bio *, unsigned int); |
|---|
| 107 | 39 | int (*request_merge)(struct request_queue *q, struct request **, struct bio *); |
|---|
| 108 | 40 | void (*request_merged)(struct request_queue *, struct request *, enum elv_merge); |
|---|
| 109 | 41 | void (*requests_merged)(struct request_queue *, struct request *, struct request *); |
|---|
| 110 | 42 | void (*limit_depth)(unsigned int, struct blk_mq_alloc_data *); |
|---|
| 111 | | - void (*prepare_request)(struct request *, struct bio *bio); |
|---|
| 43 | + void (*prepare_request)(struct request *); |
|---|
| 112 | 44 | void (*finish_request)(struct request *); |
|---|
| 113 | 45 | void (*insert_requests)(struct blk_mq_hw_ctx *, struct list_head *, bool); |
|---|
| 114 | 46 | struct request *(*dispatch_request)(struct blk_mq_hw_ctx *); |
|---|
| 115 | 47 | bool (*has_work)(struct blk_mq_hw_ctx *); |
|---|
| 116 | | - void (*completed_request)(struct request *); |
|---|
| 117 | | - void (*started_request)(struct request *); |
|---|
| 48 | + void (*completed_request)(struct request *, u64); |
|---|
| 118 | 49 | void (*requeue_request)(struct request *); |
|---|
| 119 | 50 | struct request *(*former_request)(struct request_queue *, struct request *); |
|---|
| 120 | 51 | struct request *(*next_request)(struct request_queue *, struct request *); |
|---|
| 121 | 52 | void (*init_icq)(struct io_cq *); |
|---|
| 122 | 53 | void (*exit_icq)(struct io_cq *); |
|---|
| 123 | | - void (*elevator_registered_fn)(struct request_queue *q); |
|---|
| 124 | 54 | |
|---|
| 125 | 55 | ANDROID_KABI_RESERVE(1); |
|---|
| 126 | 56 | ANDROID_KABI_RESERVE(2); |
|---|
| .. | .. |
|---|
| 145 | 75 | struct kmem_cache *icq_cache; |
|---|
| 146 | 76 | |
|---|
| 147 | 77 | /* fields provided by elevator implementation */ |
|---|
| 148 | | - union { |
|---|
| 149 | | - struct elevator_ops sq; |
|---|
| 150 | | - struct elevator_mq_ops mq; |
|---|
| 151 | | - } ops; |
|---|
| 78 | + struct elevator_mq_ops ops; |
|---|
| 79 | + |
|---|
| 152 | 80 | size_t icq_size; /* see iocontext.h */ |
|---|
| 153 | 81 | size_t icq_align; /* ditto */ |
|---|
| 154 | 82 | struct elv_fs_entry *elevator_attrs; |
|---|
| 155 | | - char elevator_name[ELV_NAME_MAX]; |
|---|
| 83 | + const char *elevator_name; |
|---|
| 156 | 84 | const char *elevator_alias; |
|---|
| 85 | + const unsigned int elevator_features; |
|---|
| 157 | 86 | struct module *elevator_owner; |
|---|
| 158 | | - bool uses_mq; |
|---|
| 159 | 87 | #ifdef CONFIG_BLK_DEBUG_FS |
|---|
| 160 | 88 | const struct blk_mq_debugfs_attr *queue_debugfs_attrs; |
|---|
| 161 | 89 | const struct blk_mq_debugfs_attr *hctx_debugfs_attrs; |
|---|
| .. | .. |
|---|
| 186 | 114 | struct kobject kobj; |
|---|
| 187 | 115 | struct mutex sysfs_lock; |
|---|
| 188 | 116 | unsigned int registered:1; |
|---|
| 189 | | - unsigned int uses_mq:1; |
|---|
| 190 | 117 | DECLARE_HASHTABLE(hash, ELV_HASH_BITS); |
|---|
| 191 | 118 | }; |
|---|
| 192 | 119 | |
|---|
| 193 | 120 | /* |
|---|
| 194 | 121 | * block elevator interface |
|---|
| 195 | 122 | */ |
|---|
| 196 | | -extern void elv_dispatch_sort(struct request_queue *, struct request *); |
|---|
| 197 | | -extern void elv_dispatch_add_tail(struct request_queue *, struct request *); |
|---|
| 198 | | -extern void elv_add_request(struct request_queue *, struct request *, int); |
|---|
| 199 | | -extern void __elv_add_request(struct request_queue *, struct request *, int); |
|---|
| 200 | 123 | extern enum elv_merge elv_merge(struct request_queue *, struct request **, |
|---|
| 201 | 124 | struct bio *); |
|---|
| 202 | 125 | extern void elv_merge_requests(struct request_queue *, struct request *, |
|---|
| 203 | 126 | struct request *); |
|---|
| 204 | 127 | extern void elv_merged_request(struct request_queue *, struct request *, |
|---|
| 205 | 128 | enum elv_merge); |
|---|
| 206 | | -extern void elv_bio_merged(struct request_queue *q, struct request *, |
|---|
| 207 | | - struct bio *); |
|---|
| 208 | 129 | extern bool elv_attempt_insert_merge(struct request_queue *, struct request *); |
|---|
| 209 | | -extern void elv_requeue_request(struct request_queue *, struct request *); |
|---|
| 210 | 130 | extern struct request *elv_former_request(struct request_queue *, struct request *); |
|---|
| 211 | 131 | extern struct request *elv_latter_request(struct request_queue *, struct request *); |
|---|
| 212 | | -extern int elv_may_queue(struct request_queue *, unsigned int); |
|---|
| 213 | | -extern void elv_completed_request(struct request_queue *, struct request *); |
|---|
| 214 | | -extern int elv_set_request(struct request_queue *q, struct request *rq, |
|---|
| 215 | | - struct bio *bio, gfp_t gfp_mask); |
|---|
| 216 | | -extern void elv_put_request(struct request_queue *, struct request *); |
|---|
| 217 | | -extern void elv_drain_elevator(struct request_queue *); |
|---|
| 218 | 132 | |
|---|
| 219 | 133 | /* |
|---|
| 220 | 134 | * io scheduler registration |
|---|
| 221 | 135 | */ |
|---|
| 222 | | -extern void __init load_default_elevator_module(void); |
|---|
| 223 | 136 | extern int elv_register(struct elevator_type *); |
|---|
| 224 | 137 | extern void elv_unregister(struct elevator_type *); |
|---|
| 225 | 138 | |
|---|
| .. | .. |
|---|
| 256 | 169 | #define ELEVATOR_INSERT_FLUSH 5 |
|---|
| 257 | 170 | #define ELEVATOR_INSERT_SORT_MERGE 6 |
|---|
| 258 | 171 | |
|---|
| 259 | | -/* |
|---|
| 260 | | - * return values from elevator_may_queue_fn |
|---|
| 261 | | - */ |
|---|
| 262 | | -enum { |
|---|
| 263 | | - ELV_MQUEUE_MAY, |
|---|
| 264 | | - ELV_MQUEUE_NO, |
|---|
| 265 | | - ELV_MQUEUE_MUST, |
|---|
| 266 | | -}; |
|---|
| 267 | | - |
|---|
| 268 | 172 | #define rq_end_sector(rq) (blk_rq_pos(rq) + blk_rq_sectors(rq)) |
|---|
| 269 | 173 | #define rb_entry_rq(node) rb_entry((node), struct request, rb_node) |
|---|
| 270 | 174 | |
|---|
| 271 | 175 | #define rq_entry_fifo(ptr) list_entry((ptr), struct request, queuelist) |
|---|
| 272 | 176 | #define rq_fifo_clear(rq) list_del_init(&(rq)->queuelist) |
|---|
| 273 | 177 | |
|---|
| 274 | | -#else /* CONFIG_BLOCK */ |
|---|
| 178 | +/* |
|---|
| 179 | + * Elevator features. |
|---|
| 180 | + */ |
|---|
| 275 | 181 | |
|---|
| 276 | | -static inline void load_default_elevator_module(void) { } |
|---|
| 182 | +/* Supports zoned block devices sequential write constraint */ |
|---|
| 183 | +#define ELEVATOR_F_ZBD_SEQ_WRITE (1U << 0) |
|---|
| 184 | +/* Supports scheduling on multiple hardware queues */ |
|---|
| 185 | +#define ELEVATOR_F_MQ_AWARE (1U << 1) |
|---|
| 277 | 186 | |
|---|
| 278 | 187 | #endif /* CONFIG_BLOCK */ |
|---|
| 279 | 188 | #endif |
|---|