forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2023-12-09 b22da3d8526a935aa31e086e63f60ff3246cb61c 
 kernel/drivers/nvme/host/nvme.h
....@@ -1,14 +1,6 @@
1
+/* SPDX-License-Identifier: GPL-2.0 */
12 /*
23  * Copyright (c) 2011-2014, Intel Corporation.
3
- *
4
- * This program is free software; you can redistribute it and/or modify it
5
- * under the terms and conditions of the GNU General Public License,
6
- * version 2, as published by the Free Software Foundation.
7
- *
8
- * This program is distributed in the hope it will be useful, but WITHOUT
9
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
11
- * more details.
124  */
135 
146 #ifndef _NVME_H
....@@ -23,6 +15,10 @@
2315 #include <linux/sed-opal.h>
2416 #include <linux/fault-inject.h>
2517 #include <linux/rcupdate.h>
18
+#include <linux/wait.h>
19
+#include <linux/t10-pi.h>
20
+
21
+#include <trace/events/block.h>
2622 
2723 extern unsigned int nvme_io_timeout;
2824 #define NVME_IO_TIMEOUT	(nvme_io_timeout * HZ)
....@@ -32,6 +28,22 @@
3228 
3329 #define NVME_DEFAULT_KATO	5
3430 #define NVME_KATO_GRACE		10
31
+
32
+#ifdef CONFIG_ARCH_NO_SG_CHAIN
33
+#define  NVME_INLINE_SG_CNT  0
34
+#define  NVME_INLINE_METADATA_SG_CNT  0
35
+#else
36
+#define  NVME_INLINE_SG_CNT  2
37
+#define  NVME_INLINE_METADATA_SG_CNT  1
38
+#endif
39
+
40
+/*
41
+ * Default to a 4K page size, with the intention to update this
42
+ * path in the future to accommodate architectures with differing
43
+ * kernel and IO page sizes.
44
+ */
45
+#define NVME_CTRL_PAGE_SHIFT	12
46
+#define NVME_CTRL_PAGE_SIZE	(1 << NVME_CTRL_PAGE_SHIFT)
3547 
3648 extern struct workqueue_struct *nvme_wq;
3749 extern struct workqueue_struct *nvme_reset_wq;
....@@ -90,6 +102,59 @@
90102 	 * Set MEDIUM priority on SQ creation
91103 	 */
92104 	NVME_QUIRK_MEDIUM_PRIO_SQ		= (1 << 7),
105
+
106
+	/*
107
+	 * Ignore device provided subnqn.
108
+	 */
109
+	NVME_QUIRK_IGNORE_DEV_SUBNQN		= (1 << 8),
110
+
111
+	/*
112
+	 * Broken Write Zeroes.
113
+	 */
114
+	NVME_QUIRK_DISABLE_WRITE_ZEROES		= (1 << 9),
115
+
116
+	/*
117
+	 * Force simple suspend/resume path.
118
+	 */
119
+	NVME_QUIRK_SIMPLE_SUSPEND		= (1 << 10),
120
+
121
+	/*
122
+	 * Use only one interrupt vector for all queues
123
+	 */
124
+	NVME_QUIRK_SINGLE_VECTOR		= (1 << 11),
125
+
126
+	/*
127
+	 * Use non-standard 128 bytes SQEs.
128
+	 */
129
+	NVME_QUIRK_128_BYTES_SQES		= (1 << 12),
130
+
131
+	/*
132
+	 * Prevent tag overlap between queues
133
+	 */
134
+	NVME_QUIRK_SHARED_TAGS                  = (1 << 13),
135
+
136
+	/*
137
+	 * Don't change the value of the temperature threshold feature
138
+	 */
139
+	NVME_QUIRK_NO_TEMP_THRESH_CHANGE	= (1 << 14),
140
+
141
+	/*
142
+	 * The controller doesn't handle the Identify Namespace
143
+	 * Identification Descriptor list subcommand despite claiming
144
+	 * NVMe 1.3 compliance.
145
+	 */
146
+	NVME_QUIRK_NO_NS_DESC_LIST		= (1 << 15),
147
+
148
+	/*
149
+	 * The controller requires the command_id value be be limited, so skip
150
+	 * encoding the generation sequence number.
151
+	 */
152
+	NVME_QUIRK_SKIP_CID_GEN			= (1 << 17),
153
+
154
+	/*
155
+	 * Reports garbage in the namespace identifiers (eui64, nguid, uuid).
156
+	 */
157
+	NVME_QUIRK_BOGUS_NID			= (1 << 18),
93158 };
94159 
95160 /*
....@@ -99,6 +164,7 @@
99164 struct nvme_request {
100165 	struct nvme_command	*cmd;
101166 	union nvme_result	result;
167
+	u8			genctr;
102168 	u8			retries;
103169 	u8			flags;
104170 	u16			status;
....@@ -122,7 +188,7 @@
122188 
123189 static inline u16 nvme_req_qid(struct request *req)
124190 {
125
-	if (!req->rq_disk)
191
+	if (!req->q->queuedata)
126192 		return 0;
127193 	return blk_mq_unique_tag_to_hwq(blk_mq_unique_tag(req)) + 1;
128194 }
....@@ -134,17 +200,46 @@
134200  */
135201 #define NVME_QUIRK_DELAY_AMOUNT		2300
136202 
203
+/*
204
+ * enum nvme_ctrl_state: Controller state
205
+ *
206
+ * @NVME_CTRL_NEW:		New controller just allocated, initial state
207
+ * @NVME_CTRL_LIVE:		Controller is connected and I/O capable
208
+ * @NVME_CTRL_RESETTING:	Controller is resetting (or scheduled reset)
209
+ * @NVME_CTRL_CONNECTING:	Controller is disconnected, now connecting the
210
+ *				transport
211
+ * @NVME_CTRL_DELETING:		Controller is deleting (or scheduled deletion)
212
+ * @NVME_CTRL_DELETING_NOIO:	Controller is deleting and I/O is not
213
+ *				disabled/failed immediately. This state comes
214
+ * 				after all async event processing took place and
215
+ * 				before ns removal and the controller deletion
216
+ * 				progress
217
+ * @NVME_CTRL_DEAD:		Controller is non-present/unresponsive during
218
+ *				shutdown or removal. In this case we forcibly
219
+ *				kill all inflight I/O as they have no chance to
220
+ *				complete
221
+ */
137222 enum nvme_ctrl_state {
138223 	NVME_CTRL_NEW,
139224 	NVME_CTRL_LIVE,
140
-	NVME_CTRL_ADMIN_ONLY,    /* Only admin queue live */
141225 	NVME_CTRL_RESETTING,
142226 	NVME_CTRL_CONNECTING,
143227 	NVME_CTRL_DELETING,
228
+	NVME_CTRL_DELETING_NOIO,
144229 	NVME_CTRL_DEAD,
145230 };
146231 
232
+struct nvme_fault_inject {
233
+#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
234
+	struct fault_attr attr;
235
+	struct dentry *parent;
236
+	bool dont_retry;	/* DNR, do not retry */
237
+	u16 status;		/* status code */
238
+#endif
239
+};
240
+
147241 struct nvme_ctrl {
242
+	bool comp_seen;
148243 	enum nvme_ctrl_state state;
149244 	bool identified;
150245 	spinlock_t lock;
....@@ -152,17 +247,23 @@
152247 	const struct nvme_ctrl_ops *ops;
153248 	struct request_queue *admin_q;
154249 	struct request_queue *connect_q;
250
+	struct request_queue *fabrics_q;
155251 	struct device *dev;
156252 	int instance;
253
+	int numa_node;
157254 	struct blk_mq_tag_set *tagset;
158255 	struct blk_mq_tag_set *admin_tagset;
159256 	struct list_head namespaces;
160257 	struct rw_semaphore namespaces_rwsem;
161258 	struct device ctrl_device;
162259 	struct device *device;	/* char device */
260
+#ifdef CONFIG_NVME_HWMON
261
+	struct device *hwmon_device;
262
+#endif
163263 	struct cdev cdev;
164264 	struct work_struct reset_work;
165265 	struct work_struct delete_work;
266
+	wait_queue_head_t state_wq;
166267 
167268 	struct nvme_subsystem *subsys;
168269 	struct list_head subsys_entry;
....@@ -177,13 +278,18 @@
177278 	u32 queue_count;
178279 
179280 	u64 cap;
180
-	u32 page_size;
181281 	u32 max_hw_sectors;
182282 	u32 max_segments;
283
+	u32 max_integrity_segments;
284
+#ifdef CONFIG_BLK_DEV_ZONED
285
+	u32 max_zone_append;
286
+#endif
287
+	u16 crdt[3];
183288 	u16 oncs;
184289 	u16 oacs;
185290 	u16 nssa;
186291 	u16 nr_streams;
292
+	u16 sqsize;
187293 	u32 max_namespaces;
188294 	atomic_t abort_limit;
189295 	u8 vwc;
....@@ -192,21 +298,24 @@
192298 	u16 kas;
193299 	u8 npss;
194300 	u8 apsta;
301
+	u16 wctemp;
302
+	u16 cctemp;
195303 	u32 oaes;
196304 	u32 aen_result;
305
+	u32 ctratt;
197306 	unsigned int shutdown_timeout;
198307 	unsigned int kato;
199308 	bool subsystem;
200309 	unsigned long quirks;
201310 	struct nvme_id_power_state psd[32];
202311 	struct nvme_effects_log *effects;
312
+	struct xarray cels;
203313 	struct work_struct scan_work;
204314 	struct work_struct async_event_work;
205315 	struct delayed_work ka_work;
206316 	struct nvme_command ka_cmd;
207317 	struct work_struct fw_act_work;
208318 	unsigned long events;
209
-	bool created;
210319 
211320 #ifdef CONFIG_NVME_MULTIPATH
212321 	/* asymmetric namespace access: */
....@@ -232,7 +341,6 @@
232341 	u16 hmmaxd;
233342 
234343 	/* Fabrics only */
235
-	u16 sqsize;
236344 	u32 ioccsz;
237345 	u32 iorcsz;
238346 	u16 icdoff;
....@@ -242,6 +350,13 @@
242350 
243351 	struct page *discard_page;
244352 	unsigned long discard_page_busy;
353
+
354
+	struct nvme_fault_inject fault_inject;
355
+};
356
+
357
+enum nvme_iopolicy {
358
+	NVME_IOPOLICY_NUMA,
359
+	NVME_IOPOLICY_RR,
245360 };
246361 
247362 struct nvme_subsystem {
....@@ -262,7 +377,11 @@
262377 	char			firmware_rev[8];
263378 	u8			cmic;
264379 	u16			vendor_id;
380
+	u16			awupf;	/* 0's based awupf value. */
265381 	struct ida		ns_ida;
382
+#ifdef CONFIG_NVME_MULTIPATH
383
+	enum nvme_iopolicy	iopolicy;
384
+#endif
266385 };
267386 
268387 /*
....@@ -272,6 +391,7 @@
272391 	u8	eui64[8];
273392 	u8	nguid[16];
274393 	uuid_t	uuid;
394
+	u8	csi;
275395 };
276396 
277397 /*
....@@ -282,14 +402,6 @@
282402  * only ever has a single entry for private namespaces.
283403  */
284404 struct nvme_ns_head {
285
-#ifdef CONFIG_NVME_MULTIPATH
286
-	struct gendisk		*disk;
287
-	struct nvme_ns __rcu	*current_path;
288
-	struct bio_list		requeue_list;
289
-	spinlock_t		requeue_lock;
290
-	struct work_struct	requeue_work;
291
-	struct mutex		lock;
292
-#endif
293405 	struct list_head	list;
294406 	struct srcu_struct      srcu;
295407 	struct nvme_subsystem	*subsys;
....@@ -297,17 +409,25 @@
297409 	struct nvme_ns_ids	ids;
298410 	struct list_head	entry;
299411 	struct kref		ref;
412
+	bool			shared;
300413 	int			instance;
414
+	struct nvme_effects_log *effects;
415
+#ifdef CONFIG_NVME_MULTIPATH
416
+	struct gendisk		*disk;
417
+	struct bio_list		requeue_list;
418
+	spinlock_t		requeue_lock;
419
+	struct work_struct	requeue_work;
420
+	struct mutex		lock;
421
+	unsigned long		flags;
422
+#define NVME_NSHEAD_DISK_LIVE	0
423
+	struct nvme_ns __rcu	*current_path[];
424
+#endif
301425 };
302426 
303
-#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
304
-struct nvme_fault_inject {
305
-	struct fault_attr attr;
306
-	struct dentry *parent;
307
-	bool dont_retry;	/* DNR, do not retry */
308
-	u16 status;		/* status code */
427
+enum nvme_ns_features {
428
+	NVME_NS_EXT_LBAS = 1 << 0, /* support extended LBA format */
429
+	NVME_NS_METADATA_SUPPORTED = 1 << 1, /* support getting generated md */
309430 };
310
-#endif
311431 
312432 struct nvme_ns {
313433 	struct list_head list;
....@@ -328,19 +448,25 @@
328448 	u16 ms;
329449 	u16 sgs;
330450 	u32 sws;
331
-	bool ext;
332451 	u8 pi_type;
452
+#ifdef CONFIG_BLK_DEV_ZONED
453
+	u64 zsze;
454
+#endif
455
+	unsigned long features;
333456 	unsigned long flags;
334457 #define NVME_NS_REMOVING	0
335458 #define NVME_NS_DEAD     	1
336459 #define NVME_NS_ANA_PENDING	2
337
-	u16 noiob;
338460 
339
-#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
340461 	struct nvme_fault_inject fault_inject;
341
-#endif
342462 
343463 };
464
+
465
+/* NVMe ns supports metadata actions by the controller (generate/strip) */
466
+static inline bool nvme_ns_has_pi(struct nvme_ns *ns)
467
+{
468
+	return ns->pi_type && ns->ms == sizeof(struct t10_pi_tuple);
469
+}
344470 
345471 struct nvme_ctrl_ops {
346472 	const char *name;
....@@ -348,48 +474,144 @@
348474 	unsigned int flags;
349475 #define NVME_F_FABRICS			(1 << 0)
350476 #define NVME_F_METADATA_SUPPORTED	(1 << 1)
477
+#define NVME_F_PCI_P2PDMA		(1 << 2)
351478 	int (*reg_read32)(struct nvme_ctrl *ctrl, u32 off, u32 *val);
352479 	int (*reg_write32)(struct nvme_ctrl *ctrl, u32 off, u32 val);
353480 	int (*reg_read64)(struct nvme_ctrl *ctrl, u32 off, u64 *val);
354481 	void (*free_ctrl)(struct nvme_ctrl *ctrl);
355482 	void (*submit_async_event)(struct nvme_ctrl *ctrl);
356483 	void (*delete_ctrl)(struct nvme_ctrl *ctrl);
357
-	int (*get_address)(struct nvme_ctrl *ctrl, char *buf, int size);
358484 	void (*stop_ctrl)(struct nvme_ctrl *ctrl);
485
+	int (*get_address)(struct nvme_ctrl *ctrl, char *buf, int size);
359486 };
360487 
488
+/*
489
+ * nvme command_id is constructed as such:
490
+ * | xxxx | xxxxxxxxxxxx |
491
+ *   gen    request tag
492
+ */
493
+#define nvme_genctr_mask(gen)			(gen & 0xf)
494
+#define nvme_cid_install_genctr(gen)		(nvme_genctr_mask(gen) << 12)
495
+#define nvme_genctr_from_cid(cid)		((cid & 0xf000) >> 12)
496
+#define nvme_tag_from_cid(cid)			(cid & 0xfff)
497
+
498
+static inline u16 nvme_cid(struct request *rq)
499
+{
500
+	return nvme_cid_install_genctr(nvme_req(rq)->genctr) | rq->tag;
501
+}
502
+
503
+static inline struct request *nvme_find_rq(struct blk_mq_tags *tags,
504
+		u16 command_id)
505
+{
506
+	u8 genctr = nvme_genctr_from_cid(command_id);
507
+	u16 tag = nvme_tag_from_cid(command_id);
508
+	struct request *rq;
509
+
510
+	rq = blk_mq_tag_to_rq(tags, tag);
511
+	if (unlikely(!rq)) {
512
+		pr_err("could not locate request for tag %#x\n",
513
+			tag);
514
+		return NULL;
515
+	}
516
+	if (unlikely(nvme_genctr_mask(nvme_req(rq)->genctr) != genctr)) {
517
+		dev_err(nvme_req(rq)->ctrl->device,
518
+			"request %#x genctr mismatch (got %#x expected %#x)\n",
519
+			tag, genctr, nvme_genctr_mask(nvme_req(rq)->genctr));
520
+		return NULL;
521
+	}
522
+	return rq;
523
+}
524
+
525
+static inline struct request *nvme_cid_to_rq(struct blk_mq_tags *tags,
526
+                u16 command_id)
527
+{
528
+	return blk_mq_tag_to_rq(tags, nvme_tag_from_cid(command_id));
529
+}
530
+
361531 #ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
362
-void nvme_fault_inject_init(struct nvme_ns *ns);
363
-void nvme_fault_inject_fini(struct nvme_ns *ns);
532
+void nvme_fault_inject_init(struct nvme_fault_inject *fault_inj,
533
+			    const char *dev_name);
534
+void nvme_fault_inject_fini(struct nvme_fault_inject *fault_inject);
364535 void nvme_should_fail(struct request *req);
365536 #else
366
-static inline void nvme_fault_inject_init(struct nvme_ns *ns) {}
367
-static inline void nvme_fault_inject_fini(struct nvme_ns *ns) {}
537
+static inline void nvme_fault_inject_init(struct nvme_fault_inject *fault_inj,
538
+					  const char *dev_name)
539
+{
540
+}
541
+static inline void nvme_fault_inject_fini(struct nvme_fault_inject *fault_inj)
542
+{
543
+}
368544 static inline void nvme_should_fail(struct request *req) {}
369545 #endif
370546 
371
-static inline bool nvme_ctrl_ready(struct nvme_ctrl *ctrl)
372
-{
373
-	u32 val = 0;
374
-
375
-	if (ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &val))
376
-		return false;
377
-	return val & NVME_CSTS_RDY;
378
-}
547
+bool nvme_wait_reset(struct nvme_ctrl *ctrl);
548
+int nvme_try_sched_reset(struct nvme_ctrl *ctrl);
379549 
380550 static inline int nvme_reset_subsystem(struct nvme_ctrl *ctrl)
381551 {
552
+	int ret;
553
+
382554 	if (!ctrl->subsystem)
383555 		return -ENOTTY;
384
-	return ctrl->ops->reg_write32(ctrl, NVME_REG_NSSR, 0x4E564D65);
556
+	if (!nvme_wait_reset(ctrl))
557
+		return -EBUSY;
558
+
559
+	ret = ctrl->ops->reg_write32(ctrl, NVME_REG_NSSR, 0x4E564D65);
560
+	if (ret)
561
+		return ret;
562
+
563
+	return nvme_try_sched_reset(ctrl);
385564 }
386565 
387
-static inline u64 nvme_block_nr(struct nvme_ns *ns, sector_t sector)
566
+/*
567
+ * Convert a 512B sector number to a device logical block number.
568
+ */
569
+static inline u64 nvme_sect_to_lba(struct nvme_ns *ns, sector_t sector)
388570 {
389
-	return (sector >> (ns->lba_shift - 9));
571
+	return sector >> (ns->lba_shift - SECTOR_SHIFT);
390572 }
391573 
392
-static inline void nvme_end_request(struct request *req, __le16 status,
574
+/*
575
+ * Convert a device logical block number to a 512B sector number.
576
+ */
577
+static inline sector_t nvme_lba_to_sect(struct nvme_ns *ns, u64 lba)
578
+{
579
+	return lba << (ns->lba_shift - SECTOR_SHIFT);
580
+}
581
+
582
+/*
583
+ * Convert byte length to nvme's 0-based num dwords
584
+ */
585
+static inline u32 nvme_bytes_to_numd(size_t len)
586
+{
587
+	return (len >> 2) - 1;
588
+}
589
+
590
+static inline bool nvme_is_ana_error(u16 status)
591
+{
592
+	switch (status & 0x7ff) {
593
+	case NVME_SC_ANA_TRANSITION:
594
+	case NVME_SC_ANA_INACCESSIBLE:
595
+	case NVME_SC_ANA_PERSISTENT_LOSS:
596
+		return true;
597
+	default:
598
+		return false;
599
+	}
600
+}
601
+
602
+static inline bool nvme_is_path_error(u16 status)
603
+{
604
+	/* check for a status code type of 'path related status' */
605
+	return (status & 0x700) == 0x300;
606
+}
607
+
608
+/*
609
+ * Fill in the status and result information from the CQE, and then figure out
610
+ * if blk-mq will need to use IPI magic to complete the request, and if yes do
611
+ * so.  If not let the caller complete the request without an indirect function
612
+ * call.
613
+ */
614
+static inline bool nvme_try_complete_req(struct request *req, __le16 status,
393615 		union nvme_result result)
394616 {
395617 	struct nvme_request *rq = nvme_req(req);
....@@ -398,7 +620,9 @@
398620 	rq->result = result;
399621 	/* inject error when permitted by fault injection framework */
400622 	nvme_should_fail(req);
401
-	blk_mq_complete_request(req);
623
+	if (unlikely(blk_should_fake_timeout(req->q)))
624
+		return true;
625
+	return blk_mq_complete_request_remote(req);
402626 }
403627 
404628 static inline void nvme_get_ctrl(struct nvme_ctrl *ctrl)
....@@ -411,19 +635,26 @@
411635 	put_device(ctrl->device);
412636 }
413637 
638
+static inline bool nvme_is_aen_req(u16 qid, __u16 command_id)
639
+{
640
+	return !qid &&
641
+		nvme_tag_from_cid(command_id) >= NVME_AQ_BLK_MQ_DEPTH;
642
+}
643
+
414644 void nvme_complete_rq(struct request *req);
415
-void nvme_cancel_request(struct request *req, void *data, bool reserved);
645
+bool nvme_cancel_request(struct request *req, void *data, bool reserved);
646
+void nvme_cancel_tagset(struct nvme_ctrl *ctrl);
647
+void nvme_cancel_admin_tagset(struct nvme_ctrl *ctrl);
416648 bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
417649 		enum nvme_ctrl_state new_state);
418
-int nvme_disable_ctrl(struct nvme_ctrl *ctrl, u64 cap);
419
-int nvme_enable_ctrl(struct nvme_ctrl *ctrl, u64 cap);
650
+int nvme_disable_ctrl(struct nvme_ctrl *ctrl);
651
+int nvme_enable_ctrl(struct nvme_ctrl *ctrl);
420652 int nvme_shutdown_ctrl(struct nvme_ctrl *ctrl);
421653 int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,
422654 		const struct nvme_ctrl_ops *ops, unsigned long quirks);
423655 void nvme_uninit_ctrl(struct nvme_ctrl *ctrl);
424656 void nvme_start_ctrl(struct nvme_ctrl *ctrl);
425657 void nvme_stop_ctrl(struct nvme_ctrl *ctrl);
426
-void nvme_put_ctrl(struct nvme_ctrl *ctrl);
427658 int nvme_init_identify(struct nvme_ctrl *ctrl);
428659 
429660 void nvme_remove_namespaces(struct nvme_ctrl *ctrl);
....@@ -437,13 +668,17 @@
437668 void nvme_stop_queues(struct nvme_ctrl *ctrl);
438669 void nvme_start_queues(struct nvme_ctrl *ctrl);
439670 void nvme_kill_queues(struct nvme_ctrl *ctrl);
671
+void nvme_sync_queues(struct nvme_ctrl *ctrl);
672
+void nvme_sync_io_queues(struct nvme_ctrl *ctrl);
440673 void nvme_unfreeze(struct nvme_ctrl *ctrl);
441674 void nvme_wait_freeze(struct nvme_ctrl *ctrl);
442
-void nvme_wait_freeze_timeout(struct nvme_ctrl *ctrl, long timeout);
675
+int nvme_wait_freeze_timeout(struct nvme_ctrl *ctrl, long timeout);
443676 void nvme_start_freeze(struct nvme_ctrl *ctrl);
444677 
445678 #define NVME_QID_ANY -1
446679 struct request *nvme_alloc_request(struct request_queue *q,
680
+		struct nvme_command *cmd, blk_mq_req_flags_t flags);
681
+struct request *nvme_alloc_request_qid(struct request_queue *q,
447682 		struct nvme_command *cmd, blk_mq_req_flags_t flags, int qid);
448683 void nvme_cleanup_cmd(struct request *req);
449684 blk_status_t nvme_setup_cmd(struct nvme_ns *ns, struct request *req,
....@@ -453,18 +688,26 @@
453688 int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
454689 		union nvme_result *result, void *buffer, unsigned bufflen,
455690 		unsigned timeout, int qid, int at_head,
456
-		blk_mq_req_flags_t flags);
691
+		blk_mq_req_flags_t flags, bool poll);
692
+int nvme_set_features(struct nvme_ctrl *dev, unsigned int fid,
693
+		      unsigned int dword11, void *buffer, size_t buflen,
694
+		      u32 *result);
695
+int nvme_get_features(struct nvme_ctrl *dev, unsigned int fid,
696
+		      unsigned int dword11, void *buffer, size_t buflen,
697
+		      u32 *result);
457698 int nvme_set_queue_count(struct nvme_ctrl *ctrl, int *count);
458699 void nvme_stop_keep_alive(struct nvme_ctrl *ctrl);
459700 int nvme_reset_ctrl(struct nvme_ctrl *ctrl);
460701 int nvme_reset_ctrl_sync(struct nvme_ctrl *ctrl);
461702 int nvme_delete_ctrl(struct nvme_ctrl *ctrl);
462
-int nvme_delete_ctrl_sync(struct nvme_ctrl *ctrl);
463703 
464
-int nvme_get_log(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp,
704
+int nvme_get_log(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp, u8 csi,
465705 		void *log, size_t size, u64 offset);
706
+struct nvme_ns *nvme_get_ns_from_disk(struct gendisk *disk,
707
+		struct nvme_ns_head **head, int *srcu_idx);
708
+void nvme_put_ns_from_disk(struct nvme_ns_head *head, int idx);
466709 
467
-extern const struct attribute_group nvme_ns_id_attr_group;
710
+extern const struct attribute_group *nvme_ns_id_attr_groups[];
468711 extern const struct block_device_operations nvme_ns_head_ops;
469712 
470713 #ifdef CONFIG_NVME_MULTIPATH
....@@ -478,23 +721,20 @@
478721 void nvme_mpath_start_freeze(struct nvme_subsystem *subsys);
479722 void nvme_set_disk_name(char *disk_name, struct nvme_ns *ns,
480723 			struct nvme_ctrl *ctrl, int *flags);
481
-bool nvme_failover_req(struct request *req);
724
+void nvme_failover_req(struct request *req);
482725 void nvme_kick_requeue_lists(struct nvme_ctrl *ctrl);
483726 int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl,struct nvme_ns_head *head);
484727 void nvme_mpath_add_disk(struct nvme_ns *ns, struct nvme_id_ns *id);
485728 void nvme_mpath_remove_disk(struct nvme_ns_head *head);
486
-int nvme_mpath_init(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id);
729
+int nvme_mpath_init_identify(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id);
730
+void nvme_mpath_init_ctrl(struct nvme_ctrl *ctrl);
731
+void nvme_mpath_update(struct nvme_ctrl *ctrl);
487732 void nvme_mpath_uninit(struct nvme_ctrl *ctrl);
488733 void nvme_mpath_stop(struct nvme_ctrl *ctrl);
489
-
490
-static inline void nvme_mpath_clear_current_path(struct nvme_ns *ns)
491
-{
492
-	struct nvme_ns_head *head = ns->head;
493
-
494
-	if (head && ns == rcu_access_pointer(head->current_path))
495
-		rcu_assign_pointer(head->current_path, NULL);
496
-}
734
+bool nvme_mpath_clear_current_path(struct nvme_ns *ns);
735
+void nvme_mpath_clear_ctrl_paths(struct nvme_ctrl *ctrl);
497736 struct nvme_ns *nvme_find_path(struct nvme_ns_head *head);
737
+blk_qc_t nvme_ns_head_submit_bio(struct bio *bio);
498738 
499739 static inline void nvme_mpath_check_last_path(struct nvme_ns *ns)
500740 {
....@@ -504,18 +744,18 @@
504744 		kblockd_schedule_work(&head->requeue_work);
505745 }
506746 
507
-static inline void nvme_mpath_update_disk_size(struct gendisk *disk)
747
+static inline void nvme_trace_bio_complete(struct request *req,
748
+        blk_status_t status)
508749 {
509
-	struct block_device *bdev = bdget_disk(disk, 0);
750
+	struct nvme_ns *ns = req->q->queuedata;
510751 
511
-	if (bdev) {
512
-		bd_set_size(bdev, get_capacity(disk) << SECTOR_SHIFT);
513
-		bdput(bdev);
514
-	}
752
+	if (req->cmd_flags & REQ_NVME_MPATH)
753
+		trace_block_bio_complete(ns->head->disk->queue, req->bio);
515754 }
516755 
517756 extern struct device_attribute dev_attr_ana_grpid;
518757 extern struct device_attribute dev_attr_ana_state;
758
+extern struct device_attribute subsys_attr_iopolicy;
519759 
520760 #else
521761 static inline bool nvme_ctrl_use_ana(struct nvme_ctrl *ctrl)
....@@ -532,9 +772,8 @@
532772 	sprintf(disk_name, "nvme%dn%d", ctrl->instance, ns->head->instance);
533773 }
534774 
535
-static inline bool nvme_failover_req(struct request *req)
775
+static inline void nvme_failover_req(struct request *req)
536776 {
537
-	return false;
538777 }
539778 static inline void nvme_kick_requeue_lists(struct nvme_ctrl *ctrl)
540779 {
....@@ -551,19 +790,33 @@
551790 static inline void nvme_mpath_remove_disk(struct nvme_ns_head *head)
552791 {
553792 }
554
-static inline void nvme_mpath_clear_current_path(struct nvme_ns *ns)
793
+static inline bool nvme_mpath_clear_current_path(struct nvme_ns *ns)
794
+{
795
+	return false;
796
+}
797
+static inline void nvme_mpath_clear_ctrl_paths(struct nvme_ctrl *ctrl)
555798 {
556799 }
557800 static inline void nvme_mpath_check_last_path(struct nvme_ns *ns)
558801 {
559802 }
560
-static inline int nvme_mpath_init(struct nvme_ctrl *ctrl,
803
+static inline void nvme_trace_bio_complete(struct request *req,
804
+        blk_status_t status)
805
+{
806
+}
807
+static inline void nvme_mpath_init_ctrl(struct nvme_ctrl *ctrl)
808
+{
809
+}
810
+static inline int nvme_mpath_init_identify(struct nvme_ctrl *ctrl,
561811 		struct nvme_id_ctrl *id)
562812 {
563813 	if (ctrl->subsys->cmic & (1 << 3))
564814 		dev_warn(ctrl->device,
565815 "Please enable CONFIG_NVME_MULTIPATH for full support of multi-port devices.\n");
566816 	return 0;
817
+}
818
+static inline void nvme_mpath_update(struct nvme_ctrl *ctrl)
819
+{
567820 }
568821 static inline void nvme_mpath_uninit(struct nvme_ctrl *ctrl)
569822 {
....@@ -580,20 +833,41 @@
580833 static inline void nvme_mpath_start_freeze(struct nvme_subsystem *subsys)
581834 {
582835 }
583
-static inline void nvme_mpath_update_disk_size(struct gendisk *disk)
584
-{
585
-}
586836 #endif /* CONFIG_NVME_MULTIPATH */
587837 
838
+int nvme_revalidate_zones(struct nvme_ns *ns);
839
+#ifdef CONFIG_BLK_DEV_ZONED
840
+int nvme_update_zone_info(struct nvme_ns *ns, unsigned lbaf);
841
+int nvme_report_zones(struct gendisk *disk, sector_t sector,
842
+		      unsigned int nr_zones, report_zones_cb cb, void *data);
843
+
844
+blk_status_t nvme_setup_zone_mgmt_send(struct nvme_ns *ns, struct request *req,
845
+				       struct nvme_command *cmnd,
846
+				       enum nvme_zone_mgmt_action action);
847
+#else
848
+#define nvme_report_zones NULL
849
+
850
+static inline blk_status_t nvme_setup_zone_mgmt_send(struct nvme_ns *ns,
851
+		struct request *req, struct nvme_command *cmnd,
852
+		enum nvme_zone_mgmt_action action)
853
+{
854
+	return BLK_STS_NOTSUPP;
855
+}
856
+
857
+static inline int nvme_update_zone_info(struct nvme_ns *ns, unsigned lbaf)
858
+{
859
+	dev_warn(ns->ctrl->device,
860
+		 "Please enable CONFIG_BLK_DEV_ZONED to support ZNS devices\n");
861
+	return -EPROTONOSUPPORT;
862
+}
863
+#endif
864
+
588865 #ifdef CONFIG_NVM
589
-void nvme_nvm_update_nvm_info(struct nvme_ns *ns);
590866 int nvme_nvm_register(struct nvme_ns *ns, char *disk_name, int node);
591867 void nvme_nvm_unregister(struct nvme_ns *ns);
592
-int nvme_nvm_register_sysfs(struct nvme_ns *ns);
593
-void nvme_nvm_unregister_sysfs(struct nvme_ns *ns);
868
+extern const struct attribute_group nvme_nvm_attr_group;
594869 int nvme_nvm_ioctl(struct nvme_ns *ns, unsigned int cmd, unsigned long arg);
595870 #else
596
-static inline void nvme_nvm_update_nvm_info(struct nvme_ns *ns) {};
597871 static inline int nvme_nvm_register(struct nvme_ns *ns, char *disk_name,
598872 				    int node)
599873 {
....@@ -601,11 +875,6 @@
601875 }
602876 
603877 static inline void nvme_nvm_unregister(struct nvme_ns *ns) {};
604
-static inline int nvme_nvm_register_sysfs(struct nvme_ns *ns)
605
-{
606
-	return 0;
607
-}
608
-static inline void nvme_nvm_unregister_sysfs(struct nvme_ns *ns) {};
609878 static inline int nvme_nvm_ioctl(struct nvme_ns *ns, unsigned int cmd,
610879 							unsigned long arg)
611880 {
....@@ -618,7 +887,25 @@
618887 	return dev_to_disk(dev)->private_data;
619888 }
620889 
621
-int __init nvme_core_init(void);
622
-void nvme_core_exit(void);
890
+#ifdef CONFIG_NVME_HWMON
891
+int nvme_hwmon_init(struct nvme_ctrl *ctrl);
892
+void nvme_hwmon_exit(struct nvme_ctrl *ctrl);
893
+#else
894
+static inline int nvme_hwmon_init(struct nvme_ctrl *ctrl)
895
+{
896
+	return 0;
897
+}
898
+
899
+static inline void nvme_hwmon_exit(struct nvme_ctrl *ctrl)
900
+{
901
+}
902
+#endif
903
+
904
+u32 nvme_command_effects(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
905
+			 u8 opcode);
906
+void nvme_execute_passthru_rq(struct request *rq);
907
+struct nvme_ctrl *nvme_ctrl_from_file(struct file *file);
908
+struct nvme_ns *nvme_find_get_ns(struct nvme_ctrl *ctrl, unsigned nsid);
909
+void nvme_put_ns(struct nvme_ns *ns);
623910 
624911 #endif /* _NVME_H */