~ljy/RK356X_SDK_RELEASE.git

..	..	@@ -1,14 +1,6 @@
	1	+/* SPDX-License-Identifier: GPL-2.0 */
1	2	/*
2	3	* Copyright (c) 2011-2014, Intel Corporation.
3		- *
4		- * This program is free software; you can redistribute it and/or modify it
5		- * under the terms and conditions of the GNU General Public License,
6		- * version 2, as published by the Free Software Foundation.
7		- *
8		- * This program is distributed in the hope it will be useful, but WITHOUT
9		- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10		- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11		- * more details.
12	4	*/
13	5
14	6	#ifndef _NVME_H
..	..	@@ -23,6 +15,10 @@
23	15	#include <linux/sed-opal.h>
24	16	#include <linux/fault-inject.h>
25	17	#include <linux/rcupdate.h>
	18	+#include <linux/wait.h>
	19	+#include <linux/t10-pi.h>
	20	+
	21	+#include <trace/events/block.h>
26	22
27	23	extern unsigned int nvme_io_timeout;
28	24	#define NVME_IO_TIMEOUT (nvme_io_timeout * HZ)
..	..	@@ -32,6 +28,22 @@
32	28
33	29	#define NVME_DEFAULT_KATO 5
34	30	#define NVME_KATO_GRACE 10
	31	+
	32	+#ifdef CONFIG_ARCH_NO_SG_CHAIN
	33	+#define NVME_INLINE_SG_CNT 0
	34	+#define NVME_INLINE_METADATA_SG_CNT 0
	35	+#else
	36	+#define NVME_INLINE_SG_CNT 2
	37	+#define NVME_INLINE_METADATA_SG_CNT 1
	38	+#endif
	39	+
	40	+/*
	41	+ * Default to a 4K page size, with the intention to update this
	42	+ * path in the future to accommodate architectures with differing
	43	+ * kernel and IO page sizes.
	44	+ */
	45	+#define NVME_CTRL_PAGE_SHIFT 12
	46	+#define NVME_CTRL_PAGE_SIZE (1 << NVME_CTRL_PAGE_SHIFT)
35	47
36	48	extern struct workqueue_struct *nvme_wq;
37	49	extern struct workqueue_struct *nvme_reset_wq;
..	..	@@ -90,6 +102,64 @@
90	102	* Set MEDIUM priority on SQ creation
91	103	*/
92	104	NVME_QUIRK_MEDIUM_PRIO_SQ = (1 << 7),
	105	+
	106	+ /*
	107	+ * Ignore device provided subnqn.
	108	+ */
	109	+ NVME_QUIRK_IGNORE_DEV_SUBNQN = (1 << 8),
	110	+
	111	+ /*
	112	+ * Broken Write Zeroes.
	113	+ */
	114	+ NVME_QUIRK_DISABLE_WRITE_ZEROES = (1 << 9),
	115	+
	116	+ /*
	117	+ * Force simple suspend/resume path.
	118	+ */
	119	+ NVME_QUIRK_SIMPLE_SUSPEND = (1 << 10),
	120	+
	121	+ /*
	122	+ * Use only one interrupt vector for all queues
	123	+ */
	124	+ NVME_QUIRK_SINGLE_VECTOR = (1 << 11),
	125	+
	126	+ /*
	127	+ * Use non-standard 128 bytes SQEs.
	128	+ */
	129	+ NVME_QUIRK_128_BYTES_SQES = (1 << 12),
	130	+
	131	+ /*
	132	+ * Prevent tag overlap between queues
	133	+ */
	134	+ NVME_QUIRK_SHARED_TAGS = (1 << 13),
	135	+
	136	+ /*
	137	+ * Don't change the value of the temperature threshold feature
	138	+ */
	139	+ NVME_QUIRK_NO_TEMP_THRESH_CHANGE = (1 << 14),
	140	+
	141	+ /*
	142	+ * The controller doesn't handle the Identify Namespace
	143	+ * Identification Descriptor list subcommand despite claiming
	144	+ * NVMe 1.3 compliance.
	145	+ */
	146	+ NVME_QUIRK_NO_NS_DESC_LIST = (1 << 15),
	147	+
	148	+ /*
	149	+ * The controller requires the command_id value be be limited, so skip
	150	+ * encoding the generation sequence number.
	151	+ */
	152	+ NVME_QUIRK_SKIP_CID_GEN = (1 << 17),
	153	+
	154	+ /*
	155	+ * Reports garbage in the namespace identifiers (eui64, nguid, uuid).
	156	+ */
	157	+ NVME_QUIRK_BOGUS_NID = (1 << 18),
	158	+
	159	+ /*
	160	+ * Limit io queue depth to 32
	161	+ */
	162	+ NVME_QUIRK_LIMIT_IOQD32 = (1 << 31),
93	163	};
94	164
95	165	/*
..	..	@@ -99,6 +169,7 @@
99	169	struct nvme_request {
100	170	struct nvme_command *cmd;
101	171	union nvme_result result;
	172	+ u8 genctr;
102	173	u8 retries;
103	174	u8 flags;
104	175	u16 status;
..	..	@@ -122,7 +193,7 @@
122	193
123	194	static inline u16 nvme_req_qid(struct request *req)
124	195	{
125		- if (!req->rq_disk)
	196	+ if (!req->q->queuedata)
126	197	return 0;
127	198	return blk_mq_unique_tag_to_hwq(blk_mq_unique_tag(req)) + 1;
128	199	}
..	..	@@ -134,17 +205,46 @@
134	205	*/
135	206	#define NVME_QUIRK_DELAY_AMOUNT 2300
136	207
	208	+/*
	209	+ * enum nvme_ctrl_state: Controller state
	210	+ *
	211	+ * @NVME_CTRL_NEW: New controller just allocated, initial state
	212	+ * @NVME_CTRL_LIVE: Controller is connected and I/O capable
	213	+ * @NVME_CTRL_RESETTING: Controller is resetting (or scheduled reset)
	214	+ * @NVME_CTRL_CONNECTING: Controller is disconnected, now connecting the
	215	+ * transport
	216	+ * @NVME_CTRL_DELETING: Controller is deleting (or scheduled deletion)
	217	+ * @NVME_CTRL_DELETING_NOIO: Controller is deleting and I/O is not
	218	+ * disabled/failed immediately. This state comes
	219	+ * after all async event processing took place and
	220	+ * before ns removal and the controller deletion
	221	+ * progress
	222	+ * @NVME_CTRL_DEAD: Controller is non-present/unresponsive during
	223	+ * shutdown or removal. In this case we forcibly
	224	+ * kill all inflight I/O as they have no chance to
	225	+ * complete
	226	+ */
137	227	enum nvme_ctrl_state {
138	228	NVME_CTRL_NEW,
139	229	NVME_CTRL_LIVE,
140		- NVME_CTRL_ADMIN_ONLY, /* Only admin queue live */
141	230	NVME_CTRL_RESETTING,
142	231	NVME_CTRL_CONNECTING,
143	232	NVME_CTRL_DELETING,
	233	+ NVME_CTRL_DELETING_NOIO,
144	234	NVME_CTRL_DEAD,
145	235	};
146	236
	237	+struct nvme_fault_inject {
	238	+#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
	239	+ struct fault_attr attr;
	240	+ struct dentry *parent;
	241	+ bool dont_retry; /* DNR, do not retry */
	242	+ u16 status; /* status code */
	243	+#endif
	244	+};
	245	+
147	246	struct nvme_ctrl {
	247	+ bool comp_seen;
148	248	enum nvme_ctrl_state state;
149	249	bool identified;
150	250	spinlock_t lock;
..	..	@@ -152,17 +252,23 @@
152	252	const struct nvme_ctrl_ops *ops;
153	253	struct request_queue *admin_q;
154	254	struct request_queue *connect_q;
	255	+ struct request_queue *fabrics_q;
155	256	struct device *dev;
156	257	int instance;
	258	+ int numa_node;
157	259	struct blk_mq_tag_set *tagset;
158	260	struct blk_mq_tag_set *admin_tagset;
159	261	struct list_head namespaces;
160	262	struct rw_semaphore namespaces_rwsem;
161	263	struct device ctrl_device;
162	264	struct device device; / char device */
	265	+#ifdef CONFIG_NVME_HWMON
	266	+ struct device *hwmon_device;
	267	+#endif
163	268	struct cdev cdev;
164	269	struct work_struct reset_work;
165	270	struct work_struct delete_work;
	271	+ wait_queue_head_t state_wq;
166	272
167	273	struct nvme_subsystem *subsys;
168	274	struct list_head subsys_entry;
..	..	@@ -177,13 +283,18 @@
177	283	u32 queue_count;
178	284
179	285	u64 cap;
180		- u32 page_size;
181	286	u32 max_hw_sectors;
182	287	u32 max_segments;
	288	+ u32 max_integrity_segments;
	289	+#ifdef CONFIG_BLK_DEV_ZONED
	290	+ u32 max_zone_append;
	291	+#endif
	292	+ u16 crdt[3];
183	293	u16 oncs;
184	294	u16 oacs;
185	295	u16 nssa;
186	296	u16 nr_streams;
	297	+ u16 sqsize;
187	298	u32 max_namespaces;
188	299	atomic_t abort_limit;
189	300	u8 vwc;
..	..	@@ -192,21 +303,24 @@
192	303	u16 kas;
193	304	u8 npss;
194	305	u8 apsta;
	306	+ u16 wctemp;
	307	+ u16 cctemp;
195	308	u32 oaes;
196	309	u32 aen_result;
	310	+ u32 ctratt;
197	311	unsigned int shutdown_timeout;
198	312	unsigned int kato;
199	313	bool subsystem;
200	314	unsigned long quirks;
201	315	struct nvme_id_power_state psd[32];
202	316	struct nvme_effects_log *effects;
	317	+ struct xarray cels;
203	318	struct work_struct scan_work;
204	319	struct work_struct async_event_work;
205	320	struct delayed_work ka_work;
206	321	struct nvme_command ka_cmd;
207	322	struct work_struct fw_act_work;
208	323	unsigned long events;
209		- bool created;
210	324
211	325	#ifdef CONFIG_NVME_MULTIPATH
212	326	/* asymmetric namespace access: */
..	..	@@ -232,7 +346,6 @@
232	346	u16 hmmaxd;
233	347
234	348	/* Fabrics only */
235		- u16 sqsize;
236	349	u32 ioccsz;
237	350	u32 iorcsz;
238	351	u16 icdoff;
..	..	@@ -242,6 +355,13 @@
242	355
243	356	struct page *discard_page;
244	357	unsigned long discard_page_busy;
	358	+
	359	+ struct nvme_fault_inject fault_inject;
	360	+};
	361	+
	362	+enum nvme_iopolicy {
	363	+ NVME_IOPOLICY_NUMA,
	364	+ NVME_IOPOLICY_RR,
245	365	};
246	366
247	367	struct nvme_subsystem {
..	..	@@ -262,7 +382,11 @@
262	382	char firmware_rev[8];
263	383	u8 cmic;
264	384	u16 vendor_id;
	385	+ u16 awupf; /* 0's based awupf value. */
265	386	struct ida ns_ida;
	387	+#ifdef CONFIG_NVME_MULTIPATH
	388	+ enum nvme_iopolicy iopolicy;
	389	+#endif
266	390	};
267	391
268	392	/*
..	..	@@ -272,6 +396,7 @@
272	396	u8 eui64[8];
273	397	u8 nguid[16];
274	398	uuid_t uuid;
	399	+ u8 csi;
275	400	};
276	401
277	402	/*
..	..	@@ -282,14 +407,6 @@
282	407	* only ever has a single entry for private namespaces.
283	408	*/
284	409	struct nvme_ns_head {
285		-#ifdef CONFIG_NVME_MULTIPATH
286		- struct gendisk *disk;
287		- struct nvme_ns __rcu *current_path;
288		- struct bio_list requeue_list;
289		- spinlock_t requeue_lock;
290		- struct work_struct requeue_work;
291		- struct mutex lock;
292		-#endif
293	410	struct list_head list;
294	411	struct srcu_struct srcu;
295	412	struct nvme_subsystem *subsys;
..	..	@@ -297,17 +414,25 @@
297	414	struct nvme_ns_ids ids;
298	415	struct list_head entry;
299	416	struct kref ref;
	417	+ bool shared;
300	418	int instance;
	419	+ struct nvme_effects_log *effects;
	420	+#ifdef CONFIG_NVME_MULTIPATH
	421	+ struct gendisk *disk;
	422	+ struct bio_list requeue_list;
	423	+ spinlock_t requeue_lock;
	424	+ struct work_struct requeue_work;
	425	+ struct mutex lock;
	426	+ unsigned long flags;
	427	+#define NVME_NSHEAD_DISK_LIVE 0
	428	+ struct nvme_ns __rcu *current_path[];
	429	+#endif
301	430	};
302	431
303		-#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
304		-struct nvme_fault_inject {
305		- struct fault_attr attr;
306		- struct dentry *parent;
307		- bool dont_retry; /* DNR, do not retry */
308		- u16 status; /* status code */
	432	+enum nvme_ns_features {
	433	+ NVME_NS_EXT_LBAS = 1 << 0, /* support extended LBA format */
	434	+ NVME_NS_METADATA_SUPPORTED = 1 << 1, /* support getting generated md */
309	435	};
310		-#endif
311	436
312	437	struct nvme_ns {
313	438	struct list_head list;
..	..	@@ -328,19 +453,25 @@
328	453	u16 ms;
329	454	u16 sgs;
330	455	u32 sws;
331		- bool ext;
332	456	u8 pi_type;
	457	+#ifdef CONFIG_BLK_DEV_ZONED
	458	+ u64 zsze;
	459	+#endif
	460	+ unsigned long features;
333	461	unsigned long flags;
334	462	#define NVME_NS_REMOVING 0
335	463	#define NVME_NS_DEAD 1
336	464	#define NVME_NS_ANA_PENDING 2
337		- u16 noiob;
338	465
339		-#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
340	466	struct nvme_fault_inject fault_inject;
341		-#endif
342	467
343	468	};
	469	+
	470	+/* NVMe ns supports metadata actions by the controller (generate/strip) */
	471	+static inline bool nvme_ns_has_pi(struct nvme_ns *ns)
	472	+{
	473	+ return ns->pi_type && ns->ms == sizeof(struct t10_pi_tuple);
	474	+}
344	475
345	476	struct nvme_ctrl_ops {
346	477	const char *name;
..	..	@@ -348,48 +479,144 @@
348	479	unsigned int flags;
349	480	#define NVME_F_FABRICS (1 << 0)
350	481	#define NVME_F_METADATA_SUPPORTED (1 << 1)
	482	+#define NVME_F_PCI_P2PDMA (1 << 2)
351	483	int (reg_read32)(struct nvme_ctrl ctrl, u32 off, u32 *val);
352	484	int (reg_write32)(struct nvme_ctrl ctrl, u32 off, u32 val);
353	485	int (reg_read64)(struct nvme_ctrl ctrl, u32 off, u64 *val);
354	486	void (free_ctrl)(struct nvme_ctrl ctrl);
355	487	void (submit_async_event)(struct nvme_ctrl ctrl);
356	488	void (delete_ctrl)(struct nvme_ctrl ctrl);
357		- int (get_address)(struct nvme_ctrl ctrl, char *buf, int size);
358	489	void (stop_ctrl)(struct nvme_ctrl ctrl);
	490	+ int (get_address)(struct nvme_ctrl ctrl, char *buf, int size);
359	491	};
360	492
	493	+/*
	494	+ * nvme command_id is constructed as such:
	495	+ * \| xxxx \| xxxxxxxxxxxx \|
	496	+ * gen request tag
	497	+ */
	498	+#define nvme_genctr_mask(gen) (gen & 0xf)
	499	+#define nvme_cid_install_genctr(gen) (nvme_genctr_mask(gen) << 12)
	500	+#define nvme_genctr_from_cid(cid) ((cid & 0xf000) >> 12)
	501	+#define nvme_tag_from_cid(cid) (cid & 0xfff)
	502	+
	503	+static inline u16 nvme_cid(struct request *rq)
	504	+{
	505	+ return nvme_cid_install_genctr(nvme_req(rq)->genctr) \| rq->tag;
	506	+}
	507	+
	508	+static inline struct request nvme_find_rq(struct blk_mq_tags tags,
	509	+ u16 command_id)
	510	+{
	511	+ u8 genctr = nvme_genctr_from_cid(command_id);
	512	+ u16 tag = nvme_tag_from_cid(command_id);
	513	+ struct request *rq;
	514	+
	515	+ rq = blk_mq_tag_to_rq(tags, tag);
	516	+ if (unlikely(!rq)) {
	517	+ pr_err("could not locate request for tag %#x\n",
	518	+ tag);
	519	+ return NULL;
	520	+ }
	521	+ if (unlikely(nvme_genctr_mask(nvme_req(rq)->genctr) != genctr)) {
	522	+ dev_err(nvme_req(rq)->ctrl->device,
	523	+ "request %#x genctr mismatch (got %#x expected %#x)\n",
	524	+ tag, genctr, nvme_genctr_mask(nvme_req(rq)->genctr));
	525	+ return NULL;
	526	+ }
	527	+ return rq;
	528	+}
	529	+
	530	+static inline struct request nvme_cid_to_rq(struct blk_mq_tags tags,
	531	+ u16 command_id)
	532	+{
	533	+ return blk_mq_tag_to_rq(tags, nvme_tag_from_cid(command_id));
	534	+}
	535	+
361	536	#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
362		-void nvme_fault_inject_init(struct nvme_ns *ns);
363		-void nvme_fault_inject_fini(struct nvme_ns *ns);
	537	+void nvme_fault_inject_init(struct nvme_fault_inject *fault_inj,
	538	+ const char *dev_name);
	539	+void nvme_fault_inject_fini(struct nvme_fault_inject *fault_inject);
364	540	void nvme_should_fail(struct request *req);
365	541	#else
366		-static inline void nvme_fault_inject_init(struct nvme_ns *ns) {}
367		-static inline void nvme_fault_inject_fini(struct nvme_ns *ns) {}
	542	+static inline void nvme_fault_inject_init(struct nvme_fault_inject *fault_inj,
	543	+ const char *dev_name)
	544	+{
	545	+}
	546	+static inline void nvme_fault_inject_fini(struct nvme_fault_inject *fault_inj)
	547	+{
	548	+}
368	549	static inline void nvme_should_fail(struct request *req) {}
369	550	#endif
370	551
371		-static inline bool nvme_ctrl_ready(struct nvme_ctrl *ctrl)
372		-{
373		- u32 val = 0;
374		-
375		- if (ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &val))
376		- return false;
377		- return val & NVME_CSTS_RDY;
378		-}
	552	+bool nvme_wait_reset(struct nvme_ctrl *ctrl);
	553	+int nvme_try_sched_reset(struct nvme_ctrl *ctrl);
379	554
380	555	static inline int nvme_reset_subsystem(struct nvme_ctrl *ctrl)
381	556	{
	557	+ int ret;
	558	+
382	559	if (!ctrl->subsystem)
383	560	return -ENOTTY;
384		- return ctrl->ops->reg_write32(ctrl, NVME_REG_NSSR, 0x4E564D65);
	561	+ if (!nvme_wait_reset(ctrl))
	562	+ return -EBUSY;
	563	+
	564	+ ret = ctrl->ops->reg_write32(ctrl, NVME_REG_NSSR, 0x4E564D65);
	565	+ if (ret)
	566	+ return ret;
	567	+
	568	+ return nvme_try_sched_reset(ctrl);
385	569	}
386	570
387		-static inline u64 nvme_block_nr(struct nvme_ns *ns, sector_t sector)
	571	+/*
	572	+ * Convert a 512B sector number to a device logical block number.
	573	+ */
	574	+static inline u64 nvme_sect_to_lba(struct nvme_ns *ns, sector_t sector)
388	575	{
389		- return (sector >> (ns->lba_shift - 9));
	576	+ return sector >> (ns->lba_shift - SECTOR_SHIFT);
390	577	}
391	578
392		-static inline void nvme_end_request(struct request *req, __le16 status,
	579	+/*
	580	+ * Convert a device logical block number to a 512B sector number.
	581	+ */
	582	+static inline sector_t nvme_lba_to_sect(struct nvme_ns *ns, u64 lba)
	583	+{
	584	+ return lba << (ns->lba_shift - SECTOR_SHIFT);
	585	+}
	586	+
	587	+/*
	588	+ * Convert byte length to nvme's 0-based num dwords
	589	+ */
	590	+static inline u32 nvme_bytes_to_numd(size_t len)
	591	+{
	592	+ return (len >> 2) - 1;
	593	+}
	594	+
	595	+static inline bool nvme_is_ana_error(u16 status)
	596	+{
	597	+ switch (status & 0x7ff) {
	598	+ case NVME_SC_ANA_TRANSITION:
	599	+ case NVME_SC_ANA_INACCESSIBLE:
	600	+ case NVME_SC_ANA_PERSISTENT_LOSS:
	601	+ return true;
	602	+ default:
	603	+ return false;
	604	+ }
	605	+}
	606	+
	607	+static inline bool nvme_is_path_error(u16 status)
	608	+{
	609	+ /* check for a status code type of 'path related status' */
	610	+ return (status & 0x700) == 0x300;
	611	+}
	612	+
	613	+/*
	614	+ * Fill in the status and result information from the CQE, and then figure out
	615	+ * if blk-mq will need to use IPI magic to complete the request, and if yes do
	616	+ * so. If not let the caller complete the request without an indirect function
	617	+ * call.
	618	+ */
	619	+static inline bool nvme_try_complete_req(struct request *req, __le16 status,
393	620	union nvme_result result)
394	621	{
395	622	struct nvme_request *rq = nvme_req(req);
..	..	@@ -398,7 +625,9 @@
398	625	rq->result = result;
399	626	/* inject error when permitted by fault injection framework */
400	627	nvme_should_fail(req);
401		- blk_mq_complete_request(req);
	628	+ if (unlikely(blk_should_fake_timeout(req->q)))
	629	+ return true;
	630	+ return blk_mq_complete_request_remote(req);
402	631	}
403	632
404	633	static inline void nvme_get_ctrl(struct nvme_ctrl *ctrl)
..	..	@@ -411,19 +640,26 @@
411	640	put_device(ctrl->device);
412	641	}
413	642
	643	+static inline bool nvme_is_aen_req(u16 qid, __u16 command_id)
	644	+{
	645	+ return !qid &&
	646	+ nvme_tag_from_cid(command_id) >= NVME_AQ_BLK_MQ_DEPTH;
	647	+}
	648	+
414	649	void nvme_complete_rq(struct request *req);
415		-void nvme_cancel_request(struct request req, void data, bool reserved);
	650	+bool nvme_cancel_request(struct request req, void data, bool reserved);
	651	+void nvme_cancel_tagset(struct nvme_ctrl *ctrl);
	652	+void nvme_cancel_admin_tagset(struct nvme_ctrl *ctrl);
416	653	bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
417	654	enum nvme_ctrl_state new_state);
418		-int nvme_disable_ctrl(struct nvme_ctrl *ctrl, u64 cap);
419		-int nvme_enable_ctrl(struct nvme_ctrl *ctrl, u64 cap);
	655	+int nvme_disable_ctrl(struct nvme_ctrl *ctrl);
	656	+int nvme_enable_ctrl(struct nvme_ctrl *ctrl);
420	657	int nvme_shutdown_ctrl(struct nvme_ctrl *ctrl);
421	658	int nvme_init_ctrl(struct nvme_ctrl ctrl, struct device dev,
422	659	const struct nvme_ctrl_ops *ops, unsigned long quirks);
423	660	void nvme_uninit_ctrl(struct nvme_ctrl *ctrl);
424	661	void nvme_start_ctrl(struct nvme_ctrl *ctrl);
425	662	void nvme_stop_ctrl(struct nvme_ctrl *ctrl);
426		-void nvme_put_ctrl(struct nvme_ctrl *ctrl);
427	663	int nvme_init_identify(struct nvme_ctrl *ctrl);
428	664
429	665	void nvme_remove_namespaces(struct nvme_ctrl *ctrl);
..	..	@@ -437,13 +673,17 @@
437	673	void nvme_stop_queues(struct nvme_ctrl *ctrl);
438	674	void nvme_start_queues(struct nvme_ctrl *ctrl);
439	675	void nvme_kill_queues(struct nvme_ctrl *ctrl);
	676	+void nvme_sync_queues(struct nvme_ctrl *ctrl);
	677	+void nvme_sync_io_queues(struct nvme_ctrl *ctrl);
440	678	void nvme_unfreeze(struct nvme_ctrl *ctrl);
441	679	void nvme_wait_freeze(struct nvme_ctrl *ctrl);
442		-void nvme_wait_freeze_timeout(struct nvme_ctrl *ctrl, long timeout);
	680	+int nvme_wait_freeze_timeout(struct nvme_ctrl *ctrl, long timeout);
443	681	void nvme_start_freeze(struct nvme_ctrl *ctrl);
444	682
445	683	#define NVME_QID_ANY -1
446	684	struct request nvme_alloc_request(struct request_queue q,
	685	+ struct nvme_command *cmd, blk_mq_req_flags_t flags);
	686	+struct request nvme_alloc_request_qid(struct request_queue q,
447	687	struct nvme_command *cmd, blk_mq_req_flags_t flags, int qid);
448	688	void nvme_cleanup_cmd(struct request *req);
449	689	blk_status_t nvme_setup_cmd(struct nvme_ns ns, struct request req,
..	..	@@ -453,18 +693,26 @@
453	693	int __nvme_submit_sync_cmd(struct request_queue q, struct nvme_command cmd,
454	694	union nvme_result result, void buffer, unsigned bufflen,
455	695	unsigned timeout, int qid, int at_head,
456		- blk_mq_req_flags_t flags);
	696	+ blk_mq_req_flags_t flags, bool poll);
	697	+int nvme_set_features(struct nvme_ctrl *dev, unsigned int fid,
	698	+ unsigned int dword11, void *buffer, size_t buflen,
	699	+ u32 *result);
	700	+int nvme_get_features(struct nvme_ctrl *dev, unsigned int fid,
	701	+ unsigned int dword11, void *buffer, size_t buflen,
	702	+ u32 *result);
457	703	int nvme_set_queue_count(struct nvme_ctrl ctrl, int count);
458	704	void nvme_stop_keep_alive(struct nvme_ctrl *ctrl);
459	705	int nvme_reset_ctrl(struct nvme_ctrl *ctrl);
460	706	int nvme_reset_ctrl_sync(struct nvme_ctrl *ctrl);
461	707	int nvme_delete_ctrl(struct nvme_ctrl *ctrl);
462		-int nvme_delete_ctrl_sync(struct nvme_ctrl *ctrl);
463	708
464		-int nvme_get_log(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp,
	709	+int nvme_get_log(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp, u8 csi,
465	710	void *log, size_t size, u64 offset);
	711	+struct nvme_ns nvme_get_ns_from_disk(struct gendisk disk,
	712	+ struct nvme_ns_head *head, int srcu_idx);
	713	+void nvme_put_ns_from_disk(struct nvme_ns_head *head, int idx);
466	714
467		-extern const struct attribute_group nvme_ns_id_attr_group;
	715	+extern const struct attribute_group *nvme_ns_id_attr_groups[];
468	716	extern const struct block_device_operations nvme_ns_head_ops;
469	717
470	718	#ifdef CONFIG_NVME_MULTIPATH
..	..	@@ -478,23 +726,20 @@
478	726	void nvme_mpath_start_freeze(struct nvme_subsystem *subsys);
479	727	void nvme_set_disk_name(char disk_name, struct nvme_ns ns,
480	728	struct nvme_ctrl ctrl, int flags);
481		-bool nvme_failover_req(struct request *req);
	729	+void nvme_failover_req(struct request *req);
482	730	void nvme_kick_requeue_lists(struct nvme_ctrl *ctrl);
483	731	int nvme_mpath_alloc_disk(struct nvme_ctrl ctrl,struct nvme_ns_head head);
484	732	void nvme_mpath_add_disk(struct nvme_ns ns, struct nvme_id_ns id);
485	733	void nvme_mpath_remove_disk(struct nvme_ns_head *head);
486		-int nvme_mpath_init(struct nvme_ctrl ctrl, struct nvme_id_ctrl id);
	734	+int nvme_mpath_init_identify(struct nvme_ctrl ctrl, struct nvme_id_ctrl id);
	735	+void nvme_mpath_init_ctrl(struct nvme_ctrl *ctrl);
	736	+void nvme_mpath_update(struct nvme_ctrl *ctrl);
487	737	void nvme_mpath_uninit(struct nvme_ctrl *ctrl);
488	738	void nvme_mpath_stop(struct nvme_ctrl *ctrl);
489		-
490		-static inline void nvme_mpath_clear_current_path(struct nvme_ns *ns)
491		-{
492		- struct nvme_ns_head *head = ns->head;
493		-
494		- if (head && ns == rcu_access_pointer(head->current_path))
495		- rcu_assign_pointer(head->current_path, NULL);
496		-}
	739	+bool nvme_mpath_clear_current_path(struct nvme_ns *ns);
	740	+void nvme_mpath_clear_ctrl_paths(struct nvme_ctrl *ctrl);
497	741	struct nvme_ns nvme_find_path(struct nvme_ns_head head);
	742	+blk_qc_t nvme_ns_head_submit_bio(struct bio *bio);
498	743
499	744	static inline void nvme_mpath_check_last_path(struct nvme_ns *ns)
500	745	{
..	..	@@ -504,18 +749,18 @@
504	749	kblockd_schedule_work(&head->requeue_work);
505	750	}
506	751
507		-static inline void nvme_mpath_update_disk_size(struct gendisk *disk)
	752	+static inline void nvme_trace_bio_complete(struct request *req,
	753	+ blk_status_t status)
508	754	{
509		- struct block_device *bdev = bdget_disk(disk, 0);
	755	+ struct nvme_ns *ns = req->q->queuedata;
510	756
511		- if (bdev) {
512		- bd_set_size(bdev, get_capacity(disk) << SECTOR_SHIFT);
513		- bdput(bdev);
514		- }
	757	+ if ((req->cmd_flags & REQ_NVME_MPATH) && req->bio)
	758	+ trace_block_bio_complete(ns->head->disk->queue, req->bio);
515	759	}
516	760
517	761	extern struct device_attribute dev_attr_ana_grpid;
518	762	extern struct device_attribute dev_attr_ana_state;
	763	+extern struct device_attribute subsys_attr_iopolicy;
519	764
520	765	#else
521	766	static inline bool nvme_ctrl_use_ana(struct nvme_ctrl *ctrl)
..	..	@@ -532,9 +777,8 @@
532	777	sprintf(disk_name, "nvme%dn%d", ctrl->instance, ns->head->instance);
533	778	}
534	779
535		-static inline bool nvme_failover_req(struct request *req)
	780	+static inline void nvme_failover_req(struct request *req)
536	781	{
537		- return false;
538	782	}
539	783	static inline void nvme_kick_requeue_lists(struct nvme_ctrl *ctrl)
540	784	{
..	..	@@ -551,19 +795,33 @@
551	795	static inline void nvme_mpath_remove_disk(struct nvme_ns_head *head)
552	796	{
553	797	}
554		-static inline void nvme_mpath_clear_current_path(struct nvme_ns *ns)
	798	+static inline bool nvme_mpath_clear_current_path(struct nvme_ns *ns)
	799	+{
	800	+ return false;
	801	+}
	802	+static inline void nvme_mpath_clear_ctrl_paths(struct nvme_ctrl *ctrl)
555	803	{
556	804	}
557	805	static inline void nvme_mpath_check_last_path(struct nvme_ns *ns)
558	806	{
559	807	}
560		-static inline int nvme_mpath_init(struct nvme_ctrl *ctrl,
	808	+static inline void nvme_trace_bio_complete(struct request *req,
	809	+ blk_status_t status)
	810	+{
	811	+}
	812	+static inline void nvme_mpath_init_ctrl(struct nvme_ctrl *ctrl)
	813	+{
	814	+}
	815	+static inline int nvme_mpath_init_identify(struct nvme_ctrl *ctrl,
561	816	struct nvme_id_ctrl *id)
562	817	{
563	818	if (ctrl->subsys->cmic & (1 << 3))
564	819	dev_warn(ctrl->device,
565	820	"Please enable CONFIG_NVME_MULTIPATH for full support of multi-port devices.\n");
566	821	return 0;
	822	+}
	823	+static inline void nvme_mpath_update(struct nvme_ctrl *ctrl)
	824	+{
567	825	}
568	826	static inline void nvme_mpath_uninit(struct nvme_ctrl *ctrl)
569	827	{
..	..	@@ -580,20 +838,41 @@
580	838	static inline void nvme_mpath_start_freeze(struct nvme_subsystem *subsys)
581	839	{
582	840	}
583		-static inline void nvme_mpath_update_disk_size(struct gendisk *disk)
584		-{
585		-}
586	841	#endif /* CONFIG_NVME_MULTIPATH */
587	842
	843	+int nvme_revalidate_zones(struct nvme_ns *ns);
	844	+#ifdef CONFIG_BLK_DEV_ZONED
	845	+int nvme_update_zone_info(struct nvme_ns *ns, unsigned lbaf);
	846	+int nvme_report_zones(struct gendisk *disk, sector_t sector,
	847	+ unsigned int nr_zones, report_zones_cb cb, void *data);
	848	+
	849	+blk_status_t nvme_setup_zone_mgmt_send(struct nvme_ns ns, struct request req,
	850	+ struct nvme_command *cmnd,
	851	+ enum nvme_zone_mgmt_action action);
	852	+#else
	853	+#define nvme_report_zones NULL
	854	+
	855	+static inline blk_status_t nvme_setup_zone_mgmt_send(struct nvme_ns *ns,
	856	+ struct request req, struct nvme_command cmnd,
	857	+ enum nvme_zone_mgmt_action action)
	858	+{
	859	+ return BLK_STS_NOTSUPP;
	860	+}
	861	+
	862	+static inline int nvme_update_zone_info(struct nvme_ns *ns, unsigned lbaf)
	863	+{
	864	+ dev_warn(ns->ctrl->device,
	865	+ "Please enable CONFIG_BLK_DEV_ZONED to support ZNS devices\n");
	866	+ return -EPROTONOSUPPORT;
	867	+}
	868	+#endif
	869	+
588	870	#ifdef CONFIG_NVM
589		-void nvme_nvm_update_nvm_info(struct nvme_ns *ns);
590	871	int nvme_nvm_register(struct nvme_ns ns, char disk_name, int node);
591	872	void nvme_nvm_unregister(struct nvme_ns *ns);
592		-int nvme_nvm_register_sysfs(struct nvme_ns *ns);
593		-void nvme_nvm_unregister_sysfs(struct nvme_ns *ns);
	873	+extern const struct attribute_group nvme_nvm_attr_group;
594	874	int nvme_nvm_ioctl(struct nvme_ns *ns, unsigned int cmd, unsigned long arg);
595	875	#else
596		-static inline void nvme_nvm_update_nvm_info(struct nvme_ns *ns) {};
597	876	static inline int nvme_nvm_register(struct nvme_ns ns, char disk_name,
598	877	int node)
599	878	{
..	..	@@ -601,11 +880,6 @@
601	880	}
602	881
603	882	static inline void nvme_nvm_unregister(struct nvme_ns *ns) {};
604		-static inline int nvme_nvm_register_sysfs(struct nvme_ns *ns)
605		-{
606		- return 0;
607		-}
608		-static inline void nvme_nvm_unregister_sysfs(struct nvme_ns *ns) {};
609	883	static inline int nvme_nvm_ioctl(struct nvme_ns *ns, unsigned int cmd,
610	884	unsigned long arg)
611	885	{
..	..	@@ -618,7 +892,25 @@
618	892	return dev_to_disk(dev)->private_data;
619	893	}
620	894
621		-int __init nvme_core_init(void);
622		-void nvme_core_exit(void);
	895	+#ifdef CONFIG_NVME_HWMON
	896	+int nvme_hwmon_init(struct nvme_ctrl *ctrl);
	897	+void nvme_hwmon_exit(struct nvme_ctrl *ctrl);
	898	+#else
	899	+static inline int nvme_hwmon_init(struct nvme_ctrl *ctrl)
	900	+{
	901	+ return 0;
	902	+}
	903	+
	904	+static inline void nvme_hwmon_exit(struct nvme_ctrl *ctrl)
	905	+{
	906	+}
	907	+#endif
	908	+
	909	+u32 nvme_command_effects(struct nvme_ctrl ctrl, struct nvme_ns ns,
	910	+ u8 opcode);
	911	+void nvme_execute_passthru_rq(struct request *rq);
	912	+struct nvme_ctrl nvme_ctrl_from_file(struct file file);
	913	+struct nvme_ns nvme_find_get_ns(struct nvme_ctrl ctrl, unsigned nsid);
	914	+void nvme_put_ns(struct nvme_ns *ns);
623	915
624	916	#endif /* _NVME_H */