~ljy/RK356X_SDK_RELEASE.git

..	..	@@ -1,14 +1,6 @@
	1	+/* SPDX-License-Identifier: GPL-2.0 */
1	2	/*
2	3	* Copyright (c) 2011-2014, Intel Corporation.
3		- *
4		- * This program is free software; you can redistribute it and/or modify it
5		- * under the terms and conditions of the GNU General Public License,
6		- * version 2, as published by the Free Software Foundation.
7		- *
8		- * This program is distributed in the hope it will be useful, but WITHOUT
9		- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10		- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11		- * more details.
12	4	*/
13	5
14	6	#ifndef _NVME_H
..	..	@@ -23,6 +15,10 @@
23	15	#include <linux/sed-opal.h>
24	16	#include <linux/fault-inject.h>
25	17	#include <linux/rcupdate.h>
	18	+#include <linux/wait.h>
	19	+#include <linux/t10-pi.h>
	20	+
	21	+#include <trace/events/block.h>
26	22
27	23	extern unsigned int nvme_io_timeout;
28	24	#define NVME_IO_TIMEOUT (nvme_io_timeout * HZ)
..	..	@@ -32,6 +28,22 @@
32	28
33	29	#define NVME_DEFAULT_KATO 5
34	30	#define NVME_KATO_GRACE 10
	31	+
	32	+#ifdef CONFIG_ARCH_NO_SG_CHAIN
	33	+#define NVME_INLINE_SG_CNT 0
	34	+#define NVME_INLINE_METADATA_SG_CNT 0
	35	+#else
	36	+#define NVME_INLINE_SG_CNT 2
	37	+#define NVME_INLINE_METADATA_SG_CNT 1
	38	+#endif
	39	+
	40	+/*
	41	+ * Default to a 4K page size, with the intention to update this
	42	+ * path in the future to accommodate architectures with differing
	43	+ * kernel and IO page sizes.
	44	+ */
	45	+#define NVME_CTRL_PAGE_SHIFT 12
	46	+#define NVME_CTRL_PAGE_SIZE (1 << NVME_CTRL_PAGE_SHIFT)
35	47
36	48	extern struct workqueue_struct *nvme_wq;
37	49	extern struct workqueue_struct *nvme_reset_wq;
..	..	@@ -90,6 +102,59 @@
90	102	* Set MEDIUM priority on SQ creation
91	103	*/
92	104	NVME_QUIRK_MEDIUM_PRIO_SQ = (1 << 7),
	105	+
	106	+ /*
	107	+ * Ignore device provided subnqn.
	108	+ */
	109	+ NVME_QUIRK_IGNORE_DEV_SUBNQN = (1 << 8),
	110	+
	111	+ /*
	112	+ * Broken Write Zeroes.
	113	+ */
	114	+ NVME_QUIRK_DISABLE_WRITE_ZEROES = (1 << 9),
	115	+
	116	+ /*
	117	+ * Force simple suspend/resume path.
	118	+ */
	119	+ NVME_QUIRK_SIMPLE_SUSPEND = (1 << 10),
	120	+
	121	+ /*
	122	+ * Use only one interrupt vector for all queues
	123	+ */
	124	+ NVME_QUIRK_SINGLE_VECTOR = (1 << 11),
	125	+
	126	+ /*
	127	+ * Use non-standard 128 bytes SQEs.
	128	+ */
	129	+ NVME_QUIRK_128_BYTES_SQES = (1 << 12),
	130	+
	131	+ /*
	132	+ * Prevent tag overlap between queues
	133	+ */
	134	+ NVME_QUIRK_SHARED_TAGS = (1 << 13),
	135	+
	136	+ /*
	137	+ * Don't change the value of the temperature threshold feature
	138	+ */
	139	+ NVME_QUIRK_NO_TEMP_THRESH_CHANGE = (1 << 14),
	140	+
	141	+ /*
	142	+ * The controller doesn't handle the Identify Namespace
	143	+ * Identification Descriptor list subcommand despite claiming
	144	+ * NVMe 1.3 compliance.
	145	+ */
	146	+ NVME_QUIRK_NO_NS_DESC_LIST = (1 << 15),
	147	+
	148	+ /*
	149	+ * The controller requires the command_id value be be limited, so skip
	150	+ * encoding the generation sequence number.
	151	+ */
	152	+ NVME_QUIRK_SKIP_CID_GEN = (1 << 17),
	153	+
	154	+ /*
	155	+ * Reports garbage in the namespace identifiers (eui64, nguid, uuid).
	156	+ */
	157	+ NVME_QUIRK_BOGUS_NID = (1 << 18),
93	158	};
94	159
95	160	/*
..	..	@@ -99,6 +164,7 @@
99	164	struct nvme_request {
100	165	struct nvme_command *cmd;
101	166	union nvme_result result;
	167	+ u8 genctr;
102	168	u8 retries;
103	169	u8 flags;
104	170	u16 status;
..	..	@@ -122,7 +188,7 @@
122	188
123	189	static inline u16 nvme_req_qid(struct request *req)
124	190	{
125		- if (!req->rq_disk)
	191	+ if (!req->q->queuedata)
126	192	return 0;
127	193	return blk_mq_unique_tag_to_hwq(blk_mq_unique_tag(req)) + 1;
128	194	}
..	..	@@ -134,17 +200,46 @@
134	200	*/
135	201	#define NVME_QUIRK_DELAY_AMOUNT 2300
136	202
	203	+/*
	204	+ * enum nvme_ctrl_state: Controller state
	205	+ *
	206	+ * @NVME_CTRL_NEW: New controller just allocated, initial state
	207	+ * @NVME_CTRL_LIVE: Controller is connected and I/O capable
	208	+ * @NVME_CTRL_RESETTING: Controller is resetting (or scheduled reset)
	209	+ * @NVME_CTRL_CONNECTING: Controller is disconnected, now connecting the
	210	+ * transport
	211	+ * @NVME_CTRL_DELETING: Controller is deleting (or scheduled deletion)
	212	+ * @NVME_CTRL_DELETING_NOIO: Controller is deleting and I/O is not
	213	+ * disabled/failed immediately. This state comes
	214	+ * after all async event processing took place and
	215	+ * before ns removal and the controller deletion
	216	+ * progress
	217	+ * @NVME_CTRL_DEAD: Controller is non-present/unresponsive during
	218	+ * shutdown or removal. In this case we forcibly
	219	+ * kill all inflight I/O as they have no chance to
	220	+ * complete
	221	+ */
137	222	enum nvme_ctrl_state {
138	223	NVME_CTRL_NEW,
139	224	NVME_CTRL_LIVE,
140		- NVME_CTRL_ADMIN_ONLY, /* Only admin queue live */
141	225	NVME_CTRL_RESETTING,
142	226	NVME_CTRL_CONNECTING,
143	227	NVME_CTRL_DELETING,
	228	+ NVME_CTRL_DELETING_NOIO,
144	229	NVME_CTRL_DEAD,
145	230	};
146	231
	232	+struct nvme_fault_inject {
	233	+#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
	234	+ struct fault_attr attr;
	235	+ struct dentry *parent;
	236	+ bool dont_retry; /* DNR, do not retry */
	237	+ u16 status; /* status code */
	238	+#endif
	239	+};
	240	+
147	241	struct nvme_ctrl {
	242	+ bool comp_seen;
148	243	enum nvme_ctrl_state state;
149	244	bool identified;
150	245	spinlock_t lock;
..	..	@@ -152,17 +247,23 @@
152	247	const struct nvme_ctrl_ops *ops;
153	248	struct request_queue *admin_q;
154	249	struct request_queue *connect_q;
	250	+ struct request_queue *fabrics_q;
155	251	struct device *dev;
156	252	int instance;
	253	+ int numa_node;
157	254	struct blk_mq_tag_set *tagset;
158	255	struct blk_mq_tag_set *admin_tagset;
159	256	struct list_head namespaces;
160	257	struct rw_semaphore namespaces_rwsem;
161	258	struct device ctrl_device;
162	259	struct device device; / char device */
	260	+#ifdef CONFIG_NVME_HWMON
	261	+ struct device *hwmon_device;
	262	+#endif
163	263	struct cdev cdev;
164	264	struct work_struct reset_work;
165	265	struct work_struct delete_work;
	266	+ wait_queue_head_t state_wq;
166	267
167	268	struct nvme_subsystem *subsys;
168	269	struct list_head subsys_entry;
..	..	@@ -177,13 +278,18 @@
177	278	u32 queue_count;
178	279
179	280	u64 cap;
180		- u32 page_size;
181	281	u32 max_hw_sectors;
182	282	u32 max_segments;
	283	+ u32 max_integrity_segments;
	284	+#ifdef CONFIG_BLK_DEV_ZONED
	285	+ u32 max_zone_append;
	286	+#endif
	287	+ u16 crdt[3];
183	288	u16 oncs;
184	289	u16 oacs;
185	290	u16 nssa;
186	291	u16 nr_streams;
	292	+ u16 sqsize;
187	293	u32 max_namespaces;
188	294	atomic_t abort_limit;
189	295	u8 vwc;
..	..	@@ -192,21 +298,24 @@
192	298	u16 kas;
193	299	u8 npss;
194	300	u8 apsta;
	301	+ u16 wctemp;
	302	+ u16 cctemp;
195	303	u32 oaes;
196	304	u32 aen_result;
	305	+ u32 ctratt;
197	306	unsigned int shutdown_timeout;
198	307	unsigned int kato;
199	308	bool subsystem;
200	309	unsigned long quirks;
201	310	struct nvme_id_power_state psd[32];
202	311	struct nvme_effects_log *effects;
	312	+ struct xarray cels;
203	313	struct work_struct scan_work;
204	314	struct work_struct async_event_work;
205	315	struct delayed_work ka_work;
206	316	struct nvme_command ka_cmd;
207	317	struct work_struct fw_act_work;
208	318	unsigned long events;
209		- bool created;
210	319
211	320	#ifdef CONFIG_NVME_MULTIPATH
212	321	/* asymmetric namespace access: */
..	..	@@ -232,7 +341,6 @@
232	341	u16 hmmaxd;
233	342
234	343	/* Fabrics only */
235		- u16 sqsize;
236	344	u32 ioccsz;
237	345	u32 iorcsz;
238	346	u16 icdoff;
..	..	@@ -242,6 +350,13 @@
242	350
243	351	struct page *discard_page;
244	352	unsigned long discard_page_busy;
	353	+
	354	+ struct nvme_fault_inject fault_inject;
	355	+};
	356	+
	357	+enum nvme_iopolicy {
	358	+ NVME_IOPOLICY_NUMA,
	359	+ NVME_IOPOLICY_RR,
245	360	};
246	361
247	362	struct nvme_subsystem {
..	..	@@ -262,7 +377,11 @@
262	377	char firmware_rev[8];
263	378	u8 cmic;
264	379	u16 vendor_id;
	380	+ u16 awupf; /* 0's based awupf value. */
265	381	struct ida ns_ida;
	382	+#ifdef CONFIG_NVME_MULTIPATH
	383	+ enum nvme_iopolicy iopolicy;
	384	+#endif
266	385	};
267	386
268	387	/*
..	..	@@ -272,6 +391,7 @@
272	391	u8 eui64[8];
273	392	u8 nguid[16];
274	393	uuid_t uuid;
	394	+ u8 csi;
275	395	};
276	396
277	397	/*
..	..	@@ -282,14 +402,6 @@
282	402	* only ever has a single entry for private namespaces.
283	403	*/
284	404	struct nvme_ns_head {
285		-#ifdef CONFIG_NVME_MULTIPATH
286		- struct gendisk *disk;
287		- struct nvme_ns __rcu *current_path;
288		- struct bio_list requeue_list;
289		- spinlock_t requeue_lock;
290		- struct work_struct requeue_work;
291		- struct mutex lock;
292		-#endif
293	405	struct list_head list;
294	406	struct srcu_struct srcu;
295	407	struct nvme_subsystem *subsys;
..	..	@@ -297,17 +409,25 @@
297	409	struct nvme_ns_ids ids;
298	410	struct list_head entry;
299	411	struct kref ref;
	412	+ bool shared;
300	413	int instance;
	414	+ struct nvme_effects_log *effects;
	415	+#ifdef CONFIG_NVME_MULTIPATH
	416	+ struct gendisk *disk;
	417	+ struct bio_list requeue_list;
	418	+ spinlock_t requeue_lock;
	419	+ struct work_struct requeue_work;
	420	+ struct mutex lock;
	421	+ unsigned long flags;
	422	+#define NVME_NSHEAD_DISK_LIVE 0
	423	+ struct nvme_ns __rcu *current_path[];
	424	+#endif
301	425	};
302	426
303		-#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
304		-struct nvme_fault_inject {
305		- struct fault_attr attr;
306		- struct dentry *parent;
307		- bool dont_retry; /* DNR, do not retry */
308		- u16 status; /* status code */
	427	+enum nvme_ns_features {
	428	+ NVME_NS_EXT_LBAS = 1 << 0, /* support extended LBA format */
	429	+ NVME_NS_METADATA_SUPPORTED = 1 << 1, /* support getting generated md */
309	430	};
310		-#endif
311	431
312	432	struct nvme_ns {
313	433	struct list_head list;
..	..	@@ -328,19 +448,25 @@
328	448	u16 ms;
329	449	u16 sgs;
330	450	u32 sws;
331		- bool ext;
332	451	u8 pi_type;
	452	+#ifdef CONFIG_BLK_DEV_ZONED
	453	+ u64 zsze;
	454	+#endif
	455	+ unsigned long features;
333	456	unsigned long flags;
334	457	#define NVME_NS_REMOVING 0
335	458	#define NVME_NS_DEAD 1
336	459	#define NVME_NS_ANA_PENDING 2
337		- u16 noiob;
338	460
339		-#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
340	461	struct nvme_fault_inject fault_inject;
341		-#endif
342	462
343	463	};
	464	+
	465	+/* NVMe ns supports metadata actions by the controller (generate/strip) */
	466	+static inline bool nvme_ns_has_pi(struct nvme_ns *ns)
	467	+{
	468	+ return ns->pi_type && ns->ms == sizeof(struct t10_pi_tuple);
	469	+}
344	470
345	471	struct nvme_ctrl_ops {
346	472	const char *name;
..	..	@@ -348,48 +474,144 @@
348	474	unsigned int flags;
349	475	#define NVME_F_FABRICS (1 << 0)
350	476	#define NVME_F_METADATA_SUPPORTED (1 << 1)
	477	+#define NVME_F_PCI_P2PDMA (1 << 2)
351	478	int (reg_read32)(struct nvme_ctrl ctrl, u32 off, u32 *val);
352	479	int (reg_write32)(struct nvme_ctrl ctrl, u32 off, u32 val);
353	480	int (reg_read64)(struct nvme_ctrl ctrl, u32 off, u64 *val);
354	481	void (free_ctrl)(struct nvme_ctrl ctrl);
355	482	void (submit_async_event)(struct nvme_ctrl ctrl);
356	483	void (delete_ctrl)(struct nvme_ctrl ctrl);
357		- int (get_address)(struct nvme_ctrl ctrl, char *buf, int size);
358	484	void (stop_ctrl)(struct nvme_ctrl ctrl);
	485	+ int (get_address)(struct nvme_ctrl ctrl, char *buf, int size);
359	486	};
360	487
	488	+/*
	489	+ * nvme command_id is constructed as such:
	490	+ * \| xxxx \| xxxxxxxxxxxx \|
	491	+ * gen request tag
	492	+ */
	493	+#define nvme_genctr_mask(gen) (gen & 0xf)
	494	+#define nvme_cid_install_genctr(gen) (nvme_genctr_mask(gen) << 12)
	495	+#define nvme_genctr_from_cid(cid) ((cid & 0xf000) >> 12)
	496	+#define nvme_tag_from_cid(cid) (cid & 0xfff)
	497	+
	498	+static inline u16 nvme_cid(struct request *rq)
	499	+{
	500	+ return nvme_cid_install_genctr(nvme_req(rq)->genctr) \| rq->tag;
	501	+}
	502	+
	503	+static inline struct request nvme_find_rq(struct blk_mq_tags tags,
	504	+ u16 command_id)
	505	+{
	506	+ u8 genctr = nvme_genctr_from_cid(command_id);
	507	+ u16 tag = nvme_tag_from_cid(command_id);
	508	+ struct request *rq;
	509	+
	510	+ rq = blk_mq_tag_to_rq(tags, tag);
	511	+ if (unlikely(!rq)) {
	512	+ pr_err("could not locate request for tag %#x\n",
	513	+ tag);
	514	+ return NULL;
	515	+ }
	516	+ if (unlikely(nvme_genctr_mask(nvme_req(rq)->genctr) != genctr)) {
	517	+ dev_err(nvme_req(rq)->ctrl->device,
	518	+ "request %#x genctr mismatch (got %#x expected %#x)\n",
	519	+ tag, genctr, nvme_genctr_mask(nvme_req(rq)->genctr));
	520	+ return NULL;
	521	+ }
	522	+ return rq;
	523	+}
	524	+
	525	+static inline struct request nvme_cid_to_rq(struct blk_mq_tags tags,
	526	+ u16 command_id)
	527	+{
	528	+ return blk_mq_tag_to_rq(tags, nvme_tag_from_cid(command_id));
	529	+}
	530	+
361	531	#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
362		-void nvme_fault_inject_init(struct nvme_ns *ns);
363		-void nvme_fault_inject_fini(struct nvme_ns *ns);
	532	+void nvme_fault_inject_init(struct nvme_fault_inject *fault_inj,
	533	+ const char *dev_name);
	534	+void nvme_fault_inject_fini(struct nvme_fault_inject *fault_inject);
364	535	void nvme_should_fail(struct request *req);
365	536	#else
366		-static inline void nvme_fault_inject_init(struct nvme_ns *ns) {}
367		-static inline void nvme_fault_inject_fini(struct nvme_ns *ns) {}
	537	+static inline void nvme_fault_inject_init(struct nvme_fault_inject *fault_inj,
	538	+ const char *dev_name)
	539	+{
	540	+}
	541	+static inline void nvme_fault_inject_fini(struct nvme_fault_inject *fault_inj)
	542	+{
	543	+}
368	544	static inline void nvme_should_fail(struct request *req) {}
369	545	#endif
370	546
371		-static inline bool nvme_ctrl_ready(struct nvme_ctrl *ctrl)
372		-{
373		- u32 val = 0;
374		-
375		- if (ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &val))
376		- return false;
377		- return val & NVME_CSTS_RDY;
378		-}
	547	+bool nvme_wait_reset(struct nvme_ctrl *ctrl);
	548	+int nvme_try_sched_reset(struct nvme_ctrl *ctrl);
379	549
380	550	static inline int nvme_reset_subsystem(struct nvme_ctrl *ctrl)
381	551	{
	552	+ int ret;
	553	+
382	554	if (!ctrl->subsystem)
383	555	return -ENOTTY;
384		- return ctrl->ops->reg_write32(ctrl, NVME_REG_NSSR, 0x4E564D65);
	556	+ if (!nvme_wait_reset(ctrl))
	557	+ return -EBUSY;
	558	+
	559	+ ret = ctrl->ops->reg_write32(ctrl, NVME_REG_NSSR, 0x4E564D65);
	560	+ if (ret)
	561	+ return ret;
	562	+
	563	+ return nvme_try_sched_reset(ctrl);
385	564	}
386	565
387		-static inline u64 nvme_block_nr(struct nvme_ns *ns, sector_t sector)
	566	+/*
	567	+ * Convert a 512B sector number to a device logical block number.
	568	+ */
	569	+static inline u64 nvme_sect_to_lba(struct nvme_ns *ns, sector_t sector)
388	570	{
389		- return (sector >> (ns->lba_shift - 9));
	571	+ return sector >> (ns->lba_shift - SECTOR_SHIFT);
390	572	}
391	573
392		-static inline void nvme_end_request(struct request *req, __le16 status,
	574	+/*
	575	+ * Convert a device logical block number to a 512B sector number.
	576	+ */
	577	+static inline sector_t nvme_lba_to_sect(struct nvme_ns *ns, u64 lba)
	578	+{
	579	+ return lba << (ns->lba_shift - SECTOR_SHIFT);
	580	+}
	581	+
	582	+/*
	583	+ * Convert byte length to nvme's 0-based num dwords
	584	+ */
	585	+static inline u32 nvme_bytes_to_numd(size_t len)
	586	+{
	587	+ return (len >> 2) - 1;
	588	+}
	589	+
	590	+static inline bool nvme_is_ana_error(u16 status)
	591	+{
	592	+ switch (status & 0x7ff) {
	593	+ case NVME_SC_ANA_TRANSITION:
	594	+ case NVME_SC_ANA_INACCESSIBLE:
	595	+ case NVME_SC_ANA_PERSISTENT_LOSS:
	596	+ return true;
	597	+ default:
	598	+ return false;
	599	+ }
	600	+}
	601	+
	602	+static inline bool nvme_is_path_error(u16 status)
	603	+{
	604	+ /* check for a status code type of 'path related status' */
	605	+ return (status & 0x700) == 0x300;
	606	+}
	607	+
	608	+/*
	609	+ * Fill in the status and result information from the CQE, and then figure out
	610	+ * if blk-mq will need to use IPI magic to complete the request, and if yes do
	611	+ * so. If not let the caller complete the request without an indirect function
	612	+ * call.
	613	+ */
	614	+static inline bool nvme_try_complete_req(struct request *req, __le16 status,
393	615	union nvme_result result)
394	616	{
395	617	struct nvme_request *rq = nvme_req(req);
..	..	@@ -398,7 +620,9 @@
398	620	rq->result = result;
399	621	/* inject error when permitted by fault injection framework */
400	622	nvme_should_fail(req);
401		- blk_mq_complete_request(req);
	623	+ if (unlikely(blk_should_fake_timeout(req->q)))
	624	+ return true;
	625	+ return blk_mq_complete_request_remote(req);
402	626	}
403	627
404	628	static inline void nvme_get_ctrl(struct nvme_ctrl *ctrl)
..	..	@@ -411,19 +635,26 @@
411	635	put_device(ctrl->device);
412	636	}
413	637
	638	+static inline bool nvme_is_aen_req(u16 qid, __u16 command_id)
	639	+{
	640	+ return !qid &&
	641	+ nvme_tag_from_cid(command_id) >= NVME_AQ_BLK_MQ_DEPTH;
	642	+}
	643	+
414	644	void nvme_complete_rq(struct request *req);
415		-void nvme_cancel_request(struct request req, void data, bool reserved);
	645	+bool nvme_cancel_request(struct request req, void data, bool reserved);
	646	+void nvme_cancel_tagset(struct nvme_ctrl *ctrl);
	647	+void nvme_cancel_admin_tagset(struct nvme_ctrl *ctrl);
416	648	bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
417	649	enum nvme_ctrl_state new_state);
418		-int nvme_disable_ctrl(struct nvme_ctrl *ctrl, u64 cap);
419		-int nvme_enable_ctrl(struct nvme_ctrl *ctrl, u64 cap);
	650	+int nvme_disable_ctrl(struct nvme_ctrl *ctrl);
	651	+int nvme_enable_ctrl(struct nvme_ctrl *ctrl);
420	652	int nvme_shutdown_ctrl(struct nvme_ctrl *ctrl);
421	653	int nvme_init_ctrl(struct nvme_ctrl ctrl, struct device dev,
422	654	const struct nvme_ctrl_ops *ops, unsigned long quirks);
423	655	void nvme_uninit_ctrl(struct nvme_ctrl *ctrl);
424	656	void nvme_start_ctrl(struct nvme_ctrl *ctrl);
425	657	void nvme_stop_ctrl(struct nvme_ctrl *ctrl);
426		-void nvme_put_ctrl(struct nvme_ctrl *ctrl);
427	658	int nvme_init_identify(struct nvme_ctrl *ctrl);
428	659
429	660	void nvme_remove_namespaces(struct nvme_ctrl *ctrl);
..	..	@@ -437,13 +668,17 @@
437	668	void nvme_stop_queues(struct nvme_ctrl *ctrl);
438	669	void nvme_start_queues(struct nvme_ctrl *ctrl);
439	670	void nvme_kill_queues(struct nvme_ctrl *ctrl);
	671	+void nvme_sync_queues(struct nvme_ctrl *ctrl);
	672	+void nvme_sync_io_queues(struct nvme_ctrl *ctrl);
440	673	void nvme_unfreeze(struct nvme_ctrl *ctrl);
441	674	void nvme_wait_freeze(struct nvme_ctrl *ctrl);
442		-void nvme_wait_freeze_timeout(struct nvme_ctrl *ctrl, long timeout);
	675	+int nvme_wait_freeze_timeout(struct nvme_ctrl *ctrl, long timeout);
443	676	void nvme_start_freeze(struct nvme_ctrl *ctrl);
444	677
445	678	#define NVME_QID_ANY -1
446	679	struct request nvme_alloc_request(struct request_queue q,
	680	+ struct nvme_command *cmd, blk_mq_req_flags_t flags);
	681	+struct request nvme_alloc_request_qid(struct request_queue q,
447	682	struct nvme_command *cmd, blk_mq_req_flags_t flags, int qid);
448	683	void nvme_cleanup_cmd(struct request *req);
449	684	blk_status_t nvme_setup_cmd(struct nvme_ns ns, struct request req,
..	..	@@ -453,18 +688,26 @@
453	688	int __nvme_submit_sync_cmd(struct request_queue q, struct nvme_command cmd,
454	689	union nvme_result result, void buffer, unsigned bufflen,
455	690	unsigned timeout, int qid, int at_head,
456		- blk_mq_req_flags_t flags);
	691	+ blk_mq_req_flags_t flags, bool poll);
	692	+int nvme_set_features(struct nvme_ctrl *dev, unsigned int fid,
	693	+ unsigned int dword11, void *buffer, size_t buflen,
	694	+ u32 *result);
	695	+int nvme_get_features(struct nvme_ctrl *dev, unsigned int fid,
	696	+ unsigned int dword11, void *buffer, size_t buflen,
	697	+ u32 *result);
457	698	int nvme_set_queue_count(struct nvme_ctrl ctrl, int count);
458	699	void nvme_stop_keep_alive(struct nvme_ctrl *ctrl);
459	700	int nvme_reset_ctrl(struct nvme_ctrl *ctrl);
460	701	int nvme_reset_ctrl_sync(struct nvme_ctrl *ctrl);
461	702	int nvme_delete_ctrl(struct nvme_ctrl *ctrl);
462		-int nvme_delete_ctrl_sync(struct nvme_ctrl *ctrl);
463	703
464		-int nvme_get_log(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp,
	704	+int nvme_get_log(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp, u8 csi,
465	705	void *log, size_t size, u64 offset);
	706	+struct nvme_ns nvme_get_ns_from_disk(struct gendisk disk,
	707	+ struct nvme_ns_head *head, int srcu_idx);
	708	+void nvme_put_ns_from_disk(struct nvme_ns_head *head, int idx);
466	709
467		-extern const struct attribute_group nvme_ns_id_attr_group;
	710	+extern const struct attribute_group *nvme_ns_id_attr_groups[];
468	711	extern const struct block_device_operations nvme_ns_head_ops;
469	712
470	713	#ifdef CONFIG_NVME_MULTIPATH
..	..	@@ -478,23 +721,20 @@
478	721	void nvme_mpath_start_freeze(struct nvme_subsystem *subsys);
479	722	void nvme_set_disk_name(char disk_name, struct nvme_ns ns,
480	723	struct nvme_ctrl ctrl, int flags);
481		-bool nvme_failover_req(struct request *req);
	724	+void nvme_failover_req(struct request *req);
482	725	void nvme_kick_requeue_lists(struct nvme_ctrl *ctrl);
483	726	int nvme_mpath_alloc_disk(struct nvme_ctrl ctrl,struct nvme_ns_head head);
484	727	void nvme_mpath_add_disk(struct nvme_ns ns, struct nvme_id_ns id);
485	728	void nvme_mpath_remove_disk(struct nvme_ns_head *head);
486		-int nvme_mpath_init(struct nvme_ctrl ctrl, struct nvme_id_ctrl id);
	729	+int nvme_mpath_init_identify(struct nvme_ctrl ctrl, struct nvme_id_ctrl id);
	730	+void nvme_mpath_init_ctrl(struct nvme_ctrl *ctrl);
	731	+void nvme_mpath_update(struct nvme_ctrl *ctrl);
487	732	void nvme_mpath_uninit(struct nvme_ctrl *ctrl);
488	733	void nvme_mpath_stop(struct nvme_ctrl *ctrl);
489		-
490		-static inline void nvme_mpath_clear_current_path(struct nvme_ns *ns)
491		-{
492		- struct nvme_ns_head *head = ns->head;
493		-
494		- if (head && ns == rcu_access_pointer(head->current_path))
495		- rcu_assign_pointer(head->current_path, NULL);
496		-}
	734	+bool nvme_mpath_clear_current_path(struct nvme_ns *ns);
	735	+void nvme_mpath_clear_ctrl_paths(struct nvme_ctrl *ctrl);
497	736	struct nvme_ns nvme_find_path(struct nvme_ns_head head);
	737	+blk_qc_t nvme_ns_head_submit_bio(struct bio *bio);
498	738
499	739	static inline void nvme_mpath_check_last_path(struct nvme_ns *ns)
500	740	{
..	..	@@ -504,18 +744,18 @@
504	744	kblockd_schedule_work(&head->requeue_work);
505	745	}
506	746
507		-static inline void nvme_mpath_update_disk_size(struct gendisk *disk)
	747	+static inline void nvme_trace_bio_complete(struct request *req,
	748	+ blk_status_t status)
508	749	{
509		- struct block_device *bdev = bdget_disk(disk, 0);
	750	+ struct nvme_ns *ns = req->q->queuedata;
510	751
511		- if (bdev) {
512		- bd_set_size(bdev, get_capacity(disk) << SECTOR_SHIFT);
513		- bdput(bdev);
514		- }
	752	+ if (req->cmd_flags & REQ_NVME_MPATH)
	753	+ trace_block_bio_complete(ns->head->disk->queue, req->bio);
515	754	}
516	755
517	756	extern struct device_attribute dev_attr_ana_grpid;
518	757	extern struct device_attribute dev_attr_ana_state;
	758	+extern struct device_attribute subsys_attr_iopolicy;
519	759
520	760	#else
521	761	static inline bool nvme_ctrl_use_ana(struct nvme_ctrl *ctrl)
..	..	@@ -532,9 +772,8 @@
532	772	sprintf(disk_name, "nvme%dn%d", ctrl->instance, ns->head->instance);
533	773	}
534	774
535		-static inline bool nvme_failover_req(struct request *req)
	775	+static inline void nvme_failover_req(struct request *req)
536	776	{
537		- return false;
538	777	}
539	778	static inline void nvme_kick_requeue_lists(struct nvme_ctrl *ctrl)
540	779	{
..	..	@@ -551,19 +790,33 @@
551	790	static inline void nvme_mpath_remove_disk(struct nvme_ns_head *head)
552	791	{
553	792	}
554		-static inline void nvme_mpath_clear_current_path(struct nvme_ns *ns)
	793	+static inline bool nvme_mpath_clear_current_path(struct nvme_ns *ns)
	794	+{
	795	+ return false;
	796	+}
	797	+static inline void nvme_mpath_clear_ctrl_paths(struct nvme_ctrl *ctrl)
555	798	{
556	799	}
557	800	static inline void nvme_mpath_check_last_path(struct nvme_ns *ns)
558	801	{
559	802	}
560		-static inline int nvme_mpath_init(struct nvme_ctrl *ctrl,
	803	+static inline void nvme_trace_bio_complete(struct request *req,
	804	+ blk_status_t status)
	805	+{
	806	+}
	807	+static inline void nvme_mpath_init_ctrl(struct nvme_ctrl *ctrl)
	808	+{
	809	+}
	810	+static inline int nvme_mpath_init_identify(struct nvme_ctrl *ctrl,
561	811	struct nvme_id_ctrl *id)
562	812	{
563	813	if (ctrl->subsys->cmic & (1 << 3))
564	814	dev_warn(ctrl->device,
565	815	"Please enable CONFIG_NVME_MULTIPATH for full support of multi-port devices.\n");
566	816	return 0;
	817	+}
	818	+static inline void nvme_mpath_update(struct nvme_ctrl *ctrl)
	819	+{
567	820	}
568	821	static inline void nvme_mpath_uninit(struct nvme_ctrl *ctrl)
569	822	{
..	..	@@ -580,20 +833,41 @@
580	833	static inline void nvme_mpath_start_freeze(struct nvme_subsystem *subsys)
581	834	{
582	835	}
583		-static inline void nvme_mpath_update_disk_size(struct gendisk *disk)
584		-{
585		-}
586	836	#endif /* CONFIG_NVME_MULTIPATH */
587	837
	838	+int nvme_revalidate_zones(struct nvme_ns *ns);
	839	+#ifdef CONFIG_BLK_DEV_ZONED
	840	+int nvme_update_zone_info(struct nvme_ns *ns, unsigned lbaf);
	841	+int nvme_report_zones(struct gendisk *disk, sector_t sector,
	842	+ unsigned int nr_zones, report_zones_cb cb, void *data);
	843	+
	844	+blk_status_t nvme_setup_zone_mgmt_send(struct nvme_ns ns, struct request req,
	845	+ struct nvme_command *cmnd,
	846	+ enum nvme_zone_mgmt_action action);
	847	+#else
	848	+#define nvme_report_zones NULL
	849	+
	850	+static inline blk_status_t nvme_setup_zone_mgmt_send(struct nvme_ns *ns,
	851	+ struct request req, struct nvme_command cmnd,
	852	+ enum nvme_zone_mgmt_action action)
	853	+{
	854	+ return BLK_STS_NOTSUPP;
	855	+}
	856	+
	857	+static inline int nvme_update_zone_info(struct nvme_ns *ns, unsigned lbaf)
	858	+{
	859	+ dev_warn(ns->ctrl->device,
	860	+ "Please enable CONFIG_BLK_DEV_ZONED to support ZNS devices\n");
	861	+ return -EPROTONOSUPPORT;
	862	+}
	863	+#endif
	864	+
588	865	#ifdef CONFIG_NVM
589		-void nvme_nvm_update_nvm_info(struct nvme_ns *ns);
590	866	int nvme_nvm_register(struct nvme_ns ns, char disk_name, int node);
591	867	void nvme_nvm_unregister(struct nvme_ns *ns);
592		-int nvme_nvm_register_sysfs(struct nvme_ns *ns);
593		-void nvme_nvm_unregister_sysfs(struct nvme_ns *ns);
	868	+extern const struct attribute_group nvme_nvm_attr_group;
594	869	int nvme_nvm_ioctl(struct nvme_ns *ns, unsigned int cmd, unsigned long arg);
595	870	#else
596		-static inline void nvme_nvm_update_nvm_info(struct nvme_ns *ns) {};
597	871	static inline int nvme_nvm_register(struct nvme_ns ns, char disk_name,
598	872	int node)
599	873	{
..	..	@@ -601,11 +875,6 @@
601	875	}
602	876
603	877	static inline void nvme_nvm_unregister(struct nvme_ns *ns) {};
604		-static inline int nvme_nvm_register_sysfs(struct nvme_ns *ns)
605		-{
606		- return 0;
607		-}
608		-static inline void nvme_nvm_unregister_sysfs(struct nvme_ns *ns) {};
609	878	static inline int nvme_nvm_ioctl(struct nvme_ns *ns, unsigned int cmd,
610	879	unsigned long arg)
611	880	{
..	..	@@ -618,7 +887,25 @@
618	887	return dev_to_disk(dev)->private_data;
619	888	}
620	889
621		-int __init nvme_core_init(void);
622		-void nvme_core_exit(void);
	890	+#ifdef CONFIG_NVME_HWMON
	891	+int nvme_hwmon_init(struct nvme_ctrl *ctrl);
	892	+void nvme_hwmon_exit(struct nvme_ctrl *ctrl);
	893	+#else
	894	+static inline int nvme_hwmon_init(struct nvme_ctrl *ctrl)
	895	+{
	896	+ return 0;
	897	+}
	898	+
	899	+static inline void nvme_hwmon_exit(struct nvme_ctrl *ctrl)
	900	+{
	901	+}
	902	+#endif
	903	+
	904	+u32 nvme_command_effects(struct nvme_ctrl ctrl, struct nvme_ns ns,
	905	+ u8 opcode);
	906	+void nvme_execute_passthru_rq(struct request *rq);
	907	+struct nvme_ctrl nvme_ctrl_from_file(struct file file);
	908	+struct nvme_ns nvme_find_get_ns(struct nvme_ctrl ctrl, unsigned nsid);
	909	+void nvme_put_ns(struct nvme_ns *ns);
623	910
624	911	#endif /* _NVME_H */