~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,39 +1,31 @@
	1	+// SPDX-License-Identifier: GPL-2.0
1	2	/*
2	3	* NVM Express device driver
3	4	* Copyright (c) 2011-2014, Intel Corporation.
4		- *
5		- * This program is free software; you can redistribute it and/or modify it
6		- * under the terms and conditions of the GNU General Public License,
7		- * version 2, as published by the Free Software Foundation.
8		- *
9		- * This program is distributed in the hope it will be useful, but WITHOUT
10		- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11		- * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12		- * more details.
13	5	*/
14	6
15	7	#include <linux/blkdev.h>
16	8	#include <linux/blk-mq.h>
	9	+#include <linux/compat.h>
17	10	#include <linux/delay.h>
18	11	#include <linux/errno.h>
19	12	#include <linux/hdreg.h>
20	13	#include <linux/kernel.h>
21	14	#include <linux/module.h>
22		-#include <linux/list_sort.h>
	15	+#include <linux/backing-dev.h>
23	16	#include <linux/slab.h>
24	17	#include <linux/types.h>
25	18	#include <linux/pr.h>
26	19	#include <linux/ptrace.h>
27	20	#include <linux/nvme_ioctl.h>
28		-#include <linux/t10-pi.h>
29	21	#include <linux/pm_qos.h>
30	22	#include <asm/unaligned.h>
31	23
32		-#define CREATE_TRACE_POINTS
33		-#include "trace.h"
34		-
35	24	#include "nvme.h"
36	25	#include "fabrics.h"
	26	+
	27	+#define CREATE_TRACE_POINTS
	28	+#include "trace.h"
37	29
38	30	#define NVME_MINORS (1U << MINORBITS)
39	31
..	..	@@ -73,8 +65,8 @@
73	65	* nvme_reset_wq - hosts nvme reset works
74	66	* nvme_delete_wq - hosts nvme delete works
75	67	*
76		- * nvme_wq will host works such are scan, aen handling, fw activation,
77		- * keep-alive error recovery, periodic reconnects etc. nvme_reset_wq
	68	+ * nvme_wq will host works such as scan, aen handling, fw activation,
	69	+ * keep-alive, periodic reconnects etc. nvme_reset_wq
78	70	* runs reset works which also flush works hosted on nvme_wq for
79	71	* serialization purposes. nvme_delete_wq host controller deletion
80	72	* works which flush reset works for serialization.
..	..	@@ -88,7 +80,6 @@
88	80	struct workqueue_struct *nvme_delete_wq;
89	81	EXPORT_SYMBOL_GPL(nvme_delete_wq);
90	82
91		-static DEFINE_IDA(nvme_subsystems_ida);
92	83	static LIST_HEAD(nvme_subsystems);
93	84	static DEFINE_MUTEX(nvme_subsystems_lock);
94	85
..	..	@@ -97,27 +88,38 @@
97	88	static struct class *nvme_class;
98	89	static struct class *nvme_subsys_class;
99	90
100		-static void nvme_ns_remove(struct nvme_ns *ns);
101		-static int nvme_revalidate_disk(struct gendisk *disk);
102	91	static void nvme_put_subsystem(struct nvme_subsystem *subsys);
103	92	static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl,
104	93	unsigned nsid);
105	94
	95	+static void nvme_update_bdev_size(struct gendisk *disk)
	96	+{
	97	+ struct block_device *bdev = bdget_disk(disk, 0);
	98	+
	99	+ if (bdev) {
	100	+ bd_set_nr_sectors(bdev, get_capacity(disk));
	101	+ bdput(bdev);
	102	+ }
	103	+}
	104	+
	105	+/*
	106	+ * Prepare a queue for teardown.
	107	+ *
	108	+ * This must forcibly unquiesce queues to avoid blocking dispatch, and only set
	109	+ * the capacity to 0 after that to avoid blocking dispatchers that may be
	110	+ * holding bd_butex. This will end buffered writers dirtying pages that can't
	111	+ * be synced.
	112	+ */
106	113	static void nvme_set_queue_dying(struct nvme_ns *ns)
107	114	{
108		- /*
109		- * Revalidating a dead namespace sets capacity to 0. This will end
110		- * buffered writers dirtying pages that can't be synced.
111		- */
112		- if (!ns->disk \|\| test_and_set_bit(NVME_NS_DEAD, &ns->flags))
	115	+ if (test_and_set_bit(NVME_NS_DEAD, &ns->flags))
113	116	return;
	117	+
114	118	blk_set_queue_dying(ns->queue);
115		- /* Forcibly unquiesce queues to avoid blocking dispatch */
116	119	blk_mq_unquiesce_queue(ns->queue);
117		- /*
118		- * Revalidate after unblocking dispatchers that may be holding bd_butex
119		- */
120		- revalidate_disk(ns->disk);
	120	+
	121	+ set_capacity(ns->disk, 0);
	122	+ nvme_update_bdev_size(ns->disk);
121	123	}
122	124
123	125	static void nvme_queue_scan(struct nvme_ctrl *ctrl)
..	..	@@ -125,9 +127,25 @@
125	127	/*
126	128	* Only new queue scan work when admin and IO queues are both alive
127	129	*/
128		- if (ctrl->state == NVME_CTRL_LIVE)
	130	+ if (ctrl->state == NVME_CTRL_LIVE && ctrl->tagset)
129	131	queue_work(nvme_wq, &ctrl->scan_work);
130	132	}
	133	+
	134	+/*
	135	+ * Use this function to proceed with scheduling reset_work for a controller
	136	+ * that had previously been set to the resetting state. This is intended for
	137	+ * code paths that can't be interrupted by other reset attempts. A hot removal
	138	+ * may prevent this from succeeding.
	139	+ */
	140	+int nvme_try_sched_reset(struct nvme_ctrl *ctrl)
	141	+{
	142	+ if (ctrl->state != NVME_CTRL_RESETTING)
	143	+ return -EBUSY;
	144	+ if (!queue_work(nvme_reset_wq, &ctrl->reset_work))
	145	+ return -EBUSY;
	146	+ return 0;
	147	+}
	148	+EXPORT_SYMBOL_GPL(nvme_try_sched_reset);
131	149
132	150	int nvme_reset_ctrl(struct nvme_ctrl *ctrl)
133	151	{
..	..	@@ -146,8 +164,7 @@
146	164	ret = nvme_reset_ctrl(ctrl);
147	165	if (!ret) {
148	166	flush_work(&ctrl->reset_work);
149		- if (ctrl->state != NVME_CTRL_LIVE &&
150		- ctrl->state != NVME_CTRL_ADMIN_ONLY)
	167	+ if (ctrl->state != NVME_CTRL_LIVE)
151	168	ret = -ENETRESET;
152	169	}
153	170
..	..	@@ -155,11 +172,8 @@
155	172	}
156	173	EXPORT_SYMBOL_GPL(nvme_reset_ctrl_sync);
157	174
158		-static void nvme_delete_ctrl_work(struct work_struct *work)
	175	+static void nvme_do_delete_ctrl(struct nvme_ctrl *ctrl)
159	176	{
160		- struct nvme_ctrl *ctrl =
161		- container_of(work, struct nvme_ctrl, delete_work);
162		-
163	177	dev_info(ctrl->device,
164	178	"Removing ctrl: NQN \"%s\"\n", ctrl->opts->subsysnqn);
165	179
..	..	@@ -168,7 +182,14 @@
168	182	nvme_remove_namespaces(ctrl);
169	183	ctrl->ops->delete_ctrl(ctrl);
170	184	nvme_uninit_ctrl(ctrl);
171		- nvme_put_ctrl(ctrl);
	185	+}
	186	+
	187	+static void nvme_delete_ctrl_work(struct work_struct *work)
	188	+{
	189	+ struct nvme_ctrl *ctrl =
	190	+ container_of(work, struct nvme_ctrl, delete_work);
	191	+
	192	+ nvme_do_delete_ctrl(ctrl);
172	193	}
173	194
174	195	int nvme_delete_ctrl(struct nvme_ctrl *ctrl)
..	..	@@ -181,36 +202,28 @@
181	202	}
182	203	EXPORT_SYMBOL_GPL(nvme_delete_ctrl);
183	204
184		-int nvme_delete_ctrl_sync(struct nvme_ctrl *ctrl)
	205	+static void nvme_delete_ctrl_sync(struct nvme_ctrl *ctrl)
185	206	{
186		- int ret = 0;
187		-
188	207	/*
189		- * Keep a reference until the work is flushed since ->delete_ctrl
190		- * can free the controller.
	208	+ * Keep a reference until nvme_do_delete_ctrl() complete,
	209	+ * since ->delete_ctrl can free the controller.
191	210	*/
192	211	nvme_get_ctrl(ctrl);
193		- ret = nvme_delete_ctrl(ctrl);
194		- if (!ret)
195		- flush_work(&ctrl->delete_work);
	212	+ if (nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING))
	213	+ nvme_do_delete_ctrl(ctrl);
196	214	nvme_put_ctrl(ctrl);
197		- return ret;
198		-}
199		-EXPORT_SYMBOL_GPL(nvme_delete_ctrl_sync);
200		-
201		-static inline bool nvme_ns_has_pi(struct nvme_ns *ns)
202		-{
203		- return ns->pi_type && ns->ms == sizeof(struct t10_pi_tuple);
204	215	}
205	216
206		-static blk_status_t nvme_error_status(struct request *req)
	217	+static blk_status_t nvme_error_status(u16 status)
207	218	{
208		- switch (nvme_req(req)->status & 0x7ff) {
	219	+ switch (status & 0x7ff) {
209	220	case NVME_SC_SUCCESS:
210	221	return BLK_STS_OK;
211	222	case NVME_SC_CAP_EXCEEDED:
212	223	return BLK_STS_NOSPC;
213	224	case NVME_SC_LBA_RANGE:
	225	+ case NVME_SC_CMD_INTERRUPTED:
	226	+ case NVME_SC_NS_NOT_READY:
214	227	return BLK_STS_TARGET;
215	228	case NVME_SC_BAD_ATTRIBUTES:
216	229	case NVME_SC_ONCS_NOT_SUPPORTED:
..	..	@@ -232,52 +245,131 @@
232	245	return BLK_STS_PROTECTION;
233	246	case NVME_SC_RESERVATION_CONFLICT:
234	247	return BLK_STS_NEXUS;
	248	+ case NVME_SC_HOST_PATH_ERROR:
	249	+ return BLK_STS_TRANSPORT;
	250	+ case NVME_SC_ZONE_TOO_MANY_ACTIVE:
	251	+ return BLK_STS_ZONE_ACTIVE_RESOURCE;
	252	+ case NVME_SC_ZONE_TOO_MANY_OPEN:
	253	+ return BLK_STS_ZONE_OPEN_RESOURCE;
235	254	default:
236	255	return BLK_STS_IOERR;
237	256	}
238	257	}
239	258
240		-static inline bool nvme_req_needs_retry(struct request *req)
	259	+static void nvme_retry_req(struct request *req)
241	260	{
242		- if (blk_noretry_request(req))
243		- return false;
244		- if (nvme_req(req)->status & NVME_SC_DNR)
245		- return false;
246		- if (nvme_req(req)->retries >= nvme_max_retries)
247		- return false;
248		- return true;
	261	+ struct nvme_ns *ns = req->q->queuedata;
	262	+ unsigned long delay = 0;
	263	+ u16 crd;
	264	+
	265	+ /* The mask and shift result must be <= 3 */
	266	+ crd = (nvme_req(req)->status & NVME_SC_CRD) >> 11;
	267	+ if (ns && crd)
	268	+ delay = ns->ctrl->crdt[crd - 1] * 100;
	269	+
	270	+ nvme_req(req)->retries++;
	271	+ blk_mq_requeue_request(req, false);
	272	+ blk_mq_delay_kick_requeue_list(req->q, delay);
	273	+}
	274	+
	275	+enum nvme_disposition {
	276	+ COMPLETE,
	277	+ RETRY,
	278	+ FAILOVER,
	279	+};
	280	+
	281	+static inline enum nvme_disposition nvme_decide_disposition(struct request *req)
	282	+{
	283	+ if (likely(nvme_req(req)->status == 0))
	284	+ return COMPLETE;
	285	+
	286	+ if (blk_noretry_request(req) \|\|
	287	+ (nvme_req(req)->status & NVME_SC_DNR) \|\|
	288	+ nvme_req(req)->retries >= nvme_max_retries)
	289	+ return COMPLETE;
	290	+
	291	+ if (req->cmd_flags & REQ_NVME_MPATH) {
	292	+ if (nvme_is_path_error(nvme_req(req)->status) \|\|
	293	+ blk_queue_dying(req->q))
	294	+ return FAILOVER;
	295	+ } else {
	296	+ if (blk_queue_dying(req->q))
	297	+ return COMPLETE;
	298	+ }
	299	+
	300	+ return RETRY;
	301	+}
	302	+
	303	+static inline void nvme_end_req(struct request *req)
	304	+{
	305	+ blk_status_t status = nvme_error_status(nvme_req(req)->status);
	306	+
	307	+ if (IS_ENABLED(CONFIG_BLK_DEV_ZONED) &&
	308	+ req_op(req) == REQ_OP_ZONE_APPEND)
	309	+ req->__sector = nvme_lba_to_sect(req->q->queuedata,
	310	+ le64_to_cpu(nvme_req(req)->result.u64));
	311	+
	312	+ nvme_trace_bio_complete(req, status);
	313	+ blk_mq_end_request(req, status);
249	314	}
250	315
251	316	void nvme_complete_rq(struct request *req)
252	317	{
253		- blk_status_t status = nvme_error_status(req);
254		-
255	318	trace_nvme_complete_rq(req);
	319	+ nvme_cleanup_cmd(req);
256	320
257		- if (unlikely(status != BLK_STS_OK && nvme_req_needs_retry(req))) {
258		- if ((req->cmd_flags & REQ_NVME_MPATH) && nvme_failover_req(req))
259		- return;
	321	+ if (nvme_req(req)->ctrl->kas)
	322	+ nvme_req(req)->ctrl->comp_seen = true;
260	323
261		- if (!blk_queue_dying(req->q)) {
262		- nvme_req(req)->retries++;
263		- blk_mq_requeue_request(req, true);
264		- return;
265		- }
	324	+ switch (nvme_decide_disposition(req)) {
	325	+ case COMPLETE:
	326	+ nvme_end_req(req);
	327	+ return;
	328	+ case RETRY:
	329	+ nvme_retry_req(req);
	330	+ return;
	331	+ case FAILOVER:
	332	+ nvme_failover_req(req);
	333	+ return;
266	334	}
267		- blk_mq_end_request(req, status);
268	335	}
269	336	EXPORT_SYMBOL_GPL(nvme_complete_rq);
270	337
271		-void nvme_cancel_request(struct request req, void data, bool reserved)
	338	+bool nvme_cancel_request(struct request req, void data, bool reserved)
272	339	{
273	340	dev_dbg_ratelimited(((struct nvme_ctrl *) data)->device,
274	341	"Cancelling I/O %d", req->tag);
275	342
276		- nvme_req(req)->status = NVME_SC_ABORT_REQ;
277		- blk_mq_complete_request(req);
	343	+ /* don't abort one completed request */
	344	+ if (blk_mq_request_completed(req))
	345	+ return true;
278	346
	347	+ nvme_req(req)->status = NVME_SC_HOST_ABORTED_CMD;
	348	+ nvme_req(req)->flags \|= NVME_REQ_CANCELLED;
	349	+ blk_mq_complete_request(req);
	350	+ return true;
279	351	}
280	352	EXPORT_SYMBOL_GPL(nvme_cancel_request);
	353	+
	354	+void nvme_cancel_tagset(struct nvme_ctrl *ctrl)
	355	+{
	356	+ if (ctrl->tagset) {
	357	+ blk_mq_tagset_busy_iter(ctrl->tagset,
	358	+ nvme_cancel_request, ctrl);
	359	+ blk_mq_tagset_wait_completed_request(ctrl->tagset);
	360	+ }
	361	+}
	362	+EXPORT_SYMBOL_GPL(nvme_cancel_tagset);
	363	+
	364	+void nvme_cancel_admin_tagset(struct nvme_ctrl *ctrl)
	365	+{
	366	+ if (ctrl->admin_tagset) {
	367	+ blk_mq_tagset_busy_iter(ctrl->admin_tagset,
	368	+ nvme_cancel_request, ctrl);
	369	+ blk_mq_tagset_wait_completed_request(ctrl->admin_tagset);
	370	+ }
	371	+}
	372	+EXPORT_SYMBOL_GPL(nvme_cancel_admin_tagset);
281	373
282	374	bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
283	375	enum nvme_ctrl_state new_state)
..	..	@@ -290,22 +382,13 @@
290	382
291	383	old_state = ctrl->state;
292	384	switch (new_state) {
293		- case NVME_CTRL_ADMIN_ONLY:
294		- switch (old_state) {
295		- case NVME_CTRL_CONNECTING:
296		- changed = true;
297		- /* FALLTHRU */
298		- default:
299		- break;
300		- }
301		- break;
302	385	case NVME_CTRL_LIVE:
303	386	switch (old_state) {
304	387	case NVME_CTRL_NEW:
305	388	case NVME_CTRL_RESETTING:
306	389	case NVME_CTRL_CONNECTING:
307	390	changed = true;
308		- /* FALLTHRU */
	391	+ fallthrough;
309	392	default:
310	393	break;
311	394	}
..	..	@@ -314,9 +397,8 @@
314	397	switch (old_state) {
315	398	case NVME_CTRL_NEW:
316	399	case NVME_CTRL_LIVE:
317		- case NVME_CTRL_ADMIN_ONLY:
318	400	changed = true;
319		- /* FALLTHRU */
	401	+ fallthrough;
320	402	default:
321	403	break;
322	404	}
..	..	@@ -326,7 +408,7 @@
326	408	case NVME_CTRL_NEW:
327	409	case NVME_CTRL_RESETTING:
328	410	changed = true;
329		- /* FALLTHRU */
	411	+ fallthrough;
330	412	default:
331	413	break;
332	414	}
..	..	@@ -334,11 +416,20 @@
334	416	case NVME_CTRL_DELETING:
335	417	switch (old_state) {
336	418	case NVME_CTRL_LIVE:
337		- case NVME_CTRL_ADMIN_ONLY:
338	419	case NVME_CTRL_RESETTING:
339	420	case NVME_CTRL_CONNECTING:
340	421	changed = true;
341		- /* FALLTHRU */
	422	+ fallthrough;
	423	+ default:
	424	+ break;
	425	+ }
	426	+ break;
	427	+ case NVME_CTRL_DELETING_NOIO:
	428	+ switch (old_state) {
	429	+ case NVME_CTRL_DELETING:
	430	+ case NVME_CTRL_DEAD:
	431	+ changed = true;
	432	+ fallthrough;
342	433	default:
343	434	break;
344	435	}
..	..	@@ -347,7 +438,7 @@
347	438	switch (old_state) {
348	439	case NVME_CTRL_DELETING:
349	440	changed = true;
350		- /* FALLTHRU */
	441	+ fallthrough;
351	442	default:
352	443	break;
353	444	}
..	..	@@ -356,8 +447,10 @@
356	447	break;
357	448	}
358	449
359		- if (changed)
	450	+ if (changed) {
360	451	ctrl->state = new_state;
	452	+ wake_up_all(&ctrl->state_wq);
	453	+ }
361	454
362	455	spin_unlock_irqrestore(&ctrl->lock, flags);
363	456	if (changed && ctrl->state == NVME_CTRL_LIVE)
..	..	@@ -366,6 +459,40 @@
366	459	}
367	460	EXPORT_SYMBOL_GPL(nvme_change_ctrl_state);
368	461
	462	+/*
	463	+ * Returns true for sink states that can't ever transition back to live.
	464	+ */
	465	+static bool nvme_state_terminal(struct nvme_ctrl *ctrl)
	466	+{
	467	+ switch (ctrl->state) {
	468	+ case NVME_CTRL_NEW:
	469	+ case NVME_CTRL_LIVE:
	470	+ case NVME_CTRL_RESETTING:
	471	+ case NVME_CTRL_CONNECTING:
	472	+ return false;
	473	+ case NVME_CTRL_DELETING:
	474	+ case NVME_CTRL_DELETING_NOIO:
	475	+ case NVME_CTRL_DEAD:
	476	+ return true;
	477	+ default:
	478	+ WARN_ONCE(1, "Unhandled ctrl state:%d", ctrl->state);
	479	+ return true;
	480	+ }
	481	+}
	482	+
	483	+/*
	484	+ * Waits for the controller state to be resetting, or returns false if it is
	485	+ * not possible to ever transition to that state.
	486	+ */
	487	+bool nvme_wait_reset(struct nvme_ctrl *ctrl)
	488	+{
	489	+ wait_event(ctrl->state_wq,
	490	+ nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING) \|\|
	491	+ nvme_state_terminal(ctrl));
	492	+ return ctrl->state == NVME_CTRL_RESETTING;
	493	+}
	494	+EXPORT_SYMBOL_GPL(nvme_wait_reset);
	495	+
369	496	static void nvme_free_ns_head(struct kref *ref)
370	497	{
371	498	struct nvme_ns_head *head =
..	..	@@ -373,8 +500,7 @@
373	500
374	501	nvme_mpath_remove_disk(head);
375	502	ida_simple_remove(&head->subsys->ns_ida, head->instance);
376		- list_del_init(&head->entry);
377		- cleanup_srcu_struct_quiesced(&head->srcu);
	503	+ cleanup_srcu_struct(&head->srcu);
378	504	nvme_put_subsystem(head->subsys);
379	505	kfree(head);
380	506	}
..	..	@@ -397,42 +523,61 @@
397	523	kfree(ns);
398	524	}
399	525
400		-static void nvme_put_ns(struct nvme_ns *ns)
	526	+void nvme_put_ns(struct nvme_ns *ns)
401	527	{
402	528	kref_put(&ns->kref, nvme_free_ns);
403	529	}
	530	+EXPORT_SYMBOL_NS_GPL(nvme_put_ns, NVME_TARGET_PASSTHRU);
404	531
405	532	static inline void nvme_clear_nvme_request(struct request *req)
406	533	{
407		- if (!(req->rq_flags & RQF_DONTPREP)) {
408		- nvme_req(req)->retries = 0;
409		- nvme_req(req)->flags = 0;
410		- req->rq_flags \|= RQF_DONTPREP;
411		- }
	534	+ nvme_req(req)->retries = 0;
	535	+ nvme_req(req)->flags = 0;
	536	+ req->rq_flags \|= RQF_DONTPREP;
412	537	}
413	538
414		-struct request nvme_alloc_request(struct request_queue q,
415		- struct nvme_command *cmd, blk_mq_req_flags_t flags, int qid)
	539	+static inline unsigned int nvme_req_op(struct nvme_command *cmd)
416	540	{
417		- unsigned op = nvme_is_write(cmd) ? REQ_OP_DRV_OUT : REQ_OP_DRV_IN;
418		- struct request *req;
	541	+ return nvme_is_write(cmd) ? REQ_OP_DRV_OUT : REQ_OP_DRV_IN;
	542	+}
419	543
420		- if (qid == NVME_QID_ANY) {
421		- req = blk_mq_alloc_request(q, op, flags);
422		- } else {
423		- req = blk_mq_alloc_request_hctx(q, op, flags,
424		- qid ? qid - 1 : 0);
425		- }
426		- if (IS_ERR(req))
427		- return req;
	544	+static inline void nvme_init_request(struct request *req,
	545	+ struct nvme_command *cmd)
	546	+{
	547	+ if (req->q->queuedata)
	548	+ req->timeout = NVME_IO_TIMEOUT;
	549	+ else /* no queuedata implies admin queue */
	550	+ req->timeout = ADMIN_TIMEOUT;
428	551
429	552	req->cmd_flags \|= REQ_FAILFAST_DRIVER;
430	553	nvme_clear_nvme_request(req);
431	554	nvme_req(req)->cmd = cmd;
	555	+}
432	556
	557	+struct request nvme_alloc_request(struct request_queue q,
	558	+ struct nvme_command *cmd, blk_mq_req_flags_t flags)
	559	+{
	560	+ struct request *req;
	561	+
	562	+ req = blk_mq_alloc_request(q, nvme_req_op(cmd), flags);
	563	+ if (!IS_ERR(req))
	564	+ nvme_init_request(req, cmd);
433	565	return req;
434	566	}
435	567	EXPORT_SYMBOL_GPL(nvme_alloc_request);
	568	+
	569	+struct request nvme_alloc_request_qid(struct request_queue q,
	570	+ struct nvme_command *cmd, blk_mq_req_flags_t flags, int qid)
	571	+{
	572	+ struct request *req;
	573	+
	574	+ req = blk_mq_alloc_request_hctx(q, nvme_req_op(cmd), flags,
	575	+ qid ? qid - 1 : 0);
	576	+ if (!IS_ERR(req))
	577	+ nvme_init_request(req, cmd);
	578	+ return req;
	579	+}
	580	+EXPORT_SYMBOL_GPL(nvme_alloc_request_qid);
436	581
437	582	static int nvme_toggle_streams(struct nvme_ctrl *ctrl, bool enable)
438	583	{
..	..	@@ -470,7 +615,7 @@
470	615
471	616	c.directive.opcode = nvme_admin_directive_recv;
472	617	c.directive.nsid = cpu_to_le32(nsid);
473		- c.directive.numd = cpu_to_le32((sizeof(*s) >> 2) - 1);
	618	+ c.directive.numd = cpu_to_le32(nvme_bytes_to_numd(sizeof(*s)));
474	619	c.directive.doper = NVME_DIR_RCV_ST_OP_PARAM;
475	620	c.directive.dtype = NVME_DIR_STREAMS;
476	621
..	..	@@ -493,19 +638,22 @@
493	638
494	639	ret = nvme_get_stream_params(ctrl, &s, NVME_NSID_ALL);
495	640	if (ret)
496		- return ret;
	641	+ goto out_disable_stream;
497	642
498	643	ctrl->nssa = le16_to_cpu(s.nssa);
499	644	if (ctrl->nssa < BLK_MAX_WRITE_HINTS - 1) {
500	645	dev_info(ctrl->device, "too few streams (%u) available\n",
501	646	ctrl->nssa);
502		- nvme_disable_streams(ctrl);
503		- return 0;
	647	+ goto out_disable_stream;
504	648	}
505	649
506		- ctrl->nr_streams = min_t(unsigned, ctrl->nssa, BLK_MAX_WRITE_HINTS - 1);
	650	+ ctrl->nr_streams = min_t(u16, ctrl->nssa, BLK_MAX_WRITE_HINTS - 1);
507	651	dev_info(ctrl->device, "Using %u streams\n", ctrl->nr_streams);
508	652	return 0;
	653	+
	654	+out_disable_stream:
	655	+ nvme_disable_streams(ctrl);
	656	+ return ret;
509	657	}
510	658
511	659	/*
..	..	@@ -533,10 +681,17 @@
533	681	req->q->write_hints[streamid] += blk_rq_bytes(req) >> 9;
534	682	}
535	683
	684	+static inline void nvme_setup_passthrough(struct request *req,
	685	+ struct nvme_command *cmd)
	686	+{
	687	+ memcpy(cmd, nvme_req(req)->cmd, sizeof(*cmd));
	688	+ /* passthru commands should let the driver set the SGL flags */
	689	+ cmd->common.flags &= ~NVME_CMD_SGL_ALL;
	690	+}
	691	+
536	692	static inline void nvme_setup_flush(struct nvme_ns *ns,
537	693	struct nvme_command *cmnd)
538	694	{
539		- memset(cmnd, 0, sizeof(*cmnd));
540	695	cmnd->common.opcode = nvme_cmd_flush;
541	696	cmnd->common.nsid = cpu_to_le32(ns->head->ns_id);
542	697	}
..	..	@@ -568,16 +723,26 @@
568	723	range = page_address(ns->ctrl->discard_page);
569	724	}
570	725
571		- __rq_for_each_bio(bio, req) {
572		- u64 slba = nvme_block_nr(ns, bio->bi_iter.bi_sector);
573		- u32 nlb = bio->bi_iter.bi_size >> ns->lba_shift;
	726	+ if (queue_max_discard_segments(req->q) == 1) {
	727	+ u64 slba = nvme_sect_to_lba(ns, blk_rq_pos(req));
	728	+ u32 nlb = blk_rq_sectors(req) >> (ns->lba_shift - 9);
574	729
575		- if (n < segments) {
576		- range[n].cattr = cpu_to_le32(0);
577		- range[n].nlb = cpu_to_le32(nlb);
578		- range[n].slba = cpu_to_le64(slba);
	730	+ range[0].cattr = cpu_to_le32(0);
	731	+ range[0].nlb = cpu_to_le32(nlb);
	732	+ range[0].slba = cpu_to_le64(slba);
	733	+ n = 1;
	734	+ } else {
	735	+ __rq_for_each_bio(bio, req) {
	736	+ u64 slba = nvme_sect_to_lba(ns, bio->bi_iter.bi_sector);
	737	+ u32 nlb = bio->bi_iter.bi_size >> ns->lba_shift;
	738	+
	739	+ if (n < segments) {
	740	+ range[n].cattr = cpu_to_le32(0);
	741	+ range[n].nlb = cpu_to_le32(nlb);
	742	+ range[n].slba = cpu_to_le64(slba);
	743	+ }
	744	+ n++;
579	745	}
580		- n++;
581	746	}
582	747
583	748	if (WARN_ON_ONCE(n != segments)) {
..	..	@@ -588,7 +753,6 @@
588	753	return BLK_STS_IOERR;
589	754	}
590	755
591		- memset(cmnd, 0, sizeof(*cmnd));
592	756	cmnd->dsm.opcode = nvme_cmd_dsm;
593	757	cmnd->dsm.nsid = cpu_to_le32(ns->head->ns_id);
594	758	cmnd->dsm.nr = cpu_to_le32(segments - 1);
..	..	@@ -602,8 +766,28 @@
602	766	return BLK_STS_OK;
603	767	}
604	768
605		-static inline blk_status_t nvme_setup_rw(struct nvme_ns *ns,
	769	+static inline blk_status_t nvme_setup_write_zeroes(struct nvme_ns *ns,
606	770	struct request req, struct nvme_command cmnd)
	771	+{
	772	+ if (ns->ctrl->quirks & NVME_QUIRK_DEALLOCATE_ZEROES)
	773	+ return nvme_setup_discard(ns, req, cmnd);
	774	+
	775	+ cmnd->write_zeroes.opcode = nvme_cmd_write_zeroes;
	776	+ cmnd->write_zeroes.nsid = cpu_to_le32(ns->head->ns_id);
	777	+ cmnd->write_zeroes.slba =
	778	+ cpu_to_le64(nvme_sect_to_lba(ns, blk_rq_pos(req)));
	779	+ cmnd->write_zeroes.length =
	780	+ cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1);
	781	+ if (nvme_ns_has_pi(ns))
	782	+ cmnd->write_zeroes.control = cpu_to_le16(NVME_RW_PRINFO_PRACT);
	783	+ else
	784	+ cmnd->write_zeroes.control = 0;
	785	+ return BLK_STS_OK;
	786	+}
	787	+
	788	+static inline blk_status_t nvme_setup_rw(struct nvme_ns *ns,
	789	+ struct request req, struct nvme_command cmnd,
	790	+ enum nvme_opcode op)
607	791	{
608	792	struct nvme_ctrl *ctrl = ns->ctrl;
609	793	u16 control = 0;
..	..	@@ -617,10 +801,9 @@
617	801	if (req->cmd_flags & REQ_RAHEAD)
618	802	dsmgmt \|= NVME_RW_DSM_FREQ_PREFETCH;
619	803
620		- memset(cmnd, 0, sizeof(*cmnd));
621		- cmnd->rw.opcode = (rq_data_dir(req) ? nvme_cmd_write : nvme_cmd_read);
	804	+ cmnd->rw.opcode = op;
622	805	cmnd->rw.nsid = cpu_to_le32(ns->head->ns_id);
623		- cmnd->rw.slba = cpu_to_le64(nvme_block_nr(ns, blk_rq_pos(req)));
	806	+ cmnd->rw.slba = cpu_to_le64(nvme_sect_to_lba(ns, blk_rq_pos(req)));
624	807	cmnd->rw.length = cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1);
625	808
626	809	if (req_op(req) == REQ_OP_WRITE && ctrl->nr_streams)
..	..	@@ -637,8 +820,6 @@
637	820	if (WARN_ON_ONCE(!nvme_ns_has_pi(ns)))
638	821	return BLK_STS_NOTSUPP;
639	822	control \|= NVME_RW_PRINFO_PRACT;
640		- } else if (req_op(req) == REQ_OP_WRITE) {
641		- t10_pi_prepare(req, ns->pi_type);
642	823	}
643	824
644	825	switch (ns->pi_type) {
..	..	@@ -649,6 +830,8 @@
649	830	case NVME_NS_DPS_PI_TYPE2:
650	831	control \|= NVME_RW_PRINFO_PRCHK_GUARD \|
651	832	NVME_RW_PRINFO_PRCHK_REF;
	833	+ if (op == nvme_cmd_zone_append)
	834	+ control \|= NVME_RW_APPEND_PIREMAP;
652	835	cmnd->rw.reftag = cpu_to_le32(t10_pi_ref_tag(req));
653	836	break;
654	837	}
..	..	@@ -661,13 +844,6 @@
661	844
662	845	void nvme_cleanup_cmd(struct request *req)
663	846	{
664		- if (blk_integrity_rq(req) && req_op(req) == REQ_OP_READ &&
665		- nvme_req(req)->status == 0) {
666		- struct nvme_ns *ns = req->rq_disk->private_data;
667		-
668		- t10_pi_complete(req, ns->pi_type,
669		- blk_rq_bytes(req) >> ns->lba_shift);
670		- }
671	847	if (req->rq_flags & RQF_SPECIAL_PAYLOAD) {
672	848	struct nvme_ns *ns = req->rq_disk->private_data;
673	849	struct page *page = req->special_vec.bv_page;
..	..	@@ -683,37 +859,86 @@
683	859	blk_status_t nvme_setup_cmd(struct nvme_ns ns, struct request req,
684	860	struct nvme_command *cmd)
685	861	{
	862	+ struct nvme_ctrl *ctrl = nvme_req(req)->ctrl;
686	863	blk_status_t ret = BLK_STS_OK;
687	864
688		- nvme_clear_nvme_request(req);
	865	+ if (!(req->rq_flags & RQF_DONTPREP))
	866	+ nvme_clear_nvme_request(req);
689	867
	868	+ memset(cmd, 0, sizeof(*cmd));
690	869	switch (req_op(req)) {
691	870	case REQ_OP_DRV_IN:
692	871	case REQ_OP_DRV_OUT:
693		- memcpy(cmd, nvme_req(req)->cmd, sizeof(*cmd));
	872	+ nvme_setup_passthrough(req, cmd);
694	873	break;
695	874	case REQ_OP_FLUSH:
696	875	nvme_setup_flush(ns, cmd);
697	876	break;
	877	+ case REQ_OP_ZONE_RESET_ALL:
	878	+ case REQ_OP_ZONE_RESET:
	879	+ ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_RESET);
	880	+ break;
	881	+ case REQ_OP_ZONE_OPEN:
	882	+ ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_OPEN);
	883	+ break;
	884	+ case REQ_OP_ZONE_CLOSE:
	885	+ ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_CLOSE);
	886	+ break;
	887	+ case REQ_OP_ZONE_FINISH:
	888	+ ret = nvme_setup_zone_mgmt_send(ns, req, cmd, NVME_ZONE_FINISH);
	889	+ break;
698	890	case REQ_OP_WRITE_ZEROES:
699		- /* currently only aliased to deallocate for a few ctrls: */
	891	+ ret = nvme_setup_write_zeroes(ns, req, cmd);
	892	+ break;
700	893	case REQ_OP_DISCARD:
701	894	ret = nvme_setup_discard(ns, req, cmd);
702	895	break;
703	896	case REQ_OP_READ:
	897	+ ret = nvme_setup_rw(ns, req, cmd, nvme_cmd_read);
	898	+ break;
704	899	case REQ_OP_WRITE:
705		- ret = nvme_setup_rw(ns, req, cmd);
	900	+ ret = nvme_setup_rw(ns, req, cmd, nvme_cmd_write);
	901	+ break;
	902	+ case REQ_OP_ZONE_APPEND:
	903	+ ret = nvme_setup_rw(ns, req, cmd, nvme_cmd_zone_append);
706	904	break;
707	905	default:
708	906	WARN_ON_ONCE(1);
709	907	return BLK_STS_IOERR;
710	908	}
711	909
712		- cmd->common.command_id = req->tag;
	910	+ if (!(ctrl->quirks & NVME_QUIRK_SKIP_CID_GEN))
	911	+ nvme_req(req)->genctr++;
	912	+ cmd->common.command_id = nvme_cid(req);
713	913	trace_nvme_setup_cmd(req, cmd);
714	914	return ret;
715	915	}
716	916	EXPORT_SYMBOL_GPL(nvme_setup_cmd);
	917	+
	918	+static void nvme_end_sync_rq(struct request *rq, blk_status_t error)
	919	+{
	920	+ struct completion *waiting = rq->end_io_data;
	921	+
	922	+ rq->end_io_data = NULL;
	923	+ complete(waiting);
	924	+}
	925	+
	926	+static void nvme_execute_rq_polled(struct request_queue *q,
	927	+ struct gendisk bd_disk, struct request rq, int at_head)
	928	+{
	929	+ DECLARE_COMPLETION_ONSTACK(wait);
	930	+
	931	+ WARN_ON_ONCE(!test_bit(QUEUE_FLAG_POLL, &q->queue_flags));
	932	+
	933	+ rq->cmd_flags \|= REQ_HIPRI;
	934	+ rq->end_io_data = &wait;
	935	+ blk_execute_rq_nowait(q, bd_disk, rq, at_head, nvme_end_sync_rq);
	936	+
	937	+ while (!completion_done(&wait)) {
	938	+ blk_poll(q, request_to_qc_t(rq->mq_hctx, rq), true);
	939	+ cond_resched();
	940	+ }
	941	+}
717	942
718	943	/*
719	944	* Returns 0 on success. If the result is negative, it's a Linux error code;
..	..	@@ -722,16 +947,20 @@
722	947	int __nvme_submit_sync_cmd(struct request_queue q, struct nvme_command cmd,
723	948	union nvme_result result, void buffer, unsigned bufflen,
724	949	unsigned timeout, int qid, int at_head,
725		- blk_mq_req_flags_t flags)
	950	+ blk_mq_req_flags_t flags, bool poll)
726	951	{
727	952	struct request *req;
728	953	int ret;
729	954
730		- req = nvme_alloc_request(q, cmd, flags, qid);
	955	+ if (qid == NVME_QID_ANY)
	956	+ req = nvme_alloc_request(q, cmd, flags);
	957	+ else
	958	+ req = nvme_alloc_request_qid(q, cmd, flags, qid);
731	959	if (IS_ERR(req))
732	960	return PTR_ERR(req);
733	961
734		- req->timeout = timeout ? timeout : ADMIN_TIMEOUT;
	962	+ if (timeout)
	963	+ req->timeout = timeout;
735	964
736	965	if (buffer && bufflen) {
737	966	ret = blk_rq_map_kern(q, req, buffer, bufflen, GFP_KERNEL);
..	..	@@ -739,7 +968,10 @@
739	968	goto out;
740	969	}
741	970
742		- blk_execute_rq(req->q, NULL, req, at_head);
	971	+ if (poll)
	972	+ nvme_execute_rq_polled(req->q, NULL, req, at_head);
	973	+ else
	974	+ blk_execute_rq(req->q, NULL, req, at_head);
743	975	if (result)
744	976	*result = nvme_req(req)->result;
745	977	if (nvme_req(req)->flags & NVME_REQ_CANCELLED)
..	..	@@ -756,7 +988,7 @@
756	988	void *buffer, unsigned bufflen)
757	989	{
758	990	return __nvme_submit_sync_cmd(q, cmd, NULL, buffer, bufflen, 0,
759		- NVME_QID_ANY, 0, 0);
	991	+ NVME_QID_ANY, 0, 0, false);
760	992	}
761	993	EXPORT_SYMBOL_GPL(nvme_submit_sync_cmd);
762	994
..	..	@@ -794,10 +1026,97 @@
794	1026	return ERR_PTR(ret);
795	1027	}
796	1028
	1029	+static u32 nvme_known_admin_effects(u8 opcode)
	1030	+{
	1031	+ switch (opcode) {
	1032	+ case nvme_admin_format_nvm:
	1033	+ return NVME_CMD_EFFECTS_LBCC \| NVME_CMD_EFFECTS_NCC \|
	1034	+ NVME_CMD_EFFECTS_CSE_MASK;
	1035	+ case nvme_admin_sanitize_nvm:
	1036	+ return NVME_CMD_EFFECTS_LBCC \| NVME_CMD_EFFECTS_CSE_MASK;
	1037	+ default:
	1038	+ break;
	1039	+ }
	1040	+ return 0;
	1041	+}
	1042	+
	1043	+u32 nvme_command_effects(struct nvme_ctrl ctrl, struct nvme_ns ns, u8 opcode)
	1044	+{
	1045	+ u32 effects = 0;
	1046	+
	1047	+ if (ns) {
	1048	+ if (ns->head->effects)
	1049	+ effects = le32_to_cpu(ns->head->effects->iocs[opcode]);
	1050	+ if (effects & ~(NVME_CMD_EFFECTS_CSUPP \| NVME_CMD_EFFECTS_LBCC))
	1051	+ dev_warn(ctrl->device,
	1052	+ "IO command:%02x has unhandled effects:%08x\n",
	1053	+ opcode, effects);
	1054	+ return 0;
	1055	+ }
	1056	+
	1057	+ if (ctrl->effects)
	1058	+ effects = le32_to_cpu(ctrl->effects->acs[opcode]);
	1059	+ effects \|= nvme_known_admin_effects(opcode);
	1060	+
	1061	+ return effects;
	1062	+}
	1063	+EXPORT_SYMBOL_NS_GPL(nvme_command_effects, NVME_TARGET_PASSTHRU);
	1064	+
	1065	+static u32 nvme_passthru_start(struct nvme_ctrl ctrl, struct nvme_ns ns,
	1066	+ u8 opcode)
	1067	+{
	1068	+ u32 effects = nvme_command_effects(ctrl, ns, opcode);
	1069	+
	1070	+ /*
	1071	+ * For simplicity, IO to all namespaces is quiesced even if the command
	1072	+ * effects say only one namespace is affected.
	1073	+ */
	1074	+ if (effects & NVME_CMD_EFFECTS_CSE_MASK) {
	1075	+ mutex_lock(&ctrl->scan_lock);
	1076	+ mutex_lock(&ctrl->subsys->lock);
	1077	+ nvme_mpath_start_freeze(ctrl->subsys);
	1078	+ nvme_mpath_wait_freeze(ctrl->subsys);
	1079	+ nvme_start_freeze(ctrl);
	1080	+ nvme_wait_freeze(ctrl);
	1081	+ }
	1082	+ return effects;
	1083	+}
	1084	+
	1085	+static void nvme_passthru_end(struct nvme_ctrl *ctrl, u32 effects)
	1086	+{
	1087	+ if (effects & NVME_CMD_EFFECTS_CSE_MASK) {
	1088	+ nvme_unfreeze(ctrl);
	1089	+ nvme_mpath_unfreeze(ctrl->subsys);
	1090	+ mutex_unlock(&ctrl->subsys->lock);
	1091	+ nvme_remove_invalid_namespaces(ctrl, NVME_NSID_ALL);
	1092	+ mutex_unlock(&ctrl->scan_lock);
	1093	+ }
	1094	+ if (effects & NVME_CMD_EFFECTS_CCC)
	1095	+ nvme_init_identify(ctrl);
	1096	+ if (effects & (NVME_CMD_EFFECTS_NIC \| NVME_CMD_EFFECTS_NCC)) {
	1097	+ nvme_queue_scan(ctrl);
	1098	+ flush_work(&ctrl->scan_work);
	1099	+ }
	1100	+}
	1101	+
	1102	+void nvme_execute_passthru_rq(struct request *rq)
	1103	+{
	1104	+ struct nvme_command *cmd = nvme_req(rq)->cmd;
	1105	+ struct nvme_ctrl *ctrl = nvme_req(rq)->ctrl;
	1106	+ struct nvme_ns *ns = rq->q->queuedata;
	1107	+ struct gendisk *disk = ns ? ns->disk : NULL;
	1108	+ u32 effects;
	1109	+
	1110	+ effects = nvme_passthru_start(ctrl, ns, cmd->common.opcode);
	1111	+ blk_execute_rq(rq->q, disk, rq, 0);
	1112	+ nvme_passthru_end(ctrl, effects);
	1113	+}
	1114	+EXPORT_SYMBOL_NS_GPL(nvme_execute_passthru_rq, NVME_TARGET_PASSTHRU);
	1115	+
797	1116	static int nvme_submit_user_cmd(struct request_queue *q,
798	1117	struct nvme_command cmd, void __user ubuffer,
799	1118	unsigned bufflen, void __user *meta_buffer, unsigned meta_len,
800		- u32 meta_seed, u32 *result, unsigned timeout)
	1119	+ u32 meta_seed, u64 *result, unsigned timeout)
801	1120	{
802	1121	bool write = nvme_is_write(cmd);
803	1122	struct nvme_ns *ns = q->queuedata;
..	..	@@ -807,11 +1126,12 @@
807	1126	void *meta = NULL;
808	1127	int ret;
809	1128
810		- req = nvme_alloc_request(q, cmd, 0, NVME_QID_ANY);
	1129	+ req = nvme_alloc_request(q, cmd, 0);
811	1130	if (IS_ERR(req))
812	1131	return PTR_ERR(req);
813	1132
814		- req->timeout = timeout ? timeout : ADMIN_TIMEOUT;
	1133	+ if (timeout)
	1134	+ req->timeout = timeout;
815	1135	nvme_req(req)->flags \|= NVME_REQ_USERCMD;
816	1136
817	1137	if (ubuffer && bufflen) {
..	..	@@ -832,13 +1152,13 @@
832	1152	}
833	1153	}
834	1154
835		- blk_execute_rq(req->q, disk, req, 0);
	1155	+ nvme_execute_passthru_rq(req);
836	1156	if (nvme_req(req)->flags & NVME_REQ_CANCELLED)
837	1157	ret = -EINTR;
838	1158	else
839	1159	ret = nvme_req(req)->status;
840	1160	if (result)
841		- *result = le32_to_cpu(nvme_req(req)->result.u32);
	1161	+ *result = le64_to_cpu(nvme_req(req)->result.u64);
842	1162	if (meta && !ret && !write) {
843	1163	if (copy_to_user(meta_buffer, meta, meta_len))
844	1164	ret = -EFAULT;
..	..	@@ -867,21 +1187,22 @@
867	1187	return;
868	1188	}
869	1189
	1190	+ ctrl->comp_seen = false;
870	1191	spin_lock_irqsave(&ctrl->lock, flags);
871	1192	if (ctrl->state == NVME_CTRL_LIVE \|\|
872	1193	ctrl->state == NVME_CTRL_CONNECTING)
873	1194	startka = true;
874	1195	spin_unlock_irqrestore(&ctrl->lock, flags);
875	1196	if (startka)
876		- schedule_delayed_work(&ctrl->ka_work, ctrl->kato * HZ);
	1197	+ queue_delayed_work(nvme_wq, &ctrl->ka_work, ctrl->kato * HZ);
877	1198	}
878	1199
879	1200	static int nvme_keep_alive(struct nvme_ctrl *ctrl)
880	1201	{
881	1202	struct request *rq;
882	1203
883		- rq = nvme_alloc_request(ctrl->admin_q, &ctrl->ka_cmd, BLK_MQ_REQ_RESERVED,
884		- NVME_QID_ANY);
	1204	+ rq = nvme_alloc_request(ctrl->admin_q, &ctrl->ka_cmd,
	1205	+ BLK_MQ_REQ_RESERVED);
885	1206	if (IS_ERR(rq))
886	1207	return PTR_ERR(rq);
887	1208
..	..	@@ -897,6 +1218,15 @@
897	1218	{
898	1219	struct nvme_ctrl *ctrl = container_of(to_delayed_work(work),
899	1220	struct nvme_ctrl, ka_work);
	1221	+ bool comp_seen = ctrl->comp_seen;
	1222	+
	1223	+ if ((ctrl->ctratt & NVME_CTRL_ATTR_TBKAS) && comp_seen) {
	1224	+ dev_dbg(ctrl->device,
	1225	+ "reschedule traffic based keep-alive timer\n");
	1226	+ ctrl->comp_seen = false;
	1227	+ queue_delayed_work(nvme_wq, &ctrl->ka_work, ctrl->kato * HZ);
	1228	+ return;
	1229	+ }
900	1230
901	1231	if (nvme_keep_alive(ctrl)) {
902	1232	/* allocation failure, reset the controller */
..	..	@@ -911,7 +1241,7 @@
911	1241	if (unlikely(ctrl->kato == 0))
912	1242	return;
913	1243
914		- schedule_delayed_work(&ctrl->ka_work, ctrl->kato * HZ);
	1244	+ queue_delayed_work(nvme_wq, &ctrl->ka_work, ctrl->kato * HZ);
915	1245	}
916	1246
917	1247	void nvme_stop_keep_alive(struct nvme_ctrl *ctrl)
..	..	@@ -956,14 +1286,75 @@
956	1286	return error;
957	1287	}
958	1288
	1289	+static bool nvme_multi_css(struct nvme_ctrl *ctrl)
	1290	+{
	1291	+ return (ctrl->ctrl_config & NVME_CC_CSS_MASK) == NVME_CC_CSS_CSI;
	1292	+}
	1293	+
	1294	+static int nvme_process_ns_desc(struct nvme_ctrl ctrl, struct nvme_ns_ids ids,
	1295	+ struct nvme_ns_id_desc cur, bool csi_seen)
	1296	+{
	1297	+ const char *warn_str = "ctrl returned bogus length:";
	1298	+ void *data = cur;
	1299	+
	1300	+ switch (cur->nidt) {
	1301	+ case NVME_NIDT_EUI64:
	1302	+ if (cur->nidl != NVME_NIDT_EUI64_LEN) {
	1303	+ dev_warn(ctrl->device, "%s %d for NVME_NIDT_EUI64\n",
	1304	+ warn_str, cur->nidl);
	1305	+ return -1;
	1306	+ }
	1307	+ if (ctrl->quirks & NVME_QUIRK_BOGUS_NID)
	1308	+ return NVME_NIDT_EUI64_LEN;
	1309	+ memcpy(ids->eui64, data + sizeof(*cur), NVME_NIDT_EUI64_LEN);
	1310	+ return NVME_NIDT_EUI64_LEN;
	1311	+ case NVME_NIDT_NGUID:
	1312	+ if (cur->nidl != NVME_NIDT_NGUID_LEN) {
	1313	+ dev_warn(ctrl->device, "%s %d for NVME_NIDT_NGUID\n",
	1314	+ warn_str, cur->nidl);
	1315	+ return -1;
	1316	+ }
	1317	+ if (ctrl->quirks & NVME_QUIRK_BOGUS_NID)
	1318	+ return NVME_NIDT_NGUID_LEN;
	1319	+ memcpy(ids->nguid, data + sizeof(*cur), NVME_NIDT_NGUID_LEN);
	1320	+ return NVME_NIDT_NGUID_LEN;
	1321	+ case NVME_NIDT_UUID:
	1322	+ if (cur->nidl != NVME_NIDT_UUID_LEN) {
	1323	+ dev_warn(ctrl->device, "%s %d for NVME_NIDT_UUID\n",
	1324	+ warn_str, cur->nidl);
	1325	+ return -1;
	1326	+ }
	1327	+ if (ctrl->quirks & NVME_QUIRK_BOGUS_NID)
	1328	+ return NVME_NIDT_UUID_LEN;
	1329	+ uuid_copy(&ids->uuid, data + sizeof(*cur));
	1330	+ return NVME_NIDT_UUID_LEN;
	1331	+ case NVME_NIDT_CSI:
	1332	+ if (cur->nidl != NVME_NIDT_CSI_LEN) {
	1333	+ dev_warn(ctrl->device, "%s %d for NVME_NIDT_CSI\n",
	1334	+ warn_str, cur->nidl);
	1335	+ return -1;
	1336	+ }
	1337	+ memcpy(&ids->csi, data + sizeof(*cur), NVME_NIDT_CSI_LEN);
	1338	+ *csi_seen = true;
	1339	+ return NVME_NIDT_CSI_LEN;
	1340	+ default:
	1341	+ /* Skip unknown types */
	1342	+ return cur->nidl;
	1343	+ }
	1344	+}
	1345	+
959	1346	static int nvme_identify_ns_descs(struct nvme_ctrl *ctrl, unsigned nsid,
960	1347	struct nvme_ns_ids *ids)
961	1348	{
962	1349	struct nvme_command c = { };
963		- int status;
	1350	+ bool csi_seen = false;
	1351	+ int status, pos, len;
964	1352	void *data;
965		- int pos;
966		- int len;
	1353	+
	1354	+ if (ctrl->vs < NVME_VS(1, 3, 0) && !nvme_multi_css(ctrl))
	1355	+ return 0;
	1356	+ if (ctrl->quirks & NVME_QUIRK_NO_NS_DESC_LIST)
	1357	+ return 0;
967	1358
968	1359	c.identify.opcode = nvme_admin_identify;
969	1360	c.identify.nsid = cpu_to_le32(nsid);
..	..	@@ -975,8 +1366,11 @@
975	1366
976	1367	status = nvme_submit_sync_cmd(ctrl->admin_q, &c, data,
977	1368	NVME_IDENTIFY_DATA_SIZE);
978		- if (status)
	1369	+ if (status) {
	1370	+ dev_warn(ctrl->device,
	1371	+ "Identify Descriptors failed (%d)\n", status);
979	1372	goto free_data;
	1373	+ }
980	1374
981	1375	for (pos = 0; pos < NVME_IDENTIFY_DATA_SIZE; pos += len) {
982	1376	struct nvme_ns_id_desc *cur = data + pos;
..	..	@@ -984,65 +1378,27 @@
984	1378	if (cur->nidl == 0)
985	1379	break;
986	1380
987		- switch (cur->nidt) {
988		- case NVME_NIDT_EUI64:
989		- if (cur->nidl != NVME_NIDT_EUI64_LEN) {
990		- dev_warn(ctrl->device,
991		- "ctrl returned bogus length: %d for NVME_NIDT_EUI64\n",
992		- cur->nidl);
993		- goto free_data;
994		- }
995		- len = NVME_NIDT_EUI64_LEN;
996		- memcpy(ids->eui64, data + pos + sizeof(*cur), len);
	1381	+ len = nvme_process_ns_desc(ctrl, ids, cur, &csi_seen);
	1382	+ if (len < 0)
997	1383	break;
998		- case NVME_NIDT_NGUID:
999		- if (cur->nidl != NVME_NIDT_NGUID_LEN) {
1000		- dev_warn(ctrl->device,
1001		- "ctrl returned bogus length: %d for NVME_NIDT_NGUID\n",
1002		- cur->nidl);
1003		- goto free_data;
1004		- }
1005		- len = NVME_NIDT_NGUID_LEN;
1006		- memcpy(ids->nguid, data + pos + sizeof(*cur), len);
1007		- break;
1008		- case NVME_NIDT_UUID:
1009		- if (cur->nidl != NVME_NIDT_UUID_LEN) {
1010		- dev_warn(ctrl->device,
1011		- "ctrl returned bogus length: %d for NVME_NIDT_UUID\n",
1012		- cur->nidl);
1013		- goto free_data;
1014		- }
1015		- len = NVME_NIDT_UUID_LEN;
1016		- uuid_copy(&ids->uuid, data + pos + sizeof(*cur));
1017		- break;
1018		- default:
1019		- /* Skip unnkown types */
1020		- len = cur->nidl;
1021		- break;
1022		- }
1023	1384
1024	1385	len += sizeof(*cur);
1025	1386	}
	1387	+
	1388	+ if (nvme_multi_css(ctrl) && !csi_seen) {
	1389	+ dev_warn(ctrl->device, "Command set not reported for nsid:%d\n",
	1390	+ nsid);
	1391	+ status = -EINVAL;
	1392	+ }
	1393	+
1026	1394	free_data:
1027	1395	kfree(data);
1028	1396	return status;
1029	1397	}
1030	1398
1031		-static int nvme_identify_ns_list(struct nvme_ctrl dev, unsigned nsid, __le32 ns_list)
	1399	+static int nvme_identify_ns(struct nvme_ctrl *ctrl, unsigned nsid,
	1400	+ struct nvme_ns_ids ids, struct nvme_id_ns *id)
1032	1401	{
1033		- struct nvme_command c = { };
1034		-
1035		- c.identify.opcode = nvme_admin_identify;
1036		- c.identify.cns = NVME_ID_CNS_NS_ACTIVE_LIST;
1037		- c.identify.nsid = cpu_to_le32(nsid);
1038		- return nvme_submit_sync_cmd(dev->admin_q, &c, ns_list,
1039		- NVME_IDENTIFY_DATA_SIZE);
1040		-}
1041		-
1042		-static struct nvme_id_ns nvme_identify_ns(struct nvme_ctrl ctrl,
1043		- unsigned nsid)
1044		-{
1045		- struct nvme_id_ns *id;
1046	1402	struct nvme_command c = { };
1047	1403	int error;
1048	1404
..	..	@@ -1051,38 +1407,76 @@
1051	1407	c.identify.nsid = cpu_to_le32(nsid);
1052	1408	c.identify.cns = NVME_ID_CNS_NS;
1053	1409
1054		- id = kmalloc(sizeof(*id), GFP_KERNEL);
1055		- if (!id)
1056		- return NULL;
	1410	+ id = kmalloc(sizeof(*id), GFP_KERNEL);
	1411	+ if (!*id)
	1412	+ return -ENOMEM;
1057	1413
1058		- error = nvme_submit_sync_cmd(ctrl->admin_q, &c, id, sizeof(*id));
	1414	+ error = nvme_submit_sync_cmd(ctrl->admin_q, &c, id, sizeof(*id));
1059	1415	if (error) {
1060		- dev_warn(ctrl->device, "Identify namespace failed\n");
1061		- kfree(id);
1062		- return NULL;
	1416	+ dev_warn(ctrl->device, "Identify namespace failed (%d)\n", error);
	1417	+ goto out_free_id;
1063	1418	}
1064	1419
1065		- return id;
	1420	+ error = NVME_SC_INVALID_NS \| NVME_SC_DNR;
	1421	+ if ((id)->ncap == 0) / namespace not allocated or attached */
	1422	+ goto out_free_id;
	1423	+
	1424	+
	1425	+ if (ctrl->quirks & NVME_QUIRK_BOGUS_NID) {
	1426	+ dev_info(ctrl->device,
	1427	+ "Ignoring bogus Namespace Identifiers\n");
	1428	+ } else {
	1429	+ if (ctrl->vs >= NVME_VS(1, 1, 0) &&
	1430	+ !memchr_inv(ids->eui64, 0, sizeof(ids->eui64)))
	1431	+ memcpy(ids->eui64, (*id)->eui64, sizeof(ids->eui64));
	1432	+ if (ctrl->vs >= NVME_VS(1, 2, 0) &&
	1433	+ !memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
	1434	+ memcpy(ids->nguid, (*id)->nguid, sizeof(ids->nguid));
	1435	+ }
	1436	+
	1437	+ return 0;
	1438	+
	1439	+out_free_id:
	1440	+ kfree(*id);
	1441	+ return error;
1066	1442	}
1067	1443
1068		-static int nvme_set_features(struct nvme_ctrl *dev, unsigned fid, unsigned dword11,
1069		- void buffer, size_t buflen, u32 result)
	1444	+static int nvme_features(struct nvme_ctrl *dev, u8 op, unsigned int fid,
	1445	+ unsigned int dword11, void buffer, size_t buflen, u32 result)
1070	1446	{
1071	1447	union nvme_result res = { 0 };
1072	1448	struct nvme_command c;
1073	1449	int ret;
1074	1450
1075	1451	memset(&c, 0, sizeof(c));
1076		- c.features.opcode = nvme_admin_set_features;
	1452	+ c.features.opcode = op;
1077	1453	c.features.fid = cpu_to_le32(fid);
1078	1454	c.features.dword11 = cpu_to_le32(dword11);
1079	1455
1080	1456	ret = __nvme_submit_sync_cmd(dev->admin_q, &c, &res,
1081		- buffer, buflen, 0, NVME_QID_ANY, 0, 0);
	1457	+ buffer, buflen, 0, NVME_QID_ANY, 0, 0, false);
1082	1458	if (ret >= 0 && result)
1083	1459	*result = le32_to_cpu(res.u32);
1084	1460	return ret;
1085	1461	}
	1462	+
	1463	+int nvme_set_features(struct nvme_ctrl *dev, unsigned int fid,
	1464	+ unsigned int dword11, void *buffer, size_t buflen,
	1465	+ u32 *result)
	1466	+{
	1467	+ return nvme_features(dev, nvme_admin_set_features, fid, dword11, buffer,
	1468	+ buflen, result);
	1469	+}
	1470	+EXPORT_SYMBOL_GPL(nvme_set_features);
	1471	+
	1472	+int nvme_get_features(struct nvme_ctrl *dev, unsigned int fid,
	1473	+ unsigned int dword11, void *buffer, size_t buflen,
	1474	+ u32 *result)
	1475	+{
	1476	+ return nvme_features(dev, nvme_admin_get_features, fid, dword11, buffer,
	1477	+ buflen, result);
	1478	+}
	1479	+EXPORT_SYMBOL_GPL(nvme_get_features);
1086	1480
1087	1481	int nvme_set_queue_count(struct nvme_ctrl ctrl, int count)
1088	1482	{
..	..	@@ -1113,7 +1507,8 @@
1113	1507	EXPORT_SYMBOL_GPL(nvme_set_queue_count);
1114	1508
1115	1509	#define NVME_AEN_SUPPORTED \
1116		- (NVME_AEN_CFG_NS_ATTR \| NVME_AEN_CFG_FW_ACT \| NVME_AEN_CFG_ANA_CHANGE)
	1510	+ (NVME_AEN_CFG_NS_ATTR \| NVME_AEN_CFG_FW_ACT \| \
	1511	+ NVME_AEN_CFG_ANA_CHANGE \| NVME_AEN_CFG_DISC_CHANGE)
1117	1512
1118	1513	static void nvme_enable_aen(struct nvme_ctrl *ctrl)
1119	1514	{
..	..	@@ -1128,6 +1523,20 @@
1128	1523	if (status)
1129	1524	dev_warn(ctrl->device, "Failed to configure AEN (cfg %x)\n",
1130	1525	supported_aens);
	1526	+
	1527	+ queue_work(nvme_wq, &ctrl->async_event_work);
	1528	+}
	1529	+
	1530	+/*
	1531	+ * Convert integer values from ioctl structures to user pointers, silently
	1532	+ * ignoring the upper bits in the compat case to match behaviour of 32-bit
	1533	+ * kernels.
	1534	+ */
	1535	+static void __user *nvme_to_user_ptr(uintptr_t ptrval)
	1536	+{
	1537	+ if (in_compat_syscall())
	1538	+ ptrval = (compat_uptr_t)ptrval;
	1539	+ return (void __user *)ptrval;
1131	1540	}
1132	1541
1133	1542	static int nvme_submit_io(struct nvme_ns ns, struct nvme_user_io __user uio)
..	..	@@ -1152,10 +1561,23 @@
1152	1561	}
1153	1562
1154	1563	length = (io.nblocks + 1) << ns->lba_shift;
1155		- meta_len = (io.nblocks + 1) * ns->ms;
1156		- metadata = (void __user *)(uintptr_t)io.metadata;
1157	1564
1158		- if (ns->ext) {
	1565	+ if ((io.control & NVME_RW_PRINFO_PRACT) &&
	1566	+ ns->ms == sizeof(struct t10_pi_tuple)) {
	1567	+ /*
	1568	+ * Protection information is stripped/inserted by the
	1569	+ * controller.
	1570	+ */
	1571	+ if (nvme_to_user_ptr(io.metadata))
	1572	+ return -EINVAL;
	1573	+ meta_len = 0;
	1574	+ metadata = NULL;
	1575	+ } else {
	1576	+ meta_len = (io.nblocks + 1) * ns->ms;
	1577	+ metadata = nvme_to_user_ptr(io.metadata);
	1578	+ }
	1579	+
	1580	+ if (ns->features & NVME_NS_EXT_LBAS) {
1159	1581	length += meta_len;
1160	1582	meta_len = 0;
1161	1583	} else if (meta_len) {
..	..	@@ -1176,91 +1598,8 @@
1176	1598	c.rw.appmask = cpu_to_le16(io.appmask);
1177	1599
1178	1600	return nvme_submit_user_cmd(ns->queue, &c,
1179		- (void __user *)(uintptr_t)io.addr, length,
1180		- metadata, meta_len, io.slba, NULL, 0);
1181		-}
1182		-
1183		-static u32 nvme_known_admin_effects(u8 opcode)
1184		-{
1185		- switch (opcode) {
1186		- case nvme_admin_format_nvm:
1187		- return NVME_CMD_EFFECTS_CSUPP \| NVME_CMD_EFFECTS_LBCC \|
1188		- NVME_CMD_EFFECTS_CSE_MASK;
1189		- case nvme_admin_sanitize_nvm:
1190		- return NVME_CMD_EFFECTS_CSE_MASK;
1191		- default:
1192		- break;
1193		- }
1194		- return 0;
1195		-}
1196		-
1197		-static u32 nvme_passthru_start(struct nvme_ctrl ctrl, struct nvme_ns ns,
1198		- u8 opcode)
1199		-{
1200		- u32 effects = 0;
1201		-
1202		- if (ns) {
1203		- if (ctrl->effects)
1204		- effects = le32_to_cpu(ctrl->effects->iocs[opcode]);
1205		- if (effects & ~NVME_CMD_EFFECTS_CSUPP)
1206		- dev_warn(ctrl->device,
1207		- "IO command:%02x has unhandled effects:%08x\n",
1208		- opcode, effects);
1209		- return 0;
1210		- }
1211		-
1212		- if (ctrl->effects)
1213		- effects = le32_to_cpu(ctrl->effects->acs[opcode]);
1214		- else
1215		- effects = nvme_known_admin_effects(opcode);
1216		-
1217		- /*
1218		- * For simplicity, IO to all namespaces is quiesced even if the command
1219		- * effects say only one namespace is affected.
1220		- */
1221		- if (effects & (NVME_CMD_EFFECTS_LBCC \| NVME_CMD_EFFECTS_CSE_MASK)) {
1222		- mutex_lock(&ctrl->scan_lock);
1223		- mutex_lock(&ctrl->subsys->lock);
1224		- nvme_mpath_start_freeze(ctrl->subsys);
1225		- nvme_mpath_wait_freeze(ctrl->subsys);
1226		- nvme_start_freeze(ctrl);
1227		- nvme_wait_freeze(ctrl);
1228		- }
1229		- return effects;
1230		-}
1231		-
1232		-static void nvme_update_formats(struct nvme_ctrl *ctrl)
1233		-{
1234		- struct nvme_ns *ns;
1235		-
1236		- down_read(&ctrl->namespaces_rwsem);
1237		- list_for_each_entry(ns, &ctrl->namespaces, list)
1238		- if (ns->disk && nvme_revalidate_disk(ns->disk))
1239		- nvme_set_queue_dying(ns);
1240		- up_read(&ctrl->namespaces_rwsem);
1241		-
1242		- nvme_remove_invalid_namespaces(ctrl, NVME_NSID_ALL);
1243		-}
1244		-
1245		-static void nvme_passthru_end(struct nvme_ctrl *ctrl, u32 effects)
1246		-{
1247		- /*
1248		- * Revalidate LBA changes prior to unfreezing. This is necessary to
1249		- * prevent memory corruption if a logical block size was changed by
1250		- * this command.
1251		- */
1252		- if (effects & NVME_CMD_EFFECTS_LBCC)
1253		- nvme_update_formats(ctrl);
1254		- if (effects & (NVME_CMD_EFFECTS_LBCC \| NVME_CMD_EFFECTS_CSE_MASK)) {
1255		- nvme_unfreeze(ctrl);
1256		- nvme_mpath_unfreeze(ctrl->subsys);
1257		- mutex_unlock(&ctrl->subsys->lock);
1258		- mutex_unlock(&ctrl->scan_lock);
1259		- }
1260		- if (effects & NVME_CMD_EFFECTS_CCC)
1261		- nvme_init_identify(ctrl);
1262		- if (effects & (NVME_CMD_EFFECTS_NIC \| NVME_CMD_EFFECTS_NCC))
1263		- nvme_queue_scan(ctrl);
	1601	+ nvme_to_user_ptr(io.addr), length,
	1602	+ metadata, meta_len, lower_32_bits(io.slba), NULL, 0);
1264	1603	}
1265	1604
1266	1605	static int nvme_user_cmd(struct nvme_ctrl ctrl, struct nvme_ns ns,
..	..	@@ -1269,7 +1608,7 @@
1269	1608	struct nvme_passthru_cmd cmd;
1270	1609	struct nvme_command c;
1271	1610	unsigned timeout = 0;
1272		- u32 effects;
	1611	+ u64 result;
1273	1612	int status;
1274	1613
1275	1614	if (!capable(CAP_SYS_ADMIN))
..	..	@@ -1285,22 +1624,64 @@
1285	1624	c.common.nsid = cpu_to_le32(cmd.nsid);
1286	1625	c.common.cdw2[0] = cpu_to_le32(cmd.cdw2);
1287	1626	c.common.cdw2[1] = cpu_to_le32(cmd.cdw3);
1288		- c.common.cdw10[0] = cpu_to_le32(cmd.cdw10);
1289		- c.common.cdw10[1] = cpu_to_le32(cmd.cdw11);
1290		- c.common.cdw10[2] = cpu_to_le32(cmd.cdw12);
1291		- c.common.cdw10[3] = cpu_to_le32(cmd.cdw13);
1292		- c.common.cdw10[4] = cpu_to_le32(cmd.cdw14);
1293		- c.common.cdw10[5] = cpu_to_le32(cmd.cdw15);
	1627	+ c.common.cdw10 = cpu_to_le32(cmd.cdw10);
	1628	+ c.common.cdw11 = cpu_to_le32(cmd.cdw11);
	1629	+ c.common.cdw12 = cpu_to_le32(cmd.cdw12);
	1630	+ c.common.cdw13 = cpu_to_le32(cmd.cdw13);
	1631	+ c.common.cdw14 = cpu_to_le32(cmd.cdw14);
	1632	+ c.common.cdw15 = cpu_to_le32(cmd.cdw15);
1294	1633
1295	1634	if (cmd.timeout_ms)
1296	1635	timeout = msecs_to_jiffies(cmd.timeout_ms);
1297	1636
1298		- effects = nvme_passthru_start(ctrl, ns, cmd.opcode);
1299	1637	status = nvme_submit_user_cmd(ns ? ns->queue : ctrl->admin_q, &c,
1300		- (void __user *)(uintptr_t)cmd.addr, cmd.data_len,
1301		- (void __user *)(uintptr_t)cmd.metadata, cmd.metadata_len,
	1638	+ nvme_to_user_ptr(cmd.addr), cmd.data_len,
	1639	+ nvme_to_user_ptr(cmd.metadata), cmd.metadata_len,
	1640	+ 0, &result, timeout);
	1641	+
	1642	+ if (status >= 0) {
	1643	+ if (put_user(result, &ucmd->result))
	1644	+ return -EFAULT;
	1645	+ }
	1646	+
	1647	+ return status;
	1648	+}
	1649	+
	1650	+static int nvme_user_cmd64(struct nvme_ctrl ctrl, struct nvme_ns ns,
	1651	+ struct nvme_passthru_cmd64 __user *ucmd)
	1652	+{
	1653	+ struct nvme_passthru_cmd64 cmd;
	1654	+ struct nvme_command c;
	1655	+ unsigned timeout = 0;
	1656	+ int status;
	1657	+
	1658	+ if (!capable(CAP_SYS_ADMIN))
	1659	+ return -EACCES;
	1660	+ if (copy_from_user(&cmd, ucmd, sizeof(cmd)))
	1661	+ return -EFAULT;
	1662	+ if (cmd.flags)
	1663	+ return -EINVAL;
	1664	+
	1665	+ memset(&c, 0, sizeof(c));
	1666	+ c.common.opcode = cmd.opcode;
	1667	+ c.common.flags = cmd.flags;
	1668	+ c.common.nsid = cpu_to_le32(cmd.nsid);
	1669	+ c.common.cdw2[0] = cpu_to_le32(cmd.cdw2);
	1670	+ c.common.cdw2[1] = cpu_to_le32(cmd.cdw3);
	1671	+ c.common.cdw10 = cpu_to_le32(cmd.cdw10);
	1672	+ c.common.cdw11 = cpu_to_le32(cmd.cdw11);
	1673	+ c.common.cdw12 = cpu_to_le32(cmd.cdw12);
	1674	+ c.common.cdw13 = cpu_to_le32(cmd.cdw13);
	1675	+ c.common.cdw14 = cpu_to_le32(cmd.cdw14);
	1676	+ c.common.cdw15 = cpu_to_le32(cmd.cdw15);
	1677	+
	1678	+ if (cmd.timeout_ms)
	1679	+ timeout = msecs_to_jiffies(cmd.timeout_ms);
	1680	+
	1681	+ status = nvme_submit_user_cmd(ns ? ns->queue : ctrl->admin_q, &c,
	1682	+ nvme_to_user_ptr(cmd.addr), cmd.data_len,
	1683	+ nvme_to_user_ptr(cmd.metadata), cmd.metadata_len,
1302	1684	0, &cmd.result, timeout);
1303		- nvme_passthru_end(ctrl, effects);
1304	1685
1305	1686	if (status >= 0) {
1306	1687	if (put_user(cmd.result, &ucmd->result))
..	..	@@ -1314,7 +1695,7 @@
1314	1695	* Issue ioctl requests on the first available path. Note that unlike normal
1315	1696	* block layer requests we will not retry failed request on another controller.
1316	1697	*/
1317		-static struct nvme_ns nvme_get_ns_from_disk(struct gendisk disk,
	1698	+struct nvme_ns nvme_get_ns_from_disk(struct gendisk disk,
1318	1699	struct nvme_ns_head *head, int srcu_idx)
1319	1700	{
1320	1701	#ifdef CONFIG_NVME_MULTIPATH
..	..	@@ -1334,10 +1715,45 @@
1334	1715	return disk->private_data;
1335	1716	}
1336	1717
1337		-static void nvme_put_ns_from_disk(struct nvme_ns_head *head, int idx)
	1718	+void nvme_put_ns_from_disk(struct nvme_ns_head *head, int idx)
1338	1719	{
1339	1720	if (head)
1340	1721	srcu_read_unlock(&head->srcu, idx);
	1722	+}
	1723	+
	1724	+static bool is_ctrl_ioctl(unsigned int cmd)
	1725	+{
	1726	+ if (cmd == NVME_IOCTL_ADMIN_CMD \|\| cmd == NVME_IOCTL_ADMIN64_CMD)
	1727	+ return true;
	1728	+ if (is_sed_ioctl(cmd))
	1729	+ return true;
	1730	+ return false;
	1731	+}
	1732	+
	1733	+static int nvme_handle_ctrl_ioctl(struct nvme_ns *ns, unsigned int cmd,
	1734	+ void __user *argp,
	1735	+ struct nvme_ns_head *head,
	1736	+ int srcu_idx)
	1737	+{
	1738	+ struct nvme_ctrl *ctrl = ns->ctrl;
	1739	+ int ret;
	1740	+
	1741	+ nvme_get_ctrl(ns->ctrl);
	1742	+ nvme_put_ns_from_disk(head, srcu_idx);
	1743	+
	1744	+ switch (cmd) {
	1745	+ case NVME_IOCTL_ADMIN_CMD:
	1746	+ ret = nvme_user_cmd(ctrl, NULL, argp);
	1747	+ break;
	1748	+ case NVME_IOCTL_ADMIN64_CMD:
	1749	+ ret = nvme_user_cmd64(ctrl, NULL, argp);
	1750	+ break;
	1751	+ default:
	1752	+ ret = sed_ioctl(ctrl->opal_dev, cmd, argp);
	1753	+ break;
	1754	+ }
	1755	+ nvme_put_ctrl(ctrl);
	1756	+ return ret;
1341	1757	}
1342	1758
1343	1759	static int nvme_ioctl(struct block_device *bdev, fmode_t mode,
..	..	@@ -1357,20 +1773,8 @@
1357	1773	* seperately and drop the ns SRCU reference early. This avoids a
1358	1774	* deadlock when deleting namespaces using the passthrough interface.
1359	1775	*/
1360		- if (cmd == NVME_IOCTL_ADMIN_CMD \|\| is_sed_ioctl(cmd)) {
1361		- struct nvme_ctrl *ctrl = ns->ctrl;
1362		-
1363		- nvme_get_ctrl(ns->ctrl);
1364		- nvme_put_ns_from_disk(head, srcu_idx);
1365		-
1366		- if (cmd == NVME_IOCTL_ADMIN_CMD)
1367		- ret = nvme_user_cmd(ctrl, NULL, argp);
1368		- else
1369		- ret = sed_ioctl(ctrl->opal_dev, cmd, argp);
1370		-
1371		- nvme_put_ctrl(ctrl);
1372		- return ret;
1373		- }
	1776	+ if (is_ctrl_ioctl(cmd))
	1777	+ return nvme_handle_ctrl_ioctl(ns, cmd, argp, head, srcu_idx);
1374	1778
1375	1779	switch (cmd) {
1376	1780	case NVME_IOCTL_ID:
..	..	@@ -1383,6 +1787,9 @@
1383	1787	case NVME_IOCTL_SUBMIT_IO:
1384	1788	ret = nvme_submit_io(ns, argp);
1385	1789	break;
	1790	+ case NVME_IOCTL_IO64_CMD:
	1791	+ ret = nvme_user_cmd64(ns->ctrl, ns, argp);
	1792	+ break;
1386	1793	default:
1387	1794	if (ns->ndev)
1388	1795	ret = nvme_nvm_ioctl(ns, cmd, arg);
..	..	@@ -1393,6 +1800,47 @@
1393	1800	nvme_put_ns_from_disk(head, srcu_idx);
1394	1801	return ret;
1395	1802	}
	1803	+
	1804	+#ifdef CONFIG_COMPAT
	1805	+struct nvme_user_io32 {
	1806	+ __u8 opcode;
	1807	+ __u8 flags;
	1808	+ __u16 control;
	1809	+ __u16 nblocks;
	1810	+ __u16 rsvd;
	1811	+ __u64 metadata;
	1812	+ __u64 addr;
	1813	+ __u64 slba;
	1814	+ __u32 dsmgmt;
	1815	+ __u32 reftag;
	1816	+ __u16 apptag;
	1817	+ __u16 appmask;
	1818	+} __attribute__((__packed__));
	1819	+
	1820	+#define NVME_IOCTL_SUBMIT_IO32 _IOW('N', 0x42, struct nvme_user_io32)
	1821	+
	1822	+static int nvme_compat_ioctl(struct block_device *bdev, fmode_t mode,
	1823	+ unsigned int cmd, unsigned long arg)
	1824	+{
	1825	+ /*
	1826	+ * Corresponds to the difference of NVME_IOCTL_SUBMIT_IO
	1827	+ * between 32 bit programs and 64 bit kernel.
	1828	+ * The cause is that the results of sizeof(struct nvme_user_io),
	1829	+ * which is used to define NVME_IOCTL_SUBMIT_IO,
	1830	+ * are not same between 32 bit compiler and 64 bit compiler.
	1831	+ * NVME_IOCTL_SUBMIT_IO32 is for 64 bit kernel handling
	1832	+ * NVME_IOCTL_SUBMIT_IO issued from 32 bit programs.
	1833	+ * Other IOCTL numbers are same between 32 bit and 64 bit.
	1834	+ * So there is nothing to do regarding to other IOCTL numbers.
	1835	+ */
	1836	+ if (cmd == NVME_IOCTL_SUBMIT_IO32)
	1837	+ return nvme_ioctl(bdev, mode, NVME_IOCTL_SUBMIT_IO, arg);
	1838	+
	1839	+ return nvme_ioctl(bdev, mode, cmd, arg);
	1840	+}
	1841	+#else
	1842	+#define nvme_compat_ioctl NULL
	1843	+#endif /* CONFIG_COMPAT */
1396	1844
1397	1845	static int nvme_open(struct block_device *bdev, fmode_t mode)
1398	1846	{
..	..	@@ -1434,7 +1882,8 @@
1434	1882	}
1435	1883
1436	1884	#ifdef CONFIG_BLK_DEV_INTEGRITY
1437		-static void nvme_init_integrity(struct gendisk *disk, u16 ms, u8 pi_type)
	1885	+static void nvme_init_integrity(struct gendisk *disk, u16 ms, u8 pi_type,
	1886	+ u32 max_integrity_segments)
1438	1887	{
1439	1888	struct blk_integrity integrity;
1440	1889
..	..	@@ -1457,24 +1906,19 @@
1457	1906	}
1458	1907	integrity.tuple_size = ms;
1459	1908	blk_integrity_register(disk, &integrity);
1460		- blk_queue_max_integrity_segments(disk->queue, 1);
	1909	+ blk_queue_max_integrity_segments(disk->queue, max_integrity_segments);
1461	1910	}
1462	1911	#else
1463		-static void nvme_init_integrity(struct gendisk *disk, u16 ms, u8 pi_type)
	1912	+static void nvme_init_integrity(struct gendisk *disk, u16 ms, u8 pi_type,
	1913	+ u32 max_integrity_segments)
1464	1914	{
1465	1915	}
1466	1916	#endif /* CONFIG_BLK_DEV_INTEGRITY */
1467	1917
1468		-static void nvme_set_chunk_size(struct nvme_ns *ns)
1469		-{
1470		- u32 chunk_size = (((u32)ns->noiob) << (ns->lba_shift - 9));
1471		- blk_queue_chunk_sectors(ns->queue, rounddown_pow_of_two(chunk_size));
1472		-}
1473		-
1474		-static void nvme_config_discard(struct nvme_ns *ns)
	1918	+static void nvme_config_discard(struct gendisk disk, struct nvme_ns ns)
1475	1919	{
1476	1920	struct nvme_ctrl *ctrl = ns->ctrl;
1477		- struct request_queue *queue = ns->queue;
	1921	+ struct request_queue *queue = disk->queue;
1478	1922	u32 size = queue_logical_block_size(queue);
1479	1923
1480	1924	if (!(ctrl->oncs & NVME_CTRL_ONCS_DSM)) {
..	..	@@ -1502,23 +1946,18 @@
1502	1946	blk_queue_max_write_zeroes_sectors(queue, UINT_MAX);
1503	1947	}
1504	1948
1505		-static void nvme_report_ns_ids(struct nvme_ctrl *ctrl, unsigned int nsid,
1506		- struct nvme_id_ns id, struct nvme_ns_ids ids)
	1949	+/*
	1950	+ * Even though NVMe spec explicitly states that MDTS is not applicable to the
	1951	+ * write-zeroes, we are cautious and limit the size to the controllers
	1952	+ * max_hw_sectors value, which is based on the MDTS field and possibly other
	1953	+ * limiting factors.
	1954	+ */
	1955	+static void nvme_config_write_zeroes(struct request_queue *q,
	1956	+ struct nvme_ctrl *ctrl)
1507	1957	{
1508		- memset(ids, 0, sizeof(*ids));
1509		-
1510		- if (ctrl->vs >= NVME_VS(1, 1, 0))
1511		- memcpy(ids->eui64, id->eui64, sizeof(id->eui64));
1512		- if (ctrl->vs >= NVME_VS(1, 2, 0))
1513		- memcpy(ids->nguid, id->nguid, sizeof(id->nguid));
1514		- if (ctrl->vs >= NVME_VS(1, 3, 0)) {
1515		- /* Don't treat error as fatal we potentially
1516		- * already have a NGUID or EUI-64
1517		- */
1518		- if (nvme_identify_ns_descs(ctrl, nsid, ids))
1519		- dev_warn(ctrl->device,
1520		- "%s: Identify Descriptors failed\n", __func__);
1521		- }
	1958	+ if ((ctrl->oncs & NVME_CTRL_ONCS_WRITE_ZEROES) &&
	1959	+ !(ctrl->quirks & NVME_QUIRK_DISABLE_WRITE_ZEROES))
	1960	+ blk_queue_max_write_zeroes_sectors(q, ctrl->max_hw_sectors);
1522	1961	}
1523	1962
1524	1963	static bool nvme_ns_ids_valid(struct nvme_ns_ids *ids)
..	..	@@ -1532,110 +1971,250 @@
1532	1971	{
1533	1972	return uuid_equal(&a->uuid, &b->uuid) &&
1534	1973	memcmp(&a->nguid, &b->nguid, sizeof(a->nguid)) == 0 &&
1535		- memcmp(&a->eui64, &b->eui64, sizeof(a->eui64)) == 0;
	1974	+ memcmp(&a->eui64, &b->eui64, sizeof(a->eui64)) == 0 &&
	1975	+ a->csi == b->csi;
1536	1976	}
1537	1977
1538		-static void nvme_update_disk_info(struct gendisk *disk,
1539		- struct nvme_ns ns, struct nvme_id_ns id)
	1978	+static int nvme_setup_streams_ns(struct nvme_ctrl ctrl, struct nvme_ns ns,
	1979	+ u32 phys_bs, u32 io_opt)
1540	1980	{
1541		- sector_t capacity = le64_to_cpup(&id->nsze) << (ns->lba_shift - 9);
1542		- unsigned short bs = 1 << ns->lba_shift;
	1981	+ struct streams_directive_params s;
	1982	+ int ret;
1543	1983
1544		- if (ns->lba_shift > PAGE_SHIFT) {
1545		- /* unsupported block size, set capacity to 0 later */
1546		- bs = (1 << 9);
	1984	+ if (!ctrl->nr_streams)
	1985	+ return 0;
	1986	+
	1987	+ ret = nvme_get_stream_params(ctrl, &s, ns->head->ns_id);
	1988	+ if (ret)
	1989	+ return ret;
	1990	+
	1991	+ ns->sws = le32_to_cpu(s.sws);
	1992	+ ns->sgs = le16_to_cpu(s.sgs);
	1993	+
	1994	+ if (ns->sws) {
	1995	+ phys_bs = ns->sws (1 << ns->lba_shift);
	1996	+ if (ns->sgs)
	1997	+ io_opt = phys_bs * ns->sgs;
1547	1998	}
1548		- blk_mq_freeze_queue(disk->queue);
1549		- blk_integrity_unregister(disk);
1550	1999
1551		- blk_queue_logical_block_size(disk->queue, bs);
1552		- blk_queue_physical_block_size(disk->queue, bs);
1553		- blk_queue_io_min(disk->queue, bs);
1554		-
1555		- if (ns->ms && !ns->ext &&
1556		- (ns->ctrl->ops->flags & NVME_F_METADATA_SUPPORTED))
1557		- nvme_init_integrity(disk, ns->ms, ns->pi_type);
1558		- if ((ns->ms && !nvme_ns_has_pi(ns) && !blk_get_integrity(disk)) \|\|
1559		- ns->lba_shift > PAGE_SHIFT)
1560		- capacity = 0;
1561		-
1562		- set_capacity(disk, capacity);
1563		- nvme_config_discard(ns);
1564		-
1565		- if (id->nsattr & (1 << 0))
1566		- set_disk_ro(disk, true);
1567		- else
1568		- set_disk_ro(disk, false);
1569		-
1570		- blk_mq_unfreeze_queue(disk->queue);
	2000	+ return 0;
1571	2001	}
1572	2002
1573		-static void __nvme_revalidate_disk(struct gendisk disk, struct nvme_id_ns id)
	2003	+static int nvme_configure_metadata(struct nvme_ns ns, struct nvme_id_ns id)
1574	2004	{
1575		- struct nvme_ns *ns = disk->private_data;
	2005	+ struct nvme_ctrl *ctrl = ns->ctrl;
1576	2006
1577	2007	/*
1578		- * If identify namespace failed, use default 512 byte block size so
1579		- * block layer can use before failing read/write for 0 capacity.
	2008	+ * The PI implementation requires the metadata size to be equal to the
	2009	+ * t10 pi tuple size.
1580	2010	*/
1581		- ns->lba_shift = id->lbaf[id->flbas & NVME_NS_FLBAS_LBA_MASK].ds;
1582		- if (ns->lba_shift == 0)
1583		- ns->lba_shift = 9;
1584		- ns->noiob = le16_to_cpu(id->noiob);
1585	2011	ns->ms = le16_to_cpu(id->lbaf[id->flbas & NVME_NS_FLBAS_LBA_MASK].ms);
1586		- ns->ext = ns->ms && (id->flbas & NVME_NS_FLBAS_META_EXT);
1587		- /* the PI implementation requires metadata equal t10 pi tuple size */
1588	2012	if (ns->ms == sizeof(struct t10_pi_tuple))
1589	2013	ns->pi_type = id->dps & NVME_NS_DPS_PI_MASK;
1590	2014	else
1591	2015	ns->pi_type = 0;
1592	2016
1593		- if (ns->noiob)
1594		- nvme_set_chunk_size(ns);
1595		- nvme_update_disk_info(disk, ns, id);
1596		- if (ns->ndev)
1597		- nvme_nvm_update_nvm_info(ns);
1598		-#ifdef CONFIG_NVME_MULTIPATH
1599		- if (ns->head->disk) {
1600		- nvme_update_disk_info(ns->head->disk, ns, id);
1601		- blk_queue_stack_limits(ns->head->disk->queue, ns->queue);
1602		- nvme_mpath_update_disk_size(ns->head->disk);
	2017	+ ns->features &= ~(NVME_NS_METADATA_SUPPORTED \| NVME_NS_EXT_LBAS);
	2018	+ if (!ns->ms \|\| !(ctrl->ops->flags & NVME_F_METADATA_SUPPORTED))
	2019	+ return 0;
	2020	+ if (ctrl->ops->flags & NVME_F_FABRICS) {
	2021	+ /*
	2022	+ * The NVMe over Fabrics specification only supports metadata as
	2023	+ * part of the extended data LBA. We rely on HCA/HBA support to
	2024	+ * remap the separate metadata buffer from the block layer.
	2025	+ */
	2026	+ if (WARN_ON_ONCE(!(id->flbas & NVME_NS_FLBAS_META_EXT)))
	2027	+ return -EINVAL;
	2028	+ if (ctrl->max_integrity_segments)
	2029	+ ns->features \|=
	2030	+ (NVME_NS_METADATA_SUPPORTED \| NVME_NS_EXT_LBAS);
	2031	+ } else {
	2032	+ /*
	2033	+ * For PCIe controllers, we can't easily remap the separate
	2034	+ * metadata buffer from the block layer and thus require a
	2035	+ * separate metadata buffer for block layer metadata/PI support.
	2036	+ * We allow extended LBAs for the passthrough interface, though.
	2037	+ */
	2038	+ if (id->flbas & NVME_NS_FLBAS_META_EXT)
	2039	+ ns->features \|= NVME_NS_EXT_LBAS;
	2040	+ else
	2041	+ ns->features \|= NVME_NS_METADATA_SUPPORTED;
1603	2042	}
1604		-#endif
	2043	+
	2044	+ return 0;
1605	2045	}
1606	2046
1607		-static int nvme_revalidate_disk(struct gendisk *disk)
	2047	+static void nvme_set_queue_limits(struct nvme_ctrl *ctrl,
	2048	+ struct request_queue *q)
1608	2049	{
1609		- struct nvme_ns *ns = disk->private_data;
	2050	+ bool vwc = ctrl->vwc & NVME_CTRL_VWC_PRESENT;
	2051	+
	2052	+ if (ctrl->max_hw_sectors) {
	2053	+ u32 max_segments =
	2054	+ (ctrl->max_hw_sectors / (NVME_CTRL_PAGE_SIZE >> 9)) + 1;
	2055	+
	2056	+ max_segments = min_not_zero(max_segments, ctrl->max_segments);
	2057	+ blk_queue_max_hw_sectors(q, ctrl->max_hw_sectors);
	2058	+ blk_queue_max_segments(q, min_t(u32, max_segments, USHRT_MAX));
	2059	+ }
	2060	+ blk_queue_virt_boundary(q, NVME_CTRL_PAGE_SIZE - 1);
	2061	+ blk_queue_dma_alignment(q, 3);
	2062	+ blk_queue_write_cache(q, vwc, vwc);
	2063	+}
	2064	+
	2065	+static void nvme_update_disk_info(struct gendisk *disk,
	2066	+ struct nvme_ns ns, struct nvme_id_ns id)
	2067	+{
	2068	+ sector_t capacity = nvme_lba_to_sect(ns, le64_to_cpu(id->nsze));
	2069	+ unsigned short bs = 1 << ns->lba_shift;
	2070	+ u32 atomic_bs, phys_bs, io_opt = 0;
	2071	+
	2072	+ /*
	2073	+ * The block layer can't support LBA sizes larger than the page size
	2074	+ * yet, so catch this early and don't allow block I/O.
	2075	+ */
	2076	+ if (ns->lba_shift > PAGE_SHIFT) {
	2077	+ capacity = 0;
	2078	+ bs = (1 << 9);
	2079	+ }
	2080	+
	2081	+ blk_integrity_unregister(disk);
	2082	+
	2083	+ atomic_bs = phys_bs = bs;
	2084	+ nvme_setup_streams_ns(ns->ctrl, ns, &phys_bs, &io_opt);
	2085	+ if (id->nabo == 0) {
	2086	+ /*
	2087	+ * Bit 1 indicates whether NAWUPF is defined for this namespace
	2088	+ * and whether it should be used instead of AWUPF. If NAWUPF ==
	2089	+ * 0 then AWUPF must be used instead.
	2090	+ */
	2091	+ if (id->nsfeat & NVME_NS_FEAT_ATOMICS && id->nawupf)
	2092	+ atomic_bs = (1 + le16_to_cpu(id->nawupf)) * bs;
	2093	+ else
	2094	+ atomic_bs = (1 + ns->ctrl->subsys->awupf) * bs;
	2095	+ }
	2096	+
	2097	+ if (id->nsfeat & NVME_NS_FEAT_IO_OPT) {
	2098	+ /* NPWG = Namespace Preferred Write Granularity */
	2099	+ phys_bs = bs * (1 + le16_to_cpu(id->npwg));
	2100	+ /* NOWS = Namespace Optimal Write Size */
	2101	+ io_opt = bs * (1 + le16_to_cpu(id->nows));
	2102	+ }
	2103	+
	2104	+ blk_queue_logical_block_size(disk->queue, bs);
	2105	+ /*
	2106	+ * Linux filesystems assume writing a single physical block is
	2107	+ * an atomic operation. Hence limit the physical block size to the
	2108	+ * value of the Atomic Write Unit Power Fail parameter.
	2109	+ */
	2110	+ blk_queue_physical_block_size(disk->queue, min(phys_bs, atomic_bs));
	2111	+ blk_queue_io_min(disk->queue, phys_bs);
	2112	+ blk_queue_io_opt(disk->queue, io_opt);
	2113	+
	2114	+ /*
	2115	+ * Register a metadata profile for PI, or the plain non-integrity NVMe
	2116	+ * metadata masquerading as Type 0 if supported, otherwise reject block
	2117	+ * I/O to namespaces with metadata except when the namespace supports
	2118	+ * PI, as it can strip/insert in that case.
	2119	+ */
	2120	+ if (ns->ms) {
	2121	+ if (IS_ENABLED(CONFIG_BLK_DEV_INTEGRITY) &&
	2122	+ (ns->features & NVME_NS_METADATA_SUPPORTED))
	2123	+ nvme_init_integrity(disk, ns->ms, ns->pi_type,
	2124	+ ns->ctrl->max_integrity_segments);
	2125	+ else if (!nvme_ns_has_pi(ns))
	2126	+ capacity = 0;
	2127	+ }
	2128	+
	2129	+ set_capacity_revalidate_and_notify(disk, capacity, false);
	2130	+
	2131	+ nvme_config_discard(disk, ns);
	2132	+ nvme_config_write_zeroes(disk->queue, ns->ctrl);
	2133	+
	2134	+ if (id->nsattr & NVME_NS_ATTR_RO)
	2135	+ set_disk_ro(disk, true);
	2136	+}
	2137	+
	2138	+static inline bool nvme_first_scan(struct gendisk *disk)
	2139	+{
	2140	+ /* nvme_alloc_ns() scans the disk prior to adding it */
	2141	+ return !(disk->flags & GENHD_FL_UP);
	2142	+}
	2143	+
	2144	+static void nvme_set_chunk_sectors(struct nvme_ns ns, struct nvme_id_ns id)
	2145	+{
1610	2146	struct nvme_ctrl *ctrl = ns->ctrl;
1611		- struct nvme_id_ns *id;
1612		- struct nvme_ns_ids ids;
1613		- int ret = 0;
	2147	+ u32 iob;
1614	2148
1615		- if (test_bit(NVME_NS_DEAD, &ns->flags)) {
1616		- set_capacity(disk, 0);
1617		- return -ENODEV;
	2149	+ if ((ctrl->quirks & NVME_QUIRK_STRIPE_SIZE) &&
	2150	+ is_power_of_2(ctrl->max_hw_sectors))
	2151	+ iob = ctrl->max_hw_sectors;
	2152	+ else
	2153	+ iob = nvme_lba_to_sect(ns, le16_to_cpu(id->noiob));
	2154	+
	2155	+ if (!iob)
	2156	+ return;
	2157	+
	2158	+ if (!is_power_of_2(iob)) {
	2159	+ if (nvme_first_scan(ns->disk))
	2160	+ pr_warn("%s: ignoring unaligned IO boundary:%u\n",
	2161	+ ns->disk->disk_name, iob);
	2162	+ return;
1618	2163	}
1619	2164
1620		- id = nvme_identify_ns(ctrl, ns->head->ns_id);
1621		- if (!id)
1622		- return -ENODEV;
1623		-
1624		- if (id->ncap == 0) {
1625		- ret = -ENODEV;
1626		- goto out;
	2165	+ if (blk_queue_is_zoned(ns->disk->queue)) {
	2166	+ if (nvme_first_scan(ns->disk))
	2167	+ pr_warn("%s: ignoring zoned namespace IO boundary\n",
	2168	+ ns->disk->disk_name);
	2169	+ return;
1627	2170	}
1628	2171
1629		- __nvme_revalidate_disk(disk, id);
1630		- nvme_report_ns_ids(ctrl, ns->head->ns_id, id, &ids);
1631		- if (!nvme_ns_ids_equal(&ns->head->ids, &ids)) {
1632		- dev_err(ctrl->device,
1633		- "identifiers changed for nsid %d\n", ns->head->ns_id);
1634		- ret = -ENODEV;
	2172	+ blk_queue_chunk_sectors(ns->queue, iob);
	2173	+}
	2174	+
	2175	+static int nvme_update_ns_info(struct nvme_ns ns, struct nvme_id_ns id)
	2176	+{
	2177	+ unsigned lbaf = id->flbas & NVME_NS_FLBAS_LBA_MASK;
	2178	+ int ret;
	2179	+
	2180	+ blk_mq_freeze_queue(ns->disk->queue);
	2181	+ ns->lba_shift = id->lbaf[lbaf].ds;
	2182	+ nvme_set_queue_limits(ns->ctrl, ns->queue);
	2183	+
	2184	+ if (ns->head->ids.csi == NVME_CSI_ZNS) {
	2185	+ ret = nvme_update_zone_info(ns, lbaf);
	2186	+ if (ret)
	2187	+ goto out_unfreeze;
1635	2188	}
1636	2189
1637		-out:
1638		- kfree(id);
	2190	+ ret = nvme_configure_metadata(ns, id);
	2191	+ if (ret)
	2192	+ goto out_unfreeze;
	2193	+ nvme_set_chunk_sectors(ns, id);
	2194	+ nvme_update_disk_info(ns->disk, ns, id);
	2195	+ blk_mq_unfreeze_queue(ns->disk->queue);
	2196	+
	2197	+ if (blk_queue_is_zoned(ns->queue)) {
	2198	+ ret = nvme_revalidate_zones(ns);
	2199	+ if (ret && !nvme_first_scan(ns->disk))
	2200	+ return ret;
	2201	+ }
	2202	+
	2203	+#ifdef CONFIG_NVME_MULTIPATH
	2204	+ if (ns->head->disk) {
	2205	+ blk_mq_freeze_queue(ns->head->disk->queue);
	2206	+ nvme_update_disk_info(ns->head->disk, ns, id);
	2207	+ blk_stack_limits(&ns->head->disk->queue->limits,
	2208	+ &ns->queue->limits, 0);
	2209	+ blk_queue_update_readahead(ns->head->disk->queue);
	2210	+ nvme_update_bdev_size(ns->head->disk);
	2211	+ blk_mq_unfreeze_queue(ns->head->disk->queue);
	2212	+ }
	2213	+#endif
	2214	+ return 0;
	2215	+
	2216	+out_unfreeze:
	2217	+ blk_mq_unfreeze_queue(ns->disk->queue);
1639	2218	return ret;
1640	2219	}
1641	2220
..	..	@@ -1678,7 +2257,7 @@
1678	2257	memset(&c, 0, sizeof(c));
1679	2258	c.common.opcode = op;
1680	2259	c.common.nsid = cpu_to_le32(ns->head->ns_id);
1681		- c.common.cdw10[0] = cpu_to_le32(cdw10);
	2260	+ c.common.cdw10 = cpu_to_le32(cdw10);
1682	2261
1683	2262	ret = nvme_submit_sync_cmd(ns->queue, &c, data, 16);
1684	2263	nvme_put_ns_from_disk(head, srcu_idx);
..	..	@@ -1716,18 +2295,21 @@
1716	2295	enum pr_type type, bool abort)
1717	2296	{
1718	2297	u32 cdw10 = nvme_pr_type(type) << 8 \| (abort ? 2 : 1);
	2298	+
1719	2299	return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_acquire);
1720	2300	}
1721	2301
1722	2302	static int nvme_pr_clear(struct block_device *bdev, u64 key)
1723	2303	{
1724		- u32 cdw10 = 1 \| (key ? 1 << 3 : 0);
1725		- return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_register);
	2304	+ u32 cdw10 = 1 \| (key ? 0 : 1 << 3);
	2305	+
	2306	+ return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_release);
1726	2307	}
1727	2308
1728	2309	static int nvme_pr_release(struct block_device *bdev, u64 key, enum pr_type type)
1729	2310	{
1730		- u32 cdw10 = nvme_pr_type(type) << 8 \| (key ? 1 << 3 : 0);
	2311	+ u32 cdw10 = nvme_pr_type(type) << 8 \| (key ? 0 : 1 << 3);
	2312	+
1731	2313	return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_release);
1732	2314	}
1733	2315
..	..	@@ -1752,11 +2334,11 @@
1752	2334	else
1753	2335	cmd.common.opcode = nvme_admin_security_recv;
1754	2336	cmd.common.nsid = 0;
1755		- cmd.common.cdw10[0] = cpu_to_le32(((u32)secp) << 24 \| ((u32)spsp) << 8);
1756		- cmd.common.cdw10[1] = cpu_to_le32(len);
	2337	+ cmd.common.cdw10 = cpu_to_le32(((u32)secp) << 24 \| ((u32)spsp) << 8);
	2338	+ cmd.common.cdw11 = cpu_to_le32(len);
1757	2339
1758	2340	return __nvme_submit_sync_cmd(ctrl->admin_q, &cmd, NULL, buffer, len,
1759		- ADMIN_TIMEOUT, NVME_QID_ANY, 1, 0);
	2341	+ ADMIN_TIMEOUT, NVME_QID_ANY, 1, 0, false);
1760	2342	}
1761	2343	EXPORT_SYMBOL_GPL(nvme_sec_submit);
1762	2344	#endif /* CONFIG_BLK_SED_OPAL */
..	..	@@ -1764,11 +2346,11 @@
1764	2346	static const struct block_device_operations nvme_fops = {
1765	2347	.owner = THIS_MODULE,
1766	2348	.ioctl = nvme_ioctl,
1767		- .compat_ioctl = nvme_ioctl,
	2349	+ .compat_ioctl = nvme_compat_ioctl,
1768	2350	.open = nvme_open,
1769	2351	.release = nvme_release,
1770	2352	.getgeo = nvme_getgeo,
1771		- .revalidate_disk= nvme_revalidate_disk,
	2353	+ .report_zones = nvme_report_zones,
1772	2354	.pr_ops = &nvme_pr_ops,
1773	2355	};
1774	2356
..	..	@@ -1789,11 +2371,13 @@
1789	2371
1790	2372	const struct block_device_operations nvme_ns_head_ops = {
1791	2373	.owner = THIS_MODULE,
	2374	+ .submit_bio = nvme_ns_head_submit_bio,
1792	2375	.open = nvme_ns_head_open,
1793	2376	.release = nvme_ns_head_release,
1794	2377	.ioctl = nvme_ioctl,
1795		- .compat_ioctl = nvme_ioctl,
	2378	+ .compat_ioctl = nvme_compat_ioctl,
1796	2379	.getgeo = nvme_getgeo,
	2380	+ .report_zones = nvme_report_zones,
1797	2381	.pr_ops = &nvme_pr_ops,
1798	2382	};
1799	2383	#endif /* CONFIG_NVME_MULTIPATH */
..	..	@@ -1811,13 +2395,13 @@
1811	2395	if ((csts & NVME_CSTS_RDY) == bit)
1812	2396	break;
1813	2397
1814		- msleep(100);
	2398	+ usleep_range(1000, 2000);
1815	2399	if (fatal_signal_pending(current))
1816	2400	return -EINTR;
1817	2401	if (time_after(jiffies, timeout)) {
1818	2402	dev_err(ctrl->device,
1819		- "Device not ready; aborting %s\n", enabled ?
1820		- "initialisation" : "reset");
	2403	+ "Device not ready; aborting %s, CSTS=0x%x\n",
	2404	+ enabled ? "initialisation" : "reset", csts);
1821	2405	return -ENODEV;
1822	2406	}
1823	2407	}
..	..	@@ -1831,7 +2415,7 @@
1831	2415	* bits', but doing so may cause the device to complete commands to the
1832	2416	* admin queue ... and we don't know what memory that might be pointing at!
1833	2417	*/
1834		-int nvme_disable_ctrl(struct nvme_ctrl *ctrl, u64 cap)
	2418	+int nvme_disable_ctrl(struct nvme_ctrl *ctrl)
1835	2419	{
1836	2420	int ret;
1837	2421
..	..	@@ -1845,31 +2429,34 @@
1845	2429	if (ctrl->quirks & NVME_QUIRK_DELAY_BEFORE_CHK_RDY)
1846	2430	msleep(NVME_QUIRK_DELAY_AMOUNT);
1847	2431
1848		- return nvme_wait_ready(ctrl, cap, false);
	2432	+ return nvme_wait_ready(ctrl, ctrl->cap, false);
1849	2433	}
1850	2434	EXPORT_SYMBOL_GPL(nvme_disable_ctrl);
1851	2435
1852		-int nvme_enable_ctrl(struct nvme_ctrl *ctrl, u64 cap)
	2436	+int nvme_enable_ctrl(struct nvme_ctrl *ctrl)
1853	2437	{
1854		- /*
1855		- * Default to a 4K page size, with the intention to update this
1856		- * path in the future to accomodate architectures with differing
1857		- * kernel and IO page sizes.
1858		- */
1859		- unsigned dev_page_min = NVME_CAP_MPSMIN(cap) + 12, page_shift = 12;
	2438	+ unsigned dev_page_min;
1860	2439	int ret;
1861	2440
1862		- if (page_shift < dev_page_min) {
	2441	+ ret = ctrl->ops->reg_read64(ctrl, NVME_REG_CAP, &ctrl->cap);
	2442	+ if (ret) {
	2443	+ dev_err(ctrl->device, "Reading CAP failed (%d)\n", ret);
	2444	+ return ret;
	2445	+ }
	2446	+ dev_page_min = NVME_CAP_MPSMIN(ctrl->cap) + 12;
	2447	+
	2448	+ if (NVME_CTRL_PAGE_SHIFT < dev_page_min) {
1863	2449	dev_err(ctrl->device,
1864	2450	"Minimum device page size %u too large for host (%u)\n",
1865		- 1 << dev_page_min, 1 << page_shift);
	2451	+ 1 << dev_page_min, 1 << NVME_CTRL_PAGE_SHIFT);
1866	2452	return -ENODEV;
1867	2453	}
1868	2454
1869		- ctrl->page_size = 1 << page_shift;
1870		-
1871		- ctrl->ctrl_config = NVME_CC_CSS_NVM;
1872		- ctrl->ctrl_config \|= (page_shift - 12) << NVME_CC_MPS_SHIFT;
	2455	+ if (NVME_CAP_CSS(ctrl->cap) & NVME_CAP_CSS_CSI)
	2456	+ ctrl->ctrl_config = NVME_CC_CSS_CSI;
	2457	+ else
	2458	+ ctrl->ctrl_config = NVME_CC_CSS_NVM;
	2459	+ ctrl->ctrl_config \|= (NVME_CTRL_PAGE_SHIFT - 12) << NVME_CC_MPS_SHIFT;
1873	2460	ctrl->ctrl_config \|= NVME_CC_AMS_RR \| NVME_CC_SHN_NONE;
1874	2461	ctrl->ctrl_config \|= NVME_CC_IOSQES \| NVME_CC_IOCQES;
1875	2462	ctrl->ctrl_config \|= NVME_CC_ENABLE;
..	..	@@ -1877,7 +2464,7 @@
1877	2464	ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
1878	2465	if (ret)
1879	2466	return ret;
1880		- return nvme_wait_ready(ctrl, cap, true);
	2467	+ return nvme_wait_ready(ctrl, ctrl->cap, true);
1881	2468	}
1882	2469	EXPORT_SYMBOL_GPL(nvme_enable_ctrl);
1883	2470
..	..	@@ -1912,28 +2499,6 @@
1912	2499	}
1913	2500	EXPORT_SYMBOL_GPL(nvme_shutdown_ctrl);
1914	2501
1915		-static void nvme_set_queue_limits(struct nvme_ctrl *ctrl,
1916		- struct request_queue *q)
1917		-{
1918		- bool vwc = false;
1919		-
1920		- if (ctrl->max_hw_sectors) {
1921		- u32 max_segments =
1922		- (ctrl->max_hw_sectors / (ctrl->page_size >> 9)) + 1;
1923		-
1924		- max_segments = min_not_zero(max_segments, ctrl->max_segments);
1925		- blk_queue_max_hw_sectors(q, ctrl->max_hw_sectors);
1926		- blk_queue_max_segments(q, min_t(u32, max_segments, USHRT_MAX));
1927		- }
1928		- if ((ctrl->quirks & NVME_QUIRK_STRIPE_SIZE) &&
1929		- is_power_of_2(ctrl->max_hw_sectors))
1930		- blk_queue_chunk_sectors(q, ctrl->max_hw_sectors);
1931		- blk_queue_virt_boundary(q, ctrl->page_size - 1);
1932		- if (ctrl->vwc & NVME_CTRL_VWC_PRESENT)
1933		- vwc = true;
1934		- blk_queue_write_cache(q, vwc, vwc);
1935		-}
1936		-
1937	2502	static int nvme_configure_timestamp(struct nvme_ctrl *ctrl)
1938	2503	{
1939	2504	__le64 ts;
..	..	@@ -1948,6 +2513,26 @@
1948	2513	if (ret)
1949	2514	dev_warn_once(ctrl->device,
1950	2515	"could not set timestamp (%d)\n", ret);
	2516	+ return ret;
	2517	+}
	2518	+
	2519	+static int nvme_configure_acre(struct nvme_ctrl *ctrl)
	2520	+{
	2521	+ struct nvme_feat_host_behavior *host;
	2522	+ int ret;
	2523	+
	2524	+ /* Don't bother enabling the feature if retry delay is not reported */
	2525	+ if (!ctrl->crdt[0])
	2526	+ return 0;
	2527	+
	2528	+ host = kzalloc(sizeof(*host), GFP_KERNEL);
	2529	+ if (!host)
	2530	+ return 0;
	2531	+
	2532	+ host->acre = NVME_ENABLE_ACRE;
	2533	+ ret = nvme_set_features(ctrl, NVME_FEAT_HOST_BEHAVIOR, 0,
	2534	+ host, sizeof(*host), NULL);
	2535	+ kfree(host);
1951	2536	return ret;
1952	2537	}
1953	2538
..	..	@@ -2117,6 +2702,44 @@
2117	2702	.vid = 0x1179,
2118	2703	.mn = "THNSF5256GPUK TOSHIBA",
2119	2704	.quirks = NVME_QUIRK_NO_APST,
	2705	+ },
	2706	+ {
	2707	+ /*
	2708	+ * This LiteON CL1-3D*-Q11 firmware version has a race
	2709	+ * condition associated with actions related to suspend to idle
	2710	+ * LiteON has resolved the problem in future firmware
	2711	+ */
	2712	+ .vid = 0x14a4,
	2713	+ .fr = "22301111",
	2714	+ .quirks = NVME_QUIRK_SIMPLE_SUSPEND,
	2715	+ },
	2716	+ {
	2717	+ /*
	2718	+ * This Kioxia CD6-V Series / HPE PE8030 device times out and
	2719	+ * aborts I/O during any load, but more easily reproducible
	2720	+ * with discards (fstrim).
	2721	+ *
	2722	+ * The device is left in a state where it is also not possible
	2723	+ * to use "nvme set-feature" to disable APST, but booting with
	2724	+ * nvme_core.default_ps_max_latency=0 works.
	2725	+ */
	2726	+ .vid = 0x1e0f,
	2727	+ .mn = "KCD6XVUL6T40",
	2728	+ .quirks = NVME_QUIRK_NO_APST,
	2729	+ },
	2730	+ {
	2731	+ /*
	2732	+ * The external Samsung X5 SSD fails initialization without a
	2733	+ * delay before checking if it is ready and has a whole set of
	2734	+ * other problems. To make this even more interesting, it
	2735	+ * shares the PCI ID with internal Samsung 970 Evo Plus that
	2736	+ * does not need or want these quirks.
	2737	+ */
	2738	+ .vid = 0x144d,
	2739	+ .mn = "Samsung Portable SSD X5",
	2740	+ .quirks = NVME_QUIRK_DELAY_BEFORE_CHK_RDY \|
	2741	+ NVME_QUIRK_NO_DEEPEST_PS \|
	2742	+ NVME_QUIRK_IGNORE_DEV_SUBNQN,
2120	2743	}
2121	2744	};
2122	2745
..	..	@@ -2155,14 +2778,16 @@
2155	2778	size_t nqnlen;
2156	2779	int off;
2157	2780
2158		- nqnlen = strnlen(id->subnqn, NVMF_NQN_SIZE);
2159		- if (nqnlen > 0 && nqnlen < NVMF_NQN_SIZE) {
2160		- strncpy(subsys->subnqn, id->subnqn, NVMF_NQN_SIZE);
2161		- return;
2162		- }
	2781	+ if(!(ctrl->quirks & NVME_QUIRK_IGNORE_DEV_SUBNQN)) {
	2782	+ nqnlen = strnlen(id->subnqn, NVMF_NQN_SIZE);
	2783	+ if (nqnlen > 0 && nqnlen < NVMF_NQN_SIZE) {
	2784	+ strlcpy(subsys->subnqn, id->subnqn, NVMF_NQN_SIZE);
	2785	+ return;
	2786	+ }
2163	2787
2164		- if (ctrl->vs >= NVME_VS(1, 2, 1))
2165		- dev_warn(ctrl->device, "missing or invalid SUBNQN field.\n");
	2788	+ if (ctrl->vs >= NVME_VS(1, 2, 1))
	2789	+ dev_warn(ctrl->device, "missing or invalid SUBNQN field.\n");
	2790	+ }
2166	2791
2167	2792	/* Generate a "fake" NQN per Figure 254 in NVMe 1.3 + ECN 001 */
2168	2793	off = snprintf(subsys->subnqn, NVMF_NQN_SIZE,
..	..	@@ -2175,15 +2800,14 @@
2175	2800	memset(subsys->subnqn + off, 0, sizeof(subsys->subnqn) - off);
2176	2801	}
2177	2802
2178		-static void __nvme_release_subsystem(struct nvme_subsystem *subsys)
2179		-{
2180		- ida_simple_remove(&nvme_subsystems_ida, subsys->instance);
2181		- kfree(subsys);
2182		-}
2183		-
2184	2803	static void nvme_release_subsystem(struct device *dev)
2185	2804	{
2186		- __nvme_release_subsystem(container_of(dev, struct nvme_subsystem, dev));
	2805	+ struct nvme_subsystem *subsys =
	2806	+ container_of(dev, struct nvme_subsystem, dev);
	2807	+
	2808	+ if (subsys->instance >= 0)
	2809	+ ida_simple_remove(&nvme_instance_ida, subsys->instance);
	2810	+ kfree(subsys);
2187	2811	}
2188	2812
2189	2813	static void nvme_destroy_subsystem(struct kref *ref)
..	..	@@ -2254,8 +2878,8 @@
2254	2878	{ \
2255	2879	struct nvme_subsystem *subsys = \
2256	2880	container_of(dev, struct nvme_subsystem, dev); \
2257		- return sprintf(buf, "%.*s\n", \
2258		- (int)sizeof(subsys->field), subsys->field); \
	2881	+ return sysfs_emit(buf, "%.*s\n", \
	2882	+ (int)sizeof(subsys->field), subsys->field); \
2259	2883	} \
2260	2884	static SUBSYS_ATTR_RO(field, S_IRUGO, subsys_##field##_show);
2261	2885
..	..	@@ -2268,6 +2892,9 @@
2268	2892	&subsys_attr_serial.attr,
2269	2893	&subsys_attr_firmware_rev.attr,
2270	2894	&subsys_attr_subsysnqn.attr,
	2895	+#ifdef CONFIG_NVME_MULTIPATH
	2896	+ &subsys_attr_iopolicy.attr,
	2897	+#endif
2271	2898	NULL,
2272	2899	};
2273	2900
..	..	@@ -2280,20 +2907,39 @@
2280	2907	NULL,
2281	2908	};
2282	2909
2283		-static int nvme_active_ctrls(struct nvme_subsystem *subsys)
	2910	+static inline bool nvme_discovery_ctrl(struct nvme_ctrl *ctrl)
2284	2911	{
2285		- int count = 0;
2286		- struct nvme_ctrl *ctrl;
	2912	+ return ctrl->opts && ctrl->opts->discovery_nqn;
	2913	+}
2287	2914
2288		- mutex_lock(&subsys->lock);
2289		- list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) {
2290		- if (ctrl->state != NVME_CTRL_DELETING &&
2291		- ctrl->state != NVME_CTRL_DEAD)
2292		- count++;
	2915	+static bool nvme_validate_cntlid(struct nvme_subsystem *subsys,
	2916	+ struct nvme_ctrl ctrl, struct nvme_id_ctrl id)
	2917	+{
	2918	+ struct nvme_ctrl *tmp;
	2919	+
	2920	+ lockdep_assert_held(&nvme_subsystems_lock);
	2921	+
	2922	+ list_for_each_entry(tmp, &subsys->ctrls, subsys_entry) {
	2923	+ if (nvme_state_terminal(tmp))
	2924	+ continue;
	2925	+
	2926	+ if (tmp->cntlid == ctrl->cntlid) {
	2927	+ dev_err(ctrl->device,
	2928	+ "Duplicate cntlid %u with %s, rejecting\n",
	2929	+ ctrl->cntlid, dev_name(tmp->device));
	2930	+ return false;
	2931	+ }
	2932	+
	2933	+ if ((id->cmic & NVME_CTRL_CMIC_MULTI_CTRL) \|\|
	2934	+ nvme_discovery_ctrl(ctrl))
	2935	+ continue;
	2936	+
	2937	+ dev_err(ctrl->device,
	2938	+ "Subsystem does not support multiple controllers\n");
	2939	+ return false;
2293	2940	}
2294		- mutex_unlock(&subsys->lock);
2295	2941
2296		- return count;
	2942	+ return true;
2297	2943	}
2298	2944
2299	2945	static int nvme_init_subsystem(struct nvme_ctrl ctrl, struct nvme_id_ctrl id)
..	..	@@ -2304,12 +2950,8 @@
2304	2950	subsys = kzalloc(sizeof(*subsys), GFP_KERNEL);
2305	2951	if (!subsys)
2306	2952	return -ENOMEM;
2307		- ret = ida_simple_get(&nvme_subsystems_ida, 0, 0, GFP_KERNEL);
2308		- if (ret < 0) {
2309		- kfree(subsys);
2310		- return ret;
2311		- }
2312		- subsys->instance = ret;
	2953	+
	2954	+ subsys->instance = -1;
2313	2955	mutex_init(&subsys->lock);
2314	2956	kref_init(&subsys->ref);
2315	2957	INIT_LIST_HEAD(&subsys->ctrls);
..	..	@@ -2317,74 +2959,68 @@
2317	2959	nvme_init_subnqn(subsys, ctrl, id);
2318	2960	memcpy(subsys->serial, id->sn, sizeof(subsys->serial));
2319	2961	memcpy(subsys->model, id->mn, sizeof(subsys->model));
2320		- memcpy(subsys->firmware_rev, id->fr, sizeof(subsys->firmware_rev));
2321	2962	subsys->vendor_id = le16_to_cpu(id->vid);
2322	2963	subsys->cmic = id->cmic;
	2964	+ subsys->awupf = le16_to_cpu(id->awupf);
	2965	+#ifdef CONFIG_NVME_MULTIPATH
	2966	+ subsys->iopolicy = NVME_IOPOLICY_NUMA;
	2967	+#endif
2323	2968
2324	2969	subsys->dev.class = nvme_subsys_class;
2325	2970	subsys->dev.release = nvme_release_subsystem;
2326	2971	subsys->dev.groups = nvme_subsys_attrs_groups;
2327		- dev_set_name(&subsys->dev, "nvme-subsys%d", subsys->instance);
	2972	+ dev_set_name(&subsys->dev, "nvme-subsys%d", ctrl->instance);
2328	2973	device_initialize(&subsys->dev);
2329	2974
2330	2975	mutex_lock(&nvme_subsystems_lock);
2331	2976	found = __nvme_find_get_subsystem(subsys->subnqn);
2332	2977	if (found) {
2333		- /*
2334		- * Verify that the subsystem actually supports multiple
2335		- * controllers, else bail out.
2336		- */
2337		- if (!(ctrl->opts && ctrl->opts->discovery_nqn) &&
2338		- nvme_active_ctrls(found) && !(id->cmic & (1 << 1))) {
2339		- dev_err(ctrl->device,
2340		- "ignoring ctrl due to duplicate subnqn (%s).\n",
2341		- found->subnqn);
2342		- nvme_put_subsystem(found);
2343		- ret = -EINVAL;
2344		- goto out_unlock;
2345		- }
2346		-
2347		- __nvme_release_subsystem(subsys);
	2978	+ put_device(&subsys->dev);
2348	2979	subsys = found;
	2980	+
	2981	+ if (!nvme_validate_cntlid(subsys, ctrl, id)) {
	2982	+ ret = -EINVAL;
	2983	+ goto out_put_subsystem;
	2984	+ }
2349	2985	} else {
2350	2986	ret = device_add(&subsys->dev);
2351	2987	if (ret) {
2352	2988	dev_err(ctrl->device,
2353	2989	"failed to register subsystem device.\n");
	2990	+ put_device(&subsys->dev);
2354	2991	goto out_unlock;
2355	2992	}
2356	2993	ida_init(&subsys->ns_ida);
2357	2994	list_add_tail(&subsys->entry, &nvme_subsystems);
2358	2995	}
2359	2996
2360		- ctrl->subsys = subsys;
2361		- mutex_unlock(&nvme_subsystems_lock);
2362		-
2363		- if (sysfs_create_link(&subsys->dev.kobj, &ctrl->device->kobj,
2364		- dev_name(ctrl->device))) {
	2997	+ ret = sysfs_create_link(&subsys->dev.kobj, &ctrl->device->kobj,
	2998	+ dev_name(ctrl->device));
	2999	+ if (ret) {
2365	3000	dev_err(ctrl->device,
2366	3001	"failed to create sysfs link from subsystem.\n");
2367		- /* the transport driver will eventually put the subsystem */
2368		- return -EINVAL;
	3002	+ goto out_put_subsystem;
2369	3003	}
2370	3004
2371		- mutex_lock(&subsys->lock);
	3005	+ if (!found)
	3006	+ subsys->instance = ctrl->instance;
	3007	+ ctrl->subsys = subsys;
2372	3008	list_add_tail(&ctrl->subsys_entry, &subsys->ctrls);
2373		- mutex_unlock(&subsys->lock);
2374		-
	3009	+ mutex_unlock(&nvme_subsystems_lock);
2375	3010	return 0;
2376	3011
	3012	+out_put_subsystem:
	3013	+ nvme_put_subsystem(subsys);
2377	3014	out_unlock:
2378	3015	mutex_unlock(&nvme_subsystems_lock);
2379		- put_device(&subsys->dev);
2380	3016	return ret;
2381	3017	}
2382	3018
2383		-int nvme_get_log(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp,
	3019	+int nvme_get_log(struct nvme_ctrl *ctrl, u32 nsid, u8 log_page, u8 lsp, u8 csi,
2384	3020	void *log, size_t size, u64 offset)
2385	3021	{
2386	3022	struct nvme_command c = { };
2387		- unsigned long dwlen = size / 4 - 1;
	3023	+ u32 dwlen = nvme_bytes_to_numd(size);
2388	3024
2389	3025	c.get_log_page.opcode = nvme_admin_get_log_page;
2390	3026	c.get_log_page.nsid = cpu_to_le32(nsid);
..	..	@@ -2394,27 +3030,35 @@
2394	3030	c.get_log_page.numdu = cpu_to_le16(dwlen >> 16);
2395	3031	c.get_log_page.lpol = cpu_to_le32(lower_32_bits(offset));
2396	3032	c.get_log_page.lpou = cpu_to_le32(upper_32_bits(offset));
	3033	+ c.get_log_page.csi = csi;
2397	3034
2398	3035	return nvme_submit_sync_cmd(ctrl->admin_q, &c, log, size);
2399	3036	}
2400	3037
2401		-static int nvme_get_effects_log(struct nvme_ctrl *ctrl)
	3038	+static int nvme_get_effects_log(struct nvme_ctrl *ctrl, u8 csi,
	3039	+ struct nvme_effects_log **log)
2402	3040	{
	3041	+ struct nvme_effects_log *cel = xa_load(&ctrl->cels, csi);
2403	3042	int ret;
2404	3043
2405		- if (!ctrl->effects)
2406		- ctrl->effects = kzalloc(sizeof(*ctrl->effects), GFP_KERNEL);
	3044	+ if (cel)
	3045	+ goto out;
2407	3046
2408		- if (!ctrl->effects)
2409		- return 0;
	3047	+ cel = kzalloc(sizeof(*cel), GFP_KERNEL);
	3048	+ if (!cel)
	3049	+ return -ENOMEM;
2410	3050
2411		- ret = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_CMD_EFFECTS, 0,
2412		- ctrl->effects, sizeof(*ctrl->effects), 0);
	3051	+ ret = nvme_get_log(ctrl, 0x00, NVME_LOG_CMD_EFFECTS, 0, csi,
	3052	+ cel, sizeof(*cel), 0);
2413	3053	if (ret) {
2414		- kfree(ctrl->effects);
2415		- ctrl->effects = NULL;
	3054	+ kfree(cel);
	3055	+ return ret;
2416	3056	}
2417		- return ret;
	3057	+
	3058	+ xa_store(&ctrl->cels, csi, cel, GFP_KERNEL);
	3059	+out:
	3060	+ *log = cel;
	3061	+ return 0;
2418	3062	}
2419	3063
2420	3064	/*
..	..	@@ -2425,7 +3069,6 @@
2425	3069	int nvme_init_identify(struct nvme_ctrl *ctrl)
2426	3070	{
2427	3071	struct nvme_id_ctrl *id;
2428		- u64 cap;
2429	3072	int ret, page_shift;
2430	3073	u32 max_hw_sectors;
2431	3074	bool prev_apst_enabled;
..	..	@@ -2435,16 +3078,11 @@
2435	3078	dev_err(ctrl->device, "Reading VS failed (%d)\n", ret);
2436	3079	return ret;
2437	3080	}
2438		-
2439		- ret = ctrl->ops->reg_read64(ctrl, NVME_REG_CAP, &cap);
2440		- if (ret) {
2441		- dev_err(ctrl->device, "Reading CAP failed (%d)\n", ret);
2442		- return ret;
2443		- }
2444		- page_shift = NVME_CAP_MPSMIN(cap) + 12;
	3081	+ page_shift = NVME_CAP_MPSMIN(ctrl->cap) + 12;
	3082	+ ctrl->sqsize = min_t(u16, NVME_CAP_MQES(ctrl->cap), ctrl->sqsize);
2445	3083
2446	3084	if (ctrl->vs >= NVME_VS(1, 1, 0))
2447		- ctrl->subsystem = NVME_CAP_NSSRC(cap);
	3085	+ ctrl->subsystem = NVME_CAP_NSSRC(ctrl->cap);
2448	3086
2449	3087	ret = nvme_identify_ctrl(ctrl, &id);
2450	3088	if (ret) {
..	..	@@ -2453,17 +3091,16 @@
2453	3091	}
2454	3092
2455	3093	if (id->lpa & NVME_CTRL_LPA_CMD_EFFECTS_LOG) {
2456		- ret = nvme_get_effects_log(ctrl);
	3094	+ ret = nvme_get_effects_log(ctrl, NVME_CSI_NVM, &ctrl->effects);
2457	3095	if (ret < 0)
2458	3096	goto out_free;
2459	3097	}
2460	3098
	3099	+ if (!(ctrl->ops->flags & NVME_F_FABRICS))
	3100	+ ctrl->cntlid = le16_to_cpu(id->cntlid);
	3101	+
2461	3102	if (!ctrl->identified) {
2462	3103	int i;
2463		-
2464		- ret = nvme_init_subsystem(ctrl, id);
2465		- if (ret)
2466		- goto out_free;
2467	3104
2468	3105	/*
2469	3106	* Check for quirks. Quirk can depend on firmware version,
..	..	@@ -2477,19 +3114,32 @@
2477	3114	if (quirk_matches(id, &core_quirks[i]))
2478	3115	ctrl->quirks \|= core_quirks[i].quirks;
2479	3116	}
	3117	+
	3118	+ ret = nvme_init_subsystem(ctrl, id);
	3119	+ if (ret)
	3120	+ goto out_free;
2480	3121	}
	3122	+ memcpy(ctrl->subsys->firmware_rev, id->fr,
	3123	+ sizeof(ctrl->subsys->firmware_rev));
2481	3124
2482	3125	if (force_apst && (ctrl->quirks & NVME_QUIRK_NO_DEEPEST_PS)) {
2483	3126	dev_warn(ctrl->device, "forcibly allowing all power states due to nvme_core.force_apst -- use at your own risk\n");
2484	3127	ctrl->quirks &= ~NVME_QUIRK_NO_DEEPEST_PS;
2485	3128	}
2486	3129
	3130	+ ctrl->crdt[0] = le16_to_cpu(id->crdt1);
	3131	+ ctrl->crdt[1] = le16_to_cpu(id->crdt2);
	3132	+ ctrl->crdt[2] = le16_to_cpu(id->crdt3);
	3133	+
2487	3134	ctrl->oacs = le16_to_cpu(id->oacs);
2488		- ctrl->oncs = le16_to_cpup(&id->oncs);
	3135	+ ctrl->oncs = le16_to_cpu(id->oncs);
	3136	+ ctrl->mtfa = le16_to_cpu(id->mtfa);
2489	3137	ctrl->oaes = le32_to_cpu(id->oaes);
	3138	+ ctrl->wctemp = le16_to_cpu(id->wctemp);
	3139	+ ctrl->cctemp = le16_to_cpu(id->cctemp);
	3140	+
2490	3141	atomic_set(&ctrl->abort_limit, id->acl + 1);
2491	3142	ctrl->vwc = id->vwc;
2492		- ctrl->cntlid = le16_to_cpup(&id->cntlid);
2493	3143	if (id->mdts)
2494	3144	max_hw_sectors = 1 << (id->mdts + page_shift - 9);
2495	3145	else
..	..	@@ -2501,10 +3151,11 @@
2501	3151	ctrl->sgls = le32_to_cpu(id->sgls);
2502	3152	ctrl->kas = le16_to_cpu(id->kas);
2503	3153	ctrl->max_namespaces = le32_to_cpu(id->mnan);
	3154	+ ctrl->ctratt = le32_to_cpu(id->ctratt);
2504	3155
2505	3156	if (id->rtd3e) {
2506	3157	/* us -> s */
2507		- u32 transition_time = le32_to_cpu(id->rtd3e) / 1000000;
	3158	+ u32 transition_time = le32_to_cpu(id->rtd3e) / USEC_PER_SEC;
2508	3159
2509	3160	ctrl->shutdown_timeout = clamp_t(unsigned int, transition_time,
2510	3161	shutdown_timeout, 60);
..	..	@@ -2542,25 +3193,28 @@
2542	3193	* admin connect
2543	3194	*/
2544	3195	if (ctrl->cntlid != le16_to_cpu(id->cntlid)) {
	3196	+ dev_err(ctrl->device,
	3197	+ "Mismatching cntlid: Connect %u vs Identify "
	3198	+ "%u, rejecting\n",
	3199	+ ctrl->cntlid, le16_to_cpu(id->cntlid));
2545	3200	ret = -EINVAL;
2546	3201	goto out_free;
2547	3202	}
2548	3203
2549		- if (!ctrl->opts->discovery_nqn && !ctrl->kas) {
	3204	+ if (!nvme_discovery_ctrl(ctrl) && !ctrl->kas) {
2550	3205	dev_err(ctrl->device,
2551	3206	"keep-alive support is mandatory for fabrics\n");
2552	3207	ret = -EINVAL;
2553	3208	goto out_free;
2554	3209	}
2555	3210	} else {
2556		- ctrl->cntlid = le16_to_cpu(id->cntlid);
2557	3211	ctrl->hmpre = le32_to_cpu(id->hmpre);
2558	3212	ctrl->hmmin = le32_to_cpu(id->hmmin);
2559	3213	ctrl->hmminds = le32_to_cpu(id->hmminds);
2560	3214	ctrl->hmmaxd = le16_to_cpu(id->hmmaxd);
2561	3215	}
2562	3216
2563		- ret = nvme_mpath_init(ctrl, id);
	3217	+ ret = nvme_mpath_init_identify(ctrl, id);
2564	3218	kfree(id);
2565	3219
2566	3220	if (ret < 0)
..	..	@@ -2583,6 +3237,20 @@
2583	3237	if (ret < 0)
2584	3238	return ret;
2585	3239
	3240	+ ret = nvme_configure_acre(ctrl);
	3241	+ if (ret < 0)
	3242	+ return ret;
	3243	+
	3244	+ if (!ctrl->identified && !nvme_discovery_ctrl(ctrl)) {
	3245	+ /*
	3246	+ * Do not return errors unless we are in a controller reset,
	3247	+ * the controller works perfectly fine without hwmon.
	3248	+ */
	3249	+ ret = nvme_hwmon_init(ctrl);
	3250	+ if (ret == -EINTR)
	3251	+ return ret;
	3252	+ }
	3253	+
2586	3254	ctrl->identified = true;
2587	3255
2588	3256	return 0;
..	..	@@ -2600,7 +3268,6 @@
2600	3268
2601	3269	switch (ctrl->state) {
2602	3270	case NVME_CTRL_LIVE:
2603		- case NVME_CTRL_ADMIN_ONLY:
2604	3271	break;
2605	3272	default:
2606	3273	return -EWOULDBLOCK;
..	..	@@ -2668,14 +3335,22 @@
2668	3335	switch (cmd) {
2669	3336	case NVME_IOCTL_ADMIN_CMD:
2670	3337	return nvme_user_cmd(ctrl, NULL, argp);
	3338	+ case NVME_IOCTL_ADMIN64_CMD:
	3339	+ return nvme_user_cmd64(ctrl, NULL, argp);
2671	3340	case NVME_IOCTL_IO_CMD:
2672	3341	return nvme_dev_user_cmd(ctrl, argp);
2673	3342	case NVME_IOCTL_RESET:
	3343	+ if (!capable(CAP_SYS_ADMIN))
	3344	+ return -EACCES;
2674	3345	dev_warn(ctrl->device, "resetting controller\n");
2675	3346	return nvme_reset_ctrl_sync(ctrl);
2676	3347	case NVME_IOCTL_SUBSYS_RESET:
	3348	+ if (!capable(CAP_SYS_ADMIN))
	3349	+ return -EACCES;
2677	3350	return nvme_reset_subsystem(ctrl);
2678	3351	case NVME_IOCTL_RESCAN:
	3352	+ if (!capable(CAP_SYS_ADMIN))
	3353	+ return -EACCES;
2679	3354	nvme_queue_scan(ctrl);
2680	3355	return 0;
2681	3356	default:
..	..	@@ -2688,7 +3363,7 @@
2688	3363	.open = nvme_dev_open,
2689	3364	.release = nvme_dev_release,
2690	3365	.unlocked_ioctl = nvme_dev_ioctl,
2691		- .compat_ioctl = nvme_dev_ioctl,
	3366	+ .compat_ioctl = compat_ptr_ioctl,
2692	3367	};
2693	3368
2694	3369	static ssize_t nvme_sysfs_reset(struct device *dev,
..	..	@@ -2736,13 +3411,13 @@
2736	3411	int model_len = sizeof(subsys->model);
2737	3412
2738	3413	if (!uuid_is_null(&ids->uuid))
2739		- return sprintf(buf, "uuid.%pU\n", &ids->uuid);
	3414	+ return sysfs_emit(buf, "uuid.%pU\n", &ids->uuid);
2740	3415
2741	3416	if (memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
2742		- return sprintf(buf, "eui.%16phN\n", ids->nguid);
	3417	+ return sysfs_emit(buf, "eui.%16phN\n", ids->nguid);
2743	3418
2744	3419	if (memchr_inv(ids->eui64, 0, sizeof(ids->eui64)))
2745		- return sprintf(buf, "eui.%8phN\n", ids->eui64);
	3420	+ return sysfs_emit(buf, "eui.%8phN\n", ids->eui64);
2746	3421
2747	3422	while (serial_len > 0 && (subsys->serial[serial_len - 1] == ' ' \|\|
2748	3423	subsys->serial[serial_len - 1] == '\0'))
..	..	@@ -2751,7 +3426,7 @@
2751	3426	subsys->model[model_len - 1] == '\0'))
2752	3427	model_len--;
2753	3428
2754		- return sprintf(buf, "nvme.%04x-%phN-%phN-%08x\n", subsys->vendor_id,
	3429	+ return sysfs_emit(buf, "nvme.%04x-%phN-%phN-%08x\n", subsys->vendor_id,
2755	3430	serial_len, subsys->serial, model_len, subsys->model,
2756	3431	head->ns_id);
2757	3432	}
..	..	@@ -2760,7 +3435,7 @@
2760	3435	static ssize_t nguid_show(struct device dev, struct device_attribute attr,
2761	3436	char *buf)
2762	3437	{
2763		- return sprintf(buf, "%pU\n", dev_to_ns_head(dev)->ids.nguid);
	3438	+ return sysfs_emit(buf, "%pU\n", dev_to_ns_head(dev)->ids.nguid);
2764	3439	}
2765	3440	static DEVICE_ATTR_RO(nguid);
2766	3441
..	..	@@ -2773,25 +3448,25 @@
2773	3448	* we have no UUID set
2774	3449	*/
2775	3450	if (uuid_is_null(&ids->uuid)) {
2776		- printk_ratelimited(KERN_WARNING
2777		- "No UUID available providing old NGUID\n");
2778		- return sprintf(buf, "%pU\n", ids->nguid);
	3451	+ dev_warn_ratelimited(dev,
	3452	+ "No UUID available providing old NGUID\n");
	3453	+ return sysfs_emit(buf, "%pU\n", ids->nguid);
2779	3454	}
2780		- return sprintf(buf, "%pU\n", &ids->uuid);
	3455	+ return sysfs_emit(buf, "%pU\n", &ids->uuid);
2781	3456	}
2782	3457	static DEVICE_ATTR_RO(uuid);
2783	3458
2784	3459	static ssize_t eui_show(struct device dev, struct device_attribute attr,
2785	3460	char *buf)
2786	3461	{
2787		- return sprintf(buf, "%8ph\n", dev_to_ns_head(dev)->ids.eui64);
	3462	+ return sysfs_emit(buf, "%8ph\n", dev_to_ns_head(dev)->ids.eui64);
2788	3463	}
2789	3464	static DEVICE_ATTR_RO(eui);
2790	3465
2791	3466	static ssize_t nsid_show(struct device dev, struct device_attribute attr,
2792	3467	char *buf)
2793	3468	{
2794		- return sprintf(buf, "%d\n", dev_to_ns_head(dev)->ns_id);
	3469	+ return sysfs_emit(buf, "%d\n", dev_to_ns_head(dev)->ns_id);
2795	3470	}
2796	3471	static DEVICE_ATTR_RO(nsid);
2797	3472
..	..	@@ -2838,9 +3513,17 @@
2838	3513	return a->mode;
2839	3514	}
2840	3515
2841		-const struct attribute_group nvme_ns_id_attr_group = {
	3516	+static const struct attribute_group nvme_ns_id_attr_group = {
2842	3517	.attrs = nvme_ns_id_attrs,
2843	3518	.is_visible = nvme_ns_id_attrs_are_visible,
	3519	+};
	3520	+
	3521	+const struct attribute_group *nvme_ns_id_attr_groups[] = {
	3522	+ &nvme_ns_id_attr_group,
	3523	+#ifdef CONFIG_NVM
	3524	+ &nvme_nvm_attr_group,
	3525	+#endif
	3526	+ NULL,
2844	3527	};
2845	3528
2846	3529	#define nvme_show_str_function(field) \
..	..	@@ -2848,7 +3531,7 @@
2848	3531	struct device_attribute attr, char buf) \
2849	3532	{ \
2850	3533	struct nvme_ctrl *ctrl = dev_get_drvdata(dev); \
2851		- return sprintf(buf, "%.*s\n", \
	3534	+ return sysfs_emit(buf, "%.*s\n", \
2852	3535	(int)sizeof(ctrl->subsys->field), ctrl->subsys->field); \
2853	3536	} \
2854	3537	static DEVICE_ATTR(field, S_IRUGO, field##_show, NULL);
..	..	@@ -2862,21 +3545,20 @@
2862	3545	struct device_attribute attr, char buf) \
2863	3546	{ \
2864	3547	struct nvme_ctrl *ctrl = dev_get_drvdata(dev); \
2865		- return sprintf(buf, "%d\n", ctrl->field); \
	3548	+ return sysfs_emit(buf, "%d\n", ctrl->field); \
2866	3549	} \
2867	3550	static DEVICE_ATTR(field, S_IRUGO, field##_show, NULL);
2868	3551
2869	3552	nvme_show_int_function(cntlid);
	3553	+nvme_show_int_function(numa_node);
	3554	+nvme_show_int_function(queue_count);
	3555	+nvme_show_int_function(sqsize);
2870	3556
2871	3557	static ssize_t nvme_sysfs_delete(struct device *dev,
2872	3558	struct device_attribute attr, const char buf,
2873	3559	size_t count)
2874	3560	{
2875	3561	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
2876		-
2877		- /* Can't delete non-created controllers */
2878		- if (!ctrl->created)
2879		- return -EBUSY;
2880	3562
2881	3563	if (device_remove_file_self(dev, attr))
2882	3564	nvme_delete_ctrl_sync(ctrl);
..	..	@@ -2902,18 +3584,18 @@
2902	3584	static const char *const state_name[] = {
2903	3585	[NVME_CTRL_NEW] = "new",
2904	3586	[NVME_CTRL_LIVE] = "live",
2905		- [NVME_CTRL_ADMIN_ONLY] = "only-admin",
2906	3587	[NVME_CTRL_RESETTING] = "resetting",
2907	3588	[NVME_CTRL_CONNECTING] = "connecting",
2908	3589	[NVME_CTRL_DELETING] = "deleting",
	3590	+ [NVME_CTRL_DELETING_NOIO]= "deleting (no IO)",
2909	3591	[NVME_CTRL_DEAD] = "dead",
2910	3592	};
2911	3593
2912	3594	if ((unsigned)ctrl->state < ARRAY_SIZE(state_name) &&
2913	3595	state_name[ctrl->state])
2914		- return sprintf(buf, "%s\n", state_name[ctrl->state]);
	3596	+ return sysfs_emit(buf, "%s\n", state_name[ctrl->state]);
2915	3597
2916		- return sprintf(buf, "unknown state\n");
	3598	+ return sysfs_emit(buf, "unknown state\n");
2917	3599	}
2918	3600
2919	3601	static DEVICE_ATTR(state, S_IRUGO, nvme_sysfs_show_state, NULL);
..	..	@@ -2928,6 +3610,26 @@
2928	3610	}
2929	3611	static DEVICE_ATTR(subsysnqn, S_IRUGO, nvme_sysfs_show_subsysnqn, NULL);
2930	3612
	3613	+static ssize_t nvme_sysfs_show_hostnqn(struct device *dev,
	3614	+ struct device_attribute *attr,
	3615	+ char *buf)
	3616	+{
	3617	+ struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
	3618	+
	3619	+ return snprintf(buf, PAGE_SIZE, "%s\n", ctrl->opts->host->nqn);
	3620	+}
	3621	+static DEVICE_ATTR(hostnqn, S_IRUGO, nvme_sysfs_show_hostnqn, NULL);
	3622	+
	3623	+static ssize_t nvme_sysfs_show_hostid(struct device *dev,
	3624	+ struct device_attribute *attr,
	3625	+ char *buf)
	3626	+{
	3627	+ struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
	3628	+
	3629	+ return snprintf(buf, PAGE_SIZE, "%pU\n", &ctrl->opts->host->id);
	3630	+}
	3631	+static DEVICE_ATTR(hostid, S_IRUGO, nvme_sysfs_show_hostid, NULL);
	3632	+
2931	3633	static ssize_t nvme_sysfs_show_address(struct device *dev,
2932	3634	struct device_attribute *attr,
2933	3635	char *buf)
..	..	@@ -2937,6 +3639,66 @@
2937	3639	return ctrl->ops->get_address(ctrl, buf, PAGE_SIZE);
2938	3640	}
2939	3641	static DEVICE_ATTR(address, S_IRUGO, nvme_sysfs_show_address, NULL);
	3642	+
	3643	+static ssize_t nvme_ctrl_loss_tmo_show(struct device *dev,
	3644	+ struct device_attribute attr, char buf)
	3645	+{
	3646	+ struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
	3647	+ struct nvmf_ctrl_options *opts = ctrl->opts;
	3648	+
	3649	+ if (ctrl->opts->max_reconnects == -1)
	3650	+ return sysfs_emit(buf, "off\n");
	3651	+ return sysfs_emit(buf, "%d\n",
	3652	+ opts->max_reconnects * opts->reconnect_delay);
	3653	+}
	3654	+
	3655	+static ssize_t nvme_ctrl_loss_tmo_store(struct device *dev,
	3656	+ struct device_attribute attr, const char buf, size_t count)
	3657	+{
	3658	+ struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
	3659	+ struct nvmf_ctrl_options *opts = ctrl->opts;
	3660	+ int ctrl_loss_tmo, err;
	3661	+
	3662	+ err = kstrtoint(buf, 10, &ctrl_loss_tmo);
	3663	+ if (err)
	3664	+ return -EINVAL;
	3665	+
	3666	+ else if (ctrl_loss_tmo < 0)
	3667	+ opts->max_reconnects = -1;
	3668	+ else
	3669	+ opts->max_reconnects = DIV_ROUND_UP(ctrl_loss_tmo,
	3670	+ opts->reconnect_delay);
	3671	+ return count;
	3672	+}
	3673	+static DEVICE_ATTR(ctrl_loss_tmo, S_IRUGO \| S_IWUSR,
	3674	+ nvme_ctrl_loss_tmo_show, nvme_ctrl_loss_tmo_store);
	3675	+
	3676	+static ssize_t nvme_ctrl_reconnect_delay_show(struct device *dev,
	3677	+ struct device_attribute attr, char buf)
	3678	+{
	3679	+ struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
	3680	+
	3681	+ if (ctrl->opts->reconnect_delay == -1)
	3682	+ return sysfs_emit(buf, "off\n");
	3683	+ return sysfs_emit(buf, "%d\n", ctrl->opts->reconnect_delay);
	3684	+}
	3685	+
	3686	+static ssize_t nvme_ctrl_reconnect_delay_store(struct device *dev,
	3687	+ struct device_attribute attr, const char buf, size_t count)
	3688	+{
	3689	+ struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
	3690	+ unsigned int v;
	3691	+ int err;
	3692	+
	3693	+ err = kstrtou32(buf, 10, &v);
	3694	+ if (err)
	3695	+ return err;
	3696	+
	3697	+ ctrl->opts->reconnect_delay = v;
	3698	+ return count;
	3699	+}
	3700	+static DEVICE_ATTR(reconnect_delay, S_IRUGO \| S_IWUSR,
	3701	+ nvme_ctrl_reconnect_delay_show, nvme_ctrl_reconnect_delay_store);
2940	3702
2941	3703	static struct attribute *nvme_dev_attrs[] = {
2942	3704	&dev_attr_reset_controller.attr,
..	..	@@ -2950,6 +3712,13 @@
2950	3712	&dev_attr_subsysnqn.attr,
2951	3713	&dev_attr_address.attr,
2952	3714	&dev_attr_state.attr,
	3715	+ &dev_attr_numa_node.attr,
	3716	+ &dev_attr_queue_count.attr,
	3717	+ &dev_attr_sqsize.attr,
	3718	+ &dev_attr_hostnqn.attr,
	3719	+ &dev_attr_hostid.attr,
	3720	+ &dev_attr_ctrl_loss_tmo.attr,
	3721	+ &dev_attr_reconnect_delay.attr,
2953	3722	NULL
2954	3723	};
2955	3724
..	..	@@ -2962,6 +3731,14 @@
2962	3731	if (a == &dev_attr_delete_controller.attr && !ctrl->ops->delete_ctrl)
2963	3732	return 0;
2964	3733	if (a == &dev_attr_address.attr && !ctrl->ops->get_address)
	3734	+ return 0;
	3735	+ if (a == &dev_attr_hostnqn.attr && !ctrl->opts)
	3736	+ return 0;
	3737	+ if (a == &dev_attr_hostid.attr && !ctrl->opts)
	3738	+ return 0;
	3739	+ if (a == &dev_attr_ctrl_loss_tmo.attr && !ctrl->opts)
	3740	+ return 0;
	3741	+ if (a == &dev_attr_reconnect_delay.attr && !ctrl->opts)
2965	3742	return 0;
2966	3743
2967	3744	return a->mode;
..	..	@@ -2977,7 +3754,7 @@
2977	3754	NULL,
2978	3755	};
2979	3756
2980		-static struct nvme_ns_head __nvme_find_ns_head(struct nvme_subsystem subsys,
	3757	+static struct nvme_ns_head nvme_find_ns_head(struct nvme_subsystem subsys,
2981	3758	unsigned nsid)
2982	3759	{
2983	3760	struct nvme_ns_head *h;
..	..	@@ -2992,17 +3769,15 @@
2992	3769	return NULL;
2993	3770	}
2994	3771
2995		-static int __nvme_check_ids(struct nvme_subsystem *subsys,
2996		- struct nvme_ns_head *new)
	3772	+static int nvme_subsys_check_duplicate_ids(struct nvme_subsystem *subsys,
	3773	+ struct nvme_ns_ids *ids)
2997	3774	{
2998	3775	struct nvme_ns_head *h;
2999	3776
3000	3777	lockdep_assert_held(&subsys->lock);
3001	3778
3002	3779	list_for_each_entry(h, &subsys->nsheads, entry) {
3003		- if (nvme_ns_ids_valid(&new->ids) &&
3004		- !list_empty(&h->list) &&
3005		- nvme_ns_ids_equal(&new->ids, &h->ids))
	3780	+ if (nvme_ns_ids_valid(ids) && nvme_ns_ids_equal(ids, &h->ids))
3006	3781	return -EINVAL;
3007	3782	}
3008	3783
..	..	@@ -3010,12 +3785,17 @@
3010	3785	}
3011	3786
3012	3787	static struct nvme_ns_head nvme_alloc_ns_head(struct nvme_ctrl ctrl,
3013		- unsigned nsid, struct nvme_id_ns *id)
	3788	+ unsigned nsid, struct nvme_ns_ids *ids)
3014	3789	{
3015	3790	struct nvme_ns_head *head;
	3791	+ size_t size = sizeof(*head);
3016	3792	int ret = -ENOMEM;
3017	3793
3018		- head = kzalloc(sizeof(*head), GFP_KERNEL);
	3794	+#ifdef CONFIG_NVME_MULTIPATH
	3795	+ size += num_possible_nodes() * sizeof(struct nvme_ns *);
	3796	+#endif
	3797	+
	3798	+ head = kzalloc(size, GFP_KERNEL);
3019	3799	if (!head)
3020	3800	goto out;
3021	3801	ret = ida_simple_get(&ctrl->subsys->ns_ida, 1, 0, GFP_KERNEL);
..	..	@@ -3028,16 +3808,22 @@
3028	3808	goto out_ida_remove;
3029	3809	head->subsys = ctrl->subsys;
3030	3810	head->ns_id = nsid;
	3811	+ head->ids = *ids;
3031	3812	kref_init(&head->ref);
3032	3813
3033		- nvme_report_ns_ids(ctrl, nsid, id, &head->ids);
3034		-
3035		- ret = __nvme_check_ids(ctrl->subsys, head);
	3814	+ ret = nvme_subsys_check_duplicate_ids(ctrl->subsys, &head->ids);
3036	3815	if (ret) {
3037	3816	dev_err(ctrl->device,
3038	3817	"duplicate IDs for nsid %d\n", nsid);
3039	3818	goto out_cleanup_srcu;
3040	3819	}
	3820	+
	3821	+ if (head->ids.csi) {
	3822	+ ret = nvme_get_effects_log(ctrl, head->ids.csi, &head->effects);
	3823	+ if (ret)
	3824	+ goto out_cleanup_srcu;
	3825	+ } else
	3826	+ head->effects = ctrl->effects;
3041	3827
3042	3828	ret = nvme_mpath_alloc_disk(ctrl, head);
3043	3829	if (ret)
..	..	@@ -3055,56 +3841,55 @@
3055	3841	out_free_head:
3056	3842	kfree(head);
3057	3843	out:
	3844	+ if (ret > 0)
	3845	+ ret = blk_status_to_errno(nvme_error_status(ret));
3058	3846	return ERR_PTR(ret);
3059	3847	}
3060	3848
3061	3849	static int nvme_init_ns_head(struct nvme_ns *ns, unsigned nsid,
3062		- struct nvme_id_ns *id)
	3850	+ struct nvme_ns_ids *ids, bool is_shared)
3063	3851	{
3064	3852	struct nvme_ctrl *ctrl = ns->ctrl;
3065		- bool is_shared = id->nmic & (1 << 0);
3066	3853	struct nvme_ns_head *head = NULL;
3067	3854	int ret = 0;
3068	3855
3069	3856	mutex_lock(&ctrl->subsys->lock);
3070		- if (is_shared)
3071		- head = __nvme_find_ns_head(ctrl->subsys, nsid);
	3857	+ head = nvme_find_ns_head(ctrl->subsys, nsid);
3072	3858	if (!head) {
3073		- head = nvme_alloc_ns_head(ctrl, nsid, id);
	3859	+ head = nvme_alloc_ns_head(ctrl, nsid, ids);
3074	3860	if (IS_ERR(head)) {
3075	3861	ret = PTR_ERR(head);
3076	3862	goto out_unlock;
3077	3863	}
	3864	+ head->shared = is_shared;
3078	3865	} else {
3079		- struct nvme_ns_ids ids;
3080		-
3081		- nvme_report_ns_ids(ctrl, nsid, id, &ids);
3082		- if (!nvme_ns_ids_equal(&head->ids, &ids)) {
	3866	+ ret = -EINVAL;
	3867	+ if (!is_shared \|\| !head->shared) {
	3868	+ dev_err(ctrl->device,
	3869	+ "Duplicate unshared namespace %d\n", nsid);
	3870	+ goto out_put_ns_head;
	3871	+ }
	3872	+ if (!nvme_ns_ids_equal(&head->ids, ids)) {
3083	3873	dev_err(ctrl->device,
3084	3874	"IDs don't match for shared namespace %d\n",
3085	3875	nsid);
3086		- ret = -EINVAL;
3087		- goto out_unlock;
	3876	+ goto out_put_ns_head;
3088	3877	}
3089	3878	}
3090	3879
3091	3880	list_add_tail(&ns->siblings, &head->list);
3092	3881	ns->head = head;
	3882	+ mutex_unlock(&ctrl->subsys->lock);
	3883	+ return 0;
3093	3884
	3885	+out_put_ns_head:
	3886	+ nvme_put_ns_head(head);
3094	3887	out_unlock:
3095	3888	mutex_unlock(&ctrl->subsys->lock);
3096	3889	return ret;
3097	3890	}
3098	3891
3099		-static int ns_cmp(void priv, struct list_head a, struct list_head *b)
3100		-{
3101		- struct nvme_ns *nsa = container_of(a, struct nvme_ns, list);
3102		- struct nvme_ns *nsb = container_of(b, struct nvme_ns, list);
3103		-
3104		- return nsa->head->ns_id - nsb->head->ns_id;
3105		-}
3106		-
3107		-static struct nvme_ns nvme_find_get_ns(struct nvme_ctrl ctrl, unsigned nsid)
	3892	+struct nvme_ns nvme_find_get_ns(struct nvme_ctrl ctrl, unsigned nsid)
3108	3893	{
3109	3894	struct nvme_ns ns, ret = NULL;
3110	3895
..	..	@@ -3122,76 +3907,59 @@
3122	3907	up_read(&ctrl->namespaces_rwsem);
3123	3908	return ret;
3124	3909	}
	3910	+EXPORT_SYMBOL_NS_GPL(nvme_find_get_ns, NVME_TARGET_PASSTHRU);
3125	3911
3126		-static int nvme_setup_streams_ns(struct nvme_ctrl ctrl, struct nvme_ns ns)
	3912	+/*
	3913	+ * Add the namespace to the controller list while keeping the list ordered.
	3914	+ */
	3915	+static void nvme_ns_add_to_ctrl_list(struct nvme_ns *ns)
3127	3916	{
3128		- struct streams_directive_params s;
3129		- int ret;
	3917	+ struct nvme_ns *tmp;
3130	3918
3131		- if (!ctrl->nr_streams)
3132		- return 0;
3133		-
3134		- ret = nvme_get_stream_params(ctrl, &s, ns->head->ns_id);
3135		- if (ret)
3136		- return ret;
3137		-
3138		- ns->sws = le32_to_cpu(s.sws);
3139		- ns->sgs = le16_to_cpu(s.sgs);
3140		-
3141		- if (ns->sws) {
3142		- unsigned int bs = 1 << ns->lba_shift;
3143		-
3144		- blk_queue_io_min(ns->queue, bs * ns->sws);
3145		- if (ns->sgs)
3146		- blk_queue_io_opt(ns->queue, bs * ns->sws * ns->sgs);
	3919	+ list_for_each_entry_reverse(tmp, &ns->ctrl->namespaces, list) {
	3920	+ if (tmp->head->ns_id < ns->head->ns_id) {
	3921	+ list_add(&ns->list, &tmp->list);
	3922	+ return;
	3923	+ }
3147	3924	}
3148		-
3149		- return 0;
	3925	+ list_add(&ns->list, &ns->ctrl->namespaces);
3150	3926	}
3151	3927
3152		-static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid)
	3928	+static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid,
	3929	+ struct nvme_ns_ids *ids)
3153	3930	{
3154	3931	struct nvme_ns *ns;
3155	3932	struct gendisk *disk;
3156	3933	struct nvme_id_ns *id;
3157	3934	char disk_name[DISK_NAME_LEN];
3158		- int node = dev_to_node(ctrl->dev), flags = GENHD_FL_EXT_DEVT;
	3935	+ int node = ctrl->numa_node, flags = GENHD_FL_EXT_DEVT, ret;
	3936	+
	3937	+ if (nvme_identify_ns(ctrl, nsid, ids, &id))
	3938	+ return;
3159	3939
3160	3940	ns = kzalloc_node(sizeof(*ns), GFP_KERNEL, node);
3161	3941	if (!ns)
3162		- return;
	3942	+ goto out_free_id;
3163	3943
3164	3944	ns->queue = blk_mq_init_queue(ctrl->tagset);
3165	3945	if (IS_ERR(ns->queue))
3166	3946	goto out_free_ns;
	3947	+
	3948	+ if (ctrl->opts && ctrl->opts->data_digest)
	3949	+ blk_queue_flag_set(QUEUE_FLAG_STABLE_WRITES, ns->queue);
	3950	+
3167	3951	blk_queue_flag_set(QUEUE_FLAG_NONROT, ns->queue);
	3952	+ if (ctrl->ops->flags & NVME_F_PCI_P2PDMA)
	3953	+ blk_queue_flag_set(QUEUE_FLAG_PCI_P2PDMA, ns->queue);
	3954	+
3168	3955	ns->queue->queuedata = ns;
3169	3956	ns->ctrl = ctrl;
3170		-
3171	3957	kref_init(&ns->kref);
3172		- ns->lba_shift = 9; /* set to a default value for 512 until disk is validated */
3173	3958
3174		- blk_queue_logical_block_size(ns->queue, 1 << ns->lba_shift);
3175		- nvme_set_queue_limits(ctrl, ns->queue);
3176		-
3177		- id = nvme_identify_ns(ctrl, nsid);
3178		- if (!id)
	3959	+ ret = nvme_init_ns_head(ns, nsid, ids, id->nmic & NVME_NS_NMIC_SHARED);
	3960	+ if (ret)
3179	3961	goto out_free_queue;
3180		-
3181		- if (id->ncap == 0)
3182		- goto out_free_id;
3183		-
3184		- if (nvme_init_ns_head(ns, nsid, id))
3185		- goto out_free_id;
3186		- nvme_setup_streams_ns(ctrl, ns);
3187	3962	nvme_set_disk_name(disk_name, ns, ctrl, &flags);
3188		-
3189		- if ((ctrl->quirks & NVME_QUIRK_LIGHTNVM) && id->vs[0] == 0x1) {
3190		- if (nvme_nvm_register(ns, disk_name, node)) {
3191		- dev_warn(ctrl->device, "LightNVM init failure\n");
3192		- goto out_unlink_ns;
3193		- }
3194		- }
3195	3963
3196	3964	disk = alloc_disk_node(0, node);
3197	3965	if (!disk)
..	..	@@ -3204,38 +3972,46 @@
3204	3972	memcpy(disk->disk_name, disk_name, DISK_NAME_LEN);
3205	3973	ns->disk = disk;
3206	3974
3207		- __nvme_revalidate_disk(disk, id);
	3975	+ if (nvme_update_ns_info(ns, id))
	3976	+ goto out_put_disk;
	3977	+
	3978	+ if ((ctrl->quirks & NVME_QUIRK_LIGHTNVM) && id->vs[0] == 0x1) {
	3979	+ ret = nvme_nvm_register(ns, disk_name, node);
	3980	+ if (ret) {
	3981	+ dev_warn(ctrl->device, "LightNVM init failure\n");
	3982	+ goto out_put_disk;
	3983	+ }
	3984	+ }
3208	3985
3209	3986	down_write(&ctrl->namespaces_rwsem);
3210		- list_add_tail(&ns->list, &ctrl->namespaces);
	3987	+ nvme_ns_add_to_ctrl_list(ns);
3211	3988	up_write(&ctrl->namespaces_rwsem);
3212		-
3213	3989	nvme_get_ctrl(ctrl);
3214	3990
3215		- device_add_disk(ctrl->device, ns->disk);
3216		- if (sysfs_create_group(&disk_to_dev(ns->disk)->kobj,
3217		- &nvme_ns_id_attr_group))
3218		- pr_warn("%s: failed to create sysfs group for identification\n",
3219		- ns->disk->disk_name);
3220		- if (ns->ndev && nvme_nvm_register_sysfs(ns))
3221		- pr_warn("%s: failed to register lightnvm sysfs group for identification\n",
3222		- ns->disk->disk_name);
	3991	+ device_add_disk(ctrl->device, ns->disk, nvme_ns_id_attr_groups);
3223	3992
3224	3993	nvme_mpath_add_disk(ns, id);
3225		- nvme_fault_inject_init(ns);
	3994	+ nvme_fault_inject_init(&ns->fault_inject, ns->disk->disk_name);
3226	3995	kfree(id);
3227	3996
3228	3997	return;
	3998	+ out_put_disk:
	3999	+ /* prevent double queue cleanup */
	4000	+ ns->disk->queue = NULL;
	4001	+ put_disk(ns->disk);
3229	4002	out_unlink_ns:
3230	4003	mutex_lock(&ctrl->subsys->lock);
3231	4004	list_del_rcu(&ns->siblings);
	4005	+ if (list_empty(&ns->head->list))
	4006	+ list_del_init(&ns->head->entry);
3232	4007	mutex_unlock(&ctrl->subsys->lock);
3233		- out_free_id:
3234		- kfree(id);
	4008	+ nvme_put_ns_head(ns->head);
3235	4009	out_free_queue:
3236	4010	blk_cleanup_queue(ns->queue);
3237	4011	out_free_ns:
3238	4012	kfree(ns);
	4013	+ out_free_id:
	4014	+ kfree(id);
3239	4015	}
3240	4016
3241	4017	static void nvme_ns_remove(struct nvme_ns *ns)
..	..	@@ -3243,20 +4019,20 @@
3243	4019	if (test_and_set_bit(NVME_NS_REMOVING, &ns->flags))
3244	4020	return;
3245	4021
3246		- nvme_fault_inject_fini(ns);
	4022	+ set_capacity(ns->disk, 0);
	4023	+ nvme_fault_inject_fini(&ns->fault_inject);
3247	4024
3248	4025	mutex_lock(&ns->ctrl->subsys->lock);
3249	4026	list_del_rcu(&ns->siblings);
	4027	+ if (list_empty(&ns->head->list))
	4028	+ list_del_init(&ns->head->entry);
3250	4029	mutex_unlock(&ns->ctrl->subsys->lock);
	4030	+
3251	4031	synchronize_rcu(); /* guarantee not available in head->list */
3252	4032	nvme_mpath_clear_current_path(ns);
3253	4033	synchronize_srcu(&ns->head->srcu); /* wait for concurrent submissions */
3254	4034
3255		- if (ns->disk && ns->disk->flags & GENHD_FL_UP) {
3256		- sysfs_remove_group(&disk_to_dev(ns->disk)->kobj,
3257		- &nvme_ns_id_attr_group);
3258		- if (ns->ndev)
3259		- nvme_nvm_unregister_sysfs(ns);
	4035	+ if (ns->disk->flags & GENHD_FL_UP) {
3260	4036	del_gendisk(ns->disk);
3261	4037	blk_cleanup_queue(ns->queue);
3262	4038	if (blk_get_integrity(ns->disk))
..	..	@@ -3271,17 +4047,91 @@
3271	4047	nvme_put_ns(ns);
3272	4048	}
3273	4049
3274		-static void nvme_validate_ns(struct nvme_ctrl *ctrl, unsigned nsid)
	4050	+static void nvme_ns_remove_by_nsid(struct nvme_ctrl *ctrl, u32 nsid)
3275	4051	{
	4052	+ struct nvme_ns *ns = nvme_find_get_ns(ctrl, nsid);
	4053	+
	4054	+ if (ns) {
	4055	+ nvme_ns_remove(ns);
	4056	+ nvme_put_ns(ns);
	4057	+ }
	4058	+}
	4059	+
	4060	+static void nvme_validate_ns(struct nvme_ns ns, struct nvme_ns_ids ids)
	4061	+{
	4062	+ struct nvme_id_ns *id;
	4063	+ int ret = NVME_SC_INVALID_NS \| NVME_SC_DNR;
	4064	+
	4065	+ if (test_bit(NVME_NS_DEAD, &ns->flags))
	4066	+ goto out;
	4067	+
	4068	+ ret = nvme_identify_ns(ns->ctrl, ns->head->ns_id, ids, &id);
	4069	+ if (ret)
	4070	+ goto out;
	4071	+
	4072	+ ret = NVME_SC_INVALID_NS \| NVME_SC_DNR;
	4073	+ if (!nvme_ns_ids_equal(&ns->head->ids, ids)) {
	4074	+ dev_err(ns->ctrl->device,
	4075	+ "identifiers changed for nsid %d\n", ns->head->ns_id);
	4076	+ goto out_free_id;
	4077	+ }
	4078	+
	4079	+ ret = nvme_update_ns_info(ns, id);
	4080	+
	4081	+out_free_id:
	4082	+ kfree(id);
	4083	+out:
	4084	+ /*
	4085	+ * Only remove the namespace if we got a fatal error back from the
	4086	+ * device, otherwise ignore the error and just move on.
	4087	+ *
	4088	+ * TODO: we should probably schedule a delayed retry here.
	4089	+ */
	4090	+ if (ret > 0 && (ret & NVME_SC_DNR))
	4091	+ nvme_ns_remove(ns);
	4092	+ else
	4093	+ revalidate_disk_size(ns->disk, true);
	4094	+}
	4095	+
	4096	+static void nvme_validate_or_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid)
	4097	+{
	4098	+ struct nvme_ns_ids ids = { };
3276	4099	struct nvme_ns *ns;
	4100	+
	4101	+ if (nvme_identify_ns_descs(ctrl, nsid, &ids))
	4102	+ return;
3277	4103
3278	4104	ns = nvme_find_get_ns(ctrl, nsid);
3279	4105	if (ns) {
3280		- if (ns->disk && revalidate_disk(ns->disk))
3281		- nvme_ns_remove(ns);
	4106	+ nvme_validate_ns(ns, &ids);
3282	4107	nvme_put_ns(ns);
3283		- } else
3284		- nvme_alloc_ns(ctrl, nsid);
	4108	+ return;
	4109	+ }
	4110	+
	4111	+ switch (ids.csi) {
	4112	+ case NVME_CSI_NVM:
	4113	+ nvme_alloc_ns(ctrl, nsid, &ids);
	4114	+ break;
	4115	+ case NVME_CSI_ZNS:
	4116	+ if (!IS_ENABLED(CONFIG_BLK_DEV_ZONED)) {
	4117	+ dev_warn(ctrl->device,
	4118	+ "nsid %u not supported without CONFIG_BLK_DEV_ZONED\n",
	4119	+ nsid);
	4120	+ break;
	4121	+ }
	4122	+ if (!nvme_multi_css(ctrl)) {
	4123	+ dev_warn(ctrl->device,
	4124	+ "command set not reported for nsid: %d\n",
	4125	+ nsid);
	4126	+ break;
	4127	+ }
	4128	+ nvme_alloc_ns(ctrl, nsid, &ids);
	4129	+ break;
	4130	+ default:
	4131	+ dev_warn(ctrl->device, "unknown csi %u for nsid %u\n",
	4132	+ ids.csi, nsid);
	4133	+ break;
	4134	+ }
3285	4135	}
3286	4136
3287	4137	static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl,
..	..	@@ -3302,39 +4152,41 @@
3302	4152
3303	4153	}
3304	4154
3305		-static int nvme_scan_ns_list(struct nvme_ctrl *ctrl, unsigned nn)
	4155	+static int nvme_scan_ns_list(struct nvme_ctrl *ctrl)
3306	4156	{
3307		- struct nvme_ns *ns;
	4157	+ const int nr_entries = NVME_IDENTIFY_DATA_SIZE / sizeof(__le32);
3308	4158	__le32 *ns_list;
3309		- unsigned i, j, nsid, prev = 0;
3310		- unsigned num_lists = DIV_ROUND_UP_ULL((u64)nn, 1024);
3311		- int ret = 0;
	4159	+ u32 prev = 0;
	4160	+ int ret = 0, i;
	4161	+
	4162	+ if (nvme_ctrl_limited_cns(ctrl))
	4163	+ return -EOPNOTSUPP;
3312	4164
3313	4165	ns_list = kzalloc(NVME_IDENTIFY_DATA_SIZE, GFP_KERNEL);
3314	4166	if (!ns_list)
3315	4167	return -ENOMEM;
3316	4168
3317		- for (i = 0; i < num_lists; i++) {
3318		- ret = nvme_identify_ns_list(ctrl, prev, ns_list);
	4169	+ for (;;) {
	4170	+ struct nvme_command cmd = {
	4171	+ .identify.opcode = nvme_admin_identify,
	4172	+ .identify.cns = NVME_ID_CNS_NS_ACTIVE_LIST,
	4173	+ .identify.nsid = cpu_to_le32(prev),
	4174	+ };
	4175	+
	4176	+ ret = nvme_submit_sync_cmd(ctrl->admin_q, &cmd, ns_list,
	4177	+ NVME_IDENTIFY_DATA_SIZE);
3319	4178	if (ret)
3320	4179	goto free;
3321	4180
3322		- for (j = 0; j < min(nn, 1024U); j++) {
3323		- nsid = le32_to_cpu(ns_list[j]);
3324		- if (!nsid)
	4181	+ for (i = 0; i < nr_entries; i++) {
	4182	+ u32 nsid = le32_to_cpu(ns_list[i]);
	4183	+
	4184	+ if (!nsid) /* end of the list? */
3325	4185	goto out;
3326		-
3327		- nvme_validate_ns(ctrl, nsid);
3328		-
3329		- while (++prev < nsid) {
3330		- ns = nvme_find_get_ns(ctrl, prev);
3331		- if (ns) {
3332		- nvme_ns_remove(ns);
3333		- nvme_put_ns(ns);
3334		- }
3335		- }
	4186	+ nvme_validate_or_alloc_ns(ctrl, nsid);
	4187	+ while (++prev < nsid)
	4188	+ nvme_ns_remove_by_nsid(ctrl, prev);
3336	4189	}
3337		- nn -= j;
3338	4190	}
3339	4191	out:
3340	4192	nvme_remove_invalid_namespaces(ctrl, prev);
..	..	@@ -3343,12 +4195,18 @@
3343	4195	return ret;
3344	4196	}
3345	4197
3346		-static void nvme_scan_ns_sequential(struct nvme_ctrl *ctrl, unsigned nn)
	4198	+static void nvme_scan_ns_sequential(struct nvme_ctrl *ctrl)
3347	4199	{
3348		- unsigned i;
	4200	+ struct nvme_id_ctrl *id;
	4201	+ u32 nn, i;
	4202	+
	4203	+ if (nvme_identify_ctrl(ctrl, &id))
	4204	+ return;
	4205	+ nn = le32_to_cpu(id->nn);
	4206	+ kfree(id);
3349	4207
3350	4208	for (i = 1; i <= nn; i++)
3351		- nvme_validate_ns(ctrl, i);
	4209	+ nvme_validate_or_alloc_ns(ctrl, i);
3352	4210
3353	4211	nvme_remove_invalid_namespaces(ctrl, nn);
3354	4212	}
..	..	@@ -3369,8 +4227,8 @@
3369	4227	* raced with us in reading the log page, which could cause us to miss
3370	4228	* updates.
3371	4229	*/
3372		- error = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_CHANGED_NS, 0, log,
3373		- log_size, 0);
	4230	+ error = nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_CHANGED_NS, 0,
	4231	+ NVME_CSI_NVM, log, log_size, 0);
3374	4232	if (error)
3375	4233	dev_warn(ctrl->device,
3376	4234	"reading changed ns log failed: %d\n", error);
..	..	@@ -3382,35 +4240,20 @@
3382	4240	{
3383	4241	struct nvme_ctrl *ctrl =
3384	4242	container_of(work, struct nvme_ctrl, scan_work);
3385		- struct nvme_id_ctrl *id;
3386		- unsigned nn;
3387	4243
3388		- if (ctrl->state != NVME_CTRL_LIVE)
	4244	+ /* No tagset on a live ctrl means IO queues could not created */
	4245	+ if (ctrl->state != NVME_CTRL_LIVE \|\| !ctrl->tagset)
3389	4246	return;
3390		-
3391		- WARN_ON_ONCE(!ctrl->tagset);
3392	4247
3393	4248	if (test_and_clear_bit(NVME_AER_NOTICE_NS_CHANGED, &ctrl->events)) {
3394	4249	dev_info(ctrl->device, "rescanning namespaces.\n");
3395	4250	nvme_clear_changed_ns_log(ctrl);
3396	4251	}
3397	4252
3398		- if (nvme_identify_ctrl(ctrl, &id))
3399		- return;
3400		-
3401	4253	mutex_lock(&ctrl->scan_lock);
3402		- nn = le32_to_cpu(id->nn);
3403		- if (!nvme_ctrl_limited_cns(ctrl)) {
3404		- if (!nvme_scan_ns_list(ctrl, nn))
3405		- goto out_free_id;
3406		- }
3407		- nvme_scan_ns_sequential(ctrl, nn);
3408		-out_free_id:
	4254	+ if (nvme_scan_ns_list(ctrl) != 0)
	4255	+ nvme_scan_ns_sequential(ctrl);
3409	4256	mutex_unlock(&ctrl->scan_lock);
3410		- kfree(id);
3411		- down_write(&ctrl->namespaces_rwsem);
3412		- list_sort(NULL, &ctrl->namespaces, ns_cmp);
3413		- up_write(&ctrl->namespaces_rwsem);
3414	4257	}
3415	4258
3416	4259	/*
..	..	@@ -3422,6 +4265,13 @@
3422	4265	{
3423	4266	struct nvme_ns ns, next;
3424	4267	LIST_HEAD(ns_list);
	4268	+
	4269	+ /*
	4270	+ * make sure to requeue I/O to all namespaces as these
	4271	+ * might result from the scan itself and must complete
	4272	+ * for the scan_work to make progress
	4273	+ */
	4274	+ nvme_mpath_clear_ctrl_paths(ctrl);
3425	4275
3426	4276	/* prevent racing with ns scanning */
3427	4277	flush_work(&ctrl->scan_work);
..	..	@@ -3435,6 +4285,9 @@
3435	4285	if (ctrl->state == NVME_CTRL_DEAD)
3436	4286	nvme_kill_queues(ctrl);
3437	4287
	4288	+ /* this is a no-op when called from the controller reset handler */
	4289	+ nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING_NOIO);
	4290	+
3438	4291	down_write(&ctrl->namespaces_rwsem);
3439	4292	list_splice_init(&ctrl->namespaces, &ns_list);
3440	4293	up_write(&ctrl->namespaces_rwsem);
..	..	@@ -3443,6 +4296,33 @@
3443	4296	nvme_ns_remove(ns);
3444	4297	}
3445	4298	EXPORT_SYMBOL_GPL(nvme_remove_namespaces);
	4299	+
	4300	+static int nvme_class_uevent(struct device dev, struct kobj_uevent_env env)
	4301	+{
	4302	+ struct nvme_ctrl *ctrl =
	4303	+ container_of(dev, struct nvme_ctrl, ctrl_device);
	4304	+ struct nvmf_ctrl_options *opts = ctrl->opts;
	4305	+ int ret;
	4306	+
	4307	+ ret = add_uevent_var(env, "NVME_TRTYPE=%s", ctrl->ops->name);
	4308	+ if (ret)
	4309	+ return ret;
	4310	+
	4311	+ if (opts) {
	4312	+ ret = add_uevent_var(env, "NVME_TRADDR=%s", opts->traddr);
	4313	+ if (ret)
	4314	+ return ret;
	4315	+
	4316	+ ret = add_uevent_var(env, "NVME_TRSVCID=%s",
	4317	+ opts->trsvcid ?: "none");
	4318	+ if (ret)
	4319	+ return ret;
	4320	+
	4321	+ ret = add_uevent_var(env, "NVME_HOST_TRADDR=%s",
	4322	+ opts->host_traddr ?: "none");
	4323	+ }
	4324	+ return ret;
	4325	+}
3446	4326
3447	4327	static void nvme_aen_uevent(struct nvme_ctrl *ctrl)
3448	4328	{
..	..	@@ -3498,8 +4378,8 @@
3498	4378	if (!log)
3499	4379	return;
3500	4380
3501		- if (nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_FW_SLOT, 0, log,
3502		- sizeof(*log), 0))
	4381	+ if (nvme_get_log(ctrl, NVME_NSID_ALL, NVME_LOG_FW_SLOT, 0, NVME_CSI_NVM,
	4382	+ log, sizeof(*log), 0))
3503	4383	dev_warn(ctrl->device, "Get FW SLOT INFO log error\n");
3504	4384	kfree(log);
3505	4385	}
..	..	@@ -3522,13 +4402,13 @@
3522	4402	if (time_after(jiffies, fw_act_timeout)) {
3523	4403	dev_warn(ctrl->device,
3524	4404	"Fw activation timeout, reset controller\n");
3525		- nvme_reset_ctrl(ctrl);
3526		- break;
	4405	+ nvme_try_sched_reset(ctrl);
	4406	+ return;
3527	4407	}
3528	4408	msleep(100);
3529	4409	}
3530	4410
3531		- if (ctrl->state != NVME_CTRL_LIVE)
	4411	+ if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_LIVE))
3532	4412	return;
3533	4413
3534	4414	nvme_start_queues(ctrl);
..	..	@@ -3536,15 +4416,33 @@
3536	4416	nvme_get_fw_slot_info(ctrl);
3537	4417	}
3538	4418
	4419	+static u32 nvme_aer_type(u32 result)
	4420	+{
	4421	+ return result & 0x7;
	4422	+}
	4423	+
	4424	+static u32 nvme_aer_subtype(u32 result)
	4425	+{
	4426	+ return (result & 0xff00) >> 8;
	4427	+}
	4428	+
3539	4429	static void nvme_handle_aen_notice(struct nvme_ctrl *ctrl, u32 result)
3540	4430	{
3541		- switch ((result & 0xff00) >> 8) {
	4431	+ u32 aer_notice_type = nvme_aer_subtype(result);
	4432	+
	4433	+ switch (aer_notice_type) {
3542	4434	case NVME_AER_NOTICE_NS_CHANGED:
3543	4435	set_bit(NVME_AER_NOTICE_NS_CHANGED, &ctrl->events);
3544	4436	nvme_queue_scan(ctrl);
3545	4437	break;
3546	4438	case NVME_AER_NOTICE_FW_ACT_STARTING:
3547		- queue_work(nvme_wq, &ctrl->fw_act_work);
	4439	+ /*
	4440	+ * We are (ab)using the RESETTING state to prevent subsequent
	4441	+ * recovery actions from interfering with the controller's
	4442	+ * firmware activation.
	4443	+ */
	4444	+ if (nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING))
	4445	+ queue_work(nvme_wq, &ctrl->fw_act_work);
3548	4446	break;
3549	4447	#ifdef CONFIG_NVME_MULTIPATH
3550	4448	case NVME_AER_NOTICE_ANA:
..	..	@@ -3553,24 +4451,45 @@
3553	4451	queue_work(nvme_wq, &ctrl->ana_work);
3554	4452	break;
3555	4453	#endif
	4454	+ case NVME_AER_NOTICE_DISC_CHANGED:
	4455	+ ctrl->aen_result = result;
	4456	+ break;
3556	4457	default:
3557	4458	dev_warn(ctrl->device, "async event result %08x\n", result);
3558	4459	}
	4460	+}
	4461	+
	4462	+static void nvme_handle_aer_persistent_error(struct nvme_ctrl *ctrl)
	4463	+{
	4464	+ dev_warn(ctrl->device, "resetting controller due to AER\n");
	4465	+ nvme_reset_ctrl(ctrl);
3559	4466	}
3560	4467
3561	4468	void nvme_complete_async_event(struct nvme_ctrl *ctrl, __le16 status,
3562	4469	volatile union nvme_result *res)
3563	4470	{
3564	4471	u32 result = le32_to_cpu(res->u32);
	4472	+ u32 aer_type = nvme_aer_type(result);
	4473	+ u32 aer_subtype = nvme_aer_subtype(result);
3565	4474
3566	4475	if (le16_to_cpu(status) >> 1 != NVME_SC_SUCCESS)
3567	4476	return;
3568	4477
3569		- switch (result & 0x7) {
	4478	+ trace_nvme_async_event(ctrl, result);
	4479	+ switch (aer_type) {
3570	4480	case NVME_AER_NOTICE:
3571	4481	nvme_handle_aen_notice(ctrl, result);
3572	4482	break;
3573	4483	case NVME_AER_ERROR:
	4484	+ /*
	4485	+ * For a persistent internal error, don't run async_event_work
	4486	+ * to submit a new AER. The controller reset will do it.
	4487	+ */
	4488	+ if (aer_subtype == NVME_AER_ERROR_PERSIST_INT_ERR) {
	4489	+ nvme_handle_aer_persistent_error(ctrl);
	4490	+ return;
	4491	+ }
	4492	+ fallthrough;
3574	4493	case NVME_AER_SMART:
3575	4494	case NVME_AER_CSS:
3576	4495	case NVME_AER_VS:
..	..	@@ -3596,25 +4515,40 @@
3596	4515
3597	4516	void nvme_start_ctrl(struct nvme_ctrl *ctrl)
3598	4517	{
3599		- if (ctrl->kato)
3600		- nvme_start_keep_alive(ctrl);
	4518	+ nvme_start_keep_alive(ctrl);
	4519	+
	4520	+ nvme_enable_aen(ctrl);
3601	4521
3602	4522	if (ctrl->queue_count > 1) {
3603	4523	nvme_queue_scan(ctrl);
3604		- nvme_enable_aen(ctrl);
3605		- queue_work(nvme_wq, &ctrl->async_event_work);
3606	4524	nvme_start_queues(ctrl);
	4525	+ nvme_mpath_update(ctrl);
3607	4526	}
3608		- ctrl->created = true;
3609	4527	}
3610	4528	EXPORT_SYMBOL_GPL(nvme_start_ctrl);
3611	4529
3612	4530	void nvme_uninit_ctrl(struct nvme_ctrl *ctrl)
3613	4531	{
	4532	+ nvme_hwmon_exit(ctrl);
	4533	+ nvme_fault_inject_fini(&ctrl->fault_inject);
3614	4534	dev_pm_qos_hide_latency_tolerance(ctrl->device);
3615	4535	cdev_device_del(&ctrl->cdev, ctrl->device);
	4536	+ nvme_put_ctrl(ctrl);
3616	4537	}
3617	4538	EXPORT_SYMBOL_GPL(nvme_uninit_ctrl);
	4539	+
	4540	+static void nvme_free_cels(struct nvme_ctrl *ctrl)
	4541	+{
	4542	+ struct nvme_effects_log *cel;
	4543	+ unsigned long i;
	4544	+
	4545	+ xa_for_each (&ctrl->cels, i, cel) {
	4546	+ xa_erase(&ctrl->cels, i);
	4547	+ kfree(cel);
	4548	+ }
	4549	+
	4550	+ xa_destroy(&ctrl->cels);
	4551	+}
3618	4552
3619	4553	static void nvme_free_ctrl(struct device *dev)
3620	4554	{
..	..	@@ -3622,16 +4556,18 @@
3622	4556	container_of(dev, struct nvme_ctrl, ctrl_device);
3623	4557	struct nvme_subsystem *subsys = ctrl->subsys;
3624	4558
3625		- ida_simple_remove(&nvme_instance_ida, ctrl->instance);
3626		- kfree(ctrl->effects);
	4559	+ if (!subsys \|\| ctrl->instance != subsys->instance)
	4560	+ ida_simple_remove(&nvme_instance_ida, ctrl->instance);
	4561	+
	4562	+ nvme_free_cels(ctrl);
3627	4563	nvme_mpath_uninit(ctrl);
3628	4564	__free_page(ctrl->discard_page);
3629	4565
3630	4566	if (subsys) {
3631		- mutex_lock(&subsys->lock);
	4567	+ mutex_lock(&nvme_subsystems_lock);
3632	4568	list_del(&ctrl->subsys_entry);
3633		- mutex_unlock(&subsys->lock);
3634	4569	sysfs_remove_link(&subsys->dev.kobj, dev_name(ctrl->device));
	4570	+ mutex_unlock(&nvme_subsystems_lock);
3635	4571	}
3636	4572
3637	4573	ctrl->ops->free_ctrl(ctrl);
..	..	@@ -3654,14 +4590,17 @@
3654	4590	spin_lock_init(&ctrl->lock);
3655	4591	mutex_init(&ctrl->scan_lock);
3656	4592	INIT_LIST_HEAD(&ctrl->namespaces);
	4593	+ xa_init(&ctrl->cels);
3657	4594	init_rwsem(&ctrl->namespaces_rwsem);
3658	4595	ctrl->dev = dev;
3659	4596	ctrl->ops = ops;
3660	4597	ctrl->quirks = quirks;
	4598	+ ctrl->numa_node = NUMA_NO_NODE;
3661	4599	INIT_WORK(&ctrl->scan_work, nvme_scan_work);
3662	4600	INIT_WORK(&ctrl->async_event_work, nvme_async_event_work);
3663	4601	INIT_WORK(&ctrl->fw_act_work, nvme_fw_act_work);
3664	4602	INIT_WORK(&ctrl->delete_work, nvme_delete_ctrl_work);
	4603	+ init_waitqueue_head(&ctrl->state_wq);
3665	4604
3666	4605	INIT_DELAYED_WORK(&ctrl->ka_work, nvme_keep_alive_work);
3667	4606	memset(&ctrl->ka_cmd, 0, sizeof(ctrl->ka_cmd));
..	..	@@ -3692,6 +4631,7 @@
3692	4631	if (ret)
3693	4632	goto out_release_instance;
3694	4633
	4634	+ nvme_get_ctrl(ctrl);
3695	4635	cdev_init(&ctrl->cdev, &nvme_dev_fops);
3696	4636	ctrl->cdev.owner = ops->module;
3697	4637	ret = cdev_device_add(&ctrl->cdev, ctrl->device);
..	..	@@ -3706,8 +4646,12 @@
3706	4646	dev_pm_qos_update_user_latency_tolerance(ctrl->device,
3707	4647	min(default_ps_max_latency_us, (unsigned long)S32_MAX));
3708	4648
	4649	+ nvme_fault_inject_init(&ctrl->fault_inject, dev_name(ctrl->device));
	4650	+ nvme_mpath_init_ctrl(ctrl);
	4651	+
3709	4652	return 0;
3710	4653	out_free_name:
	4654	+ nvme_put_ctrl(ctrl);
3711	4655	kfree_const(ctrl->device->kobj.name);
3712	4656	out_release_instance:
3713	4657	ida_simple_remove(&nvme_instance_ida, ctrl->instance);
..	..	@@ -3753,7 +4697,7 @@
3753	4697	}
3754	4698	EXPORT_SYMBOL_GPL(nvme_unfreeze);
3755	4699
3756		-void nvme_wait_freeze_timeout(struct nvme_ctrl *ctrl, long timeout)
	4700	+int nvme_wait_freeze_timeout(struct nvme_ctrl *ctrl, long timeout)
3757	4701	{
3758	4702	struct nvme_ns *ns;
3759	4703
..	..	@@ -3764,6 +4708,7 @@
3764	4708	break;
3765	4709	}
3766	4710	up_read(&ctrl->namespaces_rwsem);
	4711	+ return timeout;
3767	4712	}
3768	4713	EXPORT_SYMBOL_GPL(nvme_wait_freeze_timeout);
3769	4714
..	..	@@ -3811,9 +4756,65 @@
3811	4756	}
3812	4757	EXPORT_SYMBOL_GPL(nvme_start_queues);
3813	4758
3814		-int __init nvme_core_init(void)
	4759	+void nvme_sync_io_queues(struct nvme_ctrl *ctrl)
	4760	+{
	4761	+ struct nvme_ns *ns;
	4762	+
	4763	+ down_read(&ctrl->namespaces_rwsem);
	4764	+ list_for_each_entry(ns, &ctrl->namespaces, list)
	4765	+ blk_sync_queue(ns->queue);
	4766	+ up_read(&ctrl->namespaces_rwsem);
	4767	+}
	4768	+EXPORT_SYMBOL_GPL(nvme_sync_io_queues);
	4769	+
	4770	+void nvme_sync_queues(struct nvme_ctrl *ctrl)
	4771	+{
	4772	+ nvme_sync_io_queues(ctrl);
	4773	+ if (ctrl->admin_q)
	4774	+ blk_sync_queue(ctrl->admin_q);
	4775	+}
	4776	+EXPORT_SYMBOL_GPL(nvme_sync_queues);
	4777	+
	4778	+struct nvme_ctrl nvme_ctrl_from_file(struct file file)
	4779	+{
	4780	+ if (file->f_op != &nvme_dev_fops)
	4781	+ return NULL;
	4782	+ return file->private_data;
	4783	+}
	4784	+EXPORT_SYMBOL_NS_GPL(nvme_ctrl_from_file, NVME_TARGET_PASSTHRU);
	4785	+
	4786	+/*
	4787	+ * Check we didn't inadvertently grow the command structure sizes:
	4788	+ */
	4789	+static inline void _nvme_check_size(void)
	4790	+{
	4791	+ BUILD_BUG_ON(sizeof(struct nvme_common_command) != 64);
	4792	+ BUILD_BUG_ON(sizeof(struct nvme_rw_command) != 64);
	4793	+ BUILD_BUG_ON(sizeof(struct nvme_identify) != 64);
	4794	+ BUILD_BUG_ON(sizeof(struct nvme_features) != 64);
	4795	+ BUILD_BUG_ON(sizeof(struct nvme_download_firmware) != 64);
	4796	+ BUILD_BUG_ON(sizeof(struct nvme_format_cmd) != 64);
	4797	+ BUILD_BUG_ON(sizeof(struct nvme_dsm_cmd) != 64);
	4798	+ BUILD_BUG_ON(sizeof(struct nvme_write_zeroes_cmd) != 64);
	4799	+ BUILD_BUG_ON(sizeof(struct nvme_abort_cmd) != 64);
	4800	+ BUILD_BUG_ON(sizeof(struct nvme_get_log_page_command) != 64);
	4801	+ BUILD_BUG_ON(sizeof(struct nvme_command) != 64);
	4802	+ BUILD_BUG_ON(sizeof(struct nvme_id_ctrl) != NVME_IDENTIFY_DATA_SIZE);
	4803	+ BUILD_BUG_ON(sizeof(struct nvme_id_ns) != NVME_IDENTIFY_DATA_SIZE);
	4804	+ BUILD_BUG_ON(sizeof(struct nvme_id_ns_zns) != NVME_IDENTIFY_DATA_SIZE);
	4805	+ BUILD_BUG_ON(sizeof(struct nvme_id_ctrl_zns) != NVME_IDENTIFY_DATA_SIZE);
	4806	+ BUILD_BUG_ON(sizeof(struct nvme_lba_range_type) != 64);
	4807	+ BUILD_BUG_ON(sizeof(struct nvme_smart_log) != 512);
	4808	+ BUILD_BUG_ON(sizeof(struct nvme_dbbuf) != 64);
	4809	+ BUILD_BUG_ON(sizeof(struct nvme_directive_cmd) != 64);
	4810	+}
	4811	+
	4812	+
	4813	+static int __init nvme_core_init(void)
3815	4814	{
3816	4815	int result = -ENOMEM;
	4816	+
	4817	+ _nvme_check_size();
3817	4818
3818	4819	nvme_wq = alloc_workqueue("nvme-wq",
3819	4820	WQ_UNBOUND \| WQ_MEM_RECLAIM \| WQ_SYSFS, 0);
..	..	@@ -3839,6 +4840,7 @@
3839	4840	result = PTR_ERR(nvme_class);
3840	4841	goto unregister_chrdev;
3841	4842	}
	4843	+ nvme_class->dev_uevent = nvme_class_uevent;
3842	4844
3843	4845	nvme_subsys_class = class_create(THIS_MODULE, "nvme-subsystem");
3844	4846	if (IS_ERR(nvme_subsys_class)) {
..	..	@@ -3861,15 +4863,15 @@
3861	4863	return result;
3862	4864	}
3863	4865
3864		-void nvme_core_exit(void)
	4866	+static void __exit nvme_core_exit(void)
3865	4867	{
3866		- ida_destroy(&nvme_subsystems_ida);
3867	4868	class_destroy(nvme_subsys_class);
3868	4869	class_destroy(nvme_class);
3869	4870	unregister_chrdev_region(nvme_chr_devt, NVME_MINORS);
3870	4871	destroy_workqueue(nvme_delete_wq);
3871	4872	destroy_workqueue(nvme_reset_wq);
3872	4873	destroy_workqueue(nvme_wq);
	4874	+ ida_destroy(&nvme_instance_ida);
3873	4875	}
3874	4876
3875	4877	MODULE_LICENSE("GPL");