~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,3 +1,4 @@
	1	+// SPDX-License-Identifier: GPL-2.0-only
1	2	/*
2	3	* Copyright (C) 1999 Eric Youngdale
3	4	* Copyright (C) 2014 Christoph Hellwig
..	..	@@ -39,7 +40,18 @@
39	40	#include "scsi_priv.h"
40	41	#include "scsi_logging.h"
41	42
42		-static struct kmem_cache *scsi_sdb_cache;
	43	+/*
	44	+ * Size of integrity metadata is usually small, 1 inline sg should
	45	+ * cover normal cases.
	46	+ */
	47	+#ifdef CONFIG_ARCH_NO_SG_CHAIN
	48	+#define SCSI_INLINE_PROT_SG_CNT 0
	49	+#define SCSI_INLINE_SG_CNT 0
	50	+#else
	51	+#define SCSI_INLINE_PROT_SG_CNT 1
	52	+#define SCSI_INLINE_SG_CNT 2
	53	+#endif
	54	+
43	55	static struct kmem_cache *scsi_sense_cache;
44	56	static struct kmem_cache *scsi_sense_isadma_cache;
45	57	static DEFINE_MUTEX(scsi_sense_cache_mutex);
..	..	@@ -141,8 +153,6 @@
141	153
142	154	static void scsi_mq_requeue_cmd(struct scsi_cmnd *cmd)
143	155	{
144		- struct scsi_device *sdev = cmd->device;
145		-
146	156	if (cmd->request->rq_flags & RQF_DONTPREP) {
147	157	cmd->request->rq_flags &= ~RQF_DONTPREP;
148	158	scsi_mq_uninit_cmd(cmd);
..	..	@@ -150,7 +160,6 @@
150	160	WARN_ON_ONCE(true);
151	161	}
152	162	blk_mq_requeue_request(cmd->request, true);
153		- put_device(&sdev->sdev_gendev);
154	163	}
155	164
156	165	/**
..	..	@@ -168,8 +177,6 @@
168	177	static void __scsi_queue_insert(struct scsi_cmnd *cmd, int reason, bool unbusy)
169	178	{
170	179	struct scsi_device *device = cmd->device;
171		- struct request_queue *q = device->request_queue;
172		- unsigned long flags;
173	180
174	181	SCSI_LOG_MLQUEUE(1, scmd_printk(KERN_INFO, cmd,
175	182	"Inserting command %p into mlqueue\n", cmd));
..	..	@@ -181,7 +188,7 @@
181	188	* active on the host/device.
182	189	*/
183	190	if (unbusy)
184		- scsi_device_unbusy(device);
	191	+ scsi_device_unbusy(device, cmd);
185	192
186	193	/*
187	194	* Requeue this command. It will go before all other commands
..	..	@@ -190,46 +197,21 @@
190	197	* before blk_cleanup_queue() finishes.
191	198	*/
192	199	cmd->result = 0;
193		- if (q->mq_ops) {
194		- /*
195		- * Before a SCSI command is dispatched,
196		- * get_device(&sdev->sdev_gendev) is called and the host,
197		- * target and device busy counters are increased. Since
198		- * requeuing a request causes these actions to be repeated and
199		- * since scsi_device_unbusy() has already been called,
200		- * put_device(&device->sdev_gendev) must still be called. Call
201		- * put_device() after blk_mq_requeue_request() to avoid that
202		- * removal of the SCSI device can start before requeueing has
203		- * happened.
204		- */
205		- blk_mq_requeue_request(cmd->request, true);
206		- put_device(&device->sdev_gendev);
207		- return;
208		- }
209		- spin_lock_irqsave(q->queue_lock, flags);
210		- blk_requeue_request(q, cmd->request);
211		- kblockd_schedule_work(&device->requeue_work);
212		- spin_unlock_irqrestore(q->queue_lock, flags);
	200	+
	201	+ blk_mq_requeue_request(cmd->request, true);
213	202	}
214	203
215		-/*
216		- * Function: scsi_queue_insert()
	204	+/**
	205	+ * scsi_queue_insert - Reinsert a command in the queue.
	206	+ * @cmd: command that we are adding to queue.
	207	+ * @reason: why we are inserting command to queue.
217	208	*
218		- * Purpose: Insert a command in the midlevel queue.
	209	+ * We do this for one of two cases. Either the host is busy and it cannot accept
	210	+ * any more commands for the time being, or the device returned QUEUE_FULL and
	211	+ * can accept no more commands.
219	212	*
220		- * Arguments: cmd - command that we are adding to queue.
221		- * reason - why we are inserting command to queue.
222		- *
223		- * Lock status: Assumed that lock is not held upon entry.
224		- *
225		- * Returns: Nothing.
226		- *
227		- * Notes: We do this for one of two cases. Either the host is busy
228		- * and it cannot accept any more commands for the time being,
229		- * or the device returned QUEUE_FULL and can accept no more
230		- * commands.
231		- * Notes: This could be called either from an interrupt context or a
232		- * normal process context.
	213	+ * Context: This could be called either from an interrupt context or a normal
	214	+ * process context.
233	215	*/
234	216	void scsi_queue_insert(struct scsi_cmnd *cmd, int reason)
235	217	{
..	..	@@ -267,7 +249,8 @@
267	249
268	250	req = blk_get_request(sdev->request_queue,
269	251	data_direction == DMA_TO_DEVICE ?
270		- REQ_OP_SCSI_OUT : REQ_OP_SCSI_IN, BLK_MQ_REQ_PREEMPT);
	252	+ REQ_OP_SCSI_OUT : REQ_OP_SCSI_IN,
	253	+ rq_flags & RQF_PM ? BLK_MQ_REQ_PM : 0);
271	254	if (IS_ERR(req))
272	255	return ret;
273	256	rq = scsi_req(req);
..	..	@@ -279,11 +262,7 @@
279	262	rq->cmd_len = COMMAND_SIZE(cmd[0]);
280	263	memcpy(rq->cmd, cmd, rq->cmd_len);
281	264	rq->retries = retries;
282		- if (likely(!sdev->timeout_override))
283		- req->timeout = timeout;
284		- else
285		- req->timeout = sdev->timeout_override;
286		-
	265	+ req->timeout = timeout;
287	266	req->cmd_flags \|= flags;
288	267	req->rq_flags \|= rq_flags \| RQF_QUIET;
289	268
..	..	@@ -316,40 +295,20 @@
316	295	EXPORT_SYMBOL(__scsi_execute);
317	296
318	297	/*
319		- * Function: scsi_init_cmd_errh()
320		- *
321		- * Purpose: Initialize cmd fields related to error handling.
322		- *
323		- * Arguments: cmd - command that is ready to be queued.
324		- *
325		- * Notes: This function has the job of initializing a number of
326		- * fields related to error handling. Typically this will
327		- * be called once for each command, as required.
328		- */
329		-static void scsi_init_cmd_errh(struct scsi_cmnd *cmd)
330		-{
331		- cmd->serial_number = 0;
332		- scsi_set_resid(cmd, 0);
333		- memset(cmd->sense_buffer, 0, SCSI_SENSE_BUFFERSIZE);
334		- if (cmd->cmd_len == 0)
335		- cmd->cmd_len = scsi_command_size(cmd->cmnd);
336		-}
337		-
338		-/*
339		- * Decrement the host_busy counter and wake up the error handler if necessary.
340		- * Avoid as follows that the error handler is not woken up if shost->host_busy
341		- * == shost->host_failed: use call_rcu() in scsi_eh_scmd_add() in combination
342		- * with an RCU read lock in this function to ensure that this function in its
343		- * entirety either finishes before scsi_eh_scmd_add() increases the
	298	+ * Wake up the error handler if necessary. Avoid as follows that the error
	299	+ * handler is not woken up if host in-flight requests number ==
	300	+ * shost->host_failed: use call_rcu() in scsi_eh_scmd_add() in combination
	301	+ * with an RCU read lock in this function to ensure that this function in
	302	+ * its entirety either finishes before scsi_eh_scmd_add() increases the
344	303	* host_failed counter or that it notices the shost state change made by
345	304	* scsi_eh_scmd_add().
346	305	*/
347		-static void scsi_dec_host_busy(struct Scsi_Host *shost)
	306	+static void scsi_dec_host_busy(struct Scsi_Host shost, struct scsi_cmnd cmd)
348	307	{
349	308	unsigned long flags;
350	309
351	310	rcu_read_lock();
352		- atomic_dec(&shost->host_busy);
	311	+ __clear_bit(SCMD_STATE_INFLIGHT, &cmd->state);
353	312	if (unlikely(scsi_host_in_recovery(shost))) {
354	313	spin_lock_irqsave(shost->host_lock, flags);
355	314	if (shost->host_failed \|\| shost->host_eh_scheduled)
..	..	@@ -359,12 +318,12 @@
359	318	rcu_read_unlock();
360	319	}
361	320
362		-void scsi_device_unbusy(struct scsi_device *sdev)
	321	+void scsi_device_unbusy(struct scsi_device sdev, struct scsi_cmnd cmd)
363	322	{
364	323	struct Scsi_Host *shost = sdev->host;
365	324	struct scsi_target *starget = scsi_target(sdev);
366	325
367		- scsi_dec_host_busy(shost);
	326	+ scsi_dec_host_busy(shost, cmd);
368	327
369	328	if (starget->can_queue > 0)
370	329	atomic_dec(&starget->target_busy);
..	..	@@ -374,10 +333,7 @@
374	333
375	334	static void scsi_kick_queue(struct request_queue *q)
376	335	{
377		- if (q->mq_ops)
378		- blk_mq_run_hw_queues(q, false);
379		- else
380		- blk_run_queue(q);
	336	+ blk_mq_run_hw_queues(q, false);
381	337	}
382	338
383	339	/*
..	..	@@ -419,7 +375,7 @@
419	375	spin_unlock_irqrestore(shost->host_lock, flags);
420	376	scsi_kick_queue(sdev->request_queue);
421	377	spin_lock_irqsave(shost->host_lock, flags);
422		-
	378	+
423	379	scsi_device_put(sdev);
424	380	}
425	381	out:
..	..	@@ -448,9 +404,6 @@
448	404
449	405	static inline bool scsi_host_is_busy(struct Scsi_Host *shost)
450	406	{
451		- if (shost->can_queue > 0 &&
452		- atomic_read(&shost->host_busy) >= shost->can_queue)
453		- return true;
454	407	if (atomic_read(&shost->host_blocked) > 0)
455	408	return true;
456	409	if (shost->host_self_blocked)
..	..	@@ -517,17 +470,11 @@
517	470	spin_unlock_irqrestore(shost->host_lock, flags);
518	471	}
519	472
520		-/*
521		- * Function: scsi_run_queue()
	473	+/**
	474	+ * scsi_run_queue - Select a proper request queue to serve next.
	475	+ * @q: last request's queue
522	476	*
523		- * Purpose: Select a proper request queue to serve next
524		- *
525		- * Arguments: q - last request's queue
526		- *
527		- * Returns: Nothing
528		- *
529		- * Notes: The previous command was completely finished, start
530		- * a new one if possible.
	477	+ * The previous command was completely finished, start a new one if possible.
531	478	*/
532	479	static void scsi_run_queue(struct request_queue *q)
533	480	{
..	..	@@ -538,10 +485,7 @@
538	485	if (!list_empty(&sdev->host->starved_list))
539	486	scsi_starved_list_run(sdev->host);
540	487
541		- if (q->mq_ops)
542		- blk_mq_run_hw_queues(q, false);
543		- else
544		- blk_run_queue(q);
	488	+ blk_mq_run_hw_queues(q, false);
545	489	}
546	490
547	491	void scsi_requeue_run_queue(struct work_struct *work)
..	..	@@ -552,42 +496,6 @@
552	496	sdev = container_of(work, struct scsi_device, requeue_work);
553	497	q = sdev->request_queue;
554	498	scsi_run_queue(q);
555		-}
556		-
557		-/*
558		- * Function: scsi_requeue_command()
559		- *
560		- * Purpose: Handle post-processing of completed commands.
561		- *
562		- * Arguments: q - queue to operate on
563		- * cmd - command that may need to be requeued.
564		- *
565		- * Returns: Nothing
566		- *
567		- * Notes: After command completion, there may be blocks left
568		- * over which weren't finished by the previous command
569		- * this can be for a number of reasons - the main one is
570		- * I/O errors in the middle of the request, in which case
571		- * we need to request the blocks that come after the bad
572		- * sector.
573		- * Notes: Upon return, cmd is a stale pointer.
574		- */
575		-static void scsi_requeue_command(struct request_queue q, struct scsi_cmnd cmd)
576		-{
577		- struct scsi_device *sdev = cmd->device;
578		- struct request *req = cmd->request;
579		- unsigned long flags;
580		-
581		- spin_lock_irqsave(q->queue_lock, flags);
582		- blk_unprep_request(req);
583		- req->special = NULL;
584		- scsi_put_command(cmd);
585		- blk_requeue_request(q, req);
586		- spin_unlock_irqrestore(q->queue_lock, flags);
587		-
588		- scsi_run_queue(q);
589		-
590		- put_device(&sdev->sdev_gendev);
591	499	}
592	500
593	501	void scsi_run_host_queues(struct Scsi_Host *shost)
..	..	@@ -608,67 +516,52 @@
608	516	}
609	517	}
610	518
611		-static void scsi_mq_free_sgtables(struct scsi_cmnd *cmd)
	519	+void scsi_free_sgtables(struct scsi_cmnd *cmd)
612	520	{
613		- struct scsi_data_buffer *sdb;
614		-
615	521	if (cmd->sdb.table.nents)
616		- sg_free_table_chained(&cmd->sdb.table, true);
617		- if (cmd->request->next_rq) {
618		- sdb = cmd->request->next_rq->special;
619		- if (sdb)
620		- sg_free_table_chained(&sdb->table, true);
621		- }
	522	+ sg_free_table_chained(&cmd->sdb.table,
	523	+ SCSI_INLINE_SG_CNT);
622	524	if (scsi_prot_sg_count(cmd))
623		- sg_free_table_chained(&cmd->prot_sdb->table, true);
	525	+ sg_free_table_chained(&cmd->prot_sdb->table,
	526	+ SCSI_INLINE_PROT_SG_CNT);
624	527	}
	528	+EXPORT_SYMBOL_GPL(scsi_free_sgtables);
625	529
626	530	static void scsi_mq_uninit_cmd(struct scsi_cmnd *cmd)
627	531	{
628		- scsi_mq_free_sgtables(cmd);
	532	+ scsi_free_sgtables(cmd);
629	533	scsi_uninit_cmd(cmd);
630		- scsi_del_cmd_from_list(cmd);
631	534	}
632	535
633		-/*
634		- * Function: scsi_release_buffers()
635		- *
636		- * Purpose: Free resources allocate for a scsi_command.
637		- *
638		- * Arguments: cmd - command that we are bailing.
639		- *
640		- * Lock status: Assumed that no lock is held upon entry.
641		- *
642		- * Returns: Nothing
643		- *
644		- * Notes: In the event that an upper level driver rejects a
645		- * command, we must release resources allocated during
646		- * the __init_io() function. Primarily this would involve
647		- * the scatter-gather table.
648		- */
649		-static void scsi_release_buffers(struct scsi_cmnd *cmd)
	536	+static void scsi_run_queue_async(struct scsi_device *sdev)
650	537	{
651		- if (cmd->sdb.table.nents)
652		- sg_free_table_chained(&cmd->sdb.table, false);
	538	+ if (scsi_target(sdev)->single_lun \|\|
	539	+ !list_empty(&sdev->host->starved_list)) {
	540	+ kblockd_schedule_work(&sdev->requeue_work);
	541	+ } else {
	542	+ /*
	543	+ * smp_mb() present in sbitmap_queue_clear() or implied in
	544	+ * .end_io is for ordering writing .device_busy in
	545	+ * scsi_device_unbusy() and reading sdev->restarts.
	546	+ */
	547	+ int old = atomic_read(&sdev->restarts);
653	548
654		- memset(&cmd->sdb, 0, sizeof(cmd->sdb));
655		-
656		- if (scsi_prot_sg_count(cmd))
657		- sg_free_table_chained(&cmd->prot_sdb->table, false);
658		-}
659		-
660		-static void scsi_release_bidi_buffers(struct scsi_cmnd *cmd)
661		-{
662		- struct scsi_data_buffer *bidi_sdb = cmd->request->next_rq->special;
663		-
664		- sg_free_table_chained(&bidi_sdb->table, false);
665		- kmem_cache_free(scsi_sdb_cache, bidi_sdb);
666		- cmd->request->next_rq->special = NULL;
	549	+ /*
	550	+ * ->restarts has to be kept as non-zero if new budget
	551	+ * contention occurs.
	552	+ *
	553	+ * No need to run queue when either another re-run
	554	+ * queue wins in updating ->restarts or a new budget
	555	+ * contention occurs.
	556	+ */
	557	+ if (old && atomic_cmpxchg(&sdev->restarts, old, 0) == old)
	558	+ blk_mq_run_hw_queues(sdev->request_queue, true);
	559	+ }
667	560	}
668	561
669	562	/* Returns false when no more bytes to process, true if there are more */
670	563	static bool scsi_end_request(struct request *req, blk_status_t error,
671		- unsigned int bytes, unsigned int bidi_bytes)
	564	+ unsigned int bytes)
672	565	{
673	566	struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req);
674	567	struct scsi_device *sdev = cmd->device;
..	..	@@ -677,61 +570,42 @@
677	570	if (blk_update_request(req, error, bytes))
678	571	return true;
679	572
680		- /* Bidi request must be completed as a whole */
681		- if (unlikely(bidi_bytes) &&
682		- blk_update_request(req->next_rq, error, bidi_bytes))
683		- return true;
684		-
685	573	if (blk_queue_add_random(q))
686	574	add_disk_randomness(req->rq_disk);
687	575
688	576	if (!blk_rq_is_scsi(req)) {
689	577	WARN_ON_ONCE(!(cmd->flags & SCMD_INITIALIZED));
690	578	cmd->flags &= ~SCMD_INITIALIZED;
691		- destroy_rcu_head(&cmd->rcu);
692	579	}
693	580
694		- if (req->mq_ctx) {
695		- /*
696		- * In the MQ case the command gets freed by __blk_mq_end_request,
697		- * so we have to do all cleanup that depends on it earlier.
698		- *
699		- * We also can't kick the queues from irq context, so we
700		- * will have to defer it to a workqueue.
701		- */
702		- scsi_mq_uninit_cmd(cmd);
	581	+ /*
	582	+ * Calling rcu_barrier() is not necessary here because the
	583	+ * SCSI error handler guarantees that the function called by
	584	+ * call_rcu() has been called before scsi_end_request() is
	585	+ * called.
	586	+ */
	587	+ destroy_rcu_head(&cmd->rcu);
703	588
704		- /*
705		- * queue is still alive, so grab the ref for preventing it
706		- * from being cleaned up during running queue.
707		- */
708		- percpu_ref_get(&q->q_usage_counter);
	589	+ /*
	590	+ * In the MQ case the command gets freed by __blk_mq_end_request,
	591	+ * so we have to do all cleanup that depends on it earlier.
	592	+ *
	593	+ * We also can't kick the queues from irq context, so we
	594	+ * will have to defer it to a workqueue.
	595	+ */
	596	+ scsi_mq_uninit_cmd(cmd);
709	597
710		- __blk_mq_end_request(req, error);
	598	+ /*
	599	+ * queue is still alive, so grab the ref for preventing it
	600	+ * from being cleaned up during running queue.
	601	+ */
	602	+ percpu_ref_get(&q->q_usage_counter);
711	603
712		- if (scsi_target(sdev)->single_lun \|\|
713		- !list_empty(&sdev->host->starved_list))
714		- kblockd_schedule_work(&sdev->requeue_work);
715		- else
716		- blk_mq_run_hw_queues(q, true);
	604	+ __blk_mq_end_request(req, error);
717	605
718		- percpu_ref_put(&q->q_usage_counter);
719		- } else {
720		- unsigned long flags;
	606	+ scsi_run_queue_async(sdev);
721	607
722		- if (bidi_bytes)
723		- scsi_release_bidi_buffers(cmd);
724		- scsi_release_buffers(cmd);
725		- scsi_put_command(cmd);
726		-
727		- spin_lock_irqsave(q->queue_lock, flags);
728		- blk_finish_request(req, error);
729		- spin_unlock_irqrestore(q->queue_lock, flags);
730		-
731		- scsi_run_queue(q);
732		- }
733		-
734		- put_device(&sdev->sdev_gendev);
	608	+ percpu_ref_put(&q->q_usage_counter);
735	609	return false;
736	610	}
737	611
..	..	@@ -779,13 +653,24 @@
779	653	struct request_queue *q)
780	654	{
781	655	/* A new command will be prepared and issued. */
782		- if (q->mq_ops) {
783		- scsi_mq_requeue_cmd(cmd);
784		- } else {
785		- /* Unprep request and put it back at head of the queue. */
786		- scsi_release_buffers(cmd);
787		- scsi_requeue_command(q, cmd);
	656	+ scsi_mq_requeue_cmd(cmd);
	657	+}
	658	+
	659	+static bool scsi_cmd_runtime_exceeced(struct scsi_cmnd *cmd)
	660	+{
	661	+ struct request *req = cmd->request;
	662	+ unsigned long wait_for;
	663	+
	664	+ if (cmd->allowed == SCSI_CMD_RETRIES_NO_LIMIT)
	665	+ return false;
	666	+
	667	+ wait_for = (cmd->allowed + 1) * req->timeout;
	668	+ if (time_before(cmd->jiffies_at_alloc + wait_for, jiffies)) {
	669	+ scmd_printk(KERN_ERR, cmd, "timing out command, waited %lus\n",
	670	+ wait_for/HZ);
	671	+ return true;
788	672	}
	673	+ return false;
789	674	}
790	675
791	676	/* Helper for scsi_io_completion() when special action required. */
..	..	@@ -796,7 +681,6 @@
796	681	int level = 0;
797	682	enum {ACTION_FAIL, ACTION_REPREP, ACTION_RETRY,
798	683	ACTION_DELAYED_RETRY} action;
799		- unsigned long wait_for = (cmd->allowed + 1) * req->timeout;
800	684	struct scsi_sense_hdr sshdr;
801	685	bool sense_valid;
802	686	bool sense_current = true; /* false implies "deferred sense" */
..	..	@@ -895,6 +779,15 @@
895	779	/* See SSC3rXX or current. */
896	780	action = ACTION_FAIL;
897	781	break;
	782	+ case DATA_PROTECT:
	783	+ action = ACTION_FAIL;
	784	+ if ((sshdr.asc == 0x0C && sshdr.ascq == 0x12) \|\|
	785	+ (sshdr.asc == 0x55 &&
	786	+ (sshdr.ascq == 0x0E \|\| sshdr.ascq == 0x0F))) {
	787	+ /* Insufficient zone resources */
	788	+ blk_stat = BLK_STS_ZONE_OPEN_RESOURCE;
	789	+ }
	790	+ break;
898	791	default:
899	792	action = ACTION_FAIL;
900	793	break;
..	..	@@ -902,8 +795,7 @@
902	795	} else
903	796	action = ACTION_FAIL;
904	797
905		- if (action != ACTION_FAIL &&
906		- time_before(cmd->jiffies_at_alloc + wait_for, jiffies))
	798	+ if (action != ACTION_FAIL && scsi_cmd_runtime_exceeced(cmd))
907	799	action = ACTION_FAIL;
908	800
909	801	switch (action) {
..	..	@@ -930,9 +822,9 @@
930	822	scsi_print_command(cmd);
931	823	}
932	824	}
933		- if (!scsi_end_request(req, blk_stat, blk_rq_err_bytes(req), 0))
	825	+ if (!scsi_end_request(req, blk_stat, blk_rq_err_bytes(req)))
934	826	return;
935		- /FALLTHRU/
	827	+ fallthrough;
936	828	case ACTION_REPREP:
937	829	scsi_io_completion_reprep(cmd, q);
938	830	break;
..	..	@@ -1020,34 +912,27 @@
1020	912	return result;
1021	913	}
1022	914
1023		-/*
1024		- * Function: scsi_io_completion()
	915	+/**
	916	+ * scsi_io_completion - Completion processing for SCSI commands.
	917	+ * @cmd: command that is finished.
	918	+ * @good_bytes: number of processed bytes.
1025	919	*
1026		- * Purpose: Completion processing for block device I/O requests.
	920	+ * We will finish off the specified number of sectors. If we are done, the
	921	+ * command block will be released and the queue function will be goosed. If we
	922	+ * are not done then we have to figure out what to do next:
1027	923	*
1028		- * Arguments: cmd - command that is finished.
	924	+ * a) We can call scsi_io_completion_reprep(). The request will be
	925	+ * unprepared and put back on the queue. Then a new command will
	926	+ * be created for it. This should be used if we made forward
	927	+ * progress, or if we want to switch from READ(10) to READ(6) for
	928	+ * example.
1029	929	*
1030		- * Lock status: Assumed that no lock is held upon entry.
	930	+ * b) We can call scsi_io_completion_action(). The request will be
	931	+ * put back on the queue and retried using the same command as
	932	+ * before, possibly after a delay.
1031	933	*
1032		- * Returns: Nothing
1033		- *
1034		- * Notes: We will finish off the specified number of sectors. If we
1035		- * are done, the command block will be released and the queue
1036		- * function will be goosed. If we are not done then we have to
1037		- * figure out what to do next:
1038		- *
1039		- * a) We can call scsi_requeue_command(). The request
1040		- * will be unprepared and put back on the queue. Then
1041		- * a new command will be created for it. This should
1042		- * be used if we made forward progress, or if we want
1043		- * to switch from READ(10) to READ(6) for example.
1044		- *
1045		- * b) We can call __scsi_queue_insert(). The request will
1046		- * be put back on the queue and retried using the same
1047		- * command as before, possibly after a delay.
1048		- *
1049		- * c) We can call scsi_end_request() with blk_stat other than
1050		- * BLK_STS_OK, to fail the remainder of the request.
	934	+ * c) We can call scsi_end_request() with blk_stat other than
	935	+ * BLK_STS_OK, to fail the remainder of the request.
1051	936	*/
1052	937	void scsi_io_completion(struct scsi_cmnd *cmd, unsigned int good_bytes)
1053	938	{
..	..	@@ -1064,30 +949,6 @@
1064	949	* scsi_result_to_blk_status may have reset the host_byte
1065	950	*/
1066	951	scsi_req(req)->result = cmd->result;
1067		- scsi_req(req)->resid_len = scsi_get_resid(cmd);
1068		-
1069		- if (unlikely(scsi_bidi_cmnd(cmd))) {
1070		- /*
1071		- * Bidi commands Must be complete as a whole,
1072		- * both sides at once.
1073		- */
1074		- scsi_req(req->next_rq)->resid_len = scsi_in(cmd)->resid;
1075		- if (scsi_end_request(req, BLK_STS_OK, blk_rq_bytes(req),
1076		- blk_rq_bytes(req->next_rq)))
1077		- WARN_ONCE(true,
1078		- "Bidi command with remaining bytes");
1079		- return;
1080		- }
1081		- }
1082		-
1083		- /* no bidi support yet, other than in pass-through */
1084		- if (unlikely(blk_bidi_rq(req))) {
1085		- WARN_ONCE(true, "Only support bidi command in passthrough");
1086		- scmd_printk(KERN_ERR, cmd, "Killing bidi command\n");
1087		- if (scsi_end_request(req, BLK_STS_IOERR, blk_rq_bytes(req),
1088		- blk_rq_bytes(req->next_rq)))
1089		- WARN_ONCE(true, "Bidi command with remaining bytes");
1090		- return;
1091	952	}
1092	953
1093	954	/*
..	..	@@ -1099,18 +960,17 @@
1099	960	blk_rq_sectors(req), good_bytes));
1100	961
1101	962	/*
1102		- * Next deal with any sectors which we were able to correctly
1103		- * handle. Failed, zero length commands always need to drop down
	963	+ * Failed, zero length commands always need to drop down
1104	964	* to retry code. Fast path should return in this block.
1105	965	*/
1106	966	if (likely(blk_rq_bytes(req) > 0 \|\| blk_stat == BLK_STS_OK)) {
1107		- if (likely(!scsi_end_request(req, blk_stat, good_bytes, 0)))
	967	+ if (likely(!scsi_end_request(req, blk_stat, good_bytes)))
1108	968	return; /* no bytes remaining */
1109	969	}
1110	970
1111	971	/* Kill remainder if no retries. */
1112	972	if (unlikely(blk_stat && scsi_noretry_cmd(cmd))) {
1113		- if (scsi_end_request(req, blk_stat, blk_rq_bytes(req), 0))
	973	+ if (scsi_end_request(req, blk_stat, blk_rq_bytes(req)))
1114	974	WARN_ONCE(true,
1115	975	"Bytes remaining after failed, no-retry command");
1116	976	return;
..	..	@@ -1126,115 +986,116 @@
1126	986	scsi_io_completion_action(cmd, result);
1127	987	}
1128	988
1129		-static int scsi_init_sgtable(struct request req, struct scsi_data_buffer sdb)
	989	+static inline bool scsi_cmd_needs_dma_drain(struct scsi_device *sdev,
	990	+ struct request *rq)
1130	991	{
	992	+ return sdev->dma_drain_len && blk_rq_is_passthrough(rq) &&
	993	+ !op_is_write(req_op(rq)) &&
	994	+ sdev->host->hostt->dma_need_drain(rq);
	995	+}
	996	+
	997	+/**
	998	+ * scsi_alloc_sgtables - allocate S/G tables for a command
	999	+ * @cmd: command descriptor we wish to initialize
	1000	+ *
	1001	+ * Returns:
	1002	+ * * BLK_STS_OK - on success
	1003	+ * * BLK_STS_RESOURCE - if the failure is retryable
	1004	+ * * BLK_STS_IOERR - if the failure is fatal
	1005	+ */
	1006	+blk_status_t scsi_alloc_sgtables(struct scsi_cmnd *cmd)
	1007	+{
	1008	+ struct scsi_device *sdev = cmd->device;
	1009	+ struct request *rq = cmd->request;
	1010	+ unsigned short nr_segs = blk_rq_nr_phys_segments(rq);
	1011	+ struct scatterlist *last_sg = NULL;
	1012	+ blk_status_t ret;
	1013	+ bool need_drain = scsi_cmd_needs_dma_drain(sdev, rq);
1131	1014	int count;
	1015	+
	1016	+ if (WARN_ON_ONCE(!nr_segs))
	1017	+ return BLK_STS_IOERR;
	1018	+
	1019	+ /*
	1020	+ * Make sure there is space for the drain. The driver must adjust
	1021	+ * max_hw_segments to be prepared for this.
	1022	+ */
	1023	+ if (need_drain)
	1024	+ nr_segs++;
1132	1025
1133	1026	/*
1134	1027	* If sg table allocation fails, requeue request later.
1135	1028	*/
1136		- if (unlikely(sg_alloc_table_chained(&sdb->table,
1137		- blk_rq_nr_phys_segments(req), sdb->table.sgl)))
1138		- return BLKPREP_DEFER;
	1029	+ if (unlikely(sg_alloc_table_chained(&cmd->sdb.table, nr_segs,
	1030	+ cmd->sdb.table.sgl, SCSI_INLINE_SG_CNT)))
	1031	+ return BLK_STS_RESOURCE;
1139	1032
1140		- /*
	1033	+ /*
1141	1034	* Next, walk the list, and fill in the addresses and sizes of
1142	1035	* each segment.
1143	1036	*/
1144		- count = blk_rq_map_sg(req->q, req, sdb->table.sgl);
1145		- BUG_ON(count > sdb->table.nents);
1146		- sdb->table.nents = count;
1147		- sdb->length = blk_rq_payload_bytes(req);
1148		- return BLKPREP_OK;
1149		-}
	1037	+ count = __blk_rq_map_sg(rq->q, rq, cmd->sdb.table.sgl, &last_sg);
1150	1038
1151		-/*
1152		- * Function: scsi_init_io()
1153		- *
1154		- * Purpose: SCSI I/O initialize function.
1155		- *
1156		- * Arguments: cmd - Command descriptor we wish to initialize
1157		- *
1158		- * Returns: 0 on success
1159		- * BLKPREP_DEFER if the failure is retryable
1160		- * BLKPREP_KILL if the failure is fatal
1161		- */
1162		-int scsi_init_io(struct scsi_cmnd *cmd)
1163		-{
1164		- struct scsi_device *sdev = cmd->device;
1165		- struct request *rq = cmd->request;
1166		- bool is_mq = (rq->mq_ctx != NULL);
1167		- int error = BLKPREP_KILL;
	1039	+ if (blk_rq_bytes(rq) & rq->q->dma_pad_mask) {
	1040	+ unsigned int pad_len =
	1041	+ (rq->q->dma_pad_mask & ~blk_rq_bytes(rq)) + 1;
1168	1042
1169		- if (WARN_ON_ONCE(!blk_rq_nr_phys_segments(rq)))
1170		- goto err_exit;
1171		-
1172		- error = scsi_init_sgtable(rq, &cmd->sdb);
1173		- if (error)
1174		- goto err_exit;
1175		-
1176		- if (blk_bidi_rq(rq)) {
1177		- if (!rq->q->mq_ops) {
1178		- struct scsi_data_buffer *bidi_sdb =
1179		- kmem_cache_zalloc(scsi_sdb_cache, GFP_ATOMIC);
1180		- if (!bidi_sdb) {
1181		- error = BLKPREP_DEFER;
1182		- goto err_exit;
1183		- }
1184		-
1185		- rq->next_rq->special = bidi_sdb;
1186		- }
1187		-
1188		- error = scsi_init_sgtable(rq->next_rq, rq->next_rq->special);
1189		- if (error)
1190		- goto err_exit;
	1043	+ last_sg->length += pad_len;
	1044	+ cmd->extra_len += pad_len;
1191	1045	}
	1046	+
	1047	+ if (need_drain) {
	1048	+ sg_unmark_end(last_sg);
	1049	+ last_sg = sg_next(last_sg);
	1050	+ sg_set_buf(last_sg, sdev->dma_drain_buf, sdev->dma_drain_len);
	1051	+ sg_mark_end(last_sg);
	1052	+
	1053	+ cmd->extra_len += sdev->dma_drain_len;
	1054	+ count++;
	1055	+ }
	1056	+
	1057	+ BUG_ON(count > cmd->sdb.table.nents);
	1058	+ cmd->sdb.table.nents = count;
	1059	+ cmd->sdb.length = blk_rq_payload_bytes(rq);
1192	1060
1193	1061	if (blk_integrity_rq(rq)) {
1194	1062	struct scsi_data_buffer *prot_sdb = cmd->prot_sdb;
1195		- int ivecs, count;
	1063	+ int ivecs;
1196	1064
1197		- if (prot_sdb == NULL) {
	1065	+ if (WARN_ON_ONCE(!prot_sdb)) {
1198	1066	/*
1199	1067	* This can happen if someone (e.g. multipath)
1200	1068	* queues a command to a device on an adapter
1201	1069	* that does not support DIX.
1202	1070	*/
1203		- WARN_ON_ONCE(1);
1204		- error = BLKPREP_KILL;
1205		- goto err_exit;
	1071	+ ret = BLK_STS_IOERR;
	1072	+ goto out_free_sgtables;
1206	1073	}
1207	1074
1208	1075	ivecs = blk_rq_count_integrity_sg(rq->q, rq->bio);
1209	1076
1210	1077	if (sg_alloc_table_chained(&prot_sdb->table, ivecs,
1211		- prot_sdb->table.sgl)) {
1212		- error = BLKPREP_DEFER;
1213		- goto err_exit;
	1078	+ prot_sdb->table.sgl,
	1079	+ SCSI_INLINE_PROT_SG_CNT)) {
	1080	+ ret = BLK_STS_RESOURCE;
	1081	+ goto out_free_sgtables;
1214	1082	}
1215	1083
1216	1084	count = blk_rq_map_integrity_sg(rq->q, rq->bio,
1217	1085	prot_sdb->table.sgl);
1218		- BUG_ON(unlikely(count > ivecs));
1219		- BUG_ON(unlikely(count > queue_max_integrity_segments(rq->q)));
	1086	+ BUG_ON(count > ivecs);
	1087	+ BUG_ON(count > queue_max_integrity_segments(rq->q));
1220	1088
1221	1089	cmd->prot_sdb = prot_sdb;
1222	1090	cmd->prot_sdb->table.nents = count;
1223	1091	}
1224	1092
1225		- return BLKPREP_OK;
1226		-err_exit:
1227		- if (is_mq) {
1228		- scsi_mq_free_sgtables(cmd);
1229		- } else {
1230		- scsi_release_buffers(cmd);
1231		- cmd->request->special = NULL;
1232		- scsi_put_command(cmd);
1233		- put_device(&sdev->sdev_gendev);
1234		- }
1235		- return error;
	1093	+ return BLK_STS_OK;
	1094	+out_free_sgtables:
	1095	+ scsi_free_sgtables(cmd);
	1096	+ return ret;
1236	1097	}
1237		-EXPORT_SYMBOL(scsi_init_io);
	1098	+EXPORT_SYMBOL(scsi_alloc_sgtables);
1238	1099
1239	1100	/**
1240	1101	* scsi_initialize_rq - initialize struct scsi_cmnd partially
..	..	@@ -1269,36 +1130,7 @@
1269	1130	}
1270	1131	}
1271	1132
1272		-/* Add a command to the list used by the aacraid and dpt_i2o drivers */
1273		-void scsi_add_cmd_to_list(struct scsi_cmnd *cmd)
1274		-{
1275		- struct scsi_device *sdev = cmd->device;
1276		- struct Scsi_Host *shost = sdev->host;
1277		- unsigned long flags;
1278		-
1279		- if (shost->use_cmd_list) {
1280		- spin_lock_irqsave(&sdev->list_lock, flags);
1281		- list_add_tail(&cmd->list, &sdev->cmd_list);
1282		- spin_unlock_irqrestore(&sdev->list_lock, flags);
1283		- }
1284		-}
1285		-
1286		-/* Remove a command from the list used by the aacraid and dpt_i2o drivers */
1287		-void scsi_del_cmd_from_list(struct scsi_cmnd *cmd)
1288		-{
1289		- struct scsi_device *sdev = cmd->device;
1290		- struct Scsi_Host *shost = sdev->host;
1291		- unsigned long flags;
1292		-
1293		- if (shost->use_cmd_list) {
1294		- spin_lock_irqsave(&sdev->list_lock, flags);
1295		- BUG_ON(list_empty(&cmd->list));
1296		- list_del_init(&cmd->list);
1297		- spin_unlock_irqrestore(&sdev->list_lock, flags);
1298		- }
1299		-}
1300		-
1301		-/* Called after a request has been started. */
	1133	+/* Called before a request is prepared. See also scsi_mq_prep_fn(). */
1302	1134	void scsi_init_command(struct scsi_device dev, struct scsi_cmnd cmd)
1303	1135	{
1304	1136	void *buf = cmd->sense_buffer;
..	..	@@ -1306,7 +1138,8 @@
1306	1138	struct request *rq = blk_mq_rq_from_pdu(cmd);
1307	1139	unsigned int flags = cmd->flags & SCMD_PRESERVED_FLAGS;
1308	1140	unsigned long jiffies_at_alloc;
1309		- int retries;
	1141	+ int retries, to_clear;
	1142	+ bool in_flight;
1310	1143
1311	1144	if (!blk_rq_is_scsi(rq) && !(flags & SCMD_INITIALIZED)) {
1312	1145	flags \|= SCMD_INITIALIZED;
..	..	@@ -1315,9 +1148,16 @@
1315	1148
1316	1149	jiffies_at_alloc = cmd->jiffies_at_alloc;
1317	1150	retries = cmd->retries;
1318		- /* zero out the cmd, except for the embedded scsi_request */
1319		- memset((char *)cmd + sizeof(cmd->req), 0,
1320		- sizeof(*cmd) - sizeof(cmd->req) + dev->host->hostt->cmd_size);
	1151	+ in_flight = test_bit(SCMD_STATE_INFLIGHT, &cmd->state);
	1152	+ /*
	1153	+ * Zero out the cmd, except for the embedded scsi_request. Only clear
	1154	+ * the driver-private command data if the LLD does not supply a
	1155	+ * function to initialize that data.
	1156	+ */
	1157	+ to_clear = sizeof(*cmd) - sizeof(cmd->req);
	1158	+ if (!dev->host->hostt->init_cmd_priv)
	1159	+ to_clear += dev->host->hostt->cmd_size;
	1160	+ memset((char *)cmd + sizeof(cmd->req), 0, to_clear);
1321	1161
1322	1162	cmd->device = dev;
1323	1163	cmd->sense_buffer = buf;
..	..	@@ -1326,11 +1166,13 @@
1326	1166	INIT_DELAYED_WORK(&cmd->abort_work, scmd_eh_abort_handler);
1327	1167	cmd->jiffies_at_alloc = jiffies_at_alloc;
1328	1168	cmd->retries = retries;
	1169	+ if (in_flight)
	1170	+ __set_bit(SCMD_STATE_INFLIGHT, &cmd->state);
1329	1171
1330		- scsi_add_cmd_to_list(cmd);
1331	1172	}
1332	1173
1333		-static int scsi_setup_scsi_cmnd(struct scsi_device sdev, struct request req)
	1174	+static blk_status_t scsi_setup_scsi_cmnd(struct scsi_device *sdev,
	1175	+ struct request *req)
1334	1176	{
1335	1177	struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req);
1336	1178
..	..	@@ -1341,8 +1183,8 @@
1341	1183	* submit a request without an attached bio.
1342	1184	*/
1343	1185	if (req->bio) {
1344		- int ret = scsi_init_io(cmd);
1345		- if (unlikely(ret))
	1186	+ blk_status_t ret = scsi_alloc_sgtables(cmd);
	1187	+ if (unlikely(ret != BLK_STS_OK))
1346	1188	return ret;
1347	1189	} else {
1348	1190	BUG_ON(blk_rq_bytes(req));
..	..	@@ -1354,168 +1196,56 @@
1354	1196	cmd->cmnd = scsi_req(req)->cmd;
1355	1197	cmd->transfersize = blk_rq_bytes(req);
1356	1198	cmd->allowed = scsi_req(req)->retries;
1357		- return BLKPREP_OK;
	1199	+ return BLK_STS_OK;
1358	1200	}
1359	1201
1360		-/*
1361		- * Setup a normal block command. These are simple request from filesystems
1362		- * that still need to be translated to SCSI CDBs from the ULD.
1363		- */
1364		-static int scsi_setup_fs_cmnd(struct scsi_device sdev, struct request req)
	1202	+static blk_status_t
	1203	+scsi_device_state_check(struct scsi_device sdev, struct request req)
1365	1204	{
1366		- struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req);
1367		-
1368		- if (unlikely(sdev->handler && sdev->handler->prep_fn)) {
1369		- int ret = sdev->handler->prep_fn(sdev, req);
1370		- if (ret != BLKPREP_OK)
1371		- return ret;
1372		- }
1373		-
1374		- cmd->cmnd = scsi_req(req)->cmd = scsi_req(req)->__cmd;
1375		- memset(cmd->cmnd, 0, BLK_MAX_CDB);
1376		- return scsi_cmd_to_driver(cmd)->init_command(cmd);
1377		-}
1378		-
1379		-static int scsi_setup_cmnd(struct scsi_device sdev, struct request req)
1380		-{
1381		- struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req);
1382		-
1383		- if (!blk_rq_bytes(req))
1384		- cmd->sc_data_direction = DMA_NONE;
1385		- else if (rq_data_dir(req) == WRITE)
1386		- cmd->sc_data_direction = DMA_TO_DEVICE;
1387		- else
1388		- cmd->sc_data_direction = DMA_FROM_DEVICE;
1389		-
1390		- if (blk_rq_is_scsi(req))
1391		- return scsi_setup_scsi_cmnd(sdev, req);
1392		- else
1393		- return scsi_setup_fs_cmnd(sdev, req);
1394		-}
1395		-
1396		-static int
1397		-scsi_prep_state_check(struct scsi_device sdev, struct request req)
1398		-{
1399		- int ret = BLKPREP_OK;
1400		-
1401		- /*
1402		- * If the device is not in running state we will reject some
1403		- * or all commands.
1404		- */
1405		- if (unlikely(sdev->sdev_state != SDEV_RUNNING)) {
1406		- switch (sdev->sdev_state) {
1407		- case SDEV_OFFLINE:
1408		- case SDEV_TRANSPORT_OFFLINE:
1409		- /*
1410		- * If the device is offline we refuse to process any
1411		- * commands. The device must be brought online
1412		- * before trying any recovery commands.
1413		- */
	1205	+ switch (sdev->sdev_state) {
	1206	+ case SDEV_CREATED:
	1207	+ return BLK_STS_OK;
	1208	+ case SDEV_OFFLINE:
	1209	+ case SDEV_TRANSPORT_OFFLINE:
	1210	+ /*
	1211	+ * If the device is offline we refuse to process any
	1212	+ * commands. The device must be brought online
	1213	+ * before trying any recovery commands.
	1214	+ */
	1215	+ if (!sdev->offline_already) {
	1216	+ sdev->offline_already = true;
1414	1217	sdev_printk(KERN_ERR, sdev,
1415	1218	"rejecting I/O to offline device\n");
1416		- ret = BLKPREP_KILL;
1417		- break;
1418		- case SDEV_DEL:
1419		- /*
1420		- * If the device is fully deleted, we refuse to
1421		- * process any commands as well.
1422		- */
1423		- sdev_printk(KERN_ERR, sdev,
1424		- "rejecting I/O to dead device\n");
1425		- ret = BLKPREP_KILL;
1426		- break;
1427		- case SDEV_BLOCK:
1428		- case SDEV_CREATED_BLOCK:
1429		- ret = BLKPREP_DEFER;
1430		- break;
1431		- case SDEV_QUIESCE:
1432		- /*
1433		- * If the devices is blocked we defer normal commands.
1434		- */
1435		- if (req && !(req->rq_flags & RQF_PREEMPT))
1436		- ret = BLKPREP_DEFER;
1437		- break;
1438		- default:
1439		- /*
1440		- * For any other not fully online state we only allow
1441		- * special commands. In particular any user initiated
1442		- * command is not allowed.
1443		- */
1444		- if (req && !(req->rq_flags & RQF_PREEMPT))
1445		- ret = BLKPREP_KILL;
1446		- break;
1447	1219	}
1448		- }
1449		- return ret;
1450		-}
1451		-
1452		-static int
1453		-scsi_prep_return(struct request_queue q, struct request req, int ret)
1454		-{
1455		- struct scsi_device *sdev = q->queuedata;
1456		-
1457		- switch (ret) {
1458		- case BLKPREP_KILL:
1459		- case BLKPREP_INVALID:
1460		- scsi_req(req)->result = DID_NO_CONNECT << 16;
1461		- /* release the command and kill it */
1462		- if (req->special) {
1463		- struct scsi_cmnd *cmd = req->special;
1464		- scsi_release_buffers(cmd);
1465		- scsi_put_command(cmd);
1466		- put_device(&sdev->sdev_gendev);
1467		- req->special = NULL;
1468		- }
1469		- break;
1470		- case BLKPREP_DEFER:
	1220	+ return BLK_STS_IOERR;
	1221	+ case SDEV_DEL:
1471	1222	/*
1472		- * If we defer, the blk_peek_request() returns NULL, but the
1473		- * queue must be restarted, so we schedule a callback to happen
1474		- * shortly.
	1223	+ * If the device is fully deleted, we refuse to
	1224	+ * process any commands as well.
1475	1225	*/
1476		- if (atomic_read(&sdev->device_busy) == 0)
1477		- blk_delay_queue(q, SCSI_QUEUE_DELAY);
1478		- break;
	1226	+ sdev_printk(KERN_ERR, sdev,
	1227	+ "rejecting I/O to dead device\n");
	1228	+ return BLK_STS_IOERR;
	1229	+ case SDEV_BLOCK:
	1230	+ case SDEV_CREATED_BLOCK:
	1231	+ return BLK_STS_RESOURCE;
	1232	+ case SDEV_QUIESCE:
	1233	+ /*
	1234	+ * If the device is blocked we only accept power management
	1235	+ * commands.
	1236	+ */
	1237	+ if (req && WARN_ON_ONCE(!(req->rq_flags & RQF_PM)))
	1238	+ return BLK_STS_RESOURCE;
	1239	+ return BLK_STS_OK;
1479	1240	default:
1480		- req->rq_flags \|= RQF_DONTPREP;
	1241	+ /*
	1242	+ * For any other not fully online state we only allow
	1243	+ * power management commands.
	1244	+ */
	1245	+ if (req && !(req->rq_flags & RQF_PM))
	1246	+ return BLK_STS_IOERR;
	1247	+ return BLK_STS_OK;
1481	1248	}
1482		-
1483		- return ret;
1484		-}
1485		-
1486		-static int scsi_prep_fn(struct request_queue q, struct request req)
1487		-{
1488		- struct scsi_device *sdev = q->queuedata;
1489		- struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req);
1490		- int ret;
1491		-
1492		- ret = scsi_prep_state_check(sdev, req);
1493		- if (ret != BLKPREP_OK)
1494		- goto out;
1495		-
1496		- if (!req->special) {
1497		- /* Bail if we can't get a reference to the device */
1498		- if (unlikely(!get_device(&sdev->sdev_gendev))) {
1499		- ret = BLKPREP_DEFER;
1500		- goto out;
1501		- }
1502		-
1503		- scsi_init_command(sdev, cmd);
1504		- req->special = cmd;
1505		- }
1506		-
1507		- cmd->tag = req->tag;
1508		- cmd->request = req;
1509		- cmd->prot_op = SCSI_PROT_NORMAL;
1510		-
1511		- ret = scsi_setup_cmnd(sdev, req);
1512		-out:
1513		- return scsi_prep_return(q, req, ret);
1514		-}
1515		-
1516		-static void scsi_unprep_fn(struct request_queue q, struct request req)
1517		-{
1518		- scsi_uninit_cmd(blk_mq_rq_to_pdu(req));
1519	1249	}
1520	1250
1521	1251	/*
..	..	@@ -1537,14 +1267,8 @@
1537	1267	/*
1538	1268	* unblock after device_blocked iterates to zero
1539	1269	*/
1540		- if (atomic_dec_return(&sdev->device_blocked) > 0) {
1541		- /*
1542		- * For the MQ case we take care of this in the caller.
1543		- */
1544		- if (!q->mq_ops)
1545		- blk_delay_queue(q, SCSI_QUEUE_DELAY);
	1270	+ if (atomic_dec_return(&sdev->device_blocked) > 0)
1546	1271	goto out_dec;
1547		- }
1548	1272	SCSI_LOG_MLQUEUE(3, sdev_printk(KERN_INFO, sdev,
1549	1273	"unblocking device at zero depth\n"));
1550	1274	}
..	..	@@ -1619,16 +1343,14 @@
1619	1343	*/
1620	1344	static inline int scsi_host_queue_ready(struct request_queue *q,
1621	1345	struct Scsi_Host *shost,
1622		- struct scsi_device *sdev)
	1346	+ struct scsi_device *sdev,
	1347	+ struct scsi_cmnd *cmd)
1623	1348	{
1624		- unsigned int busy;
1625		-
1626	1349	if (scsi_host_in_recovery(shost))
1627	1350	return 0;
1628	1351
1629		- busy = atomic_inc_return(&shost->host_busy) - 1;
1630	1352	if (atomic_read(&shost->host_blocked) > 0) {
1631		- if (busy)
	1353	+ if (scsi_host_busy(shost) > 0)
1632	1354	goto starved;
1633	1355
1634	1356	/*
..	..	@@ -1642,8 +1364,6 @@
1642	1364	"unblocking host at zero depth\n"));
1643	1365	}
1644	1366
1645		- if (shost->can_queue > 0 && busy >= shost->can_queue)
1646		- goto starved;
1647	1367	if (shost->host_self_blocked)
1648	1368	goto starved;
1649	1369
..	..	@@ -1655,6 +1375,8 @@
1655	1375	spin_unlock_irq(shost->host_lock);
1656	1376	}
1657	1377
	1378	+ __set_bit(SCMD_STATE_INFLIGHT, &cmd->state);
	1379	+
1658	1380	return 1;
1659	1381
1660	1382	starved:
..	..	@@ -1663,7 +1385,7 @@
1663	1385	list_add_tail(&sdev->starved_entry, &shost->starved_list);
1664	1386	spin_unlock_irq(shost->host_lock);
1665	1387	out_dec:
1666		- scsi_dec_host_busy(shost);
	1388	+ scsi_dec_host_busy(shost, cmd);
1667	1389	return 0;
1668	1390	}
1669	1391
..	..	@@ -1679,13 +1401,13 @@
1679	1401	* needs to return 'not busy'. Otherwise, request stacking drivers
1680	1402	* may hold requests forever.
1681	1403	*/
1682		-static int scsi_lld_busy(struct request_queue *q)
	1404	+static bool scsi_mq_lld_busy(struct request_queue *q)
1683	1405	{
1684	1406	struct scsi_device *sdev = q->queuedata;
1685	1407	struct Scsi_Host *shost;
1686	1408
1687	1409	if (blk_queue_dying(q))
1688		- return 0;
	1410	+ return false;
1689	1411
1690	1412	shost = sdev->host;
1691	1413
..	..	@@ -1696,49 +1418,14 @@
1696	1418	* in SCSI layer.
1697	1419	*/
1698	1420	if (scsi_host_in_recovery(shost) \|\| scsi_device_is_busy(sdev))
1699		- return 1;
	1421	+ return true;
1700	1422
1701		- return 0;
1702		-}
1703		-
1704		-/*
1705		- * Kill a request for a dead device
1706		- */
1707		-static void scsi_kill_request(struct request req, struct request_queue q)
1708		-{
1709		- struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req);
1710		- struct scsi_device *sdev;
1711		- struct scsi_target *starget;
1712		- struct Scsi_Host *shost;
1713		-
1714		- blk_start_request(req);
1715		-
1716		- scmd_printk(KERN_INFO, cmd, "killing request\n");
1717		-
1718		- sdev = cmd->device;
1719		- starget = scsi_target(sdev);
1720		- shost = sdev->host;
1721		- scsi_init_cmd_errh(cmd);
1722		- cmd->result = DID_NO_CONNECT << 16;
1723		- atomic_inc(&cmd->device->iorequest_cnt);
1724		-
1725		- /*
1726		- * SCSI request completion path will do scsi_device_unbusy(),
1727		- * bump busy counts. To bump the counters, we need to dance
1728		- * with the locks as normal issue path does.
1729		- */
1730		- atomic_inc(&sdev->device_busy);
1731		- atomic_inc(&shost->host_busy);
1732		- if (starget->can_queue > 0)
1733		- atomic_inc(&starget->target_busy);
1734		-
1735		- blk_complete_request(req);
	1423	+ return false;
1736	1424	}
1737	1425
1738	1426	static void scsi_softirq_done(struct request *rq)
1739	1427	{
1740	1428	struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq);
1741		- unsigned long wait_for = (cmd->allowed + 1) * rq->timeout;
1742	1429	int disposition;
1743	1430
1744	1431	INIT_LIST_HEAD(&cmd->eh_entry);
..	..	@@ -1748,29 +1435,24 @@
1748	1435	atomic_inc(&cmd->device->ioerr_cnt);
1749	1436
1750	1437	disposition = scsi_decide_disposition(cmd);
1751		- if (disposition != SUCCESS &&
1752		- time_before(cmd->jiffies_at_alloc + wait_for, jiffies)) {
1753		- sdev_printk(KERN_ERR, cmd->device,
1754		- "timing out command, waited %lus\n",
1755		- wait_for/HZ);
	1438	+ if (disposition != SUCCESS && scsi_cmd_runtime_exceeced(cmd))
1756	1439	disposition = SUCCESS;
1757		- }
1758	1440
1759	1441	scsi_log_completion(cmd, disposition);
1760	1442
1761	1443	switch (disposition) {
1762		- case SUCCESS:
1763		- scsi_finish_command(cmd);
1764		- break;
1765		- case NEEDS_RETRY:
1766		- scsi_queue_insert(cmd, SCSI_MLQUEUE_EH_RETRY);
1767		- break;
1768		- case ADD_TO_MLQUEUE:
1769		- scsi_queue_insert(cmd, SCSI_MLQUEUE_DEVICE_BUSY);
1770		- break;
1771		- default:
1772		- scsi_eh_scmd_add(cmd);
1773		- break;
	1444	+ case SUCCESS:
	1445	+ scsi_finish_command(cmd);
	1446	+ break;
	1447	+ case NEEDS_RETRY:
	1448	+ scsi_queue_insert(cmd, SCSI_MLQUEUE_EH_RETRY);
	1449	+ break;
	1450	+ case ADD_TO_MLQUEUE:
	1451	+ scsi_queue_insert(cmd, SCSI_MLQUEUE_DEVICE_BUSY);
	1452	+ break;
	1453	+ default:
	1454	+ scsi_eh_scmd_add(cmd);
	1455	+ break;
1774	1456	}
1775	1457	}
1776	1458
..	..	@@ -1808,6 +1490,7 @@
1808	1490	*/
1809	1491	SCSI_LOG_MLQUEUE(3, scmd_printk(KERN_INFO, cmd,
1810	1492	"queuecommand : device blocked\n"));
	1493	+ atomic_dec(&cmd->device->iorequest_cnt);
1811	1494	return SCSI_MLQUEUE_DEVICE_BUSY;
1812	1495	}
1813	1496
..	..	@@ -1840,6 +1523,7 @@
1840	1523	trace_scsi_dispatch_cmd_start(cmd);
1841	1524	rtn = host->hostt->queuecommand(host, cmd);
1842	1525	if (rtn) {
	1526	+ atomic_dec(&cmd->device->iorequest_cnt);
1843	1527	trace_scsi_dispatch_cmd_error(cmd, rtn);
1844	1528	if (rtn != SCSI_MLQUEUE_DEVICE_BUSY &&
1845	1529	rtn != SCSI_MLQUEUE_TARGET_BUSY)
..	..	@@ -1855,178 +1539,14 @@
1855	1539	return 0;
1856	1540	}
1857	1541
1858		-/**
1859		- * scsi_done - Invoke completion on finished SCSI command.
1860		- * @cmd: The SCSI Command for which a low-level device driver (LLDD) gives
1861		- * ownership back to SCSI Core -- i.e. the LLDD has finished with it.
1862		- *
1863		- * Description: This function is the mid-level's (SCSI Core) interrupt routine,
1864		- * which regains ownership of the SCSI command (de facto) from a LLDD, and
1865		- * calls blk_complete_request() for further processing.
1866		- *
1867		- * This function is interrupt context safe.
1868		- */
1869		-static void scsi_done(struct scsi_cmnd *cmd)
1870		-{
1871		- trace_scsi_dispatch_cmd_done(cmd);
1872		- blk_complete_request(cmd->request);
1873		-}
1874		-
1875		-/*
1876		- * Function: scsi_request_fn()
1877		- *
1878		- * Purpose: Main strategy routine for SCSI.
1879		- *
1880		- * Arguments: q - Pointer to actual queue.
1881		- *
1882		- * Returns: Nothing
1883		- *
1884		- * Lock status: request queue lock assumed to be held when called.
1885		- *
1886		- * Note: See sd_zbc.c sd_zbc_write_lock_zone() for write order
1887		- * protection for ZBC disks.
1888		- */
1889		-static void scsi_request_fn(struct request_queue *q)
1890		- __releases(q->queue_lock)
1891		- __acquires(q->queue_lock)
1892		-{
1893		- struct scsi_device *sdev = q->queuedata;
1894		- struct Scsi_Host *shost;
1895		- struct scsi_cmnd *cmd;
1896		- struct request *req;
1897		-
1898		- /*
1899		- * To start with, we keep looping until the queue is empty, or until
1900		- * the host is no longer able to accept any more requests.
1901		- */
1902		- shost = sdev->host;
1903		- for (;;) {
1904		- int rtn;
1905		- /*
1906		- * get next queueable request. We do this early to make sure
1907		- * that the request is fully prepared even if we cannot
1908		- * accept it.
1909		- */
1910		- req = blk_peek_request(q);
1911		- if (!req)
1912		- break;
1913		-
1914		- if (unlikely(!scsi_device_online(sdev))) {
1915		- sdev_printk(KERN_ERR, sdev,
1916		- "rejecting I/O to offline device\n");
1917		- scsi_kill_request(req, q);
1918		- continue;
1919		- }
1920		-
1921		- if (!scsi_dev_queue_ready(q, sdev))
1922		- break;
1923		-
1924		- /*
1925		- * Remove the request from the request list.
1926		- */
1927		- if (!(blk_queue_tagged(q) && !blk_queue_start_tag(q, req)))
1928		- blk_start_request(req);
1929		-
1930		- spin_unlock_irq(q->queue_lock);
1931		- cmd = blk_mq_rq_to_pdu(req);
1932		- if (cmd != req->special) {
1933		- printk(KERN_CRIT "impossible request in %s.\n"
1934		- "please mail a stack trace to "
1935		- "linux-scsi@vger.kernel.org\n",
1936		- __func__);
1937		- blk_dump_rq_flags(req, "foo");
1938		- BUG();
1939		- }
1940		-
1941		- /*
1942		- * We hit this when the driver is using a host wide
1943		- * tag map. For device level tag maps the queue_depth check
1944		- * in the device ready fn would prevent us from trying
1945		- * to allocate a tag. Since the map is a shared host resource
1946		- * we add the dev to the starved list so it eventually gets
1947		- * a run when a tag is freed.
1948		- */
1949		- if (blk_queue_tagged(q) && !(req->rq_flags & RQF_QUEUED)) {
1950		- spin_lock_irq(shost->host_lock);
1951		- if (list_empty(&sdev->starved_entry))
1952		- list_add_tail(&sdev->starved_entry,
1953		- &shost->starved_list);
1954		- spin_unlock_irq(shost->host_lock);
1955		- goto not_ready;
1956		- }
1957		-
1958		- if (!scsi_target_queue_ready(shost, sdev))
1959		- goto not_ready;
1960		-
1961		- if (!scsi_host_queue_ready(q, shost, sdev))
1962		- goto host_not_ready;
1963		-
1964		- if (sdev->simple_tags)
1965		- cmd->flags \|= SCMD_TAGGED;
1966		- else
1967		- cmd->flags &= ~SCMD_TAGGED;
1968		-
1969		- /*
1970		- * Finally, initialize any error handling parameters, and set up
1971		- * the timers for timeouts.
1972		- */
1973		- scsi_init_cmd_errh(cmd);
1974		-
1975		- /*
1976		- * Dispatch the command to the low-level driver.
1977		- */
1978		- cmd->scsi_done = scsi_done;
1979		- rtn = scsi_dispatch_cmd(cmd);
1980		- if (rtn) {
1981		- scsi_queue_insert(cmd, rtn);
1982		- spin_lock_irq(q->queue_lock);
1983		- goto out_delay;
1984		- }
1985		- spin_lock_irq(q->queue_lock);
1986		- }
1987		-
1988		- return;
1989		-
1990		- host_not_ready:
1991		- if (scsi_target(sdev)->can_queue > 0)
1992		- atomic_dec(&scsi_target(sdev)->target_busy);
1993		- not_ready:
1994		- /*
1995		- * lock q, handle tag, requeue req, and decrement device_busy. We
1996		- * must return with queue_lock held.
1997		- *
1998		- * Decrementing device_busy without checking it is OK, as all such
1999		- * cases (host limits or settings) should run the queue at some
2000		- * later time.
2001		- */
2002		- spin_lock_irq(q->queue_lock);
2003		- blk_requeue_request(q, req);
2004		- atomic_dec(&sdev->device_busy);
2005		-out_delay:
2006		- if (!atomic_read(&sdev->device_busy) && !scsi_device_blocked(sdev))
2007		- blk_delay_queue(q, SCSI_QUEUE_DELAY);
2008		-}
2009		-
2010		-static inline blk_status_t prep_to_mq(int ret)
2011		-{
2012		- switch (ret) {
2013		- case BLKPREP_OK:
2014		- return BLK_STS_OK;
2015		- case BLKPREP_DEFER:
2016		- return BLK_STS_RESOURCE;
2017		- default:
2018		- return BLK_STS_IOERR;
2019		- }
2020		-}
2021		-
2022	1542	/* Size in bytes of the sg-list stored in the scsi-mq command-private data. */
2023		-static unsigned int scsi_mq_sgl_size(struct Scsi_Host *shost)
	1543	+static unsigned int scsi_mq_inline_sgl_size(struct Scsi_Host *shost)
2024	1544	{
2025		- return min_t(unsigned int, shost->sg_tablesize, SG_CHUNK_SIZE) *
	1545	+ return min_t(unsigned int, shost->sg_tablesize, SCSI_INLINE_SG_CNT) *
2026	1546	sizeof(struct scatterlist);
2027	1547	}
2028	1548
2029		-static int scsi_mq_prep_fn(struct request *req)
	1549	+static blk_status_t scsi_prepare_cmd(struct request *req)
2030	1550	{
2031	1551	struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(req);
2032	1552	struct scsi_device *sdev = req->q->queuedata;
..	..	@@ -2035,12 +1555,13 @@
2035	1555
2036	1556	scsi_init_command(sdev, cmd);
2037	1557
2038		- req->special = cmd;
2039		-
2040	1558	cmd->request = req;
2041		-
2042	1559	cmd->tag = req->tag;
2043	1560	cmd->prot_op = SCSI_PROT_NORMAL;
	1561	+ if (blk_rq_bytes(req))
	1562	+ cmd->sc_data_direction = rq_dma_dir(req);
	1563	+ else
	1564	+ cmd->sc_data_direction = DMA_NONE;
2044	1565
2045	1566	sg = (void *)cmd + sizeof(struct scsi_cmnd) + shost->hostt->cmd_size;
2046	1567	cmd->sdb.table.sgl = sg;
..	..	@@ -2052,54 +1573,69 @@
2052	1573	(struct scatterlist *)(cmd->prot_sdb + 1);
2053	1574	}
2054	1575
2055		- if (blk_bidi_rq(req)) {
2056		- struct request *next_rq = req->next_rq;
2057		- struct scsi_data_buffer *bidi_sdb = blk_mq_rq_to_pdu(next_rq);
	1576	+ /*
	1577	+ * Special handling for passthrough commands, which don't go to the ULP
	1578	+ * at all:
	1579	+ */
	1580	+ if (blk_rq_is_scsi(req))
	1581	+ return scsi_setup_scsi_cmnd(sdev, req);
2058	1582
2059		- memset(bidi_sdb, 0, sizeof(struct scsi_data_buffer));
2060		- bidi_sdb->table.sgl =
2061		- (struct scatterlist *)(bidi_sdb + 1);
	1583	+ if (sdev->handler && sdev->handler->prep_fn) {
	1584	+ blk_status_t ret = sdev->handler->prep_fn(sdev, req);
2062	1585
2063		- next_rq->special = bidi_sdb;
	1586	+ if (ret != BLK_STS_OK)
	1587	+ return ret;
2064	1588	}
2065	1589
2066		- blk_mq_start_request(req);
2067		-
2068		- return scsi_setup_cmnd(sdev, req);
	1590	+ cmd->cmnd = scsi_req(req)->cmd = scsi_req(req)->__cmd;
	1591	+ memset(cmd->cmnd, 0, BLK_MAX_CDB);
	1592	+ return scsi_cmd_to_driver(cmd)->init_command(cmd);
2069	1593	}
2070	1594
2071	1595	static void scsi_mq_done(struct scsi_cmnd *cmd)
2072	1596	{
	1597	+ if (unlikely(blk_should_fake_timeout(cmd->request->q)))
	1598	+ return;
	1599	+ if (unlikely(test_and_set_bit(SCMD_STATE_COMPLETE, &cmd->state)))
	1600	+ return;
2073	1601	trace_scsi_dispatch_cmd_done(cmd);
2074	1602	blk_mq_complete_request(cmd->request);
2075	1603	}
2076	1604
2077		-static void scsi_mq_put_budget(struct blk_mq_hw_ctx *hctx)
	1605	+static void scsi_mq_put_budget(struct request_queue *q)
2078	1606	{
2079		- struct request_queue *q = hctx->queue;
2080	1607	struct scsi_device *sdev = q->queuedata;
2081	1608
2082	1609	atomic_dec(&sdev->device_busy);
2083		- put_device(&sdev->sdev_gendev);
2084	1610	}
2085	1611
2086		-static bool scsi_mq_get_budget(struct blk_mq_hw_ctx *hctx)
	1612	+static bool scsi_mq_get_budget(struct request_queue *q)
2087	1613	{
2088		- struct request_queue *q = hctx->queue;
2089	1614	struct scsi_device *sdev = q->queuedata;
2090	1615
2091		- if (!get_device(&sdev->sdev_gendev))
2092		- goto out;
2093		- if (!scsi_dev_queue_ready(q, sdev))
2094		- goto out_put_device;
	1616	+ if (scsi_dev_queue_ready(q, sdev))
	1617	+ return true;
2095	1618
2096		- return true;
	1619	+ atomic_inc(&sdev->restarts);
2097	1620
2098		-out_put_device:
2099		- put_device(&sdev->sdev_gendev);
2100		-out:
2101		- if (atomic_read(&sdev->device_busy) == 0 && !scsi_device_blocked(sdev))
2102		- blk_mq_delay_run_hw_queue(hctx, SCSI_QUEUE_DELAY);
	1621	+ /*
	1622	+ * Orders atomic_inc(&sdev->restarts) and atomic_read(&sdev->device_busy).
	1623	+ * .restarts must be incremented before .device_busy is read because the
	1624	+ * code in scsi_run_queue_async() depends on the order of these operations.
	1625	+ */
	1626	+ smp_mb__after_atomic();
	1627	+
	1628	+ /*
	1629	+ * If all in-flight requests originated from this LUN are completed
	1630	+ * before reading .device_busy, sdev->device_busy will be observed as
	1631	+ * zero, then blk_mq_delay_run_hw_queues() will dispatch this request
	1632	+ * soon. Otherwise, completion of one of these requests will observe
	1633	+ * the .restarts flag, and the request queue will be run for handling
	1634	+ * this request, see scsi_end_request().
	1635	+ */
	1636	+ if (unlikely(atomic_read(&sdev->device_busy) == 0 &&
	1637	+ !scsi_device_blocked(sdev)))
	1638	+ blk_mq_delay_run_hw_queues(sdev->request_queue, SCSI_QUEUE_DELAY);
2103	1639	return false;
2104	1640	}
2105	1641
..	..	@@ -2114,33 +1650,42 @@
2114	1650	blk_status_t ret;
2115	1651	int reason;
2116	1652
2117		- ret = prep_to_mq(scsi_prep_state_check(sdev, req));
2118		- if (ret != BLK_STS_OK)
2119		- goto out_put_budget;
	1653	+ /*
	1654	+ * If the device is not in running state we will reject some or all
	1655	+ * commands.
	1656	+ */
	1657	+ if (unlikely(sdev->sdev_state != SDEV_RUNNING)) {
	1658	+ ret = scsi_device_state_check(sdev, req);
	1659	+ if (ret != BLK_STS_OK)
	1660	+ goto out_put_budget;
	1661	+ }
2120	1662
2121	1663	ret = BLK_STS_RESOURCE;
2122	1664	if (!scsi_target_queue_ready(shost, sdev))
2123	1665	goto out_put_budget;
2124		- if (!scsi_host_queue_ready(q, shost, sdev))
	1666	+ if (!scsi_host_queue_ready(q, shost, sdev, cmd))
2125	1667	goto out_dec_target_busy;
2126	1668
2127	1669	if (!(req->rq_flags & RQF_DONTPREP)) {
2128		- ret = prep_to_mq(scsi_mq_prep_fn(req));
	1670	+ ret = scsi_prepare_cmd(req);
2129	1671	if (ret != BLK_STS_OK)
2130	1672	goto out_dec_host_busy;
2131	1673	req->rq_flags \|= RQF_DONTPREP;
2132	1674	} else {
2133		- blk_mq_start_request(req);
	1675	+ clear_bit(SCMD_STATE_COMPLETE, &cmd->state);
2134	1676	}
2135	1677
	1678	+ cmd->flags &= SCMD_PRESERVED_FLAGS;
2136	1679	if (sdev->simple_tags)
2137	1680	cmd->flags \|= SCMD_TAGGED;
2138		- else
2139		- cmd->flags &= ~SCMD_TAGGED;
	1681	+ if (bd->last)
	1682	+ cmd->flags \|= SCMD_LAST;
2140	1683
2141		- scsi_init_cmd_errh(cmd);
	1684	+ scsi_set_resid(cmd, 0);
	1685	+ memset(cmd->sense_buffer, 0, SCSI_SENSE_BUFFERSIZE);
2142	1686	cmd->scsi_done = scsi_mq_done;
2143	1687
	1688	+ blk_mq_start_request(req);
2144	1689	reason = scsi_dispatch_cmd(cmd);
2145	1690	if (reason) {
2146	1691	scsi_set_blocked(cmd, reason);
..	..	@@ -2151,18 +1696,18 @@
2151	1696	return BLK_STS_OK;
2152	1697
2153	1698	out_dec_host_busy:
2154		- scsi_dec_host_busy(shost);
	1699	+ scsi_dec_host_busy(shost, cmd);
2155	1700	out_dec_target_busy:
2156	1701	if (scsi_target(sdev)->can_queue > 0)
2157	1702	atomic_dec(&scsi_target(sdev)->target_busy);
2158	1703	out_put_budget:
2159		- scsi_mq_put_budget(hctx);
	1704	+ scsi_mq_put_budget(q);
2160	1705	switch (ret) {
2161	1706	case BLK_STS_OK:
2162	1707	break;
2163	1708	case BLK_STS_RESOURCE:
2164		- if (atomic_read(&sdev->device_busy) \|\|
2165		- scsi_device_blocked(sdev))
	1709	+ case BLK_STS_ZONE_RESOURCE:
	1710	+ if (scsi_device_blocked(sdev))
2166	1711	ret = BLK_STS_DEV_RESOURCE;
2167	1712	break;
2168	1713	default:
..	..	@@ -2177,6 +1722,7 @@
2177	1722	*/
2178	1723	if (req->rq_flags & RQF_DONTPREP)
2179	1724	scsi_mq_uninit_cmd(cmd);
	1725	+ scsi_run_queue_async(sdev);
2180	1726	break;
2181	1727	}
2182	1728	return ret;
..	..	@@ -2197,6 +1743,7 @@
2197	1743	const bool unchecked_isa_dma = shost->unchecked_isa_dma;
2198	1744	struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq);
2199	1745	struct scatterlist *sg;
	1746	+ int ret = 0;
2200	1747
2201	1748	if (unchecked_isa_dma)
2202	1749	cmd->flags \|= SCMD_UNCHECKED_ISA_DMA;
..	..	@@ -2209,17 +1756,27 @@
2209	1756	if (scsi_host_get_prot(shost)) {
2210	1757	sg = (void *)cmd + sizeof(struct scsi_cmnd) +
2211	1758	shost->hostt->cmd_size;
2212		- cmd->prot_sdb = (void *)sg + scsi_mq_sgl_size(shost);
	1759	+ cmd->prot_sdb = (void *)sg + scsi_mq_inline_sgl_size(shost);
2213	1760	}
2214	1761
2215		- return 0;
	1762	+ if (shost->hostt->init_cmd_priv) {
	1763	+ ret = shost->hostt->init_cmd_priv(shost, cmd);
	1764	+ if (ret < 0)
	1765	+ scsi_free_sense_buffer(unchecked_isa_dma,
	1766	+ cmd->sense_buffer);
	1767	+ }
	1768	+
	1769	+ return ret;
2216	1770	}
2217	1771
2218	1772	static void scsi_mq_exit_request(struct blk_mq_tag_set set, struct request rq,
2219	1773	unsigned int hctx_idx)
2220	1774	{
	1775	+ struct Scsi_Host *shost = set->driver_data;
2221	1776	struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq);
2222	1777
	1778	+ if (shost->hostt->exit_cmd_priv)
	1779	+ shost->hostt->exit_cmd_priv(shost, cmd);
2223	1780	scsi_free_sense_buffer(cmd->flags & SCMD_UNCHECKED_ISA_DMA,
2224	1781	cmd->sense_buffer);
2225	1782	}
..	..	@@ -2230,7 +1787,7 @@
2230	1787
2231	1788	if (shost->hostt->map_queues)
2232	1789	return shost->hostt->map_queues(shost);
2233		- return blk_mq_map_queues(set);
	1790	+ return blk_mq_map_queues(&set->map[HCTX_TYPE_DEFAULT]);
2234	1791	}
2235	1792
2236	1793	void __scsi_init_queue(struct Scsi_Host shost, struct request_queue q)
..	..	@@ -2251,16 +1808,19 @@
2251	1808	blk_queue_max_integrity_segments(q, shost->sg_prot_tablesize);
2252	1809	}
2253	1810
	1811	+ if (dev->dma_mask) {
	1812	+ shost->max_sectors = min_t(unsigned int, shost->max_sectors,
	1813	+ dma_max_mapping_size(dev) >> SECTOR_SHIFT);
	1814	+ }
2254	1815	blk_queue_max_hw_sectors(q, shost->max_sectors);
2255	1816	if (shost->unchecked_isa_dma)
2256	1817	blk_queue_bounce_limit(q, BLK_BOUNCE_ISA);
2257	1818	blk_queue_segment_boundary(q, shost->dma_boundary);
2258	1819	dma_set_seg_boundary(dev, shost->dma_boundary);
2259	1820
2260		- blk_queue_max_segment_size(q, dma_get_max_seg_size(dev));
2261		-
2262		- if (!shost->use_clustering)
2263		- q->limits.cluster = 0;
	1821	+ blk_queue_max_segment_size(q, shost->max_segment_size);
	1822	+ blk_queue_virt_boundary(q, shost->virt_boundary_mask);
	1823	+ dma_set_max_seg_size(dev, queue_max_segment_size(q));
2264	1824
2265	1825	/*
2266	1826	* Set a reasonable default alignment: The larger of 32-byte (dword),
..	..	@@ -2273,78 +1833,7 @@
2273	1833	}
2274	1834	EXPORT_SYMBOL_GPL(__scsi_init_queue);
2275	1835
2276		-static int scsi_old_init_rq(struct request_queue q, struct request rq,
2277		- gfp_t gfp)
2278		-{
2279		- struct Scsi_Host *shost = q->rq_alloc_data;
2280		- const bool unchecked_isa_dma = shost->unchecked_isa_dma;
2281		- struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq);
2282		-
2283		- memset(cmd, 0, sizeof(*cmd));
2284		-
2285		- if (unchecked_isa_dma)
2286		- cmd->flags \|= SCMD_UNCHECKED_ISA_DMA;
2287		- cmd->sense_buffer = scsi_alloc_sense_buffer(unchecked_isa_dma, gfp,
2288		- NUMA_NO_NODE);
2289		- if (!cmd->sense_buffer)
2290		- goto fail;
2291		- cmd->req.sense = cmd->sense_buffer;
2292		-
2293		- if (scsi_host_get_prot(shost) >= SHOST_DIX_TYPE0_PROTECTION) {
2294		- cmd->prot_sdb = kmem_cache_zalloc(scsi_sdb_cache, gfp);
2295		- if (!cmd->prot_sdb)
2296		- goto fail_free_sense;
2297		- }
2298		-
2299		- return 0;
2300		-
2301		-fail_free_sense:
2302		- scsi_free_sense_buffer(unchecked_isa_dma, cmd->sense_buffer);
2303		-fail:
2304		- return -ENOMEM;
2305		-}
2306		-
2307		-static void scsi_old_exit_rq(struct request_queue q, struct request rq)
2308		-{
2309		- struct scsi_cmnd *cmd = blk_mq_rq_to_pdu(rq);
2310		-
2311		- if (cmd->prot_sdb)
2312		- kmem_cache_free(scsi_sdb_cache, cmd->prot_sdb);
2313		- scsi_free_sense_buffer(cmd->flags & SCMD_UNCHECKED_ISA_DMA,
2314		- cmd->sense_buffer);
2315		-}
2316		-
2317		-struct request_queue scsi_old_alloc_queue(struct scsi_device sdev)
2318		-{
2319		- struct Scsi_Host *shost = sdev->host;
2320		- struct request_queue *q;
2321		-
2322		- q = blk_alloc_queue_node(GFP_KERNEL, NUMA_NO_NODE, NULL);
2323		- if (!q)
2324		- return NULL;
2325		- q->cmd_size = sizeof(struct scsi_cmnd) + shost->hostt->cmd_size;
2326		- q->rq_alloc_data = shost;
2327		- q->request_fn = scsi_request_fn;
2328		- q->init_rq_fn = scsi_old_init_rq;
2329		- q->exit_rq_fn = scsi_old_exit_rq;
2330		- q->initialize_rq_fn = scsi_initialize_rq;
2331		-
2332		- if (blk_init_allocated_queue(q) < 0) {
2333		- blk_cleanup_queue(q);
2334		- return NULL;
2335		- }
2336		-
2337		- __scsi_init_queue(shost, q);
2338		- blk_queue_flag_set(QUEUE_FLAG_SCSI_PASSTHROUGH, q);
2339		- blk_queue_prep_rq(q, scsi_prep_fn);
2340		- blk_queue_unprep_rq(q, scsi_unprep_fn);
2341		- blk_queue_softirq_done(q, scsi_softirq_done);
2342		- blk_queue_rq_timed_out(q, scsi_times_out);
2343		- blk_queue_lld_busy(q, scsi_lld_busy);
2344		- return q;
2345		-}
2346		-
2347		-static const struct blk_mq_ops scsi_mq_ops = {
	1836	+static const struct blk_mq_ops scsi_mq_ops_no_commit = {
2348	1837	.get_budget = scsi_mq_get_budget,
2349	1838	.put_budget = scsi_mq_put_budget,
2350	1839	.queue_rq = scsi_queue_rq,
..	..	@@ -2357,6 +1846,35 @@
2357	1846	.exit_request = scsi_mq_exit_request,
2358	1847	.initialize_rq_fn = scsi_initialize_rq,
2359	1848	.cleanup_rq = scsi_cleanup_rq,
	1849	+ .busy = scsi_mq_lld_busy,
	1850	+ .map_queues = scsi_map_queues,
	1851	+};
	1852	+
	1853	+
	1854	+static void scsi_commit_rqs(struct blk_mq_hw_ctx *hctx)
	1855	+{
	1856	+ struct request_queue *q = hctx->queue;
	1857	+ struct scsi_device *sdev = q->queuedata;
	1858	+ struct Scsi_Host *shost = sdev->host;
	1859	+
	1860	+ shost->hostt->commit_rqs(shost, hctx->queue_num);
	1861	+}
	1862	+
	1863	+static const struct blk_mq_ops scsi_mq_ops = {
	1864	+ .get_budget = scsi_mq_get_budget,
	1865	+ .put_budget = scsi_mq_put_budget,
	1866	+ .queue_rq = scsi_queue_rq,
	1867	+ .commit_rqs = scsi_commit_rqs,
	1868	+ .complete = scsi_softirq_done,
	1869	+ .timeout = scsi_timeout,
	1870	+#ifdef CONFIG_BLK_DEBUG_FS
	1871	+ .show_rq = scsi_show_rq,
	1872	+#endif
	1873	+ .init_request = scsi_mq_init_request,
	1874	+ .exit_request = scsi_mq_exit_request,
	1875	+ .initialize_rq_fn = scsi_initialize_rq,
	1876	+ .cleanup_rq = scsi_cleanup_rq,
	1877	+ .busy = scsi_mq_lld_busy,
2360	1878	.map_queues = scsi_map_queues,
2361	1879	};
2362	1880
..	..	@@ -2375,25 +1893,32 @@
2375	1893	int scsi_mq_setup_tags(struct Scsi_Host *shost)
2376	1894	{
2377	1895	unsigned int cmd_size, sgl_size;
	1896	+ struct blk_mq_tag_set *tag_set = &shost->tag_set;
2378	1897
2379	1898	sgl_size = max_t(unsigned int, sizeof(struct scatterlist),
2380		- scsi_mq_sgl_size(shost));
	1899	+ scsi_mq_inline_sgl_size(shost));
2381	1900	cmd_size = sizeof(struct scsi_cmnd) + shost->hostt->cmd_size + sgl_size;
2382	1901	if (scsi_host_get_prot(shost))
2383		- cmd_size += sizeof(struct scsi_data_buffer) + sgl_size;
	1902	+ cmd_size += sizeof(struct scsi_data_buffer) +
	1903	+ sizeof(struct scatterlist) * SCSI_INLINE_PROT_SG_CNT;
2384	1904
2385		- memset(&shost->tag_set, 0, sizeof(shost->tag_set));
2386		- shost->tag_set.ops = &scsi_mq_ops;
2387		- shost->tag_set.nr_hw_queues = shost->nr_hw_queues ? : 1;
2388		- shost->tag_set.queue_depth = shost->can_queue;
2389		- shost->tag_set.cmd_size = cmd_size;
2390		- shost->tag_set.numa_node = NUMA_NO_NODE;
2391		- shost->tag_set.flags = BLK_MQ_F_SHOULD_MERGE \| BLK_MQ_F_SG_MERGE;
2392		- shost->tag_set.flags \|=
	1905	+ memset(tag_set, 0, sizeof(*tag_set));
	1906	+ if (shost->hostt->commit_rqs)
	1907	+ tag_set->ops = &scsi_mq_ops;
	1908	+ else
	1909	+ tag_set->ops = &scsi_mq_ops_no_commit;
	1910	+ tag_set->nr_hw_queues = shost->nr_hw_queues ? : 1;
	1911	+ tag_set->queue_depth = shost->can_queue;
	1912	+ tag_set->cmd_size = cmd_size;
	1913	+ tag_set->numa_node = NUMA_NO_NODE;
	1914	+ tag_set->flags = BLK_MQ_F_SHOULD_MERGE;
	1915	+ tag_set->flags \|=
2393	1916	BLK_ALLOC_POLICY_TO_MQ_FLAG(shost->hostt->tag_alloc_policy);
2394		- shost->tag_set.driver_data = shost;
	1917	+ tag_set->driver_data = shost;
	1918	+ if (shost->host_tagset)
	1919	+ tag_set->flags \|= BLK_MQ_F_TAG_HCTX_SHARED;
2395	1920
2396		- return blk_mq_alloc_tag_set(&shost->tag_set);
	1921	+ return blk_mq_alloc_tag_set(tag_set);
2397	1922	}
2398	1923
2399	1924	void scsi_mq_destroy_tags(struct Scsi_Host *shost)
..	..	@@ -2412,33 +1937,22 @@
2412	1937	{
2413	1938	struct scsi_device *sdev = NULL;
2414	1939
2415		- if (q->mq_ops) {
2416		- if (q->mq_ops == &scsi_mq_ops)
2417		- sdev = q->queuedata;
2418		- } else if (q->request_fn == scsi_request_fn)
	1940	+ if (q->mq_ops == &scsi_mq_ops_no_commit \|\|
	1941	+ q->mq_ops == &scsi_mq_ops)
2419	1942	sdev = q->queuedata;
2420	1943	if (!sdev \|\| !get_device(&sdev->sdev_gendev))
2421	1944	sdev = NULL;
2422	1945
2423	1946	return sdev;
2424	1947	}
2425		-EXPORT_SYMBOL_GPL(scsi_device_from_queue);
2426	1948
2427		-/*
2428		- * Function: scsi_block_requests()
	1949	+/**
	1950	+ * scsi_block_requests - Utility function used by low-level drivers to prevent
	1951	+ * further commands from being queued to the device.
	1952	+ * @shost: host in question
2429	1953	*
2430		- * Purpose: Utility function used by low-level drivers to prevent further
2431		- * commands from being queued to the device.
2432		- *
2433		- * Arguments: shost - Host in question
2434		- *
2435		- * Returns: Nothing
2436		- *
2437		- * Lock status: No locks are assumed held.
2438		- *
2439		- * Notes: There is no timer nor any other means by which the requests
2440		- * get unblocked other than the low-level driver calling
2441		- * scsi_unblock_requests().
	1954	+ * There is no timer nor any other means by which the requests get unblocked
	1955	+ * other than the low-level driver calling scsi_unblock_requests().
2442	1956	*/
2443	1957	void scsi_block_requests(struct Scsi_Host *shost)
2444	1958	{
..	..	@@ -2446,25 +1960,15 @@
2446	1960	}
2447	1961	EXPORT_SYMBOL(scsi_block_requests);
2448	1962
2449		-/*
2450		- * Function: scsi_unblock_requests()
	1963	+/**
	1964	+ * scsi_unblock_requests - Utility function used by low-level drivers to allow
	1965	+ * further commands to be queued to the device.
	1966	+ * @shost: host in question
2451	1967	*
2452		- * Purpose: Utility function used by low-level drivers to allow further
2453		- * commands from being queued to the device.
2454		- *
2455		- * Arguments: shost - Host in question
2456		- *
2457		- * Returns: Nothing
2458		- *
2459		- * Lock status: No locks are assumed held.
2460		- *
2461		- * Notes: There is no timer nor any other means by which the requests
2462		- * get unblocked other than the low-level driver calling
2463		- * scsi_unblock_requests().
2464		- *
2465		- * This is done as an API function so that changes to the
2466		- * internals of the scsi mid-layer won't require wholesale
2467		- * changes to drivers that use this feature.
	1968	+ * There is no timer nor any other means by which the requests get unblocked
	1969	+ * other than the low-level driver calling scsi_unblock_requests(). This is done
	1970	+ * as an API function so that changes to the internals of the scsi mid-layer
	1971	+ * won't require wholesale changes to drivers that use this feature.
2468	1972	*/
2469	1973	void scsi_unblock_requests(struct Scsi_Host *shost)
2470	1974	{
..	..	@@ -2473,51 +1977,10 @@
2473	1977	}
2474	1978	EXPORT_SYMBOL(scsi_unblock_requests);
2475	1979
2476		-/*
2477		- * Function: scsi_set_cmd_timeout_override()
2478		- *
2479		- * Purpose: Utility function used by low-level drivers to override
2480		- timeout for the scsi commands.
2481		- *
2482		- * Arguments: sdev - scsi device in question
2483		- * timeout - timeout in jiffies
2484		- *
2485		- * Returns: Nothing
2486		- *
2487		- * Lock status: No locks are assumed held.
2488		- *
2489		- * Notes: Some platforms might be very slow and command completion may
2490		- * take much longer than default scsi command timeouts.
2491		- * SCSI Read/Write command timeout can be changed by
2492		- * blk_queue_rq_timeout() but there is no option to override
2493		- * timeout for rest of the scsi commands. This function would
2494		- * would allow this.
2495		- */
2496		-void scsi_set_cmd_timeout_override(struct scsi_device *sdev,
2497		- unsigned int timeout)
2498		-{
2499		- sdev->timeout_override = timeout;
2500		-}
2501		-EXPORT_SYMBOL(scsi_set_cmd_timeout_override);
2502		-
2503		-int __init scsi_init_queue(void)
2504		-{
2505		- scsi_sdb_cache = kmem_cache_create("scsi_data_buffer",
2506		- sizeof(struct scsi_data_buffer),
2507		- 0, 0, NULL);
2508		- if (!scsi_sdb_cache) {
2509		- printk(KERN_ERR "SCSI: can't init scsi sdb cache\n");
2510		- return -ENOMEM;
2511		- }
2512		-
2513		- return 0;
2514		-}
2515		-
2516	1980	void scsi_exit_queue(void)
2517	1981	{
2518	1982	kmem_cache_destroy(scsi_sense_cache);
2519	1983	kmem_cache_destroy(scsi_sense_isadma_cache);
2520		- kmem_cache_destroy(scsi_sdb_cache);
2521	1984	}
2522	1985
2523	1986	/**
..	..	@@ -2584,7 +2047,6 @@
2584	2047	real_buffer[1] = data->medium_type;
2585	2048	real_buffer[2] = data->device_specific;
2586	2049	real_buffer[3] = data->block_descriptor_length;
2587		-
2588	2050
2589	2051	cmd[0] = MODE_SELECT;
2590	2052	cmd[4] = len;
..	..	@@ -2610,9 +2072,7 @@
2610	2072	* @sshdr: place to put sense data (or NULL if no sense to be collected).
2611	2073	* must be SCSI_SENSE_BUFFERSIZE big.
2612	2074	*
2613		- * Returns zero if unsuccessful, or the header offset (either 4
2614		- * or 8 depending on whether a six or ten byte command was
2615		- * issued) if successful.
	2075	+ * Returns zero if successful, or a negative error number on failure
2616	2076	*/
2617	2077	int
2618	2078	scsi_mode_sense(struct scsi_device *sdev, int dbd, int modepage,
..	..	@@ -2627,6 +2087,8 @@
2627	2087
2628	2088	memset(data, 0, sizeof(*data));
2629	2089	memset(&cmd[0], 0, 12);
	2090	+
	2091	+ dbd = sdev->set_dbd_for_ms ? 8 : dbd;
2630	2092	cmd[1] = dbd & 0x18; /* allows DBD and LLBA bits */
2631	2093	cmd[2] = modepage;
2632	2094
..	..	@@ -2657,6 +2119,8 @@
2657	2119
2658	2120	result = scsi_execute_req(sdev, cmd, DMA_FROM_DEVICE, buffer, len,
2659	2121	sshdr, timeout, retries, NULL);
	2122	+ if (result < 0)
	2123	+ return result;
2660	2124
2661	2125	/* This code looks awful: what it's doing is making sure an
2662	2126	* ILLEGAL REQUEST sense return identifies the actual command
..	..	@@ -2668,7 +2132,7 @@
2668	2132	if (scsi_sense_valid(sshdr)) {
2669	2133	if ((sshdr->sense_key == ILLEGAL_REQUEST) &&
2670	2134	(sshdr->asc == 0x20) && (sshdr->ascq == 0)) {
2671		- /*
	2135	+ /*
2672	2136	* Invalid command operation code
2673	2137	*/
2674	2138	sdev->use_10_for_ms = 0;
..	..	@@ -2677,7 +2141,7 @@
2677	2141	}
2678	2142	}
2679	2143
2680		- if(scsi_status_is_good(result)) {
	2144	+ if (scsi_status_is_good(result)) {
2681	2145	if (unlikely(buffer[0] == 0x86 && buffer[1] == 0x0b &&
2682	2146	(modepage == 6 \|\| modepage == 8))) {
2683	2147	/* Initio breakage? */
..	..	@@ -2687,7 +2151,7 @@
2687	2151	data->device_specific = 0;
2688	2152	data->longlba = 0;
2689	2153	data->block_descriptor_length = 0;
2690		- } else if(use_10_for_ms) {
	2154	+ } else if (use_10_for_ms) {
2691	2155	data->length = buffer[0]*256 + buffer[1] + 2;
2692	2156	data->medium_type = buffer[2];
2693	2157	data->device_specific = buffer[3];
..	..	@@ -2701,13 +2165,15 @@
2701	2165	data->block_descriptor_length = buffer[3];
2702	2166	}
2703	2167	data->header_length = header_length;
	2168	+ result = 0;
2704	2169	} else if ((status_byte(result) == CHECK_CONDITION) &&
2705	2170	scsi_sense_valid(sshdr) &&
2706	2171	sshdr->sense_key == UNIT_ATTENTION && retry_count) {
2707	2172	retry_count--;
2708	2173	goto retry;
2709	2174	}
2710		-
	2175	+ if (result > 0)
	2176	+ result = -EIO;
2711	2177	return result;
2712	2178	}
2713	2179	EXPORT_SYMBOL(scsi_mode_sense);
..	..	@@ -2770,7 +2236,7 @@
2770	2236	goto illegal;
2771	2237	}
2772	2238	break;
2773		-
	2239	+
2774	2240	case SDEV_RUNNING:
2775	2241	switch (oldstate) {
2776	2242	case SDEV_CREATED:
..	..	@@ -2812,6 +2278,8 @@
2812	2278	switch (oldstate) {
2813	2279	case SDEV_RUNNING:
2814	2280	case SDEV_CREATED_BLOCK:
	2281	+ case SDEV_QUIESCE:
	2282	+ case SDEV_OFFLINE:
2815	2283	break;
2816	2284	default:
2817	2285	goto illegal;
..	..	@@ -2856,6 +2324,7 @@
2856	2324	break;
2857	2325
2858	2326	}
	2327	+ sdev->offline_already = false;
2859	2328	sdev->sdev_state = state;
2860	2329	return 0;
2861	2330
..	..	@@ -3045,48 +2514,13 @@
3045	2514	EXPORT_SYMBOL_GPL(sdev_evt_send_simple);
3046	2515
3047	2516	/**
3048		- * scsi_request_fn_active() - number of kernel threads inside scsi_request_fn()
3049		- * @sdev: SCSI device to count the number of scsi_request_fn() callers for.
3050		- */
3051		-static int scsi_request_fn_active(struct scsi_device *sdev)
3052		-{
3053		- struct request_queue *q = sdev->request_queue;
3054		- int request_fn_active;
3055		-
3056		- WARN_ON_ONCE(sdev->host->use_blk_mq);
3057		-
3058		- spin_lock_irq(q->queue_lock);
3059		- request_fn_active = q->request_fn_active;
3060		- spin_unlock_irq(q->queue_lock);
3061		-
3062		- return request_fn_active;
3063		-}
3064		-
3065		-/**
3066		- * scsi_wait_for_queuecommand() - wait for ongoing queuecommand() calls
3067		- * @sdev: SCSI device pointer.
3068		- *
3069		- * Wait until the ongoing shost->hostt->queuecommand() calls that are
3070		- * invoked from scsi_request_fn() have finished.
3071		- */
3072		-static void scsi_wait_for_queuecommand(struct scsi_device *sdev)
3073		-{
3074		- WARN_ON_ONCE(sdev->host->use_blk_mq);
3075		-
3076		- while (scsi_request_fn_active(sdev))
3077		- msleep(20);
3078		-}
3079		-
3080		-/**
3081		- * scsi_device_quiesce - Block user issued commands.
	2517	+ * scsi_device_quiesce - Block all commands except power management.
3082	2518	* @sdev: scsi device to quiesce.
3083	2519	*
3084	2520	* This works by trying to transition to the SDEV_QUIESCE state
3085	2521	* (which must be a legal transition). When the device is in this
3086		- * state, only special requests will be accepted, all others will
3087		- * be deferred. Since special requests may also be requeued requests,
3088		- * a successful return doesn't guarantee the device will be
3089		- * totally quiescent.
	2522	+ * state, only power management requests will be accepted, all others will
	2523	+ * be deferred.
3090	2524	*
3091	2525	* Must be called with user context, may sleep.
3092	2526	*
..	..	@@ -3148,12 +2582,12 @@
3148	2582	* device deleted during suspend)
3149	2583	*/
3150	2584	mutex_lock(&sdev->state_mutex);
	2585	+ if (sdev->sdev_state == SDEV_QUIESCE)
	2586	+ scsi_device_set_state(sdev, SDEV_RUNNING);
3151	2587	if (sdev->quiesced_by) {
3152	2588	sdev->quiesced_by = NULL;
3153	2589	blk_clear_pm_only(sdev->request_queue);
3154	2590	}
3155		- if (sdev->sdev_state == SDEV_QUIESCE)
3156		- scsi_device_set_state(sdev, SDEV_RUNNING);
3157	2591	mutex_unlock(&sdev->state_mutex);
3158	2592	}
3159	2593	EXPORT_SYMBOL(scsi_device_resume);
..	..	@@ -3201,7 +2635,6 @@
3201	2635	int scsi_internal_device_block_nowait(struct scsi_device *sdev)
3202	2636	{
3203	2637	struct request_queue *q = sdev->request_queue;
3204		- unsigned long flags;
3205	2638	int err = 0;
3206	2639
3207	2640	err = scsi_device_set_state(sdev, SDEV_BLOCK);
..	..	@@ -3212,19 +2645,12 @@
3212	2645	return err;
3213	2646	}
3214	2647
3215		- /*
	2648	+ /*
3216	2649	* The device has transitioned to SDEV_BLOCK. Stop the
3217	2650	* block layer from calling the midlayer with this device's
3218		- * request queue.
	2651	+ * request queue.
3219	2652	*/
3220		- if (q->mq_ops) {
3221		- blk_mq_quiesce_queue_nowait(q);
3222		- } else {
3223		- spin_lock_irqsave(q->queue_lock, flags);
3224		- blk_stop_queue(q);
3225		- spin_unlock_irqrestore(q->queue_lock, flags);
3226		- }
3227		-
	2653	+ blk_mq_quiesce_queue_nowait(q);
3228	2654	return 0;
3229	2655	}
3230	2656	EXPORT_SYMBOL_GPL(scsi_internal_device_block_nowait);
..	..	@@ -3243,10 +2669,6 @@
3243	2669	* a legal transition). When the device is in this state, command processing
3244	2670	* is paused until the device leaves the SDEV_BLOCK state. See also
3245	2671	* scsi_internal_device_unblock().
3246		- *
3247		- * To do: avoid that scsi_send_eh_cmnd() calls queuecommand() after
3248		- * scsi_internal_device_block() has blocked a SCSI device and also
3249		- * remove the rport mutex lock and unlock calls from srp_queuecommand().
3250	2672	*/
3251	2673	static int scsi_internal_device_block(struct scsi_device *sdev)
3252	2674	{
..	..	@@ -3255,29 +2677,18 @@
3255	2677
3256	2678	mutex_lock(&sdev->state_mutex);
3257	2679	err = scsi_internal_device_block_nowait(sdev);
3258		- if (err == 0) {
3259		- if (q->mq_ops)
3260		- blk_mq_quiesce_queue(q);
3261		- else
3262		- scsi_wait_for_queuecommand(sdev);
3263		- }
	2680	+ if (err == 0)
	2681	+ blk_mq_quiesce_queue(q);
3264	2682	mutex_unlock(&sdev->state_mutex);
3265	2683
3266	2684	return err;
3267	2685	}
3268		-
	2686	+
3269	2687	void scsi_start_queue(struct scsi_device *sdev)
3270	2688	{
3271	2689	struct request_queue *q = sdev->request_queue;
3272		- unsigned long flags;
3273	2690
3274		- if (q->mq_ops) {
3275		- blk_mq_unquiesce_queue(q);
3276		- } else {
3277		- spin_lock_irqsave(q->queue_lock, flags);
3278		- blk_start_queue(q);
3279		- spin_unlock_irqrestore(q->queue_lock, flags);
3280		- }
	2691	+ blk_mq_unquiesce_queue(q);
3281	2692	}
3282	2693
3283	2694	/**
..	..	@@ -3298,6 +2709,14 @@
3298	2709	int scsi_internal_device_unblock_nowait(struct scsi_device *sdev,
3299	2710	enum scsi_device_state new_state)
3300	2711	{
	2712	+ switch (new_state) {
	2713	+ case SDEV_RUNNING:
	2714	+ case SDEV_TRANSPORT_OFFLINE:
	2715	+ break;
	2716	+ default:
	2717	+ return -EINVAL;
	2718	+ }
	2719	+
3301	2720	/*
3302	2721	* Try to transition the scsi device to SDEV_RUNNING or one of the
3303	2722	* offlined states and goose the device queue if successful.
..	..	@@ -3355,7 +2774,12 @@
3355	2774	static void
3356	2775	device_block(struct scsi_device sdev, void data)
3357	2776	{
3358		- scsi_internal_device_block(sdev);
	2777	+ int ret;
	2778	+
	2779	+ ret = scsi_internal_device_block(sdev);
	2780	+
	2781	+ WARN_ONCE(ret, "scsi_internal_device_block(%s) failed: ret = %d\n",
	2782	+ dev_name(&sdev->sdev_gendev), ret);
3359	2783	}
3360	2784
3361	2785	static int
..	..	@@ -3404,6 +2828,56 @@
3404	2828	}
3405	2829	EXPORT_SYMBOL_GPL(scsi_target_unblock);
3406	2830
	2831	+int
	2832	+scsi_host_block(struct Scsi_Host *shost)
	2833	+{
	2834	+ struct scsi_device *sdev;
	2835	+ int ret = 0;
	2836	+
	2837	+ /*
	2838	+ * Call scsi_internal_device_block_nowait so we can avoid
	2839	+ * calling synchronize_rcu() for each LUN.
	2840	+ */
	2841	+ shost_for_each_device(sdev, shost) {
	2842	+ mutex_lock(&sdev->state_mutex);
	2843	+ ret = scsi_internal_device_block_nowait(sdev);
	2844	+ mutex_unlock(&sdev->state_mutex);
	2845	+ if (ret) {
	2846	+ scsi_device_put(sdev);
	2847	+ break;
	2848	+ }
	2849	+ }
	2850	+
	2851	+ /*
	2852	+ * SCSI never enables blk-mq's BLK_MQ_F_BLOCKING flag so
	2853	+ * calling synchronize_rcu() once is enough.
	2854	+ */
	2855	+ WARN_ON_ONCE(shost->tag_set.flags & BLK_MQ_F_BLOCKING);
	2856	+
	2857	+ if (!ret)
	2858	+ synchronize_rcu();
	2859	+
	2860	+ return ret;
	2861	+}
	2862	+EXPORT_SYMBOL_GPL(scsi_host_block);
	2863	+
	2864	+int
	2865	+scsi_host_unblock(struct Scsi_Host *shost, int new_state)
	2866	+{
	2867	+ struct scsi_device *sdev;
	2868	+ int ret = 0;
	2869	+
	2870	+ shost_for_each_device(sdev, shost) {
	2871	+ ret = scsi_internal_device_unblock(sdev, new_state);
	2872	+ if (ret) {
	2873	+ scsi_device_put(sdev);
	2874	+ break;
	2875	+ }
	2876	+ }
	2877	+ return ret;
	2878	+}
	2879	+EXPORT_SYMBOL_GPL(scsi_host_unblock);
	2880	+
3407	2881	/**
3408	2882	* scsi_kmap_atomic_sg - find and atomically map an sg-elemnt
3409	2883	* @sgl: scatter-gather list