~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,3 +1,4 @@
	1	+// SPDX-License-Identifier: GPL-2.0-or-later
1	2	/*******************************************************************************
2	3	* Filename: target_core_transport.c
3	4	*
..	..	@@ -6,20 +7,6 @@
6	7	* (c) Copyright 2002-2013 Datera, Inc.
7	8	*
8	9	* Nicholas A. Bellinger <nab@kernel.org>
9		- *
10		- * This program is free software; you can redistribute it and/or modify
11		- * it under the terms of the GNU General Public License as published by
12		- * the Free Software Foundation; either version 2 of the License, or
13		- * (at your option) any later version.
14		- *
15		- * This program is distributed in the hope that it will be useful,
16		- * but WITHOUT ANY WARRANTY; without even the implied warranty of
17		- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18		- * GNU General Public License for more details.
19		- *
20		- * You should have received a copy of the GNU General Public License
21		- * along with this program; if not, write to the Free Software
22		- * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
23	10	*
24	11	******************************************************************************/
25	12
..	..	@@ -205,19 +192,19 @@
205	192	if (sub_api_initialized)
206	193	return;
207	194
208		- ret = request_module("target_core_iblock");
	195	+ ret = IS_ENABLED(CONFIG_TCM_IBLOCK) && request_module("target_core_iblock");
209	196	if (ret != 0)
210	197	pr_err("Unable to load target_core_iblock\n");
211	198
212		- ret = request_module("target_core_file");
	199	+ ret = IS_ENABLED(CONFIG_TCM_FILEIO) && request_module("target_core_file");
213	200	if (ret != 0)
214	201	pr_err("Unable to load target_core_file\n");
215	202
216		- ret = request_module("target_core_pscsi");
	203	+ ret = IS_ENABLED(CONFIG_TCM_PSCSI) && request_module("target_core_pscsi");
217	204	if (ret != 0)
218	205	pr_err("Unable to load target_core_pscsi\n");
219	206
220		- ret = request_module("target_core_user");
	207	+ ret = IS_ENABLED(CONFIG_TCM_USER2) && request_module("target_core_user");
221	208	if (ret != 0)
222	209	pr_err("Unable to load target_core_user\n");
223	210
..	..	@@ -248,6 +235,11 @@
248	235	target_release_sess_cmd_refcnt, 0, GFP_KERNEL);
249	236	}
250	237	EXPORT_SYMBOL(transport_init_session);
	238	+
	239	+void transport_uninit_session(struct se_session *se_sess)
	240	+{
	241	+ percpu_ref_exit(&se_sess->cmd_count);
	242	+}
251	243
252	244	/**
253	245	* transport_alloc_session - allocate a session object and initialize it
..	..	@@ -287,14 +279,11 @@
287	279	{
288	280	int rc;
289	281
290		- se_sess->sess_cmd_map = kcalloc(tag_size, tag_num,
291		- GFP_KERNEL \| __GFP_NOWARN \| __GFP_RETRY_MAYFAIL);
	282	+ se_sess->sess_cmd_map = kvcalloc(tag_size, tag_num,
	283	+ GFP_KERNEL \| __GFP_RETRY_MAYFAIL);
292	284	if (!se_sess->sess_cmd_map) {
293		- se_sess->sess_cmd_map = vzalloc(array_size(tag_size, tag_num));
294		- if (!se_sess->sess_cmd_map) {
295		- pr_err("Unable to allocate se_sess->sess_cmd_map\n");
296		- return -ENOMEM;
297		- }
	285	+ pr_err("Unable to allocate se_sess->sess_cmd_map\n");
	286	+ return -ENOMEM;
298	287	}
299	288
300	289	rc = sbitmap_queue_init_node(&se_sess->sess_tag_pool, tag_num, -1,
..	..	@@ -411,7 +400,7 @@
411	400	list_add_tail(&se_sess->sess_list, &se_tpg->tpg_sess_list);
412	401
413	402	pr_debug("TARGET_CORE[%s]: Registered fabric_sess_ptr: %p\n",
414		- se_tpg->se_tpg_tfo->get_fabric_name(), se_sess->fabric_sess_ptr);
	403	+ se_tpg->se_tpg_tfo->fabric_name, se_sess->fabric_sess_ptr);
415	404	}
416	405	EXPORT_SYMBOL(__transport_register_session);
417	406
..	..	@@ -595,10 +584,19 @@
595	584	sbitmap_queue_free(&se_sess->sess_tag_pool);
596	585	kvfree(se_sess->sess_cmd_map);
597	586	}
598		- percpu_ref_exit(&se_sess->cmd_count);
	587	+ transport_uninit_session(se_sess);
599	588	kmem_cache_free(se_sess_cache, se_sess);
600	589	}
601	590	EXPORT_SYMBOL(transport_free_session);
	591	+
	592	+static int target_release_res(struct se_device dev, void data)
	593	+{
	594	+ struct se_session *sess = data;
	595	+
	596	+ if (dev->reservation_holder == sess)
	597	+ target_release_reservation(dev);
	598	+ return 0;
	599	+}
602	600
603	601	void transport_deregister_session(struct se_session *se_sess)
604	602	{
..	..	@@ -616,8 +614,14 @@
616	614	se_sess->fabric_sess_ptr = NULL;
617	615	spin_unlock_irqrestore(&se_tpg->session_lock, flags);
618	616
	617	+ /*
	618	+ * Since the session is being removed, release SPC-2
	619	+ * reservations held by the session that is disappearing.
	620	+ */
	621	+ target_for_each_device(target_release_res, se_sess);
	622	+
619	623	pr_debug("TARGET_CORE[%s]: Deregistered fabric_sess\n",
620		- se_tpg->se_tpg_tfo->get_fabric_name());
	624	+ se_tpg->se_tpg_tfo->fabric_name);
621	625	/*
622	626	* If last kref is dropping now for an explicit NodeACL, awake sleeping
623	627	* ->acl_free_comp caller to wakeup configfs se_node_acl->acl_group
..	..	@@ -710,32 +714,6 @@
710	714	percpu_ref_put(&lun->lun_ref);
711	715	}
712	716
713		-int transport_cmd_finish_abort(struct se_cmd *cmd)
714		-{
715		- bool send_tas = cmd->transport_state & CMD_T_TAS;
716		- bool ack_kref = (cmd->se_cmd_flags & SCF_ACK_KREF);
717		- int ret = 0;
718		-
719		- if (send_tas)
720		- transport_send_task_abort(cmd);
721		-
722		- if (cmd->se_cmd_flags & SCF_SE_LUN_CMD)
723		- transport_lun_remove_cmd(cmd);
724		- /*
725		- * Allow the fabric driver to unmap any resources before
726		- * releasing the descriptor via TFO->release_cmd()
727		- */
728		- if (!send_tas)
729		- cmd->se_tfo->aborted_task(cmd);
730		-
731		- if (transport_cmd_check_stop_to_fabric(cmd))
732		- return 1;
733		- if (!send_tas && ack_kref)
734		- ret = target_put_sess_cmd(cmd);
735		-
736		- return ret;
737		-}
738		-
739	717	static void target_complete_failure_work(struct work_struct *work)
740	718	{
741	719	struct se_cmd *cmd = container_of(work, struct se_cmd, work);
..	..	@@ -785,11 +763,87 @@
785	763	}
786	764	EXPORT_SYMBOL(transport_copy_sense_to_cmd);
787	765
	766	+static void target_handle_abort(struct se_cmd *cmd)
	767	+{
	768	+ bool tas = cmd->transport_state & CMD_T_TAS;
	769	+ bool ack_kref = cmd->se_cmd_flags & SCF_ACK_KREF;
	770	+ int ret;
	771	+
	772	+ pr_debug("tag %#llx: send_abort_response = %d\n", cmd->tag, tas);
	773	+
	774	+ if (tas) {
	775	+ if (!(cmd->se_cmd_flags & SCF_SCSI_TMR_CDB)) {
	776	+ cmd->scsi_status = SAM_STAT_TASK_ABORTED;
	777	+ pr_debug("Setting SAM_STAT_TASK_ABORTED status for CDB: 0x%02x, ITT: 0x%08llx\n",
	778	+ cmd->t_task_cdb[0], cmd->tag);
	779	+ trace_target_cmd_complete(cmd);
	780	+ ret = cmd->se_tfo->queue_status(cmd);
	781	+ if (ret) {
	782	+ transport_handle_queue_full(cmd, cmd->se_dev,
	783	+ ret, false);
	784	+ return;
	785	+ }
	786	+ } else {
	787	+ cmd->se_tmr_req->response = TMR_FUNCTION_REJECTED;
	788	+ cmd->se_tfo->queue_tm_rsp(cmd);
	789	+ }
	790	+ } else {
	791	+ /*
	792	+ * Allow the fabric driver to unmap any resources before
	793	+ * releasing the descriptor via TFO->release_cmd().
	794	+ */
	795	+ cmd->se_tfo->aborted_task(cmd);
	796	+ if (ack_kref)
	797	+ WARN_ON_ONCE(target_put_sess_cmd(cmd) != 0);
	798	+ /*
	799	+ * To do: establish a unit attention condition on the I_T
	800	+ * nexus associated with cmd. See also the paragraph "Aborting
	801	+ * commands" in SAM.
	802	+ */
	803	+ }
	804	+
	805	+ WARN_ON_ONCE(kref_read(&cmd->cmd_kref) == 0);
	806	+
	807	+ transport_lun_remove_cmd(cmd);
	808	+
	809	+ transport_cmd_check_stop_to_fabric(cmd);
	810	+}
	811	+
	812	+static void target_abort_work(struct work_struct *work)
	813	+{
	814	+ struct se_cmd *cmd = container_of(work, struct se_cmd, work);
	815	+
	816	+ target_handle_abort(cmd);
	817	+}
	818	+
	819	+static bool target_cmd_interrupted(struct se_cmd *cmd)
	820	+{
	821	+ int post_ret;
	822	+
	823	+ if (cmd->transport_state & CMD_T_ABORTED) {
	824	+ if (cmd->transport_complete_callback)
	825	+ cmd->transport_complete_callback(cmd, false, &post_ret);
	826	+ INIT_WORK(&cmd->work, target_abort_work);
	827	+ queue_work(target_completion_wq, &cmd->work);
	828	+ return true;
	829	+ } else if (cmd->transport_state & CMD_T_STOP) {
	830	+ if (cmd->transport_complete_callback)
	831	+ cmd->transport_complete_callback(cmd, false, &post_ret);
	832	+ complete_all(&cmd->t_transport_stop_comp);
	833	+ return true;
	834	+ }
	835	+
	836	+ return false;
	837	+}
	838	+
	839	+/* May be called from interrupt context so must not sleep. */
788	840	void target_complete_cmd(struct se_cmd *cmd, u8 scsi_status)
789	841	{
790		- struct se_device *dev = cmd->se_dev;
791	842	int success;
792	843	unsigned long flags;
	844	+
	845	+ if (target_cmd_interrupted(cmd))
	846	+ return;
793	847
794	848	cmd->scsi_status = scsi_status;
795	849
..	..	@@ -806,34 +860,12 @@
806	860	break;
807	861	}
808	862
809		- /*
810		- * Check for case where an explicit ABORT_TASK has been received
811		- * and transport_wait_for_tasks() will be waiting for completion..
812		- */
813		- if (cmd->transport_state & CMD_T_ABORTED \|\|
814		- cmd->transport_state & CMD_T_STOP) {
815		- spin_unlock_irqrestore(&cmd->t_state_lock, flags);
816		- /*
817		- * If COMPARE_AND_WRITE was stopped by __transport_wait_for_tasks(),
818		- * release se_device->caw_sem obtained by sbc_compare_and_write()
819		- * since target_complete_ok_work() or target_complete_failure_work()
820		- * won't be called to invoke the normal CAW completion callbacks.
821		- */
822		- if (cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE) {
823		- up(&dev->caw_sem);
824		- }
825		- complete_all(&cmd->t_transport_stop_comp);
826		- return;
827		- } else if (!success) {
828		- INIT_WORK(&cmd->work, target_complete_failure_work);
829		- } else {
830		- INIT_WORK(&cmd->work, target_complete_ok_work);
831		- }
832		-
833	863	cmd->t_state = TRANSPORT_COMPLETE;
834	864	cmd->transport_state \|= (CMD_T_COMPLETE \| CMD_T_ACTIVE);
835	865	spin_unlock_irqrestore(&cmd->t_state_lock, flags);
836	866
	867	+ INIT_WORK(&cmd->work, success ? target_complete_ok_work :
	868	+ target_complete_failure_work);
837	869	if (cmd->se_cmd_flags & SCF_USE_CPUID)
838	870	queue_work_on(cmd->cpuid, target_completion_wq, &cmd->work);
839	871	else
..	..	@@ -902,7 +934,7 @@
902	934	atomic_dec_mb(&dev->dev_qf_count);
903	935
904	936	pr_debug("Processing %s cmd: %p QUEUE_FULL in work queue"
905		- " context: %s\n", cmd->se_tfo->get_fabric_name(), cmd,
	937	+ " context: %s\n", cmd->se_tfo->fabric_name, cmd,
906	938	(cmd->t_state == TRANSPORT_COMPLETE_QF_OK) ? "COMPLETE_OK" :
907	939	(cmd->t_state == TRANSPORT_COMPLETE_QF_WP) ? "WRITE_PENDING"
908	940	: "UNKNOWN");
..	..	@@ -1256,6 +1288,19 @@
1256	1288	return TCM_NO_SENSE;
1257	1289	}
1258	1290
	1291	+/**
	1292	+ * target_cmd_size_check - Check whether there will be a residual.
	1293	+ * @cmd: SCSI command.
	1294	+ * @size: Data buffer size derived from CDB. The data buffer size provided by
	1295	+ * the SCSI transport driver is available in @cmd->data_length.
	1296	+ *
	1297	+ * Compare the data buffer size from the CDB with the data buffer limit from the transport
	1298	+ * header. Set @cmd->residual_count and SCF_OVERFLOW_BIT or SCF_UNDERFLOW_BIT if necessary.
	1299	+ *
	1300	+ * Note: target drivers set @cmd->data_length by calling transport_init_se_cmd().
	1301	+ *
	1302	+ * Return: TCM_NO_SENSE
	1303	+ */
1259	1304	sense_reason_t
1260	1305	target_cmd_size_check(struct se_cmd *cmd, unsigned int size)
1261	1306	{
..	..	@@ -1266,7 +1311,7 @@
1266	1311	} else if (size != cmd->data_length) {
1267	1312	pr_warn_ratelimited("TARGET_CORE[%s]: Expected Transfer Length:"
1268	1313	" %u does not match SCSI CDB Length: %u for SAM Opcode:"
1269		- " 0x%02x\n", cmd->se_tfo->get_fabric_name(),
	1314	+ " 0x%02x\n", cmd->se_tfo->fabric_name,
1270	1315	cmd->data_length, size, cmd->t_task_cdb[0]);
1271	1316
1272	1317	if (cmd->data_direction == DMA_TO_DEVICE) {
..	..	@@ -1331,14 +1376,15 @@
1331	1376	u32 data_length,
1332	1377	int data_direction,
1333	1378	int task_attr,
1334		- unsigned char *sense_buffer)
	1379	+ unsigned char *sense_buffer, u64 unpacked_lun)
1335	1380	{
1336	1381	INIT_LIST_HEAD(&cmd->se_delayed_node);
1337	1382	INIT_LIST_HEAD(&cmd->se_qf_node);
1338	1383	INIT_LIST_HEAD(&cmd->se_cmd_list);
1339	1384	INIT_LIST_HEAD(&cmd->state_list);
1340	1385	init_completion(&cmd->t_transport_stop_comp);
1341		- cmd->compl = NULL;
	1386	+ cmd->free_compl = NULL;
	1387	+ cmd->abrt_compl = NULL;
1342	1388	spin_lock_init(&cmd->t_state_lock);
1343	1389	INIT_WORK(&cmd->work, NULL);
1344	1390	kref_init(&cmd->cmd_kref);
..	..	@@ -1349,6 +1395,7 @@
1349	1395	cmd->data_direction = data_direction;
1350	1396	cmd->sam_task_attr = task_attr;
1351	1397	cmd->sense_buffer = sense_buffer;
	1398	+ cmd->orig_fe_lun = unpacked_lun;
1352	1399
1353	1400	cmd->state_active = false;
1354	1401	}
..	..	@@ -1363,7 +1410,7 @@
1363	1410	* Check if SAM Task Attribute emulation is enabled for this
1364	1411	* struct se_device storage object
1365	1412	*/
1366		- if (dev->transport->transport_flags & TRANSPORT_FLAG_PASSTHROUGH)
	1413	+ if (dev->transport_flags & TRANSPORT_FLAG_PASSTHROUGH)
1367	1414	return 0;
1368	1415
1369	1416	if (cmd->sam_task_attr == TCM_ACA_TAG) {
..	..	@@ -1376,11 +1423,11 @@
1376	1423	}
1377	1424
1378	1425	sense_reason_t
1379		-target_setup_cmd_from_cdb(struct se_cmd cmd, unsigned char cdb)
	1426	+target_cmd_init_cdb(struct se_cmd cmd, unsigned char cdb)
1380	1427	{
1381		- struct se_device *dev = cmd->se_dev;
1382	1428	sense_reason_t ret;
1383	1429
	1430	+ cmd->t_task_cdb = &cmd->__t_task_cdb[0];
1384	1431	/*
1385	1432	* Ensure that the received CDB is less than the max (252 + 8) bytes
1386	1433	* for VARIABLE_LENGTH_CMD
..	..	@@ -1389,7 +1436,8 @@
1389	1436	pr_err("Received SCSI CDB with command_size: %d that"
1390	1437	" exceeds SCSI_MAX_VARLEN_CDB_SIZE: %d\n",
1391	1438	scsi_command_size(cdb), SCSI_MAX_VARLEN_CDB_SIZE);
1392		- return TCM_INVALID_CDB_FIELD;
	1439	+ ret = TCM_INVALID_CDB_FIELD;
	1440	+ goto err;
1393	1441	}
1394	1442	/*
1395	1443	* If the received CDB is larger than TCM_MAX_COMMAND_SIZE,
..	..	@@ -1404,21 +1452,39 @@
1404	1452	" %u > sizeof(cmd->__t_task_cdb): %lu ops\n",
1405	1453	scsi_command_size(cdb),
1406	1454	(unsigned long)sizeof(cmd->__t_task_cdb));
1407		- return TCM_OUT_OF_RESOURCES;
	1455	+ ret = TCM_OUT_OF_RESOURCES;
	1456	+ goto err;
1408	1457	}
1409		- } else
1410		- cmd->t_task_cdb = &cmd->__t_task_cdb[0];
	1458	+ }
1411	1459	/*
1412	1460	* Copy the original CDB into cmd->
1413	1461	*/
1414	1462	memcpy(cmd->t_task_cdb, cdb, scsi_command_size(cdb));
1415	1463
1416	1464	trace_target_sequencer_start(cmd);
	1465	+ return 0;
	1466	+
	1467	+err:
	1468	+ /*
	1469	+ * Copy the CDB here to allow trace_target_cmd_complete() to
	1470	+ * print the cdb to the trace buffers.
	1471	+ */
	1472	+ memcpy(cmd->t_task_cdb, cdb, min(scsi_command_size(cdb),
	1473	+ (unsigned int)TCM_MAX_COMMAND_SIZE));
	1474	+ return ret;
	1475	+}
	1476	+EXPORT_SYMBOL(target_cmd_init_cdb);
	1477	+
	1478	+sense_reason_t
	1479	+target_cmd_parse_cdb(struct se_cmd *cmd)
	1480	+{
	1481	+ struct se_device *dev = cmd->se_dev;
	1482	+ sense_reason_t ret;
1417	1483
1418	1484	ret = dev->transport->parse_cdb(cmd);
1419	1485	if (ret == TCM_UNSUPPORTED_SCSI_OPCODE)
1420	1486	pr_warn_ratelimited("%s/%s: Unsupported SCSI Opcode 0x%02x, sending CHECK_CONDITION.\n",
1421		- cmd->se_tfo->get_fabric_name(),
	1487	+ cmd->se_tfo->fabric_name,
1422	1488	cmd->se_sess->se_node_acl->initiatorname,
1423	1489	cmd->t_task_cdb[0]);
1424	1490	if (ret)
..	..	@@ -1432,7 +1498,7 @@
1432	1498	atomic_long_inc(&cmd->se_lun->lun_stats.cmd_pdus);
1433	1499	return 0;
1434	1500	}
1435		-EXPORT_SYMBOL(target_setup_cmd_from_cdb);
	1501	+EXPORT_SYMBOL(target_cmd_parse_cdb);
1436	1502
1437	1503	/*
1438	1504	* Used by fabric module frontends to queue tasks directly.
..	..	@@ -1554,7 +1620,8 @@
1554	1620	* target_core_fabric_ops->queue_status() callback
1555	1621	*/
1556	1622	transport_init_se_cmd(se_cmd, se_tpg->se_tpg_tfo, se_sess,
1557		- data_length, data_dir, task_attr, sense);
	1623	+ data_length, data_dir, task_attr, sense,
	1624	+ unpacked_lun);
1558	1625
1559	1626	if (flags & TARGET_SCF_USE_CPUID)
1560	1627	se_cmd->se_cmd_flags \|= SCF_USE_CPUID;
..	..	@@ -1577,17 +1644,25 @@
1577	1644	*/
1578	1645	if (flags & TARGET_SCF_BIDI_OP)
1579	1646	se_cmd->se_cmd_flags \|= SCF_BIDI;
1580		- /*
1581		- * Locate se_lun pointer and attach it to struct se_cmd
1582		- */
1583		- rc = transport_lookup_cmd_lun(se_cmd, unpacked_lun);
	1647	+
	1648	+ rc = target_cmd_init_cdb(se_cmd, cdb);
1584	1649	if (rc) {
1585	1650	transport_send_check_condition_and_sense(se_cmd, rc, 0);
1586	1651	target_put_sess_cmd(se_cmd);
1587	1652	return 0;
1588	1653	}
1589	1654
1590		- rc = target_setup_cmd_from_cdb(se_cmd, cdb);
	1655	+ /*
	1656	+ * Locate se_lun pointer and attach it to struct se_cmd
	1657	+ */
	1658	+ rc = transport_lookup_cmd_lun(se_cmd);
	1659	+ if (rc) {
	1660	+ transport_send_check_condition_and_sense(se_cmd, rc, 0);
	1661	+ target_put_sess_cmd(se_cmd);
	1662	+ return 0;
	1663	+ }
	1664	+
	1665	+ rc = target_cmd_parse_cdb(se_cmd);
1591	1666	if (rc != 0) {
1592	1667	transport_generic_request_failure(se_cmd, rc);
1593	1668	return 0;
..	..	@@ -1748,7 +1823,7 @@
1748	1823	BUG_ON(!se_tpg);
1749	1824
1750	1825	transport_init_se_cmd(se_cmd, se_tpg->se_tpg_tfo, se_sess,
1751		- 0, DMA_NONE, TCM_SIMPLE_TAG, sense);
	1826	+ 0, DMA_NONE, TCM_SIMPLE_TAG, sense, unpacked_lun);
1752	1827	/*
1753	1828	* FIXME: Currently expect caller to handle se_cmd->se_tmr_req
1754	1829	* allocation failure.
..	..	@@ -1772,11 +1847,12 @@
1772	1847	* out unpacked_lun for the original se_cmd.
1773	1848	*/
1774	1849	if (tm_type == TMR_ABORT_TASK && (flags & TARGET_SCF_LOOKUP_LUN_FROM_TAG)) {
1775		- if (!target_lookup_lun_from_tag(se_sess, tag, &unpacked_lun))
	1850	+ if (!target_lookup_lun_from_tag(se_sess, tag,
	1851	+ &se_cmd->orig_fe_lun))
1776	1852	goto failure;
1777	1853	}
1778	1854
1779		- ret = transport_lookup_tmr_lun(se_cmd, unpacked_lun);
	1855	+ ret = transport_lookup_tmr_lun(se_cmd);
1780	1856	if (ret)
1781	1857	goto failure;
1782	1858
..	..	@@ -1800,7 +1876,7 @@
1800	1876	void transport_generic_request_failure(struct se_cmd *cmd,
1801	1877	sense_reason_t sense_reason)
1802	1878	{
1803		- int ret = 0, post_ret = 0;
	1879	+ int ret = 0, post_ret;
1804	1880
1805	1881	pr_debug("-----[ Storage Engine Exception; sense_reason %d\n",
1806	1882	sense_reason);
..	..	@@ -1811,16 +1887,14 @@
1811	1887	*/
1812	1888	transport_complete_task_attr(cmd);
1813	1889
1814		- /*
1815		- * Handle special case for COMPARE_AND_WRITE failure, where the
1816		- * callback is expected to drop the per device ->caw_sem.
1817		- */
1818		- if ((cmd->se_cmd_flags & SCF_COMPARE_AND_WRITE) &&
1819		- cmd->transport_complete_callback)
	1890	+ if (cmd->transport_complete_callback)
1820	1891	cmd->transport_complete_callback(cmd, false, &post_ret);
1821	1892
1822		- if (transport_check_aborted_status(cmd, 1))
	1893	+ if (cmd->transport_state & CMD_T_ABORTED) {
	1894	+ INIT_WORK(&cmd->work, target_abort_work);
	1895	+ queue_work(target_completion_wq, &cmd->work);
1823	1896	return;
	1897	+ }
1824	1898
1825	1899	switch (sense_reason) {
1826	1900	case TCM_NON_EXISTENT_LUN:
..	..	@@ -1866,7 +1940,8 @@
1866	1940	* See spc4r17, section 7.4.6 Control Mode Page, Table 349
1867	1941	*/
1868	1942	if (cmd->se_sess &&
1869		- cmd->se_dev->dev_attrib.emulate_ua_intlck_ctrl == 2) {
	1943	+ cmd->se_dev->dev_attrib.emulate_ua_intlck_ctrl
	1944	+ == TARGET_UA_INTLCK_CTRL_ESTABLISH_UA) {
1870	1945	target_ua_allocate_lun(cmd->se_sess->se_node_acl,
1871	1946	cmd->orig_fe_lun, 0x2C,
1872	1947	ASCQ_2CH_PREVIOUS_RESERVATION_CONFLICT_STATUS);
..	..	@@ -1979,7 +2054,7 @@
1979	2054	{
1980	2055	struct se_device *dev = cmd->se_dev;
1981	2056
1982		- if (dev->transport->transport_flags & TRANSPORT_FLAG_PASSTHROUGH)
	2057	+ if (dev->transport_flags & TRANSPORT_FLAG_PASSTHROUGH)
1983	2058	return false;
1984	2059
1985	2060	cmd->se_cmd_flags \|= SCF_TASK_ATTR_SET;
..	..	@@ -2035,32 +2110,19 @@
2035	2110	return true;
2036	2111	}
2037	2112
2038		-static int __transport_check_aborted_status(struct se_cmd *, int);
2039		-
2040	2113	void target_execute_cmd(struct se_cmd *cmd)
2041	2114	{
2042	2115	/*
2043	2116	* Determine if frontend context caller is requesting the stopping of
2044	2117	* this command for frontend exceptions.
2045	2118	*
2046		- * If the received CDB has aleady been aborted stop processing it here.
	2119	+ * If the received CDB has already been aborted stop processing it here.
2047	2120	*/
	2121	+ if (target_cmd_interrupted(cmd))
	2122	+ return;
	2123	+
2048	2124	spin_lock_irq(&cmd->t_state_lock);
2049		- if (__transport_check_aborted_status(cmd, 1)) {
2050		- spin_unlock_irq(&cmd->t_state_lock);
2051		- return;
2052		- }
2053		- if (cmd->transport_state & CMD_T_STOP) {
2054		- pr_debug("%s:%d CMD_T_STOP for ITT: 0x%08llx\n",
2055		- __func__, __LINE__, cmd->tag);
2056		-
2057		- spin_unlock_irq(&cmd->t_state_lock);
2058		- complete_all(&cmd->t_transport_stop_comp);
2059		- return;
2060		- }
2061		-
2062	2125	cmd->t_state = TRANSPORT_PROCESSING;
2063		- cmd->transport_state &= ~CMD_T_PRE_EXECUTE;
2064	2126	cmd->transport_state \|= CMD_T_ACTIVE \| CMD_T_SENT;
2065	2127	spin_unlock_irq(&cmd->t_state_lock);
2066	2128
..	..	@@ -2134,7 +2196,7 @@
2134	2196	{
2135	2197	struct se_device *dev = cmd->se_dev;
2136	2198
2137		- if (dev->transport->transport_flags & TRANSPORT_FLAG_PASSTHROUGH)
	2199	+ if (dev->transport_flags & TRANSPORT_FLAG_PASSTHROUGH)
2138	2200	return;
2139	2201
2140	2202	if (!(cmd->se_cmd_flags & SCF_TASK_ATTR_SET))
..	..	@@ -2214,7 +2276,7 @@
2214	2276	ret = cmd->se_tfo->queue_data_in(cmd);
2215	2277	break;
2216	2278	}
2217		- /* fall through */
	2279	+ fallthrough;
2218	2280	case DMA_NONE:
2219	2281	queue_status:
2220	2282	trace_target_cmd_complete(cmd);
..	..	@@ -2409,7 +2471,7 @@
2409	2471	goto queue_full;
2410	2472	break;
2411	2473	}
2412		- /* fall through */
	2474	+ fallthrough;
2413	2475	case DMA_NONE:
2414	2476	queue_status:
2415	2477	trace_target_cmd_complete(cmd);
..	..	@@ -2570,7 +2632,7 @@
2570	2632	}
2571	2633
2572	2634	/*
2573		- * Determine is the TCM fabric module has already allocated physical
	2635	+ * Determine if the TCM fabric module has already allocated physical
2574	2636	* memory, and is directly calling transport_generic_map_mem_to_cmd()
2575	2637	* beforehand.
2576	2638	*/
..	..	@@ -2630,7 +2692,8 @@
2630	2692	* Determine if frontend context caller is requesting the stopping of
2631	2693	* this command for frontend exceptions.
2632	2694	*/
2633		- if (cmd->transport_state & CMD_T_STOP) {
	2695	+ if (cmd->transport_state & CMD_T_STOP &&
	2696	+ !cmd->se_tfo->write_pending_must_be_called) {
2634	2697	pr_debug("%s:%d CMD_T_STOP for ITT: 0x%08llx\n",
2635	2698	__func__, __LINE__, cmd->tag);
2636	2699
..	..	@@ -2694,13 +2757,29 @@
2694	2757	}
2695	2758
2696	2759	/*
	2760	+ * Call target_put_sess_cmd() and wait until target_release_cmd_kref(@cmd) has
	2761	+ * finished.
	2762	+ */
	2763	+void target_put_cmd_and_wait(struct se_cmd *cmd)
	2764	+{
	2765	+ DECLARE_COMPLETION_ONSTACK(compl);
	2766	+
	2767	+ WARN_ON_ONCE(cmd->abrt_compl);
	2768	+ cmd->abrt_compl = &compl;
	2769	+ target_put_sess_cmd(cmd);
	2770	+ wait_for_completion(&compl);
	2771	+}
	2772	+
	2773	+/*
2697	2774	* This function is called by frontend drivers after processing of a command
2698	2775	* has finished.
2699	2776	*
2700		- * The protocol for ensuring that either the regular flow or the TMF
2701		- * code drops one reference is as follows:
	2777	+ * The protocol for ensuring that either the regular frontend command
	2778	+ * processing flow or target_handle_abort() code drops one reference is as
	2779	+ * follows:
2702	2780	* - Calling .queue_data_in(), .queue_status() or queue_tm_rsp() will cause
2703		- * the frontend driver to drop one reference, synchronously or asynchronously.
	2781	+ * the frontend driver to call this function synchronously or asynchronously.
	2782	+ * That will cause one reference to be dropped.
2704	2783	* - During regular command processing the target core sets CMD_T_COMPLETE
2705	2784	* before invoking one of the .queue_*() functions.
2706	2785	* - The code that aborts commands skips commands and TMFs for which
..	..	@@ -2712,7 +2791,7 @@
2712	2791	* - For aborted commands for which CMD_T_TAS has been set .queue_status() will
2713	2792	* be called and will drop a reference.
2714	2793	* - For aborted commands for which CMD_T_TAS has not been set .aborted_task()
2715		- * will be called. transport_cmd_finish_abort() will drop the final reference.
	2794	+ * will be called. target_handle_abort() will drop the final reference.
2716	2795	*/
2717	2796	int transport_generic_free_cmd(struct se_cmd *cmd, int wait_for_tasks)
2718	2797	{
..	..	@@ -2736,9 +2815,8 @@
2736	2815	transport_lun_remove_cmd(cmd);
2737	2816	}
2738	2817	if (aborted)
2739		- cmd->compl = &compl;
2740		- if (!aborted \|\| tas)
2741		- ret = target_put_sess_cmd(cmd);
	2818	+ cmd->free_compl = &compl;
	2819	+ ret = target_put_sess_cmd(cmd);
2742	2820	if (aborted) {
2743	2821	pr_debug("Detected CMD_T_ABORTED for ITT: %llu\n", cmd->tag);
2744	2822	wait_for_completion(&compl);
..	..	@@ -2776,7 +2854,6 @@
2776	2854	ret = -ESHUTDOWN;
2777	2855	goto out;
2778	2856	}
2779		- se_cmd->transport_state \|= CMD_T_PRE_EXECUTE;
2780	2857	list_add_tail(&se_cmd->se_cmd_list, &se_sess->sess_cmd_list);
2781	2858	percpu_ref_get(&se_sess->cmd_count);
2782	2859	out:
..	..	@@ -2803,7 +2880,8 @@
2803	2880	{
2804	2881	struct se_cmd *se_cmd = container_of(kref, struct se_cmd, cmd_kref);
2805	2882	struct se_session *se_sess = se_cmd->se_sess;
2806		- struct completion *compl = se_cmd->compl;
	2883	+ struct completion *free_compl = se_cmd->free_compl;
	2884	+ struct completion *abrt_compl = se_cmd->abrt_compl;
2807	2885	unsigned long flags;
2808	2886
2809	2887	if (se_sess) {
..	..	@@ -2814,8 +2892,10 @@
2814	2892
2815	2893	target_free_cmd_mem(se_cmd);
2816	2894	se_cmd->se_tfo->release_cmd(se_cmd);
2817		- if (compl)
2818		- complete(compl);
	2895	+ if (free_compl)
	2896	+ complete(free_compl);
	2897	+ if (abrt_compl)
	2898	+ complete(abrt_compl);
2819	2899
2820	2900	percpu_ref_put(&se_sess->cmd_count);
2821	2901	}
..	..	@@ -2906,6 +2986,7 @@
2906	2986	case TMR_LUN_RESET: return "LUN_RESET";
2907	2987	case TMR_TARGET_WARM_RESET: return "TARGET_WARM_RESET";
2908	2988	case TMR_TARGET_COLD_RESET: return "TARGET_COLD_RESET";
	2989	+ case TMR_LUN_RESET_PRO: return "LUN_RESET_PRO";
2909	2990	case TMR_UNKNOWN: break;
2910	2991	}
2911	2992	return "(?)";
..	..	@@ -2973,38 +3054,13 @@
2973	3054	}
2974	3055	EXPORT_SYMBOL(target_wait_for_sess_cmds);
2975	3056
2976		-static void target_lun_confirm(struct percpu_ref *ref)
2977		-{
2978		- struct se_lun *lun = container_of(ref, struct se_lun, lun_ref);
2979		-
2980		- complete(&lun->lun_ref_comp);
2981		-}
2982		-
	3057	+/*
	3058	+ * Prevent that new percpu_ref_tryget_live() calls succeed and wait until
	3059	+ * all references to the LUN have been released. Called during LUN shutdown.
	3060	+ */
2983	3061	void transport_clear_lun_ref(struct se_lun *lun)
2984	3062	{
2985		- /*
2986		- * Mark the percpu-ref as DEAD, switch to atomic_t mode, drop
2987		- * the initial reference and schedule confirm kill to be
2988		- * executed after one full RCU grace period has completed.
2989		- */
2990		- percpu_ref_kill_and_confirm(&lun->lun_ref, target_lun_confirm);
2991		- /*
2992		- * The first completion waits for percpu_ref_switch_to_atomic_rcu()
2993		- * to call target_lun_confirm after lun->lun_ref has been marked
2994		- * as __PERCPU_REF_DEAD on all CPUs, and switches to atomic_t
2995		- * mode so that percpu_ref_tryget_live() lookup of lun->lun_ref
2996		- * fails for all new incoming I/O.
2997		- */
2998		- wait_for_completion(&lun->lun_ref_comp);
2999		- /*
3000		- * The second completion waits for percpu_ref_put_many() to
3001		- * invoke ->release() after lun->lun_ref has switched to
3002		- * atomic_t mode, and lun->lun_ref.count has reached zero.
3003		- *
3004		- * At this point all target-core lun->lun_ref references have
3005		- * been dropped via transport_lun_remove_cmd(), and it's safe
3006		- * to proceed with the remaining LUN shutdown.
3007		- */
	3063	+ percpu_ref_kill(&lun->lun_ref);
3008	3064	wait_for_completion(&lun->lun_shutdown_comp);
3009	3065	}
3010	3066
..	..	@@ -3286,6 +3342,8 @@
3286	3342	{
3287	3343	unsigned long flags;
3288	3344
	3345	+ WARN_ON_ONCE(cmd->se_cmd_flags & SCF_SCSI_TMR_CDB);
	3346	+
3289	3347	spin_lock_irqsave(&cmd->t_state_lock, flags);
3290	3348	if (cmd->se_cmd_flags & SCF_SENT_CHECK_CONDITION) {
3291	3349	spin_unlock_irqrestore(&cmd->t_state_lock, flags);
..	..	@@ -3302,114 +3360,31 @@
3302	3360	}
3303	3361	EXPORT_SYMBOL(transport_send_check_condition_and_sense);
3304	3362
3305		-static int __transport_check_aborted_status(struct se_cmd *cmd, int send_status)
3306		- __releases(&cmd->t_state_lock)
3307		- __acquires(&cmd->t_state_lock)
	3363	+/**
	3364	+ * target_send_busy - Send SCSI BUSY status back to the initiator
	3365	+ * @cmd: SCSI command for which to send a BUSY reply.
	3366	+ *
	3367	+ * Note: Only call this function if target_submit_cmd*() failed.
	3368	+ */
	3369	+int target_send_busy(struct se_cmd *cmd)
3308	3370	{
3309		- int ret;
	3371	+ WARN_ON_ONCE(cmd->se_cmd_flags & SCF_SCSI_TMR_CDB);
3310	3372
3311		- assert_spin_locked(&cmd->t_state_lock);
3312		- WARN_ON_ONCE(!irqs_disabled());
3313		-
3314		- if (!(cmd->transport_state & CMD_T_ABORTED))
3315		- return 0;
3316		- /*
3317		- * If cmd has been aborted but either no status is to be sent or it has
3318		- * already been sent, just return
3319		- */
3320		- if (!send_status \|\| !(cmd->se_cmd_flags & SCF_SEND_DELAYED_TAS)) {
3321		- if (send_status)
3322		- cmd->se_cmd_flags \|= SCF_SEND_DELAYED_TAS;
3323		- return 1;
3324		- }
3325		-
3326		- pr_debug("Sending delayed SAM_STAT_TASK_ABORTED status for CDB:"
3327		- " 0x%02x ITT: 0x%08llx\n", cmd->t_task_cdb[0], cmd->tag);
3328		-
3329		- cmd->se_cmd_flags &= ~SCF_SEND_DELAYED_TAS;
3330		- cmd->scsi_status = SAM_STAT_TASK_ABORTED;
	3373	+ cmd->scsi_status = SAM_STAT_BUSY;
3331	3374	trace_target_cmd_complete(cmd);
3332		-
3333		- spin_unlock_irq(&cmd->t_state_lock);
3334		- ret = cmd->se_tfo->queue_status(cmd);
3335		- if (ret)
3336		- transport_handle_queue_full(cmd, cmd->se_dev, ret, false);
3337		- spin_lock_irq(&cmd->t_state_lock);
3338		-
3339		- return 1;
	3375	+ return cmd->se_tfo->queue_status(cmd);
3340	3376	}
3341		-
3342		-int transport_check_aborted_status(struct se_cmd *cmd, int send_status)
3343		-{
3344		- int ret;
3345		-
3346		- spin_lock_irq(&cmd->t_state_lock);
3347		- ret = __transport_check_aborted_status(cmd, send_status);
3348		- spin_unlock_irq(&cmd->t_state_lock);
3349		-
3350		- return ret;
3351		-}
3352		-EXPORT_SYMBOL(transport_check_aborted_status);
3353		-
3354		-void transport_send_task_abort(struct se_cmd *cmd)
3355		-{
3356		- unsigned long flags;
3357		- int ret;
3358		-
3359		- spin_lock_irqsave(&cmd->t_state_lock, flags);
3360		- if (cmd->se_cmd_flags & (SCF_SENT_CHECK_CONDITION)) {
3361		- spin_unlock_irqrestore(&cmd->t_state_lock, flags);
3362		- return;
3363		- }
3364		- spin_unlock_irqrestore(&cmd->t_state_lock, flags);
3365		-
3366		- /*
3367		- * If there are still expected incoming fabric WRITEs, we wait
3368		- * until until they have completed before sending a TASK_ABORTED
3369		- * response. This response with TASK_ABORTED status will be
3370		- * queued back to fabric module by transport_check_aborted_status().
3371		- */
3372		- if (cmd->data_direction == DMA_TO_DEVICE) {
3373		- if (cmd->se_tfo->write_pending_status(cmd) != 0) {
3374		- spin_lock_irqsave(&cmd->t_state_lock, flags);
3375		- if (cmd->se_cmd_flags & SCF_SEND_DELAYED_TAS) {
3376		- spin_unlock_irqrestore(&cmd->t_state_lock, flags);
3377		- goto send_abort;
3378		- }
3379		- cmd->se_cmd_flags \|= SCF_SEND_DELAYED_TAS;
3380		- spin_unlock_irqrestore(&cmd->t_state_lock, flags);
3381		- return;
3382		- }
3383		- }
3384		-send_abort:
3385		- cmd->scsi_status = SAM_STAT_TASK_ABORTED;
3386		-
3387		- transport_lun_remove_cmd(cmd);
3388		-
3389		- pr_debug("Setting SAM_STAT_TASK_ABORTED status for CDB: 0x%02x, ITT: 0x%08llx\n",
3390		- cmd->t_task_cdb[0], cmd->tag);
3391		-
3392		- trace_target_cmd_complete(cmd);
3393		- ret = cmd->se_tfo->queue_status(cmd);
3394		- if (ret)
3395		- transport_handle_queue_full(cmd, cmd->se_dev, ret, false);
3396		-}
	3377	+EXPORT_SYMBOL(target_send_busy);
3397	3378
3398	3379	static void target_tmr_work(struct work_struct *work)
3399	3380	{
3400	3381	struct se_cmd *cmd = container_of(work, struct se_cmd, work);
3401	3382	struct se_device *dev = cmd->se_dev;
3402	3383	struct se_tmr_req *tmr = cmd->se_tmr_req;
3403		- unsigned long flags;
3404	3384	int ret;
3405	3385
3406		- spin_lock_irqsave(&cmd->t_state_lock, flags);
3407		- if (cmd->transport_state & CMD_T_ABORTED) {
3408		- tmr->response = TMR_FUNCTION_REJECTED;
3409		- spin_unlock_irqrestore(&cmd->t_state_lock, flags);
3410		- goto check_stop;
3411		- }
3412		- spin_unlock_irqrestore(&cmd->t_state_lock, flags);
	3386	+ if (cmd->transport_state & CMD_T_ABORTED)
	3387	+ goto aborted;
3413	3388
3414	3389	switch (tmr->function) {
3415	3390	case TMR_ABORT_TASK:
..	..	@@ -3443,18 +3418,17 @@
3443	3418	break;
3444	3419	}
3445	3420
3446		- spin_lock_irqsave(&cmd->t_state_lock, flags);
3447		- if (cmd->transport_state & CMD_T_ABORTED) {
3448		- spin_unlock_irqrestore(&cmd->t_state_lock, flags);
3449		- goto check_stop;
3450		- }
3451		- spin_unlock_irqrestore(&cmd->t_state_lock, flags);
	3421	+ if (cmd->transport_state & CMD_T_ABORTED)
	3422	+ goto aborted;
3452	3423
3453	3424	cmd->se_tfo->queue_tm_rsp(cmd);
3454	3425
3455		-check_stop:
3456	3426	transport_lun_remove_cmd(cmd);
3457	3427	transport_cmd_check_stop_to_fabric(cmd);
	3428	+ return;
	3429	+
	3430	+aborted:
	3431	+ target_handle_abort(cmd);
3458	3432	}
3459	3433
3460	3434	int transport_generic_handle_tmr(
..	..	@@ -3473,16 +3447,15 @@
3473	3447	spin_unlock_irqrestore(&cmd->t_state_lock, flags);
3474	3448
3475	3449	if (aborted) {
3476		- pr_warn_ratelimited("handle_tmr caught CMD_T_ABORTED TMR %d"
3477		- "ref_tag: %llu tag: %llu\n", cmd->se_tmr_req->function,
3478		- cmd->se_tmr_req->ref_task_tag, cmd->tag);
3479		- transport_lun_remove_cmd(cmd);
3480		- transport_cmd_check_stop_to_fabric(cmd);
	3450	+ pr_warn_ratelimited("handle_tmr caught CMD_T_ABORTED TMR %d ref_tag: %llu tag: %llu\n",
	3451	+ cmd->se_tmr_req->function,
	3452	+ cmd->se_tmr_req->ref_task_tag, cmd->tag);
	3453	+ target_handle_abort(cmd);
3481	3454	return 0;
3482	3455	}
3483	3456
3484	3457	INIT_WORK(&cmd->work, target_tmr_work);
3485		- queue_work(cmd->se_dev->tmr_wq, &cmd->work);
	3458	+ schedule_work(&cmd->work);
3486	3459	return 0;
3487	3460	}
3488	3461	EXPORT_SYMBOL(transport_generic_handle_tmr);