~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,10 +1,7 @@
	1	+// SPDX-License-Identifier: GPL-2.0-only
1	2	/*
2	3	* QLogic iSCSI Offload Driver
3	4	* Copyright (c) 2016 Cavium Inc.
4		- *
5		- * This software is available under the terms of the GNU General Public License
6		- * (GPL) Version 2, available from the file COPYING in the main directory of
7		- * this source tree.
8	5	*/
9	6
10	7	#include <linux/module.h>
..	..	@@ -31,6 +28,10 @@
31	28	#include "qedi_gbl.h"
32	29	#include "qedi_iscsi.h"
33	30
	31	+static uint qedi_qed_debug;
	32	+module_param(qedi_qed_debug, uint, 0644);
	33	+MODULE_PARM_DESC(qedi_qed_debug, " QED debug level 0 (default)");
	34	+
34	35	static uint qedi_fw_debug;
35	36	module_param(qedi_fw_debug, uint, 0644);
36	37	MODULE_PARM_DESC(qedi_fw_debug, " Firmware debug level 0(default) to 3");
..	..	@@ -44,6 +45,15 @@
44	45	MODULE_PARM_DESC(qedi_io_tracing,
45	46	" Enable logging of SCSI requests/completions into trace buffer. (default off).");
46	47
	48	+static uint qedi_ll2_buf_size = 0x400;
	49	+module_param(qedi_ll2_buf_size, uint, 0644);
	50	+MODULE_PARM_DESC(qedi_ll2_buf_size,
	51	+ "parameter to set ping packet size, default - 0x400, Jumbo packets - 0x2400.");
	52	+
	53	+static uint qedi_flags_override;
	54	+module_param(qedi_flags_override, uint, 0644);
	55	+MODULE_PARM_DESC(qedi_flags_override, "Disable/Enable MFW error flags bits action.");
	56	+
47	57	const struct qed_iscsi_ops *qedi_ops;
48	58	static struct scsi_transport_template *qedi_scsi_transport;
49	59	static struct pci_driver qedi_pci_driver;
..	..	@@ -56,6 +66,10 @@
56	66	static void qedi_reset_uio_rings(struct qedi_uio_dev *udev);
57	67	static void qedi_ll2_free_skbs(struct qedi_ctx *qedi);
58	68	static struct nvm_iscsi_block qedi_get_nvram_block(struct qedi_ctx qedi);
	69	+static void qedi_recovery_handler(struct work_struct *work);
	70	+static void qedi_schedule_hw_err_handler(void *dev,
	71	+ enum qed_hw_err_type err_type);
	72	+static int qedi_suspend(struct pci_dev *pdev, pm_message_t state);
59	73
60	74	static int qedi_iscsi_event_cb(void context, u8 fw_event_code, void fw_handle)
61	75	{
..	..	@@ -228,7 +242,7 @@
228	242	}
229	243
230	244	/* Allocating memory for Tx/Rx pkt buffer */
231		- udev->ll2_buf_size = TX_RX_RING * LL2_SINGLE_BUF_SIZE;
	245	+ udev->ll2_buf_size = TX_RX_RING * qedi_ll2_buf_size;
232	246	udev->ll2_buf_size = QEDI_PAGE_ALIGN(udev->ll2_buf_size);
233	247	udev->ll2_buf = (void *)__get_free_pages(GFP_KERNEL \| __GFP_COMP \|
234	248	__GFP_ZERO, 2);
..	..	@@ -283,7 +297,7 @@
283	297	qedi->udev = udev;
284	298
285	299	udev->tx_pkt = udev->ll2_buf;
286		- udev->rx_pkt = udev->ll2_buf + LL2_SINGLE_BUF_SIZE;
	300	+ udev->rx_pkt = udev->ll2_buf + qedi_ll2_buf_size;
287	301	return 0;
288	302
289	303	err_uctrl:
..	..	@@ -644,8 +658,7 @@
644	658	qedi->max_active_conns = ISCSI_MAX_SESS_PER_HBA;
645	659	qedi->max_sqes = QEDI_SQ_SIZE;
646	660
647		- if (shost_use_blk_mq(shost))
648		- shost->nr_hw_queues = MIN_NUM_CPUS_MSIX(qedi);
	661	+ shost->nr_hw_queues = MIN_NUM_CPUS_MSIX(qedi);
649	662
650	663	pci_set_drvdata(pdev, qedi);
651	664
..	..	@@ -656,10 +669,8 @@
656	669	static int qedi_ll2_rx(void cookie, struct sk_buff skb, u32 arg1, u32 arg2)
657	670	{
658	671	struct qedi_ctx qedi = (struct qedi_ctx )cookie;
659		- struct qedi_uio_dev *udev;
660		- struct qedi_uio_ctrl *uctrl;
661	672	struct skb_work_list *work;
662		- u32 prod;
	673	+ struct ethhdr *eh;
663	674
664	675	if (!qedi) {
665	676	QEDI_ERR(NULL, "qedi is NULL\n");
..	..	@@ -673,8 +684,28 @@
673	684	return 0;
674	685	}
675	686
676		- udev = qedi->udev;
677		- uctrl = udev->uctrl;
	687	+ eh = (struct ethhdr *)skb->data;
	688	+ /* Undo VLAN encapsulation */
	689	+ if (eh->h_proto == htons(ETH_P_8021Q)) {
	690	+ memmove((u8 )eh + VLAN_HLEN, eh, ETH_ALEN 2);
	691	+ eh = (struct ethhdr *)skb_pull(skb, VLAN_HLEN);
	692	+ skb_reset_mac_header(skb);
	693	+ }
	694	+
	695	+ /* Filter out non FIP/FCoE frames here to free them faster */
	696	+ if (eh->h_proto != htons(ETH_P_ARP) &&
	697	+ eh->h_proto != htons(ETH_P_IP) &&
	698	+ eh->h_proto != htons(ETH_P_IPV6)) {
	699	+ QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_LL2,
	700	+ "Dropping frame ethertype [0x%x] len [0x%x].\n",
	701	+ eh->h_proto, skb->len);
	702	+ kfree_skb(skb);
	703	+ return 0;
	704	+ }
	705	+
	706	+ QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_LL2,
	707	+ "Allowed frame ethertype [0x%x] len [0x%x].\n",
	708	+ eh->h_proto, skb->len);
678	709
679	710	work = kzalloc(sizeof(*work), GFP_ATOMIC);
680	711	if (!work) {
..	..	@@ -695,17 +726,10 @@
695	726
696	727	spin_lock_bh(&qedi->ll2_lock);
697	728	list_add_tail(&work->list, &qedi->ll2_skb_list);
698		-
699		- ++uctrl->hw_rx_prod_cnt;
700		- prod = (uctrl->hw_rx_prod + 1) % RX_RING;
701		- if (prod != uctrl->host_rx_cons) {
702		- uctrl->hw_rx_prod = prod;
703		- spin_unlock_bh(&qedi->ll2_lock);
704		- wake_up_process(qedi->ll2_recv_thread);
705		- return 0;
706		- }
707		-
708	729	spin_unlock_bh(&qedi->ll2_lock);
	730	+
	731	+ wake_up_process(qedi->ll2_recv_thread);
	732	+
709	733	return 0;
710	734	}
711	735
..	..	@@ -720,6 +744,7 @@
720	744	u32 rx_bd_prod;
721	745	void *pkt;
722	746	int len = 0;
	747	+ u32 prod;
723	748
724	749	if (!qedi) {
725	750	QEDI_ERR(NULL, "qedi is NULL\n");
..	..	@@ -728,12 +753,16 @@
728	753
729	754	udev = qedi->udev;
730	755	uctrl = udev->uctrl;
731		- pkt = udev->rx_pkt + (uctrl->hw_rx_prod * LL2_SINGLE_BUF_SIZE);
732		- len = min_t(u32, skb->len, (u32)LL2_SINGLE_BUF_SIZE);
	756	+
	757	+ ++uctrl->hw_rx_prod_cnt;
	758	+ prod = (uctrl->hw_rx_prod + 1) % RX_RING;
	759	+
	760	+ pkt = udev->rx_pkt + (prod * qedi_ll2_buf_size);
	761	+ len = min_t(u32, skb->len, (u32)qedi_ll2_buf_size);
733	762	memcpy(pkt, skb->data, len);
734	763
735	764	memset(&rxbd, 0, sizeof(rxbd));
736		- rxbd.rx_pkt_index = uctrl->hw_rx_prod;
	765	+ rxbd.rx_pkt_index = prod;
737	766	rxbd.rx_pkt_len = len;
738	767	rxbd.vlan_id = vlan_id;
739	768
..	..	@@ -743,6 +772,16 @@
743	772	p_rxbd += rx_bd_prod;
744	773
745	774	memcpy(p_rxbd, &rxbd, sizeof(rxbd));
	775	+
	776	+ QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_LL2,
	777	+ "hw_rx_prod [%d] prod [%d] hw_rx_bd_prod [%d] rx_pkt_idx [%d] rx_len [%d].\n",
	778	+ uctrl->hw_rx_prod, prod, uctrl->hw_rx_bd_prod,
	779	+ rxbd.rx_pkt_index, rxbd.rx_pkt_len);
	780	+ QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_LL2,
	781	+ "host_rx_cons [%d] hw_rx_bd_cons [%d].\n",
	782	+ uctrl->host_rx_cons, uctrl->host_rx_bd_cons);
	783	+
	784	+ uctrl->hw_rx_prod = prod;
746	785
747	786	/* notify the iscsiuio about new packet */
748	787	uio_event_notify(&udev->qedi_uinfo);
..	..	@@ -757,8 +796,7 @@
757	796	spin_lock_bh(&qedi->ll2_lock);
758	797	list_for_each_entry_safe(work, work_tmp, &qedi->ll2_skb_list, list) {
759	798	list_del(&work->list);
760		- if (work->skb)
761		- kfree_skb(work->skb);
	799	+ kfree_skb(work->skb);
762	800	kfree(work);
763	801	}
764	802	spin_unlock_bh(&qedi->ll2_lock);
..	..	@@ -796,7 +834,7 @@
796	834	int rval = 0;
797	835
798	836
799		- num_sq_pages = (MAX_OUSTANDING_TASKS_PER_CON * 8) / PAGE_SIZE;
	837	+ num_sq_pages = (MAX_OUTSTANDING_TASKS_PER_CON * 8) / QEDI_PAGE_SIZE;
800	838
801	839	qedi->num_queues = MIN_NUM_CPUS_MSIX(qedi);
802	840
..	..	@@ -806,11 +844,11 @@
806	844	memset(&qedi->pf_params.iscsi_pf_params, 0,
807	845	sizeof(qedi->pf_params.iscsi_pf_params));
808	846
809		- qedi->p_cpuq = pci_alloc_consistent(qedi->pdev,
	847	+ qedi->p_cpuq = dma_alloc_coherent(&qedi->pdev->dev,
810	848	qedi->num_queues * sizeof(struct qedi_glbl_q_params),
811		- &qedi->hw_p_cpuq);
	849	+ &qedi->hw_p_cpuq, GFP_KERNEL);
812	850	if (!qedi->p_cpuq) {
813		- QEDI_ERR(&qedi->dbg_ctx, "pci_alloc_consistent fail\n");
	851	+ QEDI_ERR(&qedi->dbg_ctx, "dma_alloc_coherent fail\n");
814	852	rval = -1;
815	853	goto err_alloc_mem;
816	854	}
..	..	@@ -834,7 +872,7 @@
834	872	qedi->pf_params.iscsi_pf_params.max_fin_rt = 2;
835	873
836	874	for (log_page_size = 0 ; log_page_size < 32 ; log_page_size++) {
837		- if ((1 << log_page_size) == PAGE_SIZE)
	875	+ if ((1 << log_page_size) == QEDI_PAGE_SIZE)
838	876	break;
839	877	}
840	878	qedi->pf_params.iscsi_pf_params.log_page_size = log_page_size;
..	..	@@ -871,7 +909,7 @@
871	909
872	910	if (qedi->p_cpuq) {
873	911	size = qedi->num_queues * sizeof(struct qedi_glbl_q_params);
874		- pci_free_consistent(qedi->pdev, size, qedi->p_cpuq,
	912	+ dma_free_coherent(&qedi->pdev->dev, size, qedi->p_cpuq,
875	913	qedi->hw_p_cpuq);
876	914	}
877	915
..	..	@@ -888,7 +926,7 @@
888	926	ipv6_en = !!(block->generic.ctrl_flags &
889	927	NVM_ISCSI_CFG_GEN_IPV6_ENABLED);
890	928
891		- snprintf(tgt->iscsi_name, sizeof(tgt->iscsi_name), "%s\n",
	929	+ snprintf(tgt->iscsi_name, sizeof(tgt->iscsi_name), "%s",
892	930	block->target[index].target_name.byte);
893	931
894	932	tgt->ipv6_en = ipv6_en;
..	..	@@ -1081,6 +1119,65 @@
1081	1119	return;
1082	1120	}
1083	1121
	1122	+void qedi_schedule_hw_err_handler(void *dev,
	1123	+ enum qed_hw_err_type err_type)
	1124	+{
	1125	+ struct qedi_ctx qedi = (struct qedi_ctx )dev;
	1126	+ unsigned long override_flags = qedi_flags_override;
	1127	+
	1128	+ if (override_flags && test_bit(QEDI_ERR_OVERRIDE_EN, &override_flags))
	1129	+ qedi->qedi_err_flags = qedi_flags_override;
	1130	+
	1131	+ QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
	1132	+ "HW error handler scheduled, err=%d err_flags=0x%x\n",
	1133	+ err_type, qedi->qedi_err_flags);
	1134	+
	1135	+ switch (err_type) {
	1136	+ case QED_HW_ERR_FAN_FAIL:
	1137	+ schedule_delayed_work(&qedi->board_disable_work, 0);
	1138	+ break;
	1139	+ case QED_HW_ERR_MFW_RESP_FAIL:
	1140	+ case QED_HW_ERR_HW_ATTN:
	1141	+ case QED_HW_ERR_DMAE_FAIL:
	1142	+ case QED_HW_ERR_RAMROD_FAIL:
	1143	+ case QED_HW_ERR_FW_ASSERT:
	1144	+ /* Prevent HW attentions from being reasserted */
	1145	+ if (test_bit(QEDI_ERR_ATTN_CLR_EN, &qedi->qedi_err_flags))
	1146	+ qedi_ops->common->attn_clr_enable(qedi->cdev, true);
	1147	+
	1148	+ if (err_type == QED_HW_ERR_RAMROD_FAIL &&
	1149	+ test_bit(QEDI_ERR_IS_RECOVERABLE, &qedi->qedi_err_flags))
	1150	+ qedi_ops->common->recovery_process(qedi->cdev);
	1151	+
	1152	+ break;
	1153	+ default:
	1154	+ break;
	1155	+ }
	1156	+}
	1157	+
	1158	+static void qedi_schedule_recovery_handler(void *dev)
	1159	+{
	1160	+ struct qedi_ctx *qedi = dev;
	1161	+
	1162	+ QEDI_ERR(&qedi->dbg_ctx, "Recovery handler scheduled.\n");
	1163	+
	1164	+ if (test_and_set_bit(QEDI_IN_RECOVERY, &qedi->flags))
	1165	+ return;
	1166	+
	1167	+ atomic_set(&qedi->link_state, QEDI_LINK_DOWN);
	1168	+
	1169	+ schedule_delayed_work(&qedi->recovery_work, 0);
	1170	+}
	1171	+
	1172	+static void qedi_set_conn_recovery(struct iscsi_cls_session *cls_session)
	1173	+{
	1174	+ struct iscsi_session *session = cls_session->dd_data;
	1175	+ struct iscsi_conn *conn = session->leadconn;
	1176	+ struct qedi_conn *qedi_conn = conn->dd_data;
	1177	+
	1178	+ qedi_start_conn_recovery(qedi_conn->qedi, qedi_conn);
	1179	+}
	1180	+
1084	1181	static void qedi_link_update(void dev, struct qed_link_output link)
1085	1182	{
1086	1183	struct qedi_ctx qedi = (struct qedi_ctx )dev;
..	..	@@ -1092,12 +1189,15 @@
1092	1189	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
1093	1190	"Link Down event.\n");
1094	1191	atomic_set(&qedi->link_state, QEDI_LINK_DOWN);
	1192	+ iscsi_host_for_each_session(qedi->shost, qedi_set_conn_recovery);
1095	1193	}
1096	1194	}
1097	1195
1098	1196	static struct qed_iscsi_cb_ops qedi_cb_ops = {
1099	1197	{
1100	1198	.link_update = qedi_link_update,
	1199	+ .schedule_recovery_handler = qedi_schedule_recovery_handler,
	1200	+ .schedule_hw_err_handler = qedi_schedule_hw_err_handler,
1101	1201	.get_protocol_tlv_data = qedi_get_protocol_tlv_data,
1102	1202	.get_generic_tlv_data = qedi_get_generic_tlv_data,
1103	1203	}
..	..	@@ -1108,7 +1208,6 @@
1108	1208	{
1109	1209	struct qedi_work *qedi_work;
1110	1210	struct qedi_conn *q_conn;
1111		- struct iscsi_conn *conn;
1112	1211	struct qedi_cmd *qedi_cmd;
1113	1212	u32 iscsi_cid;
1114	1213	int rc = 0;
..	..	@@ -1121,7 +1220,6 @@
1121	1220	iscsi_cid);
1122	1221	return -1;
1123	1222	}
1124		- conn = q_conn->cls_conn->dd_data;
1125	1223
1126	1224	switch (cqe->cqe_common.cqe_type) {
1127	1225	case ISCSI_CQE_TYPE_SOLICITED:
..	..	@@ -1254,13 +1352,13 @@
1254	1352	"process already running\n");
1255	1353	}
1256	1354
1257		- if (qedi_fp_has_work(fp) == 0)
	1355	+ if (!qedi_fp_has_work(fp))
1258	1356	qed_sb_update_sb_idx(fp->sb_info);
1259	1357
1260	1358	/* Check for more work */
1261	1359	rmb();
1262	1360
1263		- if (qedi_fp_has_work(fp) == 0)
	1361	+ if (!qedi_fp_has_work(fp))
1264	1362	qed_sb_ack(fp->sb_info, IGU_INT_ENABLE, 1);
1265	1363	else
1266	1364	goto process_again;
..	..	@@ -1281,13 +1379,20 @@
1281	1379	static void qedi_sync_free_irqs(struct qedi_ctx *qedi)
1282	1380	{
1283	1381	int i;
	1382	+ u16 idx;
1284	1383
1285	1384	if (qedi->int_info.msix_cnt) {
1286	1385	for (i = 0; i < qedi->int_info.used_cnt; i++) {
1287		- synchronize_irq(qedi->int_info.msix[i].vector);
1288		- irq_set_affinity_hint(qedi->int_info.msix[i].vector,
	1386	+ idx = i * qedi->dev_info.common.num_hwfns +
	1387	+ qedi_ops->common->get_affin_hwfn_idx(qedi->cdev);
	1388	+
	1389	+ QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
	1390	+ "Freeing IRQ #%d vector_idx=%d.\n", i, idx);
	1391	+
	1392	+ synchronize_irq(qedi->int_info.msix[idx].vector);
	1393	+ irq_set_affinity_hint(qedi->int_info.msix[idx].vector,
1289	1394	NULL);
1290		- free_irq(qedi->int_info.msix[i].vector,
	1395	+ free_irq(qedi->int_info.msix[idx].vector,
1291	1396	&qedi->fp_array[i]);
1292	1397	}
1293	1398	} else {
..	..	@@ -1302,20 +1407,28 @@
1302	1407	static int qedi_request_msix_irq(struct qedi_ctx *qedi)
1303	1408	{
1304	1409	int i, rc, cpu;
	1410	+ u16 idx;
1305	1411
1306	1412	cpu = cpumask_first(cpu_online_mask);
1307		- for (i = 0; i < MIN_NUM_CPUS_MSIX(qedi); i++) {
1308		- rc = request_irq(qedi->int_info.msix[i].vector,
	1413	+ for (i = 0; i < qedi->msix_count; i++) {
	1414	+ idx = i * qedi->dev_info.common.num_hwfns +
	1415	+ qedi_ops->common->get_affin_hwfn_idx(qedi->cdev);
	1416	+
	1417	+ QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
	1418	+ "dev_info: num_hwfns=%d affin_hwfn_idx=%d.\n",
	1419	+ qedi->dev_info.common.num_hwfns,
	1420	+ qedi_ops->common->get_affin_hwfn_idx(qedi->cdev));
	1421	+
	1422	+ rc = request_irq(qedi->int_info.msix[idx].vector,
1309	1423	qedi_msix_handler, 0, "qedi",
1310	1424	&qedi->fp_array[i]);
1311		-
1312	1425	if (rc) {
1313	1426	QEDI_WARN(&qedi->dbg_ctx, "request_irq failed.\n");
1314	1427	qedi_sync_free_irqs(qedi);
1315	1428	return rc;
1316	1429	}
1317	1430	qedi->int_info.used_cnt++;
1318		- rc = irq_set_affinity_hint(qedi->int_info.msix[i].vector,
	1431	+ rc = irq_set_affinity_hint(qedi->int_info.msix[idx].vector,
1319	1432	get_cpu_mask(cpu));
1320	1433	cpu = cpumask_next(cpu, cpu_online_mask);
1321	1434	}
..	..	@@ -1327,7 +1440,12 @@
1327	1440	{
1328	1441	int rc = 0;
1329	1442
1330		- rc = qedi_ops->common->set_fp_int(qedi->cdev, num_online_cpus());
	1443	+ rc = qedi_ops->common->set_fp_int(qedi->cdev, qedi->num_queues);
	1444	+ if (rc < 0)
	1445	+ goto exit_setup_int;
	1446	+
	1447	+ qedi->msix_count = rc;
	1448	+
1331	1449	rc = qedi_ops->common->get_fp_int(qedi->cdev, &qedi->int_info);
1332	1450	if (rc)
1333	1451	goto exit_setup_int;
..	..	@@ -1360,12 +1478,9 @@
1360	1478
1361	1479	static int qedi_alloc_nvm_iscsi_cfg(struct qedi_ctx *qedi)
1362	1480	{
1363		- struct qedi_nvm_iscsi_image nvm_image;
1364		-
1365		- qedi->iscsi_image = dma_zalloc_coherent(&qedi->pdev->dev,
1366		- sizeof(nvm_image),
1367		- &qedi->nvm_buf_dma,
1368		- GFP_KERNEL);
	1481	+ qedi->iscsi_image = dma_alloc_coherent(&qedi->pdev->dev,
	1482	+ sizeof(struct qedi_nvm_iscsi_image),
	1483	+ &qedi->nvm_buf_dma, GFP_KERNEL);
1369	1484	if (!qedi->iscsi_image) {
1370	1485	QEDI_ERR(&qedi->dbg_ctx, "Could not allocate NVM BUF.\n");
1371	1486	return -ENOMEM;
..	..	@@ -1382,7 +1497,7 @@
1382	1497	int i;
1383	1498
1384	1499	if (qedi->bdq_pbl_list)
1385		- dma_free_coherent(&qedi->pdev->dev, PAGE_SIZE,
	1500	+ dma_free_coherent(&qedi->pdev->dev, QEDI_PAGE_SIZE,
1386	1501	qedi->bdq_pbl_list, qedi->bdq_pbl_list_dma);
1387	1502
1388	1503	if (qedi->bdq_pbl)
..	..	@@ -1443,7 +1558,7 @@
1443	1558
1444	1559	/* Alloc dma memory for BDQ page buffer list */
1445	1560	qedi->bdq_pbl_mem_size = QEDI_BDQ_NUM * sizeof(struct scsi_bd);
1446		- qedi->bdq_pbl_mem_size = ALIGN(qedi->bdq_pbl_mem_size, PAGE_SIZE);
	1561	+ qedi->bdq_pbl_mem_size = ALIGN(qedi->bdq_pbl_mem_size, QEDI_PAGE_SIZE);
1447	1562	qedi->rq_num_entries = qedi->bdq_pbl_mem_size / sizeof(struct scsi_bd);
1448	1563
1449	1564	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_CONN, "rq_num_entries = %d.\n",
..	..	@@ -1478,9 +1593,10 @@
1478	1593	}
1479	1594
1480	1595	/* Allocate list of PBL pages */
1481		- qedi->bdq_pbl_list = dma_zalloc_coherent(&qedi->pdev->dev, PAGE_SIZE,
1482		- &qedi->bdq_pbl_list_dma,
1483		- GFP_KERNEL);
	1596	+ qedi->bdq_pbl_list = dma_alloc_coherent(&qedi->pdev->dev,
	1597	+ QEDI_PAGE_SIZE,
	1598	+ &qedi->bdq_pbl_list_dma,
	1599	+ GFP_KERNEL);
1484	1600	if (!qedi->bdq_pbl_list) {
1485	1601	QEDI_ERR(&qedi->dbg_ctx,
1486	1602	"Could not allocate list of PBL pages.\n");
..	..	@@ -1491,13 +1607,14 @@
1491	1607	* Now populate PBL list with pages that contain pointers to the
1492	1608	* individual buffers.
1493	1609	*/
1494		- qedi->bdq_pbl_list_num_entries = qedi->bdq_pbl_mem_size / PAGE_SIZE;
	1610	+ qedi->bdq_pbl_list_num_entries = qedi->bdq_pbl_mem_size /
	1611	+ QEDI_PAGE_SIZE;
1495	1612	list = (u64 *)qedi->bdq_pbl_list;
1496	1613	page = qedi->bdq_pbl_list_dma;
1497	1614	for (i = 0; i < qedi->bdq_pbl_list_num_entries; i++) {
1498	1615	*list = qedi->bdq_pbl_dma;
1499	1616	list++;
1500		- page += PAGE_SIZE;
	1617	+ page += QEDI_PAGE_SIZE;
1501	1618	}
1502	1619
1503	1620	return 0;
..	..	@@ -1576,10 +1693,10 @@
1576	1693	(qedi->global_queues[i]->cq_pbl_size +
1577	1694	(QEDI_PAGE_SIZE - 1));
1578	1695
1579		- qedi->global_queues[i]->cq = dma_zalloc_coherent(&qedi->pdev->dev,
1580		- qedi->global_queues[i]->cq_mem_size,
1581		- &qedi->global_queues[i]->cq_dma,
1582		- GFP_KERNEL);
	1696	+ qedi->global_queues[i]->cq = dma_alloc_coherent(&qedi->pdev->dev,
	1697	+ qedi->global_queues[i]->cq_mem_size,
	1698	+ &qedi->global_queues[i]->cq_dma,
	1699	+ GFP_KERNEL);
1583	1700
1584	1701	if (!qedi->global_queues[i]->cq) {
1585	1702	QEDI_WARN(&qedi->dbg_ctx,
..	..	@@ -1587,10 +1704,10 @@
1587	1704	status = -ENOMEM;
1588	1705	goto mem_alloc_failure;
1589	1706	}
1590		- qedi->global_queues[i]->cq_pbl = dma_zalloc_coherent(&qedi->pdev->dev,
1591		- qedi->global_queues[i]->cq_pbl_size,
1592		- &qedi->global_queues[i]->cq_pbl_dma,
1593		- GFP_KERNEL);
	1707	+ qedi->global_queues[i]->cq_pbl = dma_alloc_coherent(&qedi->pdev->dev,
	1708	+ qedi->global_queues[i]->cq_pbl_size,
	1709	+ &qedi->global_queues[i]->cq_pbl_dma,
	1710	+ GFP_KERNEL);
1594	1711
1595	1712	if (!qedi->global_queues[i]->cq_pbl) {
1596	1713	QEDI_WARN(&qedi->dbg_ctx,
..	..	@@ -1658,16 +1775,16 @@
1658	1775	ep->sq_pbl_size = (ep->sq_mem_size / QEDI_PAGE_SIZE) * sizeof(void *);
1659	1776	ep->sq_pbl_size = ep->sq_pbl_size + QEDI_PAGE_SIZE;
1660	1777
1661		- ep->sq = dma_zalloc_coherent(&qedi->pdev->dev, ep->sq_mem_size,
1662		- &ep->sq_dma, GFP_KERNEL);
	1778	+ ep->sq = dma_alloc_coherent(&qedi->pdev->dev, ep->sq_mem_size,
	1779	+ &ep->sq_dma, GFP_KERNEL);
1663	1780	if (!ep->sq) {
1664	1781	QEDI_WARN(&qedi->dbg_ctx,
1665	1782	"Could not allocate send queue.\n");
1666	1783	rval = -ENOMEM;
1667	1784	goto out;
1668	1785	}
1669		- ep->sq_pbl = dma_zalloc_coherent(&qedi->pdev->dev, ep->sq_pbl_size,
1670		- &ep->sq_pbl_dma, GFP_KERNEL);
	1786	+ ep->sq_pbl = dma_alloc_coherent(&qedi->pdev->dev, ep->sq_pbl_size,
	1787	+ &ep->sq_pbl_dma, GFP_KERNEL);
1671	1788	if (!ep->sq_pbl) {
1672	1789	QEDI_WARN(&qedi->dbg_ctx,
1673	1790	"Could not allocate send queue PBL.\n");
..	..	@@ -1864,8 +1981,9 @@
1864	1981	struct qedi_percpu_s *p = this_cpu_ptr(&qedi_percpu);
1865	1982	struct qedi_work work, tmp;
1866	1983	struct task_struct *thread;
	1984	+ unsigned long flags;
1867	1985
1868		- spin_lock_bh(&p->p_work_lock);
	1986	+ spin_lock_irqsave(&p->p_work_lock, flags);
1869	1987	thread = p->iothread;
1870	1988	p->iothread = NULL;
1871	1989
..	..	@@ -1876,7 +1994,7 @@
1876	1994	kfree(work);
1877	1995	}
1878	1996
1879		- spin_unlock_bh(&p->p_work_lock);
	1997	+ spin_unlock_irqrestore(&p->p_work_lock, flags);
1880	1998	if (thread)
1881	1999	kthread_stop(thread);
1882	2000	return 0;
..	..	@@ -1902,7 +2020,7 @@
1902	2020	qedi_ops->ll2->start(qedi->cdev, &params);
1903	2021	}
1904	2022
1905		-/**
	2023	+/*
1906	2024	* qedi_get_nvram_block: - Scan through the iSCSI NVRAM block (while accounting
1907	2025	* for gaps) for the matching absolute-pf-id of the QEDI device.
1908	2026	*/
..	..	@@ -2204,14 +2322,13 @@
2204	2322	static int qedi_get_boot_info(struct qedi_ctx *qedi)
2205	2323	{
2206	2324	int ret = 1;
2207		- struct qedi_nvm_iscsi_image nvm_image;
2208	2325
2209	2326	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
2210	2327	"Get NVM iSCSI CFG image\n");
2211	2328	ret = qedi_ops->common->nvm_get_image(qedi->cdev,
2212	2329	QED_NVM_IMAGE_ISCSI_CFG,
2213	2330	(char *)qedi->iscsi_image,
2214		- sizeof(nvm_image));
	2331	+ sizeof(struct qedi_nvm_iscsi_image));
2215	2332	if (ret)
2216	2333	QEDI_ERR(&qedi->dbg_ctx,
2217	2334	"Could not get NVM image. ret = %d\n", ret);
..	..	@@ -2279,21 +2396,47 @@
2279	2396	return -ENOMEM;
2280	2397	}
2281	2398
	2399	+static pci_ers_result_t qedi_io_error_detected(struct pci_dev *pdev,
	2400	+ pci_channel_state_t state)
	2401	+{
	2402	+ struct qedi_ctx *qedi = pci_get_drvdata(pdev);
	2403	+
	2404	+ QEDI_ERR(&qedi->dbg_ctx, "%s: PCI error detected [%d]\n",
	2405	+ __func__, state);
	2406	+
	2407	+ if (test_and_set_bit(QEDI_IN_RECOVERY, &qedi->flags)) {
	2408	+ QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
	2409	+ "Recovery already in progress.\n");
	2410	+ return PCI_ERS_RESULT_NONE;
	2411	+ }
	2412	+
	2413	+ qedi_ops->common->recovery_process(qedi->cdev);
	2414	+
	2415	+ return PCI_ERS_RESULT_CAN_RECOVER;
	2416	+}
	2417	+
2282	2418	static void __qedi_remove(struct pci_dev *pdev, int mode)
2283	2419	{
2284	2420	struct qedi_ctx *qedi = pci_get_drvdata(pdev);
2285	2421	int rval;
	2422	+ u16 retry = 10;
2286	2423
2287		- if (qedi->tmf_thread) {
2288		- flush_workqueue(qedi->tmf_thread);
2289		- destroy_workqueue(qedi->tmf_thread);
2290		- qedi->tmf_thread = NULL;
2291		- }
	2424	+ if (mode == QEDI_MODE_SHUTDOWN)
	2425	+ iscsi_host_for_each_session(qedi->shost,
	2426	+ qedi_clear_session_ctx);
2292	2427
2293		- if (qedi->offload_thread) {
2294		- flush_workqueue(qedi->offload_thread);
2295		- destroy_workqueue(qedi->offload_thread);
2296		- qedi->offload_thread = NULL;
	2428	+ if (mode == QEDI_MODE_NORMAL \|\| mode == QEDI_MODE_SHUTDOWN) {
	2429	+ if (qedi->tmf_thread) {
	2430	+ flush_workqueue(qedi->tmf_thread);
	2431	+ destroy_workqueue(qedi->tmf_thread);
	2432	+ qedi->tmf_thread = NULL;
	2433	+ }
	2434	+
	2435	+ if (qedi->offload_thread) {
	2436	+ flush_workqueue(qedi->offload_thread);
	2437	+ destroy_workqueue(qedi->offload_thread);
	2438	+ qedi->offload_thread = NULL;
	2439	+ }
2297	2440	}
2298	2441
2299	2442	#ifdef CONFIG_DEBUG_FS
..	..	@@ -2305,12 +2448,20 @@
2305	2448	qedi_sync_free_irqs(qedi);
2306	2449
2307	2450	if (!test_bit(QEDI_IN_OFFLINE, &qedi->flags)) {
2308		- qedi_ops->stop(qedi->cdev);
	2451	+ while (retry--) {
	2452	+ rval = qedi_ops->stop(qedi->cdev);
	2453	+ if (rval < 0)
	2454	+ msleep(1000);
	2455	+ else
	2456	+ break;
	2457	+ }
2309	2458	qedi_ops->ll2->stop(qedi->cdev);
2310	2459	}
2311	2460
2312		- if (mode == QEDI_MODE_NORMAL)
2313		- qedi_free_iscsi_pf_param(qedi);
	2461	+ cancel_delayed_work_sync(&qedi->recovery_work);
	2462	+ cancel_delayed_work_sync(&qedi->board_disable_work);
	2463	+
	2464	+ qedi_free_iscsi_pf_param(qedi);
2314	2465
2315	2466	rval = qedi_ops->common->update_drv_state(qedi->cdev, false);
2316	2467	if (rval)
..	..	@@ -2323,14 +2474,11 @@
2323	2474
2324	2475	qedi_destroy_fp(qedi);
2325	2476
2326		- if (mode == QEDI_MODE_NORMAL) {
	2477	+ if (mode == QEDI_MODE_NORMAL \|\| mode == QEDI_MODE_SHUTDOWN) {
2327	2478	qedi_release_cid_que(qedi);
2328	2479	qedi_cm_free_mem(qedi);
2329	2480	qedi_free_uio(qedi->udev);
2330	2481	qedi_free_itt(qedi);
2331		-
2332		- iscsi_host_remove(qedi->shost);
2333		- iscsi_host_free(qedi->shost);
2334	2482
2335	2483	if (qedi->ll2_recv_thread) {
2336	2484	kthread_stop(qedi->ll2_recv_thread);
..	..	@@ -2340,14 +2488,57 @@
2340	2488
2341	2489	if (qedi->boot_kset)
2342	2490	iscsi_boot_destroy_kset(qedi->boot_kset);
	2491	+
	2492	+ iscsi_host_remove(qedi->shost);
	2493	+ iscsi_host_free(qedi->shost);
2343	2494	}
	2495	+}
	2496	+
	2497	+static void qedi_board_disable_work(struct work_struct *work)
	2498	+{
	2499	+ struct qedi_ctx *qedi =
	2500	+ container_of(work, struct qedi_ctx,
	2501	+ board_disable_work.work);
	2502	+
	2503	+ QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
	2504	+ "Fan failure, Unloading firmware context.\n");
	2505	+
	2506	+ if (test_and_set_bit(QEDI_IN_SHUTDOWN, &qedi->flags))
	2507	+ return;
	2508	+
	2509	+ __qedi_remove(qedi->pdev, QEDI_MODE_SHUTDOWN);
	2510	+}
	2511	+
	2512	+static void qedi_shutdown(struct pci_dev *pdev)
	2513	+{
	2514	+ struct qedi_ctx *qedi = pci_get_drvdata(pdev);
	2515	+
	2516	+ QEDI_ERR(&qedi->dbg_ctx, "%s: Shutdown qedi\n", __func__);
	2517	+ if (test_and_set_bit(QEDI_IN_SHUTDOWN, &qedi->flags))
	2518	+ return;
	2519	+ __qedi_remove(pdev, QEDI_MODE_SHUTDOWN);
	2520	+}
	2521	+
	2522	+static int qedi_suspend(struct pci_dev *pdev, pm_message_t state)
	2523	+{
	2524	+ struct qedi_ctx *qedi;
	2525	+
	2526	+ if (!pdev) {
	2527	+ QEDI_ERR(NULL, "pdev is NULL.\n");
	2528	+ return -ENODEV;
	2529	+ }
	2530	+
	2531	+ qedi = pci_get_drvdata(pdev);
	2532	+
	2533	+ QEDI_ERR(&qedi->dbg_ctx, "%s: Device does not support suspend operation\n", __func__);
	2534	+
	2535	+ return -EPERM;
2344	2536	}
2345	2537
2346	2538	static int __qedi_probe(struct pci_dev *pdev, int mode)
2347	2539	{
2348	2540	struct qedi_ctx *qedi;
2349	2541	struct qed_ll2_params params;
2350		- u32 dp_module = 0;
2351	2542	u8 dp_level = 0;
2352	2543	bool is_vf = false;
2353	2544	char host_buf[16];
..	..	@@ -2356,7 +2547,7 @@
2356	2547	struct qed_probe_params qed_params;
2357	2548	void task_start, task_end;
2358	2549	int rc;
2359		- u16 tmp;
	2550	+ u16 retry = 10;
2360	2551
2361	2552	if (mode != QEDI_MODE_RECOVERY) {
2362	2553	qedi = qedi_host_alloc(pdev);
..	..	@@ -2368,32 +2559,48 @@
2368	2559	qedi = pci_get_drvdata(pdev);
2369	2560	}
2370	2561
	2562	+retry_probe:
	2563	+ if (mode == QEDI_MODE_RECOVERY)
	2564	+ msleep(2000);
	2565	+
2371	2566	memset(&qed_params, 0, sizeof(qed_params));
2372	2567	qed_params.protocol = QED_PROTOCOL_ISCSI;
2373		- qed_params.dp_module = dp_module;
	2568	+ qed_params.dp_module = qedi_qed_debug;
2374	2569	qed_params.dp_level = dp_level;
2375	2570	qed_params.is_vf = is_vf;
2376	2571	qedi->cdev = qedi_ops->common->probe(pdev, &qed_params);
2377	2572	if (!qedi->cdev) {
	2573	+ if (mode == QEDI_MODE_RECOVERY && retry) {
	2574	+ QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
	2575	+ "Retry %d initialize hardware\n", retry);
	2576	+ retry--;
	2577	+ goto retry_probe;
	2578	+ }
	2579	+
2378	2580	rc = -ENODEV;
2379	2581	QEDI_ERR(&qedi->dbg_ctx, "Cannot initialize hardware\n");
2380	2582	goto free_host;
2381	2583	}
2382	2584
	2585	+ set_bit(QEDI_ERR_ATTN_CLR_EN, &qedi->qedi_err_flags);
	2586	+ set_bit(QEDI_ERR_IS_RECOVERABLE, &qedi->qedi_err_flags);
2383	2587	atomic_set(&qedi->link_state, QEDI_LINK_DOWN);
2384	2588
2385	2589	rc = qedi_ops->fill_dev_info(qedi->cdev, &qedi->dev_info);
2386	2590	if (rc)
2387	2591	goto free_host;
2388	2592
2389		- if (mode != QEDI_MODE_RECOVERY) {
2390		- rc = qedi_set_iscsi_pf_param(qedi);
2391		- if (rc) {
2392		- rc = -ENOMEM;
2393		- QEDI_ERR(&qedi->dbg_ctx,
2394		- "Set iSCSI pf param fail\n");
2395		- goto free_host;
2396		- }
	2593	+ QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
	2594	+ "dev_info: num_hwfns=%d affin_hwfn_idx=%d.\n",
	2595	+ qedi->dev_info.common.num_hwfns,
	2596	+ qedi_ops->common->get_affin_hwfn_idx(qedi->cdev));
	2597	+
	2598	+ rc = qedi_set_iscsi_pf_param(qedi);
	2599	+ if (rc) {
	2600	+ rc = -ENOMEM;
	2601	+ QEDI_ERR(&qedi->dbg_ctx,
	2602	+ "Set iSCSI pf param fail\n");
	2603	+ goto free_host;
2397	2604	}
2398	2605
2399	2606	qedi_ops->common->update_pf_params(qedi->cdev, &qedi->pf_params);
..	..	@@ -2452,15 +2659,15 @@
2452	2659	"Writing %d to primary and secondary BDQ doorbell registers.\n",
2453	2660	qedi->bdq_prod_idx);
2454	2661	writew(qedi->bdq_prod_idx, qedi->bdq_primary_prod);
2455		- tmp = readw(qedi->bdq_primary_prod);
	2662	+ readw(qedi->bdq_primary_prod);
2456	2663	writew(qedi->bdq_prod_idx, qedi->bdq_secondary_prod);
2457		- tmp = readw(qedi->bdq_secondary_prod);
	2664	+ readw(qedi->bdq_secondary_prod);
2458	2665
2459	2666	ether_addr_copy(qedi->mac, qedi->dev_info.common.hw_mac);
2460	2667	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_DISC, "MAC address is %pM.\n",
2461	2668	qedi->mac);
2462	2669
2463		- sprintf(host_buf, "host_%d", qedi->shost->host_no);
	2670	+ snprintf(host_buf, sizeof(host_buf), "host_%d", qedi->shost->host_no);
2464	2671	qedi_ops->common->set_name(qedi->cdev, host_buf);
2465	2672
2466	2673	qedi_ops->register_ops(qedi->cdev, &qedi_cb_ops, qedi);
..	..	@@ -2584,6 +2791,10 @@
2584	2791	goto free_tmf_thread;
2585	2792	}
2586	2793
	2794	+ INIT_DELAYED_WORK(&qedi->recovery_work, qedi_recovery_handler);
	2795	+ INIT_DELAYED_WORK(&qedi->board_disable_work,
	2796	+ qedi_board_disable_work);
	2797	+
2587	2798	/* F/w needs 1st task context memory entry for performance */
2588	2799	set_bit(QEDI_RESERVE_TASK_ID, qedi->task_idx_map);
2589	2800	atomic_set(&qedi->num_offloads, 0);
..	..	@@ -2626,6 +2837,32 @@
2626	2837	return rc;
2627	2838	}
2628	2839
	2840	+static void qedi_mark_conn_recovery(struct iscsi_cls_session *cls_session)
	2841	+{
	2842	+ struct iscsi_session *session = cls_session->dd_data;
	2843	+ struct iscsi_conn *conn = session->leadconn;
	2844	+ struct qedi_conn *qedi_conn = conn->dd_data;
	2845	+
	2846	+ iscsi_conn_failure(qedi_conn->cls_conn->dd_data, ISCSI_ERR_CONN_FAILED);
	2847	+}
	2848	+
	2849	+static void qedi_recovery_handler(struct work_struct *work)
	2850	+{
	2851	+ struct qedi_ctx *qedi =
	2852	+ container_of(work, struct qedi_ctx, recovery_work.work);
	2853	+
	2854	+ iscsi_host_for_each_session(qedi->shost, qedi_mark_conn_recovery);
	2855	+
	2856	+ /* Call common_ops->recovery_prolog to allow the MFW to quiesce
	2857	+ * any PCI transactions.
	2858	+ */
	2859	+ qedi_ops->common->recovery_prolog(qedi->cdev);
	2860	+
	2861	+ __qedi_remove(qedi->pdev, QEDI_MODE_RECOVERY);
	2862	+ __qedi_probe(qedi->pdev, QEDI_MODE_RECOVERY);
	2863	+ clear_bit(QEDI_IN_RECOVERY, &qedi->flags);
	2864	+}
	2865	+
2629	2866	static int qedi_probe(struct pci_dev pdev, const struct pci_device_id id)
2630	2867	{
2631	2868	return __qedi_probe(pdev, QEDI_MODE_NORMAL);
..	..	@@ -2645,11 +2882,18 @@
2645	2882
2646	2883	static enum cpuhp_state qedi_cpuhp_state;
2647	2884
	2885	+static struct pci_error_handlers qedi_err_handler = {
	2886	+ .error_detected = qedi_io_error_detected,
	2887	+};
	2888	+
2648	2889	static struct pci_driver qedi_pci_driver = {
2649	2890	.name = QEDI_MODULE_NAME,
2650	2891	.id_table = qedi_pci_tbl,
2651	2892	.probe = qedi_probe,
2652	2893	.remove = qedi_remove,
	2894	+ .shutdown = qedi_shutdown,
	2895	+ .err_handler = &qedi_err_handler,
	2896	+ .suspend = qedi_suspend,
2653	2897	};
2654	2898
2655	2899	static int __init qedi_init(void)