~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,5 +1,5 @@
1	1	/*
2		- * Copyright(c) 2015-2018 Intel Corporation.
	2	+ * Copyright(c) 2015-2020 Intel Corporation.
3	3	*
4	4	* This file is provided under a dual BSD/GPLv2 license. When using or
5	5	* redistributing this file, you may do so under either license.
..	..	@@ -54,6 +54,7 @@
54	54	#include <linux/module.h>
55	55	#include <linux/prefetch.h>
56	56	#include <rdma/ib_verbs.h>
	57	+#include <linux/etherdevice.h>
57	58
58	59	#include "hfi.h"
59	60	#include "trace.h"
..	..	@@ -62,6 +63,9 @@
62	63	#include "debugfs.h"
63	64	#include "vnic.h"
64	65	#include "fault.h"
	66	+
	67	+#include "ipoib.h"
	68	+#include "netdev.h"
65	69
66	70	#undef pr_fmt
67	71	#define pr_fmt(fmt) DRIVER_NAME ": " fmt
..	..	@@ -72,8 +76,6 @@
72	76	*/
73	77	const char ib_hfi1_version[] = HFI1_DRIVER_VERSION "\n";
74	78
75		-DEFINE_SPINLOCK(hfi1_devs_lock);
76		-LIST_HEAD(hfi1_dev_list);
77	79	DEFINE_MUTEX(hfi1_mutex); /* general driver use */
78	80
79	81	unsigned int hfi1_max_mtu = HFI1_DEFAULT_MAX_MTU;
..	..	@@ -175,11 +177,11 @@
175	177	{
176	178	struct hfi1_devdata *dd;
177	179	struct hfi1_pportdata *ppd;
178		- unsigned long flags;
	180	+ unsigned long index, flags;
179	181	int pidx, nunits_active = 0;
180	182
181		- spin_lock_irqsave(&hfi1_devs_lock, flags);
182		- list_for_each_entry(dd, &hfi1_dev_list, list) {
	183	+ xa_lock_irqsave(&hfi1_dev_table, flags);
	184	+ xa_for_each(&hfi1_dev_table, index, dd) {
183	185	if (!(dd->flags & HFI1_PRESENT) \|\| !dd->kregbase1)
184	186	continue;
185	187	for (pidx = 0; pidx < dd->num_pports; ++pidx) {
..	..	@@ -190,7 +192,7 @@
190	192	}
191	193	}
192	194	}
193		- spin_unlock_irqrestore(&hfi1_devs_lock, flags);
	195	+ xa_unlock_irqrestore(&hfi1_dev_table, flags);
194	196	return nunits_active;
195	197	}
196	198
..	..	@@ -264,7 +266,7 @@
264	266	hfi1_dbg_fault_suppress_err(verbs_dev))
265	267	return;
266	268
267		- if (packet->rhf & (RHF_VCRC_ERR \| RHF_ICRC_ERR))
	269	+ if (packet->rhf & RHF_ICRC_ERR)
268	270	return;
269	271
270	272	if (packet->etype == RHF_RCV_TYPE_BYPASS) {
..	..	@@ -413,14 +415,14 @@
413	415	static inline void init_packet(struct hfi1_ctxtdata *rcd,
414	416	struct hfi1_packet *packet)
415	417	{
416		- packet->rsize = rcd->rcvhdrqentsize; /* words */
417		- packet->maxcnt = rcd->rcvhdrq_cnt * packet->rsize; /* words */
	418	+ packet->rsize = get_hdrqentsize(rcd); /* words */
	419	+ packet->maxcnt = get_hdrq_cnt(rcd) * packet->rsize; /* words */
418	420	packet->rcd = rcd;
419	421	packet->updegr = 0;
420	422	packet->etail = -1;
421	423	packet->rhf_addr = get_rhf_addr(rcd);
422	424	packet->rhf = rhf_to_cpu(packet->rhf_addr);
423		- packet->rhqoff = rcd->head;
	425	+ packet->rhqoff = hfi1_rcd_head(rcd);
424	426	packet->numpkt = 0;
425	427	}
426	428
..	..	@@ -516,7 +518,9 @@
516	518	*/
517	519	do_cnp = prescan \|\|
518	520	(opcode >= IB_OPCODE_RC_RDMA_READ_RESPONSE_FIRST &&
519		- opcode <= IB_OPCODE_RC_ATOMIC_ACKNOWLEDGE);
	521	+ opcode <= IB_OPCODE_RC_ATOMIC_ACKNOWLEDGE) \|\|
	522	+ opcode == TID_OP(READ_RESP) \|\|
	523	+ opcode == TID_OP(ACK);
520	524
521	525	/* Call appropriate CNP handler */
522	526	if (!ignore_fecn && do_cnp && fecn)
..	..	@@ -551,22 +555,22 @@
551	555	mdata->maxcnt = packet->maxcnt;
552	556	mdata->ps_head = packet->rhqoff;
553	557
554		- if (HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL)) {
	558	+ if (get_dma_rtail_setting(rcd)) {
555	559	mdata->ps_tail = get_rcvhdrtail(rcd);
556	560	if (rcd->ctxt == HFI1_CTRL_CTXT)
557		- mdata->ps_seq = rcd->seq_cnt;
	561	+ mdata->ps_seq = hfi1_seq_cnt(rcd);
558	562	else
559	563	mdata->ps_seq = 0; /* not used with DMA_RTAIL */
560	564	} else {
561	565	mdata->ps_tail = 0; /* used only with DMA_RTAIL*/
562		- mdata->ps_seq = rcd->seq_cnt;
	566	+ mdata->ps_seq = hfi1_seq_cnt(rcd);
563	567	}
564	568	}
565	569
566	570	static inline int ps_done(struct ps_mdata *mdata, u64 rhf,
567	571	struct hfi1_ctxtdata *rcd)
568	572	{
569		- if (HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL))
	573	+ if (get_dma_rtail_setting(rcd))
570	574	return mdata->ps_head == mdata->ps_tail;
571	575	return mdata->ps_seq != rhf_rcv_seq(rhf);
572	576	}
..	..	@@ -592,11 +596,9 @@
592	596	mdata->ps_head = 0;
593	597
594	598	/* Control context must do seq counting */
595		- if (!HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL) \|\|
596		- (rcd->ctxt == HFI1_CTRL_CTXT)) {
597		- if (++mdata->ps_seq > 13)
598		- mdata->ps_seq = 1;
599		- }
	599	+ if (!get_dma_rtail_setting(rcd) \|\|
	600	+ rcd->ctxt == HFI1_CTRL_CTXT)
	601	+ mdata->ps_seq = hfi1_seq_incr_wrap(mdata->ps_seq);
600	602	}
601	603
602	604	/*
..	..	@@ -750,6 +752,39 @@
750	752	return ret;
751	753	}
752	754
	755	+static void process_rcv_packet_napi(struct hfi1_packet *packet)
	756	+{
	757	+ packet->etype = rhf_rcv_type(packet->rhf);
	758	+
	759	+ /* total length */
	760	+ packet->tlen = rhf_pkt_len(packet->rhf); /* in bytes */
	761	+ /* retrieve eager buffer details */
	762	+ packet->etail = rhf_egr_index(packet->rhf);
	763	+ packet->ebuf = get_egrbuf(packet->rcd, packet->rhf,
	764	+ &packet->updegr);
	765	+ /*
	766	+ * Prefetch the contents of the eager buffer. It is
	767	+ * OK to send a negative length to prefetch_range().
	768	+ * The +2 is the size of the RHF.
	769	+ */
	770	+ prefetch_range(packet->ebuf,
	771	+ packet->tlen - ((packet->rcd->rcvhdrqentsize -
	772	+ (rhf_hdrq_offset(packet->rhf)
	773	+ + 2)) * 4));
	774	+
	775	+ packet->rcd->rhf_rcv_function_map[packet->etype](packet);
	776	+ packet->numpkt++;
	777	+
	778	+ /* Set up for the next packet */
	779	+ packet->rhqoff += packet->rsize;
	780	+ if (packet->rhqoff >= packet->maxcnt)
	781	+ packet->rhqoff = 0;
	782	+
	783	+ packet->rhf_addr = (__le32 *)packet->rcd->rcvhdrq + packet->rhqoff +
	784	+ packet->rcd->rhf_offset;
	785	+ packet->rhf = rhf_to_cpu(packet->rhf_addr);
	786	+}
	787	+
753	788	static inline int process_rcv_packet(struct hfi1_packet *packet, int thread)
754	789	{
755	790	int ret;
..	..	@@ -770,7 +805,7 @@
770	805	* The +2 is the size of the RHF.
771	806	*/
772	807	prefetch_range(packet->ebuf,
773		- packet->tlen - ((packet->rcd->rcvhdrqentsize -
	808	+ packet->tlen - ((get_hdrqentsize(packet->rcd) -
774	809	(rhf_hdrq_offset(packet->rhf)
775	810	+ 2)) * 4));
776	811	}
..	..	@@ -824,8 +859,38 @@
824	859	* The only thing we need to do is a final update and call for an
825	860	* interrupt
826	861	*/
827		- update_usrhead(packet->rcd, packet->rcd->head, packet->updegr,
	862	+ update_usrhead(packet->rcd, hfi1_rcd_head(packet->rcd), packet->updegr,
828	863	packet->etail, rcv_intr_dynamic, packet->numpkt);
	864	+}
	865	+
	866	+/*
	867	+ * handle_receive_interrupt_napi_fp - receive a packet
	868	+ * @rcd: the context
	869	+ * @budget: polling budget
	870	+ *
	871	+ * Called from interrupt handler for receive interrupt.
	872	+ * This is the fast path interrupt handler
	873	+ * when executing napi soft irq environment.
	874	+ */
	875	+int handle_receive_interrupt_napi_fp(struct hfi1_ctxtdata *rcd, int budget)
	876	+{
	877	+ struct hfi1_packet packet;
	878	+
	879	+ init_packet(rcd, &packet);
	880	+ if (last_rcv_seq(rcd, rhf_rcv_seq(packet.rhf)))
	881	+ goto bail;
	882	+
	883	+ while (packet.numpkt < budget) {
	884	+ process_rcv_packet_napi(&packet);
	885	+ if (hfi1_seq_incr(rcd, rhf_rcv_seq(packet.rhf)))
	886	+ break;
	887	+
	888	+ process_rcv_update(0, &packet);
	889	+ }
	890	+ hfi1_set_rcd_head(rcd, packet.rhqoff);
	891	+bail:
	892	+ finish_packet(&packet);
	893	+ return packet.numpkt;
829	894	}
830	895
831	896	/*
..	..	@@ -833,13 +898,11 @@
833	898	*/
834	899	int handle_receive_interrupt_nodma_rtail(struct hfi1_ctxtdata *rcd, int thread)
835	900	{
836		- u32 seq;
837	901	int last = RCV_PKT_OK;
838	902	struct hfi1_packet packet;
839	903
840	904	init_packet(rcd, &packet);
841		- seq = rhf_rcv_seq(packet.rhf);
842		- if (seq != rcd->seq_cnt) {
	905	+ if (last_rcv_seq(rcd, rhf_rcv_seq(packet.rhf))) {
843	906	last = RCV_PKT_DONE;
844	907	goto bail;
845	908	}
..	..	@@ -848,15 +911,12 @@
848	911
849	912	while (last == RCV_PKT_OK) {
850	913	last = process_rcv_packet(&packet, thread);
851		- seq = rhf_rcv_seq(packet.rhf);
852		- if (++rcd->seq_cnt > 13)
853		- rcd->seq_cnt = 1;
854		- if (seq != rcd->seq_cnt)
	914	+ if (hfi1_seq_incr(rcd, rhf_rcv_seq(packet.rhf)))
855	915	last = RCV_PKT_DONE;
856	916	process_rcv_update(last, &packet);
857	917	}
858	918	process_rcv_qp_work(&packet);
859		- rcd->head = packet.rhqoff;
	919	+ hfi1_set_rcd_head(rcd, packet.rhqoff);
860	920	bail:
861	921	finish_packet(&packet);
862	922	return last;
..	..	@@ -885,15 +945,14 @@
885	945	process_rcv_update(last, &packet);
886	946	}
887	947	process_rcv_qp_work(&packet);
888		- rcd->head = packet.rhqoff;
	948	+ hfi1_set_rcd_head(rcd, packet.rhqoff);
889	949	bail:
890	950	finish_packet(&packet);
891	951	return last;
892	952	}
893	953
894		-static inline void set_nodma_rtail(struct hfi1_devdata *dd, u16 ctxt)
	954	+static void set_all_fastpath(struct hfi1_devdata dd, struct hfi1_ctxtdata rcd)
895	955	{
896		- struct hfi1_ctxtdata *rcd;
897	956	u16 i;
898	957
899	958	/*
..	..	@@ -901,50 +960,17 @@
901	960	* interrupt handler only for that context. Otherwise, switch
902	961	* interrupt handler for all statically allocated kernel contexts.
903	962	*/
904		- if (ctxt >= dd->first_dyn_alloc_ctxt) {
905		- rcd = hfi1_rcd_get_by_index_safe(dd, ctxt);
906		- if (rcd) {
907		- rcd->do_interrupt =
908		- &handle_receive_interrupt_nodma_rtail;
909		- hfi1_rcd_put(rcd);
910		- }
911		- return;
912		- }
913		-
914		- for (i = HFI1_CTRL_CTXT + 1; i < dd->first_dyn_alloc_ctxt; i++) {
915		- rcd = hfi1_rcd_get_by_index(dd, i);
916		- if (rcd)
917		- rcd->do_interrupt =
918		- &handle_receive_interrupt_nodma_rtail;
	963	+ if (rcd->ctxt >= dd->first_dyn_alloc_ctxt && !rcd->is_vnic) {
	964	+ hfi1_rcd_get(rcd);
	965	+ hfi1_set_fast(rcd);
919	966	hfi1_rcd_put(rcd);
920		- }
921		-}
922		-
923		-static inline void set_dma_rtail(struct hfi1_devdata *dd, u16 ctxt)
924		-{
925		- struct hfi1_ctxtdata *rcd;
926		- u16 i;
927		-
928		- /*
929		- * For dynamically allocated kernel contexts (like vnic) switch
930		- * interrupt handler only for that context. Otherwise, switch
931		- * interrupt handler for all statically allocated kernel contexts.
932		- */
933		- if (ctxt >= dd->first_dyn_alloc_ctxt) {
934		- rcd = hfi1_rcd_get_by_index_safe(dd, ctxt);
935		- if (rcd) {
936		- rcd->do_interrupt =
937		- &handle_receive_interrupt_dma_rtail;
938		- hfi1_rcd_put(rcd);
939		- }
940	967	return;
941	968	}
942	969
943		- for (i = HFI1_CTRL_CTXT + 1; i < dd->first_dyn_alloc_ctxt; i++) {
	970	+ for (i = HFI1_CTRL_CTXT + 1; i < dd->num_rcv_contexts; i++) {
944	971	rcd = hfi1_rcd_get_by_index(dd, i);
945		- if (rcd)
946		- rcd->do_interrupt =
947		- &handle_receive_interrupt_dma_rtail;
	972	+ if (rcd && (i < dd->first_dyn_alloc_ctxt \|\| rcd->is_vnic))
	973	+ hfi1_set_fast(rcd);
948	974	hfi1_rcd_put(rcd);
949	975	}
950	976	}
..	..	@@ -960,17 +986,14 @@
960	986	if (!rcd)
961	987	continue;
962	988	if (i < dd->first_dyn_alloc_ctxt \|\| rcd->is_vnic)
963		- rcd->do_interrupt = &handle_receive_interrupt;
	989	+ rcd->do_interrupt = rcd->slow_handler;
964	990
965	991	hfi1_rcd_put(rcd);
966	992	}
967	993	}
968	994
969		-static inline int set_armed_to_active(struct hfi1_ctxtdata *rcd,
970		- struct hfi1_packet *packet,
971		- struct hfi1_devdata *dd)
	995	+static bool __set_armed_to_active(struct hfi1_packet *packet)
972	996	{
973		- struct work_struct *lsaw = &rcd->ppd->linkstate_active_work;
974	997	u8 etype = rhf_rcv_type(packet->rhf);
975	998	u8 sc = SC15_PACKET;
976	999
..	..	@@ -985,19 +1008,34 @@
985	1008	sc = hfi1_16B_get_sc(hdr);
986	1009	}
987	1010	if (sc != SC15_PACKET) {
988		- int hwstate = driver_lstate(rcd->ppd);
	1011	+ int hwstate = driver_lstate(packet->rcd->ppd);
	1012	+ struct work_struct *lsaw =
	1013	+ &packet->rcd->ppd->linkstate_active_work;
989	1014
990	1015	if (hwstate != IB_PORT_ACTIVE) {
991		- dd_dev_info(dd,
	1016	+ dd_dev_info(packet->rcd->dd,
992	1017	"Unexpected link state %s\n",
993	1018	opa_lstate_name(hwstate));
994		- return 0;
	1019	+ return false;
995	1020	}
996	1021
997		- queue_work(rcd->ppd->link_wq, lsaw);
998		- return 1;
	1022	+ queue_work(packet->rcd->ppd->link_wq, lsaw);
	1023	+ return true;
999	1024	}
1000		- return 0;
	1025	+ return false;
	1026	+}
	1027	+
	1028	+/**
	1029	+ * armed to active - the fast path for armed to active
	1030	+ * @packet: the packet structure
	1031	+ *
	1032	+ * Return true if packet processing needs to bail.
	1033	+ */
	1034	+static bool set_armed_to_active(struct hfi1_packet *packet)
	1035	+{
	1036	+ if (likely(packet->rcd->ppd->host_link_state != HLS_UP_ARMED))
	1037	+ return false;
	1038	+ return __set_armed_to_active(packet);
1001	1039	}
1002	1040
1003	1041	/*
..	..	@@ -1015,15 +1053,15 @@
1015	1053	struct hfi1_packet packet;
1016	1054	int skip_pkt = 0;
1017	1055
	1056	+ if (!rcd->rcvhdrq)
	1057	+ return RCV_PKT_OK;
1018	1058	/* Control context will always use the slow path interrupt handler */
1019	1059	needset = (rcd->ctxt == HFI1_CTRL_CTXT) ? 0 : 1;
1020	1060
1021	1061	init_packet(rcd, &packet);
1022	1062
1023		- if (!HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL)) {
1024		- u32 seq = rhf_rcv_seq(packet.rhf);
1025		-
1026		- if (seq != rcd->seq_cnt) {
	1063	+ if (!get_dma_rtail_setting(rcd)) {
	1064	+ if (last_rcv_seq(rcd, rhf_rcv_seq(packet.rhf))) {
1027	1065	last = RCV_PKT_DONE;
1028	1066	goto bail;
1029	1067	}
..	..	@@ -1040,22 +1078,15 @@
1040	1078	* Control context can potentially receive an invalid
1041	1079	* rhf. Drop such packets.
1042	1080	*/
1043		- if (rcd->ctxt == HFI1_CTRL_CTXT) {
1044		- u32 seq = rhf_rcv_seq(packet.rhf);
1045		-
1046		- if (seq != rcd->seq_cnt)
	1081	+ if (rcd->ctxt == HFI1_CTRL_CTXT)
	1082	+ if (last_rcv_seq(rcd, rhf_rcv_seq(packet.rhf)))
1047	1083	skip_pkt = 1;
1048		- }
1049	1084	}
1050	1085
1051	1086	prescan_rxq(rcd, &packet);
1052	1087
1053	1088	while (last == RCV_PKT_OK) {
1054		- if (unlikely(dd->do_drop &&
1055		- atomic_xchg(&dd->drop_packet, DROP_PACKET_OFF) ==
1056		- DROP_PACKET_ON)) {
1057		- dd->do_drop = 0;
1058		-
	1089	+ if (hfi1_need_drop(dd)) {
1059	1090	/* On to the next packet */
1060	1091	packet.rhqoff += packet.rsize;
1061	1092	packet.rhf_addr = (__le32 *)rcd->rcvhdrq +
..	..	@@ -1067,26 +1098,14 @@
1067	1098	last = skip_rcv_packet(&packet, thread);
1068	1099	skip_pkt = 0;
1069	1100	} else {
1070		- /* Auto activate link on non-SC15 packet receive */
1071		- if (unlikely(rcd->ppd->host_link_state ==
1072		- HLS_UP_ARMED) &&
1073		- set_armed_to_active(rcd, &packet, dd))
	1101	+ if (set_armed_to_active(&packet))
1074	1102	goto bail;
1075	1103	last = process_rcv_packet(&packet, thread);
1076	1104	}
1077	1105
1078		- if (!HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL)) {
1079		- u32 seq = rhf_rcv_seq(packet.rhf);
1080		-
1081		- if (++rcd->seq_cnt > 13)
1082		- rcd->seq_cnt = 1;
1083		- if (seq != rcd->seq_cnt)
	1106	+ if (!get_dma_rtail_setting(rcd)) {
	1107	+ if (hfi1_seq_incr(rcd, rhf_rcv_seq(packet.rhf)))
1084	1108	last = RCV_PKT_DONE;
1085		- if (needset) {
1086		- dd_dev_info(dd, "Switching to NO_DMA_RTAIL\n");
1087		- set_nodma_rtail(dd, rcd->ctxt);
1088		- needset = 0;
1089		- }
1090	1109	} else {
1091	1110	if (packet.rhqoff == hdrqtail)
1092	1111	last = RCV_PKT_DONE;
..	..	@@ -1095,27 +1114,24 @@
1095	1114	* rhf. Drop such packets.
1096	1115	*/
1097	1116	if (rcd->ctxt == HFI1_CTRL_CTXT) {
1098		- u32 seq = rhf_rcv_seq(packet.rhf);
	1117	+ bool lseq;
1099	1118
1100		- if (++rcd->seq_cnt > 13)
1101		- rcd->seq_cnt = 1;
1102		- if (!last && (seq != rcd->seq_cnt))
	1119	+ lseq = hfi1_seq_incr(rcd,
	1120	+ rhf_rcv_seq(packet.rhf));
	1121	+ if (!last && lseq)
1103	1122	skip_pkt = 1;
1104		- }
1105		-
1106		- if (needset) {
1107		- dd_dev_info(dd,
1108		- "Switching to DMA_RTAIL\n");
1109		- set_dma_rtail(dd, rcd->ctxt);
1110		- needset = 0;
1111	1123	}
1112	1124	}
1113	1125
	1126	+ if (needset) {
	1127	+ needset = false;
	1128	+ set_all_fastpath(dd, rcd);
	1129	+ }
1114	1130	process_rcv_update(last, &packet);
1115	1131	}
1116	1132
1117	1133	process_rcv_qp_work(&packet);
1118		- rcd->head = packet.rhqoff;
	1134	+ hfi1_set_rcd_head(rcd, packet.rhqoff);
1119	1135
1120	1136	bail:
1121	1137	/*
..	..	@@ -1124,6 +1140,63 @@
1124	1140	*/
1125	1141	finish_packet(&packet);
1126	1142	return last;
	1143	+}
	1144	+
	1145	+/*
	1146	+ * handle_receive_interrupt_napi_sp - receive a packet
	1147	+ * @rcd: the context
	1148	+ * @budget: polling budget
	1149	+ *
	1150	+ * Called from interrupt handler for errors or receive interrupt.
	1151	+ * This is the slow path interrupt handler
	1152	+ * when executing napi soft irq environment.
	1153	+ */
	1154	+int handle_receive_interrupt_napi_sp(struct hfi1_ctxtdata *rcd, int budget)
	1155	+{
	1156	+ struct hfi1_devdata *dd = rcd->dd;
	1157	+ int last = RCV_PKT_OK;
	1158	+ bool needset = true;
	1159	+ struct hfi1_packet packet;
	1160	+
	1161	+ init_packet(rcd, &packet);
	1162	+ if (last_rcv_seq(rcd, rhf_rcv_seq(packet.rhf)))
	1163	+ goto bail;
	1164	+
	1165	+ while (last != RCV_PKT_DONE && packet.numpkt < budget) {
	1166	+ if (hfi1_need_drop(dd)) {
	1167	+ /* On to the next packet */
	1168	+ packet.rhqoff += packet.rsize;
	1169	+ packet.rhf_addr = (__le32 *)rcd->rcvhdrq +
	1170	+ packet.rhqoff +
	1171	+ rcd->rhf_offset;
	1172	+ packet.rhf = rhf_to_cpu(packet.rhf_addr);
	1173	+
	1174	+ } else {
	1175	+ if (set_armed_to_active(&packet))
	1176	+ goto bail;
	1177	+ process_rcv_packet_napi(&packet);
	1178	+ }
	1179	+
	1180	+ if (hfi1_seq_incr(rcd, rhf_rcv_seq(packet.rhf)))
	1181	+ last = RCV_PKT_DONE;
	1182	+
	1183	+ if (needset) {
	1184	+ needset = false;
	1185	+ set_all_fastpath(dd, rcd);
	1186	+ }
	1187	+
	1188	+ process_rcv_update(last, &packet);
	1189	+ }
	1190	+
	1191	+ hfi1_set_rcd_head(rcd, packet.rhqoff);
	1192	+
	1193	+bail:
	1194	+ /*
	1195	+ * Always write head at end, and setup rcv interrupt, even
	1196	+ * if no packets were processed.
	1197	+ */
	1198	+ finish_packet(&packet);
	1199	+ return packet.numpkt;
1127	1200	}
1128	1201
1129	1202	/*
..	..	@@ -1427,7 +1500,7 @@
1427	1500	if ((!(hfi1_is_16B_mcast(packet->dlid))) &&
1428	1501	(packet->dlid !=
1429	1502	opa_get_lid(be32_to_cpu(OPA_LID_PERMISSIVE), 16B))) {
1430		- if (packet->dlid != ppd->lid)
	1503	+ if ((packet->dlid & ~((1 << ppd->lmc) - 1)) != ppd->lid)
1431	1504	return -EINVAL;
1432	1505	}
1433	1506
..	..	@@ -1576,80 +1649,143 @@
1576	1649	return -EINVAL;
1577	1650	}
1578	1651
1579		-void handle_eflags(struct hfi1_packet *packet)
	1652	+static void show_eflags_errs(struct hfi1_packet *packet)
1580	1653	{
1581	1654	struct hfi1_ctxtdata *rcd = packet->rcd;
1582	1655	u32 rte = rhf_rcv_type_err(packet->rhf);
1583	1656
	1657	+ dd_dev_err(rcd->dd,
	1658	+ "receive context %d: rhf 0x%016llx, errs [ %s%s%s%s%s%s%s] rte 0x%x\n",
	1659	+ rcd->ctxt, packet->rhf,
	1660	+ packet->rhf & RHF_K_HDR_LEN_ERR ? "k_hdr_len " : "",
	1661	+ packet->rhf & RHF_DC_UNC_ERR ? "dc_unc " : "",
	1662	+ packet->rhf & RHF_DC_ERR ? "dc " : "",
	1663	+ packet->rhf & RHF_TID_ERR ? "tid " : "",
	1664	+ packet->rhf & RHF_LEN_ERR ? "len " : "",
	1665	+ packet->rhf & RHF_ECC_ERR ? "ecc " : "",
	1666	+ packet->rhf & RHF_ICRC_ERR ? "icrc " : "",
	1667	+ rte);
	1668	+}
	1669	+
	1670	+void handle_eflags(struct hfi1_packet *packet)
	1671	+{
	1672	+ struct hfi1_ctxtdata *rcd = packet->rcd;
	1673	+
1584	1674	rcv_hdrerr(rcd, rcd->ppd, packet);
1585	1675	if (rhf_err_flags(packet->rhf))
1586		- dd_dev_err(rcd->dd,
1587		- "receive context %d: rhf 0x%016llx, errs [ %s%s%s%s%s%s%s%s] rte 0x%x\n",
1588		- rcd->ctxt, packet->rhf,
1589		- packet->rhf & RHF_K_HDR_LEN_ERR ? "k_hdr_len " : "",
1590		- packet->rhf & RHF_DC_UNC_ERR ? "dc_unc " : "",
1591		- packet->rhf & RHF_DC_ERR ? "dc " : "",
1592		- packet->rhf & RHF_TID_ERR ? "tid " : "",
1593		- packet->rhf & RHF_LEN_ERR ? "len " : "",
1594		- packet->rhf & RHF_ECC_ERR ? "ecc " : "",
1595		- packet->rhf & RHF_VCRC_ERR ? "vcrc " : "",
1596		- packet->rhf & RHF_ICRC_ERR ? "icrc " : "",
1597		- rte);
	1676	+ show_eflags_errs(packet);
	1677	+}
	1678	+
	1679	+static void hfi1_ipoib_ib_rcv(struct hfi1_packet *packet)
	1680	+{
	1681	+ struct hfi1_ibport *ibp;
	1682	+ struct net_device *netdev;
	1683	+ struct hfi1_ctxtdata *rcd = packet->rcd;
	1684	+ struct napi_struct *napi = rcd->napi;
	1685	+ struct sk_buff *skb;
	1686	+ struct hfi1_netdev_rxq *rxq = container_of(napi,
	1687	+ struct hfi1_netdev_rxq, napi);
	1688	+ u32 extra_bytes;
	1689	+ u32 tlen, qpnum;
	1690	+ bool do_work, do_cnp;
	1691	+ struct hfi1_ipoib_dev_priv *priv;
	1692	+
	1693	+ trace_hfi1_rcvhdr(packet);
	1694	+
	1695	+ hfi1_setup_ib_header(packet);
	1696	+
	1697	+ packet->ohdr = &((struct ib_header *)packet->hdr)->u.oth;
	1698	+ packet->grh = NULL;
	1699	+
	1700	+ if (unlikely(rhf_err_flags(packet->rhf))) {
	1701	+ handle_eflags(packet);
	1702	+ return;
	1703	+ }
	1704	+
	1705	+ qpnum = ib_bth_get_qpn(packet->ohdr);
	1706	+ netdev = hfi1_netdev_get_data(rcd->dd, qpnum);
	1707	+ if (!netdev)
	1708	+ goto drop_no_nd;
	1709	+
	1710	+ trace_input_ibhdr(rcd->dd, packet, !!(rhf_dc_info(packet->rhf)));
	1711	+ trace_ctxt_rsm_hist(rcd->ctxt);
	1712	+
	1713	+ /* handle congestion notifications */
	1714	+ do_work = hfi1_may_ecn(packet);
	1715	+ if (unlikely(do_work)) {
	1716	+ do_cnp = (packet->opcode != IB_OPCODE_CNP);
	1717	+ (void)hfi1_process_ecn_slowpath(hfi1_ipoib_priv(netdev)->qp,
	1718	+ packet, do_cnp);
	1719	+ }
	1720	+
	1721	+ /*
	1722	+ * We have split point after last byte of DETH
	1723	+ * lets strip padding and CRC and ICRC.
	1724	+ * tlen is whole packet len so we need to
	1725	+ * subtract header size as well.
	1726	+ */
	1727	+ tlen = packet->tlen;
	1728	+ extra_bytes = ib_bth_get_pad(packet->ohdr) + (SIZE_OF_CRC << 2) +
	1729	+ packet->hlen;
	1730	+ if (unlikely(tlen < extra_bytes))
	1731	+ goto drop;
	1732	+
	1733	+ tlen -= extra_bytes;
	1734	+
	1735	+ skb = hfi1_ipoib_prepare_skb(rxq, tlen, packet->ebuf);
	1736	+ if (unlikely(!skb))
	1737	+ goto drop;
	1738	+
	1739	+ priv = hfi1_ipoib_priv(netdev);
	1740	+ hfi1_ipoib_update_rx_netstats(priv, 1, skb->len);
	1741	+
	1742	+ skb->dev = netdev;
	1743	+ skb->pkt_type = PACKET_HOST;
	1744	+ netif_receive_skb(skb);
	1745	+
	1746	+ return;
	1747	+
	1748	+drop:
	1749	+ ++netdev->stats.rx_dropped;
	1750	+drop_no_nd:
	1751	+ ibp = rcd_to_iport(packet->rcd);
	1752	+ ++ibp->rvp.n_pkt_drops;
1598	1753	}
1599	1754
1600	1755	/*
1601	1756	* The following functions are called by the interrupt handler. They are type
1602	1757	* specific handlers for each packet type.
1603	1758	*/
1604		-static int process_receive_ib(struct hfi1_packet *packet)
	1759	+static void process_receive_ib(struct hfi1_packet *packet)
1605	1760	{
1606	1761	if (hfi1_setup_9B_packet(packet))
1607		- return RHF_RCV_CONTINUE;
	1762	+ return;
1608	1763
1609	1764	if (unlikely(hfi1_dbg_should_fault_rx(packet)))
1610		- return RHF_RCV_CONTINUE;
	1765	+ return;
1611	1766
1612	1767	trace_hfi1_rcvhdr(packet);
1613	1768
1614	1769	if (unlikely(rhf_err_flags(packet->rhf))) {
1615	1770	handle_eflags(packet);
1616		- return RHF_RCV_CONTINUE;
	1771	+ return;
1617	1772	}
1618	1773
1619	1774	hfi1_ib_rcv(packet);
1620		- return RHF_RCV_CONTINUE;
1621	1775	}
1622	1776
1623		-static inline bool hfi1_is_vnic_packet(struct hfi1_packet *packet)
1624		-{
1625		- /* Packet received in VNIC context via RSM */
1626		- if (packet->rcd->is_vnic)
1627		- return true;
1628		-
1629		- if ((hfi1_16B_get_l2(packet->ebuf) == OPA_16B_L2_TYPE) &&
1630		- (hfi1_16B_get_l4(packet->ebuf) == OPA_16B_L4_ETHR))
1631		- return true;
1632		-
1633		- return false;
1634		-}
1635		-
1636		-static int process_receive_bypass(struct hfi1_packet *packet)
	1777	+static void process_receive_bypass(struct hfi1_packet *packet)
1637	1778	{
1638	1779	struct hfi1_devdata *dd = packet->rcd->dd;
1639	1780
1640		- if (hfi1_is_vnic_packet(packet)) {
1641		- hfi1_vnic_bypass_rcv(packet);
1642		- return RHF_RCV_CONTINUE;
1643		- }
1644		-
1645	1781	if (hfi1_setup_bypass_packet(packet))
1646		- return RHF_RCV_CONTINUE;
	1782	+ return;
1647	1783
1648	1784	trace_hfi1_rcvhdr(packet);
1649	1785
1650	1786	if (unlikely(rhf_err_flags(packet->rhf))) {
1651	1787	handle_eflags(packet);
1652		- return RHF_RCV_CONTINUE;
	1788	+ return;
1653	1789	}
1654	1790
1655	1791	if (hfi1_16B_get_l2(packet->hdr) == 0x2) {
..	..	@@ -1672,17 +1808,16 @@
1672	1808	(OPA_EI_STATUS_SMASK \| BAD_L2_ERR);
1673	1809	}
1674	1810	}
1675		- return RHF_RCV_CONTINUE;
1676	1811	}
1677	1812
1678		-static int process_receive_error(struct hfi1_packet *packet)
	1813	+static void process_receive_error(struct hfi1_packet *packet)
1679	1814	{
1680	1815	/* KHdrHCRCErr -- KDETH packet with a bad HCRC */
1681	1816	if (unlikely(
1682	1817	hfi1_dbg_fault_suppress_err(&packet->rcd->dd->verbs_dev) &&
1683	1818	(rhf_rcv_type_err(packet->rhf) == RHF_RCV_TYPE_ERROR \|\|
1684	1819	packet->rhf & RHF_DC_ERR)))
1685		- return RHF_RCV_CONTINUE;
	1820	+ return;
1686	1821
1687	1822	hfi1_setup_ib_header(packet);
1688	1823	handle_eflags(packet);
..	..	@@ -1690,61 +1825,71 @@
1690	1825	if (unlikely(rhf_err_flags(packet->rhf)))
1691	1826	dd_dev_err(packet->rcd->dd,
1692	1827	"Unhandled error packet received. Dropping.\n");
1693		-
1694		- return RHF_RCV_CONTINUE;
1695	1828	}
1696	1829
1697		-static int kdeth_process_expected(struct hfi1_packet *packet)
	1830	+static void kdeth_process_expected(struct hfi1_packet *packet)
1698	1831	{
1699	1832	hfi1_setup_9B_packet(packet);
1700	1833	if (unlikely(hfi1_dbg_should_fault_rx(packet)))
1701		- return RHF_RCV_CONTINUE;
	1834	+ return;
1702	1835
1703		- if (unlikely(rhf_err_flags(packet->rhf)))
1704		- handle_eflags(packet);
	1836	+ if (unlikely(rhf_err_flags(packet->rhf))) {
	1837	+ struct hfi1_ctxtdata *rcd = packet->rcd;
1705	1838
1706		- dd_dev_err(packet->rcd->dd,
1707		- "Unhandled expected packet received. Dropping.\n");
1708		- return RHF_RCV_CONTINUE;
	1839	+ if (hfi1_handle_kdeth_eflags(rcd, rcd->ppd, packet))
	1840	+ return;
	1841	+ }
	1842	+
	1843	+ hfi1_kdeth_expected_rcv(packet);
1709	1844	}
1710	1845
1711		-static int kdeth_process_eager(struct hfi1_packet *packet)
	1846	+static void kdeth_process_eager(struct hfi1_packet *packet)
1712	1847	{
1713	1848	hfi1_setup_9B_packet(packet);
1714	1849	if (unlikely(hfi1_dbg_should_fault_rx(packet)))
1715		- return RHF_RCV_CONTINUE;
1716		- if (unlikely(rhf_err_flags(packet->rhf)))
1717		- handle_eflags(packet);
	1850	+ return;
1718	1851
1719		- dd_dev_err(packet->rcd->dd,
1720		- "Unhandled eager packet received. Dropping.\n");
1721		- return RHF_RCV_CONTINUE;
	1852	+ trace_hfi1_rcvhdr(packet);
	1853	+ if (unlikely(rhf_err_flags(packet->rhf))) {
	1854	+ struct hfi1_ctxtdata *rcd = packet->rcd;
	1855	+
	1856	+ show_eflags_errs(packet);
	1857	+ if (hfi1_handle_kdeth_eflags(rcd, rcd->ppd, packet))
	1858	+ return;
	1859	+ }
	1860	+
	1861	+ hfi1_kdeth_eager_rcv(packet);
1722	1862	}
1723	1863
1724		-static int process_receive_invalid(struct hfi1_packet *packet)
	1864	+static void process_receive_invalid(struct hfi1_packet *packet)
1725	1865	{
1726	1866	dd_dev_err(packet->rcd->dd, "Invalid packet type %d. Dropping\n",
1727	1867	rhf_rcv_type(packet->rhf));
1728		- return RHF_RCV_CONTINUE;
1729	1868	}
	1869	+
	1870	+#define HFI1_RCVHDR_DUMP_MAX 5
1730	1871
1731	1872	void seqfile_dump_rcd(struct seq_file s, struct hfi1_ctxtdata rcd)
1732	1873	{
1733	1874	struct hfi1_packet packet;
1734	1875	struct ps_mdata mdata;
	1876	+ int i;
1735	1877
1736		- seq_printf(s, "Rcd %u: RcvHdr cnt %u entsize %u %s head %llu tail %llu\n",
1737		- rcd->ctxt, rcd->rcvhdrq_cnt, rcd->rcvhdrqentsize,
1738		- HFI1_CAP_KGET_MASK(rcd->flags, DMA_RTAIL) ?
	1878	+ seq_printf(s, "Rcd %u: RcvHdr cnt %u entsize %u %s ctrl 0x%08llx status 0x%08llx, head %llu tail %llu sw head %u\n",
	1879	+ rcd->ctxt, get_hdrq_cnt(rcd), get_hdrqentsize(rcd),
	1880	+ get_dma_rtail_setting(rcd) ?
1739	1881	"dma_rtail" : "nodma_rtail",
	1882	+ read_kctxt_csr(rcd->dd, rcd->ctxt, RCV_CTXT_CTRL),
	1883	+ read_kctxt_csr(rcd->dd, rcd->ctxt, RCV_CTXT_STATUS),
1740	1884	read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_HEAD) &
1741	1885	RCV_HDR_HEAD_HEAD_MASK,
1742		- read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_TAIL));
	1886	+ read_uctxt_csr(rcd->dd, rcd->ctxt, RCV_HDR_TAIL),
	1887	+ rcd->head);
1743	1888
1744	1889	init_packet(rcd, &packet);
1745	1890	init_ps_mdata(&mdata, &packet);
1746	1891
1747		- while (1) {
	1892	+ for (i = 0; i < HFI1_RCVHDR_DUMP_MAX; i++) {
1748	1893	__le32 rhf_addr = (__le32 )rcd->rcvhdrq + mdata.ps_head +
1749	1894	rcd->rhf_offset;
1750	1895	struct ib_header *hdr;
..	..	@@ -1796,3 +1941,14 @@
1796	1941	[RHF_RCV_TYPE_INVALID6] = process_receive_invalid,
1797	1942	[RHF_RCV_TYPE_INVALID7] = process_receive_invalid,
1798	1943	};
	1944	+
	1945	+const rhf_rcv_function_ptr netdev_rhf_rcv_functions[] = {
	1946	+ [RHF_RCV_TYPE_EXPECTED] = process_receive_invalid,
	1947	+ [RHF_RCV_TYPE_EAGER] = process_receive_invalid,
	1948	+ [RHF_RCV_TYPE_IB] = hfi1_ipoib_ib_rcv,
	1949	+ [RHF_RCV_TYPE_ERROR] = process_receive_error,
	1950	+ [RHF_RCV_TYPE_BYPASS] = hfi1_vnic_bypass_rcv,
	1951	+ [RHF_RCV_TYPE_INVALID5] = process_receive_invalid,
	1952	+ [RHF_RCV_TYPE_INVALID6] = process_receive_invalid,
	1953	+ [RHF_RCV_TYPE_INVALID7] = process_receive_invalid,
	1954	+};