~hc/RK356X_SDK_RELEASE.git

..	..	@@ -1,17 +1,18 @@
1		-/* SPDX-License-Identifier: GPL-2.0 */
2	1	/**
3	2	* @file definition of host message ring functionality
4	3	* Provides type definitions and function prototypes used to link the
5	4	* DHD OS, bus, and protocol modules.
6	5	*
7		- * Copyright (C) 1999-2019, Broadcom Corporation
8		- *
	6	+ * Portions of this code are copyright (c) 2022 Cypress Semiconductor Corporation
	7	+ *
	8	+ * Copyright (C) 1999-2017, Broadcom Corporation
	9	+ *
9	10	* Unless you and Broadcom execute a separate written software license
10	11	* agreement governing use of this software, this software is licensed to you
11	12	* under the terms of the GNU General Public License version 2 (the "GPL"),
12	13	* available at http://www.broadcom.com/licenses/GPLv2.php, with the
13	14	* following added to such license:
14		- *
	15	+ *
15	16	* As a special exception, the copyright holders of this software give you
16	17	* permission to link this software with independent modules, and to copy and
17	18	* distribute the resulting executable under terms of your choice, provided that
..	..	@@ -19,7 +20,7 @@
19	20	* the license of that module. An independent module is a module which is not
20	21	* derived from this software. The special exception does not apply to any
21	22	* modifications of the software.
22		- *
	23	+ *
23	24	* Notwithstanding the above, under no circumstances may you combine this
24	25	* software in any way with any other Broadcom software provided under a license
25	26	* other than the GPL, without Broadcom's express prior written consent.
..	..	@@ -27,9 +28,8 @@
27	28	*
28	29	* <<Broadcom-WL-IPTag/Open:>>
29	30	*
30		- * $Id: dhd_msgbuf.c 608659 2015-12-29 01:18:33Z $
	31	+ * $Id: dhd_msgbuf.c 701962 2017-05-30 06:13:15Z $
31	32	*/
32		-
33	33
34	34	#include <typedefs.h>
35	35	#include <osl.h>
..	..	@@ -37,20 +37,17 @@
37	37	#include <bcmutils.h>
38	38	#include <bcmmsgbuf.h>
39	39	#include <bcmendian.h>
	40	+#include <bcmstdlib_s.h>
40	41
41	42	#include <dngl_stats.h>
42	43	#include <dhd.h>
43	44	#include <dhd_proto.h>
44	45
45		-#ifdef BCMDBUS
46		-#include <dbus.h>
47		-#else
48	46	#include <dhd_bus.h>
49		-#endif /* BCMDBUS */
50	47
51	48	#include <dhd_dbg.h>
52	49	#include <siutils.h>
53		-
	50	+#include <dhd_debug.h>
54	51
55	52	#include <dhd_flowring.h>
56	53
..	..	@@ -61,11 +58,27 @@
61	58	#if defined(DHD_LB)
62	59	#include <linux/cpu.h>
63	60	#include <bcm_ring.h>
64		-#define DHD_LB_WORKQ_SZ (8192)
	61	+#define DHD_LB_WORKQ_SZ (8192)
65	62	#define DHD_LB_WORKQ_SYNC (16)
66	63	#define DHD_LB_WORK_SCHED (DHD_LB_WORKQ_SYNC * 2)
67	64	#endif /* DHD_LB */
68	65
	66	+#include <etd.h>
	67	+#include <hnd_debug.h>
	68	+#include <bcmtlv.h>
	69	+#include <hnd_armtrap.h>
	70	+#include <dnglevent.h>
	71	+
	72	+#ifdef DHD_PKT_LOGGING
	73	+#include <dhd_pktlog.h>
	74	+#include <dhd_linux_pktdump.h>
	75	+#endif /* DHD_PKT_LOGGING */
	76	+#ifdef DHD_EWPR_VER2
	77	+#include <dhd_bitpack.h>
	78	+#endif /* DHD_EWPR_VER2 */
	79	+
	80	+extern char dhd_version[];
	81	+extern char fw_version[];
69	82
70	83	/**
71	84	* Host configures a soft doorbell for d2h rings, by specifying a 32bit host
..	..	@@ -95,11 +108,13 @@
95	108
96	109	#define RX_DMA_OFFSET 8 /* Mem2mem DMA inserts an extra 8 */
97	110	#define IOCT_RETBUF_SIZE (RX_DMA_OFFSET + WLC_IOCTL_MAXLEN)
98		-#define FLOWRING_SIZE (H2DRING_TXPOST_MAX_ITEM * H2DRING_TXPOST_ITEMSIZE)
99	111
100	112	/* flags for ioctl pending status */
101	113	#define MSGBUF_IOCTL_ACK_PENDING (1<<0)
102	114	#define MSGBUF_IOCTL_RESP_PENDING (1<<1)
	115	+
	116	+#define DHD_IOCTL_REQ_PKTBUFSZ 2048
	117	+#define MSGBUF_IOCTL_MAX_RQSTLEN (DHD_IOCTL_REQ_PKTBUFSZ - H2DRING_CTRL_SUB_ITEMSIZE)
103	118
104	119	#define DMA_ALIGN_LEN 4
105	120
..	..	@@ -108,16 +123,19 @@
108	123
109	124	#ifdef BCM_HOST_BUF
110	125	#ifndef DMA_HOST_BUFFER_LEN
111		-#define DMA_HOST_BUFFER_LEN 0x80000
112		-#endif
	126	+#define DMA_HOST_BUFFER_LEN 0x200000
	127	+#endif // endif
113	128	#endif /* BCM_HOST_BUF */
114	129
115	130	#define DHD_FLOWRING_IOCTL_BUFPOST_PKTSZ 8192
116	131
117		-#define DHD_FLOWRING_MAX_EVENTBUF_POST 8
	132	+#define DHD_FLOWRING_DEFAULT_NITEMS_POSTED_H2D 1
	133	+#define DHD_FLOWRING_MAX_EVENTBUF_POST 32
118	134	#define DHD_FLOWRING_MAX_IOCTLRESPBUF_POST 8
	135	+#define DHD_H2D_INFORING_MAX_BUF_POST 32
	136	+#define DHD_MAX_TSBUF_POST 8
119	137
120		-#define DHD_PROT_FUNCS 37
	138	+#define DHD_PROT_FUNCS 43
121	139
122	140	/* Length of buffer in host for bus throughput measurement */
123	141	#define DHD_BUS_TPUT_BUF_LEN 2048
..	..	@@ -128,7 +146,12 @@
128	146	#define TXP_FLUSH_MAX_ITEMS_FLUSH_CNT 48
129	147
130	148	#define RING_NAME_MAX_LENGTH 24
	149	+#define CTRLSUB_HOSTTS_MEESAGE_SIZE 1024
	150	+/* Giving room before ioctl_trans_id rollsover. */
	151	+#define BUFFER_BEFORE_ROLLOVER 300
131	152
	153	+/* 512K memory + 32K registers */
	154	+#define SNAPSHOT_UPLOAD_BUF_SIZE ((512 + 32) * 1024)
132	155
133	156	struct msgbuf_ring; /* ring context for common and flow rings */
134	157
..	..	@@ -152,10 +175,19 @@
152	175	*
153	176	* Dongle advertizes host side sync mechanism requirements.
154	177	*/
155		-#define PCIE_D2H_SYNC
156	178
157		-#if defined(PCIE_D2H_SYNC)
158		-#define PCIE_D2H_SYNC_WAIT_TRIES 512
	179	+#define PCIE_D2H_SYNC_WAIT_TRIES (512U)
	180	+#define PCIE_D2H_SYNC_NUM_OF_STEPS (5U)
	181	+#define PCIE_D2H_SYNC_DELAY (100UL) /* in terms of usecs */
	182	+
	183	+#define HWA_DB_TYPE_RXPOST (0x0050)
	184	+#define HWA_DB_TYPE_TXCPLT (0x0060)
	185	+#define HWA_DB_TYPE_RXCPLT (0x0170)
	186	+#define HWA_DB_INDEX_VALUE(val) ((uint32)(val) << 16)
	187	+
	188	+#define HWA_ENAB_BITMAP_RXPOST (1U << 0) /* 1A */
	189	+#define HWA_ENAB_BITMAP_RXCPLT (1U << 1) /* 2B */
	190	+#define HWA_ENAB_BITMAP_TXCPLT (1U << 2) /* 4B */
159	191
160	192	/**
161	193	* Custom callback attached based upon D2H DMA Sync mode advertized by dongle.
..	..	@@ -165,8 +197,18 @@
165	197	*/
166	198	typedef uint8 (* d2h_sync_cb_t)(dhd_pub_t dhd, struct msgbuf_ring ring,
167	199	volatile cmn_msg_hdr_t *msg, int msglen);
168		-#endif /* PCIE_D2H_SYNC */
169	200
	201	+/**
	202	+ * Custom callback attached based upon D2H DMA Sync mode advertized by dongle.
	203	+ * For EDL messages.
	204	+ *
	205	+ * On success: return cmn_msg_hdr_t::msg_type
	206	+ * On failure: return 0 (invalid msg_type)
	207	+ */
	208	+#ifdef EWP_EDL
	209	+typedef int (* d2h_edl_sync_cb_t)(dhd_pub_t dhd, struct msgbuf_ring ring,
	210	+ volatile cmn_msg_hdr_t *msg);
	211	+#endif /* EWP_EDL */
170	212
171	213	/*
172	214	* +----------------------------------------------------------------------------
..	..	@@ -220,8 +262,9 @@
220	262	#define DHD_FLOWRING_START_FLOWID BCMPCIE_H2D_COMMON_MSGRINGS
221	263
222	264	/* Determine whether a ringid belongs to a TxPost flowring */
223		-#define DHD_IS_FLOWRING(ringid) \
224		- ((ringid) >= BCMPCIE_COMMON_MSGRINGS)
	265	+#define DHD_IS_FLOWRING(ringid, max_flow_rings) \
	266	+ ((ringid) >= BCMPCIE_COMMON_MSGRINGS && \
	267	+ (ringid) < ((max_flow_rings) + BCMPCIE_COMMON_MSGRINGS))
225	268
226	269	/* Convert a H2D TxPost FlowId to a MsgBuf RingId */
227	270	#define DHD_FLOWID_TO_RINGID(flowid) \
..	..	@@ -236,19 +279,28 @@
236	279	* any array of H2D rings.
237	280	*/
238	281	#define DHD_H2D_RING_OFFSET(ringid) \
239		- ((DHD_IS_FLOWRING(ringid)) ? DHD_RINGID_TO_FLOWID(ringid) : (ringid))
	282	+ (((ringid) >= BCMPCIE_COMMON_MSGRINGS) ? DHD_RINGID_TO_FLOWID(ringid) : (ringid))
	283	+
	284	+/* Convert a H2D MsgBuf Flowring Id to an offset index into the H2D DMA indices array
	285	+ * This may be used for IFRM.
	286	+ */
	287	+#define DHD_H2D_FRM_FLOW_RING_OFFSET(ringid) \
	288	+ ((ringid) - BCMPCIE_COMMON_MSGRINGS)
240	289
241	290	/* Convert a D2H MsgBuf RingId to an offset index into the D2H DMA indices array
242	291	* This may be used for the D2H DMA WR index array or D2H DMA RD index array or
243	292	* any array of D2H rings.
	293	+ * d2h debug ring is located at the end, i.e. after all the tx flow rings and h2d debug ring
	294	+ * max_h2d_rings: total number of h2d rings
244	295	*/
245		-#define DHD_D2H_RING_OFFSET(ringid) \
246		- ((ringid) - BCMPCIE_H2D_COMMON_MSGRINGS)
	296	+#define DHD_D2H_RING_OFFSET(ringid, max_h2d_rings) \
	297	+ ((ringid) > (max_h2d_rings) ? \
	298	+ ((ringid) - max_h2d_rings) : \
	299	+ ((ringid) - BCMPCIE_H2D_COMMON_MSGRINGS))
247	300
248	301	/* Convert a D2H DMA Indices Offset to a RingId */
249	302	#define DHD_D2H_RINGID(offset) \
250	303	((offset) + BCMPCIE_H2D_COMMON_MSGRINGS)
251		-
252	304
253	305	#define DHD_DMAH_NULL ((void*)NULL)
254	306
..	..	@@ -263,7 +315,65 @@
263	315	#define DHD_DMA_PAD (L1_CACHE_BYTES)
264	316	#else
265	317	#define DHD_DMA_PAD (128)
266		-#endif
	318	+#endif // endif
	319	+
	320	+/*
	321	+ * +----------------------------------------------------------------------------
	322	+ * Flowring Pool
	323	+ *
	324	+ * Unlike common rings, which are attached very early on (dhd_prot_attach),
	325	+ * flowrings are dynamically instantiated. Moreover, flowrings may require a
	326	+ * larger DMA-able buffer. To avoid issues with fragmented cache coherent
	327	+ * DMA-able memory, a pre-allocated pool of msgbuf_ring_t is allocated once.
	328	+ * The DMA-able buffers are attached to these pre-allocated msgbuf_ring.
	329	+ *
	330	+ * Each DMA-able buffer may be allocated independently, or may be carved out
	331	+ * of a single large contiguous region that is registered with the protocol
	332	+ * layer into flowrings_dma_buf. On a 64bit platform, this contiguous region
	333	+ * may not span 0x00000000FFFFFFFF (avoid dongle side 64bit ptr arithmetic).
	334	+ *
	335	+ * No flowring pool action is performed in dhd_prot_attach(), as the number
	336	+ * of h2d rings is not yet known.
	337	+ *
	338	+ * In dhd_prot_init(), the dongle advertized number of h2d rings is used to
	339	+ * determine the number of flowrings required, and a pool of msgbuf_rings are
	340	+ * allocated and a DMA-able buffer (carved or allocated) is attached.
	341	+ * See: dhd_prot_flowrings_pool_attach()
	342	+ *
	343	+ * A flowring msgbuf_ring object may be fetched from this pool during flowring
	344	+ * creation, using the flowid. Likewise, flowrings may be freed back into the
	345	+ * pool on flowring deletion.
	346	+ * See: dhd_prot_flowrings_pool_fetch(), dhd_prot_flowrings_pool_release()
	347	+ *
	348	+ * In dhd_prot_detach(), the flowring pool is detached. The DMA-able buffers
	349	+ * are detached (returned back to the carved region or freed), and the pool of
	350	+ * msgbuf_ring and any objects allocated against it are freed.
	351	+ * See: dhd_prot_flowrings_pool_detach()
	352	+ *
	353	+ * In dhd_prot_reset(), the flowring pool is simply reset by returning it to a
	354	+ * state as-if upon an attach. All DMA-able buffers are retained.
	355	+ * Following a dhd_prot_reset(), in a subsequent dhd_prot_init(), the flowring
	356	+ * pool attach will notice that the pool persists and continue to use it. This
	357	+ * will avoid the case of a fragmented DMA-able region.
	358	+ *
	359	+ * +----------------------------------------------------------------------------
	360	+ */
	361	+
	362	+/* Conversion of a flowid to a flowring pool index */
	363	+#define DHD_FLOWRINGS_POOL_OFFSET(flowid) \
	364	+ ((flowid) - BCMPCIE_H2D_COMMON_MSGRINGS)
	365	+
	366	+/* Fetch the msgbuf_ring_t from the flowring pool given a flowid */
	367	+#define DHD_RING_IN_FLOWRINGS_POOL(prot, flowid) \
	368	+ (msgbuf_ring_t*)((prot)->h2d_flowrings_pool) + \
	369	+ DHD_FLOWRINGS_POOL_OFFSET(flowid)
	370	+
	371	+/* Traverse each flowring in the flowring pool, assigning ring and flowid */
	372	+#define FOREACH_RING_IN_FLOWRINGS_POOL(prot, ring, flowid, total_flowrings) \
	373	+ for ((flowid) = DHD_FLOWRING_START_FLOWID, \
	374	+ (ring) = DHD_RING_IN_FLOWRINGS_POOL(prot, flowid); \
	375	+ (flowid) < ((total_flowrings) + DHD_FLOWRING_START_FLOWID); \
	376	+ (ring)++, (flowid)++)
267	377
268	378	/* Used in loopback tests */
269	379	typedef struct dhd_dmaxfer {
..	..	@@ -273,6 +383,10 @@
273	383	uint32 destdelay;
274	384	uint32 len;
275	385	bool in_progress;
	386	+ uint64 start_usec;
	387	+ uint64 time_taken;
	388	+ uint32 d11_lpbk;
	389	+ int status;
276	390	} dhd_dmaxfer_t;
277	391
278	392	/**
..	..	@@ -290,6 +404,7 @@
290	404	bool inited;
291	405	uint16 idx; /* ring id */
292	406	uint16 rd; /* read index */
	407	+ uint16 curr_rd; /* read index for debug */
293	408	uint16 wr; /* write index */
294	409	uint16 max_items; /* maximum number of items in ring */
295	410	uint16 item_len; /* length of each item in the ring */
..	..	@@ -301,7 +416,17 @@
301	416	/* # of messages on ring not yet announced to dongle */
302	417	uint16 pend_items_count;
303	418	#endif /* TXP_FLUSH_NITEMS */
	419	+
	420	+ uint8 ring_type;
	421	+ uint16 hwa_db_type; /* hwa type non-zero for Data path rings */
	422	+ uint8 n_completion_ids;
	423	+ bool create_pending;
	424	+ uint16 create_req_id;
	425	+ uint8 current_phase;
	426	+ uint16 compeltion_ring_ids[MAX_COMPLETION_RING_IDS_ASSOCIATED];
304	427	uchar name[RING_NAME_MAX_LENGTH];
	428	+ uint32 ring_mem_allocated;
	429	+ void *ring_lock;
305	430	} msgbuf_ring_t;
306	431
307	432	#define DHD_RING_BGN_VA(ring) ((ring)->dma_buf.va)
..	..	@@ -309,21 +434,29 @@
309	434	((uint8 *)(DHD_RING_BGN_VA((ring))) + \
310	435	(((ring)->max_items - 1) * (ring)->item_len))
311	436
312		-
	437	+/* This can be overwritten by module parameter defined in dhd_linux.c
	438	+ * or by dhd iovar h2d_max_txpost.
	439	+ */
	440	+int h2d_max_txpost = H2DRING_TXPOST_MAX_ITEM;
313	441
314	442	/** DHD protocol handle. Is an opaque type to other DHD software layers. */
315	443	typedef struct dhd_prot {
316	444	osl_t osh; / OSL handle */
	445	+ uint16 rxbufpost_sz;
317	446	uint16 rxbufpost;
318	447	uint16 max_rxbufpost;
319	448	uint16 max_eventbufpost;
320	449	uint16 max_ioctlrespbufpost;
	450	+ uint16 max_tsbufpost;
	451	+ uint16 max_infobufpost;
	452	+ uint16 infobufpost;
321	453	uint16 cur_event_bufs_posted;
322	454	uint16 cur_ioctlresp_bufs_posted;
	455	+ uint16 cur_ts_bufs_posted;
323	456
324	457	/* Flow control mechanism based on active transmits pending */
325		- uint16 active_tx_count; /* increments on every packet tx, and decrements on tx_status */
326		- uint16 max_tx_count;
	458	+ osl_atomic_t active_tx_count; /* increments/decrements on every packet tx/tx_status */
	459	+ uint16 h2d_max_txpost;
327	460	uint16 txp_threshold; /* optimization to write "n" tx items at a time to ring */
328	461
329	462	/* MsgBuf Ring info: has a dhd_dma_buf that is dynamically allocated */
..	..	@@ -332,6 +465,9 @@
332	465	msgbuf_ring_t d2hring_ctrl_cpln; /* D2H ctrl completion ring */
333	466	msgbuf_ring_t d2hring_tx_cpln; /* D2H Tx complete message ring */
334	467	msgbuf_ring_t d2hring_rx_cpln; /* D2H Rx complete message ring */
	468	+ msgbuf_ring_t h2dring_info_subn; / H2D info submission ring */
	469	+ msgbuf_ring_t d2hring_info_cpln; / D2H info completion ring */
	470	+ msgbuf_ring_t d2hring_edl; / D2H Enhanced Debug Lane (EDL) ring */
335	471
336	472	msgbuf_ring_t h2d_flowrings_pool; / Pool of preallocated flowings */
337	473	dhd_dma_buf_t flowrings_dma_buf; /* Contiguous DMA buffer for flowrings */
..	..	@@ -340,6 +476,7 @@
340	476	uint32 rx_dataoffset;
341	477
342	478	dhd_mb_ring_t mb_ring_fn; /* called when dongle needs to be notified of new msg */
	479	+ dhd_mb_ring_2_t mb_2_ring_fn; /* called when dongle needs to be notified of new msg */
343	480
344	481	/* ioctl related resources */
345	482	uint8 ioctl_state;
..	..	@@ -358,37 +495,52 @@
358	495	dhd_dma_buf_t h2d_dma_indx_rd_buf; /* Array of H2D RD indices */
359	496	dhd_dma_buf_t d2h_dma_indx_wr_buf; /* Array of D2H WR indices */
360	497	dhd_dma_buf_t d2h_dma_indx_rd_buf; /* Array of D2H RD indices */
	498	+ dhd_dma_buf_t h2d_ifrm_indx_wr_buf; /* Array of H2D WR indices for ifrm */
361	499
362	500	dhd_dma_buf_t host_bus_throughput_buf; /* bus throughput measure buffer */
363	501
364	502	dhd_dma_buf_t flowring_buf; / pool of flow ring buf */
365	503	uint32 flowring_num;
366	504
367		-#if defined(PCIE_D2H_SYNC)
368	505	d2h_sync_cb_t d2h_sync_cb; /* Sync on D2H DMA done: SEQNUM or XORCSUM */
	506	+#ifdef EWP_EDL
	507	+ d2h_edl_sync_cb_t d2h_edl_sync_cb; /* Sync on EDL D2H DMA done: SEQNUM or XORCSUM */
	508	+#endif /* EWP_EDL */
369	509	ulong d2h_sync_wait_max; /* max number of wait loops to receive one msg */
370	510	ulong d2h_sync_wait_tot; /* total wait loops */
371		-#endif /* PCIE_D2H_SYNC */
372	511
373	512	dhd_dmaxfer_t dmaxfer; /* for test/DMA loopback */
374	513
375	514	uint16 ioctl_seq_no;
376	515	uint16 data_seq_no;
377	516	uint16 ioctl_trans_id;
378		- void pktid_map_handle; / a pktid maps to a packet and its metadata */
	517	+ void pktid_ctrl_map; / a pktid maps to a packet and its metadata */
	518	+ void pktid_rx_map; / pktid map for rx path */
	519	+ void pktid_tx_map; / pktid map for tx path */
379	520	bool metadata_dbg;
380	521	void *pktid_map_handle_ioctl;
	522	+#ifdef DHD_MAP_PKTID_LOGGING
	523	+ void pktid_dma_map; / pktid map for DMA MAP */
	524	+ void pktid_dma_unmap; / pktid map for DMA UNMAP */
	525	+#endif /* DHD_MAP_PKTID_LOGGING */
	526	+ uint32 pktid_depleted_cnt; /* pktid depleted count */
	527	+ /* netif tx queue stop count */
	528	+ uint8 pktid_txq_stop_cnt;
	529	+ /* netif tx queue start count */
	530	+ uint8 pktid_txq_start_cnt;
	531	+ uint64 ioctl_fillup_time; /* timestamp for ioctl fillup */
	532	+ uint64 ioctl_ack_time; /* timestamp for ioctl ack */
	533	+ uint64 ioctl_cmplt_time; /* timestamp for ioctl completion */
381	534
382	535	/* Applications/utilities can read tx and rx metadata using IOVARs */
383	536	uint16 rx_metadata_offset;
384	537	uint16 tx_metadata_offset;
385	538
386		-
387	539	#if defined(DHD_D2H_SOFT_DOORBELL_SUPPORT)
388	540	/* Host's soft doorbell configuration */
389	541	bcmpcie_soft_doorbell_t soft_doorbell[BCMPCIE_D2H_COMMON_MSGRINGS];
390	542	#endif /* DHD_D2H_SOFT_DOORBELL_SUPPORT */
391		-#if defined(DHD_LB)
	543	+
392	544	/* Work Queues to be used by the producer and the consumer, and threshold
393	545	* when the WRITE index must be synced to consumer's workq
394	546	*/
..	..	@@ -400,17 +552,50 @@
400	552	uint32 rx_compl_prod_sync ____cacheline_aligned;
401	553	bcm_workq_t rx_compl_prod, rx_compl_cons;
402	554	#endif /* DHD_LB_RXC */
403		-#endif /* DHD_LB */
	555	+
	556	+ dhd_dma_buf_t fw_trap_buf; /* firmware trap buffer */
	557	+
	558	+ uint32 host_ipc_version; /* Host sypported IPC rev */
	559	+ uint32 device_ipc_version; /* FW supported IPC rev */
	560	+ uint32 active_ipc_version; /* Host advertised IPC rev */
	561	+ dhd_dma_buf_t hostts_req_buf; /* For holding host timestamp request buf */
	562	+ bool hostts_req_buf_inuse;
	563	+ bool rx_ts_log_enabled;
	564	+ bool tx_ts_log_enabled;
	565	+ bool no_retry;
	566	+ bool no_aggr;
	567	+ bool fixed_rate;
	568	+ dhd_dma_buf_t host_scb_buf; /* scb host offload buffer */
	569	+#ifdef DHD_HP2P
	570	+ msgbuf_ring_t d2hring_hp2p_txcpl; / D2H HPP Tx completion ring */
	571	+ msgbuf_ring_t d2hring_hp2p_rxcpl; / D2H HPP Rx completion ring */
	572	+#endif /* DHD_HP2P */
	573	+ bool no_tx_resource;
404	574	} dhd_prot_t;
405	575
	576	+#ifdef DHD_EWPR_VER2
	577	+#define HANG_INFO_BASE64_BUFFER_SIZE 640
	578	+#endif // endif
	579	+
	580	+#ifdef DHD_DUMP_PCIE_RINGS
	581	+static
	582	+int dhd_ring_write(dhd_pub_t dhd, msgbuf_ring_t ring, void *file,
	583	+ const void user_buf, unsigned long file_posn);
	584	+#ifdef EWP_EDL
	585	+static
	586	+int dhd_edl_ring_hdr_write(dhd_pub_t dhd, msgbuf_ring_t ring, void file, const void user_buf,
	587	+ unsigned long *file_posn);
	588	+#endif /* EWP_EDL */
	589	+#endif /* DHD_DUMP_PCIE_RINGS */
	590	+
	591	+extern bool dhd_timesync_delay_post_bufs(dhd_pub_t *dhdp);
	592	+extern void dhd_schedule_dmaxfer_free(dhd_pub_t* dhdp, dmaxref_mem_map_t *dmmap);
406	593	/* Convert a dmaaddr_t to a base_addr with htol operations */
407	594	static INLINE void dhd_base_addr_htolpa(sh_addr_t *base_addr, dmaaddr_t pa);
408	595
409	596	/* APIs for managing a DMA-able buffer */
410	597	static int dhd_dma_buf_audit(dhd_pub_t dhd, dhd_dma_buf_t dma_buf);
411		-static int dhd_dma_buf_alloc(dhd_pub_t dhd, dhd_dma_buf_t dma_buf, uint32 buf_len);
412	598	static void dhd_dma_buf_reset(dhd_pub_t dhd, dhd_dma_buf_t dma_buf);
413		-static void dhd_dma_buf_free(dhd_pub_t dhd, dhd_dma_buf_t dma_buf);
414	599
415	600	/* msgbuf ring management */
416	601	static int dhd_prot_ring_attach(dhd_pub_t dhd, msgbuf_ring_t ring,
..	..	@@ -418,6 +603,7 @@
418	603	static void dhd_prot_ring_init(dhd_pub_t dhd, msgbuf_ring_t ring);
419	604	static void dhd_prot_ring_reset(dhd_pub_t dhd, msgbuf_ring_t ring);
420	605	static void dhd_prot_ring_detach(dhd_pub_t dhd, msgbuf_ring_t ring);
	606	+static void dhd_prot_process_fw_timestamp(dhd_pub_t dhd, void buf);
421	607
422	608	/* Pool of pre-allocated msgbuf_ring_t with DMA-able buffers for Flowrings */
423	609	static int dhd_prot_flowrings_pool_attach(dhd_pub_t *dhd);
..	..	@@ -444,13 +630,12 @@
444	630	void *p, uint16 len);
445	631	static void dhd_prot_upd_read_idx(dhd_pub_t dhd, msgbuf_ring_t ring);
446	632
447		-/* Allocate DMA-able memory for saving H2D/D2H WR/RD indices */
448	633	static INLINE int dhd_prot_dma_indx_alloc(dhd_pub_t *dhd, uint8 type,
449	634	dhd_dma_buf_t *dma_buf, uint32 bufsz);
450	635
451	636	/* Set/Get a RD or WR index in the array of indices */
452	637	/* See also: dhd_prot_dma_indx_init() */
453		-static void dhd_prot_dma_indx_set(dhd_pub_t *dhd, uint16 new_index, uint8 type,
	638	+void dhd_prot_dma_indx_set(dhd_pub_t *dhd, uint16 new_index, uint8 type,
454	639	uint16 ringid);
455	640	static uint16 dhd_prot_dma_indx_get(dhd_pub_t *dhd, uint8 type, uint16 ringid);
456	641
..	..	@@ -469,11 +654,12 @@
469	654	void *buf, int ifidx);
470	655
471	656	/* Post buffers for Rx, control ioctl response and events */
472		-static uint16 dhd_msgbuf_rxbuf_post_ctrlpath(dhd_pub_t *dhd, bool event_buf, uint32 max_to_post);
	657	+static uint16 dhd_msgbuf_rxbuf_post_ctrlpath(dhd_pub_t *dhd, uint8 msgid, uint32 max_to_post);
473	658	static void dhd_msgbuf_rxbuf_post_ioctlresp_bufs(dhd_pub_t *pub);
474	659	static void dhd_msgbuf_rxbuf_post_event_bufs(dhd_pub_t *pub);
475	660	static void dhd_msgbuf_rxbuf_post(dhd_pub_t *dhd, bool use_rsv_pktid);
476	661	static int dhd_prot_rxbuf_post(dhd_pub_t *dhd, uint16 count, bool use_rsv_pktid);
	662	+static int dhd_msgbuf_rxbuf_post_ts_bufs(dhd_pub_t *pub);
477	663
478	664	static void dhd_prot_return_rxbuf(dhd_pub_t *dhd, uint32 pktid, uint32 rxcnt);
479	665
..	..	@@ -487,7 +673,6 @@
487	673	static void dhd_prot_ioctack_process(dhd_pub_t dhd, void msg);
488	674	static void dhd_prot_ringstatus_process(dhd_pub_t dhd, void msg);
489	675	static void dhd_prot_genstatus_process(dhd_pub_t dhd, void msg);
490		-static void dhd_prot_rxcmplt_process(dhd_pub_t dhd, void msg);
491	676	static void dhd_prot_event_process(dhd_pub_t dhd, void msg);
492	677
493	678	/* Loopback test with dongle */
..	..	@@ -500,11 +685,38 @@
500	685	static void dhd_prot_flow_ring_create_response_process(dhd_pub_t dhd, void msg);
501	686	static void dhd_prot_flow_ring_delete_response_process(dhd_pub_t dhd, void msg);
502	687	static void dhd_prot_flow_ring_flush_response_process(dhd_pub_t dhd, void msg);
	688	+static void dhd_prot_process_flow_ring_resume_response(dhd_pub_t dhd, void msg);
	689	+static void dhd_prot_process_flow_ring_suspend_response(dhd_pub_t dhd, void msg);
	690	+
	691	+/* Monitor Mode */
	692	+#ifdef WL_MONITOR
	693	+extern bool dhd_monitor_enabled(dhd_pub_t *dhd, int ifidx);
	694	+extern void dhd_rx_mon_pkt(dhd_pub_t dhdp, host_rxbuf_cmpl_t msg, void *pkt, int ifidx);
	695	+#endif /* WL_MONITOR */
503	696
504	697	/* Configure a soft doorbell per D2H ring */
505	698	static void dhd_msgbuf_ring_config_d2h_soft_doorbell(dhd_pub_t *dhd);
506		-static void dhd_prot_d2h_ring_config_cmplt_process(dhd_pub_t dhd, void msg);
	699	+static void dhd_prot_process_d2h_ring_config_complete(dhd_pub_t dhd, void msg);
	700	+static void dhd_prot_process_d2h_ring_create_complete(dhd_pub_t dhd, void buf);
	701	+static void dhd_prot_process_h2d_ring_create_complete(dhd_pub_t dhd, void buf);
	702	+static void dhd_prot_process_infobuf_complete(dhd_pub_t dhd, void buf);
	703	+static void dhd_prot_process_d2h_mb_data(dhd_pub_t dhd, void buf);
	704	+static void dhd_prot_detach_info_rings(dhd_pub_t *dhd);
	705	+#ifdef DHD_HP2P
	706	+static void dhd_prot_detach_hp2p_rings(dhd_pub_t *dhd);
	707	+#endif /* DHD_HP2P */
	708	+#ifdef EWP_EDL
	709	+static void dhd_prot_detach_edl_rings(dhd_pub_t *dhd);
	710	+#endif // endif
	711	+static void dhd_prot_process_d2h_host_ts_complete(dhd_pub_t dhd, void buf);
	712	+static void dhd_prot_process_snapshot_complete(dhd_pub_t dhd, void buf);
507	713
	714	+#ifdef DHD_HP2P
	715	+static void dhd_update_hp2p_rxstats(dhd_pub_t dhd, host_rxbuf_cmpl_t rxstatus);
	716	+static void dhd_update_hp2p_txstats(dhd_pub_t dhd, host_txbuf_cmpl_t txstatus);
	717	+static void dhd_calc_hp2p_burst(dhd_pub_t dhd, msgbuf_ring_t ring, uint16 flowid);
	718	+static void dhd_update_hp2p_txdesc(dhd_pub_t dhd, host_txbuf_post_t txdesc);
	719	+#endif // endif
508	720	typedef void (dhd_msgbuf_func_t)(dhd_pub_t dhd, void *msg);
509	721
510	722	/** callback functions for messages generated by the dongle */
..	..	@@ -529,32 +741,38 @@
529	741	NULL,
530	742	dhd_prot_txstatus_process, /* MSG_TYPE_TX_STATUS */
531	743	NULL,
532		- dhd_prot_rxcmplt_process, /* MSG_TYPE_RX_CMPLT */
	744	+ NULL, /* MSG_TYPE_RX_CMPLT use dedicated handler */
533	745	NULL,
534	746	dhd_msgbuf_dmaxfer_process, /* MSG_TYPE_LPBK_DMAXFER_CMPLT */
535	747	NULL, /* MSG_TYPE_FLOW_RING_RESUME */
536		- NULL, /* MSG_TYPE_FLOW_RING_RESUME_CMPLT */
	748	+ dhd_prot_process_flow_ring_resume_response, /* MSG_TYPE_FLOW_RING_RESUME_CMPLT */
537	749	NULL, /* MSG_TYPE_FLOW_RING_SUSPEND */
538		- NULL, /* MSG_TYPE_FLOW_RING_SUSPEND_CMPLT */
	750	+ dhd_prot_process_flow_ring_suspend_response, /* MSG_TYPE_FLOW_RING_SUSPEND_CMPLT */
539	751	NULL, /* MSG_TYPE_INFO_BUF_POST */
540		- NULL, /* MSG_TYPE_INFO_BUF_CMPLT */
	752	+ dhd_prot_process_infobuf_complete, /* MSG_TYPE_INFO_BUF_CMPLT */
541	753	NULL, /* MSG_TYPE_H2D_RING_CREATE */
542	754	NULL, /* MSG_TYPE_D2H_RING_CREATE */
543		- NULL, /* MSG_TYPE_H2D_RING_CREATE_CMPLT */
544		- NULL, /* MSG_TYPE_D2H_RING_CREATE_CMPLT */
	755	+ dhd_prot_process_h2d_ring_create_complete, /* MSG_TYPE_H2D_RING_CREATE_CMPLT */
	756	+ dhd_prot_process_d2h_ring_create_complete, /* MSG_TYPE_D2H_RING_CREATE_CMPLT */
545	757	NULL, /* MSG_TYPE_H2D_RING_CONFIG */
546	758	NULL, /* MSG_TYPE_D2H_RING_CONFIG */
547	759	NULL, /* MSG_TYPE_H2D_RING_CONFIG_CMPLT */
548		- dhd_prot_d2h_ring_config_cmplt_process, /* MSG_TYPE_D2H_RING_CONFIG_CMPLT */
	760	+ dhd_prot_process_d2h_ring_config_complete, /* MSG_TYPE_D2H_RING_CONFIG_CMPLT */
549	761	NULL, /* MSG_TYPE_H2D_MAILBOX_DATA */
550		- NULL, /* MSG_TYPE_D2H_MAILBOX_DATA */
	762	+ dhd_prot_process_d2h_mb_data, /* MSG_TYPE_D2H_MAILBOX_DATA */
	763	+ NULL, /* MSG_TYPE_TIMSTAMP_BUFPOST */
	764	+ NULL, /* MSG_TYPE_HOSTTIMSTAMP */
	765	+ dhd_prot_process_d2h_host_ts_complete, /* MSG_TYPE_HOSTTIMSTAMP_CMPLT */
	766	+ dhd_prot_process_fw_timestamp, /* MSG_TYPE_FIRMWARE_TIMESTAMP */
	767	+ NULL, /* MSG_TYPE_SNAPSHOT_UPLOAD */
	768	+ dhd_prot_process_snapshot_complete, /* MSG_TYPE_SNAPSHOT_CMPLT */
551	769	};
552		-
553	770
554	771	#ifdef DHD_RX_CHAINING
555	772
556	773	#define PKT_CTF_CHAINABLE(dhd, ifidx, evh, prio, h_sa, h_da, h_prio) \
557		- (dhd_rx_pkt_chainable((dhd), (ifidx)) && \
	774	+ (dhd_wet_chainable(dhd) && \
	775	+ dhd_rx_pkt_chainable((dhd), (ifidx)) && \
558	776	!ETHER_ISNULLDEST(((struct ether_header *)(evh))->ether_dhost) && \
559	777	!ETHER_ISMULTI(((struct ether_header *)(evh))->ether_dhost) && \
560	778	!eacmp((h_da), ((struct ether_header *)(evh))->ether_dhost) && \
..	..	@@ -571,10 +789,49 @@
571	789
572	790	#endif /* DHD_RX_CHAINING */
573	791
	792	+#define DHD_LPBKDTDUMP_ON() (dhd_msg_level & DHD_LPBKDTDUMP_VAL)
	793	+
574	794	static void dhd_prot_h2d_sync_init(dhd_pub_t *dhd);
575	795
576		-#if defined(PCIE_D2H_SYNC) /* avoids problems related to host CPU cache */
	796	+bool
	797	+dhd_prot_is_cmpl_ring_empty(dhd_pub_t dhd, void prot_info)
	798	+{
	799	+ msgbuf_ring_t flow_ring = (msgbuf_ring_t )prot_info;
	800	+ uint16 rd, wr;
	801	+ bool ret;
577	802
	803	+ if (dhd->dma_d2h_ring_upd_support) {
	804	+ wr = flow_ring->wr;
	805	+ } else {
	806	+ dhd_bus_cmn_readshared(dhd->bus, &wr, RING_WR_UPD, flow_ring->idx);
	807	+ }
	808	+ if (dhd->dma_h2d_ring_upd_support) {
	809	+ rd = flow_ring->rd;
	810	+ } else {
	811	+ dhd_bus_cmn_readshared(dhd->bus, &rd, RING_RD_UPD, flow_ring->idx);
	812	+ }
	813	+ ret = (wr == rd) ? TRUE : FALSE;
	814	+ return ret;
	815	+}
	816	+
	817	+void
	818	+dhd_prot_dump_ring_ptrs(void *prot_info)
	819	+{
	820	+ msgbuf_ring_t ring = (msgbuf_ring_t )prot_info;
	821	+ DHD_ERROR(("%s curr_rd: %d rd: %d wr: %d \n", __FUNCTION__,
	822	+ ring->curr_rd, ring->rd, ring->wr));
	823	+}
	824	+
	825	+uint16
	826	+dhd_prot_get_h2d_max_txpost(dhd_pub_t *dhd)
	827	+{
	828	+ return (uint16)h2d_max_txpost;
	829	+}
	830	+void
	831	+dhd_prot_set_h2d_max_txpost(dhd_pub_t *dhd, uint16 max_txpost)
	832	+{
	833	+ h2d_max_txpost = max_txpost;
	834	+}
578	835	/**
579	836	* D2H DMA to completion callback handlers. Based on the mode advertised by the
580	837	* dongle through the PCIE shared region, the appropriate callback will be
..	..	@@ -583,8 +840,8 @@
583	840	* does not require host participation, then a noop callback handler will be
584	841	* bound that simply returns the msg_type.
585	842	*/
586		-static void dhd_prot_d2h_sync_livelock(dhd_pub_t dhd, msgbuf_ring_t ring,
587		- uint32 tries, uchar *msg, int msglen);
	843	+static void dhd_prot_d2h_sync_livelock(dhd_pub_t dhd, uint32 msg_seqnum, msgbuf_ring_t ring,
	844	+ uint32 tries, volatile uchar *msg, int msglen);
588	845	static uint8 dhd_prot_d2h_sync_seqnum(dhd_pub_t dhd, msgbuf_ring_t ring,
589	846	volatile cmn_msg_hdr_t *msg, int msglen);
590	847	static uint8 dhd_prot_d2h_sync_xorcsum(dhd_pub_t dhd, msgbuf_ring_t ring,
..	..	@@ -592,6 +849,10 @@
592	849	static uint8 dhd_prot_d2h_sync_none(dhd_pub_t dhd, msgbuf_ring_t ring,
593	850	volatile cmn_msg_hdr_t *msg, int msglen);
594	851	static void dhd_prot_d2h_sync_init(dhd_pub_t *dhd);
	852	+static int dhd_send_d2h_ringcreate(dhd_pub_t dhd, msgbuf_ring_t ring_to_create,
	853	+ uint16 ring_type, uint32 id);
	854	+static int dhd_send_h2d_ringcreate(dhd_pub_t dhd, msgbuf_ring_t ring_to_create,
	855	+ uint8 type, uint32 id);
595	856
596	857	/**
597	858	* dhd_prot_d2h_sync_livelock - when the host determines that a DMA transfer has
..	..	@@ -603,22 +864,60 @@
603	864	*
604	865	*/
605	866	static void
606		-dhd_prot_d2h_sync_livelock(dhd_pub_t dhd, msgbuf_ring_t ring, uint32 tries,
607		- uchar *msg, int msglen)
	867	+dhd_prot_d2h_sync_livelock(dhd_pub_t dhd, uint32 msg_seqnum, msgbuf_ring_t ring, uint32 tries,
	868	+ volatile uchar *msg, int msglen)
608	869	{
609		- uint32 seqnum = ring->seqnum;
	870	+ uint32 ring_seqnum = ring->seqnum;
610	871
611		- DHD_ERROR(("LIVELOCK DHD<%p> seqnum<%u:%u> tries<%u> max<%lu> tot<%lu>"
612		- "dma_buf va<%p> msg<%p>\n",
613		- dhd, seqnum, seqnum% D2H_EPOCH_MODULO, tries,
	872	+ if (dhd_query_bus_erros(dhd)) {
	873	+ return;
	874	+ }
	875	+
	876	+ DHD_ERROR((
	877	+ "LIVELOCK DHD<%p> ring<%s> msg_seqnum<%u> ring_seqnum<%u:%u> tries<%u> max<%lu>"
	878	+ " tot<%lu> dma_buf va<%p> msg<%p> curr_rd<%d> rd<%d> wr<%d>\n",
	879	+ dhd, ring->name, msg_seqnum, ring_seqnum, ring_seqnum% D2H_EPOCH_MODULO, tries,
614	880	dhd->prot->d2h_sync_wait_max, dhd->prot->d2h_sync_wait_tot,
615		- ring->dma_buf.va, msg));
616		- prhex("D2H MsgBuf Failure", (uchar *)msg, msglen);
	881	+ ring->dma_buf.va, msg, ring->curr_rd, ring->rd, ring->wr));
617	882
618		-#if defined(SUPPORT_LINKDOWN_RECOVERY) && defined(CONFIG_ARCH_MSM)
619		- dhd->bus->islinkdown = 1;
620		- dhd_os_check_hang(dhd, 0, -ETIMEDOUT);
621		-#endif /* SUPPORT_LINKDOWN_RECOVERY && CONFIG_ARCH_MSM */
	883	+ dhd_prhex("D2H MsgBuf Failure", msg, msglen, DHD_ERROR_VAL);
	884	+
	885	+ /* Try to resume if already suspended or suspend in progress */
	886	+#ifdef DHD_PCIE_RUNTIMEPM
	887	+ dhdpcie_runtime_bus_wake(dhd, CAN_SLEEP(), __builtin_return_address(0));
	888	+#endif /* DHD_PCIE_RUNTIMEPM */
	889	+
	890	+ /* Skip if still in suspended or suspend in progress */
	891	+ if (DHD_BUS_CHECK_SUSPEND_OR_ANY_SUSPEND_IN_PROGRESS(dhd)) {
	892	+ DHD_ERROR(("%s: bus is in suspend(%d) or suspending(0x%x) state, so skip\n",
	893	+ __FUNCTION__, dhd->busstate, dhd->dhd_bus_busy_state));
	894	+ goto exit;
	895	+ }
	896	+
	897	+ dhd_bus_dump_console_buffer(dhd->bus);
	898	+ dhd_prot_debug_info_print(dhd);
	899	+
	900	+#ifdef DHD_FW_COREDUMP
	901	+ if (dhd->memdump_enabled) {
	902	+ /* collect core dump */
	903	+ dhd->memdump_type = DUMP_TYPE_BY_LIVELOCK;
	904	+ dhd_bus_mem_dump(dhd);
	905	+ }
	906	+#endif /* DHD_FW_COREDUMP */
	907	+
	908	+exit:
	909	+ dhd_schedule_reset(dhd);
	910	+
	911	+#ifdef OEM_ANDROID
	912	+#ifdef SUPPORT_LINKDOWN_RECOVERY
	913	+#ifdef CONFIG_ARCH_MSM
	914	+ dhd->bus->no_cfg_restore = 1;
	915	+#endif /* CONFIG_ARCH_MSM */
	916	+ dhd->hang_reason = HANG_REASON_MSGBUF_LIVELOCK;
	917	+ dhd_os_send_hang_message(dhd);
	918	+#endif /* SUPPORT_LINKDOWN_RECOVERY */
	919	+#endif /* OEM_ANDROID */
	920	+ dhd->livelock_occured = TRUE;
622	921	}
623	922
624	923	/**
..	..	@@ -632,27 +931,63 @@
632	931	uint32 tries;
633	932	uint32 ring_seqnum = ring->seqnum % D2H_EPOCH_MODULO;
634	933	int num_words = msglen / sizeof(uint32); /* num of 32bit words */
635		- volatile uint32 marker = (uint32 )msg + (num_words - 1); /* last word */
	934	+ volatile uint32 marker = (volatile uint32 )msg + (num_words - 1); /* last word */
636	935	dhd_prot_t *prot = dhd->prot;
	936	+ uint32 msg_seqnum;
	937	+ uint32 step = 0;
	938	+ uint32 delay = PCIE_D2H_SYNC_DELAY;
	939	+ uint32 total_tries = 0;
637	940
638	941	ASSERT(msglen == ring->item_len);
639	942
640		- for (tries = 0; tries < PCIE_D2H_SYNC_WAIT_TRIES; tries++) {
641		- uint32 msg_seqnum = *marker;
642		- if (ltoh32(msg_seqnum) == ring_seqnum) { /* dma upto last word done */
643		- ring->seqnum++; /* next expected sequence number */
644		- goto dma_completed;
645		- }
	943	+ BCM_REFERENCE(delay);
	944	+ /*
	945	+ * For retries we have to make some sort of stepper algorithm.
	946	+ * We see that every time when the Dongle comes out of the D3
	947	+ * Cold state, the first D2H mem2mem DMA takes more time to
	948	+ * complete, leading to livelock issues.
	949	+ *
	950	+ * Case 1 - Apart from Host CPU some other bus master is
	951	+ * accessing the DDR port, probably page close to the ring
	952	+ * so, PCIE does not get a change to update the memory.
	953	+ * Solution - Increase the number of tries.
	954	+ *
	955	+ * Case 2 - The 50usec delay given by the Host CPU is not
	956	+ * sufficient for the PCIe RC to start its work.
	957	+ * In this case the breathing time of 50usec given by
	958	+ * the Host CPU is not sufficient.
	959	+ * Solution: Increase the delay in a stepper fashion.
	960	+ * This is done to ensure that there are no
	961	+ * unwanted extra delay introdcued in normal conditions.
	962	+ */
	963	+ for (step = 1; step <= PCIE_D2H_SYNC_NUM_OF_STEPS; step++) {
	964	+ for (tries = 0; tries < PCIE_D2H_SYNC_WAIT_TRIES; tries++) {
	965	+ msg_seqnum = *marker;
	966	+ if (ltoh32(msg_seqnum) == ring_seqnum) { /* dma upto last word done */
	967	+ ring->seqnum++; /* next expected sequence number */
	968	+ /* Check for LIVELOCK induce flag, which is set by firing
	969	+ * dhd iovar to induce LIVELOCK error. If flag is set,
	970	+ * MSG_TYPE_INVALID is returned, which results in to LIVELOCK error.
	971	+ */
	972	+ if (dhd->dhd_induce_error != DHD_INDUCE_LIVELOCK) {
	973	+ goto dma_completed;
	974	+ }
	975	+ }
646	976
647		- if (tries > prot->d2h_sync_wait_max)
648		- prot->d2h_sync_wait_max = tries;
	977	+ total_tries = (uint32)(((step-1) * PCIE_D2H_SYNC_WAIT_TRIES) + tries);
649	978
650		- OSL_CACHE_INV(msg, msglen); /* invalidate and try again */
651		- OSL_CPU_RELAX(); /* CPU relax for msg_seqnum value to update */
	979	+ if (total_tries > prot->d2h_sync_wait_max)
	980	+ prot->d2h_sync_wait_max = total_tries;
652	981
653		- } /* for PCIE_D2H_SYNC_WAIT_TRIES */
	982	+ OSL_CACHE_INV(msg, msglen); /* invalidate and try again */
	983	+ OSL_CPU_RELAX(); /* CPU relax for msg_seqnum value to update */
	984	+ OSL_DELAY(delay * step); /* Add stepper delay */
654	985
655		- dhd_prot_d2h_sync_livelock(dhd, ring, tries, (uchar *)msg, msglen);
	986	+ } /* for PCIE_D2H_SYNC_WAIT_TRIES */
	987	+ } /* for PCIE_D2H_SYNC_NUM_OF_STEPS */
	988	+
	989	+ dhd_prot_d2h_sync_livelock(dhd, msg_seqnum, ring, total_tries,
	990	+ (volatile uchar *) msg, msglen);
656	991
657	992	ring->seqnum++; /* skip this message ... leak of a pktid */
658	993	return MSG_TYPE_INVALID; /* invalid msg_type 0 -> noop callback */
..	..	@@ -677,27 +1012,70 @@
677	1012	int num_words = msglen / sizeof(uint32); /* num of 32bit words */
678	1013	uint8 ring_seqnum = ring->seqnum % D2H_EPOCH_MODULO;
679	1014	dhd_prot_t *prot = dhd->prot;
	1015	+ uint32 step = 0;
	1016	+ uint32 delay = PCIE_D2H_SYNC_DELAY;
	1017	+ uint32 total_tries = 0;
680	1018
681	1019	ASSERT(msglen == ring->item_len);
682	1020
683		- for (tries = 0; tries < PCIE_D2H_SYNC_WAIT_TRIES; tries++) {
684		- prot_checksum = bcm_compute_xor32((volatile uint32 *)msg, num_words);
685		- if (prot_checksum == 0U) { /* checksum is OK */
	1021	+ BCM_REFERENCE(delay);
	1022	+ /*
	1023	+ * For retries we have to make some sort of stepper algorithm.
	1024	+ * We see that every time when the Dongle comes out of the D3
	1025	+ * Cold state, the first D2H mem2mem DMA takes more time to
	1026	+ * complete, leading to livelock issues.
	1027	+ *
	1028	+ * Case 1 - Apart from Host CPU some other bus master is
	1029	+ * accessing the DDR port, probably page close to the ring
	1030	+ * so, PCIE does not get a change to update the memory.
	1031	+ * Solution - Increase the number of tries.
	1032	+ *
	1033	+ * Case 2 - The 50usec delay given by the Host CPU is not
	1034	+ * sufficient for the PCIe RC to start its work.
	1035	+ * In this case the breathing time of 50usec given by
	1036	+ * the Host CPU is not sufficient.
	1037	+ * Solution: Increase the delay in a stepper fashion.
	1038	+ * This is done to ensure that there are no
	1039	+ * unwanted extra delay introdcued in normal conditions.
	1040	+ */
	1041	+ for (step = 1; step <= PCIE_D2H_SYNC_NUM_OF_STEPS; step++) {
	1042	+ for (tries = 0; tries < PCIE_D2H_SYNC_WAIT_TRIES; tries++) {
	1043	+ /* First verify if the seqnumber has been update,
	1044	+ * if yes, then only check xorcsum.
	1045	+ * Once seqnum and xorcsum is proper that means
	1046	+ * complete message has arrived.
	1047	+ */
686	1048	if (msg->epoch == ring_seqnum) {
687		- ring->seqnum++; /* next expected sequence number */
688		- goto dma_completed;
	1049	+ prot_checksum = bcm_compute_xor32((volatile uint32 *)msg,
	1050	+ num_words);
	1051	+ if (prot_checksum == 0U) { /* checksum is OK */
	1052	+ ring->seqnum++; /* next expected sequence number */
	1053	+ /* Check for LIVELOCK induce flag, which is set by firing
	1054	+ * dhd iovar to induce LIVELOCK error. If flag is set,
	1055	+ * MSG_TYPE_INVALID is returned, which results in to
	1056	+ * LIVELOCK error.
	1057	+ */
	1058	+ if (dhd->dhd_induce_error != DHD_INDUCE_LIVELOCK) {
	1059	+ goto dma_completed;
	1060	+ }
	1061	+ }
689	1062	}
690		- }
691	1063
692		- if (tries > prot->d2h_sync_wait_max)
693		- prot->d2h_sync_wait_max = tries;
	1064	+ total_tries = ((step-1) * PCIE_D2H_SYNC_WAIT_TRIES) + tries;
694	1065
695		- OSL_CACHE_INV(msg, msglen); /* invalidate and try again */
696		- OSL_CPU_RELAX(); /* CPU relax for msg_seqnum value to update */
	1066	+ if (total_tries > prot->d2h_sync_wait_max)
	1067	+ prot->d2h_sync_wait_max = total_tries;
697	1068
698		- } /* for PCIE_D2H_SYNC_WAIT_TRIES */
	1069	+ OSL_CACHE_INV(msg, msglen); /* invalidate and try again */
	1070	+ OSL_CPU_RELAX(); /* CPU relax for msg_seqnum value to update */
	1071	+ OSL_DELAY(delay * step); /* Add stepper delay */
699	1072
700		- dhd_prot_d2h_sync_livelock(dhd, ring, tries, (uchar *)msg, msglen);
	1073	+ } /* for PCIE_D2H_SYNC_WAIT_TRIES */
	1074	+ } /* for PCIE_D2H_SYNC_NUM_OF_STEPS */
	1075	+
	1076	+ DHD_ERROR(("%s: prot_checksum = 0x%x\n", __FUNCTION__, prot_checksum));
	1077	+ dhd_prot_d2h_sync_livelock(dhd, msg->epoch, ring, total_tries,
	1078	+ (volatile uchar *) msg, msglen);
701	1079
702	1080	ring->seqnum++; /* skip this message ... leak of a pktid */
703	1081	return MSG_TYPE_INVALID; /* invalid msg_type 0 -> noop callback */
..	..	@@ -717,7 +1095,197 @@
717	1095	dhd_prot_d2h_sync_none(dhd_pub_t dhd, msgbuf_ring_t ring,
718	1096	volatile cmn_msg_hdr_t *msg, int msglen)
719	1097	{
720		- return msg->msg_type;
	1098	+ /* Check for LIVELOCK induce flag, which is set by firing
	1099	+ * dhd iovar to induce LIVELOCK error. If flag is set,
	1100	+ * MSG_TYPE_INVALID is returned, which results in to LIVELOCK error.
	1101	+ */
	1102	+ if (dhd->dhd_induce_error == DHD_INDUCE_LIVELOCK) {
	1103	+ DHD_ERROR(("%s: Inducing livelock\n", __FUNCTION__));
	1104	+ return MSG_TYPE_INVALID;
	1105	+ } else {
	1106	+ return msg->msg_type;
	1107	+ }
	1108	+}
	1109	+
	1110	+#ifdef EWP_EDL
	1111	+/**
	1112	+ * dhd_prot_d2h_sync_edl - Sync on a D2H DMA completion by validating the cmn_msg_hdr_t
	1113	+ * header values at both the beginning and end of the payload.
	1114	+ * The cmn_msg_hdr_t is placed at the start and end of the payload
	1115	+ * in each work item in the EDL ring.
	1116	+ * Dongle will place a seqnum inside the cmn_msg_hdr_t 'epoch' field
	1117	+ * and the length of the payload in the 'request_id' field.
	1118	+ * Structure of each work item in the EDL ring:
	1119	+ * \| cmn_msg_hdr_t \| payload (var len) \| cmn_msg_hdr_t \|
	1120	+ * NOTE: - it was felt that calculating xorcsum for the entire payload (max length of 1648 bytes) is
	1121	+ * too costly on the dongle side and might take up too many ARM cycles,
	1122	+ * hence the xorcsum sync method is not being used for EDL ring.
	1123	+ */
	1124	+static int
	1125	+BCMFASTPATH(dhd_prot_d2h_sync_edl)(dhd_pub_t dhd, msgbuf_ring_t ring,
	1126	+ volatile cmn_msg_hdr_t *msg)
	1127	+{
	1128	+ uint32 tries;
	1129	+ int msglen = 0, len = 0;
	1130	+ uint32 ring_seqnum = ring->seqnum % D2H_EPOCH_MODULO;
	1131	+ dhd_prot_t *prot = dhd->prot;
	1132	+ uint32 step = 0;
	1133	+ uint32 delay = PCIE_D2H_SYNC_DELAY;
	1134	+ uint32 total_tries = 0;
	1135	+ volatile cmn_msg_hdr_t *trailer = NULL;
	1136	+ volatile uint8 *buf = NULL;
	1137	+ bool valid_msg = FALSE;
	1138	+
	1139	+ BCM_REFERENCE(delay);
	1140	+ /*
	1141	+ * For retries we have to make some sort of stepper algorithm.
	1142	+ * We see that every time when the Dongle comes out of the D3
	1143	+ * Cold state, the first D2H mem2mem DMA takes more time to
	1144	+ * complete, leading to livelock issues.
	1145	+ *
	1146	+ * Case 1 - Apart from Host CPU some other bus master is
	1147	+ * accessing the DDR port, probably page close to the ring
	1148	+ * so, PCIE does not get a change to update the memory.
	1149	+ * Solution - Increase the number of tries.
	1150	+ *
	1151	+ * Case 2 - The 50usec delay given by the Host CPU is not
	1152	+ * sufficient for the PCIe RC to start its work.
	1153	+ * In this case the breathing time of 50usec given by
	1154	+ * the Host CPU is not sufficient.
	1155	+ * Solution: Increase the delay in a stepper fashion.
	1156	+ * This is done to ensure that there are no
	1157	+ * unwanted extra delay introdcued in normal conditions.
	1158	+ */
	1159	+ for (step = 1; step <= PCIE_D2H_SYNC_NUM_OF_STEPS; step++) {
	1160	+ for (tries = 0; tries < PCIE_D2H_SYNC_WAIT_TRIES; tries++) {
	1161	+ /* First verify if the seqnumber has been updated,
	1162	+ * if yes, only then validate the header and trailer.
	1163	+ * Once seqnum, header and trailer have been validated, it means
	1164	+ * that the complete message has arrived.
	1165	+ */
	1166	+ valid_msg = FALSE;
	1167	+ if (msg->epoch == ring_seqnum &&
	1168	+ msg->msg_type == MSG_TYPE_INFO_PYLD &&
	1169	+ msg->request_id > 0 &&
	1170	+ msg->request_id <= ring->item_len) {
	1171	+ /* proceed to check trailer only if header is valid */
	1172	+ buf = (volatile uint8 *)msg;
	1173	+ msglen = sizeof(cmn_msg_hdr_t) + msg->request_id;
	1174	+ buf += msglen;
	1175	+ if (msglen + sizeof(cmn_msg_hdr_t) <= ring->item_len) {
	1176	+ trailer = (volatile cmn_msg_hdr_t *)buf;
	1177	+ valid_msg = (trailer->epoch == ring_seqnum) &&
	1178	+ (trailer->msg_type == msg->msg_type) &&
	1179	+ (trailer->request_id == msg->request_id);
	1180	+ if (!valid_msg) {
	1181	+ DHD_TRACE(("%s:invalid trailer! seqnum=%u;reqid=%u"
	1182	+ " expected, seqnum=%u; reqid=%u. Retrying... \n",
	1183	+ __FUNCTION__, trailer->epoch, trailer->request_id,
	1184	+ msg->epoch, msg->request_id));
	1185	+ }
	1186	+ } else {
	1187	+ DHD_TRACE(("%s: invalid payload length (%u)! Retrying.. \n",
	1188	+ __FUNCTION__, msg->request_id));
	1189	+ }
	1190	+
	1191	+ if (valid_msg) {
	1192	+ /* data is OK */
	1193	+ ring->seqnum++; /* next expected sequence number */
	1194	+ if (dhd->dhd_induce_error != DHD_INDUCE_LIVELOCK) {
	1195	+ goto dma_completed;
	1196	+ }
	1197	+ }
	1198	+ } else {
	1199	+ DHD_TRACE(("%s: wrong hdr, seqnum expected %u, got %u."
	1200	+ " msg_type=0x%x, request_id=%u."
	1201	+ " Retrying...\n",
	1202	+ __FUNCTION__, ring_seqnum, msg->epoch,
	1203	+ msg->msg_type, msg->request_id));
	1204	+ }
	1205	+
	1206	+ total_tries = ((step-1) * PCIE_D2H_SYNC_WAIT_TRIES) + tries;
	1207	+
	1208	+ if (total_tries > prot->d2h_sync_wait_max)
	1209	+ prot->d2h_sync_wait_max = total_tries;
	1210	+
	1211	+ OSL_CACHE_INV(msg, msglen); /* invalidate and try again */
	1212	+ OSL_CPU_RELAX(); /* CPU relax for msg_seqnum value to update */
	1213	+ OSL_DELAY(delay * step); /* Add stepper delay */
	1214	+
	1215	+ } /* for PCIE_D2H_SYNC_WAIT_TRIES */
	1216	+ } /* for PCIE_D2H_SYNC_NUM_OF_STEPS */
	1217	+
	1218	+ DHD_ERROR(("%s: EDL header check fails !\n", __FUNCTION__));
	1219	+ DHD_ERROR(("%s: header: seqnum=%u; expected-seqnum=%u"
	1220	+ " msgtype=0x%x; expected-msgtype=0x%x"
	1221	+ " length=%u; expected-max-length=%u", __FUNCTION__,
	1222	+ msg->epoch, ring_seqnum, msg->msg_type, MSG_TYPE_INFO_PYLD,
	1223	+ msg->request_id, ring->item_len));
	1224	+ dhd_prhex("msg header bytes: ", (volatile uchar )msg, sizeof(msg), DHD_ERROR_VAL);
	1225	+ if (trailer && msglen > 0 &&
	1226	+ (msglen + sizeof(cmn_msg_hdr_t)) <= ring->item_len) {
	1227	+ DHD_ERROR(("%s: trailer: seqnum=%u; expected-seqnum=%u"
	1228	+ " msgtype=0x%x; expected-msgtype=0x%x"
	1229	+ " length=%u; expected-length=%u", __FUNCTION__,
	1230	+ trailer->epoch, ring_seqnum, trailer->msg_type, MSG_TYPE_INFO_PYLD,
	1231	+ trailer->request_id, msg->request_id));
	1232	+ dhd_prhex("msg trailer bytes: ", (volatile uchar *)trailer,
	1233	+ sizeof(*trailer), DHD_ERROR_VAL);
	1234	+ }
	1235	+
	1236	+ if ((msglen + sizeof(cmn_msg_hdr_t)) <= ring->item_len)
	1237	+ len = msglen + sizeof(cmn_msg_hdr_t);
	1238	+ else
	1239	+ len = ring->item_len;
	1240	+
	1241	+ dhd_prot_d2h_sync_livelock(dhd, msg->epoch, ring, total_tries,
	1242	+ (volatile uchar *) msg, len);
	1243	+
	1244	+ ring->seqnum++; /* skip this message */
	1245	+ return BCME_ERROR; /* invalid msg_type 0 -> noop callback */
	1246	+
	1247	+dma_completed:
	1248	+ DHD_TRACE(("%s: EDL header check pass, seqnum=%u; reqid=%u\n", __FUNCTION__,
	1249	+ msg->epoch, msg->request_id));
	1250	+
	1251	+ prot->d2h_sync_wait_tot += tries;
	1252	+ return BCME_OK;
	1253	+}
	1254	+
	1255	+/**
	1256	+ * dhd_prot_d2h_sync_edl_none - Dongle ensure that the DMA will complete and host
	1257	+ * need to try to sync. This noop sync handler will be bound when the dongle
	1258	+ * advertises that neither the SEQNUM nor XORCSUM mode of DMA sync is required.
	1259	+ */
	1260	+static int BCMFASTPATH
	1261	+dhd_prot_d2h_sync_edl_none(dhd_pub_t dhd, msgbuf_ring_t ring,
	1262	+ volatile cmn_msg_hdr_t *msg)
	1263	+{
	1264	+ /* Check for LIVELOCK induce flag, which is set by firing
	1265	+ * dhd iovar to induce LIVELOCK error. If flag is set,
	1266	+ * MSG_TYPE_INVALID is returned, which results in to LIVELOCK error.
	1267	+ */
	1268	+ if (dhd->dhd_induce_error == DHD_INDUCE_LIVELOCK) {
	1269	+ DHD_ERROR(("%s: Inducing livelock\n", __FUNCTION__));
	1270	+ return BCME_ERROR;
	1271	+ } else {
	1272	+ if (msg->msg_type == MSG_TYPE_INFO_PYLD)
	1273	+ return BCME_OK;
	1274	+ else
	1275	+ return msg->msg_type;
	1276	+ }
	1277	+}
	1278	+#endif /* EWP_EDL */
	1279	+
	1280	+INLINE void
	1281	+dhd_wakeup_ioctl_event(dhd_pub_t *dhd, dhd_ioctl_recieved_status_t reason)
	1282	+{
	1283	+ /* To synchronize with the previous memory operations call wmb() */
	1284	+ OSL_SMP_WMB();
	1285	+ dhd->prot->ioctl_received = reason;
	1286	+ /* Call another wmb() to make sure before waking up the other event value gets updated */
	1287	+ OSL_SMP_WMB();
	1288	+ dhd_os_ioctl_resp_wake(dhd);
721	1289	}
722	1290
723	1291	/**
..	..	@@ -732,29 +1300,43 @@
732	1300	prot->d2h_sync_wait_tot = 0UL;
733	1301
734	1302	prot->d2hring_ctrl_cpln.seqnum = D2H_EPOCH_INIT_VAL;
	1303	+ prot->d2hring_ctrl_cpln.current_phase = BCMPCIE_CMNHDR_PHASE_BIT_INIT;
	1304	+
735	1305	prot->d2hring_tx_cpln.seqnum = D2H_EPOCH_INIT_VAL;
	1306	+ prot->d2hring_tx_cpln.current_phase = BCMPCIE_CMNHDR_PHASE_BIT_INIT;
	1307	+
736	1308	prot->d2hring_rx_cpln.seqnum = D2H_EPOCH_INIT_VAL;
	1309	+ prot->d2hring_rx_cpln.current_phase = BCMPCIE_CMNHDR_PHASE_BIT_INIT;
	1310	+
	1311	+ if (HWA_ACTIVE(dhd)) {
	1312	+ prot->d2hring_tx_cpln.hwa_db_type =
	1313	+ (dhd->bus->hwa_enab_bmap & HWA_ENAB_BITMAP_TXCPLT) ? HWA_DB_TYPE_TXCPLT : 0;
	1314	+ prot->d2hring_rx_cpln.hwa_db_type =
	1315	+ (dhd->bus->hwa_enab_bmap & HWA_ENAB_BITMAP_RXCPLT) ? HWA_DB_TYPE_RXCPLT : 0;
	1316	+ DHD_ERROR(("%s: TXCPLT hwa_db_type:0x%x RXCPLT hwa_db_type:0x%x\n",
	1317	+ __FUNCTION__, prot->d2hring_tx_cpln.hwa_db_type,
	1318	+ prot->d2hring_rx_cpln.hwa_db_type));
	1319	+ }
737	1320
738	1321	if (dhd->d2h_sync_mode & PCIE_SHARED_D2H_SYNC_SEQNUM) {
739	1322	prot->d2h_sync_cb = dhd_prot_d2h_sync_seqnum;
	1323	+#ifdef EWP_EDL
	1324	+ prot->d2h_edl_sync_cb = dhd_prot_d2h_sync_edl;
	1325	+#endif /* EWP_EDL */
	1326	+ DHD_ERROR(("%s(): D2H sync mechanism is SEQNUM \r\n", __FUNCTION__));
740	1327	} else if (dhd->d2h_sync_mode & PCIE_SHARED_D2H_SYNC_XORCSUM) {
741	1328	prot->d2h_sync_cb = dhd_prot_d2h_sync_xorcsum;
	1329	+#ifdef EWP_EDL
	1330	+ prot->d2h_edl_sync_cb = dhd_prot_d2h_sync_edl;
	1331	+#endif /* EWP_EDL */
	1332	+ DHD_ERROR(("%s(): D2H sync mechanism is XORCSUM \r\n", __FUNCTION__));
742	1333	} else {
743	1334	prot->d2h_sync_cb = dhd_prot_d2h_sync_none;
	1335	+#ifdef EWP_EDL
	1336	+ prot->d2h_edl_sync_cb = dhd_prot_d2h_sync_edl_none;
	1337	+#endif /* EWP_EDL */
	1338	+ DHD_ERROR(("%s(): D2H sync mechanism is NONE \r\n", __FUNCTION__));
744	1339	}
745		-}
746		-
747		-#endif /* PCIE_D2H_SYNC */
748		-
749		-INLINE void
750		-dhd_wakeup_ioctl_event(dhd_pub_t *dhd, dhd_ioctl_recieved_status_t reason)
751		-{
752		- /* To synchronize with the previous memory operations call wmb() */
753		- OSL_SMP_WMB();
754		- dhd->prot->ioctl_received = reason;
755		- /* Call another wmb() to make sure before waking up the other event value gets updated */
756		- OSL_SMP_WMB();
757		- dhd_os_ioctl_resp_wake(dhd);
758	1340	}
759	1341
760	1342	/**
..	..	@@ -765,11 +1347,21 @@
765	1347	{
766	1348	dhd_prot_t *prot = dhd->prot;
767	1349	prot->h2dring_rxp_subn.seqnum = H2D_EPOCH_INIT_VAL;
	1350	+
	1351	+ if (HWA_ACTIVE(dhd)) {
	1352	+ prot->h2dring_rxp_subn.hwa_db_type =
	1353	+ (dhd->bus->hwa_enab_bmap & HWA_ENAB_BITMAP_RXPOST) ? HWA_DB_TYPE_RXPOST : 0;
	1354	+ DHD_ERROR(("%s: RXPOST hwa_db_type:0x%x\n",
	1355	+ __FUNCTION__, prot->d2hring_tx_cpln.hwa_db_type));
	1356	+ }
	1357	+
	1358	+ prot->h2dring_rxp_subn.current_phase = 0;
	1359	+
768	1360	prot->h2dring_ctrl_subn.seqnum = H2D_EPOCH_INIT_VAL;
	1361	+ prot->h2dring_ctrl_subn.current_phase = 0;
769	1362	}
770	1363
771	1364	/* +----------------- End of PCIE DHD H2D DMA SYNC ------------------------+ */
772		-
773	1365
774	1366	/*
775	1367	* +---------------------------------------------------------------------------+
..	..	@@ -786,27 +1378,25 @@
786	1378	base_addr->high_addr = htol32(PHYSADDRHI(pa));
787	1379	}
788	1380
789		-
790	1381	/**
791	1382	* dhd_dma_buf_audit - Any audits on a DHD DMA Buffer.
792	1383	*/
793	1384	static int
794	1385	dhd_dma_buf_audit(dhd_pub_t dhd, dhd_dma_buf_t dma_buf)
795	1386	{
796		- uint32 base, end; /* dongle uses 32bit ptr arithmetic */
797		-
	1387	+ uint32 pa_lowaddr, end; /* dongle uses 32bit ptr arithmetic */
798	1388	ASSERT(dma_buf);
799		- base = PHYSADDRLO(dma_buf->pa);
800		- ASSERT(base);
801		- ASSERT(ISALIGNED(base, DMA_ALIGN_LEN));
	1389	+ pa_lowaddr = PHYSADDRLO(dma_buf->pa);
	1390	+ ASSERT(PHYSADDRLO(dma_buf->pa) \|\| PHYSADDRHI(dma_buf->pa));
	1391	+ ASSERT(ISALIGNED(pa_lowaddr, DMA_ALIGN_LEN));
802	1392	ASSERT(dma_buf->len != 0);
803	1393
804	1394	/* test 32bit offset arithmetic over dma buffer for loss of carry-over */
805		- end = (base + dma_buf->len); /* end address */
	1395	+ end = (pa_lowaddr + dma_buf->len); /* end address */
806	1396
807		- if ((end & 0xFFFFFFFF) < (base & 0xFFFFFFFF)) { /* exclude carryover */
	1397	+ if ((end & 0xFFFFFFFF) < (pa_lowaddr & 0xFFFFFFFF)) { /* exclude carryover */
808	1398	DHD_ERROR(("%s: dma_buf %x len %d spans dongle 32bit ptr arithmetic\n",
809		- __FUNCTION__, base, dma_buf->len));
	1399	+ __FUNCTION__, pa_lowaddr, dma_buf->len));
810	1400	return BCME_ERROR;
811	1401	}
812	1402
..	..	@@ -818,22 +1408,22 @@
818	1408	* returns BCME_OK=0 on success
819	1409	* returns non-zero negative error value on failure.
820	1410	*/
821		-static int
	1411	+int
822	1412	dhd_dma_buf_alloc(dhd_pub_t dhd, dhd_dma_buf_t dma_buf, uint32 buf_len)
823	1413	{
824	1414	uint32 dma_pad = 0;
825	1415	osl_t *osh = dhd->osh;
826		- int dma_align = DMA_ALIGN_LEN;
827		-
	1416	+ uint16 dma_align = DMA_ALIGN_LEN;
	1417	+ uint32 rem = 0;
828	1418
829	1419	ASSERT(dma_buf != NULL);
830	1420	ASSERT(dma_buf->va == NULL);
831	1421	ASSERT(dma_buf->len == 0);
832	1422
833		- /* Pad the buffer length by one extra cacheline size.
834		- * Required for D2H direction.
835		- */
836		- dma_pad = (buf_len % DHD_DMA_PAD) ? DHD_DMA_PAD : 0;
	1423	+ /* Pad the buffer length to align to cacheline size. */
	1424	+ rem = (buf_len % DHD_DMA_PAD);
	1425	+ dma_pad = rem ? (DHD_DMA_PAD - rem) : 0;
	1426	+
837	1427	dma_buf->va = DMA_ALLOC_CONSISTENT(osh, buf_len + dma_pad,
838	1428	dma_align, &dma_buf->_alloced, &dma_buf->pa, &dma_buf->dmah);
839	1429
..	..	@@ -861,9 +1451,8 @@
861	1451	static void
862	1452	dhd_dma_buf_reset(dhd_pub_t dhd, dhd_dma_buf_t dma_buf)
863	1453	{
864		- if ((dma_buf == NULL) \|\| (dma_buf->va == NULL)) {
	1454	+ if ((dma_buf == NULL) \|\| (dma_buf->va == NULL))
865	1455	return;
866		- }
867	1456
868	1457	(void)dhd_dma_buf_audit(dhd, dma_buf);
869	1458
..	..	@@ -876,16 +1465,15 @@
876	1465	* dhd_dma_buf_free - Free a DMA-able buffer that was previously allocated using
877	1466	* dhd_dma_buf_alloc().
878	1467	*/
879		-static void
	1468	+void
880	1469	dhd_dma_buf_free(dhd_pub_t dhd, dhd_dma_buf_t dma_buf)
881	1470	{
882	1471	osl_t *osh = dhd->osh;
883	1472
884	1473	ASSERT(dma_buf);
885	1474
886		- if (dma_buf->va == NULL) {
	1475	+ if (dma_buf->va == NULL)
887	1476	return; /* Allow for free invocation, when alloc failed */
888		- }
889	1477
890	1478	/* DEBUG: dhd_dma_buf_reset(dhd, dma_buf) */
891	1479	(void)dhd_dma_buf_audit(dhd, dma_buf);
..	..	@@ -922,6 +1510,133 @@
922	1510
923	1511	/*
924	1512	* +---------------------------------------------------------------------------+
	1513	+ * DHD_MAP_PKTID_LOGGING
	1514	+ * Logging the PKTID and DMA map/unmap information for the SMMU fault issue
	1515	+ * debugging in customer platform.
	1516	+ * +---------------------------------------------------------------------------+
	1517	+ */
	1518	+
	1519	+#ifdef DHD_MAP_PKTID_LOGGING
	1520	+typedef struct dhd_pktid_log_item {
	1521	+ dmaaddr_t pa; /* DMA bus address */
	1522	+ uint64 ts_nsec; /* Timestamp: nsec */
	1523	+ uint32 size; /* DMA map/unmap size */
	1524	+ uint32 pktid; /* Packet ID */
	1525	+ uint8 pkttype; /* Packet Type */
	1526	+ uint8 rsvd[7]; /* Reserved for future use */
	1527	+} dhd_pktid_log_item_t;
	1528	+
	1529	+typedef struct dhd_pktid_log {
	1530	+ uint32 items; /* number of total items */
	1531	+ uint32 index; /* index of pktid_log_item */
	1532	+ dhd_pktid_log_item_t map[0]; /* metadata storage */
	1533	+} dhd_pktid_log_t;
	1534	+
	1535	+typedef void * dhd_pktid_log_handle_t; /* opaque handle to pktid log */
	1536	+
	1537	+#define MAX_PKTID_LOG (2048)
	1538	+#define DHD_PKTID_LOG_ITEM_SZ (sizeof(dhd_pktid_log_item_t))
	1539	+#define DHD_PKTID_LOG_SZ(items) (uint32)((sizeof(dhd_pktid_log_t)) + \
	1540	+ ((DHD_PKTID_LOG_ITEM_SZ) * (items)))
	1541	+
	1542	+#define DHD_PKTID_LOG_INIT(dhd, hdl) dhd_pktid_logging_init((dhd), (hdl))
	1543	+#define DHD_PKTID_LOG_FINI(dhd, hdl) dhd_pktid_logging_fini((dhd), (hdl))
	1544	+#define DHD_PKTID_LOG(dhd, hdl, pa, pktid, len, pkttype) \
	1545	+ dhd_pktid_logging((dhd), (hdl), (pa), (pktid), (len), (pkttype))
	1546	+#define DHD_PKTID_LOG_DUMP(dhd) dhd_pktid_logging_dump((dhd))
	1547	+
	1548	+static dhd_pktid_log_handle_t *
	1549	+dhd_pktid_logging_init(dhd_pub_t *dhd, uint32 num_items)
	1550	+{
	1551	+ dhd_pktid_log_t *log;
	1552	+ uint32 log_size;
	1553	+
	1554	+ log_size = DHD_PKTID_LOG_SZ(num_items);
	1555	+ log = (dhd_pktid_log_t *)MALLOCZ(dhd->osh, log_size);
	1556	+ if (log == NULL) {
	1557	+ DHD_ERROR(("%s: MALLOC failed for size %d\n",
	1558	+ __FUNCTION__, log_size));
	1559	+ return (dhd_pktid_log_handle_t *)NULL;
	1560	+ }
	1561	+
	1562	+ log->items = num_items;
	1563	+ log->index = 0;
	1564	+
	1565	+ return (dhd_pktid_log_handle_t )log; / opaque handle */
	1566	+}
	1567	+
	1568	+static void
	1569	+dhd_pktid_logging_fini(dhd_pub_t dhd, dhd_pktid_log_handle_t handle)
	1570	+{
	1571	+ dhd_pktid_log_t *log;
	1572	+ uint32 log_size;
	1573	+
	1574	+ if (handle == NULL) {
	1575	+ DHD_ERROR(("%s: handle is NULL\n", __FUNCTION__));
	1576	+ return;
	1577	+ }
	1578	+
	1579	+ log = (dhd_pktid_log_t *)handle;
	1580	+ log_size = DHD_PKTID_LOG_SZ(log->items);
	1581	+ MFREE(dhd->osh, handle, log_size);
	1582	+}
	1583	+
	1584	+static void
	1585	+dhd_pktid_logging(dhd_pub_t dhd, dhd_pktid_log_handle_t handle, dmaaddr_t pa,
	1586	+ uint32 pktid, uint32 len, uint8 pkttype)
	1587	+{
	1588	+ dhd_pktid_log_t *log;
	1589	+ uint32 idx;
	1590	+
	1591	+ if (handle == NULL) {
	1592	+ DHD_ERROR(("%s: handle is NULL\n", __FUNCTION__));
	1593	+ return;
	1594	+ }
	1595	+
	1596	+ log = (dhd_pktid_log_t *)handle;
	1597	+ idx = log->index;
	1598	+ log->map[idx].ts_nsec = OSL_LOCALTIME_NS();
	1599	+ log->map[idx].pa = pa;
	1600	+ log->map[idx].pktid = pktid;
	1601	+ log->map[idx].size = len;
	1602	+ log->map[idx].pkttype = pkttype;
	1603	+ log->index = (idx + 1) % (log->items); /* update index */
	1604	+}
	1605	+
	1606	+void
	1607	+dhd_pktid_logging_dump(dhd_pub_t *dhd)
	1608	+{
	1609	+ dhd_prot_t *prot = dhd->prot;
	1610	+ dhd_pktid_log_t map_log, unmap_log;
	1611	+ uint64 ts_sec, ts_usec;
	1612	+
	1613	+ if (prot == NULL) {
	1614	+ DHD_ERROR(("%s: prot is NULL\n", __FUNCTION__));
	1615	+ return;
	1616	+ }
	1617	+
	1618	+ map_log = (dhd_pktid_log_t *)(prot->pktid_dma_map);
	1619	+ unmap_log = (dhd_pktid_log_t *)(prot->pktid_dma_unmap);
	1620	+ OSL_GET_LOCALTIME(&ts_sec, &ts_usec);
	1621	+ if (map_log && unmap_log) {
	1622	+ DHD_ERROR(("%s: map_idx=%d unmap_idx=%d "
	1623	+ "current time=[%5lu.%06lu]\n", __FUNCTION__,
	1624	+ map_log->index, unmap_log->index,
	1625	+ (unsigned long)ts_sec, (unsigned long)ts_usec));
	1626	+ DHD_ERROR(("%s: pktid_map_log(pa)=0x%llx size=%d, "
	1627	+ "pktid_unmap_log(pa)=0x%llx size=%d\n", __FUNCTION__,
	1628	+ (uint64)__virt_to_phys((ulong)(map_log->map)),
	1629	+ (uint32)(DHD_PKTID_LOG_ITEM_SZ * map_log->items),
	1630	+ (uint64)__virt_to_phys((ulong)(unmap_log->map)),
	1631	+ (uint32)(DHD_PKTID_LOG_ITEM_SZ * unmap_log->items)));
	1632	+ }
	1633	+}
	1634	+#endif /* DHD_MAP_PKTID_LOGGING */
	1635	+
	1636	+/* +----------------- End of DHD_MAP_PKTID_LOGGING -----------------------+ */
	1637	+
	1638	+/*
	1639	+ * +---------------------------------------------------------------------------+
925	1640	* PktId Map: Provides a native packet pointer to unique 32bit PktId mapping.
926	1641	* Main purpose is to save memory on the dongle, has other purposes as well.
927	1642	* The packet id map, also includes storage for some packet parameters that
..	..	@@ -931,14 +1646,15 @@
931	1646	* +---------------------------------------------------------------------------+
932	1647	*/
933	1648	#define DHD_PCIE_PKTID
934		-#define MAX_PKTID_ITEMS (3072 * 2) /* Maximum number of pktids supported */
	1649	+#define MAX_CTRL_PKTID (1024) /* Maximum number of pktids supported */
	1650	+#define MAX_RX_PKTID (1024)
	1651	+#define MAX_TX_PKTID (3072 * 12)
935	1652
936	1653	/* On Router, the pktptr serves as a pktid. */
937	1654
938		-
939	1655	#if defined(PROP_TXSTATUS) && !defined(DHD_PCIE_PKTID)
940	1656	#error "PKTIDMAP must be supported with PROP_TXSTATUS/WLFC"
941		-#endif
	1657	+#endif // endif
942	1658
943	1659	/* Enum for marking the buffer color based on usage */
944	1660	typedef enum dhd_pkttype {
..	..	@@ -946,47 +1662,68 @@
946	1662	PKTTYPE_DATA_RX,
947	1663	PKTTYPE_IOCTL_RX,
948	1664	PKTTYPE_EVENT_RX,
	1665	+ PKTTYPE_INFO_RX,
949	1666	/* dhd_prot_pkt_free no check, if pktid reserved and no space avail case */
950		- PKTTYPE_NO_CHECK
	1667	+ PKTTYPE_NO_CHECK,
	1668	+ PKTTYPE_TSBUF_RX
951	1669	} dhd_pkttype_t;
952	1670
953		-#define DHD_PKTID_INVALID (0U)
954		-#define DHD_IOCTL_REQ_PKTID (0xFFFE)
955		-#define DHD_FAKE_PKTID (0xFACE)
	1671	+#define DHD_PKTID_MIN_AVAIL_COUNT 512U
	1672	+#define DHD_PKTID_DEPLETED_MAX_COUNT (DHD_PKTID_MIN_AVAIL_COUNT * 2U)
	1673	+#define DHD_PKTID_INVALID (0U)
	1674	+#define DHD_IOCTL_REQ_PKTID (0xFFFE)
	1675	+#define DHD_FAKE_PKTID (0xFACE)
	1676	+#define DHD_H2D_DBGRING_REQ_PKTID 0xFFFD
	1677	+#define DHD_D2H_DBGRING_REQ_PKTID 0xFFFC
	1678	+#define DHD_H2D_HOSTTS_REQ_PKTID 0xFFFB
	1679	+#define DHD_H2D_BTLOGRING_REQ_PKTID 0xFFFA
	1680	+#define DHD_D2H_BTLOGRING_REQ_PKTID 0xFFF9
	1681	+#define DHD_H2D_SNAPSHOT_UPLOAD_REQ_PKTID 0xFFF8
	1682	+#ifdef DHD_HP2P
	1683	+#define DHD_D2H_HPPRING_TXREQ_PKTID 0xFFF7
	1684	+#define DHD_D2H_HPPRING_RXREQ_PKTID 0xFFF6
	1685	+#endif /* DHD_HP2P */
956	1686
957		-#define DHD_PKTID_FREE_LOCKER (FALSE)
958		-#define DHD_PKTID_RSV_LOCKER (TRUE)
	1687	+#define IS_FLOWRING(ring) \
	1688	+ ((strncmp(ring->name, "h2dflr", sizeof("h2dflr"))) == (0))
959	1689
960	1690	typedef void * dhd_pktid_map_handle_t; /* opaque handle to a pktid map */
961	1691
962	1692	/* Construct a packet id mapping table, returning an opaque map handle */
963		-static dhd_pktid_map_handle_t dhd_pktid_map_init(dhd_pub_t dhd, uint32 num_items, uint32 index);
	1693	+static dhd_pktid_map_handle_t dhd_pktid_map_init(dhd_pub_t dhd, uint32 num_items);
964	1694
965	1695	/* Destroy a packet id mapping table, freeing all packets active in the table */
966	1696	static void dhd_pktid_map_fini(dhd_pub_t dhd, dhd_pktid_map_handle_t map);
967	1697
968		-#define PKTID_MAP_HANDLE (0)
969		-#define PKTID_MAP_HANDLE_IOCTL (1)
970		-
971		-#define DHD_NATIVE_TO_PKTID_INIT(dhd, items, index) dhd_pktid_map_init((dhd), (items), (index))
	1698	+#define DHD_NATIVE_TO_PKTID_INIT(dhd, items) dhd_pktid_map_init((dhd), (items))
	1699	+#define DHD_NATIVE_TO_PKTID_RESET(dhd, map) dhd_pktid_map_reset((dhd), (map))
972	1700	#define DHD_NATIVE_TO_PKTID_FINI(dhd, map) dhd_pktid_map_fini((dhd), (map))
	1701	+#define DHD_NATIVE_TO_PKTID_FINI_IOCTL(osh, map) dhd_pktid_map_fini_ioctl((osh), (map))
	1702	+
	1703	+#ifdef MACOSX_DHD
	1704	+#undef DHD_PCIE_PKTID
	1705	+#define DHD_PCIE_PKTID 1
	1706	+#endif /* MACOSX_DHD */
973	1707
974	1708	#if defined(DHD_PCIE_PKTID)
975		-
	1709	+#if defined(MACOSX_DHD)
	1710	+#define IOCTLRESP_USE_CONSTMEM
	1711	+static void free_ioctl_return_buffer(dhd_pub_t dhd, dhd_dma_buf_t retbuf);
	1712	+static int alloc_ioctl_return_buffer(dhd_pub_t dhd, dhd_dma_buf_t retbuf);
	1713	+#endif // endif
976	1714
977	1715	/* Determine number of pktids that are available */
978	1716	static INLINE uint32 dhd_pktid_map_avail_cnt(dhd_pktid_map_handle_t *handle);
979	1717
980	1718	/* Allocate a unique pktid against which a pkt and some metadata is saved */
981	1719	static INLINE uint32 dhd_pktid_map_reserve(dhd_pub_t dhd, dhd_pktid_map_handle_t handle,
982		- void *pkt);
	1720	+ void *pkt, dhd_pkttype_t pkttype);
983	1721	static INLINE void dhd_pktid_map_save(dhd_pub_t dhd, dhd_pktid_map_handle_t handle,
984	1722	void *pkt, uint32 nkey, dmaaddr_t pa, uint32 len, uint8 dma,
985	1723	void dmah, void secdma, dhd_pkttype_t pkttype);
986	1724	static uint32 dhd_pktid_map_alloc(dhd_pub_t dhd, dhd_pktid_map_handle_t map,
987	1725	void *pkt, dmaaddr_t pa, uint32 len, uint8 dma,
988	1726	void dmah, void secdma, dhd_pkttype_t pkttype);
989		-
990	1727	/* Return an allocated pktid, retrieving previously saved pkt and metadata */
991	1728	static void dhd_pktid_map_free(dhd_pub_t dhd, dhd_pktid_map_handle_t *map,
992	1729	uint32 id, dmaaddr_t pa, uint32 len, void **dmah,
..	..	@@ -1001,11 +1738,6 @@
1001	1738	* CAUTION: When DHD_PKTID_AUDIT_ENABLED is defined,
1002	1739	* either DHD_PKTID_AUDIT_MAP or DHD_PKTID_AUDIT_RING may be selected.
1003	1740	*/
1004		-#ifndef DHD_PKTID_AUDIT_ENABLED
1005		-#define DHD_PKTID_AUDIT_ENABLED 1
1006		-#endif /* DHD_PKTID_AUDIT_ENABLED */
1007		-
1008		-
1009	1741	#if defined(DHD_PKTID_AUDIT_ENABLED)
1010	1742	#define USE_DHD_PKTID_AUDIT_LOCK 1
1011	1743	/* Audit the pktidmap allocator */
..	..	@@ -1023,6 +1755,13 @@
1023	1755	#define DHD_TEST_IS_ALLOC 3
1024	1756	#define DHD_TEST_IS_FREE 4
1025	1757
	1758	+typedef enum dhd_pktid_map_type {
	1759	+ DHD_PKTID_MAP_TYPE_CTRL = 1,
	1760	+ DHD_PKTID_MAP_TYPE_TX,
	1761	+ DHD_PKTID_MAP_TYPE_RX,
	1762	+ DHD_PKTID_MAP_TYPE_UNKNOWN
	1763	+} dhd_pktid_map_type_t;
	1764	+
1026	1765	#ifdef USE_DHD_PKTID_AUDIT_LOCK
1027	1766	#define DHD_PKTID_AUDIT_LOCK_INIT(osh) dhd_os_spin_lock_init(osh)
1028	1767	#define DHD_PKTID_AUDIT_LOCK_DEINIT(osh, lock) dhd_os_spin_lock_deinit(osh, lock)
..	..	@@ -1037,12 +1776,12 @@
1037	1776
1038	1777	#endif /* DHD_PKTID_AUDIT_ENABLED */
1039	1778
1040		-/* #define USE_DHD_PKTID_LOCK 1 */
	1779	+#define USE_DHD_PKTID_LOCK 1
1041	1780
1042	1781	#ifdef USE_DHD_PKTID_LOCK
1043	1782	#define DHD_PKTID_LOCK_INIT(osh) dhd_os_spin_lock_init(osh)
1044	1783	#define DHD_PKTID_LOCK_DEINIT(osh, lock) dhd_os_spin_lock_deinit(osh, lock)
1045		-#define DHD_PKTID_LOCK(lock) dhd_os_spin_lock(lock)
	1784	+#define DHD_PKTID_LOCK(lock, flags) (flags) = dhd_os_spin_lock(lock)
1046	1785	#define DHD_PKTID_UNLOCK(lock, flags) dhd_os_spin_unlock(lock, flags)
1047	1786	#else
1048	1787	#define DHD_PKTID_LOCK_INIT(osh) (void *)(1)
..	..	@@ -1059,37 +1798,26 @@
1059	1798	} while (0)
1060	1799	#endif /* !USE_DHD_PKTID_LOCK */
1061	1800
1062		-/* Packet metadata saved in packet id mapper */
1063		-
1064		-/* The Locker can be 3 states
1065		- * LOCKER_IS_FREE - Locker is free and can be allocated
1066		- * LOCKER_IS_BUSY - Locker is assigned and is being used, values in the
1067		- * locker (buffer address, len, phy addr etc) are populated
1068		- * with valid values
1069		- * LOCKER_IS_RSVD - The locker is reserved for future use, but the values
1070		- * in the locker are not valid. Especially pkt should be
1071		- * NULL in this state. When the user wants to re-use the
1072		- * locker dhd_pktid_map_free can be called with a flag
1073		- * to reserve the pktid for future use, which will clear
1074		- * the contents of the locker. When the user calls
1075		- * dhd_pktid_map_save the locker would move to LOCKER_IS_BUSY
1076		- */
1077	1801	typedef enum dhd_locker_state {
1078	1802	LOCKER_IS_FREE,
1079	1803	LOCKER_IS_BUSY,
1080	1804	LOCKER_IS_RSVD
1081	1805	} dhd_locker_state_t;
1082	1806
	1807	+/* Packet metadata saved in packet id mapper */
	1808	+
1083	1809	typedef struct dhd_pktid_item {
1084	1810	dhd_locker_state_t state; /* tag a locker to be free, busy or reserved */
1085		- uint8 dir; /* dma map direction (Tx=flush or Rx=invalidate) */
1086		- dhd_pkttype_t pkttype; /* pktlists are maintained based on pkttype */
1087		- uint16 len; /* length of mapped packet's buffer */
1088		- void pkt; / opaque native pointer to a packet */
1089		- dmaaddr_t pa; /* physical address of mapped packet's buffer */
1090		- void dmah; / handle to OS specific DMA map */
1091		- void *secdma;
	1811	+ uint8 dir; /* dma map direction (Tx=flush or Rx=invalidate) */
	1812	+ dhd_pkttype_t pkttype; /* pktlists are maintained based on pkttype */
	1813	+ uint16 len; /* length of mapped packet's buffer */
	1814	+ void pkt; / opaque native pointer to a packet */
	1815	+ dmaaddr_t pa; /* physical address of mapped packet's buffer */
	1816	+ void dmah; / handle to OS specific DMA map */
	1817	+ void *secdma;
1092	1818	} dhd_pktid_item_t;
	1819	+
	1820	+typedef uint32 dhd_pktid_key_t;
1093	1821
1094	1822	typedef struct dhd_pktid_map {
1095	1823	uint32 items; /* total items in map */
..	..	@@ -1099,11 +1827,10 @@
1099	1827	void pktid_lock; / Used when USE_DHD_PKTID_LOCK is defined */
1100	1828
1101	1829	#if defined(DHD_PKTID_AUDIT_ENABLED)
1102		- void *pktid_audit_lock;
	1830	+ void *pktid_audit_lock;
1103	1831	struct bcm_mwbmap pktid_audit; / multi word bitmap based audit */
1104	1832	#endif /* DHD_PKTID_AUDIT_ENABLED */
1105		-
1106		- uint32 keys[MAX_PKTID_ITEMS + 1]; /* stack of unique pkt ids */
	1833	+ dhd_pktid_key_t keys; / map_items +1 unique pkt ids */
1107	1834	dhd_pktid_item_t lockers[0]; /* metadata storage */
1108	1835	} dhd_pktid_map_t;
1109	1836
..	..	@@ -1116,65 +1843,78 @@
1116	1843	* Likewise, a caller must never free a pktid of value DHD_PKTID_INVALID.
1117	1844	*/
1118	1845
	1846	+#define DHD_PKTID_FREE_LOCKER (FALSE)
	1847	+#define DHD_PKTID_RSV_LOCKER (TRUE)
	1848	+
1119	1849	#define DHD_PKTID_ITEM_SZ (sizeof(dhd_pktid_item_t))
1120	1850	#define DHD_PKIDMAP_ITEMS(items) (items)
1121	1851	#define DHD_PKTID_MAP_SZ(items) (sizeof(dhd_pktid_map_t) + \
1122		- (DHD_PKTID_ITEM_SZ * ((items) + 1)))
	1852	+ (DHD_PKTID_ITEM_SZ * ((items) + 1)))
	1853	+#define DHD_PKTIDMAP_KEYS_SZ(items) (sizeof(dhd_pktid_key_t) * ((items) + 1))
1123	1854
1124		-#define DHD_NATIVE_TO_PKTID_FINI_IOCTL(dhd, map) dhd_pktid_map_fini_ioctl((dhd), (map))
	1855	+#define DHD_NATIVE_TO_PKTID_RESET_IOCTL(dhd, map) dhd_pktid_map_reset_ioctl((dhd), (map))
1125	1856
1126	1857	/* Convert a packet to a pktid, and save pkt pointer in busy locker */
1127		-#define DHD_NATIVE_TO_PKTID_RSV(dhd, map, pkt) dhd_pktid_map_reserve((dhd), (map), (pkt))
1128		-
	1858	+#define DHD_NATIVE_TO_PKTID_RSV(dhd, map, pkt, pkttype) \
	1859	+ dhd_pktid_map_reserve((dhd), (map), (pkt), (pkttype))
1129	1860	/* Reuse a previously reserved locker to save packet params */
1130	1861	#define DHD_NATIVE_TO_PKTID_SAVE(dhd, map, pkt, nkey, pa, len, dir, dmah, secdma, pkttype) \
1131	1862	dhd_pktid_map_save((dhd), (map), (void *)(pkt), (nkey), (pa), (uint32)(len), \
1132		- (uint8)(dir), (void )(dmah), (void )(secdma), \
1133		- (dhd_pkttype_t)(pkttype))
1134		-
	1863	+ (uint8)(dir), (void )(dmah), (void )(secdma), \
	1864	+ (dhd_pkttype_t)(pkttype))
1135	1865	/* Convert a packet to a pktid, and save packet params in locker */
1136	1866	#define DHD_NATIVE_TO_PKTID(dhd, map, pkt, pa, len, dir, dmah, secdma, pkttype) \
1137	1867	dhd_pktid_map_alloc((dhd), (map), (void *)(pkt), (pa), (uint32)(len), \
1138		- (uint8)(dir), (void )(dmah), (void )(secdma), \
1139		- (dhd_pkttype_t)(pkttype))
	1868	+ (uint8)(dir), (void )(dmah), (void )(secdma), \
	1869	+ (dhd_pkttype_t)(pkttype))
1140	1870
1141	1871	/* Convert pktid to a packet, and free the locker */
1142	1872	#define DHD_PKTID_TO_NATIVE(dhd, map, pktid, pa, len, dmah, secdma, pkttype) \
1143	1873	dhd_pktid_map_free((dhd), (map), (uint32)(pktid), \
1144		- (dmaaddr_t )&(pa), (uint32 )&(len), (void **)&(dmah), \
1145		- (void **) &secdma, (dhd_pkttype_t)(pkttype), DHD_PKTID_FREE_LOCKER)
	1874	+ (dmaaddr_t )&(pa), (uint32 )&(len), (void **)&(dmah), \
	1875	+ (void **)&(secdma), (dhd_pkttype_t)(pkttype), DHD_PKTID_FREE_LOCKER)
1146	1876
1147	1877	/* Convert the pktid to a packet, empty locker, but keep it reserved */
1148	1878	#define DHD_PKTID_TO_NATIVE_RSV(dhd, map, pktid, pa, len, dmah, secdma, pkttype) \
1149	1879	dhd_pktid_map_free((dhd), (map), (uint32)(pktid), \
1150		- (dmaaddr_t )&(pa), (uint32 )&(len), (void **)&(dmah), \
1151		- (void **) &secdma, (dhd_pkttype_t)(pkttype), DHD_PKTID_RSV_LOCKER)
	1880	+ (dmaaddr_t )&(pa), (uint32 )&(len), (void **)&(dmah), \
	1881	+ (void **)&(secdma), (dhd_pkttype_t)(pkttype), DHD_PKTID_RSV_LOCKER)
1152	1882
1153	1883	#define DHD_PKTID_AVAIL(map) dhd_pktid_map_avail_cnt(map)
1154	1884
1155	1885	#if defined(DHD_PKTID_AUDIT_ENABLED)
1156	1886
1157		-static int dhd_pktid_audit(dhd_pub_t dhd, dhd_pktid_map_t pktid_map, uint32 pktid,
1158		- const int test_for, const char *errmsg);
	1887	+static int
	1888	+dhd_get_pktid_map_type(dhd_pub_t dhd, dhd_pktid_map_t pktid_map)
	1889	+{
	1890	+ dhd_prot_t *prot = dhd->prot;
	1891	+ int pktid_map_type;
1159	1892
1160		-/* Call back into OS layer to take the dongle dump and panic */
1161		-#ifdef DHD_DEBUG_PAGEALLOC
1162		-extern void dhd_pktid_audit_fail_cb(dhd_pub_t *dhdp);
1163		-#endif /* DHD_DEBUG_PAGEALLOC */
	1893	+ if (pktid_map == prot->pktid_ctrl_map) {
	1894	+ pktid_map_type = DHD_PKTID_MAP_TYPE_CTRL;
	1895	+ } else if (pktid_map == prot->pktid_tx_map) {
	1896	+ pktid_map_type = DHD_PKTID_MAP_TYPE_TX;
	1897	+ } else if (pktid_map == prot->pktid_rx_map) {
	1898	+ pktid_map_type = DHD_PKTID_MAP_TYPE_RX;
	1899	+ } else {
	1900	+ pktid_map_type = DHD_PKTID_MAP_TYPE_UNKNOWN;
	1901	+ }
	1902	+
	1903	+ return pktid_map_type;
	1904	+}
1164	1905
1165	1906	/**
1166		-* dhd_pktid_audit - Use the mwbmap to audit validity of a pktid.
	1907	+* __dhd_pktid_audit - Use the mwbmap to audit validity of a pktid.
1167	1908	*/
1168	1909	static int
1169		-dhd_pktid_audit(dhd_pub_t dhd, dhd_pktid_map_t pktid_map, uint32 pktid,
	1910	+__dhd_pktid_audit(dhd_pub_t dhd, dhd_pktid_map_t pktid_map, uint32 pktid,
1170	1911	const int test_for, const char *errmsg)
1171	1912	{
1172	1913	#define DHD_PKT_AUDIT_STR "ERROR: %16s Host PktId Audit: "
1173		-
1174		- const uint32 max_pktid_items = (MAX_PKTID_ITEMS);
1175	1914	struct bcm_mwbmap *handle;
1176	1915	uint32 flags;
1177	1916	bool ignore_audit;
	1917	+ int error = BCME_OK;
1178	1918
1179	1919	if (pktid_map == (dhd_pktid_map_t *)NULL) {
1180	1920	DHD_ERROR((DHD_PKT_AUDIT_STR "Pkt id map NULL\n", errmsg));
..	..	@@ -1186,21 +1926,19 @@
1186	1926	handle = pktid_map->pktid_audit;
1187	1927	if (handle == (struct bcm_mwbmap *)NULL) {
1188	1928	DHD_ERROR((DHD_PKT_AUDIT_STR "Handle NULL\n", errmsg));
1189		- DHD_PKTID_AUDIT_UNLOCK(pktid_map->pktid_audit_lock, flags);
1190		- return BCME_OK;
	1929	+ goto out;
1191	1930	}
1192	1931
1193	1932	/* Exclude special pktids from audit */
1194	1933	ignore_audit = (pktid == DHD_IOCTL_REQ_PKTID) \| (pktid == DHD_FAKE_PKTID);
1195	1934	if (ignore_audit) {
1196		- DHD_PKTID_AUDIT_UNLOCK(pktid_map->pktid_audit_lock, flags);
1197		- return BCME_OK;
	1935	+ goto out;
1198	1936	}
1199	1937
1200		- if ((pktid == DHD_PKTID_INVALID) \|\| (pktid > max_pktid_items)) {
	1938	+ if ((pktid == DHD_PKTID_INVALID) \|\| (pktid > pktid_map->items)) {
1201	1939	DHD_ERROR((DHD_PKT_AUDIT_STR "PktId<%d> invalid\n", errmsg, pktid));
1202		- /* lock is released in "error" */
1203		- goto error;
	1940	+ error = BCME_ERROR;
	1941	+ goto out;
1204	1942	}
1205	1943
1206	1944	/* Perform audit */
..	..	@@ -1209,25 +1947,27 @@
1209	1947	if (!bcm_mwbmap_isfree(handle, pktid)) {
1210	1948	DHD_ERROR((DHD_PKT_AUDIT_STR "PktId<%d> alloc duplicate\n",
1211	1949	errmsg, pktid));
1212		- goto error;
	1950	+ error = BCME_ERROR;
	1951	+ } else {
	1952	+ bcm_mwbmap_force(handle, pktid);
1213	1953	}
1214		- bcm_mwbmap_force(handle, pktid);
1215	1954	break;
1216	1955
1217	1956	case DHD_DUPLICATE_FREE:
1218	1957	if (bcm_mwbmap_isfree(handle, pktid)) {
1219	1958	DHD_ERROR((DHD_PKT_AUDIT_STR "PktId<%d> free duplicate\n",
1220	1959	errmsg, pktid));
1221		- goto error;
	1960	+ error = BCME_ERROR;
	1961	+ } else {
	1962	+ bcm_mwbmap_free(handle, pktid);
1222	1963	}
1223		- bcm_mwbmap_free(handle, pktid);
1224	1964	break;
1225	1965
1226	1966	case DHD_TEST_IS_ALLOC:
1227	1967	if (bcm_mwbmap_isfree(handle, pktid)) {
1228	1968	DHD_ERROR((DHD_PKT_AUDIT_STR "PktId<%d> is not allocated\n",
1229	1969	errmsg, pktid));
1230		- goto error;
	1970	+ error = BCME_ERROR;
1231	1971	}
1232	1972	break;
1233	1973
..	..	@@ -1235,42 +1975,74 @@
1235	1975	if (!bcm_mwbmap_isfree(handle, pktid)) {
1236	1976	DHD_ERROR((DHD_PKT_AUDIT_STR "PktId<%d> is not free",
1237	1977	errmsg, pktid));
1238		- goto error;
	1978	+ error = BCME_ERROR;
1239	1979	}
1240	1980	break;
1241	1981
1242	1982	default:
1243		- goto error;
	1983	+ DHD_ERROR(("%s: Invalid test case: %d\n", __FUNCTION__, test_for));
	1984	+ error = BCME_ERROR;
	1985	+ break;
1244	1986	}
1245	1987
	1988	+out:
1246	1989	DHD_PKTID_AUDIT_UNLOCK(pktid_map->pktid_audit_lock, flags);
1247		- return BCME_OK;
1248	1990
1249		-error:
	1991	+ if (error != BCME_OK) {
	1992	+ dhd->pktid_audit_failed = TRUE;
	1993	+ }
1250	1994
1251		- DHD_PKTID_AUDIT_UNLOCK(pktid_map->pktid_audit_lock, flags);
1252		- /* May insert any trap mechanism here ! */
1253		-#ifdef DHD_DEBUG_PAGEALLOC
1254		- dhd_pktid_audit_fail_cb(dhd);
1255		-#else
1256		- ASSERT(0);
1257		-#endif /* DHD_DEBUG_PAGEALLOC */
1258		- return BCME_ERROR;
	1995	+ return error;
	1996	+}
	1997	+
	1998	+static int
	1999	+dhd_pktid_audit(dhd_pub_t dhd, dhd_pktid_map_t pktid_map, uint32 pktid,
	2000	+ const int test_for, const char *errmsg)
	2001	+{
	2002	+ int ret = BCME_OK;
	2003	+ ret = __dhd_pktid_audit(dhd, pktid_map, pktid, test_for, errmsg);
	2004	+ if (ret == BCME_ERROR) {
	2005	+ DHD_ERROR(("%s: Got Pkt Id Audit failure: PKTID<%d> PKTID MAP TYPE<%d>\n",
	2006	+ __FUNCTION__, pktid, dhd_get_pktid_map_type(dhd, pktid_map)));
	2007	+ dhd_pktid_error_handler(dhd);
	2008	+ }
	2009	+
	2010	+ return ret;
1259	2011	}
1260	2012
1261	2013	#define DHD_PKTID_AUDIT(dhdp, map, pktid, test_for) \
1262	2014	dhd_pktid_audit((dhdp), (dhd_pktid_map_t *)(map), (pktid), (test_for), __FUNCTION__)
1263	2015
	2016	+static int
	2017	+dhd_pktid_audit_ring_debug(dhd_pub_t dhdp, dhd_pktid_map_t map, uint32 pktid,
	2018	+ const int test_for, void msg, uint32 msg_len, const char func)
	2019	+{
	2020	+ int ret = BCME_OK;
	2021	+
	2022	+ if (dhd_query_bus_erros(dhdp)) {
	2023	+ return BCME_ERROR;
	2024	+ }
	2025	+
	2026	+ ret = __dhd_pktid_audit(dhdp, map, pktid, test_for, func);
	2027	+ if (ret == BCME_ERROR) {
	2028	+ DHD_ERROR(("%s: Got Pkt Id Audit failure: PKTID<%d> PKTID MAP TYPE<%d>\n",
	2029	+ __FUNCTION__, pktid, dhd_get_pktid_map_type(dhdp, map)));
	2030	+ prhex(func, (uchar *)msg, msg_len);
	2031	+ dhd_pktid_error_handler(dhdp);
	2032	+ }
	2033	+ return ret;
	2034	+}
	2035	+#define DHD_PKTID_AUDIT_RING_DEBUG(dhdp, map, pktid, test_for, msg, msg_len) \
	2036	+ dhd_pktid_audit_ring_debug((dhdp), (dhd_pktid_map_t *)(map), \
	2037	+ (pktid), (test_for), msg, msg_len, __FUNCTION__)
	2038	+
1264	2039	#endif /* DHD_PKTID_AUDIT_ENABLED */
1265		-
1266		-/* +------------------ End of PCIE DHD PKTID AUDIT ------------------------+ */
1267		-
1268	2040
1269	2041	/**
1270	2042	* +---------------------------------------------------------------------------+
1271	2043	* Packet to Packet Id mapper using a <numbered_key, locker> paradigm.
1272	2044	*
1273		- * dhd_pktid_map manages a set of unique Packet Ids range[1..MAX_PKTID_ITEMS].
	2045	+ * dhd_pktid_map manages a set of unique Packet Ids range[1..MAX_xxx_PKTID].
1274	2046	*
1275	2047	* dhd_pktid_map_alloc() may be used to save some packet metadata, and a unique
1276	2048	* packet id is returned. This unique packet id may be used to retrieve the
..	..	@@ -1289,40 +2061,31 @@
1289	2061	/** Allocate and initialize a mapper of num_items <numbered_key, locker> */
1290	2062
1291	2063	static dhd_pktid_map_handle_t *
1292		-dhd_pktid_map_init(dhd_pub_t *dhd, uint32 num_items, uint32 index)
	2064	+dhd_pktid_map_init(dhd_pub_t *dhd, uint32 num_items)
1293	2065	{
1294		- void *osh;
	2066	+ void* osh;
1295	2067	uint32 nkey;
1296	2068	dhd_pktid_map_t *map;
1297	2069	uint32 dhd_pktid_map_sz;
1298	2070	uint32 map_items;
1299		-#ifdef DHD_USE_STATIC_PKTIDMAP
1300		- uint32 section;
1301		-#endif /* DHD_USE_STATIC_PKTIDMAP */
	2071	+ uint32 map_keys_sz;
1302	2072	osh = dhd->osh;
1303	2073
1304		- ASSERT((num_items >= 1) && (num_items <= MAX_PKTID_ITEMS));
1305	2074	dhd_pktid_map_sz = DHD_PKTID_MAP_SZ(num_items);
1306	2075
1307		-#ifdef DHD_USE_STATIC_PKTIDMAP
1308		- if (index == PKTID_MAP_HANDLE) {
1309		- section = DHD_PREALLOC_PKTID_MAP;
1310		- } else {
1311		- section = DHD_PREALLOC_PKTID_MAP_IOCTL;
1312		- }
1313		-
1314		- map = (dhd_pktid_map_t *)DHD_OS_PREALLOC(dhd, section, dhd_pktid_map_sz);
1315		-#else
1316		- map = (dhd_pktid_map_t *)MALLOC(osh, dhd_pktid_map_sz);
1317		-#endif /* DHD_USE_STATIC_PKTIDMAP */
1318		-
	2076	+ map = (dhd_pktid_map_t *)VMALLOCZ(osh, dhd_pktid_map_sz);
1319	2077	if (map == NULL) {
1320	2078	DHD_ERROR(("%s:%d: MALLOC failed for size %d\n",
1321	2079	__FUNCTION__, __LINE__, dhd_pktid_map_sz));
1322		- goto error;
	2080	+ return (dhd_pktid_map_handle_t *)NULL;
1323	2081	}
1324	2082
1325		- bzero(map, dhd_pktid_map_sz);
	2083	+ map->items = num_items;
	2084	+ map->avail = num_items;
	2085	+
	2086	+ map_items = DHD_PKIDMAP_ITEMS(map->items);
	2087	+
	2088	+ map_keys_sz = DHD_PKTIDMAP_KEYS_SZ(map->items);
1326	2089
1327	2090	/* Initialize the lock that protects this structure */
1328	2091	map->pktid_lock = DHD_PKTID_LOCK_INIT(osh);
..	..	@@ -1331,24 +2094,24 @@
1331	2094	goto error;
1332	2095	}
1333	2096
1334		- map->items = num_items;
1335		- map->avail = num_items;
1336		-
1337		- map_items = DHD_PKIDMAP_ITEMS(map->items);
1338		-
1339		-#if defined(DHD_PKTID_AUDIT_ENABLED)
1340		- /* Incarnate a hierarchical multiword bitmap for auditing pktid allocator */
1341		- map->pktid_audit = bcm_mwbmap_init(osh, map_items + 1);
1342		- if (map->pktid_audit == (struct bcm_mwbmap *)NULL) {
1343		- DHD_ERROR(("%s:%d: pktid_audit init failed\r\n", __FUNCTION__, __LINE__));
	2097	+ map->keys = (dhd_pktid_key_t *)MALLOC(osh, map_keys_sz);
	2098	+ if (map->keys == NULL) {
	2099	+ DHD_ERROR(("%s:%d: MALLOC failed for map->keys size %d\n",
	2100	+ __FUNCTION__, __LINE__, map_keys_sz));
1344	2101	goto error;
1345		- } else {
1346		- DHD_ERROR(("%s:%d: pktid_audit init succeeded %d\n",
1347		- __FUNCTION__, __LINE__, map_items + 1));
1348	2102	}
1349	2103
1350		- map->pktid_audit_lock = DHD_PKTID_AUDIT_LOCK_INIT(osh);
1351		-
	2104	+#if defined(DHD_PKTID_AUDIT_ENABLED)
	2105	+ /* Incarnate a hierarchical multiword bitmap for auditing pktid allocator */
	2106	+ map->pktid_audit = bcm_mwbmap_init(osh, map_items + 1);
	2107	+ if (map->pktid_audit == (struct bcm_mwbmap *)NULL) {
	2108	+ DHD_ERROR(("%s:%d: pktid_audit init failed\r\n", __FUNCTION__, __LINE__));
	2109	+ goto error;
	2110	+ } else {
	2111	+ DHD_ERROR(("%s:%d: pktid_audit init succeeded %d\n",
	2112	+ __FUNCTION__, __LINE__, map_items + 1));
	2113	+ }
	2114	+ map->pktid_audit_lock = DHD_PKTID_AUDIT_LOCK_INIT(osh);
1352	2115	#endif /* DHD_PKTID_AUDIT_ENABLED */
1353	2116
1354	2117	for (nkey = 1; nkey <= map_items; nkey++) { /* locker #0 is reserved */
..	..	@@ -1358,8 +2121,8 @@
1358	2121	map->lockers[nkey].len = 0;
1359	2122	}
1360	2123
1361		- /* Reserve pktid #0, i.e. DHD_PKTID_INVALID to be busy */
1362		- map->lockers[DHD_PKTID_INVALID].state = LOCKER_IS_BUSY;
	2124	+ /* Reserve pktid #0, i.e. DHD_PKTID_INVALID to be inuse */
	2125	+ map->lockers[DHD_PKTID_INVALID].state = LOCKER_IS_BUSY; /* tag locker #0 as inuse */
1363	2126	map->lockers[DHD_PKTID_INVALID].pkt = NULL; /* bzero: redundant */
1364	2127	map->lockers[DHD_PKTID_INVALID].len = 0;
1365	2128
..	..	@@ -1371,9 +2134,7 @@
1371	2134	return (dhd_pktid_map_handle_t )map; / opaque handle */
1372	2135
1373	2136	error:
1374		-
1375	2137	if (map) {
1376		-
1377	2138	#if defined(DHD_PKTID_AUDIT_ENABLED)
1378	2139	if (map->pktid_audit != (struct bcm_mwbmap *)NULL) {
1379	2140	bcm_mwbmap_fini(osh, map->pktid_audit); /* Destruct pktid_audit */
..	..	@@ -1383,12 +2144,16 @@
1383	2144	}
1384	2145	#endif /* DHD_PKTID_AUDIT_ENABLED */
1385	2146
1386		- if (map->pktid_lock)
	2147	+ if (map->keys) {
	2148	+ MFREE(osh, map->keys, map_keys_sz);
	2149	+ }
	2150	+
	2151	+ if (map->pktid_lock) {
1387	2152	DHD_PKTID_LOCK_DEINIT(osh, map->pktid_lock);
	2153	+ }
1388	2154
1389		- MFREE(osh, map, dhd_pktid_map_sz);
	2155	+ VMFREE(osh, map, dhd_pktid_map_sz);
1390	2156	}
1391		-
1392	2157	return (dhd_pktid_map_handle_t *)NULL;
1393	2158	}
1394	2159
..	..	@@ -1397,159 +2162,180 @@
1397	2162	* Freeing implies: unmapping the buffers and freeing the native packet
1398	2163	* This could have been a callback registered with the pktid mapper.
1399	2164	*/
1400		-
1401	2165	static void
1402		-dhd_pktid_map_fini(dhd_pub_t dhd, dhd_pktid_map_handle_t handle)
	2166	+dhd_pktid_map_reset(dhd_pub_t dhd, dhd_pktid_map_handle_t handle)
1403	2167	{
1404	2168	void *osh;
1405	2169	uint32 nkey;
1406	2170	dhd_pktid_map_t *map;
1407		- uint32 dhd_pktid_map_sz;
1408	2171	dhd_pktid_item_t *locker;
1409	2172	uint32 map_items;
1410		- uint32 flags;
1411		-
1412		- if (handle == NULL) {
1413		- return;
1414		- }
	2173	+ unsigned long flags;
	2174	+ bool data_tx = FALSE;
1415	2175
1416	2176	map = (dhd_pktid_map_t *)handle;
1417		- flags = DHD_PKTID_LOCK(map->pktid_lock);
	2177	+ DHD_PKTID_LOCK(map->pktid_lock, flags);
1418	2178	osh = dhd->osh;
1419	2179
1420		- dhd_pktid_map_sz = DHD_PKTID_MAP_SZ(map->items);
1421		-
1422		- nkey = 1; /* skip reserved KEY #0, and start from 1 */
1423		- locker = &map->lockers[nkey];
1424		-
1425	2180	map_items = DHD_PKIDMAP_ITEMS(map->items);
	2181	+ /* skip reserved KEY #0, and start from 1 */
1426	2182
1427		- for (; nkey <= map_items; nkey++, locker++) {
1428		-
1429		- if (locker->state == LOCKER_IS_BUSY) { /* numbered key still in use */
1430		-
1431		- locker->state = LOCKER_IS_FREE; /* force open the locker */
1432		-
1433		-#if defined(DHD_PKTID_AUDIT_ENABLED)
1434		- DHD_PKTID_AUDIT(dhd, map, nkey, DHD_DUPLICATE_FREE); /* duplicate frees */
1435		-#endif /* DHD_PKTID_AUDIT_ENABLED */
1436		-
1437		- { /* This could be a callback registered with dhd_pktid_map */
1438		- DMA_UNMAP(osh, locker->pa, locker->len,
1439		- locker->dir, 0, DHD_DMAH_NULL);
1440		- dhd_prot_packet_free(dhd, (ulong*)locker->pkt,
1441		- locker->pkttype, TRUE);
	2183	+ for (nkey = 1; nkey <= map_items; nkey++) {
	2184	+ if (map->lockers[nkey].state == LOCKER_IS_BUSY) {
	2185	+ locker = &map->lockers[nkey];
	2186	+ locker->state = LOCKER_IS_FREE;
	2187	+ data_tx = (locker->pkttype == PKTTYPE_DATA_TX);
	2188	+ if (data_tx) {
	2189	+ OSL_ATOMIC_DEC(dhd->osh, &dhd->prot->active_tx_count);
1442	2190	}
	2191	+
	2192	+#ifdef DHD_PKTID_AUDIT_RING
	2193	+ DHD_PKTID_AUDIT(dhd, map, nkey, DHD_DUPLICATE_FREE); /* duplicate frees */
	2194	+#endif /* DHD_PKTID_AUDIT_RING */
	2195	+#ifdef DHD_MAP_PKTID_LOGGING
	2196	+ DHD_PKTID_LOG(dhd, dhd->prot->pktid_dma_unmap,
	2197	+ locker->pa, nkey, locker->len,
	2198	+ locker->pkttype);
	2199	+#endif /* DHD_MAP_PKTID_LOGGING */
	2200	+
	2201	+ {
	2202	+ if (SECURE_DMA_ENAB(dhd->osh))
	2203	+ SECURE_DMA_UNMAP(osh, locker->pa,
	2204	+ locker->len, locker->dir, 0,
	2205	+ locker->dmah, locker->secdma, 0);
	2206	+ else
	2207	+ DMA_UNMAP(osh, locker->pa, locker->len,
	2208	+ locker->dir, 0, locker->dmah);
	2209	+ }
	2210	+ dhd_prot_packet_free(dhd, (ulong*)locker->pkt,
	2211	+ locker->pkttype, data_tx);
1443	2212	}
1444		-#if defined(DHD_PKTID_AUDIT_ENABLED)
1445	2213	else {
	2214	+#ifdef DHD_PKTID_AUDIT_RING
1446	2215	DHD_PKTID_AUDIT(dhd, map, nkey, DHD_TEST_IS_FREE);
	2216	+#endif /* DHD_PKTID_AUDIT_RING */
1447	2217	}
1448		-#endif /* DHD_PKTID_AUDIT_ENABLED */
1449		-
1450		- locker->pkt = NULL; /* clear saved pkt */
1451		- locker->len = 0;
	2218	+ map->keys[nkey] = nkey; /* populate with unique keys */
1452	2219	}
1453	2220
1454		-#if defined(DHD_PKTID_AUDIT_ENABLED)
1455		- if (map->pktid_audit != (struct bcm_mwbmap *)NULL) {
1456		- bcm_mwbmap_fini(osh, map->pktid_audit); /* Destruct pktid_audit */
1457		- map->pktid_audit = (struct bcm_mwbmap *)NULL;
1458		- if (map->pktid_audit_lock) {
1459		- DHD_PKTID_AUDIT_LOCK_DEINIT(osh, map->pktid_audit_lock);
1460		- }
1461		- }
1462		-#endif /* DHD_PKTID_AUDIT_ENABLED */
1463		-
	2221	+ map->avail = map_items;
	2222	+ memset(&map->lockers[1], 0, sizeof(dhd_pktid_item_t) * map_items);
1464	2223	DHD_PKTID_UNLOCK(map->pktid_lock, flags);
1465		- DHD_PKTID_LOCK_DEINIT(osh, map->pktid_lock);
1466		-
1467		-#ifdef DHD_USE_STATIC_PKTIDMAP
1468		- DHD_OS_PREFREE(dhd, handle, dhd_pktid_map_sz);
1469		-#else
1470		- MFREE(osh, handle, dhd_pktid_map_sz);
1471		-#endif /* DHD_USE_STATIC_PKTIDMAP */
1472	2224	}
1473	2225
1474	2226	#ifdef IOCTLRESP_USE_CONSTMEM
1475	2227	/** Called in detach scenario. Releasing IOCTL buffers. */
1476	2228	static void
1477		-dhd_pktid_map_fini_ioctl(dhd_pub_t dhd, dhd_pktid_map_handle_t handle)
	2229	+dhd_pktid_map_reset_ioctl(dhd_pub_t dhd, dhd_pktid_map_handle_t handle)
1478	2230	{
1479	2231	uint32 nkey;
1480	2232	dhd_pktid_map_t *map;
1481		- uint32 dhd_pktid_map_sz;
1482	2233	dhd_pktid_item_t *locker;
1483	2234	uint32 map_items;
1484		- uint32 flags;
1485		- osl_t *osh = dhd->osh;
1486		-
1487		- if (handle == NULL) {
1488		- return;
1489		- }
	2235	+ unsigned long flags;
1490	2236
1491	2237	map = (dhd_pktid_map_t *)handle;
1492		- flags = DHD_PKTID_LOCK(map->pktid_lock);
1493		-
1494		- dhd_pktid_map_sz = DHD_PKTID_MAP_SZ(map->items);
1495		-
1496		- nkey = 1; /* skip reserved KEY #0, and start from 1 */
1497		- locker = &map->lockers[nkey];
	2238	+ DHD_PKTID_LOCK(map->pktid_lock, flags);
1498	2239
1499	2240	map_items = DHD_PKIDMAP_ITEMS(map->items);
	2241	+ /* skip reserved KEY #0, and start from 1 */
	2242	+ for (nkey = 1; nkey <= map_items; nkey++) {
	2243	+ if (map->lockers[nkey].state == LOCKER_IS_BUSY) {
	2244	+ dhd_dma_buf_t retbuf;
1500	2245
1501		- for (; nkey <= map_items; nkey++, locker++) {
1502		-
1503		- if (locker->state == LOCKER_IS_BUSY) { /* numbered key still in use */
1504		-
1505		- locker->state = LOCKER_IS_FREE; /* force open the locker */
1506		-
1507		-#if defined(DHD_PKTID_AUDIT_ENABLED)
	2246	+#ifdef DHD_PKTID_AUDIT_RING
1508	2247	DHD_PKTID_AUDIT(dhd, map, nkey, DHD_DUPLICATE_FREE); /* duplicate frees */
1509		-#endif /* DHD_PKTID_AUDIT_ENABLED */
	2248	+#endif /* DHD_PKTID_AUDIT_RING */
1510	2249
1511		- {
1512		- dhd_dma_buf_t retbuf;
1513		- retbuf.va = locker->pkt;
1514		- retbuf.len = locker->len;
1515		- retbuf.pa = locker->pa;
1516		- retbuf.dmah = locker->dmah;
1517		- retbuf.secdma = locker->secdma;
	2250	+ locker = &map->lockers[nkey];
	2251	+ retbuf.va = locker->pkt;
	2252	+ retbuf.len = locker->len;
	2253	+ retbuf.pa = locker->pa;
	2254	+ retbuf.dmah = locker->dmah;
	2255	+ retbuf.secdma = locker->secdma;
1518	2256
1519		- /* This could be a callback registered with dhd_pktid_map */
1520		- DHD_PKTID_UNLOCK(map->pktid_lock, flags);
1521		- free_ioctl_return_buffer(dhd, &retbuf);
1522		- flags = DHD_PKTID_LOCK(map->pktid_lock);
1523		- }
	2257	+ free_ioctl_return_buffer(dhd, &retbuf);
1524	2258	}
1525		-#if defined(DHD_PKTID_AUDIT_ENABLED)
1526	2259	else {
	2260	+#ifdef DHD_PKTID_AUDIT_RING
1527	2261	DHD_PKTID_AUDIT(dhd, map, nkey, DHD_TEST_IS_FREE);
	2262	+#endif /* DHD_PKTID_AUDIT_RING */
1528	2263	}
1529		-#endif /* DHD_PKTID_AUDIT_ENABLED */
1530		-
1531		- locker->pkt = NULL; /* clear saved pkt */
1532		- locker->len = 0;
	2264	+ map->keys[nkey] = nkey; /* populate with unique keys */
1533	2265	}
	2266	+
	2267	+ map->avail = map_items;
	2268	+ memset(&map->lockers[1], 0, sizeof(dhd_pktid_item_t) * map_items);
	2269	+ DHD_PKTID_UNLOCK(map->pktid_lock, flags);
	2270	+}
	2271	+#endif /* IOCTLRESP_USE_CONSTMEM */
	2272	+
	2273	+/**
	2274	+ * Free the pktid map.
	2275	+ */
	2276	+static void
	2277	+dhd_pktid_map_fini(dhd_pub_t dhd, dhd_pktid_map_handle_t handle)
	2278	+{
	2279	+ dhd_pktid_map_t *map;
	2280	+ uint32 dhd_pktid_map_sz;
	2281	+ uint32 map_keys_sz;
	2282	+
	2283	+ if (handle == NULL)
	2284	+ return;
	2285	+
	2286	+ /* Free any pending packets */
	2287	+ dhd_pktid_map_reset(dhd, handle);
	2288	+
	2289	+ map = (dhd_pktid_map_t *)handle;
	2290	+ dhd_pktid_map_sz = DHD_PKTID_MAP_SZ(map->items);
	2291	+ map_keys_sz = DHD_PKTIDMAP_KEYS_SZ(map->items);
	2292	+
	2293	+ DHD_PKTID_LOCK_DEINIT(dhd->osh, map->pktid_lock);
1534	2294
1535	2295	#if defined(DHD_PKTID_AUDIT_ENABLED)
1536	2296	if (map->pktid_audit != (struct bcm_mwbmap *)NULL) {
1537		- bcm_mwbmap_fini(osh, map->pktid_audit); /* Destruct pktid_audit */
	2297	+ bcm_mwbmap_fini(dhd->osh, map->pktid_audit); /* Destruct pktid_audit */
1538	2298	map->pktid_audit = (struct bcm_mwbmap *)NULL;
1539	2299	if (map->pktid_audit_lock) {
1540		- DHD_PKTID_AUDIT_LOCK_DEINIT(osh, map->pktid_audit_lock);
	2300	+ DHD_PKTID_AUDIT_LOCK_DEINIT(dhd->osh, map->pktid_audit_lock);
	2301	+ }
	2302	+ }
	2303	+#endif /* DHD_PKTID_AUDIT_ENABLED */
	2304	+ MFREE(dhd->osh, map->keys, map_keys_sz);
	2305	+ VMFREE(dhd->osh, handle, dhd_pktid_map_sz);
	2306	+}
	2307	+#ifdef IOCTLRESP_USE_CONSTMEM
	2308	+static void
	2309	+dhd_pktid_map_fini_ioctl(dhd_pub_t dhd, dhd_pktid_map_handle_t handle)
	2310	+{
	2311	+ dhd_pktid_map_t *map;
	2312	+ uint32 dhd_pktid_map_sz;
	2313	+ uint32 map_keys_sz;
	2314	+
	2315	+ if (handle == NULL)
	2316	+ return;
	2317	+
	2318	+ /* Free any pending packets */
	2319	+ dhd_pktid_map_reset_ioctl(dhd, handle);
	2320	+
	2321	+ map = (dhd_pktid_map_t *)handle;
	2322	+ dhd_pktid_map_sz = DHD_PKTID_MAP_SZ(map->items);
	2323	+ map_keys_sz = DHD_PKTIDMAP_KEYS_SZ(map->items);
	2324	+
	2325	+ DHD_PKTID_LOCK_DEINIT(dhd->osh, map->pktid_lock);
	2326	+
	2327	+#if defined(DHD_PKTID_AUDIT_ENABLED)
	2328	+ if (map->pktid_audit != (struct bcm_mwbmap *)NULL) {
	2329	+ bcm_mwbmap_fini(dhd->osh, map->pktid_audit); /* Destruct pktid_audit */
	2330	+ map->pktid_audit = (struct bcm_mwbmap *)NULL;
	2331	+ if (map->pktid_audit_lock) {
	2332	+ DHD_PKTID_AUDIT_LOCK_DEINIT(dhd->osh, map->pktid_audit_lock);
1541	2333	}
1542	2334	}
1543	2335	#endif /* DHD_PKTID_AUDIT_ENABLED */
1544	2336
1545		- DHD_PKTID_UNLOCK(map->pktid_lock, flags);
1546		- DHD_PKTID_LOCK_DEINIT(osh, map->pktid_lock);
1547		-
1548		-#ifdef DHD_USE_STATIC_PKTIDMAP
1549		- DHD_OS_PREFREE(dhd, handle, dhd_pktid_map_sz);
1550		-#else
1551		- MFREE(osh, handle, dhd_pktid_map_sz);
1552		-#endif /* DHD_USE_STATIC_PKTIDMAP */
	2337	+ MFREE(dhd->osh, map->keys, map_keys_sz);
	2338	+ VMFREE(dhd->osh, handle, dhd_pktid_map_sz);
1553	2339	}
1554	2340	#endif /* IOCTLRESP_USE_CONSTMEM */
1555	2341
..	..	@@ -1558,13 +2344,13 @@
1558	2344	dhd_pktid_map_avail_cnt(dhd_pktid_map_handle_t *handle)
1559	2345	{
1560	2346	dhd_pktid_map_t *map;
1561		- uint32 flags;
1562	2347	uint32 avail;
	2348	+ unsigned long flags;
1563	2349
1564	2350	ASSERT(handle != NULL);
1565	2351	map = (dhd_pktid_map_t *)handle;
1566	2352
1567		- flags = DHD_PKTID_LOCK(map->pktid_lock);
	2353	+ DHD_PKTID_LOCK(map->pktid_lock, flags);
1568	2354	avail = map->avail;
1569	2355	DHD_PKTID_UNLOCK(map->pktid_lock, flags);
1570	2356
..	..	@@ -1572,88 +2358,97 @@
1572	2358	}
1573	2359
1574	2360	/**
1575		- * Allocate locker, save pkt contents, and return the locker's numbered key.
1576		- * dhd_pktid_map_alloc() is not reentrant, and is the caller's responsibility.
1577		- * Caller must treat a returned value DHD_PKTID_INVALID as a failure case,
1578		- * implying a depleted pool of pktids.
	2361	+ * dhd_pktid_map_reserve - reserve a unique numbered key. Reserved locker is not
	2362	+ * yet populated. Invoke the pktid save api to populate the packet parameters
	2363	+ * into the locker. This function is not reentrant, and is the caller's
	2364	+ * responsibility. Caller must treat a returned value DHD_PKTID_INVALID as
	2365	+ * a failure case, implying a depleted pool of pktids.
1579	2366	*/
1580		-
1581	2367	static INLINE uint32
1582		-__dhd_pktid_map_reserve(dhd_pub_t dhd, dhd_pktid_map_handle_t handle, void *pkt)
	2368	+dhd_pktid_map_reserve(dhd_pub_t dhd, dhd_pktid_map_handle_t handle,
	2369	+ void *pkt, dhd_pkttype_t pkttype)
1583	2370	{
1584	2371	uint32 nkey;
1585	2372	dhd_pktid_map_t *map;
1586	2373	dhd_pktid_item_t *locker;
	2374	+ unsigned long flags;
1587	2375
1588	2376	ASSERT(handle != NULL);
1589	2377	map = (dhd_pktid_map_t *)handle;
1590	2378
1591		- if (map->avail <= 0) { /* no more pktids to allocate */
	2379	+ DHD_PKTID_LOCK(map->pktid_lock, flags);
	2380	+
	2381	+ if ((int)(map->avail) <= 0) { /* no more pktids to allocate */
1592	2382	map->failures++;
1593	2383	DHD_INFO(("%s:%d: failed, no free keys\n", __FUNCTION__, __LINE__));
	2384	+ DHD_PKTID_UNLOCK(map->pktid_lock, flags);
1594	2385	return DHD_PKTID_INVALID; /* failed alloc request */
1595	2386	}
1596	2387
1597	2388	ASSERT(map->avail <= map->items);
1598	2389	nkey = map->keys[map->avail]; /* fetch a free locker, pop stack */
	2390	+
	2391	+ if ((map->avail > map->items) \|\| (nkey > map->items)) {
	2392	+ map->failures++;
	2393	+ DHD_ERROR(("%s:%d: failed to allocate a new pktid,"
	2394	+ " map->avail<%u>, nkey<%u>, pkttype<%u>\n",
	2395	+ __FUNCTION__, __LINE__, map->avail, nkey,
	2396	+ pkttype));
	2397	+ DHD_PKTID_UNLOCK(map->pktid_lock, flags);
	2398	+ return DHD_PKTID_INVALID; /* failed alloc request */
	2399	+ }
	2400	+
1599	2401	locker = &map->lockers[nkey]; /* save packet metadata in locker */
1600	2402	map->avail--;
1601	2403	locker->pkt = pkt; /* pkt is saved, other params not yet saved. */
1602	2404	locker->len = 0;
1603	2405	locker->state = LOCKER_IS_BUSY; /* reserve this locker */
1604	2406
1605		-#if defined(DHD_PKTID_AUDIT_MAP)
1606		- DHD_PKTID_AUDIT(dhd, map, nkey, DHD_DUPLICATE_ALLOC); /* Audit duplicate alloc */
1607		-#endif /* DHD_PKTID_AUDIT_MAP */
	2407	+ DHD_PKTID_UNLOCK(map->pktid_lock, flags);
1608	2408
1609	2409	ASSERT(nkey != DHD_PKTID_INVALID);
	2410	+
1610	2411	return nkey; /* return locker's numbered key */
1611	2412	}
1612	2413
1613		-
1614		-/**
1615		- * dhd_pktid_map_reserve - reserve a unique numbered key. Reserved locker is not
1616		- * yet populated. Invoke the pktid save api to populate the packet parameters
1617		- * into the locker.
1618		- * Wrapper that takes the required lock when called directly.
	2414	+/*
	2415	+ * dhd_pktid_map_save - Save a packet's parameters into a locker
	2416	+ * corresponding to a previously reserved unique numbered key.
1619	2417	*/
1620		-static INLINE uint32
1621		-dhd_pktid_map_reserve(dhd_pub_t dhd, dhd_pktid_map_handle_t handle, void *pkt)
1622		-{
1623		- dhd_pktid_map_t *map;
1624		- uint32 flags;
1625		- uint32 ret;
1626		-
1627		- ASSERT(handle != NULL);
1628		- map = (dhd_pktid_map_t *)handle;
1629		- flags = DHD_PKTID_LOCK(map->pktid_lock);
1630		- ret = __dhd_pktid_map_reserve(dhd, handle, pkt);
1631		- DHD_PKTID_UNLOCK(map->pktid_lock, flags);
1632		-
1633		- return ret;
1634		-}
1635		-
1636	2418	static INLINE void
1637		-__dhd_pktid_map_save(dhd_pub_t dhd, dhd_pktid_map_handle_t handle, void *pkt,
	2419	+dhd_pktid_map_save(dhd_pub_t dhd, dhd_pktid_map_handle_t handle, void *pkt,
1638	2420	uint32 nkey, dmaaddr_t pa, uint32 len, uint8 dir, void dmah, void secdma,
1639	2421	dhd_pkttype_t pkttype)
1640	2422	{
1641	2423	dhd_pktid_map_t *map;
1642	2424	dhd_pktid_item_t *locker;
	2425	+ unsigned long flags;
1643	2426
1644	2427	ASSERT(handle != NULL);
1645	2428	map = (dhd_pktid_map_t *)handle;
1646	2429
1647		- ASSERT((nkey != DHD_PKTID_INVALID) && (nkey <= DHD_PKIDMAP_ITEMS(map->items)));
	2430	+ DHD_PKTID_LOCK(map->pktid_lock, flags);
	2431	+
	2432	+ if ((nkey == DHD_PKTID_INVALID) \|\| (nkey > DHD_PKIDMAP_ITEMS(map->items))) {
	2433	+ DHD_ERROR(("%s:%d: Error! saving invalid pktid<%u> pkttype<%u>\n",
	2434	+ __FUNCTION__, __LINE__, nkey, pkttype));
	2435	+ DHD_PKTID_UNLOCK(map->pktid_lock, flags);
	2436	+#ifdef DHD_FW_COREDUMP
	2437	+ if (dhd->memdump_enabled) {
	2438	+ /* collect core dump */
	2439	+ dhd->memdump_type = DUMP_TYPE_PKTID_INVALID;
	2440	+ dhd_bus_mem_dump(dhd);
	2441	+ }
	2442	+#else
	2443	+ ASSERT(0);
	2444	+#endif /* DHD_FW_COREDUMP */
	2445	+ return;
	2446	+ }
1648	2447
1649	2448	locker = &map->lockers[nkey];
1650	2449
1651	2450	ASSERT(((locker->state == LOCKER_IS_BUSY) && (locker->pkt == pkt)) \|\|
1652	2451	((locker->state == LOCKER_IS_RSVD) && (locker->pkt == NULL)));
1653		-
1654		-#if defined(DHD_PKTID_AUDIT_MAP)
1655		- DHD_PKTID_AUDIT(dhd, map, nkey, DHD_TEST_IS_ALLOC); /* apriori, reservation */
1656		-#endif /* DHD_PKTID_AUDIT_MAP */
1657	2452
1658	2453	/* store contents in locker */
1659	2454	locker->dir = dir;
..	..	@@ -1664,26 +2459,9 @@
1664	2459	locker->pkttype = pkttype;
1665	2460	locker->pkt = pkt;
1666	2461	locker->state = LOCKER_IS_BUSY; /* make this locker busy */
1667		-}
1668		-
1669		-/**
1670		- * dhd_pktid_map_save - Save a packet's parameters into a locker corresponding
1671		- * to a previously reserved unique numbered key.
1672		- * Wrapper that takes the required lock when called directly.
1673		- */
1674		-static INLINE void
1675		-dhd_pktid_map_save(dhd_pub_t dhd, dhd_pktid_map_handle_t handle, void *pkt,
1676		- uint32 nkey, dmaaddr_t pa, uint32 len, uint8 dir, void dmah, void secdma,
1677		- dhd_pkttype_t pkttype)
1678		-{
1679		- dhd_pktid_map_t *map;
1680		- uint32 flags;
1681		-
1682		- ASSERT(handle != NULL);
1683		- map = (dhd_pktid_map_t *)handle;
1684		- flags = DHD_PKTID_LOCK(map->pktid_lock);
1685		- __dhd_pktid_map_save(dhd, handle, pkt, nkey, pa, len,
1686		- dir, dmah, secdma, pkttype);
	2462	+#ifdef DHD_MAP_PKTID_LOGGING
	2463	+ DHD_PKTID_LOG(dhd, dhd->prot->pktid_dma_map, pa, nkey, len, pkttype);
	2464	+#endif /* DHD_MAP_PKTID_LOGGING */
1687	2465	DHD_PKTID_UNLOCK(map->pktid_lock, flags);
1688	2466	}
1689	2467
..	..	@@ -1697,30 +2475,12 @@
1697	2475	dhd_pkttype_t pkttype)
1698	2476	{
1699	2477	uint32 nkey;
1700		- uint32 flags;
1701		- dhd_pktid_map_t *map;
1702	2478
1703		- ASSERT(handle != NULL);
1704		- map = (dhd_pktid_map_t *)handle;
1705		-
1706		- flags = DHD_PKTID_LOCK(map->pktid_lock);
1707		-
1708		- nkey = __dhd_pktid_map_reserve(dhd, handle, pkt);
	2479	+ nkey = dhd_pktid_map_reserve(dhd, handle, pkt, pkttype);
1709	2480	if (nkey != DHD_PKTID_INVALID) {
1710		- __dhd_pktid_map_save(dhd, handle, pkt, nkey, pa,
	2481	+ dhd_pktid_map_save(dhd, handle, pkt, nkey, pa,
1711	2482	len, dir, dmah, secdma, pkttype);
1712		-#if defined(DHD_PKTID_AUDIT_MAP)
1713		- DHD_PKTID_AUDIT(dhd, map, nkey, DHD_TEST_IS_ALLOC); /* apriori, reservation */
1714		-#endif /* DHD_PKTID_AUDIT_MAP */
1715	2483	}
1716		-
1717		-#ifdef CUSTOMER_HW_31_2
1718		- /* Need to do the flush at buffer allocation time */
1719		- DHD_TRACE(("%s: flush buffer 0x%x len %d\n", __FUNCTION__,
1720		- PKTDATA(dhd->osh, pkt), PKTLEN(dhd->osh, pkt)));
1721		- OSL_CACHE_FLUSH(PKTDATA(dhd->osh, pkt), PKTLEN(dhd->osh, pkt));
1722		-#endif
1723		- DHD_PKTID_UNLOCK(map->pktid_lock, flags);
1724	2484
1725	2485	return nkey;
1726	2486	}
..	..	@@ -1733,21 +2493,36 @@
1733	2493	*/
1734	2494	static void * BCMFASTPATH
1735	2495	dhd_pktid_map_free(dhd_pub_t dhd, dhd_pktid_map_handle_t handle, uint32 nkey,
1736		- dmaaddr_t pa, uint32 len, void dmah, void secdma,
1737		- dhd_pkttype_t pkttype, bool rsv_locker)
	2496	+ dmaaddr_t pa, uint32 len, void dmah, void secdma, dhd_pkttype_t pkttype,
	2497	+ bool rsv_locker)
1738	2498	{
1739	2499	dhd_pktid_map_t *map;
1740	2500	dhd_pktid_item_t *locker;
1741	2501	void * pkt;
1742		- uint32 flags;
	2502	+ unsigned long long locker_addr;
	2503	+ unsigned long flags;
1743	2504
1744	2505	ASSERT(handle != NULL);
1745	2506
1746	2507	map = (dhd_pktid_map_t *)handle;
1747	2508
1748		- flags = DHD_PKTID_LOCK(map->pktid_lock);
	2509	+ DHD_PKTID_LOCK(map->pktid_lock, flags);
1749	2510
1750		- ASSERT((nkey != DHD_PKTID_INVALID) && (nkey <= DHD_PKIDMAP_ITEMS(map->items)));
	2511	+ if ((nkey == DHD_PKTID_INVALID) \|\| (nkey > DHD_PKIDMAP_ITEMS(map->items))) {
	2512	+ DHD_ERROR(("%s:%d: Error! Try to free invalid pktid<%u>, pkttype<%d>\n",
	2513	+ __FUNCTION__, __LINE__, nkey, pkttype));
	2514	+ DHD_PKTID_UNLOCK(map->pktid_lock, flags);
	2515	+#ifdef DHD_FW_COREDUMP
	2516	+ if (dhd->memdump_enabled) {
	2517	+ /* collect core dump */
	2518	+ dhd->memdump_type = DUMP_TYPE_PKTID_INVALID;
	2519	+ dhd_bus_mem_dump(dhd);
	2520	+ }
	2521	+#else
	2522	+ ASSERT(0);
	2523	+#endif /* DHD_FW_COREDUMP */
	2524	+ return NULL;
	2525	+ }
1751	2526
1752	2527	locker = &map->lockers[nkey];
1753	2528
..	..	@@ -1755,12 +2530,20 @@
1755	2530	DHD_PKTID_AUDIT(dhd, map, nkey, DHD_DUPLICATE_FREE); /* Audit duplicate FREE */
1756	2531	#endif /* DHD_PKTID_AUDIT_MAP */
1757	2532
1758		- if (locker->state == LOCKER_IS_FREE) { /* Debug check for cloned numbered key */
1759		- DHD_ERROR(("%s:%d: Error! freeing invalid pktid<%u>\n",
1760		- __FUNCTION__, __LINE__, nkey));
1761		- ASSERT(locker->state != LOCKER_IS_FREE);
1762		-
	2533	+ /* Debug check for cloned numbered key */
	2534	+ if (locker->state == LOCKER_IS_FREE) {
	2535	+ DHD_ERROR(("%s:%d: Error! freeing already freed invalid pktid<%u>\n",
	2536	+ __FUNCTION__, __LINE__, nkey));
1763	2537	DHD_PKTID_UNLOCK(map->pktid_lock, flags);
	2538	+#ifdef DHD_FW_COREDUMP
	2539	+ if (dhd->memdump_enabled) {
	2540	+ /* collect core dump */
	2541	+ dhd->memdump_type = DUMP_TYPE_PKTID_INVALID;
	2542	+ dhd_bus_mem_dump(dhd);
	2543	+ }
	2544	+#else
	2545	+ ASSERT(0);
	2546	+#endif /* DHD_FW_COREDUMP */
1764	2547	return NULL;
1765	2548	}
1766	2549
..	..	@@ -1770,12 +2553,27 @@
1770	2553	*/
1771	2554	if ((pkttype != PKTTYPE_NO_CHECK) && (locker->pkttype != pkttype)) {
1772	2555
1773		- DHD_PKTID_UNLOCK(map->pktid_lock, flags);
1774		-
1775	2556	DHD_ERROR(("%s:%d: Error! Invalid Buffer Free for pktid<%u> \n",
1776	2557	__FUNCTION__, __LINE__, nkey));
1777		- ASSERT(locker->pkttype == pkttype);
1778		-
	2558	+#ifdef BCMDMA64OSL
	2559	+ PHYSADDRTOULONG(locker->pa, locker_addr);
	2560	+#else
	2561	+ locker_addr = PHYSADDRLO(locker->pa);
	2562	+#endif /* BCMDMA64OSL */
	2563	+ DHD_ERROR(("%s:%d: locker->state <%d>, locker->pkttype <%d>,"
	2564	+ "pkttype <%d> locker->pa <0x%llx> \n",
	2565	+ __FUNCTION__, __LINE__, locker->state, locker->pkttype,
	2566	+ pkttype, locker_addr));
	2567	+ DHD_PKTID_UNLOCK(map->pktid_lock, flags);
	2568	+#ifdef DHD_FW_COREDUMP
	2569	+ if (dhd->memdump_enabled) {
	2570	+ /* collect core dump */
	2571	+ dhd->memdump_type = DUMP_TYPE_PKTID_INVALID;
	2572	+ dhd_bus_mem_dump(dhd);
	2573	+ }
	2574	+#else
	2575	+ ASSERT(0);
	2576	+#endif /* DHD_FW_COREDUMP */
1779	2577	return NULL;
1780	2578	}
1781	2579
..	..	@@ -1791,6 +2589,10 @@
1791	2589	#if defined(DHD_PKTID_AUDIT_MAP)
1792	2590	DHD_PKTID_AUDIT(dhd, map, nkey, DHD_TEST_IS_FREE);
1793	2591	#endif /* DHD_PKTID_AUDIT_MAP */
	2592	+#ifdef DHD_MAP_PKTID_LOGGING
	2593	+ DHD_PKTID_LOG(dhd, dhd->prot->pktid_dma_unmap, locker->pa, nkey,
	2594	+ (uint32)locker->len, pkttype);
	2595	+#endif /* DHD_MAP_PKTID_LOGGING */
1794	2596
1795	2597	pa = locker->pa; / return contents of locker */
1796	2598	*len = (uint32)locker->len;
..	..	@@ -1801,17 +2603,12 @@
1801	2603	locker->pkt = NULL; /* Clear pkt */
1802	2604	locker->len = 0;
1803	2605
1804		-#ifdef CUSTOMER_HW_31_2
1805		- /* need to do to ensure all packet are flushed */
1806		- OSL_CACHE_INV(PKTDATA(dhd->osh, pkt), PKTLEN(dhd->osh, pkt));
1807		-#endif
1808		-
1809	2606	DHD_PKTID_UNLOCK(map->pktid_lock, flags);
	2607	+
1810	2608	return pkt;
1811	2609	}
1812	2610
1813	2611	#else /* ! DHD_PCIE_PKTID */
1814		-
1815	2612
1816	2613	typedef struct pktlist {
1817	2614	PKT_LIST tx_pkt_list; / list for tx packets */
..	..	@@ -1831,7 +2628,6 @@
1831	2628	#define DHD_PKTID32(pktptr32) ((uint32)(pktptr32))
1832	2629	#define DHD_PKTPTR32(pktid32) ((void *)(pktid32))
1833	2630
1834		-
1835	2631	static INLINE uint32 dhd_native_to_pktid(dhd_pktid_map_handle_t map, void pktptr32,
1836	2632	dmaaddr_t pa, uint32 dma_len, void dmah, void secdma,
1837	2633	dhd_pkttype_t pkttype);
..	..	@@ -1840,7 +2636,7 @@
1840	2636	dhd_pkttype_t pkttype);
1841	2637
1842	2638	static dhd_pktid_map_handle_t *
1843		-dhd_pktid_map_init(dhd_pub_t *dhd, uint32 num_items, uint32 index)
	2639	+dhd_pktid_map_init(dhd_pub_t *dhd, uint32 num_items)
1844	2640	{
1845	2641	osl_t *osh = dhd->osh;
1846	2642	pktlists_t *handle = NULL;
..	..	@@ -1897,15 +2693,9 @@
1897	2693	}
1898	2694
1899	2695	static void
1900		-dhd_pktid_map_fini(dhd_pub_t dhd, dhd_pktid_map_handle_t map)
	2696	+dhd_pktid_map_reset(dhd_pub_t dhd, pktlists_t handle)
1901	2697	{
1902	2698	osl_t *osh = dhd->osh;
1903		- pktlists_t handle = (pktlists_t ) map;
1904		-
1905		- ASSERT(handle != NULL);
1906		- if (handle == (pktlists_t *)NULL) {
1907		- return;
1908		- }
1909	2699
1910	2700	if (handle->ctrl_pkt_list) {
1911	2701	PKTLIST_FINI(handle->ctrl_pkt_list);
..	..	@@ -1921,6 +2711,20 @@
1921	2711	PKTLIST_FINI(handle->tx_pkt_list);
1922	2712	MFREE(osh, handle->tx_pkt_list, sizeof(PKT_LIST));
1923	2713	}
	2714	+}
	2715	+
	2716	+static void
	2717	+dhd_pktid_map_fini(dhd_pub_t dhd, dhd_pktid_map_handle_t map)
	2718	+{
	2719	+ osl_t *osh = dhd->osh;
	2720	+ pktlists_t handle = (pktlists_t ) map;
	2721	+
	2722	+ ASSERT(handle != NULL);
	2723	+ if (handle == (pktlists_t *)NULL) {
	2724	+ return;
	2725	+ }
	2726	+
	2727	+ dhd_pktid_map_reset(dhd, handle);
1924	2728
1925	2729	if (handle) {
1926	2730	MFREE(osh, handle, sizeof(pktlists_t));
..	..	@@ -1978,7 +2782,7 @@
1978	2782	return pktptr32;
1979	2783	}
1980	2784
1981		-#define DHD_NATIVE_TO_PKTID_RSV(dhd, map, pkt) DHD_PKTID32(pkt)
	2785	+#define DHD_NATIVE_TO_PKTID_RSV(dhd, map, pkt, pkttype) DHD_PKTID32(pkt)
1982	2786
1983	2787	#define DHD_NATIVE_TO_PKTID_SAVE(dhd, map, pkt, nkey, pa, len, dma_dir, dmah, secdma, pkttype) \
1984	2788	({ BCM_REFERENCE(dhd); BCM_REFERENCE(nkey); BCM_REFERENCE(dma_dir); \
..	..	@@ -2004,7 +2808,6 @@
2004	2808	#endif /* ! DHD_PCIE_PKTID */
2005	2809
2006	2810	/* +------------------ End of PCIE DHD PKTID MAPPER -----------------------+ */
2007		-
2008	2811
2009	2812	/**
2010	2813	* The PCIE FD protocol layer is constructed in two phases:
..	..	@@ -2033,6 +2836,11 @@
2033	2836	osl_t *osh = dhd->osh;
2034	2837	dhd_prot_t *prot;
2035	2838
	2839	+ /* FW going to DMA extended trap data,
	2840	+ * allocate buffer for the maximum extended trap data.
	2841	+ */
	2842	+ uint32 trap_buf_len = BCMPCIE_EXT_TRAP_DATA_MAXLEN;
	2843	+
2036	2844	/* Allocate prot structure */
2037	2845	if (!(prot = (dhd_prot_t *)DHD_OS_PREALLOC(dhd, DHD_PREALLOC_PROT,
2038	2846	sizeof(dhd_prot_t)))) {
..	..	@@ -2047,6 +2855,12 @@
2047	2855	/* DMAing ring completes supported? FALSE by default */
2048	2856	dhd->dma_d2h_ring_upd_support = FALSE;
2049	2857	dhd->dma_h2d_ring_upd_support = FALSE;
	2858	+ dhd->dma_ring_upd_overwrite = FALSE;
	2859	+
	2860	+ dhd->hwa_inited = 0;
	2861	+ dhd->idma_inited = 0;
	2862	+ dhd->ifrm_inited = 0;
	2863	+ dhd->dar_inited = 0;
2050	2864
2051	2865	/* Common Ring Allocations */
2052	2866
..	..	@@ -2112,6 +2926,12 @@
2112	2926	goto fail;
2113	2927	}
2114	2928
	2929	+ /* Host TS request buffer one buffer for now */
	2930	+ if (dhd_dma_buf_alloc(dhd, &prot->hostts_req_buf, CTRLSUB_HOSTTS_MEESAGE_SIZE)) {
	2931	+ goto fail;
	2932	+ }
	2933	+ prot->hostts_req_buf_inuse = FALSE;
	2934	+
2115	2935	/* Scratch buffer for dma rx offset */
2116	2936	#ifdef BCM_HOST_BUF
2117	2937	if (dhd_dma_buf_alloc(dhd, &prot->d2h_dma_scratch_buf,
..	..	@@ -2120,6 +2940,7 @@
2120	2940	if (dhd_dma_buf_alloc(dhd, &prot->d2h_dma_scratch_buf, DMA_D2H_SCRATCH_BUF_LEN)) {
2121	2941
2122	2942	#endif /* BCM_HOST_BUF */
	2943	+
2123	2944	goto fail;
2124	2945	}
2125	2946
..	..	@@ -2130,50 +2951,245 @@
2130	2951
2131	2952	#ifdef DHD_RX_CHAINING
2132	2953	dhd_rxchain_reset(&prot->rxchain);
2133		-#endif
	2954	+#endif // endif
2134	2955
2135		-#if defined(DHD_LB)
	2956	+ prot->pktid_ctrl_map = DHD_NATIVE_TO_PKTID_INIT(dhd, MAX_CTRL_PKTID);
	2957	+ if (prot->pktid_ctrl_map == NULL) {
	2958	+ goto fail;
	2959	+ }
	2960	+
	2961	+ prot->pktid_rx_map = DHD_NATIVE_TO_PKTID_INIT(dhd, MAX_RX_PKTID);
	2962	+ if (prot->pktid_rx_map == NULL)
	2963	+ goto fail;
	2964	+
	2965	+ prot->pktid_tx_map = DHD_NATIVE_TO_PKTID_INIT(dhd, MAX_TX_PKTID);
	2966	+ if (prot->pktid_tx_map == NULL)
	2967	+ goto fail;
	2968	+
	2969	+#ifdef IOCTLRESP_USE_CONSTMEM
	2970	+ prot->pktid_map_handle_ioctl = DHD_NATIVE_TO_PKTID_INIT(dhd,
	2971	+ DHD_FLOWRING_MAX_IOCTLRESPBUF_POST);
	2972	+ if (prot->pktid_map_handle_ioctl == NULL) {
	2973	+ goto fail;
	2974	+ }
	2975	+#endif /* IOCTLRESP_USE_CONSTMEM */
	2976	+
	2977	+#ifdef DHD_MAP_PKTID_LOGGING
	2978	+ prot->pktid_dma_map = DHD_PKTID_LOG_INIT(dhd, MAX_PKTID_LOG);
	2979	+ if (prot->pktid_dma_map == NULL) {
	2980	+ DHD_ERROR(("%s: failed to allocate pktid_dma_map\n",
	2981	+ __FUNCTION__));
	2982	+ }
	2983	+
	2984	+ prot->pktid_dma_unmap = DHD_PKTID_LOG_INIT(dhd, MAX_PKTID_LOG);
	2985	+ if (prot->pktid_dma_unmap == NULL) {
	2986	+ DHD_ERROR(("%s: failed to allocate pktid_dma_unmap\n",
	2987	+ __FUNCTION__));
	2988	+ }
	2989	+#endif /* DHD_MAP_PKTID_LOGGING */
2136	2990
2137	2991	/* Initialize the work queues to be used by the Load Balancing logic */
2138	2992	#if defined(DHD_LB_TXC)
2139	2993	{
2140	2994	void *buffer;
2141	2995	buffer = MALLOC(dhd->osh, sizeof(void) DHD_LB_WORKQ_SZ);
	2996	+ if (buffer == NULL) {
	2997	+ DHD_ERROR(("%s: failed to allocate RXC work buffer\n", __FUNCTION__));
	2998	+ goto fail;
	2999	+ }
2142	3000	bcm_workq_init(&prot->tx_compl_prod, &prot->tx_compl_cons,
2143	3001	buffer, DHD_LB_WORKQ_SZ);
2144	3002	prot->tx_compl_prod_sync = 0;
2145	3003	DHD_INFO(("%s: created tx_compl_workq <%p,%d>\n",
2146	3004	__FUNCTION__, buffer, DHD_LB_WORKQ_SZ));
2147		- }
	3005	+ }
2148	3006	#endif /* DHD_LB_TXC */
2149	3007
2150	3008	#if defined(DHD_LB_RXC)
2151		- {
	3009	+ {
2152	3010	void *buffer;
2153		- buffer = MALLOC(dhd->osh, sizeof(uint32) * DHD_LB_WORKQ_SZ);
	3011	+ buffer = MALLOC(dhd->osh, sizeof(void) DHD_LB_WORKQ_SZ);
	3012	+ if (buffer == NULL) {
	3013	+ DHD_ERROR(("%s: failed to allocate RXC work buffer\n", __FUNCTION__));
	3014	+ goto fail;
	3015	+ }
2154	3016	bcm_workq_init(&prot->rx_compl_prod, &prot->rx_compl_cons,
2155	3017	buffer, DHD_LB_WORKQ_SZ);
2156	3018	prot->rx_compl_prod_sync = 0;
2157	3019	DHD_INFO(("%s: created rx_compl_workq <%p,%d>\n",
2158	3020	__FUNCTION__, buffer, DHD_LB_WORKQ_SZ));
2159		- }
	3021	+ }
2160	3022	#endif /* DHD_LB_RXC */
2161	3023
2162		-#endif /* DHD_LB */
	3024	+ /* Initialize trap buffer */
	3025	+ if (dhd_dma_buf_alloc(dhd, &dhd->prot->fw_trap_buf, trap_buf_len)) {
	3026	+ DHD_ERROR(("%s: dhd_init_trap_buffer falied\n", __FUNCTION__));
	3027	+ goto fail;
	3028	+ }
2163	3029
2164	3030	return BCME_OK;
2165	3031
2166	3032	fail:
2167	3033
2168		-#ifndef CONFIG_DHD_USE_STATIC_BUF
2169		- if (prot != NULL) {
	3034	+ if (prot) {
	3035	+ /* Free up all allocated memories */
2170	3036	dhd_prot_detach(dhd);
2171	3037	}
2172		-#endif /* CONFIG_DHD_USE_STATIC_BUF */
2173	3038
2174	3039	return BCME_NOMEM;
2175	3040	} /* dhd_prot_attach */
2176	3041
	3042	+static int
	3043	+dhd_alloc_host_scbs(dhd_pub_t *dhd)
	3044	+{
	3045	+ int ret = BCME_OK;
	3046	+ sh_addr_t base_addr;
	3047	+ dhd_prot_t *prot = dhd->prot;
	3048	+ uint32 host_scb_size = 0;
	3049	+
	3050	+ if (dhd->hscb_enable) {
	3051	+ /* read number of bytes to allocate from F/W */
	3052	+ dhd_bus_cmn_readshared(dhd->bus, &host_scb_size, HOST_SCB_ADDR, 0);
	3053	+ if (host_scb_size) {
	3054	+ /* alloc array of host scbs */
	3055	+ ret = dhd_dma_buf_alloc(dhd, &prot->host_scb_buf, host_scb_size);
	3056	+ /* write host scb address to F/W */
	3057	+ if (ret == BCME_OK) {
	3058	+ dhd_base_addr_htolpa(&base_addr, prot->host_scb_buf.pa);
	3059	+ dhd_bus_cmn_writeshared(dhd->bus, &base_addr, sizeof(base_addr),
	3060	+ HOST_SCB_ADDR, 0);
	3061	+ } else {
	3062	+ DHD_TRACE(("dhd_alloc_host_scbs: dhd_dma_buf_alloc error\n"));
	3063	+ }
	3064	+ } else {
	3065	+ DHD_TRACE(("dhd_alloc_host_scbs: host_scb_size is 0.\n"));
	3066	+ }
	3067	+ } else {
	3068	+ DHD_TRACE(("dhd_alloc_host_scbs: Host scb not supported in F/W.\n"));
	3069	+ }
	3070	+
	3071	+ return ret;
	3072	+}
	3073	+
	3074	+void
	3075	+dhd_set_host_cap(dhd_pub_t *dhd)
	3076	+{
	3077	+ uint32 data = 0;
	3078	+ dhd_prot_t *prot = dhd->prot;
	3079	+
	3080	+ if (dhd->bus->api.fw_rev >= PCIE_SHARED_VERSION_6) {
	3081	+ if (dhd->h2d_phase_supported) {
	3082	+ data \|= HOSTCAP_H2D_VALID_PHASE;
	3083	+ if (dhd->force_dongletrap_on_bad_h2d_phase)
	3084	+ data \|= HOSTCAP_H2D_ENABLE_TRAP_ON_BADPHASE;
	3085	+ }
	3086	+ if (prot->host_ipc_version > prot->device_ipc_version)
	3087	+ prot->active_ipc_version = prot->device_ipc_version;
	3088	+ else
	3089	+ prot->active_ipc_version = prot->host_ipc_version;
	3090	+
	3091	+ data \|= prot->active_ipc_version;
	3092	+
	3093	+ if (dhdpcie_bus_get_pcie_hostready_supported(dhd->bus)) {
	3094	+ DHD_INFO(("Advertise Hostready Capability\n"));
	3095	+ data \|= HOSTCAP_H2D_ENABLE_HOSTRDY;
	3096	+ }
	3097	+ {
	3098	+ /* Disable DS altogether */
	3099	+ data \|= HOSTCAP_DS_NO_OOB_DW;
	3100	+ dhdpcie_bus_enab_pcie_dw(dhd->bus, DEVICE_WAKE_NONE);
	3101	+ }
	3102	+
	3103	+ /* Indicate support for extended trap data */
	3104	+ data \|= HOSTCAP_EXTENDED_TRAP_DATA;
	3105	+
	3106	+ /* Indicate support for TX status metadata */
	3107	+ if (dhd->pcie_txs_metadata_enable != 0)
	3108	+ data \|= HOSTCAP_TXSTATUS_METADATA;
	3109	+
	3110	+ /* Enable fast delete ring in firmware if supported */
	3111	+ if (dhd->fast_delete_ring_support) {
	3112	+ data \|= HOSTCAP_FAST_DELETE_RING;
	3113	+ }
	3114	+
	3115	+ if (dhdpcie_bus_get_pcie_hwa_supported(dhd->bus)) {
	3116	+ DHD_ERROR(("HWA inited\n"));
	3117	+ /* TODO: Is hostcap needed? */
	3118	+ dhd->hwa_inited = TRUE;
	3119	+ }
	3120	+
	3121	+ if (dhdpcie_bus_get_pcie_idma_supported(dhd->bus)) {
	3122	+ DHD_ERROR(("IDMA inited\n"));
	3123	+ data \|= HOSTCAP_H2D_IDMA;
	3124	+ dhd->idma_inited = TRUE;
	3125	+ }
	3126	+
	3127	+ if (dhdpcie_bus_get_pcie_ifrm_supported(dhd->bus)) {
	3128	+ DHD_ERROR(("IFRM Inited\n"));
	3129	+ data \|= HOSTCAP_H2D_IFRM;
	3130	+ dhd->ifrm_inited = TRUE;
	3131	+ dhd->dma_h2d_ring_upd_support = FALSE;
	3132	+ dhd_prot_dma_indx_free(dhd);
	3133	+ }
	3134	+
	3135	+ if (dhdpcie_bus_get_pcie_dar_supported(dhd->bus)) {
	3136	+ DHD_ERROR(("DAR doorbell Use\n"));
	3137	+ data \|= HOSTCAP_H2D_DAR;
	3138	+ dhd->dar_inited = TRUE;
	3139	+ }
	3140	+
	3141	+ data \|= HOSTCAP_UR_FW_NO_TRAP;
	3142	+
	3143	+ if (dhd->hscb_enable) {
	3144	+ data \|= HOSTCAP_HSCB;
	3145	+ }
	3146	+
	3147	+#ifdef EWP_EDL
	3148	+ if (dhd->dongle_edl_support) {
	3149	+ data \|= HOSTCAP_EDL_RING;
	3150	+ DHD_ERROR(("Enable EDL host cap\n"));
	3151	+ } else {
	3152	+ DHD_ERROR(("DO NOT SET EDL host cap\n"));
	3153	+ }
	3154	+#endif /* EWP_EDL */
	3155	+
	3156	+#ifdef DHD_HP2P
	3157	+ if (dhd->hp2p_capable) {
	3158	+ data \|= HOSTCAP_PKT_TIMESTAMP;
	3159	+ data \|= HOSTCAP_PKT_HP2P;
	3160	+ DHD_ERROR(("Enable HP2P in host cap\n"));
	3161	+ } else {
	3162	+ DHD_ERROR(("HP2P not enabled in host cap\n"));
	3163	+ }
	3164	+#endif // endif
	3165	+
	3166	+#ifdef DHD_DB0TS
	3167	+ if (dhd->db0ts_capable) {
	3168	+ data \|= HOSTCAP_DB0_TIMESTAMP;
	3169	+ DHD_ERROR(("Enable DB0 TS in host cap\n"));
	3170	+ } else {
	3171	+ DHD_ERROR(("DB0 TS not enabled in host cap\n"));
	3172	+ }
	3173	+#endif /* DHD_DB0TS */
	3174	+ if (dhd->extdtxs_in_txcpl) {
	3175	+ DHD_ERROR(("Enable hostcap: EXTD TXS in txcpl\n"));
	3176	+ data \|= HOSTCAP_PKT_TXSTATUS;
	3177	+ }
	3178	+ else {
	3179	+ DHD_ERROR(("Enable hostcap: EXTD TXS in txcpl\n"));
	3180	+ }
	3181	+
	3182	+ DHD_INFO(("%s:Active Ver:%d, Host Ver:%d, FW Ver:%d\n",
	3183	+ __FUNCTION__,
	3184	+ prot->active_ipc_version, prot->host_ipc_version,
	3185	+ prot->device_ipc_version));
	3186	+
	3187	+ dhd_bus_cmn_writeshared(dhd->bus, &data, sizeof(uint32), HOST_API_VERSION, 0);
	3188	+ dhd_bus_cmn_writeshared(dhd->bus, &prot->fw_trap_buf.pa,
	3189	+ sizeof(prot->fw_trap_buf.pa), DNGL_TO_HOST_TRAP_ADDR, 0);
	3190	+ }
	3191	+
	3192	+}
2177	3193
2178	3194	/**
2179	3195	* dhd_prot_init - second stage of dhd_prot_attach. Now that the dongle has
..	..	@@ -2187,43 +3203,24 @@
2187	3203	{
2188	3204	sh_addr_t base_addr;
2189	3205	dhd_prot_t *prot = dhd->prot;
	3206	+ int ret = 0;
	3207	+ uint32 idmacontrol;
	3208	+ uint32 waitcount = 0;
2190	3209
2191		- /* PKTID handle INIT */
2192		- if (prot->pktid_map_handle != NULL) {
2193		- DHD_ERROR(("%s: pktid_map_handle already set!\n", __FUNCTION__));
2194		- ASSERT(0);
2195		- return BCME_ERROR;
2196		- }
	3210	+#ifdef WL_MONITOR
	3211	+ dhd->monitor_enable = FALSE;
	3212	+#endif /* WL_MONITOR */
2197	3213
2198		-#ifdef IOCTLRESP_USE_CONSTMEM
2199		- if (prot->pktid_map_handle_ioctl != NULL) {
2200		- DHD_ERROR(("%s: pktid_map_handle_ioctl already set!\n", __FUNCTION__));
2201		- ASSERT(0);
2202		- return BCME_ERROR;
2203		- }
2204		-#endif /* IOCTLRESP_USE_CONSTMEM */
	3214	+ /**
	3215	+ * A user defined value can be assigned to global variable h2d_max_txpost via
	3216	+ * 1. DHD IOVAR h2d_max_txpost, before firmware download
	3217	+ * 2. module parameter h2d_max_txpost
	3218	+ * prot->h2d_max_txpost is assigned with H2DRING_TXPOST_MAX_ITEM,
	3219	+ * if user has not defined any buffers by one of the above methods.
	3220	+ */
	3221	+ prot->h2d_max_txpost = (uint16)h2d_max_txpost;
2205	3222
2206		- prot->pktid_map_handle = DHD_NATIVE_TO_PKTID_INIT(dhd, MAX_PKTID_ITEMS, PKTID_MAP_HANDLE);
2207		- if (prot->pktid_map_handle == NULL) {
2208		- DHD_ERROR(("%s: Unable to map packet id's\n", __FUNCTION__));
2209		- ASSERT(0);
2210		- return BCME_NOMEM;
2211		- }
2212		-
2213		-#ifdef IOCTLRESP_USE_CONSTMEM
2214		- prot->pktid_map_handle_ioctl = DHD_NATIVE_TO_PKTID_INIT(dhd,
2215		- DHD_FLOWRING_MAX_IOCTLRESPBUF_POST, PKTID_MAP_HANDLE_IOCTL);
2216		- if (prot->pktid_map_handle_ioctl == NULL) {
2217		- DHD_ERROR(("%s: Unable to map ioctl response buffers\n", __FUNCTION__));
2218		- ASSERT(0);
2219		- return BCME_NOMEM;
2220		- }
2221		-#endif /* IOCTLRESP_USE_CONSTMEM */
2222		-
2223		- /* Max pkts in ring */
2224		- prot->max_tx_count = H2DRING_TXPOST_MAX_ITEM;
2225		-
2226		- DHD_INFO(("%s:%d: MAX_TX_COUNT = %d\n", __FUNCTION__, __LINE__, prot->max_tx_count));
	3223	+ DHD_ERROR(("%s:%d: h2d_max_txpost = %d\n", __FUNCTION__, __LINE__, prot->h2d_max_txpost));
2227	3224
2228	3225	/* Read max rx packets supported by dongle */
2229	3226	dhd_bus_cmn_readshared(dhd->bus, &prot->max_rxbufpost, MAX_HOST_RXBUFS, 0);
..	..	@@ -2232,21 +3229,24 @@
2232	3229	/* using the latest shared structure template */
2233	3230	prot->max_rxbufpost = DEFAULT_RX_BUFFERS_TO_POST;
2234	3231	}
2235		- DHD_INFO(("%s:%d: MAX_RXBUFPOST = %d\n", __FUNCTION__, __LINE__, prot->max_rxbufpost));
	3232	+ DHD_ERROR(("%s:%d: MAX_RXBUFPOST = %d\n", __FUNCTION__, __LINE__, prot->max_rxbufpost));
2236	3233
2237	3234	/* Initialize. bzero() would blow away the dma pointers. */
2238	3235	prot->max_eventbufpost = DHD_FLOWRING_MAX_EVENTBUF_POST;
2239	3236	prot->max_ioctlrespbufpost = DHD_FLOWRING_MAX_IOCTLRESPBUF_POST;
	3237	+ prot->max_infobufpost = DHD_H2D_INFORING_MAX_BUF_POST;
	3238	+ prot->max_tsbufpost = DHD_MAX_TSBUF_POST;
2240	3239
2241	3240	prot->cur_ioctlresp_bufs_posted = 0;
2242		- prot->active_tx_count = 0;
	3241	+ OSL_ATOMIC_INIT(dhd->osh, &prot->active_tx_count);
2243	3242	prot->data_seq_no = 0;
2244	3243	prot->ioctl_seq_no = 0;
2245	3244	prot->rxbufpost = 0;
2246	3245	prot->cur_event_bufs_posted = 0;
2247	3246	prot->ioctl_state = 0;
2248	3247	prot->curr_ioctl_cmd = 0;
2249		- prot->ioctl_received = IOCTL_WAIT;
	3248	+ prot->cur_ts_bufs_posted = 0;
	3249	+ prot->infobufpost = 0;
2250	3250
2251	3251	prot->dmaxfer.srcmem.va = NULL;
2252	3252	prot->dmaxfer.dstmem.va = NULL;
..	..	@@ -2257,23 +3257,50 @@
2257	3257	prot->tx_metadata_offset = 0;
2258	3258	prot->txp_threshold = TXP_FLUSH_MAX_ITEMS_FLUSH_CNT;
2259	3259
2260		- prot->ioctl_trans_id = 0;
	3260	+ /* To catch any rollover issues fast, starting with higher ioctl_trans_id */
	3261	+ prot->ioctl_trans_id = MAXBITVAL(NBITS(prot->ioctl_trans_id)) - BUFFER_BEFORE_ROLLOVER;
	3262	+ prot->ioctl_state = 0;
	3263	+ prot->ioctl_status = 0;
	3264	+ prot->ioctl_resplen = 0;
	3265	+ prot->ioctl_received = IOCTL_WAIT;
	3266	+
	3267	+ /* Initialize Common MsgBuf Rings */
	3268	+
	3269	+ prot->device_ipc_version = dhd->bus->api.fw_rev;
	3270	+ prot->host_ipc_version = PCIE_SHARED_VERSION;
	3271	+ prot->no_tx_resource = FALSE;
	3272	+
	3273	+ /* Init the host API version */
	3274	+ dhd_set_host_cap(dhd);
	3275	+
	3276	+ /* alloc and configure scb host address for dongle */
	3277	+ if ((ret = dhd_alloc_host_scbs(dhd))) {
	3278	+ return ret;
	3279	+ }
2261	3280
2262	3281	/* Register the interrupt function upfront */
2263	3282	/* remove corerev checks in data path */
	3283	+ /* do this after host/fw negotiation for DAR */
2264	3284	prot->mb_ring_fn = dhd_bus_get_mbintr_fn(dhd->bus);
	3285	+ prot->mb_2_ring_fn = dhd_bus_get_mbintr_2_fn(dhd->bus);
2265	3286
2266		- /* Initialize Common MsgBuf Rings */
	3287	+ dhd->bus->_dar_war = (dhd->bus->sih->buscorerev < 64) ? TRUE : FALSE;
2267	3288
2268	3289	dhd_prot_ring_init(dhd, &prot->h2dring_ctrl_subn);
2269	3290	dhd_prot_ring_init(dhd, &prot->h2dring_rxp_subn);
2270	3291	dhd_prot_ring_init(dhd, &prot->d2hring_ctrl_cpln);
	3292	+
	3293	+ /* Make it compatibile with pre-rev7 Firmware */
	3294	+ if (prot->active_ipc_version < PCIE_SHARED_VERSION_7) {
	3295	+ prot->d2hring_tx_cpln.item_len =
	3296	+ D2HRING_TXCMPLT_ITEMSIZE_PREREV7;
	3297	+ prot->d2hring_rx_cpln.item_len =
	3298	+ D2HRING_RXCMPLT_ITEMSIZE_PREREV7;
	3299	+ }
2271	3300	dhd_prot_ring_init(dhd, &prot->d2hring_tx_cpln);
2272	3301	dhd_prot_ring_init(dhd, &prot->d2hring_rx_cpln);
2273	3302
2274		-#if defined(PCIE_D2H_SYNC)
2275	3303	dhd_prot_d2h_sync_init(dhd);
2276		-#endif /* PCIE_D2H_SYNC */
2277	3304
2278	3305	dhd_prot_h2d_sync_init(dhd);
2279	3306
..	..	@@ -2287,7 +3314,7 @@
2287	3314	/* If supported by the host, indicate the memory block
2288	3315	* for completion writes / submission reads to shared space
2289	3316	*/
2290		- if (DMA_INDX_ENAB(dhd->dma_d2h_ring_upd_support)) {
	3317	+ if (dhd->dma_d2h_ring_upd_support) {
2291	3318	dhd_base_addr_htolpa(&base_addr, prot->d2h_dma_indx_wr_buf.pa);
2292	3319	dhd_bus_cmn_writeshared(dhd->bus, &base_addr, sizeof(base_addr),
2293	3320	D2H_DMA_INDX_WR_BUF, 0);
..	..	@@ -2296,7 +3323,7 @@
2296	3323	H2D_DMA_INDX_RD_BUF, 0);
2297	3324	}
2298	3325
2299		- if (DMA_INDX_ENAB(dhd->dma_h2d_ring_upd_support)) {
	3326	+ if (dhd->dma_h2d_ring_upd_support \|\| IDMA_ENAB(dhd)) {
2300	3327	dhd_base_addr_htolpa(&base_addr, prot->h2d_dma_indx_wr_buf.pa);
2301	3328	dhd_bus_cmn_writeshared(dhd->bus, &base_addr, sizeof(base_addr),
2302	3329	H2D_DMA_INDX_WR_BUF, 0);
..	..	@@ -2304,6 +3331,9 @@
2304	3331	dhd_bus_cmn_writeshared(dhd->bus, &base_addr, sizeof(base_addr),
2305	3332	D2H_DMA_INDX_RD_BUF, 0);
2306	3333	}
	3334	+ /* Signal to the dongle that common ring init is complete */
	3335	+ if (dhd->hostrdy_after_init)
	3336	+ dhd_bus_hostready(dhd->bus);
2307	3337
2308	3338	/*
2309	3339	* If the DMA-able buffers for flowring needs to come from a specific
..	..	@@ -2317,26 +3347,123 @@
2317	3347	return BCME_ERROR;
2318	3348	}
2319	3349
	3350	+ /* If IFRM is enabled, wait for FW to setup the DMA channel */
	3351	+ if (IFRM_ENAB(dhd)) {
	3352	+ dhd_base_addr_htolpa(&base_addr, prot->h2d_ifrm_indx_wr_buf.pa);
	3353	+ dhd_bus_cmn_writeshared(dhd->bus, &base_addr, sizeof(base_addr),
	3354	+ H2D_IFRM_INDX_WR_BUF, 0);
	3355	+ }
	3356	+
	3357	+ /* If IDMA is enabled and initied, wait for FW to setup the IDMA descriptors
	3358	+ * Waiting just before configuring doorbell
	3359	+ */
	3360	+#ifdef BCMQT
	3361	+#define IDMA_ENABLE_WAIT 100
	3362	+#else
	3363	+#define IDMA_ENABLE_WAIT 10
	3364	+#endif // endif
	3365	+ if (IDMA_ACTIVE(dhd)) {
	3366	+ /* wait for idma_en bit in IDMAcontrol register to be set */
	3367	+ /* Loop till idma_en is not set */
	3368	+ uint buscorerev = dhd->bus->sih->buscorerev;
	3369	+ idmacontrol = si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx,
	3370	+ IDMAControl(buscorerev), 0, 0);
	3371	+ while (!(idmacontrol & PCIE_IDMA_MODE_EN(buscorerev)) &&
	3372	+ (waitcount++ < IDMA_ENABLE_WAIT)) {
	3373	+
	3374	+ DHD_ERROR(("iDMA not enabled yet,waiting 1 ms c=%d IDMAControl = %08x\n",
	3375	+ waitcount, idmacontrol));
	3376	+#ifdef BCMQT
	3377	+ OSL_DELAY(200000); /* 200msec for BCMQT */
	3378	+#else
	3379	+ OSL_DELAY(1000); /* 1ms as its onetime only */
	3380	+#endif // endif
	3381	+ idmacontrol = si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx,
	3382	+ IDMAControl(buscorerev), 0, 0);
	3383	+ }
	3384	+
	3385	+ if (waitcount < IDMA_ENABLE_WAIT) {
	3386	+ DHD_ERROR(("iDMA enabled PCIEControl = %08x\n", idmacontrol));
	3387	+ } else {
	3388	+ DHD_ERROR(("Error: wait for iDMA timed out wait=%d IDMAControl = %08x\n",
	3389	+ waitcount, idmacontrol));
	3390	+ return BCME_ERROR;
	3391	+ }
	3392	+ }
	3393	+
2320	3394	/* Host should configure soft doorbells if needed ... here */
2321	3395
2322	3396	/* Post to dongle host configured soft doorbells */
2323	3397	dhd_msgbuf_ring_config_d2h_soft_doorbell(dhd);
2324	3398
2325		- /* Post buffers for packet reception and ioctl/event responses */
2326		- dhd_msgbuf_rxbuf_post(dhd, FALSE); /* alloc pkt ids */
2327	3399	dhd_msgbuf_rxbuf_post_ioctlresp_bufs(dhd);
2328	3400	dhd_msgbuf_rxbuf_post_event_bufs(dhd);
2329	3401
	3402	+ prot->no_retry = FALSE;
	3403	+ prot->no_aggr = FALSE;
	3404	+ prot->fixed_rate = FALSE;
	3405	+
	3406	+ /*
	3407	+ * Note that any communication with the Dongle should be added
	3408	+ * below this point. Any other host data structure initialiation that
	3409	+ * needs to be done prior to the DPC starts executing should be done
	3410	+ * befor this point.
	3411	+ * Because once we start sending H2D requests to Dongle, the Dongle
	3412	+ * respond immediately. So the DPC context to handle this
	3413	+ * D2H response could preempt the context in which dhd_prot_init is running.
	3414	+ * We want to ensure that all the Host part of dhd_prot_init is
	3415	+ * done before that.
	3416	+ */
	3417	+
	3418	+ /* See if info rings could be created, info rings should be created
	3419	+ * only if dongle does not support EDL
	3420	+ */
	3421	+#ifdef EWP_EDL
	3422	+ if (dhd->bus->api.fw_rev >= PCIE_SHARED_VERSION_6 && !dhd->dongle_edl_support)
	3423	+#else
	3424	+ if (dhd->bus->api.fw_rev >= PCIE_SHARED_VERSION_6)
	3425	+#endif /* EWP_EDL */
	3426	+ {
	3427	+ if ((ret = dhd_prot_init_info_rings(dhd)) != BCME_OK) {
	3428	+ /* For now log and proceed, further clean up action maybe necessary
	3429	+ * when we have more clarity.
	3430	+ */
	3431	+ DHD_ERROR(("%s Info rings couldn't be created: Err Code%d",
	3432	+ __FUNCTION__, ret));
	3433	+ }
	3434	+ }
	3435	+
	3436	+#ifdef EWP_EDL
	3437	+ /* Create Enhanced Debug Lane rings (EDL) if dongle supports it */
	3438	+ if (dhd->dongle_edl_support) {
	3439	+ if ((ret = dhd_prot_init_edl_rings(dhd)) != BCME_OK) {
	3440	+ DHD_ERROR(("%s EDL rings couldn't be created: Err Code%d",
	3441	+ __FUNCTION__, ret));
	3442	+ }
	3443	+ }
	3444	+#endif /* EWP_EDL */
	3445	+
	3446	+#ifdef DHD_HP2P
	3447	+ /* create HPP txcmpl/rxcmpl rings */
	3448	+ if (dhd->bus->api.fw_rev >= PCIE_SHARED_VERSION_7 && dhd->hp2p_capable) {
	3449	+ if ((ret = dhd_prot_init_hp2p_rings(dhd)) != BCME_OK) {
	3450	+ /* For now log and proceed, further clean up action maybe necessary
	3451	+ * when we have more clarity.
	3452	+ */
	3453	+ DHD_ERROR(("%s HP2P rings couldn't be created: Err Code%d",
	3454	+ __FUNCTION__, ret));
	3455	+ }
	3456	+ }
	3457	+#endif /* DHD_HP2P */
	3458	+
2330	3459	return BCME_OK;
2331	3460	} /* dhd_prot_init */
2332		-
2333	3461
2334	3462	/**
2335	3463	* dhd_prot_detach - PCIE FD protocol layer destructor.
2336	3464	* Unlink, frees allocated protocol memory (including dhd_prot)
2337	3465	*/
2338		-void
2339		-dhd_prot_detach(dhd_pub_t *dhd)
	3466	+void dhd_prot_detach(dhd_pub_t *dhd)
2340	3467	{
2341	3468	dhd_prot_t *prot = dhd->prot;
2342	3469
..	..	@@ -2346,15 +3473,20 @@
2346	3473	/* free up all DMA-able buffers allocated during prot attach/init */
2347	3474
2348	3475	dhd_dma_buf_free(dhd, &prot->d2h_dma_scratch_buf);
2349		- dhd_dma_buf_free(dhd, &prot->retbuf); /* ioctl return buffer */
	3476	+ dhd_dma_buf_free(dhd, &prot->retbuf);
2350	3477	dhd_dma_buf_free(dhd, &prot->ioctbuf);
2351	3478	dhd_dma_buf_free(dhd, &prot->host_bus_throughput_buf);
	3479	+ dhd_dma_buf_free(dhd, &prot->hostts_req_buf);
	3480	+ dhd_dma_buf_free(dhd, &prot->fw_trap_buf);
	3481	+ dhd_dma_buf_free(dhd, &prot->host_scb_buf);
2352	3482
2353	3483	/* DMA-able buffers for DMAing H2D/D2H WR/RD indices */
2354	3484	dhd_dma_buf_free(dhd, &prot->h2d_dma_indx_wr_buf);
2355	3485	dhd_dma_buf_free(dhd, &prot->h2d_dma_indx_rd_buf);
2356	3486	dhd_dma_buf_free(dhd, &prot->d2h_dma_indx_wr_buf);
2357	3487	dhd_dma_buf_free(dhd, &prot->d2h_dma_indx_rd_buf);
	3488	+
	3489	+ dhd_dma_buf_free(dhd, &prot->h2d_ifrm_indx_wr_buf);
2358	3490
2359	3491	/* Common MsgBuf Rings */
2360	3492	dhd_prot_ring_detach(dhd, &prot->h2dring_ctrl_subn);
..	..	@@ -2366,41 +3498,61 @@
2366	3498	/* Detach each DMA-able buffer and free the pool of msgbuf_ring_t */
2367	3499	dhd_prot_flowrings_pool_detach(dhd);
2368	3500
2369		- if (dhd->prot->pktid_map_handle) {
2370		- DHD_NATIVE_TO_PKTID_FINI(dhd, dhd->prot->pktid_map_handle);
2371		- }
	3501	+ /* detach info rings */
	3502	+ dhd_prot_detach_info_rings(dhd);
2372	3503
2373		-#ifndef CONFIG_DHD_USE_STATIC_BUF
2374		- MFREE(dhd->osh, dhd->prot, sizeof(dhd_prot_t));
2375		-#endif /* CONFIG_DHD_USE_STATIC_BUF */
	3504	+#ifdef EWP_EDL
	3505	+ dhd_prot_detach_edl_rings(dhd);
	3506	+#endif // endif
	3507	+#ifdef DHD_HP2P
	3508	+ /* detach HPP rings */
	3509	+ dhd_prot_detach_hp2p_rings(dhd);
	3510	+#endif /* DHD_HP2P */
2376	3511
2377		-#if defined(DHD_LB)
	3512	+ /* if IOCTLRESP_USE_CONSTMEM is defined IOCTL PKTs use pktid_map_handle_ioctl
	3513	+ * handler and PKT memory is allocated using alloc_ioctl_return_buffer(), Otherwise
	3514	+ * they will be part of pktid_ctrl_map handler and PKT memory is allocated using
	3515	+ * PKTGET_STATIC (if DHD_USE_STATIC_CTRLBUF is defined) OR PKGET.
	3516	+ * Similarly for freeing PKT buffers DHD_NATIVE_TO_PKTID_FINI will be used
	3517	+ * which calls PKTFREE_STATIC (if DHD_USE_STATIC_CTRLBUF is defined) OR PKFREE.
	3518	+ * Else if IOCTLRESP_USE_CONSTMEM is defined IOCTL PKTs will be freed using
	3519	+ * DHD_NATIVE_TO_PKTID_FINI_IOCTL which calls free_ioctl_return_buffer.
	3520	+ */
	3521	+ DHD_NATIVE_TO_PKTID_FINI(dhd, prot->pktid_ctrl_map);
	3522	+ DHD_NATIVE_TO_PKTID_FINI(dhd, prot->pktid_rx_map);
	3523	+ DHD_NATIVE_TO_PKTID_FINI(dhd, prot->pktid_tx_map);
	3524	+#ifdef IOCTLRESP_USE_CONSTMEM
	3525	+ DHD_NATIVE_TO_PKTID_FINI_IOCTL(dhd, prot->pktid_map_handle_ioctl);
	3526	+#endif // endif
	3527	+#ifdef DHD_MAP_PKTID_LOGGING
	3528	+ DHD_PKTID_LOG_FINI(dhd, prot->pktid_dma_map);
	3529	+ DHD_PKTID_LOG_FINI(dhd, prot->pktid_dma_unmap);
	3530	+#endif /* DHD_MAP_PKTID_LOGGING */
	3531	+
2378	3532	#if defined(DHD_LB_TXC)
2379		- if (prot->tx_compl_prod.buffer) {
	3533	+ if (prot->tx_compl_prod.buffer)
2380	3534	MFREE(dhd->osh, prot->tx_compl_prod.buffer,
2381		- sizeof(void) DHD_LB_WORKQ_SZ);
2382		- }
	3535	+ sizeof(void) DHD_LB_WORKQ_SZ);
2383	3536	#endif /* DHD_LB_TXC */
2384	3537	#if defined(DHD_LB_RXC)
2385		- if (prot->rx_compl_prod.buffer) {
	3538	+ if (prot->rx_compl_prod.buffer)
2386	3539	MFREE(dhd->osh, prot->rx_compl_prod.buffer,
2387		- sizeof(void) DHD_LB_WORKQ_SZ);
2388		- }
	3540	+ sizeof(void) DHD_LB_WORKQ_SZ);
2389	3541	#endif /* DHD_LB_RXC */
2390		-#endif /* DHD_LB */
	3542	+
	3543	+ DHD_OS_PREFREE(dhd, dhd->prot, sizeof(dhd_prot_t));
2391	3544
2392	3545	dhd->prot = NULL;
2393	3546	}
2394	3547	} /* dhd_prot_detach */
2395	3548
2396		-
2397	3549	/**
2398		- * dhd_prot_reset - Reset the protocol layer without freeing any objects. This
2399		- * may be invoked to soft reboot the dongle, without having to detach and attach
2400		- * the entire protocol layer.
	3550	+ * dhd_prot_reset - Reset the protocol layer without freeing any objects.
	3551	+ * This may be invoked to soft reboot the dongle, without having to
	3552	+ * detach and attach the entire protocol layer.
2401	3553	*
2402		- * After dhd_prot_reset(), dhd_prot_init() may be invoked without going through
2403		- * a dhd_prot_attach() phase.
	3554	+ * After dhd_prot_reset(), dhd_prot_init() may be invoked
	3555	+ * without going througha dhd_prot_attach() phase.
2404	3556	*/
2405	3557	void
2406	3558	dhd_prot_reset(dhd_pub_t *dhd)
..	..	@@ -2415,20 +3567,43 @@
2415	3567
2416	3568	dhd_prot_flowrings_pool_reset(dhd);
2417	3569
	3570	+ /* Reset Common MsgBuf Rings */
2418	3571	dhd_prot_ring_reset(dhd, &prot->h2dring_ctrl_subn);
2419	3572	dhd_prot_ring_reset(dhd, &prot->h2dring_rxp_subn);
2420	3573	dhd_prot_ring_reset(dhd, &prot->d2hring_ctrl_cpln);
2421	3574	dhd_prot_ring_reset(dhd, &prot->d2hring_tx_cpln);
2422	3575	dhd_prot_ring_reset(dhd, &prot->d2hring_rx_cpln);
2423	3576
	3577	+ /* Reset info rings */
	3578	+ if (prot->h2dring_info_subn) {
	3579	+ dhd_prot_ring_reset(dhd, prot->h2dring_info_subn);
	3580	+ }
	3581	+
	3582	+ if (prot->d2hring_info_cpln) {
	3583	+ dhd_prot_ring_reset(dhd, prot->d2hring_info_cpln);
	3584	+ }
	3585	+#ifdef EWP_EDL
	3586	+ if (prot->d2hring_edl) {
	3587	+ dhd_prot_ring_reset(dhd, prot->d2hring_edl);
	3588	+ }
	3589	+#endif /* EWP_EDL */
	3590	+
	3591	+ /* Reset all DMA-able buffers allocated during prot attach */
	3592	+ dhd_dma_buf_reset(dhd, &prot->d2h_dma_scratch_buf);
2424	3593	dhd_dma_buf_reset(dhd, &prot->retbuf);
2425	3594	dhd_dma_buf_reset(dhd, &prot->ioctbuf);
2426		- dhd_dma_buf_reset(dhd, &prot->d2h_dma_scratch_buf);
	3595	+ dhd_dma_buf_reset(dhd, &prot->host_bus_throughput_buf);
	3596	+ dhd_dma_buf_reset(dhd, &prot->hostts_req_buf);
	3597	+ dhd_dma_buf_reset(dhd, &prot->fw_trap_buf);
	3598	+ dhd_dma_buf_reset(dhd, &prot->host_scb_buf);
	3599	+
	3600	+ dhd_dma_buf_reset(dhd, &prot->h2d_ifrm_indx_wr_buf);
	3601	+
	3602	+ /* Rest all DMA-able buffers for DMAing H2D/D2H WR/RD indices */
2427	3603	dhd_dma_buf_reset(dhd, &prot->h2d_dma_indx_rd_buf);
2428	3604	dhd_dma_buf_reset(dhd, &prot->h2d_dma_indx_wr_buf);
2429	3605	dhd_dma_buf_reset(dhd, &prot->d2h_dma_indx_rd_buf);
2430	3606	dhd_dma_buf_reset(dhd, &prot->d2h_dma_indx_wr_buf);
2431		-
2432	3607
2433	3608	prot->rx_metadata_offset = 0;
2434	3609	prot->tx_metadata_offset = 0;
..	..	@@ -2437,13 +3612,14 @@
2437	3612	prot->cur_event_bufs_posted = 0;
2438	3613	prot->cur_ioctlresp_bufs_posted = 0;
2439	3614
2440		- prot->active_tx_count = 0;
	3615	+ OSL_ATOMIC_INIT(dhd->osh, &prot->active_tx_count);
2441	3616	prot->data_seq_no = 0;
2442	3617	prot->ioctl_seq_no = 0;
2443	3618	prot->ioctl_state = 0;
2444	3619	prot->curr_ioctl_cmd = 0;
2445	3620	prot->ioctl_received = IOCTL_WAIT;
2446		- prot->ioctl_trans_id = 0;
	3621	+ /* To catch any rollover issues fast, starting with higher ioctl_trans_id */
	3622	+ prot->ioctl_trans_id = MAXBITVAL(NBITS(prot->ioctl_trans_id)) - BUFFER_BEFORE_ROLLOVER;
2447	3623
2448	3624	/* dhd_flow_rings_init is located at dhd_bus_start,
2449	3625	* so when stopping bus, flowrings shall be deleted
..	..	@@ -2452,123 +3628,84 @@
2452	3628	dhd_flow_rings_deinit(dhd);
2453	3629	}
2454	3630
2455		- if (prot->pktid_map_handle) {
2456		- DHD_NATIVE_TO_PKTID_FINI(dhd, prot->pktid_map_handle);
2457		- prot->pktid_map_handle = NULL;
	3631	+#ifdef DHD_HP2P
	3632	+ if (prot->d2hring_hp2p_txcpl) {
	3633	+ dhd_prot_ring_reset(dhd, prot->d2hring_hp2p_txcpl);
2458	3634	}
	3635	+ if (prot->d2hring_hp2p_rxcpl) {
	3636	+ dhd_prot_ring_reset(dhd, prot->d2hring_hp2p_rxcpl);
	3637	+ }
	3638	+#endif /* DHD_HP2P */
2459	3639
	3640	+ /* Reset PKTID map */
	3641	+ DHD_NATIVE_TO_PKTID_RESET(dhd, prot->pktid_ctrl_map);
	3642	+ DHD_NATIVE_TO_PKTID_RESET(dhd, prot->pktid_rx_map);
	3643	+ DHD_NATIVE_TO_PKTID_RESET(dhd, prot->pktid_tx_map);
2460	3644	#ifdef IOCTLRESP_USE_CONSTMEM
2461		- if (prot->pktid_map_handle_ioctl) {
2462		- DHD_NATIVE_TO_PKTID_FINI_IOCTL(dhd, prot->pktid_map_handle_ioctl);
2463		- prot->pktid_map_handle_ioctl = NULL;
2464		- }
	3645	+ DHD_NATIVE_TO_PKTID_RESET_IOCTL(dhd, prot->pktid_map_handle_ioctl);
2465	3646	#endif /* IOCTLRESP_USE_CONSTMEM */
	3647	+#ifdef DMAMAP_STATS
	3648	+ dhd->dma_stats.txdata = dhd->dma_stats.txdata_sz = 0;
	3649	+ dhd->dma_stats.rxdata = dhd->dma_stats.rxdata_sz = 0;
	3650	+#ifndef IOCTLRESP_USE_CONSTMEM
	3651	+ dhd->dma_stats.ioctl_rx = dhd->dma_stats.ioctl_rx_sz = 0;
	3652	+#endif /* IOCTLRESP_USE_CONSTMEM */
	3653	+ dhd->dma_stats.event_rx = dhd->dma_stats.event_rx_sz = 0;
	3654	+ dhd->dma_stats.info_rx = dhd->dma_stats.info_rx_sz = 0;
	3655	+ dhd->dma_stats.tsbuf_rx = dhd->dma_stats.tsbuf_rx_sz = 0;
	3656	+#endif /* DMAMAP_STATS */
2466	3657	} /* dhd_prot_reset */
2467	3658
	3659	+#if defined(DHD_LB_RXP)
	3660	+#define DHD_LB_DISPATCH_RX_PROCESS(dhdp) dhd_lb_dispatch_rx_process(dhdp)
	3661	+#else /* !DHD_LB_RXP */
	3662	+#define DHD_LB_DISPATCH_RX_PROCESS(dhdp) do { /* noop */ } while (0)
	3663	+#endif /* !DHD_LB_RXP */
2468	3664
2469		-void
2470		-dhd_prot_rx_dataoffset(dhd_pub_t *dhd, uint32 rx_offset)
2471		-{
2472		- dhd_prot_t *prot = dhd->prot;
2473		- prot->rx_dataoffset = rx_offset;
2474		-}
	3665	+#if defined(DHD_LB_RXC)
	3666	+#define DHD_LB_DISPATCH_RX_COMPL(dhdp) dhd_lb_dispatch_rx_compl(dhdp)
	3667	+#else /* !DHD_LB_RXC */
	3668	+#define DHD_LB_DISPATCH_RX_COMPL(dhdp) do { /* noop */ } while (0)
	3669	+#endif /* !DHD_LB_RXC */
2475	3670
2476		-/**
2477		- * Initialize protocol: sync w/dongle state.
2478		- * Sets dongle media info (iswl, drv_version, mac address).
2479		- */
2480		-int
2481		-dhd_sync_with_dongle(dhd_pub_t *dhd)
2482		-{
2483		- int ret = 0;
2484		- wlc_rev_info_t revinfo;
2485		-
2486		-
2487		- DHD_TRACE(("%s: Enter\n", __FUNCTION__));
2488		-
2489		- dhd_os_set_ioctl_resp_timeout(IOCTL_RESP_TIMEOUT);
2490		-
2491		-
2492		-
2493		-#ifdef DHD_FW_COREDUMP
2494		- /* For Android Builds check memdump capability */
2495		- /* Check the memdump capability */
2496		- dhd_get_memdump_info(dhd);
2497		-#endif /* DHD_FW_COREDUMP */
2498		-
2499		- /* Get the device rev info */
2500		- memset(&revinfo, 0, sizeof(revinfo));
2501		- ret = dhd_wl_ioctl_cmd(dhd, WLC_GET_REVINFO, &revinfo, sizeof(revinfo), FALSE, 0);
2502		- if (ret < 0) {
2503		- DHD_ERROR(("%s: GET revinfo FAILED\n", __FUNCTION__));
2504		- goto done;
2505		- }
2506		- DHD_ERROR(("%s: GET_REVINFO device 0x%x, vendor 0x%x, chipnum 0x%x\n", __FUNCTION__,
2507		- revinfo.deviceid, revinfo.vendorid, revinfo.chipnum));
2508		-
2509		- dhd_process_cid_mac(dhd, TRUE);
2510		-
2511		- ret = dhd_preinit_ioctls(dhd);
2512		- if (!ret) {
2513		- dhd_process_cid_mac(dhd, FALSE);
2514		- }
2515		-
2516		- /* Always assumes wl for now */
2517		- dhd->iswl = TRUE;
2518		-done:
2519		- return ret;
2520		-} /* dhd_sync_with_dongle */
	3671	+#if defined(DHD_LB_TXC)
	3672	+#define DHD_LB_DISPATCH_TX_COMPL(dhdp) dhd_lb_dispatch_tx_compl(dhdp)
	3673	+#else /* !DHD_LB_TXC */
	3674	+#define DHD_LB_DISPATCH_TX_COMPL(dhdp) do { /* noop */ } while (0)
	3675	+#endif /* !DHD_LB_TXC */
2521	3676
2522	3677	#if defined(DHD_LB)
2523		-
2524	3678	/* DHD load balancing: deferral of work to another online CPU */
2525		-
2526	3679	/* DHD_LB_TXC DHD_LB_RXC DHD_LB_RXP dispatchers, in dhd_linux.c */
2527	3680	extern void dhd_lb_tx_compl_dispatch(dhd_pub_t *dhdp);
2528	3681	extern void dhd_lb_rx_compl_dispatch(dhd_pub_t *dhdp);
2529	3682	extern void dhd_lb_rx_napi_dispatch(dhd_pub_t *dhdp);
2530		-
2531	3683	extern void dhd_lb_rx_pkt_enqueue(dhd_pub_t dhdp, void pkt, int ifidx);
2532	3684
	3685	+#if defined(DHD_LB_RXP)
2533	3686	/**
2534		- * dhd_lb_dispatch - load balance by dispatch work to other CPU cores
2535		- * Note: rx_compl_tasklet is dispatched explicitly.
	3687	+ * dhd_lb_dispatch_rx_process - load balance by dispatch Rx processing work
	3688	+ * to other CPU cores
2536	3689	*/
2537	3690	static INLINE void
2538		-dhd_lb_dispatch(dhd_pub_t *dhdp, uint16 ring_idx)
	3691	+dhd_lb_dispatch_rx_process(dhd_pub_t *dhdp)
2539	3692	{
2540		- switch (ring_idx) {
	3693	+ dhd_lb_rx_napi_dispatch(dhdp); /* dispatch rx_process_napi */
	3694	+}
	3695	+#endif /* DHD_LB_RXP */
2541	3696
2542	3697	#if defined(DHD_LB_TXC)
2543		- case BCMPCIE_D2H_MSGRING_TX_COMPLETE:
2544		- bcm_workq_prod_sync(&dhdp->prot->tx_compl_prod); /* flush WR index */
2545		- dhd_lb_tx_compl_dispatch(dhdp); /* dispatch tx_compl_tasklet */
2546		- break;
2547		-#endif /* DHD_LB_TXC */
2548		-
2549		- case BCMPCIE_D2H_MSGRING_RX_COMPLETE:
2550		- {
2551		-#if defined(DHD_LB_RXC)
2552		- dhd_prot_t *prot = dhdp->prot;
2553		- /* Schedule the takslet only if we have to */
2554		- if (prot->rxbufpost <= (prot->max_rxbufpost - RXBUFPOST_THRESHOLD)) {
2555		- /* flush WR index */
2556		- bcm_workq_prod_sync(&dhdp->prot->rx_compl_prod);
2557		- dhd_lb_rx_compl_dispatch(dhdp); /* dispatch rx_compl_tasklet */
2558		- }
2559		-#endif /* DHD_LB_RXC */
2560		-#if defined(DHD_LB_RXP)
2561		- dhd_lb_rx_napi_dispatch(dhdp); /* dispatch rx_process_napi */
2562		-#endif /* DHD_LB_RXP */
2563		- break;
2564		- }
2565		- default:
2566		- break;
2567		- }
	3698	+/**
	3699	+ * dhd_lb_dispatch_tx_compl - load balance by dispatch Tx complition work
	3700	+ * to other CPU cores
	3701	+ */
	3702	+static INLINE void
	3703	+dhd_lb_dispatch_tx_compl(dhd_pub_t *dhdp, uint16 ring_idx)
	3704	+{
	3705	+ bcm_workq_prod_sync(&dhdp->prot->tx_compl_prod); /* flush WR index */
	3706	+ dhd_lb_tx_compl_dispatch(dhdp); /* dispatch tx_compl_tasklet */
2568	3707	}
2569	3708
2570		-
2571		-#if defined(DHD_LB_TXC)
2572	3709	/**
2573	3710	* DHD load balanced tx completion tasklet handler, that will perform the
2574	3711	* freeing of packets on the selected CPU. Packet pointers are delivered to
..	..	@@ -2585,6 +3722,10 @@
2585	3722	dhd_prot_t *prot = dhd->prot;
2586	3723	bcm_workq_t *workq = &prot->tx_compl_cons;
2587	3724	uint32 count = 0;
	3725	+
	3726	+ int curr_cpu;
	3727	+ curr_cpu = get_cpu();
	3728	+ put_cpu();
2588	3729
2589	3730	DHD_LB_STATS_TXC_PERCPU_CNT_INCR(dhd);
2590	3731
..	..	@@ -2606,11 +3747,13 @@
2606	3747	pa = DHD_PKTTAG_PA((dhd_pkttag_fr_t *)PKTTAG(pkt));
2607	3748	pa_len = DHD_PKTTAG_PA_LEN((dhd_pkttag_fr_t *)PKTTAG(pkt));
2608	3749
2609		- DMA_UNMAP(dhd->osh, pa, pa_len, DMA_RX, pkt, 0);
2610		-
	3750	+ DMA_UNMAP(dhd->osh, pa, pa_len, DMA_RX, 0, 0);
2611	3751	#if defined(BCMPCIE)
2612	3752	dhd_txcomplete(dhd, pkt, true);
2613		-#endif
	3753	+#ifdef DHD_4WAYM4_FAIL_DISCONNECT
	3754	+ dhd_eap_txcomplete(dhd, pkt, TRUE, txstatus->cmn_hdr.if_id);
	3755	+#endif /* DHD_4WAYM4_FAIL_DISCONNECT */
	3756	+#endif // endif
2614	3757
2615	3758	PKTFREE(dhd->osh, pkt, TRUE);
2616	3759	count++;
..	..	@@ -2623,6 +3766,23 @@
2623	3766	#endif /* DHD_LB_TXC */
2624	3767
2625	3768	#if defined(DHD_LB_RXC)
	3769	+
	3770	+/**
	3771	+ * dhd_lb_dispatch_rx_compl - load balance by dispatch rx complition work
	3772	+ * to other CPU cores
	3773	+ */
	3774	+static INLINE void
	3775	+dhd_lb_dispatch_rx_compl(dhd_pub_t *dhdp)
	3776	+{
	3777	+ dhd_prot_t *prot = dhdp->prot;
	3778	+ /* Schedule the takslet only if we have to */
	3779	+ if (prot->rxbufpost <= (prot->max_rxbufpost - RXBUFPOST_THRESHOLD)) {
	3780	+ /* flush WR index */
	3781	+ bcm_workq_prod_sync(&dhdp->prot->rx_compl_prod);
	3782	+ dhd_lb_rx_compl_dispatch(dhdp); /* dispatch rx_compl_tasklet */
	3783	+ }
	3784	+}
	3785	+
2626	3786	void
2627	3787	dhd_lb_rx_compl_handler(unsigned long data)
2628	3788	{
..	..	@@ -2635,8 +3795,486 @@
2635	3795	bcm_workq_cons_sync(workq);
2636	3796	}
2637	3797	#endif /* DHD_LB_RXC */
2638		-
2639	3798	#endif /* DHD_LB */
	3799	+
	3800	+void
	3801	+dhd_prot_rx_dataoffset(dhd_pub_t *dhd, uint32 rx_offset)
	3802	+{
	3803	+ dhd_prot_t *prot = dhd->prot;
	3804	+ prot->rx_dataoffset = rx_offset;
	3805	+}
	3806	+
	3807	+static int
	3808	+dhd_check_create_info_rings(dhd_pub_t *dhd)
	3809	+{
	3810	+ dhd_prot_t *prot = dhd->prot;
	3811	+ int ret = BCME_ERROR;
	3812	+ uint16 ringid;
	3813	+
	3814	+ {
	3815	+ /* dongle may increase max_submission_rings so keep
	3816	+ * ringid at end of dynamic rings
	3817	+ */
	3818	+ ringid = dhd->bus->max_tx_flowrings +
	3819	+ (dhd->bus->max_submission_rings - dhd->bus->max_tx_flowrings) +
	3820	+ BCMPCIE_H2D_COMMON_MSGRINGS;
	3821	+ }
	3822	+
	3823	+ if (prot->d2hring_info_cpln) {
	3824	+ /* for d2hring re-entry case, clear inited flag */
	3825	+ prot->d2hring_info_cpln->inited = FALSE;
	3826	+ }
	3827	+
	3828	+ if (prot->h2dring_info_subn && prot->d2hring_info_cpln) {
	3829	+ return BCME_OK; /* dhd_prot_init rentry after a dhd_prot_reset */
	3830	+ }
	3831	+
	3832	+ if (prot->h2dring_info_subn == NULL) {
	3833	+ prot->h2dring_info_subn = MALLOCZ(prot->osh, sizeof(msgbuf_ring_t));
	3834	+
	3835	+ if (prot->h2dring_info_subn == NULL) {
	3836	+ DHD_ERROR(("%s: couldn't alloc memory for h2dring_info_subn\n",
	3837	+ __FUNCTION__));
	3838	+ return BCME_NOMEM;
	3839	+ }
	3840	+
	3841	+ DHD_INFO(("%s: about to create debug submit ring\n", __FUNCTION__));
	3842	+ ret = dhd_prot_ring_attach(dhd, prot->h2dring_info_subn, "h2dinfo",
	3843	+ H2DRING_DYNAMIC_INFO_MAX_ITEM, H2DRING_INFO_BUFPOST_ITEMSIZE,
	3844	+ ringid);
	3845	+ if (ret != BCME_OK) {
	3846	+ DHD_ERROR(("%s: couldn't alloc resources for dbg submit ring\n",
	3847	+ __FUNCTION__));
	3848	+ goto err;
	3849	+ }
	3850	+ }
	3851	+
	3852	+ if (prot->d2hring_info_cpln == NULL) {
	3853	+ prot->d2hring_info_cpln = MALLOCZ(prot->osh, sizeof(msgbuf_ring_t));
	3854	+
	3855	+ if (prot->d2hring_info_cpln == NULL) {
	3856	+ DHD_ERROR(("%s: couldn't alloc memory for h2dring_info_subn\n",
	3857	+ __FUNCTION__));
	3858	+ return BCME_NOMEM;
	3859	+ }
	3860	+
	3861	+ /* create the debug info completion ring next to debug info submit ring
	3862	+ * ringid = id next to debug info submit ring
	3863	+ */
	3864	+ ringid = ringid + 1;
	3865	+
	3866	+ DHD_INFO(("%s: about to create debug cpl ring\n", __FUNCTION__));
	3867	+ ret = dhd_prot_ring_attach(dhd, prot->d2hring_info_cpln, "d2hinfo",
	3868	+ D2HRING_DYNAMIC_INFO_MAX_ITEM, D2HRING_INFO_BUFCMPLT_ITEMSIZE,
	3869	+ ringid);
	3870	+ if (ret != BCME_OK) {
	3871	+ DHD_ERROR(("%s: couldn't alloc resources for dbg cpl ring\n",
	3872	+ __FUNCTION__));
	3873	+ dhd_prot_ring_detach(dhd, prot->h2dring_info_subn);
	3874	+ goto err;
	3875	+ }
	3876	+ }
	3877	+
	3878	+ return ret;
	3879	+err:
	3880	+ MFREE(prot->osh, prot->h2dring_info_subn, sizeof(msgbuf_ring_t));
	3881	+ prot->h2dring_info_subn = NULL;
	3882	+
	3883	+ if (prot->d2hring_info_cpln) {
	3884	+ MFREE(prot->osh, prot->d2hring_info_cpln, sizeof(msgbuf_ring_t));
	3885	+ prot->d2hring_info_cpln = NULL;
	3886	+ }
	3887	+ return ret;
	3888	+} /* dhd_check_create_info_rings */
	3889	+
	3890	+int
	3891	+dhd_prot_init_info_rings(dhd_pub_t *dhd)
	3892	+{
	3893	+ dhd_prot_t *prot = dhd->prot;
	3894	+ int ret = BCME_OK;
	3895	+
	3896	+ if ((ret = dhd_check_create_info_rings(dhd)) != BCME_OK) {
	3897	+ DHD_ERROR(("%s: info rings aren't created! \n",
	3898	+ __FUNCTION__));
	3899	+ return ret;
	3900	+ }
	3901	+
	3902	+ if ((prot->d2hring_info_cpln->inited) \|\| (prot->d2hring_info_cpln->create_pending)) {
	3903	+ DHD_INFO(("Info completion ring was created!\n"));
	3904	+ return ret;
	3905	+ }
	3906	+
	3907	+ DHD_TRACE(("trying to send create d2h info ring: id %d\n", prot->d2hring_info_cpln->idx));
	3908	+ ret = dhd_send_d2h_ringcreate(dhd, prot->d2hring_info_cpln,
	3909	+ BCMPCIE_D2H_RING_TYPE_DBGBUF_CPL, DHD_D2H_DBGRING_REQ_PKTID);
	3910	+ if (ret != BCME_OK)
	3911	+ return ret;
	3912	+
	3913	+ prot->h2dring_info_subn->seqnum = H2D_EPOCH_INIT_VAL;
	3914	+ prot->h2dring_info_subn->current_phase = 0;
	3915	+ prot->d2hring_info_cpln->seqnum = D2H_EPOCH_INIT_VAL;
	3916	+ prot->d2hring_info_cpln->current_phase = BCMPCIE_CMNHDR_PHASE_BIT_INIT;
	3917	+
	3918	+ DHD_TRACE(("trying to send create h2d info ring id %d\n", prot->h2dring_info_subn->idx));
	3919	+ prot->h2dring_info_subn->n_completion_ids = 1;
	3920	+ prot->h2dring_info_subn->compeltion_ring_ids[0] = prot->d2hring_info_cpln->idx;
	3921	+
	3922	+ ret = dhd_send_h2d_ringcreate(dhd, prot->h2dring_info_subn,
	3923	+ BCMPCIE_H2D_RING_TYPE_DBGBUF_SUBMIT, DHD_H2D_DBGRING_REQ_PKTID);
	3924	+
	3925	+ /* Note that there is no way to delete d2h or h2d ring deletion incase either fails,
	3926	+ * so can not cleanup if one ring was created while the other failed
	3927	+ */
	3928	+ return ret;
	3929	+} /* dhd_prot_init_info_rings */
	3930	+
	3931	+static void
	3932	+dhd_prot_detach_info_rings(dhd_pub_t *dhd)
	3933	+{
	3934	+ if (dhd->prot->h2dring_info_subn) {
	3935	+ dhd_prot_ring_detach(dhd, dhd->prot->h2dring_info_subn);
	3936	+ MFREE(dhd->prot->osh, dhd->prot->h2dring_info_subn, sizeof(msgbuf_ring_t));
	3937	+ dhd->prot->h2dring_info_subn = NULL;
	3938	+ }
	3939	+ if (dhd->prot->d2hring_info_cpln) {
	3940	+ dhd_prot_ring_detach(dhd, dhd->prot->d2hring_info_cpln);
	3941	+ MFREE(dhd->prot->osh, dhd->prot->d2hring_info_cpln, sizeof(msgbuf_ring_t));
	3942	+ dhd->prot->d2hring_info_cpln = NULL;
	3943	+ }
	3944	+}
	3945	+
	3946	+#ifdef DHD_HP2P
	3947	+static int
	3948	+dhd_check_create_hp2p_rings(dhd_pub_t *dhd)
	3949	+{
	3950	+ dhd_prot_t *prot = dhd->prot;
	3951	+ int ret = BCME_ERROR;
	3952	+ uint16 ringid;
	3953	+
	3954	+ /* Last 2 dynamic ring indices are used by hp2p rings */
	3955	+ ringid = dhd->bus->max_submission_rings + dhd->bus->max_completion_rings - 2;
	3956	+
	3957	+ if (prot->d2hring_hp2p_txcpl == NULL) {
	3958	+ prot->d2hring_hp2p_txcpl = MALLOCZ(prot->osh, sizeof(msgbuf_ring_t));
	3959	+
	3960	+ if (prot->d2hring_hp2p_txcpl == NULL) {
	3961	+ DHD_ERROR(("%s: couldn't alloc memory for d2hring_hp2p_txcpl\n",
	3962	+ __FUNCTION__));
	3963	+ return BCME_NOMEM;
	3964	+ }
	3965	+
	3966	+ DHD_INFO(("%s: about to create hp2p txcpl ring\n", __FUNCTION__));
	3967	+ ret = dhd_prot_ring_attach(dhd, prot->d2hring_hp2p_txcpl, "d2hhp2p_txcpl",
	3968	+ dhd_bus_get_hp2p_ring_max_size(dhd->bus, TRUE), D2HRING_TXCMPLT_ITEMSIZE,
	3969	+ ringid);
	3970	+ if (ret != BCME_OK) {
	3971	+ DHD_ERROR(("%s: couldn't alloc resources for hp2p txcpl ring\n",
	3972	+ __FUNCTION__));
	3973	+ goto err2;
	3974	+ }
	3975	+ } else {
	3976	+ /* for re-entry case, clear inited flag */
	3977	+ prot->d2hring_hp2p_txcpl->inited = FALSE;
	3978	+ }
	3979	+ if (prot->d2hring_hp2p_rxcpl == NULL) {
	3980	+ prot->d2hring_hp2p_rxcpl = MALLOCZ(prot->osh, sizeof(msgbuf_ring_t));
	3981	+
	3982	+ if (prot->d2hring_hp2p_rxcpl == NULL) {
	3983	+ DHD_ERROR(("%s: couldn't alloc memory for d2hring_hp2p_rxcpl\n",
	3984	+ __FUNCTION__));
	3985	+ return BCME_NOMEM;
	3986	+ }
	3987	+
	3988	+ /* create the hp2p rx completion ring next to hp2p tx compl ring
	3989	+ * ringid = id next to hp2p tx compl ring
	3990	+ */
	3991	+ ringid = ringid + 1;
	3992	+
	3993	+ DHD_INFO(("%s: about to create hp2p rxcpl ring\n", __FUNCTION__));
	3994	+ ret = dhd_prot_ring_attach(dhd, prot->d2hring_hp2p_rxcpl, "d2hhp2p_rxcpl",
	3995	+ dhd_bus_get_hp2p_ring_max_size(dhd->bus, FALSE), D2HRING_RXCMPLT_ITEMSIZE,
	3996	+ ringid);
	3997	+ if (ret != BCME_OK) {
	3998	+ DHD_ERROR(("%s: couldn't alloc resources for hp2p rxcpl ring\n",
	3999	+ __FUNCTION__));
	4000	+ goto err1;
	4001	+ }
	4002	+ } else {
	4003	+ /* for re-entry case, clear inited flag */
	4004	+ prot->d2hring_hp2p_rxcpl->inited = FALSE;
	4005	+ }
	4006	+
	4007	+ return ret;
	4008	+err1:
	4009	+ MFREE(prot->osh, prot->d2hring_hp2p_rxcpl, sizeof(msgbuf_ring_t));
	4010	+ prot->d2hring_hp2p_rxcpl = NULL;
	4011	+
	4012	+err2:
	4013	+ MFREE(prot->osh, prot->d2hring_hp2p_txcpl, sizeof(msgbuf_ring_t));
	4014	+ prot->d2hring_hp2p_txcpl = NULL;
	4015	+ return ret;
	4016	+} /* dhd_check_create_hp2p_rings */
	4017	+
	4018	+int
	4019	+dhd_prot_init_hp2p_rings(dhd_pub_t *dhd)
	4020	+{
	4021	+ dhd_prot_t *prot = dhd->prot;
	4022	+ int ret = BCME_OK;
	4023	+
	4024	+ dhd->hp2p_ring_active = FALSE;
	4025	+
	4026	+ if ((ret = dhd_check_create_hp2p_rings(dhd)) != BCME_OK) {
	4027	+ DHD_ERROR(("%s: hp2p rings aren't created! \n",
	4028	+ __FUNCTION__));
	4029	+ return ret;
	4030	+ }
	4031	+
	4032	+ if ((prot->d2hring_hp2p_txcpl->inited) \|\| (prot->d2hring_hp2p_txcpl->create_pending)) {
	4033	+ DHD_INFO(("hp2p tx completion ring was created!\n"));
	4034	+ return ret;
	4035	+ }
	4036	+
	4037	+ DHD_TRACE(("trying to send create d2h hp2p txcpl ring: id %d\n",
	4038	+ prot->d2hring_hp2p_txcpl->idx));
	4039	+ ret = dhd_send_d2h_ringcreate(dhd, prot->d2hring_hp2p_txcpl,
	4040	+ BCMPCIE_D2H_RING_TYPE_HPP_TX_CPL, DHD_D2H_HPPRING_TXREQ_PKTID);
	4041	+ if (ret != BCME_OK)
	4042	+ return ret;
	4043	+
	4044	+ prot->d2hring_hp2p_txcpl->seqnum = D2H_EPOCH_INIT_VAL;
	4045	+ prot->d2hring_hp2p_txcpl->current_phase = BCMPCIE_CMNHDR_PHASE_BIT_INIT;
	4046	+
	4047	+ if ((prot->d2hring_hp2p_rxcpl->inited) \|\| (prot->d2hring_hp2p_rxcpl->create_pending)) {
	4048	+ DHD_INFO(("hp2p rx completion ring was created!\n"));
	4049	+ return ret;
	4050	+ }
	4051	+
	4052	+ DHD_TRACE(("trying to send create d2h hp2p rxcpl ring: id %d\n",
	4053	+ prot->d2hring_hp2p_rxcpl->idx));
	4054	+ ret = dhd_send_d2h_ringcreate(dhd, prot->d2hring_hp2p_rxcpl,
	4055	+ BCMPCIE_D2H_RING_TYPE_HPP_RX_CPL, DHD_D2H_HPPRING_RXREQ_PKTID);
	4056	+ if (ret != BCME_OK)
	4057	+ return ret;
	4058	+
	4059	+ prot->d2hring_hp2p_rxcpl->seqnum = D2H_EPOCH_INIT_VAL;
	4060	+ prot->d2hring_hp2p_rxcpl->current_phase = BCMPCIE_CMNHDR_PHASE_BIT_INIT;
	4061	+
	4062	+ /* Note that there is no way to delete d2h or h2d ring deletion incase either fails,
	4063	+ * so can not cleanup if one ring was created while the other failed
	4064	+ */
	4065	+ return BCME_OK;
	4066	+} /* dhd_prot_init_hp2p_rings */
	4067	+
	4068	+static void
	4069	+dhd_prot_detach_hp2p_rings(dhd_pub_t *dhd)
	4070	+{
	4071	+ if (dhd->prot->d2hring_hp2p_txcpl) {
	4072	+ dhd_prot_ring_detach(dhd, dhd->prot->d2hring_hp2p_txcpl);
	4073	+ MFREE(dhd->prot->osh, dhd->prot->d2hring_hp2p_txcpl, sizeof(msgbuf_ring_t));
	4074	+ dhd->prot->d2hring_hp2p_txcpl = NULL;
	4075	+ }
	4076	+ if (dhd->prot->d2hring_hp2p_rxcpl) {
	4077	+ dhd_prot_ring_detach(dhd, dhd->prot->d2hring_hp2p_rxcpl);
	4078	+ MFREE(dhd->prot->osh, dhd->prot->d2hring_hp2p_rxcpl, sizeof(msgbuf_ring_t));
	4079	+ dhd->prot->d2hring_hp2p_rxcpl = NULL;
	4080	+ }
	4081	+}
	4082	+#endif /* DHD_HP2P */
	4083	+
	4084	+#ifdef EWP_EDL
	4085	+static int
	4086	+dhd_check_create_edl_rings(dhd_pub_t *dhd)
	4087	+{
	4088	+ dhd_prot_t *prot = dhd->prot;
	4089	+ int ret = BCME_ERROR;
	4090	+ uint16 ringid;
	4091	+
	4092	+ {
	4093	+ /* dongle may increase max_submission_rings so keep
	4094	+ * ringid at end of dynamic rings (re-use info ring cpl ring id)
	4095	+ */
	4096	+ ringid = dhd->bus->max_tx_flowrings +
	4097	+ (dhd->bus->max_submission_rings - dhd->bus->max_tx_flowrings) +
	4098	+ BCMPCIE_H2D_COMMON_MSGRINGS + 1;
	4099	+ }
	4100	+
	4101	+ if (prot->d2hring_edl) {
	4102	+ prot->d2hring_edl->inited = FALSE;
	4103	+ return BCME_OK; /* dhd_prot_init rentry after a dhd_prot_reset */
	4104	+ }
	4105	+
	4106	+ if (prot->d2hring_edl == NULL) {
	4107	+ prot->d2hring_edl = MALLOCZ(prot->osh, sizeof(msgbuf_ring_t));
	4108	+
	4109	+ if (prot->d2hring_edl == NULL) {
	4110	+ DHD_ERROR(("%s: couldn't alloc memory for d2hring_edl\n",
	4111	+ __FUNCTION__));
	4112	+ return BCME_NOMEM;
	4113	+ }
	4114	+
	4115	+ DHD_ERROR(("%s: about to create EDL ring, ringid: %u \n", __FUNCTION__,
	4116	+ ringid));
	4117	+ ret = dhd_prot_ring_attach(dhd, prot->d2hring_edl, "d2hring_edl",
	4118	+ D2HRING_EDL_MAX_ITEM, D2HRING_EDL_ITEMSIZE,
	4119	+ ringid);
	4120	+ if (ret != BCME_OK) {
	4121	+ DHD_ERROR(("%s: couldn't alloc resources for EDL ring\n",
	4122	+ __FUNCTION__));
	4123	+ goto err;
	4124	+ }
	4125	+ }
	4126	+
	4127	+ return ret;
	4128	+err:
	4129	+ MFREE(prot->osh, prot->d2hring_edl, sizeof(msgbuf_ring_t));
	4130	+ prot->d2hring_edl = NULL;
	4131	+
	4132	+ return ret;
	4133	+} /* dhd_check_create_btlog_rings */
	4134	+
	4135	+int
	4136	+dhd_prot_init_edl_rings(dhd_pub_t *dhd)
	4137	+{
	4138	+ dhd_prot_t *prot = dhd->prot;
	4139	+ int ret = BCME_ERROR;
	4140	+
	4141	+ if ((ret = dhd_check_create_edl_rings(dhd)) != BCME_OK) {
	4142	+ DHD_ERROR(("%s: EDL rings aren't created! \n",
	4143	+ __FUNCTION__));
	4144	+ return ret;
	4145	+ }
	4146	+
	4147	+ if ((prot->d2hring_edl->inited) \|\| (prot->d2hring_edl->create_pending)) {
	4148	+ DHD_INFO(("EDL completion ring was created!\n"));
	4149	+ return ret;
	4150	+ }
	4151	+
	4152	+ DHD_ERROR(("trying to send create d2h edl ring: idx %d\n", prot->d2hring_edl->idx));
	4153	+ ret = dhd_send_d2h_ringcreate(dhd, prot->d2hring_edl,
	4154	+ BCMPCIE_D2H_RING_TYPE_EDL, DHD_D2H_DBGRING_REQ_PKTID);
	4155	+ if (ret != BCME_OK)
	4156	+ return ret;
	4157	+
	4158	+ prot->d2hring_edl->seqnum = D2H_EPOCH_INIT_VAL;
	4159	+ prot->d2hring_edl->current_phase = BCMPCIE_CMNHDR_PHASE_BIT_INIT;
	4160	+
	4161	+ return BCME_OK;
	4162	+} /* dhd_prot_init_btlog_rings */
	4163	+
	4164	+static void
	4165	+dhd_prot_detach_edl_rings(dhd_pub_t *dhd)
	4166	+{
	4167	+ if (dhd->prot->d2hring_edl) {
	4168	+ dhd_prot_ring_detach(dhd, dhd->prot->d2hring_edl);
	4169	+ MFREE(dhd->prot->osh, dhd->prot->d2hring_edl, sizeof(msgbuf_ring_t));
	4170	+ dhd->prot->d2hring_edl = NULL;
	4171	+ }
	4172	+}
	4173	+#endif /* EWP_EDL */
	4174	+
	4175	+/**
	4176	+ * Initialize protocol: sync w/dongle state.
	4177	+ * Sets dongle media info (iswl, drv_version, mac address).
	4178	+ */
	4179	+int dhd_sync_with_dongle(dhd_pub_t *dhd)
	4180	+{
	4181	+ int ret = 0;
	4182	+ wlc_rev_info_t revinfo;
	4183	+ char buf[128];
	4184	+ dhd_prot_t *prot = dhd->prot;
	4185	+
	4186	+ DHD_TRACE(("%s: Enter\n", __FUNCTION__));
	4187	+
	4188	+ dhd_os_set_ioctl_resp_timeout(IOCTL_RESP_TIMEOUT);
	4189	+
	4190	+ /* Post ts buffer after shim layer is attached */
	4191	+ ret = dhd_msgbuf_rxbuf_post_ts_bufs(dhd);
	4192	+
	4193	+#ifndef OEM_ANDROID
	4194	+ /* Get the device MAC address */
	4195	+ memset(buf, 0, sizeof(buf));
	4196	+ strncpy(buf, "cur_etheraddr", sizeof(buf) - 1);
	4197	+ ret = dhd_wl_ioctl_cmd(dhd, WLC_GET_VAR, buf, sizeof(buf), FALSE, 0);
	4198	+ if (ret < 0) {
	4199	+ DHD_ERROR(("%s: GET iovar cur_etheraddr FAILED\n", __FUNCTION__));
	4200	+ goto done;
	4201	+ }
	4202	+ memcpy(dhd->mac.octet, buf, ETHER_ADDR_LEN);
	4203	+ if (dhd_msg_level & DHD_INFO_VAL) {
	4204	+ bcm_print_bytes("CUR_ETHERADDR ", (uchar *)buf, ETHER_ADDR_LEN);
	4205	+ }
	4206	+#endif /* OEM_ANDROID */
	4207	+
	4208	+#ifdef DHD_FW_COREDUMP
	4209	+ /* Check the memdump capability */
	4210	+ dhd_get_memdump_info(dhd);
	4211	+#endif /* DHD_FW_COREDUMP */
	4212	+#ifdef BCMASSERT_LOG
	4213	+ dhd_get_assert_info(dhd);
	4214	+#endif /* BCMASSERT_LOG */
	4215	+
	4216	+ /* Get the device rev info */
	4217	+ memset(&revinfo, 0, sizeof(revinfo));
	4218	+ ret = dhd_wl_ioctl_cmd(dhd, WLC_GET_REVINFO, &revinfo, sizeof(revinfo), FALSE, 0);
	4219	+ if (ret < 0) {
	4220	+ DHD_ERROR(("%s: GET revinfo FAILED\n", __FUNCTION__));
	4221	+ goto done;
	4222	+ }
	4223	+ DHD_ERROR(("%s: GET_REVINFO device 0x%x, vendor 0x%x, chipnum 0x%x\n", __FUNCTION__,
	4224	+ revinfo.deviceid, revinfo.vendorid, revinfo.chipnum));
	4225	+
	4226	+ /* Get the RxBuf post size */
	4227	+ memset(buf, 0, sizeof(buf));
	4228	+ bcm_mkiovar("rxbufpost_sz", NULL, 0, buf, sizeof(buf));
	4229	+ ret = dhd_wl_ioctl_cmd(dhd, WLC_GET_VAR, buf, sizeof(buf), FALSE, 0);
	4230	+ if (ret < 0) {
	4231	+ DHD_ERROR(("%s: GET RxBuf post FAILED, default to %d\n",
	4232	+ __FUNCTION__, DHD_FLOWRING_RX_BUFPOST_PKTSZ));
	4233	+ prot->rxbufpost_sz = DHD_FLOWRING_RX_BUFPOST_PKTSZ;
	4234	+ } else {
	4235	+ memcpy_s(&(prot->rxbufpost_sz), sizeof(prot->rxbufpost_sz), buf, sizeof(uint16));
	4236	+ if (prot->rxbufpost_sz > DHD_FLOWRING_RX_BUFPOST_PKTSZ_MAX) {
	4237	+ DHD_ERROR(("%s: Invalid RxBuf post size : %d, default to %d\n",
	4238	+ __FUNCTION__, prot->rxbufpost_sz, DHD_FLOWRING_RX_BUFPOST_PKTSZ));
	4239	+ prot->rxbufpost_sz = DHD_FLOWRING_RX_BUFPOST_PKTSZ;
	4240	+ } else {
	4241	+ DHD_ERROR(("%s: RxBuf Post : %d\n", __FUNCTION__, prot->rxbufpost_sz));
	4242	+ }
	4243	+ }
	4244	+
	4245	+ /* Post buffers for packet reception */
	4246	+ dhd_msgbuf_rxbuf_post(dhd, FALSE); /* alloc pkt ids */
	4247	+
	4248	+ DHD_SSSR_DUMP_INIT(dhd);
	4249	+
	4250	+ dhd_process_cid_mac(dhd, TRUE);
	4251	+ ret = dhd_preinit_ioctls(dhd);
	4252	+ dhd_process_cid_mac(dhd, FALSE);
	4253	+
	4254	+#if defined(DHD_H2D_LOG_TIME_SYNC)
	4255	+#ifdef DHD_HP2P
	4256	+ if (FW_SUPPORTED(dhd, h2dlogts) \|\| dhd->hp2p_capable) {
	4257	+ if (dhd->hp2p_enable) {
	4258	+ dhd->dhd_rte_time_sync_ms = DHD_H2D_LOG_TIME_STAMP_MATCH / 40;
	4259	+ } else {
	4260	+ dhd->dhd_rte_time_sync_ms = DHD_H2D_LOG_TIME_STAMP_MATCH;
	4261	+ }
	4262	+#else
	4263	+ if (FW_SUPPORTED(dhd, h2dlogts)) {
	4264	+ dhd->dhd_rte_time_sync_ms = DHD_H2D_LOG_TIME_STAMP_MATCH;
	4265	+#endif // endif
	4266	+ dhd->bus->dhd_rte_time_sync_count = OSL_SYSUPTIME_US();
	4267	+ /* This is during initialization. */
	4268	+ dhd_h2d_log_time_sync(dhd);
	4269	+ } else {
	4270	+ dhd->dhd_rte_time_sync_ms = 0;
	4271	+ }
	4272	+#endif /* DHD_H2D_LOG_TIME_SYNC \|\| DHD_HP2P */
	4273	+ /* Always assumes wl for now */
	4274	+ dhd->iswl = TRUE;
	4275	+done:
	4276	+ return ret;
	4277	+} /* dhd_sync_with_dongle */
2640	4278
2641	4279	#define DHD_DBG_SHOW_METADATA 0
2642	4280
..	..	@@ -2736,7 +4374,9 @@
2736	4374	{
2737	4375	if (pkt) {
2738	4376	if (pkttype == PKTTYPE_IOCTL_RX \|\|
2739		- pkttype == PKTTYPE_EVENT_RX) {
	4377	+ pkttype == PKTTYPE_EVENT_RX \|\|
	4378	+ pkttype == PKTTYPE_INFO_RX \|\|
	4379	+ pkttype == PKTTYPE_TSBUF_RX) {
2740	4380	#ifdef DHD_USE_STATIC_CTRLBUF
2741	4381	PKTFREE_STATIC(dhd->osh, pkt, send);
2742	4382	#else
..	..	@@ -2748,6 +4388,11 @@
2748	4388	}
2749	4389	}
2750	4390
	4391	+/**
	4392	+ * dhd_prot_packet_get should be called only for items having pktid_ctrl_map handle
	4393	+ * and all the bottom most functions like dhd_pktid_map_free hold separate DHD_PKTID_LOCK
	4394	+ * to ensure thread safety, so no need to hold any locks for this function
	4395	+ */
2751	4396	static INLINE void * BCMFASTPATH
2752	4397	dhd_prot_packet_get(dhd_pub_t *dhd, uint32 pktid, uint8 pkttype, bool free_pktid)
2753	4398	{
..	..	@@ -2759,25 +4404,45 @@
2759	4404
2760	4405	#ifdef DHD_PCIE_PKTID
2761	4406	if (free_pktid) {
2762		- PKTBUF = DHD_PKTID_TO_NATIVE(dhd, dhd->prot->pktid_map_handle,
	4407	+ PKTBUF = DHD_PKTID_TO_NATIVE(dhd, dhd->prot->pktid_ctrl_map,
2763	4408	pktid, pa, len, dmah, secdma, pkttype);
2764	4409	} else {
2765		- PKTBUF = DHD_PKTID_TO_NATIVE_RSV(dhd, dhd->prot->pktid_map_handle,
	4410	+ PKTBUF = DHD_PKTID_TO_NATIVE_RSV(dhd, dhd->prot->pktid_ctrl_map,
2766	4411	pktid, pa, len, dmah, secdma, pkttype);
2767	4412	}
2768	4413	#else
2769		- PKTBUF = DHD_PKTID_TO_NATIVE(dhd, dhd->prot->pktid_map_handle, pktid, pa,
	4414	+ PKTBUF = DHD_PKTID_TO_NATIVE(dhd, dhd->prot->pktid_ctrl_map, pktid, pa,
2770	4415	len, dmah, secdma, pkttype);
2771	4416	#endif /* DHD_PCIE_PKTID */
2772		-
2773	4417	if (PKTBUF) {
2774	4418	{
2775		- if (SECURE_DMA_ENAB(dhd->osh)) {
	4419	+ if (SECURE_DMA_ENAB(dhd->osh))
2776	4420	SECURE_DMA_UNMAP(dhd->osh, pa, (uint) len, DMA_RX, 0, dmah,
2777	4421	secdma, 0);
2778		- } else {
2779		- DMA_UNMAP(dhd->osh, pa, (uint) len, DMA_RX, PKTBUF, dmah);
	4422	+ else
	4423	+ DMA_UNMAP(dhd->osh, pa, (uint) len, DMA_RX, 0, dmah);
	4424	+#ifdef DMAMAP_STATS
	4425	+ switch (pkttype) {
	4426	+#ifndef IOCTLRESP_USE_CONSTMEM
	4427	+ case PKTTYPE_IOCTL_RX:
	4428	+ dhd->dma_stats.ioctl_rx--;
	4429	+ dhd->dma_stats.ioctl_rx_sz -= len;
	4430	+ break;
	4431	+#endif /* IOCTLRESP_USE_CONSTMEM */
	4432	+ case PKTTYPE_EVENT_RX:
	4433	+ dhd->dma_stats.event_rx--;
	4434	+ dhd->dma_stats.event_rx_sz -= len;
	4435	+ break;
	4436	+ case PKTTYPE_INFO_RX:
	4437	+ dhd->dma_stats.info_rx--;
	4438	+ dhd->dma_stats.info_rx_sz -= len;
	4439	+ break;
	4440	+ case PKTTYPE_TSBUF_RX:
	4441	+ dhd->dma_stats.tsbuf_rx--;
	4442	+ dhd->dma_stats.tsbuf_rx_sz -= len;
	4443	+ break;
2780	4444	}
	4445	+#endif /* DMAMAP_STATS */
2781	4446	}
2782	4447	}
2783	4448
..	..	@@ -2794,7 +4459,7 @@
2794	4459
2795	4460	return;
2796	4461	}
2797		-#endif /* IOCTLRESP_USE_CONSTMEM */
	4462	+#endif // endif
2798	4463
2799	4464	static void BCMFASTPATH
2800	4465	dhd_msgbuf_rxbuf_post(dhd_pub_t *dhd, bool use_rsv_pktid)
..	..	@@ -2839,91 +4504,98 @@
2839	4504	static int BCMFASTPATH
2840	4505	dhd_prot_rxbuf_post(dhd_pub_t *dhd, uint16 count, bool use_rsv_pktid)
2841	4506	{
2842		- void *p;
2843		- uint16 pktsz = DHD_FLOWRING_RX_BUFPOST_PKTSZ;
	4507	+ void p, *pktbuf;
2844	4508	uint8 *rxbuf_post_tmp;
2845	4509	host_rxbuf_post_t *rxbuf_post;
2846	4510	void *msg_start;
2847		- dmaaddr_t pa;
2848		- uint32 pktlen;
2849		- uint8 i = 0;
2850		- uint16 alloced = 0;
	4511	+ dmaaddr_t pa, *pktbuf_pa;
	4512	+ uint32 *pktlen;
	4513	+ uint16 i = 0, alloced = 0;
2851	4514	unsigned long flags;
2852	4515	uint32 pktid;
2853	4516	dhd_prot_t *prot = dhd->prot;
2854	4517	msgbuf_ring_t *ring = &prot->h2dring_rxp_subn;
	4518	+ void *lcl_buf;
	4519	+ uint16 lcl_buf_size;
	4520	+ uint16 pktsz = prot->rxbufpost_sz;
2855	4521
2856		- DHD_GENERAL_LOCK(dhd, flags);
2857		-
2858		- /* Claim space for exactly 'count' no of messages, for mitigation purpose */
2859		- msg_start = (void *)
2860		- dhd_prot_alloc_ring_space(dhd, ring, count, &alloced, TRUE);
2861		-
2862		- DHD_GENERAL_UNLOCK(dhd, flags);
2863		-
2864		- if (msg_start == NULL) {
2865		- DHD_INFO(("%s:%d: Rxbufpost Msgbuf Not available\n", __FUNCTION__, __LINE__));
2866		- return -1;
	4522	+ /* allocate a local buffer to store pkt buffer va, pa and length */
	4523	+ lcl_buf_size = (sizeof(void ) + sizeof(dmaaddr_t) + sizeof(uint32))
	4524	+ RX_BUF_BURST;
	4525	+ lcl_buf = MALLOC(dhd->osh, lcl_buf_size);
	4526	+ if (!lcl_buf) {
	4527	+ DHD_ERROR(("%s: local scratch buffer allocation failed\n", __FUNCTION__));
	4528	+ return 0;
2867	4529	}
2868		- /* if msg_start != NULL, we should have alloced space for atleast 1 item */
2869		- ASSERT(alloced > 0);
	4530	+ pktbuf = lcl_buf;
	4531	+ pktbuf_pa = (dmaaddr_t )((uint8 )pktbuf + sizeof(void ) RX_BUF_BURST);
	4532	+ pktlen = (uint32 )((uint8 )pktbuf_pa + sizeof(dmaaddr_t) * RX_BUF_BURST);
2870	4533
2871		- rxbuf_post_tmp = (uint8*)msg_start;
2872		-
2873		- /* loop through each allocated message in the rxbuf post msgbuf_ring */
2874		- for (i = 0; i < alloced; i++) {
2875		- rxbuf_post = (host_rxbuf_post_t *)rxbuf_post_tmp;
2876		- /* Create a rx buffer */
	4534	+ for (i = 0; i < count; i++) {
2877	4535	if ((p = PKTGET(dhd->osh, pktsz, FALSE)) == NULL) {
2878	4536	DHD_ERROR(("%s:%d: PKTGET for rxbuf failed\n", __FUNCTION__, __LINE__));
2879	4537	dhd->rx_pktgetfail++;
2880	4538	break;
2881	4539	}
2882	4540
2883		- pktlen = PKTLEN(dhd->osh, p);
	4541	+ pktlen[i] = PKTLEN(dhd->osh, p);
2884	4542	if (SECURE_DMA_ENAB(dhd->osh)) {
2885		- DHD_GENERAL_LOCK(dhd, flags);
2886		- pa = SECURE_DMA_MAP(dhd->osh, PKTDATA(dhd->osh, p), pktlen,
	4543	+ pa = SECURE_DMA_MAP(dhd->osh, PKTDATA(dhd->osh, p), pktlen[i],
2887	4544	DMA_RX, p, 0, ring->dma_buf.secdma, 0);
2888		- DHD_GENERAL_UNLOCK(dhd, flags);
2889	4545	}
2890	4546	#ifndef BCM_SECURE_DMA
2891		- else {
2892		- pa = DMA_MAP(dhd->osh, PKTDATA(dhd->osh, p), pktlen, DMA_RX, p, 0);
2893		- }
2894		-#endif /* BCM_SECURE_DMA */
	4547	+ else
	4548	+ pa = DMA_MAP(dhd->osh, PKTDATA(dhd->osh, p), pktlen[i], DMA_RX, p, 0);
	4549	+#endif /* #ifndef BCM_SECURE_DMA */
2895	4550
2896	4551	if (PHYSADDRISZERO(pa)) {
2897		- if (SECURE_DMA_ENAB(dhd->osh)) {
2898		- DHD_GENERAL_LOCK(dhd, flags);
2899		- SECURE_DMA_UNMAP(dhd->osh, pa, pktlen, DMA_RX, 0, DHD_DMAH_NULL,
2900		- ring->dma_buf.secdma, 0);
2901		- DHD_GENERAL_UNLOCK(dhd, flags);
2902		- } else {
2903		- DMA_UNMAP(dhd->osh, pa, pktlen, DMA_RX, p, DHD_DMAH_NULL);
2904		- }
2905		-
2906	4552	PKTFREE(dhd->osh, p, FALSE);
2907	4553	DHD_ERROR(("Invalid phyaddr 0\n"));
2908	4554	ASSERT(0);
2909	4555	break;
2910	4556	}
	4557	+#ifdef DMAMAP_STATS
	4558	+ dhd->dma_stats.rxdata++;
	4559	+ dhd->dma_stats.rxdata_sz += pktlen[i];
	4560	+#endif /* DMAMAP_STATS */
2911	4561
2912	4562	PKTPULL(dhd->osh, p, prot->rx_metadata_offset);
2913		- pktlen = PKTLEN(dhd->osh, p);
	4563	+ pktlen[i] = PKTLEN(dhd->osh, p);
	4564	+ pktbuf[i] = p;
	4565	+ pktbuf_pa[i] = pa;
	4566	+ }
2914	4567
2915		- /* Common msg header */
2916		- rxbuf_post->cmn_hdr.msg_type = MSG_TYPE_RXBUF_POST;
2917		- rxbuf_post->cmn_hdr.if_id = 0;
2918		- rxbuf_post->cmn_hdr.epoch = ring->seqnum % H2D_EPOCH_MODULO;
2919		- ring->seqnum++;
	4568	+ /* only post what we have */
	4569	+ count = i;
	4570	+
	4571	+ /* grab the ring lock to allocate pktid and post on ring */
	4572	+ DHD_RING_LOCK(ring->ring_lock, flags);
	4573	+
	4574	+ /* Claim space for exactly 'count' no of messages, for mitigation purpose */
	4575	+ msg_start = (void *)
	4576	+ dhd_prot_alloc_ring_space(dhd, ring, count, &alloced, TRUE);
	4577	+ if (msg_start == NULL) {
	4578	+ DHD_INFO(("%s:%d: Rxbufpost Msgbuf Not available\n", __FUNCTION__, __LINE__));
	4579	+ DHD_RING_UNLOCK(ring->ring_lock, flags);
	4580	+ goto cleanup;
	4581	+ }
	4582	+ /* if msg_start != NULL, we should have alloced space for atleast 1 item */
	4583	+ ASSERT(alloced > 0);
	4584	+
	4585	+ rxbuf_post_tmp = (uint8*)msg_start;
	4586	+
	4587	+ for (i = 0; i < alloced; i++) {
	4588	+ rxbuf_post = (host_rxbuf_post_t *)rxbuf_post_tmp;
	4589	+ p = pktbuf[i];
	4590	+ pa = pktbuf_pa[i];
2920	4591
2921	4592	#if defined(DHD_LB_RXC)
2922	4593	if (use_rsv_pktid == TRUE) {
2923	4594	bcm_workq_t *workq = &prot->rx_compl_cons;
2924	4595	int elem_ix = bcm_ring_cons(WORKQ_RING(workq), DHD_LB_WORKQ_SZ);
	4596	+
2925	4597	if (elem_ix == BCM_RING_EMPTY) {
2926		- DHD_ERROR(("%s rx_compl_cons ring is empty\n", __FUNCTION__));
	4598	+ DHD_INFO(("%s rx_compl_cons ring is empty\n", __FUNCTION__));
2927	4599	pktid = DHD_PKTID_INVALID;
2928	4600	goto alloc_pkt_id;
2929	4601	} else {
..	..	@@ -2931,49 +4603,36 @@
2931	4603	pktid = *elem;
2932	4604	}
2933	4605
	4606	+ rxbuf_post->cmn_hdr.request_id = htol32(pktid);
	4607	+
2934	4608	/* Now populate the previous locker with valid information */
2935	4609	if (pktid != DHD_PKTID_INVALID) {
2936		- rxbuf_post->cmn_hdr.request_id = htol32(pktid);
2937		- DHD_NATIVE_TO_PKTID_SAVE(dhd, dhd->prot->pktid_map_handle, p, pktid,
2938		- pa, pktlen, DMA_RX, NULL, ring->dma_buf.secdma,
	4610	+ DHD_NATIVE_TO_PKTID_SAVE(dhd, dhd->prot->pktid_rx_map,
	4611	+ p, pktid, pa, pktlen[i], DMA_RX, NULL, NULL,
2939	4612	PKTTYPE_DATA_RX);
2940	4613	}
2941	4614	} else
2942		-#endif /* DHD_LB_RXC */
	4615	+#endif /* ! DHD_LB_RXC */
2943	4616	{
2944	4617	#if defined(DHD_LB_RXC)
2945	4618	alloc_pkt_id:
2946		-#endif
	4619	+#endif /* DHD_LB_RXC */
	4620	+ pktid = DHD_NATIVE_TO_PKTID(dhd, dhd->prot->pktid_rx_map, p, pa,
	4621	+ pktlen[i], DMA_RX, NULL, ring->dma_buf.secdma, PKTTYPE_DATA_RX);
2947	4622	#if defined(DHD_PCIE_PKTID)
2948		- /* get the lock before calling DHD_NATIVE_TO_PKTID */
2949		- DHD_GENERAL_LOCK(dhd, flags);
2950		-#endif
2951		- pktid = DHD_NATIVE_TO_PKTID(dhd, dhd->prot->pktid_map_handle, p, pa,
2952		- pktlen, DMA_RX, NULL, ring->dma_buf.secdma, PKTTYPE_DATA_RX);
2953		-
2954		-#if defined(DHD_PCIE_PKTID)
2955		- /* free lock */
2956		- DHD_GENERAL_UNLOCK(dhd, flags);
2957		-
2958	4623	if (pktid == DHD_PKTID_INVALID) {
2959		-
2960		- if (SECURE_DMA_ENAB(dhd->osh)) {
2961		- DHD_GENERAL_LOCK(dhd, flags);
2962		- SECURE_DMA_UNMAP(dhd->osh, pa, pktlen, DMA_RX, 0, DHD_DMAH_NULL,
2963		- ring->dma_buf.secdma, 0);
2964		- DHD_GENERAL_UNLOCK(dhd, flags);
2965		- } else {
2966		- DMA_UNMAP(dhd->osh, pa, pktlen, DMA_RX, p, DHD_DMAH_NULL);
2967		- }
2968		-
2969		- PKTFREE(dhd->osh, p, FALSE);
2970		- DHD_ERROR(("Pktid pool depleted.\n"));
2971	4624	break;
2972	4625	}
2973	4626	#endif /* DHD_PCIE_PKTID */
2974	4627	}
2975	4628
2976		- rxbuf_post->data_buf_len = htol16((uint16)pktlen);
	4629	+ /* Common msg header */
	4630	+ rxbuf_post->cmn_hdr.msg_type = MSG_TYPE_RXBUF_POST;
	4631	+ rxbuf_post->cmn_hdr.if_id = 0;
	4632	+ rxbuf_post->cmn_hdr.epoch = ring->seqnum % H2D_EPOCH_MODULO;
	4633	+ rxbuf_post->cmn_hdr.flags = ring->current_phase;
	4634	+ ring->seqnum++;
	4635	+ rxbuf_post->data_buf_len = htol16((uint16)pktlen[i]);
2977	4636	rxbuf_post->data_buf_addr.high_addr = htol32(PHYSADDRHI(pa));
2978	4637	rxbuf_post->data_buf_addr.low_addr =
2979	4638	htol32(PHYSADDRLO(pa) + prot->rx_metadata_offset);
..	..	@@ -2988,33 +4647,235 @@
2988	4647	rxbuf_post->metadata_buf_addr.low_addr = 0;
2989	4648	}
2990	4649
2991		-#if defined(DHD_PKTID_AUDIT_RING)
2992		- DHD_PKTID_AUDIT(dhd, prot->pktid_map_handle, pktid, DHD_DUPLICATE_ALLOC);
	4650	+#ifdef DHD_PKTID_AUDIT_RING
	4651	+ DHD_PKTID_AUDIT(dhd, prot->pktid_rx_map, pktid, DHD_DUPLICATE_ALLOC);
2993	4652	#endif /* DHD_PKTID_AUDIT_RING */
2994	4653
2995	4654	rxbuf_post->cmn_hdr.request_id = htol32(pktid);
2996	4655
2997	4656	/* Move rxbuf_post_tmp to next item */
2998	4657	rxbuf_post_tmp = rxbuf_post_tmp + ring->item_len;
	4658	+
	4659	+#ifdef DHD_LBUF_AUDIT
	4660	+ PKTAUDIT(dhd->osh, p);
	4661	+#endif // endif
2999	4662	}
3000	4663
3001	4664	if (i < alloced) {
3002		- if (ring->wr < (alloced - i)) {
	4665	+ if (ring->wr < (alloced - i))
3003	4666	ring->wr = ring->max_items - (alloced - i);
3004		- } else {
	4667	+ else
3005	4668	ring->wr -= (alloced - i);
	4669	+
	4670	+ if (ring->wr == 0) {
	4671	+ DHD_INFO(("%s: flipping the phase now\n", ring->name));
	4672	+ ring->current_phase = ring->current_phase ?
	4673	+ 0 : BCMPCIE_CMNHDR_PHASE_BIT_INIT;
3006	4674	}
3007	4675
3008	4676	alloced = i;
3009	4677	}
3010	4678
3011		- /* Update ring's WR index and ring doorbell to dongle */
	4679	+ /* update ring's WR index and ring doorbell to dongle */
3012	4680	if (alloced > 0) {
3013	4681	dhd_prot_ring_write_complete(dhd, ring, msg_start, alloced);
3014	4682	}
3015	4683
	4684	+ DHD_RING_UNLOCK(ring->ring_lock, flags);
	4685	+
	4686	+cleanup:
	4687	+ for (i = alloced; i < count; i++) {
	4688	+ p = pktbuf[i];
	4689	+ pa = pktbuf_pa[i];
	4690	+
	4691	+ if (SECURE_DMA_ENAB(dhd->osh))
	4692	+ SECURE_DMA_UNMAP(dhd->osh, pa, pktlen[i], DMA_RX, 0,
	4693	+ DHD_DMAH_NULL, ring->dma_buf.secdma, 0);
	4694	+ else
	4695	+ DMA_UNMAP(dhd->osh, pa, pktlen[i], DMA_RX, 0, DHD_DMAH_NULL);
	4696	+ PKTFREE(dhd->osh, p, FALSE);
	4697	+ }
	4698	+
	4699	+ MFREE(dhd->osh, lcl_buf, lcl_buf_size);
	4700	+
3016	4701	return alloced;
3017		-} /* dhd_prot_rxbuf_post */
	4702	+} /* dhd_prot_rxbufpost */
	4703	+
	4704	+static int
	4705	+dhd_prot_infobufpost(dhd_pub_t dhd, msgbuf_ring_t ring)
	4706	+{
	4707	+ unsigned long flags;
	4708	+ uint32 pktid;
	4709	+ dhd_prot_t *prot = dhd->prot;
	4710	+ uint16 alloced = 0;
	4711	+ uint16 pktsz = DHD_INFOBUF_RX_BUFPOST_PKTSZ;
	4712	+ uint32 pktlen;
	4713	+ info_buf_post_msg_t *infobuf_post;
	4714	+ uint8 *infobuf_post_tmp;
	4715	+ void *p;
	4716	+ void* msg_start;
	4717	+ uint8 i = 0;
	4718	+ dmaaddr_t pa;
	4719	+ int16 count = 0;
	4720	+
	4721	+ if (ring == NULL)
	4722	+ return 0;
	4723	+
	4724	+ if (ring->inited != TRUE)
	4725	+ return 0;
	4726	+ if (ring == dhd->prot->h2dring_info_subn) {
	4727	+ if (prot->max_infobufpost == 0)
	4728	+ return 0;
	4729	+
	4730	+ count = prot->max_infobufpost - prot->infobufpost;
	4731	+ }
	4732	+ else {
	4733	+ DHD_ERROR(("Unknown ring\n"));
	4734	+ return 0;
	4735	+ }
	4736	+
	4737	+ if (count <= 0) {
	4738	+ DHD_INFO(("%s: Cannot post more than max info resp buffers\n",
	4739	+ __FUNCTION__));
	4740	+ return 0;
	4741	+ }
	4742	+
	4743	+ /* grab the ring lock to allocate pktid and post on ring */
	4744	+ DHD_RING_LOCK(ring->ring_lock, flags);
	4745	+
	4746	+ /* Claim space for exactly 'count' no of messages, for mitigation purpose */
	4747	+ msg_start = (void *) dhd_prot_alloc_ring_space(dhd, ring, count, &alloced, FALSE);
	4748	+
	4749	+ if (msg_start == NULL) {
	4750	+ DHD_INFO(("%s:%d: infobufpost Msgbuf Not available\n", __FUNCTION__, __LINE__));
	4751	+ DHD_RING_UNLOCK(ring->ring_lock, flags);
	4752	+ return -1;
	4753	+ }
	4754	+
	4755	+ /* if msg_start != NULL, we should have alloced space for atleast 1 item */
	4756	+ ASSERT(alloced > 0);
	4757	+
	4758	+ infobuf_post_tmp = (uint8*) msg_start;
	4759	+
	4760	+ /* loop through each allocated message in the host ring */
	4761	+ for (i = 0; i < alloced; i++) {
	4762	+ infobuf_post = (info_buf_post_msg_t *) infobuf_post_tmp;
	4763	+ /* Create a rx buffer */
	4764	+#ifdef DHD_USE_STATIC_CTRLBUF
	4765	+ p = PKTGET_STATIC(dhd->osh, pktsz, FALSE);
	4766	+#else
	4767	+ p = PKTGET(dhd->osh, pktsz, FALSE);
	4768	+#endif /* DHD_USE_STATIC_CTRLBUF */
	4769	+ if (p == NULL) {
	4770	+ DHD_ERROR(("%s:%d: PKTGET for infobuf failed\n", __FUNCTION__, __LINE__));
	4771	+ dhd->rx_pktgetfail++;
	4772	+ break;
	4773	+ }
	4774	+ pktlen = PKTLEN(dhd->osh, p);
	4775	+ if (SECURE_DMA_ENAB(dhd->osh)) {
	4776	+ pa = SECURE_DMA_MAP(dhd->osh, PKTDATA(dhd->osh, p), pktlen,
	4777	+ DMA_RX, p, 0, ring->dma_buf.secdma, 0);
	4778	+ }
	4779	+#ifndef BCM_SECURE_DMA
	4780	+ else
	4781	+ pa = DMA_MAP(dhd->osh, PKTDATA(dhd->osh, p), pktlen, DMA_RX, p, 0);
	4782	+#endif /* #ifndef BCM_SECURE_DMA */
	4783	+ if (PHYSADDRISZERO(pa)) {
	4784	+ if (SECURE_DMA_ENAB(dhd->osh)) {
	4785	+ SECURE_DMA_UNMAP(dhd->osh, pa, pktlen, DMA_RX, 0, DHD_DMAH_NULL,
	4786	+ ring->dma_buf.secdma, 0);
	4787	+ }
	4788	+ else
	4789	+ DMA_UNMAP(dhd->osh, pa, pktlen, DMA_RX, 0, DHD_DMAH_NULL);
	4790	+#ifdef DHD_USE_STATIC_CTRLBUF
	4791	+ PKTFREE_STATIC(dhd->osh, p, FALSE);
	4792	+#else
	4793	+ PKTFREE(dhd->osh, p, FALSE);
	4794	+#endif /* DHD_USE_STATIC_CTRLBUF */
	4795	+ DHD_ERROR(("Invalid phyaddr 0\n"));
	4796	+ ASSERT(0);
	4797	+ break;
	4798	+ }
	4799	+#ifdef DMAMAP_STATS
	4800	+ dhd->dma_stats.info_rx++;
	4801	+ dhd->dma_stats.info_rx_sz += pktlen;
	4802	+#endif /* DMAMAP_STATS */
	4803	+ pktlen = PKTLEN(dhd->osh, p);
	4804	+
	4805	+ /* Common msg header */
	4806	+ infobuf_post->cmn_hdr.msg_type = MSG_TYPE_INFO_BUF_POST;
	4807	+ infobuf_post->cmn_hdr.if_id = 0;
	4808	+ infobuf_post->cmn_hdr.epoch = ring->seqnum % H2D_EPOCH_MODULO;
	4809	+ infobuf_post->cmn_hdr.flags = ring->current_phase;
	4810	+ ring->seqnum++;
	4811	+
	4812	+ pktid = DHD_NATIVE_TO_PKTID(dhd, dhd->prot->pktid_ctrl_map, p, pa,
	4813	+ pktlen, DMA_RX, NULL, ring->dma_buf.secdma, PKTTYPE_INFO_RX);
	4814	+
	4815	+#if defined(DHD_PCIE_PKTID)
	4816	+ if (pktid == DHD_PKTID_INVALID) {
	4817	+ if (SECURE_DMA_ENAB(dhd->osh)) {
	4818	+ SECURE_DMA_UNMAP(dhd->osh, pa, pktlen, DMA_RX, 0, 0,
	4819	+ ring->dma_buf.secdma, 0);
	4820	+ } else
	4821	+ DMA_UNMAP(dhd->osh, pa, pktlen, DMA_RX, 0, 0);
	4822	+
	4823	+#ifdef DHD_USE_STATIC_CTRLBUF
	4824	+ PKTFREE_STATIC(dhd->osh, p, FALSE);
	4825	+#else
	4826	+ PKTFREE(dhd->osh, p, FALSE);
	4827	+#endif /* DHD_USE_STATIC_CTRLBUF */
	4828	+ DHD_ERROR_RLMT(("%s: Pktid pool depleted.\n", __FUNCTION__));
	4829	+ break;
	4830	+ }
	4831	+#endif /* DHD_PCIE_PKTID */
	4832	+
	4833	+ infobuf_post->host_buf_len = htol16((uint16)pktlen);
	4834	+ infobuf_post->host_buf_addr.high_addr = htol32(PHYSADDRHI(pa));
	4835	+ infobuf_post->host_buf_addr.low_addr = htol32(PHYSADDRLO(pa));
	4836	+
	4837	+#ifdef DHD_PKTID_AUDIT_RING
	4838	+ DHD_PKTID_AUDIT(dhd, prot->pktid_ctrl_map, pktid, DHD_DUPLICATE_ALLOC);
	4839	+#endif /* DHD_PKTID_AUDIT_RING */
	4840	+
	4841	+ DHD_INFO(("ID %d, low_addr 0x%08x, high_addr 0x%08x\n",
	4842	+ infobuf_post->cmn_hdr.request_id, infobuf_post->host_buf_addr.low_addr,
	4843	+ infobuf_post->host_buf_addr.high_addr));
	4844	+
	4845	+ infobuf_post->cmn_hdr.request_id = htol32(pktid);
	4846	+ /* Move rxbuf_post_tmp to next item */
	4847	+ infobuf_post_tmp = infobuf_post_tmp + ring->item_len;
	4848	+#ifdef DHD_LBUF_AUDIT
	4849	+ PKTAUDIT(dhd->osh, p);
	4850	+#endif // endif
	4851	+ }
	4852	+
	4853	+ if (i < alloced) {
	4854	+ if (ring->wr < (alloced - i))
	4855	+ ring->wr = ring->max_items - (alloced - i);
	4856	+ else
	4857	+ ring->wr -= (alloced - i);
	4858	+
	4859	+ alloced = i;
	4860	+ if (alloced && ring->wr == 0) {
	4861	+ DHD_INFO(("%s: flipping the phase now\n", ring->name));
	4862	+ ring->current_phase = ring->current_phase ?
	4863	+ 0 : BCMPCIE_CMNHDR_PHASE_BIT_INIT;
	4864	+ }
	4865	+ }
	4866	+
	4867	+ /* Update the write pointer in TCM & ring bell */
	4868	+ if (alloced > 0) {
	4869	+ if (ring == dhd->prot->h2dring_info_subn) {
	4870	+ prot->infobufpost += alloced;
	4871	+ }
	4872	+ dhd_prot_ring_write_complete(dhd, ring, msg_start, alloced);
	4873	+ }
	4874	+
	4875	+ DHD_RING_UNLOCK(ring->ring_lock, flags);
	4876	+
	4877	+ return alloced;
	4878	+} /* dhd_prot_infobufpost */
3018	4879
3019	4880	#ifdef IOCTLRESP_USE_CONSTMEM
3020	4881	static int
..	..	@@ -3041,10 +4902,6 @@
3041	4902	dma_pad = (IOCT_RETBUF_SIZE % DHD_DMA_PAD) ? DHD_DMA_PAD : 0;
3042	4903	retbuf->len = IOCT_RETBUF_SIZE;
3043	4904	retbuf->_alloced = retbuf->len + dma_pad;
3044		- /* JIRA:SWWLAN-70021 The pa value would be overwritten by the dongle.
3045		- * Need to reassign before free to pass the check in dhd_dma_buf_audit().
3046		- */
3047		- retbuf->pa = DMA_MAP(dhd->osh, retbuf->va, retbuf->len, DMA_RX, NULL, NULL);
3048	4905	}
3049	4906
3050	4907	dhd_dma_buf_free(dhd, retbuf);
..	..	@@ -3053,7 +4910,7 @@
3053	4910	#endif /* IOCTLRESP_USE_CONSTMEM */
3054	4911
3055	4912	static int
3056		-dhd_prot_rxbufpost_ctrl(dhd_pub_t *dhd, bool event_buf)
	4913	+dhd_prot_rxbufpost_ctrl(dhd_pub_t *dhd, uint8 msg_type)
3057	4914	{
3058	4915	void *p;
3059	4916	uint16 pktsz;
..	..	@@ -3068,16 +4925,33 @@
3068	4925	uint32 pktid;
3069	4926	void *map_handle;
3070	4927	msgbuf_ring_t *ring = &prot->h2dring_ctrl_subn;
	4928	+ bool non_ioctl_resp_buf = 0;
	4929	+ dhd_pkttype_t buf_type;
3071	4930
3072	4931	if (dhd->busstate == DHD_BUS_DOWN) {
3073	4932	DHD_ERROR(("%s: bus is already down.\n", __FUNCTION__));
3074	4933	return -1;
3075	4934	}
3076		-
3077	4935	memset(&retbuf, 0, sizeof(dhd_dma_buf_t));
3078	4936
3079		- if (event_buf) {
3080		- /* Allocate packet for event buffer post */
	4937	+ if (msg_type == MSG_TYPE_IOCTLRESP_BUF_POST)
	4938	+ buf_type = PKTTYPE_IOCTL_RX;
	4939	+ else if (msg_type == MSG_TYPE_EVENT_BUF_POST)
	4940	+ buf_type = PKTTYPE_EVENT_RX;
	4941	+ else if (msg_type == MSG_TYPE_TIMSTAMP_BUFPOST)
	4942	+ buf_type = PKTTYPE_TSBUF_RX;
	4943	+ else {
	4944	+ DHD_ERROR(("invalid message type to be posted to Ctrl ring %d\n", msg_type));
	4945	+ return -1;
	4946	+ }
	4947	+
	4948	+ if ((msg_type == MSG_TYPE_EVENT_BUF_POST) \|\| (msg_type == MSG_TYPE_TIMSTAMP_BUFPOST))
	4949	+ non_ioctl_resp_buf = TRUE;
	4950	+ else
	4951	+ non_ioctl_resp_buf = FALSE;
	4952	+
	4953	+ if (non_ioctl_resp_buf) {
	4954	+ /* Allocate packet for not ioctl resp buffer post */
3081	4955	pktsz = DHD_FLOWRING_RX_BUFPOST_PKTSZ;
3082	4956	} else {
3083	4957	/* Allocate packet for ctrl/ioctl buffer post */
..	..	@@ -3085,7 +4959,7 @@
3085	4959	}
3086	4960
3087	4961	#ifdef IOCTLRESP_USE_CONSTMEM
3088		- if (!event_buf) {
	4962	+ if (!non_ioctl_resp_buf) {
3089	4963	if (alloc_ioctl_return_buffer(dhd, &retbuf) != BCME_OK) {
3090	4964	DHD_ERROR(("Could not allocate IOCTL response buffer\n"));
3091	4965	return -1;
..	..	@@ -3105,7 +4979,7 @@
3105	4979	#endif /* DHD_USE_STATIC_CTRLBUF */
3106	4980	if (p == NULL) {
3107	4981	DHD_ERROR(("%s:%d: PKTGET for %s buf failed\n",
3108		- __FUNCTION__, __LINE__, event_buf ?
	4982	+ __FUNCTION__, __LINE__, non_ioctl_resp_buf ?
3109	4983	"EVENT" : "IOCTL RESP"));
3110	4984	dhd->rx_pktgetfail++;
3111	4985	return -1;
..	..	@@ -3114,10 +4988,8 @@
3114	4988	pktlen = PKTLEN(dhd->osh, p);
3115	4989
3116	4990	if (SECURE_DMA_ENAB(dhd->osh)) {
3117		- DHD_GENERAL_LOCK(dhd, flags);
3118	4991	pa = SECURE_DMA_MAP(dhd->osh, PKTDATA(dhd->osh, p), pktlen,
3119	4992	DMA_RX, p, 0, ring->dma_buf.secdma, 0);
3120		- DHD_GENERAL_UNLOCK(dhd, flags);
3121	4993	}
3122	4994	#ifndef BCM_SECURE_DMA
3123	4995	else
..	..	@@ -3129,53 +5001,70 @@
3129	5001	ASSERT(0);
3130	5002	goto free_pkt_return;
3131	5003	}
	5004	+
	5005	+#ifdef DMAMAP_STATS
	5006	+ switch (buf_type) {
	5007	+#ifndef IOCTLRESP_USE_CONSTMEM
	5008	+ case PKTTYPE_IOCTL_RX:
	5009	+ dhd->dma_stats.ioctl_rx++;
	5010	+ dhd->dma_stats.ioctl_rx_sz += pktlen;
	5011	+ break;
	5012	+#endif /* !IOCTLRESP_USE_CONSTMEM */
	5013	+ case PKTTYPE_EVENT_RX:
	5014	+ dhd->dma_stats.event_rx++;
	5015	+ dhd->dma_stats.event_rx_sz += pktlen;
	5016	+ break;
	5017	+ case PKTTYPE_TSBUF_RX:
	5018	+ dhd->dma_stats.tsbuf_rx++;
	5019	+ dhd->dma_stats.tsbuf_rx_sz += pktlen;
	5020	+ break;
	5021	+ default:
	5022	+ break;
	5023	+ }
	5024	+#endif /* DMAMAP_STATS */
	5025	+
3132	5026	}
3133	5027
3134		- DHD_GENERAL_LOCK(dhd, flags);
	5028	+ /* grab the ring lock to allocate pktid and post on ring */
	5029	+ DHD_RING_LOCK(ring->ring_lock, flags);
3135	5030
3136	5031	rxbuf_post = (ioctl_resp_evt_buf_post_msg_t *)
3137	5032	dhd_prot_alloc_ring_space(dhd, ring, 1, &alloced, FALSE);
3138	5033
3139	5034	if (rxbuf_post == NULL) {
3140		- DHD_GENERAL_UNLOCK(dhd, flags);
	5035	+ DHD_RING_UNLOCK(ring->ring_lock, flags);
3141	5036	DHD_ERROR(("%s:%d: Ctrl submit Msgbuf Not available to post buffer \n",
3142	5037	__FUNCTION__, __LINE__));
3143	5038
3144	5039	#ifdef IOCTLRESP_USE_CONSTMEM
3145		- if (event_buf)
	5040	+ if (non_ioctl_resp_buf)
3146	5041	#endif /* IOCTLRESP_USE_CONSTMEM */
3147	5042	{
3148	5043	if (SECURE_DMA_ENAB(dhd->osh)) {
3149		- DHD_GENERAL_LOCK(dhd, flags);
3150	5044	SECURE_DMA_UNMAP(dhd->osh, pa, pktlen, DMA_RX, 0, DHD_DMAH_NULL,
3151	5045	ring->dma_buf.secdma, 0);
3152		- DHD_GENERAL_UNLOCK(dhd, flags);
3153	5046	} else {
3154		- DMA_UNMAP(dhd->osh, pa, pktlen, DMA_RX, p, DHD_DMAH_NULL);
	5047	+ DMA_UNMAP(dhd->osh, pa, pktlen, DMA_RX, 0, DHD_DMAH_NULL);
3155	5048	}
3156	5049	}
3157	5050	goto free_pkt_return;
3158	5051	}
3159	5052
3160	5053	/* CMN msg header */
3161		- if (event_buf) {
3162		- rxbuf_post->cmn_hdr.msg_type = MSG_TYPE_EVENT_BUF_POST;
3163		- } else {
3164		- rxbuf_post->cmn_hdr.msg_type = MSG_TYPE_IOCTLRESP_BUF_POST;
3165		- }
	5054	+ rxbuf_post->cmn_hdr.msg_type = msg_type;
3166	5055
3167	5056	#ifdef IOCTLRESP_USE_CONSTMEM
3168		- if (!event_buf) {
	5057	+ if (!non_ioctl_resp_buf) {
3169	5058	map_handle = dhd->prot->pktid_map_handle_ioctl;
3170		- pktid = DHD_NATIVE_TO_PKTID(dhd, map_handle, p, pa, pktlen,
3171		- DMA_RX, dmah, ring->dma_buf.secdma, PKTTYPE_IOCTL_RX);
	5059	+ pktid = DHD_NATIVE_TO_PKTID(dhd, map_handle, p, pa, pktlen, DMA_RX, dmah,
	5060	+ ring->dma_buf.secdma, buf_type);
3172	5061	} else
3173	5062	#endif /* IOCTLRESP_USE_CONSTMEM */
3174	5063	{
3175		- map_handle = dhd->prot->pktid_map_handle;
3176		- pktid = DHD_NATIVE_TO_PKTID(dhd, map_handle,
	5064	+ map_handle = dhd->prot->pktid_ctrl_map;
	5065	+ pktid = DHD_NATIVE_TO_PKTID(dhd, map_handle,
3177	5066	p, pa, pktlen, DMA_RX, dmah, ring->dma_buf.secdma,
3178		- event_buf ? PKTTYPE_EVENT_RX : PKTTYPE_IOCTL_RX);
	5067	+ buf_type);
3179	5068	}
3180	5069
3181	5070	if (pktid == DHD_PKTID_INVALID) {
..	..	@@ -3183,47 +5072,52 @@
3183	5072	ring->wr = ring->max_items - 1;
3184	5073	} else {
3185	5074	ring->wr--;
	5075	+ if (ring->wr == 0) {
	5076	+ ring->current_phase = ring->current_phase ? 0 :
	5077	+ BCMPCIE_CMNHDR_PHASE_BIT_INIT;
	5078	+ }
3186	5079	}
3187		- DHD_GENERAL_UNLOCK(dhd, flags);
3188		- DMA_UNMAP(dhd->osh, pa, pktlen, DMA_RX, p, DHD_DMAH_NULL);
	5080	+ DHD_RING_UNLOCK(ring->ring_lock, flags);
	5081	+ DMA_UNMAP(dhd->osh, pa, pktlen, DMA_RX, 0, DHD_DMAH_NULL);
	5082	+ DHD_ERROR_RLMT(("%s: Pktid pool depleted.\n", __FUNCTION__));
3189	5083	goto free_pkt_return;
3190	5084	}
3191	5085
3192		-#if defined(DHD_PKTID_AUDIT_RING)
	5086	+#ifdef DHD_PKTID_AUDIT_RING
3193	5087	DHD_PKTID_AUDIT(dhd, map_handle, pktid, DHD_DUPLICATE_ALLOC);
3194	5088	#endif /* DHD_PKTID_AUDIT_RING */
3195	5089
3196	5090	rxbuf_post->cmn_hdr.request_id = htol32(pktid);
3197	5091	rxbuf_post->cmn_hdr.if_id = 0;
3198		- rxbuf_post->cmn_hdr.epoch = ring->seqnum % H2D_EPOCH_MODULO;
	5092	+ rxbuf_post->cmn_hdr.epoch = ring->seqnum % H2D_EPOCH_MODULO;
3199	5093	ring->seqnum++;
	5094	+ rxbuf_post->cmn_hdr.flags = ring->current_phase;
3200	5095
3201	5096	#if defined(DHD_PCIE_PKTID)
3202	5097	if (rxbuf_post->cmn_hdr.request_id == DHD_PKTID_INVALID) {
3203	5098	if (ring->wr == 0) {
3204	5099	ring->wr = ring->max_items - 1;
3205	5100	} else {
3206		- ring->wr--;
	5101	+ if (ring->wr == 0) {
	5102	+ ring->current_phase = ring->current_phase ? 0 :
	5103	+ BCMPCIE_CMNHDR_PHASE_BIT_INIT;
	5104	+ }
3207	5105	}
3208		- DHD_GENERAL_UNLOCK(dhd, flags);
	5106	+ DHD_RING_UNLOCK(ring->ring_lock, flags);
3209	5107	#ifdef IOCTLRESP_USE_CONSTMEM
3210		- if (event_buf)
	5108	+ if (non_ioctl_resp_buf)
3211	5109	#endif /* IOCTLRESP_USE_CONSTMEM */
3212	5110	{
3213	5111	if (SECURE_DMA_ENAB(dhd->osh)) {
3214		- DHD_GENERAL_LOCK(dhd, flags);
3215	5112	SECURE_DMA_UNMAP(dhd->osh, pa, pktlen, DMA_RX, 0, DHD_DMAH_NULL,
3216	5113	ring->dma_buf.secdma, 0);
3217		- DHD_GENERAL_UNLOCK(dhd, flags);
3218		- } else {
3219		- DMA_UNMAP(dhd->osh, pa, pktlen, DMA_RX, p, DHD_DMAH_NULL);
3220		- }
	5114	+ } else
	5115	+ DMA_UNMAP(dhd->osh, pa, pktlen, DMA_RX, 0, DHD_DMAH_NULL);
3221	5116	}
3222	5117	goto free_pkt_return;
3223	5118	}
3224	5119	#endif /* DHD_PCIE_PKTID */
3225	5120
3226		- rxbuf_post->cmn_hdr.flags = 0;
3227	5121	#ifndef IOCTLRESP_USE_CONSTMEM
3228	5122	rxbuf_post->host_buf_len = htol16((uint16)PKTLEN(dhd->osh, p));
3229	5123	#else
..	..	@@ -3232,34 +5126,39 @@
3232	5126	rxbuf_post->host_buf_addr.high_addr = htol32(PHYSADDRHI(pa));
3233	5127	rxbuf_post->host_buf_addr.low_addr = htol32(PHYSADDRLO(pa));
3234	5128
	5129	+#ifdef DHD_LBUF_AUDIT
	5130	+ if (non_ioctl_resp_buf)
	5131	+ PKTAUDIT(dhd->osh, p);
	5132	+#endif // endif
	5133	+
3235	5134	/* update ring's WR index and ring doorbell to dongle */
3236	5135	dhd_prot_ring_write_complete(dhd, ring, rxbuf_post, 1);
3237		- DHD_GENERAL_UNLOCK(dhd, flags);
	5136	+
	5137	+ DHD_RING_UNLOCK(ring->ring_lock, flags);
3238	5138
3239	5139	return 1;
3240	5140
3241	5141	free_pkt_return:
	5142	+ if (!non_ioctl_resp_buf) {
3242	5143	#ifdef IOCTLRESP_USE_CONSTMEM
3243		- if (!event_buf) {
3244	5144	free_ioctl_return_buffer(dhd, &retbuf);
3245		- } else
	5145	+#else
	5146	+ dhd_prot_packet_free(dhd, p, buf_type, FALSE);
3246	5147	#endif /* IOCTLRESP_USE_CONSTMEM */
3247		- {
3248		- dhd_prot_packet_free(dhd, p,
3249		- event_buf ? PKTTYPE_EVENT_RX : PKTTYPE_IOCTL_RX,
3250		- FALSE);
	5148	+ } else {
	5149	+ dhd_prot_packet_free(dhd, p, buf_type, FALSE);
3251	5150	}
3252	5151
3253	5152	return -1;
3254	5153	} /* dhd_prot_rxbufpost_ctrl */
3255	5154
3256	5155	static uint16
3257		-dhd_msgbuf_rxbuf_post_ctrlpath(dhd_pub_t *dhd, bool event_buf, uint32 max_to_post)
	5156	+dhd_msgbuf_rxbuf_post_ctrlpath(dhd_pub_t *dhd, uint8 msg_type, uint32 max_to_post)
3258	5157	{
3259	5158	uint32 i = 0;
3260	5159	int32 ret_val;
3261	5160
3262		- DHD_INFO(("max to post %d, event %d \n", max_to_post, event_buf));
	5161	+ DHD_INFO(("max to post %d, event %d \n", max_to_post, msg_type));
3263	5162
3264	5163	if (dhd->busstate == DHD_BUS_DOWN) {
3265	5164	DHD_ERROR(("%s: bus is already down.\n", __FUNCTION__));
..	..	@@ -3267,13 +5166,12 @@
3267	5166	}
3268	5167
3269	5168	while (i < max_to_post) {
3270		- ret_val = dhd_prot_rxbufpost_ctrl(dhd, event_buf);
3271		- if (ret_val < 0) {
	5169	+ ret_val = dhd_prot_rxbufpost_ctrl(dhd, msg_type);
	5170	+ if (ret_val < 0)
3272	5171	break;
3273		- }
3274	5172	i++;
3275	5173	}
3276		- DHD_INFO(("posted %d buffers to event_pool/ioctl_resp_pool %d\n", i, event_buf));
	5174	+ DHD_INFO(("posted %d buffers of type %d\n", i, msg_type));
3277	5175	return (uint16)i;
3278	5176	}
3279	5177
..	..	@@ -3291,7 +5189,7 @@
3291	5189	return;
3292	5190	}
3293	5191	prot->cur_ioctlresp_bufs_posted += dhd_msgbuf_rxbuf_post_ctrlpath(dhd,
3294		- FALSE, max_to_post);
	5192	+ MSG_TYPE_IOCTLRESP_BUF_POST, max_to_post);
3295	5193	}
3296	5194
3297	5195	static void
..	..	@@ -3302,34 +5200,58 @@
3302	5200
3303	5201	max_to_post = prot->max_eventbufpost - prot->cur_event_bufs_posted;
3304	5202	if (max_to_post <= 0) {
3305		- DHD_INFO(("%s: Cannot post more than max event buffers\n",
	5203	+ DHD_ERROR(("%s: Cannot post more than max event buffers\n",
3306	5204	__FUNCTION__));
3307	5205	return;
3308	5206	}
3309	5207	prot->cur_event_bufs_posted += dhd_msgbuf_rxbuf_post_ctrlpath(dhd,
3310		- TRUE, max_to_post);
	5208	+ MSG_TYPE_EVENT_BUF_POST, max_to_post);
3311	5209	}
3312	5210
3313		-/** called when DHD needs to check for 'receive complete' messages from the dongle */
3314		-bool BCMFASTPATH
3315		-dhd_prot_process_msgbuf_rxcpl(dhd_pub_t *dhd, uint bound)
	5211	+static int
	5212	+dhd_msgbuf_rxbuf_post_ts_bufs(dhd_pub_t *dhd)
3316	5213	{
	5214	+ return 0;
	5215	+}
	5216	+
	5217	+bool BCMFASTPATH
	5218	+dhd_prot_process_msgbuf_infocpl(dhd_pub_t *dhd, uint bound)
	5219	+{
	5220	+ dhd_prot_t *prot = dhd->prot;
3317	5221	bool more = TRUE;
3318	5222	uint n = 0;
3319		- msgbuf_ring_t *ring = &dhd->prot->d2hring_rx_cpln;
	5223	+ msgbuf_ring_t *ring = prot->d2hring_info_cpln;
	5224	+ unsigned long flags;
	5225	+
	5226	+ if (ring == NULL)
	5227	+ return FALSE;
	5228	+ if (ring->inited != TRUE)
	5229	+ return FALSE;
3320	5230
3321	5231	/* Process all the messages - DTOH direction */
3322	5232	while (!dhd_is_device_removed(dhd)) {
3323	5233	uint8 *msg_addr;
3324	5234	uint32 msg_len;
3325	5235
	5236	+ if (dhd_query_bus_erros(dhd)) {
	5237	+ more = FALSE;
	5238	+ break;
	5239	+ }
	5240	+
3326	5241	if (dhd->hang_was_sent) {
3327	5242	more = FALSE;
3328	5243	break;
3329	5244	}
3330	5245
3331		- /* Get the address of the next message to be read from ring */
	5246	+ if (dhd->smmu_fault_occurred) {
	5247	+ more = FALSE;
	5248	+ break;
	5249	+ }
	5250	+
	5251	+ DHD_RING_LOCK(ring->ring_lock, flags);
	5252	+ /* Get the message from ring */
3332	5253	msg_addr = dhd_prot_get_read_addr(dhd, ring, &msg_len);
	5254	+ DHD_RING_UNLOCK(ring->ring_lock, flags);
3333	5255	if (msg_addr == NULL) {
3334	5256	more = FALSE;
3335	5257	break;
..	..	@@ -3339,8 +5261,8 @@
3339	5261	OSL_PREFETCH(msg_addr);
3340	5262
3341	5263	if (dhd_prot_process_msgtype(dhd, ring, msg_addr, msg_len) != BCME_OK) {
3342		- DHD_ERROR(("%s: process %s msg addr %p len %d\n",
3343		- __FUNCTION__, ring->name, msg_addr, msg_len));
	5264	+ DHD_ERROR(("%s: Error at process rxpl msgbuf of len %d\n",
	5265	+ __FUNCTION__, msg_len));
3344	5266	}
3345	5267
3346	5268	/* Update read pointer */
..	..	@@ -3356,6 +5278,536 @@
3356	5278	return more;
3357	5279	}
3358	5280
	5281	+#ifdef EWP_EDL
	5282	+bool
	5283	+dhd_prot_process_msgbuf_edl(dhd_pub_t *dhd)
	5284	+{
	5285	+ dhd_prot_t *prot = dhd->prot;
	5286	+ msgbuf_ring_t *ring = prot->d2hring_edl;
	5287	+ unsigned long flags = 0;
	5288	+ uint32 items = 0;
	5289	+ uint16 rd = 0;
	5290	+ uint16 depth = 0;
	5291	+
	5292	+ if (ring == NULL)
	5293	+ return FALSE;
	5294	+ if (ring->inited != TRUE)
	5295	+ return FALSE;
	5296	+ if (ring->item_len == 0) {
	5297	+ DHD_ERROR(("%s: Bad ring ! ringidx %d, item_len %d \n",
	5298	+ __FUNCTION__, ring->idx, ring->item_len));
	5299	+ return FALSE;
	5300	+ }
	5301	+
	5302	+ if (dhd_query_bus_erros(dhd)) {
	5303	+ return FALSE;
	5304	+ }
	5305	+
	5306	+ if (dhd->hang_was_sent) {
	5307	+ return FALSE;
	5308	+ }
	5309	+
	5310	+ /* in this DPC context just check if wr index has moved
	5311	+ * and schedule deferred context to actually process the
	5312	+ * work items.
	5313	+ */
	5314	+ /* update the write index */
	5315	+ DHD_RING_LOCK(ring->ring_lock, flags);
	5316	+ if (dhd->dma_d2h_ring_upd_support) {
	5317	+ /* DMAing write/read indices supported */
	5318	+ ring->wr = dhd_prot_dma_indx_get(dhd, D2H_DMA_INDX_WR_UPD, ring->idx);
	5319	+ } else {
	5320	+ dhd_bus_cmn_readshared(dhd->bus, &ring->wr, RING_WR_UPD, ring->idx);
	5321	+ }
	5322	+ rd = ring->rd;
	5323	+ DHD_RING_UNLOCK(ring->ring_lock, flags);
	5324	+
	5325	+ depth = ring->max_items;
	5326	+ /* check for avail space, in number of ring items */
	5327	+ items = READ_AVAIL_SPACE(ring->wr, rd, depth);
	5328	+ if (items == 0) {
	5329	+ /* no work items in edl ring */
	5330	+ return FALSE;
	5331	+ }
	5332	+ if (items > ring->max_items) {
	5333	+ DHD_ERROR(("\r\n======================= \r\n"));
	5334	+ DHD_ERROR(("%s(): ring %p, ring->name %s, ring->max_items %d, items %d \r\n",
	5335	+ __FUNCTION__, ring, ring->name, ring->max_items, items));
	5336	+ DHD_ERROR(("wr: %d, rd: %d, depth: %d \r\n",
	5337	+ ring->wr, ring->rd, depth));
	5338	+ DHD_ERROR(("dhd->busstate %d bus->wait_for_d3_ack %d \r\n",
	5339	+ dhd->busstate, dhd->bus->wait_for_d3_ack));
	5340	+ DHD_ERROR(("\r\n======================= \r\n"));
	5341	+#ifdef SUPPORT_LINKDOWN_RECOVERY
	5342	+ if (ring->wr >= ring->max_items) {
	5343	+ dhd->bus->read_shm_fail = TRUE;
	5344	+ }
	5345	+#else
	5346	+#ifdef DHD_FW_COREDUMP
	5347	+ if (dhd->memdump_enabled) {
	5348	+ /* collect core dump */
	5349	+ dhd->memdump_type = DUMP_TYPE_RESUMED_ON_INVALID_RING_RDWR;
	5350	+ dhd_bus_mem_dump(dhd);
	5351	+
	5352	+ }
	5353	+#endif /* DHD_FW_COREDUMP */
	5354	+#endif /* SUPPORT_LINKDOWN_RECOVERY */
	5355	+ dhd_schedule_reset(dhd);
	5356	+
	5357	+ return FALSE;
	5358	+ }
	5359	+
	5360	+ if (items > D2HRING_EDL_WATERMARK) {
	5361	+ DHD_ERROR_RLMT(("%s: WARNING! EDL watermark hit, num items=%u;"
	5362	+ " rd=%u; wr=%u; depth=%u;\n", __FUNCTION__, items,
	5363	+ ring->rd, ring->wr, depth));
	5364	+ }
	5365	+
	5366	+ dhd_schedule_logtrace(dhd->info);
	5367	+
	5368	+ return FALSE;
	5369	+}
	5370	+
	5371	+/* This is called either from work queue context of 'event_log_dispatcher_work' or
	5372	+* from the kthread context of dhd_logtrace_thread
	5373	+*/
	5374	+int
	5375	+dhd_prot_process_edl_complete(dhd_pub_t dhd, void evt_decode_data)
	5376	+{
	5377	+ dhd_prot_t *prot = NULL;
	5378	+ msgbuf_ring_t *ring = NULL;
	5379	+ int err = 0;
	5380	+ unsigned long flags = 0;
	5381	+ cmn_msg_hdr_t *msg = NULL;
	5382	+ uint8 *msg_addr = NULL;
	5383	+ uint32 max_items_to_process = 0, n = 0;
	5384	+ uint32 num_items = 0, new_items = 0;
	5385	+ uint16 depth = 0;
	5386	+ volatile uint16 wr = 0;
	5387	+
	5388	+ if (!dhd \|\| !dhd->prot)
	5389	+ return 0;
	5390	+
	5391	+ prot = dhd->prot;
	5392	+ ring = prot->d2hring_edl;
	5393	+ if (!ring \|\| !evt_decode_data) {
	5394	+ return 0;
	5395	+ }
	5396	+
	5397	+ if (dhd->hang_was_sent) {
	5398	+ return FALSE;
	5399	+ }
	5400	+
	5401	+ DHD_RING_LOCK(ring->ring_lock, flags);
	5402	+ ring->curr_rd = ring->rd;
	5403	+ wr = ring->wr;
	5404	+ depth = ring->max_items;
	5405	+ /* check for avail space, in number of ring items
	5406	+ * Note, that this will only give the # of items
	5407	+ * from rd to wr if wr>=rd, or from rd to ring end
	5408	+ * if wr < rd. So in the latter case strictly speaking
	5409	+ * not all the items are read. But this is OK, because
	5410	+ * these will be processed in the next doorbell as rd
	5411	+ * would have wrapped around. Processing in the next
	5412	+ * doorbell is acceptable since EDL only contains debug data
	5413	+ */
	5414	+ num_items = READ_AVAIL_SPACE(wr, ring->rd, depth);
	5415	+
	5416	+ if (num_items == 0) {
	5417	+ /* no work items in edl ring */
	5418	+ DHD_RING_UNLOCK(ring->ring_lock, flags);
	5419	+ return 0;
	5420	+ }
	5421	+
	5422	+ DHD_INFO(("%s: EDL work items [%u] available \n",
	5423	+ __FUNCTION__, num_items));
	5424	+
	5425	+ /* if space is available, calculate address to be read */
	5426	+ msg_addr = (char)ring->dma_buf.va + (ring->rd ring->item_len);
	5427	+
	5428	+ max_items_to_process = MIN(num_items, DHD_EVENT_LOGTRACE_BOUND);
	5429	+
	5430	+ DHD_RING_UNLOCK(ring->ring_lock, flags);
	5431	+
	5432	+ /* Prefetch data to populate the cache */
	5433	+ OSL_PREFETCH(msg_addr);
	5434	+
	5435	+ n = max_items_to_process;
	5436	+ while (n > 0) {
	5437	+ msg = (cmn_msg_hdr_t *)msg_addr;
	5438	+ /* wait for DMA of work item to complete */
	5439	+ if ((err = prot->d2h_edl_sync_cb(dhd, ring, msg)) != BCME_OK) {
	5440	+ DHD_ERROR(("%s: Error waiting for DMA to cmpl in EDL "
	5441	+ "ring; err = %d\n", __FUNCTION__, err));
	5442	+ }
	5443	+
	5444	+ /*
	5445	+ * Update the curr_rd to the current index in the ring, from where
	5446	+ * the work item is fetched. This way if the fetched work item
	5447	+ * fails in LIVELOCK, we can print the exact read index in the ring
	5448	+ * that shows up the corrupted work item.
	5449	+ */
	5450	+ if ((ring->curr_rd + 1) >= ring->max_items) {
	5451	+ ring->curr_rd = 0;
	5452	+ } else {
	5453	+ ring->curr_rd += 1;
	5454	+ }
	5455	+
	5456	+ if (err != BCME_OK) {
	5457	+ return 0;
	5458	+ }
	5459	+
	5460	+ /* process the edl work item, i.e, the event log */
	5461	+ err = dhd_event_logtrace_process_edl(dhd, msg_addr, evt_decode_data);
	5462	+
	5463	+ /* Dummy sleep so that scheduler kicks in after processing any logprints */
	5464	+ OSL_SLEEP(0);
	5465	+
	5466	+ /* Prefetch data to populate the cache */
	5467	+ OSL_PREFETCH(msg_addr + ring->item_len);
	5468	+
	5469	+ msg_addr += ring->item_len;
	5470	+ --n;
	5471	+ }
	5472	+
	5473	+ DHD_RING_LOCK(ring->ring_lock, flags);
	5474	+ /* update host ring read pointer */
	5475	+ if ((ring->rd + max_items_to_process) >= ring->max_items)
	5476	+ ring->rd = 0;
	5477	+ else
	5478	+ ring->rd += max_items_to_process;
	5479	+ DHD_RING_UNLOCK(ring->ring_lock, flags);
	5480	+
	5481	+ /* Now after processing max_items_to_process update dongle rd index.
	5482	+ * The TCM rd index is updated only if bus is not
	5483	+ * in D3. Else, the rd index is updated from resume
	5484	+ * context in - 'dhdpcie_bus_suspend'
	5485	+ */
	5486	+ DHD_GENERAL_LOCK(dhd, flags);
	5487	+ if (DHD_BUS_CHECK_SUSPEND_OR_ANY_SUSPEND_IN_PROGRESS(dhd)) {
	5488	+ DHD_INFO(("%s: bus is in suspend(%d) or suspending(0x%x) state!!\n",
	5489	+ __FUNCTION__, dhd->busstate, dhd->dhd_bus_busy_state));
	5490	+ DHD_GENERAL_UNLOCK(dhd, flags);
	5491	+ } else {
	5492	+ DHD_GENERAL_UNLOCK(dhd, flags);
	5493	+ DHD_EDL_RING_TCM_RD_UPDATE(dhd);
	5494	+ }
	5495	+
	5496	+ /* if num_items > bound, then anyway we will reschedule and
	5497	+ * this function runs again, so that if in between the DPC has
	5498	+ * updated the wr index, then the updated wr is read. But if
	5499	+ * num_items <= bound, and if DPC executes and updates the wr index
	5500	+ * when the above while loop is running, then the updated 'wr' index
	5501	+ * needs to be re-read from here, If we don't do so, then till
	5502	+ * the next time this function is scheduled
	5503	+ * the event logs will not be processed.
	5504	+ */
	5505	+ if (num_items <= DHD_EVENT_LOGTRACE_BOUND) {
	5506	+ /* read the updated wr index if reqd. and update num_items */
	5507	+ DHD_RING_LOCK(ring->ring_lock, flags);
	5508	+ if (wr != (volatile uint16)ring->wr) {
	5509	+ wr = (volatile uint16)ring->wr;
	5510	+ new_items = READ_AVAIL_SPACE(wr, ring->rd, depth);
	5511	+ DHD_INFO(("%s: new items [%u] avail in edl\n",
	5512	+ __FUNCTION__, new_items));
	5513	+ num_items += new_items;
	5514	+ }
	5515	+ DHD_RING_UNLOCK(ring->ring_lock, flags);
	5516	+ }
	5517	+
	5518	+ /* if # of items processed is less than num_items, need to re-schedule
	5519	+ * the deferred ctx
	5520	+ */
	5521	+ if (max_items_to_process < num_items) {
	5522	+ DHD_INFO(("%s: EDL bound hit / new items found, "
	5523	+ "items processed=%u; remaining=%u, "
	5524	+ "resched deferred ctx...\n",
	5525	+ __FUNCTION__, max_items_to_process,
	5526	+ num_items - max_items_to_process));
	5527	+ return (num_items - max_items_to_process);
	5528	+ }
	5529	+
	5530	+ return 0;
	5531	+
	5532	+}
	5533	+
	5534	+void
	5535	+dhd_prot_edl_ring_tcm_rd_update(dhd_pub_t *dhd)
	5536	+{
	5537	+ dhd_prot_t *prot = NULL;
	5538	+ unsigned long flags = 0;
	5539	+ msgbuf_ring_t *ring = NULL;
	5540	+
	5541	+ if (!dhd)
	5542	+ return;
	5543	+
	5544	+ prot = dhd->prot;
	5545	+ if (!prot \|\| !prot->d2hring_edl)
	5546	+ return;
	5547	+
	5548	+ ring = prot->d2hring_edl;
	5549	+ DHD_RING_LOCK(ring->ring_lock, flags);
	5550	+ dhd_prot_upd_read_idx(dhd, ring);
	5551	+ DHD_RING_UNLOCK(ring->ring_lock, flags);
	5552	+}
	5553	+#endif /* EWP_EDL */
	5554	+
	5555	+/* called when DHD needs to check for 'receive complete' messages from the dongle */
	5556	+bool BCMFASTPATH
	5557	+dhd_prot_process_msgbuf_rxcpl(dhd_pub_t *dhd, uint bound, int ringtype)
	5558	+{
	5559	+ bool more = FALSE;
	5560	+ uint n = 0;
	5561	+ dhd_prot_t *prot = dhd->prot;
	5562	+ msgbuf_ring_t *ring;
	5563	+ uint16 item_len;
	5564	+ host_rxbuf_cmpl_t *msg = NULL;
	5565	+ uint8 *msg_addr;
	5566	+ uint32 msg_len;
	5567	+ uint16 pkt_cnt, pkt_cnt_newidx;
	5568	+ unsigned long flags;
	5569	+ dmaaddr_t pa;
	5570	+ uint32 len;
	5571	+ void *dmah;
	5572	+ void *secdma;
	5573	+ int ifidx = 0, if_newidx = 0;
	5574	+ void pkt, pktqhead = NULL, prevpkt = NULL, pkt_newidx, *nextpkt;
	5575	+ uint32 pktid;
	5576	+ int i;
	5577	+ uint8 sync;
	5578	+ ts_timestamp_t *ts;
	5579	+
	5580	+ BCM_REFERENCE(ts);
	5581	+#ifdef DHD_HP2P
	5582	+ if (ringtype == DHD_HP2P_RING && prot->d2hring_hp2p_rxcpl)
	5583	+ ring = prot->d2hring_hp2p_rxcpl;
	5584	+ else
	5585	+#endif /* DHD_HP2P */
	5586	+ ring = &prot->d2hring_rx_cpln;
	5587	+ item_len = ring->item_len;
	5588	+ while (1) {
	5589	+ if (dhd_is_device_removed(dhd))
	5590	+ break;
	5591	+
	5592	+ if (dhd_query_bus_erros(dhd))
	5593	+ break;
	5594	+
	5595	+ if (dhd->hang_was_sent)
	5596	+ break;
	5597	+
	5598	+ if (dhd->smmu_fault_occurred) {
	5599	+ break;
	5600	+ }
	5601	+
	5602	+ pkt_cnt = 0;
	5603	+ pktqhead = pkt_newidx = NULL;
	5604	+ pkt_cnt_newidx = 0;
	5605	+
	5606	+ DHD_RING_LOCK(ring->ring_lock, flags);
	5607	+
	5608	+ /* Get the address of the next message to be read from ring */
	5609	+ msg_addr = dhd_prot_get_read_addr(dhd, ring, &msg_len);
	5610	+ if (msg_addr == NULL) {
	5611	+ DHD_RING_UNLOCK(ring->ring_lock, flags);
	5612	+ break;
	5613	+ }
	5614	+
	5615	+ while (msg_len > 0) {
	5616	+ msg = (host_rxbuf_cmpl_t *)msg_addr;
	5617	+
	5618	+ /* Wait until DMA completes, then fetch msg_type */
	5619	+ sync = prot->d2h_sync_cb(dhd, ring, &msg->cmn_hdr, item_len);
	5620	+ /*
	5621	+ * Update the curr_rd to the current index in the ring, from where
	5622	+ * the work item is fetched. This way if the fetched work item
	5623	+ * fails in LIVELOCK, we can print the exact read index in the ring
	5624	+ * that shows up the corrupted work item.
	5625	+ */
	5626	+ if ((ring->curr_rd + 1) >= ring->max_items) {
	5627	+ ring->curr_rd = 0;
	5628	+ } else {
	5629	+ ring->curr_rd += 1;
	5630	+ }
	5631	+
	5632	+ if (!sync) {
	5633	+ msg_len -= item_len;
	5634	+ msg_addr += item_len;
	5635	+ continue;
	5636	+ }
	5637	+
	5638	+ pktid = ltoh32(msg->cmn_hdr.request_id);
	5639	+
	5640	+#ifdef DHD_PKTID_AUDIT_RING
	5641	+ DHD_PKTID_AUDIT_RING_DEBUG(dhd, dhd->prot->pktid_rx_map, pktid,
	5642	+ DHD_DUPLICATE_FREE, msg, D2HRING_RXCMPLT_ITEMSIZE);
	5643	+#endif /* DHD_PKTID_AUDIT_RING */
	5644	+
	5645	+ pkt = DHD_PKTID_TO_NATIVE(dhd, prot->pktid_rx_map, pktid, pa,
	5646	+ len, dmah, secdma, PKTTYPE_DATA_RX);
	5647	+ if (!pkt) {
	5648	+ msg_len -= item_len;
	5649	+ msg_addr += item_len;
	5650	+ continue;
	5651	+ }
	5652	+
	5653	+ if (SECURE_DMA_ENAB(dhd->osh))
	5654	+ SECURE_DMA_UNMAP(dhd->osh, pa, (uint) len, DMA_RX, 0,
	5655	+ dmah, secdma, 0);
	5656	+ else
	5657	+ DMA_UNMAP(dhd->osh, pa, (uint) len, DMA_RX, 0, dmah);
	5658	+
	5659	+#ifdef DMAMAP_STATS
	5660	+ dhd->dma_stats.rxdata--;
	5661	+ dhd->dma_stats.rxdata_sz -= len;
	5662	+#endif /* DMAMAP_STATS */
	5663	+ DHD_INFO(("id 0x%04x, offset %d, len %d, idx %d, phase 0x%02x, "
	5664	+ "pktdata %p, metalen %d\n",
	5665	+ ltoh32(msg->cmn_hdr.request_id),
	5666	+ ltoh16(msg->data_offset),
	5667	+ ltoh16(msg->data_len), msg->cmn_hdr.if_id,
	5668	+ msg->cmn_hdr.flags, PKTDATA(dhd->osh, pkt),
	5669	+ ltoh16(msg->metadata_len)));
	5670	+
	5671	+ pkt_cnt++;
	5672	+ msg_len -= item_len;
	5673	+ msg_addr += item_len;
	5674	+
	5675	+#if DHD_DBG_SHOW_METADATA
	5676	+ if (prot->metadata_dbg && prot->rx_metadata_offset &&
	5677	+ msg->metadata_len) {
	5678	+ uchar *ptr;
	5679	+ ptr = PKTDATA(dhd->osh, pkt) - (prot->rx_metadata_offset);
	5680	+ /* header followed by data */
	5681	+ bcm_print_bytes("rxmetadata", ptr, msg->metadata_len);
	5682	+ dhd_prot_print_metadata(dhd, ptr, msg->metadata_len);
	5683	+ }
	5684	+#endif /* DHD_DBG_SHOW_METADATA */
	5685	+
	5686	+ /* data_offset from buf start */
	5687	+ if (ltoh16(msg->data_offset)) {
	5688	+ /* data offset given from dongle after split rx */
	5689	+ PKTPULL(dhd->osh, pkt, ltoh16(msg->data_offset));
	5690	+ }
	5691	+ else if (prot->rx_dataoffset) {
	5692	+ /* DMA RX offset updated through shared area */
	5693	+ PKTPULL(dhd->osh, pkt, prot->rx_dataoffset);
	5694	+ }
	5695	+ /* Actual length of the packet */
	5696	+ PKTSETLEN(dhd->osh, pkt, ltoh16(msg->data_len));
	5697	+
	5698	+#if defined(WL_MONITOR)
	5699	+ if (dhd_monitor_enabled(dhd, ifidx)) {
	5700	+ if (msg->flags & BCMPCIE_PKT_FLAGS_FRAME_802_11) {
	5701	+ dhd_rx_mon_pkt(dhd, msg, pkt, ifidx);
	5702	+ continue;
	5703	+ } else {
	5704	+ DHD_ERROR(("Received non 802.11 packet, "
	5705	+ "when monitor mode is enabled\n"));
	5706	+ }
	5707	+ }
	5708	+#endif /* WL_MONITOR */
	5709	+
	5710	+ if (msg->flags & BCMPCIE_PKT_FLAGS_NO_FORWARD) {
	5711	+ DHD_PKT_FLAGS_SET_NO_FWD(pkt);
	5712	+ }
	5713	+
	5714	+ if (!pktqhead) {
	5715	+ pktqhead = prevpkt = pkt;
	5716	+ ifidx = msg->cmn_hdr.if_id;
	5717	+ } else {
	5718	+ if (ifidx != msg->cmn_hdr.if_id) {
	5719	+ pkt_newidx = pkt;
	5720	+ if_newidx = msg->cmn_hdr.if_id;
	5721	+ pkt_cnt--;
	5722	+ pkt_cnt_newidx = 1;
	5723	+ break;
	5724	+ } else {
	5725	+ PKTSETNEXT(dhd->osh, prevpkt, pkt);
	5726	+ prevpkt = pkt;
	5727	+ }
	5728	+ }
	5729	+
	5730	+#ifdef DHD_HP2P
	5731	+ if (dhd->hp2p_capable && ring == prot->d2hring_hp2p_rxcpl) {
	5732	+#ifdef DHD_HP2P_DEBUG
	5733	+ bcm_print_bytes("Rxcpl", (uchar *)msg, sizeof(host_rxbuf_cmpl_t));
	5734	+#endif /* DHD_HP2P_DEBUG */
	5735	+ dhd_update_hp2p_rxstats(dhd, msg);
	5736	+ }
	5737	+#endif /* DHD_HP2P */
	5738	+
	5739	+#ifdef DHD_LBUF_AUDIT
	5740	+ PKTAUDIT(dhd->osh, pkt);
	5741	+#endif // endif
	5742	+ }
	5743	+
	5744	+ /* roll back read pointer for unprocessed message */
	5745	+ if (msg_len > 0) {
	5746	+ if (ring->rd < msg_len / item_len)
	5747	+ ring->rd = ring->max_items - msg_len / item_len;
	5748	+ else
	5749	+ ring->rd -= msg_len / item_len;
	5750	+ }
	5751	+
	5752	+ /* Update read pointer */
	5753	+ dhd_prot_upd_read_idx(dhd, ring);
	5754	+
	5755	+ DHD_RING_UNLOCK(ring->ring_lock, flags);
	5756	+
	5757	+ pkt = pktqhead;
	5758	+ for (i = 0; pkt && i < pkt_cnt; i++, pkt = nextpkt) {
	5759	+ nextpkt = PKTNEXT(dhd->osh, pkt);
	5760	+ PKTSETNEXT(dhd->osh, pkt, NULL);
	5761	+#ifdef DHD_LB_RXP
	5762	+ dhd_lb_rx_pkt_enqueue(dhd, pkt, ifidx);
	5763	+#elif defined(DHD_RX_CHAINING)
	5764	+ dhd_rxchain_frame(dhd, pkt, ifidx);
	5765	+#else
	5766	+ dhd_bus_rx_frame(dhd->bus, pkt, ifidx, 1);
	5767	+#endif /* DHD_LB_RXP */
	5768	+ }
	5769	+
	5770	+ if (pkt_newidx) {
	5771	+#ifdef DHD_LB_RXP
	5772	+ dhd_lb_rx_pkt_enqueue(dhd, pkt_newidx, if_newidx);
	5773	+#elif defined(DHD_RX_CHAINING)
	5774	+ dhd_rxchain_frame(dhd, pkt_newidx, if_newidx);
	5775	+#else
	5776	+ dhd_bus_rx_frame(dhd->bus, pkt_newidx, if_newidx, 1);
	5777	+#endif /* DHD_LB_RXP */
	5778	+ }
	5779	+
	5780	+ pkt_cnt += pkt_cnt_newidx;
	5781	+
	5782	+ /* Post another set of rxbufs to the device */
	5783	+ dhd_prot_return_rxbuf(dhd, 0, pkt_cnt);
	5784	+
	5785	+#ifdef DHD_RX_CHAINING
	5786	+ dhd_rxchain_commit(dhd);
	5787	+#endif // endif
	5788	+
	5789	+ /* After batch processing, check RX bound */
	5790	+ n += pkt_cnt;
	5791	+ if (n >= bound) {
	5792	+ more = TRUE;
	5793	+ break;
	5794	+ }
	5795	+ }
	5796	+
	5797	+ /* Call lb_dispatch only if packets are queued */
	5798	+ if (n &&
	5799	+#ifdef WL_MONITOR
	5800	+ !(dhd_monitor_enabled(dhd, ifidx)) &&
	5801	+#endif /* WL_MONITOR */
	5802	+ TRUE) {
	5803	+ DHD_LB_DISPATCH_RX_COMPL(dhd);
	5804	+ DHD_LB_DISPATCH_RX_PROCESS(dhd);
	5805	+ }
	5806	+
	5807	+ return more;
	5808	+
	5809	+}
	5810	+
3359	5811	/**
3360	5812	* Hands transmit packets (with a caller provided flow_id) over to dongle territory (the flow ring)
3361	5813	*/
..	..	@@ -3364,8 +5816,12 @@
3364	5816	{
3365	5817	msgbuf_ring_t ring = (msgbuf_ring_t )msgring;
3366	5818
	5819	+ if (ring == NULL) {
	5820	+ DHD_ERROR(("%s: NULL txflowring. exiting...\n", __FUNCTION__));
	5821	+ return;
	5822	+ }
3367	5823	/* Update read pointer */
3368		- if (DMA_INDX_ENAB(dhd->dma_d2h_ring_upd_support)) {
	5824	+ if (dhd->dma_d2h_ring_upd_support) {
3369	5825	ring->rd = dhd_prot_dma_indx_get(dhd, H2D_DMA_INDX_RD_UPD, ring->idx);
3370	5826	}
3371	5827
..	..	@@ -3378,24 +5834,45 @@
3378	5834
3379	5835	/** called when DHD needs to check for 'transmit complete' messages from the dongle */
3380	5836	bool BCMFASTPATH
3381		-dhd_prot_process_msgbuf_txcpl(dhd_pub_t *dhd, uint bound)
	5837	+dhd_prot_process_msgbuf_txcpl(dhd_pub_t *dhd, uint bound, int ringtype)
3382	5838	{
3383	5839	bool more = TRUE;
3384	5840	uint n = 0;
3385		- msgbuf_ring_t *ring = &dhd->prot->d2hring_tx_cpln;
	5841	+ msgbuf_ring_t *ring;
	5842	+ unsigned long flags;
	5843	+
	5844	+#ifdef DHD_HP2P
	5845	+ if (ringtype == DHD_HP2P_RING && dhd->prot->d2hring_hp2p_txcpl)
	5846	+ ring = dhd->prot->d2hring_hp2p_txcpl;
	5847	+ else
	5848	+#endif /* DHD_HP2P */
	5849	+ ring = &dhd->prot->d2hring_tx_cpln;
3386	5850
3387	5851	/* Process all the messages - DTOH direction */
3388	5852	while (!dhd_is_device_removed(dhd)) {
3389	5853	uint8 *msg_addr;
3390	5854	uint32 msg_len;
3391	5855
	5856	+ if (dhd_query_bus_erros(dhd)) {
	5857	+ more = FALSE;
	5858	+ break;
	5859	+ }
	5860	+
3392	5861	if (dhd->hang_was_sent) {
3393	5862	more = FALSE;
3394	5863	break;
3395	5864	}
3396	5865
	5866	+ if (dhd->smmu_fault_occurred) {
	5867	+ more = FALSE;
	5868	+ break;
	5869	+ }
	5870	+
	5871	+ DHD_RING_LOCK(ring->ring_lock, flags);
3397	5872	/* Get the address of the next message to be read from ring */
3398	5873	msg_addr = dhd_prot_get_read_addr(dhd, ring, &msg_len);
	5874	+ DHD_RING_UNLOCK(ring->ring_lock, flags);
	5875	+
3399	5876	if (msg_addr == NULL) {
3400	5877	more = FALSE;
3401	5878	break;
..	..	@@ -3419,7 +5896,44 @@
3419	5896	}
3420	5897	}
3421	5898
	5899	+ DHD_LB_DISPATCH_TX_COMPL(dhd);
	5900	+
3422	5901	return more;
	5902	+}
	5903	+
	5904	+int BCMFASTPATH
	5905	+dhd_prot_process_trapbuf(dhd_pub_t *dhd)
	5906	+{
	5907	+ uint32 data;
	5908	+ dhd_dma_buf_t *trap_addr = &dhd->prot->fw_trap_buf;
	5909	+
	5910	+ /* Interrupts can come in before this struct
	5911	+ * has been initialized.
	5912	+ */
	5913	+ if (trap_addr->va == NULL) {
	5914	+ DHD_ERROR(("%s: trap_addr->va is NULL\n", __FUNCTION__));
	5915	+ return 0;
	5916	+ }
	5917	+
	5918	+ OSL_CACHE_INV((void *)trap_addr->va, sizeof(uint32));
	5919	+ data = (uint32 )(trap_addr->va);
	5920	+
	5921	+ if (data & D2H_DEV_FWHALT) {
	5922	+ DHD_ERROR(("Firmware trapped and trap_data is 0x%04x\n", data));
	5923	+
	5924	+ if (data & D2H_DEV_EXT_TRAP_DATA)
	5925	+ {
	5926	+ if (dhd->extended_trap_data) {
	5927	+ OSL_CACHE_INV((void *)trap_addr->va,
	5928	+ BCMPCIE_EXT_TRAP_DATA_MAXLEN);
	5929	+ memcpy(dhd->extended_trap_data, (uint32 *)trap_addr->va,
	5930	+ BCMPCIE_EXT_TRAP_DATA_MAXLEN);
	5931	+ }
	5932	+ DHD_ERROR(("Extended trap data available\n"));
	5933	+ }
	5934	+ return data;
	5935	+ }
	5936	+ return 0;
3423	5937	}
3424	5938
3425	5939	/** called when DHD needs to check for 'ioctl complete' messages from the dongle */
..	..	@@ -3428,18 +5942,30 @@
3428	5942	{
3429	5943	dhd_prot_t *prot = dhd->prot;
3430	5944	msgbuf_ring_t *ring = &prot->d2hring_ctrl_cpln;
	5945	+ unsigned long flags;
3431	5946
3432	5947	/* Process all the messages - DTOH direction */
3433	5948	while (!dhd_is_device_removed(dhd)) {
3434	5949	uint8 *msg_addr;
3435	5950	uint32 msg_len;
3436	5951
	5952	+ if (dhd_query_bus_erros(dhd)) {
	5953	+ break;
	5954	+ }
	5955	+
3437	5956	if (dhd->hang_was_sent) {
3438	5957	break;
3439	5958	}
3440	5959
	5960	+ if (dhd->smmu_fault_occurred) {
	5961	+ break;
	5962	+ }
	5963	+
	5964	+ DHD_RING_LOCK(ring->ring_lock, flags);
3441	5965	/* Get the address of the next message to be read from ring */
3442	5966	msg_addr = dhd_prot_get_read_addr(dhd, ring, &msg_len);
	5967	+ DHD_RING_UNLOCK(ring->ring_lock, flags);
	5968	+
3443	5969	if (msg_addr == NULL) {
3444	5970	break;
3445	5971	}
..	..	@@ -3466,7 +5992,7 @@
3466	5992	static int BCMFASTPATH
3467	5993	dhd_prot_process_msgtype(dhd_pub_t dhd, msgbuf_ring_t ring, uint8 *buf, uint32 len)
3468	5994	{
3469		- int buf_len = len;
	5995	+ uint32 buf_len = len;
3470	5996	uint16 item_len;
3471	5997	uint8 msg_type;
3472	5998	cmn_msg_hdr_t *msg = NULL;
..	..	@@ -3475,7 +6001,7 @@
3475	6001	ASSERT(ring);
3476	6002	item_len = ring->item_len;
3477	6003	if (item_len == 0) {
3478		- DHD_ERROR(("%s: ringidx %d item_len %d buf_len %d\n",
	6004	+ DHD_ERROR(("%s: ringidx %d, item_len %d buf_len %d \n",
3479	6005	__FUNCTION__, ring->idx, item_len, buf_len));
3480	6006	return BCME_ERROR;
3481	6007	}
..	..	@@ -3486,14 +6012,27 @@
3486	6012	goto done;
3487	6013	}
3488	6014
	6015	+ if (dhd->smmu_fault_occurred) {
	6016	+ ret = BCME_ERROR;
	6017	+ goto done;
	6018	+ }
	6019	+
3489	6020	msg = (cmn_msg_hdr_t *)buf;
3490	6021
3491		-#if defined(PCIE_D2H_SYNC)
3492	6022	/* Wait until DMA completes, then fetch msg_type */
3493	6023	msg_type = dhd->prot->d2h_sync_cb(dhd, ring, msg, item_len);
3494		-#else
3495		- msg_type = msg->msg_type;
3496		-#endif /* !PCIE_D2H_SYNC */
	6024	+
	6025	+ /*
	6026	+ * Update the curr_rd to the current index in the ring, from where
	6027	+ * the work item is fetched. This way if the fetched work item
	6028	+ * fails in LIVELOCK, we can print the exact read index in the ring
	6029	+ * that shows up the corrupted work item.
	6030	+ */
	6031	+ if ((ring->curr_rd + 1) >= ring->max_items) {
	6032	+ ring->curr_rd = 0;
	6033	+ } else {
	6034	+ ring->curr_rd += 1;
	6035	+ }
3497	6036
3498	6037	/* Prefetch data to populate the cache */
3499	6038	OSL_PREFETCH(buf + item_len);
..	..	@@ -3508,12 +6047,24 @@
3508	6047
3509	6048	ASSERT(msg_type < DHD_PROT_FUNCS);
3510	6049	if (msg_type >= DHD_PROT_FUNCS) {
3511		- DHD_ERROR(("%s: msg_type %d item_len %d buf_len %d\n",
	6050	+ DHD_ERROR(("%s: msg_type %d, item_len %d buf_len %d\n",
3512	6051	__FUNCTION__, msg_type, item_len, buf_len));
3513	6052	ret = BCME_ERROR;
3514	6053	goto done;
3515	6054	}
3516	6055
	6056	+ if (msg_type == MSG_TYPE_INFO_BUF_CMPLT) {
	6057	+ if (ring == dhd->prot->d2hring_info_cpln) {
	6058	+ if (!dhd->prot->infobufpost) {
	6059	+ DHD_ERROR(("infobuf posted are zero,"
	6060	+ "but there is a completion\n"));
	6061	+ goto done;
	6062	+ }
	6063	+ dhd->prot->infobufpost--;
	6064	+ dhd_prot_infobufpost(dhd, dhd->prot->h2dring_info_subn);
	6065	+ dhd_prot_process_infobuf_complete(dhd, buf);
	6066	+ }
	6067	+ } else
3517	6068	if (table_lookup[msg_type]) {
3518	6069	table_lookup[msg_type](dhd, buf);
3519	6070	}
..	..	@@ -3530,10 +6081,8 @@
3530	6081
3531	6082	#ifdef DHD_RX_CHAINING
3532	6083	dhd_rxchain_commit(dhd);
3533		-#endif
3534		-#if defined(DHD_LB)
3535		- dhd_lb_dispatch(dhd, ring->idx);
3536		-#endif
	6084	+#endif // endif
	6085	+
3537	6086	return ret;
3538	6087	} /* dhd_prot_process_msgtype */
3539	6088
..	..	@@ -3547,10 +6096,76 @@
3547	6096	static void
3548	6097	dhd_prot_ringstatus_process(dhd_pub_t dhd, void msg)
3549	6098	{
3550		- pcie_ring_status_t ring_status = (pcie_ring_status_t )msg;
	6099	+ pcie_ring_status_t ring_status = (pcie_ring_status_t ) msg;
	6100	+ uint32 request_id = ltoh32(ring_status->cmn_hdr.request_id);
	6101	+ uint16 status = ltoh16(ring_status->compl_hdr.status);
	6102	+ uint16 ring_id = ltoh16(ring_status->compl_hdr.flow_ring_id);
	6103	+
3551	6104	DHD_ERROR(("ring status: request_id %d, status 0x%04x, flow ring %d, write_idx %d \n",
3552		- ring_status->cmn_hdr.request_id, ring_status->compl_hdr.status,
3553		- ring_status->compl_hdr.flow_ring_id, ring_status->write_idx));
	6105	+ request_id, status, ring_id, ltoh16(ring_status->write_idx)));
	6106	+
	6107	+ if (ltoh16(ring_status->compl_hdr.ring_id) != BCMPCIE_H2D_MSGRING_CONTROL_SUBMIT)
	6108	+ return;
	6109	+ if (status == BCMPCIE_BAD_PHASE) {
	6110	+ /* bad phase report from */
	6111	+ DHD_ERROR(("Bad phase\n"));
	6112	+ }
	6113	+ if (status != BCMPCIE_BADOPTION)
	6114	+ return;
	6115	+
	6116	+ if (request_id == DHD_H2D_DBGRING_REQ_PKTID) {
	6117	+ if (dhd->prot->h2dring_info_subn != NULL) {
	6118	+ if (dhd->prot->h2dring_info_subn->create_pending == TRUE) {
	6119	+ DHD_ERROR(("H2D ring create failed for info ring\n"));
	6120	+ dhd->prot->h2dring_info_subn->create_pending = FALSE;
	6121	+ }
	6122	+ else
	6123	+ DHD_ERROR(("ring create ID for a ring, create not pending\n"));
	6124	+ } else {
	6125	+ DHD_ERROR(("%s info submit ring doesn't exist\n", __FUNCTION__));
	6126	+ }
	6127	+ }
	6128	+ else if (request_id == DHD_D2H_DBGRING_REQ_PKTID) {
	6129	+ if (dhd->prot->d2hring_info_cpln != NULL) {
	6130	+ if (dhd->prot->d2hring_info_cpln->create_pending == TRUE) {
	6131	+ DHD_ERROR(("D2H ring create failed for info ring\n"));
	6132	+ dhd->prot->d2hring_info_cpln->create_pending = FALSE;
	6133	+ }
	6134	+ else
	6135	+ DHD_ERROR(("ring create ID for info ring, create not pending\n"));
	6136	+ } else {
	6137	+ DHD_ERROR(("%s info cpl ring doesn't exist\n", __FUNCTION__));
	6138	+ }
	6139	+ }
	6140	+#ifdef DHD_HP2P
	6141	+ else if (request_id == DHD_D2H_HPPRING_TXREQ_PKTID) {
	6142	+ if (dhd->prot->d2hring_hp2p_txcpl != NULL) {
	6143	+ if (dhd->prot->d2hring_hp2p_txcpl->create_pending == TRUE) {
	6144	+ DHD_ERROR(("H2D ring create failed for hp2p ring\n"));
	6145	+ dhd->prot->d2hring_hp2p_txcpl->create_pending = FALSE;
	6146	+ }
	6147	+ else
	6148	+ DHD_ERROR(("ring create ID for a ring, create not pending\n"));
	6149	+ } else {
	6150	+ DHD_ERROR(("%s hp2p txcmpl ring doesn't exist\n", __FUNCTION__));
	6151	+ }
	6152	+ }
	6153	+ else if (request_id == DHD_D2H_HPPRING_RXREQ_PKTID) {
	6154	+ if (dhd->prot->d2hring_hp2p_rxcpl != NULL) {
	6155	+ if (dhd->prot->d2hring_hp2p_rxcpl->create_pending == TRUE) {
	6156	+ DHD_ERROR(("D2H ring create failed for hp2p rxcmpl ring\n"));
	6157	+ dhd->prot->d2hring_hp2p_rxcpl->create_pending = FALSE;
	6158	+ }
	6159	+ else
	6160	+ DHD_ERROR(("ring create ID for hp2p rxcmpl ring, not pending\n"));
	6161	+ } else {
	6162	+ DHD_ERROR(("%s hp2p rxcpl ring doesn't exist\n", __FUNCTION__));
	6163	+ }
	6164	+ }
	6165	+#endif /* DHD_HP2P */
	6166	+ else {
	6167	+ DHD_ERROR(("don;t know how to pair with original request\n"));
	6168	+ }
3554	6169	/* How do we track this to pair it with ??? */
3555	6170	return;
3556	6171	}
..	..	@@ -3575,19 +6190,26 @@
3575	6190	static void
3576	6191	dhd_prot_ioctack_process(dhd_pub_t dhd, void msg)
3577	6192	{
3578		- uint32 pktid;
3579	6193	ioctl_req_ack_msg_t ioct_ack = (ioctl_req_ack_msg_t )msg;
3580	6194	unsigned long flags;
3581		-
3582		- pktid = ltoh32(ioct_ack->cmn_hdr.request_id);
	6195	+#if defined(DHD_PKTID_AUDIT_RING)
	6196	+ uint32 pktid = ltoh32(ioct_ack->cmn_hdr.request_id);
	6197	+#endif // endif
3583	6198
3584	6199	#if defined(DHD_PKTID_AUDIT_RING)
3585		- /* Skip DHD_IOCTL_REQ_PKTID = 0xFFFE */
	6200	+ /* Skip audit for ADHD_IOCTL_REQ_PKTID = 0xFFFE */
3586	6201	if (pktid != DHD_IOCTL_REQ_PKTID) {
3587		- DHD_PKTID_AUDIT(dhd, dhd->prot->pktid_map_handle, pktid,
3588		- DHD_TEST_IS_ALLOC);
	6202	+#ifndef IOCTLRESP_USE_CONSTMEM
	6203	+ DHD_PKTID_AUDIT_RING_DEBUG(dhd, dhd->prot->pktid_ctrl_map, pktid,
	6204	+ DHD_TEST_IS_ALLOC, msg, D2HRING_CTRL_CMPLT_ITEMSIZE);
	6205	+#else
	6206	+ DHD_PKTID_AUDIT_RING_DEBUG(dhd, dhd->prot->pktid_map_handle_ioctl, pktid,
	6207	+ DHD_TEST_IS_ALLOC, msg, D2HRING_CTRL_CMPLT_ITEMSIZE);
	6208	+#endif /* !IOCTLRESP_USE_CONSTMEM */
3589	6209	}
3590		-#endif /* DHD_PKTID_AUDIT_RING */
	6210	+#endif // endif
	6211	+
	6212	+ dhd->prot->ioctl_ack_time = OSL_LOCALTIME_NS();
3591	6213
3592	6214	DHD_GENERAL_LOCK(dhd, flags);
3593	6215	if ((dhd->prot->ioctl_state & MSGBUF_IOCTL_ACK_PENDING) &&
..	..	@@ -3596,6 +6218,8 @@
3596	6218	} else {
3597	6219	DHD_ERROR(("%s: received ioctl ACK with state %02x trans_id = %d\n",
3598	6220	__FUNCTION__, dhd->prot->ioctl_state, dhd->prot->ioctl_trans_id));
	6221	+ prhex("dhd_prot_ioctack_process:",
	6222	+ (uchar *)msg, D2HRING_CTRL_CMPLT_ITEMSIZE);
3599	6223	}
3600	6224	DHD_GENERAL_UNLOCK(dhd, flags);
3601	6225
..	..	@@ -3618,28 +6242,46 @@
3618	6242	unsigned long flags;
3619	6243	dhd_dma_buf_t retbuf;
3620	6244
	6245	+ /* Check for ioctl timeout induce flag, which is set by firing
	6246	+ * dhd iovar to induce IOCTL timeout. If flag is set,
	6247	+ * return from here, which results in to IOCTL timeout.
	6248	+ */
	6249	+ if (dhd->dhd_induce_error == DHD_INDUCE_IOCTL_TIMEOUT) {
	6250	+ DHD_ERROR(("%s: Inducing resumed on timeout\n", __FUNCTION__));
	6251	+ return;
	6252	+ }
	6253	+
3621	6254	memset(&retbuf, 0, sizeof(dhd_dma_buf_t));
3622	6255
3623	6256	pkt_id = ltoh32(ioct_resp->cmn_hdr.request_id);
3624	6257
3625	6258	#if defined(DHD_PKTID_AUDIT_RING)
3626	6259	#ifndef IOCTLRESP_USE_CONSTMEM
3627		- DHD_PKTID_AUDIT(dhd, prot->pktid_map_handle, pkt_id, DHD_DUPLICATE_FREE);
	6260	+ DHD_PKTID_AUDIT_RING_DEBUG(dhd, prot->pktid_ctrl_map, pkt_id,
	6261	+ DHD_DUPLICATE_FREE, msg, D2HRING_CTRL_CMPLT_ITEMSIZE);
3628	6262	#else
3629		- DHD_PKTID_AUDIT(dhd, prot->pktid_map_handle_ioctl, pkt_id, DHD_DUPLICATE_FREE);
	6263	+ DHD_PKTID_AUDIT_RING_DEBUG(dhd, prot->pktid_map_handle_ioctl, pkt_id,
	6264	+ DHD_DUPLICATE_FREE, msg, D2HRING_CTRL_CMPLT_ITEMSIZE);
3630	6265	#endif /* !IOCTLRESP_USE_CONSTMEM */
3631		-#endif /* DHD_PKTID_AUDIT_RING */
	6266	+#endif // endif
3632	6267
3633	6268	DHD_GENERAL_LOCK(dhd, flags);
3634	6269	if ((prot->ioctl_state & MSGBUF_IOCTL_ACK_PENDING) \|\|
3635	6270	!(prot->ioctl_state & MSGBUF_IOCTL_RESP_PENDING)) {
3636	6271	DHD_ERROR(("%s: received ioctl response with state %02x trans_id = %d\n",
3637	6272	__FUNCTION__, dhd->prot->ioctl_state, dhd->prot->ioctl_trans_id));
3638		- /* reset ioctl state */
3639		- prot->ioctl_state = 0;
	6273	+ prhex("dhd_prot_ioctcmplt_process:",
	6274	+ (uchar *)msg, D2HRING_CTRL_CMPLT_ITEMSIZE);
3640	6275	DHD_GENERAL_UNLOCK(dhd, flags);
3641	6276	return;
3642	6277	}
	6278	+
	6279	+ dhd->prot->ioctl_cmplt_time = OSL_LOCALTIME_NS();
	6280	+
	6281	+ /* Clear Response pending bit */
	6282	+ prot->ioctl_state &= ~MSGBUF_IOCTL_RESP_PENDING;
	6283	+ DHD_GENERAL_UNLOCK(dhd, flags);
	6284	+
3643	6285	#ifndef IOCTLRESP_USE_CONSTMEM
3644	6286	pkt = dhd_prot_packet_get(dhd, pkt_id, PKTTYPE_IOCTL_RX, TRUE);
3645	6287	#else
..	..	@@ -3647,21 +6289,34 @@
3647	6289	pkt = retbuf.va;
3648	6290	#endif /* !IOCTLRESP_USE_CONSTMEM */
3649	6291	if (!pkt) {
3650		- prot->ioctl_state = 0;
3651		- DHD_GENERAL_UNLOCK(dhd, flags);
3652	6292	DHD_ERROR(("%s: received ioctl response with NULL pkt\n", __FUNCTION__));
	6293	+ prhex("dhd_prot_ioctcmplt_process:",
	6294	+ (uchar *)msg, D2HRING_CTRL_CMPLT_ITEMSIZE);
3653	6295	return;
3654	6296	}
3655		- DHD_GENERAL_UNLOCK(dhd, flags);
3656	6297
3657	6298	prot->ioctl_resplen = ltoh16(ioct_resp->resp_len);
3658	6299	prot->ioctl_status = ltoh16(ioct_resp->compl_hdr.status);
3659	6300	xt_id = ltoh16(ioct_resp->trans_id);
3660		- if (xt_id != prot->ioctl_trans_id) {
	6301	+
	6302	+ if (xt_id != prot->ioctl_trans_id \|\| prot->curr_ioctl_cmd != ioct_resp->cmd) {
	6303	+ DHD_ERROR(("%s: transaction id(%d %d) or cmd(%d %d) mismatch\n",
	6304	+ __FUNCTION__, xt_id, prot->ioctl_trans_id,
	6305	+ prot->curr_ioctl_cmd, ioct_resp->cmd));
	6306	+ dhd_wakeup_ioctl_event(dhd, IOCTL_RETURN_ON_ERROR);
	6307	+ dhd_prot_debug_info_print(dhd);
	6308	+#ifdef DHD_FW_COREDUMP
	6309	+ if (dhd->memdump_enabled) {
	6310	+ /* collect core dump */
	6311	+ dhd->memdump_type = DUMP_TYPE_TRANS_ID_MISMATCH;
	6312	+ dhd_bus_mem_dump(dhd);
	6313	+ }
	6314	+#else
3661	6315	ASSERT(0);
	6316	+#endif /* DHD_FW_COREDUMP */
	6317	+ dhd_schedule_reset(dhd);
3662	6318	goto exit;
3663	6319	}
3664		-
3665	6320	DHD_CTL(("IOCTL_COMPLETE: req_id %x transid %d status %x resplen %d\n",
3666	6321	pkt_id, xt_id, prot->ioctl_status, prot->ioctl_resplen));
3667	6322
..	..	@@ -3683,6 +6338,31 @@
3683	6338	#else
3684	6339	free_ioctl_return_buffer(dhd, &retbuf);
3685	6340	#endif /* !IOCTLRESP_USE_CONSTMEM */
	6341	+
	6342	+ /* Post another ioctl buf to the device */
	6343	+ if (prot->cur_ioctlresp_bufs_posted > 0) {
	6344	+ prot->cur_ioctlresp_bufs_posted--;
	6345	+ }
	6346	+
	6347	+ dhd_msgbuf_rxbuf_post_ioctlresp_bufs(dhd);
	6348	+}
	6349	+
	6350	+int
	6351	+dhd_prot_check_tx_resource(dhd_pub_t *dhd)
	6352	+{
	6353	+ return dhd->prot->no_tx_resource;
	6354	+}
	6355	+
	6356	+void
	6357	+dhd_prot_update_pktid_txq_stop_cnt(dhd_pub_t *dhd)
	6358	+{
	6359	+ dhd->prot->pktid_txq_stop_cnt++;
	6360	+}
	6361	+
	6362	+void
	6363	+dhd_prot_update_pktid_txq_start_cnt(dhd_pub_t *dhd)
	6364	+{
	6365	+ dhd->prot->pktid_txq_start_cnt++;
3686	6366	}
3687	6367
3688	6368	/** called on MSG_TYPE_TX_STATUS message received from dongle */
..	..	@@ -3693,45 +6373,94 @@
3693	6373	host_txbuf_cmpl_t * txstatus;
3694	6374	unsigned long flags;
3695	6375	uint32 pktid;
3696		- void *pkt = NULL;
3697		- ulong pa;
	6376	+ void *pkt;
	6377	+ dmaaddr_t pa;
3698	6378	uint32 len;
3699	6379	void *dmah;
3700	6380	void *secdma;
	6381	+ bool pkt_fate;
	6382	+ msgbuf_ring_t *ring = &dhd->prot->d2hring_tx_cpln;
	6383	+#if defined(TX_STATUS_LATENCY_STATS) \|\| defined(DHD_HP2P)
	6384	+ flow_info_t *flow_info;
	6385	+ uint64 tx_status_latency;
	6386	+#endif /* TX_STATUS_LATENCY_STATS \|\| DHD_HP2P */
	6387	+#if defined(TX_STATUS_LATENCY_STATS)
	6388	+ flow_ring_node_t *flow_ring_node;
	6389	+ uint16 flowid;
	6390	+#endif // endif
	6391	+ ts_timestamp_t *ts;
	6392	+
	6393	+ BCM_REFERENCE(ts);
	6394	+ txstatus = (host_txbuf_cmpl_t *)msg;
	6395	+#if defined(TX_STATUS_LATENCY_STATS)
	6396	+ flowid = txstatus->compl_hdr.flow_ring_id;
	6397	+ flow_ring_node = DHD_FLOW_RING(dhd, flowid);
	6398	+#endif // endif
3701	6399
3702	6400	/* locks required to protect circular buffer accesses */
3703		- DHD_GENERAL_LOCK(dhd, flags);
3704		-
3705		- txstatus = (host_txbuf_cmpl_t *)msg;
	6401	+ DHD_RING_LOCK(ring->ring_lock, flags);
3706	6402	pktid = ltoh32(txstatus->cmn_hdr.request_id);
	6403	+ pkt_fate = TRUE;
3707	6404
3708	6405	#if defined(DHD_PKTID_AUDIT_RING)
3709		- DHD_PKTID_AUDIT(dhd, dhd->prot->pktid_map_handle, pktid,
3710		- DHD_DUPLICATE_FREE);
3711		-#endif /* DHD_PKTID_AUDIT_RING */
	6406	+ DHD_PKTID_AUDIT_RING_DEBUG(dhd, dhd->prot->pktid_tx_map, pktid,
	6407	+ DHD_DUPLICATE_FREE, msg, D2HRING_TXCMPLT_ITEMSIZE);
	6408	+#endif // endif
3712	6409
3713	6410	DHD_INFO(("txstatus for pktid 0x%04x\n", pktid));
3714		- if (prot->active_tx_count) {
3715		- prot->active_tx_count--;
3716		-
3717		- /* Release the Lock when no more tx packets are pending */
3718		- if (prot->active_tx_count == 0)
3719		- DHD_TXFL_WAKE_UNLOCK(dhd);
3720		-
3721		- } else {
	6411	+ if (OSL_ATOMIC_DEC_RETURN(dhd->osh, &prot->active_tx_count) < 0) {
3722	6412	DHD_ERROR(("Extra packets are freed\n"));
3723	6413	}
3724		-
3725	6414	ASSERT(pktid != 0);
3726	6415
	6416	+ pkt = DHD_PKTID_TO_NATIVE(dhd, dhd->prot->pktid_tx_map, pktid,
	6417	+ pa, len, dmah, secdma, PKTTYPE_DATA_TX);
	6418	+ if (!pkt) {
	6419	+ DHD_RING_UNLOCK(ring->ring_lock, flags);
	6420	+ DHD_ERROR(("%s: received txstatus with NULL pkt\n", __FUNCTION__));
	6421	+ prhex("dhd_prot_txstatus_process:", (uchar *)msg, D2HRING_TXCMPLT_ITEMSIZE);
	6422	+#ifdef DHD_FW_COREDUMP
	6423	+ if (dhd->memdump_enabled) {
	6424	+ /* collect core dump */
	6425	+ dhd->memdump_type = DUMP_TYPE_PKTID_INVALID;
	6426	+ dhd_bus_mem_dump(dhd);
	6427	+ }
	6428	+#else
	6429	+ ASSERT(0);
	6430	+#endif /* DHD_FW_COREDUMP */
	6431	+ return;
	6432	+ }
	6433	+
	6434	+ if (DHD_PKTID_AVAIL(dhd->prot->pktid_tx_map) == DHD_PKTID_MIN_AVAIL_COUNT) {
	6435	+ dhd->prot->no_tx_resource = FALSE;
	6436	+ dhd_bus_start_queue(dhd->bus);
	6437	+ }
	6438	+
	6439	+ if (SECURE_DMA_ENAB(dhd->osh)) {
	6440	+ int offset = 0;
	6441	+ BCM_REFERENCE(offset);
	6442	+
	6443	+ if (dhd->prot->tx_metadata_offset)
	6444	+ offset = dhd->prot->tx_metadata_offset + ETHER_HDR_LEN;
	6445	+ SECURE_DMA_UNMAP(dhd->osh, (uint) pa,
	6446	+ (uint) dhd->prot->tx_metadata_offset, DMA_RX, 0, dmah,
	6447	+ secdma, offset);
	6448	+ } else {
	6449	+ DMA_UNMAP(dhd->osh, pa, (uint) len, DMA_RX, 0, dmah);
	6450	+ }
	6451	+
	6452	+#ifdef TX_STATUS_LATENCY_STATS
	6453	+ /* update the tx status latency for flowid */
	6454	+ flow_info = &flow_ring_node->flow_info;
	6455	+ tx_status_latency = OSL_SYSUPTIME_US() - DHD_PKT_GET_QTIME(pkt);
	6456	+ flow_info->cum_tx_status_latency += tx_status_latency;
	6457	+ flow_info->num_tx_status++;
	6458	+#endif /* TX_STATUS_LATENCY_STATS */
3727	6459	#if defined(DHD_LB_TXC) && !defined(BCM_SECURE_DMA)
3728	6460	{
3729	6461	int elem_ix;
3730	6462	void **elem;
3731	6463	bcm_workq_t *workq;
3732		-
3733		- pkt = DHD_PKTID_TO_NATIVE(dhd, dhd->prot->pktid_map_handle,
3734		- pktid, pa, len, dmah, secdma, PKTTYPE_DATA_TX);
3735	6464
3736	6465	workq = &prot->tx_compl_prod;
3737	6466	/*
..	..	@@ -3763,66 +6492,72 @@
3763	6492	}
3764	6493
3765	6494	DHD_INFO(("%s: tx_compl_prod pkt<%p> sync<%d>\n",
3766		- __FUNCTION__, pkt, prot->tx_compl_prod_sync));
	6495	+ __FUNCTION__, pkt, prot->tx_compl_prod_sync));
3767	6496
3768		- DHD_GENERAL_UNLOCK(dhd, flags);
	6497	+ DHD_RING_UNLOCK(ring->ring_lock, flags);
3769	6498	return;
3770		- }
	6499	+ }
3771	6500
3772	6501	workq_ring_full:
3773	6502
3774	6503	#endif /* !DHD_LB_TXC */
3775	6504
3776		- /*
3777		- * We can come here if no DHD_LB_TXC is enabled and in case where DHD_LB_TXC is
3778		- * defined but the tx_compl queue is full.
3779		- */
3780		- if (pkt == NULL) {
3781		- pkt = DHD_PKTID_TO_NATIVE(dhd, dhd->prot->pktid_map_handle,
3782		- pktid, pa, len, dmah, secdma, PKTTYPE_DATA_TX);
	6505	+#ifdef DMAMAP_STATS
	6506	+ dhd->dma_stats.txdata--;
	6507	+ dhd->dma_stats.txdata_sz -= len;
	6508	+#endif /* DMAMAP_STATS */
	6509	+ pkt_fate = dhd_dbg_process_tx_status(dhd, pkt, pktid,
	6510	+ ltoh16(txstatus->compl_hdr.status) & WLFC_CTL_PKTFLAG_MASK);
	6511	+#ifdef DHD_PKT_LOGGING
	6512	+ if (dhd->d11_tx_status) {
	6513	+ uint16 status = ltoh16(txstatus->compl_hdr.status) &
	6514	+ WLFC_CTL_PKTFLAG_MASK;
	6515	+ uint32 pkthash = __dhd_dbg_pkt_hash((uintptr_t)pkt, pktid);
	6516	+ DHD_PKTLOG_TXS(dhd, pkt, pktid, status);
	6517	+ dhd_dump_pkt(dhd, ltoh32(txstatus->cmn_hdr.if_id),
	6518	+ (uint8 *)PKTDATA(dhd->osh, pkt), len, TRUE,
	6519	+ &pkthash, &status);
3783	6520	}
	6521	+#endif /* DHD_PKT_LOGGING */
3784	6522
3785		- if (pkt) {
3786		- if (SECURE_DMA_ENAB(dhd->osh)) {
3787		- int offset = 0;
3788		- BCM_REFERENCE(offset);
3789		-
3790		- if (dhd->prot->tx_metadata_offset)
3791		- offset = dhd->prot->tx_metadata_offset + ETHER_HDR_LEN;
3792		- SECURE_DMA_UNMAP(dhd->osh, (uint) pa,
3793		- (uint) dhd->prot->tx_metadata_offset, DMA_RX, 0, dmah,
3794		- secdma, offset);
3795		- } else {
3796		- DMA_UNMAP(dhd->osh, pa, (uint) len, DMA_RX, pkt, dmah);
3797		- }
3798	6523	#if defined(BCMPCIE)
3799		- dhd_txcomplete(dhd, pkt, true);
3800		-#endif
	6524	+ dhd_txcomplete(dhd, pkt, pkt_fate);
	6525	+#ifdef DHD_4WAYM4_FAIL_DISCONNECT
	6526	+ dhd_eap_txcomplete(dhd, pkt, pkt_fate, txstatus->cmn_hdr.if_id);
	6527	+#endif /* DHD_4WAYM4_FAIL_DISCONNECT */
	6528	+#endif // endif
3801	6529
3802	6530	#if DHD_DBG_SHOW_METADATA
3803		- if (dhd->prot->metadata_dbg &&
3804		- dhd->prot->tx_metadata_offset && txstatus->metadata_len) {
3805		- uchar *ptr;
3806		- /* The Ethernet header of TX frame was copied and removed.
3807		- * Here, move the data pointer forward by Ethernet header size.
3808		- */
3809		- PKTPULL(dhd->osh, pkt, ETHER_HDR_LEN);
3810		- ptr = PKTDATA(dhd->osh, pkt) - (dhd->prot->tx_metadata_offset);
3811		- bcm_print_bytes("txmetadata", ptr, txstatus->metadata_len);
3812		- dhd_prot_print_metadata(dhd, ptr, txstatus->metadata_len);
3813		- }
3814		-#endif /* DHD_DBG_SHOW_METADATA */
3815		-#ifndef CUSTOMER_HW_31_2
3816		- DHD_GENERAL_UNLOCK(dhd, flags);
3817		- PKTFREE(dhd->osh, pkt, TRUE);
3818		- DHD_GENERAL_LOCK(dhd, flags);
3819		-#endif /* CUSTOMER_HW_31_2 */
3820		- DHD_FLOWRING_TXSTATUS_CNT_UPDATE(dhd->bus, txstatus->compl_hdr.flow_ring_id,
3821		- txstatus->tx_status);
	6531	+ if (dhd->prot->metadata_dbg &&
	6532	+ dhd->prot->tx_metadata_offset && txstatus->metadata_len) {
	6533	+ uchar *ptr;
	6534	+ /* The Ethernet header of TX frame was copied and removed.
	6535	+ * Here, move the data pointer forward by Ethernet header size.
	6536	+ */
	6537	+ PKTPULL(dhd->osh, pkt, ETHER_HDR_LEN);
	6538	+ ptr = PKTDATA(dhd->osh, pkt) - (dhd->prot->tx_metadata_offset);
	6539	+ bcm_print_bytes("txmetadata", ptr, txstatus->metadata_len);
	6540	+ dhd_prot_print_metadata(dhd, ptr, txstatus->metadata_len);
3822	6541	}
	6542	+#endif /* DHD_DBG_SHOW_METADATA */
3823	6543
3824		- DHD_GENERAL_UNLOCK(dhd, flags);
	6544	+#ifdef DHD_HP2P
	6545	+ if (dhd->hp2p_capable && flow_ring_node->flow_info.tid == HP2P_PRIO) {
	6546	+#ifdef DHD_HP2P_DEBUG
	6547	+ bcm_print_bytes("txcpl", (uint8 *)txstatus, sizeof(host_txbuf_cmpl_t));
	6548	+#endif /* DHD_HP2P_DEBUG */
	6549	+ dhd_update_hp2p_txstats(dhd, txstatus);
	6550	+ }
	6551	+#endif /* DHD_HP2P */
3825	6552
	6553	+#ifdef DHD_LBUF_AUDIT
	6554	+ PKTAUDIT(dhd->osh, pkt);
	6555	+#endif // endif
	6556	+
	6557	+ DHD_FLOWRING_TXSTATUS_CNT_UPDATE(dhd->bus, txstatus->compl_hdr.flow_ring_id,
	6558	+ txstatus->tx_status);
	6559	+ DHD_RING_UNLOCK(ring->ring_lock, flags);
	6560	+ PKTFREE(dhd->osh, pkt, TRUE);
3826	6561	return;
3827	6562	} /* dhd_prot_txstatus_process */
3828	6563
..	..	@@ -3835,7 +6570,6 @@
3835	6570	uint16 buflen;
3836	6571	int ifidx = 0;
3837	6572	void* pkt;
3838		- unsigned long flags;
3839	6573	dhd_prot_t *prot = dhd->prot;
3840	6574
3841	6575	/* Event complete header */
..	..	@@ -3843,134 +6577,91 @@
3843	6577	bufid = ltoh32(evnt->cmn_hdr.request_id);
3844	6578
3845	6579	#if defined(DHD_PKTID_AUDIT_RING)
3846		- DHD_PKTID_AUDIT(dhd, dhd->prot->pktid_map_handle, bufid,
3847		- DHD_DUPLICATE_FREE);
3848		-#endif /* DHD_PKTID_AUDIT_RING */
	6580	+ DHD_PKTID_AUDIT_RING_DEBUG(dhd, dhd->prot->pktid_ctrl_map, bufid,
	6581	+ DHD_DUPLICATE_FREE, msg, D2HRING_CTRL_CMPLT_ITEMSIZE);
	6582	+#endif // endif
3849	6583
3850	6584	buflen = ltoh16(evnt->event_data_len);
3851	6585
3852	6586	ifidx = BCMMSGBUF_API_IFIDX(&evnt->cmn_hdr);
3853	6587
3854	6588	/* Post another rxbuf to the device */
3855		- if (prot->cur_event_bufs_posted) {
	6589	+ if (prot->cur_event_bufs_posted)
3856	6590	prot->cur_event_bufs_posted--;
3857		- }
3858	6591	dhd_msgbuf_rxbuf_post_event_bufs(dhd);
3859	6592
3860		- /* locks required to protect pktid_map */
3861		- DHD_GENERAL_LOCK(dhd, flags);
3862	6593	pkt = dhd_prot_packet_get(dhd, bufid, PKTTYPE_EVENT_RX, TRUE);
3863		- DHD_GENERAL_UNLOCK(dhd, flags);
3864	6594
3865	6595	if (!pkt) {
	6596	+ DHD_ERROR(("%s: pkt is NULL for pktid %d\n", __FUNCTION__, bufid));
3866	6597	return;
3867	6598	}
3868	6599
3869	6600	/* DMA RX offset updated through shared area */
3870		- if (dhd->prot->rx_dataoffset) {
	6601	+ if (dhd->prot->rx_dataoffset)
3871	6602	PKTPULL(dhd->osh, pkt, dhd->prot->rx_dataoffset);
3872		- }
3873	6603
3874	6604	PKTSETLEN(dhd->osh, pkt, buflen);
3875		-
	6605	+#ifdef DHD_LBUF_AUDIT
	6606	+ PKTAUDIT(dhd->osh, pkt);
	6607	+#endif // endif
3876	6608	dhd_bus_rx_frame(dhd->bus, pkt, ifidx, 1);
3877	6609	}
3878	6610
3879		-extern bool dhd_monitor_enabled(dhd_pub_t *dhd, int ifidx);
3880		-extern void dhd_rx_mon_pkt(dhd_pub_t dhdp, void pkt, int ifidx);
3881		-
3882		-/** called on MSG_TYPE_RX_CMPLT message received from dongle */
	6611	+/** called on MSG_TYPE_INFO_BUF_CMPLT message received from dongle */
3883	6612	static void BCMFASTPATH
3884		-dhd_prot_rxcmplt_process(dhd_pub_t dhd, void msg)
	6613	+dhd_prot_process_infobuf_complete(dhd_pub_t dhd, void buf)
3885	6614	{
3886		- host_rxbuf_cmpl_t *rxcmplt_h;
3887		- uint16 data_offset; /* offset at which data starts */
3888		- void *pkt;
3889		- unsigned long flags;
3890		- uint ifidx;
	6615	+ info_buf_resp_t *resp;
3891	6616	uint32 pktid;
3892		-#if defined(DHD_LB_RXC)
3893		- const bool free_pktid = FALSE;
3894		-#else
3895		- const bool free_pktid = TRUE;
3896		-#endif /* DHD_LB_RXC */
	6617	+ uint16 buflen;
	6618	+ void * pkt;
3897	6619
3898		- /* RXCMPLT HDR */
3899		- rxcmplt_h = (host_rxbuf_cmpl_t *)msg;
	6620	+ resp = (info_buf_resp_t *)buf;
	6621	+ pktid = ltoh32(resp->cmn_hdr.request_id);
	6622	+ buflen = ltoh16(resp->info_data_len);
3900	6623
3901		- /* offset from which data starts is populated in rxstatus0 */
3902		- data_offset = ltoh16(rxcmplt_h->data_offset);
3903		-
3904		- pktid = ltoh32(rxcmplt_h->cmn_hdr.request_id);
3905		-
3906		-#if defined(DHD_PKTID_AUDIT_RING)
3907		- DHD_PKTID_AUDIT(dhd, dhd->prot->pktid_map_handle, pktid,
3908		- DHD_DUPLICATE_FREE);
	6624	+#ifdef DHD_PKTID_AUDIT_RING
	6625	+ DHD_PKTID_AUDIT_RING_DEBUG(dhd, dhd->prot->pktid_ctrl_map, pktid,
	6626	+ DHD_DUPLICATE_FREE, buf, D2HRING_INFO_BUFCMPLT_ITEMSIZE);
3909	6627	#endif /* DHD_PKTID_AUDIT_RING */
3910	6628
3911		- DHD_GENERAL_LOCK(dhd, flags);
3912		- pkt = dhd_prot_packet_get(dhd, pktid, PKTTYPE_DATA_RX, free_pktid);
3913		- DHD_GENERAL_UNLOCK(dhd, flags);
	6629	+ DHD_INFO(("id 0x%04x, len %d, phase 0x%02x, seqnum %d, rx_dataoffset %d\n",
	6630	+ pktid, buflen, resp->cmn_hdr.flags, ltoh16(resp->seqnum),
	6631	+ dhd->prot->rx_dataoffset));
3914	6632
3915		- if (!pkt) {
3916		- return;
3917		- }
3918		-
3919		- /* Post another set of rxbufs to the device */
3920		- dhd_prot_return_rxbuf(dhd, pktid, 1);
3921		-
3922		- DHD_INFO(("id 0x%04x, offset %d, len %d, idx %d, phase 0x%02x, pktdata %p, metalen %d\n",
3923		- ltoh32(rxcmplt_h->cmn_hdr.request_id), data_offset, ltoh16(rxcmplt_h->data_len),
3924		- rxcmplt_h->cmn_hdr.if_id, rxcmplt_h->cmn_hdr.flags, PKTDATA(dhd->osh, pkt),
3925		- ltoh16(rxcmplt_h->metadata_len)));
3926		-#if DHD_DBG_SHOW_METADATA
3927		- if (dhd->prot->metadata_dbg &&
3928		- dhd->prot->rx_metadata_offset && rxcmplt_h->metadata_len) {
3929		- uchar *ptr;
3930		- ptr = PKTDATA(dhd->osh, pkt) - (dhd->prot->rx_metadata_offset);
3931		- /* header followed by data */
3932		- bcm_print_bytes("rxmetadata", ptr, rxcmplt_h->metadata_len);
3933		- dhd_prot_print_metadata(dhd, ptr, rxcmplt_h->metadata_len);
3934		- }
3935		-#endif /* DHD_DBG_SHOW_METADATA */
3936		-
3937		- if (rxcmplt_h->flags & BCMPCIE_PKT_FLAGS_FRAME_802_11) {
3938		- DHD_INFO(("D11 frame rxed \n"));
3939		- }
3940		-
3941		- /* data_offset from buf start */
3942		- if (data_offset) {
3943		- /* data offset given from dongle after split rx */
3944		- PKTPULL(dhd->osh, pkt, data_offset); /* data offset */
3945		- } else {
3946		- /* DMA RX offset updated through shared area */
3947		- if (dhd->prot->rx_dataoffset) {
3948		- PKTPULL(dhd->osh, pkt, dhd->prot->rx_dataoffset);
	6633	+ if (dhd->debug_buf_dest_support) {
	6634	+ if (resp->dest < DEBUG_BUF_DEST_MAX) {
	6635	+ dhd->debug_buf_dest_stat[resp->dest]++;
3949	6636	}
3950	6637	}
3951		- /* Actual length of the packet */
3952		- PKTSETLEN(dhd->osh, pkt, ltoh16(rxcmplt_h->data_len));
3953	6638
3954		- ifidx = rxcmplt_h->cmn_hdr.if_id;
3955		-
3956		- if (dhd_monitor_enabled(dhd, ifidx) &&
3957		- (rxcmplt_h->flags & BCMPCIE_PKT_FLAGS_FRAME_802_11)) {
3958		- dhd_rx_mon_pkt(dhd, pkt, ifidx);
	6639	+ pkt = dhd_prot_packet_get(dhd, pktid, PKTTYPE_INFO_RX, TRUE);
	6640	+ if (!pkt)
3959	6641	return;
3960		- }
3961	6642
3962		-#if defined(DHD_LB_RXP)
3963		- dhd_lb_rx_pkt_enqueue(dhd, pkt, ifidx);
3964		-#else /* ! DHD_LB_RXP */
3965		-#ifdef DHD_RX_CHAINING
3966		- /* Chain the packets */
3967		- dhd_rxchain_frame(dhd, pkt, ifidx);
3968		-#else /* ! DHD_RX_CHAINING */
3969		- /* offset from which data starts is populated in rxstatus0 */
3970		- dhd_bus_rx_frame(dhd->bus, pkt, ifidx, 1);
3971		-#endif /* ! DHD_RX_CHAINING */
3972		-#endif /* ! DHD_LB_RXP */
3973		-} /* dhd_prot_rxcmplt_process */
	6643	+ /* DMA RX offset updated through shared area */
	6644	+ if (dhd->prot->rx_dataoffset)
	6645	+ PKTPULL(dhd->osh, pkt, dhd->prot->rx_dataoffset);
	6646	+
	6647	+ PKTSETLEN(dhd->osh, pkt, buflen);
	6648	+
	6649	+#ifdef DHD_LBUF_AUDIT
	6650	+ PKTAUDIT(dhd->osh, pkt);
	6651	+#endif // endif
	6652	+
	6653	+ /* info ring "debug" data, which is not a 802.3 frame, is sent/hacked with a
	6654	+ * special ifidx of -1. This is just internal to dhd to get the data to
	6655	+ * dhd_linux.c:dhd_rx_frame() from here (dhd_prot_infobuf_cmplt_process).
	6656	+ */
	6657	+ dhd_bus_rx_frame(dhd->bus, pkt, DHD_DUMMY_INFO_IF /* ifidx HACK */, 1);
	6658	+}
	6659	+
	6660	+/** called on MSG_TYPE_SNAPSHOT_CMPLT message received from dongle */
	6661	+static void BCMFASTPATH
	6662	+dhd_prot_process_snapshot_complete(dhd_pub_t dhd, void buf)
	6663	+{
	6664	+}
3974	6665
3975	6666	/** Stop protocol: sync w/dongle state. */
3976	6667	void dhd_prot_stop(dhd_pub_t *dhd)
..	..	@@ -3995,6 +6686,7 @@
3995	6686	return 0;
3996	6687	}
3997	6688
	6689	+#define MAX_MTU_SZ (1600u)
3998	6690
3999	6691	#define PKTBUF pktbuf
4000	6692
..	..	@@ -4019,26 +6711,41 @@
4019	6711	msgbuf_ring_t *ring;
4020	6712	flow_ring_table_t *flow_ring_table;
4021	6713	flow_ring_node_t *flow_ring_node;
	6714	+#ifdef DHD_PKT_LOGGING
	6715	+ uint32 pkthash;
	6716	+#endif /* DHD_PKT_LOGGING */
4022	6717
4023	6718	if (dhd->flow_ring_table == NULL) {
	6719	+ DHD_ERROR(("dhd flow_ring_table is NULL\n"));
4024	6720	return BCME_NORESOURCE;
4025	6721	}
	6722	+#ifdef DHD_PCIE_PKTID
	6723	+ if (!DHD_PKTID_AVAIL(dhd->prot->pktid_tx_map)) {
	6724	+ if (dhd->prot->pktid_depleted_cnt == DHD_PKTID_DEPLETED_MAX_COUNT) {
	6725	+ dhd_bus_stop_queue(dhd->bus);
	6726	+ dhd->prot->no_tx_resource = TRUE;
	6727	+ }
	6728	+ dhd->prot->pktid_depleted_cnt++;
	6729	+ goto err_no_res;
	6730	+ } else {
	6731	+ dhd->prot->pktid_depleted_cnt = 0;
	6732	+ }
	6733	+#endif /* DHD_PCIE_PKTID */
4026	6734
4027	6735	flowid = DHD_PKT_GET_FLOWID(PKTBUF);
4028		-
4029	6736	flow_ring_table = (flow_ring_table_t *)dhd->flow_ring_table;
4030	6737	flow_ring_node = (flow_ring_node_t *)&flow_ring_table[flowid];
4031	6738
4032	6739	ring = (msgbuf_ring_t *)flow_ring_node->prot_info;
4033	6740
4034		-
4035		- DHD_GENERAL_LOCK(dhd, flags);
	6741	+ DHD_RING_LOCK(ring->ring_lock, flags);
4036	6742
4037	6743	/* Create a unique 32-bit packet id */
4038		- pktid = DHD_NATIVE_TO_PKTID_RSV(dhd, dhd->prot->pktid_map_handle, PKTBUF);
	6744	+ pktid = DHD_NATIVE_TO_PKTID_RSV(dhd, dhd->prot->pktid_tx_map,
	6745	+ PKTBUF, PKTTYPE_DATA_TX);
4039	6746	#if defined(DHD_PCIE_PKTID)
4040	6747	if (pktid == DHD_PKTID_INVALID) {
4041		- DHD_ERROR(("Pktid pool depleted.\n"));
	6748	+ DHD_ERROR_RLMT(("%s: Pktid pool depleted.\n", __FUNCTION__));
4042	6749	/*
4043	6750	* If we return error here, the caller would queue the packet
4044	6751	* again. So we'll just free the skb allocated in DMA Zone.
..	..	@@ -4053,21 +6760,22 @@
4053	6760	txdesc = (host_txbuf_post_t *)
4054	6761	dhd_prot_alloc_ring_space(dhd, ring, 1, &alloced, FALSE);
4055	6762	if (txdesc == NULL) {
4056		-#if defined(DHD_PCIE_PKTID)
4057		- void *dmah;
4058		- void *secdma;
4059		- /* Free up the PKTID. physaddr and pktlen will be garbage. */
4060		- DHD_PKTID_TO_NATIVE(dhd, dhd->prot->pktid_map_handle, pktid,
4061		- pa, pktlen, dmah, secdma, PKTTYPE_NO_CHECK);
4062		-#endif /* DHD_PCIE_PKTID */
4063	6763	DHD_INFO(("%s:%d: HTOD Msgbuf Not available TxCount = %d\n",
4064		- __FUNCTION__, __LINE__, prot->active_tx_count));
4065		- goto err_no_res_pktfree;
	6764	+ __FUNCTION__, __LINE__, OSL_ATOMIC_READ(dhd->osh, &prot->active_tx_count)));
	6765	+ goto err_free_pktid;
4066	6766	}
4067	6767
4068	6768	/* Extract the data pointer and length information */
4069	6769	pktdata = PKTDATA(dhd->osh, PKTBUF);
4070	6770	pktlen = PKTLEN(dhd->osh, PKTBUF);
	6771	+
	6772	+ DHD_DBG_PKT_MON_TX(dhd, PKTBUF, pktid);
	6773	+#ifdef DHD_PKT_LOGGING
	6774	+ DHD_PKTLOG_TX(dhd, PKTBUF, pktid);
	6775	+ /* Dump TX packet */
	6776	+ pkthash = __dhd_dbg_pkt_hash((uintptr_t)PKTBUF, pktid);
	6777	+ dhd_dump_pkt(dhd, ifidx, pktdata, pktlen, TRUE, &pkthash, NULL);
	6778	+#endif /* DHD_PKT_LOGGING */
4071	6779
4072	6780	/* Ethernet header: Copy before we cache flush packet using DMA_MAP */
4073	6781	bcopy(pktdata, txdesc->txhdr, ETHER_HDR_LEN);
..	..	@@ -4081,9 +6789,8 @@
4081	6789	int offset = 0;
4082	6790	BCM_REFERENCE(offset);
4083	6791
4084		- if (prot->tx_metadata_offset) {
	6792	+ if (prot->tx_metadata_offset)
4085	6793	offset = prot->tx_metadata_offset + ETHER_HDR_LEN;
4086		- }
4087	6794
4088	6795	pa = SECURE_DMA_MAP(dhd->osh, PKTDATA(dhd->osh, PKTBUF), pktlen,
4089	6796	DMA_TX, PKTBUF, 0, ring->dma_buf.secdma, offset);
..	..	@@ -4093,31 +6800,36 @@
4093	6800	pa = DMA_MAP(dhd->osh, PKTDATA(dhd->osh, PKTBUF), pktlen, DMA_TX, PKTBUF, 0);
4094	6801	#endif /* #ifndef BCM_SECURE_DMA */
4095	6802
4096		- if ((PHYSADDRHI(pa) == 0) && (PHYSADDRLO(pa) == 0)) {
4097		- DHD_ERROR(("Something really bad, unless 0 is a valid phyaddr\n"));
	6803	+ if (PHYSADDRISZERO(pa)) {
	6804	+ DHD_ERROR(("%s: Something really bad, unless 0 is "
	6805	+ "a valid phyaddr for pa\n", __FUNCTION__));
4098	6806	ASSERT(0);
	6807	+ goto err_rollback_idx;
4099	6808	}
4100	6809
	6810	+#ifdef DMAMAP_STATS
	6811	+ dhd->dma_stats.txdata++;
	6812	+ dhd->dma_stats.txdata_sz += pktlen;
	6813	+#endif /* DMAMAP_STATS */
4101	6814	/* No need to lock. Save the rest of the packet's metadata */
4102		- DHD_NATIVE_TO_PKTID_SAVE(dhd, dhd->prot->pktid_map_handle, PKTBUF, pktid,
	6815	+ DHD_NATIVE_TO_PKTID_SAVE(dhd, dhd->prot->pktid_tx_map, PKTBUF, pktid,
4103	6816	pa, pktlen, DMA_TX, NULL, ring->dma_buf.secdma, PKTTYPE_DATA_TX);
4104	6817
4105	6818	#ifdef TXP_FLUSH_NITEMS
4106		- if (ring->pend_items_count == 0) {
	6819	+ if (ring->pend_items_count == 0)
4107	6820	ring->start_addr = (void *)txdesc;
4108		- }
4109	6821	ring->pend_items_count++;
4110		-#endif
	6822	+#endif // endif
4111	6823
4112	6824	/* Form the Tx descriptor message buffer */
4113	6825
4114	6826	/* Common message hdr */
4115	6827	txdesc->cmn_hdr.msg_type = MSG_TYPE_TX_POST;
4116	6828	txdesc->cmn_hdr.if_id = ifidx;
	6829	+ txdesc->cmn_hdr.flags = ring->current_phase;
4117	6830
4118	6831	txdesc->flags = BCMPCIE_PKT_FLAGS_FRAME_802_3;
4119	6832	prio = (uint8)PKTPRIO(PKTBUF);
4120		-
4121	6833
4122	6834	txdesc->flags \|= (prio & 0x7) << BCMPCIE_PKT_FLAGS_PRIO_SHIFT;
4123	6835	txdesc->seg_cnt = 1;
..	..	@@ -4131,10 +6843,9 @@
4131	6843
4132	6844	/* Handle Tx metadata */
4133	6845	headroom = (uint16)PKTHEADROOM(dhd->osh, PKTBUF);
4134		- if (prot->tx_metadata_offset && (headroom < prot->tx_metadata_offset)) {
	6846	+ if (prot->tx_metadata_offset && (headroom < prot->tx_metadata_offset))
4135	6847	DHD_ERROR(("No headroom for Metadata tx %d %d\n",
4136	6848	prot->tx_metadata_offset, headroom));
4137		- }
4138	6849
4139	6850	if (prot->tx_metadata_offset && (headroom >= prot->tx_metadata_offset)) {
4140	6851	DHD_TRACE(("Metadata in tx %d\n", prot->tx_metadata_offset));
..	..	@@ -4154,8 +6865,32 @@
4154	6865	#endif /* #ifndef BCM_SECURE_DMA */
4155	6866
4156	6867	if (PHYSADDRISZERO(meta_pa)) {
4157		- DHD_ERROR(("Something really bad, unless 0 is a valid phyaddr\n"));
	6868	+ /* Unmap the data pointer to a DMA-able address */
	6869	+ if (SECURE_DMA_ENAB(dhd->osh)) {
	6870	+ int offset = 0;
	6871	+ BCM_REFERENCE(offset);
	6872	+
	6873	+ if (prot->tx_metadata_offset) {
	6874	+ offset = prot->tx_metadata_offset + ETHER_HDR_LEN;
	6875	+ }
	6876	+
	6877	+ SECURE_DMA_UNMAP(dhd->osh, pa, pktlen,
	6878	+ DMA_TX, 0, DHD_DMAH_NULL, ring->dma_buf.secdma, offset);
	6879	+ }
	6880	+#ifndef BCM_SECURE_DMA
	6881	+ else {
	6882	+ DMA_UNMAP(dhd->osh, pa, pktlen, DMA_TX, 0, DHD_DMAH_NULL);
	6883	+ }
	6884	+#endif /* #ifndef BCM_SECURE_DMA */
	6885	+#ifdef TXP_FLUSH_NITEMS
	6886	+ /* update pend_items_count */
	6887	+ ring->pend_items_count--;
	6888	+#endif /* TXP_FLUSH_NITEMS */
	6889	+
	6890	+ DHD_ERROR(("%s: Something really bad, unless 0 is "
	6891	+ "a valid phyaddr for meta_pa\n", __FUNCTION__));
4158	6892	ASSERT(0);
	6893	+ goto err_rollback_idx;
4159	6894	}
4160	6895
4161	6896	/* Adjust the data pointer back to original value */
..	..	@@ -4165,14 +6900,21 @@
4165	6900	txdesc->metadata_buf_addr.high_addr = htol32(PHYSADDRHI(meta_pa));
4166	6901	txdesc->metadata_buf_addr.low_addr = htol32(PHYSADDRLO(meta_pa));
4167	6902	} else {
4168		- txdesc->metadata_buf_len = htol16(0);
4169		- txdesc->metadata_buf_addr.high_addr = 0;
4170		- txdesc->metadata_buf_addr.low_addr = 0;
	6903	+#ifdef DHD_HP2P
	6904	+ if (dhd->hp2p_capable && flow_ring_node->flow_info.tid == HP2P_PRIO) {
	6905	+ dhd_update_hp2p_txdesc(dhd, txdesc);
	6906	+ } else
	6907	+#endif /* DHD_HP2P */
	6908	+ if (1)
	6909	+ {
	6910	+ txdesc->metadata_buf_len = htol16(0);
	6911	+ txdesc->metadata_buf_addr.high_addr = 0;
	6912	+ txdesc->metadata_buf_addr.low_addr = 0;
	6913	+ }
4171	6914	}
4172	6915
4173		-#if defined(DHD_PKTID_AUDIT_RING)
4174		- DHD_PKTID_AUDIT(dhd, prot->pktid_map_handle, pktid,
4175		- DHD_DUPLICATE_ALLOC);
	6916	+#ifdef DHD_PKTID_AUDIT_RING
	6917	+ DHD_PKTID_AUDIT(dhd, prot->pktid_tx_map, pktid, DHD_DUPLICATE_ALLOC);
4176	6918	#endif /* DHD_PKTID_AUDIT_RING */
4177	6919
4178	6920	txdesc->cmn_hdr.request_id = htol32(pktid);
..	..	@@ -4180,57 +6922,101 @@
4180	6922	DHD_TRACE(("txpost: data_len %d, pktid 0x%04x\n", txdesc->data_len,
4181	6923	txdesc->cmn_hdr.request_id));
4182	6924
	6925	+#ifdef DHD_LBUF_AUDIT
	6926	+ PKTAUDIT(dhd->osh, PKTBUF);
	6927	+#endif // endif
	6928	+
	6929	+ if (pktlen > MAX_MTU_SZ) {
	6930	+ DHD_ERROR(("%s: ######## pktlen(%d) > MAX_MTU_SZ(%d) #######\n",
	6931	+ __FUNCTION__, pktlen, MAX_MTU_SZ));
	6932	+ dhd_prhex("txringitem", (volatile uchar*)txdesc,
	6933	+ sizeof(host_txbuf_post_t), DHD_ERROR_VAL);
	6934	+ }
	6935	+
4183	6936	/* Update the write pointer in TCM & ring bell */
4184		-#ifdef TXP_FLUSH_NITEMS
	6937	+#if defined(DHD_HP2P) && defined(TXP_FLUSH_NITEMS)
	6938	+ if (dhd->hp2p_capable && flow_ring_node->flow_info.tid == HP2P_PRIO) {
	6939	+ dhd_calc_hp2p_burst(dhd, ring, flowid);
	6940	+ } else {
	6941	+ if ((ring->pend_items_count == prot->txp_threshold) \|\|
	6942	+ ((uint8 ) txdesc == (uint8 ) DHD_RING_END_VA(ring))) {
	6943	+ dhd_prot_txdata_write_flush(dhd, flowid);
	6944	+ }
	6945	+ }
	6946	+#elif defined(TXP_FLUSH_NITEMS)
4185	6947	/* Flush if we have either hit the txp_threshold or if this msg is */
4186	6948	/* occupying the last slot in the flow_ring - before wrap around. */
4187	6949	if ((ring->pend_items_count == prot->txp_threshold) \|\|
4188	6950	((uint8 ) txdesc == (uint8 ) DHD_RING_END_VA(ring))) {
4189		- dhd_prot_txdata_write_flush(dhd, flowid, TRUE);
	6951	+ dhd_prot_txdata_write_flush(dhd, flowid);
4190	6952	}
4191	6953	#else
4192	6954	/* update ring's WR index and ring doorbell to dongle */
4193	6955	dhd_prot_ring_write_complete(dhd, ring, txdesc, 1);
4194		-#endif
	6956	+#endif /* DHD_HP2P && TXP_FLUSH_NITEMS */
4195	6957
4196		- prot->active_tx_count++;
	6958	+#if defined(TX_STATUS_LATENCY_STATS)
	6959	+ /* set the time when pkt is queued to flowring */
	6960	+ DHD_PKT_SET_QTIME(PKTBUF, OSL_SYSUPTIME_US());
	6961	+#endif // endif
4197	6962
	6963	+ OSL_ATOMIC_INC(dhd->osh, &prot->active_tx_count);
4198	6964	/*
4199	6965	* Take a wake lock, do not sleep if we have atleast one packet
4200	6966	* to finish.
4201	6967	*/
4202		- if (prot->active_tx_count == 1)
4203		- DHD_TXFL_WAKE_LOCK_TIMEOUT(dhd, MAX_TX_TIMEOUT);
	6968	+ DHD_TXFL_WAKE_LOCK_TIMEOUT(dhd, MAX_TX_TIMEOUT);
4204	6969
4205		- DHD_GENERAL_UNLOCK(dhd, flags);
	6970	+ DHD_RING_UNLOCK(ring->ring_lock, flags);
4206	6971
	6972	+#ifdef TX_STATUS_LATENCY_STATS
	6973	+ flow_ring_node->flow_info.num_tx_pkts++;
	6974	+#endif /* TX_STATUS_LATENCY_STATS */
4207	6975	return BCME_OK;
4208	6976
	6977	+err_rollback_idx:
	6978	+ /* roll back write pointer for unprocessed message */
	6979	+ if (ring->wr == 0) {
	6980	+ ring->wr = ring->max_items - 1;
	6981	+ } else {
	6982	+ ring->wr--;
	6983	+ if (ring->wr == 0) {
	6984	+ DHD_INFO(("%s: flipping the phase now\n", ring->name));
	6985	+ ring->current_phase = ring->current_phase ?
	6986	+ 0 : BCMPCIE_CMNHDR_PHASE_BIT_INIT;
	6987	+ }
	6988	+ }
	6989	+
	6990	+err_free_pktid:
	6991	+#if defined(DHD_PCIE_PKTID)
	6992	+ {
	6993	+ void *dmah;
	6994	+ void *secdma;
	6995	+ /* Free up the PKTID. physaddr and pktlen will be garbage. */
	6996	+ DHD_PKTID_TO_NATIVE(dhd, dhd->prot->pktid_tx_map, pktid,
	6997	+ pa, pktlen, dmah, secdma, PKTTYPE_NO_CHECK);
	6998	+ }
	6999	+
4209	7000	err_no_res_pktfree:
	7001	+#endif /* DHD_PCIE_PKTID */
4210	7002
4211		-
4212		-
4213		- DHD_GENERAL_UNLOCK(dhd, flags);
	7003	+ DHD_RING_UNLOCK(ring->ring_lock, flags);
	7004	+err_no_res:
4214	7005	return BCME_NORESOURCE;
4215	7006	} /* dhd_prot_txdata */
4216	7007
4217		-/* called with a lock */
	7008	+/* called with a ring_lock */
4218	7009	/** optimization to write "n" tx items at a time to ring */
4219	7010	void BCMFASTPATH
4220		-dhd_prot_txdata_write_flush(dhd_pub_t *dhd, uint16 flowid, bool in_lock)
	7011	+dhd_prot_txdata_write_flush(dhd_pub_t *dhd, uint16 flowid)
4221	7012	{
4222	7013	#ifdef TXP_FLUSH_NITEMS
4223		- unsigned long flags = 0;
4224	7014	flow_ring_table_t *flow_ring_table;
4225	7015	flow_ring_node_t *flow_ring_node;
4226	7016	msgbuf_ring_t *ring;
4227	7017
4228	7018	if (dhd->flow_ring_table == NULL) {
4229	7019	return;
4230		- }
4231		-
4232		- if (!in_lock) {
4233		- DHD_GENERAL_LOCK(dhd, flags);
4234	7020	}
4235	7021
4236	7022	flow_ring_table = (flow_ring_table_t *)dhd->flow_ring_table;
..	..	@@ -4243,10 +7029,6 @@
4243	7029	ring->pend_items_count);
4244	7030	ring->pend_items_count = 0;
4245	7031	ring->start_addr = NULL;
4246		- }
4247		-
4248		- if (!in_lock) {
4249		- DHD_GENERAL_UNLOCK(dhd, flags);
4250	7032	}
4251	7033	#endif /* TXP_FLUSH_NITEMS */
4252	7034	}
..	..	@@ -4295,19 +7077,18 @@
4295	7077
4296	7078	#endif /* DHD_LB_RXC */
4297	7079
4298		-
4299	7080	if (prot->rxbufpost >= rxcnt) {
4300		- prot->rxbufpost -= rxcnt;
	7081	+ prot->rxbufpost -= (uint16)rxcnt;
4301	7082	} else {
4302	7083	/* ASSERT(0); */
4303	7084	prot->rxbufpost = 0;
4304	7085	}
4305	7086
4306	7087	#if !defined(DHD_LB_RXC)
4307		- if (prot->rxbufpost <= (prot->max_rxbufpost - RXBUFPOST_THRESHOLD)) {
	7088	+ if (prot->rxbufpost <= (prot->max_rxbufpost - RXBUFPOST_THRESHOLD))
4308	7089	dhd_msgbuf_rxbuf_post(dhd, FALSE); /* alloc pkt ids */
4309		- }
4310	7090	#endif /* !DHD_LB_RXC */
	7091	+ return;
4311	7092	}
4312	7093
4313	7094	/* called before an ioctl is sent to the dongle */
..	..	@@ -4315,9 +7096,9 @@
4315	7096	dhd_prot_wlioctl_intercept(dhd_pub_t dhd, wl_ioctl_t ioc, void * buf)
4316	7097	{
4317	7098	dhd_prot_t *prot = dhd->prot;
	7099	+ int slen = 0;
4318	7100
4319	7101	if (ioc->cmd == WLC_SET_VAR && buf != NULL && !strcmp(buf, "pcie_bus_tput")) {
4320		- int slen = 0;
4321	7102	pcie_bus_tput_params_t *tput_params;
4322	7103
4323	7104	slen = strlen("pcie_bus_tput") + 1;
..	..	@@ -4326,8 +7107,33 @@
4326	7107	sizeof(tput_params->host_buf_addr));
4327	7108	tput_params->host_buf_len = DHD_BUS_TPUT_BUF_LEN;
4328	7109	}
	7110	+
4329	7111	}
4330	7112
	7113	+/* called after an ioctl returns from dongle */
	7114	+static void
	7115	+dhd_prot_wl_ioctl_ret_intercept(dhd_pub_t dhd, wl_ioctl_t ioc, void * buf,
	7116	+ int ifidx, int ret, int len)
	7117	+{
	7118	+
	7119	+ if (!ret && ioc->cmd == WLC_SET_VAR && buf != NULL) {
	7120	+ /* Intercept the wme_dp ioctl here */
	7121	+ if (!strcmp(buf, "wme_dp")) {
	7122	+ int slen, val = 0;
	7123	+
	7124	+ slen = strlen("wme_dp") + 1;
	7125	+ if (len >= (int)(slen + sizeof(int)))
	7126	+ bcopy(((char *)buf + slen), &val, sizeof(int));
	7127	+ dhd->wme_dp = (uint8) ltoh32(val);
	7128	+ }
	7129	+
	7130	+ }
	7131	+
	7132	+}
	7133	+
	7134	+#ifdef DHD_PM_CONTROL_FROM_FILE
	7135	+extern bool g_pm_control;
	7136	+#endif /* DHD_PM_CONTROL_FROM_FILE */
4331	7137
4332	7138	/** Use protocol to issue ioctl to dongle. Only one ioctl may be in transit. */
4333	7139	int dhd_prot_ioctl(dhd_pub_t dhd, int ifidx, wl_ioctl_t ioc, void * buf, int len)
..	..	@@ -4335,8 +7141,20 @@
4335	7141	int ret = -1;
4336	7142	uint8 action;
4337	7143
	7144	+ if (dhd->bus->is_linkdown) {
	7145	+ DHD_ERROR_RLMT(("%s : PCIe link is down. we have nothing to do\n", __FUNCTION__));
	7146	+ goto done;
	7147	+ }
	7148	+
	7149	+ if (dhd_query_bus_erros(dhd)) {
	7150	+ DHD_ERROR_RLMT(("%s : some BUS error. we have nothing to do\n", __FUNCTION__));
	7151	+ goto done;
	7152	+ }
	7153	+
4338	7154	if ((dhd->busstate == DHD_BUS_DOWN) \|\| dhd->hang_was_sent) {
4339		- DHD_ERROR(("%s : bus is down. we have nothing to do\n", __FUNCTION__));
	7155	+ DHD_ERROR_RLMT(("%s : bus is down. we have nothing to do -"
	7156	+ " bus state: %d, sent hang: %d\n", __FUNCTION__,
	7157	+ dhd->busstate, dhd->hang_was_sent));
4340	7158	goto done;
4341	7159	}
4342	7160
..	..	@@ -4347,12 +7165,21 @@
4347	7165
4348	7166	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
4349	7167
	7168	+ if (ioc->cmd == WLC_SET_PM) {
	7169	+#ifdef DHD_PM_CONTROL_FROM_FILE
	7170	+ if (g_pm_control == TRUE) {
	7171	+ DHD_ERROR(("%s: SET PM ignored!(Requested:%d)\n",
	7172	+ __FUNCTION__, buf ? (char )buf : 0));
	7173	+ goto done;
	7174	+ }
	7175	+#endif /* DHD_PM_CONTROL_FROM_FILE */
	7176	+ DHD_TRACE_HW4(("%s: SET PM to %d\n", __FUNCTION__, buf ? (char )buf : 0));
	7177	+ }
4350	7178
4351	7179	ASSERT(len <= WLC_IOCTL_MAXLEN);
4352	7180
4353		- if (len > WLC_IOCTL_MAXLEN) {
	7181	+ if (len > WLC_IOCTL_MAXLEN)
4354	7182	goto done;
4355		- }
4356	7183
4357	7184	action = ioc->set;
4358	7185
..	..	@@ -4362,32 +7189,19 @@
4362	7189	ret = dhd_msgbuf_set_ioctl(dhd, ifidx, ioc->cmd, buf, len, action);
4363	7190	} else {
4364	7191	ret = dhd_msgbuf_query_ioctl(dhd, ifidx, ioc->cmd, buf, len, action);
4365		- if (ret > 0) {
	7192	+ if (ret > 0)
4366	7193	ioc->used = ret;
4367		- }
4368	7194	}
4369	7195
4370	7196	/* Too many programs assume ioctl() returns 0 on success */
4371	7197	if (ret >= 0) {
4372	7198	ret = 0;
4373	7199	} else {
4374		- DHD_ERROR(("%s: status ret value is %d \n", __FUNCTION__, ret));
	7200	+ DHD_INFO(("%s: status ret value is %d \n", __FUNCTION__, ret));
4375	7201	dhd->dongle_error = ret;
4376	7202	}
4377	7203
4378		- if (!ret && ioc->cmd == WLC_SET_VAR && buf != NULL) {
4379		- /* Intercept the wme_dp ioctl here */
4380		- if (!strcmp(buf, "wme_dp")) {
4381		- int slen, val = 0;
4382		-
4383		- slen = strlen("wme_dp") + 1;
4384		- if (len >= (int)(slen + sizeof(int))) {
4385		- bcopy(((char *)buf + slen), &val, sizeof(int));
4386		- }
4387		- dhd->wme_dp = (uint8) ltoh32(val);
4388		- }
4389		-
4390		- }
	7204	+ dhd_prot_wl_ioctl_ret_intercept(dhd, ioc, buf, ifidx, ret, len);
4391	7205
4392	7206	done:
4393	7207	return ret;
..	..	@@ -4412,13 +7226,13 @@
4412	7226	msglen = ALIGN_SIZE(msglen, DMA_ALIGN_LEN);
4413	7227	msglen = LIMIT_TO_MAX(msglen, MSGBUF_MAX_MSG_SIZE);
4414	7228
4415		- DHD_GENERAL_LOCK(dhd, flags);
	7229	+ DHD_RING_LOCK(ring->ring_lock, flags);
4416	7230
4417	7231	ioct_rqst = (ioct_reqst_hdr_t *)
4418	7232	dhd_prot_alloc_ring_space(dhd, ring, 1, &alloced, FALSE);
4419	7233
4420	7234	if (ioct_rqst == NULL) {
4421		- DHD_GENERAL_UNLOCK(dhd, flags);
	7235	+ DHD_RING_UNLOCK(ring->ring_lock, flags);
4422	7236	return 0;
4423	7237	}
4424	7238
..	..	@@ -4438,12 +7252,14 @@
4438	7252
4439	7253	ioct_rqst->msg.msg_type = MSG_TYPE_LOOPBACK;
4440	7254	ioct_rqst->msg.if_id = 0;
	7255	+ ioct_rqst->msg.flags = ring->current_phase;
4441	7256
4442	7257	bcm_print_bytes("LPBK REQ: ", (uint8 *)ioct_rqst, msglen);
4443	7258
4444	7259	/* update ring's WR index and ring doorbell to dongle */
4445	7260	dhd_prot_ring_write_complete(dhd, ring, ioct_rqst, 1);
4446		- DHD_GENERAL_UNLOCK(dhd, flags);
	7261	+
	7262	+ DHD_RING_UNLOCK(ring->ring_lock, flags);
4447	7263
4448	7264	return 0;
4449	7265	}
..	..	@@ -4451,22 +7267,64 @@
4451	7267	/** test / loopback */
4452	7268	void dmaxfer_free_dmaaddr(dhd_pub_t dhd, dhd_dmaxfer_t dmaxfer)
4453	7269	{
4454		- if (dmaxfer == NULL) {
	7270	+ if (dmaxfer == NULL)
4455	7271	return;
4456		- }
4457	7272
4458	7273	dhd_dma_buf_free(dhd, &dmaxfer->srcmem);
4459	7274	dhd_dma_buf_free(dhd, &dmaxfer->dstmem);
4460	7275	}
4461	7276
4462	7277	/** test / loopback */
	7278	+int
	7279	+dhd_prepare_schedule_dmaxfer_free(dhd_pub_t *dhdp)
	7280	+{
	7281	+ dhd_prot_t *prot = dhdp->prot;
	7282	+ dhd_dmaxfer_t *dmaxfer = &prot->dmaxfer;
	7283	+ dmaxref_mem_map_t *dmap = NULL;
	7284	+
	7285	+ dmap = MALLOCZ(dhdp->osh, sizeof(dmaxref_mem_map_t));
	7286	+ if (!dmap) {
	7287	+ DHD_ERROR(("%s: dmap alloc failed\n", __FUNCTION__));
	7288	+ goto mem_alloc_fail;
	7289	+ }
	7290	+ dmap->srcmem = &(dmaxfer->srcmem);
	7291	+ dmap->dstmem = &(dmaxfer->dstmem);
	7292	+
	7293	+ DMAXFER_FREE(dhdp, dmap);
	7294	+ return BCME_OK;
	7295	+
	7296	+mem_alloc_fail:
	7297	+ if (dmap) {
	7298	+ MFREE(dhdp->osh, dmap, sizeof(dmaxref_mem_map_t));
	7299	+ dmap = NULL;
	7300	+ }
	7301	+ return BCME_NOMEM;
	7302	+} /* dhd_prepare_schedule_dmaxfer_free */
	7303	+
	7304	+/** test / loopback */
	7305	+void
	7306	+dmaxfer_free_prev_dmaaddr(dhd_pub_t dhdp, dmaxref_mem_map_t dmmap)
	7307	+{
	7308	+
	7309	+ dhd_dma_buf_free(dhdp, dmmap->srcmem);
	7310	+ dhd_dma_buf_free(dhdp, dmmap->dstmem);
	7311	+
	7312	+ MFREE(dhdp->osh, dmmap, sizeof(dmaxref_mem_map_t));
	7313	+
	7314	+ dhdp->bus->dmaxfer_complete = TRUE;
	7315	+ dhd_os_dmaxfer_wake(dhdp);
	7316	+
	7317	+ dmmap = NULL;
	7318	+
	7319	+} /* dmaxfer_free_prev_dmaaddr */
	7320	+
	7321	+/** test / loopback */
4463	7322	int dmaxfer_prepare_dmaaddr(dhd_pub_t *dhd, uint len,
4464	7323	uint srcdelay, uint destdelay, dhd_dmaxfer_t *dmaxfer)
4465	7324	{
4466		- uint i;
4467		- if (!dmaxfer) {
	7325	+ uint i = 0, j = 0;
	7326	+ if (!dmaxfer)
4468	7327	return BCME_ERROR;
4469		- }
4470	7328
4471	7329	/* First free up existing buffers */
4472	7330	dmaxfer_free_dmaaddr(dhd, dmaxfer);
..	..	@@ -4482,10 +7340,24 @@
4482	7340
4483	7341	dmaxfer->len = len;
4484	7342
4485		- /* Populate source with a pattern */
4486		- for (i = 0; i < dmaxfer->len; i++) {
4487		- ((uint8*)dmaxfer->srcmem.va)[i] = i % 256;
	7343	+ /* Populate source with a pattern like below
	7344	+ * 0x00000000
	7345	+ * 0x01010101
	7346	+ * 0x02020202
	7347	+ * 0x03030303
	7348	+ * 0x04040404
	7349	+ * 0x05050505
	7350	+ * ...
	7351	+ * 0xFFFFFFFF
	7352	+ */
	7353	+ while (i < dmaxfer->len) {
	7354	+ ((uint8*)dmaxfer->srcmem.va)[i] = j % 256;
	7355	+ i++;
	7356	+ if (i % 4 == 0) {
	7357	+ j++;
	7358	+ }
4488	7359	}
	7360	+
4489	7361	OSL_CACHE_FLUSH(dmaxfer->srcmem.va, dmaxfer->len);
4490	7362
4491	7363	dmaxfer->srcdelay = srcdelay;
..	..	@@ -4498,21 +7370,81 @@
4498	7370	dhd_msgbuf_dmaxfer_process(dhd_pub_t dhd, void msg)
4499	7371	{
4500	7372	dhd_prot_t *prot = dhd->prot;
	7373	+ uint64 end_usec;
	7374	+ pcie_dmaxfer_cmplt_t cmplt = (pcie_dmaxfer_cmplt_t )msg;
	7375	+ int buf_free_scheduled;
4501	7376
	7377	+ BCM_REFERENCE(cmplt);
	7378	+ end_usec = OSL_SYSUPTIME_US();
	7379	+
	7380	+ DHD_ERROR(("DMA loopback status: %d\n", cmplt->compl_hdr.status));
	7381	+ prot->dmaxfer.status = cmplt->compl_hdr.status;
4502	7382	OSL_CACHE_INV(prot->dmaxfer.dstmem.va, prot->dmaxfer.len);
4503	7383	if (prot->dmaxfer.srcmem.va && prot->dmaxfer.dstmem.va) {
4504	7384	if (memcmp(prot->dmaxfer.srcmem.va,
4505		- prot->dmaxfer.dstmem.va, prot->dmaxfer.len)) {
4506		- bcm_print_bytes("XFER SRC: ",
	7385	+ prot->dmaxfer.dstmem.va, prot->dmaxfer.len) \|\|
	7386	+ cmplt->compl_hdr.status != BCME_OK) {
	7387	+ DHD_ERROR(("DMA loopback failed\n"));
	7388	+ /* it is observed that some times the completion
	7389	+ * header status is set as OK, but the memcmp fails
	7390	+ * hence always explicitly set the dmaxfer status
	7391	+ * as error if this happens.
	7392	+ */
	7393	+ prot->dmaxfer.status = BCME_ERROR;
	7394	+ prhex("XFER SRC: ",
4507	7395	prot->dmaxfer.srcmem.va, prot->dmaxfer.len);
4508		- bcm_print_bytes("XFER DST: ",
	7396	+ prhex("XFER DST: ",
4509	7397	prot->dmaxfer.dstmem.va, prot->dmaxfer.len);
4510		- } else {
4511		- DHD_INFO(("DMA successful\n"));
	7398	+ }
	7399	+ else {
	7400	+ switch (prot->dmaxfer.d11_lpbk) {
	7401	+ case M2M_DMA_LPBK: {
	7402	+ DHD_ERROR(("DMA successful pcie m2m DMA loopback\n"));
	7403	+ } break;
	7404	+ case D11_LPBK: {
	7405	+ DHD_ERROR(("DMA successful with d11 loopback\n"));
	7406	+ } break;
	7407	+ case BMC_LPBK: {
	7408	+ DHD_ERROR(("DMA successful with bmc loopback\n"));
	7409	+ } break;
	7410	+ case M2M_NON_DMA_LPBK: {
	7411	+ DHD_ERROR(("DMA successful pcie m2m NON DMA loopback\n"));
	7412	+ } break;
	7413	+ case D11_HOST_MEM_LPBK: {
	7414	+ DHD_ERROR(("DMA successful d11 host mem loopback\n"));
	7415	+ } break;
	7416	+ case BMC_HOST_MEM_LPBK: {
	7417	+ DHD_ERROR(("DMA successful bmc host mem loopback\n"));
	7418	+ } break;
	7419	+ default: {
	7420	+ DHD_ERROR(("Invalid loopback option\n"));
	7421	+ } break;
	7422	+ }
	7423	+
	7424	+ if (DHD_LPBKDTDUMP_ON()) {
	7425	+ /* debug info print of the Tx and Rx buffers */
	7426	+ dhd_prhex("XFER SRC: ", prot->dmaxfer.srcmem.va,
	7427	+ prot->dmaxfer.len, DHD_INFO_VAL);
	7428	+ dhd_prhex("XFER DST: ", prot->dmaxfer.dstmem.va,
	7429	+ prot->dmaxfer.len, DHD_INFO_VAL);
	7430	+ }
4512	7431	}
4513	7432	}
4514		- dmaxfer_free_dmaaddr(dhd, &prot->dmaxfer);
	7433	+
	7434	+ buf_free_scheduled = dhd_prepare_schedule_dmaxfer_free(dhd);
	7435	+ end_usec -= prot->dmaxfer.start_usec;
	7436	+ if (end_usec) {
	7437	+ prot->dmaxfer.time_taken = end_usec;
	7438	+ DHD_ERROR(("DMA loopback %d bytes in %lu usec, %u kBps\n",
	7439	+ prot->dmaxfer.len, (unsigned long)end_usec,
	7440	+ (prot->dmaxfer.len * (1000 * 1000 / 1024) / (uint32)end_usec)));
	7441	+ }
4515	7442	dhd->prot->dmaxfer.in_progress = FALSE;
	7443	+
	7444	+ if (buf_free_scheduled != BCME_OK) {
	7445	+ dhd->bus->dmaxfer_complete = TRUE;
	7446	+ dhd_os_dmaxfer_wake(dhd);
	7447	+ }
4516	7448	}
4517	7449
4518	7450	/** Test functionality.
..	..	@@ -4521,7 +7453,8 @@
4521	7453	* by a spinlock.
4522	7454	*/
4523	7455	int
4524		-dhdmsgbuf_dmaxfer_req(dhd_pub_t *dhd, uint len, uint srcdelay, uint destdelay)
	7456	+dhdmsgbuf_dmaxfer_req(dhd_pub_t *dhd, uint len, uint srcdelay, uint destdelay,
	7457	+ uint d11_lpbk, uint core_num)
4525	7458	{
4526	7459	unsigned long flags;
4527	7460	int ret = BCME_OK;
..	..	@@ -4533,17 +7466,24 @@
4533	7466
4534	7467	if (prot->dmaxfer.in_progress) {
4535	7468	DHD_ERROR(("DMA is in progress...\n"));
4536		- return ret;
	7469	+ return BCME_ERROR;
4537	7470	}
	7471	+
	7472	+ if (d11_lpbk >= MAX_LPBK) {
	7473	+ DHD_ERROR(("loopback mode should be either"
	7474	+ " 0-PCIE_M2M_DMA, 1-D11, 2-BMC or 3-PCIE_M2M_NonDMA\n"));
	7475	+ return BCME_ERROR;
	7476	+ }
	7477	+
	7478	+ DHD_RING_LOCK(ring->ring_lock, flags);
4538	7479
4539	7480	prot->dmaxfer.in_progress = TRUE;
4540	7481	if ((ret = dmaxfer_prepare_dmaaddr(dhd, xferlen, srcdelay, destdelay,
4541	7482	&prot->dmaxfer)) != BCME_OK) {
4542	7483	prot->dmaxfer.in_progress = FALSE;
	7484	+ DHD_RING_UNLOCK(ring->ring_lock, flags);
4543	7485	return ret;
4544	7486	}
4545		-
4546		- DHD_GENERAL_LOCK(dhd, flags);
4547	7487
4548	7488	dmap = (pcie_dma_xfer_params_t *)
4549	7489	dhd_prot_alloc_ring_space(dhd, ring, 1, &alloced, FALSE);
..	..	@@ -4551,7 +7491,7 @@
4551	7491	if (dmap == NULL) {
4552	7492	dmaxfer_free_dmaaddr(dhd, &prot->dmaxfer);
4553	7493	prot->dmaxfer.in_progress = FALSE;
4554		- DHD_GENERAL_UNLOCK(dhd, flags);
	7494	+ DHD_RING_UNLOCK(ring->ring_lock, flags);
4555	7495	return BCME_NOMEM;
4556	7496	}
4557	7497
..	..	@@ -4559,6 +7499,7 @@
4559	7499	dmap->cmn_hdr.msg_type = MSG_TYPE_LPBK_DMAXFER;
4560	7500	dmap->cmn_hdr.request_id = htol32(DHD_FAKE_PKTID);
4561	7501	dmap->cmn_hdr.epoch = ring->seqnum % H2D_EPOCH_MODULO;
	7502	+ dmap->cmn_hdr.flags = ring->current_phase;
4562	7503	ring->seqnum++;
4563	7504
4564	7505	dmap->host_input_buf_addr.high = htol32(PHYSADDRHI(prot->dmaxfer.srcmem.pa));
..	..	@@ -4568,49 +7509,170 @@
4568	7509	dmap->xfer_len = htol32(prot->dmaxfer.len);
4569	7510	dmap->srcdelay = htol32(prot->dmaxfer.srcdelay);
4570	7511	dmap->destdelay = htol32(prot->dmaxfer.destdelay);
	7512	+ prot->dmaxfer.d11_lpbk = d11_lpbk;
	7513	+ dmap->flags = (((core_num & PCIE_DMA_XFER_FLG_CORE_NUMBER_MASK)
	7514	+ << PCIE_DMA_XFER_FLG_CORE_NUMBER_SHIFT) \|
	7515	+ ((prot->dmaxfer.d11_lpbk & PCIE_DMA_XFER_FLG_D11_LPBK_MASK)
	7516	+ << PCIE_DMA_XFER_FLG_D11_LPBK_SHIFT));
	7517	+ prot->dmaxfer.start_usec = OSL_SYSUPTIME_US();
4571	7518
4572	7519	/* update ring's WR index and ring doorbell to dongle */
4573	7520	dhd_prot_ring_write_complete(dhd, ring, dmap, 1);
4574		- DHD_GENERAL_UNLOCK(dhd, flags);
4575	7521
4576		- DHD_ERROR(("DMA Started...\n"));
	7522	+ DHD_RING_UNLOCK(ring->ring_lock, flags);
	7523	+
	7524	+ DHD_ERROR(("DMA loopback Started...\n"));
4577	7525
4578	7526	return BCME_OK;
4579	7527	} /* dhdmsgbuf_dmaxfer_req */
	7528	+
	7529	+int
	7530	+dhdmsgbuf_dmaxfer_status(dhd_pub_t dhd, dma_xfer_info_t result)
	7531	+{
	7532	+ dhd_prot_t *prot = dhd->prot;
	7533	+
	7534	+ if (prot->dmaxfer.in_progress)
	7535	+ result->status = DMA_XFER_IN_PROGRESS;
	7536	+ else if (prot->dmaxfer.status == 0)
	7537	+ result->status = DMA_XFER_SUCCESS;
	7538	+ else
	7539	+ result->status = DMA_XFER_FAILED;
	7540	+
	7541	+ result->type = prot->dmaxfer.d11_lpbk;
	7542	+ result->error_code = prot->dmaxfer.status;
	7543	+ result->num_bytes = prot->dmaxfer.len;
	7544	+ result->time_taken = prot->dmaxfer.time_taken;
	7545	+ if (prot->dmaxfer.time_taken) {
	7546	+ /* throughput in kBps */
	7547	+ result->tput =
	7548	+ (prot->dmaxfer.len * (1000 * 1000 / 1024)) /
	7549	+ (uint32)prot->dmaxfer.time_taken;
	7550	+ }
	7551	+
	7552	+ return BCME_OK;
	7553	+}
4580	7554
4581	7555	/** Called in the process of submitting an ioctl to the dongle */
4582	7556	static int
4583	7557	dhd_msgbuf_query_ioctl(dhd_pub_t dhd, int ifidx, uint cmd, void buf, uint len, uint8 action)
4584	7558	{
4585	7559	int ret = 0;
	7560	+ uint copylen = 0;
4586	7561
4587	7562	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
4588	7563
4589		- /* Respond "bcmerror" and "bcmerrorstr" with local cache */
	7564	+ if (dhd->bus->is_linkdown) {
	7565	+ DHD_ERROR(("%s : PCIe link is down. we have nothing to do\n",
	7566	+ __FUNCTION__));
	7567	+ return -EIO;
	7568	+ }
	7569	+
	7570	+ if (dhd->busstate == DHD_BUS_DOWN) {
	7571	+ DHD_ERROR(("%s : bus is down. we have nothing to do\n", __FUNCTION__));
	7572	+ return -EIO;
	7573	+ }
	7574	+
	7575	+ /* don't talk to the dongle if fw is about to be reloaded */
	7576	+ if (dhd->hang_was_sent) {
	7577	+ DHD_ERROR(("%s: HANG was sent up earlier. Not talking to the chip\n",
	7578	+ __FUNCTION__));
	7579	+ return -EIO;
	7580	+ }
	7581	+
4590	7582	if (cmd == WLC_GET_VAR && buf)
4591	7583	{
4592		- if (!strcmp((char *)buf, "bcmerrorstr"))
4593		- {
4594		- strncpy((char *)buf, bcmerrorstr(dhd->dongle_error), BCME_STRLEN);
	7584	+ if (!len \|\| !(uint8 )buf) {
	7585	+ DHD_ERROR(("%s(): Zero length bailing\n", __FUNCTION__));
	7586	+ ret = BCME_BADARG;
4595	7587	goto done;
4596	7588	}
4597		- else if (!strcmp((char *)buf, "bcmerror"))
4598		- {
4599		- (int )buf = dhd->dongle_error;
	7589	+
	7590	+ /* Respond "bcmerror" and "bcmerrorstr" with local cache */
	7591	+ copylen = MIN(len, BCME_STRLEN);
	7592	+
	7593	+ if ((len >= strlen("bcmerrorstr")) &&
	7594	+ (!strcmp((char *)buf, "bcmerrorstr"))) {
	7595	+ strncpy((char *)buf, bcmerrorstr(dhd->dongle_error), copylen);
	7596	+ (uint8 )((uint8 *)buf + (copylen - 1)) = '\0';
	7597	+ goto done;
	7598	+ } else if ((len >= strlen("bcmerror")) &&
	7599	+ !strcmp((char *)buf, "bcmerror")) {
	7600	+ (uint32 )(uint32 *)buf = dhd->dongle_error;
4600	7601	goto done;
4601	7602	}
4602	7603	}
4603	7604
	7605	+ DHD_CTL(("query_ioctl: ACTION %d ifdix %d cmd %d len %d \n",
	7606	+ action, ifidx, cmd, len));
	7607	+
4604	7608	ret = dhd_fillup_ioct_reqst(dhd, (uint16)len, cmd, buf, ifidx);
4605	7609
4606		- DHD_CTL(("query_ioctl: ACTION %d ifdix %d cmd %d len %d \n",
4607		- action, ifidx, cmd, len));
	7610	+ if (ret < 0) {
	7611	+ DHD_ERROR(("%s(): dhd_fillup_ioct_reqst failed \r\n", __FUNCTION__));
	7612	+ goto done;
	7613	+ }
4608	7614
4609	7615	/* wait for IOCTL completion message from dongle and get first fragment */
4610	7616	ret = dhd_msgbuf_wait_ioctl_cmplt(dhd, len, buf);
4611	7617
4612	7618	done:
4613	7619	return ret;
	7620	+}
	7621	+
	7622	+void
	7623	+dhd_msgbuf_iovar_timeout_dump(dhd_pub_t *dhd)
	7624	+{
	7625	+ uint32 intstatus;
	7626	+ dhd_prot_t *prot = dhd->prot;
	7627	+ dhd->rxcnt_timeout++;
	7628	+ dhd->rx_ctlerrs++;
	7629	+ dhd->iovar_timeout_occured = TRUE;
	7630	+ DHD_ERROR(("%s: resumed on timeout rxcnt_timeout%s %d ioctl_cmd %d "
	7631	+ "trans_id %d state %d busstate=%d ioctl_received=%d\n", __FUNCTION__,
	7632	+ dhd->is_sched_error ? " due to scheduling problem" : "",
	7633	+ dhd->rxcnt_timeout, prot->curr_ioctl_cmd, prot->ioctl_trans_id,
	7634	+ prot->ioctl_state, dhd->busstate, prot->ioctl_received));
	7635	+#if defined(DHD_KERNEL_SCHED_DEBUG) && defined(DHD_FW_COREDUMP)
	7636	+ if (dhd->is_sched_error && dhd->memdump_enabled == DUMP_MEMFILE_BUGON) {
	7637	+ /* change g_assert_type to trigger Kernel panic */
	7638	+ g_assert_type = 2;
	7639	+ /* use ASSERT() to trigger panic */
	7640	+ ASSERT(0);
	7641	+ }
	7642	+#endif /* DHD_KERNEL_SCHED_DEBUG && DHD_FW_COREDUMP */
	7643	+
	7644	+ if (prot->curr_ioctl_cmd == WLC_SET_VAR \|\|
	7645	+ prot->curr_ioctl_cmd == WLC_GET_VAR) {
	7646	+ char iovbuf[32];
	7647	+ int i;
	7648	+ int dump_size = 128;
	7649	+ uint8 ioctl_buf = (uint8 )prot->ioctbuf.va;
	7650	+ memset(iovbuf, 0, sizeof(iovbuf));
	7651	+ strncpy(iovbuf, ioctl_buf, sizeof(iovbuf) - 1);
	7652	+ iovbuf[sizeof(iovbuf) - 1] = '\0';
	7653	+ DHD_ERROR(("Current IOVAR (%s): %s\n",
	7654	+ prot->curr_ioctl_cmd == WLC_SET_VAR ?
	7655	+ "WLC_SET_VAR" : "WLC_GET_VAR", iovbuf));
	7656	+ DHD_ERROR(("========== START IOCTL REQBUF DUMP ==========\n"));
	7657	+ for (i = 0; i < dump_size; i++) {
	7658	+ DHD_ERROR(("%02X ", ioctl_buf[i]));
	7659	+ if ((i % 32) == 31) {
	7660	+ DHD_ERROR(("\n"));
	7661	+ }
	7662	+ }
	7663	+ DHD_ERROR(("\n========== END IOCTL REQBUF DUMP ==========\n"));
	7664	+ }
	7665	+
	7666	+ /* Check the PCIe link status by reading intstatus register */
	7667	+ intstatus = si_corereg(dhd->bus->sih,
	7668	+ dhd->bus->sih->buscoreidx, dhd->bus->pcie_mailbox_int, 0, 0);
	7669	+ if (intstatus == (uint32)-1) {
	7670	+ DHD_ERROR(("%s : PCIe link might be down\n", __FUNCTION__));
	7671	+ dhd->bus->is_linkdown = TRUE;
	7672	+ }
	7673	+
	7674	+ dhd_bus_dump_console_buffer(dhd->bus);
	7675	+ dhd_prot_debug_info_print(dhd);
4614	7676	}
4615	7677
4616	7678	/**
..	..	@@ -4627,53 +7689,61 @@
4627	7689
4628	7690	DHD_TRACE(("%s: Enter\n", __FUNCTION__));
4629	7691
4630		- if (dhd->dongle_reset) {
	7692	+ if (dhd_query_bus_erros(dhd)) {
4631	7693	ret = -EIO;
4632	7694	goto out;
4633	7695	}
4634	7696
4635		- if (prot->cur_ioctlresp_bufs_posted) {
4636		- prot->cur_ioctlresp_bufs_posted--;
	7697	+ timeleft = dhd_os_ioctl_resp_wait(dhd, (uint *)&prot->ioctl_received);
	7698	+
	7699	+ if (prot->ioctl_received == 0) {
	7700	+ uint32 intstatus = si_corereg(dhd->bus->sih,
	7701	+ dhd->bus->sih->buscoreidx, dhd->bus->pcie_mailbox_int, 0, 0);
	7702	+ int host_irq_disbled = dhdpcie_irq_disabled(dhd->bus);
	7703	+ if ((intstatus) && (intstatus != (uint32)-1) &&
	7704	+ (timeleft == 0) && (!dhd_query_bus_erros(dhd))) {
	7705	+ DHD_ERROR(("%s: iovar timeout trying again intstatus=%x"
	7706	+ " host_irq_disabled=%d\n",
	7707	+ __FUNCTION__, intstatus, host_irq_disbled));
	7708	+ dhd_pcie_intr_count_dump(dhd);
	7709	+ dhd_print_tasklet_status(dhd);
	7710	+ dhd_prot_process_ctrlbuf(dhd);
	7711	+ timeleft = dhd_os_ioctl_resp_wait(dhd, (uint *)&prot->ioctl_received);
	7712	+ /* Clear Interrupts */
	7713	+ dhdpcie_bus_clear_intstatus(dhd->bus);
	7714	+ }
4637	7715	}
4638	7716
4639		- dhd_msgbuf_rxbuf_post_ioctlresp_bufs(dhd);
	7717	+ if (timeleft == 0 && (!dhd_query_bus_erros(dhd))) {
	7718	+ /* check if resumed on time out related to scheduling issue */
	7719	+ dhd->is_sched_error = FALSE;
	7720	+ if (dhd->bus->isr_entry_time > prot->ioctl_fillup_time) {
	7721	+ dhd->is_sched_error = dhd_bus_query_dpc_sched_errors(dhd);
	7722	+ }
4640	7723
4641		- timeleft = dhd_os_ioctl_resp_wait(dhd, &prot->ioctl_received);
4642		- if (timeleft == 0) {
4643		- dhd->rxcnt_timeout++;
4644		- dhd->rx_ctlerrs++;
4645		- DHD_ERROR(("%s: resumed on timeout rxcnt_timeout %d ioctl_cmd %d "
4646		- "trans_id %d state %d busstate=%d ioctl_received=%d\n",
4647		- __FUNCTION__, dhd->rxcnt_timeout, prot->curr_ioctl_cmd,
4648		- prot->ioctl_trans_id, prot->ioctl_state & ~MSGBUF_IOCTL_RESP_PENDING,
4649		- dhd->busstate, prot->ioctl_received));
	7724	+ dhd_msgbuf_iovar_timeout_dump(dhd);
4650	7725
4651		- dhd_prot_debug_info_print(dhd);
4652		-
4653		-#if defined(DHD_FW_COREDUMP)
4654		- /* Collect socram dump for CUSTOMER_HW4 OR Brix Android */
4655		- /* As soon as FW TRAP occurs, FW dump will be collected from dhdpcie_checkdied */
4656		- if (dhd->memdump_enabled && !dhd->dongle_trap_occured) {
	7726	+#ifdef DHD_FW_COREDUMP
	7727	+ /* Collect socram dump */
	7728	+ if (dhd->memdump_enabled) {
4657	7729	/* collect core dump */
4658	7730	dhd->memdump_type = DUMP_TYPE_RESUMED_ON_TIMEOUT;
4659	7731	dhd_bus_mem_dump(dhd);
4660	7732	}
4661		-#endif /* DHD_FW_COREDUMP && OEM_ANDROID */
4662		- if (dhd->rxcnt_timeout >= MAX_CNTL_RX_TIMEOUT) {
	7733	+#endif /* DHD_FW_COREDUMP */
	7734	+
4663	7735	#ifdef SUPPORT_LINKDOWN_RECOVERY
4664	7736	#ifdef CONFIG_ARCH_MSM
4665		- dhd->bus->islinkdown = 1;
	7737	+ dhd->bus->no_cfg_restore = 1;
4666	7738	#endif /* CONFIG_ARCH_MSM */
4667	7739	#endif /* SUPPORT_LINKDOWN_RECOVERY */
4668		- DHD_ERROR(("%s: timeout > MAX_CNTL_TX_TIMEOUT\n", __FUNCTION__));
4669		- }
4670	7740	ret = -ETIMEDOUT;
4671	7741	goto out;
4672	7742	} else {
4673	7743	if (prot->ioctl_received != IOCTL_RETURN_ON_SUCCESS) {
4674	7744	DHD_ERROR(("%s: IOCTL failure due to ioctl_received = %d\n",
4675	7745	__FUNCTION__, prot->ioctl_received));
4676		- ret = -ECONNABORTED;
	7746	+ ret = -EINVAL;
4677	7747	goto out;
4678	7748	}
4679	7749	dhd->rxcnt_timeout = 0;
..	..	@@ -4682,28 +7752,16 @@
4682	7752	__FUNCTION__, prot->ioctl_resplen));
4683	7753	}
4684	7754
4685		- if (dhd->dongle_trap_occured) {
4686		-#ifdef SUPPORT_LINKDOWN_RECOVERY
4687		-#ifdef CONFIG_ARCH_MSM
4688		- dhd->bus->islinkdown = 1;
4689		-#endif /* CONFIG_ARCH_MSM */
4690		-#endif /* SUPPORT_LINKDOWN_RECOVERY */
4691		- DHD_ERROR(("%s: TRAP occurred!!\n", __FUNCTION__));
4692		- ret = -EREMOTEIO;
4693		- goto out;
4694		- }
4695		-
4696		- if (dhd->prot->ioctl_resplen > len) {
	7755	+ if (dhd->prot->ioctl_resplen > len)
4697	7756	dhd->prot->ioctl_resplen = (uint16)len;
4698		- }
4699		- if (buf) {
	7757	+ if (buf)
4700	7758	bcopy(dhd->prot->retbuf.va, buf, dhd->prot->ioctl_resplen);
4701		- }
4702	7759
4703	7760	ret = (int)(dhd->prot->ioctl_status);
	7761	+
4704	7762	out:
4705	7763	DHD_GENERAL_LOCK(dhd, flags);
4706		- dhd->prot->ioctl_state &= ~MSGBUF_IOCTL_RESP_PENDING;
	7764	+ dhd->prot->ioctl_state = 0;
4707	7765	dhd->prot->ioctl_resplen = 0;
4708	7766	dhd->prot->ioctl_received = IOCTL_WAIT;
4709	7767	dhd->prot->curr_ioctl_cmd = 0;
..	..	@@ -4719,6 +7777,12 @@
4719	7777
4720	7778	DHD_TRACE(("%s: Enter \n", __FUNCTION__));
4721	7779
	7780	+ if (dhd->bus->is_linkdown) {
	7781	+ DHD_ERROR(("%s : PCIe link is down. we have nothing to do\n",
	7782	+ __FUNCTION__));
	7783	+ return -EIO;
	7784	+ }
	7785	+
4722	7786	if (dhd->busstate == DHD_BUS_DOWN) {
4723	7787	DHD_ERROR(("%s : bus is down. we have nothing to do\n", __FUNCTION__));
4724	7788	return -EIO;
..	..	@@ -4731,14 +7795,20 @@
4731	7795	return -EIO;
4732	7796	}
4733	7797
4734		- /* Fill up msgbuf for ioctl req */
4735		- ret = dhd_fillup_ioct_reqst(dhd, (uint16)len, cmd, buf, ifidx);
4736		-
4737	7798	DHD_CTL(("ACTION %d ifdix %d cmd %d len %d \n",
4738	7799	action, ifidx, cmd, len));
4739	7800
	7801	+ /* Fill up msgbuf for ioctl req */
	7802	+ ret = dhd_fillup_ioct_reqst(dhd, (uint16)len, cmd, buf, ifidx);
	7803	+
	7804	+ if (ret < 0) {
	7805	+ DHD_ERROR(("%s(): dhd_fillup_ioct_reqst failed \r\n", __FUNCTION__));
	7806	+ goto done;
	7807	+ }
	7808	+
4740	7809	ret = dhd_msgbuf_wait_ioctl_cmplt(dhd, len, buf);
4741	7810
	7811	+done:
4742	7812	return ret;
4743	7813	}
4744	7814
..	..	@@ -4750,16 +7820,161 @@
4750	7820
4751	7821	/** Called by upper DHD layer. Check for and handle local prot-specific iovar commands */
4752	7822	int dhd_prot_iovar_op(dhd_pub_t dhd, const char name,
4753		- void params, int plen, void arg, int len, bool set)
	7823	+ void params, int plen, void arg, int len, bool set)
4754	7824	{
4755	7825	return BCME_UNSUPPORTED;
4756	7826	}
	7827	+
	7828	+#ifdef DHD_DUMP_PCIE_RINGS
	7829	+int dhd_d2h_h2d_ring_dump(dhd_pub_t dhd, void file, const void *user_buf,
	7830	+ unsigned long *file_posn, bool file_write)
	7831	+{
	7832	+ dhd_prot_t *prot;
	7833	+ msgbuf_ring_t *ring;
	7834	+ int ret = 0;
	7835	+ uint16 h2d_flowrings_total;
	7836	+ uint16 flowid;
	7837	+
	7838	+ if (!(dhd) \|\| !(dhd->prot)) {
	7839	+ goto exit;
	7840	+ }
	7841	+ prot = dhd->prot;
	7842	+
	7843	+ /* Below is the same ring dump sequence followed in parser as well. */
	7844	+ ring = &prot->h2dring_ctrl_subn;
	7845	+ if ((ret = dhd_ring_write(dhd, ring, file, user_buf, file_posn)) < 0)
	7846	+ goto exit;
	7847	+
	7848	+ ring = &prot->h2dring_rxp_subn;
	7849	+ if ((ret = dhd_ring_write(dhd, ring, file, user_buf, file_posn)) < 0)
	7850	+ goto exit;
	7851	+
	7852	+ ring = &prot->d2hring_ctrl_cpln;
	7853	+ if ((ret = dhd_ring_write(dhd, ring, file, user_buf, file_posn)) < 0)
	7854	+ goto exit;
	7855	+
	7856	+ ring = &prot->d2hring_tx_cpln;
	7857	+ if ((ret = dhd_ring_write(dhd, ring, file, user_buf, file_posn)) < 0)
	7858	+ goto exit;
	7859	+
	7860	+ ring = &prot->d2hring_rx_cpln;
	7861	+ if ((ret = dhd_ring_write(dhd, ring, file, user_buf, file_posn)) < 0)
	7862	+ goto exit;
	7863	+
	7864	+ h2d_flowrings_total = dhd_get_max_flow_rings(dhd);
	7865	+ FOREACH_RING_IN_FLOWRINGS_POOL(prot, ring, flowid, h2d_flowrings_total) {
	7866	+ if ((ret = dhd_ring_write(dhd, ring, file, user_buf, file_posn)) < 0) {
	7867	+ goto exit;
	7868	+ }
	7869	+ }
	7870	+
	7871	+#ifdef EWP_EDL
	7872	+ if (dhd->dongle_edl_support) {
	7873	+ ring = prot->d2hring_edl;
	7874	+ if ((ret = dhd_edl_ring_hdr_write(dhd, ring, file, user_buf, file_posn)) < 0)
	7875	+ goto exit;
	7876	+ }
	7877	+ else if (dhd->bus->api.fw_rev >= PCIE_SHARED_VERSION_6 && !dhd->dongle_edl_support)
	7878	+#else
	7879	+ if (dhd->bus->api.fw_rev >= PCIE_SHARED_VERSION_6)
	7880	+#endif /* EWP_EDL */
	7881	+ {
	7882	+ ring = prot->h2dring_info_subn;
	7883	+ if ((ret = dhd_ring_write(dhd, ring, file, user_buf, file_posn)) < 0)
	7884	+ goto exit;
	7885	+
	7886	+ ring = prot->d2hring_info_cpln;
	7887	+ if ((ret = dhd_ring_write(dhd, ring, file, user_buf, file_posn)) < 0)
	7888	+ goto exit;
	7889	+ }
	7890	+
	7891	+exit :
	7892	+ return ret;
	7893	+}
	7894	+
	7895	+/* Write to file */
	7896	+static
	7897	+int dhd_ring_write(dhd_pub_t dhd, msgbuf_ring_t ring, void *file,
	7898	+ const void user_buf, unsigned long file_posn)
	7899	+{
	7900	+ int ret = 0;
	7901	+
	7902	+ if (ring == NULL) {
	7903	+ DHD_ERROR(("%s: Ring not initialised, failed to dump ring contents\n",
	7904	+ __FUNCTION__));
	7905	+ return BCME_ERROR;
	7906	+ }
	7907	+ if (file) {
	7908	+ ret = dhd_os_write_file_posn(file, file_posn, (char *)(ring->dma_buf.va),
	7909	+ ((unsigned long)(ring->max_items) * (ring->item_len)));
	7910	+ if (ret < 0) {
	7911	+ DHD_ERROR(("%s: write file error !\n", __FUNCTION__));
	7912	+ ret = BCME_ERROR;
	7913	+ }
	7914	+ } else if (user_buf) {
	7915	+ ret = dhd_export_debug_data((char *)(ring->dma_buf.va), NULL, user_buf,
	7916	+ ((unsigned long)(ring->max_items) * (ring->item_len)), (int *)file_posn);
	7917	+ }
	7918	+ return ret;
	7919	+}
	7920	+#endif /* DHD_DUMP_PCIE_RINGS */
	7921	+
	7922	+#ifdef EWP_EDL
	7923	+/* Write to file */
	7924	+static
	7925	+int dhd_edl_ring_hdr_write(dhd_pub_t dhd, msgbuf_ring_t ring, void file, const void user_buf,
	7926	+ unsigned long *file_posn)
	7927	+{
	7928	+ int ret = 0, nitems = 0;
	7929	+ char buf = NULL, ptr = NULL;
	7930	+ uint8 *msg_addr = NULL;
	7931	+ uint16 rd = 0;
	7932	+
	7933	+ if (ring == NULL) {
	7934	+ DHD_ERROR(("%s: Ring not initialised, failed to dump ring contents\n",
	7935	+ __FUNCTION__));
	7936	+ ret = BCME_ERROR;
	7937	+ goto done;
	7938	+ }
	7939	+
	7940	+ buf = MALLOCZ(dhd->osh, (D2HRING_EDL_MAX_ITEM * D2HRING_EDL_HDR_SIZE));
	7941	+ if (buf == NULL) {
	7942	+ DHD_ERROR(("%s: buffer allocation failed\n", __FUNCTION__));
	7943	+ ret = BCME_ERROR;
	7944	+ goto done;
	7945	+ }
	7946	+ ptr = buf;
	7947	+
	7948	+ for (; nitems < D2HRING_EDL_MAX_ITEM; nitems++, rd++) {
	7949	+ msg_addr = (uint8 )ring->dma_buf.va + (rd ring->item_len);
	7950	+ memcpy(ptr, (char *)msg_addr, D2HRING_EDL_HDR_SIZE);
	7951	+ ptr += D2HRING_EDL_HDR_SIZE;
	7952	+ }
	7953	+ if (file) {
	7954	+ ret = dhd_os_write_file_posn(file, file_posn, buf,
	7955	+ (D2HRING_EDL_HDR_SIZE * D2HRING_EDL_MAX_ITEM));
	7956	+ if (ret < 0) {
	7957	+ DHD_ERROR(("%s: write file error !\n", __FUNCTION__));
	7958	+ goto done;
	7959	+ }
	7960	+ }
	7961	+ else {
	7962	+ ret = dhd_export_debug_data(buf, NULL, user_buf,
	7963	+ (D2HRING_EDL_HDR_SIZE * D2HRING_EDL_MAX_ITEM), file_posn);
	7964	+ }
	7965	+
	7966	+done:
	7967	+ if (buf) {
	7968	+ MFREE(dhd->osh, buf, (D2HRING_EDL_MAX_ITEM * D2HRING_EDL_HDR_SIZE));
	7969	+ }
	7970	+ return ret;
	7971	+}
	7972	+#endif /* EWP_EDL */
4757	7973
4758	7974	/** Add prot dump output to a buffer */
4759	7975	void dhd_prot_dump(dhd_pub_t dhd, struct bcmstrbuf b)
4760	7976	{
4761	7977
4762		-#if defined(PCIE_D2H_SYNC)
4763	7978	if (dhd->d2h_sync_mode & PCIE_SHARED_D2H_SYNC_SEQNUM)
4764	7979	bcm_bprintf(b, "\nd2h_sync: SEQNUM:");
4765	7980	else if (dhd->d2h_sync_mode & PCIE_SHARED_D2H_SYNC_XORCSUM)
..	..	@@ -4768,12 +7983,16 @@
4768	7983	bcm_bprintf(b, "\nd2h_sync: NONE:");
4769	7984	bcm_bprintf(b, " d2h_sync_wait max<%lu> tot<%lu>\n",
4770	7985	dhd->prot->d2h_sync_wait_max, dhd->prot->d2h_sync_wait_tot);
4771		-#endif /* PCIE_D2H_SYNC */
4772	7986
4773	7987	bcm_bprintf(b, "\nDongle DMA Indices: h2d %d d2h %d index size %d bytes\n",
4774		- DMA_INDX_ENAB(dhd->dma_h2d_ring_upd_support),
4775		- DMA_INDX_ENAB(dhd->dma_d2h_ring_upd_support),
	7988	+ dhd->dma_h2d_ring_upd_support,
	7989	+ dhd->dma_d2h_ring_upd_support,
4776	7990	dhd->prot->rw_index_sz);
	7991	+ bcm_bprintf(b, "h2d_max_txpost: %d, prot->h2d_max_txpost: %d\n",
	7992	+ h2d_max_txpost, dhd->prot->h2d_max_txpost);
	7993	+ bcm_bprintf(b, "pktid_txq_start_cnt: %d\n", dhd->prot->pktid_txq_start_cnt);
	7994	+ bcm_bprintf(b, "pktid_txq_stop_cnt: %d\n", dhd->prot->pktid_txq_stop_cnt);
	7995	+ bcm_bprintf(b, "pktid_depleted_cnt: %d\n", dhd->prot->pktid_depleted_cnt);
4777	7996	}
4778	7997
4779	7998	/* Update local copy of dongle statistics */
..	..	@@ -4800,13 +8019,13 @@
4800	8019	dhd_prot_t *prot = dhd->prot;
4801	8020	msgbuf_ring_t *ring = &prot->h2dring_ctrl_subn;
4802	8021
4803		- DHD_GENERAL_LOCK(dhd, flags);
	8022	+ DHD_RING_LOCK(ring->ring_lock, flags);
4804	8023
4805	8024	hevent = (hostevent_hdr_t *)
4806	8025	dhd_prot_alloc_ring_space(dhd, ring, 1, &alloced, FALSE);
4807	8026
4808	8027	if (hevent == NULL) {
4809		- DHD_GENERAL_UNLOCK(dhd, flags);
	8028	+ DHD_RING_UNLOCK(ring->ring_lock, flags);
4810	8029	return -1;
4811	8030	}
4812	8031
..	..	@@ -4815,6 +8034,7 @@
4815	8034	ring->seqnum++;
4816	8035	hevent->msg.msg_type = MSG_TYPE_HOST_EVNT;
4817	8036	hevent->msg.if_id = 0;
	8037	+ hevent->msg.flags = ring->current_phase;
4818	8038
4819	8039	/* Event payload */
4820	8040	hevent->evnt_pyld = htol32(HOST_EVENT_CONS_CMD);
..	..	@@ -4823,7 +8043,8 @@
4823	8043	* from the msgbuf, we can directly call the write_complete
4824	8044	*/
4825	8045	dhd_prot_ring_write_complete(dhd, ring, hevent, 1);
4826		- DHD_GENERAL_UNLOCK(dhd, flags);
	8046	+
	8047	+ DHD_RING_UNLOCK(ring->ring_lock, flags);
4827	8048
4828	8049	return 0;
4829	8050	}
..	..	@@ -4842,11 +8063,22 @@
4842	8063	ret_buf = dhd_prot_get_ring_space(ring, nitems, alloced, exactly_nitems);
4843	8064
4844	8065	if (ret_buf == NULL) {
	8066	+ /* HWA TODO, need to get RD pointer from different array
	8067	+ * which HWA will directly write into host memory
	8068	+ */
4845	8069	/* if alloc failed , invalidate cached read ptr */
4846		- if (DMA_INDX_ENAB(dhd->dma_d2h_ring_upd_support)) {
	8070	+ if (dhd->dma_d2h_ring_upd_support) {
4847	8071	ring->rd = dhd_prot_dma_indx_get(dhd, H2D_DMA_INDX_RD_UPD, ring->idx);
4848	8072	} else {
4849	8073	dhd_bus_cmn_readshared(dhd->bus, &(ring->rd), RING_RD_UPD, ring->idx);
	8074	+#ifdef SUPPORT_LINKDOWN_RECOVERY
	8075	+ /* Check if ring->rd is valid */
	8076	+ if (ring->rd >= ring->max_items) {
	8077	+ DHD_ERROR(("%s: Invalid rd idx=%d\n", ring->name, ring->rd));
	8078	+ dhd->bus->read_shm_fail = TRUE;
	8079	+ return NULL;
	8080	+ }
	8081	+#endif /* SUPPORT_LINKDOWN_RECOVERY */
4850	8082	}
4851	8083
4852	8084	/* Try allocating once more */
..	..	@@ -4856,6 +8088,11 @@
4856	8088	DHD_INFO(("%s: Ring space not available \n", ring->name));
4857	8089	return NULL;
4858	8090	}
	8091	+ }
	8092	+
	8093	+ if (ret_buf == HOST_RING_BASE(ring)) {
	8094	+ DHD_INFO(("%s: setting the phase now\n", ring->name));
	8095	+ ring->current_phase = ring->current_phase ? 0 : BCMPCIE_CMNHDR_PHASE_BIT_INIT;
4859	8096	}
4860	8097
4861	8098	/* Return alloced space */
..	..	@@ -4879,20 +8116,24 @@
4879	8116	uint16 alloced = 0;
4880	8117	msgbuf_ring_t *ring = &prot->h2dring_ctrl_subn;
4881	8118
	8119	+ if (dhd_query_bus_erros(dhd)) {
	8120	+ return -EIO;
	8121	+ }
	8122	+
4882	8123	rqstlen = len;
4883	8124	resplen = len;
4884	8125
4885	8126	/* Limit ioct request to MSGBUF_MAX_MSG_SIZE bytes including hdrs */
4886	8127	/* 8K allocation of dongle buffer fails */
4887	8128	/* dhd doesnt give separate input & output buf lens */
4888		- /* so making the assumption that input length can never be more than 1.5k */
4889		- rqstlen = MIN(rqstlen, MSGBUF_MAX_MSG_SIZE);
	8129	+ /* so making the assumption that input length can never be more than 2k */
	8130	+ rqstlen = MIN(rqstlen, MSGBUF_IOCTL_MAX_RQSTLEN);
4890	8131
4891		- DHD_GENERAL_LOCK(dhd, flags);
	8132	+ DHD_RING_LOCK(ring->ring_lock, flags);
4892	8133
4893	8134	if (prot->ioctl_state) {
4894	8135	DHD_ERROR(("%s: pending ioctl %02x\n", __FUNCTION__, prot->ioctl_state));
4895		- DHD_GENERAL_UNLOCK(dhd, flags);
	8136	+ DHD_RING_UNLOCK(ring->ring_lock, flags);
4896	8137	return BCME_BUSY;
4897	8138	} else {
4898	8139	prot->ioctl_state = MSGBUF_IOCTL_ACK_PENDING \| MSGBUF_IOCTL_RESP_PENDING;
..	..	@@ -4906,14 +8147,14 @@
4906	8147	prot->ioctl_state = 0;
4907	8148	prot->curr_ioctl_cmd = 0;
4908	8149	prot->ioctl_received = IOCTL_WAIT;
4909		- DHD_GENERAL_UNLOCK(dhd, flags);
	8150	+ DHD_RING_UNLOCK(ring->ring_lock, flags);
4910	8151	return -1;
4911	8152	}
4912	8153
4913	8154	/* Common msg buf hdr */
4914	8155	ioct_rqst->cmn_hdr.msg_type = MSG_TYPE_IOCTLPTR_REQ;
4915	8156	ioct_rqst->cmn_hdr.if_id = (uint8)ifidx;
4916		- ioct_rqst->cmn_hdr.flags = 0;
	8157	+ ioct_rqst->cmn_hdr.flags = ring->current_phase;
4917	8158	ioct_rqst->cmn_hdr.request_id = htol32(DHD_IOCTL_REQ_PKTID);
4918	8159	ioct_rqst->cmn_hdr.epoch = ring->seqnum % H2D_EPOCH_MODULO;
4919	8160	ring->seqnum++;
..	..	@@ -4931,15 +8172,15 @@
4931	8172	/* copy ioct payload */
4932	8173	ioct_buf = (void *) prot->ioctbuf.va;
4933	8174
4934		- if (buf) {
	8175	+ prot->ioctl_fillup_time = OSL_LOCALTIME_NS();
	8176	+
	8177	+ if (buf)
4935	8178	memcpy(ioct_buf, buf, len);
4936		- }
4937	8179
4938	8180	OSL_CACHE_FLUSH((void *) prot->ioctbuf.va, len);
4939	8181
4940		- if (!ISALIGNED(ioct_buf, DMA_ALIGN_LEN)) {
	8182	+ if (!ISALIGNED(ioct_buf, DMA_ALIGN_LEN))
4941	8183	DHD_ERROR(("host ioct address unaligned !!!!! \n"));
4942		- }
4943	8184
4944	8185	DHD_CTL(("submitted IOCTL request request_id %d, cmd %d, output_buf_len %d, tx_id %d\n",
4945	8186	ioct_rqst->cmn_hdr.request_id, cmd, ioct_rqst->output_buf_len,
..	..	@@ -4947,11 +8188,11 @@
4947	8188
4948	8189	/* update ring's WR index and ring doorbell to dongle */
4949	8190	dhd_prot_ring_write_complete(dhd, ring, ioct_rqst, 1);
4950		- DHD_GENERAL_UNLOCK(dhd, flags);
	8191	+
	8192	+ DHD_RING_UNLOCK(ring->ring_lock, flags);
4951	8193
4952	8194	return 0;
4953	8195	} /* dhd_fillup_ioct_reqst */
4954		-
4955	8196
4956	8197	/**
4957	8198	* dhd_prot_ring_attach - Initialize the msgbuf_ring object and attach a
..	..	@@ -4971,6 +8212,8 @@
4971	8212	int dma_buf_alloced = BCME_NOMEM;
4972	8213	uint32 dma_buf_len = max_items * item_len;
4973	8214	dhd_prot_t *prot = dhd->prot;
	8215	+ uint16 max_flowrings = dhd->bus->max_tx_flowrings;
	8216	+ dhd_dma_buf_t *dma_buf = NULL;
4974	8217
4975	8218	ASSERT(ring);
4976	8219	ASSERT(name);
..	..	@@ -4986,13 +8229,13 @@
4986	8229	ring->item_len = item_len;
4987	8230
4988	8231	/* A contiguous space may be reserved for all flowrings */
4989		- if (DHD_IS_FLOWRING(ringid) && (prot->flowrings_dma_buf.va)) {
	8232	+ if (DHD_IS_FLOWRING(ringid, max_flowrings) && (prot->flowrings_dma_buf.va)) {
4990	8233	/* Carve out from the contiguous DMA-able flowring buffer */
4991	8234	uint16 flowid;
4992	8235	uint32 base_offset;
4993	8236
4994		- dhd_dma_buf_t *dma_buf = &ring->dma_buf;
4995	8237	dhd_dma_buf_t *rsv_buf = &prot->flowrings_dma_buf;
	8238	+ dma_buf = &ring->dma_buf;
4996	8239
4997	8240	flowid = DHD_RINGID_TO_FLOWID(ringid);
4998	8241	base_offset = (flowid - BCMPCIE_H2D_COMMON_MSGRINGS) * dma_buf_len;
..	..	@@ -5012,10 +8255,24 @@
5012	8255
5013	8256	(void)dhd_dma_buf_audit(dhd, &ring->dma_buf);
5014	8257	} else {
5015		- /* Allocate a dhd_dma_buf */
5016		- dma_buf_alloced = dhd_dma_buf_alloc(dhd, &ring->dma_buf, dma_buf_len);
5017		- if (dma_buf_alloced != BCME_OK) {
5018		- return BCME_NOMEM;
	8258	+#ifdef EWP_EDL
	8259	+ if (ring == dhd->prot->d2hring_edl) {
	8260	+ /* For EDL ring, memory is alloced during attach,
	8261	+ * so just need to copy the dma_buf to the ring's dma_buf
	8262	+ */
	8263	+ memcpy(&ring->dma_buf, &dhd->edl_ring_mem, sizeof(ring->dma_buf));
	8264	+ dma_buf = &ring->dma_buf;
	8265	+ if (dma_buf->va == NULL) {
	8266	+ return BCME_NOMEM;
	8267	+ }
	8268	+ } else
	8269	+#endif /* EWP_EDL */
	8270	+ {
	8271	+ /* Allocate a dhd_dma_buf */
	8272	+ dma_buf_alloced = dhd_dma_buf_alloc(dhd, &ring->dma_buf, dma_buf_len);
	8273	+ if (dma_buf_alloced != BCME_OK) {
	8274	+ return BCME_NOMEM;
	8275	+ }
5019	8276	}
5020	8277	}
5021	8278
..	..	@@ -5030,6 +8287,8 @@
5030	8287	}
5031	8288	}
5032	8289	#endif /* BCM_SECURE_DMA */
	8290	+
	8291	+ ring->ring_lock = dhd_os_spin_lock_init(dhd->osh);
5033	8292
5034	8293	DHD_INFO(("RING_ATTACH : %s Max item %d len item %d total size %d "
5035	8294	"ring start %p buf phys addr %x:%x \n",
..	..	@@ -5050,7 +8309,6 @@
5050	8309
5051	8310	} /* dhd_prot_ring_attach */
5052	8311
5053		-
5054	8312	/**
5055	8313	* dhd_prot_ring_init - Post the common ring information to dongle.
5056	8314	*
..	..	@@ -5065,6 +8323,12 @@
5065	8323	{
5066	8324	ring->wr = 0;
5067	8325	ring->rd = 0;
	8326	+ ring->curr_rd = 0;
	8327	+ /* Reset hwa_db_type for all rings,
	8328	+ * for data path rings, it will be assigned separately post init
	8329	+ * from dhd_prot_d2h_sync_init and dhd_prot_h2d_sync_init
	8330	+ */
	8331	+ ring->hwa_db_type = 0;
5068	8332
5069	8333	/* CAUTION: ring::base_addr already in Little Endian */
5070	8334	dhd_bus_cmn_writeshared(dhd->bus, &ring->base_addr,
..	..	@@ -5084,7 +8348,6 @@
5084	8348
5085	8349	} /* dhd_prot_ring_init */
5086	8350
5087		-
5088	8351	/**
5089	8352	* dhd_prot_ring_reset - bzero a ring's DMA-ble buffer and cache flush
5090	8353	* Reset WR and RD indices to 0.
..	..	@@ -5097,8 +8360,10 @@
5097	8360	dhd_dma_buf_reset(dhd, &ring->dma_buf);
5098	8361
5099	8362	ring->rd = ring->wr = 0;
	8363	+ ring->curr_rd = 0;
	8364	+ ring->inited = FALSE;
	8365	+ ring->create_pending = FALSE;
5100	8366	}
5101		-
5102	8367
5103	8368	/**
5104	8369	* dhd_prot_ring_detach - Detach the DMA-able buffer and any other objects
..	..	@@ -5108,6 +8373,7 @@
5108	8373	dhd_prot_ring_detach(dhd_pub_t dhd, msgbuf_ring_t ring)
5109	8374	{
5110	8375	dhd_prot_t *prot = dhd->prot;
	8376	+ uint16 max_flowrings = dhd->bus->max_tx_flowrings;
5111	8377	ASSERT(ring);
5112	8378
5113	8379	ring->inited = FALSE;
..	..	@@ -5126,76 +8392,26 @@
5126	8392	/* If the DMA-able buffer was carved out of a pre-reserved contiguous
5127	8393	* memory, then simply stop using it.
5128	8394	*/
5129		- if (DHD_IS_FLOWRING(ring->idx) && (prot->flowrings_dma_buf.va)) {
	8395	+ if (DHD_IS_FLOWRING(ring->idx, max_flowrings) && (prot->flowrings_dma_buf.va)) {
5130	8396	(void)dhd_dma_buf_audit(dhd, &ring->dma_buf);
5131	8397	memset(&ring->dma_buf, 0, sizeof(dhd_dma_buf_t));
5132	8398	} else {
5133	8399	dhd_dma_buf_free(dhd, &ring->dma_buf);
5134	8400	}
5135	8401
	8402	+ dhd_os_spin_lock_deinit(dhd->osh, ring->ring_lock);
	8403	+
5136	8404	} /* dhd_prot_ring_detach */
5137	8405
5138		-
5139		-/*
5140		- * +----------------------------------------------------------------------------
5141		- * Flowring Pool
5142		- *
5143		- * Unlike common rings, which are attached very early on (dhd_prot_attach),
5144		- * flowrings are dynamically instantiated. Moreover, flowrings may require a
5145		- * larger DMA-able buffer. To avoid issues with fragmented cache coherent
5146		- * DMA-able memory, a pre-allocated pool of msgbuf_ring_t is allocated once.
5147		- * The DMA-able buffers are attached to these pre-allocated msgbuf_ring.
5148		- *
5149		- * Each DMA-able buffer may be allocated independently, or may be carved out
5150		- * of a single large contiguous region that is registered with the protocol
5151		- * layer into flowrings_dma_buf. On a 64bit platform, this contiguous region
5152		- * may not span 0x00000000FFFFFFFF (avoid dongle side 64bit ptr arithmetic).
5153		- *
5154		- * No flowring pool action is performed in dhd_prot_attach(), as the number
5155		- * of h2d rings is not yet known.
5156		- *
5157		- * In dhd_prot_init(), the dongle advertized number of h2d rings is used to
5158		- * determine the number of flowrings required, and a pool of msgbuf_rings are
5159		- * allocated and a DMA-able buffer (carved or allocated) is attached.
5160		- * See: dhd_prot_flowrings_pool_attach()
5161		- *
5162		- * A flowring msgbuf_ring object may be fetched from this pool during flowring
5163		- * creation, using the flowid. Likewise, flowrings may be freed back into the
5164		- * pool on flowring deletion.
5165		- * See: dhd_prot_flowrings_pool_fetch(), dhd_prot_flowrings_pool_release()
5166		- *
5167		- * In dhd_prot_detach(), the flowring pool is detached. The DMA-able buffers
5168		- * are detached (returned back to the carved region or freed), and the pool of
5169		- * msgbuf_ring and any objects allocated against it are freed.
5170		- * See: dhd_prot_flowrings_pool_detach()
5171		- *
5172		- * In dhd_prot_reset(), the flowring pool is simply reset by returning it to a
5173		- * state as-if upon an attach. All DMA-able buffers are retained.
5174		- * Following a dhd_prot_reset(), in a subsequent dhd_prot_init(), the flowring
5175		- * pool attach will notice that the pool persists and continue to use it. This
5176		- * will avoid the case of a fragmented DMA-able region.
5177		- *
5178		- * +----------------------------------------------------------------------------
5179		- */
5180		-
5181	8406	/* Fetch number of H2D flowrings given the total number of h2d rings */
5182		-#define DHD_FLOWRINGS_POOL_TOTAL(h2d_rings_total) \
5183		- ((h2d_rings_total) - BCMPCIE_H2D_COMMON_MSGRINGS)
5184		-
5185		-/* Conversion of a flowid to a flowring pool index */
5186		-#define DHD_FLOWRINGS_POOL_OFFSET(flowid) \
5187		- ((flowid) - BCMPCIE_H2D_COMMON_MSGRINGS)
5188		-
5189		-/* Fetch the msgbuf_ring_t from the flowring pool given a flowid */
5190		-#define DHD_RING_IN_FLOWRINGS_POOL(prot, flowid) \
5191		- (msgbuf_ring_t*)((prot)->h2d_flowrings_pool) + DHD_FLOWRINGS_POOL_OFFSET(flowid)
5192		-
5193		-/* Traverse each flowring in the flowring pool, assigning ring and flowid */
5194		-#define FOREACH_RING_IN_FLOWRINGS_POOL(prot, ring, flowid) \
5195		- for ((flowid) = DHD_FLOWRING_START_FLOWID, \
5196		- (ring) = DHD_RING_IN_FLOWRINGS_POOL(prot, flowid); \
5197		- (flowid) < (prot)->h2d_rings_total; \
5198		- (flowid)++, (ring)++)
	8407	+uint16
	8408	+dhd_get_max_flow_rings(dhd_pub_t *dhd)
	8409	+{
	8410	+ if (dhd->bus->api.fw_rev >= PCIE_SHARED_VERSION_6)
	8411	+ return dhd->bus->max_tx_flowrings;
	8412	+ else
	8413	+ return (dhd->bus->max_tx_flowrings - BCMPCIE_H2D_COMMON_MSGRINGS);
	8414	+}
5199	8415
5200	8416	/**
5201	8417	* dhd_prot_flowrings_pool_attach - Initialize a pool of flowring msgbuf_ring_t.
..	..	@@ -5223,9 +8439,8 @@
5223	8439	dhd_prot_t *prot = dhd->prot;
5224	8440	char ring_name[RING_NAME_MAX_LENGTH];
5225	8441
5226		- if (prot->h2d_flowrings_pool != NULL) {
	8442	+ if (prot->h2d_flowrings_pool != NULL)
5227	8443	return BCME_OK; /* dhd_prot_init rentry after a dhd_prot_reset */
5228		- }
5229	8444
5230	8445	ASSERT(prot->h2d_rings_total == 0);
5231	8446
..	..	@@ -5239,7 +8454,7 @@
5239	8454	}
5240	8455
5241	8456	/* Subtract number of H2D common rings, to determine number of flowrings */
5242		- h2d_flowrings_total = DHD_FLOWRINGS_POOL_TOTAL(prot->h2d_rings_total);
	8457	+ h2d_flowrings_total = dhd_get_max_flow_rings(dhd);
5243	8458
5244	8459	DHD_ERROR(("Attach flowrings pool for %d rings\n", h2d_flowrings_total));
5245	8460
..	..	@@ -5254,14 +8469,18 @@
5254	8469	}
5255	8470
5256	8471	/* Setup & Attach a DMA-able buffer to each flowring in the flowring pool */
5257		- FOREACH_RING_IN_FLOWRINGS_POOL(prot, ring, flowid) {
	8472	+ FOREACH_RING_IN_FLOWRINGS_POOL(prot, ring, flowid, h2d_flowrings_total) {
5258	8473	snprintf(ring_name, sizeof(ring_name), "h2dflr_%03u", flowid);
5259		- ring_name[RING_NAME_MAX_LENGTH - 1] = '\0';
5260	8474	if (dhd_prot_ring_attach(dhd, ring, ring_name,
5261		- H2DRING_TXPOST_MAX_ITEM, H2DRING_TXPOST_ITEMSIZE,
	8475	+ prot->h2d_max_txpost, H2DRING_TXPOST_ITEMSIZE,
5262	8476	DHD_FLOWID_TO_RINGID(flowid)) != BCME_OK) {
5263	8477	goto attach_fail;
5264	8478	}
	8479	+ /*
	8480	+ * TOD0 - Currently flowrings hwa is disabled and can be enabled like below
	8481	+ * (dhd->bus->hwa_enab_bmap & HWA_ENAB_BITMAP_TXPOSTS) ? HWA_DB_TYPE_TXPOSTS : 0;
	8482	+ */
	8483	+ ring->hwa_db_type = 0;
5265	8484	}
5266	8485
5267	8486	return BCME_OK;
..	..	@@ -5274,7 +8493,6 @@
5274	8493	return BCME_NOMEM;
5275	8494
5276	8495	} /* dhd_prot_flowrings_pool_attach */
5277		-
5278	8496
5279	8497	/**
5280	8498	* dhd_prot_flowrings_pool_reset - Reset all msgbuf_ring_t objects in the pool.
..	..	@@ -5294,7 +8512,7 @@
5294	8512	static void
5295	8513	dhd_prot_flowrings_pool_reset(dhd_pub_t *dhd)
5296	8514	{
5297		- uint16 flowid;
	8515	+ uint16 flowid, h2d_flowrings_total;
5298	8516	msgbuf_ring_t *ring;
5299	8517	dhd_prot_t *prot = dhd->prot;
5300	8518
..	..	@@ -5302,16 +8520,15 @@
5302	8520	ASSERT(prot->h2d_rings_total == 0);
5303	8521	return;
5304	8522	}
5305		-
	8523	+ h2d_flowrings_total = dhd_get_max_flow_rings(dhd);
5306	8524	/* Reset each flowring in the flowring pool */
5307		- FOREACH_RING_IN_FLOWRINGS_POOL(prot, ring, flowid) {
	8525	+ FOREACH_RING_IN_FLOWRINGS_POOL(prot, ring, flowid, h2d_flowrings_total) {
5308	8526	dhd_prot_ring_reset(dhd, ring);
5309	8527	ring->inited = FALSE;
5310	8528	}
5311	8529
5312	8530	/* Flowring pool state must be as-if dhd_prot_flowrings_pool_attach */
5313	8531	}
5314		-
5315	8532
5316	8533	/**
5317	8534	* dhd_prot_flowrings_pool_detach - Free pool of msgbuf_ring along with
..	..	@@ -5324,7 +8541,7 @@
5324	8541	{
5325	8542	int flowid;
5326	8543	msgbuf_ring_t *ring;
5327		- int h2d_flowrings_total; /* exclude H2D common rings */
	8544	+ uint16 h2d_flowrings_total; /* exclude H2D common rings */
5328	8545	dhd_prot_t *prot = dhd->prot;
5329	8546
5330	8547	if (prot->h2d_flowrings_pool == NULL) {
..	..	@@ -5332,12 +8549,11 @@
5332	8549	return;
5333	8550	}
5334	8551
	8552	+ h2d_flowrings_total = dhd_get_max_flow_rings(dhd);
5335	8553	/* Detach the DMA-able buffer for each flowring in the flowring pool */
5336		- FOREACH_RING_IN_FLOWRINGS_POOL(prot, ring, flowid) {
	8554	+ FOREACH_RING_IN_FLOWRINGS_POOL(prot, ring, flowid, h2d_flowrings_total) {
5337	8555	dhd_prot_ring_detach(dhd, ring);
5338	8556	}
5339		-
5340		- h2d_flowrings_total = DHD_FLOWRINGS_POOL_TOTAL(prot->h2d_rings_total);
5341	8557
5342	8558	MFREE(prot->osh, prot->h2d_flowrings_pool,
5343	8559	(h2d_flowrings_total * sizeof(msgbuf_ring_t)));
..	..	@@ -5346,7 +8562,6 @@
5346	8562	prot->h2d_rings_total = 0;
5347	8563
5348	8564	} /* dhd_prot_flowrings_pool_detach */
5349		-
5350	8565
5351	8566	/**
5352	8567	* dhd_prot_flowrings_pool_fetch - Fetch a preallocated and initialized
..	..	@@ -5374,11 +8589,15 @@
5374	8589
5375	8590	ring->wr = 0;
5376	8591	ring->rd = 0;
	8592	+ ring->curr_rd = 0;
5377	8593	ring->inited = TRUE;
5378		-
	8594	+ /**
	8595	+ * Every time a flowring starts dynamically, initialize current_phase with 0
	8596	+ * then flip to BCMPCIE_CMNHDR_PHASE_BIT_INIT
	8597	+ */
	8598	+ ring->current_phase = 0;
5379	8599	return ring;
5380	8600	}
5381		-
5382	8601
5383	8602	/**
5384	8603	* dhd_prot_flowrings_pool_release - release a previously fetched flowring's
..	..	@@ -5404,8 +8623,9 @@
5404	8623	ring->wr = 0;
5405	8624	ring->rd = 0;
5406	8625	ring->inited = FALSE;
5407		-}
5408	8626
	8627	+ ring->curr_rd = 0;
	8628	+}
5409	8629
5410	8630	/* Assumes only one index is updated at a time */
5411	8631	/* If exactly_nitems is true, this function will allocate space for nitems or fail */
..	..	@@ -5435,11 +8655,11 @@
5435	8655	ret_ptr = (char )DHD_RING_BGN_VA(ring) + (ring->wr ring->item_len);
5436	8656
5437	8657	/* Update write index */
5438		- if ((ring->wr + *alloced) == ring->max_items) {
	8658	+ if ((ring->wr + *alloced) == ring->max_items)
5439	8659	ring->wr = 0;
5440		- } else if ((ring->wr + *alloced) < ring->max_items) {
	8660	+ else if ((ring->wr + *alloced) < ring->max_items)
5441	8661	ring->wr += *alloced;
5442		- } else {
	8662	+ else {
5443	8663	/* Should never hit this */
5444	8664	ASSERT(0);
5445	8665	return NULL;
..	..	@@ -5448,35 +8668,97 @@
5448	8668	return ret_ptr;
5449	8669	} /* dhd_prot_get_ring_space */
5450	8670
5451		-
5452	8671	/**
5453	8672	* dhd_prot_ring_write_complete - Host updates the new WR index on producing
5454	8673	* new messages in a H2D ring. The messages are flushed from cache prior to
5455	8674	* posting the new WR index. The new WR index will be updated in the DMA index
5456	8675	* array or directly in the dongle's ring state memory.
5457	8676	* A PCIE doorbell will be generated to wake up the dongle.
	8677	+ * This is a non-atomic function, make sure the callers
	8678	+ * always hold appropriate locks.
5458	8679	*/
5459	8680	static void BCMFASTPATH
5460		-dhd_prot_ring_write_complete(dhd_pub_t dhd, msgbuf_ring_t ring, void* p,
	8681	+__dhd_prot_ring_write_complete(dhd_pub_t dhd, msgbuf_ring_t ring, void* p,
5461	8682	uint16 nitems)
5462	8683	{
5463	8684	dhd_prot_t *prot = dhd->prot;
	8685	+ uint32 db_index;
	8686	+ uint16 max_flowrings = dhd->bus->max_tx_flowrings;
	8687	+ uint corerev;
5464	8688
5465	8689	/* cache flush */
5466	8690	OSL_CACHE_FLUSH(p, ring->item_len * nitems);
5467	8691
5468		- if (DMA_INDX_ENAB(dhd->dma_h2d_ring_upd_support)) {
5469		- dhd_prot_dma_indx_set(dhd, ring->wr,
5470		- H2D_DMA_INDX_WR_UPD, ring->idx);
	8692	+ /* For HWA, update db_index and ring mb2 DB and return */
	8693	+ if (HWA_ACTIVE(dhd) && ring->hwa_db_type) {
	8694	+ db_index = HWA_DB_INDEX_VALUE(ring->wr) \| ring->hwa_db_type;
	8695	+ DHD_TRACE(("%s: ring(%s) wr(%d) hwa_db_type(0x%x) db_index(0x%x)\n",
	8696	+ __FUNCTION__, ring->name, ring->wr, ring->hwa_db_type, db_index));
	8697	+ prot->mb_2_ring_fn(dhd->bus, db_index, TRUE);
	8698	+ return;
	8699	+ }
	8700	+
	8701	+ if (IDMA_ACTIVE(dhd) \|\| dhd->dma_h2d_ring_upd_support) {
	8702	+ dhd_prot_dma_indx_set(dhd, ring->wr,
	8703	+ H2D_DMA_INDX_WR_UPD, ring->idx);
	8704	+ } else if (IFRM_ACTIVE(dhd) && DHD_IS_FLOWRING(ring->idx, max_flowrings)) {
	8705	+ dhd_prot_dma_indx_set(dhd, ring->wr,
	8706	+ H2D_IFRM_INDX_WR_UPD, ring->idx);
5471	8707	} else {
5472		- dhd_bus_cmn_writeshared(dhd->bus, &(ring->wr),
5473		- sizeof(uint16), RING_WR_UPD, ring->idx);
	8708	+ dhd_bus_cmn_writeshared(dhd->bus, &(ring->wr),
	8709	+ sizeof(uint16), RING_WR_UPD, ring->idx);
5474	8710	}
5475	8711
5476	8712	/* raise h2d interrupt */
5477		- prot->mb_ring_fn(dhd->bus, ring->wr);
	8713	+ if (IDMA_ACTIVE(dhd) \|\|
	8714	+ (IFRM_ACTIVE(dhd) && DHD_IS_FLOWRING(ring->idx, max_flowrings))) {
	8715	+ db_index = IDMA_IDX0;
	8716	+ /* this api is called in wl down path..in that case sih is freed already */
	8717	+ if (dhd->bus->sih) {
	8718	+ corerev = dhd->bus->sih->buscorerev;
	8719	+ /* We need to explictly configure the type of DMA for core rev >= 24 */
	8720	+ if (corerev >= 24) {
	8721	+ db_index \|= (DMA_TYPE_IDMA << DMA_TYPE_SHIFT);
	8722	+ }
	8723	+ }
	8724	+ prot->mb_2_ring_fn(dhd->bus, db_index, TRUE);
	8725	+ } else {
	8726	+ prot->mb_ring_fn(dhd->bus, ring->wr);
	8727	+ }
5478	8728	}
5479	8729
	8730	+static void BCMFASTPATH
	8731	+dhd_prot_ring_write_complete(dhd_pub_t dhd, msgbuf_ring_t ring, void* p,
	8732	+ uint16 nitems)
	8733	+{
	8734	+ unsigned long flags_bus;
	8735	+ DHD_BUS_LOCK(dhd->bus->bus_lock, flags_bus);
	8736	+ __dhd_prot_ring_write_complete(dhd, ring, p, nitems);
	8737	+ DHD_BUS_UNLOCK(dhd->bus->bus_lock, flags_bus);
	8738	+}
	8739	+
	8740	+/**
	8741	+ * dhd_prot_ring_write_complete_mbdata - will be called from dhd_prot_h2d_mbdata_send_ctrlmsg,
	8742	+ * which will hold DHD_BUS_LOCK to update WR pointer, Ring DB and also update bus_low_power_state
	8743	+ * to indicate D3_INFORM sent in the same BUS_LOCK.
	8744	+ */
	8745	+static void BCMFASTPATH
	8746	+dhd_prot_ring_write_complete_mbdata(dhd_pub_t dhd, msgbuf_ring_t ring, void *p,
	8747	+ uint16 nitems, uint32 mb_data)
	8748	+{
	8749	+ unsigned long flags_bus;
	8750	+
	8751	+ DHD_BUS_LOCK(dhd->bus->bus_lock, flags_bus);
	8752	+
	8753	+ __dhd_prot_ring_write_complete(dhd, ring, p, nitems);
	8754	+
	8755	+ /* Mark D3_INFORM in the same context to skip ringing H2D DB after D3_INFORM */
	8756	+ if (mb_data == H2D_HOST_D3_INFORM) {
	8757	+ dhd->bus->bus_low_power_state = DHD_BUS_D3_INFORM_SENT;
	8758	+ }
	8759	+
	8760	+ DHD_BUS_UNLOCK(dhd->bus->bus_lock, flags_bus);
	8761	+}
5480	8762
5481	8763	/**
5482	8764	* dhd_prot_upd_read_idx - Host updates the new RD index on consuming messages
..	..	@@ -5486,32 +8768,205 @@
5486	8768	static void
5487	8769	dhd_prot_upd_read_idx(dhd_pub_t dhd, msgbuf_ring_t ring)
5488	8770	{
	8771	+ dhd_prot_t *prot = dhd->prot;
	8772	+ uint32 db_index;
	8773	+ uint corerev;
	8774	+
	8775	+ /* For HWA, update db_index and ring mb2 DB and return */
	8776	+ if (HWA_ACTIVE(dhd) && ring->hwa_db_type) {
	8777	+ db_index = HWA_DB_INDEX_VALUE(ring->rd) \| ring->hwa_db_type;
	8778	+ DHD_TRACE(("%s: ring(%s) rd(0x%x) hwa_db_type(0x%x) db_index(0x%x)\n",
	8779	+ __FUNCTION__, ring->name, ring->rd, ring->hwa_db_type, db_index));
	8780	+ prot->mb_2_ring_fn(dhd->bus, db_index, FALSE);
	8781	+ return;
	8782	+ }
	8783	+
5489	8784	/* update read index */
5490	8785	/* If dma'ing h2d indices supported
5491	8786	* update the r -indices in the
5492	8787	* host memory o/w in TCM
5493	8788	*/
5494		- if (DMA_INDX_ENAB(dhd->dma_h2d_ring_upd_support)) {
	8789	+ if (IDMA_ACTIVE(dhd)) {
5495	8790	dhd_prot_dma_indx_set(dhd, ring->rd,
5496	8791	D2H_DMA_INDX_RD_UPD, ring->idx);
	8792	+ db_index = IDMA_IDX1;
	8793	+ if (dhd->bus->sih) {
	8794	+ corerev = dhd->bus->sih->buscorerev;
	8795	+ /* We need to explictly configure the type of DMA for core rev >= 24 */
	8796	+ if (corerev >= 24) {
	8797	+ db_index \|= (DMA_TYPE_IDMA << DMA_TYPE_SHIFT);
	8798	+ }
	8799	+ }
	8800	+ prot->mb_2_ring_fn(dhd->bus, db_index, FALSE);
	8801	+ } else if (dhd->dma_h2d_ring_upd_support) {
	8802	+ dhd_prot_dma_indx_set(dhd, ring->rd,
	8803	+ D2H_DMA_INDX_RD_UPD, ring->idx);
5497	8804	} else {
5498	8805	dhd_bus_cmn_writeshared(dhd->bus, &(ring->rd),
5499	8806	sizeof(uint16), RING_RD_UPD, ring->idx);
5500	8807	}
5501	8808	}
5502	8809
	8810	+static int
	8811	+dhd_send_d2h_ringcreate(dhd_pub_t dhd, msgbuf_ring_t ring_to_create,
	8812	+ uint16 ring_type, uint32 req_id)
	8813	+{
	8814	+ unsigned long flags;
	8815	+ d2h_ring_create_req_t *d2h_ring;
	8816	+ uint16 alloced = 0;
	8817	+ int ret = BCME_OK;
	8818	+ uint16 max_h2d_rings = dhd->bus->max_submission_rings;
	8819	+ msgbuf_ring_t *ctrl_ring = &dhd->prot->h2dring_ctrl_subn;
	8820	+
	8821	+ DHD_RING_LOCK(ctrl_ring->ring_lock, flags);
	8822	+
	8823	+ DHD_TRACE(("%s trying to send D2H ring create Req\n", __FUNCTION__));
	8824	+
	8825	+ if (ring_to_create == NULL) {
	8826	+ DHD_ERROR(("%s: FATAL: ring_to_create is NULL\n", __FUNCTION__));
	8827	+ ret = BCME_ERROR;
	8828	+ goto err;
	8829	+ }
	8830	+
	8831	+ /* Request for ring buffer space */
	8832	+ d2h_ring = (d2h_ring_create_req_t *) dhd_prot_alloc_ring_space(dhd,
	8833	+ ctrl_ring, DHD_FLOWRING_DEFAULT_NITEMS_POSTED_H2D,
	8834	+ &alloced, FALSE);
	8835	+
	8836	+ if (d2h_ring == NULL) {
	8837	+ DHD_ERROR(("%s: FATAL: No space in control ring to send D2H ring create\n",
	8838	+ __FUNCTION__));
	8839	+ ret = BCME_NOMEM;
	8840	+ goto err;
	8841	+ }
	8842	+ ring_to_create->create_req_id = (uint16)req_id;
	8843	+ ring_to_create->create_pending = TRUE;
	8844	+
	8845	+ /* Common msg buf hdr */
	8846	+ d2h_ring->msg.msg_type = MSG_TYPE_D2H_RING_CREATE;
	8847	+ d2h_ring->msg.if_id = 0;
	8848	+ d2h_ring->msg.flags = ctrl_ring->current_phase;
	8849	+ d2h_ring->msg.request_id = htol32(ring_to_create->create_req_id);
	8850	+ d2h_ring->ring_id = htol16(DHD_D2H_RING_OFFSET(ring_to_create->idx, max_h2d_rings));
	8851	+ DHD_ERROR(("%s ringid: %d idx: %d max_h2d: %d\n", __FUNCTION__, d2h_ring->ring_id,
	8852	+ ring_to_create->idx, max_h2d_rings));
	8853	+
	8854	+ d2h_ring->ring_type = ring_type;
	8855	+ d2h_ring->max_items = htol16(ring_to_create->max_items);
	8856	+ d2h_ring->len_item = htol16(ring_to_create->item_len);
	8857	+ d2h_ring->ring_ptr.low_addr = ring_to_create->base_addr.low_addr;
	8858	+ d2h_ring->ring_ptr.high_addr = ring_to_create->base_addr.high_addr;
	8859	+
	8860	+ d2h_ring->flags = 0;
	8861	+ d2h_ring->msg.epoch =
	8862	+ ctrl_ring->seqnum % H2D_EPOCH_MODULO;
	8863	+ ctrl_ring->seqnum++;
	8864	+#ifdef EWP_EDL
	8865	+ if (ring_type == BCMPCIE_D2H_RING_TYPE_EDL) {
	8866	+ DHD_ERROR(("%s: sending d2h EDL ring create: "
	8867	+ "\n max items=%u; len_item=%u; ring_id=%u; low_addr=0x%x; high_addr=0x%x\n",
	8868	+ __FUNCTION__, ltoh16(d2h_ring->max_items),
	8869	+ ltoh16(d2h_ring->len_item),
	8870	+ ltoh16(d2h_ring->ring_id),
	8871	+ d2h_ring->ring_ptr.low_addr,
	8872	+ d2h_ring->ring_ptr.high_addr));
	8873	+ }
	8874	+#endif /* EWP_EDL */
	8875	+
	8876	+ /* Update the flow_ring's WRITE index */
	8877	+ dhd_prot_ring_write_complete(dhd, ctrl_ring, d2h_ring,
	8878	+ DHD_FLOWRING_DEFAULT_NITEMS_POSTED_H2D);
	8879	+
	8880	+ DHD_RING_UNLOCK(ctrl_ring->ring_lock, flags);
	8881	+
	8882	+ return ret;
	8883	+err:
	8884	+ DHD_RING_UNLOCK(ctrl_ring->ring_lock, flags);
	8885	+
	8886	+ return ret;
	8887	+}
	8888	+
	8889	+static int
	8890	+dhd_send_h2d_ringcreate(dhd_pub_t dhd, msgbuf_ring_t ring_to_create, uint8 ring_type, uint32 id)
	8891	+{
	8892	+ unsigned long flags;
	8893	+ h2d_ring_create_req_t *h2d_ring;
	8894	+ uint16 alloced = 0;
	8895	+ uint8 i = 0;
	8896	+ int ret = BCME_OK;
	8897	+ msgbuf_ring_t *ctrl_ring = &dhd->prot->h2dring_ctrl_subn;
	8898	+
	8899	+ DHD_RING_LOCK(ctrl_ring->ring_lock, flags);
	8900	+
	8901	+ DHD_TRACE(("%s trying to send H2D ring create Req\n", __FUNCTION__));
	8902	+
	8903	+ if (ring_to_create == NULL) {
	8904	+ DHD_ERROR(("%s: FATAL: ring_to_create is NULL\n", __FUNCTION__));
	8905	+ ret = BCME_ERROR;
	8906	+ goto err;
	8907	+ }
	8908	+
	8909	+ /* Request for ring buffer space */
	8910	+ h2d_ring = (h2d_ring_create_req_t *)dhd_prot_alloc_ring_space(dhd,
	8911	+ ctrl_ring, DHD_FLOWRING_DEFAULT_NITEMS_POSTED_H2D,
	8912	+ &alloced, FALSE);
	8913	+
	8914	+ if (h2d_ring == NULL) {
	8915	+ DHD_ERROR(("%s: FATAL: No space in control ring to send H2D ring create\n",
	8916	+ __FUNCTION__));
	8917	+ ret = BCME_NOMEM;
	8918	+ goto err;
	8919	+ }
	8920	+ ring_to_create->create_req_id = (uint16)id;
	8921	+ ring_to_create->create_pending = TRUE;
	8922	+
	8923	+ /* Common msg buf hdr */
	8924	+ h2d_ring->msg.msg_type = MSG_TYPE_H2D_RING_CREATE;
	8925	+ h2d_ring->msg.if_id = 0;
	8926	+ h2d_ring->msg.request_id = htol32(ring_to_create->create_req_id);
	8927	+ h2d_ring->msg.flags = ctrl_ring->current_phase;
	8928	+ h2d_ring->ring_id = htol16(DHD_H2D_RING_OFFSET(ring_to_create->idx));
	8929	+ h2d_ring->ring_type = ring_type;
	8930	+ h2d_ring->max_items = htol16(H2DRING_DYNAMIC_INFO_MAX_ITEM);
	8931	+ h2d_ring->n_completion_ids = ring_to_create->n_completion_ids;
	8932	+ h2d_ring->len_item = htol16(H2DRING_INFO_BUFPOST_ITEMSIZE);
	8933	+ h2d_ring->ring_ptr.low_addr = ring_to_create->base_addr.low_addr;
	8934	+ h2d_ring->ring_ptr.high_addr = ring_to_create->base_addr.high_addr;
	8935	+
	8936	+ for (i = 0; i < ring_to_create->n_completion_ids; i++) {
	8937	+ h2d_ring->completion_ring_ids[i] = htol16(ring_to_create->compeltion_ring_ids[i]);
	8938	+ }
	8939	+
	8940	+ h2d_ring->flags = 0;
	8941	+ h2d_ring->msg.epoch =
	8942	+ ctrl_ring->seqnum % H2D_EPOCH_MODULO;
	8943	+ ctrl_ring->seqnum++;
	8944	+
	8945	+ /* Update the flow_ring's WRITE index */
	8946	+ dhd_prot_ring_write_complete(dhd, ctrl_ring, h2d_ring,
	8947	+ DHD_FLOWRING_DEFAULT_NITEMS_POSTED_H2D);
	8948	+
	8949	+ DHD_RING_UNLOCK(ctrl_ring->ring_lock, flags);
	8950	+
	8951	+ return ret;
	8952	+err:
	8953	+ DHD_RING_UNLOCK(ctrl_ring->ring_lock, flags);
	8954	+
	8955	+ return ret;
	8956	+}
5503	8957
5504	8958	/**
5505	8959	* dhd_prot_dma_indx_set - set a new WR or RD index in the DMA index array.
5506	8960	* Dongle will DMA the entire array (if DMA_INDX feature is enabled).
5507	8961	* See dhd_prot_dma_indx_init()
5508	8962	*/
5509		-static void
	8963	+void
5510	8964	dhd_prot_dma_indx_set(dhd_pub_t *dhd, uint16 new_index, uint8 type, uint16 ringid)
5511	8965	{
5512	8966	uint8 *ptr;
5513	8967	uint16 offset;
5514	8968	dhd_prot_t *prot = dhd->prot;
	8969	+ uint16 max_h2d_rings = dhd->bus->max_submission_rings;
5515	8970
5516	8971	switch (type) {
5517	8972	case H2D_DMA_INDX_WR_UPD:
..	..	@@ -5521,7 +8976,12 @@
5521	8976
5522	8977	case D2H_DMA_INDX_RD_UPD:
5523	8978	ptr = (uint8 *)(prot->d2h_dma_indx_rd_buf.va);
5524		- offset = DHD_D2H_RING_OFFSET(ringid);
	8979	+ offset = DHD_D2H_RING_OFFSET(ringid, max_h2d_rings);
	8980	+ break;
	8981	+
	8982	+ case H2D_IFRM_INDX_WR_UPD:
	8983	+ ptr = (uint8 *)(prot->h2d_ifrm_indx_wr_buf.va);
	8984	+ offset = DHD_H2D_FRM_FLOW_RING_OFFSET(ringid);
5525	8985	break;
5526	8986
5527	8987	default:
..	..	@@ -5542,7 +9002,6 @@
5542	9002
5543	9003	} /* dhd_prot_dma_indx_set */
5544	9004
5545		-
5546	9005	/**
5547	9006	* dhd_prot_dma_indx_get - Fetch a WR or RD index from the dongle DMA-ed index
5548	9007	* array.
..	..	@@ -5556,6 +9015,7 @@
5556	9015	uint16 data;
5557	9016	uint16 offset;
5558	9017	dhd_prot_t *prot = dhd->prot;
	9018	+ uint16 max_h2d_rings = dhd->bus->max_submission_rings;
5559	9019
5560	9020	switch (type) {
5561	9021	case H2D_DMA_INDX_WR_UPD:
..	..	@@ -5570,12 +9030,12 @@
5570	9030
5571	9031	case D2H_DMA_INDX_WR_UPD:
5572	9032	ptr = (uint8 *)(prot->d2h_dma_indx_wr_buf.va);
5573		- offset = DHD_D2H_RING_OFFSET(ringid);
	9033	+ offset = DHD_D2H_RING_OFFSET(ringid, max_h2d_rings);
5574	9034	break;
5575	9035
5576	9036	case D2H_DMA_INDX_RD_UPD:
5577	9037	ptr = (uint8 *)(prot->d2h_dma_indx_rd_buf.va);
5578		- offset = DHD_D2H_RING_OFFSET(ringid);
	9038	+ offset = DHD_D2H_RING_OFFSET(ringid, max_h2d_rings);
5579	9039	break;
5580	9040
5581	9041	default:
..	..	@@ -5642,37 +9102,41 @@
5642	9102	switch (type) {
5643	9103	case H2D_DMA_INDX_WR_BUF:
5644	9104	dma_buf = &prot->h2d_dma_indx_wr_buf;
5645		- if (dhd_prot_dma_indx_alloc(dhd, type, dma_buf, bufsz)) {
	9105	+ if (dhd_prot_dma_indx_alloc(dhd, type, dma_buf, bufsz))
5646	9106	goto ret_no_mem;
5647		- }
5648	9107	DHD_ERROR(("H2D DMA WR INDX : array size %d = %d * %d\n",
5649	9108	dma_buf->len, rw_index_sz, length));
5650	9109	break;
5651	9110
5652	9111	case H2D_DMA_INDX_RD_BUF:
5653	9112	dma_buf = &prot->h2d_dma_indx_rd_buf;
5654		- if (dhd_prot_dma_indx_alloc(dhd, type, dma_buf, bufsz)) {
	9113	+ if (dhd_prot_dma_indx_alloc(dhd, type, dma_buf, bufsz))
5655	9114	goto ret_no_mem;
5656		- }
5657	9115	DHD_ERROR(("H2D DMA RD INDX : array size %d = %d * %d\n",
5658	9116	dma_buf->len, rw_index_sz, length));
5659	9117	break;
5660	9118
5661	9119	case D2H_DMA_INDX_WR_BUF:
5662	9120	dma_buf = &prot->d2h_dma_indx_wr_buf;
5663		- if (dhd_prot_dma_indx_alloc(dhd, type, dma_buf, bufsz)) {
	9121	+ if (dhd_prot_dma_indx_alloc(dhd, type, dma_buf, bufsz))
5664	9122	goto ret_no_mem;
5665		- }
5666	9123	DHD_ERROR(("D2H DMA WR INDX : array size %d = %d * %d\n",
5667	9124	dma_buf->len, rw_index_sz, length));
5668	9125	break;
5669	9126
5670	9127	case D2H_DMA_INDX_RD_BUF:
5671	9128	dma_buf = &prot->d2h_dma_indx_rd_buf;
5672		- if (dhd_prot_dma_indx_alloc(dhd, type, dma_buf, bufsz)) {
	9129	+ if (dhd_prot_dma_indx_alloc(dhd, type, dma_buf, bufsz))
5673	9130	goto ret_no_mem;
5674		- }
5675	9131	DHD_ERROR(("D2H DMA RD INDX : array size %d = %d * %d\n",
	9132	+ dma_buf->len, rw_index_sz, length));
	9133	+ break;
	9134	+
	9135	+ case H2D_IFRM_INDX_WR_BUF:
	9136	+ dma_buf = &prot->h2d_ifrm_indx_wr_buf;
	9137	+ if (dhd_prot_dma_indx_alloc(dhd, type, dma_buf, bufsz))
	9138	+ goto ret_no_mem;
	9139	+ DHD_ERROR(("H2D IFRM WR INDX : array size %d = %d * %d\n",
5676	9140	dma_buf->len, rw_index_sz, length));
5677	9141	break;
5678	9142
..	..	@@ -5689,7 +9153,6 @@
5689	9153	return BCME_NOMEM;
5690	9154
5691	9155	} /* dhd_prot_dma_indx_init */
5692		-
5693	9156
5694	9157	/**
5695	9158	* Called on checking for 'completion' messages from the dongle. Returns next host buffer to read
..	..	@@ -5709,8 +9172,15 @@
5709	9172	__FUNCTION__, (uint32 *)(dhd->prot->d2h_dma_indx_rd_buf.va),
5710	9173	(uint32 *)(dhd->prot->d2h_dma_indx_wr_buf.va)));
5711	9174
	9175	+ /* Remember the read index in a variable.
	9176	+ * This is becuase ring->rd gets updated in the end of this function
	9177	+ * So if we have to print the exact read index from which the
	9178	+ * message is read its not possible.
	9179	+ */
	9180	+ ring->curr_rd = ring->rd;
	9181	+
5712	9182	/* update write pointer */
5713		- if (DMA_INDX_ENAB(dhd->dma_d2h_ring_upd_support)) {
	9183	+ if (dhd->dma_d2h_ring_upd_support) {
5714	9184	/* DMAing write/read indices supported */
5715	9185	d2h_wr = dhd_prot_dma_indx_get(dhd, D2H_DMA_INDX_WR_UPD, ring->idx);
5716	9186	ring->wr = d2h_wr;
..	..	@@ -5724,11 +9194,8 @@
5724	9194
5725	9195	/* check for avail space, in number of ring items */
5726	9196	items = READ_AVAIL_SPACE(wr, rd, depth);
5727		- if (items == 0) {
	9197	+ if (items == 0)
5728	9198	return NULL;
5729		- }
5730		-
5731		- ASSERT(items < ring->max_items);
5732	9199
5733	9200	/*
5734	9201	* Note that there are builds where Assert translates to just printk
..	..	@@ -5736,16 +9203,32 @@
5736	9203	* dhd_prot_process_msgtype can get into an big loop if this
5737	9204	* happens.
5738	9205	*/
5739		- if (items >= ring->max_items) {
	9206	+ if (items > ring->max_items) {
5740	9207	DHD_ERROR(("\r\n======================= \r\n"));
5741	9208	DHD_ERROR(("%s(): ring %p, ring->name %s, ring->max_items %d, items %d \r\n",
5742	9209	__FUNCTION__, ring, ring->name, ring->max_items, items));
5743	9210	DHD_ERROR(("wr: %d, rd: %d, depth: %d \r\n", wr, rd, depth));
5744		- DHD_ERROR(("dhd->busstate %d bus->suspended %d bus->wait_for_d3_ack %d \r\n",
5745		- dhd->busstate, dhd->bus->suspended, dhd->bus->wait_for_d3_ack));
	9211	+ DHD_ERROR(("dhd->busstate %d bus->wait_for_d3_ack %d \r\n",
	9212	+ dhd->busstate, dhd->bus->wait_for_d3_ack));
5746	9213	DHD_ERROR(("\r\n======================= \r\n"));
	9214	+#ifdef SUPPORT_LINKDOWN_RECOVERY
	9215	+ if (wr >= ring->max_items) {
	9216	+ dhd->bus->read_shm_fail = TRUE;
	9217	+ }
	9218	+#else
	9219	+#ifdef DHD_FW_COREDUMP
	9220	+ if (dhd->memdump_enabled) {
	9221	+ /* collect core dump */
	9222	+ dhd->memdump_type = DUMP_TYPE_RESUMED_ON_INVALID_RING_RDWR;
	9223	+ dhd_bus_mem_dump(dhd);
	9224	+
	9225	+ }
	9226	+#endif /* DHD_FW_COREDUMP */
	9227	+#endif /* SUPPORT_LINKDOWN_RECOVERY */
5747	9228
5748	9229	*available_len = 0;
	9230	+ dhd_schedule_reset(dhd);
	9231	+
5749	9232	return NULL;
5750	9233	}
5751	9234
..	..	@@ -5753,27 +9236,83 @@
5753	9236	read_addr = (char)ring->dma_buf.va + (rd ring->item_len);
5754	9237
5755	9238	/* update read pointer */
5756		- if ((ring->rd + items) >= ring->max_items) {
	9239	+ if ((ring->rd + items) >= ring->max_items)
5757	9240	ring->rd = 0;
5758		- } else {
	9241	+ else
5759	9242	ring->rd += items;
5760		- }
5761	9243
5762	9244	ASSERT(ring->rd < ring->max_items);
5763	9245
5764	9246	/* convert items to bytes : available_len must be 32bits */
5765	9247	available_len = (uint32)(items ring->item_len);
5766	9248
5767		-#ifndef CUSTOMER_HW_31_2
5768		- /* cannot use this since the dma ring is allocated as uncached,
5769		- * this will cause an assertation
5770		- */
5771	9249	OSL_CACHE_INV(read_addr, *available_len);
5772		-#endif
	9250	+
5773	9251	/* return read address */
5774	9252	return read_addr;
5775	9253
5776	9254	} /* dhd_prot_get_read_addr */
	9255	+
	9256	+/**
	9257	+ * dhd_prot_h2d_mbdata_send_ctrlmsg is a non-atomic function,
	9258	+ * make sure the callers always hold appropriate locks.
	9259	+ */
	9260	+int dhd_prot_h2d_mbdata_send_ctrlmsg(dhd_pub_t *dhd, uint32 mb_data)
	9261	+{
	9262	+ h2d_mailbox_data_t *h2d_mb_data;
	9263	+ uint16 alloced = 0;
	9264	+ msgbuf_ring_t *ctrl_ring = &dhd->prot->h2dring_ctrl_subn;
	9265	+ unsigned long flags;
	9266	+ int num_post = 1;
	9267	+ int i;
	9268	+
	9269	+ DHD_INFO(("%s Sending H2D MB data Req data 0x%04x\n",
	9270	+ __FUNCTION__, mb_data));
	9271	+ if (!ctrl_ring->inited) {
	9272	+ DHD_ERROR(("%s: Ctrl Submit Ring: not inited\n", __FUNCTION__));
	9273	+ return BCME_ERROR;
	9274	+ }
	9275	+
	9276	+ for (i = 0; i < num_post; i ++) {
	9277	+ DHD_RING_LOCK(ctrl_ring->ring_lock, flags);
	9278	+ /* Request for ring buffer space */
	9279	+ h2d_mb_data = (h2d_mailbox_data_t *)dhd_prot_alloc_ring_space(dhd,
	9280	+ ctrl_ring, DHD_FLOWRING_DEFAULT_NITEMS_POSTED_H2D,
	9281	+ &alloced, FALSE);
	9282	+
	9283	+ if (h2d_mb_data == NULL) {
	9284	+ DHD_ERROR(("%s: FATAL: No space in control ring to send H2D Mb data\n",
	9285	+ __FUNCTION__));
	9286	+ DHD_RING_UNLOCK(ctrl_ring->ring_lock, flags);
	9287	+ return BCME_NOMEM;
	9288	+ }
	9289	+
	9290	+ memset(h2d_mb_data, 0, sizeof(h2d_mailbox_data_t));
	9291	+ /* Common msg buf hdr */
	9292	+ h2d_mb_data->msg.msg_type = MSG_TYPE_H2D_MAILBOX_DATA;
	9293	+ h2d_mb_data->msg.flags = ctrl_ring->current_phase;
	9294	+
	9295	+ h2d_mb_data->msg.epoch =
	9296	+ ctrl_ring->seqnum % H2D_EPOCH_MODULO;
	9297	+ ctrl_ring->seqnum++;
	9298	+
	9299	+ /* Update flow create message */
	9300	+ h2d_mb_data->mail_box_data = htol32(mb_data);
	9301	+ {
	9302	+ h2d_mb_data->mail_box_data = htol32(mb_data);
	9303	+ }
	9304	+
	9305	+ DHD_INFO(("%s Send H2D MB data Req data 0x%04x\n", __FUNCTION__, mb_data));
	9306	+
	9307	+ /* upd wrt ptr and raise interrupt */
	9308	+ dhd_prot_ring_write_complete_mbdata(dhd, ctrl_ring, h2d_mb_data,
	9309	+ DHD_FLOWRING_DEFAULT_NITEMS_POSTED_H2D, mb_data);
	9310	+
	9311	+ DHD_RING_UNLOCK(ctrl_ring->ring_lock, flags);
	9312	+
	9313	+ }
	9314	+ return 0;
	9315	+}
5777	9316
5778	9317	/** Creates a flow ring and informs dongle of this event */
5779	9318	int
..	..	@@ -5785,6 +9324,7 @@
5785	9324	unsigned long flags;
5786	9325	uint16 alloced = 0;
5787	9326	msgbuf_ring_t *ctrl_ring = &prot->h2dring_ctrl_subn;
	9327	+ uint16 max_flowrings = dhd->bus->max_tx_flowrings;
5788	9328
5789	9329	/* Fetch a pre-initialized msgbuf_ring from the flowring pool */
5790	9330	flow_ring = dhd_prot_flowrings_pool_fetch(dhd, flow_ring_node->flowid);
..	..	@@ -5794,7 +9334,7 @@
5794	9334	return BCME_NOMEM;
5795	9335	}
5796	9336
5797		- DHD_GENERAL_LOCK(dhd, flags);
	9337	+ DHD_RING_LOCK(ctrl_ring->ring_lock, flags);
5798	9338
5799	9339	/* Request for ctrl_ring buffer space */
5800	9340	flow_create_rqst = (tx_flowring_create_request_t *)
..	..	@@ -5804,7 +9344,7 @@
5804	9344	dhd_prot_flowrings_pool_release(dhd, flow_ring_node->flowid, flow_ring);
5805	9345	DHD_ERROR(("%s: Flow Create Req flowid %d - failure ring space\n",
5806	9346	__FUNCTION__, flow_ring_node->flowid));
5807		- DHD_GENERAL_UNLOCK(dhd, flags);
	9347	+ DHD_RING_LOCK(ctrl_ring->ring_lock, flags);
5808	9348	return BCME_NOMEM;
5809	9349	}
5810	9350
..	..	@@ -5814,6 +9354,7 @@
5814	9354	flow_create_rqst->msg.msg_type = MSG_TYPE_FLOW_RING_CREATE;
5815	9355	flow_create_rqst->msg.if_id = (uint8)flow_ring_node->flow_info.ifindex;
5816	9356	flow_create_rqst->msg.request_id = htol32(0); /* TBD */
	9357	+ flow_create_rqst->msg.flags = ctrl_ring->current_phase;
5817	9358
5818	9359	flow_create_rqst->msg.epoch = ctrl_ring->seqnum % H2D_EPOCH_MODULO;
5819	9360	ctrl_ring->seqnum++;
..	..	@@ -5826,17 +9367,47 @@
5826	9367	/* CAUTION: ring::base_addr already in Little Endian */
5827	9368	flow_create_rqst->flow_ring_ptr.low_addr = flow_ring->base_addr.low_addr;
5828	9369	flow_create_rqst->flow_ring_ptr.high_addr = flow_ring->base_addr.high_addr;
5829		- flow_create_rqst->max_items = htol16(H2DRING_TXPOST_MAX_ITEM);
	9370	+ flow_create_rqst->max_items = htol16(prot->h2d_max_txpost);
5830	9371	flow_create_rqst->len_item = htol16(H2DRING_TXPOST_ITEMSIZE);
	9372	+ flow_create_rqst->if_flags = 0;
	9373	+
	9374	+#ifdef DHD_HP2P
	9375	+ /* Create HPP flow ring if HP2P is enabled and TID=7 and AWDL interface */
	9376	+ /* and traffic is not multicast */
	9377	+ /* Allow infra interface only if user enabled hp2p_infra_enable thru iovar */
	9378	+ /* Allow only one HP2P Flow active at a time */
	9379	+ if (dhd->hp2p_capable && !dhd->hp2p_ring_active &&
	9380	+ flow_ring_node->flow_info.tid == HP2P_PRIO &&
	9381	+ (dhd->hp2p_infra_enable \|\| flow_create_rqst->msg.if_id) &&
	9382	+ !ETHER_ISMULTI(flow_create_rqst->da)) {
	9383	+ flow_create_rqst->if_flags \|= BCMPCIE_FLOW_RING_INTF_HP2P;
	9384	+ flow_ring_node->hp2p_ring = TRUE;
	9385	+ dhd->hp2p_ring_active = TRUE;
	9386	+
	9387	+ DHD_ERROR(("%s: flow ring for HP2P tid = %d flowid = %d\n",
	9388	+ __FUNCTION__, flow_ring_node->flow_info.tid,
	9389	+ flow_ring_node->flowid));
	9390	+ }
	9391	+#endif /* DHD_HP2P */
	9392	+
	9393	+ /* definition for ifrm mask : bit0:d11ac core, bit1:d11ad core
	9394	+ * currently it is not used for priority. so uses solely for ifrm mask
	9395	+ */
	9396	+ if (IFRM_ACTIVE(dhd))
	9397	+ flow_create_rqst->priority_ifrmmask = (1 << IFRM_DEV_0);
	9398	+
5831	9399	DHD_ERROR(("%s: Send Flow Create Req flow ID %d for peer " MACDBG
5832	9400	" prio %d ifindex %d\n", __FUNCTION__, flow_ring_node->flowid,
5833	9401	MAC2STRDBG(flow_ring_node->flow_info.da), flow_ring_node->flow_info.tid,
5834	9402	flow_ring_node->flow_info.ifindex));
5835	9403
5836	9404	/* Update the flow_ring's WRITE index */
5837		- if (DMA_INDX_ENAB(dhd->dma_h2d_ring_upd_support)) {
	9405	+ if (IDMA_ACTIVE(dhd) \|\| dhd->dma_h2d_ring_upd_support) {
5838	9406	dhd_prot_dma_indx_set(dhd, flow_ring->wr,
5839		- H2D_DMA_INDX_WR_UPD, flow_ring->idx);
	9407	+ H2D_DMA_INDX_WR_UPD, flow_ring->idx);
	9408	+ } else if (IFRM_ACTIVE(dhd) && DHD_IS_FLOWRING(flow_ring->idx, max_flowrings)) {
	9409	+ dhd_prot_dma_indx_set(dhd, flow_ring->wr,
	9410	+ H2D_IFRM_INDX_WR_UPD, flow_ring->idx);
5840	9411	} else {
5841	9412	dhd_bus_cmn_writeshared(dhd->bus, &(flow_ring->wr),
5842	9413	sizeof(uint16), RING_WR_UPD, flow_ring->idx);
..	..	@@ -5845,7 +9416,7 @@
5845	9416	/* update control subn ring's WR index and ring doorbell to dongle */
5846	9417	dhd_prot_ring_write_complete(dhd, ctrl_ring, flow_create_rqst, 1);
5847	9418
5848		- DHD_GENERAL_UNLOCK(dhd, flags);
	9419	+ DHD_RING_UNLOCK(ctrl_ring->ring_lock, flags);
5849	9420
5850	9421	return BCME_OK;
5851	9422	} /* dhd_prot_flow_ring_create */
..	..	@@ -5865,6 +9436,143 @@
5865	9436	ltoh16(flow_create_resp->cmplt.status));
5866	9437	}
5867	9438
	9439	+static void
	9440	+dhd_prot_process_h2d_ring_create_complete(dhd_pub_t dhd, void buf)
	9441	+{
	9442	+ h2d_ring_create_response_t resp = (h2d_ring_create_response_t )buf;
	9443	+ DHD_INFO(("%s ring create Response status = %d ring %d, id 0x%04x\n", __FUNCTION__,
	9444	+ ltoh16(resp->cmplt.status),
	9445	+ ltoh16(resp->cmplt.ring_id),
	9446	+ ltoh32(resp->cmn_hdr.request_id)));
	9447	+ if ((ltoh32(resp->cmn_hdr.request_id) != DHD_H2D_DBGRING_REQ_PKTID) &&
	9448	+ (ltoh32(resp->cmn_hdr.request_id) != DHD_H2D_BTLOGRING_REQ_PKTID)) {
	9449	+ DHD_ERROR(("invalid request ID with h2d ring create complete\n"));
	9450	+ return;
	9451	+ }
	9452	+ if (dhd->prot->h2dring_info_subn->create_req_id == ltoh32(resp->cmn_hdr.request_id) &&
	9453	+ !dhd->prot->h2dring_info_subn->create_pending) {
	9454	+ DHD_ERROR(("info ring create status for not pending submit ring\n"));
	9455	+ }
	9456	+
	9457	+ if (ltoh16(resp->cmplt.status) != BCMPCIE_SUCCESS) {
	9458	+ DHD_ERROR(("info/btlog ring create failed with status %d\n",
	9459	+ ltoh16(resp->cmplt.status)));
	9460	+ return;
	9461	+ }
	9462	+ if (dhd->prot->h2dring_info_subn->create_req_id == ltoh32(resp->cmn_hdr.request_id)) {
	9463	+ dhd->prot->h2dring_info_subn->create_pending = FALSE;
	9464	+ dhd->prot->h2dring_info_subn->inited = TRUE;
	9465	+ DHD_ERROR(("info buffer post after ring create\n"));
	9466	+ dhd_prot_infobufpost(dhd, dhd->prot->h2dring_info_subn);
	9467	+ }
	9468	+}
	9469	+
	9470	+static void
	9471	+dhd_prot_process_d2h_ring_create_complete(dhd_pub_t dhd, void buf)
	9472	+{
	9473	+ d2h_ring_create_response_t resp = (d2h_ring_create_response_t )buf;
	9474	+ DHD_INFO(("%s ring create Response status = %d ring %d, id 0x%04x\n", __FUNCTION__,
	9475	+ ltoh16(resp->cmplt.status),
	9476	+ ltoh16(resp->cmplt.ring_id),
	9477	+ ltoh32(resp->cmn_hdr.request_id)));
	9478	+ if ((ltoh32(resp->cmn_hdr.request_id) != DHD_D2H_DBGRING_REQ_PKTID) &&
	9479	+ (ltoh32(resp->cmn_hdr.request_id) != DHD_D2H_BTLOGRING_REQ_PKTID) &&
	9480	+#ifdef DHD_HP2P
	9481	+ (ltoh32(resp->cmn_hdr.request_id) != DHD_D2H_HPPRING_TXREQ_PKTID) &&
	9482	+ (ltoh32(resp->cmn_hdr.request_id) != DHD_D2H_HPPRING_RXREQ_PKTID) &&
	9483	+#endif /* DHD_HP2P */
	9484	+ TRUE) {
	9485	+ DHD_ERROR(("invalid request ID with d2h ring create complete\n"));
	9486	+ return;
	9487	+ }
	9488	+ if (ltoh32(resp->cmn_hdr.request_id) == DHD_D2H_DBGRING_REQ_PKTID) {
	9489	+#ifdef EWP_EDL
	9490	+ if (!dhd->dongle_edl_support)
	9491	+#endif // endif
	9492	+ {
	9493	+ if (!dhd->prot->d2hring_info_cpln->create_pending) {
	9494	+ DHD_ERROR(("info ring create status for not pending cpl ring\n"));
	9495	+ return;
	9496	+ }
	9497	+
	9498	+ if (ltoh16(resp->cmplt.status) != BCMPCIE_SUCCESS) {
	9499	+ DHD_ERROR(("info cpl ring create failed with status %d\n",
	9500	+ ltoh16(resp->cmplt.status)));
	9501	+ return;
	9502	+ }
	9503	+ dhd->prot->d2hring_info_cpln->create_pending = FALSE;
	9504	+ dhd->prot->d2hring_info_cpln->inited = TRUE;
	9505	+ }
	9506	+#ifdef EWP_EDL
	9507	+ else {
	9508	+ if (!dhd->prot->d2hring_edl->create_pending) {
	9509	+ DHD_ERROR(("edl ring create status for not pending cpl ring\n"));
	9510	+ return;
	9511	+ }
	9512	+
	9513	+ if (ltoh16(resp->cmplt.status) != BCMPCIE_SUCCESS) {
	9514	+ DHD_ERROR(("edl cpl ring create failed with status %d\n",
	9515	+ ltoh16(resp->cmplt.status)));
	9516	+ return;
	9517	+ }
	9518	+ dhd->prot->d2hring_edl->create_pending = FALSE;
	9519	+ dhd->prot->d2hring_edl->inited = TRUE;
	9520	+ }
	9521	+#endif /* EWP_EDL */
	9522	+ }
	9523	+
	9524	+#ifdef DHD_HP2P
	9525	+ if (dhd->prot->d2hring_hp2p_txcpl &&
	9526	+ ltoh32(resp->cmn_hdr.request_id) == DHD_D2H_HPPRING_TXREQ_PKTID) {
	9527	+ if (!dhd->prot->d2hring_hp2p_txcpl->create_pending) {
	9528	+ DHD_ERROR(("HPP tx ring create status for not pending cpl ring\n"));
	9529	+ return;
	9530	+ }
	9531	+
	9532	+ if (ltoh16(resp->cmplt.status) != BCMPCIE_SUCCESS) {
	9533	+ DHD_ERROR(("HPP tx cpl ring create failed with status %d\n",
	9534	+ ltoh16(resp->cmplt.status)));
	9535	+ return;
	9536	+ }
	9537	+ dhd->prot->d2hring_hp2p_txcpl->create_pending = FALSE;
	9538	+ dhd->prot->d2hring_hp2p_txcpl->inited = TRUE;
	9539	+ }
	9540	+ if (dhd->prot->d2hring_hp2p_rxcpl &&
	9541	+ ltoh32(resp->cmn_hdr.request_id) == DHD_D2H_HPPRING_RXREQ_PKTID) {
	9542	+ if (!dhd->prot->d2hring_hp2p_rxcpl->create_pending) {
	9543	+ DHD_ERROR(("HPP rx ring create status for not pending cpl ring\n"));
	9544	+ return;
	9545	+ }
	9546	+
	9547	+ if (ltoh16(resp->cmplt.status) != BCMPCIE_SUCCESS) {
	9548	+ DHD_ERROR(("HPP rx cpl ring create failed with status %d\n",
	9549	+ ltoh16(resp->cmplt.status)));
	9550	+ return;
	9551	+ }
	9552	+ dhd->prot->d2hring_hp2p_rxcpl->create_pending = FALSE;
	9553	+ dhd->prot->d2hring_hp2p_rxcpl->inited = TRUE;
	9554	+ }
	9555	+#endif /* DHD_HP2P */
	9556	+}
	9557	+
	9558	+static void
	9559	+dhd_prot_process_d2h_mb_data(dhd_pub_t dhd, void buf)
	9560	+{
	9561	+ d2h_mailbox_data_t *d2h_data;
	9562	+
	9563	+ d2h_data = (d2h_mailbox_data_t *)buf;
	9564	+ DHD_INFO(("%s dhd_prot_process_d2h_mb_data, 0x%04x\n", __FUNCTION__,
	9565	+ d2h_data->d2h_mailbox_data));
	9566	+ dhd_bus_handle_mb_data(dhd->bus, d2h_data->d2h_mailbox_data);
	9567	+}
	9568	+
	9569	+static void
	9570	+dhd_prot_process_d2h_host_ts_complete(dhd_pub_t dhd, void buf)
	9571	+{
	9572	+ DHD_ERROR(("Timesunc feature not compiled in but GOT HOST_TS_COMPLETE\n"));
	9573	+
	9574	+}
	9575	+
5868	9576	/** called on e.g. flow ring delete */
5869	9577	void dhd_prot_clean_flow_ring(dhd_pub_t dhd, void msgbuf_flow_info)
5870	9578	{
..	..	@@ -5876,37 +9584,89 @@
5876	9584	void dhd_prot_print_flow_ring(dhd_pub_t dhd, void msgbuf_flow_info,
5877	9585	struct bcmstrbuf strbuf, const char fmt)
5878	9586	{
5879		- const char *default_fmt = "RD %d WR %d\n";
	9587	+ const char *default_fmt =
	9588	+ "RD %d WR %d BASE(VA) %p BASE(PA) %x:%x SIZE %d "
	9589	+ "WORK_ITEM_SIZE %d MAX_WORK_ITEMS %d TOTAL_SIZE %d\n";
5880	9590	msgbuf_ring_t flow_ring = (msgbuf_ring_t )msgbuf_flow_info;
5881	9591	uint16 rd, wr;
	9592	+ uint32 dma_buf_len = flow_ring->max_items * flow_ring->item_len;
5882	9593
5883	9594	if (fmt == NULL) {
5884	9595	fmt = default_fmt;
5885	9596	}
	9597	+
	9598	+ if (dhd->bus->is_linkdown) {
	9599	+ DHD_ERROR(("%s: Skip dumping flowring due to Link down\n", __FUNCTION__));
	9600	+ return;
	9601	+ }
	9602	+
5886	9603	dhd_bus_cmn_readshared(dhd->bus, &rd, RING_RD_UPD, flow_ring->idx);
5887	9604	dhd_bus_cmn_readshared(dhd->bus, &wr, RING_WR_UPD, flow_ring->idx);
5888		- bcm_bprintf(strbuf, fmt, rd, wr);
	9605	+ bcm_bprintf(strbuf, fmt, rd, wr, flow_ring->dma_buf.va,
	9606	+ ltoh32(flow_ring->base_addr.high_addr),
	9607	+ ltoh32(flow_ring->base_addr.low_addr),
	9608	+ flow_ring->item_len, flow_ring->max_items,
	9609	+ dma_buf_len);
5889	9610	}
5890	9611
5891	9612	void dhd_prot_print_info(dhd_pub_t dhd, struct bcmstrbuf strbuf)
5892	9613	{
5893	9614	dhd_prot_t *prot = dhd->prot;
5894		- bcm_bprintf(strbuf, "CtrlPost: ");
5895		- dhd_prot_print_flow_ring(dhd, &prot->h2dring_ctrl_subn, strbuf, NULL);
5896		- bcm_bprintf(strbuf, "CtrlCpl: ");
5897		- dhd_prot_print_flow_ring(dhd, &prot->d2hring_ctrl_cpln, strbuf, NULL);
	9615	+ bcm_bprintf(strbuf, "IPCrevs: Dev %d, \t Host %d, \tactive %d\n",
	9616	+ dhd->prot->device_ipc_version,
	9617	+ dhd->prot->host_ipc_version,
	9618	+ dhd->prot->active_ipc_version);
5898	9619
5899		- bcm_bprintf(strbuf, "RxPost: ");
5900		- bcm_bprintf(strbuf, "RBP %d ", prot->rxbufpost);
5901		- dhd_prot_print_flow_ring(dhd, &prot->h2dring_rxp_subn, strbuf, NULL);
5902		- bcm_bprintf(strbuf, "RxCpl: ");
5903		- dhd_prot_print_flow_ring(dhd, &prot->d2hring_rx_cpln, strbuf, NULL);
	9620	+ bcm_bprintf(strbuf, "max Host TS bufs to post: %d, \t posted %d \n",
	9621	+ dhd->prot->max_tsbufpost, dhd->prot->cur_ts_bufs_posted);
	9622	+ bcm_bprintf(strbuf, "max INFO bufs to post: %d, \t posted %d \n",
	9623	+ dhd->prot->max_infobufpost, dhd->prot->infobufpost);
	9624	+ bcm_bprintf(strbuf, "max event bufs to post: %d, \t posted %d \n",
	9625	+ dhd->prot->max_eventbufpost, dhd->prot->cur_event_bufs_posted);
	9626	+ bcm_bprintf(strbuf, "max ioctlresp bufs to post: %d, \t posted %d \n",
	9627	+ dhd->prot->max_ioctlrespbufpost, dhd->prot->cur_ioctlresp_bufs_posted);
	9628	+ bcm_bprintf(strbuf, "max RX bufs to post: %d, \t posted %d \n",
	9629	+ dhd->prot->max_rxbufpost, dhd->prot->rxbufpost);
5904	9630
5905		- bcm_bprintf(strbuf, "TxCpl: ");
5906		- dhd_prot_print_flow_ring(dhd, &prot->d2hring_tx_cpln, strbuf, NULL);
5907		- bcm_bprintf(strbuf, "active_tx_count %d pktidmap_avail %d\n",
5908		- dhd->prot->active_tx_count,
5909		- DHD_PKTID_AVAIL(dhd->prot->pktid_map_handle));
	9631	+ bcm_bprintf(strbuf,
	9632	+ "%14s %5s %5s %17s %17s %14s %14s %10s\n",
	9633	+ "Type", "RD", "WR", "BASE(VA)", "BASE(PA)",
	9634	+ "WORK_ITEM_SIZE", "MAX_WORK_ITEMS", "TOTAL_SIZE");
	9635	+ bcm_bprintf(strbuf, "%14s", "H2DCtrlPost");
	9636	+ dhd_prot_print_flow_ring(dhd, &prot->h2dring_ctrl_subn, strbuf,
	9637	+ " %5d %5d %17p %8x:%8x %14d %14d %10d\n");
	9638	+ bcm_bprintf(strbuf, "%14s", "D2HCtrlCpl");
	9639	+ dhd_prot_print_flow_ring(dhd, &prot->d2hring_ctrl_cpln, strbuf,
	9640	+ " %5d %5d %17p %8x:%8x %14d %14d %10d\n");
	9641	+ bcm_bprintf(strbuf, "%14s", "H2DRxPost", prot->rxbufpost);
	9642	+ dhd_prot_print_flow_ring(dhd, &prot->h2dring_rxp_subn, strbuf,
	9643	+ " %5d %5d %17p %8x:%8x %14d %14d %10d\n");
	9644	+ bcm_bprintf(strbuf, "%14s", "D2HRxCpl");
	9645	+ dhd_prot_print_flow_ring(dhd, &prot->d2hring_rx_cpln, strbuf,
	9646	+ " %5d %5d %17p %8x:%8x %14d %14d %10d\n");
	9647	+ bcm_bprintf(strbuf, "%14s", "D2HTxCpl");
	9648	+ dhd_prot_print_flow_ring(dhd, &prot->d2hring_tx_cpln, strbuf,
	9649	+ " %5d %5d %17p %8x:%8x %14d %14d %10d\n");
	9650	+ if (dhd->prot->h2dring_info_subn != NULL && dhd->prot->d2hring_info_cpln != NULL) {
	9651	+ bcm_bprintf(strbuf, "%14s", "H2DRingInfoSub");
	9652	+ dhd_prot_print_flow_ring(dhd, prot->h2dring_info_subn, strbuf,
	9653	+ " %5d %5d %17p %8x:%8x %14d %14d %10d\n");
	9654	+ bcm_bprintf(strbuf, "%14s", "D2HRingInfoCpl");
	9655	+ dhd_prot_print_flow_ring(dhd, prot->d2hring_info_cpln, strbuf,
	9656	+ " %5d %5d %17p %8x:%8x %14d %14d %10d\n");
	9657	+ }
	9658	+ if (dhd->prot->d2hring_edl != NULL) {
	9659	+ bcm_bprintf(strbuf, "%14s", "D2HRingEDL");
	9660	+ dhd_prot_print_flow_ring(dhd, prot->d2hring_edl, strbuf,
	9661	+ " %5d %5d %17p %8x:%8x %14d %14d %10d\n");
	9662	+ }
	9663	+
	9664	+ bcm_bprintf(strbuf, "active_tx_count %d pktidmap_avail(ctrl/rx/tx) %d %d %d\n",
	9665	+ OSL_ATOMIC_READ(dhd->osh, &dhd->prot->active_tx_count),
	9666	+ DHD_PKTID_AVAIL(dhd->prot->pktid_ctrl_map),
	9667	+ DHD_PKTID_AVAIL(dhd->prot->pktid_rx_map),
	9668	+ DHD_PKTID_AVAIL(dhd->prot->pktid_tx_map));
	9669	+
5910	9670	}
5911	9671
5912	9672	int
..	..	@@ -5918,14 +9678,14 @@
5918	9678	uint16 alloced = 0;
5919	9679	msgbuf_ring_t *ring = &prot->h2dring_ctrl_subn;
5920	9680
5921		- DHD_GENERAL_LOCK(dhd, flags);
	9681	+ DHD_RING_LOCK(ring->ring_lock, flags);
5922	9682
5923	9683	/* Request for ring buffer space */
5924	9684	flow_delete_rqst = (tx_flowring_delete_request_t *)
5925	9685	dhd_prot_alloc_ring_space(dhd, ring, 1, &alloced, FALSE);
5926	9686
5927	9687	if (flow_delete_rqst == NULL) {
5928		- DHD_GENERAL_UNLOCK(dhd, flags);
	9688	+ DHD_RING_UNLOCK(ring->ring_lock, flags);
5929	9689	DHD_ERROR(("%s: Flow Delete Req - failure ring space\n", __FUNCTION__));
5930	9690	return BCME_NOMEM;
5931	9691	}
..	..	@@ -5934,6 +9694,7 @@
5934	9694	flow_delete_rqst->msg.msg_type = MSG_TYPE_FLOW_RING_DELETE;
5935	9695	flow_delete_rqst->msg.if_id = (uint8)flow_ring_node->flow_info.ifindex;
5936	9696	flow_delete_rqst->msg.request_id = htol32(0); /* TBD */
	9697	+ flow_delete_rqst->msg.flags = ring->current_phase;
5937	9698
5938	9699	flow_delete_rqst->msg.epoch = ring->seqnum % H2D_EPOCH_MODULO;
5939	9700	ring->seqnum++;
..	..	@@ -5949,9 +9710,39 @@
5949	9710
5950	9711	/* update ring's WR index and ring doorbell to dongle */
5951	9712	dhd_prot_ring_write_complete(dhd, ring, flow_delete_rqst, 1);
5952		- DHD_GENERAL_UNLOCK(dhd, flags);
	9713	+
	9714	+ DHD_RING_UNLOCK(ring->ring_lock, flags);
5953	9715
5954	9716	return BCME_OK;
	9717	+}
	9718	+
	9719	+static void BCMFASTPATH
	9720	+dhd_prot_flow_ring_fastdelete(dhd_pub_t *dhd, uint16 flowid, uint16 rd_idx)
	9721	+{
	9722	+ flow_ring_node_t *flow_ring_node = DHD_FLOW_RING(dhd, flowid);
	9723	+ msgbuf_ring_t ring = (msgbuf_ring_t )flow_ring_node->prot_info;
	9724	+ host_txbuf_cmpl_t txstatus;
	9725	+ host_txbuf_post_t *txdesc;
	9726	+ uint16 wr_idx;
	9727	+
	9728	+ DHD_INFO(("%s: FAST delete ring, flowid=%d, rd_idx=%d, wr_idx=%d\n",
	9729	+ __FUNCTION__, flowid, rd_idx, ring->wr));
	9730	+
	9731	+ memset(&txstatus, 0, sizeof(txstatus));
	9732	+ txstatus.compl_hdr.flow_ring_id = flowid;
	9733	+ txstatus.cmn_hdr.if_id = flow_ring_node->flow_info.ifindex;
	9734	+ wr_idx = ring->wr;
	9735	+
	9736	+ while (wr_idx != rd_idx) {
	9737	+ if (wr_idx)
	9738	+ wr_idx--;
	9739	+ else
	9740	+ wr_idx = ring->max_items - 1;
	9741	+ txdesc = (host_txbuf_post_t )((char )DHD_RING_BGN_VA(ring) +
	9742	+ (wr_idx * ring->item_len));
	9743	+ txstatus.cmn_hdr.request_id = txdesc->cmn_hdr.request_id;
	9744	+ dhd_prot_txstatus_process(dhd, &txstatus);
	9745	+ }
5955	9746	}
5956	9747
5957	9748	static void
..	..	@@ -5962,8 +9753,49 @@
5962	9753	DHD_ERROR(("%s: Flow Delete Response status = %d Flow %d\n", __FUNCTION__,
5963	9754	flow_delete_resp->cmplt.status, flow_delete_resp->cmplt.flow_ring_id));
5964	9755
	9756	+ if (dhd->fast_delete_ring_support) {
	9757	+ dhd_prot_flow_ring_fastdelete(dhd, flow_delete_resp->cmplt.flow_ring_id,
	9758	+ flow_delete_resp->read_idx);
	9759	+ }
5965	9760	dhd_bus_flow_ring_delete_response(dhd->bus, flow_delete_resp->cmplt.flow_ring_id,
5966	9761	flow_delete_resp->cmplt.status);
	9762	+}
	9763	+
	9764	+static void
	9765	+dhd_prot_process_flow_ring_resume_response(dhd_pub_t dhd, void msg)
	9766	+{
	9767	+#ifdef IDLE_TX_FLOW_MGMT
	9768	+ tx_idle_flowring_resume_response_t *flow_resume_resp =
	9769	+ (tx_idle_flowring_resume_response_t *)msg;
	9770	+
	9771	+ DHD_ERROR(("%s Flow resume Response status = %d Flow %d\n", __FUNCTION__,
	9772	+ flow_resume_resp->cmplt.status, flow_resume_resp->cmplt.flow_ring_id));
	9773	+
	9774	+ dhd_bus_flow_ring_resume_response(dhd->bus, flow_resume_resp->cmplt.flow_ring_id,
	9775	+ flow_resume_resp->cmplt.status);
	9776	+#endif /* IDLE_TX_FLOW_MGMT */
	9777	+}
	9778	+
	9779	+static void
	9780	+dhd_prot_process_flow_ring_suspend_response(dhd_pub_t dhd, void msg)
	9781	+{
	9782	+#ifdef IDLE_TX_FLOW_MGMT
	9783	+ int16 status;
	9784	+ tx_idle_flowring_suspend_response_t *flow_suspend_resp =
	9785	+ (tx_idle_flowring_suspend_response_t *)msg;
	9786	+ status = flow_suspend_resp->cmplt.status;
	9787	+
	9788	+ DHD_ERROR(("%s Flow id %d suspend Response status = %d\n",
	9789	+ __FUNCTION__, flow_suspend_resp->cmplt.flow_ring_id,
	9790	+ status));
	9791	+
	9792	+ if (status != BCME_OK) {
	9793	+
	9794	+ DHD_ERROR(("%s Error in Suspending Flow rings!!"
	9795	+ "Dongle will still be polling idle rings!!Status = %d \n",
	9796	+ __FUNCTION__, status));
	9797	+ }
	9798	+#endif /* IDLE_TX_FLOW_MGMT */
5967	9799	}
5968	9800
5969	9801	int
..	..	@@ -5975,13 +9807,13 @@
5975	9807	uint16 alloced = 0;
5976	9808	msgbuf_ring_t *ring = &prot->h2dring_ctrl_subn;
5977	9809
5978		- DHD_GENERAL_LOCK(dhd, flags);
	9810	+ DHD_RING_LOCK(ring->ring_lock, flags);
5979	9811
5980	9812	/* Request for ring buffer space */
5981	9813	flow_flush_rqst = (tx_flowring_flush_request_t *)
5982	9814	dhd_prot_alloc_ring_space(dhd, ring, 1, &alloced, FALSE);
5983	9815	if (flow_flush_rqst == NULL) {
5984		- DHD_GENERAL_UNLOCK(dhd, flags);
	9816	+ DHD_RING_UNLOCK(ring->ring_lock, flags);
5985	9817	DHD_ERROR(("%s: Flow Flush Req - failure ring space\n", __FUNCTION__));
5986	9818	return BCME_NOMEM;
5987	9819	}
..	..	@@ -5990,7 +9822,7 @@
5990	9822	flow_flush_rqst->msg.msg_type = MSG_TYPE_FLOW_RING_FLUSH;
5991	9823	flow_flush_rqst->msg.if_id = (uint8)flow_ring_node->flow_info.ifindex;
5992	9824	flow_flush_rqst->msg.request_id = htol32(0); /* TBD */
5993		-
	9825	+ flow_flush_rqst->msg.flags = ring->current_phase;
5994	9826	flow_flush_rqst->msg.epoch = ring->seqnum % H2D_EPOCH_MODULO;
5995	9827	ring->seqnum++;
5996	9828
..	..	@@ -6001,7 +9833,8 @@
6001	9833
6002	9834	/* update ring's WR index and ring doorbell to dongle */
6003	9835	dhd_prot_ring_write_complete(dhd, ring, flow_flush_rqst, 1);
6004		- DHD_GENERAL_UNLOCK(dhd, flags);
	9836	+
	9837	+ DHD_RING_UNLOCK(ring->ring_lock, flags);
6005	9838
6006	9839	return BCME_OK;
6007	9840	} /* dhd_prot_flow_ring_flush */
..	..	@@ -6039,13 +9872,13 @@
6039	9872	const uint16 d2h_rings = BCMPCIE_D2H_COMMON_MSGRINGS;
6040	9873
6041	9874	/* Claim space for d2h_ring number of d2h_ring_config_req_t messages */
6042		- DHD_GENERAL_LOCK(dhd, flags);
	9875	+ DHD_RING_LOCK(ctrl_ring->ring_lock, flags);
6043	9876	msg_start = dhd_prot_alloc_ring_space(dhd, ctrl_ring, d2h_rings, &alloced, TRUE);
6044	9877
6045	9878	if (msg_start == NULL) {
6046	9879	DHD_ERROR(("%s Msgbuf no space for %d D2H ring config soft doorbells\n",
6047	9880	__FUNCTION__, d2h_rings));
6048		- DHD_GENERAL_UNLOCK(dhd, flags);
	9881	+ DHD_RING_UNLOCK(ctrl_ring->ring_lock, flags);
6049	9882	return;
6050	9883	}
6051	9884
..	..	@@ -6091,17 +9924,501 @@
6091	9924
6092	9925	/* update control subn ring's WR index and ring doorbell to dongle */
6093	9926	dhd_prot_ring_write_complete(dhd, ctrl_ring, msg_start, d2h_rings);
6094		- DHD_GENERAL_UNLOCK(dhd, flags);
	9927	+
	9928	+ DHD_RING_UNLOCK(ctrl_ring->ring_lock, flags);
	9929	+
6095	9930	#endif /* DHD_D2H_SOFT_DOORBELL_SUPPORT */
6096	9931	}
6097	9932
6098	9933	static void
6099		-dhd_prot_d2h_ring_config_cmplt_process(dhd_pub_t dhd, void msg)
	9934	+dhd_prot_process_d2h_ring_config_complete(dhd_pub_t dhd, void msg)
6100	9935	{
6101	9936	DHD_INFO(("%s: Ring Config Response - status %d ringid %d\n",
6102	9937	__FUNCTION__, ltoh16(((ring_config_resp_t *)msg)->compl_hdr.status),
6103	9938	ltoh16(((ring_config_resp_t *)msg)->compl_hdr.flow_ring_id)));
6104	9939	}
	9940	+
	9941	+#ifdef WL_CFGVENDOR_SEND_HANG_EVENT
	9942	+void
	9943	+copy_ext_trap_sig(dhd_pub_t dhd, trap_t tr)
	9944	+{
	9945	+ uint32 *ext_data = dhd->extended_trap_data;
	9946	+ hnd_ext_trap_hdr_t *hdr;
	9947	+ const bcm_tlv_t *tlv;
	9948	+
	9949	+ if (ext_data == NULL) {
	9950	+ return;
	9951	+ }
	9952	+ /* First word is original trap_data */
	9953	+ ext_data++;
	9954	+
	9955	+ /* Followed by the extended trap data header */
	9956	+ hdr = (hnd_ext_trap_hdr_t *)ext_data;
	9957	+
	9958	+ tlv = bcm_parse_tlvs(hdr->data, hdr->len, TAG_TRAP_SIGNATURE);
	9959	+ if (tlv) {
	9960	+ memcpy(tr, &tlv->data, sizeof(struct _trap_struct));
	9961	+ }
	9962	+}
	9963	+#define TRAP_T_NAME_OFFSET(var) {#var, OFFSETOF(trap_t, var)}
	9964	+
	9965	+typedef struct {
	9966	+ char name[HANG_INFO_TRAP_T_NAME_MAX];
	9967	+ uint32 offset;
	9968	+} hang_info_trap_t;
	9969	+
	9970	+#ifdef DHD_EWPR_VER2
	9971	+static hang_info_trap_t hang_info_trap_tbl[] = {
	9972	+ {"reason", 0},
	9973	+ {"ver", VENDOR_SEND_HANG_EXT_INFO_VER},
	9974	+ {"stype", 0},
	9975	+ TRAP_T_NAME_OFFSET(type),
	9976	+ TRAP_T_NAME_OFFSET(epc),
	9977	+ {"resrvd", 0},
	9978	+ {"resrvd", 0},
	9979	+ {"resrvd", 0},
	9980	+ {"resrvd", 0},
	9981	+ {"", 0}
	9982	+};
	9983	+#else
	9984	+static hang_info_trap_t hang_info_trap_tbl[] = {
	9985	+ {"reason", 0},
	9986	+ {"ver", VENDOR_SEND_HANG_EXT_INFO_VER},
	9987	+ {"stype", 0},
	9988	+ TRAP_T_NAME_OFFSET(type),
	9989	+ TRAP_T_NAME_OFFSET(epc),
	9990	+ TRAP_T_NAME_OFFSET(cpsr),
	9991	+ TRAP_T_NAME_OFFSET(spsr),
	9992	+ TRAP_T_NAME_OFFSET(r0),
	9993	+ TRAP_T_NAME_OFFSET(r1),
	9994	+ TRAP_T_NAME_OFFSET(r2),
	9995	+ TRAP_T_NAME_OFFSET(r3),
	9996	+ TRAP_T_NAME_OFFSET(r4),
	9997	+ TRAP_T_NAME_OFFSET(r5),
	9998	+ TRAP_T_NAME_OFFSET(r6),
	9999	+ TRAP_T_NAME_OFFSET(r7),
	10000	+ TRAP_T_NAME_OFFSET(r8),
	10001	+ TRAP_T_NAME_OFFSET(r9),
	10002	+ TRAP_T_NAME_OFFSET(r10),
	10003	+ TRAP_T_NAME_OFFSET(r11),
	10004	+ TRAP_T_NAME_OFFSET(r12),
	10005	+ TRAP_T_NAME_OFFSET(r13),
	10006	+ TRAP_T_NAME_OFFSET(r14),
	10007	+ TRAP_T_NAME_OFFSET(pc),
	10008	+ {"", 0}
	10009	+};
	10010	+#endif /* DHD_EWPR_VER2 */
	10011	+
	10012	+#define TAG_TRAP_IS_STATE(tag) \
	10013	+ ((tag == TAG_TRAP_MEMORY) \|\| (tag == TAG_TRAP_PCIE_Q) \|\| \
	10014	+ (tag == TAG_TRAP_WLC_STATE) \|\| (tag == TAG_TRAP_LOG_DATA) \|\| \
	10015	+ (tag == TAG_TRAP_CODE))
	10016	+
	10017	+static void
	10018	+copy_hang_info_head(char dest, trap_t src, int len, int field_name,
	10019	+ int bytes_written, int cnt, char *cookie)
	10020	+{
	10021	+ uint8 *ptr;
	10022	+ int remain_len;
	10023	+ int i;
	10024	+
	10025	+ ptr = (uint8 *)src;
	10026	+
	10027	+ memset(dest, 0, len);
	10028	+ remain_len = VENDOR_SEND_HANG_EXT_INFO_LEN - *bytes_written;
	10029	+
	10030	+ /* hang reason, hang info ver */
	10031	+ for (i = 0; (i < HANG_INFO_TRAP_T_SUBTYPE_IDX) && (*cnt < HANG_FIELD_CNT_MAX);
	10032	+ i++, (*cnt)++) {
	10033	+ if (field_name) {
	10034	+ remain_len = VENDOR_SEND_HANG_EXT_INFO_LEN - *bytes_written;
	10035	+ bytes_written += scnprintf(&dest[bytes_written], remain_len, "%s:%c",
	10036	+ hang_info_trap_tbl[i].name, HANG_KEY_DEL);
	10037	+ }
	10038	+ remain_len = VENDOR_SEND_HANG_EXT_INFO_LEN - *bytes_written;
	10039	+ bytes_written += scnprintf(&dest[bytes_written], remain_len, "%d%c",
	10040	+ hang_info_trap_tbl[i].offset, HANG_KEY_DEL);
	10041	+
	10042	+ }
	10043	+
	10044	+ if (*cnt < HANG_FIELD_CNT_MAX) {
	10045	+ if (field_name) {
	10046	+ remain_len = VENDOR_SEND_HANG_EXT_INFO_LEN - *bytes_written;
	10047	+ bytes_written += scnprintf(&dest[bytes_written], remain_len, "%s:%c",
	10048	+ "cookie", HANG_KEY_DEL);
	10049	+ }
	10050	+ remain_len = VENDOR_SEND_HANG_EXT_INFO_LEN - *bytes_written;
	10051	+ bytes_written += scnprintf(&dest[bytes_written], remain_len, "%s%c",
	10052	+ cookie, HANG_KEY_DEL);
	10053	+ (*cnt)++;
	10054	+ }
	10055	+
	10056	+ if (*cnt < HANG_FIELD_CNT_MAX) {
	10057	+ if (field_name) {
	10058	+ remain_len = VENDOR_SEND_HANG_EXT_INFO_LEN - *bytes_written;
	10059	+ bytes_written += scnprintf(&dest[bytes_written], remain_len, "%s:%c",
	10060	+ hang_info_trap_tbl[HANG_INFO_TRAP_T_SUBTYPE_IDX].name,
	10061	+ HANG_KEY_DEL);
	10062	+ }
	10063	+ remain_len = VENDOR_SEND_HANG_EXT_INFO_LEN - *bytes_written;
	10064	+ bytes_written += scnprintf(&dest[bytes_written], remain_len, "%08x%c",
	10065	+ hang_info_trap_tbl[HANG_INFO_TRAP_T_SUBTYPE_IDX].offset,
	10066	+ HANG_KEY_DEL);
	10067	+ (*cnt)++;
	10068	+ }
	10069	+
	10070	+ if (*cnt < HANG_FIELD_CNT_MAX) {
	10071	+ if (field_name) {
	10072	+ remain_len = VENDOR_SEND_HANG_EXT_INFO_LEN - *bytes_written;
	10073	+ bytes_written += scnprintf(&dest[bytes_written], remain_len, "%s:%c",
	10074	+ hang_info_trap_tbl[HANG_INFO_TRAP_T_EPC_IDX].name,
	10075	+ HANG_KEY_DEL);
	10076	+ }
	10077	+ remain_len = VENDOR_SEND_HANG_EXT_INFO_LEN - *bytes_written;
	10078	+ bytes_written += scnprintf(&dest[bytes_written], remain_len, "%08x%c",
	10079	+ (uint32 )
	10080	+ (ptr + hang_info_trap_tbl[HANG_INFO_TRAP_T_EPC_IDX].offset),
	10081	+ HANG_KEY_DEL);
	10082	+ (*cnt)++;
	10083	+ }
	10084	+#ifdef DHD_EWPR_VER2
	10085	+ /* put 0 for HG03 ~ HG06 (reserved for future use) */
	10086	+ for (i = 0; (i < HANG_INFO_BIGDATA_EXTRA_KEY) && (*cnt < HANG_FIELD_CNT_MAX);
	10087	+ i++, (*cnt)++) {
	10088	+ if (field_name) {
	10089	+ remain_len = VENDOR_SEND_HANG_EXT_INFO_LEN - *bytes_written;
	10090	+ bytes_written += scnprintf(&dest[bytes_written], remain_len, "%s:%c",
	10091	+ hang_info_trap_tbl[HANG_INFO_TRAP_T_EXTRA_KEY_IDX+i].name,
	10092	+ HANG_KEY_DEL);
	10093	+ }
	10094	+ remain_len = VENDOR_SEND_HANG_EXT_INFO_LEN - *bytes_written;
	10095	+ bytes_written += scnprintf(&dest[bytes_written], remain_len, "%d%c",
	10096	+ hang_info_trap_tbl[HANG_INFO_TRAP_T_EXTRA_KEY_IDX+i].offset,
	10097	+ HANG_KEY_DEL);
	10098	+ }
	10099	+#endif /* DHD_EWPR_VER2 */
	10100	+}
	10101	+#ifndef DHD_EWPR_VER2
	10102	+static void
	10103	+copy_hang_info_trap_t(char dest, trap_t src, int len, int field_name,
	10104	+ int bytes_written, int cnt, char *cookie)
	10105	+{
	10106	+ uint8 *ptr;
	10107	+ int remain_len;
	10108	+ int i;
	10109	+
	10110	+ ptr = (uint8 *)src;
	10111	+
	10112	+ remain_len = VENDOR_SEND_HANG_EXT_INFO_LEN - *bytes_written;
	10113	+
	10114	+ for (i = HANG_INFO_TRAP_T_OFFSET_IDX;
	10115	+ (hang_info_trap_tbl[i].name[0] != 0) && (*cnt < HANG_FIELD_CNT_MAX);
	10116	+ i++, (*cnt)++) {
	10117	+ if (field_name) {
	10118	+ remain_len = VENDOR_SEND_HANG_EXT_INFO_LEN - *bytes_written;
	10119	+ bytes_written += scnprintf(&dest[bytes_written], remain_len, "%c%s:",
	10120	+ HANG_RAW_DEL, hang_info_trap_tbl[i].name);
	10121	+ }
	10122	+ remain_len = VENDOR_SEND_HANG_EXT_INFO_LEN - *bytes_written;
	10123	+ bytes_written += scnprintf(&dest[bytes_written], remain_len, "%c%08x",
	10124	+ HANG_RAW_DEL, (uint32 )(ptr + hang_info_trap_tbl[i].offset));
	10125	+ }
	10126	+}
	10127	+
	10128	+/* Ignore compiler warnings due to -Werror=cast-qual */
	10129	+#if defined(STRICT_GCC_WARNINGS) && defined(__GNUC__)
	10130	+#pragma GCC diagnostic push
	10131	+#pragma GCC diagnostic ignored "-Wcast-qual"
	10132	+#endif // endif
	10133	+
	10134	+static void
	10135	+copy_hang_info_stack(dhd_pub_t dhd, char dest, int bytes_written, int cnt)
	10136	+{
	10137	+ int remain_len;
	10138	+ int i = 0;
	10139	+ const uint32 *stack;
	10140	+ uint32 *ext_data = dhd->extended_trap_data;
	10141	+ hnd_ext_trap_hdr_t *hdr;
	10142	+ const bcm_tlv_t *tlv;
	10143	+ int remain_stack_cnt = 0;
	10144	+ uint32 dummy_data = 0;
	10145	+ int bigdata_key_stack_cnt = 0;
	10146	+
	10147	+ if (ext_data == NULL) {
	10148	+ return;
	10149	+ }
	10150	+ /* First word is original trap_data */
	10151	+ ext_data++;
	10152	+
	10153	+ /* Followed by the extended trap data header */
	10154	+ hdr = (hnd_ext_trap_hdr_t *)ext_data;
	10155	+
	10156	+ tlv = bcm_parse_tlvs(hdr->data, hdr->len, TAG_TRAP_STACK);
	10157	+
	10158	+ remain_len = VENDOR_SEND_HANG_EXT_INFO_LEN - *bytes_written;
	10159	+
	10160	+ if (tlv) {
	10161	+ stack = (const uint32 *)tlv->data;
	10162	+
	10163	+ bytes_written += scnprintf(&dest[bytes_written], remain_len,
	10164	+ "%08x", (uint32 )(stack++));
	10165	+ (*cnt)++;
	10166	+ if (*cnt >= HANG_FIELD_CNT_MAX) {
	10167	+ return;
	10168	+ }
	10169	+ for (i = 1; i < (uint32)(tlv->len / sizeof(uint32)); i++, bigdata_key_stack_cnt++) {
	10170	+ remain_len = VENDOR_SEND_HANG_EXT_INFO_LEN - *bytes_written;
	10171	+ /* Raw data for bigdata use '_' and Key data for bigdata use space */
	10172	+ bytes_written += scnprintf(&dest[bytes_written], remain_len,
	10173	+ "%c%08x",
	10174	+ i <= HANG_INFO_BIGDATA_KEY_STACK_CNT ? HANG_KEY_DEL : HANG_RAW_DEL,
	10175	+ (uint32 )(stack++));
	10176	+
	10177	+ (*cnt)++;
	10178	+ if ((*cnt >= HANG_FIELD_CNT_MAX) \|\|
	10179	+ (i >= HANG_FIELD_TRAP_T_STACK_CNT_MAX)) {
	10180	+ return;
	10181	+ }
	10182	+ }
	10183	+ }
	10184	+
	10185	+ remain_stack_cnt = HANG_FIELD_TRAP_T_STACK_CNT_MAX - i;
	10186	+
	10187	+ for (i = 0; i < remain_stack_cnt; i++) {
	10188	+ remain_len = VENDOR_SEND_HANG_EXT_INFO_LEN - *bytes_written;
	10189	+ bytes_written += scnprintf(&dest[bytes_written], remain_len, "%c%08x",
	10190	+ HANG_RAW_DEL, dummy_data);
	10191	+ (*cnt)++;
	10192	+ if (*cnt >= HANG_FIELD_CNT_MAX) {
	10193	+ return;
	10194	+ }
	10195	+ }
	10196	+
	10197	+}
	10198	+
	10199	+static void
	10200	+copy_hang_info_specific(dhd_pub_t dhd, char dest, int bytes_written, int cnt)
	10201	+{
	10202	+ int remain_len;
	10203	+ int i;
	10204	+ const uint32 *data;
	10205	+ uint32 *ext_data = dhd->extended_trap_data;
	10206	+ hnd_ext_trap_hdr_t *hdr;
	10207	+ const bcm_tlv_t *tlv;
	10208	+ int remain_trap_data = 0;
	10209	+ uint8 buf_u8[sizeof(uint32)] = { 0, };
	10210	+ const uint8 *p_u8;
	10211	+
	10212	+ if (ext_data == NULL) {
	10213	+ return;
	10214	+ }
	10215	+ /* First word is original trap_data */
	10216	+ ext_data++;
	10217	+
	10218	+ /* Followed by the extended trap data header */
	10219	+ hdr = (hnd_ext_trap_hdr_t *)ext_data;
	10220	+
	10221	+ tlv = bcm_parse_tlvs(hdr->data, hdr->len, TAG_TRAP_SIGNATURE);
	10222	+ if (tlv) {
	10223	+ /* header include tlv hader */
	10224	+ remain_trap_data = (hdr->len - tlv->len - sizeof(uint16));
	10225	+ }
	10226	+
	10227	+ tlv = bcm_parse_tlvs(hdr->data, hdr->len, TAG_TRAP_STACK);
	10228	+ if (tlv) {
	10229	+ /* header include tlv hader */
	10230	+ remain_trap_data -= (tlv->len + sizeof(uint16));
	10231	+ }
	10232	+
	10233	+ data = (const uint32 *)(hdr->data + (hdr->len - remain_trap_data));
	10234	+
	10235	+ remain_len = VENDOR_SEND_HANG_EXT_INFO_LEN - *bytes_written;
	10236	+
	10237	+ for (i = 0; i < (uint32)(remain_trap_data / sizeof(uint32)) && *cnt < HANG_FIELD_CNT_MAX;
	10238	+ i++, (*cnt)++) {
	10239	+ remain_len = VENDOR_SEND_HANG_EXT_INFO_LEN - *bytes_written;
	10240	+ bytes_written += scnprintf(&dest[bytes_written], remain_len, "%c%08x",
	10241	+ HANG_RAW_DEL, (uint32 )(data++));
	10242	+ }
	10243	+
	10244	+ if (*cnt >= HANG_FIELD_CNT_MAX) {
	10245	+ return;
	10246	+ }
	10247	+
	10248	+ remain_trap_data -= (sizeof(uint32) * i);
	10249	+
	10250	+ if (remain_trap_data > sizeof(buf_u8)) {
	10251	+ DHD_ERROR(("%s: resize remain_trap_data\n", __FUNCTION__));
	10252	+ remain_trap_data = sizeof(buf_u8);
	10253	+ }
	10254	+
	10255	+ if (remain_trap_data) {
	10256	+ p_u8 = (const uint8 *)data;
	10257	+ for (i = 0; i < remain_trap_data; i++) {
	10258	+ buf_u8[i] = (const uint8 )(p_u8++);
	10259	+ }
	10260	+
	10261	+ remain_len = VENDOR_SEND_HANG_EXT_INFO_LEN - *bytes_written;
	10262	+ bytes_written += scnprintf(&dest[bytes_written], remain_len, "%c%08x",
	10263	+ HANG_RAW_DEL, ltoh32_ua(buf_u8));
	10264	+ (*cnt)++;
	10265	+ }
	10266	+}
	10267	+#endif /* DHD_EWPR_VER2 */
	10268	+
	10269	+static void
	10270	+get_hang_info_trap_subtype(dhd_pub_t dhd, uint32 subtype)
	10271	+{
	10272	+ uint32 i;
	10273	+ uint32 *ext_data = dhd->extended_trap_data;
	10274	+ hnd_ext_trap_hdr_t *hdr;
	10275	+ const bcm_tlv_t *tlv;
	10276	+
	10277	+ /* First word is original trap_data */
	10278	+ ext_data++;
	10279	+
	10280	+ /* Followed by the extended trap data header */
	10281	+ hdr = (hnd_ext_trap_hdr_t *)ext_data;
	10282	+
	10283	+ /* Dump a list of all tags found before parsing data */
	10284	+ for (i = TAG_TRAP_DEEPSLEEP; i < TAG_TRAP_LAST; i++) {
	10285	+ tlv = bcm_parse_tlvs(hdr->data, hdr->len, i);
	10286	+ if (tlv) {
	10287	+ if (!TAG_TRAP_IS_STATE(i)) {
	10288	+ *subtype = i;
	10289	+ return;
	10290	+ }
	10291	+ }
	10292	+ }
	10293	+}
	10294	+#ifdef DHD_EWPR_VER2
	10295	+static void
	10296	+copy_hang_info_etd_base64(dhd_pub_t dhd, char dest, int bytes_written, int cnt)
	10297	+{
	10298	+ int remain_len;
	10299	+ uint32 *ext_data = dhd->extended_trap_data;
	10300	+ hnd_ext_trap_hdr_t *hdr;
	10301	+ char *base64_out = NULL;
	10302	+ int base64_cnt;
	10303	+ int max_base64_len = HANG_INFO_BASE64_BUFFER_SIZE;
	10304	+
	10305	+ if (ext_data == NULL) {
	10306	+ return;
	10307	+ }
	10308	+ /* First word is original trap_data */
	10309	+ ext_data++;
	10310	+
	10311	+ /* Followed by the extended trap data header */
	10312	+ hdr = (hnd_ext_trap_hdr_t *)ext_data;
	10313	+
	10314	+ remain_len = VENDOR_SEND_HANG_EXT_INFO_LEN - *bytes_written;
	10315	+
	10316	+ if (remain_len <= 0) {
	10317	+ DHD_ERROR(("%s: no space to put etd\n", __FUNCTION__));
	10318	+ return;
	10319	+ }
	10320	+
	10321	+ if (remain_len < max_base64_len) {
	10322	+ DHD_ERROR(("%s: change max base64 length to remain length %d\n", __FUNCTION__,
	10323	+ remain_len));
	10324	+ max_base64_len = remain_len;
	10325	+ }
	10326	+
	10327	+ base64_out = MALLOCZ(dhd->osh, HANG_INFO_BASE64_BUFFER_SIZE);
	10328	+ if (base64_out == NULL) {
	10329	+ DHD_ERROR(("%s: MALLOC failed for size %d\n",
	10330	+ __FUNCTION__, HANG_INFO_BASE64_BUFFER_SIZE));
	10331	+ return;
	10332	+ }
	10333	+
	10334	+ if (hdr->len > 0) {
	10335	+ base64_cnt = dhd_base64_encode(hdr->data, hdr->len, base64_out, max_base64_len);
	10336	+ if (base64_cnt == 0) {
	10337	+ DHD_ERROR(("%s: base64 encoding error\n", __FUNCTION__));
	10338	+ }
	10339	+ }
	10340	+
	10341	+ bytes_written += scnprintf(&dest[bytes_written], remain_len, "%s",
	10342	+ base64_out);
	10343	+ (*cnt)++;
	10344	+ MFREE(dhd->osh, base64_out, HANG_INFO_BASE64_BUFFER_SIZE);
	10345	+}
	10346	+#endif /* DHD_EWPR_VER2 */
	10347	+
	10348	+#if defined(STRICT_GCC_WARNINGS) && defined(__GNUC__)
	10349	+#pragma GCC diagnostic pop
	10350	+#endif // endif
	10351	+
	10352	+void
	10353	+copy_hang_info_trap(dhd_pub_t *dhd)
	10354	+{
	10355	+ trap_t tr;
	10356	+ int bytes_written;
	10357	+ int trap_subtype = 0;
	10358	+
	10359	+ if (!dhd \|\| !dhd->hang_info) {
	10360	+ DHD_ERROR(("%s dhd=%p hang_info=%p\n", __FUNCTION__,
	10361	+ dhd, (dhd ? dhd->hang_info : NULL)));
	10362	+ return;
	10363	+ }
	10364	+
	10365	+ if (!dhd->dongle_trap_occured) {
	10366	+ DHD_ERROR(("%s: dongle_trap_occured is FALSE\n", __FUNCTION__));
	10367	+ return;
	10368	+ }
	10369	+
	10370	+ memset(&tr, 0x00, sizeof(struct _trap_struct));
	10371	+
	10372	+ copy_ext_trap_sig(dhd, &tr);
	10373	+ get_hang_info_trap_subtype(dhd, &trap_subtype);
	10374	+
	10375	+ hang_info_trap_tbl[HANG_INFO_TRAP_T_REASON_IDX].offset = HANG_REASON_DONGLE_TRAP;
	10376	+ hang_info_trap_tbl[HANG_INFO_TRAP_T_SUBTYPE_IDX].offset = trap_subtype;
	10377	+
	10378	+ bytes_written = 0;
	10379	+ dhd->hang_info_cnt = 0;
	10380	+ get_debug_dump_time(dhd->debug_dump_time_hang_str);
	10381	+ copy_debug_dump_time(dhd->debug_dump_time_str, dhd->debug_dump_time_hang_str);
	10382	+
	10383	+ copy_hang_info_head(dhd->hang_info, &tr, VENDOR_SEND_HANG_EXT_INFO_LEN, FALSE,
	10384	+ &bytes_written, &dhd->hang_info_cnt, dhd->debug_dump_time_hang_str);
	10385	+
	10386	+ DHD_INFO(("hang info haed cnt: %d len: %d data: %s\n",
	10387	+ dhd->hang_info_cnt, (int)strlen(dhd->hang_info), dhd->hang_info));
	10388	+
	10389	+ clear_debug_dump_time(dhd->debug_dump_time_hang_str);
	10390	+
	10391	+#ifdef DHD_EWPR_VER2
	10392	+ /* stack info & trap info are included in etd data */
	10393	+
	10394	+ /* extended trap data dump */
	10395	+ if (dhd->hang_info_cnt < HANG_FIELD_CNT_MAX) {
	10396	+ copy_hang_info_etd_base64(dhd, dhd->hang_info, &bytes_written, &dhd->hang_info_cnt);
	10397	+ DHD_INFO(("hang info specific cnt: %d len: %d data: %s\n",
	10398	+ dhd->hang_info_cnt, (int)strlen(dhd->hang_info), dhd->hang_info));
	10399	+ }
	10400	+#else
	10401	+ if (dhd->hang_info_cnt < HANG_FIELD_CNT_MAX) {
	10402	+ copy_hang_info_stack(dhd, dhd->hang_info, &bytes_written, &dhd->hang_info_cnt);
	10403	+ DHD_INFO(("hang info stack cnt: %d len: %d data: %s\n",
	10404	+ dhd->hang_info_cnt, (int)strlen(dhd->hang_info), dhd->hang_info));
	10405	+ }
	10406	+
	10407	+ if (dhd->hang_info_cnt < HANG_FIELD_CNT_MAX) {
	10408	+ copy_hang_info_trap_t(dhd->hang_info, &tr, VENDOR_SEND_HANG_EXT_INFO_LEN, FALSE,
	10409	+ &bytes_written, &dhd->hang_info_cnt, dhd->debug_dump_time_hang_str);
	10410	+ DHD_INFO(("hang info trap_t cnt: %d len: %d data: %s\n",
	10411	+ dhd->hang_info_cnt, (int)strlen(dhd->hang_info), dhd->hang_info));
	10412	+ }
	10413	+
	10414	+ if (dhd->hang_info_cnt < HANG_FIELD_CNT_MAX) {
	10415	+ copy_hang_info_specific(dhd, dhd->hang_info, &bytes_written, &dhd->hang_info_cnt);
	10416	+ DHD_INFO(("hang info specific cnt: %d len: %d data: %s\n",
	10417	+ dhd->hang_info_cnt, (int)strlen(dhd->hang_info), dhd->hang_info));
	10418	+ }
	10419	+#endif /* DHD_EWPR_VER2 */
	10420	+}
	10421	+#endif /* WL_CFGVENDOR_SEND_HANG_EVENT */
6105	10422
6106	10423	int
6107	10424	dhd_prot_debug_info_print(dhd_pub_t *dhd)
..	..	@@ -6109,93 +10426,266 @@
6109	10426	dhd_prot_t *prot = dhd->prot;
6110	10427	msgbuf_ring_t *ring;
6111	10428	uint16 rd, wr;
6112		- uint32 intstatus = 0;
6113		- uint32 intmask = 0;
6114		- uint32 mbintstatus = 0;
6115		- uint32 d2h_mb_data = 0;
6116	10429	uint32 dma_buf_len;
	10430	+ uint64 current_time;
	10431	+ ulong ring_tcm_rd_addr; /* dongle address */
	10432	+ ulong ring_tcm_wr_addr; /* dongle address */
	10433	+
	10434	+ DHD_ERROR(("\n ------- DUMPING VERSION INFORMATION ------- \r\n"));
	10435	+ DHD_ERROR(("DHD: %s\n", dhd_version));
	10436	+ DHD_ERROR(("Firmware: %s\n", fw_version));
	10437	+
	10438	+ DHD_ERROR(("\n ------- DUMPING PROTOCOL INFORMATION ------- \r\n"));
	10439	+ DHD_ERROR(("ICPrevs: Dev %d, Host %d, active %d\n",
	10440	+ prot->device_ipc_version,
	10441	+ prot->host_ipc_version,
	10442	+ prot->active_ipc_version));
	10443	+ DHD_ERROR(("d2h_intr_method -> %s\n",
	10444	+ dhd->bus->d2h_intr_method ? "PCIE_MSI" : "PCIE_INTX"));
	10445	+ DHD_ERROR(("max Host TS bufs to post: %d, posted %d\n",
	10446	+ prot->max_tsbufpost, prot->cur_ts_bufs_posted));
	10447	+ DHD_ERROR(("max INFO bufs to post: %d, posted %d\n",
	10448	+ prot->max_infobufpost, prot->infobufpost));
	10449	+ DHD_ERROR(("max event bufs to post: %d, posted %d\n",
	10450	+ prot->max_eventbufpost, prot->cur_event_bufs_posted));
	10451	+ DHD_ERROR(("max ioctlresp bufs to post: %d, posted %d\n",
	10452	+ prot->max_ioctlrespbufpost, prot->cur_ioctlresp_bufs_posted));
	10453	+ DHD_ERROR(("max RX bufs to post: %d, posted %d\n",
	10454	+ prot->max_rxbufpost, prot->rxbufpost));
	10455	+ DHD_ERROR(("h2d_max_txpost: %d, prot->h2d_max_txpost: %d\n",
	10456	+ h2d_max_txpost, prot->h2d_max_txpost));
	10457	+
	10458	+ current_time = OSL_LOCALTIME_NS();
	10459	+ DHD_ERROR(("current_time="SEC_USEC_FMT"\n", GET_SEC_USEC(current_time)));
	10460	+ DHD_ERROR(("ioctl_fillup_time="SEC_USEC_FMT
	10461	+ " ioctl_ack_time="SEC_USEC_FMT
	10462	+ " ioctl_cmplt_time="SEC_USEC_FMT"\n",
	10463	+ GET_SEC_USEC(prot->ioctl_fillup_time),
	10464	+ GET_SEC_USEC(prot->ioctl_ack_time),
	10465	+ GET_SEC_USEC(prot->ioctl_cmplt_time)));
	10466	+
	10467	+ /* Check PCIe INT registers */
	10468	+ if (!dhd_pcie_dump_int_regs(dhd)) {
	10469	+ DHD_ERROR(("%s : PCIe link might be down\n", __FUNCTION__));
	10470	+ dhd->bus->is_linkdown = TRUE;
	10471	+ }
6117	10472
6118	10473	DHD_ERROR(("\n ------- DUMPING IOCTL RING RD WR Pointers ------- \r\n"));
6119	10474
6120	10475	ring = &prot->h2dring_ctrl_subn;
6121	10476	dma_buf_len = ring->max_items * ring->item_len;
6122		- DHD_ERROR(("CtrlPost: Mem Info: BASE(VA) %p BASE(PA) %x:%x SIZE %d \r\n",
	10477	+ ring_tcm_rd_addr = dhd->bus->ring_sh[ring->idx].ring_state_r;
	10478	+ ring_tcm_wr_addr = dhd->bus->ring_sh[ring->idx].ring_state_w;
	10479	+ DHD_ERROR(("CtrlPost: Mem Info: BASE(VA) %p BASE(PA) %x:%x tcm_rd_wr 0x%lx:0x%lx "
	10480	+ "SIZE %d \r\n",
6123	10481	ring->dma_buf.va, ltoh32(ring->base_addr.high_addr),
6124		- ltoh32(ring->base_addr.low_addr), dma_buf_len));
	10482	+ ltoh32(ring->base_addr.low_addr), ring_tcm_rd_addr, ring_tcm_wr_addr, dma_buf_len));
6125	10483	DHD_ERROR(("CtrlPost: From Host mem: RD: %d WR %d \r\n", ring->rd, ring->wr));
6126		- dhd_bus_cmn_readshared(dhd->bus, &rd, RING_RD_UPD, ring->idx);
6127		- dhd_bus_cmn_readshared(dhd->bus, &wr, RING_WR_UPD, ring->idx);
6128		- DHD_ERROR(("CtrlPost: From Shared Mem: RD: %d WR %d \r\n", rd, wr));
	10484	+ if (dhd->bus->is_linkdown) {
	10485	+ DHD_ERROR(("CtrlPost: From Shared Mem: RD and WR are invalid"
	10486	+ " due to PCIe link down\r\n"));
	10487	+ } else {
	10488	+ dhd_bus_cmn_readshared(dhd->bus, &rd, RING_RD_UPD, ring->idx);
	10489	+ dhd_bus_cmn_readshared(dhd->bus, &wr, RING_WR_UPD, ring->idx);
	10490	+ DHD_ERROR(("CtrlPost: From Shared Mem: RD: %d WR %d \r\n", rd, wr));
	10491	+ }
	10492	+ DHD_ERROR(("CtrlPost: seq num: %d \r\n", ring->seqnum % H2D_EPOCH_MODULO));
6129	10493
6130	10494	ring = &prot->d2hring_ctrl_cpln;
6131	10495	dma_buf_len = ring->max_items * ring->item_len;
6132		- DHD_ERROR(("CtrlCpl: Mem Info: BASE(VA) %p BASE(PA) %x:%x SIZE %d \r\n",
	10496	+ ring_tcm_rd_addr = dhd->bus->ring_sh[ring->idx].ring_state_r;
	10497	+ ring_tcm_wr_addr = dhd->bus->ring_sh[ring->idx].ring_state_w;
	10498	+ DHD_ERROR(("CtrlCpl: Mem Info: BASE(VA) %p BASE(PA) %x:%x tcm_rd_wr 0x%lx:0x%lx "
	10499	+ "SIZE %d \r\n",
6133	10500	ring->dma_buf.va, ltoh32(ring->base_addr.high_addr),
6134		- ltoh32(ring->base_addr.low_addr), dma_buf_len));
	10501	+ ltoh32(ring->base_addr.low_addr), ring_tcm_rd_addr, ring_tcm_wr_addr, dma_buf_len));
6135	10502	DHD_ERROR(("CtrlCpl: From Host mem: RD: %d WR %d \r\n", ring->rd, ring->wr));
6136		- dhd_bus_cmn_readshared(dhd->bus, &rd, RING_RD_UPD, ring->idx);
6137		- dhd_bus_cmn_readshared(dhd->bus, &wr, RING_WR_UPD, ring->idx);
6138		- DHD_ERROR(("CtrlCpl: From Shared Mem: RD: %d WR %d \r\n", rd, wr));
6139		- DHD_ERROR(("CtrlCpl: Expected seq num: %d \r\n", ring->seqnum));
	10503	+ if (dhd->bus->is_linkdown) {
	10504	+ DHD_ERROR(("CtrlCpl: From Shared Mem: RD and WR are invalid"
	10505	+ " due to PCIe link down\r\n"));
	10506	+ } else {
	10507	+ dhd_bus_cmn_readshared(dhd->bus, &rd, RING_RD_UPD, ring->idx);
	10508	+ dhd_bus_cmn_readshared(dhd->bus, &wr, RING_WR_UPD, ring->idx);
	10509	+ DHD_ERROR(("CtrlCpl: From Shared Mem: RD: %d WR %d \r\n", rd, wr));
	10510	+ }
	10511	+ DHD_ERROR(("CtrlCpl: Expected seq num: %d \r\n", ring->seqnum % H2D_EPOCH_MODULO));
6140	10512
6141		- intstatus = si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx, PCIMailBoxInt, 0, 0);
6142		- intmask = si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx, PCIMailBoxMask, 0, 0);
6143		- mbintstatus = si_corereg(dhd->bus->sih, dhd->bus->sih->buscoreidx, PCID2H_MailBox, 0, 0);
6144		- dhd_bus_cmn_readshared(dhd->bus, &d2h_mb_data, D2H_MB_DATA, 0);
	10513	+ ring = prot->h2dring_info_subn;
	10514	+ if (ring) {
	10515	+ dma_buf_len = ring->max_items * ring->item_len;
	10516	+ ring_tcm_rd_addr = dhd->bus->ring_sh[ring->idx].ring_state_r;
	10517	+ ring_tcm_wr_addr = dhd->bus->ring_sh[ring->idx].ring_state_w;
	10518	+ DHD_ERROR(("InfoSub: Mem Info: BASE(VA) %p BASE(PA) %x:%x tcm_rd_wr 0x%lx:0x%lx "
	10519	+ "SIZE %d \r\n",
	10520	+ ring->dma_buf.va, ltoh32(ring->base_addr.high_addr),
	10521	+ ltoh32(ring->base_addr.low_addr), ring_tcm_rd_addr, ring_tcm_wr_addr,
	10522	+ dma_buf_len));
	10523	+ DHD_ERROR(("InfoSub: From Host mem: RD: %d WR %d \r\n", ring->rd, ring->wr));
	10524	+ if (dhd->bus->is_linkdown) {
	10525	+ DHD_ERROR(("InfoSub: From Shared Mem: RD and WR are invalid"
	10526	+ " due to PCIe link down\r\n"));
	10527	+ } else {
	10528	+ dhd_bus_cmn_readshared(dhd->bus, &rd, RING_RD_UPD, ring->idx);
	10529	+ dhd_bus_cmn_readshared(dhd->bus, &wr, RING_WR_UPD, ring->idx);
	10530	+ DHD_ERROR(("InfoSub: From Shared Mem: RD: %d WR %d \r\n", rd, wr));
	10531	+ }
	10532	+ DHD_ERROR(("InfoSub: seq num: %d \r\n", ring->seqnum % H2D_EPOCH_MODULO));
	10533	+ }
	10534	+ ring = prot->d2hring_info_cpln;
	10535	+ if (ring) {
	10536	+ dma_buf_len = ring->max_items * ring->item_len;
	10537	+ ring_tcm_rd_addr = dhd->bus->ring_sh[ring->idx].ring_state_r;
	10538	+ ring_tcm_wr_addr = dhd->bus->ring_sh[ring->idx].ring_state_w;
	10539	+ DHD_ERROR(("InfoCpl: Mem Info: BASE(VA) %p BASE(PA) %x:%x tcm_rd_wr 0x%lx:0x%lx "
	10540	+ "SIZE %d \r\n",
	10541	+ ring->dma_buf.va, ltoh32(ring->base_addr.high_addr),
	10542	+ ltoh32(ring->base_addr.low_addr), ring_tcm_rd_addr, ring_tcm_wr_addr,
	10543	+ dma_buf_len));
	10544	+ DHD_ERROR(("InfoCpl: From Host mem: RD: %d WR %d \r\n", ring->rd, ring->wr));
	10545	+ if (dhd->bus->is_linkdown) {
	10546	+ DHD_ERROR(("InfoCpl: From Shared Mem: RD and WR are invalid"
	10547	+ " due to PCIe link down\r\n"));
	10548	+ } else {
	10549	+ dhd_bus_cmn_readshared(dhd->bus, &rd, RING_RD_UPD, ring->idx);
	10550	+ dhd_bus_cmn_readshared(dhd->bus, &wr, RING_WR_UPD, ring->idx);
	10551	+ DHD_ERROR(("InfoCpl: From Shared Mem: RD: %d WR %d \r\n", rd, wr));
	10552	+ }
	10553	+ DHD_ERROR(("InfoCpl: Expected seq num: %d \r\n", ring->seqnum % D2H_EPOCH_MODULO));
	10554	+ }
6145	10555
6146		- DHD_ERROR(("\n ------- DUMPING INTR Status and Masks ------- \r\n"));
6147		- DHD_ERROR(("intstatus=0x%x intmask=0x%x mbintstatus=0x%x\n,",
6148		- intstatus, intmask, mbintstatus));
6149		- DHD_ERROR(("d2h_mb_data=0x%x def_intmask=0x%x \r\n", d2h_mb_data, dhd->bus->def_intmask));
	10556	+ ring = &prot->d2hring_tx_cpln;
	10557	+ if (ring) {
	10558	+ ring_tcm_rd_addr = dhd->bus->ring_sh[ring->idx].ring_state_r;
	10559	+ ring_tcm_wr_addr = dhd->bus->ring_sh[ring->idx].ring_state_w;
	10560	+ dma_buf_len = ring->max_items * ring->item_len;
	10561	+ DHD_ERROR(("TxCpl: Mem Info: BASE(VA) %p BASE(PA) %x:%x tcm_rd_wr 0x%lx:0x%lx "
	10562	+ "SIZE %d \r\n",
	10563	+ ring->dma_buf.va, ltoh32(ring->base_addr.high_addr),
	10564	+ ltoh32(ring->base_addr.low_addr), ring_tcm_rd_addr, ring_tcm_wr_addr,
	10565	+ dma_buf_len));
	10566	+ DHD_ERROR(("TxCpl: From Host mem: RD: %d WR %d \r\n", ring->rd, ring->wr));
	10567	+ if (dhd->bus->is_linkdown) {
	10568	+ DHD_ERROR(("TxCpl: From Shared Mem: RD and WR are invalid"
	10569	+ " due to PCIe link down\r\n"));
	10570	+ } else {
	10571	+ dhd_bus_cmn_readshared(dhd->bus, &rd, RING_RD_UPD, ring->idx);
	10572	+ dhd_bus_cmn_readshared(dhd->bus, &wr, RING_WR_UPD, ring->idx);
	10573	+ DHD_ERROR(("TxCpl: From Shared Mem: RD: %d WR %d \r\n", rd, wr));
	10574	+ }
	10575	+ DHD_ERROR(("TxCpl: Expected seq num: %d \r\n", ring->seqnum % D2H_EPOCH_MODULO));
	10576	+ }
	10577	+
	10578	+ ring = &prot->d2hring_rx_cpln;
	10579	+ if (ring) {
	10580	+ ring_tcm_rd_addr = dhd->bus->ring_sh[ring->idx].ring_state_r;
	10581	+ ring_tcm_wr_addr = dhd->bus->ring_sh[ring->idx].ring_state_w;
	10582	+ dma_buf_len = ring->max_items * ring->item_len;
	10583	+ DHD_ERROR(("RxCpl: Mem Info: BASE(VA) %p BASE(PA) %x:%x tcm_rd_wr 0x%lx:0x%lx "
	10584	+ "SIZE %d \r\n",
	10585	+ ring->dma_buf.va, ltoh32(ring->base_addr.high_addr),
	10586	+ ltoh32(ring->base_addr.low_addr), ring_tcm_rd_addr, ring_tcm_wr_addr,
	10587	+ dma_buf_len));
	10588	+ DHD_ERROR(("RxCpl: From Host mem: RD: %d WR %d \r\n", ring->rd, ring->wr));
	10589	+ if (dhd->bus->is_linkdown) {
	10590	+ DHD_ERROR(("RxCpl: From Shared Mem: RD and WR are invalid"
	10591	+ " due to PCIe link down\r\n"));
	10592	+ } else {
	10593	+ dhd_bus_cmn_readshared(dhd->bus, &rd, RING_RD_UPD, ring->idx);
	10594	+ dhd_bus_cmn_readshared(dhd->bus, &wr, RING_WR_UPD, ring->idx);
	10595	+ DHD_ERROR(("RxCpl: From Shared Mem: RD: %d WR %d \r\n", rd, wr));
	10596	+ }
	10597	+ DHD_ERROR(("RxCpl: Expected seq num: %d \r\n", ring->seqnum % D2H_EPOCH_MODULO));
	10598	+ }
	10599	+#ifdef EWP_EDL
	10600	+ ring = prot->d2hring_edl;
	10601	+ if (ring) {
	10602	+ ring_tcm_rd_addr = dhd->bus->ring_sh[ring->idx].ring_state_r;
	10603	+ ring_tcm_wr_addr = dhd->bus->ring_sh[ring->idx].ring_state_w;
	10604	+ dma_buf_len = ring->max_items * ring->item_len;
	10605	+ DHD_ERROR(("EdlRing: Mem Info: BASE(VA) %p BASE(PA) %x:%x tcm_rd_wr 0x%lx:0x%lx "
	10606	+ "SIZE %d \r\n",
	10607	+ ring->dma_buf.va, ltoh32(ring->base_addr.high_addr),
	10608	+ ltoh32(ring->base_addr.low_addr), ring_tcm_rd_addr, ring_tcm_wr_addr,
	10609	+ dma_buf_len));
	10610	+ DHD_ERROR(("EdlRing: From Host mem: RD: %d WR %d \r\n", ring->rd, ring->wr));
	10611	+ if (dhd->bus->is_linkdown) {
	10612	+ DHD_ERROR(("EdlRing: From Shared Mem: RD and WR are invalid"
	10613	+ " due to PCIe link down\r\n"));
	10614	+ } else {
	10615	+ dhd_bus_cmn_readshared(dhd->bus, &rd, RING_RD_UPD, ring->idx);
	10616	+ dhd_bus_cmn_readshared(dhd->bus, &wr, RING_WR_UPD, ring->idx);
	10617	+ DHD_ERROR(("EdlRing: From Shared Mem: RD: %d WR %d \r\n", rd, wr));
	10618	+ }
	10619	+ DHD_ERROR(("EdlRing: Expected seq num: %d \r\n",
	10620	+ ring->seqnum % D2H_EPOCH_MODULO));
	10621	+ }
	10622	+#endif /* EWP_EDL */
	10623	+
	10624	+ DHD_ERROR(("%s: cur_ioctlresp_bufs_posted %d cur_event_bufs_posted %d\n",
	10625	+ __FUNCTION__, prot->cur_ioctlresp_bufs_posted, prot->cur_event_bufs_posted));
	10626	+#ifdef DHD_LIMIT_MULTI_CLIENT_FLOWRINGS
	10627	+ DHD_ERROR(("%s: multi_client_flow_rings:%d max_multi_client_flow_rings:%d\n",
	10628	+ __FUNCTION__, dhd->multi_client_flow_rings, dhd->max_multi_client_flow_rings));
	10629	+#endif /* DHD_LIMIT_MULTI_CLIENT_FLOWRINGS */
	10630	+
	10631	+ DHD_ERROR(("pktid_txq_start_cnt: %d\n", prot->pktid_txq_start_cnt));
	10632	+ DHD_ERROR(("pktid_txq_stop_cnt: %d\n", prot->pktid_txq_stop_cnt));
	10633	+ DHD_ERROR(("pktid_depleted_cnt: %d\n", prot->pktid_depleted_cnt));
	10634	+
	10635	+ dhd_pcie_debug_info_dump(dhd);
6150	10636
6151	10637	return 0;
6152	10638	}
6153		-
6154	10639
6155	10640	int
6156	10641	dhd_prot_ringupd_dump(dhd_pub_t dhd, struct bcmstrbuf b)
6157	10642	{
6158	10643	uint32 *ptr;
6159	10644	uint32 value;
6160		- uint32 i;
6161		- uint32 max_h2d_queues = dhd_bus_max_h2d_queues(dhd->bus);
6162	10645
6163		- OSL_CACHE_INV((void *)dhd->prot->d2h_dma_indx_wr_buf.va,
6164		- dhd->prot->d2h_dma_indx_wr_buf.len);
	10646	+ if (dhd->prot->d2h_dma_indx_wr_buf.va) {
	10647	+ uint32 i;
	10648	+ uint32 max_h2d_queues = dhd_bus_max_h2d_queues(dhd->bus);
6165	10649
6166		- ptr = (uint32 *)(dhd->prot->d2h_dma_indx_wr_buf.va);
	10650	+ OSL_CACHE_INV((void *)dhd->prot->d2h_dma_indx_wr_buf.va,
	10651	+ dhd->prot->d2h_dma_indx_wr_buf.len);
6167	10652
6168		- bcm_bprintf(b, "\n max_tx_queues %d\n", max_h2d_queues);
	10653	+ ptr = (uint32 *)(dhd->prot->d2h_dma_indx_wr_buf.va);
6169	10654
6170		- bcm_bprintf(b, "\nRPTR block H2D common rings, 0x%04x\n", ptr);
6171		- value = ltoh32(*ptr);
6172		- bcm_bprintf(b, "\tH2D CTRL: value 0x%04x\n", value);
6173		- ptr++;
6174		- value = ltoh32(*ptr);
6175		- bcm_bprintf(b, "\tH2D RXPOST: value 0x%04x\n", value);
	10655	+ bcm_bprintf(b, "\n max_tx_queues %d\n", max_h2d_queues);
6176	10656
6177		- ptr++;
6178		- bcm_bprintf(b, "RPTR block Flow rings , 0x%04x\n", ptr);
6179		- for (i = BCMPCIE_H2D_COMMON_MSGRINGS; i < max_h2d_queues; i++) {
	10657	+ bcm_bprintf(b, "\nRPTR block H2D common rings, 0x%04x\n", ptr);
6180	10658	value = ltoh32(*ptr);
6181		- bcm_bprintf(b, "\tflowring ID %d: value 0x%04x\n", i, value);
	10659	+ bcm_bprintf(b, "\tH2D CTRL: value 0x%04x\n", value);
6182	10660	ptr++;
	10661	+ value = ltoh32(*ptr);
	10662	+ bcm_bprintf(b, "\tH2D RXPOST: value 0x%04x\n", value);
	10663	+
	10664	+ ptr++;
	10665	+ bcm_bprintf(b, "RPTR block Flow rings , 0x%04x\n", ptr);
	10666	+ for (i = BCMPCIE_H2D_COMMON_MSGRINGS; i < max_h2d_queues; i++) {
	10667	+ value = ltoh32(*ptr);
	10668	+ bcm_bprintf(b, "\tflowring ID %d: value 0x%04x\n", i, value);
	10669	+ ptr++;
	10670	+ }
6183	10671	}
6184	10672
6185		- OSL_CACHE_INV((void *)dhd->prot->h2d_dma_indx_rd_buf.va,
6186		- dhd->prot->h2d_dma_indx_rd_buf.len);
	10673	+ if (dhd->prot->h2d_dma_indx_rd_buf.va) {
	10674	+ OSL_CACHE_INV((void *)dhd->prot->h2d_dma_indx_rd_buf.va,
	10675	+ dhd->prot->h2d_dma_indx_rd_buf.len);
6187	10676
6188		- ptr = (uint32 *)(dhd->prot->h2d_dma_indx_rd_buf.va);
	10677	+ ptr = (uint32 *)(dhd->prot->h2d_dma_indx_rd_buf.va);
6189	10678
6190		- bcm_bprintf(b, "\nWPTR block D2H common rings, 0x%04x\n", ptr);
6191		- value = ltoh32(*ptr);
6192		- bcm_bprintf(b, "\tD2H CTRLCPLT: value 0x%04x\n", value);
6193		- ptr++;
6194		- value = ltoh32(*ptr);
6195		- bcm_bprintf(b, "\tD2H TXCPLT: value 0x%04x\n", value);
6196		- ptr++;
6197		- value = ltoh32(*ptr);
6198		- bcm_bprintf(b, "\tD2H RXCPLT: value 0x%04x\n", value);
	10679	+ bcm_bprintf(b, "\nWPTR block D2H common rings, 0x%04x\n", ptr);
	10680	+ value = ltoh32(*ptr);
	10681	+ bcm_bprintf(b, "\tD2H CTRLCPLT: value 0x%04x\n", value);
	10682	+ ptr++;
	10683	+ value = ltoh32(*ptr);
	10684	+ bcm_bprintf(b, "\tD2H TXCPLT: value 0x%04x\n", value);
	10685	+ ptr++;
	10686	+ value = ltoh32(*ptr);
	10687	+ bcm_bprintf(b, "\tD2H RXCPLT: value 0x%04x\n", value);
	10688	+ }
6199	10689
6200	10690	return 0;
6201	10691	}
..	..	@@ -6206,7 +10696,7 @@
6206	10696	dhd_prot_t *prot = dhd->prot;
6207	10697	#if DHD_DBG_SHOW_METADATA
6208	10698	prot->metadata_dbg = val;
6209		-#endif
	10699	+#endif // endif
6210	10700	return (uint32)prot->metadata_dbg;
6211	10701	}
6212	10702
..	..	@@ -6331,3 +10821,910 @@
6331	10821	}
6332	10822
6333	10823	#endif /* DHD_RX_CHAINING */
	10824	+
	10825	+#ifdef IDLE_TX_FLOW_MGMT
	10826	+int
	10827	+dhd_prot_flow_ring_resume(dhd_pub_t dhd, flow_ring_node_t flow_ring_node)
	10828	+{
	10829	+ tx_idle_flowring_resume_request_t *flow_resume_rqst;
	10830	+ msgbuf_ring_t *flow_ring;
	10831	+ dhd_prot_t *prot = dhd->prot;
	10832	+ unsigned long flags;
	10833	+ uint16 alloced = 0;
	10834	+ msgbuf_ring_t *ctrl_ring = &prot->h2dring_ctrl_subn;
	10835	+
	10836	+ /* Fetch a pre-initialized msgbuf_ring from the flowring pool */
	10837	+ flow_ring = dhd_prot_flowrings_pool_fetch(dhd, flow_ring_node->flowid);
	10838	+ if (flow_ring == NULL) {
	10839	+ DHD_ERROR(("%s: dhd_prot_flowrings_pool_fetch TX Flowid %d failed\n",
	10840	+ __FUNCTION__, flow_ring_node->flowid));
	10841	+ return BCME_NOMEM;
	10842	+ }
	10843	+
	10844	+ DHD_RING_LOCK(ctrl_ring->ring_lock, flags);
	10845	+
	10846	+ /* Request for ctrl_ring buffer space */
	10847	+ flow_resume_rqst = (tx_idle_flowring_resume_request_t *)
	10848	+ dhd_prot_alloc_ring_space(dhd, ctrl_ring, 1, &alloced, FALSE);
	10849	+
	10850	+ if (flow_resume_rqst == NULL) {
	10851	+ dhd_prot_flowrings_pool_release(dhd, flow_ring_node->flowid, flow_ring);
	10852	+ DHD_ERROR(("%s: Flow resume Req flowid %d - failure ring space\n",
	10853	+ __FUNCTION__, flow_ring_node->flowid));
	10854	+ DHD_RING_UNLOCK(ctrl_ring->ring_lock, flags);
	10855	+ return BCME_NOMEM;
	10856	+ }
	10857	+
	10858	+ flow_ring_node->prot_info = (void *)flow_ring;
	10859	+
	10860	+ /* Common msg buf hdr */
	10861	+ flow_resume_rqst->msg.msg_type = MSG_TYPE_FLOW_RING_RESUME;
	10862	+ flow_resume_rqst->msg.if_id = (uint8)flow_ring_node->flow_info.ifindex;
	10863	+ flow_resume_rqst->msg.request_id = htol32(0); /* TBD */
	10864	+
	10865	+ flow_resume_rqst->msg.epoch = ctrl_ring->seqnum % H2D_EPOCH_MODULO;
	10866	+ ctrl_ring->seqnum++;
	10867	+
	10868	+ flow_resume_rqst->flow_ring_id = htol16((uint16)flow_ring_node->flowid);
	10869	+ DHD_ERROR(("%s Send Flow resume Req flow ID %d\n",
	10870	+ __FUNCTION__, flow_ring_node->flowid));
	10871	+
	10872	+ /* Update the flow_ring's WRITE index */
	10873	+ if (IDMA_ACTIVE(dhd) \|\| dhd->dma_h2d_ring_upd_support) {
	10874	+ dhd_prot_dma_indx_set(dhd, flow_ring->wr,
	10875	+ H2D_DMA_INDX_WR_UPD, flow_ring->idx);
	10876	+ } else if (IFRM_ACTIVE(dhd) && (flow_ring->idx >= BCMPCIE_H2D_MSGRING_TXFLOW_IDX_START)) {
	10877	+ dhd_prot_dma_indx_set(dhd, flow_ring->wr,
	10878	+ H2D_IFRM_INDX_WR_UPD,
	10879	+ (flow_ring->idx - BCMPCIE_H2D_MSGRING_TXFLOW_IDX_START));
	10880	+ } else {
	10881	+ dhd_bus_cmn_writeshared(dhd->bus, &(flow_ring->wr),
	10882	+ sizeof(uint16), RING_WR_UPD, flow_ring->idx);
	10883	+ }
	10884	+
	10885	+ /* update control subn ring's WR index and ring doorbell to dongle */
	10886	+ dhd_prot_ring_write_complete(dhd, ctrl_ring, flow_resume_rqst, 1);
	10887	+
	10888	+ DHD_RING_UNLOCK(ctrl_ring->ring_lock, flags);
	10889	+
	10890	+ return BCME_OK;
	10891	+} /* dhd_prot_flow_ring_create */
	10892	+
	10893	+int
	10894	+dhd_prot_flow_ring_batch_suspend_request(dhd_pub_t dhd, uint16 ringid, uint16 count)
	10895	+{
	10896	+ tx_idle_flowring_suspend_request_t *flow_suspend_rqst;
	10897	+ dhd_prot_t *prot = dhd->prot;
	10898	+ unsigned long flags;
	10899	+ uint16 index;
	10900	+ uint16 alloced = 0;
	10901	+ msgbuf_ring_t *ring = &prot->h2dring_ctrl_subn;
	10902	+
	10903	+ DHD_RING_LOCK(ring->ring_lock, flags);
	10904	+
	10905	+ /* Request for ring buffer space */
	10906	+ flow_suspend_rqst = (tx_idle_flowring_suspend_request_t *)
	10907	+ dhd_prot_alloc_ring_space(dhd, ring, 1, &alloced, FALSE);
	10908	+
	10909	+ if (flow_suspend_rqst == NULL) {
	10910	+ DHD_RING_UNLOCK(ring->ring_lock, flags);
	10911	+ DHD_ERROR(("%s: Flow suspend Req - failure ring space\n", __FUNCTION__));
	10912	+ return BCME_NOMEM;
	10913	+ }
	10914	+
	10915	+ /* Common msg buf hdr */
	10916	+ flow_suspend_rqst->msg.msg_type = MSG_TYPE_FLOW_RING_SUSPEND;
	10917	+ /* flow_suspend_rqst->msg.if_id = (uint8)flow_ring_node->flow_info.ifindex; */
	10918	+ flow_suspend_rqst->msg.request_id = htol32(0); /* TBD */
	10919	+
	10920	+ flow_suspend_rqst->msg.epoch = ring->seqnum % H2D_EPOCH_MODULO;
	10921	+ ring->seqnum++;
	10922	+
	10923	+ /* Update flow id info */
	10924	+ for (index = 0; index < count; index++)
	10925	+ {
	10926	+ flow_suspend_rqst->ring_id[index] = ringid[index];
	10927	+ }
	10928	+ flow_suspend_rqst->num = count;
	10929	+
	10930	+ DHD_ERROR(("%s sending batch suspend!! count is %d\n", __FUNCTION__, count));
	10931	+
	10932	+ /* update ring's WR index and ring doorbell to dongle */
	10933	+ dhd_prot_ring_write_complete(dhd, ring, flow_suspend_rqst, 1);
	10934	+
	10935	+ DHD_RING_UNLOCK(ring->ring_lock, flags);
	10936	+
	10937	+ return BCME_OK;
	10938	+}
	10939	+#endif /* IDLE_TX_FLOW_MGMT */
	10940	+
	10941	+static const char* etd_trap_name(hnd_ext_tag_trap_t tag)
	10942	+{
	10943	+ switch (tag)
	10944	+ {
	10945	+ case TAG_TRAP_SIGNATURE: return "TAG_TRAP_SIGNATURE";
	10946	+ case TAG_TRAP_STACK: return "TAG_TRAP_STACK";
	10947	+ case TAG_TRAP_MEMORY: return "TAG_TRAP_MEMORY";
	10948	+ case TAG_TRAP_DEEPSLEEP: return "TAG_TRAP_DEEPSLEEP";
	10949	+ case TAG_TRAP_PSM_WD: return "TAG_TRAP_PSM_WD";
	10950	+ case TAG_TRAP_PHY: return "TAG_TRAP_PHY";
	10951	+ case TAG_TRAP_BUS: return "TAG_TRAP_BUS";
	10952	+ case TAG_TRAP_MAC_SUSP: return "TAG_TRAP_MAC_SUSP";
	10953	+ case TAG_TRAP_BACKPLANE: return "TAG_TRAP_BACKPLANE";
	10954	+ case TAG_TRAP_PCIE_Q: return "TAG_TRAP_PCIE_Q";
	10955	+ case TAG_TRAP_WLC_STATE: return "TAG_TRAP_WLC_STATE";
	10956	+ case TAG_TRAP_MAC_WAKE: return "TAG_TRAP_MAC_WAKE";
	10957	+ case TAG_TRAP_HMAP: return "TAG_TRAP_HMAP";
	10958	+ case TAG_TRAP_PHYTXERR_THRESH: return "TAG_TRAP_PHYTXERR_THRESH";
	10959	+ case TAG_TRAP_HC_DATA: return "TAG_TRAP_HC_DATA";
	10960	+ case TAG_TRAP_LOG_DATA: return "TAG_TRAP_LOG_DATA";
	10961	+ case TAG_TRAP_CODE: return "TAG_TRAP_CODE";
	10962	+ case TAG_TRAP_LAST:
	10963	+ default:
	10964	+ return "Unknown";
	10965	+ }
	10966	+ return "Unknown";
	10967	+}
	10968	+
	10969	+int dhd_prot_dump_extended_trap(dhd_pub_t dhdp, struct bcmstrbuf b, bool raw)
	10970	+{
	10971	+ uint32 i;
	10972	+ uint32 *ext_data;
	10973	+ hnd_ext_trap_hdr_t *hdr;
	10974	+ const bcm_tlv_t *tlv;
	10975	+ const trap_t *tr;
	10976	+ const uint32 *stack;
	10977	+ const hnd_ext_trap_bp_err_t *bpe;
	10978	+ uint32 raw_len;
	10979	+
	10980	+ ext_data = dhdp->extended_trap_data;
	10981	+
	10982	+ /* return if there is no extended trap data */
	10983	+ if (!ext_data \|\| !(dhdp->dongle_trap_data & D2H_DEV_EXT_TRAP_DATA))
	10984	+ {
	10985	+ bcm_bprintf(b, "%d (0x%x)", dhdp->dongle_trap_data, dhdp->dongle_trap_data);
	10986	+ return BCME_OK;
	10987	+ }
	10988	+
	10989	+ bcm_bprintf(b, "Extended trap data\n");
	10990	+
	10991	+ /* First word is original trap_data */
	10992	+ bcm_bprintf(b, "trap_data = 0x%08x\n", *ext_data);
	10993	+ ext_data++;
	10994	+
	10995	+ /* Followed by the extended trap data header */
	10996	+ hdr = (hnd_ext_trap_hdr_t *)ext_data;
	10997	+ bcm_bprintf(b, "version: %d, len: %d\n", hdr->version, hdr->len);
	10998	+
	10999	+ /* Dump a list of all tags found before parsing data */
	11000	+ bcm_bprintf(b, "\nTags Found:\n");
	11001	+ for (i = 0; i < TAG_TRAP_LAST; i++) {
	11002	+ tlv = bcm_parse_tlvs(hdr->data, hdr->len, i);
	11003	+ if (tlv)
	11004	+ bcm_bprintf(b, "Tag: %d (%s), Length: %d\n", i, etd_trap_name(i), tlv->len);
	11005	+ }
	11006	+
	11007	+ if (raw)
	11008	+ {
	11009	+ raw_len = sizeof(hnd_ext_trap_hdr_t) + (hdr->len / 4) + (hdr->len % 4 ? 1 : 0);
	11010	+ for (i = 0; i < raw_len; i++)
	11011	+ {
	11012	+ bcm_bprintf(b, "0x%08x ", ext_data[i]);
	11013	+ if (i % 4 == 3)
	11014	+ bcm_bprintf(b, "\n");
	11015	+ }
	11016	+ return BCME_OK;
	11017	+ }
	11018	+
	11019	+ /* Extract the various supported TLVs from the extended trap data */
	11020	+ tlv = bcm_parse_tlvs(hdr->data, hdr->len, TAG_TRAP_CODE);
	11021	+ if (tlv)
	11022	+ {
	11023	+ bcm_bprintf(b, "\n%s len: %d\n", etd_trap_name(TAG_TRAP_CODE), tlv->len);
	11024	+ bcm_bprintf(b, "ETD TYPE: %d\n", tlv->data[0]);
	11025	+ }
	11026	+
	11027	+ tlv = bcm_parse_tlvs(hdr->data, hdr->len, TAG_TRAP_SIGNATURE);
	11028	+ if (tlv)
	11029	+ {
	11030	+ bcm_bprintf(b, "\n%s len: %d\n", etd_trap_name(TAG_TRAP_SIGNATURE), tlv->len);
	11031	+ tr = (const trap_t *)tlv->data;
	11032	+
	11033	+ bcm_bprintf(b, "TRAP %x: pc %x, lr %x, sp %x, cpsr %x, spsr %x\n",
	11034	+ tr->type, tr->pc, tr->r14, tr->r13, tr->cpsr, tr->spsr);
	11035	+ bcm_bprintf(b, " r0 %x, r1 %x, r2 %x, r3 %x, r4 %x, r5 %x, r6 %x\n",
	11036	+ tr->r0, tr->r1, tr->r2, tr->r3, tr->r4, tr->r5, tr->r6);
	11037	+ bcm_bprintf(b, " r7 %x, r8 %x, r9 %x, r10 %x, r11 %x, r12 %x\n",
	11038	+ tr->r7, tr->r8, tr->r9, tr->r10, tr->r11, tr->r12);
	11039	+ }
	11040	+
	11041	+ tlv = bcm_parse_tlvs(hdr->data, hdr->len, TAG_TRAP_STACK);
	11042	+ if (tlv)
	11043	+ {
	11044	+ bcm_bprintf(b, "\n%s len: %d\n", etd_trap_name(TAG_TRAP_STACK), tlv->len);
	11045	+ stack = (const uint32 *)tlv->data;
	11046	+ for (i = 0; i < (uint32)(tlv->len / 4); i++)
	11047	+ {
	11048	+ bcm_bprintf(b, " 0x%08x\n", *stack);
	11049	+ stack++;
	11050	+ }
	11051	+ }
	11052	+
	11053	+ tlv = bcm_parse_tlvs(hdr->data, hdr->len, TAG_TRAP_BACKPLANE);
	11054	+ if (tlv)
	11055	+ {
	11056	+ bcm_bprintf(b, "\n%s len: %d\n", etd_trap_name(TAG_TRAP_BACKPLANE), tlv->len);
	11057	+ bpe = (const hnd_ext_trap_bp_err_t *)tlv->data;
	11058	+ bcm_bprintf(b, " error: %x\n", bpe->error);
	11059	+ bcm_bprintf(b, " coreid: %x\n", bpe->coreid);
	11060	+ bcm_bprintf(b, " baseaddr: %x\n", bpe->baseaddr);
	11061	+ bcm_bprintf(b, " ioctrl: %x\n", bpe->ioctrl);
	11062	+ bcm_bprintf(b, " iostatus: %x\n", bpe->iostatus);
	11063	+ bcm_bprintf(b, " resetctrl: %x\n", bpe->resetctrl);
	11064	+ bcm_bprintf(b, " resetstatus: %x\n", bpe->resetstatus);
	11065	+ bcm_bprintf(b, " errlogctrl: %x\n", bpe->errlogctrl);
	11066	+ bcm_bprintf(b, " errlogdone: %x\n", bpe->errlogdone);
	11067	+ bcm_bprintf(b, " errlogstatus: %x\n", bpe->errlogstatus);
	11068	+ bcm_bprintf(b, " errlogaddrlo: %x\n", bpe->errlogaddrlo);
	11069	+ bcm_bprintf(b, " errlogaddrhi: %x\n", bpe->errlogaddrhi);
	11070	+ bcm_bprintf(b, " errlogid: %x\n", bpe->errlogid);
	11071	+ bcm_bprintf(b, " errloguser: %x\n", bpe->errloguser);
	11072	+ bcm_bprintf(b, " errlogflags: %x\n", bpe->errlogflags);
	11073	+ }
	11074	+
	11075	+ tlv = bcm_parse_tlvs(hdr->data, hdr->len, TAG_TRAP_MEMORY);
	11076	+ if (tlv)
	11077	+ {
	11078	+ const hnd_ext_trap_heap_err_t* hme;
	11079	+
	11080	+ bcm_bprintf(b, "\n%s len: %d\n", etd_trap_name(TAG_TRAP_MEMORY), tlv->len);
	11081	+ hme = (const hnd_ext_trap_heap_err_t *)tlv->data;
	11082	+ bcm_bprintf(b, " arena total: %d\n", hme->arena_total);
	11083	+ bcm_bprintf(b, " heap free: %d\n", hme->heap_free);
	11084	+ bcm_bprintf(b, " heap in use: %d\n", hme->heap_inuse);
	11085	+ bcm_bprintf(b, " mf count: %d\n", hme->mf_count);
	11086	+ bcm_bprintf(b, " stack LWM: %x\n", hme->stack_lwm);
	11087	+
	11088	+ bcm_bprintf(b, " Histogram:\n");
	11089	+ for (i = 0; i < (HEAP_HISTOGRAM_DUMP_LEN * 2); i += 2) {
	11090	+ if (hme->heap_histogm[i] == 0xfffe)
	11091	+ bcm_bprintf(b, " Others\t%d\t?\n", hme->heap_histogm[i + 1]);
	11092	+ else if (hme->heap_histogm[i] == 0xffff)
	11093	+ bcm_bprintf(b, " >= 256K\t%d\t?\n", hme->heap_histogm[i + 1]);
	11094	+ else
	11095	+ bcm_bprintf(b, " %d\t%d\t%d\n", hme->heap_histogm[i] << 2,
	11096	+ hme->heap_histogm[i + 1], (hme->heap_histogm[i] << 2)
	11097	+ * hme->heap_histogm[i + 1]);
	11098	+ }
	11099	+
	11100	+ bcm_bprintf(b, " Max free block: %d\n", hme->max_sz_free_blk[0] << 2);
	11101	+ for (i = 1; i < HEAP_MAX_SZ_BLKS_LEN; i++) {
	11102	+ bcm_bprintf(b, " Next lgst free block: %d\n", hme->max_sz_free_blk[i] << 2);
	11103	+ }
	11104	+ }
	11105	+
	11106	+ tlv = bcm_parse_tlvs(hdr->data, hdr->len, TAG_TRAP_PCIE_Q);
	11107	+ if (tlv)
	11108	+ {
	11109	+ const hnd_ext_trap_pcie_mem_err_t* pqme;
	11110	+
	11111	+ bcm_bprintf(b, "\n%s len: %d\n", etd_trap_name(TAG_TRAP_PCIE_Q), tlv->len);
	11112	+ pqme = (const hnd_ext_trap_pcie_mem_err_t *)tlv->data;
	11113	+ bcm_bprintf(b, " d2h queue len: %x\n", pqme->d2h_queue_len);
	11114	+ bcm_bprintf(b, " d2h req queue len: %x\n", pqme->d2h_req_queue_len);
	11115	+ }
	11116	+
	11117	+ tlv = bcm_parse_tlvs(hdr->data, hdr->len, TAG_TRAP_WLC_STATE);
	11118	+ if (tlv)
	11119	+ {
	11120	+ const hnd_ext_trap_wlc_mem_err_t* wsme;
	11121	+
	11122	+ bcm_bprintf(b, "\n%s len: %d\n", etd_trap_name(TAG_TRAP_WLC_STATE), tlv->len);
	11123	+ wsme = (const hnd_ext_trap_wlc_mem_err_t *)tlv->data;
	11124	+ bcm_bprintf(b, " instance: %d\n", wsme->instance);
	11125	+ bcm_bprintf(b, " associated: %d\n", wsme->associated);
	11126	+ bcm_bprintf(b, " peer count: %d\n", wsme->peer_cnt);
	11127	+ bcm_bprintf(b, " client count: %d\n", wsme->soft_ap_client_cnt);
	11128	+ bcm_bprintf(b, " TX_AC_BK_FIFO: %d\n", wsme->txqueue_len[0]);
	11129	+ bcm_bprintf(b, " TX_AC_BE_FIFO: %d\n", wsme->txqueue_len[1]);
	11130	+ bcm_bprintf(b, " TX_AC_VI_FIFO: %d\n", wsme->txqueue_len[2]);
	11131	+ bcm_bprintf(b, " TX_AC_VO_FIFO: %d\n", wsme->txqueue_len[3]);
	11132	+
	11133	+ if (tlv->len >= (sizeof(wsme) 2)) {
	11134	+ wsme++;
	11135	+ bcm_bprintf(b, "\n instance: %d\n", wsme->instance);
	11136	+ bcm_bprintf(b, " associated: %d\n", wsme->associated);
	11137	+ bcm_bprintf(b, " peer count: %d\n", wsme->peer_cnt);
	11138	+ bcm_bprintf(b, " client count: %d\n", wsme->soft_ap_client_cnt);
	11139	+ bcm_bprintf(b, " TX_AC_BK_FIFO: %d\n", wsme->txqueue_len[0]);
	11140	+ bcm_bprintf(b, " TX_AC_BE_FIFO: %d\n", wsme->txqueue_len[1]);
	11141	+ bcm_bprintf(b, " TX_AC_VI_FIFO: %d\n", wsme->txqueue_len[2]);
	11142	+ bcm_bprintf(b, " TX_AC_VO_FIFO: %d\n", wsme->txqueue_len[3]);
	11143	+ }
	11144	+ }
	11145	+
	11146	+ tlv = bcm_parse_tlvs(hdr->data, hdr->len, TAG_TRAP_PHY);
	11147	+ if (tlv)
	11148	+ {
	11149	+ const hnd_ext_trap_phydbg_t* phydbg;
	11150	+ bcm_bprintf(b, "\n%s len: %d\n", etd_trap_name(TAG_TRAP_PHY), tlv->len);
	11151	+ phydbg = (const hnd_ext_trap_phydbg_t *)tlv->data;
	11152	+ bcm_bprintf(b, " err: 0x%x\n", phydbg->err);
	11153	+ bcm_bprintf(b, " RxFeStatus: 0x%x\n", phydbg->RxFeStatus);
	11154	+ bcm_bprintf(b, " TxFIFOStatus0: 0x%x\n", phydbg->TxFIFOStatus0);
	11155	+ bcm_bprintf(b, " TxFIFOStatus1: 0x%x\n", phydbg->TxFIFOStatus1);
	11156	+ bcm_bprintf(b, " RfseqMode: 0x%x\n", phydbg->RfseqMode);
	11157	+ bcm_bprintf(b, " RfseqStatus0: 0x%x\n", phydbg->RfseqStatus0);
	11158	+ bcm_bprintf(b, " RfseqStatus1: 0x%x\n", phydbg->RfseqStatus1);
	11159	+ bcm_bprintf(b, " RfseqStatus_Ocl: 0x%x\n", phydbg->RfseqStatus_Ocl);
	11160	+ bcm_bprintf(b, " RfseqStatus_Ocl1: 0x%x\n", phydbg->RfseqStatus_Ocl1);
	11161	+ bcm_bprintf(b, " OCLControl1: 0x%x\n", phydbg->OCLControl1);
	11162	+ bcm_bprintf(b, " TxError: 0x%x\n", phydbg->TxError);
	11163	+ bcm_bprintf(b, " bphyTxError: 0x%x\n", phydbg->bphyTxError);
	11164	+ bcm_bprintf(b, " TxCCKError: 0x%x\n", phydbg->TxCCKError);
	11165	+ bcm_bprintf(b, " TxCtrlWrd0: 0x%x\n", phydbg->TxCtrlWrd0);
	11166	+ bcm_bprintf(b, " TxCtrlWrd1: 0x%x\n", phydbg->TxCtrlWrd1);
	11167	+ bcm_bprintf(b, " TxCtrlWrd2: 0x%x\n", phydbg->TxCtrlWrd2);
	11168	+ bcm_bprintf(b, " TxLsig0: 0x%x\n", phydbg->TxLsig0);
	11169	+ bcm_bprintf(b, " TxLsig1: 0x%x\n", phydbg->TxLsig1);
	11170	+ bcm_bprintf(b, " TxVhtSigA10: 0x%x\n", phydbg->TxVhtSigA10);
	11171	+ bcm_bprintf(b, " TxVhtSigA11: 0x%x\n", phydbg->TxVhtSigA11);
	11172	+ bcm_bprintf(b, " TxVhtSigA20: 0x%x\n", phydbg->TxVhtSigA20);
	11173	+ bcm_bprintf(b, " TxVhtSigA21: 0x%x\n", phydbg->TxVhtSigA21);
	11174	+ bcm_bprintf(b, " txPktLength: 0x%x\n", phydbg->txPktLength);
	11175	+ bcm_bprintf(b, " txPsdulengthCtr: 0x%x\n", phydbg->txPsdulengthCtr);
	11176	+ bcm_bprintf(b, " gpioClkControl: 0x%x\n", phydbg->gpioClkControl);
	11177	+ bcm_bprintf(b, " gpioSel: 0x%x\n", phydbg->gpioSel);
	11178	+ bcm_bprintf(b, " pktprocdebug: 0x%x\n", phydbg->pktprocdebug);
	11179	+ for (i = 0; i < 3; i++)
	11180	+ bcm_bprintf(b, " gpioOut[%d]: 0x%x\n", i, phydbg->gpioOut[i]);
	11181	+ }
	11182	+
	11183	+ tlv = bcm_parse_tlvs(hdr->data, hdr->len, TAG_TRAP_PSM_WD);
	11184	+ if (tlv)
	11185	+ {
	11186	+ const hnd_ext_trap_psmwd_t* psmwd;
	11187	+ bcm_bprintf(b, "\n%s len: %d\n", etd_trap_name(TAG_TRAP_PSM_WD), tlv->len);
	11188	+ psmwd = (const hnd_ext_trap_psmwd_t *)tlv;
	11189	+ bcm_bprintf(b, " version: 0x%x\n", psmwd->version);
	11190	+ bcm_bprintf(b, " maccontrol: 0x%x\n", psmwd->i32_maccontrol);
	11191	+ bcm_bprintf(b, " maccommand: 0x%x\n", psmwd->i32_maccommand);
	11192	+ bcm_bprintf(b, " macintstatus: 0x%x\n", psmwd->i32_macintstatus);
	11193	+ bcm_bprintf(b, " phydebug: 0x%x\n", psmwd->i32_phydebug);
	11194	+ bcm_bprintf(b, " clk_ctl_st: 0x%x\n", psmwd->i32_clk_ctl_st);
	11195	+ for (i = 0; i < 3; i++)
	11196	+ bcm_bprintf(b, " psmdebug[%d]: 0x%x\n", i, psmwd->i32_psmdebug[i]);
	11197	+ bcm_bprintf(b, " gated clock en: 0x%x\n", psmwd->i16_0x1a8);
	11198	+ bcm_bprintf(b, " Rcv Fifo Ctrl: 0x%x\n", psmwd->i16_0x406);
	11199	+ bcm_bprintf(b, " Rx ctrl 1: 0x%x\n", psmwd->i16_0x408);
	11200	+ bcm_bprintf(b, " Rxe Status 1: 0x%x\n", psmwd->i16_0x41a);
	11201	+ bcm_bprintf(b, " Rxe Status 2: 0x%x\n", psmwd->i16_0x41c);
	11202	+ bcm_bprintf(b, " rcv wrd count 0: 0x%x\n", psmwd->i16_0x424);
	11203	+ bcm_bprintf(b, " rcv wrd count 1: 0x%x\n", psmwd->i16_0x426);
	11204	+ bcm_bprintf(b, " RCV_LFIFO_STS: 0x%x\n", psmwd->i16_0x456);
	11205	+ bcm_bprintf(b, " PSM_SLP_TMR: 0x%x\n", psmwd->i16_0x480);
	11206	+ bcm_bprintf(b, " PSM BRC: 0x%x\n", psmwd->i16_0x490);
	11207	+ bcm_bprintf(b, " TXE CTRL: 0x%x\n", psmwd->i16_0x500);
	11208	+ bcm_bprintf(b, " TXE Status: 0x%x\n", psmwd->i16_0x50e);
	11209	+ bcm_bprintf(b, " TXE_xmtdmabusy: 0x%x\n", psmwd->i16_0x55e);
	11210	+ bcm_bprintf(b, " TXE_XMTfifosuspflush: 0x%x\n", psmwd->i16_0x566);
	11211	+ bcm_bprintf(b, " IFS Stat: 0x%x\n", psmwd->i16_0x690);
	11212	+ bcm_bprintf(b, " IFS_MEDBUSY_CTR: 0x%x\n", psmwd->i16_0x692);
	11213	+ bcm_bprintf(b, " IFS_TX_DUR: 0x%x\n", psmwd->i16_0x694);
	11214	+ bcm_bprintf(b, " SLow_CTL: 0x%x\n", psmwd->i16_0x6a0);
	11215	+ bcm_bprintf(b, " TXE_AQM fifo Ready: 0x%x\n", psmwd->i16_0x838);
	11216	+ bcm_bprintf(b, " Dagg ctrl: 0x%x\n", psmwd->i16_0x8c0);
	11217	+ bcm_bprintf(b, " shm_prewds_cnt: 0x%x\n", psmwd->shm_prewds_cnt);
	11218	+ bcm_bprintf(b, " shm_txtplufl_cnt: 0x%x\n", psmwd->shm_txtplufl_cnt);
	11219	+ bcm_bprintf(b, " shm_txphyerr_cnt: 0x%x\n", psmwd->shm_txphyerr_cnt);
	11220	+ }
	11221	+
	11222	+ tlv = bcm_parse_tlvs(hdr->data, hdr->len, TAG_TRAP_MAC_SUSP);
	11223	+ if (tlv)
	11224	+ {
	11225	+ const hnd_ext_trap_macsusp_t* macsusp;
	11226	+ bcm_bprintf(b, "\n%s len: %d\n", etd_trap_name(TAG_TRAP_MAC_SUSP), tlv->len);
	11227	+ macsusp = (const hnd_ext_trap_macsusp_t *)tlv;
	11228	+ bcm_bprintf(b, " version: %d\n", macsusp->version);
	11229	+ bcm_bprintf(b, " trap_reason: %d\n", macsusp->trap_reason);
	11230	+ bcm_bprintf(b, " maccontrol: 0x%x\n", macsusp->i32_maccontrol);
	11231	+ bcm_bprintf(b, " maccommand: 0x%x\n", macsusp->i32_maccommand);
	11232	+ bcm_bprintf(b, " macintstatus: 0x%x\n", macsusp->i32_macintstatus);
	11233	+ for (i = 0; i < 4; i++)
	11234	+ bcm_bprintf(b, " phydebug[%d]: 0x%x\n", i, macsusp->i32_phydebug[i]);
	11235	+ for (i = 0; i < 8; i++)
	11236	+ bcm_bprintf(b, " psmdebug[%d]: 0x%x\n", i, macsusp->i32_psmdebug[i]);
	11237	+ bcm_bprintf(b, " Rxe Status_1: 0x%x\n", macsusp->i16_0x41a);
	11238	+ bcm_bprintf(b, " Rxe Status_2: 0x%x\n", macsusp->i16_0x41c);
	11239	+ bcm_bprintf(b, " PSM BRC: 0x%x\n", macsusp->i16_0x490);
	11240	+ bcm_bprintf(b, " TXE Status: 0x%x\n", macsusp->i16_0x50e);
	11241	+ bcm_bprintf(b, " TXE xmtdmabusy: 0x%x\n", macsusp->i16_0x55e);
	11242	+ bcm_bprintf(b, " TXE XMTfifosuspflush: 0x%x\n", macsusp->i16_0x566);
	11243	+ bcm_bprintf(b, " IFS Stat: 0x%x\n", macsusp->i16_0x690);
	11244	+ bcm_bprintf(b, " IFS MEDBUSY CTR: 0x%x\n", macsusp->i16_0x692);
	11245	+ bcm_bprintf(b, " IFS TX DUR: 0x%x\n", macsusp->i16_0x694);
	11246	+ bcm_bprintf(b, " WEP CTL: 0x%x\n", macsusp->i16_0x7c0);
	11247	+ bcm_bprintf(b, " TXE AQM fifo Ready: 0x%x\n", macsusp->i16_0x838);
	11248	+ bcm_bprintf(b, " MHP status: 0x%x\n", macsusp->i16_0x880);
	11249	+ bcm_bprintf(b, " shm_prewds_cnt: 0x%x\n", macsusp->shm_prewds_cnt);
	11250	+ bcm_bprintf(b, " shm_ucode_dbgst: 0x%x\n", macsusp->shm_ucode_dbgst);
	11251	+ }
	11252	+
	11253	+ tlv = bcm_parse_tlvs(hdr->data, hdr->len, TAG_TRAP_MAC_WAKE);
	11254	+ if (tlv)
	11255	+ {
	11256	+ const hnd_ext_trap_macenab_t* macwake;
	11257	+ bcm_bprintf(b, "\n%s len: %d\n", etd_trap_name(TAG_TRAP_MAC_WAKE), tlv->len);
	11258	+ macwake = (const hnd_ext_trap_macenab_t *)tlv;
	11259	+ bcm_bprintf(b, " version: 0x%x\n", macwake->version);
	11260	+ bcm_bprintf(b, " trap_reason: 0x%x\n", macwake->trap_reason);
	11261	+ bcm_bprintf(b, " maccontrol: 0x%x\n", macwake->i32_maccontrol);
	11262	+ bcm_bprintf(b, " maccommand: 0x%x\n", macwake->i32_maccommand);
	11263	+ bcm_bprintf(b, " macintstatus: 0x%x\n", macwake->i32_macintstatus);
	11264	+ for (i = 0; i < 8; i++)
	11265	+ bcm_bprintf(b, " psmdebug[%d]: 0x%x\n", i, macwake->i32_psmdebug[i]);
	11266	+ bcm_bprintf(b, " clk_ctl_st: 0x%x\n", macwake->i32_clk_ctl_st);
	11267	+ bcm_bprintf(b, " powerctl: 0x%x\n", macwake->i32_powerctl);
	11268	+ bcm_bprintf(b, " gated clock en: 0x%x\n", macwake->i16_0x1a8);
	11269	+ bcm_bprintf(b, " PSM_SLP_TMR: 0x%x\n", macwake->i16_0x480);
	11270	+ bcm_bprintf(b, " PSM BRC: 0x%x\n", macwake->i16_0x490);
	11271	+ bcm_bprintf(b, " TSF CTL: 0x%x\n", macwake->i16_0x600);
	11272	+ bcm_bprintf(b, " IFS Stat: 0x%x\n", macwake->i16_0x690);
	11273	+ bcm_bprintf(b, " IFS_MEDBUSY_CTR: 0x%x\n", macwake->i16_0x692);
	11274	+ bcm_bprintf(b, " Slow_CTL: 0x%x\n", macwake->i16_0x6a0);
	11275	+ bcm_bprintf(b, " Slow_FRAC: 0x%x\n", macwake->i16_0x6a6);
	11276	+ bcm_bprintf(b, " fast power up delay: 0x%x\n", macwake->i16_0x6a8);
	11277	+ bcm_bprintf(b, " Slow_PER: 0x%x\n", macwake->i16_0x6aa);
	11278	+ bcm_bprintf(b, " shm_ucode_dbgst: 0x%x\n", macwake->shm_ucode_dbgst);
	11279	+ }
	11280	+
	11281	+ tlv = bcm_parse_tlvs(hdr->data, hdr->len, TAG_TRAP_BUS);
	11282	+ if (tlv)
	11283	+ {
	11284	+ const bcm_dngl_pcie_hc_t* hc;
	11285	+ bcm_bprintf(b, "\n%s len: %d\n", etd_trap_name(TAG_TRAP_BUS), tlv->len);
	11286	+ hc = (const bcm_dngl_pcie_hc_t *)tlv->data;
	11287	+ bcm_bprintf(b, " version: 0x%x\n", hc->version);
	11288	+ bcm_bprintf(b, " reserved: 0x%x\n", hc->reserved);
	11289	+ bcm_bprintf(b, " pcie_err_ind_type: 0x%x\n", hc->pcie_err_ind_type);
	11290	+ bcm_bprintf(b, " pcie_flag: 0x%x\n", hc->pcie_flag);
	11291	+ bcm_bprintf(b, " pcie_control_reg: 0x%x\n", hc->pcie_control_reg);
	11292	+ for (i = 0; i < HC_PCIEDEV_CONFIG_REGLIST_MAX; i++)
	11293	+ bcm_bprintf(b, " pcie_config_regs[%d]: 0x%x\n", i, hc->pcie_config_regs[i]);
	11294	+ }
	11295	+
	11296	+ tlv = bcm_parse_tlvs(hdr->data, hdr->len, TAG_TRAP_HMAP);
	11297	+ if (tlv)
	11298	+ {
	11299	+ const pcie_hmapviolation_t* hmap;
	11300	+ hmap = (const pcie_hmapviolation_t *)tlv->data;
	11301	+ bcm_bprintf(b, "\n%s len: %d\n", etd_trap_name(TAG_TRAP_HMAP), tlv->len);
	11302	+ bcm_bprintf(b, " HMAP Vio Addr Low: 0x%x\n", hmap->hmap_violationaddr_lo);
	11303	+ bcm_bprintf(b, " HMAP Vio Addr Hi: 0x%x\n", hmap->hmap_violationaddr_hi);
	11304	+ bcm_bprintf(b, " HMAP Vio Info: 0x%x\n", hmap->hmap_violation_info);
	11305	+ }
	11306	+
	11307	+ return BCME_OK;
	11308	+}
	11309	+
	11310	+#ifdef BCMPCIE
	11311	+int
	11312	+dhd_prot_send_host_timestamp(dhd_pub_t dhdp, uchar tlvs, uint16 tlv_len,
	11313	+ uint16 seqnum, uint16 xt_id)
	11314	+{
	11315	+ dhd_prot_t *prot = dhdp->prot;
	11316	+ host_timestamp_msg_t *ts_req;
	11317	+ unsigned long flags;
	11318	+ uint16 alloced = 0;
	11319	+ uchar *ts_tlv_buf;
	11320	+ msgbuf_ring_t *ctrl_ring = &prot->h2dring_ctrl_subn;
	11321	+
	11322	+ if ((tlvs == NULL) \|\| (tlv_len == 0)) {
	11323	+ DHD_ERROR(("%s: argument error tlv: %p, tlv_len %d\n",
	11324	+ __FUNCTION__, tlvs, tlv_len));
	11325	+ return -1;
	11326	+ }
	11327	+
	11328	+ DHD_RING_LOCK(ctrl_ring->ring_lock, flags);
	11329	+
	11330	+ /* if Host TS req already pending go away */
	11331	+ if (prot->hostts_req_buf_inuse == TRUE) {
	11332	+ DHD_ERROR(("one host TS request already pending at device\n"));
	11333	+ DHD_RING_UNLOCK(ctrl_ring->ring_lock, flags);
	11334	+ return -1;
	11335	+ }
	11336	+
	11337	+ /* Request for cbuf space */
	11338	+ ts_req = (host_timestamp_msg_t*)dhd_prot_alloc_ring_space(dhdp, ctrl_ring,
	11339	+ DHD_FLOWRING_DEFAULT_NITEMS_POSTED_H2D, &alloced, FALSE);
	11340	+ if (ts_req == NULL) {
	11341	+ DHD_ERROR(("couldn't allocate space on msgring to send host TS request\n"));
	11342	+ DHD_RING_UNLOCK(ctrl_ring->ring_lock, flags);
	11343	+ return -1;
	11344	+ }
	11345	+
	11346	+ /* Common msg buf hdr */
	11347	+ ts_req->msg.msg_type = MSG_TYPE_HOSTTIMSTAMP;
	11348	+ ts_req->msg.if_id = 0;
	11349	+ ts_req->msg.flags = ctrl_ring->current_phase;
	11350	+ ts_req->msg.request_id = DHD_H2D_HOSTTS_REQ_PKTID;
	11351	+
	11352	+ ts_req->msg.epoch = ctrl_ring->seqnum % H2D_EPOCH_MODULO;
	11353	+ ctrl_ring->seqnum++;
	11354	+
	11355	+ ts_req->xt_id = xt_id;
	11356	+ ts_req->seqnum = seqnum;
	11357	+ /* populate TS req buffer info */
	11358	+ ts_req->input_data_len = htol16(tlv_len);
	11359	+ ts_req->host_buf_addr.high = htol32(PHYSADDRHI(prot->hostts_req_buf.pa));
	11360	+ ts_req->host_buf_addr.low = htol32(PHYSADDRLO(prot->hostts_req_buf.pa));
	11361	+ /* copy ioct payload */
	11362	+ ts_tlv_buf = (void *) prot->hostts_req_buf.va;
	11363	+ prot->hostts_req_buf_inuse = TRUE;
	11364	+ memcpy(ts_tlv_buf, tlvs, tlv_len);
	11365	+
	11366	+ OSL_CACHE_FLUSH((void *) prot->hostts_req_buf.va, tlv_len);
	11367	+
	11368	+ if (ISALIGNED(ts_tlv_buf, DMA_ALIGN_LEN) == FALSE) {
	11369	+ DHD_ERROR(("host TS req buffer address unaligned !!!!! \n"));
	11370	+ }
	11371	+
	11372	+ DHD_CTL(("submitted Host TS request request_id %d, data_len %d, tx_id %d, seq %d\n",
	11373	+ ts_req->msg.request_id, ts_req->input_data_len,
	11374	+ ts_req->xt_id, ts_req->seqnum));
	11375	+
	11376	+ /* upd wrt ptr and raise interrupt */
	11377	+ dhd_prot_ring_write_complete(dhdp, ctrl_ring, ts_req,
	11378	+ DHD_FLOWRING_DEFAULT_NITEMS_POSTED_H2D);
	11379	+
	11380	+ DHD_RING_UNLOCK(ctrl_ring->ring_lock, flags);
	11381	+
	11382	+ return 0;
	11383	+} /* dhd_prot_send_host_timestamp */
	11384	+
	11385	+bool
	11386	+dhd_prot_data_path_tx_timestamp_logging(dhd_pub_t *dhd, bool enable, bool set)
	11387	+{
	11388	+ if (set)
	11389	+ dhd->prot->tx_ts_log_enabled = enable;
	11390	+
	11391	+ return dhd->prot->tx_ts_log_enabled;
	11392	+}
	11393	+
	11394	+bool
	11395	+dhd_prot_data_path_rx_timestamp_logging(dhd_pub_t *dhd, bool enable, bool set)
	11396	+{
	11397	+ if (set)
	11398	+ dhd->prot->rx_ts_log_enabled = enable;
	11399	+
	11400	+ return dhd->prot->rx_ts_log_enabled;
	11401	+}
	11402	+
	11403	+bool
	11404	+dhd_prot_pkt_noretry(dhd_pub_t *dhd, bool enable, bool set)
	11405	+{
	11406	+ if (set)
	11407	+ dhd->prot->no_retry = enable;
	11408	+
	11409	+ return dhd->prot->no_retry;
	11410	+}
	11411	+
	11412	+bool
	11413	+dhd_prot_pkt_noaggr(dhd_pub_t *dhd, bool enable, bool set)
	11414	+{
	11415	+ if (set)
	11416	+ dhd->prot->no_aggr = enable;
	11417	+
	11418	+ return dhd->prot->no_aggr;
	11419	+}
	11420	+
	11421	+bool
	11422	+dhd_prot_pkt_fixed_rate(dhd_pub_t *dhd, bool enable, bool set)
	11423	+{
	11424	+ if (set)
	11425	+ dhd->prot->fixed_rate = enable;
	11426	+
	11427	+ return dhd->prot->fixed_rate;
	11428	+}
	11429	+#endif /* BCMPCIE */
	11430	+
	11431	+void
	11432	+dhd_prot_dma_indx_free(dhd_pub_t *dhd)
	11433	+{
	11434	+ dhd_prot_t *prot = dhd->prot;
	11435	+
	11436	+ dhd_dma_buf_free(dhd, &prot->h2d_dma_indx_wr_buf);
	11437	+ dhd_dma_buf_free(dhd, &prot->d2h_dma_indx_rd_buf);
	11438	+}
	11439	+
	11440	+void
	11441	+dhd_msgbuf_delay_post_ts_bufs(dhd_pub_t *dhd)
	11442	+{
	11443	+ if (dhd->prot->max_tsbufpost > 0)
	11444	+ dhd_msgbuf_rxbuf_post_ts_bufs(dhd);
	11445	+}
	11446	+
	11447	+static void BCMFASTPATH
	11448	+dhd_prot_process_fw_timestamp(dhd_pub_t dhd, void buf)
	11449	+{
	11450	+ DHD_ERROR(("Timesunc feature not compiled in but GOT FW TS message\n"));
	11451	+
	11452	+}
	11453	+
	11454	+uint16
	11455	+dhd_prot_get_ioctl_trans_id(dhd_pub_t *dhdp)
	11456	+{
	11457	+ return dhdp->prot->ioctl_trans_id;
	11458	+}
	11459	+
	11460	+int dhd_get_hscb_info(dhd_pub_t dhd, void * va, uint32 *len)
	11461	+{
	11462	+ if (!dhd->hscb_enable) {
	11463	+ if (len) {
	11464	+ /* prevent "Operation not supported" dhd message */
	11465	+ *len = 0;
	11466	+ return BCME_OK;
	11467	+ }
	11468	+ return BCME_UNSUPPORTED;
	11469	+ }
	11470	+
	11471	+ if (va) {
	11472	+ *va = dhd->prot->host_scb_buf.va;
	11473	+ }
	11474	+ if (len) {
	11475	+ *len = dhd->prot->host_scb_buf.len;
	11476	+ }
	11477	+
	11478	+ return BCME_OK;
	11479	+}
	11480	+
	11481	+#ifdef DHD_BUS_MEM_ACCESS
	11482	+int dhd_get_hscb_buff(dhd_pub_t dhd, uint32 offset, uint32 length, void buff)
	11483	+{
	11484	+ if (!dhd->hscb_enable) {
	11485	+ return BCME_UNSUPPORTED;
	11486	+ }
	11487	+
	11488	+ if (dhd->prot->host_scb_buf.va == NULL \|\|
	11489	+ ((uint64)offset + length > (uint64)dhd->prot->host_scb_buf.len)) {
	11490	+ return BCME_BADADDR;
	11491	+ }
	11492	+
	11493	+ memcpy(buff, (char*)dhd->prot->host_scb_buf.va + offset, length);
	11494	+
	11495	+ return BCME_OK;
	11496	+}
	11497	+#endif /* DHD_BUS_MEM_ACCESS */
	11498	+
	11499	+#ifdef DHD_HP2P
	11500	+uint32
	11501	+dhd_prot_pkt_threshold(dhd_pub_t *dhd, bool set, uint32 val)
	11502	+{
	11503	+ if (set)
	11504	+ dhd->pkt_thresh = (uint16)val;
	11505	+
	11506	+ val = dhd->pkt_thresh;
	11507	+
	11508	+ return val;
	11509	+}
	11510	+
	11511	+uint32
	11512	+dhd_prot_time_threshold(dhd_pub_t *dhd, bool set, uint32 val)
	11513	+{
	11514	+ if (set)
	11515	+ dhd->time_thresh = (uint16)val;
	11516	+
	11517	+ val = dhd->time_thresh;
	11518	+
	11519	+ return val;
	11520	+}
	11521	+
	11522	+uint32
	11523	+dhd_prot_pkt_expiry(dhd_pub_t *dhd, bool set, uint32 val)
	11524	+{
	11525	+ if (set)
	11526	+ dhd->pkt_expiry = (uint16)val;
	11527	+
	11528	+ val = dhd->pkt_expiry;
	11529	+
	11530	+ return val;
	11531	+}
	11532	+
	11533	+uint8
	11534	+dhd_prot_hp2p_enable(dhd_pub_t *dhd, bool set, int enable)
	11535	+{
	11536	+ uint8 ret = 0;
	11537	+ if (set) {
	11538	+ dhd->hp2p_enable = (enable & 0xf) ? TRUE : FALSE;
	11539	+ dhd->hp2p_infra_enable = ((enable >> 4) & 0xf) ? TRUE : FALSE;
	11540	+
	11541	+ if (enable) {
	11542	+ dhd_update_flow_prio_map(dhd, DHD_FLOW_PRIO_TID_MAP);
	11543	+ } else {
	11544	+ dhd_update_flow_prio_map(dhd, DHD_FLOW_PRIO_AC_MAP);
	11545	+ }
	11546	+ }
	11547	+ ret = dhd->hp2p_infra_enable ? 0x1:0x0;
	11548	+ ret <<= 4;
	11549	+ ret \|= dhd->hp2p_enable ? 0x1:0x0;
	11550	+
	11551	+ return ret;
	11552	+}
	11553	+
	11554	+static void
	11555	+dhd_update_hp2p_rxstats(dhd_pub_t dhd, host_rxbuf_cmpl_t rxstatus)
	11556	+{
	11557	+ ts_timestamp_t ts = (ts_timestamp_t )&rxstatus->ts;
	11558	+ hp2p_info_t *hp2p_info;
	11559	+ uint32 dur1;
	11560	+
	11561	+ hp2p_info = &dhd->hp2p_info[0];
	11562	+ dur1 = ((ts->high & 0x3FF) * HP2P_TIME_SCALE) / 100;
	11563	+
	11564	+ if (dur1 > (MAX_RX_HIST_BIN - 1)) {
	11565	+ dur1 = MAX_RX_HIST_BIN - 1;
	11566	+ DHD_ERROR(("%s: 0x%x 0x%x\n",
	11567	+ __FUNCTION__, ts->low, ts->high));
	11568	+ }
	11569	+
	11570	+ hp2p_info->rx_t0[dur1 % MAX_RX_HIST_BIN]++;
	11571	+ return;
	11572	+}
	11573	+
	11574	+static void
	11575	+dhd_update_hp2p_txstats(dhd_pub_t dhd, host_txbuf_cmpl_t txstatus)
	11576	+{
	11577	+ ts_timestamp_t ts = (ts_timestamp_t )&txstatus->ts;
	11578	+ uint16 flowid = txstatus->compl_hdr.flow_ring_id;
	11579	+ uint32 hp2p_flowid, dur1, dur2;
	11580	+ hp2p_info_t *hp2p_info;
	11581	+
	11582	+ hp2p_flowid = dhd->bus->max_submission_rings -
	11583	+ dhd->bus->max_cmn_rings - flowid + 1;
	11584	+ hp2p_info = &dhd->hp2p_info[hp2p_flowid];
	11585	+ ts = (ts_timestamp_t *)&(txstatus->ts);
	11586	+
	11587	+ dur1 = ((ts->high & 0x3FF) * HP2P_TIME_SCALE) / 1000;
	11588	+ if (dur1 > (MAX_TX_HIST_BIN - 1)) {
	11589	+ dur1 = MAX_TX_HIST_BIN - 1;
	11590	+ DHD_ERROR(("%s: 0x%x 0x%x\n", __FUNCTION__, ts->low, ts->high));
	11591	+ }
	11592	+ hp2p_info->tx_t0[dur1 % MAX_TX_HIST_BIN]++;
	11593	+
	11594	+ dur2 = (((ts->high >> 10) & 0x3FF) * HP2P_TIME_SCALE) / 1000;
	11595	+ if (dur2 > (MAX_TX_HIST_BIN - 1)) {
	11596	+ dur2 = MAX_TX_HIST_BIN - 1;
	11597	+ DHD_ERROR(("%s: 0x%x 0x%x\n", __FUNCTION__, ts->low, ts->high));
	11598	+ }
	11599	+
	11600	+ hp2p_info->tx_t1[dur2 % MAX_TX_HIST_BIN]++;
	11601	+ return;
	11602	+}
	11603	+
	11604	+enum hrtimer_restart dhd_hp2p_write(struct hrtimer *timer)
	11605	+{
	11606	+ hp2p_info_t *hp2p_info;
	11607	+ unsigned long flags;
	11608	+ dhd_pub_t *dhdp;
	11609	+
	11610	+#if defined(STRICT_GCC_WARNINGS) && defined(__GNUC__)
	11611	+#pragma GCC diagnostic push
	11612	+#pragma GCC diagnostic ignored "-Wcast-qual"
	11613	+#endif // endif
	11614	+#if LINUX_VERSION_CODE < KERNEL_VERSION(4, 15, 0)
	11615	+ hp2p_info = container_of(timer, hp2p_info_t, timer.timer);
	11616	+#else
	11617	+ hp2p_info = container_of(timer, hp2p_info_t, timer);
	11618	+#endif // endif
	11619	+#if defined(STRICT_GCC_WARNINGS) && defined(__GNUC__)
	11620	+#pragma GCC diagnostic pop
	11621	+#endif // endif
	11622	+ dhdp = hp2p_info->dhd_pub;
	11623	+ if (!dhdp) {
	11624	+ goto done;
	11625	+ }
	11626	+
	11627	+ DHD_INFO(("%s: pend_item = %d flowid = %d\n",
	11628	+ __FUNCTION__, ((msgbuf_ring_t *)hp2p_info->ring)->pend_items_count,
	11629	+ hp2p_info->flowid));
	11630	+
	11631	+ flags = dhd_os_hp2plock(dhdp);
	11632	+
	11633	+ dhd_prot_txdata_write_flush(dhdp, hp2p_info->flowid);
	11634	+ hp2p_info->hrtimer_init = FALSE;
	11635	+ hp2p_info->num_timer_limit++;
	11636	+
	11637	+ dhd_os_hp2punlock(dhdp, flags);
	11638	+done:
	11639	+ return HRTIMER_NORESTART;
	11640	+}
	11641	+
	11642	+static void
	11643	+dhd_calc_hp2p_burst(dhd_pub_t dhd, msgbuf_ring_t ring, uint16 flowid)
	11644	+{
	11645	+ hp2p_info_t *hp2p_info;
	11646	+ uint16 hp2p_flowid;
	11647	+
	11648	+ hp2p_flowid = dhd->bus->max_submission_rings -
	11649	+ dhd->bus->max_cmn_rings - flowid + 1;
	11650	+ hp2p_info = &dhd->hp2p_info[hp2p_flowid];
	11651	+
	11652	+ if (ring->pend_items_count == dhd->pkt_thresh) {
	11653	+ dhd_prot_txdata_write_flush(dhd, flowid);
	11654	+
	11655	+ hp2p_info->hrtimer_init = FALSE;
	11656	+ hp2p_info->ring = NULL;
	11657	+ hp2p_info->num_pkt_limit++;
	11658	+#if LINUX_VERSION_CODE <= KERNEL_VERSION(5, 1, 21)
	11659	+ tasklet_hrtimer_cancel(&hp2p_info->timer);
	11660	+#else
	11661	+ hrtimer_cancel(&hp2p_info->timer);
	11662	+#endif /* LINUX_VERSION_CODE <= KERNEL_VERSION(5, 1, 21) */
	11663	+ DHD_INFO(("%s: cancel hrtimer for flowid = %d \n"
	11664	+ "hp2p_flowid = %d pkt_thresh = %d\n",
	11665	+ __FUNCTION__, flowid, hp2p_flowid, dhd->pkt_thresh));
	11666	+ } else {
	11667	+ if (hp2p_info->hrtimer_init == FALSE) {
	11668	+ hp2p_info->hrtimer_init = TRUE;
	11669	+ hp2p_info->flowid = flowid;
	11670	+ hp2p_info->dhd_pub = dhd;
	11671	+ hp2p_info->ring = ring;
	11672	+ hp2p_info->num_timer_start++;
	11673	+#if LINUX_VERSION_CODE <= KERNEL_VERSION(5, 1, 21)
	11674	+ tasklet_hrtimer_start(&hp2p_info->timer,
	11675	+ ktime_set(0, dhd->time_thresh * 1000), HRTIMER_MODE_REL);
	11676	+#else
	11677	+ hrtimer_start(&hp2p_info->timer,
	11678	+ ktime_set(0, dhd->time_thresh * 1000), HRTIMER_MODE_REL_SOFT);
	11679	+#endif /* LINUX_VERSION_CODE <= KERNEL_VERSION(5, 1, 21) */
	11680	+
	11681	+ DHD_INFO(("%s: start hrtimer for flowid = %d hp2_flowid = %d\n",
	11682	+ __FUNCTION__, flowid, hp2p_flowid));
	11683	+ }
	11684	+ }
	11685	+ return;
	11686	+}
	11687	+
	11688	+static void
	11689	+dhd_update_hp2p_txdesc(dhd_pub_t dhd, host_txbuf_post_t txdesc)
	11690	+{
	11691	+ uint64 ts;
	11692	+
	11693	+ ts = local_clock();
	11694	+ do_div(ts, 1000);
	11695	+
	11696	+ txdesc->metadata_buf_len = 0;
	11697	+ txdesc->metadata_buf_addr.high_addr = htol32((ts >> 32) & 0xFFFFFFFF);
	11698	+ txdesc->metadata_buf_addr.low_addr = htol32(ts & 0xFFFFFFFF);
	11699	+ txdesc->exp_time = dhd->pkt_expiry;
	11700	+
	11701	+ DHD_INFO(("%s: metadata_high = 0x%x metadata_low = 0x%x exp_time = %x\n",
	11702	+ __FUNCTION__, txdesc->metadata_buf_addr.high_addr,
	11703	+ txdesc->metadata_buf_addr.low_addr,
	11704	+ txdesc->exp_time));
	11705	+
	11706	+ return;
	11707	+}
	11708	+#endif /* DHD_HP2P */
	11709	+
	11710	+#ifdef DHD_MAP_LOGGING
	11711	+void
	11712	+dhd_prot_smmu_fault_dump(dhd_pub_t *dhdp)
	11713	+{
	11714	+ dhd_prot_debug_info_print(dhdp);
	11715	+ OSL_DMA_MAP_DUMP(dhdp->osh);
	11716	+#ifdef DHD_MAP_PKTID_LOGGING
	11717	+ dhd_pktid_logging_dump(dhdp);
	11718	+#endif /* DHD_MAP_PKTID_LOGGING */
	11719	+#ifdef DHD_FW_COREDUMP
	11720	+ dhdp->memdump_type = DUMP_TYPE_SMMU_FAULT;
	11721	+#ifdef DNGL_AXI_ERROR_LOGGING
	11722	+ dhdp->memdump_enabled = DUMP_MEMFILE;
	11723	+ dhd_bus_get_mem_dump(dhdp);
	11724	+#else
	11725	+ dhdp->memdump_enabled = DUMP_MEMONLY;
	11726	+ dhd_bus_mem_dump(dhdp);
	11727	+#endif /* DNGL_AXI_ERROR_LOGGING */
	11728	+#endif /* DHD_FW_COREDUMP */
	11729	+}
	11730	+#endif /* DHD_MAP_LOGGING */