~hc/RK356X_SDK_RELEASE.git

..	..	@@ -8,7 +8,7 @@
8	8	* Copyright(c) 2008 - 2014 Intel Corporation. All rights reserved.
9	9	* Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
10	10	* Copyright(c) 2015 - 2017 Intel Deutschland GmbH
11		- * Copyright(c) 2018 Intel Corporation
	11	+ * Copyright(c) 2018 - 2020 Intel Corporation
12	12	*
13	13	* This program is free software; you can redistribute it and/or modify
14	14	* it under the terms of version 2 of the GNU General Public License as
..	..	@@ -18,9 +18,6 @@
18	18	* WITHOUT ANY WARRANTY; without even the implied warranty of
19	19	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20	20	* General Public License for more details.
21		- *
22		- * You should have received a copy of the GNU General Public License
23		- * along with this program;
24	21	*
25	22	* The full GNU General Public License is included in this distribution
26	23	* in the file called COPYING.
..	..	@@ -34,7 +31,7 @@
34	31	* Copyright(c) 2005 - 2014 Intel Corporation. All rights reserved.
35	32	* Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
36	33	* Copyright(c) 2015 - 2017 Intel Deutschland GmbH
37		- * Copyright(c) 2018 Intel Corporation
	34	+ * Copyright(c) 2018 - 2020 Intel Corporation
38	35	* All rights reserved.
39	36	*
40	37	* Redistribution and use in source and binary forms, with or without
..	..	@@ -228,8 +225,37 @@
228	225	dump_data = iwl_fw_error_next_data(dump_data);
229	226	}
230	227
231		-static void iwl_fw_dump_fifos(struct iwl_fw_runtime *fwrt,
232		- struct iwl_fw_error_dump_data **dump_data)
	228	+static void iwl_fw_dump_rxf(struct iwl_fw_runtime *fwrt,
	229	+ struct iwl_fw_error_dump_data **dump_data)
	230	+{
	231	+ struct iwl_fwrt_shared_mem_cfg *cfg = &fwrt->smem_cfg;
	232	+ unsigned long flags;
	233	+
	234	+ IWL_DEBUG_INFO(fwrt, "WRT RX FIFO dump\n");
	235	+
	236	+ if (!iwl_trans_grab_nic_access(fwrt->trans, &flags))
	237	+ return;
	238	+
	239	+ if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_RXF)) {
	240	+ /* Pull RXF1 */
	241	+ iwl_fwrt_dump_rxf(fwrt, dump_data,
	242	+ cfg->lmac[0].rxfifo1_size, 0, 0);
	243	+ /* Pull RXF2 */
	244	+ iwl_fwrt_dump_rxf(fwrt, dump_data, cfg->rxfifo2_size,
	245	+ RXF_DIFF_FROM_PREV +
	246	+ fwrt->trans->trans_cfg->umac_prph_offset, 1);
	247	+ /* Pull LMAC2 RXF1 */
	248	+ if (fwrt->smem_cfg.num_lmacs > 1)
	249	+ iwl_fwrt_dump_rxf(fwrt, dump_data,
	250	+ cfg->lmac[1].rxfifo1_size,
	251	+ LMAC2_PRPH_OFFSET, 2);
	252	+ }
	253	+
	254	+ iwl_trans_release_nic_access(fwrt->trans, &flags);
	255	+}
	256	+
	257	+static void iwl_fw_dump_txf(struct iwl_fw_runtime *fwrt,
	258	+ struct iwl_fw_error_dump_data **dump_data)
233	259	{
234	260	struct iwl_fw_error_dump_fifo *fifo_hdr;
235	261	struct iwl_fwrt_shared_mem_cfg *cfg = &fwrt->smem_cfg;
..	..	@@ -238,26 +264,12 @@
238	264	unsigned long flags;
239	265	int i, j;
240	266
241		- IWL_DEBUG_INFO(fwrt, "WRT FIFO dump\n");
	267	+ IWL_DEBUG_INFO(fwrt, "WRT TX FIFO dump\n");
242	268
243	269	if (!iwl_trans_grab_nic_access(fwrt->trans, &flags))
244	270	return;
245	271
246		- if (fwrt->fw->dbg_dump_mask & BIT(IWL_FW_ERROR_DUMP_RXF)) {
247		- /* Pull RXF1 */
248		- iwl_fwrt_dump_rxf(fwrt, dump_data,
249		- cfg->lmac[0].rxfifo1_size, 0, 0);
250		- /* Pull RXF2 */
251		- iwl_fwrt_dump_rxf(fwrt, dump_data, cfg->rxfifo2_size,
252		- RXF_DIFF_FROM_PREV, 1);
253		- /* Pull LMAC2 RXF1 */
254		- if (fwrt->smem_cfg.num_lmacs > 1)
255		- iwl_fwrt_dump_rxf(fwrt, dump_data,
256		- cfg->lmac[1].rxfifo1_size,
257		- LMAC2_PRPH_OFFSET, 2);
258		- }
259		-
260		- if (fwrt->fw->dbg_dump_mask & BIT(IWL_FW_ERROR_DUMP_TXF)) {
	272	+ if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_TXF)) {
261	273	/* Pull TXF data from LMAC1 */
262	274	for (i = 0; i < fwrt->smem_cfg.num_txfifo_entries; i++) {
263	275	/* Mark the number of TXF we're pulling now */
..	..	@@ -282,7 +294,7 @@
282	294	}
283	295	}
284	296
285		- if (fwrt->fw->dbg_dump_mask & BIT(IWL_FW_ERROR_DUMP_INTERNAL_TXF) &&
	297	+ if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_INTERNAL_TXF) &&
286	298	fw_has_capa(&fwrt->fw->ucode_capa,
287	299	IWL_UCODE_TLV_CAPA_EXTEND_SHARED_MEM_CFG)) {
288	300	/* Pull UMAC internal TXF data from all TXFs */
..	..	@@ -456,10 +468,107 @@
456	468	{ .start = 0x00a05400, .end = 0x00a056e8 },
457	469	{ .start = 0x00a08000, .end = 0x00a098bc },
458	470	{ .start = 0x00a02400, .end = 0x00a02758 },
	471	+ { .start = 0x00a04764, .end = 0x00a0476c },
	472	+ { .start = 0x00a04770, .end = 0x00a04774 },
	473	+ { .start = 0x00a04620, .end = 0x00a04624 },
459	474	};
460	475
461		-static void _iwl_read_prph_block(struct iwl_trans *trans, u32 start,
462		- u32 len_bytes, __le32 *data)
	476	+static const struct iwl_prph_range iwl_prph_dump_addr_22000[] = {
	477	+ { .start = 0x00a00000, .end = 0x00a00000 },
	478	+ { .start = 0x00a0000c, .end = 0x00a00024 },
	479	+ { .start = 0x00a0002c, .end = 0x00a00034 },
	480	+ { .start = 0x00a0003c, .end = 0x00a0003c },
	481	+ { .start = 0x00a00410, .end = 0x00a00418 },
	482	+ { .start = 0x00a00420, .end = 0x00a00420 },
	483	+ { .start = 0x00a00428, .end = 0x00a00428 },
	484	+ { .start = 0x00a00430, .end = 0x00a0043c },
	485	+ { .start = 0x00a00444, .end = 0x00a00444 },
	486	+ { .start = 0x00a00840, .end = 0x00a00840 },
	487	+ { .start = 0x00a00850, .end = 0x00a00858 },
	488	+ { .start = 0x00a01004, .end = 0x00a01008 },
	489	+ { .start = 0x00a01010, .end = 0x00a01010 },
	490	+ { .start = 0x00a01018, .end = 0x00a01018 },
	491	+ { .start = 0x00a01024, .end = 0x00a01024 },
	492	+ { .start = 0x00a0102c, .end = 0x00a01034 },
	493	+ { .start = 0x00a0103c, .end = 0x00a01040 },
	494	+ { .start = 0x00a01048, .end = 0x00a01050 },
	495	+ { .start = 0x00a01058, .end = 0x00a01058 },
	496	+ { .start = 0x00a01060, .end = 0x00a01070 },
	497	+ { .start = 0x00a0108c, .end = 0x00a0108c },
	498	+ { .start = 0x00a01c20, .end = 0x00a01c28 },
	499	+ { .start = 0x00a01d10, .end = 0x00a01d10 },
	500	+ { .start = 0x00a01e28, .end = 0x00a01e2c },
	501	+ { .start = 0x00a01e60, .end = 0x00a01e60 },
	502	+ { .start = 0x00a01e80, .end = 0x00a01e80 },
	503	+ { .start = 0x00a01ea0, .end = 0x00a01ea0 },
	504	+ { .start = 0x00a02000, .end = 0x00a0201c },
	505	+ { .start = 0x00a02024, .end = 0x00a02024 },
	506	+ { .start = 0x00a02040, .end = 0x00a02048 },
	507	+ { .start = 0x00a020c0, .end = 0x00a020e0 },
	508	+ { .start = 0x00a02400, .end = 0x00a02404 },
	509	+ { .start = 0x00a0240c, .end = 0x00a02414 },
	510	+ { .start = 0x00a0241c, .end = 0x00a0243c },
	511	+ { .start = 0x00a02448, .end = 0x00a024bc },
	512	+ { .start = 0x00a024c4, .end = 0x00a024cc },
	513	+ { .start = 0x00a02508, .end = 0x00a02508 },
	514	+ { .start = 0x00a02510, .end = 0x00a02514 },
	515	+ { .start = 0x00a0251c, .end = 0x00a0251c },
	516	+ { .start = 0x00a0252c, .end = 0x00a0255c },
	517	+ { .start = 0x00a02564, .end = 0x00a025a0 },
	518	+ { .start = 0x00a025a8, .end = 0x00a025b4 },
	519	+ { .start = 0x00a025c0, .end = 0x00a025c0 },
	520	+ { .start = 0x00a025e8, .end = 0x00a025f4 },
	521	+ { .start = 0x00a02c08, .end = 0x00a02c18 },
	522	+ { .start = 0x00a02c2c, .end = 0x00a02c38 },
	523	+ { .start = 0x00a02c68, .end = 0x00a02c78 },
	524	+ { .start = 0x00a03000, .end = 0x00a03000 },
	525	+ { .start = 0x00a03010, .end = 0x00a03014 },
	526	+ { .start = 0x00a0301c, .end = 0x00a0302c },
	527	+ { .start = 0x00a03034, .end = 0x00a03038 },
	528	+ { .start = 0x00a03040, .end = 0x00a03044 },
	529	+ { .start = 0x00a03060, .end = 0x00a03068 },
	530	+ { .start = 0x00a03070, .end = 0x00a03070 },
	531	+ { .start = 0x00a0307c, .end = 0x00a03084 },
	532	+ { .start = 0x00a0308c, .end = 0x00a03090 },
	533	+ { .start = 0x00a03098, .end = 0x00a03098 },
	534	+ { .start = 0x00a030a0, .end = 0x00a030a0 },
	535	+ { .start = 0x00a030a8, .end = 0x00a030b4 },
	536	+ { .start = 0x00a030bc, .end = 0x00a030c0 },
	537	+ { .start = 0x00a030c8, .end = 0x00a030f4 },
	538	+ { .start = 0x00a03100, .end = 0x00a0312c },
	539	+ { .start = 0x00a03c00, .end = 0x00a03c5c },
	540	+ { .start = 0x00a04400, .end = 0x00a04454 },
	541	+ { .start = 0x00a04460, .end = 0x00a04474 },
	542	+ { .start = 0x00a044c0, .end = 0x00a044ec },
	543	+ { .start = 0x00a04500, .end = 0x00a04504 },
	544	+ { .start = 0x00a04510, .end = 0x00a04538 },
	545	+ { .start = 0x00a04540, .end = 0x00a04548 },
	546	+ { .start = 0x00a04560, .end = 0x00a04560 },
	547	+ { .start = 0x00a04570, .end = 0x00a0457c },
	548	+ { .start = 0x00a04590, .end = 0x00a04590 },
	549	+ { .start = 0x00a04598, .end = 0x00a04598 },
	550	+ { .start = 0x00a045c0, .end = 0x00a045f4 },
	551	+ { .start = 0x00a05c18, .end = 0x00a05c1c },
	552	+ { .start = 0x00a0c000, .end = 0x00a0c018 },
	553	+ { .start = 0x00a0c020, .end = 0x00a0c028 },
	554	+ { .start = 0x00a0c038, .end = 0x00a0c094 },
	555	+ { .start = 0x00a0c0c0, .end = 0x00a0c104 },
	556	+ { .start = 0x00a0c10c, .end = 0x00a0c118 },
	557	+ { .start = 0x00a0c150, .end = 0x00a0c174 },
	558	+ { .start = 0x00a0c17c, .end = 0x00a0c188 },
	559	+ { .start = 0x00a0c190, .end = 0x00a0c198 },
	560	+ { .start = 0x00a0c1a0, .end = 0x00a0c1a8 },
	561	+ { .start = 0x00a0c1b0, .end = 0x00a0c1b8 },
	562	+};
	563	+
	564	+static const struct iwl_prph_range iwl_prph_dump_addr_ax210[] = {
	565	+ { .start = 0x00d03c00, .end = 0x00d03c64 },
	566	+ { .start = 0x00d05c18, .end = 0x00d05c1c },
	567	+ { .start = 0x00d0c000, .end = 0x00d0c174 },
	568	+};
	569	+
	570	+static void iwl_read_prph_block(struct iwl_trans *trans, u32 start,
	571	+ u32 len_bytes, __le32 *data)
463	572	{
464	573	u32 i;
465	574
..	..	@@ -467,29 +576,19 @@
467	576	*data++ = cpu_to_le32(iwl_read_prph_no_grab(trans, start + i));
468	577	}
469	578
470		-static bool iwl_read_prph_block(struct iwl_trans *trans, u32 start,
471		- u32 len_bytes, __le32 *data)
472		-{
473		- unsigned long flags;
474		- bool success = false;
475		-
476		- if (iwl_trans_grab_nic_access(trans, &flags)) {
477		- success = true;
478		- _iwl_read_prph_block(trans, start, len_bytes, data);
479		- iwl_trans_release_nic_access(trans, &flags);
480		- }
481		-
482		- return success;
483		-}
484		-
485		-static void iwl_dump_prph(struct iwl_trans *trans,
486		- struct iwl_fw_error_dump_data **data,
	579	+static void iwl_dump_prph(struct iwl_fw_runtime *fwrt,
487	580	const struct iwl_prph_range *iwl_prph_dump_addr,
488		- u32 range_len)
	581	+ u32 range_len, void *ptr)
489	582	{
490	583	struct iwl_fw_error_dump_prph *prph;
	584	+ struct iwl_trans *trans = fwrt->trans;
	585	+ struct iwl_fw_error_dump_data **data =
	586	+ (struct iwl_fw_error_dump_data **)ptr;
491	587	unsigned long flags;
492	588	u32 i;
	589	+
	590	+ if (!data)
	591	+ return;
493	592
494	593	IWL_DEBUG_INFO(trans, "WRT PRPH dump\n");
495	594
..	..	@@ -507,11 +606,11 @@
507	606	prph = (void )(data)->data;
508	607	prph->prph_start = cpu_to_le32(iwl_prph_dump_addr[i].start);
509	608
510		- _iwl_read_prph_block(trans, iwl_prph_dump_addr[i].start,
511		- /* our range is inclusive, hence + 4 */
512		- iwl_prph_dump_addr[i].end -
513		- iwl_prph_dump_addr[i].start + 4,
514		- (void *)prph->data);
	609	+ iwl_read_prph_block(trans, iwl_prph_dump_addr[i].start,
	610	+ /* our range is inclusive, hence + 4 */
	611	+ iwl_prph_dump_addr[i].end -
	612	+ iwl_prph_dump_addr[i].start + 4,
	613	+ (void *)prph->data);
515	614
516	615	data = iwl_fw_error_next_data(data);
517	616	}
..	..	@@ -557,46 +656,191 @@
557	656	return table;
558	657	}
559	658
560		-void iwl_fw_error_dump(struct iwl_fw_runtime *fwrt)
	659	+static void iwl_fw_get_prph_len(struct iwl_fw_runtime *fwrt,
	660	+ const struct iwl_prph_range *iwl_prph_dump_addr,
	661	+ u32 range_len, void *ptr)
	662	+{
	663	+ u32 prph_len = (u32 )ptr;
	664	+ int i, num_bytes_in_chunk;
	665	+
	666	+ if (!prph_len)
	667	+ return;
	668	+
	669	+ for (i = 0; i < range_len; i++) {
	670	+ /* The range includes both boundaries */
	671	+ num_bytes_in_chunk =
	672	+ iwl_prph_dump_addr[i].end -
	673	+ iwl_prph_dump_addr[i].start + 4;
	674	+
	675	+ *prph_len += sizeof(struct iwl_fw_error_dump_data) +
	676	+ sizeof(struct iwl_fw_error_dump_prph) +
	677	+ num_bytes_in_chunk;
	678	+ }
	679	+}
	680	+
	681	+static void iwl_fw_prph_handler(struct iwl_fw_runtime fwrt, void ptr,
	682	+ void (handler)(struct iwl_fw_runtime ,
	683	+ const struct iwl_prph_range *,
	684	+ u32, void *))
	685	+{
	686	+ u32 range_len;
	687	+
	688	+ if (fwrt->trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) {
	689	+ range_len = ARRAY_SIZE(iwl_prph_dump_addr_ax210);
	690	+ handler(fwrt, iwl_prph_dump_addr_ax210, range_len, ptr);
	691	+ } else if (fwrt->trans->trans_cfg->device_family >=
	692	+ IWL_DEVICE_FAMILY_22000) {
	693	+ range_len = ARRAY_SIZE(iwl_prph_dump_addr_22000);
	694	+ handler(fwrt, iwl_prph_dump_addr_22000, range_len, ptr);
	695	+ } else {
	696	+ range_len = ARRAY_SIZE(iwl_prph_dump_addr_comm);
	697	+ handler(fwrt, iwl_prph_dump_addr_comm, range_len, ptr);
	698	+
	699	+ if (fwrt->trans->trans_cfg->mq_rx_supported) {
	700	+ range_len = ARRAY_SIZE(iwl_prph_dump_addr_9000);
	701	+ handler(fwrt, iwl_prph_dump_addr_9000, range_len, ptr);
	702	+ }
	703	+ }
	704	+}
	705	+
	706	+static void iwl_fw_dump_mem(struct iwl_fw_runtime *fwrt,
	707	+ struct iwl_fw_error_dump_data **dump_data,
	708	+ u32 len, u32 ofs, u32 type)
	709	+{
	710	+ struct iwl_fw_error_dump_mem *dump_mem;
	711	+
	712	+ if (!len)
	713	+ return;
	714	+
	715	+ (*dump_data)->type = cpu_to_le32(IWL_FW_ERROR_DUMP_MEM);
	716	+ (dump_data)->len = cpu_to_le32(len + sizeof(dump_mem));
	717	+ dump_mem = (void )(dump_data)->data;
	718	+ dump_mem->type = cpu_to_le32(type);
	719	+ dump_mem->offset = cpu_to_le32(ofs);
	720	+ iwl_trans_read_mem_bytes(fwrt->trans, ofs, dump_mem->data, len);
	721	+ dump_data = iwl_fw_error_next_data(dump_data);
	722	+
	723	+ IWL_DEBUG_INFO(fwrt, "WRT memory dump. Type=%u\n", dump_mem->type);
	724	+}
	725	+
	726	+#define ADD_LEN(len, item_len, const_len) \
	727	+ do {size_t item = item_len; len += (!!item) * const_len + item; } \
	728	+ while (0)
	729	+
	730	+static int iwl_fw_rxf_len(struct iwl_fw_runtime *fwrt,
	731	+ struct iwl_fwrt_shared_mem_cfg *mem_cfg)
	732	+{
	733	+ size_t hdr_len = sizeof(struct iwl_fw_error_dump_data) +
	734	+ sizeof(struct iwl_fw_error_dump_fifo);
	735	+ u32 fifo_len = 0;
	736	+ int i;
	737	+
	738	+ if (!iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_RXF))
	739	+ return 0;
	740	+
	741	+ /* Count RXF2 size */
	742	+ ADD_LEN(fifo_len, mem_cfg->rxfifo2_size, hdr_len);
	743	+
	744	+ /* Count RXF1 sizes */
	745	+ if (WARN_ON(mem_cfg->num_lmacs > MAX_NUM_LMAC))
	746	+ mem_cfg->num_lmacs = MAX_NUM_LMAC;
	747	+
	748	+ for (i = 0; i < mem_cfg->num_lmacs; i++)
	749	+ ADD_LEN(fifo_len, mem_cfg->lmac[i].rxfifo1_size, hdr_len);
	750	+
	751	+ return fifo_len;
	752	+}
	753	+
	754	+static int iwl_fw_txf_len(struct iwl_fw_runtime *fwrt,
	755	+ struct iwl_fwrt_shared_mem_cfg *mem_cfg)
	756	+{
	757	+ size_t hdr_len = sizeof(struct iwl_fw_error_dump_data) +
	758	+ sizeof(struct iwl_fw_error_dump_fifo);
	759	+ u32 fifo_len = 0;
	760	+ int i;
	761	+
	762	+ if (!iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_TXF))
	763	+ goto dump_internal_txf;
	764	+
	765	+ /* Count TXF sizes */
	766	+ if (WARN_ON(mem_cfg->num_lmacs > MAX_NUM_LMAC))
	767	+ mem_cfg->num_lmacs = MAX_NUM_LMAC;
	768	+
	769	+ for (i = 0; i < mem_cfg->num_lmacs; i++) {
	770	+ int j;
	771	+
	772	+ for (j = 0; j < mem_cfg->num_txfifo_entries; j++)
	773	+ ADD_LEN(fifo_len, mem_cfg->lmac[i].txfifo_size[j],
	774	+ hdr_len);
	775	+ }
	776	+
	777	+dump_internal_txf:
	778	+ if (!(iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_INTERNAL_TXF) &&
	779	+ fw_has_capa(&fwrt->fw->ucode_capa,
	780	+ IWL_UCODE_TLV_CAPA_EXTEND_SHARED_MEM_CFG)))
	781	+ goto out;
	782	+
	783	+ for (i = 0; i < ARRAY_SIZE(mem_cfg->internal_txfifo_size); i++)
	784	+ ADD_LEN(fifo_len, mem_cfg->internal_txfifo_size[i], hdr_len);
	785	+
	786	+out:
	787	+ return fifo_len;
	788	+}
	789	+
	790	+static void iwl_dump_paging(struct iwl_fw_runtime *fwrt,
	791	+ struct iwl_fw_error_dump_data **data)
	792	+{
	793	+ int i;
	794	+
	795	+ IWL_DEBUG_INFO(fwrt, "WRT paging dump\n");
	796	+ for (i = 1; i < fwrt->num_of_paging_blk + 1; i++) {
	797	+ struct iwl_fw_error_dump_paging *paging;
	798	+ struct page *pages =
	799	+ fwrt->fw_paging_db[i].fw_paging_block;
	800	+ dma_addr_t addr = fwrt->fw_paging_db[i].fw_paging_phys;
	801	+
	802	+ (*data)->type = cpu_to_le32(IWL_FW_ERROR_DUMP_PAGING);
	803	+ (data)->len = cpu_to_le32(sizeof(paging) +
	804	+ PAGING_BLOCK_SIZE);
	805	+ paging = (void )(data)->data;
	806	+ paging->index = cpu_to_le32(i);
	807	+ dma_sync_single_for_cpu(fwrt->trans->dev, addr,
	808	+ PAGING_BLOCK_SIZE,
	809	+ DMA_BIDIRECTIONAL);
	810	+ memcpy(paging->data, page_address(pages),
	811	+ PAGING_BLOCK_SIZE);
	812	+ dma_sync_single_for_device(fwrt->trans->dev, addr,
	813	+ PAGING_BLOCK_SIZE,
	814	+ DMA_BIDIRECTIONAL);
	815	+ (data) = iwl_fw_error_next_data(data);
	816	+ }
	817	+}
	818	+
	819	+static struct iwl_fw_error_dump_file *
	820	+iwl_fw_error_dump_file(struct iwl_fw_runtime *fwrt,
	821	+ struct iwl_fw_dump_ptrs *fw_error_dump,
	822	+ struct iwl_fwrt_dump_data *data)
561	823	{
562	824	struct iwl_fw_error_dump_file *dump_file;
563	825	struct iwl_fw_error_dump_data *dump_data;
564	826	struct iwl_fw_error_dump_info *dump_info;
565		- struct iwl_fw_error_dump_mem *dump_mem;
566	827	struct iwl_fw_error_dump_smem_cfg *dump_smem_cfg;
567	828	struct iwl_fw_error_dump_trigger_desc *dump_trig;
568		- struct iwl_fw_dump_ptrs *fw_error_dump;
569		- struct scatterlist *sg_dump_data;
570	829	u32 sram_len, sram_ofs;
571		- const struct iwl_fw_dbg_mem_seg_tlv *fw_dbg_mem = fwrt->fw->dbg_mem_tlv;
	830	+ const struct iwl_fw_dbg_mem_seg_tlv *fw_mem = fwrt->fw->dbg.mem_tlv;
572	831	struct iwl_fwrt_shared_mem_cfg *mem_cfg = &fwrt->smem_cfg;
573		- u32 file_len, fifo_data_len = 0, prph_len = 0, radio_len = 0;
574		- u32 smem_len = fwrt->fw->n_dbg_mem_tlv ? 0 : fwrt->trans->cfg->smem_len;
575		- u32 sram2_len = fwrt->fw->n_dbg_mem_tlv ?
	832	+ u32 file_len, fifo_len = 0, prph_len = 0, radio_len = 0;
	833	+ u32 smem_len = fwrt->fw->dbg.n_mem_tlv ? 0 : fwrt->trans->cfg->smem_len;
	834	+ u32 sram2_len = fwrt->fw->dbg.n_mem_tlv ?
576	835	0 : fwrt->trans->cfg->dccm2_len;
577		- bool monitor_dump_only = false;
578	836	int i;
579		-
580		- IWL_DEBUG_INFO(fwrt, "WRT dump start\n");
581		-
582		- /* there's no point in fw dump if the bus is dead */
583		- if (test_bit(STATUS_TRANS_DEAD, &fwrt->trans->status)) {
584		- IWL_ERR(fwrt, "Skip fw error dump since bus is dead\n");
585		- goto out;
586		- }
587		-
588		- if (fwrt->dump.trig &&
589		- fwrt->dump.trig->mode & IWL_FW_DBG_TRIGGER_MONITOR_ONLY)
590		- monitor_dump_only = true;
591		-
592		- fw_error_dump = kzalloc(sizeof(*fw_error_dump), GFP_KERNEL);
593		- if (!fw_error_dump)
594		- goto out;
595	837
596	838	/* SRAM - include stack CCM if driver knows the values for it */
597	839	if (!fwrt->trans->cfg->dccm_offset \|\| !fwrt->trans->cfg->dccm_len) {
598	840	const struct fw_img *img;
599	841
	842	+ if (fwrt->cur_fw_img >= IWL_UCODE_TYPE_MAX)
	843	+ return NULL;
600	844	img = &fwrt->fw->img[fwrt->cur_fw_img];
601	845	sram_ofs = img->sec[IWL_UCODE_SECTION_DATA].offset;
602	846	sram_len = img->sec[IWL_UCODE_SECTION_DATA].len;
..	..	@@ -607,194 +851,101 @@
607	851
608	852	/* reading RXF/TXF sizes */
609	853	if (test_bit(STATUS_FW_ERROR, &fwrt->trans->status)) {
610		- fifo_data_len = 0;
611		-
612		- if (fwrt->fw->dbg_dump_mask & BIT(IWL_FW_ERROR_DUMP_RXF)) {
613		-
614		- /* Count RXF2 size */
615		- if (mem_cfg->rxfifo2_size) {
616		- /* Add header info */
617		- fifo_data_len +=
618		- mem_cfg->rxfifo2_size +
619		- sizeof(*dump_data) +
620		- sizeof(struct iwl_fw_error_dump_fifo);
621		- }
622		-
623		- /* Count RXF1 sizes */
624		- for (i = 0; i < mem_cfg->num_lmacs; i++) {
625		- if (!mem_cfg->lmac[i].rxfifo1_size)
626		- continue;
627		-
628		- /* Add header info */
629		- fifo_data_len +=
630		- mem_cfg->lmac[i].rxfifo1_size +
631		- sizeof(*dump_data) +
632		- sizeof(struct iwl_fw_error_dump_fifo);
633		- }
634		- }
635		-
636		- if (fwrt->fw->dbg_dump_mask & BIT(IWL_FW_ERROR_DUMP_TXF)) {
637		- size_t fifo_const_len = sizeof(*dump_data) +
638		- sizeof(struct iwl_fw_error_dump_fifo);
639		-
640		- /* Count TXF sizes */
641		- for (i = 0; i < mem_cfg->num_lmacs; i++) {
642		- int j;
643		-
644		- for (j = 0; j < mem_cfg->num_txfifo_entries;
645		- j++) {
646		- if (!mem_cfg->lmac[i].txfifo_size[j])
647		- continue;
648		-
649		- /* Add header info */
650		- fifo_data_len +=
651		- fifo_const_len +
652		- mem_cfg->lmac[i].txfifo_size[j];
653		- }
654		- }
655		- }
656		-
657		- if ((fwrt->fw->dbg_dump_mask &
658		- BIT(IWL_FW_ERROR_DUMP_INTERNAL_TXF)) &&
659		- fw_has_capa(&fwrt->fw->ucode_capa,
660		- IWL_UCODE_TLV_CAPA_EXTEND_SHARED_MEM_CFG)) {
661		- for (i = 0;
662		- i < ARRAY_SIZE(mem_cfg->internal_txfifo_size);
663		- i++) {
664		- if (!mem_cfg->internal_txfifo_size[i])
665		- continue;
666		-
667		- /* Add header info */
668		- fifo_data_len +=
669		- mem_cfg->internal_txfifo_size[i] +
670		- sizeof(*dump_data) +
671		- sizeof(struct iwl_fw_error_dump_fifo);
672		- }
673		- }
	854	+ fifo_len = iwl_fw_rxf_len(fwrt, mem_cfg);
	855	+ fifo_len += iwl_fw_txf_len(fwrt, mem_cfg);
674	856
675	857	/* Make room for PRPH registers */
676		- if (!fwrt->trans->cfg->gen2 &&
677		- fwrt->fw->dbg_dump_mask & BIT(IWL_FW_ERROR_DUMP_PRPH)) {
678		- for (i = 0; i < ARRAY_SIZE(iwl_prph_dump_addr_comm);
679		- i++) {
680		- /* The range includes both boundaries */
681		- int num_bytes_in_chunk =
682		- iwl_prph_dump_addr_comm[i].end -
683		- iwl_prph_dump_addr_comm[i].start + 4;
	858	+ if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_PRPH))
	859	+ iwl_fw_prph_handler(fwrt, &prph_len,
	860	+ iwl_fw_get_prph_len);
684	861
685		- prph_len += sizeof(*dump_data) +
686		- sizeof(struct iwl_fw_error_dump_prph) +
687		- num_bytes_in_chunk;
688		- }
689		- }
690		-
691		- if (!fwrt->trans->cfg->gen2 &&
692		- fwrt->trans->cfg->mq_rx_supported &&
693		- fwrt->fw->dbg_dump_mask & BIT(IWL_FW_ERROR_DUMP_PRPH)) {
694		- for (i = 0; i <
695		- ARRAY_SIZE(iwl_prph_dump_addr_9000); i++) {
696		- /* The range includes both boundaries */
697		- int num_bytes_in_chunk =
698		- iwl_prph_dump_addr_9000[i].end -
699		- iwl_prph_dump_addr_9000[i].start + 4;
700		-
701		- prph_len += sizeof(*dump_data) +
702		- sizeof(struct iwl_fw_error_dump_prph) +
703		- num_bytes_in_chunk;
704		- }
705		- }
706		-
707		- if (fwrt->trans->cfg->device_family == IWL_DEVICE_FAMILY_7000 &&
708		- fwrt->fw->dbg_dump_mask & BIT(IWL_FW_ERROR_DUMP_RADIO_REG))
	862	+ if (fwrt->trans->trans_cfg->device_family ==
	863	+ IWL_DEVICE_FAMILY_7000 &&
	864	+ iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_RADIO_REG))
709	865	radio_len = sizeof(*dump_data) + RADIO_REG_MAX_READ;
710	866	}
711	867
712		- file_len = sizeof(*dump_file) +
713		- fifo_data_len +
714		- prph_len +
715		- radio_len;
	868	+ file_len = sizeof(*dump_file) + fifo_len + prph_len + radio_len;
716	869
717		- if (fwrt->fw->dbg_dump_mask & BIT(IWL_FW_ERROR_DUMP_DEV_FW_INFO))
	870	+ if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_DEV_FW_INFO))
718	871	file_len += sizeof(dump_data) + sizeof(dump_info);
719		- if (fwrt->fw->dbg_dump_mask & BIT(IWL_FW_ERROR_DUMP_MEM_CFG))
	872	+ if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_MEM_CFG))
720	873	file_len += sizeof(dump_data) + sizeof(dump_smem_cfg);
721	874
722		- if (fwrt->fw->dbg_dump_mask & BIT(IWL_FW_ERROR_DUMP_MEM)) {
723		- /* Make room for the SMEM, if it exists */
724		- if (smem_len)
725		- file_len += sizeof(dump_data) + sizeof(dump_mem) +
726		- smem_len;
	875	+ if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_MEM)) {
	876	+ size_t hdr_len = sizeof(*dump_data) +
	877	+ sizeof(struct iwl_fw_error_dump_mem);
727	878
728		- /* Make room for the secondary SRAM, if it exists */
729		- if (sram2_len)
730		- file_len += sizeof(dump_data) + sizeof(dump_mem) +
731		- sram2_len;
	879	+ /* Dump SRAM only if no mem_tlvs */
	880	+ if (!fwrt->fw->dbg.n_mem_tlv)
	881	+ ADD_LEN(file_len, sram_len, hdr_len);
732	882
733		- /* Make room for MEM segments */
734		- for (i = 0; i < fwrt->fw->n_dbg_mem_tlv; i++) {
735		- file_len += sizeof(dump_data) + sizeof(dump_mem) +
736		- le32_to_cpu(fw_dbg_mem[i].len);
737		- }
	883	+ /* Make room for all mem types that exist */
	884	+ ADD_LEN(file_len, smem_len, hdr_len);
	885	+ ADD_LEN(file_len, sram2_len, hdr_len);
	886	+
	887	+ for (i = 0; i < fwrt->fw->dbg.n_mem_tlv; i++)
	888	+ ADD_LEN(file_len, le32_to_cpu(fw_mem[i].len), hdr_len);
738	889	}
739	890
740	891	/* Make room for fw's virtual image pages, if it exists */
741		- if (fwrt->fw->dbg_dump_mask & BIT(IWL_FW_ERROR_DUMP_PAGING) &&
742		- !fwrt->trans->cfg->gen2 &&
743		- fwrt->fw->img[fwrt->cur_fw_img].paging_mem_size &&
744		- fwrt->fw_paging_db[0].fw_paging_block)
	892	+ if (iwl_fw_dbg_is_paging_enabled(fwrt))
745	893	file_len += fwrt->num_of_paging_blk *
746	894	(sizeof(*dump_data) +
747	895	sizeof(struct iwl_fw_error_dump_paging) +
748	896	PAGING_BLOCK_SIZE);
749	897
	898	+ if (iwl_fw_dbg_is_d3_debug_enabled(fwrt) && fwrt->dump.d3_debug_data) {
	899	+ file_len += sizeof(*dump_data) +
	900	+ fwrt->trans->cfg->d3_debug_data_length * 2;
	901	+ }
	902	+
750	903	/* If we only want a monitor dump, reset the file length */
751		- if (monitor_dump_only) {
	904	+ if (data->monitor_only) {
752	905	file_len = sizeof(dump_file) + sizeof(dump_data) * 2 +
753	906	sizeof(dump_info) + sizeof(dump_smem_cfg);
754	907	}
755	908
756		- if (fwrt->fw->dbg_dump_mask & BIT(IWL_FW_ERROR_DUMP_ERROR_INFO) &&
757		- fwrt->dump.desc)
	909	+ if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_ERROR_INFO) &&
	910	+ data->desc)
758	911	file_len += sizeof(dump_data) + sizeof(dump_trig) +
759		- fwrt->dump.desc->len;
760		-
761		- if (fwrt->fw->dbg_dump_mask & BIT(IWL_FW_ERROR_DUMP_MEM) &&
762		- !fwrt->fw->n_dbg_mem_tlv)
763		- file_len += sizeof(dump_data) + sram_len + sizeof(dump_mem);
	912	+ data->desc->len;
764	913
765	914	dump_file = vzalloc(file_len);
766		- if (!dump_file) {
767		- kfree(fw_error_dump);
768		- goto out;
769		- }
	915	+ if (!dump_file)
	916	+ return NULL;
770	917
771	918	fw_error_dump->fwrt_ptr = dump_file;
772	919
773	920	dump_file->barker = cpu_to_le32(IWL_FW_ERROR_DUMP_BARKER);
774	921	dump_data = (void *)dump_file->data;
775	922
776		- if (fwrt->fw->dbg_dump_mask & BIT(IWL_FW_ERROR_DUMP_DEV_FW_INFO)) {
	923	+ if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_DEV_FW_INFO)) {
777	924	dump_data->type = cpu_to_le32(IWL_FW_ERROR_DUMP_DEV_FW_INFO);
778	925	dump_data->len = cpu_to_le32(sizeof(*dump_info));
779	926	dump_info = (void *)dump_data->data;
780		- dump_info->device_family =
781		- fwrt->trans->cfg->device_family ==
782		- IWL_DEVICE_FAMILY_7000 ?
783		- cpu_to_le32(IWL_FW_ERROR_DUMP_FAMILY_7) :
784		- cpu_to_le32(IWL_FW_ERROR_DUMP_FAMILY_8);
	927	+ dump_info->hw_type =
	928	+ cpu_to_le32(CSR_HW_REV_TYPE(fwrt->trans->hw_rev));
785	929	dump_info->hw_step =
786	930	cpu_to_le32(CSR_HW_REV_STEP(fwrt->trans->hw_rev));
787	931	memcpy(dump_info->fw_human_readable, fwrt->fw->human_readable,
788	932	sizeof(dump_info->fw_human_readable));
789		- strncpy(dump_info->dev_human_readable, fwrt->trans->cfg->name,
	933	+ strncpy(dump_info->dev_human_readable, fwrt->trans->name,
790	934	sizeof(dump_info->dev_human_readable) - 1);
791	935	strncpy(dump_info->bus_human_readable, fwrt->dev->bus->name,
792	936	sizeof(dump_info->bus_human_readable) - 1);
	937	+ dump_info->num_of_lmacs = fwrt->smem_cfg.num_lmacs;
	938	+ dump_info->lmac_err_id[0] =
	939	+ cpu_to_le32(fwrt->dump.lmac_err_id[0]);
	940	+ if (fwrt->smem_cfg.num_lmacs > 1)
	941	+ dump_info->lmac_err_id[1] =
	942	+ cpu_to_le32(fwrt->dump.lmac_err_id[1]);
	943	+ dump_info->umac_err_id = cpu_to_le32(fwrt->dump.umac_err_id);
793	944
794	945	dump_data = iwl_fw_error_next_data(dump_data);
795	946	}
796	947
797		- if (fwrt->fw->dbg_dump_mask & BIT(IWL_FW_ERROR_DUMP_MEM_CFG)) {
	948	+ if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_MEM_CFG)) {
798	949	/* Dump shared memory configuration */
799	950	dump_data->type = cpu_to_le32(IWL_FW_ERROR_DUMP_MEM_CFG);
800	951	dump_data->len = cpu_to_le32(sizeof(*dump_smem_cfg));
..	..	@@ -825,177 +976,1407 @@
825	976	}
826	977
827	978	/* We only dump the FIFOs if the FW is in error state */
828		- if (fifo_data_len) {
829		- iwl_fw_dump_fifos(fwrt, &dump_data);
830		- if (radio_len)
831		- iwl_read_radio_regs(fwrt, &dump_data);
	979	+ if (fifo_len) {
	980	+ iwl_fw_dump_rxf(fwrt, &dump_data);
	981	+ iwl_fw_dump_txf(fwrt, &dump_data);
832	982	}
833	983
834		- if (fwrt->fw->dbg_dump_mask & BIT(IWL_FW_ERROR_DUMP_ERROR_INFO) &&
835		- fwrt->dump.desc) {
	984	+ if (radio_len)
	985	+ iwl_read_radio_regs(fwrt, &dump_data);
	986	+
	987	+ if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_ERROR_INFO) &&
	988	+ data->desc) {
836	989	dump_data->type = cpu_to_le32(IWL_FW_ERROR_DUMP_ERROR_INFO);
837	990	dump_data->len = cpu_to_le32(sizeof(*dump_trig) +
838		- fwrt->dump.desc->len);
	991	+ data->desc->len);
839	992	dump_trig = (void *)dump_data->data;
840		- memcpy(dump_trig, &fwrt->dump.desc->trig_desc,
841		- sizeof(*dump_trig) + fwrt->dump.desc->len);
	993	+ memcpy(dump_trig, &data->desc->trig_desc,
	994	+ sizeof(*dump_trig) + data->desc->len);
842	995
843	996	dump_data = iwl_fw_error_next_data(dump_data);
844	997	}
845	998
846	999	/* In case we only want monitor dump, skip to dump trasport data */
847		- if (monitor_dump_only)
848		- goto dump_trans_data;
	1000	+ if (data->monitor_only)
	1001	+ goto out;
849	1002
850		- if (!fwrt->fw->n_dbg_mem_tlv &&
851		- fwrt->fw->dbg_dump_mask & BIT(IWL_FW_ERROR_DUMP_MEM)) {
852		- dump_data->type = cpu_to_le32(IWL_FW_ERROR_DUMP_MEM);
853		- dump_data->len = cpu_to_le32(sram_len + sizeof(*dump_mem));
854		- dump_mem = (void *)dump_data->data;
855		- dump_mem->type = cpu_to_le32(IWL_FW_ERROR_DUMP_MEM_SRAM);
856		- dump_mem->offset = cpu_to_le32(sram_ofs);
857		- iwl_trans_read_mem_bytes(fwrt->trans, sram_ofs, dump_mem->data,
858		- sram_len);
859		- dump_data = iwl_fw_error_next_data(dump_data);
860		- }
	1003	+ if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_MEM)) {
	1004	+ const struct iwl_fw_dbg_mem_seg_tlv *fw_dbg_mem =
	1005	+ fwrt->fw->dbg.mem_tlv;
861	1006
862		- for (i = 0; i < fwrt->fw->n_dbg_mem_tlv; i++) {
863		- u32 len = le32_to_cpu(fw_dbg_mem[i].len);
864		- u32 ofs = le32_to_cpu(fw_dbg_mem[i].ofs);
865		- bool success;
	1007	+ if (!fwrt->fw->dbg.n_mem_tlv)
	1008	+ iwl_fw_dump_mem(fwrt, &dump_data, sram_len, sram_ofs,
	1009	+ IWL_FW_ERROR_DUMP_MEM_SRAM);
866	1010
867		- if (!(fwrt->fw->dbg_dump_mask & BIT(IWL_FW_ERROR_DUMP_MEM)))
868		- break;
	1011	+ for (i = 0; i < fwrt->fw->dbg.n_mem_tlv; i++) {
	1012	+ u32 len = le32_to_cpu(fw_dbg_mem[i].len);
	1013	+ u32 ofs = le32_to_cpu(fw_dbg_mem[i].ofs);
869	1014
870		- dump_data->type = cpu_to_le32(IWL_FW_ERROR_DUMP_MEM);
871		- dump_data->len = cpu_to_le32(len + sizeof(*dump_mem));
872		- dump_mem = (void *)dump_data->data;
873		- dump_mem->type = fw_dbg_mem[i].data_type;
874		- dump_mem->offset = cpu_to_le32(ofs);
875		-
876		- IWL_DEBUG_INFO(fwrt, "WRT memory dump. Type=%u\n",
877		- dump_mem->type);
878		-
879		- switch (dump_mem->type & cpu_to_le32(FW_DBG_MEM_TYPE_MASK)) {
880		- case cpu_to_le32(FW_DBG_MEM_TYPE_REGULAR):
881		- iwl_trans_read_mem_bytes(fwrt->trans, ofs,
882		- dump_mem->data,
883		- len);
884		- success = true;
885		- break;
886		- case cpu_to_le32(FW_DBG_MEM_TYPE_PRPH):
887		- success = iwl_read_prph_block(fwrt->trans, ofs, len,
888		- (void *)dump_mem->data);
889		- break;
890		- default:
891		- /*
892		- * shouldn't get here, we ignored this kind
893		- * of TLV earlier during the TLV parsing?!
894		- */
895		- WARN_ON(1);
896		- success = false;
	1015	+ iwl_fw_dump_mem(fwrt, &dump_data, len, ofs,
	1016	+ le32_to_cpu(fw_dbg_mem[i].data_type));
897	1017	}
898	1018
899		- if (success)
900		- dump_data = iwl_fw_error_next_data(dump_data);
	1019	+ iwl_fw_dump_mem(fwrt, &dump_data, smem_len,
	1020	+ fwrt->trans->cfg->smem_offset,
	1021	+ IWL_FW_ERROR_DUMP_MEM_SMEM);
	1022	+
	1023	+ iwl_fw_dump_mem(fwrt, &dump_data, sram2_len,
	1024	+ fwrt->trans->cfg->dccm2_offset,
	1025	+ IWL_FW_ERROR_DUMP_MEM_SRAM);
901	1026	}
902	1027
903		- if (smem_len && fwrt->fw->dbg_dump_mask & BIT(IWL_FW_ERROR_DUMP_MEM)) {
904		- IWL_DEBUG_INFO(fwrt, "WRT SMEM dump\n");
905		- dump_data->type = cpu_to_le32(IWL_FW_ERROR_DUMP_MEM);
906		- dump_data->len = cpu_to_le32(smem_len + sizeof(*dump_mem));
907		- dump_mem = (void *)dump_data->data;
908		- dump_mem->type = cpu_to_le32(IWL_FW_ERROR_DUMP_MEM_SMEM);
909		- dump_mem->offset = cpu_to_le32(fwrt->trans->cfg->smem_offset);
910		- iwl_trans_read_mem_bytes(fwrt->trans,
911		- fwrt->trans->cfg->smem_offset,
912		- dump_mem->data, smem_len);
913		- dump_data = iwl_fw_error_next_data(dump_data);
914		- }
	1028	+ if (iwl_fw_dbg_is_d3_debug_enabled(fwrt) && fwrt->dump.d3_debug_data) {
	1029	+ u32 addr = fwrt->trans->cfg->d3_debug_data_base_addr;
	1030	+ size_t data_size = fwrt->trans->cfg->d3_debug_data_length;
915	1031
916		- if (sram2_len && fwrt->fw->dbg_dump_mask & BIT(IWL_FW_ERROR_DUMP_MEM)) {
917		- IWL_DEBUG_INFO(fwrt, "WRT SRAM dump\n");
918		- dump_data->type = cpu_to_le32(IWL_FW_ERROR_DUMP_MEM);
919		- dump_data->len = cpu_to_le32(sram2_len + sizeof(*dump_mem));
920		- dump_mem = (void *)dump_data->data;
921		- dump_mem->type = cpu_to_le32(IWL_FW_ERROR_DUMP_MEM_SRAM);
922		- dump_mem->offset = cpu_to_le32(fwrt->trans->cfg->dccm2_offset);
923		- iwl_trans_read_mem_bytes(fwrt->trans,
924		- fwrt->trans->cfg->dccm2_offset,
925		- dump_mem->data, sram2_len);
	1032	+ dump_data->type = cpu_to_le32(IWL_FW_ERROR_DUMP_D3_DEBUG_DATA);
	1033	+ dump_data->len = cpu_to_le32(data_size * 2);
	1034	+
	1035	+ memcpy(dump_data->data, fwrt->dump.d3_debug_data, data_size);
	1036	+
	1037	+ kfree(fwrt->dump.d3_debug_data);
	1038	+ fwrt->dump.d3_debug_data = NULL;
	1039	+
	1040	+ iwl_trans_read_mem_bytes(fwrt->trans, addr,
	1041	+ dump_data->data + data_size,
	1042	+ data_size);
	1043	+
926	1044	dump_data = iwl_fw_error_next_data(dump_data);
927	1045	}
928	1046
929	1047	/* Dump fw's virtual image */
930		- if (fwrt->fw->dbg_dump_mask & BIT(IWL_FW_ERROR_DUMP_PAGING) &&
931		- !fwrt->trans->cfg->gen2 &&
932		- fwrt->fw->img[fwrt->cur_fw_img].paging_mem_size &&
933		- fwrt->fw_paging_db[0].fw_paging_block) {
934		- IWL_DEBUG_INFO(fwrt, "WRT paging dump\n");
935		- for (i = 1; i < fwrt->num_of_paging_blk + 1; i++) {
936		- struct iwl_fw_error_dump_paging *paging;
937		- struct page *pages =
938		- fwrt->fw_paging_db[i].fw_paging_block;
939		- dma_addr_t addr = fwrt->fw_paging_db[i].fw_paging_phys;
	1048	+ if (iwl_fw_dbg_is_paging_enabled(fwrt))
	1049	+ iwl_dump_paging(fwrt, &dump_data);
940	1050
941		- dump_data->type = cpu_to_le32(IWL_FW_ERROR_DUMP_PAGING);
942		- dump_data->len = cpu_to_le32(sizeof(*paging) +
943		- PAGING_BLOCK_SIZE);
944		- paging = (void *)dump_data->data;
945		- paging->index = cpu_to_le32(i);
946		- dma_sync_single_for_cpu(fwrt->trans->dev, addr,
947		- PAGING_BLOCK_SIZE,
948		- DMA_BIDIRECTIONAL);
949		- memcpy(paging->data, page_address(pages),
950		- PAGING_BLOCK_SIZE);
951		- dump_data = iwl_fw_error_next_data(dump_data);
	1051	+ if (prph_len)
	1052	+ iwl_fw_prph_handler(fwrt, &dump_data, iwl_dump_prph);
	1053	+
	1054	+out:
	1055	+ dump_file->file_len = cpu_to_le32(file_len);
	1056	+ return dump_file;
	1057	+}
	1058	+
	1059	+/**
	1060	+ * struct iwl_dump_ini_region_data - region data
	1061	+ * @reg_tlv: region TLV
	1062	+ * @dump_data: dump data
	1063	+ */
	1064	+struct iwl_dump_ini_region_data {
	1065	+ struct iwl_ucode_tlv *reg_tlv;
	1066	+ struct iwl_fwrt_dump_data *dump_data;
	1067	+};
	1068	+
	1069	+static int iwl_dump_ini_prph_iter(struct iwl_fw_runtime *fwrt,
	1070	+ struct iwl_dump_ini_region_data *reg_data,
	1071	+ void *range_ptr, int idx)
	1072	+{
	1073	+ struct iwl_fw_ini_region_tlv reg = (void )reg_data->reg_tlv->data;
	1074	+ struct iwl_fw_ini_error_dump_range *range = range_ptr;
	1075	+ __le32 *val = range->data;
	1076	+ u32 prph_val;
	1077	+ u32 addr = le32_to_cpu(reg->addrs[idx]) +
	1078	+ le32_to_cpu(reg->dev_addr.offset);
	1079	+ int i;
	1080	+
	1081	+ range->internal_base_addr = cpu_to_le32(addr);
	1082	+ range->range_data_size = reg->dev_addr.size;
	1083	+ for (i = 0; i < le32_to_cpu(reg->dev_addr.size); i += 4) {
	1084	+ prph_val = iwl_read_prph(fwrt->trans, addr + i);
	1085	+ if (prph_val == 0x5a5a5a5a)
	1086	+ return -EBUSY;
	1087	+ *val++ = cpu_to_le32(prph_val);
	1088	+ }
	1089	+
	1090	+ return sizeof(*range) + le32_to_cpu(range->range_data_size);
	1091	+}
	1092	+
	1093	+static int iwl_dump_ini_csr_iter(struct iwl_fw_runtime *fwrt,
	1094	+ struct iwl_dump_ini_region_data *reg_data,
	1095	+ void *range_ptr, int idx)
	1096	+{
	1097	+ struct iwl_fw_ini_region_tlv reg = (void )reg_data->reg_tlv->data;
	1098	+ struct iwl_fw_ini_error_dump_range *range = range_ptr;
	1099	+ __le32 *val = range->data;
	1100	+ u32 addr = le32_to_cpu(reg->addrs[idx]) +
	1101	+ le32_to_cpu(reg->dev_addr.offset);
	1102	+ int i;
	1103	+
	1104	+ range->internal_base_addr = cpu_to_le32(addr);
	1105	+ range->range_data_size = reg->dev_addr.size;
	1106	+ for (i = 0; i < le32_to_cpu(reg->dev_addr.size); i += 4)
	1107	+ *val++ = cpu_to_le32(iwl_trans_read32(fwrt->trans, addr + i));
	1108	+
	1109	+ return sizeof(*range) + le32_to_cpu(range->range_data_size);
	1110	+}
	1111	+
	1112	+static int iwl_dump_ini_config_iter(struct iwl_fw_runtime *fwrt,
	1113	+ struct iwl_dump_ini_region_data *reg_data,
	1114	+ void *range_ptr, int idx)
	1115	+{
	1116	+ struct iwl_trans *trans = fwrt->trans;
	1117	+ struct iwl_fw_ini_region_tlv reg = (void )reg_data->reg_tlv->data;
	1118	+ struct iwl_fw_ini_error_dump_range *range = range_ptr;
	1119	+ __le32 *val = range->data;
	1120	+ u32 addr = le32_to_cpu(reg->addrs[idx]) +
	1121	+ le32_to_cpu(reg->dev_addr.offset);
	1122	+ int i;
	1123	+
	1124	+ /* we shouldn't get here if the trans doesn't have read_config32 */
	1125	+ if (WARN_ON_ONCE(!trans->ops->read_config32))
	1126	+ return -EOPNOTSUPP;
	1127	+
	1128	+ range->internal_base_addr = cpu_to_le32(addr);
	1129	+ range->range_data_size = reg->dev_addr.size;
	1130	+ for (i = 0; i < le32_to_cpu(reg->dev_addr.size); i += 4) {
	1131	+ int ret;
	1132	+ u32 tmp;
	1133	+
	1134	+ ret = trans->ops->read_config32(trans, addr + i, &tmp);
	1135	+ if (ret < 0)
	1136	+ return ret;
	1137	+
	1138	+ *val++ = cpu_to_le32(tmp);
	1139	+ }
	1140	+
	1141	+ return sizeof(*range) + le32_to_cpu(range->range_data_size);
	1142	+}
	1143	+
	1144	+static int iwl_dump_ini_dev_mem_iter(struct iwl_fw_runtime *fwrt,
	1145	+ struct iwl_dump_ini_region_data *reg_data,
	1146	+ void *range_ptr, int idx)
	1147	+{
	1148	+ struct iwl_fw_ini_region_tlv reg = (void )reg_data->reg_tlv->data;
	1149	+ struct iwl_fw_ini_error_dump_range *range = range_ptr;
	1150	+ u32 addr = le32_to_cpu(reg->addrs[idx]) +
	1151	+ le32_to_cpu(reg->dev_addr.offset);
	1152	+
	1153	+ range->internal_base_addr = cpu_to_le32(addr);
	1154	+ range->range_data_size = reg->dev_addr.size;
	1155	+ iwl_trans_read_mem_bytes(fwrt->trans, addr, range->data,
	1156	+ le32_to_cpu(reg->dev_addr.size));
	1157	+
	1158	+ return sizeof(*range) + le32_to_cpu(range->range_data_size);
	1159	+}
	1160	+
	1161	+static int _iwl_dump_ini_paging_iter(struct iwl_fw_runtime *fwrt,
	1162	+ void *range_ptr, int idx)
	1163	+{
	1164	+ /* increase idx by 1 since the pages are from 1 to
	1165	+ * fwrt->num_of_paging_blk + 1
	1166	+ */
	1167	+ struct page *page = fwrt->fw_paging_db[++idx].fw_paging_block;
	1168	+ struct iwl_fw_ini_error_dump_range *range = range_ptr;
	1169	+ dma_addr_t addr = fwrt->fw_paging_db[idx].fw_paging_phys;
	1170	+ u32 page_size = fwrt->fw_paging_db[idx].fw_paging_size;
	1171	+
	1172	+ range->page_num = cpu_to_le32(idx);
	1173	+ range->range_data_size = cpu_to_le32(page_size);
	1174	+ dma_sync_single_for_cpu(fwrt->trans->dev, addr, page_size,
	1175	+ DMA_BIDIRECTIONAL);
	1176	+ memcpy(range->data, page_address(page), page_size);
	1177	+ dma_sync_single_for_device(fwrt->trans->dev, addr, page_size,
	1178	+ DMA_BIDIRECTIONAL);
	1179	+
	1180	+ return sizeof(*range) + le32_to_cpu(range->range_data_size);
	1181	+}
	1182	+
	1183	+static int iwl_dump_ini_paging_iter(struct iwl_fw_runtime *fwrt,
	1184	+ struct iwl_dump_ini_region_data *reg_data,
	1185	+ void *range_ptr, int idx)
	1186	+{
	1187	+ struct iwl_fw_ini_error_dump_range *range;
	1188	+ u32 page_size;
	1189	+
	1190	+ if (!fwrt->trans->trans_cfg->gen2)
	1191	+ return _iwl_dump_ini_paging_iter(fwrt, range_ptr, idx);
	1192	+
	1193	+ range = range_ptr;
	1194	+ page_size = fwrt->trans->init_dram.paging[idx].size;
	1195	+
	1196	+ range->page_num = cpu_to_le32(idx);
	1197	+ range->range_data_size = cpu_to_le32(page_size);
	1198	+ memcpy(range->data, fwrt->trans->init_dram.paging[idx].block,
	1199	+ page_size);
	1200	+
	1201	+ return sizeof(*range) + le32_to_cpu(range->range_data_size);
	1202	+}
	1203	+
	1204	+static int
	1205	+iwl_dump_ini_mon_dram_iter(struct iwl_fw_runtime *fwrt,
	1206	+ struct iwl_dump_ini_region_data *reg_data,
	1207	+ void *range_ptr, int idx)
	1208	+{
	1209	+ struct iwl_fw_ini_region_tlv reg = (void )reg_data->reg_tlv->data;
	1210	+ struct iwl_fw_ini_error_dump_range *range = range_ptr;
	1211	+ struct iwl_dram_data *frag;
	1212	+ u32 alloc_id = le32_to_cpu(reg->dram_alloc_id);
	1213	+
	1214	+ frag = &fwrt->trans->dbg.fw_mon_ini[alloc_id].frags[idx];
	1215	+
	1216	+ range->dram_base_addr = cpu_to_le64(frag->physical);
	1217	+ range->range_data_size = cpu_to_le32(frag->size);
	1218	+
	1219	+ memcpy(range->data, frag->block, frag->size);
	1220	+
	1221	+ return sizeof(*range) + le32_to_cpu(range->range_data_size);
	1222	+}
	1223	+
	1224	+static int iwl_dump_ini_mon_smem_iter(struct iwl_fw_runtime *fwrt,
	1225	+ struct iwl_dump_ini_region_data *reg_data,
	1226	+ void *range_ptr, int idx)
	1227	+{
	1228	+ struct iwl_fw_ini_region_tlv reg = (void )reg_data->reg_tlv->data;
	1229	+ struct iwl_fw_ini_error_dump_range *range = range_ptr;
	1230	+ u32 addr = le32_to_cpu(reg->internal_buffer.base_addr);
	1231	+
	1232	+ range->internal_base_addr = cpu_to_le32(addr);
	1233	+ range->range_data_size = reg->internal_buffer.size;
	1234	+ iwl_trans_read_mem_bytes(fwrt->trans, addr, range->data,
	1235	+ le32_to_cpu(reg->internal_buffer.size));
	1236	+
	1237	+ return sizeof(*range) + le32_to_cpu(range->range_data_size);
	1238	+}
	1239	+
	1240	+static bool iwl_ini_txf_iter(struct iwl_fw_runtime *fwrt,
	1241	+ struct iwl_dump_ini_region_data *reg_data, int idx)
	1242	+{
	1243	+ struct iwl_fw_ini_region_tlv reg = (void )reg_data->reg_tlv->data;
	1244	+ struct iwl_txf_iter_data *iter = &fwrt->dump.txf_iter_data;
	1245	+ struct iwl_fwrt_shared_mem_cfg *cfg = &fwrt->smem_cfg;
	1246	+ int txf_num = cfg->num_txfifo_entries;
	1247	+ int int_txf_num = ARRAY_SIZE(cfg->internal_txfifo_size);
	1248	+ u32 lmac_bitmap = le32_to_cpu(reg->fifos.fid[0]);
	1249	+
	1250	+ if (!idx) {
	1251	+ if (le32_to_cpu(reg->fifos.offset) && cfg->num_lmacs == 1) {
	1252	+ IWL_ERR(fwrt, "WRT: Invalid lmac offset 0x%x\n",
	1253	+ le32_to_cpu(reg->fifos.offset));
	1254	+ return false;
	1255	+ }
	1256	+
	1257	+ iter->internal_txf = 0;
	1258	+ iter->fifo_size = 0;
	1259	+ iter->fifo = -1;
	1260	+ if (le32_to_cpu(reg->fifos.offset))
	1261	+ iter->lmac = 1;
	1262	+ else
	1263	+ iter->lmac = 0;
	1264	+ }
	1265	+
	1266	+ if (!iter->internal_txf) {
	1267	+ for (iter->fifo++; iter->fifo < txf_num; iter->fifo++) {
	1268	+ iter->fifo_size =
	1269	+ cfg->lmac[iter->lmac].txfifo_size[iter->fifo];
	1270	+ if (iter->fifo_size && (lmac_bitmap & BIT(iter->fifo)))
	1271	+ return true;
	1272	+ }
	1273	+ iter->fifo--;
	1274	+ }
	1275	+
	1276	+ iter->internal_txf = 1;
	1277	+
	1278	+ if (!fw_has_capa(&fwrt->fw->ucode_capa,
	1279	+ IWL_UCODE_TLV_CAPA_EXTEND_SHARED_MEM_CFG))
	1280	+ return false;
	1281	+
	1282	+ for (iter->fifo++; iter->fifo < int_txf_num + txf_num; iter->fifo++) {
	1283	+ iter->fifo_size =
	1284	+ cfg->internal_txfifo_size[iter->fifo - txf_num];
	1285	+ if (iter->fifo_size && (lmac_bitmap & BIT(iter->fifo)))
	1286	+ return true;
	1287	+ }
	1288	+
	1289	+ return false;
	1290	+}
	1291	+
	1292	+static int iwl_dump_ini_txf_iter(struct iwl_fw_runtime *fwrt,
	1293	+ struct iwl_dump_ini_region_data *reg_data,
	1294	+ void *range_ptr, int idx)
	1295	+{
	1296	+ struct iwl_fw_ini_region_tlv reg = (void )reg_data->reg_tlv->data;
	1297	+ struct iwl_fw_ini_error_dump_range *range = range_ptr;
	1298	+ struct iwl_txf_iter_data *iter = &fwrt->dump.txf_iter_data;
	1299	+ struct iwl_fw_ini_error_dump_register reg_dump = (void )range->data;
	1300	+ u32 offs = le32_to_cpu(reg->fifos.offset), addr;
	1301	+ u32 registers_num = iwl_tlv_array_len(reg_data->reg_tlv, reg, addrs);
	1302	+ u32 registers_size = registers_num * sizeof(*reg_dump);
	1303	+ __le32 *data;
	1304	+ unsigned long flags;
	1305	+ int i;
	1306	+
	1307	+ if (!iwl_ini_txf_iter(fwrt, reg_data, idx))
	1308	+ return -EIO;
	1309	+
	1310	+ if (!iwl_trans_grab_nic_access(fwrt->trans, &flags))
	1311	+ return -EBUSY;
	1312	+
	1313	+ range->fifo_hdr.fifo_num = cpu_to_le32(iter->fifo);
	1314	+ range->fifo_hdr.num_of_registers = cpu_to_le32(registers_num);
	1315	+ range->range_data_size = cpu_to_le32(iter->fifo_size + registers_size);
	1316	+
	1317	+ iwl_write_prph_no_grab(fwrt->trans, TXF_LARC_NUM + offs, iter->fifo);
	1318	+
	1319	+ /*
	1320	+ * read txf registers. for each register, write to the dump the
	1321	+ * register address and its value
	1322	+ */
	1323	+ for (i = 0; i < registers_num; i++) {
	1324	+ addr = le32_to_cpu(reg->addrs[i]) + offs;
	1325	+
	1326	+ reg_dump->addr = cpu_to_le32(addr);
	1327	+ reg_dump->data = cpu_to_le32(iwl_read_prph_no_grab(fwrt->trans,
	1328	+ addr));
	1329	+
	1330	+ reg_dump++;
	1331	+ }
	1332	+
	1333	+ if (reg->fifos.hdr_only) {
	1334	+ range->range_data_size = cpu_to_le32(registers_size);
	1335	+ goto out;
	1336	+ }
	1337	+
	1338	+ /* Set the TXF_READ_MODIFY_ADDR to TXF_WR_PTR */
	1339	+ iwl_write_prph_no_grab(fwrt->trans, TXF_READ_MODIFY_ADDR + offs,
	1340	+ TXF_WR_PTR + offs);
	1341	+
	1342	+ /* Dummy-read to advance the read pointer to the head */
	1343	+ iwl_read_prph_no_grab(fwrt->trans, TXF_READ_MODIFY_DATA + offs);
	1344	+
	1345	+ /* Read FIFO */
	1346	+ addr = TXF_READ_MODIFY_DATA + offs;
	1347	+ data = (void *)reg_dump;
	1348	+ for (i = 0; i < iter->fifo_size; i += sizeof(*data))
	1349	+ *data++ = cpu_to_le32(iwl_read_prph_no_grab(fwrt->trans, addr));
	1350	+
	1351	+out:
	1352	+ iwl_trans_release_nic_access(fwrt->trans, &flags);
	1353	+
	1354	+ return sizeof(*range) + le32_to_cpu(range->range_data_size);
	1355	+}
	1356	+
	1357	+struct iwl_ini_rxf_data {
	1358	+ u32 fifo_num;
	1359	+ u32 size;
	1360	+ u32 offset;
	1361	+};
	1362	+
	1363	+static void iwl_ini_get_rxf_data(struct iwl_fw_runtime *fwrt,
	1364	+ struct iwl_dump_ini_region_data *reg_data,
	1365	+ struct iwl_ini_rxf_data *data)
	1366	+{
	1367	+ struct iwl_fw_ini_region_tlv reg = (void )reg_data->reg_tlv->data;
	1368	+ u32 fid1 = le32_to_cpu(reg->fifos.fid[0]);
	1369	+ u32 fid2 = le32_to_cpu(reg->fifos.fid[1]);
	1370	+ u8 fifo_idx;
	1371	+
	1372	+ if (!data)
	1373	+ return;
	1374	+
	1375	+ memset(data, 0, sizeof(*data));
	1376	+
	1377	+ /* make sure only one bit is set in only one fid */
	1378	+ if (WARN_ONCE(hweight_long(fid1) + hweight_long(fid2) != 1,
	1379	+ "fid1=%x, fid2=%x\n", fid1, fid2))
	1380	+ return;
	1381	+
	1382	+ if (fid1) {
	1383	+ fifo_idx = ffs(fid1) - 1;
	1384	+ if (WARN_ONCE(fifo_idx >= MAX_NUM_LMAC, "fifo_idx=%d\n",
	1385	+ fifo_idx))
	1386	+ return;
	1387	+
	1388	+ data->size = fwrt->smem_cfg.lmac[fifo_idx].rxfifo1_size;
	1389	+ data->fifo_num = fifo_idx;
	1390	+ } else {
	1391	+ u8 max_idx;
	1392	+
	1393	+ fifo_idx = ffs(fid2) - 1;
	1394	+ if (iwl_fw_lookup_notif_ver(fwrt->fw, SYSTEM_GROUP,
	1395	+ SHARED_MEM_CFG_CMD, 0) <= 3)
	1396	+ max_idx = 0;
	1397	+ else
	1398	+ max_idx = 1;
	1399	+
	1400	+ if (WARN_ONCE(fifo_idx > max_idx,
	1401	+ "invalid umac fifo idx %d", fifo_idx))
	1402	+ return;
	1403	+
	1404	+ /* use bit 31 to distinguish between umac and lmac rxf while
	1405	+ * parsing the dump
	1406	+ */
	1407	+ data->fifo_num = fifo_idx \| IWL_RXF_UMAC_BIT;
	1408	+
	1409	+ switch (fifo_idx) {
	1410	+ case 0:
	1411	+ data->size = fwrt->smem_cfg.rxfifo2_size;
	1412	+ data->offset = iwl_umac_prph(fwrt->trans,
	1413	+ RXF_DIFF_FROM_PREV);
	1414	+ break;
	1415	+ case 1:
	1416	+ data->size = fwrt->smem_cfg.rxfifo2_control_size;
	1417	+ data->offset = iwl_umac_prph(fwrt->trans,
	1418	+ RXF2C_DIFF_FROM_PREV);
	1419	+ break;
952	1420	}
953	1421	}
	1422	+}
954	1423
955		- if (prph_len) {
956		- iwl_dump_prph(fwrt->trans, &dump_data,
957		- iwl_prph_dump_addr_comm,
958		- ARRAY_SIZE(iwl_prph_dump_addr_comm));
	1424	+static int iwl_dump_ini_rxf_iter(struct iwl_fw_runtime *fwrt,
	1425	+ struct iwl_dump_ini_region_data *reg_data,
	1426	+ void *range_ptr, int idx)
	1427	+{
	1428	+ struct iwl_fw_ini_region_tlv reg = (void )reg_data->reg_tlv->data;
	1429	+ struct iwl_fw_ini_error_dump_range *range = range_ptr;
	1430	+ struct iwl_ini_rxf_data rxf_data;
	1431	+ struct iwl_fw_ini_error_dump_register reg_dump = (void )range->data;
	1432	+ u32 offs = le32_to_cpu(reg->fifos.offset), addr;
	1433	+ u32 registers_num = iwl_tlv_array_len(reg_data->reg_tlv, reg, addrs);
	1434	+ u32 registers_size = registers_num * sizeof(*reg_dump);
	1435	+ __le32 *data;
	1436	+ unsigned long flags;
	1437	+ int i;
959	1438
960		- if (fwrt->trans->cfg->mq_rx_supported)
961		- iwl_dump_prph(fwrt->trans, &dump_data,
962		- iwl_prph_dump_addr_9000,
963		- ARRAY_SIZE(iwl_prph_dump_addr_9000));
	1439	+ iwl_ini_get_rxf_data(fwrt, reg_data, &rxf_data);
	1440	+ if (!rxf_data.size)
	1441	+ return -EIO;
	1442	+
	1443	+ if (!iwl_trans_grab_nic_access(fwrt->trans, &flags))
	1444	+ return -EBUSY;
	1445	+
	1446	+ range->fifo_hdr.fifo_num = cpu_to_le32(rxf_data.fifo_num);
	1447	+ range->fifo_hdr.num_of_registers = cpu_to_le32(registers_num);
	1448	+ range->range_data_size = cpu_to_le32(rxf_data.size + registers_size);
	1449	+
	1450	+ /*
	1451	+ * read rxf registers. for each register, write to the dump the
	1452	+ * register address and its value
	1453	+ */
	1454	+ for (i = 0; i < registers_num; i++) {
	1455	+ addr = le32_to_cpu(reg->addrs[i]) + offs;
	1456	+
	1457	+ reg_dump->addr = cpu_to_le32(addr);
	1458	+ reg_dump->data = cpu_to_le32(iwl_read_prph_no_grab(fwrt->trans,
	1459	+ addr));
	1460	+
	1461	+ reg_dump++;
964	1462	}
965	1463
966		-dump_trans_data:
967		- fw_error_dump->trans_ptr = iwl_trans_dump_data(fwrt->trans,
968		- fwrt->dump.trig);
969		- fw_error_dump->fwrt_len = file_len;
970		- if (fw_error_dump->trans_ptr)
971		- file_len += fw_error_dump->trans_ptr->len;
972		- dump_file->file_len = cpu_to_le32(file_len);
	1464	+ if (reg->fifos.hdr_only) {
	1465	+ range->range_data_size = cpu_to_le32(registers_size);
	1466	+ goto out;
	1467	+ }
	1468	+
	1469	+ offs = rxf_data.offset;
	1470	+
	1471	+ /* Lock fence */
	1472	+ iwl_write_prph_no_grab(fwrt->trans, RXF_SET_FENCE_MODE + offs, 0x1);
	1473	+ /* Set fence pointer to the same place like WR pointer */
	1474	+ iwl_write_prph_no_grab(fwrt->trans, RXF_LD_WR2FENCE + offs, 0x1);
	1475	+ /* Set fence offset */
	1476	+ iwl_write_prph_no_grab(fwrt->trans, RXF_LD_FENCE_OFFSET_ADDR + offs,
	1477	+ 0x0);
	1478	+
	1479	+ /* Read FIFO */
	1480	+ addr = RXF_FIFO_RD_FENCE_INC + offs;
	1481	+ data = (void *)reg_dump;
	1482	+ for (i = 0; i < rxf_data.size; i += sizeof(*data))
	1483	+ *data++ = cpu_to_le32(iwl_read_prph_no_grab(fwrt->trans, addr));
	1484	+
	1485	+out:
	1486	+ iwl_trans_release_nic_access(fwrt->trans, &flags);
	1487	+
	1488	+ return sizeof(*range) + le32_to_cpu(range->range_data_size);
	1489	+}
	1490	+
	1491	+static int
	1492	+iwl_dump_ini_err_table_iter(struct iwl_fw_runtime *fwrt,
	1493	+ struct iwl_dump_ini_region_data *reg_data,
	1494	+ void *range_ptr, int idx)
	1495	+{
	1496	+ struct iwl_fw_ini_region_tlv reg = (void )reg_data->reg_tlv->data;
	1497	+ struct iwl_fw_ini_region_err_table *err_table = &reg->err_table;
	1498	+ struct iwl_fw_ini_error_dump_range *range = range_ptr;
	1499	+ u32 addr = le32_to_cpu(err_table->base_addr) +
	1500	+ le32_to_cpu(err_table->offset);
	1501	+
	1502	+ range->internal_base_addr = cpu_to_le32(addr);
	1503	+ range->range_data_size = err_table->size;
	1504	+ iwl_trans_read_mem_bytes(fwrt->trans, addr, range->data,
	1505	+ le32_to_cpu(err_table->size));
	1506	+
	1507	+ return sizeof(*range) + le32_to_cpu(range->range_data_size);
	1508	+}
	1509	+
	1510	+static int
	1511	+iwl_dump_ini_special_mem_iter(struct iwl_fw_runtime *fwrt,
	1512	+ struct iwl_dump_ini_region_data *reg_data,
	1513	+ void *range_ptr, int idx)
	1514	+{
	1515	+ struct iwl_fw_ini_region_tlv reg = (void )reg_data->reg_tlv->data;
	1516	+ struct iwl_fw_ini_region_special_device_memory *special_mem =
	1517	+ &reg->special_mem;
	1518	+
	1519	+ struct iwl_fw_ini_error_dump_range *range = range_ptr;
	1520	+ u32 addr = le32_to_cpu(special_mem->base_addr) +
	1521	+ le32_to_cpu(special_mem->offset);
	1522	+
	1523	+ range->internal_base_addr = cpu_to_le32(addr);
	1524	+ range->range_data_size = special_mem->size;
	1525	+ iwl_trans_read_mem_bytes(fwrt->trans, addr, range->data,
	1526	+ le32_to_cpu(special_mem->size));
	1527	+
	1528	+ return sizeof(*range) + le32_to_cpu(range->range_data_size);
	1529	+}
	1530	+
	1531	+static int iwl_dump_ini_fw_pkt_iter(struct iwl_fw_runtime *fwrt,
	1532	+ struct iwl_dump_ini_region_data *reg_data,
	1533	+ void *range_ptr, int idx)
	1534	+{
	1535	+ struct iwl_fw_ini_error_dump_range *range = range_ptr;
	1536	+ struct iwl_rx_packet *pkt = reg_data->dump_data->fw_pkt;
	1537	+ u32 pkt_len;
	1538	+
	1539	+ if (!pkt)
	1540	+ return -EIO;
	1541	+
	1542	+ pkt_len = iwl_rx_packet_payload_len(pkt);
	1543	+
	1544	+ memcpy(&range->fw_pkt_hdr, &pkt->hdr, sizeof(range->fw_pkt_hdr));
	1545	+ range->range_data_size = cpu_to_le32(pkt_len);
	1546	+
	1547	+ memcpy(range->data, pkt->data, pkt_len);
	1548	+
	1549	+ return sizeof(*range) + le32_to_cpu(range->range_data_size);
	1550	+}
	1551	+
	1552	+static void *
	1553	+iwl_dump_ini_mem_fill_header(struct iwl_fw_runtime *fwrt,
	1554	+ struct iwl_dump_ini_region_data *reg_data,
	1555	+ void *data)
	1556	+{
	1557	+ struct iwl_fw_ini_error_dump *dump = data;
	1558	+
	1559	+ dump->header.version = cpu_to_le32(IWL_INI_DUMP_VER);
	1560	+
	1561	+ return dump->ranges;
	1562	+}
	1563	+
	1564	+/**
	1565	+ * mask_apply_and_normalize - applies mask on val and normalize the result
	1566	+ *
	1567	+ * The normalization is based on the first set bit in the mask
	1568	+ *
	1569	+ * @val: value
	1570	+ * @mask: mask to apply and to normalize with
	1571	+ */
	1572	+static u32 mask_apply_and_normalize(u32 val, u32 mask)
	1573	+{
	1574	+ return (val & mask) >> (ffs(mask) - 1);
	1575	+}
	1576	+
	1577	+static __le32 iwl_get_mon_reg(struct iwl_fw_runtime *fwrt, u32 alloc_id,
	1578	+ const struct iwl_fw_mon_reg *reg_info)
	1579	+{
	1580	+ u32 val, offs;
	1581	+
	1582	+ /* The header addresses of DBGCi is calculate as follows:
	1583	+ * DBGC1 address + (0x100 * i)
	1584	+ */
	1585	+ offs = (alloc_id - IWL_FW_INI_ALLOCATION_ID_DBGC1) * 0x100;
	1586	+
	1587	+ if (!reg_info \|\| !reg_info->addr \|\| !reg_info->mask)
	1588	+ return 0;
	1589	+
	1590	+ val = iwl_read_prph_no_grab(fwrt->trans, reg_info->addr + offs);
	1591	+
	1592	+ return cpu_to_le32(mask_apply_and_normalize(val, reg_info->mask));
	1593	+}
	1594	+
	1595	+static void *
	1596	+iwl_dump_ini_mon_fill_header(struct iwl_fw_runtime *fwrt,
	1597	+ struct iwl_dump_ini_region_data *reg_data,
	1598	+ struct iwl_fw_ini_monitor_dump *data,
	1599	+ const struct iwl_fw_mon_regs *addrs)
	1600	+{
	1601	+ struct iwl_fw_ini_region_tlv reg = (void )reg_data->reg_tlv->data;
	1602	+ u32 alloc_id = le32_to_cpu(reg->dram_alloc_id);
	1603	+ unsigned long flags;
	1604	+
	1605	+ if (!iwl_trans_grab_nic_access(fwrt->trans, &flags)) {
	1606	+ IWL_ERR(fwrt, "Failed to get monitor header\n");
	1607	+ return NULL;
	1608	+ }
	1609	+
	1610	+ data->write_ptr = iwl_get_mon_reg(fwrt, alloc_id,
	1611	+ &addrs->write_ptr);
	1612	+ data->cycle_cnt = iwl_get_mon_reg(fwrt, alloc_id,
	1613	+ &addrs->cycle_cnt);
	1614	+ data->cur_frag = iwl_get_mon_reg(fwrt, alloc_id,
	1615	+ &addrs->cur_frag);
	1616	+
	1617	+ iwl_trans_release_nic_access(fwrt->trans, &flags);
	1618	+
	1619	+ data->header.version = cpu_to_le32(IWL_INI_DUMP_VER);
	1620	+
	1621	+ return data->ranges;
	1622	+}
	1623	+
	1624	+static void *
	1625	+iwl_dump_ini_mon_dram_fill_header(struct iwl_fw_runtime *fwrt,
	1626	+ struct iwl_dump_ini_region_data *reg_data,
	1627	+ void *data)
	1628	+{
	1629	+ struct iwl_fw_ini_monitor_dump mon_dump = (void )data;
	1630	+
	1631	+ return iwl_dump_ini_mon_fill_header(fwrt, reg_data, mon_dump,
	1632	+ &fwrt->trans->cfg->mon_dram_regs);
	1633	+}
	1634	+
	1635	+static void *
	1636	+iwl_dump_ini_mon_smem_fill_header(struct iwl_fw_runtime *fwrt,
	1637	+ struct iwl_dump_ini_region_data *reg_data,
	1638	+ void *data)
	1639	+{
	1640	+ struct iwl_fw_ini_monitor_dump mon_dump = (void )data;
	1641	+
	1642	+ return iwl_dump_ini_mon_fill_header(fwrt, reg_data, mon_dump,
	1643	+ &fwrt->trans->cfg->mon_smem_regs);
	1644	+}
	1645	+
	1646	+static void *
	1647	+iwl_dump_ini_err_table_fill_header(struct iwl_fw_runtime *fwrt,
	1648	+ struct iwl_dump_ini_region_data *reg_data,
	1649	+ void *data)
	1650	+{
	1651	+ struct iwl_fw_ini_region_tlv reg = (void )reg_data->reg_tlv->data;
	1652	+ struct iwl_fw_ini_err_table_dump *dump = data;
	1653	+
	1654	+ dump->header.version = cpu_to_le32(IWL_INI_DUMP_VER);
	1655	+ dump->version = reg->err_table.version;
	1656	+
	1657	+ return dump->ranges;
	1658	+}
	1659	+
	1660	+static void *
	1661	+iwl_dump_ini_special_mem_fill_header(struct iwl_fw_runtime *fwrt,
	1662	+ struct iwl_dump_ini_region_data *reg_data,
	1663	+ void *data)
	1664	+{
	1665	+ struct iwl_fw_ini_region_tlv reg = (void )reg_data->reg_tlv->data;
	1666	+ struct iwl_fw_ini_special_device_memory *dump = data;
	1667	+
	1668	+ dump->header.version = cpu_to_le32(IWL_INI_DUMP_VER);
	1669	+ dump->type = reg->special_mem.type;
	1670	+ dump->version = reg->special_mem.version;
	1671	+
	1672	+ return dump->ranges;
	1673	+}
	1674	+
	1675	+static u32 iwl_dump_ini_mem_ranges(struct iwl_fw_runtime *fwrt,
	1676	+ struct iwl_dump_ini_region_data *reg_data)
	1677	+{
	1678	+ struct iwl_fw_ini_region_tlv reg = (void )reg_data->reg_tlv->data;
	1679	+
	1680	+ return iwl_tlv_array_len(reg_data->reg_tlv, reg, addrs);
	1681	+}
	1682	+
	1683	+static u32 iwl_dump_ini_paging_ranges(struct iwl_fw_runtime *fwrt,
	1684	+ struct iwl_dump_ini_region_data *reg_data)
	1685	+{
	1686	+ if (fwrt->trans->trans_cfg->gen2)
	1687	+ return fwrt->trans->init_dram.paging_cnt;
	1688	+
	1689	+ return fwrt->num_of_paging_blk;
	1690	+}
	1691	+
	1692	+static u32
	1693	+iwl_dump_ini_mon_dram_ranges(struct iwl_fw_runtime *fwrt,
	1694	+ struct iwl_dump_ini_region_data *reg_data)
	1695	+{
	1696	+ struct iwl_fw_ini_region_tlv reg = (void )reg_data->reg_tlv->data;
	1697	+ struct iwl_fw_mon *fw_mon;
	1698	+ u32 ranges = 0, alloc_id = le32_to_cpu(reg->dram_alloc_id);
	1699	+ int i;
	1700	+
	1701	+ fw_mon = &fwrt->trans->dbg.fw_mon_ini[alloc_id];
	1702	+
	1703	+ for (i = 0; i < fw_mon->num_frags; i++) {
	1704	+ if (!fw_mon->frags[i].size)
	1705	+ break;
	1706	+
	1707	+ ranges++;
	1708	+ }
	1709	+
	1710	+ return ranges;
	1711	+}
	1712	+
	1713	+static u32 iwl_dump_ini_txf_ranges(struct iwl_fw_runtime *fwrt,
	1714	+ struct iwl_dump_ini_region_data *reg_data)
	1715	+{
	1716	+ u32 num_of_fifos = 0;
	1717	+
	1718	+ while (iwl_ini_txf_iter(fwrt, reg_data, num_of_fifos))
	1719	+ num_of_fifos++;
	1720	+
	1721	+ return num_of_fifos;
	1722	+}
	1723	+
	1724	+static u32 iwl_dump_ini_single_range(struct iwl_fw_runtime *fwrt,
	1725	+ struct iwl_dump_ini_region_data *reg_data)
	1726	+{
	1727	+ return 1;
	1728	+}
	1729	+
	1730	+static u32 iwl_dump_ini_mem_get_size(struct iwl_fw_runtime *fwrt,
	1731	+ struct iwl_dump_ini_region_data *reg_data)
	1732	+{
	1733	+ struct iwl_fw_ini_region_tlv reg = (void )reg_data->reg_tlv->data;
	1734	+ u32 size = le32_to_cpu(reg->dev_addr.size);
	1735	+ u32 ranges = iwl_dump_ini_mem_ranges(fwrt, reg_data);
	1736	+
	1737	+ if (!size \|\| !ranges)
	1738	+ return 0;
	1739	+
	1740	+ return sizeof(struct iwl_fw_ini_error_dump) + ranges *
	1741	+ (size + sizeof(struct iwl_fw_ini_error_dump_range));
	1742	+}
	1743	+
	1744	+static u32
	1745	+iwl_dump_ini_paging_get_size(struct iwl_fw_runtime *fwrt,
	1746	+ struct iwl_dump_ini_region_data *reg_data)
	1747	+{
	1748	+ int i;
	1749	+ u32 range_header_len = sizeof(struct iwl_fw_ini_error_dump_range);
	1750	+ u32 size = sizeof(struct iwl_fw_ini_error_dump);
	1751	+
	1752	+ if (fwrt->trans->trans_cfg->gen2) {
	1753	+ for (i = 0; i < iwl_dump_ini_paging_ranges(fwrt, reg_data); i++)
	1754	+ size += range_header_len +
	1755	+ fwrt->trans->init_dram.paging[i].size;
	1756	+ } else {
	1757	+ for (i = 1; i <= iwl_dump_ini_paging_ranges(fwrt, reg_data);
	1758	+ i++)
	1759	+ size += range_header_len +
	1760	+ fwrt->fw_paging_db[i].fw_paging_size;
	1761	+ }
	1762	+
	1763	+ return size;
	1764	+}
	1765	+
	1766	+static u32
	1767	+iwl_dump_ini_mon_dram_get_size(struct iwl_fw_runtime *fwrt,
	1768	+ struct iwl_dump_ini_region_data *reg_data)
	1769	+{
	1770	+ struct iwl_fw_ini_region_tlv reg = (void )reg_data->reg_tlv->data;
	1771	+ struct iwl_fw_mon *fw_mon;
	1772	+ u32 size = 0, alloc_id = le32_to_cpu(reg->dram_alloc_id);
	1773	+ int i;
	1774	+
	1775	+ fw_mon = &fwrt->trans->dbg.fw_mon_ini[alloc_id];
	1776	+
	1777	+ for (i = 0; i < fw_mon->num_frags; i++) {
	1778	+ struct iwl_dram_data *frag = &fw_mon->frags[i];
	1779	+
	1780	+ if (!frag->size)
	1781	+ break;
	1782	+
	1783	+ size += sizeof(struct iwl_fw_ini_error_dump_range) + frag->size;
	1784	+ }
	1785	+
	1786	+ if (size)
	1787	+ size += sizeof(struct iwl_fw_ini_monitor_dump);
	1788	+
	1789	+ return size;
	1790	+}
	1791	+
	1792	+static u32
	1793	+iwl_dump_ini_mon_smem_get_size(struct iwl_fw_runtime *fwrt,
	1794	+ struct iwl_dump_ini_region_data *reg_data)
	1795	+{
	1796	+ struct iwl_fw_ini_region_tlv reg = (void )reg_data->reg_tlv->data;
	1797	+ u32 size;
	1798	+
	1799	+ size = le32_to_cpu(reg->internal_buffer.size);
	1800	+ if (!size)
	1801	+ return 0;
	1802	+
	1803	+ size += sizeof(struct iwl_fw_ini_monitor_dump) +
	1804	+ sizeof(struct iwl_fw_ini_error_dump_range);
	1805	+
	1806	+ return size;
	1807	+}
	1808	+
	1809	+static u32 iwl_dump_ini_txf_get_size(struct iwl_fw_runtime *fwrt,
	1810	+ struct iwl_dump_ini_region_data *reg_data)
	1811	+{
	1812	+ struct iwl_fw_ini_region_tlv reg = (void )reg_data->reg_tlv->data;
	1813	+ struct iwl_txf_iter_data *iter = &fwrt->dump.txf_iter_data;
	1814	+ u32 registers_num = iwl_tlv_array_len(reg_data->reg_tlv, reg, addrs);
	1815	+ u32 size = 0;
	1816	+ u32 fifo_hdr = sizeof(struct iwl_fw_ini_error_dump_range) +
	1817	+ registers_num *
	1818	+ sizeof(struct iwl_fw_ini_error_dump_register);
	1819	+
	1820	+ while (iwl_ini_txf_iter(fwrt, reg_data, size)) {
	1821	+ size += fifo_hdr;
	1822	+ if (!reg->fifos.hdr_only)
	1823	+ size += iter->fifo_size;
	1824	+ }
	1825	+
	1826	+ if (!size)
	1827	+ return 0;
	1828	+
	1829	+ return size + sizeof(struct iwl_fw_ini_error_dump);
	1830	+}
	1831	+
	1832	+static u32 iwl_dump_ini_rxf_get_size(struct iwl_fw_runtime *fwrt,
	1833	+ struct iwl_dump_ini_region_data *reg_data)
	1834	+{
	1835	+ struct iwl_fw_ini_region_tlv reg = (void )reg_data->reg_tlv->data;
	1836	+ struct iwl_ini_rxf_data rx_data;
	1837	+ u32 registers_num = iwl_tlv_array_len(reg_data->reg_tlv, reg, addrs);
	1838	+ u32 size = sizeof(struct iwl_fw_ini_error_dump) +
	1839	+ sizeof(struct iwl_fw_ini_error_dump_range) +
	1840	+ registers_num * sizeof(struct iwl_fw_ini_error_dump_register);
	1841	+
	1842	+ if (reg->fifos.hdr_only)
	1843	+ return size;
	1844	+
	1845	+ iwl_ini_get_rxf_data(fwrt, reg_data, &rx_data);
	1846	+ size += rx_data.size;
	1847	+
	1848	+ return size;
	1849	+}
	1850	+
	1851	+static u32
	1852	+iwl_dump_ini_err_table_get_size(struct iwl_fw_runtime *fwrt,
	1853	+ struct iwl_dump_ini_region_data *reg_data)
	1854	+{
	1855	+ struct iwl_fw_ini_region_tlv reg = (void )reg_data->reg_tlv->data;
	1856	+ u32 size = le32_to_cpu(reg->err_table.size);
	1857	+
	1858	+ if (size)
	1859	+ size += sizeof(struct iwl_fw_ini_err_table_dump) +
	1860	+ sizeof(struct iwl_fw_ini_error_dump_range);
	1861	+
	1862	+ return size;
	1863	+}
	1864	+
	1865	+static u32
	1866	+iwl_dump_ini_special_mem_get_size(struct iwl_fw_runtime *fwrt,
	1867	+ struct iwl_dump_ini_region_data *reg_data)
	1868	+{
	1869	+ struct iwl_fw_ini_region_tlv reg = (void )reg_data->reg_tlv->data;
	1870	+ u32 size = le32_to_cpu(reg->special_mem.size);
	1871	+
	1872	+ if (size)
	1873	+ size += sizeof(struct iwl_fw_ini_special_device_memory) +
	1874	+ sizeof(struct iwl_fw_ini_error_dump_range);
	1875	+
	1876	+ return size;
	1877	+}
	1878	+
	1879	+static u32
	1880	+iwl_dump_ini_fw_pkt_get_size(struct iwl_fw_runtime *fwrt,
	1881	+ struct iwl_dump_ini_region_data *reg_data)
	1882	+{
	1883	+ u32 size = 0;
	1884	+
	1885	+ if (!reg_data->dump_data->fw_pkt)
	1886	+ return 0;
	1887	+
	1888	+ size += iwl_rx_packet_payload_len(reg_data->dump_data->fw_pkt);
	1889	+ if (size)
	1890	+ size += sizeof(struct iwl_fw_ini_error_dump) +
	1891	+ sizeof(struct iwl_fw_ini_error_dump_range);
	1892	+
	1893	+ return size;
	1894	+}
	1895	+
	1896	+/**
	1897	+ * struct iwl_dump_ini_mem_ops - ini memory dump operations
	1898	+ * @get_num_of_ranges: returns the number of memory ranges in the region.
	1899	+ * @get_size: returns the total size of the region.
	1900	+ * @fill_mem_hdr: fills region type specific headers and returns pointer to
	1901	+ * the first range or NULL if failed to fill headers.
	1902	+ * @fill_range: copies a given memory range into the dump.
	1903	+ * Returns the size of the range or negative error value otherwise.
	1904	+ */
	1905	+struct iwl_dump_ini_mem_ops {
	1906	+ u32 (get_num_of_ranges)(struct iwl_fw_runtime fwrt,
	1907	+ struct iwl_dump_ini_region_data *reg_data);
	1908	+ u32 (get_size)(struct iwl_fw_runtime fwrt,
	1909	+ struct iwl_dump_ini_region_data *reg_data);
	1910	+ void (fill_mem_hdr)(struct iwl_fw_runtime *fwrt,
	1911	+ struct iwl_dump_ini_region_data *reg_data,
	1912	+ void *data);
	1913	+ int (fill_range)(struct iwl_fw_runtime fwrt,
	1914	+ struct iwl_dump_ini_region_data *reg_data,
	1915	+ void *range, int idx);
	1916	+};
	1917	+
	1918	+/**
	1919	+ * iwl_dump_ini_mem
	1920	+ *
	1921	+ * Creates a dump tlv and copy a memory region into it.
	1922	+ * Returns the size of the current dump tlv or 0 if failed
	1923	+ *
	1924	+ * @fwrt: fw runtime struct
	1925	+ * @list: list to add the dump tlv to
	1926	+ * @reg: memory region
	1927	+ * @ops: memory dump operations
	1928	+ */
	1929	+static u32 iwl_dump_ini_mem(struct iwl_fw_runtime fwrt, struct list_head list,
	1930	+ struct iwl_dump_ini_region_data *reg_data,
	1931	+ const struct iwl_dump_ini_mem_ops *ops)
	1932	+{
	1933	+ struct iwl_fw_ini_region_tlv reg = (void )reg_data->reg_tlv->data;
	1934	+ struct iwl_fw_ini_dump_entry *entry;
	1935	+ struct iwl_fw_error_dump_data *tlv;
	1936	+ struct iwl_fw_ini_error_dump_header *header;
	1937	+ u32 type = le32_to_cpu(reg->type), id = le32_to_cpu(reg->id);
	1938	+ u32 num_of_ranges, i, size;
	1939	+ void *range;
	1940	+
	1941	+ if (!ops->get_num_of_ranges \|\| !ops->get_size \|\| !ops->fill_mem_hdr \|\|
	1942	+ !ops->fill_range)
	1943	+ return 0;
	1944	+
	1945	+ size = ops->get_size(fwrt, reg_data);
	1946	+ if (!size)
	1947	+ return 0;
	1948	+
	1949	+ entry = vzalloc(sizeof(entry) + sizeof(tlv) + size);
	1950	+ if (!entry)
	1951	+ return 0;
	1952	+
	1953	+ entry->size = sizeof(*tlv) + size;
	1954	+
	1955	+ tlv = (void *)entry->data;
	1956	+ tlv->type = reg->type;
	1957	+ tlv->len = cpu_to_le32(size);
	1958	+
	1959	+ IWL_DEBUG_FW(fwrt, "WRT: Collecting region: id=%d, type=%d\n", id,
	1960	+ type);
	1961	+
	1962	+ num_of_ranges = ops->get_num_of_ranges(fwrt, reg_data);
	1963	+
	1964	+ header = (void *)tlv->data;
	1965	+ header->region_id = reg->id;
	1966	+ header->num_of_ranges = cpu_to_le32(num_of_ranges);
	1967	+ header->name_len = cpu_to_le32(IWL_FW_INI_MAX_NAME);
	1968	+ memcpy(header->name, reg->name, IWL_FW_INI_MAX_NAME);
	1969	+
	1970	+ range = ops->fill_mem_hdr(fwrt, reg_data, header);
	1971	+ if (!range) {
	1972	+ IWL_ERR(fwrt,
	1973	+ "WRT: Failed to fill region header: id=%d, type=%d\n",
	1974	+ id, type);
	1975	+ goto out_err;
	1976	+ }
	1977	+
	1978	+ for (i = 0; i < num_of_ranges; i++) {
	1979	+ int range_size = ops->fill_range(fwrt, reg_data, range, i);
	1980	+
	1981	+ if (range_size < 0) {
	1982	+ IWL_ERR(fwrt,
	1983	+ "WRT: Failed to dump region: id=%d, type=%d\n",
	1984	+ id, type);
	1985	+ goto out_err;
	1986	+ }
	1987	+ range = range + range_size;
	1988	+ }
	1989	+
	1990	+ list_add_tail(&entry->list, list);
	1991	+
	1992	+ return entry->size;
	1993	+
	1994	+out_err:
	1995	+ vfree(entry);
	1996	+
	1997	+ return 0;
	1998	+}
	1999	+
	2000	+static u32 iwl_dump_ini_info(struct iwl_fw_runtime *fwrt,
	2001	+ struct iwl_fw_ini_trigger_tlv *trigger,
	2002	+ struct list_head *list)
	2003	+{
	2004	+ struct iwl_fw_ini_dump_entry *entry;
	2005	+ struct iwl_fw_error_dump_data *tlv;
	2006	+ struct iwl_fw_ini_dump_info *dump;
	2007	+ struct iwl_dbg_tlv_node *node;
	2008	+ struct iwl_fw_ini_dump_cfg_name *cfg_name;
	2009	+ u32 size = sizeof(tlv) + sizeof(dump);
	2010	+ u32 num_of_cfg_names = 0;
	2011	+ u32 hw_type;
	2012	+
	2013	+ list_for_each_entry(node, &fwrt->trans->dbg.debug_info_tlv_list, list) {
	2014	+ size += sizeof(*cfg_name);
	2015	+ num_of_cfg_names++;
	2016	+ }
	2017	+
	2018	+ entry = vzalloc(sizeof(*entry) + size);
	2019	+ if (!entry)
	2020	+ return 0;
	2021	+
	2022	+ entry->size = size;
	2023	+
	2024	+ tlv = (void *)entry->data;
	2025	+ tlv->type = cpu_to_le32(IWL_INI_DUMP_INFO_TYPE);
	2026	+ tlv->len = cpu_to_le32(size - sizeof(*tlv));
	2027	+
	2028	+ dump = (void *)tlv->data;
	2029	+
	2030	+ dump->version = cpu_to_le32(IWL_INI_DUMP_VER);
	2031	+ dump->time_point = trigger->time_point;
	2032	+ dump->trigger_reason = trigger->trigger_reason;
	2033	+ dump->external_cfg_state =
	2034	+ cpu_to_le32(fwrt->trans->dbg.external_ini_cfg);
	2035	+
	2036	+ dump->ver_type = cpu_to_le32(fwrt->dump.fw_ver.type);
	2037	+ dump->ver_subtype = cpu_to_le32(fwrt->dump.fw_ver.subtype);
	2038	+
	2039	+ dump->hw_step = cpu_to_le32(CSR_HW_REV_STEP(fwrt->trans->hw_rev));
	2040	+
	2041	+ /*
	2042	+ * Several HWs all have type == 0x42, so we'll override this value
	2043	+ * according to the detected HW
	2044	+ */
	2045	+ hw_type = CSR_HW_REV_TYPE(fwrt->trans->hw_rev);
	2046	+ if (hw_type == IWL_AX210_HW_TYPE) {
	2047	+ u32 prph_val = iwl_read_prph(fwrt->trans, WFPM_OTP_CFG1_ADDR);
	2048	+ u32 is_jacket = !!(prph_val & WFPM_OTP_CFG1_IS_JACKET_BIT);
	2049	+ u32 is_cdb = !!(prph_val & WFPM_OTP_CFG1_IS_CDB_BIT);
	2050	+ u32 masked_bits = is_jacket \| (is_cdb << 1);
	2051	+
	2052	+ /*
	2053	+ * The HW type depends on certain bits in this case, so add
	2054	+ * these bits to the HW type. We won't have collisions since we
	2055	+ * add these bits after the highest possible bit in the mask.
	2056	+ */
	2057	+ hw_type \|= masked_bits << IWL_AX210_HW_TYPE_ADDITION_SHIFT;
	2058	+ }
	2059	+ dump->hw_type = cpu_to_le32(hw_type);
	2060	+
	2061	+ dump->rf_id_flavor =
	2062	+ cpu_to_le32(CSR_HW_RFID_FLAVOR(fwrt->trans->hw_rf_id));
	2063	+ dump->rf_id_dash = cpu_to_le32(CSR_HW_RFID_DASH(fwrt->trans->hw_rf_id));
	2064	+ dump->rf_id_step = cpu_to_le32(CSR_HW_RFID_STEP(fwrt->trans->hw_rf_id));
	2065	+ dump->rf_id_type = cpu_to_le32(CSR_HW_RFID_TYPE(fwrt->trans->hw_rf_id));
	2066	+
	2067	+ dump->lmac_major = cpu_to_le32(fwrt->dump.fw_ver.lmac_major);
	2068	+ dump->lmac_minor = cpu_to_le32(fwrt->dump.fw_ver.lmac_minor);
	2069	+ dump->umac_major = cpu_to_le32(fwrt->dump.fw_ver.umac_major);
	2070	+ dump->umac_minor = cpu_to_le32(fwrt->dump.fw_ver.umac_minor);
	2071	+
	2072	+ dump->fw_mon_mode = cpu_to_le32(fwrt->trans->dbg.ini_dest);
	2073	+ dump->regions_mask = trigger->regions_mask;
	2074	+
	2075	+ dump->build_tag_len = cpu_to_le32(sizeof(dump->build_tag));
	2076	+ memcpy(dump->build_tag, fwrt->fw->human_readable,
	2077	+ sizeof(dump->build_tag));
	2078	+
	2079	+ cfg_name = dump->cfg_names;
	2080	+ dump->num_of_cfg_names = cpu_to_le32(num_of_cfg_names);
	2081	+ list_for_each_entry(node, &fwrt->trans->dbg.debug_info_tlv_list, list) {
	2082	+ struct iwl_fw_ini_debug_info_tlv *debug_info =
	2083	+ (void *)node->tlv.data;
	2084	+
	2085	+ cfg_name->image_type = debug_info->image_type;
	2086	+ cfg_name->cfg_name_len =
	2087	+ cpu_to_le32(IWL_FW_INI_MAX_CFG_NAME);
	2088	+ memcpy(cfg_name->cfg_name, debug_info->debug_cfg_name,
	2089	+ sizeof(cfg_name->cfg_name));
	2090	+ cfg_name++;
	2091	+ }
	2092	+
	2093	+ /* add dump info TLV to the beginning of the list since it needs to be
	2094	+ * the first TLV in the dump
	2095	+ */
	2096	+ list_add(&entry->list, list);
	2097	+
	2098	+ return entry->size;
	2099	+}
	2100	+
	2101	+static const struct iwl_dump_ini_mem_ops iwl_dump_ini_region_ops[] = {
	2102	+ [IWL_FW_INI_REGION_INVALID] = {},
	2103	+ [IWL_FW_INI_REGION_INTERNAL_BUFFER] = {
	2104	+ .get_num_of_ranges = iwl_dump_ini_single_range,
	2105	+ .get_size = iwl_dump_ini_mon_smem_get_size,
	2106	+ .fill_mem_hdr = iwl_dump_ini_mon_smem_fill_header,
	2107	+ .fill_range = iwl_dump_ini_mon_smem_iter,
	2108	+ },
	2109	+ [IWL_FW_INI_REGION_DRAM_BUFFER] = {
	2110	+ .get_num_of_ranges = iwl_dump_ini_mon_dram_ranges,
	2111	+ .get_size = iwl_dump_ini_mon_dram_get_size,
	2112	+ .fill_mem_hdr = iwl_dump_ini_mon_dram_fill_header,
	2113	+ .fill_range = iwl_dump_ini_mon_dram_iter,
	2114	+ },
	2115	+ [IWL_FW_INI_REGION_TXF] = {
	2116	+ .get_num_of_ranges = iwl_dump_ini_txf_ranges,
	2117	+ .get_size = iwl_dump_ini_txf_get_size,
	2118	+ .fill_mem_hdr = iwl_dump_ini_mem_fill_header,
	2119	+ .fill_range = iwl_dump_ini_txf_iter,
	2120	+ },
	2121	+ [IWL_FW_INI_REGION_RXF] = {
	2122	+ .get_num_of_ranges = iwl_dump_ini_single_range,
	2123	+ .get_size = iwl_dump_ini_rxf_get_size,
	2124	+ .fill_mem_hdr = iwl_dump_ini_mem_fill_header,
	2125	+ .fill_range = iwl_dump_ini_rxf_iter,
	2126	+ },
	2127	+ [IWL_FW_INI_REGION_LMAC_ERROR_TABLE] = {
	2128	+ .get_num_of_ranges = iwl_dump_ini_single_range,
	2129	+ .get_size = iwl_dump_ini_err_table_get_size,
	2130	+ .fill_mem_hdr = iwl_dump_ini_err_table_fill_header,
	2131	+ .fill_range = iwl_dump_ini_err_table_iter,
	2132	+ },
	2133	+ [IWL_FW_INI_REGION_UMAC_ERROR_TABLE] = {
	2134	+ .get_num_of_ranges = iwl_dump_ini_single_range,
	2135	+ .get_size = iwl_dump_ini_err_table_get_size,
	2136	+ .fill_mem_hdr = iwl_dump_ini_err_table_fill_header,
	2137	+ .fill_range = iwl_dump_ini_err_table_iter,
	2138	+ },
	2139	+ [IWL_FW_INI_REGION_RSP_OR_NOTIF] = {
	2140	+ .get_num_of_ranges = iwl_dump_ini_single_range,
	2141	+ .get_size = iwl_dump_ini_fw_pkt_get_size,
	2142	+ .fill_mem_hdr = iwl_dump_ini_mem_fill_header,
	2143	+ .fill_range = iwl_dump_ini_fw_pkt_iter,
	2144	+ },
	2145	+ [IWL_FW_INI_REGION_DEVICE_MEMORY] = {
	2146	+ .get_num_of_ranges = iwl_dump_ini_mem_ranges,
	2147	+ .get_size = iwl_dump_ini_mem_get_size,
	2148	+ .fill_mem_hdr = iwl_dump_ini_mem_fill_header,
	2149	+ .fill_range = iwl_dump_ini_dev_mem_iter,
	2150	+ },
	2151	+ [IWL_FW_INI_REGION_PERIPHERY_MAC] = {
	2152	+ .get_num_of_ranges = iwl_dump_ini_mem_ranges,
	2153	+ .get_size = iwl_dump_ini_mem_get_size,
	2154	+ .fill_mem_hdr = iwl_dump_ini_mem_fill_header,
	2155	+ .fill_range = iwl_dump_ini_prph_iter,
	2156	+ },
	2157	+ [IWL_FW_INI_REGION_PERIPHERY_PHY] = {},
	2158	+ [IWL_FW_INI_REGION_PERIPHERY_AUX] = {},
	2159	+ [IWL_FW_INI_REGION_PAGING] = {
	2160	+ .fill_mem_hdr = iwl_dump_ini_mem_fill_header,
	2161	+ .get_num_of_ranges = iwl_dump_ini_paging_ranges,
	2162	+ .get_size = iwl_dump_ini_paging_get_size,
	2163	+ .fill_range = iwl_dump_ini_paging_iter,
	2164	+ },
	2165	+ [IWL_FW_INI_REGION_CSR] = {
	2166	+ .get_num_of_ranges = iwl_dump_ini_mem_ranges,
	2167	+ .get_size = iwl_dump_ini_mem_get_size,
	2168	+ .fill_mem_hdr = iwl_dump_ini_mem_fill_header,
	2169	+ .fill_range = iwl_dump_ini_csr_iter,
	2170	+ },
	2171	+ [IWL_FW_INI_REGION_DRAM_IMR] = {},
	2172	+ [IWL_FW_INI_REGION_PCI_IOSF_CONFIG] = {
	2173	+ .get_num_of_ranges = iwl_dump_ini_mem_ranges,
	2174	+ .get_size = iwl_dump_ini_mem_get_size,
	2175	+ .fill_mem_hdr = iwl_dump_ini_mem_fill_header,
	2176	+ .fill_range = iwl_dump_ini_config_iter,
	2177	+ },
	2178	+ [IWL_FW_INI_REGION_SPECIAL_DEVICE_MEMORY] = {
	2179	+ .get_num_of_ranges = iwl_dump_ini_single_range,
	2180	+ .get_size = iwl_dump_ini_special_mem_get_size,
	2181	+ .fill_mem_hdr = iwl_dump_ini_special_mem_fill_header,
	2182	+ .fill_range = iwl_dump_ini_special_mem_iter,
	2183	+ },
	2184	+};
	2185	+
	2186	+static u32 iwl_dump_ini_trigger(struct iwl_fw_runtime *fwrt,
	2187	+ struct iwl_fwrt_dump_data *dump_data,
	2188	+ struct list_head *list)
	2189	+{
	2190	+ struct iwl_fw_ini_trigger_tlv *trigger = dump_data->trig;
	2191	+ struct iwl_dump_ini_region_data reg_data = {
	2192	+ .dump_data = dump_data,
	2193	+ };
	2194	+ int i;
	2195	+ u32 size = 0;
	2196	+ u64 regions_mask = le64_to_cpu(trigger->regions_mask);
	2197	+
	2198	+ BUILD_BUG_ON(sizeof(trigger->regions_mask) != sizeof(regions_mask));
	2199	+ BUILD_BUG_ON((sizeof(trigger->regions_mask) * BITS_PER_BYTE) <
	2200	+ ARRAY_SIZE(fwrt->trans->dbg.active_regions));
	2201	+
	2202	+ for (i = 0; i < ARRAY_SIZE(fwrt->trans->dbg.active_regions); i++) {
	2203	+ u32 reg_type;
	2204	+ struct iwl_fw_ini_region_tlv *reg;
	2205	+
	2206	+ if (!(BIT_ULL(i) & regions_mask))
	2207	+ continue;
	2208	+
	2209	+ reg_data.reg_tlv = fwrt->trans->dbg.active_regions[i];
	2210	+ if (!reg_data.reg_tlv) {
	2211	+ IWL_WARN(fwrt,
	2212	+ "WRT: Unassigned region id %d, skipping\n", i);
	2213	+ continue;
	2214	+ }
	2215	+
	2216	+ reg = (void *)reg_data.reg_tlv->data;
	2217	+ reg_type = le32_to_cpu(reg->type);
	2218	+ if (reg_type >= ARRAY_SIZE(iwl_dump_ini_region_ops))
	2219	+ continue;
	2220	+
	2221	+ size += iwl_dump_ini_mem(fwrt, list, &reg_data,
	2222	+ &iwl_dump_ini_region_ops[reg_type]);
	2223	+ }
	2224	+
	2225	+ if (size)
	2226	+ size += iwl_dump_ini_info(fwrt, trigger, list);
	2227	+
	2228	+ return size;
	2229	+}
	2230	+
	2231	+static bool iwl_fw_ini_trigger_on(struct iwl_fw_runtime *fwrt,
	2232	+ struct iwl_fw_ini_trigger_tlv *trig)
	2233	+{
	2234	+ enum iwl_fw_ini_time_point tp_id = le32_to_cpu(trig->time_point);
	2235	+ u32 usec = le32_to_cpu(trig->ignore_consec);
	2236	+
	2237	+ if (!iwl_trans_dbg_ini_valid(fwrt->trans) \|\|
	2238	+ tp_id == IWL_FW_INI_TIME_POINT_INVALID \|\|
	2239	+ tp_id >= IWL_FW_INI_TIME_POINT_NUM \|\|
	2240	+ iwl_fw_dbg_no_trig_window(fwrt, tp_id, usec))
	2241	+ return false;
	2242	+
	2243	+ return true;
	2244	+}
	2245	+
	2246	+static u32 iwl_dump_ini_file_gen(struct iwl_fw_runtime *fwrt,
	2247	+ struct iwl_fwrt_dump_data *dump_data,
	2248	+ struct list_head *list)
	2249	+{
	2250	+ struct iwl_fw_ini_trigger_tlv *trigger = dump_data->trig;
	2251	+ struct iwl_fw_ini_dump_entry *entry;
	2252	+ struct iwl_fw_ini_dump_file_hdr *hdr;
	2253	+ u32 size;
	2254	+
	2255	+ if (!trigger \|\| !iwl_fw_ini_trigger_on(fwrt, trigger) \|\|
	2256	+ !le64_to_cpu(trigger->regions_mask))
	2257	+ return 0;
	2258	+
	2259	+ entry = vzalloc(sizeof(entry) + sizeof(hdr));
	2260	+ if (!entry)
	2261	+ return 0;
	2262	+
	2263	+ entry->size = sizeof(*hdr);
	2264	+
	2265	+ size = iwl_dump_ini_trigger(fwrt, dump_data, list);
	2266	+ if (!size) {
	2267	+ vfree(entry);
	2268	+ return 0;
	2269	+ }
	2270	+
	2271	+ hdr = (void *)entry->data;
	2272	+ hdr->barker = cpu_to_le32(IWL_FW_INI_ERROR_DUMP_BARKER);
	2273	+ hdr->file_len = cpu_to_le32(size + entry->size);
	2274	+
	2275	+ list_add(&entry->list, list);
	2276	+
	2277	+ return le32_to_cpu(hdr->file_len);
	2278	+}
	2279	+
	2280	+static inline void iwl_fw_free_dump_desc(struct iwl_fw_runtime *fwrt,
	2281	+ const struct iwl_fw_dump_desc *desc)
	2282	+{
	2283	+ if (desc && desc != &iwl_dump_desc_assert)
	2284	+ kfree(desc);
	2285	+
	2286	+ fwrt->dump.lmac_err_id[0] = 0;
	2287	+ if (fwrt->smem_cfg.num_lmacs > 1)
	2288	+ fwrt->dump.lmac_err_id[1] = 0;
	2289	+ fwrt->dump.umac_err_id = 0;
	2290	+}
	2291	+
	2292	+static void iwl_fw_error_dump(struct iwl_fw_runtime *fwrt,
	2293	+ struct iwl_fwrt_dump_data *dump_data)
	2294	+{
	2295	+ struct iwl_fw_dump_ptrs fw_error_dump = {};
	2296	+ struct iwl_fw_error_dump_file *dump_file;
	2297	+ struct scatterlist *sg_dump_data;
	2298	+ u32 file_len;
	2299	+ u32 dump_mask = fwrt->fw->dbg.dump_mask;
	2300	+
	2301	+ dump_file = iwl_fw_error_dump_file(fwrt, &fw_error_dump, dump_data);
	2302	+ if (!dump_file)
	2303	+ return;
	2304	+
	2305	+ if (dump_data->monitor_only)
	2306	+ dump_mask &= BIT(IWL_FW_ERROR_DUMP_FW_MONITOR);
	2307	+
	2308	+ fw_error_dump.trans_ptr = iwl_trans_dump_data(fwrt->trans, dump_mask);
	2309	+ file_len = le32_to_cpu(dump_file->file_len);
	2310	+ fw_error_dump.fwrt_len = file_len;
	2311	+
	2312	+ if (fw_error_dump.trans_ptr) {
	2313	+ file_len += fw_error_dump.trans_ptr->len;
	2314	+ dump_file->file_len = cpu_to_le32(file_len);
	2315	+ }
973	2316
974	2317	sg_dump_data = alloc_sgtable(file_len);
975	2318	if (sg_dump_data) {
976	2319	sg_pcopy_from_buffer(sg_dump_data,
977	2320	sg_nents(sg_dump_data),
978		- fw_error_dump->fwrt_ptr,
979		- fw_error_dump->fwrt_len, 0);
980		- if (fw_error_dump->trans_ptr)
	2321	+ fw_error_dump.fwrt_ptr,
	2322	+ fw_error_dump.fwrt_len, 0);
	2323	+ if (fw_error_dump.trans_ptr)
981	2324	sg_pcopy_from_buffer(sg_dump_data,
982	2325	sg_nents(sg_dump_data),
983		- fw_error_dump->trans_ptr->data,
984		- fw_error_dump->trans_ptr->len,
985		- fw_error_dump->fwrt_len);
	2326	+ fw_error_dump.trans_ptr->data,
	2327	+ fw_error_dump.trans_ptr->len,
	2328	+ fw_error_dump.fwrt_len);
986	2329	dev_coredumpsg(fwrt->trans->dev, sg_dump_data, file_len,
987	2330	GFP_KERNEL);
988	2331	}
989		- vfree(fw_error_dump->fwrt_ptr);
990		- vfree(fw_error_dump->trans_ptr);
991		- kfree(fw_error_dump);
992		-
993		-out:
994		- iwl_fw_free_dump_desc(fwrt);
995		- clear_bit(IWL_FWRT_STATUS_DUMPING, &fwrt->status);
996		- IWL_DEBUG_INFO(fwrt, "WRT dump done\n");
	2332	+ vfree(fw_error_dump.fwrt_ptr);
	2333	+ vfree(fw_error_dump.trans_ptr);
997	2334	}
998		-IWL_EXPORT_SYMBOL(iwl_fw_error_dump);
	2335	+
	2336	+static void iwl_dump_ini_list_free(struct list_head *list)
	2337	+{
	2338	+ while (!list_empty(list)) {
	2339	+ struct iwl_fw_ini_dump_entry *entry =
	2340	+ list_entry(list->next, typeof(*entry), list);
	2341	+
	2342	+ list_del(&entry->list);
	2343	+ vfree(entry);
	2344	+ }
	2345	+}
	2346	+
	2347	+static void iwl_fw_error_dump_data_free(struct iwl_fwrt_dump_data *dump_data)
	2348	+{
	2349	+ dump_data->trig = NULL;
	2350	+ kfree(dump_data->fw_pkt);
	2351	+ dump_data->fw_pkt = NULL;
	2352	+}
	2353	+
	2354	+static void iwl_fw_error_ini_dump(struct iwl_fw_runtime *fwrt,
	2355	+ struct iwl_fwrt_dump_data *dump_data)
	2356	+{
	2357	+ struct list_head dump_list = LIST_HEAD_INIT(dump_list);
	2358	+ struct scatterlist *sg_dump_data;
	2359	+ u32 file_len = iwl_dump_ini_file_gen(fwrt, dump_data, &dump_list);
	2360	+
	2361	+ if (!file_len)
	2362	+ return;
	2363	+
	2364	+ sg_dump_data = alloc_sgtable(file_len);
	2365	+ if (sg_dump_data) {
	2366	+ struct iwl_fw_ini_dump_entry *entry;
	2367	+ int sg_entries = sg_nents(sg_dump_data);
	2368	+ u32 offs = 0;
	2369	+
	2370	+ list_for_each_entry(entry, &dump_list, list) {
	2371	+ sg_pcopy_from_buffer(sg_dump_data, sg_entries,
	2372	+ entry->data, entry->size, offs);
	2373	+ offs += entry->size;
	2374	+ }
	2375	+ dev_coredumpsg(fwrt->trans->dev, sg_dump_data, file_len,
	2376	+ GFP_KERNEL);
	2377	+ }
	2378	+ iwl_dump_ini_list_free(&dump_list);
	2379	+}
999	2380
1000	2381	const struct iwl_fw_dump_desc iwl_dump_desc_assert = {
1001	2382	.trig_desc = {
..	..	@@ -1006,81 +2387,181 @@
1006	2387
1007	2388	int iwl_fw_dbg_collect_desc(struct iwl_fw_runtime *fwrt,
1008	2389	const struct iwl_fw_dump_desc *desc,
1009		- const struct iwl_fw_dbg_trigger_tlv *trigger)
	2390	+ bool monitor_only,
	2391	+ unsigned int delay)
1010	2392	{
1011		- unsigned int delay = 0;
	2393	+ struct iwl_fwrt_wk_data *wk_data;
	2394	+ unsigned long idx;
1012	2395
1013		- if (trigger)
1014		- delay = msecs_to_jiffies(le32_to_cpu(trigger->stop_delay));
	2396	+ if (iwl_trans_dbg_ini_valid(fwrt->trans)) {
	2397	+ iwl_fw_free_dump_desc(fwrt, desc);
	2398	+ return 0;
	2399	+ }
1015	2400
1016	2401	/*
1017		- * If the loading of the FW completed successfully, the next step is to
1018		- * get the SMEM config data. Thus, if fwrt->smem_cfg.num_lmacs is non
1019		- * zero, the FW was already loaded successully. If the state is "NO_FW"
1020		- * in such a case - exit, since FW may be dead. Otherwise, we
1021		- * can try to collect the data, since FW might just not be fully
1022		- * loaded (no "ALIVE" yet), and the debug data is accessible.
1023		- *
1024		- * Corner case: got the FW alive but crashed before getting the SMEM
1025		- * config. In such a case, due to HW access problems, we might
1026		- * collect garbage.
	2402	+ * Check there is an available worker.
	2403	+ * ffz return value is undefined if no zero exists,
	2404	+ * so check against ~0UL first.
1027	2405	*/
1028		- if (fwrt->trans->state == IWL_TRANS_NO_FW &&
1029		- fwrt->smem_cfg.num_lmacs)
1030		- return -EIO;
1031		-
1032		- if (test_and_set_bit(IWL_FWRT_STATUS_DUMPING, &fwrt->status))
	2406	+ if (fwrt->dump.active_wks == ~0UL)
1033	2407	return -EBUSY;
1034	2408
1035		- if (WARN_ON(fwrt->dump.desc))
1036		- iwl_fw_free_dump_desc(fwrt);
	2409	+ idx = ffz(fwrt->dump.active_wks);
	2410	+
	2411	+ if (idx >= IWL_FW_RUNTIME_DUMP_WK_NUM \|\|
	2412	+ test_and_set_bit(fwrt->dump.wks[idx].idx, &fwrt->dump.active_wks))
	2413	+ return -EBUSY;
	2414	+
	2415	+ wk_data = &fwrt->dump.wks[idx];
	2416	+
	2417	+ if (WARN_ON(wk_data->dump_data.desc))
	2418	+ iwl_fw_free_dump_desc(fwrt, wk_data->dump_data.desc);
	2419	+
	2420	+ wk_data->dump_data.desc = desc;
	2421	+ wk_data->dump_data.monitor_only = monitor_only;
1037	2422
1038	2423	IWL_WARN(fwrt, "Collecting data: trigger %d fired.\n",
1039	2424	le32_to_cpu(desc->trig_desc.type));
1040	2425
1041		- fwrt->dump.desc = desc;
1042		- fwrt->dump.trig = trigger;
1043		-
1044		- schedule_delayed_work(&fwrt->dump.wk, delay);
	2426	+ schedule_delayed_work(&wk_data->wk, usecs_to_jiffies(delay));
1045	2427
1046	2428	return 0;
1047	2429	}
1048	2430	IWL_EXPORT_SYMBOL(iwl_fw_dbg_collect_desc);
1049	2431
	2432	+int iwl_fw_dbg_error_collect(struct iwl_fw_runtime *fwrt,
	2433	+ enum iwl_fw_dbg_trigger trig_type)
	2434	+{
	2435	+ if (!test_bit(STATUS_DEVICE_ENABLED, &fwrt->trans->status))
	2436	+ return -EIO;
	2437	+
	2438	+ if (iwl_trans_dbg_ini_valid(fwrt->trans)) {
	2439	+ if (trig_type != FW_DBG_TRIGGER_ALIVE_TIMEOUT)
	2440	+ return -EIO;
	2441	+
	2442	+ iwl_dbg_tlv_time_point(fwrt,
	2443	+ IWL_FW_INI_TIME_POINT_HOST_ALIVE_TIMEOUT,
	2444	+ NULL);
	2445	+ } else {
	2446	+ struct iwl_fw_dump_desc *iwl_dump_error_desc;
	2447	+ int ret;
	2448	+
	2449	+ iwl_dump_error_desc =
	2450	+ kmalloc(sizeof(*iwl_dump_error_desc), GFP_KERNEL);
	2451	+
	2452	+ if (!iwl_dump_error_desc)
	2453	+ return -ENOMEM;
	2454	+
	2455	+ iwl_dump_error_desc->trig_desc.type = cpu_to_le32(trig_type);
	2456	+ iwl_dump_error_desc->len = 0;
	2457	+
	2458	+ ret = iwl_fw_dbg_collect_desc(fwrt, iwl_dump_error_desc,
	2459	+ false, 0);
	2460	+ if (ret) {
	2461	+ kfree(iwl_dump_error_desc);
	2462	+ return ret;
	2463	+ }
	2464	+ }
	2465	+
	2466	+ iwl_trans_sync_nmi(fwrt->trans);
	2467	+
	2468	+ return 0;
	2469	+}
	2470	+IWL_EXPORT_SYMBOL(iwl_fw_dbg_error_collect);
	2471	+
1050	2472	int iwl_fw_dbg_collect(struct iwl_fw_runtime *fwrt,
1051	2473	enum iwl_fw_dbg_trigger trig,
1052	2474	const char *str, size_t len,
1053		- const struct iwl_fw_dbg_trigger_tlv *trigger)
	2475	+ struct iwl_fw_dbg_trigger_tlv *trigger)
1054	2476	{
1055	2477	struct iwl_fw_dump_desc *desc;
	2478	+ unsigned int delay = 0;
	2479	+ bool monitor_only = false;
1056	2480
1057		- if (trigger && trigger->flags & IWL_FW_DBG_FORCE_RESTART) {
1058		- IWL_WARN(fwrt, "Force restart: trigger %d fired.\n", trig);
1059		- iwl_force_nmi(fwrt->trans);
1060		- return 0;
	2481	+ if (trigger) {
	2482	+ u16 occurrences = le16_to_cpu(trigger->occurrences) - 1;
	2483	+
	2484	+ if (!le16_to_cpu(trigger->occurrences))
	2485	+ return 0;
	2486	+
	2487	+ if (trigger->flags & IWL_FW_DBG_FORCE_RESTART) {
	2488	+ IWL_WARN(fwrt, "Force restart: trigger %d fired.\n",
	2489	+ trig);
	2490	+ iwl_force_nmi(fwrt->trans);
	2491	+ return 0;
	2492	+ }
	2493	+
	2494	+ trigger->occurrences = cpu_to_le16(occurrences);
	2495	+ monitor_only = trigger->mode & IWL_FW_DBG_TRIGGER_MONITOR_ONLY;
	2496	+
	2497	+ /* convert msec to usec */
	2498	+ delay = le32_to_cpu(trigger->stop_delay) * USEC_PER_MSEC;
1061	2499	}
1062	2500
1063	2501	desc = kzalloc(sizeof(*desc) + len, GFP_ATOMIC);
1064	2502	if (!desc)
1065	2503	return -ENOMEM;
1066	2504
	2505	+
1067	2506	desc->len = len;
1068	2507	desc->trig_desc.type = cpu_to_le32(trig);
1069	2508	memcpy(desc->trig_desc.data, str, len);
1070	2509
1071		- return iwl_fw_dbg_collect_desc(fwrt, desc, trigger);
	2510	+ return iwl_fw_dbg_collect_desc(fwrt, desc, monitor_only, delay);
1072	2511	}
1073	2512	IWL_EXPORT_SYMBOL(iwl_fw_dbg_collect);
	2513	+
	2514	+int iwl_fw_dbg_ini_collect(struct iwl_fw_runtime *fwrt,
	2515	+ struct iwl_fwrt_dump_data *dump_data)
	2516	+{
	2517	+ struct iwl_fw_ini_trigger_tlv *trig = dump_data->trig;
	2518	+ enum iwl_fw_ini_time_point tp_id = le32_to_cpu(trig->time_point);
	2519	+ u32 occur, delay;
	2520	+ unsigned long idx;
	2521	+
	2522	+ if (!iwl_fw_ini_trigger_on(fwrt, trig)) {
	2523	+ IWL_WARN(fwrt, "WRT: Trigger %d is not active, aborting dump\n",
	2524	+ tp_id);
	2525	+ return -EINVAL;
	2526	+ }
	2527	+
	2528	+ delay = le32_to_cpu(trig->dump_delay);
	2529	+ occur = le32_to_cpu(trig->occurrences);
	2530	+ if (!occur)
	2531	+ return 0;
	2532	+
	2533	+ trig->occurrences = cpu_to_le32(--occur);
	2534	+
	2535	+ /* Check there is an available worker.
	2536	+ * ffz return value is undefined if no zero exists,
	2537	+ * so check against ~0UL first.
	2538	+ */
	2539	+ if (fwrt->dump.active_wks == ~0UL)
	2540	+ return -EBUSY;
	2541	+
	2542	+ idx = ffz(fwrt->dump.active_wks);
	2543	+
	2544	+ if (idx >= IWL_FW_RUNTIME_DUMP_WK_NUM \|\|
	2545	+ test_and_set_bit(fwrt->dump.wks[idx].idx, &fwrt->dump.active_wks))
	2546	+ return -EBUSY;
	2547	+
	2548	+ fwrt->dump.wks[idx].dump_data = *dump_data;
	2549	+
	2550	+ IWL_WARN(fwrt, "WRT: Collecting data: ini trigger %d fired.\n", tp_id);
	2551	+
	2552	+ schedule_delayed_work(&fwrt->dump.wks[idx].wk, usecs_to_jiffies(delay));
	2553	+
	2554	+ return 0;
	2555	+}
1074	2556
1075	2557	int iwl_fw_dbg_collect_trig(struct iwl_fw_runtime *fwrt,
1076	2558	struct iwl_fw_dbg_trigger_tlv *trigger,
1077	2559	const char *fmt, ...)
1078	2560	{
1079		- u16 occurrences = le16_to_cpu(trigger->occurrences);
1080	2561	int ret, len = 0;
1081	2562	char buf[64];
1082	2563
1083		- if (!occurrences)
	2564	+ if (iwl_trans_dbg_ini_valid(fwrt->trans))
1084	2565	return 0;
1085	2566
1086	2567	if (fmt) {
..	..	@@ -1105,7 +2586,6 @@
1105	2586	if (ret)
1106	2587	return ret;
1107	2588
1108		- trigger->occurrences = cpu_to_le16(occurrences - 1);
1109	2589	return 0;
1110	2590	}
1111	2591	IWL_EXPORT_SYMBOL(iwl_fw_dbg_collect_trig);
..	..	@@ -1116,29 +2596,26 @@
1116	2596	int ret;
1117	2597	int i;
1118	2598
1119		- if (WARN_ONCE(conf_id >= ARRAY_SIZE(fwrt->fw->dbg_conf_tlv),
	2599	+ if (WARN_ONCE(conf_id >= ARRAY_SIZE(fwrt->fw->dbg.conf_tlv),
1120	2600	"Invalid configuration %d\n", conf_id))
1121	2601	return -EINVAL;
1122	2602
1123	2603	/* EARLY START - firmware's configuration is hard coded */
1124		- if ((!fwrt->fw->dbg_conf_tlv[conf_id] \|\|
1125		- !fwrt->fw->dbg_conf_tlv[conf_id]->num_of_hcmds) &&
	2604	+ if ((!fwrt->fw->dbg.conf_tlv[conf_id] \|\|
	2605	+ !fwrt->fw->dbg.conf_tlv[conf_id]->num_of_hcmds) &&
1126	2606	conf_id == FW_DBG_START_FROM_ALIVE)
1127	2607	return 0;
1128	2608
1129		- if (!fwrt->fw->dbg_conf_tlv[conf_id])
	2609	+ if (!fwrt->fw->dbg.conf_tlv[conf_id])
1130	2610	return -EINVAL;
1131	2611
1132	2612	if (fwrt->dump.conf != FW_DBG_INVALID)
1133		- IWL_WARN(fwrt, "FW already configured (%d) - re-configuring\n",
	2613	+ IWL_INFO(fwrt, "FW already configured (%d) - re-configuring\n",
1134	2614	fwrt->dump.conf);
1135	2615
1136		- /* start default config marker cmd for syncing logs */
1137		- iwl_fw_trigger_timestamp(fwrt, 1);
1138		-
1139	2616	/* Send all HCMDs for configuring the FW debug */
1140		- ptr = (void *)&fwrt->fw->dbg_conf_tlv[conf_id]->hcmd;
1141		- for (i = 0; i < fwrt->fw->dbg_conf_tlv[conf_id]->num_of_hcmds; i++) {
	2617	+ ptr = (void *)&fwrt->fw->dbg.conf_tlv[conf_id]->hcmd;
	2618	+ for (i = 0; i < fwrt->fw->dbg.conf_tlv[conf_id]->num_of_hcmds; i++) {
1142	2619	struct iwl_fw_dbg_conf_hcmd cmd = (void )ptr;
1143	2620	struct iwl_host_cmd hcmd = {
1144	2621	.id = cmd->id,
..	..	@@ -1160,58 +2637,224 @@
1160	2637	}
1161	2638	IWL_EXPORT_SYMBOL(iwl_fw_start_dbg_conf);
1162	2639
	2640	+/* this function assumes dump_start was called beforehand and dump_end will be
	2641	+ * called afterwards
	2642	+ */
	2643	+static void iwl_fw_dbg_collect_sync(struct iwl_fw_runtime *fwrt, u8 wk_idx)
	2644	+{
	2645	+ struct iwl_fw_dbg_params params = {0};
	2646	+ struct iwl_fwrt_dump_data *dump_data =
	2647	+ &fwrt->dump.wks[wk_idx].dump_data;
	2648	+
	2649	+ if (!test_bit(wk_idx, &fwrt->dump.active_wks))
	2650	+ return;
	2651	+
	2652	+ if (!test_bit(STATUS_DEVICE_ENABLED, &fwrt->trans->status)) {
	2653	+ IWL_ERR(fwrt, "Device is not enabled - cannot dump error\n");
	2654	+ goto out;
	2655	+ }
	2656	+
	2657	+ /* there's no point in fw dump if the bus is dead */
	2658	+ if (test_bit(STATUS_TRANS_DEAD, &fwrt->trans->status)) {
	2659	+ IWL_ERR(fwrt, "Skip fw error dump since bus is dead\n");
	2660	+ goto out;
	2661	+ }
	2662	+
	2663	+ iwl_fw_dbg_stop_restart_recording(fwrt, &params, true);
	2664	+
	2665	+ IWL_DEBUG_FW_INFO(fwrt, "WRT: Data collection start\n");
	2666	+ if (iwl_trans_dbg_ini_valid(fwrt->trans))
	2667	+ iwl_fw_error_ini_dump(fwrt, &fwrt->dump.wks[wk_idx].dump_data);
	2668	+ else
	2669	+ iwl_fw_error_dump(fwrt, &fwrt->dump.wks[wk_idx].dump_data);
	2670	+ IWL_DEBUG_FW_INFO(fwrt, "WRT: Data collection done\n");
	2671	+
	2672	+ iwl_fw_dbg_stop_restart_recording(fwrt, &params, false);
	2673	+
	2674	+out:
	2675	+ if (iwl_trans_dbg_ini_valid(fwrt->trans)) {
	2676	+ iwl_fw_error_dump_data_free(dump_data);
	2677	+ } else {
	2678	+ iwl_fw_free_dump_desc(fwrt, dump_data->desc);
	2679	+ dump_data->desc = NULL;
	2680	+ }
	2681	+
	2682	+ clear_bit(wk_idx, &fwrt->dump.active_wks);
	2683	+}
	2684	+
1163	2685	void iwl_fw_error_dump_wk(struct work_struct *work)
1164	2686	{
	2687	+ struct iwl_fwrt_wk_data *wks =
	2688	+ container_of(work, typeof(*wks), wk.work);
1165	2689	struct iwl_fw_runtime *fwrt =
1166		- container_of(work, struct iwl_fw_runtime, dump.wk.work);
	2690	+ container_of(wks, typeof(*fwrt), dump.wks[wks->idx]);
1167	2691
	2692	+ /* assumes the op mode mutex is locked in dump_start since
	2693	+ * iwl_fw_dbg_collect_sync can't run in parallel
	2694	+ */
1168	2695	if (fwrt->ops && fwrt->ops->dump_start &&
1169	2696	fwrt->ops->dump_start(fwrt->ops_ctx))
1170	2697	return;
1171	2698
1172		- if (fwrt->ops && fwrt->ops->fw_running &&
1173		- !fwrt->ops->fw_running(fwrt->ops_ctx)) {
1174		- IWL_ERR(fwrt, "Firmware not running - cannot dump error\n");
1175		- iwl_fw_free_dump_desc(fwrt);
1176		- clear_bit(IWL_FWRT_STATUS_DUMPING, &fwrt->status);
1177		- goto out;
1178		- }
	2699	+ iwl_fw_dbg_collect_sync(fwrt, wks->idx);
1179	2700
1180		- if (fwrt->trans->cfg->device_family == IWL_DEVICE_FAMILY_7000) {
1181		- /* stop recording */
1182		- iwl_fw_dbg_stop_recording(fwrt);
1183		-
1184		- iwl_fw_error_dump(fwrt);
1185		-
1186		- /* start recording again if the firmware is not crashed */
1187		- if (!test_bit(STATUS_FW_ERROR, &fwrt->trans->status) &&
1188		- fwrt->fw->dbg_dest_tlv) {
1189		- iwl_clear_bits_prph(fwrt->trans,
1190		- MON_BUFF_SAMPLE_CTL, 0x100);
1191		- iwl_clear_bits_prph(fwrt->trans,
1192		- MON_BUFF_SAMPLE_CTL, 0x1);
1193		- iwl_set_bits_prph(fwrt->trans,
1194		- MON_BUFF_SAMPLE_CTL, 0x1);
1195		- }
1196		- } else {
1197		- u32 in_sample = iwl_read_prph(fwrt->trans, DBGC_IN_SAMPLE);
1198		- u32 out_ctrl = iwl_read_prph(fwrt->trans, DBGC_OUT_CTRL);
1199		-
1200		- iwl_fw_dbg_stop_recording(fwrt);
1201		- /* wait before we collect the data till the DBGC stop */
1202		- udelay(500);
1203		-
1204		- iwl_fw_error_dump(fwrt);
1205		-
1206		- /* start recording again if the firmware is not crashed */
1207		- if (!test_bit(STATUS_FW_ERROR, &fwrt->trans->status) &&
1208		- fwrt->fw->dbg_dest_tlv) {
1209		- iwl_write_prph(fwrt->trans, DBGC_IN_SAMPLE, in_sample);
1210		- iwl_write_prph(fwrt->trans, DBGC_OUT_CTRL, out_ctrl);
1211		- }
1212		- }
1213		-out:
1214	2701	if (fwrt->ops && fwrt->ops->dump_end)
1215	2702	fwrt->ops->dump_end(fwrt->ops_ctx);
1216	2703	}
1217	2704
	2705	+void iwl_fw_dbg_read_d3_debug_data(struct iwl_fw_runtime *fwrt)
	2706	+{
	2707	+ const struct iwl_cfg *cfg = fwrt->trans->cfg;
	2708	+
	2709	+ if (!iwl_fw_dbg_is_d3_debug_enabled(fwrt))
	2710	+ return;
	2711	+
	2712	+ if (!fwrt->dump.d3_debug_data) {
	2713	+ fwrt->dump.d3_debug_data = kmalloc(cfg->d3_debug_data_length,
	2714	+ GFP_KERNEL);
	2715	+ if (!fwrt->dump.d3_debug_data) {
	2716	+ IWL_ERR(fwrt,
	2717	+ "failed to allocate memory for D3 debug data\n");
	2718	+ return;
	2719	+ }
	2720	+ }
	2721	+
	2722	+ /* if the buffer holds previous debug data it is overwritten */
	2723	+ iwl_trans_read_mem_bytes(fwrt->trans, cfg->d3_debug_data_base_addr,
	2724	+ fwrt->dump.d3_debug_data,
	2725	+ cfg->d3_debug_data_length);
	2726	+}
	2727	+IWL_EXPORT_SYMBOL(iwl_fw_dbg_read_d3_debug_data);
	2728	+
	2729	+void iwl_fw_dbg_stop_sync(struct iwl_fw_runtime *fwrt)
	2730	+{
	2731	+ int i;
	2732	+
	2733	+ iwl_dbg_tlv_del_timers(fwrt->trans);
	2734	+ for (i = 0; i < IWL_FW_RUNTIME_DUMP_WK_NUM; i++)
	2735	+ iwl_fw_dbg_collect_sync(fwrt, i);
	2736	+
	2737	+ iwl_fw_dbg_stop_restart_recording(fwrt, NULL, true);
	2738	+}
	2739	+IWL_EXPORT_SYMBOL(iwl_fw_dbg_stop_sync);
	2740	+
	2741	+#define FSEQ_REG(x) { .addr = (x), .str = #x, }
	2742	+
	2743	+void iwl_fw_error_print_fseq_regs(struct iwl_fw_runtime *fwrt)
	2744	+{
	2745	+ struct iwl_trans *trans = fwrt->trans;
	2746	+ unsigned long flags;
	2747	+ int i;
	2748	+ struct {
	2749	+ u32 addr;
	2750	+ const char *str;
	2751	+ } fseq_regs[] = {
	2752	+ FSEQ_REG(FSEQ_ERROR_CODE),
	2753	+ FSEQ_REG(FSEQ_TOP_INIT_VERSION),
	2754	+ FSEQ_REG(FSEQ_CNVIO_INIT_VERSION),
	2755	+ FSEQ_REG(FSEQ_OTP_VERSION),
	2756	+ FSEQ_REG(FSEQ_TOP_CONTENT_VERSION),
	2757	+ FSEQ_REG(FSEQ_ALIVE_TOKEN),
	2758	+ FSEQ_REG(FSEQ_CNVI_ID),
	2759	+ FSEQ_REG(FSEQ_CNVR_ID),
	2760	+ FSEQ_REG(CNVI_AUX_MISC_CHIP),
	2761	+ FSEQ_REG(CNVR_AUX_MISC_CHIP),
	2762	+ FSEQ_REG(CNVR_SCU_SD_REGS_SD_REG_DIG_DCDC_VTRIM),
	2763	+ FSEQ_REG(CNVR_SCU_SD_REGS_SD_REG_ACTIVE_VDIG_MIRROR),
	2764	+ };
	2765	+
	2766	+ if (!iwl_trans_grab_nic_access(trans, &flags))
	2767	+ return;
	2768	+
	2769	+ IWL_ERR(fwrt, "Fseq Registers:\n");
	2770	+
	2771	+ for (i = 0; i < ARRAY_SIZE(fseq_regs); i++)
	2772	+ IWL_ERR(fwrt, "0x%08X \| %s\n",
	2773	+ iwl_read_prph_no_grab(trans, fseq_regs[i].addr),
	2774	+ fseq_regs[i].str);
	2775	+
	2776	+ iwl_trans_release_nic_access(trans, &flags);
	2777	+}
	2778	+IWL_EXPORT_SYMBOL(iwl_fw_error_print_fseq_regs);
	2779	+
	2780	+static int iwl_fw_dbg_suspend_resume_hcmd(struct iwl_trans *trans, bool suspend)
	2781	+{
	2782	+ struct iwl_dbg_suspend_resume_cmd cmd = {
	2783	+ .operation = suspend ?
	2784	+ cpu_to_le32(DBGC_SUSPEND_CMD) :
	2785	+ cpu_to_le32(DBGC_RESUME_CMD),
	2786	+ };
	2787	+ struct iwl_host_cmd hcmd = {
	2788	+ .id = WIDE_ID(DEBUG_GROUP, DBGC_SUSPEND_RESUME),
	2789	+ .data[0] = &cmd,
	2790	+ .len[0] = sizeof(cmd),
	2791	+ };
	2792	+
	2793	+ return iwl_trans_send_cmd(trans, &hcmd);
	2794	+}
	2795	+
	2796	+static void iwl_fw_dbg_stop_recording(struct iwl_trans *trans,
	2797	+ struct iwl_fw_dbg_params *params)
	2798	+{
	2799	+ if (trans->trans_cfg->device_family == IWL_DEVICE_FAMILY_7000) {
	2800	+ iwl_set_bits_prph(trans, MON_BUFF_SAMPLE_CTL, 0x100);
	2801	+ return;
	2802	+ }
	2803	+
	2804	+ if (params) {
	2805	+ params->in_sample = iwl_read_umac_prph(trans, DBGC_IN_SAMPLE);
	2806	+ params->out_ctrl = iwl_read_umac_prph(trans, DBGC_OUT_CTRL);
	2807	+ }
	2808	+
	2809	+ iwl_write_umac_prph(trans, DBGC_IN_SAMPLE, 0);
	2810	+ /* wait for the DBGC to finish writing the internal buffer to DRAM to
	2811	+ * avoid halting the HW while writing
	2812	+ */
	2813	+ usleep_range(700, 1000);
	2814	+ iwl_write_umac_prph(trans, DBGC_OUT_CTRL, 0);
	2815	+}
	2816	+
	2817	+static int iwl_fw_dbg_restart_recording(struct iwl_trans *trans,
	2818	+ struct iwl_fw_dbg_params *params)
	2819	+{
	2820	+ if (!params)
	2821	+ return -EIO;
	2822	+
	2823	+ if (trans->trans_cfg->device_family == IWL_DEVICE_FAMILY_7000) {
	2824	+ iwl_clear_bits_prph(trans, MON_BUFF_SAMPLE_CTL, 0x100);
	2825	+ iwl_clear_bits_prph(trans, MON_BUFF_SAMPLE_CTL, 0x1);
	2826	+ iwl_set_bits_prph(trans, MON_BUFF_SAMPLE_CTL, 0x1);
	2827	+ } else {
	2828	+ iwl_write_umac_prph(trans, DBGC_IN_SAMPLE, params->in_sample);
	2829	+ iwl_write_umac_prph(trans, DBGC_OUT_CTRL, params->out_ctrl);
	2830	+ }
	2831	+
	2832	+ return 0;
	2833	+}
	2834	+
	2835	+void iwl_fw_dbg_stop_restart_recording(struct iwl_fw_runtime *fwrt,
	2836	+ struct iwl_fw_dbg_params *params,
	2837	+ bool stop)
	2838	+{
	2839	+ int ret __maybe_unused = 0;
	2840	+
	2841	+ if (test_bit(STATUS_FW_ERROR, &fwrt->trans->status))
	2842	+ return;
	2843	+
	2844	+ if (fw_has_capa(&fwrt->fw->ucode_capa,
	2845	+ IWL_UCODE_TLV_CAPA_DBG_SUSPEND_RESUME_CMD_SUPP))
	2846	+ ret = iwl_fw_dbg_suspend_resume_hcmd(fwrt->trans, stop);
	2847	+ else if (stop)
	2848	+ iwl_fw_dbg_stop_recording(fwrt->trans, params);
	2849	+ else
	2850	+ ret = iwl_fw_dbg_restart_recording(fwrt->trans, params);
	2851	+#ifdef CONFIG_IWLWIFI_DEBUGFS
	2852	+ if (!ret) {
	2853	+ if (stop)
	2854	+ fwrt->trans->dbg.rec_on = false;
	2855	+ else
	2856	+ iwl_fw_set_dbg_rec_on(fwrt);
	2857	+ }
	2858	+#endif
	2859	+}
	2860	+IWL_EXPORT_SYMBOL(iwl_fw_dbg_stop_restart_recording);