~hc/RK356X_SDK_RELEASE.git

..	..	@@ -6,6 +6,8 @@
6	6	* Copyright (C) 2016 Intel Corp.
7	7	*/
8	8
	9	+#define dev_fmt(fmt) "DPC: " fmt
	10	+
9	11	#include <linux/aer.h>
10	12	#include <linux/delay.h>
11	13	#include <linux/interrupt.h>
..	..	@@ -14,13 +16,6 @@
14	16
15	17	#include "portdrv.h"
16	18	#include "../pci.h"
17		-
18		-struct dpc_dev {
19		- struct pcie_device *dev;
20		- u16 cap_pos;
21		- bool rp_extensions;
22		- u8 rp_log_size;
23		-};
24	19
25	20	static const char * const rp_pio_error_string[] = {
26	21	"Configuration Request received UR Completion", /* Bit Position 0 */
..	..	@@ -44,12 +39,94 @@
44	39	"Memory Request Completion Timeout", /* Bit Position 18 */
45	40	};
46	41
47		-static int dpc_wait_rp_inactive(struct dpc_dev *dpc)
	42	+void pci_save_dpc_state(struct pci_dev *dev)
	43	+{
	44	+ struct pci_cap_saved_state *save_state;
	45	+ u16 *cap;
	46	+
	47	+ if (!pci_is_pcie(dev))
	48	+ return;
	49	+
	50	+ save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_DPC);
	51	+ if (!save_state)
	52	+ return;
	53	+
	54	+ cap = (u16 *)&save_state->cap.data[0];
	55	+ pci_read_config_word(dev, dev->dpc_cap + PCI_EXP_DPC_CTL, cap);
	56	+}
	57	+
	58	+void pci_restore_dpc_state(struct pci_dev *dev)
	59	+{
	60	+ struct pci_cap_saved_state *save_state;
	61	+ u16 *cap;
	62	+
	63	+ if (!pci_is_pcie(dev))
	64	+ return;
	65	+
	66	+ save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_DPC);
	67	+ if (!save_state)
	68	+ return;
	69	+
	70	+ cap = (u16 *)&save_state->cap.data[0];
	71	+ pci_write_config_word(dev, dev->dpc_cap + PCI_EXP_DPC_CTL, *cap);
	72	+}
	73	+
	74	+static DECLARE_WAIT_QUEUE_HEAD(dpc_completed_waitqueue);
	75	+
	76	+#ifdef CONFIG_HOTPLUG_PCI_PCIE
	77	+static bool dpc_completed(struct pci_dev *pdev)
	78	+{
	79	+ u16 status;
	80	+
	81	+ pci_read_config_word(pdev, pdev->dpc_cap + PCI_EXP_DPC_STATUS, &status);
	82	+ if ((status != 0xffff) && (status & PCI_EXP_DPC_STATUS_TRIGGER))
	83	+ return false;
	84	+
	85	+ if (test_bit(PCI_DPC_RECOVERING, &pdev->priv_flags))
	86	+ return false;
	87	+
	88	+ return true;
	89	+}
	90	+
	91	+/**
	92	+ * pci_dpc_recovered - whether DPC triggered and has recovered successfully
	93	+ * @pdev: PCI device
	94	+ *
	95	+ * Return true if DPC was triggered for @pdev and has recovered successfully.
	96	+ * Wait for recovery if it hasn't completed yet. Called from the PCIe hotplug
	97	+ * driver to recognize and ignore Link Down/Up events caused by DPC.
	98	+ */
	99	+bool pci_dpc_recovered(struct pci_dev *pdev)
	100	+{
	101	+ struct pci_host_bridge *host;
	102	+
	103	+ if (!pdev->dpc_cap)
	104	+ return false;
	105	+
	106	+ /*
	107	+ * Synchronization between hotplug and DPC is not supported
	108	+ * if DPC is owned by firmware and EDR is not enabled.
	109	+ */
	110	+ host = pci_find_host_bridge(pdev->bus);
	111	+ if (!host->native_dpc && !IS_ENABLED(CONFIG_PCIE_EDR))
	112	+ return false;
	113	+
	114	+ /*
	115	+ * Need a timeout in case DPC never completes due to failure of
	116	+ * dpc_wait_rp_inactive(). The spec doesn't mandate a time limit,
	117	+ * but reports indicate that DPC completes within 4 seconds.
	118	+ */
	119	+ wait_event_timeout(dpc_completed_waitqueue, dpc_completed(pdev),
	120	+ msecs_to_jiffies(4000));
	121	+
	122	+ return test_and_clear_bit(PCI_DPC_RECOVERED, &pdev->priv_flags);
	123	+}
	124	+#endif /* CONFIG_HOTPLUG_PCI_PCIE */
	125	+
	126	+static int dpc_wait_rp_inactive(struct pci_dev *pdev)
48	127	{
49	128	unsigned long timeout = jiffies + HZ;
50		- struct pci_dev *pdev = dpc->dev->port;
51		- struct device *dev = &dpc->dev->device;
52		- u16 cap = dpc->cap_pos, status;
	129	+ u16 cap = pdev->dpc_cap, status;
53	130
54	131	pci_read_config_word(pdev, cap + PCI_EXP_DPC_STATUS, &status);
55	132	while (status & PCI_EXP_DPC_RP_BUSY &&
..	..	@@ -58,62 +135,70 @@
58	135	pci_read_config_word(pdev, cap + PCI_EXP_DPC_STATUS, &status);
59	136	}
60	137	if (status & PCI_EXP_DPC_RP_BUSY) {
61		- dev_warn(dev, "DPC root port still busy\n");
	138	+ pci_warn(pdev, "root port still busy\n");
62	139	return -EBUSY;
63	140	}
64	141	return 0;
65	142	}
66	143
67		-static pci_ers_result_t dpc_reset_link(struct pci_dev *pdev)
	144	+pci_ers_result_t dpc_reset_link(struct pci_dev *pdev)
68	145	{
69		- struct dpc_dev *dpc;
70		- struct pcie_device *pciedev;
71		- struct device *devdpc;
72		-
	146	+ pci_ers_result_t ret;
73	147	u16 cap;
	148	+
	149	+ set_bit(PCI_DPC_RECOVERING, &pdev->priv_flags);
74	150
75	151	/*
76	152	* DPC disables the Link automatically in hardware, so it has
77	153	* already been reset by the time we get here.
78	154	*/
79		- devdpc = pcie_port_find_device(pdev, PCIE_PORT_SERVICE_DPC);
80		- pciedev = to_pcie_device(devdpc);
81		- dpc = get_service_data(pciedev);
82		- cap = dpc->cap_pos;
	155	+ cap = pdev->dpc_cap;
83	156
84	157	/*
85	158	* Wait until the Link is inactive, then clear DPC Trigger Status
86	159	* to allow the Port to leave DPC.
87	160	*/
88		- pcie_wait_for_link(pdev, false);
	161	+ if (!pcie_wait_for_link(pdev, false))
	162	+ pci_info(pdev, "Data Link Layer Link Active not cleared in 1000 msec\n");
89	163
90		- if (dpc->rp_extensions && dpc_wait_rp_inactive(dpc))
91		- return PCI_ERS_RESULT_DISCONNECT;
	164	+ if (pdev->dpc_rp_extensions && dpc_wait_rp_inactive(pdev)) {
	165	+ clear_bit(PCI_DPC_RECOVERED, &pdev->priv_flags);
	166	+ ret = PCI_ERS_RESULT_DISCONNECT;
	167	+ goto out;
	168	+ }
92	169
93	170	pci_write_config_word(pdev, cap + PCI_EXP_DPC_STATUS,
94	171	PCI_EXP_DPC_STATUS_TRIGGER);
95	172
96		- return PCI_ERS_RESULT_RECOVERED;
	173	+ if (pci_bridge_wait_for_secondary_bus(pdev, "DPC",
	174	+ PCIE_RESET_READY_POLL_MS)) {
	175	+ clear_bit(PCI_DPC_RECOVERED, &pdev->priv_flags);
	176	+ ret = PCI_ERS_RESULT_DISCONNECT;
	177	+ } else {
	178	+ set_bit(PCI_DPC_RECOVERED, &pdev->priv_flags);
	179	+ ret = PCI_ERS_RESULT_RECOVERED;
	180	+ }
	181	+out:
	182	+ clear_bit(PCI_DPC_RECOVERING, &pdev->priv_flags);
	183	+ wake_up_all(&dpc_completed_waitqueue);
	184	+ return ret;
97	185	}
98	186
99		-
100		-static void dpc_process_rp_pio_error(struct dpc_dev *dpc)
	187	+static void dpc_process_rp_pio_error(struct pci_dev *pdev)
101	188	{
102		- struct device *dev = &dpc->dev->device;
103		- struct pci_dev *pdev = dpc->dev->port;
104		- u16 cap = dpc->cap_pos, dpc_status, first_error;
	189	+ u16 cap = pdev->dpc_cap, dpc_status, first_error;
105	190	u32 status, mask, sev, syserr, exc, dw0, dw1, dw2, dw3, log, prefix;
106	191	int i;
107	192
108	193	pci_read_config_dword(pdev, cap + PCI_EXP_DPC_RP_PIO_STATUS, &status);
109	194	pci_read_config_dword(pdev, cap + PCI_EXP_DPC_RP_PIO_MASK, &mask);
110		- dev_err(dev, "rp_pio_status: %#010x, rp_pio_mask: %#010x\n",
	195	+ pci_err(pdev, "rp_pio_status: %#010x, rp_pio_mask: %#010x\n",
111	196	status, mask);
112	197
113	198	pci_read_config_dword(pdev, cap + PCI_EXP_DPC_RP_PIO_SEVERITY, &sev);
114	199	pci_read_config_dword(pdev, cap + PCI_EXP_DPC_RP_PIO_SYSERROR, &syserr);
115	200	pci_read_config_dword(pdev, cap + PCI_EXP_DPC_RP_PIO_EXCEPTION, &exc);
116		- dev_err(dev, "RP PIO severity=%#010x, syserror=%#010x, exception=%#010x\n",
	201	+ pci_err(pdev, "RP PIO severity=%#010x, syserror=%#010x, exception=%#010x\n",
117	202	sev, syserr, exc);
118	203
119	204	/* Get First Error Pointer */
..	..	@@ -122,11 +207,11 @@
122	207
123	208	for (i = 0; i < ARRAY_SIZE(rp_pio_error_string); i++) {
124	209	if ((status & ~mask) & (1 << i))
125		- dev_err(dev, "[%2d] %s%s\n", i, rp_pio_error_string[i],
	210	+ pci_err(pdev, "[%2d] %s%s\n", i, rp_pio_error_string[i],
126	211	first_error == i ? " (First)" : "");
127	212	}
128	213
129		- if (dpc->rp_log_size < 4)
	214	+ if (pdev->dpc_rp_log_size < 4)
130	215	goto clear_status;
131	216	pci_read_config_dword(pdev, cap + PCI_EXP_DPC_RP_PIO_HEADER_LOG,
132	217	&dw0);
..	..	@@ -136,18 +221,18 @@
136	221	&dw2);
137	222	pci_read_config_dword(pdev, cap + PCI_EXP_DPC_RP_PIO_HEADER_LOG + 12,
138	223	&dw3);
139		- dev_err(dev, "TLP Header: %#010x %#010x %#010x %#010x\n",
	224	+ pci_err(pdev, "TLP Header: %#010x %#010x %#010x %#010x\n",
140	225	dw0, dw1, dw2, dw3);
141	226
142		- if (dpc->rp_log_size < 5)
	227	+ if (pdev->dpc_rp_log_size < 5)
143	228	goto clear_status;
144	229	pci_read_config_dword(pdev, cap + PCI_EXP_DPC_RP_PIO_IMPSPEC_LOG, &log);
145		- dev_err(dev, "RP PIO ImpSpec Log %#010x\n", log);
	230	+ pci_err(pdev, "RP PIO ImpSpec Log %#010x\n", log);
146	231
147		- for (i = 0; i < dpc->rp_log_size - 5; i++) {
	232	+ for (i = 0; i < pdev->dpc_rp_log_size - 5; i++) {
148	233	pci_read_config_dword(pdev,
149	234	cap + PCI_EXP_DPC_RP_PIO_TLPPREFIX_LOG, &prefix);
150		- dev_err(dev, "TLP Prefix Header: dw%d, %#010x\n", i, prefix);
	235	+ pci_err(pdev, "TLP Prefix Header: dw%d, %#010x\n", i, prefix);
151	236	}
152	237	clear_status:
153	238	pci_write_config_dword(pdev, cap + PCI_EXP_DPC_RP_PIO_STATUS, status);
..	..	@@ -175,23 +260,20 @@
175	260	return 1;
176	261	}
177	262
178		-static irqreturn_t dpc_handler(int irq, void *context)
	263	+void dpc_process_error(struct pci_dev *pdev)
179	264	{
	265	+ u16 cap = pdev->dpc_cap, status, source, reason, ext_reason;
180	266	struct aer_err_info info;
181		- struct dpc_dev *dpc = context;
182		- struct pci_dev *pdev = dpc->dev->port;
183		- struct device *dev = &dpc->dev->device;
184		- u16 cap = dpc->cap_pos, status, source, reason, ext_reason;
185	267
186	268	pci_read_config_word(pdev, cap + PCI_EXP_DPC_STATUS, &status);
187	269	pci_read_config_word(pdev, cap + PCI_EXP_DPC_SOURCE_ID, &source);
188	270
189		- dev_info(dev, "DPC containment event, status:%#06x source:%#06x\n",
	271	+ pci_info(pdev, "containment event, status:%#06x source:%#06x\n",
190	272	status, source);
191	273
192	274	reason = (status & PCI_EXP_DPC_STATUS_TRIGGER_RSN) >> 1;
193	275	ext_reason = (status & PCI_EXP_DPC_STATUS_TRIGGER_RSN_EXT) >> 5;
194		- dev_warn(dev, "DPC %s detected, remove downstream devices\n",
	276	+ pci_warn(pdev, "%s detected\n",
195	277	(reason == 0) ? "unmasked uncorrectable error" :
196	278	(reason == 1) ? "ERR_NONFATAL" :
197	279	(reason == 2) ? "ERR_FATAL" :
..	..	@@ -200,27 +282,33 @@
200	282	"reserved error");
201	283
202	284	/* show RP PIO error detail information */
203		- if (dpc->rp_extensions && reason == 3 && ext_reason == 0)
204		- dpc_process_rp_pio_error(dpc);
	285	+ if (pdev->dpc_rp_extensions && reason == 3 && ext_reason == 0)
	286	+ dpc_process_rp_pio_error(pdev);
205	287	else if (reason == 0 &&
206	288	dpc_get_aer_uncorrect_severity(pdev, &info) &&
207	289	aer_get_device_error_info(pdev, &info)) {
208	290	aer_print_error(pdev, &info);
209		- pci_cleanup_aer_uncorrect_error_status(pdev);
	291	+ pci_aer_clear_nonfatal_status(pdev);
210	292	pci_aer_clear_fatal_status(pdev);
211	293	}
	294	+}
	295	+
	296	+static irqreturn_t dpc_handler(int irq, void *context)
	297	+{
	298	+ struct pci_dev *pdev = context;
	299	+
	300	+ dpc_process_error(pdev);
212	301
213	302	/* We configure DPC so it only triggers on ERR_FATAL */
214		- pcie_do_fatal_recovery(pdev, PCIE_PORT_SERVICE_DPC);
	303	+ pcie_do_recovery(pdev, pci_channel_io_frozen, dpc_reset_link);
215	304
216	305	return IRQ_HANDLED;
217	306	}
218	307
219	308	static irqreturn_t dpc_irq(int irq, void *context)
220	309	{
221		- struct dpc_dev dpc = (struct dpc_dev )context;
222		- struct pci_dev *pdev = dpc->dev->port;
223		- u16 cap = dpc->cap_pos, status;
	310	+ struct pci_dev *pdev = context;
	311	+ u16 cap = pdev->dpc_cap, status;
224	312
225	313	pci_read_config_word(pdev, cap + PCI_EXP_DPC_STATUS, &status);
226	314
..	..	@@ -234,68 +322,72 @@
234	322	return IRQ_HANDLED;
235	323	}
236	324
	325	+void pci_dpc_init(struct pci_dev *pdev)
	326	+{
	327	+ u16 cap;
	328	+
	329	+ pdev->dpc_cap = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_DPC);
	330	+ if (!pdev->dpc_cap)
	331	+ return;
	332	+
	333	+ pci_read_config_word(pdev, pdev->dpc_cap + PCI_EXP_DPC_CAP, &cap);
	334	+ if (!(cap & PCI_EXP_DPC_CAP_RP_EXT))
	335	+ return;
	336	+
	337	+ pdev->dpc_rp_extensions = true;
	338	+ pdev->dpc_rp_log_size = (cap & PCI_EXP_DPC_RP_PIO_LOG_SIZE) >> 8;
	339	+ if (pdev->dpc_rp_log_size < 4 \|\| pdev->dpc_rp_log_size > 9) {
	340	+ pci_err(pdev, "RP PIO log size %u is invalid\n",
	341	+ pdev->dpc_rp_log_size);
	342	+ pdev->dpc_rp_log_size = 0;
	343	+ }
	344	+}
	345	+
237	346	#define FLAG(x, y) (((x) & (y)) ? '+' : '-')
238	347	static int dpc_probe(struct pcie_device *dev)
239	348	{
240		- struct dpc_dev *dpc;
241	349	struct pci_dev *pdev = dev->port;
242	350	struct device *device = &dev->device;
243	351	int status;
244	352	u16 ctl, cap;
245	353
246		- if (pcie_aer_get_firmware_first(pdev))
	354	+ if (!pcie_aer_is_native(pdev) && !pcie_ports_dpc_native)
247	355	return -ENOTSUPP;
248		-
249		- dpc = devm_kzalloc(device, sizeof(*dpc), GFP_KERNEL);
250		- if (!dpc)
251		- return -ENOMEM;
252		-
253		- dpc->cap_pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_DPC);
254		- dpc->dev = dev;
255		- set_service_data(dev, dpc);
256	356
257	357	status = devm_request_threaded_irq(device, dev->irq, dpc_irq,
258	358	dpc_handler, IRQF_SHARED,
259		- "pcie-dpc", dpc);
	359	+ "pcie-dpc", pdev);
260	360	if (status) {
261		- dev_warn(device, "request IRQ%d failed: %d\n", dev->irq,
	361	+ pci_warn(pdev, "request IRQ%d failed: %d\n", dev->irq,
262	362	status);
263	363	return status;
264	364	}
265	365
266		- pci_read_config_word(pdev, dpc->cap_pos + PCI_EXP_DPC_CAP, &cap);
267		- pci_read_config_word(pdev, dpc->cap_pos + PCI_EXP_DPC_CTL, &ctl);
268		-
269		- dpc->rp_extensions = (cap & PCI_EXP_DPC_CAP_RP_EXT);
270		- if (dpc->rp_extensions) {
271		- dpc->rp_log_size = (cap & PCI_EXP_DPC_RP_PIO_LOG_SIZE) >> 8;
272		- if (dpc->rp_log_size < 4 \|\| dpc->rp_log_size > 9) {
273		- dev_err(device, "RP PIO log size %u is invalid\n",
274		- dpc->rp_log_size);
275		- dpc->rp_log_size = 0;
276		- }
277		- }
	366	+ pci_read_config_word(pdev, pdev->dpc_cap + PCI_EXP_DPC_CAP, &cap);
	367	+ pci_read_config_word(pdev, pdev->dpc_cap + PCI_EXP_DPC_CTL, &ctl);
278	368
279	369	ctl = (ctl & 0xfff4) \| PCI_EXP_DPC_CTL_EN_FATAL \| PCI_EXP_DPC_CTL_INT_EN;
280		- pci_write_config_word(pdev, dpc->cap_pos + PCI_EXP_DPC_CTL, ctl);
	370	+ pci_write_config_word(pdev, pdev->dpc_cap + PCI_EXP_DPC_CTL, ctl);
	371	+ pci_info(pdev, "enabled with IRQ %d\n", dev->irq);
281	372
282		- dev_info(device, "DPC error containment capabilities: Int Msg #%d, RPExt%c PoisonedTLP%c SwTrigger%c RP PIO Log %d, DL_ActiveErr%c\n",
283		- cap & PCI_EXP_DPC_IRQ, FLAG(cap, PCI_EXP_DPC_CAP_RP_EXT),
284		- FLAG(cap, PCI_EXP_DPC_CAP_POISONED_TLP),
285		- FLAG(cap, PCI_EXP_DPC_CAP_SW_TRIGGER), dpc->rp_log_size,
286		- FLAG(cap, PCI_EXP_DPC_CAP_DL_ACTIVE));
	373	+ pci_info(pdev, "error containment capabilities: Int Msg #%d, RPExt%c PoisonedTLP%c SwTrigger%c RP PIO Log %d, DL_ActiveErr%c\n",
	374	+ cap & PCI_EXP_DPC_IRQ, FLAG(cap, PCI_EXP_DPC_CAP_RP_EXT),
	375	+ FLAG(cap, PCI_EXP_DPC_CAP_POISONED_TLP),
	376	+ FLAG(cap, PCI_EXP_DPC_CAP_SW_TRIGGER), pdev->dpc_rp_log_size,
	377	+ FLAG(cap, PCI_EXP_DPC_CAP_DL_ACTIVE));
	378	+
	379	+ pci_add_ext_cap_save_buffer(pdev, PCI_EXT_CAP_ID_DPC, sizeof(u16));
287	380	return status;
288	381	}
289	382
290	383	static void dpc_remove(struct pcie_device *dev)
291	384	{
292		- struct dpc_dev *dpc = get_service_data(dev);
293	385	struct pci_dev *pdev = dev->port;
294	386	u16 ctl;
295	387
296		- pci_read_config_word(pdev, dpc->cap_pos + PCI_EXP_DPC_CTL, &ctl);
	388	+ pci_read_config_word(pdev, pdev->dpc_cap + PCI_EXP_DPC_CTL, &ctl);
297	389	ctl &= ~(PCI_EXP_DPC_CTL_EN_FATAL \| PCI_EXP_DPC_CTL_INT_EN);
298		- pci_write_config_word(pdev, dpc->cap_pos + PCI_EXP_DPC_CTL, ctl);
	390	+ pci_write_config_word(pdev, pdev->dpc_cap + PCI_EXP_DPC_CTL, ctl);
299	391	}
300	392
301	393	static struct pcie_port_service_driver dpcdriver = {
..	..	@@ -304,7 +396,6 @@
304	396	.service = PCIE_PORT_SERVICE_DPC,
305	397	.probe = dpc_probe,
306	398	.remove = dpc_remove,
307		- .reset_link = dpc_reset_link,
308	399	};
309	400
310	401	int __init pcie_dpc_init(void)