~hc/RK356X_SDK_RELEASE.git

..	..	@@ -12,6 +12,10 @@
12	12	* Andrew Patterson <andrew.patterson@hp.com>
13	13	*/
14	14
	15	+#define pr_fmt(fmt) "AER: " fmt
	16	+#define dev_fmt pr_fmt
	17	+
	18	+#include <linux/bitops.h>
15	19	#include <linux/cper.h>
16	20	#include <linux/pci.h>
17	21	#include <linux/pci-acpi.h>
..	..	@@ -30,10 +34,10 @@
30	34	#include "../pci.h"
31	35	#include "portdrv.h"
32	36
33		-#define AER_ERROR_SOURCES_MAX 100
	37	+#define AER_ERROR_SOURCES_MAX 128
34	38
35	39	#define AER_MAX_TYPEOF_COR_ERRS 16 /* as per PCI_ERR_COR_STATUS */
36		-#define AER_MAX_TYPEOF_UNCOR_ERRS 26 /* as per PCI_ERR_UNCOR_STATUS*/
	40	+#define AER_MAX_TYPEOF_UNCOR_ERRS 27 /* as per PCI_ERR_UNCOR_STATUS*/
37	41
38	42	struct aer_err_source {
39	43	unsigned int status;
..	..	@@ -42,21 +46,7 @@
42	46
43	47	struct aer_rpc {
44	48	struct pci_dev rpd; / Root Port device */
45		- struct work_struct dpc_handler;
46		- struct aer_err_source e_sources[AER_ERROR_SOURCES_MAX];
47		- struct aer_err_info e_info;
48		- unsigned short prod_idx; /* Error Producer Index */
49		- unsigned short cons_idx; /* Error Consumer Index */
50		- int isr;
51		- spinlock_t e_lock; /*
52		- * Lock access to Error Status/ID Regs
53		- * and error producer/consumer index
54		- */
55		- struct mutex rpc_mutex; /*
56		- * only one thread could do
57		- * recovery on the same
58		- * root port hierarchy
59		- */
	49	+ DECLARE_KFIFO(aer_fifo, struct aer_err_source, AER_ERROR_SOURCES_MAX);
60	50	};
61	51
62	52	/* AER stats for the device */
..	..	@@ -111,7 +101,13 @@
111	101	#define ERR_COR_ID(d) (d & 0xffff)
112	102	#define ERR_UNCOR_ID(d) (d >> 16)
113	103
	104	+#define AER_ERR_STATUS_MASK (PCI_ERR_ROOT_UNCOR_RCV \| \
	105	+ PCI_ERR_ROOT_COR_RCV \| \
	106	+ PCI_ERR_ROOT_MULTI_COR_RCV \| \
	107	+ PCI_ERR_ROOT_MULTI_UNCOR_RCV)
	108	+
114	109	static int pcie_aer_disable;
	110	+static pci_ers_result_t aer_root_reset(struct pci_dev *dev);
115	111
116	112	void pci_no_aer(void)
117	113	{
..	..	@@ -131,7 +127,7 @@
131	127
132	128	static int ecrc_policy = ECRC_POLICY_DEFAULT;
133	129
134		-static const char *ecrc_policy_str[] = {
	130	+static const char * const ecrc_policy_str[] = {
135	131	[ECRC_POLICY_DEFAULT] = "bios",
136	132	[ECRC_POLICY_OFF] = "off",
137	133	[ECRC_POLICY_ON] = "on"
..	..	@@ -145,22 +141,18 @@
145	141	*/
146	142	static int enable_ecrc_checking(struct pci_dev *dev)
147	143	{
148		- int pos;
	144	+ int aer = dev->aer_cap;
149	145	u32 reg32;
150	146
151		- if (!pci_is_pcie(dev))
	147	+ if (!aer)
152	148	return -ENODEV;
153	149
154		- pos = dev->aer_cap;
155		- if (!pos)
156		- return -ENODEV;
157		-
158		- pci_read_config_dword(dev, pos + PCI_ERR_CAP, &reg32);
	150	+ pci_read_config_dword(dev, aer + PCI_ERR_CAP, &reg32);
159	151	if (reg32 & PCI_ERR_CAP_ECRC_GENC)
160	152	reg32 \|= PCI_ERR_CAP_ECRC_GENE;
161	153	if (reg32 & PCI_ERR_CAP_ECRC_CHKC)
162	154	reg32 \|= PCI_ERR_CAP_ECRC_CHKE;
163		- pci_write_config_dword(dev, pos + PCI_ERR_CAP, reg32);
	155	+ pci_write_config_dword(dev, aer + PCI_ERR_CAP, reg32);
164	156
165	157	return 0;
166	158	}
..	..	@@ -173,19 +165,15 @@
173	165	*/
174	166	static int disable_ecrc_checking(struct pci_dev *dev)
175	167	{
176		- int pos;
	168	+ int aer = dev->aer_cap;
177	169	u32 reg32;
178	170
179		- if (!pci_is_pcie(dev))
	171	+ if (!aer)
180	172	return -ENODEV;
181	173
182		- pos = dev->aer_cap;
183		- if (!pos)
184		- return -ENODEV;
185		-
186		- pci_read_config_dword(dev, pos + PCI_ERR_CAP, &reg32);
	174	+ pci_read_config_dword(dev, aer + PCI_ERR_CAP, &reg32);
187	175	reg32 &= ~(PCI_ERR_CAP_ECRC_GENE \| PCI_ERR_CAP_ECRC_CHKE);
188		- pci_write_config_dword(dev, pos + PCI_ERR_CAP, reg32);
	176	+ pci_write_config_dword(dev, aer + PCI_ERR_CAP, reg32);
189	177
190	178	return 0;
191	179	}
..	..	@@ -212,264 +200,198 @@
212	200
213	201	/**
214	202	* pcie_ecrc_get_policy - parse kernel command-line ecrc option
	203	+ * @str: ECRC policy from kernel command line to use
215	204	*/
216	205	void pcie_ecrc_get_policy(char *str)
217	206	{
218	207	int i;
219	208
220		- for (i = 0; i < ARRAY_SIZE(ecrc_policy_str); i++)
221		- if (!strncmp(str, ecrc_policy_str[i],
222		- strlen(ecrc_policy_str[i])))
223		- break;
224		- if (i >= ARRAY_SIZE(ecrc_policy_str))
	209	+ i = match_string(ecrc_policy_str, ARRAY_SIZE(ecrc_policy_str), str);
	210	+ if (i < 0)
225	211	return;
226	212
227	213	ecrc_policy = i;
228	214	}
229	215	#endif /* CONFIG_PCIE_ECRC */
230	216
231		-#ifdef CONFIG_ACPI_APEI
232		-static inline int hest_match_pci(struct acpi_hest_aer_common *p,
233		- struct pci_dev *pci)
234		-{
235		- return ACPI_HEST_SEGMENT(p->bus) == pci_domain_nr(pci->bus) &&
236		- ACPI_HEST_BUS(p->bus) == pci->bus->number &&
237		- p->device == PCI_SLOT(pci->devfn) &&
238		- p->function == PCI_FUNC(pci->devfn);
239		-}
240		-
241		-static inline bool hest_match_type(struct acpi_hest_header *hest_hdr,
242		- struct pci_dev *dev)
243		-{
244		- u16 hest_type = hest_hdr->type;
245		- u8 pcie_type = pci_pcie_type(dev);
246		-
247		- if ((hest_type == ACPI_HEST_TYPE_AER_ROOT_PORT &&
248		- pcie_type == PCI_EXP_TYPE_ROOT_PORT) \|\|
249		- (hest_type == ACPI_HEST_TYPE_AER_ENDPOINT &&
250		- pcie_type == PCI_EXP_TYPE_ENDPOINT) \|\|
251		- (hest_type == ACPI_HEST_TYPE_AER_BRIDGE &&
252		- (dev->class >> 16) == PCI_BASE_CLASS_BRIDGE))
253		- return true;
254		- return false;
255		-}
256		-
257		-struct aer_hest_parse_info {
258		- struct pci_dev *pci_dev;
259		- int firmware_first;
260		-};
261		-
262		-static int hest_source_is_pcie_aer(struct acpi_hest_header *hest_hdr)
263		-{
264		- if (hest_hdr->type == ACPI_HEST_TYPE_AER_ROOT_PORT \|\|
265		- hest_hdr->type == ACPI_HEST_TYPE_AER_ENDPOINT \|\|
266		- hest_hdr->type == ACPI_HEST_TYPE_AER_BRIDGE)
267		- return 1;
268		- return 0;
269		-}
270		-
271		-static int aer_hest_parse(struct acpi_hest_header hest_hdr, void data)
272		-{
273		- struct aer_hest_parse_info *info = data;
274		- struct acpi_hest_aer_common *p;
275		- int ff;
276		-
277		- if (!hest_source_is_pcie_aer(hest_hdr))
278		- return 0;
279		-
280		- p = (struct acpi_hest_aer_common *)(hest_hdr + 1);
281		- ff = !!(p->flags & ACPI_HEST_FIRMWARE_FIRST);
282		-
283		- /*
284		- * If no specific device is supplied, determine whether
285		- * FIRMWARE_FIRST is set for any PCIe device.
286		- */
287		- if (!info->pci_dev) {
288		- info->firmware_first \|= ff;
289		- return 0;
290		- }
291		-
292		- /* Otherwise, check the specific device */
293		- if (p->flags & ACPI_HEST_GLOBAL) {
294		- if (hest_match_type(hest_hdr, info->pci_dev))
295		- info->firmware_first = ff;
296		- } else
297		- if (hest_match_pci(p, info->pci_dev))
298		- info->firmware_first = ff;
299		-
300		- return 0;
301		-}
302		-
303		-static void aer_set_firmware_first(struct pci_dev *pci_dev)
304		-{
305		- int rc;
306		- struct aer_hest_parse_info info = {
307		- .pci_dev = pci_dev,
308		- .firmware_first = 0,
309		- };
310		-
311		- rc = apei_hest_parse(aer_hest_parse, &info);
312		-
313		- if (rc)
314		- pci_dev->__aer_firmware_first = 0;
315		- else
316		- pci_dev->__aer_firmware_first = info.firmware_first;
317		- pci_dev->__aer_firmware_first_valid = 1;
318		-}
319		-
320		-int pcie_aer_get_firmware_first(struct pci_dev *dev)
321		-{
322		- if (!pci_is_pcie(dev))
323		- return 0;
324		-
325		- if (pcie_ports_native)
326		- return 0;
327		-
328		- if (!dev->__aer_firmware_first_valid)
329		- aer_set_firmware_first(dev);
330		- return dev->__aer_firmware_first;
331		-}
332		-
333		-static bool aer_firmware_first;
334		-
335		-/**
336		- * aer_acpi_firmware_first - Check if APEI should control AER.
337		- */
338		-bool aer_acpi_firmware_first(void)
339		-{
340		- static bool parsed = false;
341		- struct aer_hest_parse_info info = {
342		- .pci_dev = NULL, /* Check all PCIe devices */
343		- .firmware_first = 0,
344		- };
345		-
346		- if (pcie_ports_native)
347		- return false;
348		-
349		- if (!parsed) {
350		- apei_hest_parse(aer_hest_parse, &info);
351		- aer_firmware_first = info.firmware_first;
352		- parsed = true;
353		- }
354		- return aer_firmware_first;
355		-}
356		-#endif
357		-
358	217	#define PCI_EXP_AER_FLAGS (PCI_EXP_DEVCTL_CERE \| PCI_EXP_DEVCTL_NFERE \| \
359	218	PCI_EXP_DEVCTL_FERE \| PCI_EXP_DEVCTL_URRE)
360	219
361		-int pci_enable_pcie_error_reporting(struct pci_dev *dev)
	220	+int pcie_aer_is_native(struct pci_dev *dev)
362	221	{
363		- if (pcie_aer_get_firmware_first(dev))
364		- return -EIO;
	222	+ struct pci_host_bridge *host = pci_find_host_bridge(dev->bus);
365	223
366	224	if (!dev->aer_cap)
	225	+ return 0;
	226	+
	227	+ return pcie_ports_native \|\| host->native_aer;
	228	+}
	229	+
	230	+int pci_enable_pcie_error_reporting(struct pci_dev *dev)
	231	+{
	232	+ int rc;
	233	+
	234	+ if (!pcie_aer_is_native(dev))
367	235	return -EIO;
368	236
369		- return pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_AER_FLAGS);
	237	+ rc = pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_AER_FLAGS);
	238	+ return pcibios_err_to_errno(rc);
370	239	}
371	240	EXPORT_SYMBOL_GPL(pci_enable_pcie_error_reporting);
372	241
373	242	int pci_disable_pcie_error_reporting(struct pci_dev *dev)
374	243	{
375		- if (pcie_aer_get_firmware_first(dev))
	244	+ int rc;
	245	+
	246	+ if (!pcie_aer_is_native(dev))
376	247	return -EIO;
377	248
378		- return pcie_capability_clear_word(dev, PCI_EXP_DEVCTL,
379		- PCI_EXP_AER_FLAGS);
	249	+ rc = pcie_capability_clear_word(dev, PCI_EXP_DEVCTL, PCI_EXP_AER_FLAGS);
	250	+ return pcibios_err_to_errno(rc);
380	251	}
381	252	EXPORT_SYMBOL_GPL(pci_disable_pcie_error_reporting);
382	253
383		-void pci_aer_clear_device_status(struct pci_dev *dev)
	254	+int pci_aer_clear_nonfatal_status(struct pci_dev *dev)
384	255	{
385		- u16 sta;
386		-
387		- pcie_capability_read_word(dev, PCI_EXP_DEVSTA, &sta);
388		- pcie_capability_write_word(dev, PCI_EXP_DEVSTA, sta);
389		-}
390		-
391		-int pci_cleanup_aer_uncorrect_error_status(struct pci_dev *dev)
392		-{
393		- int pos;
	256	+ int aer = dev->aer_cap;
394	257	u32 status, sev;
395	258
396		- pos = dev->aer_cap;
397		- if (!pos)
398		- return -EIO;
399		-
400		- if (pcie_aer_get_firmware_first(dev))
	259	+ if (!pcie_aer_is_native(dev))
401	260	return -EIO;
402	261
403	262	/* Clear status bits for ERR_NONFATAL errors only */
404		- pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, &status);
405		- pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_SEVER, &sev);
	263	+ pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, &status);
	264	+ pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_SEVER, &sev);
406	265	status &= ~sev;
407	266	if (status)
408		- pci_write_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, status);
	267	+ pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, status);
409	268
410	269	return 0;
411	270	}
412		-EXPORT_SYMBOL_GPL(pci_cleanup_aer_uncorrect_error_status);
	271	+EXPORT_SYMBOL_GPL(pci_aer_clear_nonfatal_status);
413	272
414	273	void pci_aer_clear_fatal_status(struct pci_dev *dev)
415	274	{
416		- int pos;
	275	+ int aer = dev->aer_cap;
417	276	u32 status, sev;
418	277
419		- pos = dev->aer_cap;
420		- if (!pos)
421		- return;
422		-
423		- if (pcie_aer_get_firmware_first(dev))
	278	+ if (!pcie_aer_is_native(dev))
424	279	return;
425	280
426	281	/* Clear status bits for ERR_FATAL errors only */
427		- pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, &status);
428		- pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_SEVER, &sev);
	282	+ pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, &status);
	283	+ pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_SEVER, &sev);
429	284	status &= sev;
430	285	if (status)
431		- pci_write_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, status);
	286	+ pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, status);
432	287	}
433	288
434		-int pci_cleanup_aer_error_status_regs(struct pci_dev *dev)
	289	+/**
	290	+ * pci_aer_raw_clear_status - Clear AER error registers.
	291	+ * @dev: the PCI device
	292	+ *
	293	+ * Clearing AER error status registers unconditionally, regardless of
	294	+ * whether they're owned by firmware or the OS.
	295	+ *
	296	+ * Returns 0 on success, or negative on failure.
	297	+ */
	298	+int pci_aer_raw_clear_status(struct pci_dev *dev)
435	299	{
436		- int pos;
	300	+ int aer = dev->aer_cap;
437	301	u32 status;
438	302	int port_type;
439	303
440		- if (!pci_is_pcie(dev))
441		- return -ENODEV;
442		-
443		- pos = dev->aer_cap;
444		- if (!pos)
445		- return -EIO;
446		-
447		- if (pcie_aer_get_firmware_first(dev))
	304	+ if (!aer)
448	305	return -EIO;
449	306
450	307	port_type = pci_pcie_type(dev);
451		- if (port_type == PCI_EXP_TYPE_ROOT_PORT) {
452		- pci_read_config_dword(dev, pos + PCI_ERR_ROOT_STATUS, &status);
453		- pci_write_config_dword(dev, pos + PCI_ERR_ROOT_STATUS, status);
	308	+ if (port_type == PCI_EXP_TYPE_ROOT_PORT \|\|
	309	+ port_type == PCI_EXP_TYPE_RC_EC) {
	310	+ pci_read_config_dword(dev, aer + PCI_ERR_ROOT_STATUS, &status);
	311	+ pci_write_config_dword(dev, aer + PCI_ERR_ROOT_STATUS, status);
454	312	}
455	313
456		- pci_read_config_dword(dev, pos + PCI_ERR_COR_STATUS, &status);
457		- pci_write_config_dword(dev, pos + PCI_ERR_COR_STATUS, status);
	314	+ pci_read_config_dword(dev, aer + PCI_ERR_COR_STATUS, &status);
	315	+ pci_write_config_dword(dev, aer + PCI_ERR_COR_STATUS, status);
458	316
459		- pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, &status);
460		- pci_write_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, status);
	317	+ pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, &status);
	318	+ pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, status);
461	319
462	320	return 0;
	321	+}
	322	+
	323	+int pci_aer_clear_status(struct pci_dev *dev)
	324	+{
	325	+ if (!pcie_aer_is_native(dev))
	326	+ return -EIO;
	327	+
	328	+ return pci_aer_raw_clear_status(dev);
	329	+}
	330	+
	331	+void pci_save_aer_state(struct pci_dev *dev)
	332	+{
	333	+ int aer = dev->aer_cap;
	334	+ struct pci_cap_saved_state *save_state;
	335	+ u32 *cap;
	336	+
	337	+ if (!aer)
	338	+ return;
	339	+
	340	+ save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_ERR);
	341	+ if (!save_state)
	342	+ return;
	343	+
	344	+ cap = &save_state->cap.data[0];
	345	+ pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_MASK, cap++);
	346	+ pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_SEVER, cap++);
	347	+ pci_read_config_dword(dev, aer + PCI_ERR_COR_MASK, cap++);
	348	+ pci_read_config_dword(dev, aer + PCI_ERR_CAP, cap++);
	349	+ if (pcie_cap_has_rtctl(dev))
	350	+ pci_read_config_dword(dev, aer + PCI_ERR_ROOT_COMMAND, cap++);
	351	+}
	352	+
	353	+void pci_restore_aer_state(struct pci_dev *dev)
	354	+{
	355	+ int aer = dev->aer_cap;
	356	+ struct pci_cap_saved_state *save_state;
	357	+ u32 *cap;
	358	+
	359	+ if (!aer)
	360	+ return;
	361	+
	362	+ save_state = pci_find_saved_ext_cap(dev, PCI_EXT_CAP_ID_ERR);
	363	+ if (!save_state)
	364	+ return;
	365	+
	366	+ cap = &save_state->cap.data[0];
	367	+ pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_MASK, *cap++);
	368	+ pci_write_config_dword(dev, aer + PCI_ERR_UNCOR_SEVER, *cap++);
	369	+ pci_write_config_dword(dev, aer + PCI_ERR_COR_MASK, *cap++);
	370	+ pci_write_config_dword(dev, aer + PCI_ERR_CAP, *cap++);
	371	+ if (pcie_cap_has_rtctl(dev))
	372	+ pci_write_config_dword(dev, aer + PCI_ERR_ROOT_COMMAND, *cap++);
463	373	}
464	374
465	375	void pci_aer_init(struct pci_dev *dev)
466	376	{
	377	+ int n;
	378	+
467	379	dev->aer_cap = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ERR);
	380	+ if (!dev->aer_cap)
	381	+ return;
468	382
469		- if (dev->aer_cap)
470		- dev->aer_stats = kzalloc(sizeof(struct aer_stats), GFP_KERNEL);
	383	+ dev->aer_stats = kzalloc(sizeof(struct aer_stats), GFP_KERNEL);
471	384
472		- pci_cleanup_aer_error_status_regs(dev);
	385	+ /*
	386	+ * We save/restore PCI_ERR_UNCOR_MASK, PCI_ERR_UNCOR_SEVER,
	387	+ * PCI_ERR_COR_MASK, and PCI_ERR_CAP. Root and Root Complex Event
	388	+ * Collectors also implement PCI_ERR_ROOT_COMMAND (PCIe r5.0, sec
	389	+ * 7.8.4).
	390	+ */
	391	+ n = pcie_cap_has_rtctl(dev) ? 5 : 4;
	392	+ pci_add_ext_cap_save_buffer(dev, PCI_EXT_CAP_ID_ERR, sizeof(u32) * n);
	393	+
	394	+ pci_aer_clear_status(dev);
473	395	}
474	396
475	397	void pci_aer_exit(struct pci_dev *dev)
..	..	@@ -528,7 +450,7 @@
528	450	"Transaction Layer"
529	451	};
530	452
531		-static const char *aer_correctable_error_string[AER_MAX_TYPEOF_COR_ERRS] = {
	453	+static const char *aer_correctable_error_string[] = {
532	454	"RxErr", /* Bit Position 0 */
533	455	NULL,
534	456	NULL,
..	..	@@ -545,9 +467,25 @@
545	467	"NonFatalErr", /* Bit Position 13 */
546	468	"CorrIntErr", /* Bit Position 14 */
547	469	"HeaderOF", /* Bit Position 15 */
	470	+ NULL, /* Bit Position 16 */
	471	+ NULL, /* Bit Position 17 */
	472	+ NULL, /* Bit Position 18 */
	473	+ NULL, /* Bit Position 19 */
	474	+ NULL, /* Bit Position 20 */
	475	+ NULL, /* Bit Position 21 */
	476	+ NULL, /* Bit Position 22 */
	477	+ NULL, /* Bit Position 23 */
	478	+ NULL, /* Bit Position 24 */
	479	+ NULL, /* Bit Position 25 */
	480	+ NULL, /* Bit Position 26 */
	481	+ NULL, /* Bit Position 27 */
	482	+ NULL, /* Bit Position 28 */
	483	+ NULL, /* Bit Position 29 */
	484	+ NULL, /* Bit Position 30 */
	485	+ NULL, /* Bit Position 31 */
548	486	};
549	487
550		-static const char *aer_uncorrectable_error_string[AER_MAX_TYPEOF_UNCOR_ERRS] = {
	488	+static const char *aer_uncorrectable_error_string[] = {
551	489	"Undefined", /* Bit Position 0 */
552	490	NULL,
553	491	NULL,
..	..	@@ -574,6 +512,12 @@
574	512	"BlockedTLP", /* Bit Position 23 */
575	513	"AtomicOpBlocked", /* Bit Position 24 */
576	514	"TLPBlockedErr", /* Bit Position 25 */
	515	+ "PoisonTLPBlocked", /* Bit Position 26 */
	516	+ NULL, /* Bit Position 27 */
	517	+ NULL, /* Bit Position 28 */
	518	+ NULL, /* Bit Position 29 */
	519	+ NULL, /* Bit Position 30 */
	520	+ NULL, /* Bit Position 31 */
577	521	};
578	522
579	523	static const char *aer_agent_string[] = {
..	..	@@ -594,7 +538,7 @@
594	538	struct pci_dev *pdev = to_pci_dev(dev); \
595	539	u64 *stats = pdev->aer_stats->stats_array; \
596	540	\
597		- for (i = 0; i < ARRAY_SIZE(strings_array); i++) { \
	541	+ for (i = 0; i < ARRAY_SIZE(pdev->aer_stats->stats_array); i++) {\
598	542	if (strings_array[i]) \
599	543	str += sprintf(str, "%s %llu\n", \
600	544	strings_array[i], stats[i]); \
..	..	@@ -657,7 +601,8 @@
657	601	if ((a == &dev_attr_aer_rootport_total_err_cor.attr \|\|
658	602	a == &dev_attr_aer_rootport_total_err_fatal.attr \|\|
659	603	a == &dev_attr_aer_rootport_total_err_nonfatal.attr) &&
660		- pci_pcie_type(pdev) != PCI_EXP_TYPE_ROOT_PORT)
	604	+ ((pci_pcie_type(pdev) != PCI_EXP_TYPE_ROOT_PORT) &&
	605	+ (pci_pcie_type(pdev) != PCI_EXP_TYPE_RC_EC)))
661	606	return 0;
662	607
663	608	return a->mode;
..	..	@@ -671,7 +616,8 @@
671	616	static void pci_dev_aer_stats_incr(struct pci_dev *pdev,
672	617	struct aer_err_info *info)
673	618	{
674		- int status, i, max = -1;
	619	+ unsigned long status = info->status & ~info->mask;
	620	+ int i, max = -1;
675	621	u64 *counter = NULL;
676	622	struct aer_stats *aer_stats = pdev->aer_stats;
677	623
..	..	@@ -696,10 +642,8 @@
696	642	break;
697	643	}
698	644
699		- status = (info->status & ~info->mask);
700		- for (i = 0; i < max; i++)
701		- if (status & (1 << i))
702		- counter[i]++;
	645	+ for_each_set_bit(i, &status, max)
	646	+ counter[i]++;
703	647	}
704	648
705	649	static void pci_rootport_aer_stats_incr(struct pci_dev *pdev,
..	..	@@ -731,27 +675,26 @@
731	675	static void __aer_print_error(struct pci_dev *dev,
732	676	struct aer_err_info *info)
733	677	{
734		- int i, status;
735		- const char *errmsg = NULL;
736		- status = (info->status & ~info->mask);
	678	+ const char **strings;
	679	+ unsigned long status = info->status & ~info->mask;
	680	+ const char level, errmsg;
	681	+ int i;
737	682
738		- for (i = 0; i < 32; i++) {
739		- if (!(status & (1 << i)))
740		- continue;
	683	+ if (info->severity == AER_CORRECTABLE) {
	684	+ strings = aer_correctable_error_string;
	685	+ level = KERN_WARNING;
	686	+ } else {
	687	+ strings = aer_uncorrectable_error_string;
	688	+ level = KERN_ERR;
	689	+ }
741	690
742		- if (info->severity == AER_CORRECTABLE)
743		- errmsg = i < ARRAY_SIZE(aer_correctable_error_string) ?
744		- aer_correctable_error_string[i] : NULL;
745		- else
746		- errmsg = i < ARRAY_SIZE(aer_uncorrectable_error_string) ?
747		- aer_uncorrectable_error_string[i] : NULL;
	691	+ for_each_set_bit(i, &status, 32) {
	692	+ errmsg = strings[i];
	693	+ if (!errmsg)
	694	+ errmsg = "Unknown Error Bit";
748	695
749		- if (errmsg)
750		- pci_err(dev, " [%2d] %-22s%s\n", i, errmsg,
	696	+ pci_printk(level, dev, " [%2d] %-22s%s\n", i, errmsg,
751	697	info->first_error == i ? " (First)" : "");
752		- else
753		- pci_err(dev, " [%2d] Unknown Error Bit%s\n",
754		- i, info->first_error == i ? " (First)" : "");
755	698	}
756	699	pci_dev_aer_stats_incr(dev, info);
757	700	}
..	..	@@ -760,6 +703,7 @@
760	703	{
761	704	int layer, agent;
762	705	int id = ((dev->bus->number << 8) \| dev->devfn);
	706	+ const char *level;
763	707
764	708	if (!info->status) {
765	709	pci_err(dev, "PCIe Bus Error: severity=%s, type=Inaccessible, (Unregistered Agent ID)\n",
..	..	@@ -770,13 +714,14 @@
770	714	layer = AER_GET_LAYER_ERROR(info->severity, info->status);
771	715	agent = AER_GET_AGENT(info->severity, info->status);
772	716
773		- pci_err(dev, "PCIe Bus Error: severity=%s, type=%s, (%s)\n",
774		- aer_error_severity_string[info->severity],
775		- aer_error_layer[layer], aer_agent_string[agent]);
	717	+ level = (info->severity == AER_CORRECTABLE) ? KERN_WARNING : KERN_ERR;
776	718
777		- pci_err(dev, " device [%04x:%04x] error status/mask=%08x/%08x\n",
778		- dev->vendor, dev->device,
779		- info->status, info->mask);
	719	+ pci_printk(level, dev, "PCIe Bus Error: severity=%s, type=%s, (%s)\n",
	720	+ aer_error_severity_string[info->severity],
	721	+ aer_error_layer[layer], aer_agent_string[agent]);
	722	+
	723	+ pci_printk(level, dev, " device [%04x:%04x] error status/mask=%08x/%08x\n",
	724	+ dev->vendor, dev->device, info->status, info->mask);
780	725
781	726	__aer_print_error(dev, info);
782	727
..	..	@@ -796,10 +741,11 @@
796	741	u8 bus = info->id >> 8;
797	742	u8 devfn = info->id & 0xff;
798	743
799		- pci_info(dev, "AER: %s%s error received: %04x:%02x:%02x.%d\n",
800		- info->multi_error_valid ? "Multiple " : "",
801		- aer_error_severity_string[info->severity],
802		- pci_domain_nr(dev->bus), bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
	744	+ pci_info(dev, "%s%s error received: %04x:%02x:%02x.%d\n",
	745	+ info->multi_error_valid ? "Multiple " : "",
	746	+ aer_error_severity_string[info->severity],
	747	+ pci_domain_nr(dev->bus), bus, PCI_SLOT(devfn),
	748	+ PCI_FUNC(devfn));
803	749	}
804	750
805	751	#ifdef CONFIG_ACPI_APEI_PCIEAER
..	..	@@ -880,7 +826,7 @@
880	826	*/
881	827	static bool is_error_source(struct pci_dev dev, struct aer_err_info e_info)
882	828	{
883		- int pos;
	829	+ int aer = dev->aer_cap;
884	830	u32 status, mask;
885	831	u16 reg16;
886	832
..	..	@@ -915,17 +861,16 @@
915	861	if (!(reg16 & PCI_EXP_AER_FLAGS))
916	862	return false;
917	863
918		- pos = dev->aer_cap;
919		- if (!pos)
	864	+ if (!aer)
920	865	return false;
921	866
922	867	/* Check if error is recorded */
923	868	if (e_info->severity == AER_CORRECTABLE) {
924		- pci_read_config_dword(dev, pos + PCI_ERR_COR_STATUS, &status);
925		- pci_read_config_dword(dev, pos + PCI_ERR_COR_MASK, &mask);
	869	+ pci_read_config_dword(dev, aer + PCI_ERR_COR_STATUS, &status);
	870	+ pci_read_config_dword(dev, aer + PCI_ERR_COR_MASK, &mask);
926	871	} else {
927		- pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS, &status);
928		- pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_MASK, &mask);
	872	+ pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS, &status);
	873	+ pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_MASK, &mask);
929	874	}
930	875	if (status & ~mask)
931	876	return true;
..	..	@@ -981,8 +926,7 @@
981	926	pci_walk_bus(parent->subordinate, find_device_iter, e_info);
982	927
983	928	if (!e_info->error_dev_num) {
984		- pci_printk(KERN_DEBUG, parent, "can't find device of ID%04x\n",
985		- e_info->id);
	929	+ pci_info(parent, "can't find device of ID%04x\n", e_info->id);
986	930	return false;
987	931	}
988	932	return true;
..	..	@@ -997,22 +941,22 @@
997	941	*/
998	942	static void handle_error_source(struct pci_dev dev, struct aer_err_info info)
999	943	{
1000		- int pos;
	944	+ int aer = dev->aer_cap;
1001	945
1002	946	if (info->severity == AER_CORRECTABLE) {
1003	947	/*
1004	948	* Correctable error does not need software intervention.
1005	949	* No need to go through error recovery process.
1006	950	*/
1007		- pos = dev->aer_cap;
1008		- if (pos)
1009		- pci_write_config_dword(dev, pos + PCI_ERR_COR_STATUS,
	951	+ if (aer)
	952	+ pci_write_config_dword(dev, aer + PCI_ERR_COR_STATUS,
1010	953	info->status);
1011		- pci_aer_clear_device_status(dev);
	954	+ if (pcie_aer_is_native(dev))
	955	+ pcie_clear_device_status(dev);
1012	956	} else if (info->severity == AER_NONFATAL)
1013		- pcie_do_nonfatal_recovery(dev);
	957	+ pcie_do_recovery(dev, pci_channel_io_normal, aer_root_reset);
1014	958	else if (info->severity == AER_FATAL)
1015		- pcie_do_fatal_recovery(dev, PCIE_PORT_SERVICE_AER);
	959	+ pcie_do_recovery(dev, pci_channel_io_frozen, aer_root_reset);
1016	960	pci_dev_put(dev);
1017	961	}
1018	962
..	..	@@ -1048,9 +992,11 @@
1048	992	}
1049	993	cper_print_aer(pdev, entry.severity, entry.regs);
1050	994	if (entry.severity == AER_NONFATAL)
1051		- pcie_do_nonfatal_recovery(pdev);
	995	+ pcie_do_recovery(pdev, pci_channel_io_normal,
	996	+ aer_root_reset);
1052	997	else if (entry.severity == AER_FATAL)
1053		- pcie_do_fatal_recovery(pdev, PCIE_PORT_SERVICE_AER);
	998	+ pcie_do_recovery(pdev, pci_channel_io_frozen,
	999	+ aer_root_reset);
1054	1000	pci_dev_put(pdev);
1055	1001	}
1056	1002	}
..	..	@@ -1066,7 +1012,6 @@
1066	1012	void aer_recover_queue(int domain, unsigned int bus, unsigned int devfn,
1067	1013	int severity, struct aer_capability_regs *aer_regs)
1068	1014	{
1069		- unsigned long flags;
1070	1015	struct aer_recover_entry entry = {
1071	1016	.bus = bus,
1072	1017	.devfn = devfn,
..	..	@@ -1075,13 +1020,12 @@
1075	1020	.regs = aer_regs,
1076	1021	};
1077	1022
1078		- spin_lock_irqsave(&aer_recover_ring_lock, flags);
1079		- if (kfifo_put(&aer_recover_ring, entry))
	1023	+ if (kfifo_in_spinlocked(&aer_recover_ring, &entry, 1,
	1024	+ &aer_recover_ring_lock))
1080	1025	schedule_work(&aer_recover_work);
1081	1026	else
1082	1027	pr_err("AER recover: Buffer overflow when recovering AER for %04x:%02x:%02x:%x\n",
1083	1028	domain, bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1084		- spin_unlock_irqrestore(&aer_recover_ring_lock, flags);
1085	1029	}
1086	1030	EXPORT_SYMBOL_GPL(aer_recover_queue);
1087	1031	#endif
..	..	@@ -1097,51 +1041,51 @@
1097	1041	*/
1098	1042	int aer_get_device_error_info(struct pci_dev dev, struct aer_err_info info)
1099	1043	{
1100		- int pos, temp;
	1044	+ int type = pci_pcie_type(dev);
	1045	+ int aer = dev->aer_cap;
	1046	+ int temp;
1101	1047
1102	1048	/* Must reset in this function */
1103	1049	info->status = 0;
1104	1050	info->tlp_header_valid = 0;
1105	1051
1106		- pos = dev->aer_cap;
1107		-
1108	1052	/* The device might not support AER */
1109		- if (!pos)
	1053	+ if (!aer)
1110	1054	return 0;
1111	1055
1112	1056	if (info->severity == AER_CORRECTABLE) {
1113		- pci_read_config_dword(dev, pos + PCI_ERR_COR_STATUS,
	1057	+ pci_read_config_dword(dev, aer + PCI_ERR_COR_STATUS,
1114	1058	&info->status);
1115		- pci_read_config_dword(dev, pos + PCI_ERR_COR_MASK,
	1059	+ pci_read_config_dword(dev, aer + PCI_ERR_COR_MASK,
1116	1060	&info->mask);
1117	1061	if (!(info->status & ~info->mask))
1118	1062	return 0;
1119		- } else if (pci_pcie_type(dev) == PCI_EXP_TYPE_ROOT_PORT \|\|
1120		- pci_pcie_type(dev) == PCI_EXP_TYPE_DOWNSTREAM \|\|
	1063	+ } else if (type == PCI_EXP_TYPE_ROOT_PORT \|\|
	1064	+ type == PCI_EXP_TYPE_DOWNSTREAM \|\|
1121	1065	info->severity == AER_NONFATAL) {
1122	1066
1123	1067	/* Link is still healthy for IO reads */
1124		- pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_STATUS,
	1068	+ pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_STATUS,
1125	1069	&info->status);
1126		- pci_read_config_dword(dev, pos + PCI_ERR_UNCOR_MASK,
	1070	+ pci_read_config_dword(dev, aer + PCI_ERR_UNCOR_MASK,
1127	1071	&info->mask);
1128	1072	if (!(info->status & ~info->mask))
1129	1073	return 0;
1130	1074
1131	1075	/* Get First Error Pointer */
1132		- pci_read_config_dword(dev, pos + PCI_ERR_CAP, &temp);
	1076	+ pci_read_config_dword(dev, aer + PCI_ERR_CAP, &temp);
1133	1077	info->first_error = PCI_ERR_CAP_FEP(temp);
1134	1078
1135	1079	if (info->status & AER_LOG_TLP_MASKS) {
1136	1080	info->tlp_header_valid = 1;
1137	1081	pci_read_config_dword(dev,
1138		- pos + PCI_ERR_HEADER_LOG, &info->tlp.dw0);
	1082	+ aer + PCI_ERR_HEADER_LOG, &info->tlp.dw0);
1139	1083	pci_read_config_dword(dev,
1140		- pos + PCI_ERR_HEADER_LOG + 4, &info->tlp.dw1);
	1084	+ aer + PCI_ERR_HEADER_LOG + 4, &info->tlp.dw1);
1141	1085	pci_read_config_dword(dev,
1142		- pos + PCI_ERR_HEADER_LOG + 8, &info->tlp.dw2);
	1086	+ aer + PCI_ERR_HEADER_LOG + 8, &info->tlp.dw2);
1143	1087	pci_read_config_dword(dev,
1144		- pos + PCI_ERR_HEADER_LOG + 12, &info->tlp.dw3);
	1088	+ aer + PCI_ERR_HEADER_LOG + 12, &info->tlp.dw3);
1145	1089	}
1146	1090	}
1147	1091
..	..	@@ -1172,7 +1116,7 @@
1172	1116	struct aer_err_source *e_src)
1173	1117	{
1174	1118	struct pci_dev *pdev = rpc->rpd;
1175		- struct aer_err_info *e_info = &rpc->e_info;
	1119	+ struct aer_err_info e_info;
1176	1120
1177	1121	pci_rootport_aer_stats_incr(pdev, e_src);
1178	1122
..	..	@@ -1181,83 +1125,58 @@
1181	1125	* uncorrectable error being logged. Report correctable error first.
1182	1126	*/
1183	1127	if (e_src->status & PCI_ERR_ROOT_COR_RCV) {
1184		- e_info->id = ERR_COR_ID(e_src->id);
1185		- e_info->severity = AER_CORRECTABLE;
	1128	+ e_info.id = ERR_COR_ID(e_src->id);
	1129	+ e_info.severity = AER_CORRECTABLE;
1186	1130
1187	1131	if (e_src->status & PCI_ERR_ROOT_MULTI_COR_RCV)
1188		- e_info->multi_error_valid = 1;
	1132	+ e_info.multi_error_valid = 1;
1189	1133	else
1190		- e_info->multi_error_valid = 0;
1191		- aer_print_port_info(pdev, e_info);
	1134	+ e_info.multi_error_valid = 0;
	1135	+ aer_print_port_info(pdev, &e_info);
1192	1136
1193		- if (find_source_device(pdev, e_info))
1194		- aer_process_err_devices(e_info);
	1137	+ if (find_source_device(pdev, &e_info))
	1138	+ aer_process_err_devices(&e_info);
1195	1139	}
1196	1140
1197	1141	if (e_src->status & PCI_ERR_ROOT_UNCOR_RCV) {
1198		- e_info->id = ERR_UNCOR_ID(e_src->id);
	1142	+ e_info.id = ERR_UNCOR_ID(e_src->id);
1199	1143
1200	1144	if (e_src->status & PCI_ERR_ROOT_FATAL_RCV)
1201		- e_info->severity = AER_FATAL;
	1145	+ e_info.severity = AER_FATAL;
1202	1146	else
1203		- e_info->severity = AER_NONFATAL;
	1147	+ e_info.severity = AER_NONFATAL;
1204	1148
1205	1149	if (e_src->status & PCI_ERR_ROOT_MULTI_UNCOR_RCV)
1206		- e_info->multi_error_valid = 1;
	1150	+ e_info.multi_error_valid = 1;
1207	1151	else
1208		- e_info->multi_error_valid = 0;
	1152	+ e_info.multi_error_valid = 0;
1209	1153
1210		- aer_print_port_info(pdev, e_info);
	1154	+ aer_print_port_info(pdev, &e_info);
1211	1155
1212		- if (find_source_device(pdev, e_info))
1213		- aer_process_err_devices(e_info);
	1156	+ if (find_source_device(pdev, &e_info))
	1157	+ aer_process_err_devices(&e_info);
1214	1158	}
1215		-}
1216		-
1217		-/**
1218		- * get_e_source - retrieve an error source
1219		- * @rpc: pointer to the root port which holds an error
1220		- * @e_src: pointer to store retrieved error source
1221		- *
1222		- * Return 1 if an error source is retrieved, otherwise 0.
1223		- *
1224		- * Invoked by DPC handler to consume an error.
1225		- */
1226		-static int get_e_source(struct aer_rpc rpc, struct aer_err_source e_src)
1227		-{
1228		- unsigned long flags;
1229		-
1230		- /* Lock access to Root error producer/consumer index */
1231		- spin_lock_irqsave(&rpc->e_lock, flags);
1232		- if (rpc->prod_idx == rpc->cons_idx) {
1233		- spin_unlock_irqrestore(&rpc->e_lock, flags);
1234		- return 0;
1235		- }
1236		-
1237		- *e_src = rpc->e_sources[rpc->cons_idx];
1238		- rpc->cons_idx++;
1239		- if (rpc->cons_idx == AER_ERROR_SOURCES_MAX)
1240		- rpc->cons_idx = 0;
1241		- spin_unlock_irqrestore(&rpc->e_lock, flags);
1242		-
1243		- return 1;
1244	1159	}
1245	1160
1246	1161	/**
1247	1162	* aer_isr - consume errors detected by root port
1248		- * @work: definition of this work item
	1163	+ * @irq: IRQ assigned to Root Port
	1164	+ * @context: pointer to Root Port data structure
1249	1165	*
1250	1166	* Invoked, as DPC, when root port records new detected error
1251	1167	*/
1252		-static void aer_isr(struct work_struct *work)
	1168	+static irqreturn_t aer_isr(int irq, void *context)
1253	1169	{
1254		- struct aer_rpc *rpc = container_of(work, struct aer_rpc, dpc_handler);
1255		- struct aer_err_source uninitialized_var(e_src);
	1170	+ struct pcie_device dev = (struct pcie_device )context;
	1171	+ struct aer_rpc *rpc = get_service_data(dev);
	1172	+ struct aer_err_source e_src;
1256	1173
1257		- mutex_lock(&rpc->rpc_mutex);
1258		- while (get_e_source(rpc, &e_src))
	1174	+ if (kfifo_is_empty(&rpc->aer_fifo))
	1175	+ return IRQ_NONE;
	1176	+
	1177	+ while (kfifo_get(&rpc->aer_fifo, &e_src))
1259	1178	aer_isr_one_error(rpc, &e_src);
1260		- mutex_unlock(&rpc->rpc_mutex);
	1179	+ return IRQ_HANDLED;
1261	1180	}
1262	1181
1263	1182	/**
..	..	@@ -1267,56 +1186,26 @@
1267	1186	*
1268	1187	* Invoked when Root Port detects AER messages.
1269	1188	*/
1270		-irqreturn_t aer_irq(int irq, void *context)
	1189	+static irqreturn_t aer_irq(int irq, void *context)
1271	1190	{
1272		- unsigned int status, id;
1273	1191	struct pcie_device pdev = (struct pcie_device )context;
1274	1192	struct aer_rpc *rpc = get_service_data(pdev);
1275		- int next_prod_idx;
1276		- unsigned long flags;
1277		- int pos;
	1193	+ struct pci_dev *rp = rpc->rpd;
	1194	+ int aer = rp->aer_cap;
	1195	+ struct aer_err_source e_src = {};
1278	1196
1279		- pos = pdev->port->aer_cap;
1280		- /*
1281		- * Must lock access to Root Error Status Reg, Root Error ID Reg,
1282		- * and Root error producer/consumer index
1283		- */
1284		- spin_lock_irqsave(&rpc->e_lock, flags);
1285		-
1286		- /* Read error status */
1287		- pci_read_config_dword(pdev->port, pos + PCI_ERR_ROOT_STATUS, &status);
1288		- if (!(status & (PCI_ERR_ROOT_UNCOR_RCV\|PCI_ERR_ROOT_COR_RCV))) {
1289		- spin_unlock_irqrestore(&rpc->e_lock, flags);
	1197	+ pci_read_config_dword(rp, aer + PCI_ERR_ROOT_STATUS, &e_src.status);
	1198	+ if (!(e_src.status & AER_ERR_STATUS_MASK))
1290	1199	return IRQ_NONE;
1291		- }
1292	1200
1293		- /* Read error source and clear error status */
1294		- pci_read_config_dword(pdev->port, pos + PCI_ERR_ROOT_ERR_SRC, &id);
1295		- pci_write_config_dword(pdev->port, pos + PCI_ERR_ROOT_STATUS, status);
	1201	+ pci_read_config_dword(rp, aer + PCI_ERR_ROOT_ERR_SRC, &e_src.id);
	1202	+ pci_write_config_dword(rp, aer + PCI_ERR_ROOT_STATUS, e_src.status);
1296	1203
1297		- /* Store error source for later DPC handler */
1298		- next_prod_idx = rpc->prod_idx + 1;
1299		- if (next_prod_idx == AER_ERROR_SOURCES_MAX)
1300		- next_prod_idx = 0;
1301		- if (next_prod_idx == rpc->cons_idx) {
1302		- /*
1303		- * Error Storm Condition - possibly the same error occurred.
1304		- * Drop the error.
1305		- */
1306		- spin_unlock_irqrestore(&rpc->e_lock, flags);
	1204	+ if (!kfifo_put(&rpc->aer_fifo, e_src))
1307	1205	return IRQ_HANDLED;
1308		- }
1309		- rpc->e_sources[rpc->prod_idx].status = status;
1310		- rpc->e_sources[rpc->prod_idx].id = id;
1311		- rpc->prod_idx = next_prod_idx;
1312		- spin_unlock_irqrestore(&rpc->e_lock, flags);
1313	1206
1314		- /* Invoke DPC handler */
1315		- schedule_work(&rpc->dpc_handler);
1316		-
1317		- return IRQ_HANDLED;
	1207	+ return IRQ_WAKE_THREAD;
1318	1208	}
1319		-EXPORT_SYMBOL_GPL(aer_irq);
1320	1209
1321	1210	static int set_device_error_reporting(struct pci_dev dev, void data)
1322	1211	{
..	..	@@ -1324,6 +1213,7 @@
1324	1213	int type = pci_pcie_type(dev);
1325	1214
1326	1215	if ((type == PCI_EXP_TYPE_ROOT_PORT) \|\|
	1216	+ (type == PCI_EXP_TYPE_RC_EC) \|\|
1327	1217	(type == PCI_EXP_TYPE_UPSTREAM) \|\|
1328	1218	(type == PCI_EXP_TYPE_DOWNSTREAM)) {
1329	1219	if (enable)
..	..	@@ -1362,7 +1252,7 @@
1362	1252	static void aer_enable_rootport(struct aer_rpc *rpc)
1363	1253	{
1364	1254	struct pci_dev *pdev = rpc->rpd;
1365		- int aer_pos;
	1255	+ int aer = pdev->aer_cap;
1366	1256	u16 reg16;
1367	1257	u32 reg32;
1368	1258
..	..	@@ -1374,14 +1264,13 @@
1374	1264	pcie_capability_clear_word(pdev, PCI_EXP_RTCTL,
1375	1265	SYSTEM_ERROR_INTR_ON_MESG_MASK);
1376	1266
1377		- aer_pos = pdev->aer_cap;
1378	1267	/* Clear error status */
1379		- pci_read_config_dword(pdev, aer_pos + PCI_ERR_ROOT_STATUS, &reg32);
1380		- pci_write_config_dword(pdev, aer_pos + PCI_ERR_ROOT_STATUS, reg32);
1381		- pci_read_config_dword(pdev, aer_pos + PCI_ERR_COR_STATUS, &reg32);
1382		- pci_write_config_dword(pdev, aer_pos + PCI_ERR_COR_STATUS, reg32);
1383		- pci_read_config_dword(pdev, aer_pos + PCI_ERR_UNCOR_STATUS, &reg32);
1384		- pci_write_config_dword(pdev, aer_pos + PCI_ERR_UNCOR_STATUS, reg32);
	1268	+ pci_read_config_dword(pdev, aer + PCI_ERR_ROOT_STATUS, &reg32);
	1269	+ pci_write_config_dword(pdev, aer + PCI_ERR_ROOT_STATUS, reg32);
	1270	+ pci_read_config_dword(pdev, aer + PCI_ERR_COR_STATUS, &reg32);
	1271	+ pci_write_config_dword(pdev, aer + PCI_ERR_COR_STATUS, reg32);
	1272	+ pci_read_config_dword(pdev, aer + PCI_ERR_UNCOR_STATUS, &reg32);
	1273	+ pci_write_config_dword(pdev, aer + PCI_ERR_UNCOR_STATUS, reg32);
1385	1274
1386	1275	/*
1387	1276	* Enable error reporting for the root port device and downstream port
..	..	@@ -1390,9 +1279,9 @@
1390	1279	set_downstream_devices_error_reporting(pdev, true);
1391	1280
1392	1281	/* Enable Root Port's interrupt in response to error messages */
1393		- pci_read_config_dword(pdev, aer_pos + PCI_ERR_ROOT_COMMAND, &reg32);
	1282	+ pci_read_config_dword(pdev, aer + PCI_ERR_ROOT_COMMAND, &reg32);
1394	1283	reg32 \|= ROOT_PORT_INTR_ON_MESG_MASK;
1395		- pci_write_config_dword(pdev, aer_pos + PCI_ERR_ROOT_COMMAND, reg32);
	1284	+ pci_write_config_dword(pdev, aer + PCI_ERR_ROOT_COMMAND, reg32);
1396	1285	}
1397	1286
1398	1287	/**
..	..	@@ -1404,8 +1293,8 @@
1404	1293	static void aer_disable_rootport(struct aer_rpc *rpc)
1405	1294	{
1406	1295	struct pci_dev *pdev = rpc->rpd;
	1296	+ int aer = pdev->aer_cap;
1407	1297	u32 reg32;
1408		- int pos;
1409	1298
1410	1299	/*
1411	1300	* Disable error reporting for the root port device and downstream port
..	..	@@ -1413,42 +1302,14 @@
1413	1302	*/
1414	1303	set_downstream_devices_error_reporting(pdev, false);
1415	1304
1416		- pos = pdev->aer_cap;
1417	1305	/* Disable Root's interrupt in response to error messages */
1418		- pci_read_config_dword(pdev, pos + PCI_ERR_ROOT_COMMAND, &reg32);
	1306	+ pci_read_config_dword(pdev, aer + PCI_ERR_ROOT_COMMAND, &reg32);
1419	1307	reg32 &= ~ROOT_PORT_INTR_ON_MESG_MASK;
1420		- pci_write_config_dword(pdev, pos + PCI_ERR_ROOT_COMMAND, reg32);
	1308	+ pci_write_config_dword(pdev, aer + PCI_ERR_ROOT_COMMAND, reg32);
1421	1309
1422	1310	/* Clear Root's error status reg */
1423		- pci_read_config_dword(pdev, pos + PCI_ERR_ROOT_STATUS, &reg32);
1424		- pci_write_config_dword(pdev, pos + PCI_ERR_ROOT_STATUS, reg32);
1425		-}
1426		-
1427		-/**
1428		- * aer_alloc_rpc - allocate Root Port data structure
1429		- * @dev: pointer to the pcie_dev data structure
1430		- *
1431		- * Invoked when Root Port's AER service is loaded.
1432		- */
1433		-static struct aer_rpc aer_alloc_rpc(struct pcie_device dev)
1434		-{
1435		- struct aer_rpc *rpc;
1436		-
1437		- rpc = kzalloc(sizeof(struct aer_rpc), GFP_KERNEL);
1438		- if (!rpc)
1439		- return NULL;
1440		-
1441		- /* Initialize Root lock access, e_lock, to Root Error Status Reg */
1442		- spin_lock_init(&rpc->e_lock);
1443		-
1444		- rpc->rpd = dev->port;
1445		- INIT_WORK(&rpc->dpc_handler, aer_isr);
1446		- mutex_init(&rpc->rpc_mutex);
1447		-
1448		- /* Use PCIe bus function to store rpc into PCIe device */
1449		- set_service_data(dev, rpc);
1450		-
1451		- return rpc;
	1311	+ pci_read_config_dword(pdev, aer + PCI_ERR_ROOT_STATUS, &reg32);
	1312	+ pci_write_config_dword(pdev, aer + PCI_ERR_ROOT_STATUS, reg32);
1452	1313	}
1453	1314
1454	1315	/**
..	..	@@ -1461,16 +1322,7 @@
1461	1322	{
1462	1323	struct aer_rpc *rpc = get_service_data(dev);
1463	1324
1464		- if (rpc) {
1465		- /* If register interrupt service, it must be free. */
1466		- if (rpc->isr)
1467		- free_irq(dev->irq, dev);
1468		-
1469		- flush_work(&rpc->dpc_handler);
1470		- aer_disable_rootport(rpc);
1471		- kfree(rpc);
1472		- set_service_data(dev, NULL);
1473		- }
	1325	+ aer_disable_rootport(rpc);
1474	1326	}
1475	1327
1476	1328	/**
..	..	@@ -1483,87 +1335,98 @@
1483	1335	{
1484	1336	int status;
1485	1337	struct aer_rpc *rpc;
1486		- struct device *device = &dev->port->dev;
	1338	+ struct device *device = &dev->device;
	1339	+ struct pci_dev *port = dev->port;
1487	1340
1488		- /* Alloc rpc data structure */
1489		- rpc = aer_alloc_rpc(dev);
1490		- if (!rpc) {
1491		- dev_printk(KERN_DEBUG, device, "alloc AER rpc failed\n");
1492		- aer_remove(dev);
	1341	+ BUILD_BUG_ON(ARRAY_SIZE(aer_correctable_error_string) <
	1342	+ AER_MAX_TYPEOF_COR_ERRS);
	1343	+ BUILD_BUG_ON(ARRAY_SIZE(aer_uncorrectable_error_string) <
	1344	+ AER_MAX_TYPEOF_UNCOR_ERRS);
	1345	+
	1346	+ /* Limit to Root Ports or Root Complex Event Collectors */
	1347	+ if ((pci_pcie_type(port) != PCI_EXP_TYPE_RC_EC) &&
	1348	+ (pci_pcie_type(port) != PCI_EXP_TYPE_ROOT_PORT))
	1349	+ return -ENODEV;
	1350	+
	1351	+ rpc = devm_kzalloc(device, sizeof(struct aer_rpc), GFP_KERNEL);
	1352	+ if (!rpc)
1493	1353	return -ENOMEM;
1494		- }
1495	1354
1496		- /* Request IRQ ISR */
1497		- status = request_irq(dev->irq, aer_irq, IRQF_SHARED, "aerdrv", dev);
	1355	+ rpc->rpd = port;
	1356	+ INIT_KFIFO(rpc->aer_fifo);
	1357	+ set_service_data(dev, rpc);
	1358	+
	1359	+ status = devm_request_threaded_irq(device, dev->irq, aer_irq, aer_isr,
	1360	+ IRQF_SHARED, "aerdrv", dev);
1498	1361	if (status) {
1499		- dev_printk(KERN_DEBUG, device, "request AER IRQ %d failed\n",
1500		- dev->irq);
1501		- aer_remove(dev);
	1362	+ pci_err(port, "request AER IRQ %d failed\n", dev->irq);
1502	1363	return status;
1503	1364	}
1504	1365
1505		- rpc->isr = 1;
1506		-
1507	1366	aer_enable_rootport(rpc);
1508		- dev_info(device, "AER enabled with IRQ %d\n", dev->irq);
	1367	+ pci_info(port, "enabled with IRQ %d\n", dev->irq);
1509	1368	return 0;
1510	1369	}
1511	1370
1512	1371	/**
1513		- * aer_root_reset - reset link on Root Port
1514		- * @dev: pointer to Root Port's pci_dev data structure
	1372	+ * aer_root_reset - reset Root Port hierarchy or RCEC
	1373	+ * @dev: pointer to Root Port or RCEC
1515	1374	*
1516		- * Invoked by Port Bus driver when performing link reset at Root Port.
	1375	+ * Invoked by Port Bus driver when performing reset.
1517	1376	*/
1518	1377	static pci_ers_result_t aer_root_reset(struct pci_dev *dev)
1519	1378	{
	1379	+ int type = pci_pcie_type(dev);
	1380	+ struct pci_dev *root;
	1381	+ int aer;
	1382	+ struct pci_host_bridge *host = pci_find_host_bridge(dev->bus);
1520	1383	u32 reg32;
1521		- int pos;
1522	1384	int rc;
1523	1385
1524		- pos = dev->aer_cap;
	1386	+ root = dev; /* device with Root Error registers */
	1387	+ aer = root->aer_cap;
1525	1388
1526		- /* Disable Root's interrupt in response to error messages */
1527		- pci_read_config_dword(dev, pos + PCI_ERR_ROOT_COMMAND, &reg32);
1528		- reg32 &= ~ROOT_PORT_INTR_ON_MESG_MASK;
1529		- pci_write_config_dword(dev, pos + PCI_ERR_ROOT_COMMAND, reg32);
	1389	+ if ((host->native_aer \|\| pcie_ports_native) && aer) {
	1390	+ /* Disable Root's interrupt in response to error messages */
	1391	+ pci_read_config_dword(root, aer + PCI_ERR_ROOT_COMMAND, &reg32);
	1392	+ reg32 &= ~ROOT_PORT_INTR_ON_MESG_MASK;
	1393	+ pci_write_config_dword(root, aer + PCI_ERR_ROOT_COMMAND, reg32);
	1394	+ }
1530	1395
1531		- rc = pci_bus_error_reset(dev);
1532		- pci_printk(KERN_DEBUG, dev, "Root Port link has been reset\n");
	1396	+ if (type == PCI_EXP_TYPE_RC_EC) {
	1397	+ if (pcie_has_flr(dev)) {
	1398	+ rc = pcie_flr(dev);
	1399	+ pci_info(dev, "has been reset (%d)\n", rc);
	1400	+ } else {
	1401	+ pci_info(dev, "not reset (no FLR support)\n");
	1402	+ rc = -ENOTTY;
	1403	+ }
	1404	+ } else {
	1405	+ rc = pci_bus_error_reset(dev);
	1406	+ pci_info(dev, "Root Port link has been reset (%d)\n", rc);
	1407	+ }
1533	1408
1534		- /* Clear Root Error Status */
1535		- pci_read_config_dword(dev, pos + PCI_ERR_ROOT_STATUS, &reg32);
1536		- pci_write_config_dword(dev, pos + PCI_ERR_ROOT_STATUS, reg32);
	1409	+ if ((host->native_aer \|\| pcie_ports_native) && aer) {
	1410	+ /* Clear Root Error Status */
	1411	+ pci_read_config_dword(root, aer + PCI_ERR_ROOT_STATUS, &reg32);
	1412	+ pci_write_config_dword(root, aer + PCI_ERR_ROOT_STATUS, reg32);
1537	1413
1538		- /* Enable Root Port's interrupt in response to error messages */
1539		- pci_read_config_dword(dev, pos + PCI_ERR_ROOT_COMMAND, &reg32);
1540		- reg32 \|= ROOT_PORT_INTR_ON_MESG_MASK;
1541		- pci_write_config_dword(dev, pos + PCI_ERR_ROOT_COMMAND, reg32);
	1414	+ /* Enable Root Port's interrupt in response to error messages */
	1415	+ pci_read_config_dword(root, aer + PCI_ERR_ROOT_COMMAND, &reg32);
	1416	+ reg32 \|= ROOT_PORT_INTR_ON_MESG_MASK;
	1417	+ pci_write_config_dword(root, aer + PCI_ERR_ROOT_COMMAND, reg32);
	1418	+ }
1542	1419
1543	1420	return rc ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED;
1544	1421	}
1545	1422
1546		-/**
1547		- * aer_error_resume - clean up corresponding error status bits
1548		- * @dev: pointer to Root Port's pci_dev data structure
1549		- *
1550		- * Invoked by Port Bus driver during nonfatal recovery.
1551		- */
1552		-static void aer_error_resume(struct pci_dev *dev)
1553		-{
1554		- pci_aer_clear_device_status(dev);
1555		- pci_cleanup_aer_uncorrect_error_status(dev);
1556		-}
1557		-
1558	1423	static struct pcie_port_service_driver aerdriver = {
1559	1424	.name = "aer",
1560		- .port_type = PCI_EXP_TYPE_ROOT_PORT,
	1425	+ .port_type = PCIE_ANY_PORT,
1561	1426	.service = PCIE_PORT_SERVICE_AER,
1562	1427
1563	1428	.probe = aer_probe,
1564	1429	.remove = aer_remove,
1565		- .error_resume = aer_error_resume,
1566		- .reset_link = aer_root_reset,
1567	1430	};
1568	1431
1569	1432	/**
..	..	@@ -1573,7 +1436,7 @@
1573	1436	*/
1574	1437	int __init pcie_aer_init(void)
1575	1438	{
1576		- if (!pci_aer_available() \|\| aer_acpi_firmware_first())
	1439	+ if (!pci_aer_available())
1577	1440	return -ENXIO;
1578	1441	return pcie_port_service_register(&aerdriver);
1579	1442	}