forked from ~ljy/RK356X_SDK_RELEASE

blame | history | patch | commit | commitdiff
hc
2024-12-19 9370bb92b2d16684ee45cf24e879c93c509162da 
 kernel/drivers/pci/pcie/aspm.c
....@@ -18,7 +18,6 @@
1818 #include <linux/slab.h>
1919 #include <linux/jiffies.h>
2020 #include <linux/delay.h>
21
-#include <linux/pci-aspm.h>
2221 #include "../pci.h"
2322 
2423 #ifdef MODULE_PARAM_PREFIX
....@@ -53,8 +52,6 @@
5352 	struct pcie_link_state *root;	/* pointer to the root port link */
5453 	struct pcie_link_state *parent;	/* pointer to the parent Link state */
5554 	struct list_head sibling;	/* node in link_list */
56
-	struct list_head children;	/* list of child link states */
57
-	struct list_head link;		/* node in parent's children list */
5855 
5956 	/* ASPM state */
6057 	u32 aspm_support:7;		/* Supported ASPM state */
....@@ -77,14 +74,6 @@
7774 	 * has one slot under it, so at most there are 8 functions.
7875 	 */
7976 	struct aspm_latency acceptable[8];
80
-
81
-	/* L1 PM Substate info */
82
-	struct {
83
-		u32 up_cap_ptr;		/* L1SS cap ptr in upstream dev */
84
-		u32 dw_cap_ptr;		/* L1SS cap ptr in downstream dev */
85
-		u32 ctl1;		/* value to be programmed in ctl1 */
86
-		u32 ctl2;		/* value to be programmed in ctl2 */
87
-	} l1ss;
8877 };
8978 
9079 static int aspm_disabled, aspm_force;
....@@ -203,11 +192,38 @@
203192 	link->clkpm_disable = blacklist ? 1 : 0;
204193 }
205194 
206
-static bool pcie_retrain_link(struct pcie_link_state *link)
195
+static int pcie_wait_for_retrain(struct pci_dev *pdev)
196
+{
197
+	unsigned long end_jiffies;
198
+	u16 reg16;
199
+
200
+	/* Wait for Link Training to be cleared by hardware */
201
+	end_jiffies = jiffies + LINK_RETRAIN_TIMEOUT;
202
+	do {
203
+		pcie_capability_read_word(pdev, PCI_EXP_LNKSTA, &reg16);
204
+		if (!(reg16 & PCI_EXP_LNKSTA_LT))
205
+			return 0;
206
+		msleep(1);
207
+	} while (time_before(jiffies, end_jiffies));
208
+
209
+	return -ETIMEDOUT;
210
+}
211
+
212
+static int pcie_retrain_link(struct pcie_link_state *link)
207213 {
208214 	struct pci_dev *parent = link->pdev;
209
-	unsigned long start_jiffies;
215
+	int rc;
210216 	u16 reg16;
217
+
218
+	/*
219
+	 * Ensure the updated LNKCTL parameters are used during link
220
+	 * training by checking that there is no ongoing link training to
221
+	 * avoid LTSSM race as recommended in Implementation Note at the
222
+	 * end of PCIe r6.0.1 sec 7.5.3.7.
223
+	 */
224
+	rc = pcie_wait_for_retrain(parent);
225
+	if (rc)
226
+		return rc;
211227 
212228 	pcie_capability_read_word(parent, PCI_EXP_LNKCTL, &reg16);
213229 	reg16 |= PCI_EXP_LNKCTL_RL;
....@@ -222,17 +238,7 @@
222238 		pcie_capability_write_word(parent, PCI_EXP_LNKCTL, reg16);
223239 	}
224240 
225
-	/* Wait for link training end. Break out after waiting for timeout */
226
-	start_jiffies = jiffies;
227
-	for (;;) {
228
-		pcie_capability_read_word(parent, PCI_EXP_LNKSTA, &reg16);
229
-		if (!(reg16 & PCI_EXP_LNKSTA_LT))
230
-			break;
231
-		if (time_after(jiffies, start_jiffies + LINK_RETRAIN_TIMEOUT))
232
-			break;
233
-		msleep(1);
234
-	}
235
-	return !(reg16 & PCI_EXP_LNKSTA_LT);
241
+	return pcie_wait_for_retrain(parent);
236242 }
237243 
238244 /*
....@@ -243,7 +249,7 @@
243249 static void pcie_aspm_configure_common_clock(struct pcie_link_state *link)
244250 {
245251 	int same_clock = 1;
246
-	u16 reg16, parent_reg, child_reg[8];
252
+	u16 reg16, ccc, parent_old_ccc, child_old_ccc[8];
247253 	struct pci_dev *child, *parent = link->pdev;
248254 	struct pci_bus *linkbus = parent->subordinate;
249255 	/*
....@@ -265,6 +271,7 @@
265271 
266272 	/* Port might be already in common clock mode */
267273 	pcie_capability_read_word(parent, PCI_EXP_LNKCTL, &reg16);
274
+	parent_old_ccc = reg16 & PCI_EXP_LNKCTL_CCC;
268275 	if (same_clock && (reg16 & PCI_EXP_LNKCTL_CCC)) {
269276 		bool consistent = true;
270277 
....@@ -278,43 +285,40 @@
278285 		}
279286 		if (consistent)
280287 			return;
281
-		pci_warn(parent, "ASPM: current common clock configuration is broken, reconfiguring\n");
288
+		pci_info(parent, "ASPM: current common clock configuration is inconsistent, reconfiguring\n");
282289 	}
283290 
291
+	ccc = same_clock ? PCI_EXP_LNKCTL_CCC : 0;
284292 	/* Configure downstream component, all functions */
285293 	list_for_each_entry(child, &linkbus->devices, bus_list) {
286294 		pcie_capability_read_word(child, PCI_EXP_LNKCTL, &reg16);
287
-		child_reg[PCI_FUNC(child->devfn)] = reg16;
288
-		if (same_clock)
289
-			reg16 |= PCI_EXP_LNKCTL_CCC;
290
-		else
291
-			reg16 &= ~PCI_EXP_LNKCTL_CCC;
292
-		pcie_capability_write_word(child, PCI_EXP_LNKCTL, reg16);
295
+		child_old_ccc[PCI_FUNC(child->devfn)] = reg16 & PCI_EXP_LNKCTL_CCC;
296
+		pcie_capability_clear_and_set_word(child, PCI_EXP_LNKCTL,
297
+						   PCI_EXP_LNKCTL_CCC, ccc);
293298 	}
294299 
295300 	/* Configure upstream component */
296
-	pcie_capability_read_word(parent, PCI_EXP_LNKCTL, &reg16);
297
-	parent_reg = reg16;
298
-	if (same_clock)
299
-		reg16 |= PCI_EXP_LNKCTL_CCC;
300
-	else
301
-		reg16 &= ~PCI_EXP_LNKCTL_CCC;
302
-	pcie_capability_write_word(parent, PCI_EXP_LNKCTL, reg16);
301
+	pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL,
302
+					   PCI_EXP_LNKCTL_CCC, ccc);
303303 
304
-	if (pcie_retrain_link(link))
305
-		return;
304
+	if (pcie_retrain_link(link)) {
306305 
307
-	/* Training failed. Restore common clock configurations */
308
-	pci_err(parent, "ASPM: Could not configure common clock\n");
309
-	list_for_each_entry(child, &linkbus->devices, bus_list)
310
-		pcie_capability_write_word(child, PCI_EXP_LNKCTL,
311
-					   child_reg[PCI_FUNC(child->devfn)]);
312
-	pcie_capability_write_word(parent, PCI_EXP_LNKCTL, parent_reg);
306
+		/* Training failed. Restore common clock configurations */
307
+		pci_err(parent, "ASPM: Could not configure common clock\n");
308
+		list_for_each_entry(child, &linkbus->devices, bus_list)
309
+			pcie_capability_clear_and_set_word(child, PCI_EXP_LNKCTL,
310
+							   PCI_EXP_LNKCTL_CCC,
311
+							   child_old_ccc[PCI_FUNC(child->devfn)]);
312
+		pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL,
313
+						   PCI_EXP_LNKCTL_CCC, parent_old_ccc);
314
+	}
313315 }
314316 
315317 /* Convert L0s latency encoding to ns */
316
-static u32 calc_l0s_latency(u32 encoding)
318
+static u32 calc_l0s_latency(u32 lnkcap)
317319 {
320
+	u32 encoding = (lnkcap & PCI_EXP_LNKCAP_L0SEL) >> 12;
321
+
318322 	if (encoding == 0x7)
319323 		return (5 * 1000);	/* > 4us */
320324 	return (64 << encoding);
....@@ -329,8 +333,10 @@
329333 }
330334 
331335 /* Convert L1 latency encoding to ns */
332
-static u32 calc_l1_latency(u32 encoding)
336
+static u32 calc_l1_latency(u32 lnkcap)
333337 {
338
+	u32 encoding = (lnkcap & PCI_EXP_LNKCAP_L1EL) >> 15;
339
+
334340 	if (encoding == 0x7)
335341 		return (65 * 1000);	/* > 64us */
336342 	return (1000 << encoding);
....@@ -383,58 +389,6 @@
383389 		*scale = 5;
384390 		*value = threshold_ns >> 25;
385391 	}
386
-}
387
-
388
-struct aspm_register_info {
389
-	u32 support:2;
390
-	u32 enabled:2;
391
-	u32 latency_encoding_l0s;
392
-	u32 latency_encoding_l1;
393
-
394
-	/* L1 substates */
395
-	u32 l1ss_cap_ptr;
396
-	u32 l1ss_cap;
397
-	u32 l1ss_ctl1;
398
-	u32 l1ss_ctl2;
399
-};
400
-
401
-static void pcie_get_aspm_reg(struct pci_dev *pdev,
402
-			      struct aspm_register_info *info)
403
-{
404
-	u16 reg16;
405
-	u32 reg32;
406
-
407
-	pcie_capability_read_dword(pdev, PCI_EXP_LNKCAP, &reg32);
408
-	info->support = (reg32 & PCI_EXP_LNKCAP_ASPMS) >> 10;
409
-	info->latency_encoding_l0s = (reg32 & PCI_EXP_LNKCAP_L0SEL) >> 12;
410
-	info->latency_encoding_l1  = (reg32 & PCI_EXP_LNKCAP_L1EL) >> 15;
411
-	pcie_capability_read_word(pdev, PCI_EXP_LNKCTL, &reg16);
412
-	info->enabled = reg16 & PCI_EXP_LNKCTL_ASPMC;
413
-
414
-	/* Read L1 PM substate capabilities */
415
-	info->l1ss_cap = info->l1ss_ctl1 = info->l1ss_ctl2 = 0;
416
-	info->l1ss_cap_ptr = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_L1SS);
417
-	if (!info->l1ss_cap_ptr)
418
-		return;
419
-	pci_read_config_dword(pdev, info->l1ss_cap_ptr + PCI_L1SS_CAP,
420
-			      &info->l1ss_cap);
421
-	if (!(info->l1ss_cap & PCI_L1SS_CAP_L1_PM_SS)) {
422
-		info->l1ss_cap = 0;
423
-		return;
424
-	}
425
-
426
-	/*
427
-	 * If we don't have LTR for the entire path from the Root Complex
428
-	 * to this device, we can't use ASPM L1.2 because it relies on the
429
-	 * LTR_L1.2_THRESHOLD.  See PCIe r4.0, secs 5.5.4, 6.18.
430
-	 */
431
-	if (!pdev->ltr_path)
432
-		info->l1ss_cap &= ~PCI_L1SS_CAP_ASPM_L1_2;
433
-
434
-	pci_read_config_dword(pdev, info->l1ss_cap_ptr + PCI_L1SS_CTL1,
435
-			      &info->l1ss_ctl1);
436
-	pci_read_config_dword(pdev, info->l1ss_cap_ptr + PCI_L1SS_CTL2,
437
-			      &info->l1ss_ctl2);
438392 }
439393 
440394 static void pcie_aspm_check_latency(struct pci_dev *endpoint)
....@@ -498,39 +452,49 @@
498452 	return NULL;
499453 }
500454 
455
+static void pci_clear_and_set_dword(struct pci_dev *pdev, int pos,
456
+				    u32 clear, u32 set)
457
+{
458
+	u32 val;
459
+
460
+	pci_read_config_dword(pdev, pos, &val);
461
+	val &= ~clear;
462
+	val |= set;
463
+	pci_write_config_dword(pdev, pos, val);
464
+}
465
+
501466 /* Calculate L1.2 PM substate timing parameters */
502467 static void aspm_calc_l1ss_info(struct pcie_link_state *link,
503
-				struct aspm_register_info *upreg,
504
-				struct aspm_register_info *dwreg)
468
+				u32 parent_l1ss_cap, u32 child_l1ss_cap)
505469 {
470
+	struct pci_dev *child = link->downstream, *parent = link->pdev;
506471 	u32 val1, val2, scale1, scale2;
507472 	u32 t_common_mode, t_power_on, l1_2_threshold, scale, value;
508
-
509
-	link->l1ss.up_cap_ptr = upreg->l1ss_cap_ptr;
510
-	link->l1ss.dw_cap_ptr = dwreg->l1ss_cap_ptr;
511
-	link->l1ss.ctl1 = link->l1ss.ctl2 = 0;
473
+	u32 ctl1 = 0, ctl2 = 0;
474
+	u32 pctl1, pctl2, cctl1, cctl2;
475
+	u32 pl1_2_enables, cl1_2_enables;
512476 
513477 	if (!(link->aspm_support & ASPM_STATE_L1_2_MASK))
514478 		return;
515479 
516480 	/* Choose the greater of the two Port Common_Mode_Restore_Times */
517
-	val1 = (upreg->l1ss_cap & PCI_L1SS_CAP_CM_RESTORE_TIME) >> 8;
518
-	val2 = (dwreg->l1ss_cap & PCI_L1SS_CAP_CM_RESTORE_TIME) >> 8;
481
+	val1 = (parent_l1ss_cap & PCI_L1SS_CAP_CM_RESTORE_TIME) >> 8;
482
+	val2 = (child_l1ss_cap & PCI_L1SS_CAP_CM_RESTORE_TIME) >> 8;
519483 	t_common_mode = max(val1, val2);
520484 
521485 	/* Choose the greater of the two Port T_POWER_ON times */
522
-	val1   = (upreg->l1ss_cap & PCI_L1SS_CAP_P_PWR_ON_VALUE) >> 19;
523
-	scale1 = (upreg->l1ss_cap & PCI_L1SS_CAP_P_PWR_ON_SCALE) >> 16;
524
-	val2   = (dwreg->l1ss_cap & PCI_L1SS_CAP_P_PWR_ON_VALUE) >> 19;
525
-	scale2 = (dwreg->l1ss_cap & PCI_L1SS_CAP_P_PWR_ON_SCALE) >> 16;
486
+	val1   = (parent_l1ss_cap & PCI_L1SS_CAP_P_PWR_ON_VALUE) >> 19;
487
+	scale1 = (parent_l1ss_cap & PCI_L1SS_CAP_P_PWR_ON_SCALE) >> 16;
488
+	val2   = (child_l1ss_cap & PCI_L1SS_CAP_P_PWR_ON_VALUE) >> 19;
489
+	scale2 = (child_l1ss_cap & PCI_L1SS_CAP_P_PWR_ON_SCALE) >> 16;
526490 
527
-	if (calc_l1ss_pwron(link->pdev, scale1, val1) >
528
-	    calc_l1ss_pwron(link->downstream, scale2, val2)) {
529
-		link->l1ss.ctl2 |= scale1 | (val1 << 3);
530
-		t_power_on = calc_l1ss_pwron(link->pdev, scale1, val1);
491
+	if (calc_l1ss_pwron(parent, scale1, val1) >
492
+	    calc_l1ss_pwron(child, scale2, val2)) {
493
+		ctl2 |= scale1 | (val1 << 3);
494
+		t_power_on = calc_l1ss_pwron(parent, scale1, val1);
531495 	} else {
532
-		link->l1ss.ctl2 |= scale2 | (val2 << 3);
533
-		t_power_on = calc_l1ss_pwron(link->downstream, scale2, val2);
496
+		ctl2 |= scale2 | (val2 << 3);
497
+		t_power_on = calc_l1ss_pwron(child, scale2, val2);
534498 	}
535499 
536500 	/*
....@@ -545,14 +509,60 @@
545509 	 */
546510 	l1_2_threshold = 2 + 4 + t_common_mode + t_power_on;
547511 	encode_l12_threshold(l1_2_threshold, &scale, &value);
548
-	link->l1ss.ctl1 |= t_common_mode << 8 | scale << 29 | value << 16;
512
+	ctl1 |= t_common_mode << 8 | scale << 29 | value << 16;
513
+
514
+	pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1, &pctl1);
515
+	pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CTL2, &pctl2);
516
+	pci_read_config_dword(child, child->l1ss + PCI_L1SS_CTL1, &cctl1);
517
+	pci_read_config_dword(child, child->l1ss + PCI_L1SS_CTL2, &cctl2);
518
+
519
+	if (ctl1 == pctl1 && ctl1 == cctl1 &&
520
+	    ctl2 == pctl2 && ctl2 == cctl2)
521
+		return;
522
+
523
+	/* Disable L1.2 while updating.  See PCIe r5.0, sec 5.5.4, 7.8.3.3 */
524
+	pl1_2_enables = pctl1 & PCI_L1SS_CTL1_L1_2_MASK;
525
+	cl1_2_enables = cctl1 & PCI_L1SS_CTL1_L1_2_MASK;
526
+
527
+	if (pl1_2_enables || cl1_2_enables) {
528
+		pci_clear_and_set_dword(child, child->l1ss + PCI_L1SS_CTL1,
529
+					PCI_L1SS_CTL1_L1_2_MASK, 0);
530
+		pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1,
531
+					PCI_L1SS_CTL1_L1_2_MASK, 0);
532
+	}
533
+
534
+	/* Program T_POWER_ON times in both ports */
535
+	pci_write_config_dword(parent, parent->l1ss + PCI_L1SS_CTL2, ctl2);
536
+	pci_write_config_dword(child, child->l1ss + PCI_L1SS_CTL2, ctl2);
537
+
538
+	/* Program Common_Mode_Restore_Time in upstream device */
539
+	pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1,
540
+				PCI_L1SS_CTL1_CM_RESTORE_TIME, ctl1);
541
+
542
+	/* Program LTR_L1.2_THRESHOLD time in both ports */
543
+	pci_clear_and_set_dword(parent,	parent->l1ss + PCI_L1SS_CTL1,
544
+				PCI_L1SS_CTL1_LTR_L12_TH_VALUE |
545
+				PCI_L1SS_CTL1_LTR_L12_TH_SCALE, ctl1);
546
+	pci_clear_and_set_dword(child, child->l1ss + PCI_L1SS_CTL1,
547
+				PCI_L1SS_CTL1_LTR_L12_TH_VALUE |
548
+				PCI_L1SS_CTL1_LTR_L12_TH_SCALE, ctl1);
549
+
550
+	if (pl1_2_enables || cl1_2_enables) {
551
+		pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1, 0,
552
+					pl1_2_enables);
553
+		pci_clear_and_set_dword(child, child->l1ss + PCI_L1SS_CTL1, 0,
554
+					cl1_2_enables);
555
+	}
549556 }
550557 
551558 static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist)
552559 {
553560 	struct pci_dev *child = link->downstream, *parent = link->pdev;
561
+	u32 parent_lnkcap, child_lnkcap;
562
+	u16 parent_lnkctl, child_lnkctl;
563
+	u32 parent_l1ss_cap, child_l1ss_cap;
564
+	u32 parent_l1ss_ctl1 = 0, child_l1ss_ctl1 = 0;
554565 	struct pci_bus *linkbus = parent->subordinate;
555
-	struct aspm_register_info upreg, dwreg;
556566 
557567 	if (blacklist) {
558568 		/* Set enabled/disable so that we will disable ASPM later */
....@@ -561,26 +571,28 @@
561571 		return;
562572 	}
563573 
564
-	/* Get upstream/downstream components' register state */
565
-	pcie_get_aspm_reg(parent, &upreg);
566
-	pcie_get_aspm_reg(child, &dwreg);
567
-
568574 	/*
569575 	 * If ASPM not supported, don't mess with the clocks and link,
570576 	 * bail out now.
571577 	 */
572
-	if (!(upreg.support & dwreg.support))
578
+	pcie_capability_read_dword(parent, PCI_EXP_LNKCAP, &parent_lnkcap);
579
+	pcie_capability_read_dword(child, PCI_EXP_LNKCAP, &child_lnkcap);
580
+	if (!(parent_lnkcap & child_lnkcap & PCI_EXP_LNKCAP_ASPMS))
573581 		return;
574582 
575583 	/* Configure common clock before checking latencies */
576584 	pcie_aspm_configure_common_clock(link);
577585 
578586 	/*
579
-	 * Re-read upstream/downstream components' register state
580
-	 * after clock configuration
587
+	 * Re-read upstream/downstream components' register state after
588
+	 * clock configuration.  L0s & L1 exit latencies in the otherwise
589
+	 * read-only Link Capabilities may change depending on common clock
590
+	 * configuration (PCIe r5.0, sec 7.5.3.6).
581591 	 */
582
-	pcie_get_aspm_reg(parent, &upreg);
583
-	pcie_get_aspm_reg(child, &dwreg);
592
+	pcie_capability_read_dword(parent, PCI_EXP_LNKCAP, &parent_lnkcap);
593
+	pcie_capability_read_dword(child, PCI_EXP_LNKCAP, &child_lnkcap);
594
+	pcie_capability_read_word(parent, PCI_EXP_LNKCTL, &parent_lnkctl);
595
+	pcie_capability_read_word(child, PCI_EXP_LNKCTL, &child_lnkctl);
584596 
585597 	/*
586598 	 * Setup L0s state
....@@ -589,44 +601,71 @@
589601 	 * given link unless components on both sides of the link each
590602 	 * support L0s.
591603 	 */
592
-	if (dwreg.support & upreg.support & PCIE_LINK_STATE_L0S)
604
+	if (parent_lnkcap & child_lnkcap & PCI_EXP_LNKCAP_ASPM_L0S)
593605 		link->aspm_support |= ASPM_STATE_L0S;
594
-	if (dwreg.enabled & PCIE_LINK_STATE_L0S)
606
+
607
+	if (child_lnkctl & PCI_EXP_LNKCTL_ASPM_L0S)
595608 		link->aspm_enabled |= ASPM_STATE_L0S_UP;
596
-	if (upreg.enabled & PCIE_LINK_STATE_L0S)
609
+	if (parent_lnkctl & PCI_EXP_LNKCTL_ASPM_L0S)
597610 		link->aspm_enabled |= ASPM_STATE_L0S_DW;
598
-	link->latency_up.l0s = calc_l0s_latency(upreg.latency_encoding_l0s);
599
-	link->latency_dw.l0s = calc_l0s_latency(dwreg.latency_encoding_l0s);
611
+	link->latency_up.l0s = calc_l0s_latency(parent_lnkcap);
612
+	link->latency_dw.l0s = calc_l0s_latency(child_lnkcap);
600613 
601614 	/* Setup L1 state */
602
-	if (upreg.support & dwreg.support & PCIE_LINK_STATE_L1)
615
+	if (parent_lnkcap & child_lnkcap & PCI_EXP_LNKCAP_ASPM_L1)
603616 		link->aspm_support |= ASPM_STATE_L1;
604
-	if (upreg.enabled & dwreg.enabled & PCIE_LINK_STATE_L1)
617
+
618
+	if (parent_lnkctl & child_lnkctl & PCI_EXP_LNKCTL_ASPM_L1)
605619 		link->aspm_enabled |= ASPM_STATE_L1;
606
-	link->latency_up.l1 = calc_l1_latency(upreg.latency_encoding_l1);
607
-	link->latency_dw.l1 = calc_l1_latency(dwreg.latency_encoding_l1);
620
+	link->latency_up.l1 = calc_l1_latency(parent_lnkcap);
621
+	link->latency_dw.l1 = calc_l1_latency(child_lnkcap);
608622 
609623 	/* Setup L1 substate */
610
-	if (upreg.l1ss_cap & dwreg.l1ss_cap & PCI_L1SS_CAP_ASPM_L1_1)
624
+	pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CAP,
625
+			      &parent_l1ss_cap);
626
+	pci_read_config_dword(child, child->l1ss + PCI_L1SS_CAP,
627
+			      &child_l1ss_cap);
628
+
629
+	if (!(parent_l1ss_cap & PCI_L1SS_CAP_L1_PM_SS))
630
+		parent_l1ss_cap = 0;
631
+	if (!(child_l1ss_cap & PCI_L1SS_CAP_L1_PM_SS))
632
+		child_l1ss_cap = 0;
633
+
634
+	/*
635
+	 * If we don't have LTR for the entire path from the Root Complex
636
+	 * to this device, we can't use ASPM L1.2 because it relies on the
637
+	 * LTR_L1.2_THRESHOLD.  See PCIe r4.0, secs 5.5.4, 6.18.
638
+	 */
639
+	if (!child->ltr_path)
640
+		child_l1ss_cap &= ~PCI_L1SS_CAP_ASPM_L1_2;
641
+
642
+	if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_ASPM_L1_1)
611643 		link->aspm_support |= ASPM_STATE_L1_1;
612
-	if (upreg.l1ss_cap & dwreg.l1ss_cap & PCI_L1SS_CAP_ASPM_L1_2)
644
+	if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_ASPM_L1_2)
613645 		link->aspm_support |= ASPM_STATE_L1_2;
614
-	if (upreg.l1ss_cap & dwreg.l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_1)
646
+	if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_1)
615647 		link->aspm_support |= ASPM_STATE_L1_1_PCIPM;
616
-	if (upreg.l1ss_cap & dwreg.l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_2)
648
+	if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_2)
617649 		link->aspm_support |= ASPM_STATE_L1_2_PCIPM;
618650 
619
-	if (upreg.l1ss_ctl1 & dwreg.l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_1)
651
+	if (parent_l1ss_cap)
652
+		pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1,
653
+				      &parent_l1ss_ctl1);
654
+	if (child_l1ss_cap)
655
+		pci_read_config_dword(child, child->l1ss + PCI_L1SS_CTL1,
656
+				      &child_l1ss_ctl1);
657
+
658
+	if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_1)
620659 		link->aspm_enabled |= ASPM_STATE_L1_1;
621
-	if (upreg.l1ss_ctl1 & dwreg.l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_2)
660
+	if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_2)
622661 		link->aspm_enabled |= ASPM_STATE_L1_2;
623
-	if (upreg.l1ss_ctl1 & dwreg.l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_1)
662
+	if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_1)
624663 		link->aspm_enabled |= ASPM_STATE_L1_1_PCIPM;
625
-	if (upreg.l1ss_ctl1 & dwreg.l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_2)
664
+	if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_2)
626665 		link->aspm_enabled |= ASPM_STATE_L1_2_PCIPM;
627666 
628667 	if (link->aspm_support & ASPM_STATE_L1SS)
629
-		aspm_calc_l1ss_info(link, &upreg, &dwreg);
668
+		aspm_calc_l1ss_info(link, parent_l1ss_cap, child_l1ss_cap);
630669 
631670 	/* Save default state */
632671 	link->aspm_default = link->aspm_enabled;
....@@ -656,24 +695,11 @@
656695 	}
657696 }
658697 
659
-static void pci_clear_and_set_dword(struct pci_dev *pdev, int pos,
660
-				    u32 clear, u32 set)
661
-{
662
-	u32 val;
663
-
664
-	pci_read_config_dword(pdev, pos, &val);
665
-	val &= ~clear;
666
-	val |= set;
667
-	pci_write_config_dword(pdev, pos, val);
668
-}
669
-
670698 /* Configure the ASPM L1 substates */
671699 static void pcie_config_aspm_l1ss(struct pcie_link_state *link, u32 state)
672700 {
673701 	u32 val, enable_req;
674702 	struct pci_dev *child = link->downstream, *parent = link->pdev;
675
-	u32 up_cap_ptr = link->l1ss.up_cap_ptr;
676
-	u32 dw_cap_ptr = link->l1ss.dw_cap_ptr;
677703 
678704 	enable_req = (link->aspm_enabled ^ state) & state;
679705 
....@@ -691,9 +717,9 @@
691717 	 */
692718 
693719 	/* Disable all L1 substates */
694
-	pci_clear_and_set_dword(child, dw_cap_ptr + PCI_L1SS_CTL1,
720
+	pci_clear_and_set_dword(child, child->l1ss + PCI_L1SS_CTL1,
695721 				PCI_L1SS_CTL1_L1SS_MASK, 0);
696
-	pci_clear_and_set_dword(parent, up_cap_ptr + PCI_L1SS_CTL1,
722
+	pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1,
697723 				PCI_L1SS_CTL1_L1SS_MASK, 0);
698724 	/*
699725 	 * If needed, disable L1, and it gets enabled later
....@@ -704,30 +730,6 @@
704730 						   PCI_EXP_LNKCTL_ASPM_L1, 0);
705731 		pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL,
706732 						   PCI_EXP_LNKCTL_ASPM_L1, 0);
707
-	}
708
-
709
-	if (enable_req & ASPM_STATE_L1_2_MASK) {
710
-
711
-		/* Program T_POWER_ON times in both ports */
712
-		pci_write_config_dword(parent, up_cap_ptr + PCI_L1SS_CTL2,
713
-				       link->l1ss.ctl2);
714
-		pci_write_config_dword(child, dw_cap_ptr + PCI_L1SS_CTL2,
715
-				       link->l1ss.ctl2);
716
-
717
-		/* Program Common_Mode_Restore_Time in upstream device */
718
-		pci_clear_and_set_dword(parent, up_cap_ptr + PCI_L1SS_CTL1,
719
-					PCI_L1SS_CTL1_CM_RESTORE_TIME,
720
-					link->l1ss.ctl1);
721
-
722
-		/* Program LTR_L1.2_THRESHOLD time in both ports */
723
-		pci_clear_and_set_dword(parent,	up_cap_ptr + PCI_L1SS_CTL1,
724
-					PCI_L1SS_CTL1_LTR_L12_TH_VALUE |
725
-					PCI_L1SS_CTL1_LTR_L12_TH_SCALE,
726
-					link->l1ss.ctl1);
727
-		pci_clear_and_set_dword(child, dw_cap_ptr + PCI_L1SS_CTL1,
728
-					PCI_L1SS_CTL1_LTR_L12_TH_VALUE |
729
-					PCI_L1SS_CTL1_LTR_L12_TH_SCALE,
730
-					link->l1ss.ctl1);
731733 	}
732734 
733735 	val = 0;
....@@ -741,9 +743,9 @@
741743 		val |= PCI_L1SS_CTL1_PCIPM_L1_2;
742744 
743745 	/* Enable what we need to enable */
744
-	pci_clear_and_set_dword(parent, up_cap_ptr + PCI_L1SS_CTL1,
746
+	pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1,
745747 				PCI_L1SS_CTL1_L1SS_MASK, val);
746
-	pci_clear_and_set_dword(child, dw_cap_ptr + PCI_L1SS_CTL1,
748
+	pci_clear_and_set_dword(child, child->l1ss + PCI_L1SS_CTL1,
747749 				PCI_L1SS_CTL1_L1SS_MASK, val);
748750 }
749751 
....@@ -862,8 +864,6 @@
862864 		return NULL;
863865 
864866 	INIT_LIST_HEAD(&link->sibling);
865
-	INIT_LIST_HEAD(&link->children);
866
-	INIT_LIST_HEAD(&link->link);
867867 	link->pdev = pdev;
868868 	link->downstream = pci_function_0(pdev->subordinate);
869869 
....@@ -889,12 +889,19 @@
889889 
890890 		link->parent = parent;
891891 		link->root = link->parent->root;
892
-		list_add(&link->link, &parent->children);
893892 	}
894893 
895894 	list_add(&link->sibling, &link_list);
896895 	pdev->link_state = link;
897896 	return link;
897
+}
898
+
899
+static void pcie_aspm_update_sysfs_visibility(struct pci_dev *pdev)
900
+{
901
+	struct pci_dev *child;
902
+
903
+	list_for_each_entry(child, &pdev->subordinate->devices, bus_list)
904
+		sysfs_update_group(&child->dev.kobj, &aspm_ctrl_attr_group);
898905 }
899906 
900907 /*
....@@ -915,10 +922,10 @@
915922 
916923 	/*
917924 	 * We allocate pcie_link_state for the component on the upstream
918
-	 * end of a Link, so there's nothing to do unless this device has a
919
-	 * Link on its secondary side.
925
+	 * end of a Link, so there's nothing to do unless this device is
926
+	 * downstream port.
920927 	 */
921
-	if (!pdev->has_secondary_link)
928
+	if (!pcie_downstream_port(pdev))
922929 		return;
923930 
924931 	/* VIA has a strange chipset, root port is under a bridge */
....@@ -957,6 +964,8 @@
957964 		pcie_config_aspm_path(link);
958965 		pcie_set_clkpm(link, policy_to_clkpm_state(link));
959966 	}
967
+
968
+	pcie_aspm_update_sysfs_visibility(pdev);
960969 
961970 unlock:
962971 	mutex_unlock(&aspm_lock);
....@@ -999,22 +1008,24 @@
9991008 
10001009 	down_read(&pci_bus_sem);
10011010 	mutex_lock(&aspm_lock);
1002
-	/*
1003
-	 * All PCIe functions are in one slot, remove one function will remove
1004
-	 * the whole slot, so just wait until we are the last function left.
1005
-	 */
1006
-	if (!list_empty(&parent->subordinate->devices))
1007
-		goto out;
10081011 
10091012 	link = parent->link_state;
10101013 	root = link->root;
10111014 	parent_link = link->parent;
10121015 
1013
-	/* All functions are removed, so just disable ASPM for the link */
1016
+	/*
1017
+	 * link->downstream is a pointer to the pci_dev of function 0.  If
1018
+	 * we remove that function, the pci_dev is about to be deallocated,
1019
+	 * so we can't use link->downstream again.  Free the link state to
1020
+	 * avoid this.
1021
+	 *
1022
+	 * If we're removing a non-0 function, it's possible we could
1023
+	 * retain the link state, but PCIe r6.0, sec 7.5.3.7, recommends
1024
+	 * programming the same ASPM Control value for all functions of
1025
+	 * multi-function devices, so disable ASPM for all of them.
1026
+	 */
10141027 	pcie_config_aspm_link(link, 0);
10151028 	list_del(&link->sibling);
1016
-	list_del(&link->link);
1017
-	/* Clock PM is for endpoint device */
10181029 	free_link_state(link);
10191030 
10201031 	/* Recheck latencies and configure upstream links */
....@@ -1022,7 +1033,7 @@
10221033 		pcie_update_aspm_capable(root);
10231034 		pcie_config_aspm_path(parent_link);
10241035 	}
1025
-out:
1036
+
10261037 	mutex_unlock(&aspm_lock);
10271038 	up_read(&pci_bus_sem);
10281039 }
....@@ -1065,19 +1076,26 @@
10651076 	up_read(&pci_bus_sem);
10661077 }
10671078 
1068
-static void __pci_disable_link_state(struct pci_dev *pdev, int state, bool sem)
1079
+static struct pcie_link_state *pcie_aspm_get_link(struct pci_dev *pdev)
10691080 {
1070
-	struct pci_dev *parent = pdev->bus->self;
1071
-	struct pcie_link_state *link;
1081
+	struct pci_dev *bridge;
10721082 
10731083 	if (!pci_is_pcie(pdev))
1074
-		return;
1084
+		return NULL;
10751085 
1076
-	if (pdev->has_secondary_link)
1077
-		parent = pdev;
1078
-	if (!parent || !parent->link_state)
1079
-		return;
1086
+	bridge = pci_upstream_bridge(pdev);
1087
+	if (!bridge || !pci_is_pcie(bridge))
1088
+		return NULL;
10801089 
1090
+	return bridge->link_state;
1091
+}
1092
+
1093
+static int __pci_disable_link_state(struct pci_dev *pdev, int state, bool sem)
1094
+{
1095
+	struct pcie_link_state *link = pcie_aspm_get_link(pdev);
1096
+
1097
+	if (!link)
1098
+		return -EINVAL;
10811099 	/*
10821100 	 * A driver requested that ASPM be disabled on this device, but
10831101 	 * if we don't have permission to manage ASPM (e.g., on ACPI
....@@ -1088,17 +1106,25 @@
10881106 	 */
10891107 	if (aspm_disabled) {
10901108 		pci_warn(pdev, "can't disable ASPM; OS doesn't have ASPM control\n");
1091
-		return;
1109
+		return -EPERM;
10921110 	}
10931111 
10941112 	if (sem)
10951113 		down_read(&pci_bus_sem);
10961114 	mutex_lock(&aspm_lock);
1097
-	link = parent->link_state;
10981115 	if (state & PCIE_LINK_STATE_L0S)
10991116 		link->aspm_disable |= ASPM_STATE_L0S;
11001117 	if (state & PCIE_LINK_STATE_L1)
1101
-		link->aspm_disable |= ASPM_STATE_L1;
1118
+		/* L1 PM substates require L1 */
1119
+		link->aspm_disable |= ASPM_STATE_L1 | ASPM_STATE_L1SS;
1120
+	if (state & PCIE_LINK_STATE_L1_1)
1121
+		link->aspm_disable |= ASPM_STATE_L1_1;
1122
+	if (state & PCIE_LINK_STATE_L1_2)
1123
+		link->aspm_disable |= ASPM_STATE_L1_2;
1124
+	if (state & PCIE_LINK_STATE_L1_1_PCIPM)
1125
+		link->aspm_disable |= ASPM_STATE_L1_1_PCIPM;
1126
+	if (state & PCIE_LINK_STATE_L1_2_PCIPM)
1127
+		link->aspm_disable |= ASPM_STATE_L1_2_PCIPM;
11021128 	pcie_config_aspm_link(link, policy_to_aspm_state(link));
11031129 
11041130 	if (state & PCIE_LINK_STATE_CLKPM)
....@@ -1107,11 +1133,13 @@
11071133 	mutex_unlock(&aspm_lock);
11081134 	if (sem)
11091135 		up_read(&pci_bus_sem);
1136
+
1137
+	return 0;
11101138 }
11111139 
1112
-void pci_disable_link_state_locked(struct pci_dev *pdev, int state)
1140
+int pci_disable_link_state_locked(struct pci_dev *pdev, int state)
11131141 {
1114
-	__pci_disable_link_state(pdev, state, false);
1142
+	return __pci_disable_link_state(pdev, state, false);
11151143 }
11161144 EXPORT_SYMBOL(pci_disable_link_state_locked);
11171145 
....@@ -1119,14 +1147,14 @@
11191147  * pci_disable_link_state - Disable device's link state, so the link will
11201148  * never enter specific states.  Note that if the BIOS didn't grant ASPM
11211149  * control to the OS, this does nothing because we can't touch the LNKCTL
1122
- * register.
1150
+ * register. Returns 0 or a negative errno.
11231151  *
11241152  * @pdev: PCI device
11251153  * @state: ASPM link state to disable
11261154  */
1127
-void pci_disable_link_state(struct pci_dev *pdev, int state)
1155
+int pci_disable_link_state(struct pci_dev *pdev, int state)
11281156 {
1129
-	__pci_disable_link_state(pdev, state, true);
1157
+	return __pci_disable_link_state(pdev, state, true);
11301158 }
11311159 EXPORT_SYMBOL(pci_disable_link_state);
11321160 
....@@ -1171,110 +1199,164 @@
11711199 module_param_call(policy, pcie_aspm_set_policy, pcie_aspm_get_policy,
11721200 	NULL, 0644);
11731201 
1174
-#ifdef CONFIG_PCIEASPM_DEBUG
1175
-static ssize_t link_state_show(struct device *dev,
1176
-		struct device_attribute *attr,
1177
-		char *buf)
1202
+/**
1203
+ * pcie_aspm_enabled - Check if PCIe ASPM has been enabled for a device.
1204
+ * @pdev: Target device.
1205
+ *
1206
+ * Relies on the upstream bridge's link_state being valid.  The link_state
1207
+ * is deallocated only when the last child of the bridge (i.e., @pdev or a
1208
+ * sibling) is removed, and the caller should be holding a reference to
1209
+ * @pdev, so this should be safe.
1210
+ */
1211
+bool pcie_aspm_enabled(struct pci_dev *pdev)
11781212 {
1179
-	struct pci_dev *pci_device = to_pci_dev(dev);
1180
-	struct pcie_link_state *link_state = pci_device->link_state;
1213
+	struct pcie_link_state *link = pcie_aspm_get_link(pdev);
11811214 
1182
-	return sprintf(buf, "%d\n", link_state->aspm_enabled);
1215
+	if (!link)
1216
+		return false;
1217
+
1218
+	return link->aspm_enabled;
11831219 }
1220
+EXPORT_SYMBOL_GPL(pcie_aspm_enabled);
11841221 
1185
-static ssize_t link_state_store(struct device *dev,
1186
-		struct device_attribute *attr,
1187
-		const char *buf,
1188
-		size_t n)
1222
+static ssize_t aspm_attr_show_common(struct device *dev,
1223
+				     struct device_attribute *attr,
1224
+				     char *buf, u8 state)
11891225 {
11901226 	struct pci_dev *pdev = to_pci_dev(dev);
1191
-	struct pcie_link_state *link, *root = pdev->link_state->root;
1192
-	u32 state;
1227
+	struct pcie_link_state *link = pcie_aspm_get_link(pdev);
11931228 
1194
-	if (aspm_disabled)
1195
-		return -EPERM;
1229
+	return sprintf(buf, "%d\n", (link->aspm_enabled & state) ? 1 : 0);
1230
+}
11961231 
1197
-	if (kstrtouint(buf, 10, &state))
1198
-		return -EINVAL;
1199
-	if ((state & ~ASPM_STATE_ALL) != 0)
1232
+static ssize_t aspm_attr_store_common(struct device *dev,
1233
+				      struct device_attribute *attr,
1234
+				      const char *buf, size_t len, u8 state)
1235
+{
1236
+	struct pci_dev *pdev = to_pci_dev(dev);
1237
+	struct pcie_link_state *link = pcie_aspm_get_link(pdev);
1238
+	bool state_enable;
1239
+
1240
+	if (strtobool(buf, &state_enable) < 0)
12001241 		return -EINVAL;
12011242 
12021243 	down_read(&pci_bus_sem);
12031244 	mutex_lock(&aspm_lock);
1204
-	list_for_each_entry(link, &link_list, sibling) {
1205
-		if (link->root != root)
1206
-			continue;
1207
-		pcie_config_aspm_link(link, state);
1245
+
1246
+	if (state_enable) {
1247
+		link->aspm_disable &= ~state;
1248
+		/* need to enable L1 for substates */
1249
+		if (state & ASPM_STATE_L1SS)
1250
+			link->aspm_disable &= ~ASPM_STATE_L1;
1251
+	} else {
1252
+		link->aspm_disable |= state;
12081253 	}
1254
+
1255
+	pcie_config_aspm_link(link, policy_to_aspm_state(link));
1256
+
12091257 	mutex_unlock(&aspm_lock);
12101258 	up_read(&pci_bus_sem);
1211
-	return n;
1259
+
1260
+	return len;
12121261 }
12131262 
1214
-static ssize_t clk_ctl_show(struct device *dev,
1215
-		struct device_attribute *attr,
1216
-		char *buf)
1217
-{
1218
-	struct pci_dev *pci_device = to_pci_dev(dev);
1219
-	struct pcie_link_state *link_state = pci_device->link_state;
1263
+#define ASPM_ATTR(_f, _s)						\
1264
+static ssize_t _f##_show(struct device *dev,				\
1265
+			 struct device_attribute *attr, char *buf)	\
1266
+{ return aspm_attr_show_common(dev, attr, buf, ASPM_STATE_##_s); }	\
1267
+									\
1268
+static ssize_t _f##_store(struct device *dev,				\
1269
+			  struct device_attribute *attr,		\
1270
+			  const char *buf, size_t len)			\
1271
+{ return aspm_attr_store_common(dev, attr, buf, len, ASPM_STATE_##_s); }
12201272 
1221
-	return sprintf(buf, "%d\n", link_state->clkpm_enabled);
1222
-}
1273
+ASPM_ATTR(l0s_aspm, L0S)
1274
+ASPM_ATTR(l1_aspm, L1)
1275
+ASPM_ATTR(l1_1_aspm, L1_1)
1276
+ASPM_ATTR(l1_2_aspm, L1_2)
1277
+ASPM_ATTR(l1_1_pcipm, L1_1_PCIPM)
1278
+ASPM_ATTR(l1_2_pcipm, L1_2_PCIPM)
12231279 
1224
-static ssize_t clk_ctl_store(struct device *dev,
1225
-		struct device_attribute *attr,
1226
-		const char *buf,
1227
-		size_t n)
1280
+static ssize_t clkpm_show(struct device *dev,
1281
+			  struct device_attribute *attr, char *buf)
12281282 {
12291283 	struct pci_dev *pdev = to_pci_dev(dev);
1230
-	bool state;
1284
+	struct pcie_link_state *link = pcie_aspm_get_link(pdev);
12311285 
1232
-	if (strtobool(buf, &state))
1286
+	return sprintf(buf, "%d\n", link->clkpm_enabled);
1287
+}
1288
+
1289
+static ssize_t clkpm_store(struct device *dev,
1290
+			   struct device_attribute *attr,
1291
+			   const char *buf, size_t len)
1292
+{
1293
+	struct pci_dev *pdev = to_pci_dev(dev);
1294
+	struct pcie_link_state *link = pcie_aspm_get_link(pdev);
1295
+	bool state_enable;
1296
+
1297
+	if (strtobool(buf, &state_enable) < 0)
12331298 		return -EINVAL;
12341299 
12351300 	down_read(&pci_bus_sem);
12361301 	mutex_lock(&aspm_lock);
1237
-	pcie_set_clkpm_nocheck(pdev->link_state, state);
1302
+
1303
+	link->clkpm_disable = !state_enable;
1304
+	pcie_set_clkpm(link, policy_to_clkpm_state(link));
1305
+
12381306 	mutex_unlock(&aspm_lock);
12391307 	up_read(&pci_bus_sem);
12401308 
1241
-	return n;
1309
+	return len;
12421310 }
12431311 
1244
-static DEVICE_ATTR_RW(link_state);
1245
-static DEVICE_ATTR_RW(clk_ctl);
1312
+static DEVICE_ATTR_RW(clkpm);
1313
+static DEVICE_ATTR_RW(l0s_aspm);
1314
+static DEVICE_ATTR_RW(l1_aspm);
1315
+static DEVICE_ATTR_RW(l1_1_aspm);
1316
+static DEVICE_ATTR_RW(l1_2_aspm);
1317
+static DEVICE_ATTR_RW(l1_1_pcipm);
1318
+static DEVICE_ATTR_RW(l1_2_pcipm);
12461319 
1247
-static char power_group[] = "power";
1248
-void pcie_aspm_create_sysfs_dev_files(struct pci_dev *pdev)
1320
+static struct attribute *aspm_ctrl_attrs[] = {
1321
+	&dev_attr_clkpm.attr,
1322
+	&dev_attr_l0s_aspm.attr,
1323
+	&dev_attr_l1_aspm.attr,
1324
+	&dev_attr_l1_1_aspm.attr,
1325
+	&dev_attr_l1_2_aspm.attr,
1326
+	&dev_attr_l1_1_pcipm.attr,
1327
+	&dev_attr_l1_2_pcipm.attr,
1328
+	NULL
1329
+};
1330
+
1331
+static umode_t aspm_ctrl_attrs_are_visible(struct kobject *kobj,
1332
+					   struct attribute *a, int n)
12491333 {
1250
-	struct pcie_link_state *link_state = pdev->link_state;
1334
+	struct device *dev = kobj_to_dev(kobj);
1335
+	struct pci_dev *pdev = to_pci_dev(dev);
1336
+	struct pcie_link_state *link = pcie_aspm_get_link(pdev);
1337
+	static const u8 aspm_state_map[] = {
1338
+		ASPM_STATE_L0S,
1339
+		ASPM_STATE_L1,
1340
+		ASPM_STATE_L1_1,
1341
+		ASPM_STATE_L1_2,
1342
+		ASPM_STATE_L1_1_PCIPM,
1343
+		ASPM_STATE_L1_2_PCIPM,
1344
+	};
12511345 
1252
-	if (!link_state)
1253
-		return;
1346
+	if (aspm_disabled || !link)
1347
+		return 0;
12541348 
1255
-	if (link_state->aspm_support)
1256
-		sysfs_add_file_to_group(&pdev->dev.kobj,
1257
-			&dev_attr_link_state.attr, power_group);
1258
-	if (link_state->clkpm_capable)
1259
-		sysfs_add_file_to_group(&pdev->dev.kobj,
1260
-			&dev_attr_clk_ctl.attr, power_group);
1349
+	if (n == 0)
1350
+		return link->clkpm_capable ? a->mode : 0;
1351
+
1352
+	return link->aspm_capable & aspm_state_map[n - 1] ? a->mode : 0;
12611353 }
12621354 
1263
-void pcie_aspm_remove_sysfs_dev_files(struct pci_dev *pdev)
1264
-{
1265
-	struct pcie_link_state *link_state = pdev->link_state;
1266
-
1267
-	if (!link_state)
1268
-		return;
1269
-
1270
-	if (link_state->aspm_support)
1271
-		sysfs_remove_file_from_group(&pdev->dev.kobj,
1272
-			&dev_attr_link_state.attr, power_group);
1273
-	if (link_state->clkpm_capable)
1274
-		sysfs_remove_file_from_group(&pdev->dev.kobj,
1275
-			&dev_attr_clk_ctl.attr, power_group);
1276
-}
1277
-#endif
1355
+const struct attribute_group aspm_ctrl_attr_group = {
1356
+	.name = "link",
1357
+	.attrs = aspm_ctrl_attrs,
1358
+	.is_visible = aspm_ctrl_attrs_are_visible,
1359
+};
12781360 
12791361 static int __init pcie_aspm_disable(char *str)
12801362 {