forked from ~ljy/RK356X_SDK_RELEASE
blame | history | patch | commit | commitdiff
hc
2023-12-08 01573e231f18eb2d99162747186f59511f56b64d 
 kernel/drivers/pci/pcie/aspm.c
....@@ -18,7 +18,6 @@
1818 #include <linux/slab.h>
1919 #include <linux/jiffies.h>
2020 #include <linux/delay.h>
21
-#include <linux/pci-aspm.h>
2221 #include "../pci.h"
2322 
2423 #ifdef MODULE_PARAM_PREFIX
....@@ -53,8 +52,6 @@
5352 	struct pcie_link_state *root;	/* pointer to the root port link */
5453 	struct pcie_link_state *parent;	/* pointer to the parent Link state */
5554 	struct list_head sibling;	/* node in link_list */
56
-	struct list_head children;	/* list of child link states */
57
-	struct list_head link;		/* node in parent's children list */
5855 
5956 	/* ASPM state */
6057 	u32 aspm_support:7;		/* Supported ASPM state */
....@@ -77,14 +74,6 @@
7774 	 * has one slot under it, so at most there are 8 functions.
7875 	 */
7976 	struct aspm_latency acceptable[8];
80
-
81
-	/* L1 PM Substate info */
82
-	struct {
83
-		u32 up_cap_ptr;		/* L1SS cap ptr in upstream dev */
84
-		u32 dw_cap_ptr;		/* L1SS cap ptr in downstream dev */
85
-		u32 ctl1;		/* value to be programmed in ctl1 */
86
-		u32 ctl2;		/* value to be programmed in ctl2 */
87
-	} l1ss;
8877 };
8978 
9079 static int aspm_disabled, aspm_force;
....@@ -206,7 +195,7 @@
206195 static bool pcie_retrain_link(struct pcie_link_state *link)
207196 {
208197 	struct pci_dev *parent = link->pdev;
209
-	unsigned long start_jiffies;
198
+	unsigned long end_jiffies;
210199 	u16 reg16;
211200 
212201 	pcie_capability_read_word(parent, PCI_EXP_LNKCTL, &reg16);
....@@ -223,15 +212,13 @@
223212 	}
224213 
225214 	/* Wait for link training end. Break out after waiting for timeout */
226
-	start_jiffies = jiffies;
227
-	for (;;) {
215
+	end_jiffies = jiffies + LINK_RETRAIN_TIMEOUT;
216
+	do {
228217 		pcie_capability_read_word(parent, PCI_EXP_LNKSTA, &reg16);
229218 		if (!(reg16 & PCI_EXP_LNKSTA_LT))
230219 			break;
231
-		if (time_after(jiffies, start_jiffies + LINK_RETRAIN_TIMEOUT))
232
-			break;
233220 		msleep(1);
234
-	}
221
+	} while (time_before(jiffies, end_jiffies));
235222 	return !(reg16 & PCI_EXP_LNKSTA_LT);
236223 }
237224 
....@@ -278,7 +265,7 @@
278265 		}
279266 		if (consistent)
280267 			return;
281
-		pci_warn(parent, "ASPM: current common clock configuration is broken, reconfiguring\n");
268
+		pci_info(parent, "ASPM: current common clock configuration is inconsistent, reconfiguring\n");
282269 	}
283270 
284271 	/* Configure downstream component, all functions */
....@@ -313,8 +300,10 @@
313300 }
314301 
315302 /* Convert L0s latency encoding to ns */
316
-static u32 calc_l0s_latency(u32 encoding)
303
+static u32 calc_l0s_latency(u32 lnkcap)
317304 {
305
+	u32 encoding = (lnkcap & PCI_EXP_LNKCAP_L0SEL) >> 12;
306
+
318307 	if (encoding == 0x7)
319308 		return (5 * 1000);	/* > 4us */
320309 	return (64 << encoding);
....@@ -329,8 +318,10 @@
329318 }
330319 
331320 /* Convert L1 latency encoding to ns */
332
-static u32 calc_l1_latency(u32 encoding)
321
+static u32 calc_l1_latency(u32 lnkcap)
333322 {
323
+	u32 encoding = (lnkcap & PCI_EXP_LNKCAP_L1EL) >> 15;
324
+
334325 	if (encoding == 0x7)
335326 		return (65 * 1000);	/* > 64us */
336327 	return (1000 << encoding);
....@@ -383,58 +374,6 @@
383374 		*scale = 5;
384375 		*value = threshold_ns >> 25;
385376 	}
386
-}
387
-
388
-struct aspm_register_info {
389
-	u32 support:2;
390
-	u32 enabled:2;
391
-	u32 latency_encoding_l0s;
392
-	u32 latency_encoding_l1;
393
-
394
-	/* L1 substates */
395
-	u32 l1ss_cap_ptr;
396
-	u32 l1ss_cap;
397
-	u32 l1ss_ctl1;
398
-	u32 l1ss_ctl2;
399
-};
400
-
401
-static void pcie_get_aspm_reg(struct pci_dev *pdev,
402
-			      struct aspm_register_info *info)
403
-{
404
-	u16 reg16;
405
-	u32 reg32;
406
-
407
-	pcie_capability_read_dword(pdev, PCI_EXP_LNKCAP, &reg32);
408
-	info->support = (reg32 & PCI_EXP_LNKCAP_ASPMS) >> 10;
409
-	info->latency_encoding_l0s = (reg32 & PCI_EXP_LNKCAP_L0SEL) >> 12;
410
-	info->latency_encoding_l1  = (reg32 & PCI_EXP_LNKCAP_L1EL) >> 15;
411
-	pcie_capability_read_word(pdev, PCI_EXP_LNKCTL, &reg16);
412
-	info->enabled = reg16 & PCI_EXP_LNKCTL_ASPMC;
413
-
414
-	/* Read L1 PM substate capabilities */
415
-	info->l1ss_cap = info->l1ss_ctl1 = info->l1ss_ctl2 = 0;
416
-	info->l1ss_cap_ptr = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_L1SS);
417
-	if (!info->l1ss_cap_ptr)
418
-		return;
419
-	pci_read_config_dword(pdev, info->l1ss_cap_ptr + PCI_L1SS_CAP,
420
-			      &info->l1ss_cap);
421
-	if (!(info->l1ss_cap & PCI_L1SS_CAP_L1_PM_SS)) {
422
-		info->l1ss_cap = 0;
423
-		return;
424
-	}
425
-
426
-	/*
427
-	 * If we don't have LTR for the entire path from the Root Complex
428
-	 * to this device, we can't use ASPM L1.2 because it relies on the
429
-	 * LTR_L1.2_THRESHOLD.  See PCIe r4.0, secs 5.5.4, 6.18.
430
-	 */
431
-	if (!pdev->ltr_path)
432
-		info->l1ss_cap &= ~PCI_L1SS_CAP_ASPM_L1_2;
433
-
434
-	pci_read_config_dword(pdev, info->l1ss_cap_ptr + PCI_L1SS_CTL1,
435
-			      &info->l1ss_ctl1);
436
-	pci_read_config_dword(pdev, info->l1ss_cap_ptr + PCI_L1SS_CTL2,
437
-			      &info->l1ss_ctl2);
438377 }
439378 
440379 static void pcie_aspm_check_latency(struct pci_dev *endpoint)
....@@ -498,39 +437,49 @@
498437 	return NULL;
499438 }
500439 
440
+static void pci_clear_and_set_dword(struct pci_dev *pdev, int pos,
441
+				    u32 clear, u32 set)
442
+{
443
+	u32 val;
444
+
445
+	pci_read_config_dword(pdev, pos, &val);
446
+	val &= ~clear;
447
+	val |= set;
448
+	pci_write_config_dword(pdev, pos, val);
449
+}
450
+
501451 /* Calculate L1.2 PM substate timing parameters */
502452 static void aspm_calc_l1ss_info(struct pcie_link_state *link,
503
-				struct aspm_register_info *upreg,
504
-				struct aspm_register_info *dwreg)
453
+				u32 parent_l1ss_cap, u32 child_l1ss_cap)
505454 {
455
+	struct pci_dev *child = link->downstream, *parent = link->pdev;
506456 	u32 val1, val2, scale1, scale2;
507457 	u32 t_common_mode, t_power_on, l1_2_threshold, scale, value;
508
-
509
-	link->l1ss.up_cap_ptr = upreg->l1ss_cap_ptr;
510
-	link->l1ss.dw_cap_ptr = dwreg->l1ss_cap_ptr;
511
-	link->l1ss.ctl1 = link->l1ss.ctl2 = 0;
458
+	u32 ctl1 = 0, ctl2 = 0;
459
+	u32 pctl1, pctl2, cctl1, cctl2;
460
+	u32 pl1_2_enables, cl1_2_enables;
512461 
513462 	if (!(link->aspm_support & ASPM_STATE_L1_2_MASK))
514463 		return;
515464 
516465 	/* Choose the greater of the two Port Common_Mode_Restore_Times */
517
-	val1 = (upreg->l1ss_cap & PCI_L1SS_CAP_CM_RESTORE_TIME) >> 8;
518
-	val2 = (dwreg->l1ss_cap & PCI_L1SS_CAP_CM_RESTORE_TIME) >> 8;
466
+	val1 = (parent_l1ss_cap & PCI_L1SS_CAP_CM_RESTORE_TIME) >> 8;
467
+	val2 = (child_l1ss_cap & PCI_L1SS_CAP_CM_RESTORE_TIME) >> 8;
519468 	t_common_mode = max(val1, val2);
520469 
521470 	/* Choose the greater of the two Port T_POWER_ON times */
522
-	val1   = (upreg->l1ss_cap & PCI_L1SS_CAP_P_PWR_ON_VALUE) >> 19;
523
-	scale1 = (upreg->l1ss_cap & PCI_L1SS_CAP_P_PWR_ON_SCALE) >> 16;
524
-	val2   = (dwreg->l1ss_cap & PCI_L1SS_CAP_P_PWR_ON_VALUE) >> 19;
525
-	scale2 = (dwreg->l1ss_cap & PCI_L1SS_CAP_P_PWR_ON_SCALE) >> 16;
471
+	val1   = (parent_l1ss_cap & PCI_L1SS_CAP_P_PWR_ON_VALUE) >> 19;
472
+	scale1 = (parent_l1ss_cap & PCI_L1SS_CAP_P_PWR_ON_SCALE) >> 16;
473
+	val2   = (child_l1ss_cap & PCI_L1SS_CAP_P_PWR_ON_VALUE) >> 19;
474
+	scale2 = (child_l1ss_cap & PCI_L1SS_CAP_P_PWR_ON_SCALE) >> 16;
526475 
527
-	if (calc_l1ss_pwron(link->pdev, scale1, val1) >
528
-	    calc_l1ss_pwron(link->downstream, scale2, val2)) {
529
-		link->l1ss.ctl2 |= scale1 | (val1 << 3);
530
-		t_power_on = calc_l1ss_pwron(link->pdev, scale1, val1);
476
+	if (calc_l1ss_pwron(parent, scale1, val1) >
477
+	    calc_l1ss_pwron(child, scale2, val2)) {
478
+		ctl2 |= scale1 | (val1 << 3);
479
+		t_power_on = calc_l1ss_pwron(parent, scale1, val1);
531480 	} else {
532
-		link->l1ss.ctl2 |= scale2 | (val2 << 3);
533
-		t_power_on = calc_l1ss_pwron(link->downstream, scale2, val2);
481
+		ctl2 |= scale2 | (val2 << 3);
482
+		t_power_on = calc_l1ss_pwron(child, scale2, val2);
534483 	}
535484 
536485 	/*
....@@ -545,14 +494,60 @@
545494 	 */
546495 	l1_2_threshold = 2 + 4 + t_common_mode + t_power_on;
547496 	encode_l12_threshold(l1_2_threshold, &scale, &value);
548
-	link->l1ss.ctl1 |= t_common_mode << 8 | scale << 29 | value << 16;
497
+	ctl1 |= t_common_mode << 8 | scale << 29 | value << 16;
498
+
499
+	pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1, &pctl1);
500
+	pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CTL2, &pctl2);
501
+	pci_read_config_dword(child, child->l1ss + PCI_L1SS_CTL1, &cctl1);
502
+	pci_read_config_dword(child, child->l1ss + PCI_L1SS_CTL2, &cctl2);
503
+
504
+	if (ctl1 == pctl1 && ctl1 == cctl1 &&
505
+	    ctl2 == pctl2 && ctl2 == cctl2)
506
+		return;
507
+
508
+	/* Disable L1.2 while updating.  See PCIe r5.0, sec 5.5.4, 7.8.3.3 */
509
+	pl1_2_enables = pctl1 & PCI_L1SS_CTL1_L1_2_MASK;
510
+	cl1_2_enables = cctl1 & PCI_L1SS_CTL1_L1_2_MASK;
511
+
512
+	if (pl1_2_enables || cl1_2_enables) {
513
+		pci_clear_and_set_dword(child, child->l1ss + PCI_L1SS_CTL1,
514
+					PCI_L1SS_CTL1_L1_2_MASK, 0);
515
+		pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1,
516
+					PCI_L1SS_CTL1_L1_2_MASK, 0);
517
+	}
518
+
519
+	/* Program T_POWER_ON times in both ports */
520
+	pci_write_config_dword(parent, parent->l1ss + PCI_L1SS_CTL2, ctl2);
521
+	pci_write_config_dword(child, child->l1ss + PCI_L1SS_CTL2, ctl2);
522
+
523
+	/* Program Common_Mode_Restore_Time in upstream device */
524
+	pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1,
525
+				PCI_L1SS_CTL1_CM_RESTORE_TIME, ctl1);
526
+
527
+	/* Program LTR_L1.2_THRESHOLD time in both ports */
528
+	pci_clear_and_set_dword(parent,	parent->l1ss + PCI_L1SS_CTL1,
529
+				PCI_L1SS_CTL1_LTR_L12_TH_VALUE |
530
+				PCI_L1SS_CTL1_LTR_L12_TH_SCALE, ctl1);
531
+	pci_clear_and_set_dword(child, child->l1ss + PCI_L1SS_CTL1,
532
+				PCI_L1SS_CTL1_LTR_L12_TH_VALUE |
533
+				PCI_L1SS_CTL1_LTR_L12_TH_SCALE, ctl1);
534
+
535
+	if (pl1_2_enables || cl1_2_enables) {
536
+		pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1, 0,
537
+					pl1_2_enables);
538
+		pci_clear_and_set_dword(child, child->l1ss + PCI_L1SS_CTL1, 0,
539
+					cl1_2_enables);
540
+	}
549541 }
550542 
551543 static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist)
552544 {
553545 	struct pci_dev *child = link->downstream, *parent = link->pdev;
546
+	u32 parent_lnkcap, child_lnkcap;
547
+	u16 parent_lnkctl, child_lnkctl;
548
+	u32 parent_l1ss_cap, child_l1ss_cap;
549
+	u32 parent_l1ss_ctl1 = 0, child_l1ss_ctl1 = 0;
554550 	struct pci_bus *linkbus = parent->subordinate;
555
-	struct aspm_register_info upreg, dwreg;
556551 
557552 	if (blacklist) {
558553 		/* Set enabled/disable so that we will disable ASPM later */
....@@ -561,26 +556,28 @@
561556 		return;
562557 	}
563558 
564
-	/* Get upstream/downstream components' register state */
565
-	pcie_get_aspm_reg(parent, &upreg);
566
-	pcie_get_aspm_reg(child, &dwreg);
567
-
568559 	/*
569560 	 * If ASPM not supported, don't mess with the clocks and link,
570561 	 * bail out now.
571562 	 */
572
-	if (!(upreg.support & dwreg.support))
563
+	pcie_capability_read_dword(parent, PCI_EXP_LNKCAP, &parent_lnkcap);
564
+	pcie_capability_read_dword(child, PCI_EXP_LNKCAP, &child_lnkcap);
565
+	if (!(parent_lnkcap & child_lnkcap & PCI_EXP_LNKCAP_ASPMS))
573566 		return;
574567 
575568 	/* Configure common clock before checking latencies */
576569 	pcie_aspm_configure_common_clock(link);
577570 
578571 	/*
579
-	 * Re-read upstream/downstream components' register state
580
-	 * after clock configuration
572
+	 * Re-read upstream/downstream components' register state after
573
+	 * clock configuration.  L0s & L1 exit latencies in the otherwise
574
+	 * read-only Link Capabilities may change depending on common clock
575
+	 * configuration (PCIe r5.0, sec 7.5.3.6).
581576 	 */
582
-	pcie_get_aspm_reg(parent, &upreg);
583
-	pcie_get_aspm_reg(child, &dwreg);
577
+	pcie_capability_read_dword(parent, PCI_EXP_LNKCAP, &parent_lnkcap);
578
+	pcie_capability_read_dword(child, PCI_EXP_LNKCAP, &child_lnkcap);
579
+	pcie_capability_read_word(parent, PCI_EXP_LNKCTL, &parent_lnkctl);
580
+	pcie_capability_read_word(child, PCI_EXP_LNKCTL, &child_lnkctl);
584581 
585582 	/*
586583 	 * Setup L0s state
....@@ -589,44 +586,71 @@
589586 	 * given link unless components on both sides of the link each
590587 	 * support L0s.
591588 	 */
592
-	if (dwreg.support & upreg.support & PCIE_LINK_STATE_L0S)
589
+	if (parent_lnkcap & child_lnkcap & PCI_EXP_LNKCAP_ASPM_L0S)
593590 		link->aspm_support |= ASPM_STATE_L0S;
594
-	if (dwreg.enabled & PCIE_LINK_STATE_L0S)
591
+
592
+	if (child_lnkctl & PCI_EXP_LNKCTL_ASPM_L0S)
595593 		link->aspm_enabled |= ASPM_STATE_L0S_UP;
596
-	if (upreg.enabled & PCIE_LINK_STATE_L0S)
594
+	if (parent_lnkctl & PCI_EXP_LNKCTL_ASPM_L0S)
597595 		link->aspm_enabled |= ASPM_STATE_L0S_DW;
598
-	link->latency_up.l0s = calc_l0s_latency(upreg.latency_encoding_l0s);
599
-	link->latency_dw.l0s = calc_l0s_latency(dwreg.latency_encoding_l0s);
596
+	link->latency_up.l0s = calc_l0s_latency(parent_lnkcap);
597
+	link->latency_dw.l0s = calc_l0s_latency(child_lnkcap);
600598 
601599 	/* Setup L1 state */
602
-	if (upreg.support & dwreg.support & PCIE_LINK_STATE_L1)
600
+	if (parent_lnkcap & child_lnkcap & PCI_EXP_LNKCAP_ASPM_L1)
603601 		link->aspm_support |= ASPM_STATE_L1;
604
-	if (upreg.enabled & dwreg.enabled & PCIE_LINK_STATE_L1)
602
+
603
+	if (parent_lnkctl & child_lnkctl & PCI_EXP_LNKCTL_ASPM_L1)
605604 		link->aspm_enabled |= ASPM_STATE_L1;
606
-	link->latency_up.l1 = calc_l1_latency(upreg.latency_encoding_l1);
607
-	link->latency_dw.l1 = calc_l1_latency(dwreg.latency_encoding_l1);
605
+	link->latency_up.l1 = calc_l1_latency(parent_lnkcap);
606
+	link->latency_dw.l1 = calc_l1_latency(child_lnkcap);
608607 
609608 	/* Setup L1 substate */
610
-	if (upreg.l1ss_cap & dwreg.l1ss_cap & PCI_L1SS_CAP_ASPM_L1_1)
609
+	pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CAP,
610
+			      &parent_l1ss_cap);
611
+	pci_read_config_dword(child, child->l1ss + PCI_L1SS_CAP,
612
+			      &child_l1ss_cap);
613
+
614
+	if (!(parent_l1ss_cap & PCI_L1SS_CAP_L1_PM_SS))
615
+		parent_l1ss_cap = 0;
616
+	if (!(child_l1ss_cap & PCI_L1SS_CAP_L1_PM_SS))
617
+		child_l1ss_cap = 0;
618
+
619
+	/*
620
+	 * If we don't have LTR for the entire path from the Root Complex
621
+	 * to this device, we can't use ASPM L1.2 because it relies on the
622
+	 * LTR_L1.2_THRESHOLD.  See PCIe r4.0, secs 5.5.4, 6.18.
623
+	 */
624
+	if (!child->ltr_path)
625
+		child_l1ss_cap &= ~PCI_L1SS_CAP_ASPM_L1_2;
626
+
627
+	if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_ASPM_L1_1)
611628 		link->aspm_support |= ASPM_STATE_L1_1;
612
-	if (upreg.l1ss_cap & dwreg.l1ss_cap & PCI_L1SS_CAP_ASPM_L1_2)
629
+	if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_ASPM_L1_2)
613630 		link->aspm_support |= ASPM_STATE_L1_2;
614
-	if (upreg.l1ss_cap & dwreg.l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_1)
631
+	if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_1)
615632 		link->aspm_support |= ASPM_STATE_L1_1_PCIPM;
616
-	if (upreg.l1ss_cap & dwreg.l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_2)
633
+	if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_2)
617634 		link->aspm_support |= ASPM_STATE_L1_2_PCIPM;
618635 
619
-	if (upreg.l1ss_ctl1 & dwreg.l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_1)
636
+	if (parent_l1ss_cap)
637
+		pci_read_config_dword(parent, parent->l1ss + PCI_L1SS_CTL1,
638
+				      &parent_l1ss_ctl1);
639
+	if (child_l1ss_cap)
640
+		pci_read_config_dword(child, child->l1ss + PCI_L1SS_CTL1,
641
+				      &child_l1ss_ctl1);
642
+
643
+	if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_1)
620644 		link->aspm_enabled |= ASPM_STATE_L1_1;
621
-	if (upreg.l1ss_ctl1 & dwreg.l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_2)
645
+	if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_2)
622646 		link->aspm_enabled |= ASPM_STATE_L1_2;
623
-	if (upreg.l1ss_ctl1 & dwreg.l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_1)
647
+	if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_1)
624648 		link->aspm_enabled |= ASPM_STATE_L1_1_PCIPM;
625
-	if (upreg.l1ss_ctl1 & dwreg.l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_2)
649
+	if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_2)
626650 		link->aspm_enabled |= ASPM_STATE_L1_2_PCIPM;
627651 
628652 	if (link->aspm_support & ASPM_STATE_L1SS)
629
-		aspm_calc_l1ss_info(link, &upreg, &dwreg);
653
+		aspm_calc_l1ss_info(link, parent_l1ss_cap, child_l1ss_cap);
630654 
631655 	/* Save default state */
632656 	link->aspm_default = link->aspm_enabled;
....@@ -656,24 +680,11 @@
656680 	}
657681 }
658682 
659
-static void pci_clear_and_set_dword(struct pci_dev *pdev, int pos,
660
-				    u32 clear, u32 set)
661
-{
662
-	u32 val;
663
-
664
-	pci_read_config_dword(pdev, pos, &val);
665
-	val &= ~clear;
666
-	val |= set;
667
-	pci_write_config_dword(pdev, pos, val);
668
-}
669
-
670683 /* Configure the ASPM L1 substates */
671684 static void pcie_config_aspm_l1ss(struct pcie_link_state *link, u32 state)
672685 {
673686 	u32 val, enable_req;
674687 	struct pci_dev *child = link->downstream, *parent = link->pdev;
675
-	u32 up_cap_ptr = link->l1ss.up_cap_ptr;
676
-	u32 dw_cap_ptr = link->l1ss.dw_cap_ptr;
677688 
678689 	enable_req = (link->aspm_enabled ^ state) & state;
679690 
....@@ -691,9 +702,9 @@
691702 	 */
692703 
693704 	/* Disable all L1 substates */
694
-	pci_clear_and_set_dword(child, dw_cap_ptr + PCI_L1SS_CTL1,
705
+	pci_clear_and_set_dword(child, child->l1ss + PCI_L1SS_CTL1,
695706 				PCI_L1SS_CTL1_L1SS_MASK, 0);
696
-	pci_clear_and_set_dword(parent, up_cap_ptr + PCI_L1SS_CTL1,
707
+	pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1,
697708 				PCI_L1SS_CTL1_L1SS_MASK, 0);
698709 	/*
699710 	 * If needed, disable L1, and it gets enabled later
....@@ -704,30 +715,6 @@
704715 						   PCI_EXP_LNKCTL_ASPM_L1, 0);
705716 		pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL,
706717 						   PCI_EXP_LNKCTL_ASPM_L1, 0);
707
-	}
708
-
709
-	if (enable_req & ASPM_STATE_L1_2_MASK) {
710
-
711
-		/* Program T_POWER_ON times in both ports */
712
-		pci_write_config_dword(parent, up_cap_ptr + PCI_L1SS_CTL2,
713
-				       link->l1ss.ctl2);
714
-		pci_write_config_dword(child, dw_cap_ptr + PCI_L1SS_CTL2,
715
-				       link->l1ss.ctl2);
716
-
717
-		/* Program Common_Mode_Restore_Time in upstream device */
718
-		pci_clear_and_set_dword(parent, up_cap_ptr + PCI_L1SS_CTL1,
719
-					PCI_L1SS_CTL1_CM_RESTORE_TIME,
720
-					link->l1ss.ctl1);
721
-
722
-		/* Program LTR_L1.2_THRESHOLD time in both ports */
723
-		pci_clear_and_set_dword(parent,	up_cap_ptr + PCI_L1SS_CTL1,
724
-					PCI_L1SS_CTL1_LTR_L12_TH_VALUE |
725
-					PCI_L1SS_CTL1_LTR_L12_TH_SCALE,
726
-					link->l1ss.ctl1);
727
-		pci_clear_and_set_dword(child, dw_cap_ptr + PCI_L1SS_CTL1,
728
-					PCI_L1SS_CTL1_LTR_L12_TH_VALUE |
729
-					PCI_L1SS_CTL1_LTR_L12_TH_SCALE,
730
-					link->l1ss.ctl1);
731718 	}
732719 
733720 	val = 0;
....@@ -741,9 +728,9 @@
741728 		val |= PCI_L1SS_CTL1_PCIPM_L1_2;
742729 
743730 	/* Enable what we need to enable */
744
-	pci_clear_and_set_dword(parent, up_cap_ptr + PCI_L1SS_CTL1,
731
+	pci_clear_and_set_dword(parent, parent->l1ss + PCI_L1SS_CTL1,
745732 				PCI_L1SS_CTL1_L1SS_MASK, val);
746
-	pci_clear_and_set_dword(child, dw_cap_ptr + PCI_L1SS_CTL1,
733
+	pci_clear_and_set_dword(child, child->l1ss + PCI_L1SS_CTL1,
747734 				PCI_L1SS_CTL1_L1SS_MASK, val);
748735 }
749736 
....@@ -862,8 +849,6 @@
862849 		return NULL;
863850 
864851 	INIT_LIST_HEAD(&link->sibling);
865
-	INIT_LIST_HEAD(&link->children);
866
-	INIT_LIST_HEAD(&link->link);
867852 	link->pdev = pdev;
868853 	link->downstream = pci_function_0(pdev->subordinate);
869854 
....@@ -889,12 +874,19 @@
889874 
890875 		link->parent = parent;
891876 		link->root = link->parent->root;
892
-		list_add(&link->link, &parent->children);
893877 	}
894878 
895879 	list_add(&link->sibling, &link_list);
896880 	pdev->link_state = link;
897881 	return link;
882
+}
883
+
884
+static void pcie_aspm_update_sysfs_visibility(struct pci_dev *pdev)
885
+{
886
+	struct pci_dev *child;
887
+
888
+	list_for_each_entry(child, &pdev->subordinate->devices, bus_list)
889
+		sysfs_update_group(&child->dev.kobj, &aspm_ctrl_attr_group);
898890 }
899891 
900892 /*
....@@ -915,10 +907,10 @@
915907 
916908 	/*
917909 	 * We allocate pcie_link_state for the component on the upstream
918
-	 * end of a Link, so there's nothing to do unless this device has a
919
-	 * Link on its secondary side.
910
+	 * end of a Link, so there's nothing to do unless this device is
911
+	 * downstream port.
920912 	 */
921
-	if (!pdev->has_secondary_link)
913
+	if (!pcie_downstream_port(pdev))
922914 		return;
923915 
924916 	/* VIA has a strange chipset, root port is under a bridge */
....@@ -957,6 +949,8 @@
957949 		pcie_config_aspm_path(link);
958950 		pcie_set_clkpm(link, policy_to_clkpm_state(link));
959951 	}
952
+
953
+	pcie_aspm_update_sysfs_visibility(pdev);
960954 
961955 unlock:
962956 	mutex_unlock(&aspm_lock);
....@@ -1013,7 +1007,6 @@
10131007 	/* All functions are removed, so just disable ASPM for the link */
10141008 	pcie_config_aspm_link(link, 0);
10151009 	list_del(&link->sibling);
1016
-	list_del(&link->link);
10171010 	/* Clock PM is for endpoint device */
10181011 	free_link_state(link);
10191012 
....@@ -1065,19 +1058,26 @@
10651058 	up_read(&pci_bus_sem);
10661059 }
10671060 
1068
-static void __pci_disable_link_state(struct pci_dev *pdev, int state, bool sem)
1061
+static struct pcie_link_state *pcie_aspm_get_link(struct pci_dev *pdev)
10691062 {
1070
-	struct pci_dev *parent = pdev->bus->self;
1071
-	struct pcie_link_state *link;
1063
+	struct pci_dev *bridge;
10721064 
10731065 	if (!pci_is_pcie(pdev))
1074
-		return;
1066
+		return NULL;
10751067 
1076
-	if (pdev->has_secondary_link)
1077
-		parent = pdev;
1078
-	if (!parent || !parent->link_state)
1079
-		return;
1068
+	bridge = pci_upstream_bridge(pdev);
1069
+	if (!bridge || !pci_is_pcie(bridge))
1070
+		return NULL;
10801071 
1072
+	return bridge->link_state;
1073
+}
1074
+
1075
+static int __pci_disable_link_state(struct pci_dev *pdev, int state, bool sem)
1076
+{
1077
+	struct pcie_link_state *link = pcie_aspm_get_link(pdev);
1078
+
1079
+	if (!link)
1080
+		return -EINVAL;
10811081 	/*
10821082 	 * A driver requested that ASPM be disabled on this device, but
10831083 	 * if we don't have permission to manage ASPM (e.g., on ACPI
....@@ -1088,17 +1088,25 @@
10881088 	 */
10891089 	if (aspm_disabled) {
10901090 		pci_warn(pdev, "can't disable ASPM; OS doesn't have ASPM control\n");
1091
-		return;
1091
+		return -EPERM;
10921092 	}
10931093 
10941094 	if (sem)
10951095 		down_read(&pci_bus_sem);
10961096 	mutex_lock(&aspm_lock);
1097
-	link = parent->link_state;
10981097 	if (state & PCIE_LINK_STATE_L0S)
10991098 		link->aspm_disable |= ASPM_STATE_L0S;
11001099 	if (state & PCIE_LINK_STATE_L1)
1101
-		link->aspm_disable |= ASPM_STATE_L1;
1100
+		/* L1 PM substates require L1 */
1101
+		link->aspm_disable |= ASPM_STATE_L1 | ASPM_STATE_L1SS;
1102
+	if (state & PCIE_LINK_STATE_L1_1)
1103
+		link->aspm_disable |= ASPM_STATE_L1_1;
1104
+	if (state & PCIE_LINK_STATE_L1_2)
1105
+		link->aspm_disable |= ASPM_STATE_L1_2;
1106
+	if (state & PCIE_LINK_STATE_L1_1_PCIPM)
1107
+		link->aspm_disable |= ASPM_STATE_L1_1_PCIPM;
1108
+	if (state & PCIE_LINK_STATE_L1_2_PCIPM)
1109
+		link->aspm_disable |= ASPM_STATE_L1_2_PCIPM;
11021110 	pcie_config_aspm_link(link, policy_to_aspm_state(link));
11031111 
11041112 	if (state & PCIE_LINK_STATE_CLKPM)
....@@ -1107,11 +1115,13 @@
11071115 	mutex_unlock(&aspm_lock);
11081116 	if (sem)
11091117 		up_read(&pci_bus_sem);
1118
+
1119
+	return 0;
11101120 }
11111121 
1112
-void pci_disable_link_state_locked(struct pci_dev *pdev, int state)
1122
+int pci_disable_link_state_locked(struct pci_dev *pdev, int state)
11131123 {
1114
-	__pci_disable_link_state(pdev, state, false);
1124
+	return __pci_disable_link_state(pdev, state, false);
11151125 }
11161126 EXPORT_SYMBOL(pci_disable_link_state_locked);
11171127 
....@@ -1119,14 +1129,14 @@
11191129  * pci_disable_link_state - Disable device's link state, so the link will
11201130  * never enter specific states.  Note that if the BIOS didn't grant ASPM
11211131  * control to the OS, this does nothing because we can't touch the LNKCTL
1122
- * register.
1132
+ * register. Returns 0 or a negative errno.
11231133  *
11241134  * @pdev: PCI device
11251135  * @state: ASPM link state to disable
11261136  */
1127
-void pci_disable_link_state(struct pci_dev *pdev, int state)
1137
+int pci_disable_link_state(struct pci_dev *pdev, int state)
11281138 {
1129
-	__pci_disable_link_state(pdev, state, true);
1139
+	return __pci_disable_link_state(pdev, state, true);
11301140 }
11311141 EXPORT_SYMBOL(pci_disable_link_state);
11321142 
....@@ -1171,110 +1181,164 @@
11711181 module_param_call(policy, pcie_aspm_set_policy, pcie_aspm_get_policy,
11721182 	NULL, 0644);
11731183 
1174
-#ifdef CONFIG_PCIEASPM_DEBUG
1175
-static ssize_t link_state_show(struct device *dev,
1176
-		struct device_attribute *attr,
1177
-		char *buf)
1184
+/**
1185
+ * pcie_aspm_enabled - Check if PCIe ASPM has been enabled for a device.
1186
+ * @pdev: Target device.
1187
+ *
1188
+ * Relies on the upstream bridge's link_state being valid.  The link_state
1189
+ * is deallocated only when the last child of the bridge (i.e., @pdev or a
1190
+ * sibling) is removed, and the caller should be holding a reference to
1191
+ * @pdev, so this should be safe.
1192
+ */
1193
+bool pcie_aspm_enabled(struct pci_dev *pdev)
11781194 {
1179
-	struct pci_dev *pci_device = to_pci_dev(dev);
1180
-	struct pcie_link_state *link_state = pci_device->link_state;
1195
+	struct pcie_link_state *link = pcie_aspm_get_link(pdev);
11811196 
1182
-	return sprintf(buf, "%d\n", link_state->aspm_enabled);
1197
+	if (!link)
1198
+		return false;
1199
+
1200
+	return link->aspm_enabled;
11831201 }
1202
+EXPORT_SYMBOL_GPL(pcie_aspm_enabled);
11841203 
1185
-static ssize_t link_state_store(struct device *dev,
1186
-		struct device_attribute *attr,
1187
-		const char *buf,
1188
-		size_t n)
1204
+static ssize_t aspm_attr_show_common(struct device *dev,
1205
+				     struct device_attribute *attr,
1206
+				     char *buf, u8 state)
11891207 {
11901208 	struct pci_dev *pdev = to_pci_dev(dev);
1191
-	struct pcie_link_state *link, *root = pdev->link_state->root;
1192
-	u32 state;
1209
+	struct pcie_link_state *link = pcie_aspm_get_link(pdev);
11931210 
1194
-	if (aspm_disabled)
1195
-		return -EPERM;
1211
+	return sprintf(buf, "%d\n", (link->aspm_enabled & state) ? 1 : 0);
1212
+}
11961213 
1197
-	if (kstrtouint(buf, 10, &state))
1198
-		return -EINVAL;
1199
-	if ((state & ~ASPM_STATE_ALL) != 0)
1214
+static ssize_t aspm_attr_store_common(struct device *dev,
1215
+				      struct device_attribute *attr,
1216
+				      const char *buf, size_t len, u8 state)
1217
+{
1218
+	struct pci_dev *pdev = to_pci_dev(dev);
1219
+	struct pcie_link_state *link = pcie_aspm_get_link(pdev);
1220
+	bool state_enable;
1221
+
1222
+	if (strtobool(buf, &state_enable) < 0)
12001223 		return -EINVAL;
12011224 
12021225 	down_read(&pci_bus_sem);
12031226 	mutex_lock(&aspm_lock);
1204
-	list_for_each_entry(link, &link_list, sibling) {
1205
-		if (link->root != root)
1206
-			continue;
1207
-		pcie_config_aspm_link(link, state);
1227
+
1228
+	if (state_enable) {
1229
+		link->aspm_disable &= ~state;
1230
+		/* need to enable L1 for substates */
1231
+		if (state & ASPM_STATE_L1SS)
1232
+			link->aspm_disable &= ~ASPM_STATE_L1;
1233
+	} else {
1234
+		link->aspm_disable |= state;
12081235 	}
1236
+
1237
+	pcie_config_aspm_link(link, policy_to_aspm_state(link));
1238
+
12091239 	mutex_unlock(&aspm_lock);
12101240 	up_read(&pci_bus_sem);
1211
-	return n;
1241
+
1242
+	return len;
12121243 }
12131244 
1214
-static ssize_t clk_ctl_show(struct device *dev,
1215
-		struct device_attribute *attr,
1216
-		char *buf)
1217
-{
1218
-	struct pci_dev *pci_device = to_pci_dev(dev);
1219
-	struct pcie_link_state *link_state = pci_device->link_state;
1245
+#define ASPM_ATTR(_f, _s)						\
1246
+static ssize_t _f##_show(struct device *dev,				\
1247
+			 struct device_attribute *attr, char *buf)	\
1248
+{ return aspm_attr_show_common(dev, attr, buf, ASPM_STATE_##_s); }	\
1249
+									\
1250
+static ssize_t _f##_store(struct device *dev,				\
1251
+			  struct device_attribute *attr,		\
1252
+			  const char *buf, size_t len)			\
1253
+{ return aspm_attr_store_common(dev, attr, buf, len, ASPM_STATE_##_s); }
12201254 
1221
-	return sprintf(buf, "%d\n", link_state->clkpm_enabled);
1222
-}
1255
+ASPM_ATTR(l0s_aspm, L0S)
1256
+ASPM_ATTR(l1_aspm, L1)
1257
+ASPM_ATTR(l1_1_aspm, L1_1)
1258
+ASPM_ATTR(l1_2_aspm, L1_2)
1259
+ASPM_ATTR(l1_1_pcipm, L1_1_PCIPM)
1260
+ASPM_ATTR(l1_2_pcipm, L1_2_PCIPM)
12231261 
1224
-static ssize_t clk_ctl_store(struct device *dev,
1225
-		struct device_attribute *attr,
1226
-		const char *buf,
1227
-		size_t n)
1262
+static ssize_t clkpm_show(struct device *dev,
1263
+			  struct device_attribute *attr, char *buf)
12281264 {
12291265 	struct pci_dev *pdev = to_pci_dev(dev);
1230
-	bool state;
1266
+	struct pcie_link_state *link = pcie_aspm_get_link(pdev);
12311267 
1232
-	if (strtobool(buf, &state))
1268
+	return sprintf(buf, "%d\n", link->clkpm_enabled);
1269
+}
1270
+
1271
+static ssize_t clkpm_store(struct device *dev,
1272
+			   struct device_attribute *attr,
1273
+			   const char *buf, size_t len)
1274
+{
1275
+	struct pci_dev *pdev = to_pci_dev(dev);
1276
+	struct pcie_link_state *link = pcie_aspm_get_link(pdev);
1277
+	bool state_enable;
1278
+
1279
+	if (strtobool(buf, &state_enable) < 0)
12331280 		return -EINVAL;
12341281 
12351282 	down_read(&pci_bus_sem);
12361283 	mutex_lock(&aspm_lock);
1237
-	pcie_set_clkpm_nocheck(pdev->link_state, state);
1284
+
1285
+	link->clkpm_disable = !state_enable;
1286
+	pcie_set_clkpm(link, policy_to_clkpm_state(link));
1287
+
12381288 	mutex_unlock(&aspm_lock);
12391289 	up_read(&pci_bus_sem);
12401290 
1241
-	return n;
1291
+	return len;
12421292 }
12431293 
1244
-static DEVICE_ATTR_RW(link_state);
1245
-static DEVICE_ATTR_RW(clk_ctl);
1294
+static DEVICE_ATTR_RW(clkpm);
1295
+static DEVICE_ATTR_RW(l0s_aspm);
1296
+static DEVICE_ATTR_RW(l1_aspm);
1297
+static DEVICE_ATTR_RW(l1_1_aspm);
1298
+static DEVICE_ATTR_RW(l1_2_aspm);
1299
+static DEVICE_ATTR_RW(l1_1_pcipm);
1300
+static DEVICE_ATTR_RW(l1_2_pcipm);
12461301 
1247
-static char power_group[] = "power";
1248
-void pcie_aspm_create_sysfs_dev_files(struct pci_dev *pdev)
1302
+static struct attribute *aspm_ctrl_attrs[] = {
1303
+	&dev_attr_clkpm.attr,
1304
+	&dev_attr_l0s_aspm.attr,
1305
+	&dev_attr_l1_aspm.attr,
1306
+	&dev_attr_l1_1_aspm.attr,
1307
+	&dev_attr_l1_2_aspm.attr,
1308
+	&dev_attr_l1_1_pcipm.attr,
1309
+	&dev_attr_l1_2_pcipm.attr,
1310
+	NULL
1311
+};
1312
+
1313
+static umode_t aspm_ctrl_attrs_are_visible(struct kobject *kobj,
1314
+					   struct attribute *a, int n)
12491315 {
1250
-	struct pcie_link_state *link_state = pdev->link_state;
1316
+	struct device *dev = kobj_to_dev(kobj);
1317
+	struct pci_dev *pdev = to_pci_dev(dev);
1318
+	struct pcie_link_state *link = pcie_aspm_get_link(pdev);
1319
+	static const u8 aspm_state_map[] = {
1320
+		ASPM_STATE_L0S,
1321
+		ASPM_STATE_L1,
1322
+		ASPM_STATE_L1_1,
1323
+		ASPM_STATE_L1_2,
1324
+		ASPM_STATE_L1_1_PCIPM,
1325
+		ASPM_STATE_L1_2_PCIPM,
1326
+	};
12511327 
1252
-	if (!link_state)
1253
-		return;
1328
+	if (aspm_disabled || !link)
1329
+		return 0;
12541330 
1255
-	if (link_state->aspm_support)
1256
-		sysfs_add_file_to_group(&pdev->dev.kobj,
1257
-			&dev_attr_link_state.attr, power_group);
1258
-	if (link_state->clkpm_capable)
1259
-		sysfs_add_file_to_group(&pdev->dev.kobj,
1260
-			&dev_attr_clk_ctl.attr, power_group);
1331
+	if (n == 0)
1332
+		return link->clkpm_capable ? a->mode : 0;
1333
+
1334
+	return link->aspm_capable & aspm_state_map[n - 1] ? a->mode : 0;
12611335 }
12621336 
1263
-void pcie_aspm_remove_sysfs_dev_files(struct pci_dev *pdev)
1264
-{
1265
-	struct pcie_link_state *link_state = pdev->link_state;
1266
-
1267
-	if (!link_state)
1268
-		return;
1269
-
1270
-	if (link_state->aspm_support)
1271
-		sysfs_remove_file_from_group(&pdev->dev.kobj,
1272
-			&dev_attr_link_state.attr, power_group);
1273
-	if (link_state->clkpm_capable)
1274
-		sysfs_remove_file_from_group(&pdev->dev.kobj,
1275
-			&dev_attr_clk_ctl.attr, power_group);
1276
-}
1277
-#endif
1337
+const struct attribute_group aspm_ctrl_attr_group = {
1338
+	.name = "link",
1339
+	.attrs = aspm_ctrl_attrs,
1340
+	.is_visible = aspm_ctrl_attrs_are_visible,
1341
+};
12781342 
12791343 static int __init pcie_aspm_disable(char *str)
12801344 {