88 */
99
1010#include <linux/bitfield.h>
11+ #include <linux/bits.h>
12+ #include <linux/build_bug.h>
1113#include <linux/kernel.h>
1214#include <linux/limits.h>
1315#include <linux/math.h>
@@ -189,21 +191,18 @@ void pci_restore_aspm_l1ss_state(struct pci_dev *pdev)
189191#endif
190192#define MODULE_PARAM_PREFIX "pcie_aspm."
191193
192- /* Note: those are not register definitions */
193- #define ASPM_STATE_L0S_UP (1) /* Upstream direction L0s state */
194- #define ASPM_STATE_L0S_DW (2) /* Downstream direction L0s state */
195- #define ASPM_STATE_L1 (4) /* L1 state */
196- #define ASPM_STATE_L1_1 (8) /* ASPM L1.1 state */
197- #define ASPM_STATE_L1_2 (0x10) /* ASPM L1.2 state */
198- #define ASPM_STATE_L1_1_PCIPM (0x20) /* PCI PM L1.1 state */
199- #define ASPM_STATE_L1_2_PCIPM (0x40) /* PCI PM L1.2 state */
200- #define ASPM_STATE_L1_SS_PCIPM (ASPM_STATE_L1_1_PCIPM | ASPM_STATE_L1_2_PCIPM)
201- #define ASPM_STATE_L1_2_MASK (ASPM_STATE_L1_2 | ASPM_STATE_L1_2_PCIPM)
202- #define ASPM_STATE_L1SS (ASPM_STATE_L1_1 | ASPM_STATE_L1_1_PCIPM |\
203- ASPM_STATE_L1_2_MASK)
204- #define ASPM_STATE_L0S (ASPM_STATE_L0S_UP | ASPM_STATE_L0S_DW)
205- #define ASPM_STATE_ALL (ASPM_STATE_L0S | ASPM_STATE_L1 | \
206- ASPM_STATE_L1SS)
194+ /* Note: these are not register definitions */
195+ #define PCIE_LINK_STATE_L0S_UP BIT(0) /* Upstream direction L0s state */
196+ #define PCIE_LINK_STATE_L0S_DW BIT(1) /* Downstream direction L0s state */
197+ static_assert (PCIE_LINK_STATE_L0S == (PCIE_LINK_STATE_L0S_UP | PCIE_LINK_STATE_L0S_DW ));
198+
199+ #define PCIE_LINK_STATE_L1_SS_PCIPM (PCIE_LINK_STATE_L1_1_PCIPM |\
200+ PCIE_LINK_STATE_L1_2_PCIPM)
201+ #define PCIE_LINK_STATE_L1_2_MASK (PCIE_LINK_STATE_L1_2 |\
202+ PCIE_LINK_STATE_L1_2_PCIPM)
203+ #define PCIE_LINK_STATE_L1SS (PCIE_LINK_STATE_L1_1 |\
204+ PCIE_LINK_STATE_L1_1_PCIPM |\
205+ PCIE_LINK_STATE_L1_2_MASK)
207206
208207struct pcie_link_state {
209208 struct pci_dev * pdev ; /* Upstream component of the Link */
@@ -275,10 +274,10 @@ static int policy_to_aspm_state(struct pcie_link_state *link)
275274 return 0 ;
276275 case POLICY_POWERSAVE :
277276 /* Enable ASPM L0s/L1 */
278- return ( ASPM_STATE_L0S | ASPM_STATE_L1 ) ;
277+ return PCIE_LINK_STATE_L0S | PCIE_LINK_STATE_L1 ;
279278 case POLICY_POWER_SUPERSAVE :
280279 /* Enable Everything */
281- return ASPM_STATE_ALL ;
280+ return PCIE_LINK_STATE_ASPM_ALL ;
282281 case POLICY_DEFAULT :
283282 return link -> aspm_default ;
284283 }
@@ -581,14 +580,14 @@ static void pcie_aspm_check_latency(struct pci_dev *endpoint)
581580 latency_dw_l1 = calc_l1_latency (lnkcap_dw );
582581
583582 /* Check upstream direction L0s latency */
584- if ((link -> aspm_capable & ASPM_STATE_L0S_UP ) &&
583+ if ((link -> aspm_capable & PCIE_LINK_STATE_L0S_UP ) &&
585584 (latency_up_l0s > acceptable_l0s ))
586- link -> aspm_capable &= ~ASPM_STATE_L0S_UP ;
585+ link -> aspm_capable &= ~PCIE_LINK_STATE_L0S_UP ;
587586
588587 /* Check downstream direction L0s latency */
589- if ((link -> aspm_capable & ASPM_STATE_L0S_DW ) &&
588+ if ((link -> aspm_capable & PCIE_LINK_STATE_L0S_DW ) &&
590589 (latency_dw_l0s > acceptable_l0s ))
591- link -> aspm_capable &= ~ASPM_STATE_L0S_DW ;
590+ link -> aspm_capable &= ~PCIE_LINK_STATE_L0S_DW ;
592591 /*
593592 * Check L1 latency.
594593 * Every switch on the path to root complex need 1
@@ -603,9 +602,9 @@ static void pcie_aspm_check_latency(struct pci_dev *endpoint)
603602 * substate latencies (and hence do not do any check).
604603 */
605604 latency = max_t (u32 , latency_up_l1 , latency_dw_l1 );
606- if ((link -> aspm_capable & ASPM_STATE_L1 ) &&
605+ if ((link -> aspm_capable & PCIE_LINK_STATE_L1 ) &&
607606 (latency + l1_switch_latency > acceptable_l1 ))
608- link -> aspm_capable &= ~ASPM_STATE_L1 ;
607+ link -> aspm_capable &= ~PCIE_LINK_STATE_L1 ;
609608 l1_switch_latency += NSEC_PER_USEC ;
610609
611610 link = link -> parent ;
@@ -741,13 +740,13 @@ static void aspm_l1ss_init(struct pcie_link_state *link)
741740 child_l1ss_cap &= ~PCI_L1SS_CAP_ASPM_L1_2 ;
742741
743742 if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_ASPM_L1_1 )
744- link -> aspm_support |= ASPM_STATE_L1_1 ;
743+ link -> aspm_support |= PCIE_LINK_STATE_L1_1 ;
745744 if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_ASPM_L1_2 )
746- link -> aspm_support |= ASPM_STATE_L1_2 ;
745+ link -> aspm_support |= PCIE_LINK_STATE_L1_2 ;
747746 if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_1 )
748- link -> aspm_support |= ASPM_STATE_L1_1_PCIPM ;
747+ link -> aspm_support |= PCIE_LINK_STATE_L1_1_PCIPM ;
749748 if (parent_l1ss_cap & child_l1ss_cap & PCI_L1SS_CAP_PCIPM_L1_2 )
750- link -> aspm_support |= ASPM_STATE_L1_2_PCIPM ;
749+ link -> aspm_support |= PCIE_LINK_STATE_L1_2_PCIPM ;
751750
752751 if (parent_l1ss_cap )
753752 pci_read_config_dword (parent , parent -> l1ss + PCI_L1SS_CTL1 ,
@@ -757,15 +756,15 @@ static void aspm_l1ss_init(struct pcie_link_state *link)
757756 & child_l1ss_ctl1 );
758757
759758 if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_1 )
760- link -> aspm_enabled |= ASPM_STATE_L1_1 ;
759+ link -> aspm_enabled |= PCIE_LINK_STATE_L1_1 ;
761760 if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_ASPM_L1_2 )
762- link -> aspm_enabled |= ASPM_STATE_L1_2 ;
761+ link -> aspm_enabled |= PCIE_LINK_STATE_L1_2 ;
763762 if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_1 )
764- link -> aspm_enabled |= ASPM_STATE_L1_1_PCIPM ;
763+ link -> aspm_enabled |= PCIE_LINK_STATE_L1_1_PCIPM ;
765764 if (parent_l1ss_ctl1 & child_l1ss_ctl1 & PCI_L1SS_CTL1_PCIPM_L1_2 )
766- link -> aspm_enabled |= ASPM_STATE_L1_2_PCIPM ;
765+ link -> aspm_enabled |= PCIE_LINK_STATE_L1_2_PCIPM ;
767766
768- if (link -> aspm_support & ASPM_STATE_L1_2_MASK )
767+ if (link -> aspm_support & PCIE_LINK_STATE_L1_2_MASK )
769768 aspm_calc_l12_info (link , parent_l1ss_cap , child_l1ss_cap );
770769}
771770
@@ -778,8 +777,8 @@ static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist)
778777
779778 if (blacklist ) {
780779 /* Set enabled/disable so that we will disable ASPM later */
781- link -> aspm_enabled = ASPM_STATE_ALL ;
782- link -> aspm_disable = ASPM_STATE_ALL ;
780+ link -> aspm_enabled = PCIE_LINK_STATE_ASPM_ALL ;
781+ link -> aspm_disable = PCIE_LINK_STATE_ASPM_ALL ;
783782 return ;
784783 }
785784
@@ -814,19 +813,19 @@ static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist)
814813 * support L0s.
815814 */
816815 if (parent_lnkcap & child_lnkcap & PCI_EXP_LNKCAP_ASPM_L0S )
817- link -> aspm_support |= ASPM_STATE_L0S ;
816+ link -> aspm_support |= PCIE_LINK_STATE_L0S ;
818817
819818 if (child_lnkctl & PCI_EXP_LNKCTL_ASPM_L0S )
820- link -> aspm_enabled |= ASPM_STATE_L0S_UP ;
819+ link -> aspm_enabled |= PCIE_LINK_STATE_L0S_UP ;
821820 if (parent_lnkctl & PCI_EXP_LNKCTL_ASPM_L0S )
822- link -> aspm_enabled |= ASPM_STATE_L0S_DW ;
821+ link -> aspm_enabled |= PCIE_LINK_STATE_L0S_DW ;
823822
824823 /* Setup L1 state */
825824 if (parent_lnkcap & child_lnkcap & PCI_EXP_LNKCAP_ASPM_L1 )
826- link -> aspm_support |= ASPM_STATE_L1 ;
825+ link -> aspm_support |= PCIE_LINK_STATE_L1 ;
827826
828827 if (parent_lnkctl & child_lnkctl & PCI_EXP_LNKCTL_ASPM_L1 )
829- link -> aspm_enabled |= ASPM_STATE_L1 ;
828+ link -> aspm_enabled |= PCIE_LINK_STATE_L1 ;
830829
831830 aspm_l1ss_init (link );
832831
@@ -876,21 +875,21 @@ static void pcie_config_aspm_l1ss(struct pcie_link_state *link, u32 state)
876875 * If needed, disable L1, and it gets enabled later
877876 * in pcie_config_aspm_link().
878877 */
879- if (enable_req & (ASPM_STATE_L1_1 | ASPM_STATE_L1_2 )) {
878+ if (enable_req & (PCIE_LINK_STATE_L1_1 | PCIE_LINK_STATE_L1_2 )) {
880879 pcie_capability_clear_word (child , PCI_EXP_LNKCTL ,
881880 PCI_EXP_LNKCTL_ASPM_L1 );
882881 pcie_capability_clear_word (parent , PCI_EXP_LNKCTL ,
883882 PCI_EXP_LNKCTL_ASPM_L1 );
884883 }
885884
886885 val = 0 ;
887- if (state & ASPM_STATE_L1_1 )
886+ if (state & PCIE_LINK_STATE_L1_1 )
888887 val |= PCI_L1SS_CTL1_ASPM_L1_1 ;
889- if (state & ASPM_STATE_L1_2 )
888+ if (state & PCIE_LINK_STATE_L1_2 )
890889 val |= PCI_L1SS_CTL1_ASPM_L1_2 ;
891- if (state & ASPM_STATE_L1_1_PCIPM )
890+ if (state & PCIE_LINK_STATE_L1_1_PCIPM )
892891 val |= PCI_L1SS_CTL1_PCIPM_L1_1 ;
893- if (state & ASPM_STATE_L1_2_PCIPM )
892+ if (state & PCIE_LINK_STATE_L1_2_PCIPM )
894893 val |= PCI_L1SS_CTL1_PCIPM_L1_2 ;
895894
896895 /* Enable what we need to enable */
@@ -916,29 +915,29 @@ static void pcie_config_aspm_link(struct pcie_link_state *link, u32 state)
916915 state &= (link -> aspm_capable & ~link -> aspm_disable );
917916
918917 /* Can't enable any substates if L1 is not enabled */
919- if (!(state & ASPM_STATE_L1 ))
920- state &= ~ASPM_STATE_L1SS ;
918+ if (!(state & PCIE_LINK_STATE_L1 ))
919+ state &= ~PCIE_LINK_STATE_L1SS ;
921920
922921 /* Spec says both ports must be in D0 before enabling PCI PM substates*/
923922 if (parent -> current_state != PCI_D0 || child -> current_state != PCI_D0 ) {
924- state &= ~ASPM_STATE_L1_SS_PCIPM ;
925- state |= (link -> aspm_enabled & ASPM_STATE_L1_SS_PCIPM );
923+ state &= ~PCIE_LINK_STATE_L1_SS_PCIPM ;
924+ state |= (link -> aspm_enabled & PCIE_LINK_STATE_L1_SS_PCIPM );
926925 }
927926
928927 /* Nothing to do if the link is already in the requested state */
929928 if (link -> aspm_enabled == state )
930929 return ;
931930 /* Convert ASPM state to upstream/downstream ASPM register state */
932- if (state & ASPM_STATE_L0S_UP )
931+ if (state & PCIE_LINK_STATE_L0S_UP )
933932 dwstream |= PCI_EXP_LNKCTL_ASPM_L0S ;
934- if (state & ASPM_STATE_L0S_DW )
933+ if (state & PCIE_LINK_STATE_L0S_DW )
935934 upstream |= PCI_EXP_LNKCTL_ASPM_L0S ;
936- if (state & ASPM_STATE_L1 ) {
935+ if (state & PCIE_LINK_STATE_L1 ) {
937936 upstream |= PCI_EXP_LNKCTL_ASPM_L1 ;
938937 dwstream |= PCI_EXP_LNKCTL_ASPM_L1 ;
939938 }
940939
941- if (link -> aspm_capable & ASPM_STATE_L1SS )
940+ if (link -> aspm_capable & PCIE_LINK_STATE_L1SS )
942941 pcie_config_aspm_l1ss (link , state );
943942
944943 /*
@@ -947,11 +946,11 @@ static void pcie_config_aspm_link(struct pcie_link_state *link, u32 state)
947946 * upstream component first and then downstream, and vice
948947 * versa for disabling ASPM L1. Spec doesn't mention L0S.
949948 */
950- if (state & ASPM_STATE_L1 )
949+ if (state & PCIE_LINK_STATE_L1 )
951950 pcie_config_aspm_dev (parent , upstream );
952951 list_for_each_entry (child , & linkbus -> devices , bus_list )
953952 pcie_config_aspm_dev (child , dwstream );
954- if (!(state & ASPM_STATE_L1 ))
953+ if (!(state & PCIE_LINK_STATE_L1 ))
955954 pcie_config_aspm_dev (parent , upstream );
956955
957956 link -> aspm_enabled = state ;
@@ -1324,6 +1323,28 @@ static struct pcie_link_state *pcie_aspm_get_link(struct pci_dev *pdev)
13241323 return bridge -> link_state ;
13251324}
13261325
1326+ static u8 pci_calc_aspm_disable_mask (int state )
1327+ {
1328+ state &= ~PCIE_LINK_STATE_CLKPM ;
1329+
1330+ /* L1 PM substates require L1 */
1331+ if (state & PCIE_LINK_STATE_L1 )
1332+ state |= PCIE_LINK_STATE_L1SS ;
1333+
1334+ return state ;
1335+ }
1336+
1337+ static u8 pci_calc_aspm_enable_mask (int state )
1338+ {
1339+ state &= ~PCIE_LINK_STATE_CLKPM ;
1340+
1341+ /* L1 PM substates require L1 */
1342+ if (state & PCIE_LINK_STATE_L1SS )
1343+ state |= PCIE_LINK_STATE_L1 ;
1344+
1345+ return state ;
1346+ }
1347+
13271348static int __pci_disable_link_state (struct pci_dev * pdev , int state , bool locked )
13281349{
13291350 struct pcie_link_state * link = pcie_aspm_get_link (pdev );
@@ -1346,19 +1367,7 @@ static int __pci_disable_link_state(struct pci_dev *pdev, int state, bool locked
13461367 if (!locked )
13471368 down_read (& pci_bus_sem );
13481369 mutex_lock (& aspm_lock );
1349- if (state & PCIE_LINK_STATE_L0S )
1350- link -> aspm_disable |= ASPM_STATE_L0S ;
1351- if (state & PCIE_LINK_STATE_L1 )
1352- /* L1 PM substates require L1 */
1353- link -> aspm_disable |= ASPM_STATE_L1 | ASPM_STATE_L1SS ;
1354- if (state & PCIE_LINK_STATE_L1_1 )
1355- link -> aspm_disable |= ASPM_STATE_L1_1 ;
1356- if (state & PCIE_LINK_STATE_L1_2 )
1357- link -> aspm_disable |= ASPM_STATE_L1_2 ;
1358- if (state & PCIE_LINK_STATE_L1_1_PCIPM )
1359- link -> aspm_disable |= ASPM_STATE_L1_1_PCIPM ;
1360- if (state & PCIE_LINK_STATE_L1_2_PCIPM )
1361- link -> aspm_disable |= ASPM_STATE_L1_2_PCIPM ;
1370+ link -> aspm_disable |= pci_calc_aspm_disable_mask (state );
13621371 pcie_config_aspm_link (link , policy_to_aspm_state (link ));
13631372
13641373 if (state & PCIE_LINK_STATE_CLKPM )
@@ -1414,20 +1423,7 @@ static int __pci_enable_link_state(struct pci_dev *pdev, int state, bool locked)
14141423 if (!locked )
14151424 down_read (& pci_bus_sem );
14161425 mutex_lock (& aspm_lock );
1417- link -> aspm_default = 0 ;
1418- if (state & PCIE_LINK_STATE_L0S )
1419- link -> aspm_default |= ASPM_STATE_L0S ;
1420- if (state & PCIE_LINK_STATE_L1 )
1421- link -> aspm_default |= ASPM_STATE_L1 ;
1422- /* L1 PM substates require L1 */
1423- if (state & PCIE_LINK_STATE_L1_1 )
1424- link -> aspm_default |= ASPM_STATE_L1_1 | ASPM_STATE_L1 ;
1425- if (state & PCIE_LINK_STATE_L1_2 )
1426- link -> aspm_default |= ASPM_STATE_L1_2 | ASPM_STATE_L1 ;
1427- if (state & PCIE_LINK_STATE_L1_1_PCIPM )
1428- link -> aspm_default |= ASPM_STATE_L1_1_PCIPM | ASPM_STATE_L1 ;
1429- if (state & PCIE_LINK_STATE_L1_2_PCIPM )
1430- link -> aspm_default |= ASPM_STATE_L1_2_PCIPM | ASPM_STATE_L1 ;
1426+ link -> aspm_default = pci_calc_aspm_enable_mask (state );
14311427 pcie_config_aspm_link (link , policy_to_aspm_state (link ));
14321428
14331429 link -> clkpm_default = (state & PCIE_LINK_STATE_CLKPM ) ? 1 : 0 ;
@@ -1563,12 +1559,12 @@ static ssize_t aspm_attr_store_common(struct device *dev,
15631559 if (state_enable ) {
15641560 link -> aspm_disable &= ~state ;
15651561 /* need to enable L1 for substates */
1566- if (state & ASPM_STATE_L1SS )
1567- link -> aspm_disable &= ~ASPM_STATE_L1 ;
1562+ if (state & PCIE_LINK_STATE_L1SS )
1563+ link -> aspm_disable &= ~PCIE_LINK_STATE_L1 ;
15681564 } else {
15691565 link -> aspm_disable |= state ;
1570- if (state & ASPM_STATE_L1 )
1571- link -> aspm_disable |= ASPM_STATE_L1SS ;
1566+ if (state & PCIE_LINK_STATE_L1 )
1567+ link -> aspm_disable |= PCIE_LINK_STATE_L1SS ;
15721568 }
15731569
15741570 pcie_config_aspm_link (link , policy_to_aspm_state (link ));
@@ -1582,12 +1578,12 @@ static ssize_t aspm_attr_store_common(struct device *dev,
15821578#define ASPM_ATTR (_f , _s ) \
15831579static ssize_t _f##_show(struct device *dev, \
15841580 struct device_attribute *attr, char *buf) \
1585- { return aspm_attr_show_common(dev, attr, buf, ASPM_STATE_ ##_s); } \
1581+ { return aspm_attr_show_common(dev, attr, buf, PCIE_LINK_STATE_ ##_s); } \
15861582 \
15871583static ssize_t _f##_store(struct device *dev, \
15881584 struct device_attribute *attr, \
15891585 const char *buf, size_t len) \
1590- { return aspm_attr_store_common(dev, attr, buf, len, ASPM_STATE_ ##_s); }
1586+ { return aspm_attr_store_common(dev, attr, buf, len, PCIE_LINK_STATE_ ##_s); }
15911587
15921588ASPM_ATTR (l0s_aspm , L0S )
15931589ASPM_ATTR (l1_aspm , L1 )
@@ -1654,12 +1650,12 @@ static umode_t aspm_ctrl_attrs_are_visible(struct kobject *kobj,
16541650 struct pci_dev * pdev = to_pci_dev (dev );
16551651 struct pcie_link_state * link = pcie_aspm_get_link (pdev );
16561652 static const u8 aspm_state_map [] = {
1657- ASPM_STATE_L0S ,
1658- ASPM_STATE_L1 ,
1659- ASPM_STATE_L1_1 ,
1660- ASPM_STATE_L1_2 ,
1661- ASPM_STATE_L1_1_PCIPM ,
1662- ASPM_STATE_L1_2_PCIPM ,
1653+ PCIE_LINK_STATE_L0S ,
1654+ PCIE_LINK_STATE_L1 ,
1655+ PCIE_LINK_STATE_L1_1 ,
1656+ PCIE_LINK_STATE_L1_2 ,
1657+ PCIE_LINK_STATE_L1_1_PCIPM ,
1658+ PCIE_LINK_STATE_L1_2_PCIPM ,
16631659 };
16641660
16651661 if (aspm_disabled || !link )
0 commit comments