Skip to content

Commit 7c10dd0

Browse files
Ravi BangoriaPeter Zijlstra
Ravi Bangoria
authored and
Peter Zijlstra
committed
perf/x86/amd: Support PERF_SAMPLE_DATA_SRC
struct perf_mem_data_src is used to pass arch specific memory access details into generic form. These details gets consumed by tools like perf mem and c2c. IBS tagged load/store sample provides most of the information needed for these tools. Add a logic to convert IBS specific raw data into perf_mem_data_src. Signed-off-by: Ravi Bangoria <ravi.bangoria@amd.com> Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Link: https://lkml.kernel.org/r/20220928095805.596-4-ravi.bangoria@amd.com
1 parent 610c238 commit 7c10dd0

1 file changed

Lines changed: 312 additions & 6 deletions

File tree

‎arch/x86/events/amd/ibs.c‎

Lines changed: 312 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -678,6 +678,312 @@ static struct perf_ibs perf_ibs_op = {
678678
.get_count = get_ibs_op_count,
679679
};
680680

681+
static void perf_ibs_get_mem_op(union ibs_op_data3 *op_data3,
682+
struct perf_sample_data *data)
683+
{
684+
union perf_mem_data_src *data_src = &data->data_src;
685+
686+
data_src->mem_op = PERF_MEM_OP_NA;
687+
688+
if (op_data3->ld_op)
689+
data_src->mem_op = PERF_MEM_OP_LOAD;
690+
else if (op_data3->st_op)
691+
data_src->mem_op = PERF_MEM_OP_STORE;
692+
}
693+
694+
/*
695+
* Processors having CPUID_Fn8000001B_EAX[11] aka IBS_CAPS_ZEN4 has
696+
* more fine granular DataSrc encodings. Others have coarse.
697+
*/
698+
static u8 perf_ibs_data_src(union ibs_op_data2 *op_data2)
699+
{
700+
if (ibs_caps & IBS_CAPS_ZEN4)
701+
return (op_data2->data_src_hi << 3) | op_data2->data_src_lo;
702+
703+
return op_data2->data_src_lo;
704+
}
705+
706+
static void perf_ibs_get_mem_lvl(union ibs_op_data2 *op_data2,
707+
union ibs_op_data3 *op_data3,
708+
struct perf_sample_data *data)
709+
{
710+
union perf_mem_data_src *data_src = &data->data_src;
711+
u8 ibs_data_src = perf_ibs_data_src(op_data2);
712+
713+
data_src->mem_lvl = 0;
714+
715+
/*
716+
* DcMiss, L2Miss, DataSrc, DcMissLat etc. are all invalid for Uncached
717+
* memory accesses. So, check DcUcMemAcc bit early.
718+
*/
719+
if (op_data3->dc_uc_mem_acc && ibs_data_src != IBS_DATA_SRC_EXT_IO) {
720+
data_src->mem_lvl = PERF_MEM_LVL_UNC | PERF_MEM_LVL_HIT;
721+
return;
722+
}
723+
724+
/* L1 Hit */
725+
if (op_data3->dc_miss == 0) {
726+
data_src->mem_lvl = PERF_MEM_LVL_L1 | PERF_MEM_LVL_HIT;
727+
return;
728+
}
729+
730+
/* L2 Hit */
731+
if (op_data3->l2_miss == 0) {
732+
/* Erratum #1293 */
733+
if (boot_cpu_data.x86 != 0x19 || boot_cpu_data.x86_model > 0xF ||
734+
!(op_data3->sw_pf || op_data3->dc_miss_no_mab_alloc)) {
735+
data_src->mem_lvl = PERF_MEM_LVL_L2 | PERF_MEM_LVL_HIT;
736+
return;
737+
}
738+
}
739+
740+
/*
741+
* OP_DATA2 is valid only for load ops. Skip all checks which
742+
* uses OP_DATA2[DataSrc].
743+
*/
744+
if (data_src->mem_op != PERF_MEM_OP_LOAD)
745+
goto check_mab;
746+
747+
/* L3 Hit */
748+
if (ibs_caps & IBS_CAPS_ZEN4) {
749+
if (ibs_data_src == IBS_DATA_SRC_EXT_LOC_CACHE) {
750+
data_src->mem_lvl = PERF_MEM_LVL_L3 | PERF_MEM_LVL_HIT;
751+
return;
752+
}
753+
} else {
754+
if (ibs_data_src == IBS_DATA_SRC_LOC_CACHE) {
755+
data_src->mem_lvl = PERF_MEM_LVL_L3 | PERF_MEM_LVL_REM_CCE1 |
756+
PERF_MEM_LVL_HIT;
757+
return;
758+
}
759+
}
760+
761+
/* A peer cache in a near CCX */
762+
if (ibs_caps & IBS_CAPS_ZEN4 &&
763+
ibs_data_src == IBS_DATA_SRC_EXT_NEAR_CCX_CACHE) {
764+
data_src->mem_lvl = PERF_MEM_LVL_REM_CCE1 | PERF_MEM_LVL_HIT;
765+
return;
766+
}
767+
768+
/* A peer cache in a far CCX */
769+
if (ibs_caps & IBS_CAPS_ZEN4) {
770+
if (ibs_data_src == IBS_DATA_SRC_EXT_FAR_CCX_CACHE) {
771+
data_src->mem_lvl = PERF_MEM_LVL_REM_CCE2 | PERF_MEM_LVL_HIT;
772+
return;
773+
}
774+
} else {
775+
if (ibs_data_src == IBS_DATA_SRC_REM_CACHE) {
776+
data_src->mem_lvl = PERF_MEM_LVL_REM_CCE2 | PERF_MEM_LVL_HIT;
777+
return;
778+
}
779+
}
780+
781+
/* DRAM */
782+
if (ibs_data_src == IBS_DATA_SRC_EXT_DRAM) {
783+
if (op_data2->rmt_node == 0)
784+
data_src->mem_lvl = PERF_MEM_LVL_LOC_RAM | PERF_MEM_LVL_HIT;
785+
else
786+
data_src->mem_lvl = PERF_MEM_LVL_REM_RAM1 | PERF_MEM_LVL_HIT;
787+
return;
788+
}
789+
790+
/* PMEM */
791+
if (ibs_caps & IBS_CAPS_ZEN4 && ibs_data_src == IBS_DATA_SRC_EXT_PMEM) {
792+
data_src->mem_lvl_num = PERF_MEM_LVLNUM_PMEM;
793+
if (op_data2->rmt_node) {
794+
data_src->mem_remote = PERF_MEM_REMOTE_REMOTE;
795+
/* IBS doesn't provide Remote socket detail */
796+
data_src->mem_hops = PERF_MEM_HOPS_1;
797+
}
798+
return;
799+
}
800+
801+
/* Extension Memory */
802+
if (ibs_caps & IBS_CAPS_ZEN4 &&
803+
ibs_data_src == IBS_DATA_SRC_EXT_EXT_MEM) {
804+
data_src->mem_lvl_num = PERF_MEM_LVLNUM_EXTN_MEM;
805+
if (op_data2->rmt_node) {
806+
data_src->mem_remote = PERF_MEM_REMOTE_REMOTE;
807+
/* IBS doesn't provide Remote socket detail */
808+
data_src->mem_hops = PERF_MEM_HOPS_1;
809+
}
810+
return;
811+
}
812+
813+
/* IO */
814+
if (ibs_data_src == IBS_DATA_SRC_EXT_IO) {
815+
data_src->mem_lvl = PERF_MEM_LVL_IO;
816+
data_src->mem_lvl_num = PERF_MEM_LVLNUM_IO;
817+
if (op_data2->rmt_node) {
818+
data_src->mem_remote = PERF_MEM_REMOTE_REMOTE;
819+
/* IBS doesn't provide Remote socket detail */
820+
data_src->mem_hops = PERF_MEM_HOPS_1;
821+
}
822+
return;
823+
}
824+
825+
check_mab:
826+
/*
827+
* MAB (Miss Address Buffer) Hit. MAB keeps track of outstanding
828+
* DC misses. However, such data may come from any level in mem
829+
* hierarchy. IBS provides detail about both MAB as well as actual
830+
* DataSrc simultaneously. Prioritize DataSrc over MAB, i.e. set
831+
* MAB only when IBS fails to provide DataSrc.
832+
*/
833+
if (op_data3->dc_miss_no_mab_alloc) {
834+
data_src->mem_lvl = PERF_MEM_LVL_LFB | PERF_MEM_LVL_HIT;
835+
return;
836+
}
837+
838+
data_src->mem_lvl = PERF_MEM_LVL_NA;
839+
}
840+
841+
static bool perf_ibs_cache_hit_st_valid(void)
842+
{
843+
/* 0: Uninitialized, 1: Valid, -1: Invalid */
844+
static int cache_hit_st_valid;
845+
846+
if (unlikely(!cache_hit_st_valid)) {
847+
if (boot_cpu_data.x86 == 0x19 &&
848+
(boot_cpu_data.x86_model <= 0xF ||
849+
(boot_cpu_data.x86_model >= 0x20 &&
850+
boot_cpu_data.x86_model <= 0x5F))) {
851+
cache_hit_st_valid = -1;
852+
} else {
853+
cache_hit_st_valid = 1;
854+
}
855+
}
856+
857+
return cache_hit_st_valid == 1;
858+
}
859+
860+
static void perf_ibs_get_mem_snoop(union ibs_op_data2 *op_data2,
861+
struct perf_sample_data *data)
862+
{
863+
union perf_mem_data_src *data_src = &data->data_src;
864+
u8 ibs_data_src;
865+
866+
data_src->mem_snoop = PERF_MEM_SNOOP_NA;
867+
868+
if (!perf_ibs_cache_hit_st_valid() ||
869+
data_src->mem_op != PERF_MEM_OP_LOAD ||
870+
data_src->mem_lvl & PERF_MEM_LVL_L1 ||
871+
data_src->mem_lvl & PERF_MEM_LVL_L2 ||
872+
op_data2->cache_hit_st)
873+
return;
874+
875+
ibs_data_src = perf_ibs_data_src(op_data2);
876+
877+
if (ibs_caps & IBS_CAPS_ZEN4) {
878+
if (ibs_data_src == IBS_DATA_SRC_EXT_LOC_CACHE ||
879+
ibs_data_src == IBS_DATA_SRC_EXT_NEAR_CCX_CACHE ||
880+
ibs_data_src == IBS_DATA_SRC_EXT_FAR_CCX_CACHE)
881+
data_src->mem_snoop = PERF_MEM_SNOOP_HITM;
882+
} else if (ibs_data_src == IBS_DATA_SRC_LOC_CACHE) {
883+
data_src->mem_snoop = PERF_MEM_SNOOP_HITM;
884+
}
885+
}
886+
887+
static void perf_ibs_get_tlb_lvl(union ibs_op_data3 *op_data3,
888+
struct perf_sample_data *data)
889+
{
890+
union perf_mem_data_src *data_src = &data->data_src;
891+
892+
data_src->mem_dtlb = PERF_MEM_TLB_NA;
893+
894+
if (!op_data3->dc_lin_addr_valid)
895+
return;
896+
897+
if (!op_data3->dc_l1tlb_miss) {
898+
data_src->mem_dtlb = PERF_MEM_TLB_L1 | PERF_MEM_TLB_HIT;
899+
return;
900+
}
901+
902+
if (!op_data3->dc_l2tlb_miss) {
903+
data_src->mem_dtlb = PERF_MEM_TLB_L2 | PERF_MEM_TLB_HIT;
904+
return;
905+
}
906+
907+
data_src->mem_dtlb = PERF_MEM_TLB_L2 | PERF_MEM_TLB_MISS;
908+
}
909+
910+
static void perf_ibs_get_mem_lock(union ibs_op_data3 *op_data3,
911+
struct perf_sample_data *data)
912+
{
913+
union perf_mem_data_src *data_src = &data->data_src;
914+
915+
data_src->mem_lock = PERF_MEM_LOCK_NA;
916+
917+
if (op_data3->dc_locked_op)
918+
data_src->mem_lock = PERF_MEM_LOCK_LOCKED;
919+
}
920+
921+
#define ibs_op_msr_idx(msr) (msr - MSR_AMD64_IBSOPCTL)
922+
923+
static void perf_ibs_get_data_src(struct perf_ibs_data *ibs_data,
924+
struct perf_sample_data *data,
925+
union ibs_op_data2 *op_data2,
926+
union ibs_op_data3 *op_data3)
927+
{
928+
perf_ibs_get_mem_lvl(op_data2, op_data3, data);
929+
perf_ibs_get_mem_snoop(op_data2, data);
930+
perf_ibs_get_tlb_lvl(op_data3, data);
931+
perf_ibs_get_mem_lock(op_data3, data);
932+
}
933+
934+
static __u64 perf_ibs_get_op_data2(struct perf_ibs_data *ibs_data,
935+
union ibs_op_data3 *op_data3)
936+
{
937+
__u64 val = ibs_data->regs[ibs_op_msr_idx(MSR_AMD64_IBSOPDATA2)];
938+
939+
/* Erratum #1293 */
940+
if (boot_cpu_data.x86 == 0x19 && boot_cpu_data.x86_model <= 0xF &&
941+
(op_data3->sw_pf || op_data3->dc_miss_no_mab_alloc)) {
942+
/*
943+
* OP_DATA2 has only two fields on Zen3: DataSrc and RmtNode.
944+
* DataSrc=0 is 'No valid status' and RmtNode is invalid when
945+
* DataSrc=0.
946+
*/
947+
val = 0;
948+
}
949+
return val;
950+
}
951+
952+
static void perf_ibs_parse_ld_st_data(__u64 sample_type,
953+
struct perf_ibs_data *ibs_data,
954+
struct perf_sample_data *data)
955+
{
956+
union ibs_op_data3 op_data3;
957+
union ibs_op_data2 op_data2;
958+
959+
data->data_src.val = PERF_MEM_NA;
960+
op_data3.val = ibs_data->regs[ibs_op_msr_idx(MSR_AMD64_IBSOPDATA3)];
961+
962+
perf_ibs_get_mem_op(&op_data3, data);
963+
if (data->data_src.mem_op != PERF_MEM_OP_LOAD &&
964+
data->data_src.mem_op != PERF_MEM_OP_STORE)
965+
return;
966+
967+
op_data2.val = perf_ibs_get_op_data2(ibs_data, &op_data3);
968+
969+
if (sample_type & PERF_SAMPLE_DATA_SRC) {
970+
perf_ibs_get_data_src(ibs_data, data, &op_data2, &op_data3);
971+
data->sample_flags |= PERF_SAMPLE_DATA_SRC;
972+
}
973+
}
974+
975+
static int perf_ibs_get_offset_max(struct perf_ibs *perf_ibs, u64 sample_type,
976+
int check_rip)
977+
{
978+
if (sample_type & PERF_SAMPLE_RAW ||
979+
(perf_ibs == &perf_ibs_op &&
980+
sample_type & PERF_SAMPLE_DATA_SRC))
981+
return perf_ibs->offset_max;
982+
else if (check_rip)
983+
return 3;
984+
return 1;
985+
}
986+
681987
static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs)
682988
{
683989
struct cpu_perf_ibs *pcpu = this_cpu_ptr(perf_ibs->pcpu);
@@ -725,12 +1031,9 @@ static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs)
7251031
size = 1;
7261032
offset = 1;
7271033
check_rip = (perf_ibs == &perf_ibs_op && (ibs_caps & IBS_CAPS_RIPINVALIDCHK));
728-
if (event->attr.sample_type & PERF_SAMPLE_RAW)
729-
offset_max = perf_ibs->offset_max;
730-
else if (check_rip)
731-
offset_max = 3;
732-
else
733-
offset_max = 1;
1034+
1035+
offset_max = perf_ibs_get_offset_max(perf_ibs, event->attr.sample_type, check_rip);
1036+
7341037
do {
7351038
rdmsrl(msr + offset, *buf++);
7361039
size++;
@@ -784,6 +1087,9 @@ static int perf_ibs_handle_irq(struct perf_ibs *perf_ibs, struct pt_regs *iregs)
7841087
data.sample_flags |= PERF_SAMPLE_RAW;
7851088
}
7861089

1090+
if (perf_ibs == &perf_ibs_op)
1091+
perf_ibs_parse_ld_st_data(event->attr.sample_type, &ibs_data, &data);
1092+
7871093
/*
7881094
* rip recorded by IbsOpRip will not be consistent with rsp and rbp
7891095
* recorded as part of interrupt regs. Thus we need to use rip from

0 commit comments

Comments
 (0)