Skip to content

Commit e08fcd9

Browse files
ssuthiku-amdjoergroedel
ssuthiku-amd
authored and
joergroedel
committed
iommu/amd: Move PPR-related functions into ppr.c
In preparation to subsequent PPR-related patches, and also remove static declaration for certain helper functions so that it can be reused in other files. Also rename below functions: alloc_ppr_log -> amd_iommu_alloc_ppr_log iommu_enable_ppr_log -> amd_iommu_enable_ppr_log free_ppr_log -> amd_iommu_free_ppr_log iommu_poll_ppr_log -> amd_iommu_poll_ppr_log Signed-off-by: Suravee Suthikulpanit <suravee.suthikulpanit@amd.com> Co-developed-by: Vasant Hegde <vasant.hegde@amd.com> Signed-off-by: Vasant Hegde <vasant.hegde@amd.com> Reviewed-by: Jason Gunthorpe <jgg@nvidia.com> Link: https://lore.kernel.org/r/20240418103400.6229-5-vasant.hegde@amd.com Signed-off-by: Joerg Roedel <jroedel@suse.de>
1 parent db44bd5 commit e08fcd9

5 files changed

Lines changed: 139 additions & 114 deletions

File tree

‎drivers/iommu/amd/Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
# SPDX-License-Identifier: GPL-2.0-only
2-
obj-$(CONFIG_AMD_IOMMU) += iommu.o init.o quirks.o io_pgtable.o io_pgtable_v2.o
2+
obj-$(CONFIG_AMD_IOMMU) += iommu.o init.o quirks.o io_pgtable.o io_pgtable_v2.o ppr.o
33
obj-$(CONFIG_AMD_IOMMU_DEBUGFS) += debugfs.o

‎drivers/iommu/amd/amd_iommu.h‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,16 @@ irqreturn_t amd_iommu_int_thread_pprlog(int irq, void *data);
1717
irqreturn_t amd_iommu_int_thread_galog(int irq, void *data);
1818
irqreturn_t amd_iommu_int_handler(int irq, void *data);
1919
void amd_iommu_apply_erratum_63(struct amd_iommu *iommu, u16 devid);
20+
void amd_iommu_restart_log(struct amd_iommu *iommu, const char *evt_type,
21+
u8 cntrl_intr, u8 cntrl_log,
22+
u32 status_run_mask, u32 status_overflow_mask);
2023
void amd_iommu_restart_event_logging(struct amd_iommu *iommu);
2124
void amd_iommu_restart_ga_log(struct amd_iommu *iommu);
2225
void amd_iommu_restart_ppr_log(struct amd_iommu *iommu);
2326
void amd_iommu_set_rlookup_table(struct amd_iommu *iommu, u16 devid);
27+
void iommu_feature_enable(struct amd_iommu *iommu, u8 bit);
28+
void *__init iommu_alloc_4k_pages(struct amd_iommu *iommu,
29+
gfp_t gfp, size_t size);
2430

2531
#ifdef CONFIG_AMD_IOMMU_DEBUGFS
2632
void amd_iommu_debugfs_setup(struct amd_iommu *iommu);
@@ -49,6 +55,14 @@ int amd_iommu_set_gcr3(struct iommu_dev_data *dev_data,
4955
ioasid_t pasid, unsigned long gcr3);
5056
int amd_iommu_clear_gcr3(struct iommu_dev_data *dev_data, ioasid_t pasid);
5157

58+
/* PPR */
59+
int __init amd_iommu_alloc_ppr_log(struct amd_iommu *iommu);
60+
void __init amd_iommu_free_ppr_log(struct amd_iommu *iommu);
61+
void amd_iommu_enable_ppr_log(struct amd_iommu *iommu);
62+
void amd_iommu_poll_ppr_log(struct amd_iommu *iommu);
63+
int amd_iommu_complete_ppr(struct pci_dev *pdev, u32 pasid,
64+
int status, int tag);
65+
5266
/*
5367
* This function flushes all internal caches of
5468
* the IOMMU used by this driver.
@@ -74,9 +88,6 @@ static inline int amd_iommu_create_irq_domain(struct amd_iommu *iommu)
7488
}
7589
#endif
7690

77-
int amd_iommu_complete_ppr(struct pci_dev *pdev, u32 pasid,
78-
int status, int tag);
79-
8091
static inline bool is_rd890_iommu(struct pci_dev *pdev)
8192
{
8293
return (pdev->vendor == PCI_VENDOR_ID_ATI) &&

‎drivers/iommu/amd/init.c‎

Lines changed: 9 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -419,7 +419,7 @@ static void iommu_set_device_table(struct amd_iommu *iommu)
419419
}
420420

421421
/* Generic functions to enable/disable certain features of the IOMMU. */
422-
static void iommu_feature_enable(struct amd_iommu *iommu, u8 bit)
422+
void iommu_feature_enable(struct amd_iommu *iommu, u8 bit)
423423
{
424424
u64 ctrl;
425425

@@ -746,9 +746,9 @@ static int __init alloc_command_buffer(struct amd_iommu *iommu)
746746
* Interrupt handler has processed all pending events and adjusted head
747747
* and tail pointer. Reset overflow mask and restart logging again.
748748
*/
749-
static void amd_iommu_restart_log(struct amd_iommu *iommu, const char *evt_type,
750-
u8 cntrl_intr, u8 cntrl_log,
751-
u32 status_run_mask, u32 status_overflow_mask)
749+
void amd_iommu_restart_log(struct amd_iommu *iommu, const char *evt_type,
750+
u8 cntrl_intr, u8 cntrl_log,
751+
u32 status_run_mask, u32 status_overflow_mask)
752752
{
753753
u32 status;
754754

@@ -789,17 +789,6 @@ void amd_iommu_restart_ga_log(struct amd_iommu *iommu)
789789
MMIO_STATUS_GALOG_OVERFLOW_MASK);
790790
}
791791

792-
/*
793-
* This function restarts ppr logging in case the IOMMU experienced
794-
* PPR log overflow.
795-
*/
796-
void amd_iommu_restart_ppr_log(struct amd_iommu *iommu)
797-
{
798-
amd_iommu_restart_log(iommu, "PPR", CONTROL_PPRINT_EN,
799-
CONTROL_PPRLOG_EN, MMIO_STATUS_PPR_RUN_MASK,
800-
MMIO_STATUS_PPR_OVERFLOW_MASK);
801-
}
802-
803792
/*
804793
* This function resets the command buffer if the IOMMU stopped fetching
805794
* commands from it.
@@ -848,8 +837,8 @@ static void __init free_command_buffer(struct amd_iommu *iommu)
848837
free_pages((unsigned long)iommu->cmd_buf, get_order(CMD_BUFFER_SIZE));
849838
}
850839

851-
static void *__init iommu_alloc_4k_pages(struct amd_iommu *iommu,
852-
gfp_t gfp, size_t size)
840+
void *__init iommu_alloc_4k_pages(struct amd_iommu *iommu, gfp_t gfp,
841+
size_t size)
853842
{
854843
int order = get_order(size);
855844
void *buf = (void *)__get_free_pages(gfp, order);
@@ -904,42 +893,6 @@ static void __init free_event_buffer(struct amd_iommu *iommu)
904893
free_pages((unsigned long)iommu->evt_buf, get_order(EVT_BUFFER_SIZE));
905894
}
906895

907-
/* allocates the memory where the IOMMU will log its events to */
908-
static int __init alloc_ppr_log(struct amd_iommu *iommu)
909-
{
910-
iommu->ppr_log = iommu_alloc_4k_pages(iommu, GFP_KERNEL | __GFP_ZERO,
911-
PPR_LOG_SIZE);
912-
913-
return iommu->ppr_log ? 0 : -ENOMEM;
914-
}
915-
916-
static void iommu_enable_ppr_log(struct amd_iommu *iommu)
917-
{
918-
u64 entry;
919-
920-
if (iommu->ppr_log == NULL)
921-
return;
922-
923-
iommu_feature_enable(iommu, CONTROL_PPR_EN);
924-
925-
entry = iommu_virt_to_phys(iommu->ppr_log) | PPR_LOG_SIZE_512;
926-
927-
memcpy_toio(iommu->mmio_base + MMIO_PPR_LOG_OFFSET,
928-
&entry, sizeof(entry));
929-
930-
/* set head and tail to zero manually */
931-
writel(0x00, iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
932-
writel(0x00, iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);
933-
934-
iommu_feature_enable(iommu, CONTROL_PPRLOG_EN);
935-
iommu_feature_enable(iommu, CONTROL_PPRINT_EN);
936-
}
937-
938-
static void __init free_ppr_log(struct amd_iommu *iommu)
939-
{
940-
free_pages((unsigned long)iommu->ppr_log, get_order(PPR_LOG_SIZE));
941-
}
942-
943896
static void free_ga_log(struct amd_iommu *iommu)
944897
{
945898
#ifdef CONFIG_IRQ_REMAP
@@ -1683,7 +1636,7 @@ static void __init free_iommu_one(struct amd_iommu *iommu)
16831636
free_cwwb_sem(iommu);
16841637
free_command_buffer(iommu);
16851638
free_event_buffer(iommu);
1686-
free_ppr_log(iommu);
1639+
amd_iommu_free_ppr_log(iommu);
16871640
free_ga_log(iommu);
16881641
iommu_unmap_mmio_space(iommu);
16891642
}
@@ -2099,7 +2052,7 @@ static int __init iommu_init_pci(struct amd_iommu *iommu)
20992052
amd_iommu_max_glx_val = min(amd_iommu_max_glx_val, glxval);
21002053
}
21012054

2102-
if (check_feature(FEATURE_PPR) && alloc_ppr_log(iommu))
2055+
if (check_feature(FEATURE_PPR) && amd_iommu_alloc_ppr_log(iommu))
21032056
return -ENOMEM;
21042057

21052058
if (iommu->cap & (1UL << IOMMU_CAP_NPCACHE)) {
@@ -2845,7 +2798,7 @@ static void enable_iommus_v2(void)
28452798
struct amd_iommu *iommu;
28462799

28472800
for_each_iommu(iommu)
2848-
iommu_enable_ppr_log(iommu);
2801+
amd_iommu_enable_ppr_log(iommu);
28492802
}
28502803

28512804
static void enable_iommus_vapic(void)

‎drivers/iommu/amd/iommu.c‎

Lines changed: 1 addition & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -818,59 +818,6 @@ static void iommu_poll_events(struct amd_iommu *iommu)
818818
writel(head, iommu->mmio_base + MMIO_EVT_HEAD_OFFSET);
819819
}
820820

821-
static void iommu_poll_ppr_log(struct amd_iommu *iommu)
822-
{
823-
u32 head, tail;
824-
825-
if (iommu->ppr_log == NULL)
826-
return;
827-
828-
head = readl(iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
829-
tail = readl(iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);
830-
831-
while (head != tail) {
832-
volatile u64 *raw;
833-
u64 entry[2];
834-
int i;
835-
836-
raw = (u64 *)(iommu->ppr_log + head);
837-
838-
/*
839-
* Hardware bug: Interrupt may arrive before the entry is
840-
* written to memory. If this happens we need to wait for the
841-
* entry to arrive.
842-
*/
843-
for (i = 0; i < LOOP_TIMEOUT; ++i) {
844-
if (PPR_REQ_TYPE(raw[0]) != 0)
845-
break;
846-
udelay(1);
847-
}
848-
849-
/* Avoid memcpy function-call overhead */
850-
entry[0] = raw[0];
851-
entry[1] = raw[1];
852-
853-
/*
854-
* To detect the hardware errata 733 we need to clear the
855-
* entry back to zero. This issue does not exist on SNP
856-
* enabled system. Also this buffer is not writeable on
857-
* SNP enabled system.
858-
*/
859-
if (!amd_iommu_snp_en)
860-
raw[0] = raw[1] = 0UL;
861-
862-
/* Update head pointer of hardware ring-buffer */
863-
head = (head + PPR_ENTRY_SIZE) % PPR_LOG_SIZE;
864-
writel(head, iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
865-
866-
/* TODO: PPR Handler will be added when we add IOPF support */
867-
868-
/* Refresh ring-buffer information */
869-
head = readl(iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
870-
tail = readl(iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);
871-
}
872-
}
873-
874821
#ifdef CONFIG_IRQ_REMAP
875822
static int (*iommu_ga_log_notifier)(u32);
876823

@@ -991,7 +938,7 @@ irqreturn_t amd_iommu_int_thread_pprlog(int irq, void *data)
991938
{
992939
amd_iommu_handle_irq(data, "PPR", MMIO_STATUS_PPR_INT_MASK,
993940
MMIO_STATUS_PPR_OVERFLOW_MASK,
994-
iommu_poll_ppr_log, amd_iommu_restart_ppr_log);
941+
amd_iommu_poll_ppr_log, amd_iommu_restart_ppr_log);
995942

996943
return IRQ_HANDLED;
997944
}

‎drivers/iommu/amd/ppr.c‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
// SPDX-License-Identifier: GPL-2.0-only
2+
/*
3+
* Copyright (C) 2023 Advanced Micro Devices, Inc.
4+
*/
5+
6+
#define pr_fmt(fmt) "AMD-Vi: " fmt
7+
#define dev_fmt(fmt) pr_fmt(fmt)
8+
9+
#include <linux/amd-iommu.h>
10+
#include <linux/delay.h>
11+
#include <linux/mmu_notifier.h>
12+
13+
#include <asm/iommu.h>
14+
15+
#include "amd_iommu.h"
16+
#include "amd_iommu_types.h"
17+
18+
int __init amd_iommu_alloc_ppr_log(struct amd_iommu *iommu)
19+
{
20+
iommu->ppr_log = iommu_alloc_4k_pages(iommu, GFP_KERNEL | __GFP_ZERO,
21+
PPR_LOG_SIZE);
22+
return iommu->ppr_log ? 0 : -ENOMEM;
23+
}
24+
25+
void amd_iommu_enable_ppr_log(struct amd_iommu *iommu)
26+
{
27+
u64 entry;
28+
29+
if (iommu->ppr_log == NULL)
30+
return;
31+
32+
iommu_feature_enable(iommu, CONTROL_PPR_EN);
33+
34+
entry = iommu_virt_to_phys(iommu->ppr_log) | PPR_LOG_SIZE_512;
35+
36+
memcpy_toio(iommu->mmio_base + MMIO_PPR_LOG_OFFSET,
37+
&entry, sizeof(entry));
38+
39+
/* set head and tail to zero manually */
40+
writel(0x00, iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
41+
writel(0x00, iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);
42+
43+
iommu_feature_enable(iommu, CONTROL_PPRINT_EN);
44+
iommu_feature_enable(iommu, CONTROL_PPRLOG_EN);
45+
}
46+
47+
void __init amd_iommu_free_ppr_log(struct amd_iommu *iommu)
48+
{
49+
free_pages((unsigned long)iommu->ppr_log, get_order(PPR_LOG_SIZE));
50+
}
51+
52+
/*
53+
* This function restarts ppr logging in case the IOMMU experienced
54+
* PPR log overflow.
55+
*/
56+
void amd_iommu_restart_ppr_log(struct amd_iommu *iommu)
57+
{
58+
amd_iommu_restart_log(iommu, "PPR", CONTROL_PPRINT_EN,
59+
CONTROL_PPRLOG_EN, MMIO_STATUS_PPR_RUN_MASK,
60+
MMIO_STATUS_PPR_OVERFLOW_MASK);
61+
}
62+
63+
void amd_iommu_poll_ppr_log(struct amd_iommu *iommu)
64+
{
65+
u32 head, tail;
66+
67+
if (iommu->ppr_log == NULL)
68+
return;
69+
70+
head = readl(iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
71+
tail = readl(iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);
72+
73+
while (head != tail) {
74+
volatile u64 *raw;
75+
u64 entry[2];
76+
int i;
77+
78+
raw = (u64 *)(iommu->ppr_log + head);
79+
80+
/*
81+
* Hardware bug: Interrupt may arrive before the entry is
82+
* written to memory. If this happens we need to wait for the
83+
* entry to arrive.
84+
*/
85+
for (i = 0; i < LOOP_TIMEOUT; ++i) {
86+
if (PPR_REQ_TYPE(raw[0]) != 0)
87+
break;
88+
udelay(1);
89+
}
90+
91+
/* Avoid memcpy function-call overhead */
92+
entry[0] = raw[0];
93+
entry[1] = raw[1];
94+
95+
/*
96+
* To detect the hardware errata 733 we need to clear the
97+
* entry back to zero. This issue does not exist on SNP
98+
* enabled system. Also this buffer is not writeable on
99+
* SNP enabled system.
100+
*/
101+
if (!amd_iommu_snp_en)
102+
raw[0] = raw[1] = 0UL;
103+
104+
/* Update head pointer of hardware ring-buffer */
105+
head = (head + PPR_ENTRY_SIZE) % PPR_LOG_SIZE;
106+
writel(head, iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
107+
108+
/* TODO: PPR Handler will be added when we add IOPF support */
109+
110+
/* Refresh ring-buffer information */
111+
head = readl(iommu->mmio_base + MMIO_PPR_HEAD_OFFSET);
112+
tail = readl(iommu->mmio_base + MMIO_PPR_TAIL_OFFSET);
113+
}
114+
}

0 commit comments

Comments
 (0)