iommu: apple-dart: Add 4-level page table support

The T8110 variant DART implementation on T602x SoCs indicates an IAS of
42, which requires an extra page table level. The extra level is
optional, but let's implement it.

Later it might be useful to restrict this based on the actual attached
devices, since most won't need that much address space anyway.

Signed-off-by: Hector Martin <marcan@marcan.st>

Author: marcan

drivers/iommu/apple-dart.c

@@ -136,6 +136,7 @@
 #define DART_T8110_TCR                  0x1000
 #define DART_T8110_TCR_REMAP            GENMASK(11, 8)
 #define DART_T8110_TCR_REMAP_EN         BIT(7)
+#define DART_T8110_TCR_FOUR_LEVEL       BIT(3)
 #define DART_T8110_TCR_BYPASS_DAPF      BIT(2)
 #define DART_T8110_TCR_BYPASS_DART      BIT(1)
 #define DART_T8110_TCR_TRANSLATE_ENABLE BIT(0)
@@ -180,6 +181,7 @@ struct apple_dart_hw {
 	u32 tcr_enabled;
 	u32 tcr_disabled;
 	u32 tcr_bypass;
+	u32 tcr_4level;
 
 	u32 ttbr;
 	u32 ttbr_valid;
@@ -222,6 +224,7 @@ struct apple_dart {
 	u32 num_streams;
 	u32 supports_bypass : 1;
 	u32 locked : 1;
+	u32 four_level : 1;
 
 	struct iommu_group *sid2group[DART_MAX_STREAMS];
 	struct iommu_device iommu;
@@ -310,14 +313,17 @@ static struct apple_dart_domain *to_dart_domain(struct iommu_domain *dom)
 }
 
 static void
-apple_dart_hw_enable_translation(struct apple_dart_stream_map *stream_map)
+apple_dart_hw_enable_translation(struct apple_dart_stream_map *stream_map, int levels)
 {
 	struct apple_dart *dart = stream_map->dart;
 	int sid;
 
+	WARN_ON(levels != 3 && levels != 4);
+	WARN_ON(levels == 4 && !dart->four_level);
 	WARN_ON(stream_map->dart->locked);
 	for_each_set_bit(sid, stream_map->sidmap, dart->num_streams)
-		writel(dart->hw->tcr_enabled, dart->regs + DART_TCR(dart, sid));
+		writel(dart->hw->tcr_enabled | (levels == 4 ? dart->hw->tcr_4level : 0),
+		       dart->regs + DART_TCR(dart, sid));
 }
 
 static void apple_dart_hw_disable_dma(struct apple_dart_stream_map *stream_map)
@@ -669,7 +675,8 @@ apple_dart_setup_translation(struct apple_dart_domain *domain,
 		for (; i < stream_map->dart->hw->ttbr_count; ++i)
 			apple_dart_hw_clear_ttbr(stream_map, i);
 
-		apple_dart_hw_enable_translation(stream_map);
+		apple_dart_hw_enable_translation(stream_map,
+						 pgtbl_cfg->apple_dart_cfg.n_levels);
 	}
 	stream_map->dart->hw->invalidate_tlb(stream_map);
 }
@@ -753,6 +760,19 @@ static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain,
 		ttbr = readl(dart->regs + DART_TTBR(dart, sid, 0));
 
 		WARN_ON(!(ttbr & dart->hw->ttbr_valid));
+
+		/* If the DART is locked, we need to keep the translation level count. */
+		if (dart->hw->tcr_4level && dart->ias > 36) {
+			if (readl(dart->regs + DART_TCR(dart, sid)) & dart->hw->tcr_4level) {
+				if (dart->ias < 37) {
+					dev_info(dart->dev, "Expanded to ias=37 due to lock\n");
+					pgtbl_cfg.ias = 37;
+				}
+			} else if (dart->ias > 36) {
+				dev_info(dart->dev, "Limited to ias=36 due to lock\n");
+				pgtbl_cfg.ias = 36;
+			}
+		}
 	}
 
 	dart_domain->pgtbl_ops = alloc_io_pgtable_ops(dart->hw->fmt, &pgtbl_cfg,
@@ -765,7 +785,7 @@ static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain,
 	dart_domain->domain.pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
 	dart_domain->domain.geometry.aperture_start = 0;
 	dart_domain->domain.geometry.aperture_end =
-		(dma_addr_t)DMA_BIT_MASK(dart->ias);
+		(dma_addr_t)DMA_BIT_MASK(pgtbl_cfg.ias);
 	dart_domain->domain.geometry.force_aperture = true;
 
 	dart_domain->finalized = true;
@@ -1336,6 +1356,7 @@ static int apple_dart_probe(struct platform_device *pdev)
 		dart->ias = FIELD_GET(DART_T8110_PARAMS3_VA_WIDTH, dart_params[2]);
 		dart->oas = FIELD_GET(DART_T8110_PARAMS3_PA_WIDTH, dart_params[2]);
 		dart->num_streams = FIELD_GET(DART_T8110_PARAMS4_NUM_SIDS, dart_params[3]);
+		dart->four_level = dart->ias > 36;
 		break;
 	}
 
@@ -1373,9 +1394,9 @@ static int apple_dart_probe(struct platform_device *pdev)
 
 	dev_info(
 		&pdev->dev,
-		"DART [pagesize %x, %d streams, bypass support: %d, bypass forced: %d, locked: %d] initialized\n",
+		"DART [pagesize %x, %d streams, bypass support: %d, bypass forced: %d, locked: %d, AS %d -> %d] initialized\n",
 		dart->pgsize, dart->num_streams, dart->supports_bypass,
-		dart->pgsize > PAGE_SIZE, dart->locked);
+		dart->pgsize > PAGE_SIZE, dart->locked, dart->ias, dart->oas);
 	return 0;
 
 err_sysfs_remove:
@@ -1499,6 +1520,7 @@ static const struct apple_dart_hw apple_dart_hw_t8110 = {
 	.tcr_enabled = DART_T8110_TCR_TRANSLATE_ENABLE,
 	.tcr_disabled = 0,
 	.tcr_bypass = DART_T8110_TCR_BYPASS_DAPF | DART_T8110_TCR_BYPASS_DART,
+	.tcr_4level = DART_T8110_TCR_FOUR_LEVEL,
 
 	.ttbr = DART_T8110_TTBR,
 	.ttbr_valid = DART_T8110_TTBR_VALID,