iommu/dart: Add iommu_ops for locked DARTs

A locked DART has partially read-only MMIO registers. Most importantly
the TTBR registers are read-only. Apple's bootloader sets the DART up
for its intended use before locking it. The single used streams has
a L1 translation table allocated in carved out memory and its TTBRs
point to this table.
In addition translation and bypass can not be disabled or
enabled so a locked DART must not offer default identity or blocked
domains.

The only observed locked DART is for the display coprocessor. It
requires careful handling as translation errors result in unrecoverable
crashes of the display coprocessor.

Signed-off-by: Janne Grunau <j@jannau.net>

Author: jannau

drivers/iommu/apple-dart.c

@@ -235,6 +235,8 @@ struct apple_dart {
 
 	u32 save_tcr[DART_MAX_STREAMS];
 	u32 save_ttbr[DART_MAX_STREAMS][DART_MAX_TTBR];
+
+	u64 *locked_ttbr[DART_MAX_STREAMS][DART_MAX_TTBR];
 };
 
 /*
@@ -383,6 +385,82 @@ apple_dart_hw_clear_all_ttbrs(struct apple_dart_stream_map *stream_map)
 		apple_dart_hw_clear_ttbr(stream_map, i);
 }
 
+static int
+apple_dart_hw_map_locked_ttbr(struct apple_dart_stream_map *stream_map, u8 idx)
+{
+	struct apple_dart *dart = stream_map->dart;
+	int sid;
+
+	for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) {
+		u32 ttbr;
+		phys_addr_t phys;
+		u64 *l1_tbl;
+
+		ttbr = readl(dart->regs + DART_TTBR(dart, sid, idx));
+
+		if (!(ttbr & dart->hw->ttbr_valid)) {
+			dev_err(dart->dev, "Invalid ttbr[%u] for locked dart\n",
+				idx);
+			return -EIO;
+		}
+
+		ttbr &= ~dart->hw->ttbr_valid;
+
+		if (dart->hw->ttbr_addr_field_shift)
+			ttbr >>= dart->hw->ttbr_addr_field_shift;
+		phys = ((phys_addr_t) ttbr) << dart->hw->ttbr_shift;
+
+		l1_tbl = devm_memremap(dart->dev, phys, dart->pgsize,
+				       MEMREMAP_WB);
+		if (!l1_tbl)
+			return -ENOMEM;
+
+		dart->locked_ttbr[sid][idx] = l1_tbl;
+	}
+
+	return 0;
+}
+
+static int
+apple_dart_hw_unmap_locked_ttbr(struct apple_dart_stream_map *stream_map,
+				u8 idx)
+{
+	struct apple_dart *dart = stream_map->dart;
+	int sid;
+
+	for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) {
+		/* TODO: locked L1 table might need to be restored to boot state */
+		if (dart->locked_ttbr[sid][idx]) {
+			memset(dart->locked_ttbr[sid][idx], 0, dart->pgsize);
+			devm_memunmap(dart->dev, dart->locked_ttbr[sid][idx]);
+		}
+		dart->locked_ttbr[sid][idx] = NULL;
+	}
+
+	return 0;
+}
+
+static int
+apple_dart_hw_sync_locked(struct io_pgtable_cfg *cfg,
+			  struct apple_dart_stream_map *stream_map)
+{
+	struct apple_dart *dart = stream_map->dart;
+	int sid;
+
+	for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) {
+		for (int idx = 0; idx < dart->hw->ttbr_count; idx++) {
+			u64 *ttbrep = dart->locked_ttbr[sid][idx];
+			u64 *ptep = cfg->apple_dart_cfg.ttbr[idx];
+			if (!ttbrep || !ptep)
+				continue;
+			for (int entry = 0; entry < dart->pgsize / sizeof(*ptep); entry++)
+				ttbrep[entry] = ptep[entry];
+		}
+	}
+
+	return 0;
+}
+
 static int
 apple_dart_t8020_hw_stream_command(struct apple_dart_stream_map *stream_map,
 			     u32 command)
@@ -504,6 +582,8 @@ static void apple_dart_domain_flush_tlb(struct apple_dart_domain *domain)
 	int i, j;
 	struct apple_dart_atomic_stream_map *domain_stream_map;
 	struct apple_dart_stream_map stream_map;
+	struct io_pgtable_cfg *pgtbl_cfg =
+		&io_pgtable_ops_to_pgtable(domain->pgtbl_ops)->cfg;
 
 	for_each_stream_map(i, domain, domain_stream_map) {
 		stream_map.dart = domain_stream_map->dart;
@@ -512,6 +592,10 @@ static void apple_dart_domain_flush_tlb(struct apple_dart_domain *domain)
 			stream_map.sidmap[j] = atomic_long_read(&domain_stream_map->sidmap[j]);
 
 		WARN_ON(pm_runtime_get_sync(stream_map.dart->dev) < 0);
+
+		if (stream_map.dart->locked)
+			apple_dart_hw_sync_locked(pgtbl_cfg, &stream_map);
+
 		stream_map.dart->hw->invalidate_tlb(&stream_map);
 		pm_runtime_put(stream_map.dart->dev);
 	}
@@ -594,6 +678,24 @@ apple_dart_setup_translation(struct apple_dart_domain *domain,
 	stream_map->dart->hw->invalidate_tlb(stream_map);
 }
 
+static void
+apple_dart_setup_translation_locked(struct apple_dart_domain *domain,
+				    struct apple_dart_stream_map *stream_map)
+{
+	int i;
+	struct io_pgtable_cfg *pgtbl_cfg =
+		&io_pgtable_ops_to_pgtable(domain->pgtbl_ops)->cfg;
+
+	/* Locked DARTs are set up by the bootloader. */
+	for (i = 0; i < pgtbl_cfg->apple_dart_cfg.n_ttbrs; ++i)
+		apple_dart_hw_map_locked_ttbr(stream_map, i);
+	for (; i < stream_map->dart->hw->ttbr_count; ++i)
+		apple_dart_hw_unmap_locked_ttbr(stream_map, i);
+
+	apple_dart_hw_sync_locked(pgtbl_cfg, stream_map);
+	stream_map->dart->hw->invalidate_tlb(stream_map);
+}
+
 static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain,
 				      struct apple_dart_master_cfg *cfg)
 {
@@ -627,6 +729,42 @@ static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain,
 		.iommu_dev = dart->dev,
 	};
 
+	if (dart->locked) {
+		unsigned long *sidmap;
+		int sid;
+		u32 ttbr;
+
+		/* Locked DARTs can only have a single stream bound */
+		sidmap = cfg->stream_maps[0].sidmap;
+		sid = find_first_bit(sidmap, dart->num_streams);
+
+		WARN_ON((sid < 0) || bitmap_weight(sidmap, dart->num_streams) > 1);
+		ttbr = readl(dart->regs + DART_TTBR(dart, sid, 0));
+
+		WARN_ON(!(ttbr & dart->hw->ttbr_valid));
+
+		/* If the DART is locked, we need to keep the translation level count. */
+		if (dart->hw->tcr_4level && dart->ias > 36) {
+			if (readl(dart->regs + DART_TCR(dart, sid)) & dart->hw->tcr_4level) {
+				if (ias < 37) {
+					dev_info(dart->dev, "Expanded to ias=37 due to lock\n");
+					pgtbl_cfg.ias = 37;
+				}
+			} else if (ias > 36) {
+				dev_info(dart->dev, "Limited to ias=36 due to lock\n");
+				pgtbl_cfg.ias = 36;
+				if (dart->dma_min == 0 && dma_max == DMA_BIT_MASK(dart->ias)) {
+					dma_max = DMA_BIT_MASK(pgtbl_cfg.ias);
+				} else if ((dart->dma_min ^ dma_max) & ~DMA_BIT_MASK(36)) {
+					dev_err(dart->dev,
+						"Invalid DMA range for locked 3-level PT\n");
+					ret = -ENOMEM;
+					goto done;
+				}
+			}
+		}
+	}
+
 	dart_domain->pgtbl_ops = alloc_io_pgtable_ops(dart->hw->fmt, &pgtbl_cfg,
 						      &dart_domain->domain);
 	if (!dart_domain->pgtbl_ops) {
@@ -707,8 +845,13 @@ static int apple_dart_attach_dev_paging(struct iommu_domain *domain,
 	if (ret)
 		goto err;
 
-	for_each_stream_map(i, cfg, stream_map)
-		apple_dart_setup_translation(dart_domain, stream_map);
+	for_each_stream_map(i, cfg, stream_map) {
+		if (!stream_map->dart->locked)
+			apple_dart_setup_translation(dart_domain, stream_map);
+		else
+			apple_dart_setup_translation_locked(dart_domain,
+							    stream_map);
+	}
 
 err:
 	for_each_stream_map(i, cfg, stream_map)
@@ -792,8 +935,16 @@ static struct iommu_device *apple_dart_probe_device(struct device *dev)
 
 static void apple_dart_release_device(struct device *dev)
 {
+	int i, j;
+	struct apple_dart_stream_map *stream_map;
 	struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev);
 
+	for_each_stream_map(j, cfg, stream_map) {
+		if (stream_map->dart->locked)
+			for (i = 0; i < stream_map->dart->hw->ttbr_count; ++i)
+				apple_dart_hw_unmap_locked_ttbr(stream_map, i);
+	}
+
 	kfree(cfg);
 }
 
@@ -1070,6 +1221,11 @@ static const struct iommu_ops apple_dart_iommu_no_bypass_ops = {
 	APPLE_DART_IOMMU_COMMON_OPS()
 };
 
+static const struct iommu_ops apple_dart_iommu_locked_ops = {
+	.def_domain_type = apple_dart_def_domain_type_dma,
+	APPLE_DART_IOMMU_COMMON_OPS()
+};
+
 static irqreturn_t apple_dart_t8020_irq(int irq, void *dev)
 {
 	struct apple_dart *dart = dev;
@@ -1280,7 +1436,9 @@ static int apple_dart_probe(struct platform_device *pdev)
 	if (ret)
 		goto err_free_irq;
 
-	if (!dart->supports_bypass)
+	if (dart->locked)
+		ret = iommu_device_register(&dart->iommu, &apple_dart_iommu_locked_ops, dev);
+	else if (!dart->supports_bypass)
 		ret = iommu_device_register(&dart->iommu, &apple_dart_iommu_no_bypass_ops, dev);
 	else
 		ret = iommu_device_register(&dart->iommu, &apple_dart_iommu_ops, dev);