|
| 1 | +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB |
| 2 | +/* |
| 3 | + * Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved |
| 4 | + */ |
| 5 | + |
| 6 | +#include <linux/mlx5/driver.h> |
| 7 | +#include <linux/mlx5/device.h> |
| 8 | + |
| 9 | +#include "mlx5_core.h" |
| 10 | +#include "lib/mlx5.h" |
| 11 | + |
| 12 | +struct mlx5_st_idx_data { |
| 13 | + refcount_t usecount; |
| 14 | + u16 tag; |
| 15 | +}; |
| 16 | + |
| 17 | +struct mlx5_st { |
| 18 | + /* serialize access upon alloc/free flows */ |
| 19 | + struct mutex lock; |
| 20 | + struct xa_limit index_limit; |
| 21 | + struct xarray idx_xa; /* key == index, value == struct mlx5_st_idx_data */ |
| 22 | +}; |
| 23 | + |
| 24 | +struct mlx5_st *mlx5_st_create(struct mlx5_core_dev *dev) |
| 25 | +{ |
| 26 | + struct pci_dev *pdev = dev->pdev; |
| 27 | + struct mlx5_st *st; |
| 28 | + u16 num_entries; |
| 29 | + int ret; |
| 30 | + |
| 31 | + if (!MLX5_CAP_GEN(dev, mkey_pcie_tph)) |
| 32 | + return NULL; |
| 33 | + |
| 34 | +#ifdef CONFIG_MLX5_SF |
| 35 | + if (mlx5_core_is_sf(dev)) |
| 36 | + return dev->priv.parent_mdev->st; |
| 37 | +#endif |
| 38 | + |
| 39 | + /* Checking whether the device is capable */ |
| 40 | + if (!pdev->tph_cap) |
| 41 | + return NULL; |
| 42 | + |
| 43 | + num_entries = pcie_tph_get_st_table_size(pdev); |
| 44 | + /* We need a reserved entry for non TPH cases */ |
| 45 | + if (num_entries < 2) |
| 46 | + return NULL; |
| 47 | + |
| 48 | + /* The OS doesn't support ST */ |
| 49 | + ret = pcie_enable_tph(pdev, PCI_TPH_ST_DS_MODE); |
| 50 | + if (ret) |
| 51 | + return NULL; |
| 52 | + |
| 53 | + st = kzalloc(sizeof(*st), GFP_KERNEL); |
| 54 | + if (!st) |
| 55 | + goto end; |
| 56 | + |
| 57 | + mutex_init(&st->lock); |
| 58 | + xa_init_flags(&st->idx_xa, XA_FLAGS_ALLOC); |
| 59 | + /* entry 0 is reserved for non TPH cases */ |
| 60 | + st->index_limit.min = MLX5_MKC_PCIE_TPH_NO_STEERING_TAG_INDEX + 1; |
| 61 | + st->index_limit.max = num_entries - 1; |
| 62 | + |
| 63 | + return st; |
| 64 | + |
| 65 | +end: |
| 66 | + pcie_disable_tph(dev->pdev); |
| 67 | + return NULL; |
| 68 | +} |
| 69 | + |
| 70 | +void mlx5_st_destroy(struct mlx5_core_dev *dev) |
| 71 | +{ |
| 72 | + struct mlx5_st *st = dev->st; |
| 73 | + |
| 74 | + if (mlx5_core_is_sf(dev) || !st) |
| 75 | + return; |
| 76 | + |
| 77 | + pcie_disable_tph(dev->pdev); |
| 78 | + WARN_ON_ONCE(!xa_empty(&st->idx_xa)); |
| 79 | + kfree(st); |
| 80 | +} |
| 81 | + |
| 82 | +int mlx5_st_alloc_index(struct mlx5_core_dev *dev, enum tph_mem_type mem_type, |
| 83 | + unsigned int cpu_uid, u16 *st_index) |
| 84 | +{ |
| 85 | + struct mlx5_st_idx_data *idx_data; |
| 86 | + struct mlx5_st *st = dev->st; |
| 87 | + unsigned long index; |
| 88 | + u32 xa_id; |
| 89 | + u16 tag; |
| 90 | + int ret; |
| 91 | + |
| 92 | + if (!st) |
| 93 | + return -EOPNOTSUPP; |
| 94 | + |
| 95 | + ret = pcie_tph_get_cpu_st(dev->pdev, mem_type, cpu_uid, &tag); |
| 96 | + if (ret) |
| 97 | + return ret; |
| 98 | + |
| 99 | + mutex_lock(&st->lock); |
| 100 | + |
| 101 | + xa_for_each(&st->idx_xa, index, idx_data) { |
| 102 | + if (tag == idx_data->tag) { |
| 103 | + refcount_inc(&idx_data->usecount); |
| 104 | + *st_index = index; |
| 105 | + goto end; |
| 106 | + } |
| 107 | + } |
| 108 | + |
| 109 | + idx_data = kzalloc(sizeof(*idx_data), GFP_KERNEL); |
| 110 | + if (!idx_data) { |
| 111 | + ret = -ENOMEM; |
| 112 | + goto end; |
| 113 | + } |
| 114 | + |
| 115 | + refcount_set(&idx_data->usecount, 1); |
| 116 | + idx_data->tag = tag; |
| 117 | + |
| 118 | + ret = xa_alloc(&st->idx_xa, &xa_id, idx_data, st->index_limit, GFP_KERNEL); |
| 119 | + if (ret) |
| 120 | + goto clean_idx_data; |
| 121 | + |
| 122 | + ret = pcie_tph_set_st_entry(dev->pdev, xa_id, tag); |
| 123 | + if (ret) |
| 124 | + goto clean_idx_xa; |
| 125 | + |
| 126 | + *st_index = xa_id; |
| 127 | + goto end; |
| 128 | + |
| 129 | +clean_idx_xa: |
| 130 | + xa_erase(&st->idx_xa, xa_id); |
| 131 | +clean_idx_data: |
| 132 | + kfree(idx_data); |
| 133 | +end: |
| 134 | + mutex_unlock(&st->lock); |
| 135 | + return ret; |
| 136 | +} |
| 137 | +EXPORT_SYMBOL_GPL(mlx5_st_alloc_index); |
| 138 | + |
| 139 | +int mlx5_st_dealloc_index(struct mlx5_core_dev *dev, u16 st_index) |
| 140 | +{ |
| 141 | + struct mlx5_st_idx_data *idx_data; |
| 142 | + struct mlx5_st *st = dev->st; |
| 143 | + int ret = 0; |
| 144 | + |
| 145 | + if (!st) |
| 146 | + return -EOPNOTSUPP; |
| 147 | + |
| 148 | + mutex_lock(&st->lock); |
| 149 | + idx_data = xa_load(&st->idx_xa, st_index); |
| 150 | + if (WARN_ON_ONCE(!idx_data)) { |
| 151 | + ret = -EINVAL; |
| 152 | + goto end; |
| 153 | + } |
| 154 | + |
| 155 | + if (refcount_dec_and_test(&idx_data->usecount)) { |
| 156 | + xa_erase(&st->idx_xa, st_index); |
| 157 | + /* We leave PCI config space as was before, no mkey will refer to it */ |
| 158 | + } |
| 159 | + |
| 160 | +end: |
| 161 | + mutex_unlock(&st->lock); |
| 162 | + return ret; |
| 163 | +} |
| 164 | +EXPORT_SYMBOL_GPL(mlx5_st_dealloc_index); |
0 commit comments