Skip to content

Commit 3fc3068

Browse files
zhaoshanemstsirkin
zhaoshane
authored and
mstsirkin
committed
vduse: Use fixed 4KB bounce pages for non-4KB page size
The allocation granularity of bounce pages is PAGE_SIZE. This may cause even small IO requests to occupy an entire bounce page exclusively. The kind of memory waste will be more significant when PAGE_SIZE is larger than 4KB (e.g. arm64 with 64KB pages). So, optimize it by using fixed 4KB bounce maps and iova allocation granularity. A single IO request occupies at least a 4KB bounce page instead of the entire memory page of PAGE_SIZE. Signed-off-by: Sheng Zhao <sheng.zhao@bytedance.com> Message-Id: <20250925113516.60305-1-sheng.zhao@bytedance.com> Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
1 parent 1c14b0e commit 3fc3068

2 files changed

Lines changed: 95 additions & 42 deletions

File tree

‎drivers/vdpa/vdpa_user/iova_domain.c‎

Lines changed: 90 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -103,19 +103,38 @@ void vduse_domain_clear_map(struct vduse_iova_domain *domain,
103103
static int vduse_domain_map_bounce_page(struct vduse_iova_domain *domain,
104104
u64 iova, u64 size, u64 paddr)
105105
{
106-
struct vduse_bounce_map *map;
106+
struct vduse_bounce_map *map, *head_map;
107+
struct page *tmp_page;
107108
u64 last = iova + size - 1;
108109

109110
while (iova <= last) {
110-
map = &domain->bounce_maps[iova >> PAGE_SHIFT];
111+
/*
112+
* When PAGE_SIZE is larger than 4KB, multiple adjacent bounce_maps will
113+
* point to the same memory page of PAGE_SIZE. Since bounce_maps originate
114+
* from IO requests, we may not be able to guarantee that the orig_phys
115+
* values of all IO requests within the same 64KB memory page are contiguous.
116+
* Therefore, we need to store them separately.
117+
*
118+
* Bounce pages are allocated on demand. As a result, it may occur that
119+
* multiple bounce pages corresponding to the same 64KB memory page attempt
120+
* to allocate memory simultaneously, so we use cmpxchg to handle this
121+
* concurrency.
122+
*/
123+
map = &domain->bounce_maps[iova >> BOUNCE_MAP_SHIFT];
111124
if (!map->bounce_page) {
112-
map->bounce_page = alloc_page(GFP_ATOMIC);
113-
if (!map->bounce_page)
114-
return -ENOMEM;
125+
head_map = &domain->bounce_maps[(iova & PAGE_MASK) >> BOUNCE_MAP_SHIFT];
126+
if (!head_map->bounce_page) {
127+
tmp_page = alloc_page(GFP_ATOMIC);
128+
if (!tmp_page)
129+
return -ENOMEM;
130+
if (cmpxchg(&head_map->bounce_page, NULL, tmp_page))
131+
__free_page(tmp_page);
132+
}
133+
map->bounce_page = head_map->bounce_page;
115134
}
116135
map->orig_phys = paddr;
117-
paddr += PAGE_SIZE;
118-
iova += PAGE_SIZE;
136+
paddr += BOUNCE_MAP_SIZE;
137+
iova += BOUNCE_MAP_SIZE;
119138
}
120139
return 0;
121140
}
@@ -127,12 +146,17 @@ static void vduse_domain_unmap_bounce_page(struct vduse_iova_domain *domain,
127146
u64 last = iova + size - 1;
128147

129148
while (iova <= last) {
130-
map = &domain->bounce_maps[iova >> PAGE_SHIFT];
149+
map = &domain->bounce_maps[iova >> BOUNCE_MAP_SHIFT];
131150
map->orig_phys = INVALID_PHYS_ADDR;
132-
iova += PAGE_SIZE;
151+
iova += BOUNCE_MAP_SIZE;
133152
}
134153
}
135154

155+
static unsigned int offset_in_bounce_page(dma_addr_t addr)
156+
{
157+
return (addr & ~BOUNCE_MAP_MASK);
158+
}
159+
136160
static void do_bounce(phys_addr_t orig, void *addr, size_t size,
137161
enum dma_data_direction dir)
138162
{
@@ -163,17 +187,18 @@ static void vduse_domain_bounce(struct vduse_iova_domain *domain,
163187
{
164188
struct vduse_bounce_map *map;
165189
struct page *page;
166-
unsigned int offset;
190+
unsigned int offset, head_offset;
167191
void *addr;
168192
size_t sz;
169193

170194
if (iova >= domain->bounce_size)
171195
return;
172196

173197
while (size) {
174-
map = &domain->bounce_maps[iova >> PAGE_SHIFT];
175-
offset = offset_in_page(iova);
176-
sz = min_t(size_t, PAGE_SIZE - offset, size);
198+
map = &domain->bounce_maps[iova >> BOUNCE_MAP_SHIFT];
199+
head_offset = offset_in_page(iova);
200+
offset = offset_in_bounce_page(iova);
201+
sz = min_t(size_t, BOUNCE_MAP_SIZE - offset, size);
177202

178203
if (WARN_ON(!map->bounce_page ||
179204
map->orig_phys == INVALID_PHYS_ADDR))
@@ -183,7 +208,7 @@ static void vduse_domain_bounce(struct vduse_iova_domain *domain,
183208
map->user_bounce_page : map->bounce_page;
184209

185210
addr = kmap_local_page(page);
186-
do_bounce(map->orig_phys + offset, addr + offset, sz, dir);
211+
do_bounce(map->orig_phys + offset, addr + head_offset, sz, dir);
187212
kunmap_local(addr);
188213
size -= sz;
189214
iova += sz;
@@ -218,7 +243,7 @@ vduse_domain_get_bounce_page(struct vduse_iova_domain *domain, u64 iova)
218243
struct page *page = NULL;
219244

220245
read_lock(&domain->bounce_lock);
221-
map = &domain->bounce_maps[iova >> PAGE_SHIFT];
246+
map = &domain->bounce_maps[iova >> BOUNCE_MAP_SHIFT];
222247
if (domain->user_bounce_pages || !map->bounce_page)
223248
goto out;
224249

@@ -236,7 +261,7 @@ vduse_domain_free_kernel_bounce_pages(struct vduse_iova_domain *domain)
236261
struct vduse_bounce_map *map;
237262
unsigned long pfn, bounce_pfns;
238263

239-
bounce_pfns = domain->bounce_size >> PAGE_SHIFT;
264+
bounce_pfns = domain->bounce_size >> BOUNCE_MAP_SHIFT;
240265

241266
for (pfn = 0; pfn < bounce_pfns; pfn++) {
242267
map = &domain->bounce_maps[pfn];
@@ -246,16 +271,21 @@ vduse_domain_free_kernel_bounce_pages(struct vduse_iova_domain *domain)
246271
if (!map->bounce_page)
247272
continue;
248273

249-
__free_page(map->bounce_page);
274+
if (!((pfn << BOUNCE_MAP_SHIFT) & ~PAGE_MASK))
275+
__free_page(map->bounce_page);
250276
map->bounce_page = NULL;
251277
}
252278
}
253279

254280
int vduse_domain_add_user_bounce_pages(struct vduse_iova_domain *domain,
255281
struct page **pages, int count)
256282
{
257-
struct vduse_bounce_map *map;
258-
int i, ret;
283+
struct vduse_bounce_map *map, *head_map;
284+
int i, j, ret;
285+
int inner_pages = PAGE_SIZE / BOUNCE_MAP_SIZE;
286+
int bounce_pfns = domain->bounce_size >> BOUNCE_MAP_SHIFT;
287+
struct page *head_page = NULL;
288+
bool need_copy;
259289

260290
/* Now we don't support partial mapping */
261291
if (count != (domain->bounce_size >> PAGE_SHIFT))
@@ -267,16 +297,23 @@ int vduse_domain_add_user_bounce_pages(struct vduse_iova_domain *domain,
267297
goto out;
268298

269299
for (i = 0; i < count; i++) {
270-
map = &domain->bounce_maps[i];
271-
if (map->bounce_page) {
300+
need_copy = false;
301+
head_map = &domain->bounce_maps[(i * inner_pages)];
302+
head_page = head_map->bounce_page;
303+
for (j = 0; j < inner_pages; j++) {
304+
if ((i * inner_pages + j) >= bounce_pfns)
305+
break;
306+
map = &domain->bounce_maps[(i * inner_pages + j)];
272307
/* Copy kernel page to user page if it's in use */
273-
if (map->orig_phys != INVALID_PHYS_ADDR)
274-
memcpy_to_page(pages[i], 0,
275-
page_address(map->bounce_page),
276-
PAGE_SIZE);
308+
if ((head_page) && (map->orig_phys != INVALID_PHYS_ADDR))
309+
need_copy = true;
310+
map->user_bounce_page = pages[i];
277311
}
278-
map->user_bounce_page = pages[i];
279312
get_page(pages[i]);
313+
if ((head_page) && (need_copy))
314+
memcpy_to_page(pages[i], 0,
315+
page_address(head_page),
316+
PAGE_SIZE);
280317
}
281318
domain->user_bounce_pages = true;
282319
ret = 0;
@@ -288,29 +325,40 @@ int vduse_domain_add_user_bounce_pages(struct vduse_iova_domain *domain,
288325

289326
void vduse_domain_remove_user_bounce_pages(struct vduse_iova_domain *domain)
290327
{
291-
struct vduse_bounce_map *map;
292-
unsigned long i, count;
328+
struct vduse_bounce_map *map, *head_map;
329+
unsigned long i, j, count;
330+
int inner_pages = PAGE_SIZE / BOUNCE_MAP_SIZE;
331+
int bounce_pfns = domain->bounce_size >> BOUNCE_MAP_SHIFT;
332+
struct page *head_page = NULL;
333+
bool need_copy;
293334

294335
write_lock(&domain->bounce_lock);
295336
if (!domain->user_bounce_pages)
296337
goto out;
297338

298339
count = domain->bounce_size >> PAGE_SHIFT;
299340
for (i = 0; i < count; i++) {
300-
struct page *page = NULL;
301-
302-
map = &domain->bounce_maps[i];
303-
if (WARN_ON(!map->user_bounce_page))
341+
need_copy = false;
342+
head_map = &domain->bounce_maps[(i * inner_pages)];
343+
if (WARN_ON(!head_map->user_bounce_page))
304344
continue;
305-
306-
/* Copy user page to kernel page if it's in use */
307-
if (map->orig_phys != INVALID_PHYS_ADDR) {
308-
page = map->bounce_page;
309-
memcpy_from_page(page_address(page),
310-
map->user_bounce_page, 0, PAGE_SIZE);
345+
head_page = head_map->user_bounce_page;
346+
347+
for (j = 0; j < inner_pages; j++) {
348+
if ((i * inner_pages + j) >= bounce_pfns)
349+
break;
350+
map = &domain->bounce_maps[(i * inner_pages + j)];
351+
if (WARN_ON(!map->user_bounce_page))
352+
continue;
353+
/* Copy user page to kernel page if it's in use */
354+
if ((map->orig_phys != INVALID_PHYS_ADDR) && (head_map->bounce_page))
355+
need_copy = true;
356+
map->user_bounce_page = NULL;
311357
}
312-
put_page(map->user_bounce_page);
313-
map->user_bounce_page = NULL;
358+
if (need_copy)
359+
memcpy_from_page(page_address(head_map->bounce_page),
360+
head_page, 0, PAGE_SIZE);
361+
put_page(head_page);
314362
}
315363
domain->user_bounce_pages = false;
316364
out:
@@ -581,7 +629,7 @@ vduse_domain_create(unsigned long iova_limit, size_t bounce_size)
581629
unsigned long pfn, bounce_pfns;
582630
int ret;
583631

584-
bounce_pfns = PAGE_ALIGN(bounce_size) >> PAGE_SHIFT;
632+
bounce_pfns = PAGE_ALIGN(bounce_size) >> BOUNCE_MAP_SHIFT;
585633
if (iova_limit <= bounce_size)
586634
return NULL;
587635

@@ -613,7 +661,7 @@ vduse_domain_create(unsigned long iova_limit, size_t bounce_size)
613661
rwlock_init(&domain->bounce_lock);
614662
spin_lock_init(&domain->iotlb_lock);
615663
init_iova_domain(&domain->stream_iovad,
616-
PAGE_SIZE, IOVA_START_PFN);
664+
BOUNCE_MAP_SIZE, IOVA_START_PFN);
617665
ret = iova_domain_init_rcaches(&domain->stream_iovad);
618666
if (ret)
619667
goto err_iovad_stream;

‎drivers/vdpa/vdpa_user/iova_domain.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,11 @@
1919

2020
#define INVALID_PHYS_ADDR (~(phys_addr_t)0)
2121

22+
#define BOUNCE_MAP_SHIFT 12
23+
#define BOUNCE_MAP_SIZE (1 << BOUNCE_MAP_SHIFT)
24+
#define BOUNCE_MAP_MASK (~(BOUNCE_MAP_SIZE - 1))
25+
#define BOUNCE_MAP_ALIGN(addr) (((addr) + BOUNCE_MAP_SIZE - 1) & ~(BOUNCE_MAP_SIZE - 1))
26+
2227
struct vduse_bounce_map {
2328
struct page *bounce_page;
2429
struct page *user_bounce_page;

0 commit comments

Comments
 (0)