Skip to content

Commit 67135f2

Browse files
yishaihAlex Williamson
yishaih
authored and
Alex Williamson
committed
vfio/mlx5: Add support for SAVING in chunk mode
Add support for SAVING in chunk mode, it includes running a work that will fill the next chunk from the device. In case the number of available chunks will reach the MAX_NUM_CHUNKS, the next chunk SAVING will be delayed till the reader will consume one chunk. The next patch from the series will add the reader part of the chunk mode. Signed-off-by: Yishai Hadas <yishaih@nvidia.com> Reviewed-by: Jason Gunthorpe <jgg@nvidia.com> Link: https://lore.kernel.org/r/20230911093856.81910-8-yishaih@nvidia.com Signed-off-by: Alex Williamson <alex.williamson@redhat.com>
1 parent 5798e4d commit 67135f2

3 files changed

Lines changed: 122 additions & 17 deletions

File tree

‎drivers/vfio/pci/mlx5/cmd.c‎

Lines changed: 36 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -435,6 +435,7 @@ mlx5vf_alloc_data_buffer(struct mlx5_vf_migration_file *migf,
435435
void mlx5vf_put_data_buffer(struct mlx5_vhca_data_buffer *buf)
436436
{
437437
spin_lock_irq(&buf->migf->list_lock);
438+
buf->stop_copy_chunk_num = 0;
438439
list_add_tail(&buf->buf_elm, &buf->migf->avail_list);
439440
spin_unlock_irq(&buf->migf->list_lock);
440441
}
@@ -551,13 +552,20 @@ static void mlx5vf_save_callback(int status, struct mlx5_async_work *context)
551552
struct mlx5_vf_migration_file, async_data);
552553

553554
if (!status) {
555+
size_t next_required_umem_size = 0;
556+
bool stop_copy_last_chunk;
554557
size_t image_size;
555558
unsigned long flags;
556559
bool initial_pre_copy = migf->state != MLX5_MIGF_STATE_PRE_COPY &&
557560
!async_data->stop_copy_chunk;
558561

559562
image_size = MLX5_GET(save_vhca_state_out, async_data->out,
560563
actual_image_size);
564+
if (async_data->buf->stop_copy_chunk_num)
565+
next_required_umem_size = MLX5_GET(save_vhca_state_out,
566+
async_data->out, next_required_umem_size);
567+
stop_copy_last_chunk = async_data->stop_copy_chunk &&
568+
!next_required_umem_size;
561569
if (async_data->header_buf) {
562570
status = add_buf_header(async_data->header_buf, image_size,
563571
initial_pre_copy);
@@ -569,12 +577,28 @@ static void mlx5vf_save_callback(int status, struct mlx5_async_work *context)
569577
migf->max_pos += async_data->buf->length;
570578
spin_lock_irqsave(&migf->list_lock, flags);
571579
list_add_tail(&async_data->buf->buf_elm, &migf->buf_list);
580+
if (async_data->buf->stop_copy_chunk_num) {
581+
migf->num_ready_chunks++;
582+
if (next_required_umem_size &&
583+
migf->num_ready_chunks >= MAX_NUM_CHUNKS) {
584+
/* Delay the next SAVE till one chunk be consumed */
585+
migf->next_required_umem_size = next_required_umem_size;
586+
next_required_umem_size = 0;
587+
}
588+
}
572589
spin_unlock_irqrestore(&migf->list_lock, flags);
573-
if (initial_pre_copy)
590+
if (initial_pre_copy) {
574591
migf->pre_copy_initial_bytes += image_size;
575-
migf->state = async_data->stop_copy_chunk ?
576-
MLX5_MIGF_STATE_COMPLETE : MLX5_MIGF_STATE_PRE_COPY;
592+
migf->state = MLX5_MIGF_STATE_PRE_COPY;
593+
}
594+
if (stop_copy_last_chunk)
595+
migf->state = MLX5_MIGF_STATE_COMPLETE;
577596
wake_up_interruptible(&migf->poll_wait);
597+
if (next_required_umem_size)
598+
mlx5vf_mig_file_set_save_work(migf,
599+
/* Picking up the next chunk num */
600+
(async_data->buf->stop_copy_chunk_num % MAX_NUM_CHUNKS) + 1,
601+
next_required_umem_size);
578602
mlx5vf_save_callback_complete(migf, async_data);
579603
return;
580604
}
@@ -632,10 +656,15 @@ int mlx5vf_cmd_save_vhca_state(struct mlx5vf_pci_core_device *mvdev,
632656
}
633657

634658
if (MLX5VF_PRE_COPY_SUPP(mvdev)) {
635-
if (async_data->stop_copy_chunk && migf->buf_header[0]) {
636-
header_buf = migf->buf_header[0];
637-
migf->buf_header[0] = NULL;
638-
} else {
659+
if (async_data->stop_copy_chunk) {
660+
u8 header_idx = buf->stop_copy_chunk_num ?
661+
buf->stop_copy_chunk_num - 1 : 0;
662+
663+
header_buf = migf->buf_header[header_idx];
664+
migf->buf_header[header_idx] = NULL;
665+
}
666+
667+
if (!header_buf) {
639668
header_buf = mlx5vf_get_data_buffer(migf,
640669
sizeof(struct mlx5_vf_migration_header), DMA_NONE);
641670
if (IS_ERR(header_buf)) {

‎drivers/vfio/pci/mlx5/cmd.h‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,13 @@ struct mlx5vf_async_data {
8383
void *out;
8484
};
8585

86+
struct mlx5vf_save_work_data {
87+
struct mlx5_vf_migration_file *migf;
88+
size_t next_required_umem_size;
89+
struct work_struct work;
90+
u8 chunk_num;
91+
};
92+
8693
#define MAX_NUM_CHUNKS 2
8794

8895
struct mlx5_vf_migration_file {
@@ -97,9 +104,12 @@ struct mlx5_vf_migration_file {
97104
u32 record_tag;
98105
u64 stop_copy_prep_size;
99106
u64 pre_copy_initial_bytes;
107+
size_t next_required_umem_size;
108+
u8 num_ready_chunks;
100109
/* Upon chunk mode preserve another set of buffers for stop_copy phase */
101110
struct mlx5_vhca_data_buffer *buf[MAX_NUM_CHUNKS];
102111
struct mlx5_vhca_data_buffer *buf_header[MAX_NUM_CHUNKS];
112+
struct mlx5vf_save_work_data save_data[MAX_NUM_CHUNKS];
103113
spinlock_t list_lock;
104114
struct list_head buf_list;
105115
struct list_head avail_list;
@@ -223,6 +233,8 @@ struct page *mlx5vf_get_migration_page(struct mlx5_vhca_data_buffer *buf,
223233
void mlx5vf_state_mutex_unlock(struct mlx5vf_pci_core_device *mvdev);
224234
void mlx5vf_disable_fds(struct mlx5vf_pci_core_device *mvdev);
225235
void mlx5vf_mig_file_cleanup_cb(struct work_struct *_work);
236+
void mlx5vf_mig_file_set_save_work(struct mlx5_vf_migration_file *migf,
237+
u8 chunk_num, size_t next_required_umem_size);
226238
int mlx5vf_start_page_tracker(struct vfio_device *vdev,
227239
struct rb_root_cached *ranges, u32 nnodes, u64 *page_size);
228240
int mlx5vf_stop_page_tracker(struct vfio_device *vdev);

‎drivers/vfio/pci/mlx5/main.c‎

Lines changed: 74 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -306,6 +306,73 @@ static void mlx5vf_mark_err(struct mlx5_vf_migration_file *migf)
306306
wake_up_interruptible(&migf->poll_wait);
307307
}
308308

309+
void mlx5vf_mig_file_set_save_work(struct mlx5_vf_migration_file *migf,
310+
u8 chunk_num, size_t next_required_umem_size)
311+
{
312+
migf->save_data[chunk_num - 1].next_required_umem_size =
313+
next_required_umem_size;
314+
migf->save_data[chunk_num - 1].migf = migf;
315+
get_file(migf->filp);
316+
queue_work(migf->mvdev->cb_wq,
317+
&migf->save_data[chunk_num - 1].work);
318+
}
319+
320+
static struct mlx5_vhca_data_buffer *
321+
mlx5vf_mig_file_get_stop_copy_buf(struct mlx5_vf_migration_file *migf,
322+
u8 index, size_t required_length)
323+
{
324+
struct mlx5_vhca_data_buffer *buf = migf->buf[index];
325+
u8 chunk_num;
326+
327+
WARN_ON(!buf);
328+
chunk_num = buf->stop_copy_chunk_num;
329+
buf->migf->buf[index] = NULL;
330+
/* Checking whether the pre-allocated buffer can fit */
331+
if (buf->allocated_length >= required_length)
332+
return buf;
333+
334+
mlx5vf_put_data_buffer(buf);
335+
buf = mlx5vf_get_data_buffer(buf->migf, required_length,
336+
DMA_FROM_DEVICE);
337+
if (IS_ERR(buf))
338+
return buf;
339+
340+
buf->stop_copy_chunk_num = chunk_num;
341+
return buf;
342+
}
343+
344+
static void mlx5vf_mig_file_save_work(struct work_struct *_work)
345+
{
346+
struct mlx5vf_save_work_data *save_data = container_of(_work,
347+
struct mlx5vf_save_work_data, work);
348+
struct mlx5_vf_migration_file *migf = save_data->migf;
349+
struct mlx5vf_pci_core_device *mvdev = migf->mvdev;
350+
struct mlx5_vhca_data_buffer *buf;
351+
352+
mutex_lock(&mvdev->state_mutex);
353+
if (migf->state == MLX5_MIGF_STATE_ERROR)
354+
goto end;
355+
356+
buf = mlx5vf_mig_file_get_stop_copy_buf(migf,
357+
save_data->chunk_num - 1,
358+
save_data->next_required_umem_size);
359+
if (IS_ERR(buf))
360+
goto err;
361+
362+
if (mlx5vf_cmd_save_vhca_state(mvdev, migf, buf, true, false))
363+
goto err_save;
364+
365+
goto end;
366+
367+
err_save:
368+
mlx5vf_put_data_buffer(buf);
369+
err:
370+
mlx5vf_mark_err(migf);
371+
end:
372+
mlx5vf_state_mutex_unlock(mvdev);
373+
fput(migf->filp);
374+
}
375+
309376
static int mlx5vf_add_stop_copy_header(struct mlx5_vf_migration_file *migf,
310377
bool track)
311378
{
@@ -400,6 +467,9 @@ static int mlx5vf_prep_stop_copy(struct mlx5vf_pci_core_device *mvdev,
400467
if (mvdev->chunk_mode) {
401468
migf->buf[i]->stop_copy_chunk_num = i + 1;
402469
migf->buf_header[i]->stop_copy_chunk_num = i + 1;
470+
INIT_WORK(&migf->save_data[i].work,
471+
mlx5vf_mig_file_save_work);
472+
migf->save_data[i].chunk_num = i + 1;
403473
}
404474
}
405475

@@ -548,16 +618,10 @@ static int mlx5vf_pci_save_device_inc_data(struct mlx5vf_pci_core_device *mvdev)
548618
if (ret)
549619
goto err;
550620

551-
/* Checking whether we have a matching pre-allocated buffer that can fit */
552-
if (migf->buf[0]->allocated_length >= length) {
553-
buf = migf->buf[0];
554-
migf->buf[0] = NULL;
555-
} else {
556-
buf = mlx5vf_get_data_buffer(migf, length, DMA_FROM_DEVICE);
557-
if (IS_ERR(buf)) {
558-
ret = PTR_ERR(buf);
559-
goto err;
560-
}
621+
buf = mlx5vf_mig_file_get_stop_copy_buf(migf, 0, length);
622+
if (IS_ERR(buf)) {
623+
ret = PTR_ERR(buf);
624+
goto err;
561625
}
562626

563627
ret = mlx5vf_cmd_save_vhca_state(mvdev, migf, buf, true, false);

0 commit comments

Comments
 (0)