@@ -1084,6 +1084,80 @@ long xe_bo_shrink(struct ttm_operation_ctx *ctx, struct ttm_buffer_object *bo,
10841084 return lret ;
10851085}
10861086
1087+ /**
1088+ * xe_bo_notifier_prepare_pinned() - Prepare a pinned VRAM object to be backed
1089+ * up in system memory.
1090+ * @bo: The buffer object to prepare.
1091+ *
1092+ * On successful completion, the object backup pages are allocated. Expectation
1093+ * is that this is called from the PM notifier, prior to suspend/hibernation.
1094+ *
1095+ * Return: 0 on success. Negative error code on failure.
1096+ */
1097+ int xe_bo_notifier_prepare_pinned (struct xe_bo * bo )
1098+ {
1099+ struct xe_device * xe = ttm_to_xe_device (bo -> ttm .bdev );
1100+ struct xe_bo * backup ;
1101+ int ret = 0 ;
1102+
1103+ xe_bo_lock (bo , false);
1104+
1105+ xe_assert (xe , !bo -> backup_obj );
1106+
1107+ /*
1108+ * Since this is called from the PM notifier we might have raced with
1109+ * someone unpinning this after we dropped the pinned list lock and
1110+ * grabbing the above bo lock.
1111+ */
1112+ if (!xe_bo_is_pinned (bo ))
1113+ goto out_unlock_bo ;
1114+
1115+ if (!xe_bo_is_vram (bo ))
1116+ goto out_unlock_bo ;
1117+
1118+ if (bo -> flags & XE_BO_FLAG_PINNED_NORESTORE )
1119+ goto out_unlock_bo ;
1120+
1121+ backup = ___xe_bo_create_locked (xe , NULL , NULL , bo -> ttm .base .resv , NULL , bo -> size ,
1122+ DRM_XE_GEM_CPU_CACHING_WB , ttm_bo_type_kernel ,
1123+ XE_BO_FLAG_SYSTEM | XE_BO_FLAG_NEEDS_CPU_ACCESS |
1124+ XE_BO_FLAG_PINNED );
1125+ if (IS_ERR (backup )) {
1126+ ret = PTR_ERR (backup );
1127+ goto out_unlock_bo ;
1128+ }
1129+
1130+ backup -> parent_obj = xe_bo_get (bo ); /* Released by bo_destroy */
1131+ ttm_bo_pin (& backup -> ttm );
1132+ bo -> backup_obj = backup ;
1133+
1134+ out_unlock_bo :
1135+ xe_bo_unlock (bo );
1136+ return ret ;
1137+ }
1138+
1139+ /**
1140+ * xe_bo_notifier_unprepare_pinned() - Undo the previous prepare operation.
1141+ * @bo: The buffer object to undo the prepare for.
1142+ *
1143+ * Always returns 0. The backup object is removed, if still present. Expectation
1144+ * it that this called from the PM notifier when undoing the prepare step.
1145+ *
1146+ * Return: Always returns 0.
1147+ */
1148+ int xe_bo_notifier_unprepare_pinned (struct xe_bo * bo )
1149+ {
1150+ xe_bo_lock (bo , false);
1151+ if (bo -> backup_obj ) {
1152+ ttm_bo_unpin (& bo -> backup_obj -> ttm );
1153+ xe_bo_put (bo -> backup_obj );
1154+ bo -> backup_obj = NULL ;
1155+ }
1156+ xe_bo_unlock (bo );
1157+
1158+ return 0 ;
1159+ }
1160+
10871161/**
10881162 * xe_bo_evict_pinned() - Evict a pinned VRAM object to system memory
10891163 * @bo: The buffer object to move.
@@ -1098,7 +1172,8 @@ long xe_bo_shrink(struct ttm_operation_ctx *ctx, struct ttm_buffer_object *bo,
10981172int xe_bo_evict_pinned (struct xe_bo * bo )
10991173{
11001174 struct xe_device * xe = ttm_to_xe_device (bo -> ttm .bdev );
1101- struct xe_bo * backup ;
1175+ struct xe_bo * backup = bo -> backup_obj ;
1176+ bool backup_created = false;
11021177 bool unmap = false;
11031178 int ret = 0 ;
11041179
@@ -1120,12 +1195,17 @@ int xe_bo_evict_pinned(struct xe_bo *bo)
11201195 if (bo -> flags & XE_BO_FLAG_PINNED_NORESTORE )
11211196 goto out_unlock_bo ;
11221197
1123- backup = xe_bo_create_locked (xe , NULL , NULL , bo -> size , ttm_bo_type_kernel ,
1124- XE_BO_FLAG_SYSTEM | XE_BO_FLAG_NEEDS_CPU_ACCESS |
1125- XE_BO_FLAG_PINNED );
1126- if (IS_ERR (backup )) {
1127- ret = PTR_ERR (backup );
1128- goto out_unlock_bo ;
1198+ if (!backup ) {
1199+ backup = ___xe_bo_create_locked (xe , NULL , NULL , bo -> ttm .base .resv , NULL , bo -> size ,
1200+ DRM_XE_GEM_CPU_CACHING_WB , ttm_bo_type_kernel ,
1201+ XE_BO_FLAG_SYSTEM | XE_BO_FLAG_NEEDS_CPU_ACCESS |
1202+ XE_BO_FLAG_PINNED );
1203+ if (IS_ERR (backup )) {
1204+ ret = PTR_ERR (backup );
1205+ goto out_unlock_bo ;
1206+ }
1207+ backup -> parent_obj = xe_bo_get (bo ); /* Released by bo_destroy */
1208+ backup_created = true;
11291209 }
11301210
11311211 if (xe_bo_is_user (bo ) || (bo -> flags & XE_BO_FLAG_PINNED_LATE_RESTORE )) {
@@ -1173,12 +1253,12 @@ int xe_bo_evict_pinned(struct xe_bo *bo)
11731253 bo -> size );
11741254 }
11751255
1176- bo -> backup_obj = backup ;
1256+ if (!bo -> backup_obj )
1257+ bo -> backup_obj = backup ;
11771258
11781259out_backup :
11791260 xe_bo_vunmap (backup );
1180- xe_bo_unlock (backup );
1181- if (ret )
1261+ if (ret && backup_created )
11821262 xe_bo_put (backup );
11831263out_unlock_bo :
11841264 if (unmap )
@@ -1212,15 +1292,12 @@ int xe_bo_restore_pinned(struct xe_bo *bo)
12121292 if (!backup )
12131293 return 0 ;
12141294
1215- xe_bo_lock (backup , false);
1216-
1217- ret = ttm_bo_validate (& backup -> ttm , & backup -> placement , & ctx );
1218- if (ret )
1219- goto out_backup ;
1295+ xe_bo_lock (bo , false);
12201296
1221- if (WARN_ON (!dma_resv_trylock (bo -> ttm .base .resv ))) {
1222- ret = - EBUSY ;
1223- goto out_backup ;
1297+ if (!xe_bo_is_pinned (backup )) {
1298+ ret = ttm_bo_validate (& backup -> ttm , & backup -> placement , & ctx );
1299+ if (ret )
1300+ goto out_unlock_bo ;
12241301 }
12251302
12261303 if (xe_bo_is_user (bo ) || (bo -> flags & XE_BO_FLAG_PINNED_LATE_RESTORE )) {
@@ -1261,7 +1338,7 @@ int xe_bo_restore_pinned(struct xe_bo *bo)
12611338 if (iosys_map_is_null (& bo -> vmap )) {
12621339 ret = xe_bo_vmap (bo );
12631340 if (ret )
1264- goto out_unlock_bo ;
1341+ goto out_backup ;
12651342 unmap = true;
12661343 }
12671344
@@ -1271,15 +1348,17 @@ int xe_bo_restore_pinned(struct xe_bo *bo)
12711348
12721349 bo -> backup_obj = NULL ;
12731350
1351+ out_backup :
1352+ xe_bo_vunmap (backup );
1353+ if (!bo -> backup_obj ) {
1354+ if (xe_bo_is_pinned (backup ))
1355+ ttm_bo_unpin (& backup -> ttm );
1356+ xe_bo_put (backup );
1357+ }
12741358out_unlock_bo :
12751359 if (unmap )
12761360 xe_bo_vunmap (bo );
12771361 xe_bo_unlock (bo );
1278- out_backup :
1279- xe_bo_vunmap (backup );
1280- xe_bo_unlock (backup );
1281- if (!bo -> backup_obj )
1282- xe_bo_put (backup );
12831362 return ret ;
12841363}
12851364
@@ -1455,16 +1534,22 @@ static int xe_ttm_access_memory(struct ttm_buffer_object *ttm_bo,
14551534 struct xe_res_cursor cursor ;
14561535 struct xe_vram_region * vram ;
14571536 int bytes_left = len ;
1537+ int err = 0 ;
14581538
14591539 xe_bo_assert_held (bo );
14601540 xe_device_assert_mem_access (xe );
14611541
14621542 if (!mem_type_is_vram (ttm_bo -> resource -> mem_type ))
14631543 return - EIO ;
14641544
1465- /* FIXME: Use GPU for non-visible VRAM */
1466- if (!xe_ttm_resource_visible (ttm_bo -> resource ))
1467- return - EIO ;
1545+ if (!xe_ttm_resource_visible (ttm_bo -> resource ) || len >= SZ_16K ) {
1546+ struct xe_migrate * migrate =
1547+ mem_type_to_migrate (xe , ttm_bo -> resource -> mem_type );
1548+
1549+ err = xe_migrate_access_memory (migrate , bo , offset , buf , len ,
1550+ write );
1551+ goto out ;
1552+ }
14681553
14691554 vram = res_to_mem_region (ttm_bo -> resource );
14701555 xe_res_first (ttm_bo -> resource , offset & PAGE_MASK ,
@@ -1488,7 +1573,8 @@ static int xe_ttm_access_memory(struct ttm_buffer_object *ttm_bo,
14881573 xe_res_next (& cursor , PAGE_SIZE );
14891574 } while (bytes_left );
14901575
1491- return len ;
1576+ out :
1577+ return err ?: len ;
14921578}
14931579
14941580const struct ttm_device_funcs xe_ttm_funcs = {
@@ -1532,6 +1618,9 @@ static void xe_ttm_bo_destroy(struct ttm_buffer_object *ttm_bo)
15321618 if (bo -> vm && xe_bo_is_user (bo ))
15331619 xe_vm_put (bo -> vm );
15341620
1621+ if (bo -> parent_obj )
1622+ xe_bo_put (bo -> parent_obj );
1623+
15351624 mutex_lock (& xe -> mem_access .vram_userfault .lock );
15361625 if (!list_empty (& bo -> vram_userfault_link ))
15371626 list_del (& bo -> vram_userfault_link );
@@ -2306,6 +2395,13 @@ void xe_bo_unpin(struct xe_bo *bo)
23062395 xe_assert (xe , !list_empty (& bo -> pinned_link ));
23072396 list_del_init (& bo -> pinned_link );
23082397 spin_unlock (& xe -> pinned .lock );
2398+
2399+ if (bo -> backup_obj ) {
2400+ if (xe_bo_is_pinned (bo -> backup_obj ))
2401+ ttm_bo_unpin (& bo -> backup_obj -> ttm );
2402+ xe_bo_put (bo -> backup_obj );
2403+ bo -> backup_obj = NULL ;
2404+ }
23092405 }
23102406 ttm_bo_unpin (& bo -> ttm );
23112407 if (bo -> ttm .ttm && ttm_tt_is_populated (bo -> ttm .ttm ))
0 commit comments