diff --git a/include/umf/memory_pool_ops.h b/include/umf/memory_pool_ops.h index c9628c77e0..7754dcbe03 100644 --- a/include/umf/memory_pool_ops.h +++ b/include/umf/memory_pool_ops.h @@ -167,7 +167,7 @@ typedef struct umf_memory_pool_ops_t { /// /// @return umf_result_t result of the control operation. /// - umf_result_t (*ext_ctl)(void *hPool, umf_ctl_query_source_t source, + umf_result_t (*ext_ctl)(void *pool, umf_ctl_query_source_t source, const char *name, void *arg, size_t size, umf_ctl_query_type_t queryType, va_list args); diff --git a/include/umf/providers/provider_level_zero.h b/include/umf/providers/provider_level_zero.h index 65d7e3e781..22c3513351 100644 --- a/include/umf/providers/provider_level_zero.h +++ b/include/umf/providers/provider_level_zero.h @@ -8,6 +8,8 @@ #ifndef UMF_LEVEL_ZERO_PROVIDER_H #define UMF_LEVEL_ZERO_PROVIDER_H +#include + #include #ifdef __cplusplus @@ -101,6 +103,17 @@ umf_result_t umfLevelZeroMemoryProviderParamsSetDeviceOrdinal( umf_result_t umfLevelZeroMemoryProviderParamsSetName( umf_level_zero_memory_provider_params_handle_t hParams, const char *name); +/// @brief Adds or removes devices on which allocations should be made +/// resident. +/// @param provider handle to the memory provider +/// @param device device handle +/// @param is_adding Boolean indicating if peer is to be removed or added +/// @return UMF_RESULT_SUCCESS on success or appropriate error code on +/// failure. +umf_result_t umfLevelZeroMemoryProviderResidentDeviceChange( + umf_memory_provider_handle_t provider, ze_device_handle_t device, + bool is_adding); + const umf_memory_provider_ops_t *umfLevelZeroMemoryProviderOps(void); #ifdef __cplusplus diff --git a/src/critnib/critnib.c b/src/critnib/critnib.c index 17a7d80be1..8ff5796684 100644 --- a/src/critnib/critnib.c +++ b/src/critnib/critnib.c @@ -1094,7 +1094,8 @@ int critnib_find(struct critnib *c, uintptr_t key, enum find_dir_t dir, * * If func() returns non-zero, the search is aborted. */ -static int iter(struct critnib_node *__restrict n, word min, word max, +static int iter(struct critnib_node *__restrict n, const word min, + const word max, int (*func)(word key, void *value, void *privdata), void *privdata) { if (is_leaf(n)) { @@ -1129,9 +1130,21 @@ static int iter(struct critnib_node *__restrict n, word min, word max, void critnib_iter(critnib *c, uintptr_t min, uintptr_t max, int (*func)(uintptr_t key, void *value, void *privdata), void *privdata) { + bool wasIterating = false; utils_mutex_lock(&c->mutex); if (c->root) { iter(c->root, min, max, func, privdata); + wasIterating = true; } utils_mutex_unlock(&c->mutex); + if (!wasIterating) { + LOG_DEBUG("there was no root, iterating critnib: %p was skipped", + (void *)c); + } +} + +void critnib_iter_all(critnib *c, + int (*func)(uintptr_t key, void *value, void *privdata), + void *privdata) { + critnib_iter(c, 0, (uintptr_t)-1, func, privdata); } diff --git a/src/critnib/critnib.h b/src/critnib/critnib.h index 690d75faef..c9f215d079 100644 --- a/src/critnib/critnib.h +++ b/src/critnib/critnib.h @@ -35,6 +35,10 @@ int critnib_insert(critnib *c, uintptr_t key, void *value, int update); void critnib_iter(critnib *c, uintptr_t min, uintptr_t max, int (*func)(uintptr_t key, void *value, void *privdata), void *privdata); +void critnib_iter_all(critnib *c, + int (*func)(uintptr_t key, void *value, void *privdata), + void *privdata); + int critnib_remove_release(critnib *c, uintptr_t key); /* diff --git a/src/libumf.def b/src/libumf.def index 68163c6b5f..397ff74aeb 100644 --- a/src/libumf.def +++ b/src/libumf.def @@ -42,6 +42,7 @@ EXPORTS umfLevelZeroMemoryProviderParamsSetDevice umfLevelZeroMemoryProviderParamsSetMemoryType umfLevelZeroMemoryProviderParamsSetResidentDevices + umfLevelZeroMemoryProviderResidentDeviceChange umfMemoryProviderAlloc umfMemoryProviderAllocationMerge umfMemoryProviderAllocationSplit diff --git a/src/libumf.map b/src/libumf.map index 98b9134995..d23c6a2e02 100644 --- a/src/libumf.map +++ b/src/libumf.map @@ -152,6 +152,7 @@ UMF_1.1 { umfGetMemoryPropertySize; umfJemallocPoolParamsSetName; umfLevelZeroMemoryProviderParamsSetName; + umfLevelZeroMemoryProviderResidentDeviceChange; umfOsMemoryProviderParamsSetName; umfPoolTrimMemory; umfScalablePoolParamsSetName; diff --git a/src/provider/provider_level_zero.c b/src/provider/provider_level_zero.c index d5ab3e8e4f..126df53478 100644 --- a/src/provider/provider_level_zero.c +++ b/src/provider/provider_level_zero.c @@ -17,6 +17,7 @@ #include "memory_provider_internal.h" #include "provider_ctl_stats_type.h" #include "provider_level_zero_internal.h" +#include "provider_tracking.h" #include "utils_load_library.h" #include "utils_log.h" @@ -67,8 +68,10 @@ typedef struct ze_memory_provider_t { ze_device_handle_t device; ze_memory_type_t memory_type; + utils_rwlock_t resident_device_rwlock; ze_device_handle_t *resident_device_handles; uint32_t resident_device_count; + uint32_t resident_device_capacity; ze_device_properties_t device_properties; @@ -174,11 +177,18 @@ static ze_memory_type_t umf2ze_memory_type(umf_usm_memory_type_t memory_type) { } static void init_ze_global_state(void) { + + const char *lib_name = getenv("UMF_ZE_LOADER_LIB_NAME"); + if (lib_name != NULL && lib_name[0] != '\0') { + LOG_INFO("Using custom ze_loader library name: %s", lib_name); + } else { #ifdef _WIN32 - const char *lib_name = "ze_loader.dll"; + lib_name = "ze_loader.dll"; #else - const char *lib_name = "libze_loader.so.1"; + lib_name = "libze_loader.so.1"; #endif + LOG_DEBUG("Using default ze_loader library name: %s", lib_name); + } // The Level Zero shared library should be already loaded by the user // of the Level Zero provider. UMF just want to reuse it // and increase the reference count to the Level Zero shared library. @@ -350,12 +360,22 @@ umf_result_t umfLevelZeroMemoryProviderParamsSetResidentDevices( LOG_ERR("Level Zero memory provider params handle is NULL"); return UMF_RESULT_ERROR_INVALID_ARGUMENT; } - - if (deviceCount && !hDevices) { + if (deviceCount > 0 && hDevices == NULL) { LOG_ERR("Resident devices array is NULL, but deviceCount is not zero"); return UMF_RESULT_ERROR_INVALID_ARGUMENT; } + for (uint32_t first_idx = 0; first_idx < deviceCount; first_idx++) { + for (uint32_t second_idx = 0; second_idx < first_idx; second_idx++) { + if (hDevices[first_idx] == hDevices[second_idx]) { + LOG_ERR("resident devices are not unique, idx:%u and " + "idx:%u both point to device:%p", + first_idx, second_idx, (void *)hDevices[first_idx]); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + } + } + hParams->resident_device_handles = hDevices; hParams->resident_device_count = deviceCount; @@ -401,6 +421,43 @@ static ze_relaxed_allocation_limits_exp_desc_t relaxed_device_allocation_desc = .pNext = NULL, .flags = ZE_RELAXED_ALLOCATION_LIMITS_EXP_FLAG_MAX_SIZE}; +static umf_result_t ze_memory_provider_free_helper(void *provider, void *ptr, + size_t bytes, + int update_stats) { + if (ptr == NULL) { + return UMF_RESULT_SUCCESS; + } + + ze_memory_provider_t *ze_provider = (ze_memory_provider_t *)provider; + umf_result_t ret; + if (ze_provider->freePolicyFlags == 0) { + ret = ze2umf_result(g_ze_ops.zeMemFree(ze_provider->context, ptr)); + } else { + ze_memory_free_ext_desc_t desc = { + .stype = ZE_STRUCTURE_TYPE_MEMORY_FREE_EXT_DESC, + .pNext = NULL, + .freePolicy = ze_provider->freePolicyFlags}; + + ret = ze2umf_result( + g_ze_ops.zeMemFreeExt(ze_provider->context, &desc, ptr)); + } + + if (ret != UMF_RESULT_SUCCESS) { + return ret; + } + + if (update_stats) { + provider_ctl_stats_free(ze_provider, bytes); + } + + return UMF_RESULT_SUCCESS; +} + +static umf_result_t ze_memory_provider_free(void *provider, void *ptr, + size_t bytes) { + return ze_memory_provider_free_helper(provider, ptr, bytes, 1); +} + static umf_result_t ze_memory_provider_alloc_helper(void *provider, size_t size, size_t alignment, int update_stats, @@ -459,14 +516,31 @@ static umf_result_t ze_memory_provider_alloc_helper(void *provider, size_t size, return ze2umf_result(ze_result); } + utils_read_lock(&ze_provider->resident_device_rwlock); for (uint32_t i = 0; i < ze_provider->resident_device_count; i++) { ze_result = g_ze_ops.zeContextMakeMemoryResident( ze_provider->context, ze_provider->resident_device_handles[i], *resultPtr, size); if (ze_result != ZE_RESULT_SUCCESS) { + utils_read_unlock(&ze_provider->resident_device_rwlock); + LOG_ERR("making resident allocation %p of size:%lu on device %p " + "failed with 0x%x", + *resultPtr, size, + (void *)ze_provider->resident_device_handles[i], ze_result); + umf_result_t free_result = + ze_memory_provider_free(ze_provider, *resultPtr, size); + if (free_result != UMF_RESULT_SUCCESS) { + LOG_ERR("failed to free memory with: 0x%x after failed making " + "resident, free fail ignored", + free_result); + } return ze2umf_result(ze_result); } + LOG_DEBUG("allocation %p of size:%lu made resident on device %p", + *resultPtr, size, + (void *)ze_provider->resident_device_handles[i]); } + utils_read_unlock(&ze_provider->resident_device_rwlock); if (update_stats) { provider_ctl_stats_alloc(ze_provider, size); @@ -481,43 +555,6 @@ static umf_result_t ze_memory_provider_alloc(void *provider, size_t size, resultPtr); } -static umf_result_t ze_memory_provider_free_helper(void *provider, void *ptr, - size_t bytes, - int update_stats) { - if (ptr == NULL) { - return UMF_RESULT_SUCCESS; - } - - ze_memory_provider_t *ze_provider = (ze_memory_provider_t *)provider; - umf_result_t ret; - if (ze_provider->freePolicyFlags == 0) { - ret = ze2umf_result(g_ze_ops.zeMemFree(ze_provider->context, ptr)); - } else { - ze_memory_free_ext_desc_t desc = { - .stype = ZE_STRUCTURE_TYPE_MEMORY_FREE_EXT_DESC, - .pNext = NULL, - .freePolicy = ze_provider->freePolicyFlags}; - - ret = ze2umf_result( - g_ze_ops.zeMemFreeExt(ze_provider->context, &desc, ptr)); - } - - if (ret != UMF_RESULT_SUCCESS) { - return ret; - } - - if (update_stats) { - provider_ctl_stats_free(ze_provider, bytes); - } - - return UMF_RESULT_SUCCESS; -} - -static umf_result_t ze_memory_provider_free(void *provider, void *ptr, - size_t bytes) { - return ze_memory_provider_free_helper(provider, ptr, bytes, 1); -} - static umf_result_t query_min_page_size(ze_memory_provider_t *ze_provider, size_t *min_page_size) { assert(min_page_size); @@ -544,9 +581,11 @@ static umf_result_t query_min_page_size(ze_memory_provider_t *ze_provider, } static umf_result_t ze_memory_provider_finalize(void *provider) { - ze_memory_provider_t *ze_provider = (ze_memory_provider_t *)provider; - umf_ba_global_free(ze_provider->resident_device_handles); - + ze_memory_provider_t *ze_provider = provider; + if (ze_provider->resident_device_handles != NULL) { + umf_ba_global_free(ze_provider->resident_device_handles); + } + utils_rwlock_destroy_not_free(&ze_provider->resident_device_rwlock); umf_ba_global_free(provider); return UMF_RESULT_SUCCESS; } @@ -571,10 +610,11 @@ static umf_result_t ze_memory_provider_initialize(const void *params, return UMF_RESULT_ERROR_INVALID_ARGUMENT; } - if ((bool)ze_params->resident_device_count && - (ze_params->resident_device_handles == NULL)) { - LOG_ERR("Resident devices handles array is NULL, but device_count is " - "not zero"); + if (ze_params->resident_device_count > 0 && + ze_params->resident_device_handles == NULL) { + LOG_ERR("Device handler should be non-NULL if device_count:%d is " + "greater than 0", + ze_params->resident_device_count); return UMF_RESULT_ERROR_INVALID_ARGUMENT; } @@ -617,27 +657,36 @@ static umf_result_t ze_memory_provider_initialize(const void *params, } } - if (ze_params->resident_device_count) { + if (utils_rwlock_init(&ze_provider->resident_device_rwlock) == NULL) { + LOG_ERR("Cannot initialize resident device rwlock"); + umf_ba_global_free(ze_provider); + return UMF_RESULT_ERROR_OUT_OF_RESOURCES; + } + + ze_provider->resident_device_count = ze_params->resident_device_count; + ze_provider->resident_device_capacity = ze_params->resident_device_count; + + if (ze_params->resident_device_count > 0) { ze_provider->resident_device_handles = umf_ba_global_alloc( sizeof(ze_device_handle_t) * ze_params->resident_device_count); - if (!ze_provider->resident_device_handles) { + if (ze_provider->resident_device_handles == NULL) { LOG_ERR("Cannot allocate memory for resident devices"); + utils_rwlock_destroy_not_free(&ze_provider->resident_device_rwlock); umf_ba_global_free(ze_provider); return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; } - ze_provider->resident_device_count = ze_params->resident_device_count; + memcpy(ze_provider->resident_device_handles, + ze_params->resident_device_handles, + sizeof(ze_device_handle_t) * ze_params->resident_device_count); - for (uint32_t i = 0; i < ze_provider->resident_device_count; i++) { - ze_provider->resident_device_handles[i] = - ze_params->resident_device_handles[i]; - } + LOG_INFO("L0 memory provider:%p have %d resident device(s)", + (void *)ze_provider, ze_params->resident_device_count); } else { - ze_provider->resident_device_handles = NULL; - ze_provider->resident_device_count = 0; + LOG_INFO("L0 memory provider has no resident devices"); } - umf_result_t result = + const umf_result_t result = query_min_page_size(ze_provider, &ze_provider->min_page_size); if (result != UMF_RESULT_SUCCESS) { ze_memory_provider_finalize(ze_provider); @@ -930,6 +979,158 @@ static umf_result_t ze_memory_provider_get_allocation_properties_size( return UMF_RESULT_ERROR_INVALID_ARGUMENT; } +struct ze_memory_provider_resident_device_change_data { + bool is_adding; + ze_device_handle_t peer_device; + ze_memory_provider_t *source_memory_provider; + uint32_t success_changes; + uint32_t failed_changes; +}; + +static int ze_memory_provider_resident_device_change_helper(uintptr_t key, + void *value, + void *privdata) { + struct ze_memory_provider_resident_device_change_data *change_data = + privdata; + tracker_alloc_info_t *info = value; + if (info->props.provider->provider_priv != + (void *)change_data->source_memory_provider) { + LOG_DEBUG("ze_memory_provider_resident_device_change found not our " + "pointer %p", + (void *)key); + return 0; + } + + assert(info->props.base == (void *)key); + + ze_result_t result; + if (change_data->is_adding) { + result = g_ze_ops.zeContextMakeMemoryResident( + change_data->source_memory_provider->context, + change_data->peer_device, info->props.base, info->props.base_size); + } else { + result = ZE_RESULT_SUCCESS; + // TODO: currently not implemented call evict here + } + + if (result != ZE_RESULT_SUCCESS) { + LOG_ERR("ze_memory_provider_resident_device_change found our pointer " + "%p but failed to make it resident on device:%p due to err:%d", + (void *)key, (void *)change_data->peer_device, result); + ++change_data->failed_changes; + return 1; + } + + LOG_DEBUG("ze_memory_provider_resident_device_change found our pointer %p " + "and made it resident on device:%p", + (void *)key, (void *)change_data->peer_device); + ++change_data->success_changes; + return 0; +} + +umf_result_t umfLevelZeroMemoryProviderResidentDeviceChange( + umf_memory_provider_handle_t provider, ze_device_handle_t device, + bool is_adding) { + ze_memory_provider_t *ze_provider = umfMemoryProviderGetPriv(provider); + + LOG_INFO("%s resident device %p, src_provider:%p, existing peers " + "count:%d", + (is_adding ? "adding" : "removing"), (void *)device, + (void *)provider, ze_provider->resident_device_count); + + uint32_t existing_peer_index = 0; + utils_write_lock(&ze_provider->resident_device_rwlock); + while (existing_peer_index < ze_provider->resident_device_count && + ze_provider->resident_device_handles[existing_peer_index] != + device) { + ++existing_peer_index; + } + + if (ze_provider->resident_device_count == 0 || + existing_peer_index == ze_provider->resident_device_count) { + // not found + if (!is_adding) { + utils_write_unlock(&ze_provider->resident_device_rwlock); + LOG_ERR("trying to remove resident device %p but the device " + "is not a peer of provider:%p currently", + (void *)device, (void *)provider); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + // adding case + if (ze_provider->resident_device_count == + ze_provider->resident_device_capacity) { + const uint32_t new_capacity = + ze_provider->resident_device_capacity + 1; + ze_device_handle_t *new_handles = + umf_ba_global_alloc(sizeof(ze_device_handle_t) * new_capacity); + if (new_handles == NULL) { + utils_write_unlock(&ze_provider->resident_device_rwlock); + LOG_ERR("enlarging resident devices array from %u to %u failed " + "due to no memory", + ze_provider->resident_device_capacity, new_capacity); + return UMF_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } + LOG_DEBUG("enlarging resident devices array from %u to %u", + ze_provider->resident_device_capacity, new_capacity); + memcpy(new_handles, ze_provider->resident_device_handles, + sizeof(ze_device_handle_t) * + ze_provider->resident_device_count); + umf_ba_global_free(ze_provider->resident_device_handles); + ze_provider->resident_device_handles = new_handles; + ze_provider->resident_device_capacity = new_capacity; + } + ze_provider->resident_device_handles[existing_peer_index] = device; + ++ze_provider->resident_device_count; + + } else { + // found + if (is_adding) { + utils_write_unlock(&ze_provider->resident_device_rwlock); + LOG_ERR("trying to add resident device:%p but the device is " + "already a peer of provider:%p", + (void *)device, (void *)provider); + return UMF_RESULT_ERROR_INVALID_ARGUMENT; + } + // removing case, put last in place of removed one + --ze_provider->resident_device_count; + ze_provider->resident_device_handles[existing_peer_index] = + ze_provider + ->resident_device_handles[ze_provider->resident_device_count]; + } + utils_write_unlock(&ze_provider->resident_device_rwlock); + + struct ze_memory_provider_resident_device_change_data privData = { + .is_adding = is_adding, + .peer_device = device, + .source_memory_provider = ze_provider, + .success_changes = 0, + .failed_changes = 0, + }; + + // This is "hacky" and it will not work if someone uses pool without tracker + // or just use provider without pool. It can be solved by keeping track of + // allocations by the provider like in os_provider. + umf_result_t result = umfMemoryTrackerIterateAll( + &ze_memory_provider_resident_device_change_helper, &privData); + if (result != UMF_RESULT_SUCCESS) { + LOG_ERR("umfMemoryTrackerIterateAll failed during resident device " + "change with result:%d numFailed:%d, numSuccess:%d", + result, privData.success_changes, privData.failed_changes); + return result; + } + + if (privData.failed_changes > 0) { + LOG_ERR("umfMemoryTrackerIterateAll did not manage to do some change " + "numFailed:%d, numSuccess:%d", + privData.success_changes, privData.failed_changes); + return UMF_RESULT_ERROR_MEMORY_PROVIDER_SPECIFIC; + } + + LOG_INFO("ze_memory_provider_resident_device_change done, numSuccess:%d", + privData.success_changes); + return UMF_RESULT_SUCCESS; +} + static umf_memory_provider_ops_t UMF_LEVEL_ZERO_MEMORY_PROVIDER_OPS = { .version = UMF_PROVIDER_OPS_VERSION_CURRENT, .initialize = ze_memory_provider_initialize, @@ -1049,4 +1250,15 @@ const umf_memory_provider_ops_t *umfLevelZeroMemoryProviderOps(void) { return NULL; } +umf_result_t umfLevelZeroMemoryProviderResidentDeviceChange( + umf_memory_provider_handle_t provider, ze_device_handle_t device, + bool is_adding) { + (void)provider; + (void)device; + (void)is_adding; + LOG_ERR("L0 memory provider is disabled! (UMF_BUILD_LEVEL_ZERO_PROVIDER is " + "OFF)"); + return UMF_RESULT_ERROR_NOT_SUPPORTED; +} + #endif // !UMF_BUILD_LEVEL_ZERO_PROVIDER diff --git a/src/provider/provider_tracking.c b/src/provider/provider_tracking.c index cf76d2be74..b8f3cfa71c 100644 --- a/src/provider/provider_tracking.c +++ b/src/provider/provider_tracking.c @@ -1527,3 +1527,26 @@ void umfMemoryTrackerDestroy(umf_memory_tracker_handle_t handle) { handle->ipc_info_allocator = NULL; umf_ba_global_free(handle); } + +umf_result_t umfMemoryTrackerIterateAll(int (*func)(uintptr_t key, void *value, + void *privdata), + void *privdata) { + if (UNLIKELY(TRACKER == NULL)) { + LOG_ERR("tracker does not exist"); + return UMF_RESULT_ERROR_NOT_SUPPORTED; + } + + if (UNLIKELY(TRACKER->alloc_segments_map[0] == NULL)) { + LOG_ERR("tracker's alloc_segments_map does not exist"); + return UMF_RESULT_ERROR_NOT_SUPPORTED; + } + + for (int level = 0; level < MAX_LEVELS_OF_ALLOC_SEGMENT_MAP; level++) { + critnib *alloc_segment = TRACKER->alloc_segments_map[level]; + LOG_DEBUG("iterating tracker's %d segment:%p", level, + (void *)alloc_segment); + critnib_iter_all(alloc_segment, func, privdata); + } + + return UMF_RESULT_SUCCESS; +} diff --git a/src/provider/provider_tracking.h b/src/provider/provider_tracking.h index cdbee3973f..254bbf6786 100644 --- a/src/provider/provider_tracking.h +++ b/src/provider/provider_tracking.h @@ -70,6 +70,10 @@ void umfTrackingMemoryProviderGetUpstreamProvider( umf_memory_provider_handle_t hTrackingProvider, umf_memory_provider_handle_t *hUpstream); +umf_result_t umfMemoryTrackerIterateAll(int (*func)(uintptr_t key, void *value, + void *privdata), + void *privdata); + #ifdef __cplusplus } #endif diff --git a/src/utils/utils_concurrency.h b/src/utils/utils_concurrency.h index a00b8bc405..922f992e83 100644 --- a/src/utils/utils_concurrency.h +++ b/src/utils/utils_concurrency.h @@ -73,10 +73,10 @@ typedef struct utils_rwlock_t { utils_rwlock_t *utils_rwlock_init(utils_rwlock_t *ptr); void utils_rwlock_destroy_not_free(utils_rwlock_t *rwlock); -int utils_read_lock(utils_rwlock_t *rwlock); -int utils_write_lock(utils_rwlock_t *rwlock); -int utils_read_unlock(utils_rwlock_t *rwlock); -int utils_write_unlock(utils_rwlock_t *rwlock); +void utils_read_lock(utils_rwlock_t *rwlock); +void utils_write_lock(utils_rwlock_t *rwlock); +void utils_read_unlock(utils_rwlock_t *rwlock); +void utils_write_unlock(utils_rwlock_t *rwlock); #if defined(_WIN32) #define UTIL_ONCE_FLAG INIT_ONCE diff --git a/src/utils/utils_level_zero.cpp b/src/utils/utils_level_zero.cpp index 6daab3e691..ee7aa85eae 100644 --- a/src/utils/utils_level_zero.cpp +++ b/src/utils/utils_level_zero.cpp @@ -13,8 +13,6 @@ #include "utils_concurrency.h" #include "utils_load_library.h" -#include "ze_api.h" - struct libze_ops { ze_result_t (*zeInit)(ze_init_flags_t flags); ze_result_t (*zeDriverGet)(uint32_t *pCount, ze_driver_handle_t *phDrivers); diff --git a/src/utils/utils_posix_concurrency.c b/src/utils/utils_posix_concurrency.c index c6f273bed2..6cbcd908ec 100644 --- a/src/utils/utils_posix_concurrency.c +++ b/src/utils/utils_posix_concurrency.c @@ -54,24 +54,36 @@ utils_rwlock_t *utils_rwlock_init(utils_rwlock_t *ptr) { void utils_rwlock_destroy_not_free(utils_rwlock_t *ptr) { pthread_rwlock_t *rwlock = (pthread_rwlock_t *)ptr; - int ret = pthread_rwlock_destroy(rwlock); - if (ret) { - LOG_ERR("pthread_rwlock_destroy failed"); + if (pthread_rwlock_destroy(rwlock) != 0) { + LOG_FATAL("pthread_rwlock_destroy failed"); + abort(); } } -int utils_read_lock(utils_rwlock_t *rwlock) { - return pthread_rwlock_rdlock((pthread_rwlock_t *)rwlock); +void utils_read_lock(utils_rwlock_t *rwlock) { + if (pthread_rwlock_rdlock((pthread_rwlock_t *)rwlock) != 0) { + LOG_FATAL("pthread_rwlock_rdlock failed"); + abort(); + } } -int utils_write_lock(utils_rwlock_t *rwlock) { - return pthread_rwlock_wrlock((pthread_rwlock_t *)rwlock); +void utils_write_lock(utils_rwlock_t *rwlock) { + if (pthread_rwlock_wrlock((pthread_rwlock_t *)rwlock) != 0) { + LOG_FATAL("pthread_rwlock_wrlock failed"); + abort(); + } } -int utils_read_unlock(utils_rwlock_t *rwlock) { - return pthread_rwlock_unlock((pthread_rwlock_t *)rwlock); +void utils_read_unlock(utils_rwlock_t *rwlock) { + if (pthread_rwlock_unlock((pthread_rwlock_t *)rwlock) != 0) { + LOG_FATAL("pthread_rwlock_unlock failed"); + abort(); + } } -int utils_write_unlock(utils_rwlock_t *rwlock) { - return pthread_rwlock_unlock((pthread_rwlock_t *)rwlock); +void utils_write_unlock(utils_rwlock_t *rwlock) { + if (pthread_rwlock_unlock((pthread_rwlock_t *)rwlock) != 0) { + LOG_FATAL("pthread_rwlock_unlock failed"); + abort(); + } } diff --git a/src/utils/utils_windows_concurrency.c b/src/utils/utils_windows_concurrency.c index faa302be36..4e2f9c3046 100644 --- a/src/utils/utils_windows_concurrency.c +++ b/src/utils/utils_windows_concurrency.c @@ -46,24 +46,20 @@ void utils_rwlock_destroy_not_free(utils_rwlock_t *rwlock) { (void)rwlock; } -int utils_read_lock(utils_rwlock_t *rwlock) { - AcquireSRWLockShared(&rwlock->lock); - return 0; // never fails +void utils_read_lock(utils_rwlock_t *rwlock) { + AcquireSRWLockShared(&rwlock->lock); // never fails } -int utils_write_lock(utils_rwlock_t *rwlock) { - AcquireSRWLockExclusive(&rwlock->lock); - return 0; // never fails +void utils_write_lock(utils_rwlock_t *rwlock) { + AcquireSRWLockExclusive(&rwlock->lock); // never fails } -int utils_read_unlock(utils_rwlock_t *rwlock) { - ReleaseSRWLockShared(&rwlock->lock); - return 0; // never fails +void utils_read_unlock(utils_rwlock_t *rwlock) { + ReleaseSRWLockShared(&rwlock->lock); // never fails } -int utils_write_unlock(utils_rwlock_t *rwlock) { - ReleaseSRWLockExclusive(&rwlock->lock); - return 0; // never fails +void utils_write_unlock(utils_rwlock_t *rwlock) { + ReleaseSRWLockExclusive(&rwlock->lock); // never fails } static BOOL CALLBACK initOnceCb(PINIT_ONCE InitOnce, PVOID Parameter, diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 8862b7b88d..b3c2ecd703 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -35,8 +35,9 @@ endif() enable_testing() set(UMF_TEST_DIR ${CMAKE_CURRENT_SOURCE_DIR}) -set(UMF_UTILS_DIR ${UMF_CMAKE_SOURCE_DIR}/src/utils) -set(UMF_BA_DIR ${UMF_CMAKE_SOURCE_DIR}/src/base_alloc) +set(UMF_SRC_DIR ${UMF_CMAKE_SOURCE_DIR}/src) +set(UMF_UTILS_DIR ${UMF_SRC_DIR}/utils) +set(UMF_BA_DIR ${UMF_SRC_DIR}/base_alloc) function(build_umf_test) # Parameters: @@ -430,6 +431,22 @@ else() LIBS ${UMF_UTILS_FOR_TEST} ${UMF_BA_FOR_TEST}) endif() +if(UMF_BUILD_LEVEL_ZERO_PROVIDER) + add_umf_test( + NAME provider_level_zero_residency + SRCS providers/provider_level_zero_residency.cpp + ${UMF_UTILS_DIR}/utils_level_zero.cpp + LIBS ${UMF_UTILS_FOR_TEST} umf_ze_loopback umf_test_mocks GTest::gmock) + add_umf_test( + NAME pool_residency + SRCS pools/pool_residency.cpp ${UMF_UTILS_DIR}/utils_level_zero.cpp + LIBS ${UMF_UTILS_FOR_TEST} + ${UMF_BA_FOR_TEST} + umf_ze_loopback + umf_test_mocks + GTest::gmock) +endif() + if(UMF_BUILD_GPU_TESTS AND UMF_LEVEL_ZERO_ENABLED) # we have two test binaries here that use the same sources, but differ in # the way they are linked to the Level Zero (statically or at runtime using diff --git a/test/common/CMakeLists.txt b/test/common/CMakeLists.txt index 6cffe5cfe8..67f20ecd28 100644 --- a/test/common/CMakeLists.txt +++ b/test/common/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (C) 2023-2024 Intel Corporation +# Copyright (C) 2023-2025 Intel Corporation # Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception @@ -20,3 +20,15 @@ add_umf_library( target_include_directories(umf_test_common PRIVATE ${UMF_CMAKE_SOURCE_DIR}/include) + +if(UMF_BUILD_LEVEL_ZERO_PROVIDER) + add_library(umf_test_mocks STATIC level_zero_mocks.cpp) + target_link_libraries(umf_test_mocks GTest::gmock umf_utils) + target_include_directories( + umf_test_mocks PUBLIC ${UMF_CMAKE_SOURCE_DIR}/include ${UMF_SRC_DIR} + ${UMF_UTILS_DIR} ${LEVEL_ZERO_INCLUDE_DIRS}) + + add_library(umf_ze_loopback SHARED ze_loopback.h ze_loopback.cpp) + target_include_directories(umf_ze_loopback + PUBLIC ${LEVEL_ZERO_INCLUDE_DIRS}) +endif() diff --git a/test/common/level_zero_mocks.cpp b/test/common/level_zero_mocks.cpp new file mode 100644 index 0000000000..f5a5e59443 --- /dev/null +++ b/test/common/level_zero_mocks.cpp @@ -0,0 +1,115 @@ +/* + * + * Copyright (C) 2025 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + */ + +#include "level_zero_mocks.h" + +#include "umf/providers/provider_level_zero.h" +#include "utils_load_library.h" +#include + +using namespace ::testing; + +umf_memory_provider_handle_t +LevelZeroMock::initializeMemoryProviderWithResidentDevices( + ze_device_handle_t device, std::vector residentDevices, + ze_context_handle_t context, ze_device_properties_t device_properties, + ze_memory_allocation_properties_t memory_allocation_properties) { + umf_level_zero_memory_provider_params_handle_t params = nullptr; + EXPECT_EQ(umfLevelZeroMemoryProviderParamsCreate(¶ms), + UMF_RESULT_SUCCESS); + EXPECT_EQ(umfLevelZeroMemoryProviderParamsSetContext(params, context), + UMF_RESULT_SUCCESS); + EXPECT_EQ(umfLevelZeroMemoryProviderParamsSetDevice(params, device), + UMF_RESULT_SUCCESS); + EXPECT_EQ(umfLevelZeroMemoryProviderParamsSetMemoryType( + params, UMF_MEMORY_TYPE_DEVICE), + UMF_RESULT_SUCCESS); + + EXPECT_EQ(umfLevelZeroMemoryProviderParamsSetResidentDevices( + params, residentDevices.data(), residentDevices.size()), + UMF_RESULT_SUCCESS); + + // query min page size operation upon provider initialization + EXPECT_CALL(*this, zeDeviceGetProperties(device, _)) + .WillRepeatedly(DoAll(SetArgPointee<1>(device_properties), + Return(ZE_RESULT_SUCCESS))); + EXPECT_CALL(*this, zeMemAllocDevice(CONTEXT, _, _, _, device, _)) + .WillOnce(Return(ZE_RESULT_SUCCESS)); + for (auto dev : residentDevices) { + EXPECT_CALL(*this, zeContextMakeMemoryResident(context, dev, _, _)) + .WillOnce(Return(ZE_RESULT_SUCCESS)); + } + EXPECT_CALL(*this, zeMemGetAllocProperties(context, _, _, _)) + .WillOnce(DoAll(SetArgPointee<2>(memory_allocation_properties), + Return(ZE_RESULT_SUCCESS))); + EXPECT_CALL(*this, zeMemFree(CONTEXT, _)) + .WillOnce(Return(ZE_RESULT_SUCCESS)); + + umf_memory_provider_handle_t provider = nullptr; + EXPECT_EQ(umfMemoryProviderCreate(umfLevelZeroMemoryProviderOps(), params, + &provider), + UMF_RESULT_SUCCESS); + EXPECT_NE(provider, nullptr); + + umfLevelZeroMemoryProviderParamsDestroy(params); + return provider; +} + +ze_device_properties_t TestCreateDeviceProperties() { + return ze_device_properties_t{ZE_STRUCTURE_TYPE_DEVICE_PROPERTIES, + nullptr, + ZE_DEVICE_TYPE_GPU, + 0, + 0, + 0, + 0, + 0, + 1024, + 100, + 20, + 16, + 256, + 8, + 2, + 4, + 1, + 8, + 8, + {123}, + "TESTGPU"}; +}; + +ze_memory_allocation_properties_t +TestCreateMemoryAllocationProperties(uint32_t modifier) { + return ze_memory_allocation_properties_t{ + ZE_STRUCTURE_TYPE_MEMORY_ALLOCATION_PROPERTIES, nullptr, + ZE_MEMORY_TYPE_DEVICE, modifier, 2048}; +} + +void MockedLevelZeroTestEnvironment::SetUp() { +#ifdef _WIN32 + const char *lib_name = "umf_ze_loopback.dll"; + _putenv_s("UMF_ZE_LOADER_LIB_NAME", lib_name); +#else + const char *lib_name = "libumf_ze_loopback.so"; + setenv("UMF_ZE_LOADER_LIB_NAME", lib_name, 1); +#endif + + void *lib_handle = + utils_open_library(lib_name, UMF_UTIL_OPEN_LIBRARY_NO_LOAD); + ASSERT_NE(lib_handle, nullptr); + + l0interface = static_cast( + utils_get_symbol_addr(lib_handle, "level_zero_mock", lib_name)); + ASSERT_NE(l0interface, nullptr); + ASSERT_EQ(*l0interface, nullptr); +} +void MockedLevelZeroTestEnvironment::TearDown() {} + +LevelZero **MockedLevelZeroTestEnvironment::l0interface; \ No newline at end of file diff --git a/test/common/level_zero_mocks.h b/test/common/level_zero_mocks.h new file mode 100644 index 0000000000..94fef6c538 --- /dev/null +++ b/test/common/level_zero_mocks.h @@ -0,0 +1,89 @@ +/* + * + * Copyright (C) 2025 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + */ + +#ifndef UMF_TEST_PROVIDER_LEVEL_ZERO_MOCKS_H +#define UMF_TEST_PROVIDER_LEVEL_ZERO_MOCKS_H + +#include "utils_log.h" +#include "ze_loopback.h" +#include +#include + +// TEST CREATE methods for objects + +template constexpr T TestCreatePointer(uintptr_t modifier = 0) { + return reinterpret_cast(static_cast(0x1000) + modifier); +} + +ze_device_properties_t TestCreateDeviceProperties(); + +ze_memory_allocation_properties_t +TestCreateMemoryAllocationProperties(uint32_t modifier = 0); + +// already created common instances for tests writing convenience + +static const auto DEVICE_0 = TestCreatePointer(0); +static const auto DEVICE_1 = TestCreatePointer(1); +static const auto DEVICE_2 = TestCreatePointer(2); +static const auto DEVICE_3 = TestCreatePointer(3); +static const auto DEVICE_4 = TestCreatePointer(4); +static const auto DEVICE_5 = TestCreatePointer(5); + +static const auto CONTEXT = TestCreatePointer(); +static const auto DEVICE_PROPS = TestCreateDeviceProperties(); +static const auto MEM_PROPS = TestCreateMemoryAllocationProperties(); + +static void *POINTER_0 = TestCreatePointer(0x90); +static void *POINTER_1 = TestCreatePointer(0x91); +static void *POINTER_2 = TestCreatePointer(0x92); +static void *POINTER_3 = TestCreatePointer(0x93); +static void *POINTER_4 = TestCreatePointer(0x94); + +class LevelZeroMock : public LevelZero { + public: + MOCK_METHOD3(zeContextCreate, + ze_result_t(ze_driver_handle_t, const ze_context_desc_t *, + ze_context_handle_t *)); + MOCK_METHOD2(zeDeviceGetProperties, + ze_result_t(ze_device_handle_t, ze_device_properties_t *)); + MOCK_METHOD6(zeMemAllocDevice, + ze_result_t(ze_context_handle_t, + const ze_device_mem_alloc_desc_t *, size_t, size_t, + ze_device_handle_t, void **)); + MOCK_METHOD4(zeMemGetAllocProperties, + ze_result_t(ze_context_handle_t, const void *, + ze_memory_allocation_properties_t *, + ze_device_handle_t *)); + MOCK_METHOD4(zeContextMakeMemoryResident, + ze_result_t(ze_context_handle_t, ze_device_handle_t, void *, + size_t)); + MOCK_METHOD2(zeMemFree, + ze_result_t(ze_context_handle_t hContext, void *ptr)); + + // helper setting all expects related to successful l0 provider creation + // and initialization and calling its creation and initialization + umf_memory_provider_handle_t initializeMemoryProviderWithResidentDevices( + ze_device_handle_t device, + std::vector residentDevices, + ze_context_handle_t context = CONTEXT, + ze_device_properties_t device_properties = DEVICE_PROPS, + ze_memory_allocation_properties_t memory_allocation_properties = + MEM_PROPS); +}; + +// important, makes UMF load ze_loopback instead of regular l0 +class MockedLevelZeroTestEnvironment : public ::testing::Environment { + public: + static LevelZero **l0interface; + + void SetUp() override; + void TearDown() override; +}; + +#endif //UMF_TEST_PROVIDER_LEVEL_ZERO_MOCKS_H diff --git a/test/common/ze_loopback.cpp b/test/common/ze_loopback.cpp new file mode 100644 index 0000000000..3ca4cd4c73 --- /dev/null +++ b/test/common/ze_loopback.cpp @@ -0,0 +1,273 @@ +/* + * + * Copyright (C) 2025 Intel Corporation + * + * Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + */ + +#include "ze_loopback.h" + +#include +#include + +LevelZero *level_zero_mock = nullptr; + +static void check_mock_present() { + if (level_zero_mock == nullptr) { + std::cerr << "level_zero_mock was not set\n"; + abort(); + } +} + +#define FAIL_NOT_IMPLEMENTED \ + std::cerr << __func__ << " not implemented in ze_loopback.cpp\n"; \ + abort(); + +// +// libze_ops from src/utils/utils_level_zero.cpp +// + +ZE_APIEXPORT ze_result_t ZE_APICALL zeInit(ze_init_flags_t flags) { + (void)flags; + return ZE_RESULT_SUCCESS; +} + +ZE_APIEXPORT ze_result_t ZE_APICALL zeDriverGet(uint32_t *pCount, + ze_driver_handle_t *phDrivers) { + (void)phDrivers; + (void)pCount; + FAIL_NOT_IMPLEMENTED +} + +ZE_APIEXPORT ze_result_t ZE_APICALL zeDeviceGet(ze_driver_handle_t hDriver, + uint32_t *pCount, + ze_device_handle_t *phDevices) { + (void)hDriver; + (void)pCount; + (void)phDevices; + FAIL_NOT_IMPLEMENTED +} + +ZE_APIEXPORT ze_result_t ZE_APICALL zeDeviceGetProperties( + ze_device_handle_t hDevice, ze_device_properties_t *pDeviceProperties) { + check_mock_present(); + return level_zero_mock->zeDeviceGetProperties(hDevice, pDeviceProperties); +} + +ZE_APIEXPORT ze_result_t ZE_APICALL +zeContextCreate(ze_driver_handle_t hDriver, const ze_context_desc_t *desc, + ze_context_handle_t *phContext) { + (void)hDriver; + (void)desc; + (void)phContext; + FAIL_NOT_IMPLEMENTED +} + +ZE_APIEXPORT ze_result_t ZE_APICALL +zeContextDestroy(ze_context_handle_t hContext) { + (void)hContext; + FAIL_NOT_IMPLEMENTED; +} + +ZE_APIEXPORT ze_result_t ZE_APICALL +zeCommandQueueCreate(ze_context_handle_t hContext, ze_device_handle_t hDevice, + const ze_command_queue_desc_t *desc, + ze_command_queue_handle_t *phCommandQueue) { + (void)hContext; + (void)hDevice; + (void)desc; + (void)phCommandQueue; + FAIL_NOT_IMPLEMENTED +} + +ZE_APIEXPORT ze_result_t ZE_APICALL +zeCommandQueueDestroy(ze_command_queue_handle_t hCommandQueue) { + (void)hCommandQueue; + FAIL_NOT_IMPLEMENTED +} + +ZE_APIEXPORT ze_result_t ZE_APICALL zeCommandQueueExecuteCommandLists( + ze_command_queue_handle_t hCommandQueue, uint32_t numCommandLists, + ze_command_list_handle_t *phCommandLists, ze_fence_handle_t hFence) { + (void)hCommandQueue; + (void)numCommandLists; + (void)phCommandLists; + (void)hFence; + FAIL_NOT_IMPLEMENTED +} + +ZE_APIEXPORT ze_result_t ZE_APICALL zeCommandQueueSynchronize( + ze_command_queue_handle_t hCommandQueue, uint64_t timeout) { + (void)hCommandQueue; + (void)timeout; + FAIL_NOT_IMPLEMENTED +} + +ZE_APIEXPORT ze_result_t ZE_APICALL +zeCommandListCreate(ze_context_handle_t hContext, ze_device_handle_t hDevice, + const ze_command_list_desc_t *desc, + ze_command_list_handle_t *phCommandList) { + (void)hContext; + (void)hDevice; + (void)desc; + (void)phCommandList; + FAIL_NOT_IMPLEMENTED +} + +ZE_APIEXPORT ze_result_t ZE_APICALL +zeCommandListDestroy(ze_command_list_handle_t hCommandList) { + (void)hCommandList; + FAIL_NOT_IMPLEMENTED +} + +ZE_APIEXPORT ze_result_t ZE_APICALL +zeCommandListClose(ze_command_list_handle_t hCommandList) { + (void)hCommandList; + FAIL_NOT_IMPLEMENTED +} + +ZE_APIEXPORT ze_result_t ZE_APICALL zeCommandListAppendMemoryCopy( + ze_command_list_handle_t hCommandList, void *dstptr, const void *srcptr, + size_t size, ze_event_handle_t hSignalEvent, uint32_t numWaitEvents, + ze_event_handle_t *phWaitEvents) { + (void)hCommandList; + (void)dstptr; + (void)srcptr; + (void)size; + (void)hSignalEvent; + (void)numWaitEvents; + (void)phWaitEvents; + FAIL_NOT_IMPLEMENTED +} + +ZE_APIEXPORT ze_result_t ZE_APICALL zeCommandListAppendMemoryFill( + ze_command_list_handle_t hCommandList, void *ptr, const void *pattern, + size_t pattern_size, size_t size, ze_event_handle_t hSignalEvent, + uint32_t numWaitEvents, ze_event_handle_t *phWaitEvents) { + (void)hCommandList; + (void)ptr; + (void)pattern; + (void)pattern_size; + (void)size; + (void)hSignalEvent; + (void)numWaitEvents; + (void)phWaitEvents; + FAIL_NOT_IMPLEMENTED +} + +ZE_APIEXPORT ze_result_t ZE_APICALL +zeMemGetAllocProperties(ze_context_handle_t hContext, const void *ptr, + ze_memory_allocation_properties_t *pMemAllocProperties, + ze_device_handle_t *phDevice) { + check_mock_present(); + return level_zero_mock->zeMemGetAllocProperties( + hContext, ptr, pMemAllocProperties, phDevice); +} + +ZE_APIEXPORT ze_result_t ZE_APICALL zeMemAllocDevice( + ze_context_handle_t hContext, const ze_device_mem_alloc_desc_t *device_desc, + size_t size, size_t alignment, ze_device_handle_t hDevice, void **pptr) { + check_mock_present(); + return level_zero_mock->zeMemAllocDevice(hContext, device_desc, size, + alignment, hDevice, pptr); +} + +ZE_APIEXPORT ze_result_t ZE_APICALL zeMemFree(ze_context_handle_t hContext, + void *ptr) { + check_mock_present(); + return level_zero_mock->zeMemFree(hContext, ptr); +} + +ZE_APIEXPORT ze_result_t ZE_APICALL +zeDeviceGetMemoryProperties(ze_device_handle_t hDevice, uint32_t *pCount, + ze_device_memory_properties_t *pMemProperties) { + (void)hDevice; + (void)pCount; + (void)pMemProperties; + FAIL_NOT_IMPLEMENTED +} + +// +// ze_ops_t operations from src/provider/provider_level_zero.c +// + +ze_result_t ZE_APICALL zeMemAllocHost(ze_context_handle_t hContext, + const ze_host_mem_alloc_desc_t *host_desc, + size_t size, size_t alignment, + void **pptr) { + (void)hContext; + (void)host_desc; + (void)size; + (void)alignment; + (void)pptr; + FAIL_NOT_IMPLEMENTED +} + +ze_result_t ZE_APICALL zeMemAllocShared( + ze_context_handle_t hContext, const ze_device_mem_alloc_desc_t *device_desc, + const ze_host_mem_alloc_desc_t *host_desc, size_t size, size_t alignment, + ze_device_handle_t hDevice, void **pptr) { + (void)hContext; + (void)device_desc; + (void)host_desc; + (void)size; + (void)alignment; + (void)hDevice; + (void)pptr; + FAIL_NOT_IMPLEMENTED +} + +ze_result_t ZE_APICALL zeMemGetIpcHandle(ze_context_handle_t hContext, + const void *ptr, + ze_ipc_mem_handle_t *pIpcHandle) { + (void)hContext; + (void)ptr; + (void)pIpcHandle; + FAIL_NOT_IMPLEMENTED +} + +ze_result_t ZE_APICALL zeMemPutIpcHandle(ze_context_handle_t hContext, + ze_ipc_mem_handle_t handle) { + (void)hContext; + (void)handle; + FAIL_NOT_IMPLEMENTED +} + +ze_result_t ZE_APICALL zeMemOpenIpcHandle(ze_context_handle_t hContext, + ze_device_handle_t hDevice, + ze_ipc_mem_handle_t handle, + ze_ipc_memory_flags_t flags, + void **pptr) { + (void)hContext; + (void)hDevice; + (void)handle; + (void)flags; + (void)pptr; + FAIL_NOT_IMPLEMENTED +} + +ze_result_t ZE_APICALL zeMemCloseIpcHandle(ze_context_handle_t hContext, + const void *ptr) { + (void)hContext; + (void)ptr; + FAIL_NOT_IMPLEMENTED +} + +ze_result_t ZE_APICALL zeContextMakeMemoryResident(ze_context_handle_t hContext, + ze_device_handle_t hDevice, + void *ptr, size_t size) { + check_mock_present(); + return level_zero_mock->zeContextMakeMemoryResident(hContext, hDevice, ptr, + size); +} + +ze_result_t ZE_APICALL +zeMemFreeExt(ze_context_handle_t hContext, + const ze_memory_free_ext_desc_t *pMemFreeDesc, void *ptr) { + (void)hContext; + (void)pMemFreeDesc; + (void)ptr; + FAIL_NOT_IMPLEMENTED +} diff --git a/test/common/ze_loopback.h b/test/common/ze_loopback.h new file mode 100644 index 0000000000..2bfa441be5 --- /dev/null +++ b/test/common/ze_loopback.h @@ -0,0 +1,33 @@ +// Copyright (C) 2025 Intel Corporation +// Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#ifndef UMF_TEST_ZE_LOOPBACK_H +#define UMF_TEST_ZE_LOOPBACK_H + +#include "ze_api.h" + +class LevelZero { + public: + virtual ~LevelZero() = default; + + virtual ze_result_t zeContextCreate(ze_driver_handle_t, + const ze_context_desc_t *, + ze_context_handle_t *) = 0; + virtual ze_result_t zeDeviceGetProperties(ze_device_handle_t, + ze_device_properties_t *) = 0; + virtual ze_result_t zeMemAllocDevice(ze_context_handle_t, + const ze_device_mem_alloc_desc_t *, + size_t, size_t, ze_device_handle_t, + void **) = 0; + virtual ze_result_t + zeMemGetAllocProperties(ze_context_handle_t, const void *, + ze_memory_allocation_properties_t *, + ze_device_handle_t *) = 0; + virtual ze_result_t zeContextMakeMemoryResident(ze_context_handle_t, + ze_device_handle_t, void *, + size_t) = 0; + virtual ze_result_t zeMemFree(ze_context_handle_t hContext, void *ptr) = 0; +}; + +#endif //UMF_TEST_ZE_LOOPBACK_H diff --git a/test/pools/pool_residency.cpp b/test/pools/pool_residency.cpp new file mode 100644 index 0000000000..8f4db27a3f --- /dev/null +++ b/test/pools/pool_residency.cpp @@ -0,0 +1,156 @@ +// Copyright (C) 2025 Intel Corporation +// Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "../common/level_zero_mocks.h" +#include "pool.hpp" +#include "umf/pools/pool_disjoint.h" +#include "umf/providers/provider_level_zero.h" + +#include "gtest/gtest.h" + +using namespace testing; + +class PoolResidencyTestFixture : public Test { + protected: + umf_memory_pool_handle_t pool = nullptr; + const ze_device_handle_t OUR_DEVICE; + StrictMock l0mock; + + PoolResidencyTestFixture() + : OUR_DEVICE(TestCreatePointer(777)) { + *MockedLevelZeroTestEnvironment::l0interface = &l0mock; + } + + void initializeMemoryPool(umf_memory_provider_handle_t provider) { + + auto *params = static_cast( + umf_test::defaultDisjointPoolConfig()); + + EXPECT_EQ(umfPoolCreate(umfDisjointPoolOps(), provider, params, + UMF_POOL_CREATE_FLAG_OWN_PROVIDER, &pool), + UMF_RESULT_SUCCESS); + + umf_test::defaultDisjointPoolConfigDestroy(params); + } + + void SetUp() override {} + void TearDown() override { + if (pool != nullptr) { + EXPECT_CALL(l0mock, zeMemFree(CONTEXT, _)) + .WillRepeatedly(Return(ZE_RESULT_SUCCESS)); + umfPoolDestroy(pool); + } + Mock::VerifyAndClearExpectations(&l0mock); + } +}; + +TEST_F(PoolResidencyTestFixture, + initialResidentDevicesShouldBeUsedDuringAllocation) { + initializeMemoryPool(l0mock.initializeMemoryProviderWithResidentDevices( + OUR_DEVICE, {DEVICE_0, DEVICE_1})); + + EXPECT_CALL(l0mock, zeMemAllocDevice(CONTEXT, _, _, _, OUR_DEVICE, _)) + .WillOnce( + DoAll(SetArgPointee<5>(POINTER_0), Return(ZE_RESULT_SUCCESS))); + EXPECT_CALL(l0mock, zeContextMakeMemoryResident(CONTEXT, DEVICE_0, _, _)) + .WillOnce(Return(ZE_RESULT_SUCCESS)); + EXPECT_CALL(l0mock, zeContextMakeMemoryResident(CONTEXT, DEVICE_1, _, _)) + .WillOnce(Return(ZE_RESULT_SUCCESS)); + + void *ptr = umfPoolMalloc(pool, 123); + EXPECT_EQ(ptr, POINTER_0); + + umfPoolFree(pool, ptr); +} + +TEST_F(PoolResidencyTestFixture, + addedResidentDevicesShouldBeUsedDuringAllocation) { + initializeMemoryPool(l0mock.initializeMemoryProviderWithResidentDevices( + OUR_DEVICE, {DEVICE_0})); + + umf_memory_provider_handle_t provider = nullptr; + EXPECT_EQ(umfPoolGetMemoryProvider(pool, &provider), UMF_RESULT_SUCCESS); + umfLevelZeroMemoryProviderResidentDeviceChange(provider, DEVICE_4, true); + + EXPECT_CALL(l0mock, zeMemAllocDevice(CONTEXT, _, _, _, OUR_DEVICE, _)) + .WillOnce( + DoAll(SetArgPointee<5>(POINTER_0), Return(ZE_RESULT_SUCCESS))); + EXPECT_CALL(l0mock, zeContextMakeMemoryResident(CONTEXT, DEVICE_0, _, _)) + .WillOnce(Return(ZE_RESULT_SUCCESS)); + EXPECT_CALL(l0mock, zeContextMakeMemoryResident(CONTEXT, DEVICE_4, _, _)) + .WillOnce(Return(ZE_RESULT_SUCCESS)); + + void *ptr = umfPoolMalloc(pool, 123); + EXPECT_EQ(ptr, POINTER_0); + + umfPoolFree(pool, ptr); +} + +TEST_F(PoolResidencyTestFixture, + existingAllocationsShouldBeMadeResidentOnAddedDevice) { + initializeMemoryPool( + l0mock.initializeMemoryProviderWithResidentDevices(OUR_DEVICE, {})); + + EXPECT_CALL(l0mock, zeMemAllocDevice(CONTEXT, _, _, _, OUR_DEVICE, _)) + .WillOnce( + DoAll(SetArgPointee<5>(POINTER_0), Return(ZE_RESULT_SUCCESS))); + + void *ptr = umfPoolMalloc(pool, 123); + EXPECT_EQ(ptr, POINTER_0); + + EXPECT_CALL(l0mock, zeContextMakeMemoryResident(CONTEXT, DEVICE_4, _, _)) + .WillOnce(Return(ZE_RESULT_SUCCESS)); + + umf_memory_provider_handle_t provider = nullptr; + EXPECT_EQ(umfPoolGetMemoryProvider(pool, &provider), UMF_RESULT_SUCCESS); + umfLevelZeroMemoryProviderResidentDeviceChange(provider, DEVICE_4, true); + + umfPoolFree(pool, ptr); +} + +TEST_F(PoolResidencyTestFixture, + allocationShouldNotBeMadeResidentOnRemovedDevice) { + initializeMemoryPool(l0mock.initializeMemoryProviderWithResidentDevices( + OUR_DEVICE, {DEVICE_2})); + + umf_memory_provider_handle_t provider = nullptr; + EXPECT_EQ(umfPoolGetMemoryProvider(pool, &provider), UMF_RESULT_SUCCESS); + umfLevelZeroMemoryProviderResidentDeviceChange(provider, DEVICE_2, false); + + EXPECT_CALL(l0mock, zeMemAllocDevice(CONTEXT, _, _, _, OUR_DEVICE, _)) + .WillOnce( + DoAll(SetArgPointee<5>(POINTER_0), Return(ZE_RESULT_SUCCESS))); + EXPECT_CALL(l0mock, zeContextMakeMemoryResident(CONTEXT, DEVICE_2, _, _)) + .Times(0); // not called + + void *ptr = umfPoolMalloc(pool, 123); + EXPECT_EQ(ptr, POINTER_0); + + umfPoolFree(pool, ptr); +} + +TEST_F(PoolResidencyTestFixture, + allocationThatFailedToBeMadeResidedShouldBeFreed) { + initializeMemoryPool(l0mock.initializeMemoryProviderWithResidentDevices( + OUR_DEVICE, {DEVICE_2})); + + EXPECT_CALL(l0mock, zeMemAllocDevice(CONTEXT, _, _, _, OUR_DEVICE, _)) + .WillOnce( + DoAll(SetArgPointee<5>(POINTER_0), Return(ZE_RESULT_SUCCESS))); + EXPECT_CALL(l0mock, zeContextMakeMemoryResident(CONTEXT, DEVICE_2, _, _)) + .WillOnce(Return(ZE_RESULT_ERROR_DEVICE_LOST)); + EXPECT_CALL(l0mock, zeMemFree(CONTEXT, _)) + .WillOnce(Return(ZE_RESULT_ERROR_DEVICE_IN_LOW_POWER_STATE)); + + void *ptr = umfPoolMalloc(pool, 16 * 1024 * 1024); + EXPECT_EQ(ptr, nullptr); + + umfPoolFree(pool, ptr); +} + +int main(int argc, char **argv) { + InitGoogleTest(&argc, argv); + AddGlobalTestEnvironment(new MockedLevelZeroTestEnvironment); + return RUN_ALL_TESTS(); +} diff --git a/test/providers/provider_level_zero.cpp b/test/providers/provider_level_zero.cpp index d91660399b..9d72e4f367 100644 --- a/test/providers/provider_level_zero.cpp +++ b/test/providers/provider_level_zero.cpp @@ -204,21 +204,37 @@ TEST_F(LevelZeroProviderInit, FailNonNullDevice) { umfLevelZeroMemoryProviderParamsDestroy(hParams); } -TEST_F(test, FailMismatchedResidentHandlesCount) { +static void +invalidResidentDevicesHandlesTestHelper(ze_device_handle_t *hDevices, + uint32_t deviceCount) { const umf_memory_provider_ops_t *ops = umfLevelZeroMemoryProviderOps(); ASSERT_NE(ops, nullptr); umf_level_zero_memory_provider_params_handle_t hParams = nullptr; - umf_result_t result = umfLevelZeroMemoryProviderParamsCreate(&hParams); - ASSERT_EQ(result, UMF_RESULT_SUCCESS); + const umf_result_t create_result = + umfLevelZeroMemoryProviderParamsCreate(&hParams); + ASSERT_EQ(create_result, UMF_RESULT_SUCCESS); - result = umfLevelZeroMemoryProviderParamsSetResidentDevices(hParams, - nullptr, 99); - ASSERT_EQ(result, UMF_RESULT_ERROR_INVALID_ARGUMENT); + const umf_result_t set_resident_result = + umfLevelZeroMemoryProviderParamsSetResidentDevices(hParams, hDevices, + deviceCount); + ASSERT_EQ(set_resident_result, UMF_RESULT_ERROR_INVALID_ARGUMENT); umfLevelZeroMemoryProviderParamsDestroy(hParams); } +TEST_F(test, FailMismatchedResidentHandlesCount) { + invalidResidentDevicesHandlesTestHelper(nullptr, 99); +} + +TEST_F(test, FailRedundantResidentDeviceHandles) { + std::vector hDevices{ + reinterpret_cast(0x100), + reinterpret_cast(0x101), + reinterpret_cast(0x100)}; + invalidResidentDevicesHandlesTestHelper(hDevices.data(), 3); +} + class LevelZeroMemoryAccessor : public MemoryAccessor { public: LevelZeroMemoryAccessor(ze_context_handle_t hContext, diff --git a/test/providers/provider_level_zero_residency.cpp b/test/providers/provider_level_zero_residency.cpp new file mode 100644 index 0000000000..808547b213 --- /dev/null +++ b/test/providers/provider_level_zero_residency.cpp @@ -0,0 +1,89 @@ +// Copyright (C) 2025 Intel Corporation +// Under the Apache License v2.0 with LLVM Exceptions. See LICENSE.TXT. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "../common/level_zero_mocks.h" +#include "utils_level_zero.h" +#include "utils_log.h" + +#include "gtest/gtest.h" + +using namespace testing; + +class LevelZeroResidencyTestFixture : public Test { + protected: + StrictMock l0mock; + umf_memory_provider_handle_t provider = nullptr; + const ze_device_handle_t OUR_DEVICE; + + LevelZeroResidencyTestFixture() + : OUR_DEVICE(TestCreatePointer(777)) { + *MockedLevelZeroTestEnvironment::l0interface = &l0mock; + } + + void SetUp() override {} + void TearDown() override { + Mock::VerifyAndClearExpectations(&l0mock); + umfMemoryProviderDestroy(provider); + } +}; + +TEST_F(LevelZeroResidencyTestFixture, addNonexistingDeviceShouldSucceed) { + provider = l0mock.initializeMemoryProviderWithResidentDevices( + OUR_DEVICE, {DEVICE_1, DEVICE_5, DEVICE_3}); + ASSERT_EQ(umfLevelZeroMemoryProviderResidentDeviceChange(provider, DEVICE_2, + true), + UMF_RESULT_SUCCESS); +} + +TEST_F(LevelZeroResidencyTestFixture, addExistingDeviceShouldFail) { + provider = l0mock.initializeMemoryProviderWithResidentDevices( + OUR_DEVICE, {DEVICE_1, DEVICE_5, DEVICE_3}); + ASSERT_EQ(umfLevelZeroMemoryProviderResidentDeviceChange(provider, DEVICE_5, + true), + UMF_RESULT_ERROR_INVALID_ARGUMENT); +} + +TEST_F(LevelZeroResidencyTestFixture, removeNonexistingDeviceShouldFail) { + provider = l0mock.initializeMemoryProviderWithResidentDevices( + OUR_DEVICE, {DEVICE_1, DEVICE_5, DEVICE_3}); + ASSERT_EQ(umfLevelZeroMemoryProviderResidentDeviceChange(provider, DEVICE_0, + false), + UMF_RESULT_ERROR_INVALID_ARGUMENT); +} + +TEST_F(LevelZeroResidencyTestFixture, removeExistingDeviceShouldSucceed) { + provider = l0mock.initializeMemoryProviderWithResidentDevices( + OUR_DEVICE, {DEVICE_1, DEVICE_5, DEVICE_3}); + ASSERT_EQ(umfLevelZeroMemoryProviderResidentDeviceChange(provider, DEVICE_1, + false), + UMF_RESULT_SUCCESS); +} + +TEST_F(LevelZeroResidencyTestFixture, addDeviceTwiceShouldFail) { + provider = l0mock.initializeMemoryProviderWithResidentDevices( + OUR_DEVICE, {DEVICE_1, DEVICE_5, DEVICE_3}); + ASSERT_EQ(umfLevelZeroMemoryProviderResidentDeviceChange(provider, DEVICE_2, + true), + UMF_RESULT_SUCCESS); + ASSERT_EQ(umfLevelZeroMemoryProviderResidentDeviceChange(provider, DEVICE_2, + true), + UMF_RESULT_ERROR_INVALID_ARGUMENT); +} + +TEST_F(LevelZeroResidencyTestFixture, removeDeviceTwiceShouldFail) { + provider = l0mock.initializeMemoryProviderWithResidentDevices( + OUR_DEVICE, {DEVICE_1, DEVICE_5, DEVICE_3}); + ASSERT_EQ(umfLevelZeroMemoryProviderResidentDeviceChange(provider, DEVICE_3, + false), + UMF_RESULT_SUCCESS); + ASSERT_EQ(umfLevelZeroMemoryProviderResidentDeviceChange(provider, DEVICE_3, + false), + UMF_RESULT_ERROR_INVALID_ARGUMENT); +} + +int main(int argc, char **argv) { + InitGoogleTest(&argc, argv); + AddGlobalTestEnvironment(new MockedLevelZeroTestEnvironment); + return RUN_ALL_TESTS(); +}