diff --git a/78_ChainedScanUnitTest/CMakeLists.txt b/78_ChainedScanUnitTest/CMakeLists.txt new file mode 100644 index 000000000..a1935bec0 --- /dev/null +++ b/78_ChainedScanUnitTest/CMakeLists.txt @@ -0,0 +1,14 @@ +include(common) + +nbl_create_executable_project("" "" "" "") + +NBL_CREATE_RESOURCE_ARCHIVE( + NAMESPACE nbl::this_example::builtin + TARGET ${EXECUTABLE_NAME}_builtins + LINK_TO ${EXECUTABLE_NAME} + BIND app_resources + BUILTINS + common.hlsl + shaderCommon.hlsl + testGlobal.comp.hlsl +) \ No newline at end of file diff --git a/78_ChainedScanUnitTest/app_resources/common.hlsl b/78_ChainedScanUnitTest/app_resources/common.hlsl new file mode 100644 index 000000000..b10f7da53 --- /dev/null +++ b/78_ChainedScanUnitTest/app_resources/common.hlsl @@ -0,0 +1,98 @@ +#include "nbl/builtin/hlsl/cpp_compat.hlsl" +#include "nbl/builtin/hlsl/functional.hlsl" + +struct PushConstantData +{ + uint64_t pInputBuf; + uint64_t pOutputBuf[8]; + uint64_t pReduceBuf; + uint64_t pWgCounterBuf; +}; + +namespace arithmetic +{ +// Thanks to our unified HLSL/C++ STD lib we're able to remove a whole load of code +template +struct bit_and : nbl::hlsl::bit_and +{ + using base_t = nbl::hlsl::bit_and; + + NBL_CONSTEXPR_STATIC_INLINE uint16_t BindingIndex = 0; +#ifndef __HLSL_VERSION + static inline constexpr const char* name = "bit_and"; +#endif +}; +template +struct bit_or : nbl::hlsl::bit_or +{ + using base_t = nbl::hlsl::bit_or; + + NBL_CONSTEXPR_STATIC_INLINE uint16_t BindingIndex = 1; +#ifndef __HLSL_VERSION + static inline constexpr const char* name = "bit_xor"; +#endif +}; +template +struct bit_xor : nbl::hlsl::bit_xor +{ + using base_t = nbl::hlsl::bit_xor; + + NBL_CONSTEXPR_STATIC_INLINE uint16_t BindingIndex = 2; +#ifndef __HLSL_VERSION + static inline constexpr const char* name = "bit_or"; +#endif +}; +template +struct plus : nbl::hlsl::plus +{ + using base_t = nbl::hlsl::plus; + + NBL_CONSTEXPR_STATIC_INLINE uint16_t BindingIndex = 3; +#ifndef __HLSL_VERSION + static inline constexpr const char* name = "plus"; +#endif +}; +template +struct multiplies : nbl::hlsl::multiplies +{ + using base_t = nbl::hlsl::multiplies; + + NBL_CONSTEXPR_STATIC_INLINE uint16_t BindingIndex = 4; +#ifndef __HLSL_VERSION + static inline constexpr const char* name = "multiplies"; +#endif +}; +template +struct minimum : nbl::hlsl::minimum +{ + using base_t = nbl::hlsl::minimum; + + NBL_CONSTEXPR_STATIC_INLINE uint16_t BindingIndex = 5; +#ifndef __HLSL_VERSION + static inline constexpr const char* name = "minimum"; +#endif +}; +template +struct maximum : nbl::hlsl::maximum +{ + using base_t = nbl::hlsl::maximum; + + NBL_CONSTEXPR_STATIC_INLINE uint16_t BindingIndex = 6; +#ifndef __HLSL_VERSION + static inline constexpr const char* name = "maximum"; +#endif +}; + +template +struct ballot : nbl::hlsl::plus +{ + using base_t = nbl::hlsl::plus; + + NBL_CONSTEXPR_STATIC_INLINE uint16_t BindingIndex = 7; +#ifndef __HLSL_VERSION + static inline constexpr const char* name = "bitcount"; +#endif +}; +} + +#include "nbl/builtin/hlsl/glsl_compat/subgroup_basic.hlsl" diff --git a/78_ChainedScanUnitTest/app_resources/shaderCommon.hlsl b/78_ChainedScanUnitTest/app_resources/shaderCommon.hlsl new file mode 100644 index 000000000..5baf9a28d --- /dev/null +++ b/78_ChainedScanUnitTest/app_resources/shaderCommon.hlsl @@ -0,0 +1,19 @@ +#include "app_resources/common.hlsl" + +using namespace nbl; +using namespace hlsl; + +[[vk::push_constant]] PushConstantData pc; + +struct device_capabilities +{ +#ifdef TEST_NATIVE + NBL_CONSTEXPR_STATIC_INLINE bool shaderSubgroupArithmetic = true; +#else + NBL_CONSTEXPR_STATIC_INLINE bool shaderSubgroupArithmetic = false; +#endif +}; + +#ifndef OPERATION +#error "Define OPERATION!" +#endif diff --git a/78_ChainedScanUnitTest/app_resources/testGlobal.comp.hlsl b/78_ChainedScanUnitTest/app_resources/testGlobal.comp.hlsl new file mode 100644 index 000000000..1afa33d95 --- /dev/null +++ b/78_ChainedScanUnitTest/app_resources/testGlobal.comp.hlsl @@ -0,0 +1,205 @@ +#pragma shader_stage(compute) + +#include "nbl/builtin/hlsl/glsl_compat/core.hlsl" +#include "nbl/builtin/hlsl/glsl_compat/subgroup_basic.hlsl" +#include "nbl/builtin/hlsl/subgroup2/arithmetic_portability.hlsl" + +#include "nbl/builtin/hlsl/bda/legacy_bda_accessor.hlsl" +#include "nbl/builtin/hlsl/scan/chained_scan.hlsl" + +using config_t = WORKGROUP_CONFIG_T; + +#include "app_resources/shaderCommon.hlsl" + +typedef vector type_t; + +groupshared uint32_t scratch[mpl::max_v]; + +struct ScratchProxy +{ + template + void get(const uint32_t ix, NBL_REF_ARG(AccessType) value) + { + value = scratch[ix]; + } + template + void set(const uint32_t ix, const AccessType value) + { + scratch[ix] = value; + } + + uint32_t atomicOr(const uint32_t ix, const uint32_t value) + { + return glsl::atomicOr(scratch[ix],value); + } + + void workgroupExecutionAndMemoryBarrier() + { + glsl::barrier(); + } +}; + +template +struct DataProxy +{ + using dtype_t = vector; + // function template AccessType should be the same as dtype_t + + static DataProxy create(const uint64_t inputBuf, const uint64_t outputBuf) + { + DataProxy retval; + const uint32_t workgroupOffset = glsl::gl_WorkGroupID().x * VirtualWorkgroupSize * sizeof(dtype_t); + retval.accessor = DoubleLegacyBdaAccessor::create(inputBuf + workgroupOffset, outputBuf + workgroupOffset); + retval.inputAddress = inputBuf; + retval.outputAddress = outputBuf; + return retval; + } + + void initAtWorkgroupID(const uint32_t workgroupID) + { + const uint32_t workgroupOffset = workgroupID * VirtualWorkgroupSize * sizeof(dtype_t); + accessor = DoubleLegacyBdaAccessor::create(inputAddress + workgroupOffset, outputAddress + workgroupOffset); + } + + template + void get(const IndexType ix, NBL_REF_ARG(AccessType) value) + { + accessor.get(ix, value); + } + template + void set(const IndexType ix, const AccessType value) + { + accessor.set(ix, value); + } + + uint64_t getInputBufAddr() + { + return inputAddress; + } + uint64_t getOutputBufAddr() + { + return outputAddress; + } + + void workgroupExecutionAndMemoryBarrier() + { + glsl::barrier(); + //glsl::memoryBarrierShared(); implied by the above + } + + DoubleLegacyBdaAccessor accessor; + uint64_t inputAddress, outputAddress; +}; + +template +struct ReduceAccessor +{ + using type_t = T; + static ReduceAccessor create(const bda::__ptr ptr) + { + ReduceAccessor retval; + retval.ptr = ptr; + return retval; + } + + template + void get(const IndexType index, NBL_REF_ARG(AccessType) value) + { + bda::__ptr target = ptr + index; + value = target.template deref().load(); + } + template + void set(const IndexType index, const AccessType value) + { + bda::__ptr target = ptr + index; + return target.template deref().store(value); + } + + T atomicMax(const uint64_t index, const T value) + { + bda::__ptr target = ptr + index; + return glsl::atomicMax(target.template deref().ptr.value, value); + } + T atomicExchange(const uint64_t index, const T value) + { + bda::__ptr target = ptr + index; + return glsl::atomicExchange(target.template deref().ptr.value, value); + } + + void memoryBarrier() + { + spirv::memoryBarrier(spv::ScopeDevice, spv::MemorySemanticsAcquireReleaseMask | spv::MemorySemanticsUniformMemoryMask); + } + + bda::__ptr ptr; +}; + +struct WorkgroupCounter +{ + static WorkgroupCounter create(const uint64_t addr) + { + WorkgroupCounter retval; + retval.ptr = bda::__ptr::create(addr); + return retval; + } + + uint32_t atomicAdd(const uint64_t index, const uint32_t value) // TODO: maybe it should be just increment + { + bda::__ptr target = ptr + index; + return glsl::atomicAdd(target.template deref().ptr.value, value); + } + + bda::__ptr ptr; +}; + +static ScratchProxy arithmeticAccessor; + +template +struct operation_t +{ + using binop_base_t = typename Binop::base_t; + using otype_t = typename Binop::type_t; + + void operator()() + { + using data_proxy_t = DataProxy; + data_proxy_t dataAccessor = data_proxy_t::create(pc.pInputBuf, pc.pOutputBuf[Binop::BindingIndex]); + + using reduce_proxy_t = ReduceAccessor; + bda::__ptr ptr = bda::__ptr::create(pc.pReduceBuf); + reduce_proxy_t reduceAccessor = reduce_proxy_t::create(ptr); + + WorkgroupCounter wgCounter = WorkgroupCounter::create(pc.pWgCounterBuf); + + OPERATION::template __call(dataAccessor,arithmeticAccessor,reduceAccessor,wgCounter); + // we barrier before because we alias the accessors for Binop + arithmeticAccessor.workgroupExecutionAndMemoryBarrier(); + } +}; + + +template +static void subtest() +{ + assert(glsl::gl_SubgroupSize() == config_t::SubgroupSize) + + operation_t func; + func(); +} + +void test() +{ + // subtest >(); + // subtest >(); + // subtest >(); + subtest >(); + // subtest >(); + // subtest >(); + // subtest >(); +} + +[numthreads(config_t::WorkgroupSize,1,1)] +void main() +{ + test(); +} diff --git a/78_ChainedScanUnitTest/main.cpp b/78_ChainedScanUnitTest/main.cpp new file mode 100644 index 000000000..c3e250b49 --- /dev/null +++ b/78_ChainedScanUnitTest/main.cpp @@ -0,0 +1,515 @@ +// TODO: copyright notice + + +#include "nbl/examples/examples.hpp" + +#include "app_resources/common.hlsl" +#include "nbl/builtin/hlsl/workgroup2/arithmetic_config.hlsl" +#include "nbl/builtin/hlsl/subgroup2/arithmetic_params.hlsl" + + +using namespace nbl; +using namespace core; +using namespace asset; +using namespace system; +using namespace video; + +// method emulations on the CPU, to verify the results of the GPU methods +template +struct emulatedReduction +{ + using type_t = typename Binop::type_t; + + static inline void impl(type_t* out, const type_t* in, const uint32_t itemCount) + { + const type_t red = std::reduce(in,in+itemCount,Binop::identity,Binop()); + std::fill(out,out+itemCount,red); + } + + static inline constexpr const char* name = "reduction"; +}; +template +struct emulatedScanInclusive +{ + using type_t = typename Binop::type_t; + + static inline void impl(type_t* out, const type_t* in, const uint32_t itemCount) + { + std::inclusive_scan(in,in+itemCount,out,Binop()); + } + static inline constexpr const char* name = "inclusive_scan"; +}; +template +struct emulatedScanExclusive +{ + using type_t = typename Binop::type_t; + + static inline void impl(type_t* out, const type_t* in, const uint32_t itemCount) + { + std::exclusive_scan(in,in+itemCount,out,Binop::identity,Binop()); + } + static inline constexpr const char* name = "exclusive_scan"; +}; + +class ChainedScanTestApp final : public application_templates::BasicMultiQueueApplication, public examples::BuiltinResourcesApplication +{ + using device_base_t = application_templates::BasicMultiQueueApplication; + using asset_base_t = examples::BuiltinResourcesApplication; + + constexpr static inline uint32_t ElementCount = 1024 * 1024; + +public: + ChainedScanTestApp(const path& _localInputCWD, const path& _localOutputCWD, const path& _sharedInputCWD, const path& _sharedOutputCWD) : + system::IApplicationFramework(_localInputCWD, _localOutputCWD, _sharedInputCWD, _sharedOutputCWD) {} + + bool onAppInitialized(smart_refctd_ptr&& system) override + { + if (!device_base_t::onAppInitialized(std::move(system))) + return false; + if (!asset_base_t::onAppInitialized(std::move(system))) + return false; + + transferDownQueue = getTransferDownQueue(); + computeQueue = getComputeQueue(); + + // TODO: get the element count from argv + // populate our random data buffer on the CPU and create a GPU copy + inputData = new uint32_t[ElementCount]; + smart_refctd_ptr gpuinputDataBuffer; + { + std::mt19937 randGenerator(0xdeadbeefu); + std::uniform_int_distribution<> distrib(0, 100); + for (uint32_t i = 0u; i < ElementCount; i++) + inputData[i] = distrib(randGenerator); // TODO: change to using xoroshiro, then we can skip having the input buffer at all + + IGPUBuffer::SCreationParams inputDataBufferCreationParams = {}; + inputDataBufferCreationParams.size = sizeof(uint32_t) * ElementCount; + inputDataBufferCreationParams.usage = IGPUBuffer::EUF_STORAGE_BUFFER_BIT | IGPUBuffer::EUF_TRANSFER_DST_BIT | IGPUBuffer::EUF_SHADER_DEVICE_ADDRESS_BIT; + m_utils->createFilledDeviceLocalBufferOnDedMem( + SIntendedSubmitInfo{.queue=getTransferUpQueue()}, + std::move(inputDataBufferCreationParams), + inputData + ).move_into(gpuinputDataBuffer); + + gpuinputDataBuffer->setObjectDebugName("Input data buffer"); + } + + // create 8 buffers for 8 operations + for (auto i=0u; igetSize(); + params.usage = bitflag(IGPUBuffer::EUF_STORAGE_BUFFER_BIT) | IGPUBuffer::EUF_TRANSFER_SRC_BIT | IGPUBuffer::EUF_SHADER_DEVICE_ADDRESS_BIT; + + outputBuffers[i] = m_device->createBuffer(std::move(params)); + auto mreq = outputBuffers[i]->getMemoryReqs(); + mreq.memoryTypeBits &= m_physicalDevice->getDeviceLocalMemoryTypeBits(); + assert(mreq.memoryTypeBits); + + auto bufferMem = m_device->allocate(mreq, { outputBuffers[i].get(), IDeviceMemoryAllocation::EMAF_DEVICE_ADDRESS_BIT }); + assert(bufferMem.isValid()); + + outputBuffers[i]->setObjectDebugName(("Output data buffer #" + std::to_string(i)).c_str()); + } + + const auto MinSubgroupSize = m_physicalDevice->getLimits().minSubgroupSize; + // buffer for workgroup reductions + { + const auto maxReductionsCount = ElementCount / MinSubgroupSize; + IGPUBuffer::SCreationParams params = {}; + params.size = sizeof(uint32_t) * maxReductionsCount; // TODO: could be smaller + params.usage = bitflag(IGPUBuffer::EUF_STORAGE_BUFFER_BIT) | IGPUBuffer::EUF_TRANSFER_DST_BIT | IGPUBuffer::EUF_SHADER_DEVICE_ADDRESS_BIT; + + reduceBuffer = m_device->createBuffer(std::move(params)); + auto mreq = reduceBuffer->getMemoryReqs(); + mreq.memoryTypeBits &= m_physicalDevice->getDeviceLocalMemoryTypeBits(); + assert(mreq.memoryTypeBits); + + auto bufferMem = m_device->allocate(mreq, { reduceBuffer.get(), IDeviceMemoryAllocation::EMAF_DEVICE_ADDRESS_BIT }); + assert(bufferMem.isValid()); + + reduceBuffer->setObjectDebugName("Workgroup reduction buffer"); + } + { + IGPUBuffer::SCreationParams params = {}; + params.size = sizeof(uint32_t); + params.usage = bitflag(IGPUBuffer::EUF_STORAGE_BUFFER_BIT) | IGPUBuffer::EUF_TRANSFER_DST_BIT | IGPUBuffer::EUF_SHADER_DEVICE_ADDRESS_BIT; + + wgCounterBuffer = m_device->createBuffer(std::move(params)); + auto mreq = wgCounterBuffer->getMemoryReqs(); + mreq.memoryTypeBits &= m_physicalDevice->getDeviceLocalMemoryTypeBits(); + assert(mreq.memoryTypeBits); + + auto bufferMem = m_device->allocate(mreq, { wgCounterBuffer.get(), IDeviceMemoryAllocation::EMAF_DEVICE_ADDRESS_BIT }); + assert(bufferMem.isValid()); + + wgCounterBuffer->setObjectDebugName("Workgroup counter buffer"); + } + + pc.pInputBuf = gpuinputDataBuffer->getDeviceAddress(); + pc.pReduceBuf = reduceBuffer->getDeviceAddress(); + pc.pWgCounterBuf = wgCounterBuffer->getDeviceAddress(); + for (uint32_t i = 0; i < OutputBufferCount; i++) + pc.pOutputBuf[i] = outputBuffers[i]->getDeviceAddress(); + + // create Pipeline Layout + { + SPushConstantRange pcRange = { .stageFlags = IShader::E_SHADER_STAGE::ESS_COMPUTE, .offset = 0,.size = sizeof(PushConstantData) }; + pipelineLayout = m_device->createPipelineLayout({&pcRange, 1}); + } + + const auto spirv_isa_cache_path = localOutputCWD / "spirv_isa_cache.bin"; + // enclose to make sure file goes out of scope and we can reopen it + { + smart_refctd_ptr spirv_isa_cache_input; + // try to load SPIR-V to ISA cache + { + ISystem::future_t> fileCreate; + m_system->createFile(fileCreate, spirv_isa_cache_path, IFile::ECF_READ | IFile::ECF_MAPPABLE | IFile::ECF_COHERENT); + if (auto lock = fileCreate.acquire()) + spirv_isa_cache_input = *lock; + } + // create the cache + { + std::span spirv_isa_cache_data = {}; + if (spirv_isa_cache_input) + spirv_isa_cache_data = { reinterpret_cast(spirv_isa_cache_input->getMappedPointer()),spirv_isa_cache_input->getSize() }; + else + m_logger->log("Failed to load SPIR-V 2 ISA cache!", ILogger::ELL_PERFORMANCE); + // Normally we'd deserialize a `ICPUPipelineCache` properly and pass that instead + m_spirv_isa_cache = m_device->createPipelineCache(spirv_isa_cache_data); + } + } + { + // TODO: rename `deleteDirectory` to just `delete`? and a `IFile::setSize()` ? + m_system->deleteDirectory(spirv_isa_cache_path); + ISystem::future_t> fileCreate; + m_system->createFile(fileCreate, spirv_isa_cache_path, IFile::ECF_WRITE); + // I can be relatively sure I'll succeed to acquire the future, the pointer to created file might be null though. + m_spirv_isa_cache_output = *fileCreate.acquire(); + if (!m_spirv_isa_cache_output) + logFail("Failed to Create SPIR-V to ISA cache file."); + } + + // load shader source from file + auto getShaderSource = [&](const char* filePath) -> auto + { + IAssetLoader::SAssetLoadParams lparams = {}; + lparams.logger = m_logger.get(); + lparams.workingDirectory = ""; + auto bundle = m_assetMgr->getAsset(filePath, lparams); + if (bundle.getContents().empty() || bundle.getAssetType()!=IAsset::ET_SHADER) + { + m_logger->log("Shader %s not found!", ILogger::ELL_ERROR, filePath); + exit(-1); + } + auto firstAssetInBundle = bundle.getContents()[0]; + return smart_refctd_ptr_static_cast(firstAssetInBundle); + }; + + auto globalTestSource = getShaderSource("app_resources/testGlobal.comp.hlsl"); + // now create or retrieve final resources to run our tests + sema = m_device->createSemaphore(timelineValue); + resultsBuffer = ICPUBuffer::create({ outputBuffers[0]->getSize() }); + { + smart_refctd_ptr cmdpool = m_device->createCommandPool(computeQueue->getFamilyIndex(),IGPUCommandPool::CREATE_FLAGS::RESET_COMMAND_BUFFER_BIT); + if (!cmdpool->createCommandBuffers(IGPUCommandPool::BUFFER_LEVEL::PRIMARY,{&cmdbuf,1})) + { + logFail("Failed to create Command Buffers!\n"); + return false; + } + } + + const auto MaxWorkgroupSize = m_physicalDevice->getLimits().maxComputeWorkGroupInvocations; + const auto MaxSubgroupSize = m_physicalDevice->getLimits().maxSubgroupSize; + for (uint32_t useNative = 0; useNative <= uint32_t(m_physicalDevice->getProperties().limits.shaderSubgroupArithmetic); useNative++) + { + if (useNative) + m_logger->log("Testing with native subgroup arithmetic", ILogger::ELL_INFO); + else + m_logger->log("Testing with emulated subgroup arithmetic", ILogger::ELL_INFO); + + for (auto subgroupSize = MinSubgroupSize; subgroupSize <= MaxSubgroupSize; subgroupSize *= 2u) + { + const uint8_t subgroupSizeLog2 = hlsl::findMSB(subgroupSize); + for (uint32_t workgroupSize = subgroupSize; workgroupSize <= MaxWorkgroupSize; workgroupSize *= 2u) + { + // make sure renderdoc captures everything for debugging + m_api->startCapture(); + m_logger->log("Testing Workgroup Size %u with Subgroup Size %u", ILogger::ELL_INFO, workgroupSize, subgroupSize); + + for (uint32_t j = 0; j < ItemsPerInvocations.size(); j++) + { + const uint32_t itemsPerInvocation = ItemsPerInvocations[j]; + bool passed = true; + + hlsl::workgroup2::SArithmeticConfiguration wgConfig; + wgConfig.init(hlsl::findMSB(workgroupSize), subgroupSizeLog2, itemsPerInvocation); + uint32_t itemsPerWG = wgConfig.VirtualWorkgroupSize * wgConfig.ItemsPerInvocation_0; + m_logger->log("Testing Item Count %u", ILogger::ELL_INFO, itemsPerWG); + passed = runTest(globalTestSource, ElementCount, subgroupSizeLog2, workgroupSize, bool(useNative), itemsPerWG, itemsPerInvocation) && passed; + logTestOutcome(passed, itemsPerWG); + passed = runTest(globalTestSource, ElementCount, subgroupSizeLog2, workgroupSize, bool(useNative), itemsPerWG, itemsPerInvocation) && passed; + logTestOutcome(passed, itemsPerWG); + } + m_api->endCapture(); + + // save cache every now and then + { + auto cpu = m_spirv_isa_cache->convertToCPUCache(); + // Normally we'd beautifully JSON serialize the thing, allow multiple devices & drivers + metadata + auto bin = cpu->getEntries().begin()->second.bin; + IFile::success_t success; + m_spirv_isa_cache_output->write(success, bin->data(), 0ull, bin->size()); + if (!success) + logFail("Could not write Create SPIR-V to ISA cache to disk!"); + } + } + } + } + + return true; + } + + virtual bool onAppTerminated() override + { + m_logger->log("==========Result==========", ILogger::ELL_INFO); + m_logger->log("Fail Count: %u", ILogger::ELL_INFO, totalFailCount); + delete[] inputData; + return true; + } + + // the unit test is carried out on init + void workLoopBody() override {} + + // + bool keepRunning() override { return false; } + +private: + void logTestOutcome(bool passed, uint32_t workgroupSize) + { + if (passed) + m_logger->log("Passed test #%u", ILogger::ELL_INFO, workgroupSize); + else + { + totalFailCount++; + m_logger->log("Failed test #%u", ILogger::ELL_ERROR, workgroupSize); + } + } + + // create pipeline (specialized every test) [TODO: turn into a future/async] + smart_refctd_ptr createPipeline(const IShader* overridenUnspecialized, const uint8_t subgroupSizeLog2) + { + auto shader = m_device->compileShader({ overridenUnspecialized }); + IGPUComputePipeline::SCreationParams params = {}; + params.layout = pipelineLayout.get(); + params.shader = { + .shader = shader.get(), + .entryPoint = "main", + .requiredSubgroupSize = static_cast(subgroupSizeLog2), + .entries = nullptr, + }; + params.cached.requireFullSubgroups = true; + core::smart_refctd_ptr pipeline; + if (!m_device->createComputePipelines(m_spirv_isa_cache.get(),{¶ms,1},&pipeline)) + return nullptr; + return pipeline; + } + + template class Arithmetic> + bool runTest(const smart_refctd_ptr& source, const uint32_t elementCount, const uint8_t subgroupSizeLog2, const uint32_t workgroupSize, bool useNative, uint32_t itemsPerWG, uint32_t itemsPerInvoc = 1u) + { + std::string arith_name = Arithmetic>::name; + const uint32_t workgroupSizeLog2 = hlsl::findMSB(workgroupSize); + + auto compiler = make_smart_refctd_ptr(smart_refctd_ptr(m_system)); + CHLSLCompiler::SOptions options = {}; + options.stage = IShader::E_SHADER_STAGE::ESS_COMPUTE; + options.preprocessorOptions.targetSpirvVersion = m_device->getPhysicalDevice()->getLimits().spirvVersion; + options.spirvOptimizer = nullptr; +#ifndef _NBL_DEBUG + ISPIRVOptimizer::E_OPTIMIZER_PASS optPasses = ISPIRVOptimizer::EOP_STRIP_DEBUG_INFO; + auto opt = make_smart_refctd_ptr(std::span(&optPasses, 1)); + options.spirvOptimizer = opt.get(); +#else + options.debugInfoFlags |= IShaderCompiler::E_DEBUG_INFO_FLAGS::EDIF_LINE_BIT; +#endif + options.preprocessorOptions.sourceIdentifier = source->getFilepathHint(); + options.preprocessorOptions.logger = m_logger.get(); + + auto* includeFinder = compiler->getDefaultIncludeFinder(); + options.preprocessorOptions.includeFinder = includeFinder; + + hlsl::workgroup2::SArithmeticConfiguration wgConfig; + wgConfig.init(hlsl::findMSB(workgroupSize), subgroupSizeLog2, itemsPerInvoc); + + const std::string definitions[3] = { + "scan::" + arith_name, + wgConfig.getConfigTemplateStructString(), + std::to_string(arith_name=="reduction") + }; + + const IShaderCompiler::SMacroDefinition defines[4] = { + { "OPERATION", definitions[0] }, + { "WORKGROUP_CONFIG_T", definitions[1] }, + { "IS_REDUCTION", definitions[2] }, + { "TEST_NATIVE", "1" } + }; + if (useNative) + options.preprocessorOptions.extraDefines = { defines, defines + 4 }; + else + options.preprocessorOptions.extraDefines = { defines, defines + 3 }; + + smart_refctd_ptr overriddenUnspecialized = compiler->compileToSPIRV((const char*)source->getContent()->getPointer(), options); + + auto pipeline = createPipeline(overriddenUnspecialized.get(),subgroupSizeLog2); + + // TODO: overlap dispatches with memory readbacks (requires multiple copies of `buffers`) + uint32_t workgroupCount = min((elementCount + itemsPerWG - 1) / itemsPerWG, m_physicalDevice->getLimits().maxComputeWorkGroupCount[0]); + + cmdbuf->begin(IGPUCommandBuffer::USAGE::NONE); + cmdbuf->fillBuffer({ .offset = 0,.size = reduceBuffer->getSize(),.buffer = reduceBuffer}, 0); + cmdbuf->fillBuffer({ .offset = 0,.size = wgCounterBuffer->getSize(),.buffer = wgCounterBuffer }, 0); + { + using buffer_barrier_t = IGPUCommandBuffer::SBufferMemoryBarrier; + buffer_barrier_t bufBarrier[2]; + bufBarrier[0] = { + .barrier = { + .dep = { + .srcStageMask = PIPELINE_STAGE_FLAGS::CLEAR_BIT | PIPELINE_STAGE_FLAGS::COPY_BIT, + .srcAccessMask = ACCESS_FLAGS::TRANSFER_WRITE_BIT, + .dstStageMask = PIPELINE_STAGE_FLAGS::COMPUTE_SHADER_BIT, + .dstAccessMask = ACCESS_FLAGS::SHADER_READ_BITS | ACCESS_FLAGS::SHADER_WRITE_BITS + } // no ownership transfers, etc. + }, + .range = {.offset = 0,.size = reduceBuffer->getSize(),.buffer = reduceBuffer} + }; + bufBarrier[1] = { + .barrier = { + .dep = { + .srcStageMask = PIPELINE_STAGE_FLAGS::CLEAR_BIT | PIPELINE_STAGE_FLAGS::COPY_BIT, + .srcAccessMask = ACCESS_FLAGS::TRANSFER_WRITE_BIT, + .dstStageMask = PIPELINE_STAGE_FLAGS::COMPUTE_SHADER_BIT, + .dstAccessMask = ACCESS_FLAGS::SHADER_READ_BITS | ACCESS_FLAGS::SHADER_WRITE_BITS + } // no ownership transfers, etc. + }, + .range = {.offset = 0,.size = wgCounterBuffer->getSize(),.buffer = wgCounterBuffer} + }; + + cmdbuf->pipelineBarrier(E_DEPENDENCY_FLAGS::EDF_NONE, { + .memBarriers = {}, + .bufBarriers = bufBarrier, + .imgBarriers = {} + }); + } + + cmdbuf->bindComputePipeline(pipeline.get()); + cmdbuf->pushConstants(pipelineLayout.get(), IShader::E_SHADER_STAGE::ESS_COMPUTE, 0, sizeof(PushConstantData), &pc); + cmdbuf->dispatch(workgroupCount, 1, 1); + { + IGPUCommandBuffer::SPipelineBarrierDependencyInfo::buffer_barrier_t memoryBarrier[OutputBufferCount]; + for (auto i=0u; igetSize(),outputBuffers[i]} + }; + } + IGPUCommandBuffer::SPipelineBarrierDependencyInfo info = {.memBarriers={},.bufBarriers=memoryBarrier}; + cmdbuf->pipelineBarrier(asset::E_DEPENDENCY_FLAGS::EDF_NONE,info); + } + cmdbuf->end(); + + const IQueue::SSubmitInfo::SSemaphoreInfo signal[1] = {{.semaphore=sema.get(),.value=++timelineValue}}; + const IQueue::SSubmitInfo::SCommandBufferInfo cmdbufs[1] = {{.cmdbuf=cmdbuf.get()}}; + const IQueue::SSubmitInfo submits[1] = {{.commandBuffers=cmdbufs,.signalSemaphores=signal}}; + computeQueue->submit(submits); + const ISemaphore::SWaitInfo wait[1] = {{.semaphore=sema.get(),.value=timelineValue}}; + m_device->blockForSemaphores(wait); + + const uint32_t subgroupSize = 1u << subgroupSizeLog2; + // check results + bool passed = true; + //passed = validateResults>(itemsPerWG, workgroupCount, subgroupSize, itemsPerInvoc); + //passed = validateResults>(itemsPerWG, workgroupCount, subgroupSize, itemsPerInvoc) && passed; + //passed = validateResults>(itemsPerWG, workgroupCount, subgroupSize, itemsPerInvoc) && passed; + passed = validateResults>(itemsPerWG, workgroupCount, subgroupSize, itemsPerInvoc) && passed; + //passed = validateResults>(itemsPerWG, workgroupCount, subgroupSize, itemsPerInvoc) && passed; + //passed = validateResults>(itemsPerWG, workgroupCount, subgroupSize, itemsPerInvoc) && passed; + //passed = validateResults>(itemsPerWG, workgroupCount, subgroupSize, itemsPerInvoc) && passed; + + return passed; + } + + //returns true if result matches + template class Arithmetic, class Binop> + bool validateResults(const uint32_t itemsPerWG, const uint32_t workgroupCount, const uint32_t subgroupSize, const uint32_t itemsPerInvoc) + { + bool success = true; + + // download data + const SBufferRange bufferRange = {0u, resultsBuffer->getSize(), outputBuffers[Binop::BindingIndex]}; + m_utils->downloadBufferRangeViaStagingBufferAutoSubmit(SIntendedSubmitInfo{.queue=transferDownQueue},bufferRange,resultsBuffer->getPointer()); + + using type_t = typename Binop::type_t; + const auto testData = reinterpret_cast(resultsBuffer->getPointer()); + + // TODO: parallel for (the temporary values need to be threadlocal or what?) + // now check if the data obtained has valid values + type_t* tmp = new type_t[ElementCount]; + { + Arithmetic::impl(tmp, inputData, ElementCount); + + for (uint32_t i = 0u; i < ElementCount; i++) + { + const auto cpuVal = tmp[i]; + const auto gpuVal = testData[i]; + if (cpuVal != gpuVal) + { + m_logger->log( + "Failed test #%d (%s) (%s) Expected %u got %u for element number %d", + ILogger::ELL_ERROR, itemsPerWG, "global", Binop::name, + cpuVal, gpuVal, i + ); + success = false; + break; + } + } + } + delete[] tmp; + + return success; + } + + IQueue* transferDownQueue; + IQueue* computeQueue; + smart_refctd_ptr m_spirv_isa_cache; + smart_refctd_ptr m_spirv_isa_cache_output; + + uint32_t* inputData = nullptr; + constexpr static inline uint32_t OutputBufferCount = 8u; + smart_refctd_ptr outputBuffers[OutputBufferCount]; + smart_refctd_ptr reduceBuffer; + smart_refctd_ptr wgCounterBuffer; + smart_refctd_ptr pipelineLayout; + PushConstantData pc; + + smart_refctd_ptr sema; + uint64_t timelineValue = 0; + smart_refctd_ptr cmdbuf; + smart_refctd_ptr resultsBuffer; + + uint32_t totalFailCount = 0; + + constexpr static inline std::array ItemsPerInvocations = { 1, 2, 3, 4 }; +}; + +NBL_MAIN_FUNC(ChainedScanTestApp) \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt index ca7794ca8..d4e2c5a72 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -115,6 +115,7 @@ if(NBL_BUILD_EXAMPLES) add_subdirectory(76_CudaInterop) endif() + add_subdirectory(78_ChainedScanUnitTest) # add new examples *before* NBL_GET_ALL_TARGETS invocation, it gathers recursively all targets created so far in this subdirectory NBL_GET_ALL_TARGETS(TARGETS)