From e43c1969faf911d79f729ce1613ee7e7e7d2ba2e Mon Sep 17 00:00:00 2001 From: Steven Johnson Date: Wed, 3 Nov 2021 11:07:41 -0700 Subject: [PATCH 1/6] Rename halide_assert -> HALIDDE_CHECK A crashing bug got mistakenly inserted because a new contributor (reasonably) assumed that the `halide_assert()` macro in our runtime code was like a C `assert()` (i.e., something that would vanish in optimized builds). This is not the case; it is a check that happens in all build modes and always triggers an `abort()` if it fires. We should remove any ambiguity about it, so this proposes to rename it to somethingmore like the Google/Abseil-style CHECK() macro, to make it stand out more. (We may want to do a followup to verify that all of the uses really are unrecoverable errors that aren't better handled by returning an error.) --- src/runtime/cache.cpp | 12 +- src/runtime/cuda.cpp | 48 ++-- src/runtime/d3d12compute.cpp | 280 ++++++++++++------------ src/runtime/device_interface.cpp | 8 +- src/runtime/gpu_context_common.h | 6 +- src/runtime/hashmap.h | 26 +-- src/runtime/hexagon_cache_allocator.cpp | 2 +- src/runtime/hexagon_dma.cpp | 48 ++-- src/runtime/hexagon_dma_pool.cpp | 12 +- src/runtime/hexagon_host.cpp | 32 +-- src/runtime/metal.cpp | 50 ++--- src/runtime/opencl.cpp | 88 ++++---- src/runtime/openglcompute.cpp | 8 +- src/runtime/profiler.cpp | 14 +- src/runtime/runtime_internal.h | 21 +- src/runtime/synchronization_common.h | 6 +- src/runtime/thread_pool_common.h | 4 +- src/runtime/tracing.cpp | 10 +- 18 files changed, 339 insertions(+), 336 deletions(-) diff --git a/src/runtime/cache.cpp b/src/runtime/cache.cpp index 689c6aa329ee..86b582cb5c76 100644 --- a/src/runtime/cache.cpp +++ b/src/runtime/cache.cpp @@ -278,7 +278,7 @@ WEAK void prune_cache() { while (prev_hash_entry != nullptr && prev_hash_entry->next != prune_candidate) { prev_hash_entry = prev_hash_entry->next; } - halide_assert(nullptr, prev_hash_entry != nullptr); + HALIDE_CHECK(nullptr, prev_hash_entry != nullptr); prev_hash_entry->next = prune_candidate->next; } @@ -367,14 +367,14 @@ WEAK int halide_memoization_cache_lookup(void *user_context, const uint8_t *cach if (all_bounds_equal) { if (entry != most_recently_used) { - halide_assert(user_context, entry->more_recent != nullptr); + HALIDE_CHECK(user_context, entry->more_recent != nullptr); if (entry->less_recent != nullptr) { entry->less_recent->more_recent = entry->more_recent; } else { - halide_assert(user_context, least_recently_used == entry); + HALIDE_CHECK(user_context, least_recently_used == entry); least_recently_used = entry->more_recent; } - halide_assert(user_context, entry->more_recent != nullptr); + HALIDE_CHECK(user_context, entry->more_recent != nullptr); entry->more_recent->less_recent = entry->less_recent; entry->more_recent = nullptr; @@ -466,7 +466,7 @@ WEAK int halide_memoization_cache_store(void *user_context, const uint8_t *cache } } if (all_bounds_equal) { - halide_assert(user_context, no_host_pointers_equal); + HALIDE_CHECK(user_context, no_host_pointers_equal); // This entry is still in use by the caller. Mark it as having no cache entry // so halide_memoization_cache_release can free the buffer. for (int32_t i = 0; i < tuple_count; i++) { @@ -544,7 +544,7 @@ WEAK void halide_memoization_cache_release(void *user_context, void *host) { } else { ScopedMutexLock lock(&memoization_lock); - halide_assert(user_context, entry->in_use_count > 0); + HALIDE_CHECK(user_context, entry->in_use_count > 0); entry->in_use_count--; #if CACHE_DEBUGGING validate_cache(); diff --git a/src/runtime/cuda.cpp b/src/runtime/cuda.cpp index eee52c1e4804..a4e300be79ca 100644 --- a/src/runtime/cuda.cpp +++ b/src/runtime/cuda.cpp @@ -72,7 +72,7 @@ ALWAYS_INLINE T get_cuda_symbol(void *user_context, const char *name, bool optio // Load a CUDA shared object/dll and get the CUDA API function pointers from it. WEAK void load_libcuda(void *user_context) { debug(user_context) << " load_libcuda (user_context: " << user_context << ")\n"; - halide_assert(user_context, cuInit == nullptr); + HALIDE_CHECK(user_context, cuInit == nullptr); // clang-format off #define CUDA_FN(ret, fn, args) fn = get_cuda_symbol(user_context, #fn); // NOLINT(bugprone-macro-parentheses) @@ -139,10 +139,10 @@ extern "C" { WEAK int halide_default_cuda_acquire_context(void *user_context, CUcontext *ctx, bool create = true) { // TODO: Should we use a more "assertive" assert? these asserts do // not block execution on failure. - halide_assert(user_context, ctx != nullptr); + HALIDE_CHECK(user_context, ctx != nullptr); // If the context has not been initialized, initialize it now. - halide_assert(user_context, &context != nullptr); + HALIDE_CHECK(user_context, &context != nullptr); // Note that this null-check of the context is *not* locked with // respect to device_release, so we may get a non-null context @@ -276,8 +276,8 @@ class Context { // overridden, we may still need to load libcuda ensure_libcuda_init(user_context); - halide_assert(user_context, context != nullptr); - halide_assert(user_context, cuInit != nullptr); + HALIDE_CHECK(user_context, context != nullptr); + HALIDE_CHECK(user_context, cuInit != nullptr); error = cuCtxPushCurrent(context); } @@ -578,7 +578,7 @@ WEAK int halide_cuda_initialize_kernels(void *user_context, void **state_ptr, co compile_kernel, user_context, ptx_src, size)) { return halide_error_code_generic_error; } - halide_assert(user_context, loaded_module != nullptr); + HALIDE_CHECK(user_context, loaded_module != nullptr); #ifdef DEBUG_RUNTIME uint64_t t_after = halide_current_time_ns(user_context); @@ -661,7 +661,7 @@ WEAK int halide_cuda_device_free(void *user_context, halide_buffer_t *buf) { uint64_t t_before = halide_current_time_ns(user_context); #endif - halide_assert(user_context, validate_device_pointer(user_context, buf)); + HALIDE_CHECK(user_context, validate_device_pointer(user_context, buf)); CUresult err = CUDA_SUCCESS; if (halide_can_reuse_device_allocations(user_context)) { @@ -735,7 +735,7 @@ WEAK int halide_cuda_device_release(void *user_context) { if (err != CUDA_SUCCESS) { err = cuCtxSynchronize(); } - halide_assert(user_context, err == CUDA_SUCCESS || err == CUDA_ERROR_DEINITIALIZED); + HALIDE_CHECK(user_context, err == CUDA_SUCCESS || err == CUDA_ERROR_DEINITIALIZED); // Dump the contents of the free list, ignoring errors. halide_cuda_release_unused_device_allocations(user_context); @@ -754,7 +754,7 @@ WEAK int halide_cuda_device_release(void *user_context) { debug(user_context) << " cuCtxDestroy " << context << "\n"; err = cuProfilerStop(); err = cuCtxDestroy(context); - halide_assert(user_context, err == CUDA_SUCCESS || err == CUDA_ERROR_DEINITIALIZED); + HALIDE_CHECK(user_context, err == CUDA_SUCCESS || err == CUDA_ERROR_DEINITIALIZED); context = nullptr; } } // spinlock @@ -779,16 +779,16 @@ WEAK int halide_cuda_device_malloc(void *user_context, halide_buffer_t *buf) { if (halide_can_reuse_device_allocations(user_context)) { size = quantize_allocation_size(size); } - halide_assert(user_context, size != 0); + HALIDE_CHECK(user_context, size != 0); if (buf->device) { // This buffer already has a device allocation - halide_assert(user_context, validate_device_pointer(user_context, buf, size)); + HALIDE_CHECK(user_context, validate_device_pointer(user_context, buf, size)); return 0; } // Check all strides positive. for (int i = 0; i < buf->dimensions; i++) { - halide_assert(user_context, buf->dim[i].stride >= 0); + HALIDE_CHECK(user_context, buf->dim[i].stride >= 0); } debug(user_context) << " allocating " << *buf << "\n"; @@ -877,7 +877,7 @@ WEAK int halide_cuda_device_malloc(void *user_context, halide_buffer_t *buf) { debug(user_context) << (void *)p << "\n"; } } - halide_assert(user_context, p); + HALIDE_CHECK(user_context, p); buf->device = p; buf->device_interface = &cuda_device_interface; buf->device_interface->impl->use_module(); @@ -957,12 +957,12 @@ WEAK int halide_cuda_buffer_copy(void *user_context, struct halide_buffer_t *src const struct halide_device_interface_t *dst_device_interface, struct halide_buffer_t *dst) { // We only handle copies to cuda or to host - halide_assert(user_context, dst_device_interface == nullptr || + HALIDE_CHECK(user_context, dst_device_interface == nullptr || dst_device_interface == &cuda_device_interface); if ((src->device_dirty() || src->host == nullptr) && src->device_interface != &cuda_device_interface) { - halide_assert(user_context, dst_device_interface == &cuda_device_interface); + HALIDE_CHECK(user_context, dst_device_interface == &cuda_device_interface); // This is handled at the higher level. return halide_error_code_incompatible_device_interface; } @@ -972,8 +972,8 @@ WEAK int halide_cuda_buffer_copy(void *user_context, struct halide_buffer_t *src (src->host_dirty() && src->host != nullptr); bool to_host = !dst_device_interface; - halide_assert(user_context, from_host || src->device); - halide_assert(user_context, to_host || dst->device); + HALIDE_CHECK(user_context, from_host || src->device); + HALIDE_CHECK(user_context, to_host || dst->device); device_copy c = make_buffer_copy(src, from_host, dst, to_host); @@ -991,10 +991,10 @@ WEAK int halide_cuda_buffer_copy(void *user_context, struct halide_buffer_t *src #ifdef DEBUG_RUNTIME uint64_t t_before = halide_current_time_ns(user_context); if (!from_host) { - halide_assert(user_context, validate_device_pointer(user_context, src)); + HALIDE_CHECK(user_context, validate_device_pointer(user_context, src)); } if (!to_host) { - halide_assert(user_context, validate_device_pointer(user_context, dst)); + HALIDE_CHECK(user_context, validate_device_pointer(user_context, dst)); } #endif @@ -1139,7 +1139,7 @@ WEAK int halide_cuda_run(void *user_context, CUmodule mod{}; bool found = compilation_cache.lookup(ctx.context, state_ptr, mod); - halide_assert(user_context, found && mod != nullptr); + HALIDE_CHECK(user_context, found && mod != nullptr); debug(user_context) << "Got module " << mod << "\n"; CUfunction f; @@ -1166,7 +1166,7 @@ WEAK int halide_cuda_run(void *user_context, uint64_t *dev_handles = (uint64_t *)malloc(num_args * sizeof(uint64_t)); for (size_t i = 0; i <= num_args; i++) { // Get nullptr at end. if (arg_is_buffer[i]) { - halide_assert(user_context, arg_sizes[i] == sizeof(uint64_t)); + HALIDE_CHECK(user_context, arg_sizes[i] == sizeof(uint64_t)); dev_handles[i] = ((halide_buffer_t *)args[i])->device; translated_args[i] = &(dev_handles[i]); debug(user_context) << " halide_cuda_run translated arg" << (int)i @@ -1226,7 +1226,7 @@ WEAK int halide_cuda_device_and_host_free(void *user_context, struct halide_buff } WEAK int halide_cuda_wrap_device_ptr(void *user_context, struct halide_buffer_t *buf, uint64_t device_ptr) { - halide_assert(user_context, buf->device == 0); + HALIDE_CHECK(user_context, buf->device == 0); if (buf->device != 0) { return -2; } @@ -1248,7 +1248,7 @@ WEAK int halide_cuda_detach_device_ptr(void *user_context, struct halide_buffer_ if (buf->device == 0) { return 0; } - halide_assert(user_context, buf->device_interface == &cuda_device_interface); + HALIDE_CHECK(user_context, buf->device_interface == &cuda_device_interface); buf->device_interface->impl->release_module(); buf->device = 0; buf->device_interface = nullptr; @@ -1259,7 +1259,7 @@ WEAK uintptr_t halide_cuda_get_device_ptr(void *user_context, struct halide_buff if (buf->device == 0) { return 0; } - halide_assert(user_context, buf->device_interface == &cuda_device_interface); + HALIDE_CHECK(user_context, buf->device_interface == &cuda_device_interface); return (uintptr_t)buf->device; } diff --git a/src/runtime/d3d12compute.cpp b/src/runtime/d3d12compute.cpp index a7173057aa41..b8d55782224a 100644 --- a/src/runtime/d3d12compute.cpp +++ b/src/runtime/d3d12compute.cpp @@ -226,7 +226,7 @@ void *d3d12_get_library_symbol(void *lib, const char *name) { #pragma message "RenderDoc might not work well alongside Direct3D debug layers..." #endif #define WIN32 -#define RenderDocAssert(expr) halide_assert(user_context, expr) +#define RenderDocAssert(expr) HALIDE_CHECK(user_context, expr) #define LoadRenderDocLibrary(dll) d3d12_load_library(dll) #define GetRenderDocProcAddr(dll, proc) d3d12_get_library_symbol(dll, proc) #define RENDERDOC_NO_STDINT @@ -420,8 +420,8 @@ static DXGI_FORMAT FindD3D12FormatForHalideType(void *user_context, halide_type_ }}, }; - halide_assert(user_context, (type.code >= 0) && (type.code <= 2)); - halide_assert(user_context, (type.lanes > 0) && (type.lanes <= 4)); + HALIDE_CHECK(user_context, (type.code >= 0) && (type.code <= 2)); + HALIDE_CHECK(user_context, (type.lanes > 0) && (type.lanes <= 4)); int i = 0; switch (type.bytes()) { @@ -438,7 +438,7 @@ static DXGI_FORMAT FindD3D12FormatForHalideType(void *user_context, halide_type_ i = 3; break; default: - halide_assert(user_context, false); + HALIDE_CHECK(user_context, false); break; } @@ -661,15 +661,15 @@ static size_t number_of_elements(void *user_context, const halide_buffer_t *buff // elements in the stride regions. size_t size_in_bytes = buffer->size_in_bytes(); - halide_assert(user_context, (size_in_bytes > 0)); + HALIDE_CHECK(user_context, (size_in_bytes > 0)); size_t element_size = 1; element_size *= buffer->type.bytes(); element_size *= buffer->type.lanes; - halide_assert(user_context, (element_size > 0)); + HALIDE_CHECK(user_context, (element_size > 0)); size_t elements = size_in_bytes / element_size; - halide_assert(user_context, (size_in_bytes % element_size) == 0); + HALIDE_CHECK(user_context, (size_in_bytes % element_size) == 0); return elements; } @@ -890,10 +890,10 @@ extern WEAK halide_device_interface_t d3d12compute_device_interface; static d3d12_buffer *peel_buffer(struct halide_buffer_t *hbuffer) { TRACELOG; - halide_assert(user_context, (hbuffer != nullptr)); - halide_assert(user_context, (hbuffer->device_interface == &d3d12compute_device_interface)); + HALIDE_CHECK(user_context, (hbuffer != nullptr)); + HALIDE_CHECK(user_context, (hbuffer->device_interface == &d3d12compute_device_interface)); d3d12_buffer *dbuffer = reinterpret_cast(hbuffer->device); - halide_assert(user_context, (dbuffer != nullptr)); + HALIDE_CHECK(user_context, (dbuffer != nullptr)); return dbuffer; } @@ -902,12 +902,12 @@ static const d3d12_buffer *peel_buffer(const struct halide_buffer_t *hbuffer) { } WEAK int wrap_buffer(void *user_context, struct halide_buffer_t *hbuffer, d3d12_buffer *dbuffer) { - halide_assert(user_context, (hbuffer->device == 0)); + HALIDE_CHECK(user_context, (hbuffer->device == 0)); if (hbuffer->device != 0) { return halide_error_code_device_wrap_native_failed; } - halide_assert(user_context, (dbuffer->resource != nullptr)); + HALIDE_CHECK(user_context, (dbuffer->resource != nullptr)); dbuffer->offset = 0; dbuffer->offsetInBytes = 0; @@ -921,7 +921,7 @@ WEAK int wrap_buffer(void *user_context, struct halide_buffer_t *hbuffer, d3d12_ dbuffer->halide_type = hbuffer->type; hbuffer->device = reinterpret_cast(dbuffer); - halide_assert(user_context, (hbuffer->device_interface == nullptr)); + HALIDE_CHECK(user_context, (hbuffer->device_interface == nullptr)); hbuffer->device_interface = &d3d12compute_device_interface; hbuffer->device_interface->impl->use_module(); @@ -972,7 +972,7 @@ static void D3D12LoadDependencies(void *user_context) { #if !RENDERDOC_AUTOINIT TRACEPRINT("Initializing RenderDoc\n"); bool rdinit = InitRenderDoc(); - halide_assert(user_context, rdinit); + HALIDE_CHECK(user_context, rdinit); #endif #endif @@ -984,33 +984,33 @@ static void D3D12LoadDependencies(void *user_context) { // Windows x64 follows the LLP64 integer type convention: // https://msdn.microsoft.com/en-us/library/windows/desktop/aa383751(v=vs.85).aspx - halide_assert(user_context, sizeof(BOOL) == (32 / 8)); // BOOL must be 32 bits - halide_assert(user_context, sizeof(CHAR) == (8 / 8)); // CHAR must be 8 bits - halide_assert(user_context, sizeof(SHORT) == (16 / 8)); // SHORT must be 16 bits - halide_assert(user_context, sizeof(LONG) == (32 / 8)); // LONG must be 32 bits - halide_assert(user_context, sizeof(ULONG) == (32 / 8)); // ULONG must be 32 bits - halide_assert(user_context, sizeof(LONGLONG) == (64 / 8)); // LONGLONG must be 16 bits - halide_assert(user_context, sizeof(BYTE) == (8 / 8)); // BYTE must be 8 bits - halide_assert(user_context, sizeof(WORD) == (16 / 8)); // WORD must be 16 bits - halide_assert(user_context, sizeof(DWORD) == (32 / 8)); // DWORD must be 32 bits - halide_assert(user_context, sizeof(WCHAR) == (16 / 8)); // WCHAR must be 16 bits - halide_assert(user_context, sizeof(INT) == (32 / 8)); // INT must be 32 bits - halide_assert(user_context, sizeof(UINT) == (32 / 8)); // UINT must be 32 bits - halide_assert(user_context, sizeof(IID) == (128 / 8)); // COM GUIDs must be 128 bits + HALIDE_CHECK(user_context, sizeof(BOOL) == (32 / 8)); // BOOL must be 32 bits + HALIDE_CHECK(user_context, sizeof(CHAR) == (8 / 8)); // CHAR must be 8 bits + HALIDE_CHECK(user_context, sizeof(SHORT) == (16 / 8)); // SHORT must be 16 bits + HALIDE_CHECK(user_context, sizeof(LONG) == (32 / 8)); // LONG must be 32 bits + HALIDE_CHECK(user_context, sizeof(ULONG) == (32 / 8)); // ULONG must be 32 bits + HALIDE_CHECK(user_context, sizeof(LONGLONG) == (64 / 8)); // LONGLONG must be 16 bits + HALIDE_CHECK(user_context, sizeof(BYTE) == (8 / 8)); // BYTE must be 8 bits + HALIDE_CHECK(user_context, sizeof(WORD) == (16 / 8)); // WORD must be 16 bits + HALIDE_CHECK(user_context, sizeof(DWORD) == (32 / 8)); // DWORD must be 32 bits + HALIDE_CHECK(user_context, sizeof(WCHAR) == (16 / 8)); // WCHAR must be 16 bits + HALIDE_CHECK(user_context, sizeof(INT) == (32 / 8)); // INT must be 32 bits + HALIDE_CHECK(user_context, sizeof(UINT) == (32 / 8)); // UINT must be 32 bits + HALIDE_CHECK(user_context, sizeof(IID) == (128 / 8)); // COM GUIDs must be 128 bits // Paranoid checks (I am not taking any chances...) - halide_assert(user_context, sizeof(INT8) == (8 / 8)); - halide_assert(user_context, sizeof(INT16) == (16 / 8)); - halide_assert(user_context, sizeof(INT32) == (32 / 8)); - halide_assert(user_context, sizeof(INT64) == (64 / 8)); - halide_assert(user_context, sizeof(UINT8) == (8 / 8)); - halide_assert(user_context, sizeof(UINT16) == (16 / 8)); - halide_assert(user_context, sizeof(UINT32) == (32 / 8)); - halide_assert(user_context, sizeof(UINT64) == (64 / 8)); + HALIDE_CHECK(user_context, sizeof(INT8) == (8 / 8)); + HALIDE_CHECK(user_context, sizeof(INT16) == (16 / 8)); + HALIDE_CHECK(user_context, sizeof(INT32) == (32 / 8)); + HALIDE_CHECK(user_context, sizeof(INT64) == (64 / 8)); + HALIDE_CHECK(user_context, sizeof(UINT8) == (8 / 8)); + HALIDE_CHECK(user_context, sizeof(UINT16) == (16 / 8)); + HALIDE_CHECK(user_context, sizeof(UINT32) == (32 / 8)); + HALIDE_CHECK(user_context, sizeof(UINT64) == (64 / 8)); #ifdef BITS_64 - halide_assert(user_context, sizeof(SIZE_T) == (64 / 8)); + HALIDE_CHECK(user_context, sizeof(SIZE_T) == (64 / 8)); #else - halide_assert(user_context, sizeof(SIZE_T) == (32 / 8)); + HALIDE_CHECK(user_context, sizeof(SIZE_T) == (32 / 8)); #endif } @@ -1117,7 +1117,7 @@ static d3d12_device *D3D12CreateSystemDefaultDevice(void *user_context) { return nullptr; } - halide_assert(user_context, (dxgiAdapter == nullptr)); + HALIDE_CHECK(user_context, (dxgiAdapter == nullptr)); size_t vram_max = 0; for (int i = 0;; ++i) { IDXGIAdapter1 *adapter = nullptr; @@ -1249,7 +1249,7 @@ ID3D12RootSignature *D3D12CreateMasterRootSignature(ID3D12Device *device) { ID3DBlob *pSignError = nullptr; HRESULT result = D3D12SerializeRootSignature(&rsd, Version, &pSignBlob, &pSignError); if (D3DErrorCheck(result, pSignBlob, nullptr, "Unable to serialize the Direct3D 12 root signature")) { - halide_assert(user_context, pSignError); + HALIDE_CHECK(user_context, pSignError); TRACEFATAL((const char *)pSignError->GetBufferPointer()); return nullptr; } @@ -1331,7 +1331,7 @@ WEAK d3d12_buffer new_buffer_resource(d3d12_device *device, size_t length, D3D12 break; default: TRACEPRINT("UNSUPPORTED D3D12 BUFFER HEAP TYPE: " << (int)heaptype << "\n"); - halide_assert(user_context, false); + HALIDE_CHECK(user_context, false); break; } @@ -1394,7 +1394,7 @@ WEAK void *map_buffer(d3d12_buffer *buffer) { break; default: TRACEPRINT("UNSUPPORTED BUFFER TYPE: " << (int)buffer->type << "\n"); - halide_assert(user_context, false); + HALIDE_CHECK(user_context, false); break; } @@ -1410,7 +1410,7 @@ WEAK void *map_buffer(d3d12_buffer *buffer) { return nullptr; } - halide_assert(user_context, pData); + HALIDE_CHECK(user_context, pData); buffer->mapped = pData; return pData; @@ -1438,7 +1438,7 @@ WEAK void unmap_buffer(d3d12_buffer *buffer) { break; default: TRACEPRINT("UNSUPPORTED BUFFER TYPE: " << (int)buffer->type << "\n"); - halide_assert(user_context, false); + HALIDE_CHECK(user_context, false); break; } @@ -1498,37 +1498,37 @@ WEAK size_t suballocate(d3d12_device *device, d3d12_buffer *staging, size_t num_ if (staging->sizeInBytes < num_bytes) { // ensure there are no pending transfers on this buffer uint64_t use_count = __atomic_add_fetch(&staging->ref_count, 1, __ATOMIC_SEQ_CST); - halide_assert(user_context, (use_count == 1)); + HALIDE_CHECK(user_context, (use_count == 1)); // find a new "ideal" size: e.g., using a cumulative 2x heuristic size_t old_capacity = staging->sizeInBytes; size_t new_capacity = 2 * (old_capacity + num_bytes); TRACEPRINT("not enough storage: growing from " << (uintptr_t)old_capacity << " bytes to " << (uintptr_t)new_capacity << " bytes.\n"); // release the old storage use_count = __atomic_sub_fetch(&staging->ref_count, 1, __ATOMIC_SEQ_CST); - halide_assert(user_context, (use_count == 0)); + HALIDE_CHECK(user_context, (use_count == 0)); release_d3d12_object(staging); // and allocate a new one switch (staging->type) { case d3d12_buffer::Upload: - halide_assert(user_context, (staging == &upload)); + HALIDE_CHECK(user_context, (staging == &upload)); *staging = new_upload_buffer(device, new_capacity); break; case d3d12_buffer::ReadBack: - halide_assert(user_context, (staging == &readback)); + HALIDE_CHECK(user_context, (staging == &readback)); *staging = new_readback_buffer(device, new_capacity); break; default: TRACEPRINT("UNSUPPORTED BUFFER TYPE: " << (int)staging->type << "\n"); - halide_assert(user_context, false); + HALIDE_CHECK(user_context, false); break; } } - halide_assert(user_context, (staging->sizeInBytes >= num_bytes)); + HALIDE_CHECK(user_context, (staging->sizeInBytes >= num_bytes)); // this reference counter will be decremented later by 'd3d12compute_device_sync_internal()' uint64_t use_count = __atomic_add_fetch(&staging->ref_count, 1, __ATOMIC_SEQ_CST); // but for now we must ensure that there are no pending transfers on this buffer already - halide_assert(user_context, (use_count == 1)); + HALIDE_CHECK(user_context, (use_count == 1)); size_t byte_offset = 0; // always zero, for now return byte_offset; } @@ -1668,7 +1668,7 @@ WEAK d3d12_command_queue *new_command_queue(d3d12_device *device) { template static d3d12_command_allocator *new_command_allocator(d3d12_device *device) { TRACELOG; - halide_assert(user_context, device); + HALIDE_CHECK(user_context, device); ID3D12CommandAllocator *commandAllocator = nullptr; HRESULT result = (*device)->CreateCommandAllocator(Type, IID_PPV_ARGS(&commandAllocator)); if (D3DErrorCheck(result, commandAllocator, nullptr, "Unable to create the Direct3D 12 command allocator")) { @@ -1941,11 +1941,11 @@ static d3d12_function *new_function_with_name(d3d12_device *device, d3d12_librar d3d12_function *function = nullptr; Printer key(nullptr); key << name << "_(" << threadsX << "," << threadsY << "," << threadsZ << ")_[" << shared_mem_bytes << "]"; - halide_assert(user_context, (key.size() < key.capacity() - 1)); // make sure key fits into the stream + HALIDE_CHECK(user_context, (key.size() < key.capacity() - 1)); // make sure key fits into the stream int not_found = library->cache.lookup(user_context, (const uint8_t *)key.str(), key.size(), &function); if (not_found) { // function has not been cached yet: must compile it - halide_assert(user_context, (function == nullptr)); + HALIDE_CHECK(user_context, (function == nullptr)); function = d3d12_compile_shader(device, library, name, shared_mem_bytes, threadsX, threadsY, threadsZ); if (function == nullptr) { return nullptr; @@ -1956,7 +1956,7 @@ static d3d12_function *new_function_with_name(d3d12_device *device, d3d12_librar TRACEPRINT("function has been found in the cache!\n"); } - halide_assert(user_context, (function != nullptr)); + HALIDE_CHECK(user_context, (function != nullptr)); return function; } @@ -1971,7 +1971,7 @@ WEAK void set_input_buffer(d3d12_binder *binder, d3d12_buffer *input_buffer, uin // NOTE(marcos): constant buffers are only used internally by the // runtime; users cannot create, control or access them, so it is // expected that no halide_buffer_t will be associated with them: - halide_assert(user_context, input_buffer->format == DXGI_FORMAT_UNKNOWN); + HALIDE_CHECK(user_context, input_buffer->format == DXGI_FORMAT_UNKNOWN); ID3D12Resource *pResource = input_buffer->resource; D3D12_GPU_VIRTUAL_ADDRESS pGPU = pResource->GetGPUVirtualAddress(); @@ -1982,7 +1982,7 @@ WEAK void set_input_buffer(d3d12_binder *binder, d3d12_buffer *input_buffer, uin cbvd.SizeInBytes = input_buffer->sizeInBytes; } - halide_assert(user_context, (index < ResourceBindingLimits[CBV])); + HALIDE_CHECK(user_context, (index < ResourceBindingLimits[CBV])); D3D12_CPU_DESCRIPTOR_HANDLE hDescCBV = binder->CPU[CBV]; binder->CPU[CBV].ptr += binder->descriptorSize; @@ -2038,7 +2038,7 @@ WEAK void set_input_buffer(d3d12_binder *binder, d3d12_buffer *input_buffer, uin uavd.Buffer.Flags = D3D12_BUFFER_UAV_FLAG_NONE; } - halide_assert(user_context, (index < ResourceBindingLimits[UAV])); + HALIDE_CHECK(user_context, (index < ResourceBindingLimits[UAV])); D3D12_CPU_DESCRIPTOR_HANDLE hDescUAV = binder->CPU[UAV]; binder->CPU[UAV].ptr += binder->descriptorSize; @@ -2055,7 +2055,7 @@ WEAK void set_input_buffer(d3d12_binder *binder, d3d12_buffer *input_buffer, uin case d3d12_buffer::ReadBack: default: TRACEPRINT("UNSUPPORTED BUFFER TYPE: " << (int)input_buffer->type << "\n"); - halide_assert(user_context, false); + HALIDE_CHECK(user_context, false); break; } } @@ -2181,10 +2181,10 @@ static void buffer_copy_command(d3d12_copy_command_list *cmdList, src_barrier.Transition.StateBefore = src->state; src_barrier.Transition.StateAfter = D3D12_RESOURCE_STATE_COPY_SOURCE; if (src->state == D3D12_RESOURCE_STATE_GENERIC_READ) { - halide_assert(user_context, src->type == d3d12_buffer::Upload); + HALIDE_CHECK(user_context, src->type == d3d12_buffer::Upload); src_barrier.Transition.StateAfter = D3D12_RESOURCE_STATE_GENERIC_READ; } else { - halide_assert(user_context, src->state == D3D12_RESOURCE_STATE_UNORDERED_ACCESS); + HALIDE_CHECK(user_context, src->state == D3D12_RESOURCE_STATE_UNORDERED_ACCESS); } } @@ -2197,9 +2197,9 @@ static void buffer_copy_command(d3d12_copy_command_list *cmdList, dst_barrier.Transition.StateBefore = dst->state; dst_barrier.Transition.StateAfter = D3D12_RESOURCE_STATE_COPY_DEST; if (dst->state == D3D12_RESOURCE_STATE_COPY_DEST) { - halide_assert(user_context, dst->type == d3d12_buffer::ReadBack); + HALIDE_CHECK(user_context, dst->type == d3d12_buffer::ReadBack); } else { - halide_assert(user_context, dst->state == D3D12_RESOURCE_STATE_UNORDERED_ACCESS); + HALIDE_CHECK(user_context, dst->state == D3D12_RESOURCE_STATE_UNORDERED_ACCESS); } } @@ -2231,7 +2231,7 @@ static void synchronize_host_and_device_buffer_contents(d3d12_copy_command_list TRACELOG; d3d12_buffer::transfer_t *xfer = buffer->xfer; - halide_assert(user_context, (xfer != nullptr)); + HALIDE_CHECK(user_context, (xfer != nullptr)); d3d12_buffer *src = nullptr; d3d12_buffer *dst = nullptr; @@ -2258,7 +2258,7 @@ static void synchronize_host_and_device_buffer_contents(d3d12_copy_command_list break; default: TRACEPRINT("UNSUPPORTED BUFFER TYPE: " << (int)buffer->type << "\n"); - halide_assert(user_context, false); + HALIDE_CHECK(user_context, false); break; } @@ -2295,7 +2295,7 @@ static void d3d12compute_device_sync_internal(d3d12_device *device, d3d12_buffer // decrement the reference counter that was incremented by 'suballocate()' uint64_t use_count = __atomic_sub_fetch(&staging_buffer->ref_count, 1, __ATOMIC_SEQ_CST); // for now, we expect to have been the only one with pending transfer on the staging buffer: - halide_assert(user_context, (use_count == 0)); + HALIDE_CHECK(user_context, (use_count == 0)); dev_buffer->xfer = nullptr; } } @@ -2308,28 +2308,28 @@ static int d3d12compute_buffer_copy(d3d12_device *device, uint64_t num_bytes) { TRACELOG; - halide_assert(user_context, device); - halide_assert(user_context, src); - halide_assert(user_context, dst); - halide_assert(user_context, (src->type != d3d12_buffer::Unknown)); - halide_assert(user_context, (dst->type != d3d12_buffer::Unknown)); + HALIDE_CHECK(user_context, device); + HALIDE_CHECK(user_context, src); + HALIDE_CHECK(user_context, dst); + HALIDE_CHECK(user_context, (src->type != d3d12_buffer::Unknown)); + HALIDE_CHECK(user_context, (dst->type != d3d12_buffer::Unknown)); // constant buffers are only used internally (Halide never expose them) // (uploads to constant buffers are managed automatically by Map/Unmap) - halide_assert(user_context, (src->type != d3d12_buffer::Constant)); - halide_assert(user_context, (dst->type != d3d12_buffer::Constant)); + HALIDE_CHECK(user_context, (src->type != d3d12_buffer::Constant)); + HALIDE_CHECK(user_context, (dst->type != d3d12_buffer::Constant)); - halide_assert(user_context, num_bytes > 0); + HALIDE_CHECK(user_context, num_bytes > 0); if (src->type == d3d12_buffer::Upload) { // host-to-device via staging buffer: - halide_assert(user_context, (dst->type != d3d12_buffer::Upload)); - halide_assert(user_context, (dst->type != d3d12_buffer::ReadBack)); + HALIDE_CHECK(user_context, (dst->type != d3d12_buffer::Upload)); + HALIDE_CHECK(user_context, (dst->type != d3d12_buffer::ReadBack)); // TODO: assert that offsets and sizes are within bounds d3d12_buffer::transfer_t xfer = {}; xfer.staging = src; xfer.offset = src_byte_offset; xfer.size = num_bytes; - halide_assert(user_context, (dst->xfer == nullptr)); + HALIDE_CHECK(user_context, (dst->xfer == nullptr)); dst->xfer = &xfer; d3d12compute_device_sync_internal(device, dst); return 0; @@ -2337,24 +2337,24 @@ static int d3d12compute_buffer_copy(d3d12_device *device, if (dst->type == d3d12_buffer::ReadBack) { // device-to-host via staging buffer: - halide_assert(user_context, (src->type != d3d12_buffer::Upload)); - halide_assert(user_context, (src->type != d3d12_buffer::ReadBack)); + HALIDE_CHECK(user_context, (src->type != d3d12_buffer::Upload)); + HALIDE_CHECK(user_context, (src->type != d3d12_buffer::ReadBack)); // TODO: assert that offsets and sizes are within bounds d3d12_buffer::transfer_t xfer = {}; xfer.staging = dst; xfer.offset = dst_byte_offset; xfer.size = num_bytes; - halide_assert(user_context, (src->xfer == nullptr)); + HALIDE_CHECK(user_context, (src->xfer == nullptr)); src->xfer = &xfer; d3d12compute_device_sync_internal(device, src); return 0; } // device-to-device: - halide_assert(user_context, (src->type != d3d12_buffer::Upload)); - halide_assert(user_context, (dst->type != d3d12_buffer::Upload)); - halide_assert(user_context, (src->type != d3d12_buffer::ReadBack)); - halide_assert(user_context, (dst->type != d3d12_buffer::ReadBack)); + HALIDE_CHECK(user_context, (src->type != d3d12_buffer::Upload)); + HALIDE_CHECK(user_context, (dst->type != d3d12_buffer::Upload)); + HALIDE_CHECK(user_context, (src->type != d3d12_buffer::ReadBack)); + HALIDE_CHECK(user_context, (dst->type != d3d12_buffer::ReadBack)); // ReadWrite, ReadOnly and WriteOnly are shader usage hints, not copy hints // (there's no need to worry about them during device-to-device transfers) @@ -2392,7 +2392,7 @@ static void *buffer_contents(d3d12_buffer *buffer) { case d3d12_buffer::WriteOnly: case d3d12_buffer::ReadWrite: { TRACEWARN("UNCHARTED TERRITORY! THIS CASE IS NOT EXPECTED TO HAPPEN FOR NOW!\n"); - halide_assert(user_context, false); + HALIDE_CHECK(user_context, false); D3D12ContextHolder d3d12_context(user_context, true); if (d3d12_context.error != 0) { @@ -2415,11 +2415,11 @@ static void *buffer_contents(d3d12_buffer *buffer) { case d3d12_buffer::Unknown: default: TRACEPRINT("UNSUPPORTED BUFFER TYPE: " << (int)buffer->type << "\n"); - halide_assert(user_context, false); + HALIDE_CHECK(user_context, false); break; } - halide_assert(user_context, pData); + HALIDE_CHECK(user_context, pData); return pData; } @@ -2458,14 +2458,14 @@ static int d3d12_create_context(void *user_context) { int status = halide_error_code_success; - halide_assert(user_context, (device == nullptr)); + HALIDE_CHECK(user_context, (device == nullptr)); device = D3D12CreateSystemDefaultDevice(user_context); if (device == nullptr) { status = halide_error_code_generic_error; } if (status == halide_error_code_success) { - halide_assert(user_context, (rootSignature == nullptr)); + HALIDE_CHECK(user_context, (rootSignature == nullptr)); rootSignature = D3D12CreateMasterRootSignature((*device)); if (rootSignature == nullptr) { status = halide_error_code_generic_error; @@ -2473,7 +2473,7 @@ static int d3d12_create_context(void *user_context) { } if (status == halide_error_code_success) { - halide_assert(user_context, (queue == nullptr)); + HALIDE_CHECK(user_context, (queue == nullptr)); queue = new_command_queue(device); if (queue == nullptr) { status = halide_error_code_generic_error; @@ -2481,7 +2481,7 @@ static int d3d12_create_context(void *user_context) { } if (status == halide_error_code_success) { - halide_assert(user_context, (cmd_allocator_main == nullptr)); + HALIDE_CHECK(user_context, (cmd_allocator_main == nullptr)); cmd_allocator_main = new_command_allocator(device); if (cmd_allocator_main == nullptr) { status = halide_error_code_generic_error; @@ -2491,8 +2491,8 @@ static int d3d12_create_context(void *user_context) { if (status == halide_error_code_success) { // NOTE(marcos): a small amount of hard-coded staging buffer storage is // sufficient to get started as suballocations will grow them as needed - halide_assert(user_context, (upload == 0)); - halide_assert(user_context, (readback == 0)); + HALIDE_CHECK(user_context, (upload == 0)); + HALIDE_CHECK(user_context, (readback == 0)); size_t heap_size = 4 * 1024 * 1024; upload = new_upload_buffer(device, heap_size); readback = new_readback_buffer(device, heap_size); @@ -2537,7 +2537,7 @@ WEAK int halide_d3d12compute_acquire_context(void *user_context, halide_d3d12com halide_start_clock(user_context); #endif - halide_assert(user_context, &thread_lock != nullptr); + HALIDE_CHECK(user_context, &thread_lock != nullptr); while (__atomic_test_and_set(&thread_lock, __ATOMIC_ACQUIRE)) { } @@ -2554,7 +2554,7 @@ WEAK int halide_d3d12compute_acquire_context(void *user_context, halide_d3d12com // If the device has already been initialized, // ensure the queue has as well. - halide_assert(user_context, (device == nullptr) || (queue != nullptr)); + HALIDE_CHECK(user_context, (device == nullptr) || (queue != nullptr)); *device_ret = device; *queue_ret = queue; @@ -2576,7 +2576,7 @@ static void d3d12_debug_dump(error &err) { return; } - halide_assert(user_context, (dxgiAdapter != nullptr)); + HALIDE_CHECK(user_context, (dxgiAdapter != nullptr)); DXGI_ADAPTER_DESC1 desc = {}; if (FAILED(dxgiAdapter->GetDesc1(&desc))) { err << "Unable to retrieve information about the device adapter.\n"; @@ -2681,11 +2681,11 @@ WEAK int halide_d3d12compute_device_malloc(void *user_context, halide_buffer_t * } size_t size = buf->size_in_bytes(); - halide_assert(user_context, size > 0); + HALIDE_CHECK(user_context, size > 0); // Check all strides positive for (int i = 0; i < buf->dimensions; i++) { - halide_assert(user_context, buf->dim[i].stride >= 0); + HALIDE_CHECK(user_context, buf->dim[i].stride >= 0); } d3d12_buffer *d3d12_buf = d3d12_allocation_cache_get_buffer(user_context, size); @@ -2860,7 +2860,7 @@ int do_multidimensional_copy(d3d12_device *device, const device_copy &c, if (dimensions == 0) { d3d12_buffer *dsrc = reinterpret_cast(c.src); d3d12_buffer *ddst = reinterpret_cast(c.dst); - halide_assert(user_context, (dsrc->halide_type == ddst->halide_type)); + HALIDE_CHECK(user_context, (dsrc->halide_type == ddst->halide_type)); TRACEPRINT("src_offset: " << src_offset << "\n"); TRACEPRINT("dst_offset: " << dst_offset << "\n"); TRACEPRINT("c.chunk_size: " << c.chunk_size << "\n"); @@ -2890,8 +2890,8 @@ int do_multidimensional_copy(d3d12_device *device, const device_copy &c, WEAK int halide_d3d12compute_copy_to_device(void *user_context, halide_buffer_t *buffer) { TRACELOG; - halide_assert(user_context, buffer); - halide_assert(user_context, buffer->host && buffer->device); + HALIDE_CHECK(user_context, buffer); + HALIDE_CHECK(user_context, buffer->host && buffer->device); D3D12ContextHolder d3d12_context(user_context, true); if (d3d12_context.error != 0) { @@ -2900,9 +2900,9 @@ WEAK int halide_d3d12compute_copy_to_device(void *user_context, halide_buffer_t // 1. memcpy from halide host memory to "upload" staging memory device_copy c = make_host_to_device_copy(buffer); - halide_assert(user_context, (c.dst == buffer->device)); + HALIDE_CHECK(user_context, (c.dst == buffer->device)); d3d12_buffer *dev_buffer = peel_buffer(buffer); - halide_assert(user_context, buffer->size_in_bytes() == dev_buffer->sizeInBytes); + HALIDE_CHECK(user_context, buffer->size_in_bytes() == dev_buffer->sizeInBytes); size_t total_size = dev_buffer->sizeInBytes; d3d12_buffer *staging = &upload; size_t staging_byte_offset = suballocate(d3d12_context.device, staging, total_size); @@ -2933,10 +2933,10 @@ WEAK int halide_d3d12compute_copy_to_device(void *user_context, halide_buffer_t WEAK int halide_d3d12compute_copy_to_host(void *user_context, halide_buffer_t *buffer) { TRACELOG; - halide_assert(user_context, buffer); - halide_assert(user_context, buffer->host && buffer->device); + HALIDE_CHECK(user_context, buffer); + HALIDE_CHECK(user_context, buffer->host && buffer->device); if (buffer->dimensions > MAX_COPY_DIMS) { - halide_assert(user_context, false); + HALIDE_CHECK(user_context, false); return halide_error_code_copy_to_host_failed; } @@ -2948,7 +2948,7 @@ WEAK int halide_d3d12compute_copy_to_host(void *user_context, halide_buffer_t *b // 1. download data from device (copy to the "readback" staging memory): d3d12_buffer *dev_buffer = peel_buffer(buffer); d3d12_buffer *staging = &readback; - halide_assert(user_context, buffer->size_in_bytes() == dev_buffer->sizeInBytes); + HALIDE_CHECK(user_context, buffer->size_in_bytes() == dev_buffer->sizeInBytes); size_t total_size = dev_buffer->sizeInBytes; size_t dev_byte_offset = dev_buffer->offsetInBytes; // handle cropping size_t staging_byte_offset = suballocate(d3d12_context.device, staging, total_size); @@ -2989,7 +2989,7 @@ WEAK int halide_d3d12compute_run(void *user_context, d3d12_library *library{}; bool found = compilation_cache.lookup(device, state_ptr, library); - halide_assert(user_context, found && library != nullptr); + HALIDE_CHECK(user_context, found && library != nullptr); d3d12_frame *frame = acquire_frame(device); d3d12_compute_command_list *cmdList = frame->cmd_list; @@ -3003,7 +3003,7 @@ WEAK int halide_d3d12compute_run(void *user_context, TRACE_SCOPE("kernel shader selection"); function = new_function_with_name(device, library, entry_name, strlen(entry_name), shared_mem_bytes, threadsX, threadsY, threadsZ); - halide_assert(user_context, function); + HALIDE_CHECK(user_context, function); pipeline_state = function->pipeline_state; set_compute_pipeline_state(cmdList, pipeline_state, function, binder); } @@ -3040,13 +3040,13 @@ WEAK int halide_d3d12compute_run(void *user_context, // multiple of that power-of-two. halide_type_t arg_type = arg_types[i]; arg_sizes[i] = arg_type.bytes(); - halide_assert(user_context, (arg_sizes[i] & (arg_sizes[i] - 1)) == 0); + HALIDE_CHECK(user_context, (arg_sizes[i] & (arg_sizes[i] - 1)) == 0); // We can ignore vector arguments since they never show up in constant // blocks. Having to worry about scalar parameters only is convenient // since in HLSL SM 5.1 all scalar types are 32bit: - halide_assert(user_context, arg_type.lanes == 1); - halide_assert(user_context, arg_sizes[i] > 0); - halide_assert(user_context, arg_sizes[i] <= 4); + HALIDE_CHECK(user_context, arg_type.lanes == 1); + HALIDE_CHECK(user_context, arg_sizes[i] > 0); + HALIDE_CHECK(user_context, arg_sizes[i] <= 4); size_t packed_size = 4; // force the final "packed" argument to be 32bit total_uniform_args_size = (total_uniform_args_size + packed_size - 1) & ~(packed_size - 1); total_uniform_args_size += packed_size; @@ -3079,7 +3079,7 @@ WEAK int halide_d3d12compute_run(void *user_context, } const halide_type_t arg_type = arg_types[i]; if (arg_type.code == halide_type_float) { - halide_assert(user_context, (arg_type.bits == 32)); + HALIDE_CHECK(user_context, (arg_type.bits == 32)); float &uniform_value = ((float &)uniform_word); uniform_value = *((float *)args[i]); TRACELEVEL(3, "args[" << i << "] -> float32 = " << uniform_value << "\n"); @@ -3095,7 +3095,7 @@ WEAK int halide_d3d12compute_run(void *user_context, } else if (arg_type.bits == 32) { uniform_value = *((int32_t *)args[i]); } else { - halide_assert(user_context, false); + HALIDE_CHECK(user_context, false); } TRACELEVEL(3, "args[" << i << "] -> int32 = " << uniform_value << "\n"); } else if (arg_type.code == halide_type_uint) { @@ -3110,17 +3110,17 @@ WEAK int halide_d3d12compute_run(void *user_context, } else if (arg_type.bits == 32) { uniform_value = *((uint32_t *)args[i]); } else { - halide_assert(user_context, false); + HALIDE_CHECK(user_context, false); } TRACELEVEL(3, "args[" << i << "] -> uint32 = " << uniform_value << "\n"); } else { - halide_assert(user_context, false); + HALIDE_CHECK(user_context, false); } memcpy(&uniform_bytes[offset], &uniform_word, uniform_size); offset = (offset + uniform_size - 1) & ~(uniform_size - 1); offset += uniform_size; } - halide_assert(user_context, offset == total_uniform_args_size); + HALIDE_CHECK(user_context, offset == total_uniform_args_size); } { @@ -3231,7 +3231,7 @@ WEAK int halide_d3d12compute_buffer_copy(void *user_context, struct halide_buffe struct halide_buffer_t *dst) { TRACELOG; - halide_assert(user_context, (src->dimensions == dst->dimensions)); + HALIDE_CHECK(user_context, (src->dimensions == dst->dimensions)); const int dimensions = dst->dimensions; if (dimensions > MAX_COPY_DIMS) { error(user_context) << "Buffer has too many dimensions to copy to/from GPU.\n"; @@ -3239,12 +3239,12 @@ WEAK int halide_d3d12compute_buffer_copy(void *user_context, struct halide_buffe } // We only handle copies to d3d12 device or to host - halide_assert(user_context, (dst_device_interface == nullptr) || + HALIDE_CHECK(user_context, (dst_device_interface == nullptr) || (dst_device_interface == &d3d12compute_device_interface)); if ((src->device_dirty() || src->host == nullptr) && src->device_interface != &d3d12compute_device_interface) { - halide_assert(user_context, dst_device_interface == &d3d12compute_device_interface); + HALIDE_CHECK(user_context, dst_device_interface == &d3d12compute_device_interface); // This is handled at the higher level. return halide_error_code_incompatible_device_interface; } @@ -3254,8 +3254,8 @@ WEAK int halide_d3d12compute_buffer_copy(void *user_context, struct halide_buffe (src->host_dirty() && src->host != nullptr); bool to_host = !dst_device_interface; - halide_assert(user_context, from_host || src->device); - halide_assert(user_context, to_host || dst->device); + HALIDE_CHECK(user_context, from_host || src->device); + HALIDE_CHECK(user_context, to_host || dst->device); device_copy c = make_buffer_copy(src, from_host, dst, to_host); MAYBE_UNUSED(c); @@ -3281,13 +3281,13 @@ WEAK int halide_d3d12compute_buffer_copy(void *user_context, struct halide_buffe if (from_host) { // host-to-device: TRACEPRINT("host-to-device case\n"); - halide_assert(user_context, !to_host); - halide_assert(user_context, (dst->device_interface == &d3d12compute_device_interface)); - halide_assert(user_context, (src->device_interface != &d3d12compute_device_interface)); - halide_assert(user_context, (src->host != nullptr)); + HALIDE_CHECK(user_context, !to_host); + HALIDE_CHECK(user_context, (dst->device_interface == &d3d12compute_device_interface)); + HALIDE_CHECK(user_context, (src->device_interface != &d3d12compute_device_interface)); + HALIDE_CHECK(user_context, (src->host != nullptr)); // it's possible for 'dst->host' to be null, so we can't always memcpy from 'src->host' // to 'dst-host' and push/sync changes with 'halide_d3d12compute_copy_to_device' ... - halide_assert(user_context, (dst->device == c.dst)); + HALIDE_CHECK(user_context, (dst->device == c.dst)); if (dst->host != nullptr) { // 1. copy 'src->host' buffer to 'dst->host' buffer: // host buffers already account for the beginning of cropped regions @@ -3316,18 +3316,18 @@ WEAK int halide_d3d12compute_buffer_copy(void *user_context, struct halide_buffe d3d12compute_buffer_copy(d3d12_context.device, staging, ddst, staging_byte_offset, dst_byte_offset, total_size); uint64_t use_count = __atomic_load_n(&staging->ref_count, __ATOMIC_SEQ_CST); - halide_assert(user_context, (use_count == 0)); + HALIDE_CHECK(user_context, (use_count == 0)); } } else { // device-to-host: TRACEPRINT("device-to-host case\n"); - halide_assert(user_context, to_host); - halide_assert(user_context, (src->device_interface == &d3d12compute_device_interface)); - halide_assert(user_context, (dst->device_interface == nullptr)); - halide_assert(user_context, (dst->host != nullptr)); + HALIDE_CHECK(user_context, to_host); + HALIDE_CHECK(user_context, (src->device_interface == &d3d12compute_device_interface)); + HALIDE_CHECK(user_context, (dst->device_interface == nullptr)); + HALIDE_CHECK(user_context, (dst->host != nullptr)); // it's possible for 'src->host' to be null, so we can't always pull/sync changes with // 'halide_d3d12compute_copy_to_host' and then memcpy from 'src->host' to 'dst-host'... - halide_assert(user_context, (src->device == c.src)); + HALIDE_CHECK(user_context, (src->device == c.src)); if (src->host != nullptr) { // 1. sync 'src->device' buffer with 'src->host' buffer: halide_d3d12compute_copy_to_host(user_context, src); @@ -3356,7 +3356,7 @@ WEAK int halide_d3d12compute_buffer_copy(void *user_context, struct halide_buffe c.dst = reinterpret_cast(dst->host) + 0; copy_memory(c, user_context); uint64_t use_count = __atomic_load_n(&staging->ref_count, __ATOMIC_SEQ_CST); - halide_assert(user_context, (use_count == 0)); + HALIDE_CHECK(user_context, (use_count == 0)); } } } @@ -3384,9 +3384,9 @@ WEAK int d3d12compute_device_crop_from_offset(void *user_context, } d3d12_buffer *new_handle = peel_buffer(dst); - halide_assert(user_context, (new_handle != nullptr)); - halide_assert(user_context, (new_handle->halide_type == dst->type)); - halide_assert(user_context, (src->device_interface == dst->device_interface)); + HALIDE_CHECK(user_context, (new_handle != nullptr)); + HALIDE_CHECK(user_context, (new_handle->halide_type == dst->type)); + HALIDE_CHECK(user_context, (src->device_interface == dst->device_interface)); new_handle->offset = old_handle->offset + offset; new_handle->offsetInBytes = new_handle->offset * dst->type.bytes() * dst->type.lanes; @@ -3417,7 +3417,7 @@ WEAK int halide_d3d12compute_device_crop(void *user_context, using namespace Halide::Runtime; int64_t offset = Internal::calc_device_crop_byte_offset(src, dst); // D3D12 buffer views are element-based, not byte-based - halide_assert(user_context, (offset % src->type.bytes()) == 0); + HALIDE_CHECK(user_context, (offset % src->type.bytes()) == 0); offset /= src->type.bytes(); return d3d12compute_device_crop_from_offset(user_context, src, offset, dst); } @@ -3430,7 +3430,7 @@ WEAK int halide_d3d12compute_device_slice(void *user_context, using namespace Halide::Runtime; int64_t offset = Internal::calc_device_slice_byte_offset(src, slice_dim, slice_pos); // D3D12 buffer views are element-based, not byte-based - halide_assert(user_context, (offset % src->type.bytes()) == 0); + HALIDE_CHECK(user_context, (offset % src->type.bytes()) == 0); offset /= src->type.bytes(); return d3d12compute_device_crop_from_offset(user_context, src, offset, dst); } @@ -3472,7 +3472,7 @@ WEAK int halide_d3d12compute_wrap_buffer(void *user_context, struct halide_buffe TRACELOG; ID3D12Resource *pResource = reinterpret_cast(d3d12_resource); - halide_assert(user_context, (pResource != nullptr)); + HALIDE_CHECK(user_context, (pResource != nullptr)); d3d12_buffer sbuffer = {}; sbuffer.resource = pResource; diff --git a/src/runtime/device_interface.cpp b/src/runtime/device_interface.cpp index a494d05a9574..bee7d201876d 100644 --- a/src/runtime/device_interface.cpp +++ b/src/runtime/device_interface.cpp @@ -249,7 +249,7 @@ WEAK int halide_device_free(void *user_context, struct halide_buffer_t *buf) { device_interface->impl->use_module(); result = device_interface->impl->device_free(user_context, buf); device_interface->impl->release_module(); - halide_assert(user_context, buf->device == 0); + HALIDE_CHECK(user_context, buf->device == 0); if (result) { return halide_error_code_device_free_failed; } else { @@ -314,7 +314,7 @@ WEAK int halide_device_and_host_free(void *user_context, struct halide_buffer_t device_interface->impl->use_module(); result = device_interface->impl->device_and_host_free(user_context, buf); device_interface->impl->release_module(); - halide_assert(user_context, buf->device == 0); + HALIDE_CHECK(user_context, buf->device == 0); if (result) { return halide_error_code_device_free_failed; } else { @@ -400,7 +400,7 @@ WEAK int halide_device_detach_native(void *user_context, struct halide_buffer_t device_interface->impl->use_module(); result = device_interface->impl->detach_native(user_context, buf); device_interface->impl->release_module(); - halide_assert(user_context, buf->device == 0); + HALIDE_CHECK(user_context, buf->device == 0); if (result) { result = halide_error_code_device_detach_native_failed; } @@ -409,7 +409,7 @@ WEAK int halide_device_detach_native(void *user_context, struct halide_buffer_t } WEAK int halide_default_device_wrap_native(void *user_context, struct halide_buffer_t *buf, uint64_t handle) { - halide_assert(user_context, buf->device == 0); + HALIDE_CHECK(user_context, buf->device == 0); if (buf->device != 0) { return halide_error_code_device_wrap_native_failed; } diff --git a/src/runtime/gpu_context_common.h b/src/runtime/gpu_context_common.h index 9b9bbaab4a65..c244f9c09859 100644 --- a/src/runtime/gpu_context_common.h +++ b/src/runtime/gpu_context_common.h @@ -66,7 +66,7 @@ class GPUCompilationCache { } // This is a logic error that should never occur. It means the table is // full, but it should have been resized. - halide_assert(nullptr, false); + HALIDE_CHECK(nullptr, false); return false; } @@ -124,7 +124,7 @@ class GPUCompilationCache { if (old_table[i].kernel_id != kInvalidId && old_table[i].kernel_id != kDeletedId) { bool result = insert(old_table[i]); - halide_assert(nullptr, result); // Resizing the table while resizing the table is a logic error. + HALIDE_CHECK(nullptr, result); // Resizing the table while resizing the table is a logic error. } } } @@ -212,7 +212,7 @@ class GPUCompilationCache { ModuleStateT *mod; uint32_t id = (uint32_t)(uintptr_t)state_ptr; bool result = find_internal(context, id, mod, -1); - halide_assert(user_context, result); // Value must be in cache to be released + HALIDE_CHECK(user_context, result); // Value must be in cache to be released } }; diff --git a/src/runtime/hashmap.h b/src/runtime/hashmap.h index a9195dcdb2b0..e1d39b509555 100644 --- a/src/runtime/hashmap.h +++ b/src/runtime/hashmap.h @@ -149,7 +149,7 @@ struct HashMap { inline bool HashMap::init(void *user_context, copy_value_func _copy_value, destroy_value_func _destroy_value) { memset(&memoization_lock, 0, sizeof(halide_mutex)); - halide_assert(nullptr, !inited); + HALIDE_CHECK(nullptr, !inited); most_recently_used = nullptr; least_recently_used = nullptr; kDefaultCacheSize = 1 << 20; @@ -158,8 +158,8 @@ inline bool HashMap::init(void *user_context, copy_value_func _copy_value, destr for (auto &cache_entry_ref : cache_entries) { cache_entry_ref = nullptr; } - halide_assert(nullptr, _copy_value); - halide_assert(nullptr, _destroy_value); + HALIDE_CHECK(nullptr, _copy_value); + HALIDE_CHECK(nullptr, _destroy_value); this->copy_value = _copy_value; this->destroy_value = _destroy_value; inited = true; @@ -188,7 +188,7 @@ inline void HashMap::prune() { while (prev_hash_entry != nullptr && prev_hash_entry->next != prune_candidate) { prev_hash_entry = prev_hash_entry->next; } - halide_assert(nullptr, prev_hash_entry != nullptr); + HALIDE_CHECK(nullptr, prev_hash_entry != nullptr); prev_hash_entry->next = prune_candidate->next; } @@ -261,14 +261,14 @@ inline int HashMap::lookup(void *user_context, keys_equal(entry->key, cache_key, size)) { if (entry != most_recently_used) { - halide_assert(user_context, entry->more_recent != nullptr); + HALIDE_CHECK(user_context, entry->more_recent != nullptr); if (entry->less_recent != nullptr) { entry->less_recent->more_recent = entry->more_recent; } else { - halide_assert(user_context, least_recently_used == entry); + HALIDE_CHECK(user_context, least_recently_used == entry); least_recently_used = entry->more_recent; } - halide_assert(user_context, entry->more_recent != nullptr); + HALIDE_CHECK(user_context, entry->more_recent != nullptr); entry->more_recent->less_recent = entry->less_recent; entry->more_recent = nullptr; @@ -279,7 +279,7 @@ inline int HashMap::lookup(void *user_context, most_recently_used = entry; } - halide_assert(user_context, (cache_value_size == entry->value_size)); + HALIDE_CHECK(user_context, (cache_value_size == entry->value_size)); copy_value(cache_value, entry->value, entry->value_size); entry->in_use_count += 1; @@ -325,7 +325,7 @@ inline int HashMap::store(void *user_context, entry = entry->next) { if (entry->hash == h && entry->key_size == (size_t)size && keys_equal(entry->key, cache_key, size)) { - halide_assert(user_context, (cache_value_size == entry->value_size)); + HALIDE_CHECK(user_context, (cache_value_size == entry->value_size)); destroy_value(entry->value, entry->value_size); copy_value(entry->value, cache_value, entry->value_size); return (0); @@ -335,7 +335,7 @@ inline int HashMap::store(void *user_context, // key not found: create new entry CacheEntry *new_entry = (CacheEntry *)hashmap_malloc(user_context, sizeof(CacheEntry)); bool inited = new_entry->init(user_context, cache_key, size, h, cache_value, cache_value_size, copy_value); - halide_assert(user_context, inited); + HALIDE_CHECK(user_context, inited); uint64_t added_size = cache_value_size; current_cache_size += added_size; @@ -365,7 +365,7 @@ inline int HashMap::store(void *user_context, inline void HashMap::release(void *user_context, void *host) { debug(user_context) << "halide_memoization_cache_release\n"; // TODO(marcos): this method does not make sense on a generic hashmap... remove it? - halide_assert(user_context, false); + HALIDE_CHECK(user_context, false); debug(user_context) << "Exited halide_memoization_cache_release.\n"; } @@ -396,14 +396,14 @@ struct THashMap : public HashMap { // member functions below... static void copy_value_func(uint8_t *dst, const uint8_t *src, size_t size) { - halide_assert(nullptr, sizeof(ValueType) == size); + HALIDE_CHECK(nullptr, sizeof(ValueType) == size); ValueType *D = reinterpret_cast(dst); const ValueType *S = reinterpret_cast(src); *D = *S; } static void destroy_value_func(uint8_t *value, size_t size) { - halide_assert(nullptr, sizeof(ValueType) == size); + HALIDE_CHECK(nullptr, sizeof(ValueType) == size); ValueType *V = reinterpret_cast(value); V->~ValueType(); } diff --git a/src/runtime/hexagon_cache_allocator.cpp b/src/runtime/hexagon_cache_allocator.cpp index 686e39b85a6c..4e642031ee83 100644 --- a/src/runtime/hexagon_cache_allocator.cpp +++ b/src/runtime/hexagon_cache_allocator.cpp @@ -119,7 +119,7 @@ inline void *hexagon_cache_pool_get(void *user_context, size_t size, bool retry) inline void hexagon_cache_pool_put(void *user_context, void *cache_mem) { ScopedMutexLock lock(&hexagon_cache_mutex); - halide_assert(user_context, cache_mem); + HALIDE_CHECK(user_context, cache_mem); pcache_pool temp = hexagon_cache_pool; while (temp != nullptr) { if (temp->l2memory == cache_mem) { diff --git a/src/runtime/hexagon_dma.cpp b/src/runtime/hexagon_dma.cpp index 60c12fbdd6a2..4f4c87ee7e96 100644 --- a/src/runtime/hexagon_dma.cpp +++ b/src/runtime/hexagon_dma.cpp @@ -127,7 +127,7 @@ void *desc_pool_get(void *user_context) { void desc_pool_put(void *user_context, void *desc) { ScopedMutexLock lock(&hexagon_desc_mutex); - halide_assert(user_context, desc); + HALIDE_CHECK(user_context, desc); pdesc_pool temp = dma_desc_pool; while (temp != nullptr) { if (temp->descriptor == desc) { @@ -219,25 +219,25 @@ int halide_hexagon_dma_wrapper(void *user_context, struct halide_buffer_t *src, // Assert if buffer dimensions do not fulfill the format requirements if (dev->fmt == eDmaFmt_RawData) { - halide_assert(user_context, src->dimensions <= 3); + HALIDE_CHECK(user_context, src->dimensions <= 3); } if ((dev->fmt == eDmaFmt_NV12_Y) || (dev->fmt == eDmaFmt_P010_Y) || (dev->fmt == eDmaFmt_TP10_Y) || (dev->fmt == eDmaFmt_NV124R_Y)) { - halide_assert(user_context, src->dimensions == 2); + HALIDE_CHECK(user_context, src->dimensions == 2); } if ((dev->fmt == eDmaFmt_NV12_UV) || (dev->fmt == eDmaFmt_P010_UV) || (dev->fmt == eDmaFmt_TP10_UV) || (dev->fmt == eDmaFmt_NV124R_UV)) { - halide_assert(user_context, src->dimensions == 3); - halide_assert(user_context, src->dim[0].stride == 2); - halide_assert(user_context, src->dim[2].stride == 1); - halide_assert(user_context, src->dim[2].min == 0); - halide_assert(user_context, src->dim[2].extent == 2); + HALIDE_CHECK(user_context, src->dimensions == 3); + HALIDE_CHECK(user_context, src->dim[0].stride == 2); + HALIDE_CHECK(user_context, src->dim[2].stride == 1); + HALIDE_CHECK(user_context, src->dim[2].min == 0); + HALIDE_CHECK(user_context, src->dim[2].extent == 2); } t_StDmaWrapper_RoiAlignInfo stWalkSize = { @@ -258,7 +258,7 @@ int halide_hexagon_dma_wrapper(void *user_context, struct halide_buffer_t *src, } // Assert if destination stride is a multipe of recommended stride - halide_assert(user_context, ((dst->dim[1].stride % roi_stride) == 0)); + HALIDE_CHECK(user_context, ((dst->dim[1].stride % roi_stride) == 0)); // Return nullptr if descriptor is not allocated void *desc_addr = desc_pool_get(user_context); @@ -371,7 +371,7 @@ WEAK int halide_hexagon_dma_device_malloc(void *user_context, halide_buffer_t *b } size_t size = buf->size_in_bytes(); - halide_assert(user_context, size != 0); + HALIDE_CHECK(user_context, size != 0); void *mem = halide_malloc(user_context, size); if (!mem) { @@ -409,7 +409,7 @@ WEAK int halide_hexagon_dma_allocate_engine(void *user_context, void **dma_engin debug(user_context) << "Hexagon: halide_hexagon_dma_allocate_engine (user_context: " << user_context << ")\n"; - halide_assert(user_context, dma_engine); + HALIDE_CHECK(user_context, dma_engine); debug(user_context) << " dma_allocate_dma_engine -> "; *dma_engine = halide_hexagon_allocate_dma_resource(user_context); debug(user_context) << " " << dma_engine << "\n"; @@ -426,7 +426,7 @@ WEAK int halide_hexagon_dma_deallocate_engine(void *user_context, void *dma_engi << "Hexagon: halide_hexagon_dma_deallocate_engine (user_context: " << user_context << ", dma_engine: " << dma_engine << ")\n"; - halide_assert(user_context, dma_engine); + HALIDE_CHECK(user_context, dma_engine); // Its safe to free descriptors here, even on 1st engine of multi-engines deallocation, since its called outside of pipeline // If descriptors are needed on pipeline re-entry, the pool will also re-populate @@ -445,7 +445,7 @@ WEAK int halide_hexagon_dma_deallocate_engine(void *user_context, void *dma_engi namespace { inline int dma_prepare_for_copy(void *user_context, struct halide_buffer_t *buf, void *dma_engine, bool is_ubwc, t_eDmaFmt fmt, bool is_write) { - halide_assert(user_context, dma_engine); + HALIDE_CHECK(user_context, dma_engine); dma_device_handle *dev = reinterpret_cast(buf->device); dev->dma_engine = dma_engine; dev->is_ubwc = is_ubwc; @@ -494,12 +494,12 @@ WEAK int halide_hexagon_dma_buffer_copy(void *user_context, struct halide_buffer const struct halide_device_interface_t *dst_device_interface, struct halide_buffer_t *dst) { - halide_assert(user_context, dst_device_interface == nullptr || + HALIDE_CHECK(user_context, dst_device_interface == nullptr || dst_device_interface == &hexagon_dma_device_interface); if (src->device_dirty() && src->device_interface != &hexagon_dma_device_interface) { - halide_assert(user_context, dst_device_interface == &hexagon_dma_device_interface); + HALIDE_CHECK(user_context, dst_device_interface == &hexagon_dma_device_interface); // If the source is not hexagon_dma or host memory, ask the source // device interface to copy to dst host memory first. debug(user_context) << "Hexagon: src->device_interface != &hexagon_dma_device_interface\n"; @@ -515,10 +515,10 @@ WEAK int halide_hexagon_dma_buffer_copy(void *user_context, struct halide_buffer bool from_host = !src->device_dirty() && src->host != nullptr; bool to_host = !dst_device_interface; - halide_assert(user_context, from_host || src->device); - halide_assert(user_context, to_host || dst->device); + HALIDE_CHECK(user_context, from_host || src->device); + HALIDE_CHECK(user_context, to_host || dst->device); - halide_assert(user_context, (!from_host && to_host) || (from_host && !to_host)); + HALIDE_CHECK(user_context, (!from_host && to_host) || (from_host && !to_host)); debug(user_context) << "Hexagon: halide_hexagon_dma_buffer_copy (user_context: " << user_context @@ -564,7 +564,7 @@ WEAK int halide_hexagon_dma_device_crop(void *user_context, const dma_device_handle *src_dev = (dma_device_handle *)src->device; dma_device_handle *dst_dev = malloc_device_handle(); - halide_assert(user_context, dst_dev); + HALIDE_CHECK(user_context, dst_dev); dst_dev->buffer = src_dev->buffer; dst_dev->offset_wrx = src_dev->offset_wrx + dst->dim[0].min - src->dim[0].min; dst_dev->offset_wry = src_dev->offset_wry + dst->dim[1].min - src->dim[1].min; @@ -588,7 +588,7 @@ WEAK int halide_hexagon_dma_device_slice(void *user_context, << "Hexagon: halide_hexagon_dma_device_slice (user_context: " << user_context << " src: " << *src << " dst: " << *dst << ")\n"; - halide_assert(user_context, 0); + HALIDE_CHECK(user_context, 0); error(user_context) << "Hexagon: halide_hexagon_dma_device_slice not implemented\n"; return halide_error_code_generic_error; @@ -599,7 +599,7 @@ WEAK int halide_hexagon_dma_device_release_crop(void *user_context, struct halid << "Hexagon: halide_hexagon_dma_device_release_crop (user_context: " << user_context << " buf: " << *buf << ")\n"; - halide_assert(user_context, buf->device); + HALIDE_CHECK(user_context, buf->device); free((dma_device_handle *)buf->device); buf->device = 0; @@ -620,7 +620,7 @@ WEAK int halide_hexagon_dma_device_wrap_native(void *user_context, struct halide << "Hexagon: halide_hexagon_dma_device_wrap_native (user_context: " << user_context << " buf: " << *buf << " handle: " << handle << ")\n"; - halide_assert(user_context, buf->device == 0); + HALIDE_CHECK(user_context, buf->device == 0); if (buf->device != 0) { error(user_context) << "Hexagon: halide_hexagon_dma_device_wrap_native buffer already has a device\n"; return halide_error_code_device_wrap_native_failed; @@ -630,7 +630,7 @@ WEAK int halide_hexagon_dma_device_wrap_native(void *user_context, struct halide buf->device_interface->impl->use_module(); dma_device_handle *dev = malloc_device_handle(); - halide_assert(user_context, dev); + HALIDE_CHECK(user_context, dev); dev->buffer = reinterpret_cast(handle); dev->dma_engine = nullptr; dev->frame_width = buf->dim[0].extent * buf->dim[0].stride; @@ -650,7 +650,7 @@ WEAK int halide_hexagon_dma_device_detach_native(void *user_context, struct hali error(user_context) << "Hexagon: halide_hexagon_dma_device_detach_native buffer without a device\n"; return halide_error_code_device_detach_native_failed; } - halide_assert(user_context, buf->device_interface == &hexagon_dma_device_interface); + HALIDE_CHECK(user_context, buf->device_interface == &hexagon_dma_device_interface); dma_device_handle *dev = (dma_device_handle *)buf->device; free(dev); buf->device_interface->impl->release_module(); diff --git a/src/runtime/hexagon_dma_pool.cpp b/src/runtime/hexagon_dma_pool.cpp index 3defeaa01eec..bdc1f46ffe01 100644 --- a/src/runtime/hexagon_dma_pool.cpp +++ b/src/runtime/hexagon_dma_pool.cpp @@ -43,8 +43,8 @@ namespace { // In this function we pick the dma engine and assign it to a virtual engine inline void *hexagon_dma_pool_get(void *user_context, void *virtual_engine_id) { - halide_assert(user_context, hexagon_dma_pool); - halide_assert(user_context, virtual_engine_id); + HALIDE_CHECK(user_context, hexagon_dma_pool); + HALIDE_CHECK(user_context, virtual_engine_id); ScopedMutexLock lock(&hexagon_dma_pool_mutex); hexagon_dma_virtual_engine_t *virtual_engine_addr = (hexagon_dma_virtual_engine_t *)virtual_engine_id; @@ -66,7 +66,7 @@ inline void *hexagon_dma_pool_get(void *user_context, void *virtual_engine_id) { virtual_engine_addr->mapped_engines[virtual_engine_addr->num_of_engines] = j + 1; if (!hexagon_dma_pool->dma_engine_list[j].engine_addr) { hexagon_dma_pool->dma_engine_list[j].engine_addr = (void *)hDmaWrapper_AllocDma(); - halide_assert(user_context, hexagon_dma_pool->dma_engine_list[j].engine_addr); + HALIDE_CHECK(user_context, hexagon_dma_pool->dma_engine_list[j].engine_addr); } virtual_engine_addr->num_of_engines++; return hexagon_dma_pool->dma_engine_list[j].engine_addr; @@ -79,7 +79,7 @@ inline void *hexagon_dma_pool_get(void *user_context, void *virtual_engine_id) { // In this function we simply mark the dma engine as free inline int hexagon_dma_pool_put(void *user_context, void *dma_engine, void *virtual_engine_id) { - halide_assert(user_context, virtual_engine_id); + HALIDE_CHECK(user_context, virtual_engine_id); ScopedMutexLock lock(&hexagon_dma_pool_mutex); hexagon_dma_virtual_engine_t *virtual_engine_addr = (hexagon_dma_virtual_engine_t *)virtual_engine_id; @@ -100,8 +100,8 @@ extern "C" { // halide_hexagon_free_dma_resource WEAK int halide_hexagon_free_dma_resource(void *user_context, void *virtual_engine_id) { - halide_assert(user_context, hexagon_dma_pool); - halide_assert(user_context, virtual_engine_id); + HALIDE_CHECK(user_context, hexagon_dma_pool); + HALIDE_CHECK(user_context, virtual_engine_id); // Free the Real DMA Engines int nRet = halide_error_code_success; diff --git a/src/runtime/hexagon_host.cpp b/src/runtime/hexagon_host.cpp index 00f5cfeba9a2..3ec021b0a0d9 100644 --- a/src/runtime/hexagon_host.cpp +++ b/src/runtime/hexagon_host.cpp @@ -250,7 +250,7 @@ WEAK int halide_hexagon_initialize_kernels(void *user_context, void **state_ptr, << ", code_size: " << (int)code_size << ")\n" << ", code: " << runtime << ", code_size: " << (int)runtime_size << ")\n"; - halide_assert(user_context, state_ptr != nullptr); + HALIDE_CHECK(user_context, state_ptr != nullptr); #ifdef DEBUG_RUNTIME uint64_t t_before = halide_current_time_ns(user_context); @@ -277,7 +277,7 @@ WEAK int halide_hexagon_initialize_kernels(void *user_context, void **state_ptr, poll_log(user_context); if (result == 0) { debug(user_context) << " " << (void *)(size_t)shared_runtime << "\n"; - halide_assert(user_context, shared_runtime != 0); + HALIDE_CHECK(user_context, shared_runtime != 0); } else { debug(user_context) << " " << result << "\n"; error(user_context) << "Initialization of Hexagon kernels failed\n"; @@ -389,8 +389,8 @@ WEAK int halide_hexagon_run(void *user_context, uint64_t arg_sizes[], void *args[], int arg_flags[]) { - halide_assert(user_context, state_ptr != nullptr); - halide_assert(user_context, function != nullptr); + HALIDE_CHECK(user_context, state_ptr != nullptr); + HALIDE_CHECK(user_context, function != nullptr); int result = init_hexagon_runtime(user_context); if (result != 0) { return result; @@ -546,7 +546,7 @@ WEAK int halide_hexagon_device_malloc(void *user_context, halide_buffer_t *buf) } size_t size = buf->size_in_bytes(); - halide_assert(user_context, size != 0); + HALIDE_CHECK(user_context, size != 0); // Hexagon code generation generates clamped ramp loads in a way // that requires up to an extra vector beyond the end of the @@ -554,7 +554,7 @@ WEAK int halide_hexagon_device_malloc(void *user_context, halide_buffer_t *buf) size += 128; for (int i = 0; i < buf->dimensions; i++) { - halide_assert(user_context, buf->dim[i].stride >= 0); + HALIDE_CHECK(user_context, buf->dim[i].stride >= 0); } debug(user_context) << " allocating buffer of " << (uint64_t)size << " bytes\n"; @@ -657,7 +657,7 @@ WEAK int halide_hexagon_copy_to_device(void *user_context, halide_buffer_t *buf) uint64_t t_before = halide_current_time_ns(user_context); #endif - halide_assert(user_context, buf->host && buf->device); + HALIDE_CHECK(user_context, buf->host && buf->device); device_copy c = make_host_to_device_copy(buf); // Get the descriptor associated with the ion buffer. @@ -681,7 +681,7 @@ WEAK int halide_hexagon_copy_to_host(void *user_context, struct halide_buffer_t uint64_t t_before = halide_current_time_ns(user_context); #endif - halide_assert(user_context, buf->host && buf->device); + HALIDE_CHECK(user_context, buf->host && buf->device); device_copy c = make_device_to_host_copy(buf); // Get the descriptor associated with the ion buffer. @@ -705,7 +705,7 @@ WEAK int halide_hexagon_device_sync(void *user_context, struct halide_buffer_t * WEAK int halide_hexagon_wrap_device_handle(void *user_context, struct halide_buffer_t *buf, void *ion_buf, uint64_t size) { - halide_assert(user_context, buf->device == 0); + HALIDE_CHECK(user_context, buf->device == 0); if (buf->device != 0) { return -2; } @@ -726,7 +726,7 @@ WEAK int halide_hexagon_detach_device_handle(void *user_context, struct halide_b if (buf->device == 0) { return 0; } - halide_assert(user_context, buf->device_interface == &hexagon_device_interface); + HALIDE_CHECK(user_context, buf->device_interface == &hexagon_device_interface); ion_device_handle *handle = uint64_to_ptr(buf->device); free(handle); @@ -740,7 +740,7 @@ WEAK void *halide_hexagon_get_device_handle(void *user_context, struct halide_bu if (buf->device == 0) { return nullptr; } - halide_assert(user_context, buf->device_interface == &hexagon_device_interface); + HALIDE_CHECK(user_context, buf->device_interface == &hexagon_device_interface); ion_device_handle *handle = uint64_to_ptr(buf->device); return handle->buffer; } @@ -749,7 +749,7 @@ WEAK uint64_t halide_hexagon_get_device_size(void *user_context, struct halide_b if (buf->device == 0) { return 0; } - halide_assert(user_context, buf->device_interface == &hexagon_device_interface); + HALIDE_CHECK(user_context, buf->device_interface == &hexagon_device_interface); ion_device_handle *handle = uint64_to_ptr(buf->device); return handle->size; } @@ -774,12 +774,12 @@ WEAK int halide_hexagon_buffer_copy(void *user_context, struct halide_buffer_t * const struct halide_device_interface_t *dst_device_interface, struct halide_buffer_t *dst) { // We only handle copies to hexagon buffers or to host - halide_assert(user_context, dst_device_interface == nullptr || + HALIDE_CHECK(user_context, dst_device_interface == nullptr || dst_device_interface == &hexagon_device_interface); if ((src->device_dirty() || src->host == nullptr) && src->device_interface != &hexagon_device_interface) { - halide_assert(user_context, dst_device_interface == &hexagon_device_interface); + HALIDE_CHECK(user_context, dst_device_interface == &hexagon_device_interface); // This is handled at the higher level. return halide_error_code_incompatible_device_interface; } @@ -789,8 +789,8 @@ WEAK int halide_hexagon_buffer_copy(void *user_context, struct halide_buffer_t * (src->host_dirty() && src->host != nullptr); bool to_host = !dst_device_interface; - halide_assert(user_context, from_host || src->device); - halide_assert(user_context, to_host || dst->device); + HALIDE_CHECK(user_context, from_host || src->device); + HALIDE_CHECK(user_context, to_host || dst->device); #ifdef DEBUG_RUNTIME uint64_t t_before = halide_current_time_ns(user_context); diff --git a/src/runtime/metal.cpp b/src/runtime/metal.cpp index 9bbe32b512fe..3d02b2433de4 100644 --- a/src/runtime/metal.cpp +++ b/src/runtime/metal.cpp @@ -340,7 +340,7 @@ extern "C" { // previous call (if any) has not yet been released via halide_release_metal_context. WEAK int halide_metal_acquire_context(void *user_context, mtl_device **device_ret, mtl_command_queue **queue_ret, bool create) { - halide_assert(user_context, &thread_lock != nullptr); + HALIDE_CHECK(user_context, &thread_lock != nullptr); while (__atomic_test_and_set(&thread_lock, __ATOMIC_ACQUIRE)) { } @@ -369,7 +369,7 @@ WEAK int halide_metal_acquire_context(void *user_context, mtl_device **device_re // If the device has already been initialized, // ensure the queue has as well. - halide_assert(user_context, (device == nullptr) || (queue != nullptr)); + HALIDE_CHECK(user_context, (device == nullptr) || (queue != nullptr)); *device_ret = device; *queue_ret = queue; @@ -452,7 +452,7 @@ WEAK int halide_metal_device_malloc(void *user_context, halide_buffer_t *buf) { << ", buf: " << buf << ")\n"; size_t size = buf->size_in_bytes(); - halide_assert(user_context, size != 0); + HALIDE_CHECK(user_context, size != 0); if (buf->device) { // This buffer already has a device allocation return 0; @@ -460,7 +460,7 @@ WEAK int halide_metal_device_malloc(void *user_context, halide_buffer_t *buf) { // Check all strides positive for (int i = 0; i < buf->dimensions; i++) { - halide_assert(user_context, buf->dim[i].stride >= 0); + HALIDE_CHECK(user_context, buf->dim[i].stride >= 0); } debug(user_context) << " allocating " << *buf << "\n"; @@ -513,7 +513,7 @@ WEAK int halide_metal_device_free(void *user_context, halide_buffer_t *buf) { #endif device_handle *handle = (device_handle *)buf->device; - halide_assert(user_context, (((device_handle *)buf->device)->offset == 0) && "halide_metal_device_free on buffer obtained from halide_device_crop"); + HALIDE_CHECK(user_context, (((device_handle *)buf->device)->offset == 0) && "halide_metal_device_free on buffer obtained from halide_device_crop"); release_ns_object(handle->buf); free(handle); @@ -545,7 +545,7 @@ WEAK int halide_metal_initialize_kernels(void *user_context, void **state_ptr, c source, source_size)) { return halide_error_code_generic_error; } - halide_assert(user_context, library != nullptr); + HALIDE_CHECK(user_context, library != nullptr); #ifdef DEBUG_RUNTIME uint64_t t_after = halide_current_time_ns(user_context); @@ -645,7 +645,7 @@ WEAK int halide_metal_copy_to_device(void *user_context, halide_buffer_t *buffer return metal_context.error; } - halide_assert(user_context, buffer->host && buffer->device); + HALIDE_CHECK(user_context, buffer->host && buffer->device); device_copy c = make_host_to_device_copy(buffer); mtl_buffer *metal_buffer = ((device_handle *)c.dst)->buf; @@ -659,7 +659,7 @@ WEAK int halide_metal_copy_to_device(void *user_context, halide_buffer_t *buffer if (is_buffer_managed(metal_buffer)) { size_t total_size = buffer->size_in_bytes(); - halide_assert(user_context, total_size != 0); + HALIDE_CHECK(user_context, total_size != 0); NSRange total_extent; total_extent.location = 0; total_extent.length = total_size; @@ -688,8 +688,8 @@ WEAK int halide_metal_copy_to_host(void *user_context, halide_buffer_t *buffer) halide_metal_device_sync_internal(metal_context.queue, buffer); - halide_assert(user_context, buffer->host && buffer->device); - halide_assert(user_context, buffer->dimensions <= MAX_COPY_DIMS); + HALIDE_CHECK(user_context, buffer->host && buffer->device); + HALIDE_CHECK(user_context, buffer->dimensions <= MAX_COPY_DIMS); if (buffer->dimensions > MAX_COPY_DIMS) { return -1; } @@ -739,7 +739,7 @@ WEAK int halide_metal_run(void *user_context, mtl_library *library{}; bool found = compilation_cache.lookup(metal_context.device, state_ptr, library); - halide_assert(user_context, found && library != nullptr); + HALIDE_CHECK(user_context, found && library != nullptr); mtl_function *function = new_function_with_name(library, entry_name, strlen(entry_name)); if (function == nullptr) { @@ -775,7 +775,7 @@ WEAK int halide_metal_run(void *user_context, // TODO(zalman): This seems fishy - if the arguments are // not already sorted in decreasing order of size, wrong // results occur. To repro, remove the sorting code in CodeGen_GPU_Host - halide_assert(user_context, (arg_sizes[i] & (arg_sizes[i] - 1)) == 0); + HALIDE_CHECK(user_context, (arg_sizes[i] & (arg_sizes[i] - 1)) == 0); total_args_size = (total_args_size + arg_sizes[i] - 1) & ~(arg_sizes[i] - 1); total_args_size += arg_sizes[i]; } @@ -799,7 +799,7 @@ WEAK int halide_metal_run(void *user_context, // in the struct, per email communication from Apple size_t padded_args_size = (total_args_size + 4 - 1) & ~((size_t)(4 - 1)); debug(user_context) << "Total args size is " << (uint64_t)total_args_size << " and with padding, size is " << (uint64_t)padded_args_size << "\n"; - halide_assert(user_context, padded_args_size >= total_args_size); + HALIDE_CHECK(user_context, padded_args_size >= total_args_size); if (padded_args_size < 4096 && metal_api_supports_set_bytes) { args_ptr = (char *)small_args_buffer; @@ -820,7 +820,7 @@ WEAK int halide_metal_run(void *user_context, offset += arg_sizes[i]; } } - halide_assert(user_context, offset == total_args_size); + HALIDE_CHECK(user_context, offset == total_args_size); if (total_args_size < 4096 && metal_api_supports_set_bytes) { set_input_buffer_from_bytes(encoder, small_args_buffer, padded_args_size, buffer_index); @@ -833,7 +833,7 @@ WEAK int halide_metal_run(void *user_context, for (size_t i = 0; arg_sizes[i] != 0; i++) { if (arg_is_buffer[i]) { - halide_assert(user_context, arg_sizes[i] == sizeof(uint64_t)); + HALIDE_CHECK(user_context, arg_sizes[i] == sizeof(uint64_t)); device_handle *handle = (device_handle *)((halide_buffer_t *)args[i])->device; set_input_buffer(encoder, handle->buf, handle->offset, buffer_index); buffer_index++; @@ -901,12 +901,12 @@ WEAK int halide_metal_buffer_copy(void *user_context, struct halide_buffer_t *sr } // We only handle copies to metal buffers or to host - halide_assert(user_context, dst_device_interface == nullptr || + HALIDE_CHECK(user_context, dst_device_interface == nullptr || dst_device_interface == &metal_device_interface); if ((src->device_dirty() || src->host == nullptr) && src->device_interface != &metal_device_interface) { - halide_assert(user_context, dst_device_interface == &metal_device_interface); + HALIDE_CHECK(user_context, dst_device_interface == &metal_device_interface); // This is handled at the higher level. return halide_error_code_incompatible_device_interface; } @@ -916,8 +916,8 @@ WEAK int halide_metal_buffer_copy(void *user_context, struct halide_buffer_t *sr (src->host_dirty() && src->host != nullptr); bool to_host = !dst_device_interface; - halide_assert(user_context, from_host || src->device); - halide_assert(user_context, to_host || dst->device); + HALIDE_CHECK(user_context, from_host || src->device); + HALIDE_CHECK(user_context, to_host || dst->device); device_copy c = make_buffer_copy(src, from_host, dst, to_host); @@ -962,7 +962,7 @@ WEAK int halide_metal_buffer_copy(void *user_context, struct halide_buffer_t *sr halide_metal_device_sync_internal(metal_context.queue, dst); dst_buffer = ((device_handle *)c.dst)->buf; - halide_assert(user_context, from_host); + HALIDE_CHECK(user_context, from_host); c.dst = (uint64_t)buffer_contents(dst_buffer) + ((device_handle *)c.dst)->offset; } @@ -971,7 +971,7 @@ WEAK int halide_metal_buffer_copy(void *user_context, struct halide_buffer_t *sr if (!to_host) { if (is_buffer_managed(dst_buffer)) { size_t total_size = dst->size_in_bytes(); - halide_assert(user_context, total_size != 0); + HALIDE_CHECK(user_context, total_size != 0); NSRange total_extent; total_extent.location = 0; total_extent.length = total_size; @@ -1062,7 +1062,7 @@ WEAK int halide_metal_device_release_crop(void *user_context, } WEAK int halide_metal_wrap_buffer(void *user_context, struct halide_buffer_t *buf, uint64_t buffer) { - halide_assert(user_context, buf->device == 0); + HALIDE_CHECK(user_context, buf->device == 0); if (buf->device != 0) { return -2; } @@ -1084,7 +1084,7 @@ WEAK int halide_metal_detach_buffer(void *user_context, struct halide_buffer_t * if (buf->device == 0) { return 0; } - halide_assert(user_context, buf->device_interface == &metal_device_interface); + HALIDE_CHECK(user_context, buf->device_interface == &metal_device_interface); buf->device_interface->impl->release_module(); buf->device_interface = nullptr; free((device_handle *)buf->device); @@ -1096,7 +1096,7 @@ WEAK uintptr_t halide_metal_get_buffer(void *user_context, struct halide_buffer_ if (buf->device == 0) { return 0; } - halide_assert(user_context, buf->device_interface == &metal_device_interface); + HALIDE_CHECK(user_context, buf->device_interface == &metal_device_interface); return (uintptr_t)(((device_handle *)buf->device)->buf); } @@ -1104,7 +1104,7 @@ WEAK uint64_t halide_metal_get_crop_offset(void *user_context, struct halide_buf if (buf->device == 0) { return 0; } - halide_assert(user_context, buf->device_interface == &metal_device_interface); + HALIDE_CHECK(user_context, buf->device_interface == &metal_device_interface); return (uint64_t)(((device_handle *)buf->device)->offset); } diff --git a/src/runtime/opencl.cpp b/src/runtime/opencl.cpp index a281a5ee3106..67fc0f2da9bd 100644 --- a/src/runtime/opencl.cpp +++ b/src/runtime/opencl.cpp @@ -64,7 +64,7 @@ ALWAYS_INLINE T get_cl_symbol(void *user_context, const char *name, bool req) { // Load an OpenCL shared object/dll, and get the function pointers for the OpenCL API from it. WEAK void load_libopencl(void *user_context) { debug(user_context) << " load_libopencl (user_context: " << user_context << ")\n"; - halide_assert(user_context, clCreateContext == nullptr); + HALIDE_CHECK(user_context, clCreateContext == nullptr); // clang-format off #define CL_FN(ret, fn, args) fn = get_cl_symbol(user_context, #fn, true); // NOLINT(bugprone-macro-parentheses) @@ -212,16 +212,16 @@ WEAK const char *halide_opencl_get_build_options(void *user_context) { WEAK int halide_acquire_cl_context(void *user_context, cl_context *ctx, cl_command_queue *q, bool create = true) { // TODO: Should we use a more "assertive" assert? These asserts do // not block execution on failure. - halide_assert(user_context, ctx != nullptr); - halide_assert(user_context, q != nullptr); + HALIDE_CHECK(user_context, ctx != nullptr); + HALIDE_CHECK(user_context, q != nullptr); - halide_assert(user_context, &thread_lock != nullptr); + HALIDE_CHECK(user_context, &thread_lock != nullptr); while (__atomic_test_and_set(&thread_lock, __ATOMIC_ACQUIRE)) { } // If the context has not been initialized, initialize it now. - halide_assert(user_context, &context != nullptr); - halide_assert(user_context, &command_queue != nullptr); + HALIDE_CHECK(user_context, &context != nullptr); + HALIDE_CHECK(user_context, &command_queue != nullptr); if (!context && create) { cl_int error = create_opencl_context(user_context, &context, &command_queue); if (error != CL_SUCCESS) { @@ -325,7 +325,7 @@ WEAK bool validate_device_pointer(void *user_context, halide_buffer_t *buf, size << ", actual allocated " << (uint64_t)real_size << "\n"; if (size) { - halide_assert(user_context, real_size >= (size + offset) && "Validating pointer with insufficient size"); + HALIDE_CHECK(user_context, real_size >= (size + offset) && "Validating pointer with insufficient size"); } return true; } @@ -336,8 +336,8 @@ WEAK int create_opencl_context(void *user_context, cl_context *ctx, cl_command_q debug(user_context) << " create_opencl_context (user_context: " << user_context << ")\n"; - halide_assert(user_context, ctx != nullptr && *ctx == nullptr); - halide_assert(user_context, q != nullptr && *q == nullptr); + HALIDE_CHECK(user_context, ctx != nullptr && *ctx == nullptr); + HALIDE_CHECK(user_context, q != nullptr && *q == nullptr); if (clGetPlatformIDs == nullptr) { error(user_context) << "CL: clGetPlatformIDs not found\n"; @@ -650,7 +650,7 @@ WEAK int halide_opencl_device_free(void *user_context, halide_buffer_t *buf) { } cl_mem dev_ptr = ((device_handle *)buf->device)->mem; - halide_assert(user_context, (((device_handle *)buf->device)->offset == 0) && "halide_opencl_device_free on buffer obtained from halide_device_crop"); + HALIDE_CHECK(user_context, (((device_handle *)buf->device)->offset == 0) && "halide_opencl_device_free on buffer obtained from halide_device_crop"); debug(user_context) << "CL: halide_opencl_device_free (user_context: " << user_context @@ -665,7 +665,7 @@ WEAK int halide_opencl_device_free(void *user_context, halide_buffer_t *buf) { uint64_t t_before = halide_current_time_ns(user_context); #endif - halide_assert(user_context, validate_device_pointer(user_context, buf)); + HALIDE_CHECK(user_context, validate_device_pointer(user_context, buf)); debug(user_context) << " clReleaseMemObject " << (void *)dev_ptr << "\n"; cl_int result = clReleaseMemObject((cl_mem)dev_ptr); // If clReleaseMemObject fails, it is unlikely to succeed in a later call, so @@ -759,7 +759,7 @@ WEAK int halide_opencl_initialize_kernels(void *user_context, void **state_ptr, compile_kernel, user_context, ctx.context, src, size)) { return halide_error_code_generic_error; } - halide_assert(user_context, program != nullptr); + HALIDE_CHECK(user_context, program != nullptr); #ifdef DEBUG_RUNTIME uint64_t t_after = halide_current_time_ns(user_context); @@ -822,7 +822,7 @@ WEAK int halide_opencl_device_release(void *user_context) { if (ctx) { err = clFinish(q); - halide_assert(user_context, err == CL_SUCCESS); + HALIDE_CHECK(user_context, err == CL_SUCCESS); compilation_cache.delete_context(user_context, ctx, clReleaseProgram); @@ -830,12 +830,12 @@ WEAK int halide_opencl_device_release(void *user_context) { if (ctx == context) { debug(user_context) << " clReleaseCommandQueue " << command_queue << "\n"; err = clReleaseCommandQueue(command_queue); - halide_assert(user_context, err == CL_SUCCESS); + HALIDE_CHECK(user_context, err == CL_SUCCESS); command_queue = nullptr; debug(user_context) << " clReleaseContext " << context << "\n"; err = clReleaseContext(context); - halide_assert(user_context, err == CL_SUCCESS); + HALIDE_CHECK(user_context, err == CL_SUCCESS); context = nullptr; } } @@ -856,14 +856,14 @@ WEAK int halide_opencl_device_malloc(void *user_context, halide_buffer_t *buf) { } size_t size = buf->size_in_bytes(); - halide_assert(user_context, size != 0); + HALIDE_CHECK(user_context, size != 0); if (buf->device) { - halide_assert(user_context, validate_device_pointer(user_context, buf, size)); + HALIDE_CHECK(user_context, validate_device_pointer(user_context, buf, size)); return 0; } for (int i = 0; i < buf->dimensions; i++) { - halide_assert(user_context, buf->dim[i].stride >= 0); + HALIDE_CHECK(user_context, buf->dim[i].stride >= 0); } debug(user_context) << " allocating " << *buf << "\n"; @@ -900,7 +900,7 @@ WEAK int halide_opencl_device_malloc(void *user_context, halide_buffer_t *buf) { << " Allocated device buffer " << (void *)buf->device << " for buffer " << buf << "\n"; - halide_assert(user_context, validate_device_pointer(user_context, buf, size)); + HALIDE_CHECK(user_context, validate_device_pointer(user_context, buf, size)); #ifdef DEBUG_RUNTIME uint64_t t_after = halide_current_time_ns(user_context); @@ -970,12 +970,12 @@ WEAK int halide_opencl_buffer_copy(void *user_context, struct halide_buffer_t *s const struct halide_device_interface_t *dst_device_interface, struct halide_buffer_t *dst) { // We only handle copies to opencl or to host - halide_assert(user_context, dst_device_interface == nullptr || + HALIDE_CHECK(user_context, dst_device_interface == nullptr || dst_device_interface == &opencl_device_interface); if ((src->device_dirty() || src->host == nullptr) && src->device_interface != &opencl_device_interface) { - halide_assert(user_context, dst_device_interface == &opencl_device_interface); + HALIDE_CHECK(user_context, dst_device_interface == &opencl_device_interface); // This is handled at the higher level. return halide_error_code_incompatible_device_interface; } @@ -985,8 +985,8 @@ WEAK int halide_opencl_buffer_copy(void *user_context, struct halide_buffer_t *s (src->host_dirty() && src->host != nullptr); bool to_host = !dst_device_interface; - halide_assert(user_context, from_host || src->device); - halide_assert(user_context, to_host || dst->device); + HALIDE_CHECK(user_context, from_host || src->device); + HALIDE_CHECK(user_context, to_host || dst->device); device_copy c = make_buffer_copy(src, from_host, dst, to_host); @@ -1004,10 +1004,10 @@ WEAK int halide_opencl_buffer_copy(void *user_context, struct halide_buffer_t *s #ifdef DEBUG_RUNTIME uint64_t t_before = halide_current_time_ns(user_context); if (!from_host) { - halide_assert(user_context, validate_device_pointer(user_context, src)); + HALIDE_CHECK(user_context, validate_device_pointer(user_context, src)); } if (!to_host) { - halide_assert(user_context, validate_device_pointer(user_context, dst)); + HALIDE_CHECK(user_context, validate_device_pointer(user_context, dst)); } #endif @@ -1062,11 +1062,11 @@ WEAK int halide_opencl_run(void *user_context, #endif // Create kernel object for entry_name from the program for this module. - halide_assert(user_context, state_ptr); + HALIDE_CHECK(user_context, state_ptr); cl_program program{}; bool found = compilation_cache.lookup(ctx.context, state_ptr, program); - halide_assert(user_context, found && program != nullptr); + HALIDE_CHECK(user_context, found && program != nullptr); debug(user_context) << " clCreateKernel " << entry_name << " -> "; cl_kernel f = clCreateKernel(program, entry_name, &err); @@ -1118,7 +1118,7 @@ WEAK int halide_opencl_run(void *user_context, cl_int err = CL_SUCCESS; if (arg_is_buffer[i]) { - halide_assert(user_context, arg_sizes[i] == sizeof(uint64_t)); + HALIDE_CHECK(user_context, arg_sizes[i] == sizeof(uint64_t)); cl_mem mem = ((device_handle *)((halide_buffer_t *)this_arg)->device)->mem; uint64_t offset = ((device_handle *)((halide_buffer_t *)this_arg)->device)->offset; @@ -1209,7 +1209,7 @@ WEAK int halide_opencl_device_and_host_free(void *user_context, struct halide_bu } WEAK int halide_opencl_wrap_cl_mem(void *user_context, struct halide_buffer_t *buf, uint64_t mem) { - halide_assert(user_context, buf->device == 0); + HALIDE_CHECK(user_context, buf->device == 0); if (buf->device != 0) { return -2; } @@ -1238,7 +1238,7 @@ WEAK int halide_opencl_detach_cl_mem(void *user_context, halide_buffer_t *buf) { if (buf->device == 0) { return 0; } - halide_assert(user_context, buf->device_interface == &opencl_device_interface || + HALIDE_CHECK(user_context, buf->device_interface == &opencl_device_interface || buf->device_interface == &opencl_image_device_interface); free((device_handle *)buf->device); buf->device = 0; @@ -1251,7 +1251,7 @@ WEAK uintptr_t halide_opencl_get_cl_mem(void *user_context, halide_buffer_t *buf if (buf->device == 0) { return 0; } - halide_assert(user_context, buf->device_interface == &opencl_device_interface || + HALIDE_CHECK(user_context, buf->device_interface == &opencl_device_interface || buf->device_interface == &opencl_image_device_interface); return (uintptr_t)((device_handle *)buf->device)->mem; } @@ -1260,7 +1260,7 @@ WEAK uint64_t halide_opencl_get_crop_offset(void *user_context, halide_buffer_t if (buf->device == 0) { return 0; } - halide_assert(user_context, buf->device_interface == &opencl_device_interface); + HALIDE_CHECK(user_context, buf->device_interface == &opencl_device_interface); return ((device_handle *)buf->device)->offset; } @@ -1329,7 +1329,7 @@ WEAK int halide_opencl_device_release_crop(void *user_context, uint64_t t_before = halide_current_time_ns(user_context); #endif - halide_assert(user_context, validate_device_pointer(user_context, buf)); + HALIDE_CHECK(user_context, validate_device_pointer(user_context, buf)); debug(user_context) << " clReleaseMemObject " << (void *)dev_ptr << "\n"; // Sub-buffers are released with clReleaseMemObject cl_int result = clReleaseMemObject((cl_mem)dev_ptr); @@ -1545,14 +1545,14 @@ WEAK int halide_opencl_image_device_malloc(void *user_context, halide_buffer_t * } size_t size = buf->size_in_bytes(); - halide_assert(user_context, size != 0); + HALIDE_CHECK(user_context, size != 0); if (buf->device) { - halide_assert(user_context, validate_device_pointer(user_context, buf, size)); + HALIDE_CHECK(user_context, validate_device_pointer(user_context, buf, size)); return 0; } for (int i = 0; i < buf->dimensions; i++) { - halide_assert(user_context, buf->dim[i].stride >= 0); + HALIDE_CHECK(user_context, buf->dim[i].stride >= 0); } debug(user_context) << " allocating " << *buf << "\n"; @@ -1664,7 +1664,7 @@ WEAK int halide_opencl_image_device_malloc(void *user_context, halide_buffer_t * << " Allocated device buffer " << (void *)buf->device << " for buffer " << buf << "\n"; - halide_assert(user_context, validate_device_pointer(user_context, buf, size)); + HALIDE_CHECK(user_context, validate_device_pointer(user_context, buf, size)); #ifdef DEBUG_RUNTIME uint64_t t_after = halide_current_time_ns(user_context); @@ -1682,12 +1682,12 @@ WEAK int halide_opencl_image_buffer_copy(void *user_context, struct halide_buffe << "CL: halide_opencl_image_buffer_copy (user_context: " << user_context << ", src: " << src << ", dst: " << dst << ")\n"; - halide_assert(user_context, dst_device_interface == nullptr || + HALIDE_CHECK(user_context, dst_device_interface == nullptr || dst_device_interface == &opencl_image_device_interface); if ((src->device_dirty() || src->host == nullptr) && src->device_interface != &opencl_image_device_interface) { - halide_assert(user_context, dst_device_interface == &opencl_image_device_interface); + HALIDE_CHECK(user_context, dst_device_interface == &opencl_image_device_interface); // This is handled at the higher level. return halide_error_code_incompatible_device_interface; } @@ -1697,8 +1697,8 @@ WEAK int halide_opencl_image_buffer_copy(void *user_context, struct halide_buffe (src->host_dirty() && src->host != nullptr); bool to_host = !dst_device_interface; - halide_assert(user_context, from_host || src->device); - halide_assert(user_context, to_host || dst->device); + HALIDE_CHECK(user_context, from_host || src->device); + HALIDE_CHECK(user_context, to_host || dst->device); device_copy c = make_buffer_copy(src, from_host, dst, to_host); @@ -1712,10 +1712,10 @@ WEAK int halide_opencl_image_buffer_copy(void *user_context, struct halide_buffe #ifdef DEBUG_RUNTIME uint64_t t_before = halide_current_time_ns(user_context); if (!from_host) { - halide_assert(user_context, validate_device_pointer(user_context, src)); + HALIDE_CHECK(user_context, validate_device_pointer(user_context, src)); } if (!to_host) { - halide_assert(user_context, validate_device_pointer(user_context, dst)); + HALIDE_CHECK(user_context, validate_device_pointer(user_context, dst)); } #endif @@ -1811,7 +1811,7 @@ WEAK int halide_opencl_image_device_and_host_free(void *user_context, struct hal } WEAK int halide_opencl_image_wrap_cl_mem(void *user_context, struct halide_buffer_t *buf, uint64_t mem) { - halide_assert(user_context, buf->device == 0); + HALIDE_CHECK(user_context, buf->device == 0); if (buf->device != 0) { return -2; } diff --git a/src/runtime/openglcompute.cpp b/src/runtime/openglcompute.cpp index 99707ab02888..77b411d39c2d 100644 --- a/src/runtime/openglcompute.cpp +++ b/src/runtime/openglcompute.cpp @@ -268,7 +268,7 @@ WEAK int halide_openglcompute_device_malloc(void *user_context, halide_buffer_t } size_t size = buf->size_in_bytes(); - halide_assert(user_context, size != 0); + HALIDE_CHECK(user_context, size != 0); if (buf->device) { // This buffer already has a device allocation @@ -278,7 +278,7 @@ WEAK int halide_openglcompute_device_malloc(void *user_context, halide_buffer_t } for (int i = 0; i < buf->dimensions; i++) { - halide_assert(user_context, buf->dim[i].stride >= 0); + HALIDE_CHECK(user_context, buf->dim[i].stride >= 0); } debug(user_context) << " allocating buffer, " @@ -314,7 +314,7 @@ WEAK int halide_openglcompute_device_malloc(void *user_context, halide_buffer_t // types, all of which are 4 bytes. We'll inflate the size for // smaller types. size *= (4 / buf->type.bytes()); - halide_assert(user_context, size != 0); + HALIDE_CHECK(user_context, size != 0); global_state.BufferData(GL_ARRAY_BUFFER, size, nullptr, GL_DYNAMIC_COPY); if (global_state.CheckAndReportError(user_context, "oglc: BufferData")) { return 1; @@ -499,7 +499,7 @@ WEAK int halide_openglcompute_copy_to_host(void *user_context, halide_buffer_t * GLuint the_buffer = (GLuint)buf->device; size_t size = buf->size_in_bytes(); - halide_assert(user_context, size != 0); + HALIDE_CHECK(user_context, size != 0); debug(user_context) << "OGLC: halide_openglcompute_copy_to_host (" << "user_context: " << user_context diff --git a/src/runtime/profiler.cpp b/src/runtime/profiler.cpp index bcf1b14ef3dc..1974e18cad88 100644 --- a/src/runtime/profiler.cpp +++ b/src/runtime/profiler.cpp @@ -204,7 +204,7 @@ WEAK void halide_profiler_stack_peak_update(void *user_context, void *pipeline_state, uint64_t *f_values) { halide_profiler_pipeline_stats *p_stats = (halide_profiler_pipeline_stats *)pipeline_state; - halide_assert(user_context, p_stats != nullptr); + HALIDE_CHECK(user_context, p_stats != nullptr); // Note: Update to the counter is done without grabbing the state's lock to // reduce lock contention. One potential issue is that other call that frees the @@ -231,9 +231,9 @@ WEAK void halide_profiler_memory_allocate(void *user_context, } halide_profiler_pipeline_stats *p_stats = (halide_profiler_pipeline_stats *)pipeline_state; - halide_assert(user_context, p_stats != nullptr); - halide_assert(user_context, func_id >= 0); - halide_assert(user_context, func_id < p_stats->num_funcs); + HALIDE_CHECK(user_context, p_stats != nullptr); + HALIDE_CHECK(user_context, func_id >= 0); + HALIDE_CHECK(user_context, func_id < p_stats->num_funcs); halide_profiler_func_stats *f_stats = &p_stats->funcs[func_id]; @@ -267,9 +267,9 @@ WEAK void halide_profiler_memory_free(void *user_context, } halide_profiler_pipeline_stats *p_stats = (halide_profiler_pipeline_stats *)pipeline_state; - halide_assert(user_context, p_stats != nullptr); - halide_assert(user_context, func_id >= 0); - halide_assert(user_context, func_id < p_stats->num_funcs); + HALIDE_CHECK(user_context, p_stats != nullptr); + HALIDE_CHECK(user_context, func_id >= 0); + HALIDE_CHECK(user_context, func_id < p_stats->num_funcs); halide_profiler_func_stats *f_stats = &p_stats->funcs[func_id]; diff --git a/src/runtime/runtime_internal.h b/src/runtime/runtime_internal.h index f0f040979fe7..f75ce4c54ce5 100644 --- a/src/runtime/runtime_internal.h +++ b/src/runtime/runtime_internal.h @@ -227,15 +227,18 @@ using namespace Halide::Runtime::Internal; /** A macro that calls halide_print if the supplied condition is * false, then aborts. Used for unrecoverable errors, or - * should-never-happen errors. */ -#define _halide_stringify(x) #x -#define _halide_expand_and_stringify(x) _halide_stringify(x) -#define halide_assert(user_context, cond) \ - do { \ - if (!(cond)) { \ - halide_print(user_context, __FILE__ ":" _halide_expand_and_stringify(__LINE__) " Assert failed: " #cond "\n"); \ - abort(); \ - } \ + * should-never-happen errors. + * + * Note that this is *NOT* a debug-only macro; + * the condition will be checked in *all* build modes! */ +#define _HALIDE_CHECK_STRINGIFY(x) #x +#define _HALIDE_CHECK_EXPAND_AND_STRINGIFY(x) _HALIDE_CHECK_STRINGIFY(x) +#define HALIDE_CHECK(user_context, cond) \ + do { \ + if (!(cond)) { \ + halide_print(user_context, __FILE__ ":" _HALIDE_CHECK_EXPAND_AND_STRINGIFY(__LINE__) " HALIDE_CHECK failed: " #cond "\n"); \ + abort(); \ + } \ } while (0) #endif diff --git a/src/runtime/synchronization_common.h b/src/runtime/synchronization_common.h index f4063e824e8b..70334949785b 100644 --- a/src/runtime/synchronization_common.h +++ b/src/runtime/synchronization_common.h @@ -384,7 +384,7 @@ WEAK void word_lock::unlock_full() { int times_through = 0; while (tail == nullptr) { word_lock_queue_data *next = current->next; - halide_assert(nullptr, next != nullptr); + HALIDE_CHECK(nullptr, next != nullptr); next->prev = current; current = next; tail = current->tail; @@ -491,7 +491,7 @@ static ALWAYS_INLINE uintptr_t addr_hash(uintptr_t addr) { #ifdef DEBUG_RUNTIME // Any hash calculated by addr_hash() should be incapable of being outside this range. ALWAYS_INLINE void check_hash(uintptr_t hash) { - halide_assert(nullptr, hash < HASH_TABLE_SIZE); + HALIDE_CHECK(nullptr, hash < HASH_TABLE_SIZE); } #endif // DEBUG_RUNTIME @@ -1004,7 +1004,7 @@ class fast_cond { // TODO: this is debug only. uintptr_t val; atomic_load_relaxed((uintptr_t *)mutex, &val); - halide_assert(nullptr, val & 0x1); + HALIDE_CHECK(nullptr, val & 0x1); if_tsan_post_lock(mutex); } diff --git a/src/runtime/thread_pool_common.h b/src/runtime/thread_pool_common.h index 657d473a2523..c73e2a1eb335 100644 --- a/src/runtime/thread_pool_common.h +++ b/src/runtime/thread_pool_common.h @@ -152,7 +152,7 @@ struct work_queue_t { while (bytes < limit && *bytes == 0) { bytes++; } - halide_assert(nullptr, bytes == limit && "Logic error in thread pool work queue initialization.\n"); + HALIDE_CHECK(nullptr, bytes == limit && "Logic error in thread pool work queue initialization.\n"); } // Return the work queue to initial state. Must be called while locked @@ -522,7 +522,7 @@ WEAK void enqueue_work_already_locked(int num_jobs, work *jobs, work *task_paren } } else { log_message("enqueue_work_already_locked job " << jobs[0].task.name << " with min_threads " << min_threads << " task_parent " << task_parent->task.name << " task_parent->task.min_threads " << task_parent->task.min_threads << " task_parent->threads_reserved " << task_parent->threads_reserved); - halide_assert(nullptr, (min_threads <= ((task_parent->task.min_threads * task_parent->active_workers) - + HALIDE_CHECK(nullptr, (min_threads <= ((task_parent->task.min_threads * task_parent->active_workers) - task_parent->threads_reserved)) && "Logic error: thread over commit.\n"); if (job_has_acquires || job_may_block) { diff --git a/src/runtime/tracing.cpp b/src/runtime/tracing.cpp index c465ef172972..a171f4217730 100644 --- a/src/runtime/tracing.cpp +++ b/src/runtime/tracing.cpp @@ -84,7 +84,7 @@ class TraceBuffer { // packet. Returns nullptr if the buffer was full. ALWAYS_INLINE halide_trace_packet_t *try_acquire_packet(void *user_context, uint32_t size) { lock.acquire_shared(); - halide_assert(user_context, size <= buffer_size); + HALIDE_CHECK(user_context, size <= buffer_size); uint32_t my_cursor = __sync_fetch_and_add(&cursor, size); if (my_cursor + size > sizeof(buf)) { // Don't try to back it out: instead, just allow this request to fail @@ -112,7 +112,7 @@ class TraceBuffer { overage = 0; } lock.release_exclusive(); - halide_assert(user_context, success && "Could not write to trace file"); + HALIDE_CHECK(user_context, success && "Could not write to trace file"); } // Acquire and return a packet's worth of space in the trace @@ -216,7 +216,7 @@ WEAK int32_t halide_default_trace(void *user_context, const halide_trace_event_t while (print_bits < e->type.bits) { print_bits <<= 1; } - halide_assert(user_context, print_bits <= 64 && "Tracing bad type"); + HALIDE_CHECK(user_context, print_bits <= 64 && "Tracing bad type"); // Otherwise, use halide_print and a plain-text format const char *event_types[] = {"Load", @@ -285,7 +285,7 @@ WEAK int32_t halide_default_trace(void *user_context, const halide_trace_event_t ss << ((uint64_t *)(e->value))[i]; } } else if (e->type.code == 2) { - halide_assert(user_context, print_bits >= 16 && "Tracing a bad type"); + HALIDE_CHECK(user_context, print_bits >= 16 && "Tracing a bad type"); if (print_bits == 32) { ss << ((float *)(e->value))[i]; } else if (print_bits == 16) { @@ -350,7 +350,7 @@ WEAK int halide_get_trace_file(void *user_context) { const char *trace_file_name = getenv("HL_TRACE_FILE"); if (trace_file_name) { void *file = fopen(trace_file_name, "ab"); - halide_assert(user_context, file && "Failed to open trace file\n"); + HALIDE_CHECK(user_context, file && "Failed to open trace file\n"); halide_set_trace_file(fileno(file)); halide_trace_file_internally_opened = file; if (!halide_trace_buffer) { From cd0b0655d5cda6f350f6f51f67bbf62a22aa4280 Mon Sep 17 00:00:00 2001 From: Steven Johnson Date: Wed, 3 Nov 2021 11:11:28 -0700 Subject: [PATCH 2/6] clang-format --- src/runtime/cuda.cpp | 2 +- src/runtime/d3d12compute.cpp | 2 +- src/runtime/hexagon_dma.cpp | 2 +- src/runtime/hexagon_host.cpp | 2 +- src/runtime/metal.cpp | 2 +- src/runtime/opencl.cpp | 8 ++++---- src/runtime/runtime_internal.h | 10 +++++----- src/runtime/thread_pool_common.h | 4 ++-- 8 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/runtime/cuda.cpp b/src/runtime/cuda.cpp index a4e300be79ca..b80792ab0e73 100644 --- a/src/runtime/cuda.cpp +++ b/src/runtime/cuda.cpp @@ -958,7 +958,7 @@ WEAK int halide_cuda_buffer_copy(void *user_context, struct halide_buffer_t *src struct halide_buffer_t *dst) { // We only handle copies to cuda or to host HALIDE_CHECK(user_context, dst_device_interface == nullptr || - dst_device_interface == &cuda_device_interface); + dst_device_interface == &cuda_device_interface); if ((src->device_dirty() || src->host == nullptr) && src->device_interface != &cuda_device_interface) { diff --git a/src/runtime/d3d12compute.cpp b/src/runtime/d3d12compute.cpp index b8d55782224a..4e28658d7354 100644 --- a/src/runtime/d3d12compute.cpp +++ b/src/runtime/d3d12compute.cpp @@ -3240,7 +3240,7 @@ WEAK int halide_d3d12compute_buffer_copy(void *user_context, struct halide_buffe // We only handle copies to d3d12 device or to host HALIDE_CHECK(user_context, (dst_device_interface == nullptr) || - (dst_device_interface == &d3d12compute_device_interface)); + (dst_device_interface == &d3d12compute_device_interface)); if ((src->device_dirty() || src->host == nullptr) && src->device_interface != &d3d12compute_device_interface) { diff --git a/src/runtime/hexagon_dma.cpp b/src/runtime/hexagon_dma.cpp index 4f4c87ee7e96..28bc8204c779 100644 --- a/src/runtime/hexagon_dma.cpp +++ b/src/runtime/hexagon_dma.cpp @@ -495,7 +495,7 @@ WEAK int halide_hexagon_dma_buffer_copy(void *user_context, struct halide_buffer struct halide_buffer_t *dst) { HALIDE_CHECK(user_context, dst_device_interface == nullptr || - dst_device_interface == &hexagon_dma_device_interface); + dst_device_interface == &hexagon_dma_device_interface); if (src->device_dirty() && src->device_interface != &hexagon_dma_device_interface) { diff --git a/src/runtime/hexagon_host.cpp b/src/runtime/hexagon_host.cpp index 3ec021b0a0d9..a744e0e355db 100644 --- a/src/runtime/hexagon_host.cpp +++ b/src/runtime/hexagon_host.cpp @@ -775,7 +775,7 @@ WEAK int halide_hexagon_buffer_copy(void *user_context, struct halide_buffer_t * struct halide_buffer_t *dst) { // We only handle copies to hexagon buffers or to host HALIDE_CHECK(user_context, dst_device_interface == nullptr || - dst_device_interface == &hexagon_device_interface); + dst_device_interface == &hexagon_device_interface); if ((src->device_dirty() || src->host == nullptr) && src->device_interface != &hexagon_device_interface) { diff --git a/src/runtime/metal.cpp b/src/runtime/metal.cpp index 3d02b2433de4..a6d8306dbb66 100644 --- a/src/runtime/metal.cpp +++ b/src/runtime/metal.cpp @@ -902,7 +902,7 @@ WEAK int halide_metal_buffer_copy(void *user_context, struct halide_buffer_t *sr // We only handle copies to metal buffers or to host HALIDE_CHECK(user_context, dst_device_interface == nullptr || - dst_device_interface == &metal_device_interface); + dst_device_interface == &metal_device_interface); if ((src->device_dirty() || src->host == nullptr) && src->device_interface != &metal_device_interface) { diff --git a/src/runtime/opencl.cpp b/src/runtime/opencl.cpp index 67fc0f2da9bd..5922cba81dc0 100644 --- a/src/runtime/opencl.cpp +++ b/src/runtime/opencl.cpp @@ -971,7 +971,7 @@ WEAK int halide_opencl_buffer_copy(void *user_context, struct halide_buffer_t *s struct halide_buffer_t *dst) { // We only handle copies to opencl or to host HALIDE_CHECK(user_context, dst_device_interface == nullptr || - dst_device_interface == &opencl_device_interface); + dst_device_interface == &opencl_device_interface); if ((src->device_dirty() || src->host == nullptr) && src->device_interface != &opencl_device_interface) { @@ -1239,7 +1239,7 @@ WEAK int halide_opencl_detach_cl_mem(void *user_context, halide_buffer_t *buf) { return 0; } HALIDE_CHECK(user_context, buf->device_interface == &opencl_device_interface || - buf->device_interface == &opencl_image_device_interface); + buf->device_interface == &opencl_image_device_interface); free((device_handle *)buf->device); buf->device = 0; buf->device_interface->impl->release_module(); @@ -1252,7 +1252,7 @@ WEAK uintptr_t halide_opencl_get_cl_mem(void *user_context, halide_buffer_t *buf return 0; } HALIDE_CHECK(user_context, buf->device_interface == &opencl_device_interface || - buf->device_interface == &opencl_image_device_interface); + buf->device_interface == &opencl_image_device_interface); return (uintptr_t)((device_handle *)buf->device)->mem; } @@ -1683,7 +1683,7 @@ WEAK int halide_opencl_image_buffer_copy(void *user_context, struct halide_buffe << ", src: " << src << ", dst: " << dst << ")\n"; HALIDE_CHECK(user_context, dst_device_interface == nullptr || - dst_device_interface == &opencl_image_device_interface); + dst_device_interface == &opencl_image_device_interface); if ((src->device_dirty() || src->host == nullptr) && src->device_interface != &opencl_image_device_interface) { diff --git a/src/runtime/runtime_internal.h b/src/runtime/runtime_internal.h index f75ce4c54ce5..b750c2e72022 100644 --- a/src/runtime/runtime_internal.h +++ b/src/runtime/runtime_internal.h @@ -233,12 +233,12 @@ using namespace Halide::Runtime::Internal; * the condition will be checked in *all* build modes! */ #define _HALIDE_CHECK_STRINGIFY(x) #x #define _HALIDE_CHECK_EXPAND_AND_STRINGIFY(x) _HALIDE_CHECK_STRINGIFY(x) -#define HALIDE_CHECK(user_context, cond) \ - do { \ - if (!(cond)) { \ +#define HALIDE_CHECK(user_context, cond) \ + do { \ + if (!(cond)) { \ halide_print(user_context, __FILE__ ":" _HALIDE_CHECK_EXPAND_AND_STRINGIFY(__LINE__) " HALIDE_CHECK failed: " #cond "\n"); \ - abort(); \ - } \ + abort(); \ + } \ } while (0) #endif diff --git a/src/runtime/thread_pool_common.h b/src/runtime/thread_pool_common.h index c73e2a1eb335..f26dd4d955f6 100644 --- a/src/runtime/thread_pool_common.h +++ b/src/runtime/thread_pool_common.h @@ -523,8 +523,8 @@ WEAK void enqueue_work_already_locked(int num_jobs, work *jobs, work *task_paren } else { log_message("enqueue_work_already_locked job " << jobs[0].task.name << " with min_threads " << min_threads << " task_parent " << task_parent->task.name << " task_parent->task.min_threads " << task_parent->task.min_threads << " task_parent->threads_reserved " << task_parent->threads_reserved); HALIDE_CHECK(nullptr, (min_threads <= ((task_parent->task.min_threads * task_parent->active_workers) - - task_parent->threads_reserved)) && - "Logic error: thread over commit.\n"); + task_parent->threads_reserved)) && + "Logic error: thread over commit.\n"); if (job_has_acquires || job_may_block) { task_parent->threads_reserved++; } From 0189766e9595f7923f867dbc9793c3940058c29d Mon Sep 17 00:00:00 2001 From: Steven Johnson Date: Wed, 3 Nov 2021 12:49:30 -0700 Subject: [PATCH 3/6] Fix for top-of-tree LLVM --- src/CodeGen_LLVM.cpp | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/src/CodeGen_LLVM.cpp b/src/CodeGen_LLVM.cpp index cf9e8e15bcf5..237730a8da45 100644 --- a/src/CodeGen_LLVM.cpp +++ b/src/CodeGen_LLVM.cpp @@ -1159,21 +1159,27 @@ void CodeGen_LLVM::optimize_module() { #endif pb.registerOptimizerLastEPCallback( [](ModulePassManager &mpm, OptimizationLevel level) { - constexpr bool compile_kernel = false; - constexpr bool recover = false; - constexpr bool use_after_scope = true; #if LLVM_VERSION >= 140 - // TODO: this is the value that the default ctor uses. - // Not sure if it's ideal for Halide. - constexpr AsanDetectStackUseAfterReturnMode use_after_return = AsanDetectStackUseAfterReturnMode::Runtime; - mpm.addPass(createModuleToFunctionPassAdaptor(AddressSanitizerPass( - AddressSanitizerOptions{compile_kernel, recover, use_after_scope, use_after_return}))); + AddressSanitizerOptions asan_options; // default values are good... + asan_options.UseAfterScope = true; // ... except this one + mpm.addPass(createModuleToFunctionPassAdaptor(AddressSanitizerPass(asan_options))); #else mpm.addPass(createModuleToFunctionPassAdaptor(AddressSanitizerPass( compile_kernel, recover, use_after_scope))); #endif }); -#if LLVM_VERSION >= 120 +#if LLVM_VERSION >= 140 + pb.registerPipelineStartEPCallback( + [](ModulePassManager &mpm, OptimizationLevel) { + AddressSanitizerOptions asan_options; // default values are good + constexpr bool use_global_gc = true; + constexpr bool use_odr_indicator = true; + constexpr auto destructor_kind = AsanDtorKind::Global; + mpm.addPass(ModuleAddressSanitizerPass( + asan_options, use_global_gc, + use_odr_indicator, destructor_kind)); + }); +#elif LLVM_VERSION >= 120 pb.registerPipelineStartEPCallback( [](ModulePassManager &mpm, OptimizationLevel) { constexpr bool compile_kernel = false; From a251f2ee8a8ffebc5a85c5849a62bb53b092eea2 Mon Sep 17 00:00:00 2001 From: Steven Johnson Date: Wed, 3 Nov 2021 13:18:03 -0700 Subject: [PATCH 4/6] Fix for older versions --- src/CodeGen_LLVM.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/CodeGen_LLVM.cpp b/src/CodeGen_LLVM.cpp index 237730a8da45..0b677a2ec5b0 100644 --- a/src/CodeGen_LLVM.cpp +++ b/src/CodeGen_LLVM.cpp @@ -1164,6 +1164,9 @@ void CodeGen_LLVM::optimize_module() { asan_options.UseAfterScope = true; // ... except this one mpm.addPass(createModuleToFunctionPassAdaptor(AddressSanitizerPass(asan_options))); #else + constexpr bool compile_kernel = false; + constexpr bool recover = false; + constexpr bool use_after_scope = true; mpm.addPass(createModuleToFunctionPassAdaptor(AddressSanitizerPass( compile_kernel, recover, use_after_scope))); #endif From d9db8e1befc0d4347a078ace1f9f9575435ac15c Mon Sep 17 00:00:00 2001 From: Steven Johnson Date: Wed, 3 Nov 2021 14:14:47 -0700 Subject: [PATCH 5/6] HALIDE_CHECK -> halide_abort_if_false --- src/runtime/cache.cpp | 12 +- src/runtime/cuda.cpp | 50 ++--- src/runtime/d3d12compute.cpp | 282 ++++++++++++------------ src/runtime/device_interface.cpp | 6 +- src/runtime/gpu_context_common.h | 6 +- src/runtime/hashmap.h | 26 +-- src/runtime/hexagon_cache_allocator.cpp | 2 +- src/runtime/hexagon_dma.cpp | 50 ++--- src/runtime/hexagon_dma_pool.cpp | 12 +- src/runtime/hexagon_host.cpp | 34 +-- src/runtime/metal.cpp | 52 ++--- src/runtime/opencl.cpp | 96 ++++---- src/runtime/openglcompute.cpp | 8 +- src/runtime/profiler.cpp | 14 +- src/runtime/runtime_internal.h | 17 +- src/runtime/synchronization_common.h | 6 +- src/runtime/thread_pool_common.h | 8 +- src/runtime/tracing.cpp | 10 +- 18 files changed, 345 insertions(+), 346 deletions(-) diff --git a/src/runtime/cache.cpp b/src/runtime/cache.cpp index 86b582cb5c76..b759b9eec45e 100644 --- a/src/runtime/cache.cpp +++ b/src/runtime/cache.cpp @@ -278,7 +278,7 @@ WEAK void prune_cache() { while (prev_hash_entry != nullptr && prev_hash_entry->next != prune_candidate) { prev_hash_entry = prev_hash_entry->next; } - HALIDE_CHECK(nullptr, prev_hash_entry != nullptr); + halide_abort_if_false(nullptr, prev_hash_entry != nullptr); prev_hash_entry->next = prune_candidate->next; } @@ -367,14 +367,14 @@ WEAK int halide_memoization_cache_lookup(void *user_context, const uint8_t *cach if (all_bounds_equal) { if (entry != most_recently_used) { - HALIDE_CHECK(user_context, entry->more_recent != nullptr); + halide_abort_if_false(user_context, entry->more_recent != nullptr); if (entry->less_recent != nullptr) { entry->less_recent->more_recent = entry->more_recent; } else { - HALIDE_CHECK(user_context, least_recently_used == entry); + halide_abort_if_false(user_context, least_recently_used == entry); least_recently_used = entry->more_recent; } - HALIDE_CHECK(user_context, entry->more_recent != nullptr); + halide_abort_if_false(user_context, entry->more_recent != nullptr); entry->more_recent->less_recent = entry->less_recent; entry->more_recent = nullptr; @@ -466,7 +466,7 @@ WEAK int halide_memoization_cache_store(void *user_context, const uint8_t *cache } } if (all_bounds_equal) { - HALIDE_CHECK(user_context, no_host_pointers_equal); + halide_abort_if_false(user_context, no_host_pointers_equal); // This entry is still in use by the caller. Mark it as having no cache entry // so halide_memoization_cache_release can free the buffer. for (int32_t i = 0; i < tuple_count; i++) { @@ -544,7 +544,7 @@ WEAK void halide_memoization_cache_release(void *user_context, void *host) { } else { ScopedMutexLock lock(&memoization_lock); - HALIDE_CHECK(user_context, entry->in_use_count > 0); + halide_abort_if_false(user_context, entry->in_use_count > 0); entry->in_use_count--; #if CACHE_DEBUGGING validate_cache(); diff --git a/src/runtime/cuda.cpp b/src/runtime/cuda.cpp index b80792ab0e73..f80c148c2e7c 100644 --- a/src/runtime/cuda.cpp +++ b/src/runtime/cuda.cpp @@ -72,7 +72,7 @@ ALWAYS_INLINE T get_cuda_symbol(void *user_context, const char *name, bool optio // Load a CUDA shared object/dll and get the CUDA API function pointers from it. WEAK void load_libcuda(void *user_context) { debug(user_context) << " load_libcuda (user_context: " << user_context << ")\n"; - HALIDE_CHECK(user_context, cuInit == nullptr); + halide_abort_if_false(user_context, cuInit == nullptr); // clang-format off #define CUDA_FN(ret, fn, args) fn = get_cuda_symbol(user_context, #fn); // NOLINT(bugprone-macro-parentheses) @@ -139,10 +139,10 @@ extern "C" { WEAK int halide_default_cuda_acquire_context(void *user_context, CUcontext *ctx, bool create = true) { // TODO: Should we use a more "assertive" assert? these asserts do // not block execution on failure. - HALIDE_CHECK(user_context, ctx != nullptr); + halide_abort_if_false(user_context, ctx != nullptr); // If the context has not been initialized, initialize it now. - HALIDE_CHECK(user_context, &context != nullptr); + halide_abort_if_false(user_context, &context != nullptr); // Note that this null-check of the context is *not* locked with // respect to device_release, so we may get a non-null context @@ -276,8 +276,8 @@ class Context { // overridden, we may still need to load libcuda ensure_libcuda_init(user_context); - HALIDE_CHECK(user_context, context != nullptr); - HALIDE_CHECK(user_context, cuInit != nullptr); + halide_abort_if_false(user_context, context != nullptr); + halide_abort_if_false(user_context, cuInit != nullptr); error = cuCtxPushCurrent(context); } @@ -578,7 +578,7 @@ WEAK int halide_cuda_initialize_kernels(void *user_context, void **state_ptr, co compile_kernel, user_context, ptx_src, size)) { return halide_error_code_generic_error; } - HALIDE_CHECK(user_context, loaded_module != nullptr); + halide_abort_if_false(user_context, loaded_module != nullptr); #ifdef DEBUG_RUNTIME uint64_t t_after = halide_current_time_ns(user_context); @@ -661,7 +661,7 @@ WEAK int halide_cuda_device_free(void *user_context, halide_buffer_t *buf) { uint64_t t_before = halide_current_time_ns(user_context); #endif - HALIDE_CHECK(user_context, validate_device_pointer(user_context, buf)); + halide_abort_if_false(user_context, validate_device_pointer(user_context, buf)); CUresult err = CUDA_SUCCESS; if (halide_can_reuse_device_allocations(user_context)) { @@ -735,7 +735,7 @@ WEAK int halide_cuda_device_release(void *user_context) { if (err != CUDA_SUCCESS) { err = cuCtxSynchronize(); } - HALIDE_CHECK(user_context, err == CUDA_SUCCESS || err == CUDA_ERROR_DEINITIALIZED); + halide_abort_if_false(user_context, err == CUDA_SUCCESS || err == CUDA_ERROR_DEINITIALIZED); // Dump the contents of the free list, ignoring errors. halide_cuda_release_unused_device_allocations(user_context); @@ -754,7 +754,7 @@ WEAK int halide_cuda_device_release(void *user_context) { debug(user_context) << " cuCtxDestroy " << context << "\n"; err = cuProfilerStop(); err = cuCtxDestroy(context); - HALIDE_CHECK(user_context, err == CUDA_SUCCESS || err == CUDA_ERROR_DEINITIALIZED); + halide_abort_if_false(user_context, err == CUDA_SUCCESS || err == CUDA_ERROR_DEINITIALIZED); context = nullptr; } } // spinlock @@ -779,16 +779,16 @@ WEAK int halide_cuda_device_malloc(void *user_context, halide_buffer_t *buf) { if (halide_can_reuse_device_allocations(user_context)) { size = quantize_allocation_size(size); } - HALIDE_CHECK(user_context, size != 0); + halide_abort_if_false(user_context, size != 0); if (buf->device) { // This buffer already has a device allocation - HALIDE_CHECK(user_context, validate_device_pointer(user_context, buf, size)); + halide_abort_if_false(user_context, validate_device_pointer(user_context, buf, size)); return 0; } // Check all strides positive. for (int i = 0; i < buf->dimensions; i++) { - HALIDE_CHECK(user_context, buf->dim[i].stride >= 0); + halide_abort_if_false(user_context, buf->dim[i].stride >= 0); } debug(user_context) << " allocating " << *buf << "\n"; @@ -877,7 +877,7 @@ WEAK int halide_cuda_device_malloc(void *user_context, halide_buffer_t *buf) { debug(user_context) << (void *)p << "\n"; } } - HALIDE_CHECK(user_context, p); + halide_abort_if_false(user_context, p); buf->device = p; buf->device_interface = &cuda_device_interface; buf->device_interface->impl->use_module(); @@ -957,12 +957,12 @@ WEAK int halide_cuda_buffer_copy(void *user_context, struct halide_buffer_t *src const struct halide_device_interface_t *dst_device_interface, struct halide_buffer_t *dst) { // We only handle copies to cuda or to host - HALIDE_CHECK(user_context, dst_device_interface == nullptr || - dst_device_interface == &cuda_device_interface); + halide_abort_if_false(user_context, dst_device_interface == nullptr || + dst_device_interface == &cuda_device_interface); if ((src->device_dirty() || src->host == nullptr) && src->device_interface != &cuda_device_interface) { - HALIDE_CHECK(user_context, dst_device_interface == &cuda_device_interface); + halide_abort_if_false(user_context, dst_device_interface == &cuda_device_interface); // This is handled at the higher level. return halide_error_code_incompatible_device_interface; } @@ -972,8 +972,8 @@ WEAK int halide_cuda_buffer_copy(void *user_context, struct halide_buffer_t *src (src->host_dirty() && src->host != nullptr); bool to_host = !dst_device_interface; - HALIDE_CHECK(user_context, from_host || src->device); - HALIDE_CHECK(user_context, to_host || dst->device); + halide_abort_if_false(user_context, from_host || src->device); + halide_abort_if_false(user_context, to_host || dst->device); device_copy c = make_buffer_copy(src, from_host, dst, to_host); @@ -991,10 +991,10 @@ WEAK int halide_cuda_buffer_copy(void *user_context, struct halide_buffer_t *src #ifdef DEBUG_RUNTIME uint64_t t_before = halide_current_time_ns(user_context); if (!from_host) { - HALIDE_CHECK(user_context, validate_device_pointer(user_context, src)); + halide_abort_if_false(user_context, validate_device_pointer(user_context, src)); } if (!to_host) { - HALIDE_CHECK(user_context, validate_device_pointer(user_context, dst)); + halide_abort_if_false(user_context, validate_device_pointer(user_context, dst)); } #endif @@ -1139,7 +1139,7 @@ WEAK int halide_cuda_run(void *user_context, CUmodule mod{}; bool found = compilation_cache.lookup(ctx.context, state_ptr, mod); - HALIDE_CHECK(user_context, found && mod != nullptr); + halide_abort_if_false(user_context, found && mod != nullptr); debug(user_context) << "Got module " << mod << "\n"; CUfunction f; @@ -1166,7 +1166,7 @@ WEAK int halide_cuda_run(void *user_context, uint64_t *dev_handles = (uint64_t *)malloc(num_args * sizeof(uint64_t)); for (size_t i = 0; i <= num_args; i++) { // Get nullptr at end. if (arg_is_buffer[i]) { - HALIDE_CHECK(user_context, arg_sizes[i] == sizeof(uint64_t)); + halide_abort_if_false(user_context, arg_sizes[i] == sizeof(uint64_t)); dev_handles[i] = ((halide_buffer_t *)args[i])->device; translated_args[i] = &(dev_handles[i]); debug(user_context) << " halide_cuda_run translated arg" << (int)i @@ -1226,7 +1226,7 @@ WEAK int halide_cuda_device_and_host_free(void *user_context, struct halide_buff } WEAK int halide_cuda_wrap_device_ptr(void *user_context, struct halide_buffer_t *buf, uint64_t device_ptr) { - HALIDE_CHECK(user_context, buf->device == 0); + halide_abort_if_false(user_context, buf->device == 0); if (buf->device != 0) { return -2; } @@ -1248,7 +1248,7 @@ WEAK int halide_cuda_detach_device_ptr(void *user_context, struct halide_buffer_ if (buf->device == 0) { return 0; } - HALIDE_CHECK(user_context, buf->device_interface == &cuda_device_interface); + halide_abort_if_false(user_context, buf->device_interface == &cuda_device_interface); buf->device_interface->impl->release_module(); buf->device = 0; buf->device_interface = nullptr; @@ -1259,7 +1259,7 @@ WEAK uintptr_t halide_cuda_get_device_ptr(void *user_context, struct halide_buff if (buf->device == 0) { return 0; } - HALIDE_CHECK(user_context, buf->device_interface == &cuda_device_interface); + halide_abort_if_false(user_context, buf->device_interface == &cuda_device_interface); return (uintptr_t)buf->device; } diff --git a/src/runtime/d3d12compute.cpp b/src/runtime/d3d12compute.cpp index 4e28658d7354..8276a6bd177e 100644 --- a/src/runtime/d3d12compute.cpp +++ b/src/runtime/d3d12compute.cpp @@ -226,7 +226,7 @@ void *d3d12_get_library_symbol(void *lib, const char *name) { #pragma message "RenderDoc might not work well alongside Direct3D debug layers..." #endif #define WIN32 -#define RenderDocAssert(expr) HALIDE_CHECK(user_context, expr) +#define RenderDocAssert(expr) halide_abort_if_false(user_context, expr) #define LoadRenderDocLibrary(dll) d3d12_load_library(dll) #define GetRenderDocProcAddr(dll, proc) d3d12_get_library_symbol(dll, proc) #define RENDERDOC_NO_STDINT @@ -420,8 +420,8 @@ static DXGI_FORMAT FindD3D12FormatForHalideType(void *user_context, halide_type_ }}, }; - HALIDE_CHECK(user_context, (type.code >= 0) && (type.code <= 2)); - HALIDE_CHECK(user_context, (type.lanes > 0) && (type.lanes <= 4)); + halide_abort_if_false(user_context, (type.code >= 0) && (type.code <= 2)); + halide_abort_if_false(user_context, (type.lanes > 0) && (type.lanes <= 4)); int i = 0; switch (type.bytes()) { @@ -438,7 +438,7 @@ static DXGI_FORMAT FindD3D12FormatForHalideType(void *user_context, halide_type_ i = 3; break; default: - HALIDE_CHECK(user_context, false); + halide_abort_if_false(user_context, false); break; } @@ -661,15 +661,15 @@ static size_t number_of_elements(void *user_context, const halide_buffer_t *buff // elements in the stride regions. size_t size_in_bytes = buffer->size_in_bytes(); - HALIDE_CHECK(user_context, (size_in_bytes > 0)); + halide_abort_if_false(user_context, (size_in_bytes > 0)); size_t element_size = 1; element_size *= buffer->type.bytes(); element_size *= buffer->type.lanes; - HALIDE_CHECK(user_context, (element_size > 0)); + halide_abort_if_false(user_context, (element_size > 0)); size_t elements = size_in_bytes / element_size; - HALIDE_CHECK(user_context, (size_in_bytes % element_size) == 0); + halide_abort_if_false(user_context, (size_in_bytes % element_size) == 0); return elements; } @@ -890,10 +890,10 @@ extern WEAK halide_device_interface_t d3d12compute_device_interface; static d3d12_buffer *peel_buffer(struct halide_buffer_t *hbuffer) { TRACELOG; - HALIDE_CHECK(user_context, (hbuffer != nullptr)); - HALIDE_CHECK(user_context, (hbuffer->device_interface == &d3d12compute_device_interface)); + halide_abort_if_false(user_context, (hbuffer != nullptr)); + halide_abort_if_false(user_context, (hbuffer->device_interface == &d3d12compute_device_interface)); d3d12_buffer *dbuffer = reinterpret_cast(hbuffer->device); - HALIDE_CHECK(user_context, (dbuffer != nullptr)); + halide_abort_if_false(user_context, (dbuffer != nullptr)); return dbuffer; } @@ -902,12 +902,12 @@ static const d3d12_buffer *peel_buffer(const struct halide_buffer_t *hbuffer) { } WEAK int wrap_buffer(void *user_context, struct halide_buffer_t *hbuffer, d3d12_buffer *dbuffer) { - HALIDE_CHECK(user_context, (hbuffer->device == 0)); + halide_abort_if_false(user_context, (hbuffer->device == 0)); if (hbuffer->device != 0) { return halide_error_code_device_wrap_native_failed; } - HALIDE_CHECK(user_context, (dbuffer->resource != nullptr)); + halide_abort_if_false(user_context, (dbuffer->resource != nullptr)); dbuffer->offset = 0; dbuffer->offsetInBytes = 0; @@ -921,7 +921,7 @@ WEAK int wrap_buffer(void *user_context, struct halide_buffer_t *hbuffer, d3d12_ dbuffer->halide_type = hbuffer->type; hbuffer->device = reinterpret_cast(dbuffer); - HALIDE_CHECK(user_context, (hbuffer->device_interface == nullptr)); + halide_abort_if_false(user_context, (hbuffer->device_interface == nullptr)); hbuffer->device_interface = &d3d12compute_device_interface; hbuffer->device_interface->impl->use_module(); @@ -972,7 +972,7 @@ static void D3D12LoadDependencies(void *user_context) { #if !RENDERDOC_AUTOINIT TRACEPRINT("Initializing RenderDoc\n"); bool rdinit = InitRenderDoc(); - HALIDE_CHECK(user_context, rdinit); + halide_abort_if_false(user_context, rdinit); #endif #endif @@ -984,33 +984,33 @@ static void D3D12LoadDependencies(void *user_context) { // Windows x64 follows the LLP64 integer type convention: // https://msdn.microsoft.com/en-us/library/windows/desktop/aa383751(v=vs.85).aspx - HALIDE_CHECK(user_context, sizeof(BOOL) == (32 / 8)); // BOOL must be 32 bits - HALIDE_CHECK(user_context, sizeof(CHAR) == (8 / 8)); // CHAR must be 8 bits - HALIDE_CHECK(user_context, sizeof(SHORT) == (16 / 8)); // SHORT must be 16 bits - HALIDE_CHECK(user_context, sizeof(LONG) == (32 / 8)); // LONG must be 32 bits - HALIDE_CHECK(user_context, sizeof(ULONG) == (32 / 8)); // ULONG must be 32 bits - HALIDE_CHECK(user_context, sizeof(LONGLONG) == (64 / 8)); // LONGLONG must be 16 bits - HALIDE_CHECK(user_context, sizeof(BYTE) == (8 / 8)); // BYTE must be 8 bits - HALIDE_CHECK(user_context, sizeof(WORD) == (16 / 8)); // WORD must be 16 bits - HALIDE_CHECK(user_context, sizeof(DWORD) == (32 / 8)); // DWORD must be 32 bits - HALIDE_CHECK(user_context, sizeof(WCHAR) == (16 / 8)); // WCHAR must be 16 bits - HALIDE_CHECK(user_context, sizeof(INT) == (32 / 8)); // INT must be 32 bits - HALIDE_CHECK(user_context, sizeof(UINT) == (32 / 8)); // UINT must be 32 bits - HALIDE_CHECK(user_context, sizeof(IID) == (128 / 8)); // COM GUIDs must be 128 bits + halide_abort_if_false(user_context, sizeof(BOOL) == (32 / 8)); // BOOL must be 32 bits + halide_abort_if_false(user_context, sizeof(CHAR) == (8 / 8)); // CHAR must be 8 bits + halide_abort_if_false(user_context, sizeof(SHORT) == (16 / 8)); // SHORT must be 16 bits + halide_abort_if_false(user_context, sizeof(LONG) == (32 / 8)); // LONG must be 32 bits + halide_abort_if_false(user_context, sizeof(ULONG) == (32 / 8)); // ULONG must be 32 bits + halide_abort_if_false(user_context, sizeof(LONGLONG) == (64 / 8)); // LONGLONG must be 16 bits + halide_abort_if_false(user_context, sizeof(BYTE) == (8 / 8)); // BYTE must be 8 bits + halide_abort_if_false(user_context, sizeof(WORD) == (16 / 8)); // WORD must be 16 bits + halide_abort_if_false(user_context, sizeof(DWORD) == (32 / 8)); // DWORD must be 32 bits + halide_abort_if_false(user_context, sizeof(WCHAR) == (16 / 8)); // WCHAR must be 16 bits + halide_abort_if_false(user_context, sizeof(INT) == (32 / 8)); // INT must be 32 bits + halide_abort_if_false(user_context, sizeof(UINT) == (32 / 8)); // UINT must be 32 bits + halide_abort_if_false(user_context, sizeof(IID) == (128 / 8)); // COM GUIDs must be 128 bits // Paranoid checks (I am not taking any chances...) - HALIDE_CHECK(user_context, sizeof(INT8) == (8 / 8)); - HALIDE_CHECK(user_context, sizeof(INT16) == (16 / 8)); - HALIDE_CHECK(user_context, sizeof(INT32) == (32 / 8)); - HALIDE_CHECK(user_context, sizeof(INT64) == (64 / 8)); - HALIDE_CHECK(user_context, sizeof(UINT8) == (8 / 8)); - HALIDE_CHECK(user_context, sizeof(UINT16) == (16 / 8)); - HALIDE_CHECK(user_context, sizeof(UINT32) == (32 / 8)); - HALIDE_CHECK(user_context, sizeof(UINT64) == (64 / 8)); + halide_abort_if_false(user_context, sizeof(INT8) == (8 / 8)); + halide_abort_if_false(user_context, sizeof(INT16) == (16 / 8)); + halide_abort_if_false(user_context, sizeof(INT32) == (32 / 8)); + halide_abort_if_false(user_context, sizeof(INT64) == (64 / 8)); + halide_abort_if_false(user_context, sizeof(UINT8) == (8 / 8)); + halide_abort_if_false(user_context, sizeof(UINT16) == (16 / 8)); + halide_abort_if_false(user_context, sizeof(UINT32) == (32 / 8)); + halide_abort_if_false(user_context, sizeof(UINT64) == (64 / 8)); #ifdef BITS_64 - HALIDE_CHECK(user_context, sizeof(SIZE_T) == (64 / 8)); + halide_abort_if_false(user_context, sizeof(SIZE_T) == (64 / 8)); #else - HALIDE_CHECK(user_context, sizeof(SIZE_T) == (32 / 8)); + halide_abort_if_false(user_context, sizeof(SIZE_T) == (32 / 8)); #endif } @@ -1117,7 +1117,7 @@ static d3d12_device *D3D12CreateSystemDefaultDevice(void *user_context) { return nullptr; } - HALIDE_CHECK(user_context, (dxgiAdapter == nullptr)); + halide_abort_if_false(user_context, (dxgiAdapter == nullptr)); size_t vram_max = 0; for (int i = 0;; ++i) { IDXGIAdapter1 *adapter = nullptr; @@ -1249,7 +1249,7 @@ ID3D12RootSignature *D3D12CreateMasterRootSignature(ID3D12Device *device) { ID3DBlob *pSignError = nullptr; HRESULT result = D3D12SerializeRootSignature(&rsd, Version, &pSignBlob, &pSignError); if (D3DErrorCheck(result, pSignBlob, nullptr, "Unable to serialize the Direct3D 12 root signature")) { - HALIDE_CHECK(user_context, pSignError); + halide_abort_if_false(user_context, pSignError); TRACEFATAL((const char *)pSignError->GetBufferPointer()); return nullptr; } @@ -1331,7 +1331,7 @@ WEAK d3d12_buffer new_buffer_resource(d3d12_device *device, size_t length, D3D12 break; default: TRACEPRINT("UNSUPPORTED D3D12 BUFFER HEAP TYPE: " << (int)heaptype << "\n"); - HALIDE_CHECK(user_context, false); + halide_abort_if_false(user_context, false); break; } @@ -1394,7 +1394,7 @@ WEAK void *map_buffer(d3d12_buffer *buffer) { break; default: TRACEPRINT("UNSUPPORTED BUFFER TYPE: " << (int)buffer->type << "\n"); - HALIDE_CHECK(user_context, false); + halide_abort_if_false(user_context, false); break; } @@ -1410,7 +1410,7 @@ WEAK void *map_buffer(d3d12_buffer *buffer) { return nullptr; } - HALIDE_CHECK(user_context, pData); + halide_abort_if_false(user_context, pData); buffer->mapped = pData; return pData; @@ -1438,7 +1438,7 @@ WEAK void unmap_buffer(d3d12_buffer *buffer) { break; default: TRACEPRINT("UNSUPPORTED BUFFER TYPE: " << (int)buffer->type << "\n"); - HALIDE_CHECK(user_context, false); + halide_abort_if_false(user_context, false); break; } @@ -1498,37 +1498,37 @@ WEAK size_t suballocate(d3d12_device *device, d3d12_buffer *staging, size_t num_ if (staging->sizeInBytes < num_bytes) { // ensure there are no pending transfers on this buffer uint64_t use_count = __atomic_add_fetch(&staging->ref_count, 1, __ATOMIC_SEQ_CST); - HALIDE_CHECK(user_context, (use_count == 1)); + halide_abort_if_false(user_context, (use_count == 1)); // find a new "ideal" size: e.g., using a cumulative 2x heuristic size_t old_capacity = staging->sizeInBytes; size_t new_capacity = 2 * (old_capacity + num_bytes); TRACEPRINT("not enough storage: growing from " << (uintptr_t)old_capacity << " bytes to " << (uintptr_t)new_capacity << " bytes.\n"); // release the old storage use_count = __atomic_sub_fetch(&staging->ref_count, 1, __ATOMIC_SEQ_CST); - HALIDE_CHECK(user_context, (use_count == 0)); + halide_abort_if_false(user_context, (use_count == 0)); release_d3d12_object(staging); // and allocate a new one switch (staging->type) { case d3d12_buffer::Upload: - HALIDE_CHECK(user_context, (staging == &upload)); + halide_abort_if_false(user_context, (staging == &upload)); *staging = new_upload_buffer(device, new_capacity); break; case d3d12_buffer::ReadBack: - HALIDE_CHECK(user_context, (staging == &readback)); + halide_abort_if_false(user_context, (staging == &readback)); *staging = new_readback_buffer(device, new_capacity); break; default: TRACEPRINT("UNSUPPORTED BUFFER TYPE: " << (int)staging->type << "\n"); - HALIDE_CHECK(user_context, false); + halide_abort_if_false(user_context, false); break; } } - HALIDE_CHECK(user_context, (staging->sizeInBytes >= num_bytes)); + halide_abort_if_false(user_context, (staging->sizeInBytes >= num_bytes)); // this reference counter will be decremented later by 'd3d12compute_device_sync_internal()' uint64_t use_count = __atomic_add_fetch(&staging->ref_count, 1, __ATOMIC_SEQ_CST); // but for now we must ensure that there are no pending transfers on this buffer already - HALIDE_CHECK(user_context, (use_count == 1)); + halide_abort_if_false(user_context, (use_count == 1)); size_t byte_offset = 0; // always zero, for now return byte_offset; } @@ -1668,7 +1668,7 @@ WEAK d3d12_command_queue *new_command_queue(d3d12_device *device) { template static d3d12_command_allocator *new_command_allocator(d3d12_device *device) { TRACELOG; - HALIDE_CHECK(user_context, device); + halide_abort_if_false(user_context, device); ID3D12CommandAllocator *commandAllocator = nullptr; HRESULT result = (*device)->CreateCommandAllocator(Type, IID_PPV_ARGS(&commandAllocator)); if (D3DErrorCheck(result, commandAllocator, nullptr, "Unable to create the Direct3D 12 command allocator")) { @@ -1941,11 +1941,11 @@ static d3d12_function *new_function_with_name(d3d12_device *device, d3d12_librar d3d12_function *function = nullptr; Printer key(nullptr); key << name << "_(" << threadsX << "," << threadsY << "," << threadsZ << ")_[" << shared_mem_bytes << "]"; - HALIDE_CHECK(user_context, (key.size() < key.capacity() - 1)); // make sure key fits into the stream + halide_abort_if_false(user_context, (key.size() < key.capacity() - 1)); // make sure key fits into the stream int not_found = library->cache.lookup(user_context, (const uint8_t *)key.str(), key.size(), &function); if (not_found) { // function has not been cached yet: must compile it - HALIDE_CHECK(user_context, (function == nullptr)); + halide_abort_if_false(user_context, (function == nullptr)); function = d3d12_compile_shader(device, library, name, shared_mem_bytes, threadsX, threadsY, threadsZ); if (function == nullptr) { return nullptr; @@ -1956,7 +1956,7 @@ static d3d12_function *new_function_with_name(d3d12_device *device, d3d12_librar TRACEPRINT("function has been found in the cache!\n"); } - HALIDE_CHECK(user_context, (function != nullptr)); + halide_abort_if_false(user_context, (function != nullptr)); return function; } @@ -1971,7 +1971,7 @@ WEAK void set_input_buffer(d3d12_binder *binder, d3d12_buffer *input_buffer, uin // NOTE(marcos): constant buffers are only used internally by the // runtime; users cannot create, control or access them, so it is // expected that no halide_buffer_t will be associated with them: - HALIDE_CHECK(user_context, input_buffer->format == DXGI_FORMAT_UNKNOWN); + halide_abort_if_false(user_context, input_buffer->format == DXGI_FORMAT_UNKNOWN); ID3D12Resource *pResource = input_buffer->resource; D3D12_GPU_VIRTUAL_ADDRESS pGPU = pResource->GetGPUVirtualAddress(); @@ -1982,7 +1982,7 @@ WEAK void set_input_buffer(d3d12_binder *binder, d3d12_buffer *input_buffer, uin cbvd.SizeInBytes = input_buffer->sizeInBytes; } - HALIDE_CHECK(user_context, (index < ResourceBindingLimits[CBV])); + halide_abort_if_false(user_context, (index < ResourceBindingLimits[CBV])); D3D12_CPU_DESCRIPTOR_HANDLE hDescCBV = binder->CPU[CBV]; binder->CPU[CBV].ptr += binder->descriptorSize; @@ -2038,7 +2038,7 @@ WEAK void set_input_buffer(d3d12_binder *binder, d3d12_buffer *input_buffer, uin uavd.Buffer.Flags = D3D12_BUFFER_UAV_FLAG_NONE; } - HALIDE_CHECK(user_context, (index < ResourceBindingLimits[UAV])); + halide_abort_if_false(user_context, (index < ResourceBindingLimits[UAV])); D3D12_CPU_DESCRIPTOR_HANDLE hDescUAV = binder->CPU[UAV]; binder->CPU[UAV].ptr += binder->descriptorSize; @@ -2055,7 +2055,7 @@ WEAK void set_input_buffer(d3d12_binder *binder, d3d12_buffer *input_buffer, uin case d3d12_buffer::ReadBack: default: TRACEPRINT("UNSUPPORTED BUFFER TYPE: " << (int)input_buffer->type << "\n"); - HALIDE_CHECK(user_context, false); + halide_abort_if_false(user_context, false); break; } } @@ -2181,10 +2181,10 @@ static void buffer_copy_command(d3d12_copy_command_list *cmdList, src_barrier.Transition.StateBefore = src->state; src_barrier.Transition.StateAfter = D3D12_RESOURCE_STATE_COPY_SOURCE; if (src->state == D3D12_RESOURCE_STATE_GENERIC_READ) { - HALIDE_CHECK(user_context, src->type == d3d12_buffer::Upload); + halide_abort_if_false(user_context, src->type == d3d12_buffer::Upload); src_barrier.Transition.StateAfter = D3D12_RESOURCE_STATE_GENERIC_READ; } else { - HALIDE_CHECK(user_context, src->state == D3D12_RESOURCE_STATE_UNORDERED_ACCESS); + halide_abort_if_false(user_context, src->state == D3D12_RESOURCE_STATE_UNORDERED_ACCESS); } } @@ -2197,9 +2197,9 @@ static void buffer_copy_command(d3d12_copy_command_list *cmdList, dst_barrier.Transition.StateBefore = dst->state; dst_barrier.Transition.StateAfter = D3D12_RESOURCE_STATE_COPY_DEST; if (dst->state == D3D12_RESOURCE_STATE_COPY_DEST) { - HALIDE_CHECK(user_context, dst->type == d3d12_buffer::ReadBack); + halide_abort_if_false(user_context, dst->type == d3d12_buffer::ReadBack); } else { - HALIDE_CHECK(user_context, dst->state == D3D12_RESOURCE_STATE_UNORDERED_ACCESS); + halide_abort_if_false(user_context, dst->state == D3D12_RESOURCE_STATE_UNORDERED_ACCESS); } } @@ -2231,7 +2231,7 @@ static void synchronize_host_and_device_buffer_contents(d3d12_copy_command_list TRACELOG; d3d12_buffer::transfer_t *xfer = buffer->xfer; - HALIDE_CHECK(user_context, (xfer != nullptr)); + halide_abort_if_false(user_context, (xfer != nullptr)); d3d12_buffer *src = nullptr; d3d12_buffer *dst = nullptr; @@ -2258,7 +2258,7 @@ static void synchronize_host_and_device_buffer_contents(d3d12_copy_command_list break; default: TRACEPRINT("UNSUPPORTED BUFFER TYPE: " << (int)buffer->type << "\n"); - HALIDE_CHECK(user_context, false); + halide_abort_if_false(user_context, false); break; } @@ -2295,7 +2295,7 @@ static void d3d12compute_device_sync_internal(d3d12_device *device, d3d12_buffer // decrement the reference counter that was incremented by 'suballocate()' uint64_t use_count = __atomic_sub_fetch(&staging_buffer->ref_count, 1, __ATOMIC_SEQ_CST); // for now, we expect to have been the only one with pending transfer on the staging buffer: - HALIDE_CHECK(user_context, (use_count == 0)); + halide_abort_if_false(user_context, (use_count == 0)); dev_buffer->xfer = nullptr; } } @@ -2308,28 +2308,28 @@ static int d3d12compute_buffer_copy(d3d12_device *device, uint64_t num_bytes) { TRACELOG; - HALIDE_CHECK(user_context, device); - HALIDE_CHECK(user_context, src); - HALIDE_CHECK(user_context, dst); - HALIDE_CHECK(user_context, (src->type != d3d12_buffer::Unknown)); - HALIDE_CHECK(user_context, (dst->type != d3d12_buffer::Unknown)); + halide_abort_if_false(user_context, device); + halide_abort_if_false(user_context, src); + halide_abort_if_false(user_context, dst); + halide_abort_if_false(user_context, (src->type != d3d12_buffer::Unknown)); + halide_abort_if_false(user_context, (dst->type != d3d12_buffer::Unknown)); // constant buffers are only used internally (Halide never expose them) // (uploads to constant buffers are managed automatically by Map/Unmap) - HALIDE_CHECK(user_context, (src->type != d3d12_buffer::Constant)); - HALIDE_CHECK(user_context, (dst->type != d3d12_buffer::Constant)); + halide_abort_if_false(user_context, (src->type != d3d12_buffer::Constant)); + halide_abort_if_false(user_context, (dst->type != d3d12_buffer::Constant)); - HALIDE_CHECK(user_context, num_bytes > 0); + halide_abort_if_false(user_context, num_bytes > 0); if (src->type == d3d12_buffer::Upload) { // host-to-device via staging buffer: - HALIDE_CHECK(user_context, (dst->type != d3d12_buffer::Upload)); - HALIDE_CHECK(user_context, (dst->type != d3d12_buffer::ReadBack)); + halide_abort_if_false(user_context, (dst->type != d3d12_buffer::Upload)); + halide_abort_if_false(user_context, (dst->type != d3d12_buffer::ReadBack)); // TODO: assert that offsets and sizes are within bounds d3d12_buffer::transfer_t xfer = {}; xfer.staging = src; xfer.offset = src_byte_offset; xfer.size = num_bytes; - HALIDE_CHECK(user_context, (dst->xfer == nullptr)); + halide_abort_if_false(user_context, (dst->xfer == nullptr)); dst->xfer = &xfer; d3d12compute_device_sync_internal(device, dst); return 0; @@ -2337,24 +2337,24 @@ static int d3d12compute_buffer_copy(d3d12_device *device, if (dst->type == d3d12_buffer::ReadBack) { // device-to-host via staging buffer: - HALIDE_CHECK(user_context, (src->type != d3d12_buffer::Upload)); - HALIDE_CHECK(user_context, (src->type != d3d12_buffer::ReadBack)); + halide_abort_if_false(user_context, (src->type != d3d12_buffer::Upload)); + halide_abort_if_false(user_context, (src->type != d3d12_buffer::ReadBack)); // TODO: assert that offsets and sizes are within bounds d3d12_buffer::transfer_t xfer = {}; xfer.staging = dst; xfer.offset = dst_byte_offset; xfer.size = num_bytes; - HALIDE_CHECK(user_context, (src->xfer == nullptr)); + halide_abort_if_false(user_context, (src->xfer == nullptr)); src->xfer = &xfer; d3d12compute_device_sync_internal(device, src); return 0; } // device-to-device: - HALIDE_CHECK(user_context, (src->type != d3d12_buffer::Upload)); - HALIDE_CHECK(user_context, (dst->type != d3d12_buffer::Upload)); - HALIDE_CHECK(user_context, (src->type != d3d12_buffer::ReadBack)); - HALIDE_CHECK(user_context, (dst->type != d3d12_buffer::ReadBack)); + halide_abort_if_false(user_context, (src->type != d3d12_buffer::Upload)); + halide_abort_if_false(user_context, (dst->type != d3d12_buffer::Upload)); + halide_abort_if_false(user_context, (src->type != d3d12_buffer::ReadBack)); + halide_abort_if_false(user_context, (dst->type != d3d12_buffer::ReadBack)); // ReadWrite, ReadOnly and WriteOnly are shader usage hints, not copy hints // (there's no need to worry about them during device-to-device transfers) @@ -2392,7 +2392,7 @@ static void *buffer_contents(d3d12_buffer *buffer) { case d3d12_buffer::WriteOnly: case d3d12_buffer::ReadWrite: { TRACEWARN("UNCHARTED TERRITORY! THIS CASE IS NOT EXPECTED TO HAPPEN FOR NOW!\n"); - HALIDE_CHECK(user_context, false); + halide_abort_if_false(user_context, false); D3D12ContextHolder d3d12_context(user_context, true); if (d3d12_context.error != 0) { @@ -2415,11 +2415,11 @@ static void *buffer_contents(d3d12_buffer *buffer) { case d3d12_buffer::Unknown: default: TRACEPRINT("UNSUPPORTED BUFFER TYPE: " << (int)buffer->type << "\n"); - HALIDE_CHECK(user_context, false); + halide_abort_if_false(user_context, false); break; } - HALIDE_CHECK(user_context, pData); + halide_abort_if_false(user_context, pData); return pData; } @@ -2458,14 +2458,14 @@ static int d3d12_create_context(void *user_context) { int status = halide_error_code_success; - HALIDE_CHECK(user_context, (device == nullptr)); + halide_abort_if_false(user_context, (device == nullptr)); device = D3D12CreateSystemDefaultDevice(user_context); if (device == nullptr) { status = halide_error_code_generic_error; } if (status == halide_error_code_success) { - HALIDE_CHECK(user_context, (rootSignature == nullptr)); + halide_abort_if_false(user_context, (rootSignature == nullptr)); rootSignature = D3D12CreateMasterRootSignature((*device)); if (rootSignature == nullptr) { status = halide_error_code_generic_error; @@ -2473,7 +2473,7 @@ static int d3d12_create_context(void *user_context) { } if (status == halide_error_code_success) { - HALIDE_CHECK(user_context, (queue == nullptr)); + halide_abort_if_false(user_context, (queue == nullptr)); queue = new_command_queue(device); if (queue == nullptr) { status = halide_error_code_generic_error; @@ -2481,7 +2481,7 @@ static int d3d12_create_context(void *user_context) { } if (status == halide_error_code_success) { - HALIDE_CHECK(user_context, (cmd_allocator_main == nullptr)); + halide_abort_if_false(user_context, (cmd_allocator_main == nullptr)); cmd_allocator_main = new_command_allocator(device); if (cmd_allocator_main == nullptr) { status = halide_error_code_generic_error; @@ -2491,8 +2491,8 @@ static int d3d12_create_context(void *user_context) { if (status == halide_error_code_success) { // NOTE(marcos): a small amount of hard-coded staging buffer storage is // sufficient to get started as suballocations will grow them as needed - HALIDE_CHECK(user_context, (upload == 0)); - HALIDE_CHECK(user_context, (readback == 0)); + halide_abort_if_false(user_context, (upload == 0)); + halide_abort_if_false(user_context, (readback == 0)); size_t heap_size = 4 * 1024 * 1024; upload = new_upload_buffer(device, heap_size); readback = new_readback_buffer(device, heap_size); @@ -2537,7 +2537,7 @@ WEAK int halide_d3d12compute_acquire_context(void *user_context, halide_d3d12com halide_start_clock(user_context); #endif - HALIDE_CHECK(user_context, &thread_lock != nullptr); + halide_abort_if_false(user_context, &thread_lock != nullptr); while (__atomic_test_and_set(&thread_lock, __ATOMIC_ACQUIRE)) { } @@ -2554,7 +2554,7 @@ WEAK int halide_d3d12compute_acquire_context(void *user_context, halide_d3d12com // If the device has already been initialized, // ensure the queue has as well. - HALIDE_CHECK(user_context, (device == nullptr) || (queue != nullptr)); + halide_abort_if_false(user_context, (device == nullptr) || (queue != nullptr)); *device_ret = device; *queue_ret = queue; @@ -2576,7 +2576,7 @@ static void d3d12_debug_dump(error &err) { return; } - HALIDE_CHECK(user_context, (dxgiAdapter != nullptr)); + halide_abort_if_false(user_context, (dxgiAdapter != nullptr)); DXGI_ADAPTER_DESC1 desc = {}; if (FAILED(dxgiAdapter->GetDesc1(&desc))) { err << "Unable to retrieve information about the device adapter.\n"; @@ -2681,11 +2681,11 @@ WEAK int halide_d3d12compute_device_malloc(void *user_context, halide_buffer_t * } size_t size = buf->size_in_bytes(); - HALIDE_CHECK(user_context, size > 0); + halide_abort_if_false(user_context, size > 0); // Check all strides positive for (int i = 0; i < buf->dimensions; i++) { - HALIDE_CHECK(user_context, buf->dim[i].stride >= 0); + halide_abort_if_false(user_context, buf->dim[i].stride >= 0); } d3d12_buffer *d3d12_buf = d3d12_allocation_cache_get_buffer(user_context, size); @@ -2860,7 +2860,7 @@ int do_multidimensional_copy(d3d12_device *device, const device_copy &c, if (dimensions == 0) { d3d12_buffer *dsrc = reinterpret_cast(c.src); d3d12_buffer *ddst = reinterpret_cast(c.dst); - HALIDE_CHECK(user_context, (dsrc->halide_type == ddst->halide_type)); + halide_abort_if_false(user_context, (dsrc->halide_type == ddst->halide_type)); TRACEPRINT("src_offset: " << src_offset << "\n"); TRACEPRINT("dst_offset: " << dst_offset << "\n"); TRACEPRINT("c.chunk_size: " << c.chunk_size << "\n"); @@ -2890,8 +2890,8 @@ int do_multidimensional_copy(d3d12_device *device, const device_copy &c, WEAK int halide_d3d12compute_copy_to_device(void *user_context, halide_buffer_t *buffer) { TRACELOG; - HALIDE_CHECK(user_context, buffer); - HALIDE_CHECK(user_context, buffer->host && buffer->device); + halide_abort_if_false(user_context, buffer); + halide_abort_if_false(user_context, buffer->host && buffer->device); D3D12ContextHolder d3d12_context(user_context, true); if (d3d12_context.error != 0) { @@ -2900,9 +2900,9 @@ WEAK int halide_d3d12compute_copy_to_device(void *user_context, halide_buffer_t // 1. memcpy from halide host memory to "upload" staging memory device_copy c = make_host_to_device_copy(buffer); - HALIDE_CHECK(user_context, (c.dst == buffer->device)); + halide_abort_if_false(user_context, (c.dst == buffer->device)); d3d12_buffer *dev_buffer = peel_buffer(buffer); - HALIDE_CHECK(user_context, buffer->size_in_bytes() == dev_buffer->sizeInBytes); + halide_abort_if_false(user_context, buffer->size_in_bytes() == dev_buffer->sizeInBytes); size_t total_size = dev_buffer->sizeInBytes; d3d12_buffer *staging = &upload; size_t staging_byte_offset = suballocate(d3d12_context.device, staging, total_size); @@ -2933,10 +2933,10 @@ WEAK int halide_d3d12compute_copy_to_device(void *user_context, halide_buffer_t WEAK int halide_d3d12compute_copy_to_host(void *user_context, halide_buffer_t *buffer) { TRACELOG; - HALIDE_CHECK(user_context, buffer); - HALIDE_CHECK(user_context, buffer->host && buffer->device); + halide_abort_if_false(user_context, buffer); + halide_abort_if_false(user_context, buffer->host && buffer->device); if (buffer->dimensions > MAX_COPY_DIMS) { - HALIDE_CHECK(user_context, false); + halide_abort_if_false(user_context, false); return halide_error_code_copy_to_host_failed; } @@ -2948,7 +2948,7 @@ WEAK int halide_d3d12compute_copy_to_host(void *user_context, halide_buffer_t *b // 1. download data from device (copy to the "readback" staging memory): d3d12_buffer *dev_buffer = peel_buffer(buffer); d3d12_buffer *staging = &readback; - HALIDE_CHECK(user_context, buffer->size_in_bytes() == dev_buffer->sizeInBytes); + halide_abort_if_false(user_context, buffer->size_in_bytes() == dev_buffer->sizeInBytes); size_t total_size = dev_buffer->sizeInBytes; size_t dev_byte_offset = dev_buffer->offsetInBytes; // handle cropping size_t staging_byte_offset = suballocate(d3d12_context.device, staging, total_size); @@ -2989,7 +2989,7 @@ WEAK int halide_d3d12compute_run(void *user_context, d3d12_library *library{}; bool found = compilation_cache.lookup(device, state_ptr, library); - HALIDE_CHECK(user_context, found && library != nullptr); + halide_abort_if_false(user_context, found && library != nullptr); d3d12_frame *frame = acquire_frame(device); d3d12_compute_command_list *cmdList = frame->cmd_list; @@ -3003,7 +3003,7 @@ WEAK int halide_d3d12compute_run(void *user_context, TRACE_SCOPE("kernel shader selection"); function = new_function_with_name(device, library, entry_name, strlen(entry_name), shared_mem_bytes, threadsX, threadsY, threadsZ); - HALIDE_CHECK(user_context, function); + halide_abort_if_false(user_context, function); pipeline_state = function->pipeline_state; set_compute_pipeline_state(cmdList, pipeline_state, function, binder); } @@ -3040,13 +3040,13 @@ WEAK int halide_d3d12compute_run(void *user_context, // multiple of that power-of-two. halide_type_t arg_type = arg_types[i]; arg_sizes[i] = arg_type.bytes(); - HALIDE_CHECK(user_context, (arg_sizes[i] & (arg_sizes[i] - 1)) == 0); + halide_abort_if_false(user_context, (arg_sizes[i] & (arg_sizes[i] - 1)) == 0); // We can ignore vector arguments since they never show up in constant // blocks. Having to worry about scalar parameters only is convenient // since in HLSL SM 5.1 all scalar types are 32bit: - HALIDE_CHECK(user_context, arg_type.lanes == 1); - HALIDE_CHECK(user_context, arg_sizes[i] > 0); - HALIDE_CHECK(user_context, arg_sizes[i] <= 4); + halide_abort_if_false(user_context, arg_type.lanes == 1); + halide_abort_if_false(user_context, arg_sizes[i] > 0); + halide_abort_if_false(user_context, arg_sizes[i] <= 4); size_t packed_size = 4; // force the final "packed" argument to be 32bit total_uniform_args_size = (total_uniform_args_size + packed_size - 1) & ~(packed_size - 1); total_uniform_args_size += packed_size; @@ -3079,7 +3079,7 @@ WEAK int halide_d3d12compute_run(void *user_context, } const halide_type_t arg_type = arg_types[i]; if (arg_type.code == halide_type_float) { - HALIDE_CHECK(user_context, (arg_type.bits == 32)); + halide_abort_if_false(user_context, (arg_type.bits == 32)); float &uniform_value = ((float &)uniform_word); uniform_value = *((float *)args[i]); TRACELEVEL(3, "args[" << i << "] -> float32 = " << uniform_value << "\n"); @@ -3095,7 +3095,7 @@ WEAK int halide_d3d12compute_run(void *user_context, } else if (arg_type.bits == 32) { uniform_value = *((int32_t *)args[i]); } else { - HALIDE_CHECK(user_context, false); + halide_abort_if_false(user_context, false); } TRACELEVEL(3, "args[" << i << "] -> int32 = " << uniform_value << "\n"); } else if (arg_type.code == halide_type_uint) { @@ -3110,17 +3110,17 @@ WEAK int halide_d3d12compute_run(void *user_context, } else if (arg_type.bits == 32) { uniform_value = *((uint32_t *)args[i]); } else { - HALIDE_CHECK(user_context, false); + halide_abort_if_false(user_context, false); } TRACELEVEL(3, "args[" << i << "] -> uint32 = " << uniform_value << "\n"); } else { - HALIDE_CHECK(user_context, false); + halide_abort_if_false(user_context, false); } memcpy(&uniform_bytes[offset], &uniform_word, uniform_size); offset = (offset + uniform_size - 1) & ~(uniform_size - 1); offset += uniform_size; } - HALIDE_CHECK(user_context, offset == total_uniform_args_size); + halide_abort_if_false(user_context, offset == total_uniform_args_size); } { @@ -3231,7 +3231,7 @@ WEAK int halide_d3d12compute_buffer_copy(void *user_context, struct halide_buffe struct halide_buffer_t *dst) { TRACELOG; - HALIDE_CHECK(user_context, (src->dimensions == dst->dimensions)); + halide_abort_if_false(user_context, (src->dimensions == dst->dimensions)); const int dimensions = dst->dimensions; if (dimensions > MAX_COPY_DIMS) { error(user_context) << "Buffer has too many dimensions to copy to/from GPU.\n"; @@ -3239,12 +3239,12 @@ WEAK int halide_d3d12compute_buffer_copy(void *user_context, struct halide_buffe } // We only handle copies to d3d12 device or to host - HALIDE_CHECK(user_context, (dst_device_interface == nullptr) || - (dst_device_interface == &d3d12compute_device_interface)); + halide_abort_if_false(user_context, (dst_device_interface == nullptr) || + (dst_device_interface == &d3d12compute_device_interface)); if ((src->device_dirty() || src->host == nullptr) && src->device_interface != &d3d12compute_device_interface) { - HALIDE_CHECK(user_context, dst_device_interface == &d3d12compute_device_interface); + halide_abort_if_false(user_context, dst_device_interface == &d3d12compute_device_interface); // This is handled at the higher level. return halide_error_code_incompatible_device_interface; } @@ -3254,8 +3254,8 @@ WEAK int halide_d3d12compute_buffer_copy(void *user_context, struct halide_buffe (src->host_dirty() && src->host != nullptr); bool to_host = !dst_device_interface; - HALIDE_CHECK(user_context, from_host || src->device); - HALIDE_CHECK(user_context, to_host || dst->device); + halide_abort_if_false(user_context, from_host || src->device); + halide_abort_if_false(user_context, to_host || dst->device); device_copy c = make_buffer_copy(src, from_host, dst, to_host); MAYBE_UNUSED(c); @@ -3281,13 +3281,13 @@ WEAK int halide_d3d12compute_buffer_copy(void *user_context, struct halide_buffe if (from_host) { // host-to-device: TRACEPRINT("host-to-device case\n"); - HALIDE_CHECK(user_context, !to_host); - HALIDE_CHECK(user_context, (dst->device_interface == &d3d12compute_device_interface)); - HALIDE_CHECK(user_context, (src->device_interface != &d3d12compute_device_interface)); - HALIDE_CHECK(user_context, (src->host != nullptr)); + halide_abort_if_false(user_context, !to_host); + halide_abort_if_false(user_context, (dst->device_interface == &d3d12compute_device_interface)); + halide_abort_if_false(user_context, (src->device_interface != &d3d12compute_device_interface)); + halide_abort_if_false(user_context, (src->host != nullptr)); // it's possible for 'dst->host' to be null, so we can't always memcpy from 'src->host' // to 'dst-host' and push/sync changes with 'halide_d3d12compute_copy_to_device' ... - HALIDE_CHECK(user_context, (dst->device == c.dst)); + halide_abort_if_false(user_context, (dst->device == c.dst)); if (dst->host != nullptr) { // 1. copy 'src->host' buffer to 'dst->host' buffer: // host buffers already account for the beginning of cropped regions @@ -3316,18 +3316,18 @@ WEAK int halide_d3d12compute_buffer_copy(void *user_context, struct halide_buffe d3d12compute_buffer_copy(d3d12_context.device, staging, ddst, staging_byte_offset, dst_byte_offset, total_size); uint64_t use_count = __atomic_load_n(&staging->ref_count, __ATOMIC_SEQ_CST); - HALIDE_CHECK(user_context, (use_count == 0)); + halide_abort_if_false(user_context, (use_count == 0)); } } else { // device-to-host: TRACEPRINT("device-to-host case\n"); - HALIDE_CHECK(user_context, to_host); - HALIDE_CHECK(user_context, (src->device_interface == &d3d12compute_device_interface)); - HALIDE_CHECK(user_context, (dst->device_interface == nullptr)); - HALIDE_CHECK(user_context, (dst->host != nullptr)); + halide_abort_if_false(user_context, to_host); + halide_abort_if_false(user_context, (src->device_interface == &d3d12compute_device_interface)); + halide_abort_if_false(user_context, (dst->device_interface == nullptr)); + halide_abort_if_false(user_context, (dst->host != nullptr)); // it's possible for 'src->host' to be null, so we can't always pull/sync changes with // 'halide_d3d12compute_copy_to_host' and then memcpy from 'src->host' to 'dst-host'... - HALIDE_CHECK(user_context, (src->device == c.src)); + halide_abort_if_false(user_context, (src->device == c.src)); if (src->host != nullptr) { // 1. sync 'src->device' buffer with 'src->host' buffer: halide_d3d12compute_copy_to_host(user_context, src); @@ -3356,7 +3356,7 @@ WEAK int halide_d3d12compute_buffer_copy(void *user_context, struct halide_buffe c.dst = reinterpret_cast(dst->host) + 0; copy_memory(c, user_context); uint64_t use_count = __atomic_load_n(&staging->ref_count, __ATOMIC_SEQ_CST); - HALIDE_CHECK(user_context, (use_count == 0)); + halide_abort_if_false(user_context, (use_count == 0)); } } } @@ -3384,9 +3384,9 @@ WEAK int d3d12compute_device_crop_from_offset(void *user_context, } d3d12_buffer *new_handle = peel_buffer(dst); - HALIDE_CHECK(user_context, (new_handle != nullptr)); - HALIDE_CHECK(user_context, (new_handle->halide_type == dst->type)); - HALIDE_CHECK(user_context, (src->device_interface == dst->device_interface)); + halide_abort_if_false(user_context, (new_handle != nullptr)); + halide_abort_if_false(user_context, (new_handle->halide_type == dst->type)); + halide_abort_if_false(user_context, (src->device_interface == dst->device_interface)); new_handle->offset = old_handle->offset + offset; new_handle->offsetInBytes = new_handle->offset * dst->type.bytes() * dst->type.lanes; @@ -3417,7 +3417,7 @@ WEAK int halide_d3d12compute_device_crop(void *user_context, using namespace Halide::Runtime; int64_t offset = Internal::calc_device_crop_byte_offset(src, dst); // D3D12 buffer views are element-based, not byte-based - HALIDE_CHECK(user_context, (offset % src->type.bytes()) == 0); + halide_abort_if_false(user_context, (offset % src->type.bytes()) == 0); offset /= src->type.bytes(); return d3d12compute_device_crop_from_offset(user_context, src, offset, dst); } @@ -3430,7 +3430,7 @@ WEAK int halide_d3d12compute_device_slice(void *user_context, using namespace Halide::Runtime; int64_t offset = Internal::calc_device_slice_byte_offset(src, slice_dim, slice_pos); // D3D12 buffer views are element-based, not byte-based - HALIDE_CHECK(user_context, (offset % src->type.bytes()) == 0); + halide_abort_if_false(user_context, (offset % src->type.bytes()) == 0); offset /= src->type.bytes(); return d3d12compute_device_crop_from_offset(user_context, src, offset, dst); } @@ -3472,7 +3472,7 @@ WEAK int halide_d3d12compute_wrap_buffer(void *user_context, struct halide_buffe TRACELOG; ID3D12Resource *pResource = reinterpret_cast(d3d12_resource); - HALIDE_CHECK(user_context, (pResource != nullptr)); + halide_abort_if_false(user_context, (pResource != nullptr)); d3d12_buffer sbuffer = {}; sbuffer.resource = pResource; diff --git a/src/runtime/device_interface.cpp b/src/runtime/device_interface.cpp index ea0edec4b4db..274351687c86 100644 --- a/src/runtime/device_interface.cpp +++ b/src/runtime/device_interface.cpp @@ -249,7 +249,7 @@ WEAK int halide_device_free(void *user_context, struct halide_buffer_t *buf) { device_interface->impl->use_module(); result = device_interface->impl->device_free(user_context, buf); device_interface->impl->release_module(); - HALIDE_CHECK(user_context, buf->device == 0); + halide_abort_if_false(user_context, buf->device == 0); if (result) { return halide_error_code_device_free_failed; } else { @@ -314,7 +314,7 @@ WEAK int halide_device_and_host_free(void *user_context, struct halide_buffer_t device_interface->impl->use_module(); result = device_interface->impl->device_and_host_free(user_context, buf); device_interface->impl->release_module(); - HALIDE_CHECK(user_context, buf->device == 0); + halide_abort_if_false(user_context, buf->device == 0); if (result) { return halide_error_code_device_free_failed; } else { @@ -400,7 +400,7 @@ WEAK int halide_device_detach_native(void *user_context, struct halide_buffer_t device_interface->impl->use_module(); result = device_interface->impl->detach_native(user_context, buf); device_interface->impl->release_module(); - HALIDE_CHECK(user_context, buf->device == 0); + halide_abort_if_false(user_context, buf->device == 0); if (result) { result = halide_error_code_device_detach_native_failed; } diff --git a/src/runtime/gpu_context_common.h b/src/runtime/gpu_context_common.h index c244f9c09859..5041b15336dd 100644 --- a/src/runtime/gpu_context_common.h +++ b/src/runtime/gpu_context_common.h @@ -66,7 +66,7 @@ class GPUCompilationCache { } // This is a logic error that should never occur. It means the table is // full, but it should have been resized. - HALIDE_CHECK(nullptr, false); + halide_abort_if_false(nullptr, false); return false; } @@ -124,7 +124,7 @@ class GPUCompilationCache { if (old_table[i].kernel_id != kInvalidId && old_table[i].kernel_id != kDeletedId) { bool result = insert(old_table[i]); - HALIDE_CHECK(nullptr, result); // Resizing the table while resizing the table is a logic error. + halide_abort_if_false(nullptr, result); // Resizing the table while resizing the table is a logic error. } } } @@ -212,7 +212,7 @@ class GPUCompilationCache { ModuleStateT *mod; uint32_t id = (uint32_t)(uintptr_t)state_ptr; bool result = find_internal(context, id, mod, -1); - HALIDE_CHECK(user_context, result); // Value must be in cache to be released + halide_abort_if_false(user_context, result); // Value must be in cache to be released } }; diff --git a/src/runtime/hashmap.h b/src/runtime/hashmap.h index e1d39b509555..56fed5044ad1 100644 --- a/src/runtime/hashmap.h +++ b/src/runtime/hashmap.h @@ -149,7 +149,7 @@ struct HashMap { inline bool HashMap::init(void *user_context, copy_value_func _copy_value, destroy_value_func _destroy_value) { memset(&memoization_lock, 0, sizeof(halide_mutex)); - HALIDE_CHECK(nullptr, !inited); + halide_abort_if_false(nullptr, !inited); most_recently_used = nullptr; least_recently_used = nullptr; kDefaultCacheSize = 1 << 20; @@ -158,8 +158,8 @@ inline bool HashMap::init(void *user_context, copy_value_func _copy_value, destr for (auto &cache_entry_ref : cache_entries) { cache_entry_ref = nullptr; } - HALIDE_CHECK(nullptr, _copy_value); - HALIDE_CHECK(nullptr, _destroy_value); + halide_abort_if_false(nullptr, _copy_value); + halide_abort_if_false(nullptr, _destroy_value); this->copy_value = _copy_value; this->destroy_value = _destroy_value; inited = true; @@ -188,7 +188,7 @@ inline void HashMap::prune() { while (prev_hash_entry != nullptr && prev_hash_entry->next != prune_candidate) { prev_hash_entry = prev_hash_entry->next; } - HALIDE_CHECK(nullptr, prev_hash_entry != nullptr); + halide_abort_if_false(nullptr, prev_hash_entry != nullptr); prev_hash_entry->next = prune_candidate->next; } @@ -261,14 +261,14 @@ inline int HashMap::lookup(void *user_context, keys_equal(entry->key, cache_key, size)) { if (entry != most_recently_used) { - HALIDE_CHECK(user_context, entry->more_recent != nullptr); + halide_abort_if_false(user_context, entry->more_recent != nullptr); if (entry->less_recent != nullptr) { entry->less_recent->more_recent = entry->more_recent; } else { - HALIDE_CHECK(user_context, least_recently_used == entry); + halide_abort_if_false(user_context, least_recently_used == entry); least_recently_used = entry->more_recent; } - HALIDE_CHECK(user_context, entry->more_recent != nullptr); + halide_abort_if_false(user_context, entry->more_recent != nullptr); entry->more_recent->less_recent = entry->less_recent; entry->more_recent = nullptr; @@ -279,7 +279,7 @@ inline int HashMap::lookup(void *user_context, most_recently_used = entry; } - HALIDE_CHECK(user_context, (cache_value_size == entry->value_size)); + halide_abort_if_false(user_context, (cache_value_size == entry->value_size)); copy_value(cache_value, entry->value, entry->value_size); entry->in_use_count += 1; @@ -325,7 +325,7 @@ inline int HashMap::store(void *user_context, entry = entry->next) { if (entry->hash == h && entry->key_size == (size_t)size && keys_equal(entry->key, cache_key, size)) { - HALIDE_CHECK(user_context, (cache_value_size == entry->value_size)); + halide_abort_if_false(user_context, (cache_value_size == entry->value_size)); destroy_value(entry->value, entry->value_size); copy_value(entry->value, cache_value, entry->value_size); return (0); @@ -335,7 +335,7 @@ inline int HashMap::store(void *user_context, // key not found: create new entry CacheEntry *new_entry = (CacheEntry *)hashmap_malloc(user_context, sizeof(CacheEntry)); bool inited = new_entry->init(user_context, cache_key, size, h, cache_value, cache_value_size, copy_value); - HALIDE_CHECK(user_context, inited); + halide_abort_if_false(user_context, inited); uint64_t added_size = cache_value_size; current_cache_size += added_size; @@ -365,7 +365,7 @@ inline int HashMap::store(void *user_context, inline void HashMap::release(void *user_context, void *host) { debug(user_context) << "halide_memoization_cache_release\n"; // TODO(marcos): this method does not make sense on a generic hashmap... remove it? - HALIDE_CHECK(user_context, false); + halide_abort_if_false(user_context, false); debug(user_context) << "Exited halide_memoization_cache_release.\n"; } @@ -396,14 +396,14 @@ struct THashMap : public HashMap { // member functions below... static void copy_value_func(uint8_t *dst, const uint8_t *src, size_t size) { - HALIDE_CHECK(nullptr, sizeof(ValueType) == size); + halide_abort_if_false(nullptr, sizeof(ValueType) == size); ValueType *D = reinterpret_cast(dst); const ValueType *S = reinterpret_cast(src); *D = *S; } static void destroy_value_func(uint8_t *value, size_t size) { - HALIDE_CHECK(nullptr, sizeof(ValueType) == size); + halide_abort_if_false(nullptr, sizeof(ValueType) == size); ValueType *V = reinterpret_cast(value); V->~ValueType(); } diff --git a/src/runtime/hexagon_cache_allocator.cpp b/src/runtime/hexagon_cache_allocator.cpp index 4e642031ee83..c8039638da88 100644 --- a/src/runtime/hexagon_cache_allocator.cpp +++ b/src/runtime/hexagon_cache_allocator.cpp @@ -119,7 +119,7 @@ inline void *hexagon_cache_pool_get(void *user_context, size_t size, bool retry) inline void hexagon_cache_pool_put(void *user_context, void *cache_mem) { ScopedMutexLock lock(&hexagon_cache_mutex); - HALIDE_CHECK(user_context, cache_mem); + halide_abort_if_false(user_context, cache_mem); pcache_pool temp = hexagon_cache_pool; while (temp != nullptr) { if (temp->l2memory == cache_mem) { diff --git a/src/runtime/hexagon_dma.cpp b/src/runtime/hexagon_dma.cpp index 28bc8204c779..4db1e81d8f93 100644 --- a/src/runtime/hexagon_dma.cpp +++ b/src/runtime/hexagon_dma.cpp @@ -127,7 +127,7 @@ void *desc_pool_get(void *user_context) { void desc_pool_put(void *user_context, void *desc) { ScopedMutexLock lock(&hexagon_desc_mutex); - HALIDE_CHECK(user_context, desc); + halide_abort_if_false(user_context, desc); pdesc_pool temp = dma_desc_pool; while (temp != nullptr) { if (temp->descriptor == desc) { @@ -219,25 +219,25 @@ int halide_hexagon_dma_wrapper(void *user_context, struct halide_buffer_t *src, // Assert if buffer dimensions do not fulfill the format requirements if (dev->fmt == eDmaFmt_RawData) { - HALIDE_CHECK(user_context, src->dimensions <= 3); + halide_abort_if_false(user_context, src->dimensions <= 3); } if ((dev->fmt == eDmaFmt_NV12_Y) || (dev->fmt == eDmaFmt_P010_Y) || (dev->fmt == eDmaFmt_TP10_Y) || (dev->fmt == eDmaFmt_NV124R_Y)) { - HALIDE_CHECK(user_context, src->dimensions == 2); + halide_abort_if_false(user_context, src->dimensions == 2); } if ((dev->fmt == eDmaFmt_NV12_UV) || (dev->fmt == eDmaFmt_P010_UV) || (dev->fmt == eDmaFmt_TP10_UV) || (dev->fmt == eDmaFmt_NV124R_UV)) { - HALIDE_CHECK(user_context, src->dimensions == 3); - HALIDE_CHECK(user_context, src->dim[0].stride == 2); - HALIDE_CHECK(user_context, src->dim[2].stride == 1); - HALIDE_CHECK(user_context, src->dim[2].min == 0); - HALIDE_CHECK(user_context, src->dim[2].extent == 2); + halide_abort_if_false(user_context, src->dimensions == 3); + halide_abort_if_false(user_context, src->dim[0].stride == 2); + halide_abort_if_false(user_context, src->dim[2].stride == 1); + halide_abort_if_false(user_context, src->dim[2].min == 0); + halide_abort_if_false(user_context, src->dim[2].extent == 2); } t_StDmaWrapper_RoiAlignInfo stWalkSize = { @@ -258,7 +258,7 @@ int halide_hexagon_dma_wrapper(void *user_context, struct halide_buffer_t *src, } // Assert if destination stride is a multipe of recommended stride - HALIDE_CHECK(user_context, ((dst->dim[1].stride % roi_stride) == 0)); + halide_abort_if_false(user_context, ((dst->dim[1].stride % roi_stride) == 0)); // Return nullptr if descriptor is not allocated void *desc_addr = desc_pool_get(user_context); @@ -371,7 +371,7 @@ WEAK int halide_hexagon_dma_device_malloc(void *user_context, halide_buffer_t *b } size_t size = buf->size_in_bytes(); - HALIDE_CHECK(user_context, size != 0); + halide_abort_if_false(user_context, size != 0); void *mem = halide_malloc(user_context, size); if (!mem) { @@ -409,7 +409,7 @@ WEAK int halide_hexagon_dma_allocate_engine(void *user_context, void **dma_engin debug(user_context) << "Hexagon: halide_hexagon_dma_allocate_engine (user_context: " << user_context << ")\n"; - HALIDE_CHECK(user_context, dma_engine); + halide_abort_if_false(user_context, dma_engine); debug(user_context) << " dma_allocate_dma_engine -> "; *dma_engine = halide_hexagon_allocate_dma_resource(user_context); debug(user_context) << " " << dma_engine << "\n"; @@ -426,7 +426,7 @@ WEAK int halide_hexagon_dma_deallocate_engine(void *user_context, void *dma_engi << "Hexagon: halide_hexagon_dma_deallocate_engine (user_context: " << user_context << ", dma_engine: " << dma_engine << ")\n"; - HALIDE_CHECK(user_context, dma_engine); + halide_abort_if_false(user_context, dma_engine); // Its safe to free descriptors here, even on 1st engine of multi-engines deallocation, since its called outside of pipeline // If descriptors are needed on pipeline re-entry, the pool will also re-populate @@ -445,7 +445,7 @@ WEAK int halide_hexagon_dma_deallocate_engine(void *user_context, void *dma_engi namespace { inline int dma_prepare_for_copy(void *user_context, struct halide_buffer_t *buf, void *dma_engine, bool is_ubwc, t_eDmaFmt fmt, bool is_write) { - HALIDE_CHECK(user_context, dma_engine); + halide_abort_if_false(user_context, dma_engine); dma_device_handle *dev = reinterpret_cast(buf->device); dev->dma_engine = dma_engine; dev->is_ubwc = is_ubwc; @@ -494,12 +494,12 @@ WEAK int halide_hexagon_dma_buffer_copy(void *user_context, struct halide_buffer const struct halide_device_interface_t *dst_device_interface, struct halide_buffer_t *dst) { - HALIDE_CHECK(user_context, dst_device_interface == nullptr || - dst_device_interface == &hexagon_dma_device_interface); + halide_abort_if_false(user_context, dst_device_interface == nullptr || + dst_device_interface == &hexagon_dma_device_interface); if (src->device_dirty() && src->device_interface != &hexagon_dma_device_interface) { - HALIDE_CHECK(user_context, dst_device_interface == &hexagon_dma_device_interface); + halide_abort_if_false(user_context, dst_device_interface == &hexagon_dma_device_interface); // If the source is not hexagon_dma or host memory, ask the source // device interface to copy to dst host memory first. debug(user_context) << "Hexagon: src->device_interface != &hexagon_dma_device_interface\n"; @@ -515,10 +515,10 @@ WEAK int halide_hexagon_dma_buffer_copy(void *user_context, struct halide_buffer bool from_host = !src->device_dirty() && src->host != nullptr; bool to_host = !dst_device_interface; - HALIDE_CHECK(user_context, from_host || src->device); - HALIDE_CHECK(user_context, to_host || dst->device); + halide_abort_if_false(user_context, from_host || src->device); + halide_abort_if_false(user_context, to_host || dst->device); - HALIDE_CHECK(user_context, (!from_host && to_host) || (from_host && !to_host)); + halide_abort_if_false(user_context, (!from_host && to_host) || (from_host && !to_host)); debug(user_context) << "Hexagon: halide_hexagon_dma_buffer_copy (user_context: " << user_context @@ -564,7 +564,7 @@ WEAK int halide_hexagon_dma_device_crop(void *user_context, const dma_device_handle *src_dev = (dma_device_handle *)src->device; dma_device_handle *dst_dev = malloc_device_handle(); - HALIDE_CHECK(user_context, dst_dev); + halide_abort_if_false(user_context, dst_dev); dst_dev->buffer = src_dev->buffer; dst_dev->offset_wrx = src_dev->offset_wrx + dst->dim[0].min - src->dim[0].min; dst_dev->offset_wry = src_dev->offset_wry + dst->dim[1].min - src->dim[1].min; @@ -588,7 +588,7 @@ WEAK int halide_hexagon_dma_device_slice(void *user_context, << "Hexagon: halide_hexagon_dma_device_slice (user_context: " << user_context << " src: " << *src << " dst: " << *dst << ")\n"; - HALIDE_CHECK(user_context, 0); + halide_abort_if_false(user_context, 0); error(user_context) << "Hexagon: halide_hexagon_dma_device_slice not implemented\n"; return halide_error_code_generic_error; @@ -599,7 +599,7 @@ WEAK int halide_hexagon_dma_device_release_crop(void *user_context, struct halid << "Hexagon: halide_hexagon_dma_device_release_crop (user_context: " << user_context << " buf: " << *buf << ")\n"; - HALIDE_CHECK(user_context, buf->device); + halide_abort_if_false(user_context, buf->device); free((dma_device_handle *)buf->device); buf->device = 0; @@ -620,7 +620,7 @@ WEAK int halide_hexagon_dma_device_wrap_native(void *user_context, struct halide << "Hexagon: halide_hexagon_dma_device_wrap_native (user_context: " << user_context << " buf: " << *buf << " handle: " << handle << ")\n"; - HALIDE_CHECK(user_context, buf->device == 0); + halide_abort_if_false(user_context, buf->device == 0); if (buf->device != 0) { error(user_context) << "Hexagon: halide_hexagon_dma_device_wrap_native buffer already has a device\n"; return halide_error_code_device_wrap_native_failed; @@ -630,7 +630,7 @@ WEAK int halide_hexagon_dma_device_wrap_native(void *user_context, struct halide buf->device_interface->impl->use_module(); dma_device_handle *dev = malloc_device_handle(); - HALIDE_CHECK(user_context, dev); + halide_abort_if_false(user_context, dev); dev->buffer = reinterpret_cast(handle); dev->dma_engine = nullptr; dev->frame_width = buf->dim[0].extent * buf->dim[0].stride; @@ -650,7 +650,7 @@ WEAK int halide_hexagon_dma_device_detach_native(void *user_context, struct hali error(user_context) << "Hexagon: halide_hexagon_dma_device_detach_native buffer without a device\n"; return halide_error_code_device_detach_native_failed; } - HALIDE_CHECK(user_context, buf->device_interface == &hexagon_dma_device_interface); + halide_abort_if_false(user_context, buf->device_interface == &hexagon_dma_device_interface); dma_device_handle *dev = (dma_device_handle *)buf->device; free(dev); buf->device_interface->impl->release_module(); diff --git a/src/runtime/hexagon_dma_pool.cpp b/src/runtime/hexagon_dma_pool.cpp index bdc1f46ffe01..30f833b8a6c2 100644 --- a/src/runtime/hexagon_dma_pool.cpp +++ b/src/runtime/hexagon_dma_pool.cpp @@ -43,8 +43,8 @@ namespace { // In this function we pick the dma engine and assign it to a virtual engine inline void *hexagon_dma_pool_get(void *user_context, void *virtual_engine_id) { - HALIDE_CHECK(user_context, hexagon_dma_pool); - HALIDE_CHECK(user_context, virtual_engine_id); + halide_abort_if_false(user_context, hexagon_dma_pool); + halide_abort_if_false(user_context, virtual_engine_id); ScopedMutexLock lock(&hexagon_dma_pool_mutex); hexagon_dma_virtual_engine_t *virtual_engine_addr = (hexagon_dma_virtual_engine_t *)virtual_engine_id; @@ -66,7 +66,7 @@ inline void *hexagon_dma_pool_get(void *user_context, void *virtual_engine_id) { virtual_engine_addr->mapped_engines[virtual_engine_addr->num_of_engines] = j + 1; if (!hexagon_dma_pool->dma_engine_list[j].engine_addr) { hexagon_dma_pool->dma_engine_list[j].engine_addr = (void *)hDmaWrapper_AllocDma(); - HALIDE_CHECK(user_context, hexagon_dma_pool->dma_engine_list[j].engine_addr); + halide_abort_if_false(user_context, hexagon_dma_pool->dma_engine_list[j].engine_addr); } virtual_engine_addr->num_of_engines++; return hexagon_dma_pool->dma_engine_list[j].engine_addr; @@ -79,7 +79,7 @@ inline void *hexagon_dma_pool_get(void *user_context, void *virtual_engine_id) { // In this function we simply mark the dma engine as free inline int hexagon_dma_pool_put(void *user_context, void *dma_engine, void *virtual_engine_id) { - HALIDE_CHECK(user_context, virtual_engine_id); + halide_abort_if_false(user_context, virtual_engine_id); ScopedMutexLock lock(&hexagon_dma_pool_mutex); hexagon_dma_virtual_engine_t *virtual_engine_addr = (hexagon_dma_virtual_engine_t *)virtual_engine_id; @@ -100,8 +100,8 @@ extern "C" { // halide_hexagon_free_dma_resource WEAK int halide_hexagon_free_dma_resource(void *user_context, void *virtual_engine_id) { - HALIDE_CHECK(user_context, hexagon_dma_pool); - HALIDE_CHECK(user_context, virtual_engine_id); + halide_abort_if_false(user_context, hexagon_dma_pool); + halide_abort_if_false(user_context, virtual_engine_id); // Free the Real DMA Engines int nRet = halide_error_code_success; diff --git a/src/runtime/hexagon_host.cpp b/src/runtime/hexagon_host.cpp index a744e0e355db..3fae48ddd603 100644 --- a/src/runtime/hexagon_host.cpp +++ b/src/runtime/hexagon_host.cpp @@ -250,7 +250,7 @@ WEAK int halide_hexagon_initialize_kernels(void *user_context, void **state_ptr, << ", code_size: " << (int)code_size << ")\n" << ", code: " << runtime << ", code_size: " << (int)runtime_size << ")\n"; - HALIDE_CHECK(user_context, state_ptr != nullptr); + halide_abort_if_false(user_context, state_ptr != nullptr); #ifdef DEBUG_RUNTIME uint64_t t_before = halide_current_time_ns(user_context); @@ -277,7 +277,7 @@ WEAK int halide_hexagon_initialize_kernels(void *user_context, void **state_ptr, poll_log(user_context); if (result == 0) { debug(user_context) << " " << (void *)(size_t)shared_runtime << "\n"; - HALIDE_CHECK(user_context, shared_runtime != 0); + halide_abort_if_false(user_context, shared_runtime != 0); } else { debug(user_context) << " " << result << "\n"; error(user_context) << "Initialization of Hexagon kernels failed\n"; @@ -389,8 +389,8 @@ WEAK int halide_hexagon_run(void *user_context, uint64_t arg_sizes[], void *args[], int arg_flags[]) { - HALIDE_CHECK(user_context, state_ptr != nullptr); - HALIDE_CHECK(user_context, function != nullptr); + halide_abort_if_false(user_context, state_ptr != nullptr); + halide_abort_if_false(user_context, function != nullptr); int result = init_hexagon_runtime(user_context); if (result != 0) { return result; @@ -546,7 +546,7 @@ WEAK int halide_hexagon_device_malloc(void *user_context, halide_buffer_t *buf) } size_t size = buf->size_in_bytes(); - HALIDE_CHECK(user_context, size != 0); + halide_abort_if_false(user_context, size != 0); // Hexagon code generation generates clamped ramp loads in a way // that requires up to an extra vector beyond the end of the @@ -554,7 +554,7 @@ WEAK int halide_hexagon_device_malloc(void *user_context, halide_buffer_t *buf) size += 128; for (int i = 0; i < buf->dimensions; i++) { - HALIDE_CHECK(user_context, buf->dim[i].stride >= 0); + halide_abort_if_false(user_context, buf->dim[i].stride >= 0); } debug(user_context) << " allocating buffer of " << (uint64_t)size << " bytes\n"; @@ -657,7 +657,7 @@ WEAK int halide_hexagon_copy_to_device(void *user_context, halide_buffer_t *buf) uint64_t t_before = halide_current_time_ns(user_context); #endif - HALIDE_CHECK(user_context, buf->host && buf->device); + halide_abort_if_false(user_context, buf->host && buf->device); device_copy c = make_host_to_device_copy(buf); // Get the descriptor associated with the ion buffer. @@ -681,7 +681,7 @@ WEAK int halide_hexagon_copy_to_host(void *user_context, struct halide_buffer_t uint64_t t_before = halide_current_time_ns(user_context); #endif - HALIDE_CHECK(user_context, buf->host && buf->device); + halide_abort_if_false(user_context, buf->host && buf->device); device_copy c = make_device_to_host_copy(buf); // Get the descriptor associated with the ion buffer. @@ -705,7 +705,7 @@ WEAK int halide_hexagon_device_sync(void *user_context, struct halide_buffer_t * WEAK int halide_hexagon_wrap_device_handle(void *user_context, struct halide_buffer_t *buf, void *ion_buf, uint64_t size) { - HALIDE_CHECK(user_context, buf->device == 0); + halide_abort_if_false(user_context, buf->device == 0); if (buf->device != 0) { return -2; } @@ -726,7 +726,7 @@ WEAK int halide_hexagon_detach_device_handle(void *user_context, struct halide_b if (buf->device == 0) { return 0; } - HALIDE_CHECK(user_context, buf->device_interface == &hexagon_device_interface); + halide_abort_if_false(user_context, buf->device_interface == &hexagon_device_interface); ion_device_handle *handle = uint64_to_ptr(buf->device); free(handle); @@ -740,7 +740,7 @@ WEAK void *halide_hexagon_get_device_handle(void *user_context, struct halide_bu if (buf->device == 0) { return nullptr; } - HALIDE_CHECK(user_context, buf->device_interface == &hexagon_device_interface); + halide_abort_if_false(user_context, buf->device_interface == &hexagon_device_interface); ion_device_handle *handle = uint64_to_ptr(buf->device); return handle->buffer; } @@ -749,7 +749,7 @@ WEAK uint64_t halide_hexagon_get_device_size(void *user_context, struct halide_b if (buf->device == 0) { return 0; } - HALIDE_CHECK(user_context, buf->device_interface == &hexagon_device_interface); + halide_abort_if_false(user_context, buf->device_interface == &hexagon_device_interface); ion_device_handle *handle = uint64_to_ptr(buf->device); return handle->size; } @@ -774,12 +774,12 @@ WEAK int halide_hexagon_buffer_copy(void *user_context, struct halide_buffer_t * const struct halide_device_interface_t *dst_device_interface, struct halide_buffer_t *dst) { // We only handle copies to hexagon buffers or to host - HALIDE_CHECK(user_context, dst_device_interface == nullptr || - dst_device_interface == &hexagon_device_interface); + halide_abort_if_false(user_context, dst_device_interface == nullptr || + dst_device_interface == &hexagon_device_interface); if ((src->device_dirty() || src->host == nullptr) && src->device_interface != &hexagon_device_interface) { - HALIDE_CHECK(user_context, dst_device_interface == &hexagon_device_interface); + halide_abort_if_false(user_context, dst_device_interface == &hexagon_device_interface); // This is handled at the higher level. return halide_error_code_incompatible_device_interface; } @@ -789,8 +789,8 @@ WEAK int halide_hexagon_buffer_copy(void *user_context, struct halide_buffer_t * (src->host_dirty() && src->host != nullptr); bool to_host = !dst_device_interface; - HALIDE_CHECK(user_context, from_host || src->device); - HALIDE_CHECK(user_context, to_host || dst->device); + halide_abort_if_false(user_context, from_host || src->device); + halide_abort_if_false(user_context, to_host || dst->device); #ifdef DEBUG_RUNTIME uint64_t t_before = halide_current_time_ns(user_context); diff --git a/src/runtime/metal.cpp b/src/runtime/metal.cpp index a6d8306dbb66..fda6ae442d86 100644 --- a/src/runtime/metal.cpp +++ b/src/runtime/metal.cpp @@ -340,7 +340,7 @@ extern "C" { // previous call (if any) has not yet been released via halide_release_metal_context. WEAK int halide_metal_acquire_context(void *user_context, mtl_device **device_ret, mtl_command_queue **queue_ret, bool create) { - HALIDE_CHECK(user_context, &thread_lock != nullptr); + halide_abort_if_false(user_context, &thread_lock != nullptr); while (__atomic_test_and_set(&thread_lock, __ATOMIC_ACQUIRE)) { } @@ -369,7 +369,7 @@ WEAK int halide_metal_acquire_context(void *user_context, mtl_device **device_re // If the device has already been initialized, // ensure the queue has as well. - HALIDE_CHECK(user_context, (device == nullptr) || (queue != nullptr)); + halide_abort_if_false(user_context, (device == nullptr) || (queue != nullptr)); *device_ret = device; *queue_ret = queue; @@ -452,7 +452,7 @@ WEAK int halide_metal_device_malloc(void *user_context, halide_buffer_t *buf) { << ", buf: " << buf << ")\n"; size_t size = buf->size_in_bytes(); - HALIDE_CHECK(user_context, size != 0); + halide_abort_if_false(user_context, size != 0); if (buf->device) { // This buffer already has a device allocation return 0; @@ -460,7 +460,7 @@ WEAK int halide_metal_device_malloc(void *user_context, halide_buffer_t *buf) { // Check all strides positive for (int i = 0; i < buf->dimensions; i++) { - HALIDE_CHECK(user_context, buf->dim[i].stride >= 0); + halide_abort_if_false(user_context, buf->dim[i].stride >= 0); } debug(user_context) << " allocating " << *buf << "\n"; @@ -513,7 +513,7 @@ WEAK int halide_metal_device_free(void *user_context, halide_buffer_t *buf) { #endif device_handle *handle = (device_handle *)buf->device; - HALIDE_CHECK(user_context, (((device_handle *)buf->device)->offset == 0) && "halide_metal_device_free on buffer obtained from halide_device_crop"); + halide_abort_if_false(user_context, (((device_handle *)buf->device)->offset == 0) && "halide_metal_device_free on buffer obtained from halide_device_crop"); release_ns_object(handle->buf); free(handle); @@ -545,7 +545,7 @@ WEAK int halide_metal_initialize_kernels(void *user_context, void **state_ptr, c source, source_size)) { return halide_error_code_generic_error; } - HALIDE_CHECK(user_context, library != nullptr); + halide_abort_if_false(user_context, library != nullptr); #ifdef DEBUG_RUNTIME uint64_t t_after = halide_current_time_ns(user_context); @@ -645,7 +645,7 @@ WEAK int halide_metal_copy_to_device(void *user_context, halide_buffer_t *buffer return metal_context.error; } - HALIDE_CHECK(user_context, buffer->host && buffer->device); + halide_abort_if_false(user_context, buffer->host && buffer->device); device_copy c = make_host_to_device_copy(buffer); mtl_buffer *metal_buffer = ((device_handle *)c.dst)->buf; @@ -659,7 +659,7 @@ WEAK int halide_metal_copy_to_device(void *user_context, halide_buffer_t *buffer if (is_buffer_managed(metal_buffer)) { size_t total_size = buffer->size_in_bytes(); - HALIDE_CHECK(user_context, total_size != 0); + halide_abort_if_false(user_context, total_size != 0); NSRange total_extent; total_extent.location = 0; total_extent.length = total_size; @@ -688,8 +688,8 @@ WEAK int halide_metal_copy_to_host(void *user_context, halide_buffer_t *buffer) halide_metal_device_sync_internal(metal_context.queue, buffer); - HALIDE_CHECK(user_context, buffer->host && buffer->device); - HALIDE_CHECK(user_context, buffer->dimensions <= MAX_COPY_DIMS); + halide_abort_if_false(user_context, buffer->host && buffer->device); + halide_abort_if_false(user_context, buffer->dimensions <= MAX_COPY_DIMS); if (buffer->dimensions > MAX_COPY_DIMS) { return -1; } @@ -739,7 +739,7 @@ WEAK int halide_metal_run(void *user_context, mtl_library *library{}; bool found = compilation_cache.lookup(metal_context.device, state_ptr, library); - HALIDE_CHECK(user_context, found && library != nullptr); + halide_abort_if_false(user_context, found && library != nullptr); mtl_function *function = new_function_with_name(library, entry_name, strlen(entry_name)); if (function == nullptr) { @@ -775,7 +775,7 @@ WEAK int halide_metal_run(void *user_context, // TODO(zalman): This seems fishy - if the arguments are // not already sorted in decreasing order of size, wrong // results occur. To repro, remove the sorting code in CodeGen_GPU_Host - HALIDE_CHECK(user_context, (arg_sizes[i] & (arg_sizes[i] - 1)) == 0); + halide_abort_if_false(user_context, (arg_sizes[i] & (arg_sizes[i] - 1)) == 0); total_args_size = (total_args_size + arg_sizes[i] - 1) & ~(arg_sizes[i] - 1); total_args_size += arg_sizes[i]; } @@ -799,7 +799,7 @@ WEAK int halide_metal_run(void *user_context, // in the struct, per email communication from Apple size_t padded_args_size = (total_args_size + 4 - 1) & ~((size_t)(4 - 1)); debug(user_context) << "Total args size is " << (uint64_t)total_args_size << " and with padding, size is " << (uint64_t)padded_args_size << "\n"; - HALIDE_CHECK(user_context, padded_args_size >= total_args_size); + halide_abort_if_false(user_context, padded_args_size >= total_args_size); if (padded_args_size < 4096 && metal_api_supports_set_bytes) { args_ptr = (char *)small_args_buffer; @@ -820,7 +820,7 @@ WEAK int halide_metal_run(void *user_context, offset += arg_sizes[i]; } } - HALIDE_CHECK(user_context, offset == total_args_size); + halide_abort_if_false(user_context, offset == total_args_size); if (total_args_size < 4096 && metal_api_supports_set_bytes) { set_input_buffer_from_bytes(encoder, small_args_buffer, padded_args_size, buffer_index); @@ -833,7 +833,7 @@ WEAK int halide_metal_run(void *user_context, for (size_t i = 0; arg_sizes[i] != 0; i++) { if (arg_is_buffer[i]) { - HALIDE_CHECK(user_context, arg_sizes[i] == sizeof(uint64_t)); + halide_abort_if_false(user_context, arg_sizes[i] == sizeof(uint64_t)); device_handle *handle = (device_handle *)((halide_buffer_t *)args[i])->device; set_input_buffer(encoder, handle->buf, handle->offset, buffer_index); buffer_index++; @@ -901,12 +901,12 @@ WEAK int halide_metal_buffer_copy(void *user_context, struct halide_buffer_t *sr } // We only handle copies to metal buffers or to host - HALIDE_CHECK(user_context, dst_device_interface == nullptr || - dst_device_interface == &metal_device_interface); + halide_abort_if_false(user_context, dst_device_interface == nullptr || + dst_device_interface == &metal_device_interface); if ((src->device_dirty() || src->host == nullptr) && src->device_interface != &metal_device_interface) { - HALIDE_CHECK(user_context, dst_device_interface == &metal_device_interface); + halide_abort_if_false(user_context, dst_device_interface == &metal_device_interface); // This is handled at the higher level. return halide_error_code_incompatible_device_interface; } @@ -916,8 +916,8 @@ WEAK int halide_metal_buffer_copy(void *user_context, struct halide_buffer_t *sr (src->host_dirty() && src->host != nullptr); bool to_host = !dst_device_interface; - HALIDE_CHECK(user_context, from_host || src->device); - HALIDE_CHECK(user_context, to_host || dst->device); + halide_abort_if_false(user_context, from_host || src->device); + halide_abort_if_false(user_context, to_host || dst->device); device_copy c = make_buffer_copy(src, from_host, dst, to_host); @@ -962,7 +962,7 @@ WEAK int halide_metal_buffer_copy(void *user_context, struct halide_buffer_t *sr halide_metal_device_sync_internal(metal_context.queue, dst); dst_buffer = ((device_handle *)c.dst)->buf; - HALIDE_CHECK(user_context, from_host); + halide_abort_if_false(user_context, from_host); c.dst = (uint64_t)buffer_contents(dst_buffer) + ((device_handle *)c.dst)->offset; } @@ -971,7 +971,7 @@ WEAK int halide_metal_buffer_copy(void *user_context, struct halide_buffer_t *sr if (!to_host) { if (is_buffer_managed(dst_buffer)) { size_t total_size = dst->size_in_bytes(); - HALIDE_CHECK(user_context, total_size != 0); + halide_abort_if_false(user_context, total_size != 0); NSRange total_extent; total_extent.location = 0; total_extent.length = total_size; @@ -1062,7 +1062,7 @@ WEAK int halide_metal_device_release_crop(void *user_context, } WEAK int halide_metal_wrap_buffer(void *user_context, struct halide_buffer_t *buf, uint64_t buffer) { - HALIDE_CHECK(user_context, buf->device == 0); + halide_abort_if_false(user_context, buf->device == 0); if (buf->device != 0) { return -2; } @@ -1084,7 +1084,7 @@ WEAK int halide_metal_detach_buffer(void *user_context, struct halide_buffer_t * if (buf->device == 0) { return 0; } - HALIDE_CHECK(user_context, buf->device_interface == &metal_device_interface); + halide_abort_if_false(user_context, buf->device_interface == &metal_device_interface); buf->device_interface->impl->release_module(); buf->device_interface = nullptr; free((device_handle *)buf->device); @@ -1096,7 +1096,7 @@ WEAK uintptr_t halide_metal_get_buffer(void *user_context, struct halide_buffer_ if (buf->device == 0) { return 0; } - HALIDE_CHECK(user_context, buf->device_interface == &metal_device_interface); + halide_abort_if_false(user_context, buf->device_interface == &metal_device_interface); return (uintptr_t)(((device_handle *)buf->device)->buf); } @@ -1104,7 +1104,7 @@ WEAK uint64_t halide_metal_get_crop_offset(void *user_context, struct halide_buf if (buf->device == 0) { return 0; } - HALIDE_CHECK(user_context, buf->device_interface == &metal_device_interface); + halide_abort_if_false(user_context, buf->device_interface == &metal_device_interface); return (uint64_t)(((device_handle *)buf->device)->offset); } diff --git a/src/runtime/opencl.cpp b/src/runtime/opencl.cpp index 5922cba81dc0..8c7236a41c34 100644 --- a/src/runtime/opencl.cpp +++ b/src/runtime/opencl.cpp @@ -64,7 +64,7 @@ ALWAYS_INLINE T get_cl_symbol(void *user_context, const char *name, bool req) { // Load an OpenCL shared object/dll, and get the function pointers for the OpenCL API from it. WEAK void load_libopencl(void *user_context) { debug(user_context) << " load_libopencl (user_context: " << user_context << ")\n"; - HALIDE_CHECK(user_context, clCreateContext == nullptr); + halide_abort_if_false(user_context, clCreateContext == nullptr); // clang-format off #define CL_FN(ret, fn, args) fn = get_cl_symbol(user_context, #fn, true); // NOLINT(bugprone-macro-parentheses) @@ -212,16 +212,16 @@ WEAK const char *halide_opencl_get_build_options(void *user_context) { WEAK int halide_acquire_cl_context(void *user_context, cl_context *ctx, cl_command_queue *q, bool create = true) { // TODO: Should we use a more "assertive" assert? These asserts do // not block execution on failure. - HALIDE_CHECK(user_context, ctx != nullptr); - HALIDE_CHECK(user_context, q != nullptr); + halide_abort_if_false(user_context, ctx != nullptr); + halide_abort_if_false(user_context, q != nullptr); - HALIDE_CHECK(user_context, &thread_lock != nullptr); + halide_abort_if_false(user_context, &thread_lock != nullptr); while (__atomic_test_and_set(&thread_lock, __ATOMIC_ACQUIRE)) { } // If the context has not been initialized, initialize it now. - HALIDE_CHECK(user_context, &context != nullptr); - HALIDE_CHECK(user_context, &command_queue != nullptr); + halide_abort_if_false(user_context, &context != nullptr); + halide_abort_if_false(user_context, &command_queue != nullptr); if (!context && create) { cl_int error = create_opencl_context(user_context, &context, &command_queue); if (error != CL_SUCCESS) { @@ -325,7 +325,7 @@ WEAK bool validate_device_pointer(void *user_context, halide_buffer_t *buf, size << ", actual allocated " << (uint64_t)real_size << "\n"; if (size) { - HALIDE_CHECK(user_context, real_size >= (size + offset) && "Validating pointer with insufficient size"); + halide_abort_if_false(user_context, real_size >= (size + offset) && "Validating pointer with insufficient size"); } return true; } @@ -336,8 +336,8 @@ WEAK int create_opencl_context(void *user_context, cl_context *ctx, cl_command_q debug(user_context) << " create_opencl_context (user_context: " << user_context << ")\n"; - HALIDE_CHECK(user_context, ctx != nullptr && *ctx == nullptr); - HALIDE_CHECK(user_context, q != nullptr && *q == nullptr); + halide_abort_if_false(user_context, ctx != nullptr && *ctx == nullptr); + halide_abort_if_false(user_context, q != nullptr && *q == nullptr); if (clGetPlatformIDs == nullptr) { error(user_context) << "CL: clGetPlatformIDs not found\n"; @@ -650,7 +650,7 @@ WEAK int halide_opencl_device_free(void *user_context, halide_buffer_t *buf) { } cl_mem dev_ptr = ((device_handle *)buf->device)->mem; - HALIDE_CHECK(user_context, (((device_handle *)buf->device)->offset == 0) && "halide_opencl_device_free on buffer obtained from halide_device_crop"); + halide_abort_if_false(user_context, (((device_handle *)buf->device)->offset == 0) && "halide_opencl_device_free on buffer obtained from halide_device_crop"); debug(user_context) << "CL: halide_opencl_device_free (user_context: " << user_context @@ -665,7 +665,7 @@ WEAK int halide_opencl_device_free(void *user_context, halide_buffer_t *buf) { uint64_t t_before = halide_current_time_ns(user_context); #endif - HALIDE_CHECK(user_context, validate_device_pointer(user_context, buf)); + halide_abort_if_false(user_context, validate_device_pointer(user_context, buf)); debug(user_context) << " clReleaseMemObject " << (void *)dev_ptr << "\n"; cl_int result = clReleaseMemObject((cl_mem)dev_ptr); // If clReleaseMemObject fails, it is unlikely to succeed in a later call, so @@ -759,7 +759,7 @@ WEAK int halide_opencl_initialize_kernels(void *user_context, void **state_ptr, compile_kernel, user_context, ctx.context, src, size)) { return halide_error_code_generic_error; } - HALIDE_CHECK(user_context, program != nullptr); + halide_abort_if_false(user_context, program != nullptr); #ifdef DEBUG_RUNTIME uint64_t t_after = halide_current_time_ns(user_context); @@ -822,7 +822,7 @@ WEAK int halide_opencl_device_release(void *user_context) { if (ctx) { err = clFinish(q); - HALIDE_CHECK(user_context, err == CL_SUCCESS); + halide_abort_if_false(user_context, err == CL_SUCCESS); compilation_cache.delete_context(user_context, ctx, clReleaseProgram); @@ -830,12 +830,12 @@ WEAK int halide_opencl_device_release(void *user_context) { if (ctx == context) { debug(user_context) << " clReleaseCommandQueue " << command_queue << "\n"; err = clReleaseCommandQueue(command_queue); - HALIDE_CHECK(user_context, err == CL_SUCCESS); + halide_abort_if_false(user_context, err == CL_SUCCESS); command_queue = nullptr; debug(user_context) << " clReleaseContext " << context << "\n"; err = clReleaseContext(context); - HALIDE_CHECK(user_context, err == CL_SUCCESS); + halide_abort_if_false(user_context, err == CL_SUCCESS); context = nullptr; } } @@ -856,14 +856,14 @@ WEAK int halide_opencl_device_malloc(void *user_context, halide_buffer_t *buf) { } size_t size = buf->size_in_bytes(); - HALIDE_CHECK(user_context, size != 0); + halide_abort_if_false(user_context, size != 0); if (buf->device) { - HALIDE_CHECK(user_context, validate_device_pointer(user_context, buf, size)); + halide_abort_if_false(user_context, validate_device_pointer(user_context, buf, size)); return 0; } for (int i = 0; i < buf->dimensions; i++) { - HALIDE_CHECK(user_context, buf->dim[i].stride >= 0); + halide_abort_if_false(user_context, buf->dim[i].stride >= 0); } debug(user_context) << " allocating " << *buf << "\n"; @@ -900,7 +900,7 @@ WEAK int halide_opencl_device_malloc(void *user_context, halide_buffer_t *buf) { << " Allocated device buffer " << (void *)buf->device << " for buffer " << buf << "\n"; - HALIDE_CHECK(user_context, validate_device_pointer(user_context, buf, size)); + halide_abort_if_false(user_context, validate_device_pointer(user_context, buf, size)); #ifdef DEBUG_RUNTIME uint64_t t_after = halide_current_time_ns(user_context); @@ -970,12 +970,12 @@ WEAK int halide_opencl_buffer_copy(void *user_context, struct halide_buffer_t *s const struct halide_device_interface_t *dst_device_interface, struct halide_buffer_t *dst) { // We only handle copies to opencl or to host - HALIDE_CHECK(user_context, dst_device_interface == nullptr || - dst_device_interface == &opencl_device_interface); + halide_abort_if_false(user_context, dst_device_interface == nullptr || + dst_device_interface == &opencl_device_interface); if ((src->device_dirty() || src->host == nullptr) && src->device_interface != &opencl_device_interface) { - HALIDE_CHECK(user_context, dst_device_interface == &opencl_device_interface); + halide_abort_if_false(user_context, dst_device_interface == &opencl_device_interface); // This is handled at the higher level. return halide_error_code_incompatible_device_interface; } @@ -985,8 +985,8 @@ WEAK int halide_opencl_buffer_copy(void *user_context, struct halide_buffer_t *s (src->host_dirty() && src->host != nullptr); bool to_host = !dst_device_interface; - HALIDE_CHECK(user_context, from_host || src->device); - HALIDE_CHECK(user_context, to_host || dst->device); + halide_abort_if_false(user_context, from_host || src->device); + halide_abort_if_false(user_context, to_host || dst->device); device_copy c = make_buffer_copy(src, from_host, dst, to_host); @@ -1004,10 +1004,10 @@ WEAK int halide_opencl_buffer_copy(void *user_context, struct halide_buffer_t *s #ifdef DEBUG_RUNTIME uint64_t t_before = halide_current_time_ns(user_context); if (!from_host) { - HALIDE_CHECK(user_context, validate_device_pointer(user_context, src)); + halide_abort_if_false(user_context, validate_device_pointer(user_context, src)); } if (!to_host) { - HALIDE_CHECK(user_context, validate_device_pointer(user_context, dst)); + halide_abort_if_false(user_context, validate_device_pointer(user_context, dst)); } #endif @@ -1062,11 +1062,11 @@ WEAK int halide_opencl_run(void *user_context, #endif // Create kernel object for entry_name from the program for this module. - HALIDE_CHECK(user_context, state_ptr); + halide_abort_if_false(user_context, state_ptr); cl_program program{}; bool found = compilation_cache.lookup(ctx.context, state_ptr, program); - HALIDE_CHECK(user_context, found && program != nullptr); + halide_abort_if_false(user_context, found && program != nullptr); debug(user_context) << " clCreateKernel " << entry_name << " -> "; cl_kernel f = clCreateKernel(program, entry_name, &err); @@ -1118,7 +1118,7 @@ WEAK int halide_opencl_run(void *user_context, cl_int err = CL_SUCCESS; if (arg_is_buffer[i]) { - HALIDE_CHECK(user_context, arg_sizes[i] == sizeof(uint64_t)); + halide_abort_if_false(user_context, arg_sizes[i] == sizeof(uint64_t)); cl_mem mem = ((device_handle *)((halide_buffer_t *)this_arg)->device)->mem; uint64_t offset = ((device_handle *)((halide_buffer_t *)this_arg)->device)->offset; @@ -1209,7 +1209,7 @@ WEAK int halide_opencl_device_and_host_free(void *user_context, struct halide_bu } WEAK int halide_opencl_wrap_cl_mem(void *user_context, struct halide_buffer_t *buf, uint64_t mem) { - HALIDE_CHECK(user_context, buf->device == 0); + halide_abort_if_false(user_context, buf->device == 0); if (buf->device != 0) { return -2; } @@ -1238,8 +1238,8 @@ WEAK int halide_opencl_detach_cl_mem(void *user_context, halide_buffer_t *buf) { if (buf->device == 0) { return 0; } - HALIDE_CHECK(user_context, buf->device_interface == &opencl_device_interface || - buf->device_interface == &opencl_image_device_interface); + halide_abort_if_false(user_context, buf->device_interface == &opencl_device_interface || + buf->device_interface == &opencl_image_device_interface); free((device_handle *)buf->device); buf->device = 0; buf->device_interface->impl->release_module(); @@ -1251,8 +1251,8 @@ WEAK uintptr_t halide_opencl_get_cl_mem(void *user_context, halide_buffer_t *buf if (buf->device == 0) { return 0; } - HALIDE_CHECK(user_context, buf->device_interface == &opencl_device_interface || - buf->device_interface == &opencl_image_device_interface); + halide_abort_if_false(user_context, buf->device_interface == &opencl_device_interface || + buf->device_interface == &opencl_image_device_interface); return (uintptr_t)((device_handle *)buf->device)->mem; } @@ -1260,7 +1260,7 @@ WEAK uint64_t halide_opencl_get_crop_offset(void *user_context, halide_buffer_t if (buf->device == 0) { return 0; } - HALIDE_CHECK(user_context, buf->device_interface == &opencl_device_interface); + halide_abort_if_false(user_context, buf->device_interface == &opencl_device_interface); return ((device_handle *)buf->device)->offset; } @@ -1329,7 +1329,7 @@ WEAK int halide_opencl_device_release_crop(void *user_context, uint64_t t_before = halide_current_time_ns(user_context); #endif - HALIDE_CHECK(user_context, validate_device_pointer(user_context, buf)); + halide_abort_if_false(user_context, validate_device_pointer(user_context, buf)); debug(user_context) << " clReleaseMemObject " << (void *)dev_ptr << "\n"; // Sub-buffers are released with clReleaseMemObject cl_int result = clReleaseMemObject((cl_mem)dev_ptr); @@ -1545,14 +1545,14 @@ WEAK int halide_opencl_image_device_malloc(void *user_context, halide_buffer_t * } size_t size = buf->size_in_bytes(); - HALIDE_CHECK(user_context, size != 0); + halide_abort_if_false(user_context, size != 0); if (buf->device) { - HALIDE_CHECK(user_context, validate_device_pointer(user_context, buf, size)); + halide_abort_if_false(user_context, validate_device_pointer(user_context, buf, size)); return 0; } for (int i = 0; i < buf->dimensions; i++) { - HALIDE_CHECK(user_context, buf->dim[i].stride >= 0); + halide_abort_if_false(user_context, buf->dim[i].stride >= 0); } debug(user_context) << " allocating " << *buf << "\n"; @@ -1664,7 +1664,7 @@ WEAK int halide_opencl_image_device_malloc(void *user_context, halide_buffer_t * << " Allocated device buffer " << (void *)buf->device << " for buffer " << buf << "\n"; - HALIDE_CHECK(user_context, validate_device_pointer(user_context, buf, size)); + halide_abort_if_false(user_context, validate_device_pointer(user_context, buf, size)); #ifdef DEBUG_RUNTIME uint64_t t_after = halide_current_time_ns(user_context); @@ -1682,12 +1682,12 @@ WEAK int halide_opencl_image_buffer_copy(void *user_context, struct halide_buffe << "CL: halide_opencl_image_buffer_copy (user_context: " << user_context << ", src: " << src << ", dst: " << dst << ")\n"; - HALIDE_CHECK(user_context, dst_device_interface == nullptr || - dst_device_interface == &opencl_image_device_interface); + halide_abort_if_false(user_context, dst_device_interface == nullptr || + dst_device_interface == &opencl_image_device_interface); if ((src->device_dirty() || src->host == nullptr) && src->device_interface != &opencl_image_device_interface) { - HALIDE_CHECK(user_context, dst_device_interface == &opencl_image_device_interface); + halide_abort_if_false(user_context, dst_device_interface == &opencl_image_device_interface); // This is handled at the higher level. return halide_error_code_incompatible_device_interface; } @@ -1697,8 +1697,8 @@ WEAK int halide_opencl_image_buffer_copy(void *user_context, struct halide_buffe (src->host_dirty() && src->host != nullptr); bool to_host = !dst_device_interface; - HALIDE_CHECK(user_context, from_host || src->device); - HALIDE_CHECK(user_context, to_host || dst->device); + halide_abort_if_false(user_context, from_host || src->device); + halide_abort_if_false(user_context, to_host || dst->device); device_copy c = make_buffer_copy(src, from_host, dst, to_host); @@ -1712,10 +1712,10 @@ WEAK int halide_opencl_image_buffer_copy(void *user_context, struct halide_buffe #ifdef DEBUG_RUNTIME uint64_t t_before = halide_current_time_ns(user_context); if (!from_host) { - HALIDE_CHECK(user_context, validate_device_pointer(user_context, src)); + halide_abort_if_false(user_context, validate_device_pointer(user_context, src)); } if (!to_host) { - HALIDE_CHECK(user_context, validate_device_pointer(user_context, dst)); + halide_abort_if_false(user_context, validate_device_pointer(user_context, dst)); } #endif @@ -1811,7 +1811,7 @@ WEAK int halide_opencl_image_device_and_host_free(void *user_context, struct hal } WEAK int halide_opencl_image_wrap_cl_mem(void *user_context, struct halide_buffer_t *buf, uint64_t mem) { - HALIDE_CHECK(user_context, buf->device == 0); + halide_abort_if_false(user_context, buf->device == 0); if (buf->device != 0) { return -2; } diff --git a/src/runtime/openglcompute.cpp b/src/runtime/openglcompute.cpp index 77b411d39c2d..85257b732116 100644 --- a/src/runtime/openglcompute.cpp +++ b/src/runtime/openglcompute.cpp @@ -268,7 +268,7 @@ WEAK int halide_openglcompute_device_malloc(void *user_context, halide_buffer_t } size_t size = buf->size_in_bytes(); - HALIDE_CHECK(user_context, size != 0); + halide_abort_if_false(user_context, size != 0); if (buf->device) { // This buffer already has a device allocation @@ -278,7 +278,7 @@ WEAK int halide_openglcompute_device_malloc(void *user_context, halide_buffer_t } for (int i = 0; i < buf->dimensions; i++) { - HALIDE_CHECK(user_context, buf->dim[i].stride >= 0); + halide_abort_if_false(user_context, buf->dim[i].stride >= 0); } debug(user_context) << " allocating buffer, " @@ -314,7 +314,7 @@ WEAK int halide_openglcompute_device_malloc(void *user_context, halide_buffer_t // types, all of which are 4 bytes. We'll inflate the size for // smaller types. size *= (4 / buf->type.bytes()); - HALIDE_CHECK(user_context, size != 0); + halide_abort_if_false(user_context, size != 0); global_state.BufferData(GL_ARRAY_BUFFER, size, nullptr, GL_DYNAMIC_COPY); if (global_state.CheckAndReportError(user_context, "oglc: BufferData")) { return 1; @@ -499,7 +499,7 @@ WEAK int halide_openglcompute_copy_to_host(void *user_context, halide_buffer_t * GLuint the_buffer = (GLuint)buf->device; size_t size = buf->size_in_bytes(); - HALIDE_CHECK(user_context, size != 0); + halide_abort_if_false(user_context, size != 0); debug(user_context) << "OGLC: halide_openglcompute_copy_to_host (" << "user_context: " << user_context diff --git a/src/runtime/profiler.cpp b/src/runtime/profiler.cpp index 1974e18cad88..1fd5b24afdbc 100644 --- a/src/runtime/profiler.cpp +++ b/src/runtime/profiler.cpp @@ -204,7 +204,7 @@ WEAK void halide_profiler_stack_peak_update(void *user_context, void *pipeline_state, uint64_t *f_values) { halide_profiler_pipeline_stats *p_stats = (halide_profiler_pipeline_stats *)pipeline_state; - HALIDE_CHECK(user_context, p_stats != nullptr); + halide_abort_if_false(user_context, p_stats != nullptr); // Note: Update to the counter is done without grabbing the state's lock to // reduce lock contention. One potential issue is that other call that frees the @@ -231,9 +231,9 @@ WEAK void halide_profiler_memory_allocate(void *user_context, } halide_profiler_pipeline_stats *p_stats = (halide_profiler_pipeline_stats *)pipeline_state; - HALIDE_CHECK(user_context, p_stats != nullptr); - HALIDE_CHECK(user_context, func_id >= 0); - HALIDE_CHECK(user_context, func_id < p_stats->num_funcs); + halide_abort_if_false(user_context, p_stats != nullptr); + halide_abort_if_false(user_context, func_id >= 0); + halide_abort_if_false(user_context, func_id < p_stats->num_funcs); halide_profiler_func_stats *f_stats = &p_stats->funcs[func_id]; @@ -267,9 +267,9 @@ WEAK void halide_profiler_memory_free(void *user_context, } halide_profiler_pipeline_stats *p_stats = (halide_profiler_pipeline_stats *)pipeline_state; - HALIDE_CHECK(user_context, p_stats != nullptr); - HALIDE_CHECK(user_context, func_id >= 0); - HALIDE_CHECK(user_context, func_id < p_stats->num_funcs); + halide_abort_if_false(user_context, p_stats != nullptr); + halide_abort_if_false(user_context, func_id >= 0); + halide_abort_if_false(user_context, func_id < p_stats->num_funcs); halide_profiler_func_stats *f_stats = &p_stats->funcs[func_id]; diff --git a/src/runtime/runtime_internal.h b/src/runtime/runtime_internal.h index b750c2e72022..fb95363b1f97 100644 --- a/src/runtime/runtime_internal.h +++ b/src/runtime/runtime_internal.h @@ -225,20 +225,19 @@ namespace Internal { } // namespace Halide using namespace Halide::Runtime::Internal; -/** A macro that calls halide_print if the supplied condition is - * false, then aborts. Used for unrecoverable errors, or - * should-never-happen errors. +/** halide_abort_if_false() is a macro that calls halide_print if the supplied condition is + * false, then aborts. Used for unrecoverable errors, or should-never-happen errors. * * Note that this is *NOT* a debug-only macro; * the condition will be checked in *all* build modes! */ #define _HALIDE_CHECK_STRINGIFY(x) #x #define _HALIDE_CHECK_EXPAND_AND_STRINGIFY(x) _HALIDE_CHECK_STRINGIFY(x) -#define HALIDE_CHECK(user_context, cond) \ - do { \ - if (!(cond)) { \ - halide_print(user_context, __FILE__ ":" _HALIDE_CHECK_EXPAND_AND_STRINGIFY(__LINE__) " HALIDE_CHECK failed: " #cond "\n"); \ - abort(); \ - } \ +#define halide_abort_if_false(user_context, cond) \ + do { \ + if (!(cond)) { \ + halide_print(user_context, __FILE__ ":" _HALIDE_CHECK_EXPAND_AND_STRINGIFY(__LINE__) " halide_abort_if_false() failed: " #cond "\n"); \ + abort(); \ + } \ } while (0) #endif diff --git a/src/runtime/synchronization_common.h b/src/runtime/synchronization_common.h index 70334949785b..ede6cd679fe8 100644 --- a/src/runtime/synchronization_common.h +++ b/src/runtime/synchronization_common.h @@ -384,7 +384,7 @@ WEAK void word_lock::unlock_full() { int times_through = 0; while (tail == nullptr) { word_lock_queue_data *next = current->next; - HALIDE_CHECK(nullptr, next != nullptr); + halide_abort_if_false(nullptr, next != nullptr); next->prev = current; current = next; tail = current->tail; @@ -491,7 +491,7 @@ static ALWAYS_INLINE uintptr_t addr_hash(uintptr_t addr) { #ifdef DEBUG_RUNTIME // Any hash calculated by addr_hash() should be incapable of being outside this range. ALWAYS_INLINE void check_hash(uintptr_t hash) { - HALIDE_CHECK(nullptr, hash < HASH_TABLE_SIZE); + halide_abort_if_false(nullptr, hash < HASH_TABLE_SIZE); } #endif // DEBUG_RUNTIME @@ -1004,7 +1004,7 @@ class fast_cond { // TODO: this is debug only. uintptr_t val; atomic_load_relaxed((uintptr_t *)mutex, &val); - HALIDE_CHECK(nullptr, val & 0x1); + halide_abort_if_false(nullptr, val & 0x1); if_tsan_post_lock(mutex); } diff --git a/src/runtime/thread_pool_common.h b/src/runtime/thread_pool_common.h index f26dd4d955f6..ba5e82af361f 100644 --- a/src/runtime/thread_pool_common.h +++ b/src/runtime/thread_pool_common.h @@ -152,7 +152,7 @@ struct work_queue_t { while (bytes < limit && *bytes == 0) { bytes++; } - HALIDE_CHECK(nullptr, bytes == limit && "Logic error in thread pool work queue initialization.\n"); + halide_abort_if_false(nullptr, bytes == limit && "Logic error in thread pool work queue initialization.\n"); } // Return the work queue to initial state. Must be called while locked @@ -522,9 +522,9 @@ WEAK void enqueue_work_already_locked(int num_jobs, work *jobs, work *task_paren } } else { log_message("enqueue_work_already_locked job " << jobs[0].task.name << " with min_threads " << min_threads << " task_parent " << task_parent->task.name << " task_parent->task.min_threads " << task_parent->task.min_threads << " task_parent->threads_reserved " << task_parent->threads_reserved); - HALIDE_CHECK(nullptr, (min_threads <= ((task_parent->task.min_threads * task_parent->active_workers) - - task_parent->threads_reserved)) && - "Logic error: thread over commit.\n"); + halide_abort_if_false(nullptr, (min_threads <= ((task_parent->task.min_threads * task_parent->active_workers) - + task_parent->threads_reserved)) && + "Logic error: thread over commit.\n"); if (job_has_acquires || job_may_block) { task_parent->threads_reserved++; } diff --git a/src/runtime/tracing.cpp b/src/runtime/tracing.cpp index a171f4217730..f755fabf378e 100644 --- a/src/runtime/tracing.cpp +++ b/src/runtime/tracing.cpp @@ -84,7 +84,7 @@ class TraceBuffer { // packet. Returns nullptr if the buffer was full. ALWAYS_INLINE halide_trace_packet_t *try_acquire_packet(void *user_context, uint32_t size) { lock.acquire_shared(); - HALIDE_CHECK(user_context, size <= buffer_size); + halide_abort_if_false(user_context, size <= buffer_size); uint32_t my_cursor = __sync_fetch_and_add(&cursor, size); if (my_cursor + size > sizeof(buf)) { // Don't try to back it out: instead, just allow this request to fail @@ -112,7 +112,7 @@ class TraceBuffer { overage = 0; } lock.release_exclusive(); - HALIDE_CHECK(user_context, success && "Could not write to trace file"); + halide_abort_if_false(user_context, success && "Could not write to trace file"); } // Acquire and return a packet's worth of space in the trace @@ -216,7 +216,7 @@ WEAK int32_t halide_default_trace(void *user_context, const halide_trace_event_t while (print_bits < e->type.bits) { print_bits <<= 1; } - HALIDE_CHECK(user_context, print_bits <= 64 && "Tracing bad type"); + halide_abort_if_false(user_context, print_bits <= 64 && "Tracing bad type"); // Otherwise, use halide_print and a plain-text format const char *event_types[] = {"Load", @@ -285,7 +285,7 @@ WEAK int32_t halide_default_trace(void *user_context, const halide_trace_event_t ss << ((uint64_t *)(e->value))[i]; } } else if (e->type.code == 2) { - HALIDE_CHECK(user_context, print_bits >= 16 && "Tracing a bad type"); + halide_abort_if_false(user_context, print_bits >= 16 && "Tracing a bad type"); if (print_bits == 32) { ss << ((float *)(e->value))[i]; } else if (print_bits == 16) { @@ -350,7 +350,7 @@ WEAK int halide_get_trace_file(void *user_context) { const char *trace_file_name = getenv("HL_TRACE_FILE"); if (trace_file_name) { void *file = fopen(trace_file_name, "ab"); - HALIDE_CHECK(user_context, file && "Failed to open trace file\n"); + halide_abort_if_false(user_context, file && "Failed to open trace file\n"); halide_set_trace_file(fileno(file)); halide_trace_file_internally_opened = file; if (!halide_trace_buffer) { From c10ba5eb1fea520dbcd4eac3c18f8319e5855082 Mon Sep 17 00:00:00 2001 From: Steven Johnson Date: Mon, 8 Nov 2021 12:18:25 -0800 Subject: [PATCH 6/6] Update runtime_internal.h --- src/runtime/runtime_internal.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/runtime/runtime_internal.h b/src/runtime/runtime_internal.h index fae0129f1b49..9dd3572a2bb9 100644 --- a/src/runtime/runtime_internal.h +++ b/src/runtime/runtime_internal.h @@ -232,12 +232,12 @@ using namespace Halide::Runtime::Internal; * the condition will be checked in *all* build modes! */ #define _halide_stringify(x) #x #define _halide_expand_and_stringify(x) _halide_stringify(x) -#define halide_abort_if_false(user_context, cond) \ - do { \ - if (!(cond)) { \ +#define halide_abort_if_false(user_context, cond) \ + do { \ + if (!(cond)) { \ halide_print(user_context, __FILE__ ":" _halide_expand_and_stringify(__LINE__) " halide_abort_if_false() failed: " #cond "\n"); \ - abort(); \ - } \ + abort(); \ + } \ } while (0) /** halide_debug_assert() is like halide_assert(), but only expands into a check when