diff --git a/include/gin/vmem_stack_frame_allocator.h b/include/gin/vmem_stack_frame_allocator.h new file mode 100644 index 0000000..ecce167 --- /dev/null +++ b/include/gin/vmem_stack_frame_allocator.h @@ -0,0 +1,566 @@ +#ifndef GIN_VMEM_STACK_FRAME_ALLOCATOR_H +#define GIN_VMEM_STACK_FRAME_ALLOCATOR_H + +//////////////////////////////////////////////////////////////////////////////// +// The MIT License (MIT) +// +// Copyright (c) 2015-2016 Nicholas Frechette +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +//////////////////////////////////////////////////////////////////////////////// + +#include "allocator.h" +#include "allocator_frame.h" +#include "utils.h" +#include "virtual_memory.h" + +#include +#include +#include + +namespace gin +{ + //////////////////////////////////////// + // A simple virtual memory aware stack frame allocator. + // Unlike StackFrameAllocator, we do not allocate multiple segments, + // and instead we allocate a single large virtual memory range + // and commit/decommit when relevant. + // + // The allocator is not thread-safe. + // + // See here for more details: + // http://nfrechette.github.io/todo/ + //////////////////////////////////////// + + template + class TVMemStackFrameAllocator : public Allocator + { + public: + inline TVMemStackFrameAllocator(); + inline TVMemStackFrameAllocator(size_t bufferSize); + inline ~TVMemStackFrameAllocator(); + + virtual void* Allocate(size_t size, size_t alignment) override; + virtual void Deallocate(void* ptr, size_t size) override; + + virtual bool IsOwnerOf(void* ptr) const override; + + AllocatorFrame PushFrame(); + bool PopFrame(AllocatorFrame& frame); + inline operator internal::AllocatorFrameFactory(); + + void Initialize(size_t bufferSize); + void Release(); + + bool DecommitSlack(size_t minSlack); + + inline bool IsInitialized() const; + inline size_t GetAllocatedSize() const; + inline size_t GetCommittedSize() const; + inline bool HasLiveFrame() const; + + inline size_t GetFrameOverhead() const; + + private: + struct FrameDescription + { + FrameDescription* prevFrame; + }; + + TVMemStackFrameAllocator(const TVMemStackFrameAllocator&) = delete; + TVMemStackFrameAllocator(TVMemStackFrameAllocator&&) = delete; + TVMemStackFrameAllocator& operator=(TVMemStackFrameAllocator) = delete; + + void* AllocateImpl(size_t size, size_t alignment); + static void* ReallocateImpl(Allocator* allocator, void* oldPtr, size_t oldSize, size_t newSize, size_t alignment); + static void PushImpl(Allocator* allocator, AllocatorFrame& outFrame); + static bool PopImpl(Allocator* allocator, void* allocatorData); + + uintptr_t m_buffer; + FrameDescription* m_liveFrame; + + SizeType m_bufferSize; + SizeType m_allocatedSize; + SizeType m_committedSize; + SizeType m_lastAllocationOffset; // For realloc support only + }; + + //////////////////////////////////////// + + template + TVMemStackFrameAllocator::TVMemStackFrameAllocator() + : Allocator(&TVMemStackFrameAllocator::ReallocateImpl) + , m_buffer(0) + , m_liveFrame(nullptr) + , m_bufferSize(0) + , m_allocatedSize(0) + , m_committedSize(0) + , m_lastAllocationOffset(0) + { + } + + template + TVMemStackFrameAllocator::TVMemStackFrameAllocator(size_t bufferSize) + : Allocator(&TVMemStackFrameAllocator::ReallocateImpl) + , m_buffer(0) + , m_liveFrame(nullptr) + , m_bufferSize(0) + , m_allocatedSize(0) + , m_committedSize(0) + , m_lastAllocationOffset(0) + { + Initialize(bufferSize); + } + + template + TVMemStackFrameAllocator::~TVMemStackFrameAllocator() + { + Release(); + } + + template + void TVMemStackFrameAllocator::Initialize(size_t bufferSize) + { + //assert(!IsInitialized()); + //assert(bufferSize >= PAGE_SIZE); + //assert(IsAlignedTo(bufferSize, PAGE_SIZE); + //assert(bufferSize <= static_cast(std::numeric_limits::max())); + + if (IsInitialized()) + { + // Invalid allocator state + return; + } + + if (bufferSize < 4096 // TODO: PAGE_SIZE + || !IsAlignedTo(bufferSize, 4096) + || bufferSize > static_cast(std::numeric_limits::max())) + { + // Invalid arguments + return; + } + + MemoryAccessFlags accessFlags = MemoryAccessFlags::eCPU_ReadWrite; + MemoryRegionFlags regionFlags = MemoryRegionFlags::ePrivate | MemoryRegionFlags::eAnonymous; + + void* ptr = VirtualReserve(bufferSize, accessFlags, regionFlags); + //assert(ptr); + if (!ptr) + { + // Failed to reserve virtual memory + return; + } + + m_buffer = reinterpret_cast(ptr); + m_liveFrame = nullptr; + m_bufferSize = static_cast(bufferSize); + m_allocatedSize = 0; + m_committedSize = 0; + m_lastAllocationOffset = static_cast(bufferSize); + } + + template + void TVMemStackFrameAllocator::Release() + { + //assert(IsInitialized()); + //assert(!HasLiveFrame()); + + if (!IsInitialized()) + { + // Invalid allocator state + return; + } + + if (HasLiveFrame()) + { + // Cannot release the allocator if we have live frames, leak memory instead + return; + } + + // No need to decommit memory, release will take care of it + + void* ptr = reinterpret_cast(m_buffer); + bool success = VirtualRelease(ptr, m_bufferSize); + //assert(success); + if (!success) + { + // Failed to release the virtual memory + return; + } + + m_buffer = 0; + m_liveFrame = nullptr; + m_bufferSize = 0; + m_allocatedSize = 0; + m_committedSize = 0; + m_lastAllocationOffset = 0; + } + + template + bool TVMemStackFrameAllocator::DecommitSlack(size_t minSlack) + { + //assert(IsInitialized()); + //assert(IsAlignedTo(minSlack, PAGE_SIZE); + //assert(minSlack <= static_cast(std::numeric_limits::max())); + + if (!IsInitialized()) + { + // Invalid allocator state + return false; + } + + if (!IsAlignedTo(minSlack, 4096) + || minSlack > static_cast(std::numeric_limits::max())) + { + // Invalid arguments + return false; + } + + SizeType slack = m_committedSize - m_allocatedSize; + + // Round down decommit size to a multiple of the page size + size_t decommitSize = (slack - minSlack) & ~(4096 - 1); // TODO: PAGE_SIZE + + if (slack > minSlack && decommitSize != 0) + { + void* ptr = reinterpret_cast(m_buffer); + + bool success = VirtualDecommit(ptr, decommitSize); + //assert(success); + + if (success) + { + m_committedSize -= decommitSize; + } + + return success; + } + + return true; + } + + template + bool TVMemStackFrameAllocator::IsInitialized() const + { + return m_buffer != 0; + } + + template + size_t TVMemStackFrameAllocator::GetAllocatedSize() const + { + return m_allocatedSize; + } + + template + size_t TVMemStackFrameAllocator::GetCommittedSize() const + { + return m_committedSize; + } + + template + bool TVMemStackFrameAllocator::HasLiveFrame() const + { + return m_liveFrame != nullptr; + } + + template + size_t TVMemStackFrameAllocator::GetFrameOverhead() const + { + return sizeof(FrameDescription); + } + + template + void* TVMemStackFrameAllocator::Allocate(size_t size, size_t alignment) + { + //assert(IsInitialized()); + //assert(size > 0); + //assert(IsPowerOfTwo(alignment)); + + if (!IsInitialized()) + { + // Invalid allocator state + return nullptr; + } + + if (size == 0 || !IsPowerOfTwo(alignment)) + { + // Invalid arguments + return nullptr; + } + + if (!HasLiveFrame()) + { + // Need at least a single live frame + return nullptr; + } + + return AllocateImpl(size, alignment); + } + + template + void TVMemStackFrameAllocator::Deallocate(void* ptr, size_t size) + { + // Not supported, does nothing + } + + template + bool TVMemStackFrameAllocator::IsOwnerOf(void* ptr) const + { + //assert(IsInitialized()); + + if (!IsInitialized()) + { + // Invalid allocator state + return false; + } + + return IsPointerInBuffer(ptr, m_buffer, m_allocatedSize); + } + + template + AllocatorFrame TVMemStackFrameAllocator::PushFrame() + { + AllocatorFrame frame; + + PushImpl(this, frame); + + return frame; + } + + template + bool TVMemStackFrameAllocator::PopFrame(AllocatorFrame& frame) + { + return frame.Pop(); + } + + template + TVMemStackFrameAllocator::operator internal::AllocatorFrameFactory() + { + return internal::AllocatorFrameFactory(this, &PushImpl); + } + + template + void* TVMemStackFrameAllocator::AllocateImpl(size_t size, size_t alignment) + { + //assert(IsInitialized()); + //assert(size > 0); + //assert(IsPowerOfTwo(alignment)); + + if (!CanSatisfyAllocation(m_buffer, m_bufferSize, m_allocatedSize, size, alignment)) + { + // Out of memory or overflow + return nullptr; + } + + SizeType allocatedSize = m_allocatedSize; + SizeType lastAllocationOffset = m_lastAllocationOffset; + SizeType committedSize = m_committedSize; + + void* ptr = AllocateFromBuffer(m_buffer, m_bufferSize, allocatedSize, size, alignment, lastAllocationOffset); + + if (allocatedSize > committedSize) + { + // We need to commit more memory + void* commitPtr = reinterpret_cast(m_buffer + committedSize); + SizeType commitSize = AlignTo(allocatedSize - committedSize, 4096); // TODO: PAGE_SIZE + + MemoryAccessFlags accessFlags = MemoryAccessFlags::eCPU_ReadWrite; + MemoryRegionFlags regionFlags = MemoryRegionFlags::ePrivate | MemoryRegionFlags::eAnonymous; + + bool success = VirtualCommit(commitPtr, commitSize, accessFlags, regionFlags); + //assert(success); + if (!success) + { + // Out of memory + return nullptr; + } + + m_committedSize = committedSize + commitSize; + } + + m_allocatedSize = allocatedSize; + m_lastAllocationOffset = lastAllocationOffset; + + return ptr; + } + + template + void* TVMemStackFrameAllocator::ReallocateImpl(Allocator* allocator, void* oldPtr, size_t oldSize, size_t newSize, size_t alignment) + { + TVMemStackFrameAllocator* allocatorImpl = static_cast*>(allocator); + + //assert(allocatorImpl->IsInitialized()); + //assert(newSize > 0); + //assert(IsPowerOfTwo(alignment)); + + if (!allocatorImpl->IsInitialized()) + { + // Invalid allocator state + return nullptr; + } + + if (newSize == 0 || !IsPowerOfTwo(alignment)) + { + // Invalid arguments + return nullptr; + } + + if (!allocatorImpl->HasLiveFrame()) + { + // Need at least a single live frame + return nullptr; + } + + // We do not support freeing + SizeType lastAllocationOffset = allocatorImpl->m_lastAllocationOffset; + uintptr_t lastAllocation = allocatorImpl->m_buffer + lastAllocationOffset; + uintptr_t rawOldPtr = reinterpret_cast(oldPtr); + + if (lastAllocation == rawOldPtr) + { + // We are reallocating the last allocation + SizeType allocatedSize = allocatorImpl->m_allocatedSize; + SizeType bufferSize = allocatorImpl->m_bufferSize; + + // If we are shrinking the allocation, deltaSize + // will be very large (negative) + SizeType deltaSize = newSize - oldSize; + + // If deltaSize is very large (negative), we will wrap around + // and newAllocatedSize should end up smaller than allocatedSize + SizeType newAllocatedSize = allocatedSize + deltaSize; + //assert(newAllocatedSize <= bufferSize); + if (newAllocatedSize > bufferSize) + { + // Out of memory + return nullptr; + } + + SizeType committedSize = allocatorImpl->m_committedSize; + if (newAllocatedSize > committedSize) + { + // We need to commit more memory + void* commitPtr = reinterpret_cast(allocatorImpl->m_buffer + committedSize); + SizeType commitSize = AlignTo(newAllocatedSize - committedSize, 4096); // TODO: PAGE_SIZE + + MemoryAccessFlags accessFlags = MemoryAccessFlags::eCPU_ReadWrite; + MemoryRegionFlags regionFlags = MemoryRegionFlags::ePrivate | MemoryRegionFlags::eAnonymous; + + bool success = VirtualCommit(commitPtr, commitSize, accessFlags, regionFlags); + //assert(success); + if (!success) + { + // Out of memory + return nullptr; + } + + allocatorImpl->m_committedSize = committedSize + commitSize; + } + + allocatorImpl->m_allocatedSize = newAllocatedSize; + + // Nothing to copy since we re-use the same memory + + return oldPtr; + } + + // We do not support reallocating an arbitrary allocation + // we simply perform a new allocation and copy the contents + void* ptr = allocatorImpl->AllocateImpl(newSize, alignment); + + if (ptr != nullptr) + { + size_t numBytesToCopy = newSize >= oldSize ? oldSize : newSize; + memcpy(ptr, oldPtr, numBytesToCopy); + } + + return ptr; + } + + template + void TVMemStackFrameAllocator::PushImpl(Allocator* allocator, AllocatorFrame& outFrame) + { + //assert(allocator); + + TVMemStackFrameAllocator* allocatorImpl = static_cast*>(allocator); + + if (!allocatorImpl->IsInitialized()) + { + // Invalid allocator state + outFrame = AllocatorFrame(); + return; + } + + void* ptr = allocatorImpl->AllocateImpl(sizeof(FrameDescription), alignof(FrameDescription)); + if (ptr == nullptr) + { + // Failed to allocate our frame, out of memory? + outFrame = AllocatorFrame(); + return; + } + + FrameDescription* frameDesc = reinterpret_cast(ptr); + frameDesc->prevFrame = allocatorImpl->m_liveFrame; + + allocatorImpl->m_liveFrame = frameDesc; + + outFrame = AllocatorFrame(allocator, &PopImpl, frameDesc); + } + + template + bool TVMemStackFrameAllocator::PopImpl(Allocator* allocator, void* allocatorData) + { + //assert(allocator); + + TVMemStackFrameAllocator* allocatorImpl = static_cast*>(allocator); + + //assert(allocatorImpl->IsInitialized()); + + if (!allocatorImpl->IsInitialized()) + { + // Invalid allocator state + return false; + } + + const FrameDescription* frameDesc = static_cast(allocatorData); + + // We can only pop the top most frame + //assert(frameDesc == allocatorImpl->m_liveFrame); + if (frameDesc != allocatorImpl->m_liveFrame) + { + return false; + } + + // Update our topmost frame + allocatorImpl->m_liveFrame = frameDesc->prevFrame; + + // Popping is noop + uintptr_t allocatedSize = reinterpret_cast(frameDesc) - allocatorImpl->m_buffer; + allocatorImpl->m_allocatedSize = static_cast(allocatedSize); + + return true; + } + + //////////////////////////////////////// + + typedef TVMemStackFrameAllocator VMemStackFrameAllocator; +} + +#endif // GIN_VMEM_STACK_FRAME_ALLOCATOR_H + diff --git a/test/test_vmem_stack_frame_allocator.cpp b/test/test_vmem_stack_frame_allocator.cpp new file mode 100644 index 0000000..89efde4 --- /dev/null +++ b/test/test_vmem_stack_frame_allocator.cpp @@ -0,0 +1,229 @@ +#include "catch.hpp" + +#include "gin/vmem_stack_frame_allocator.h" +#include "gin/utils.h" + +TEST_CASE("allocate and free from vmem stack frame allocator", "[VMemStackFrameAllocator]") +{ + using namespace gin; + + const size_t BUFFER_SIZE = 16 * 1024; + + VMemStackFrameAllocator alloc(BUFFER_SIZE); + + size_t frameOverhead = alloc.GetFrameOverhead(); + + REQUIRE(alloc.IsInitialized()); + REQUIRE(alloc.GetAllocatedSize() == 0); + REQUIRE(!alloc.HasLiveFrame()); + + SECTION("test frame push/pop") + { + { + AllocatorFrame frame = alloc.PushFrame(); + + REQUIRE(frame.CanPop()); + REQUIRE(alloc.HasLiveFrame()); + + // Pop manually + frame.Pop(); + + REQUIRE(!frame.CanPop()); + REQUIRE(!alloc.HasLiveFrame()); + } + + REQUIRE(!alloc.HasLiveFrame()); + + { + AllocatorFrame frame(alloc); + + REQUIRE(frame.CanPop()); + REQUIRE(alloc.HasLiveFrame()); + + // Pop automatically with the destructor + } + + REQUIRE(!alloc.HasLiveFrame()); + REQUIRE(alloc.GetAllocatedSize() == 0); + } + + SECTION("test IsOwnerOf()") + { + uint8_t* ptr0; + + { + AllocatorFrame frame(alloc); + + REQUIRE(!alloc.IsOwnerOf(nullptr)); + + ptr0 = static_cast(alloc.Allocate(2, 1)); + if (ptr0) memset(ptr0, 0xcd, 2); + + REQUIRE(alloc.IsOwnerOf(ptr0)); + REQUIRE(alloc.IsOwnerOf(ptr0 + 1)); + REQUIRE(!alloc.IsOwnerOf(ptr0 + 2)); + } + + REQUIRE(!alloc.IsOwnerOf(ptr0)); + REQUIRE(alloc.GetAllocatedSize() == 0); + } + + SECTION("test allocation") + { + { + AllocatorFrame frame(alloc); + + void* ptr0 = alloc.Allocate(2, 1); + if (ptr0) memset(ptr0, 0xcd, 2); + + REQUIRE(alloc.IsOwnerOf(ptr0)); + REQUIRE(alloc.GetAllocatedSize() == 2 + frameOverhead); + + void* ptr1 = alloc.Allocate(1022, 1); + if (ptr1) memset(ptr1, 0xcd, 1022); + + REQUIRE(alloc.IsOwnerOf(ptr1)); + REQUIRE(alloc.GetAllocatedSize() == 1024 + frameOverhead); + REQUIRE(ptr0 != ptr1); + + void* ptr2 = alloc.Allocate(2048, 1); + if (ptr2) memset(ptr2, 0xcd, 2048); + + REQUIRE(alloc.IsOwnerOf(ptr2)); + REQUIRE(alloc.GetAllocatedSize() == 1024 + 2048 + frameOverhead); + REQUIRE(ptr1 != ptr2); + } + + REQUIRE(alloc.GetAllocatedSize() == 0); + } + + SECTION("test commit/decommit") + { + { + AllocatorFrame frame(alloc); + + void* ptr0 = alloc.Allocate(1024, 1); + if (ptr0) memset(ptr0, 0xcd, 1024); + + REQUIRE(alloc.GetCommittedSize() == 4096); + + void* ptr1 = alloc.Allocate(8192, 1); + if (ptr1) memset(ptr1, 0xcd, 8192); + + REQUIRE(alloc.GetCommittedSize() == 4096 * 3); + } + + alloc.DecommitSlack(8192); + + REQUIRE(alloc.GetCommittedSize() == 8192); + + alloc.DecommitSlack(0); + + REQUIRE(alloc.GetCommittedSize() == 0); + + { + AllocatorFrame frame(alloc); + + void* ptr0 = alloc.Allocate(1024, 1); + if (ptr0) memset(ptr0, 0xcd, 1024); + + REQUIRE(ptr0 != nullptr); + REQUIRE(alloc.IsOwnerOf(ptr0)); + REQUIRE(alloc.GetAllocatedSize() == 1024 + frameOverhead); + } + } + + SECTION("test alignment") + { + AllocatorFrame frame(alloc); + + void* ptr0 = alloc.Allocate(2, 8); + if (ptr0) memset(ptr0, 0xcd, 2); + + REQUIRE(alloc.IsOwnerOf(ptr0)); + REQUIRE(IsAlignedTo(ptr0, 8)); + + void* ptr1 = alloc.Allocate(2, 16); + if (ptr1) memset(ptr1, 0xcd, 2); + + REQUIRE(alloc.IsOwnerOf(ptr1)); + REQUIRE(IsAlignedTo(ptr1, 16)); + REQUIRE(ptr0 != ptr1); + } + + SECTION("test realloc") + { + AllocatorFrame frame(alloc); + + void* ptr0 = alloc.Allocate(2, 1); + if (ptr0) memset(ptr0, 0xcd, 2); + + void* ptr1 = alloc.Reallocate(ptr0, 2, 8, 1); + if (ptr1) memset(ptr1, 0xcd, 8); + + REQUIRE(ptr0 == ptr1); + REQUIRE(alloc.GetAllocatedSize() == 8 + frameOverhead); + + void* ptr2 = alloc.Reallocate(nullptr, 0, 4, 1); + if (ptr2) memset(ptr2, 0xcd, 4); + + REQUIRE(ptr0 != ptr2); + REQUIRE(alloc.GetAllocatedSize() == 12 + frameOverhead); + + void* ptr3 = alloc.Reallocate(ptr0, 8, 12, 1); + if (ptr3) memset(ptr3, 0xcd, 12); + + REQUIRE(ptr0 != ptr3); + REQUIRE(ptr2 != ptr3); + REQUIRE(alloc.GetAllocatedSize() == 24 + frameOverhead); + + void* ptr4 = alloc.Reallocate(ptr3, 12, 4, 1); + if (ptr4) memset(ptr4, 0xcd, 4); + + REQUIRE(ptr3 == ptr4); + REQUIRE(alloc.GetAllocatedSize() == 16 + frameOverhead); + + void* ptr5 = alloc.Reallocate(ptr4, 4, 128 * 1024, 1); + if (ptr5) memset(ptr5, 0xcd, 128 * 1024); + + REQUIRE(ptr5 == nullptr); + REQUIRE(alloc.GetAllocatedSize() == 16 + frameOverhead); + } + + SECTION("test nop free") + { + AllocatorFrame frame(alloc); + + void* ptr0 = alloc.Allocate(2, 1); + if (ptr0) memset(ptr0, 0xcd, 2); + + REQUIRE(alloc.GetAllocatedSize() == 2 + frameOverhead); + + alloc.Deallocate(ptr0, 2); + + REQUIRE(alloc.GetAllocatedSize() == 2 + frameOverhead); + + void* ptr1 = alloc.Allocate(2, 1); + if (ptr1) memset(ptr1, 0xcd, 2); + + REQUIRE(ptr0 != ptr1); + REQUIRE(alloc.GetAllocatedSize() == 4 + frameOverhead); + } +} + +TEST_CASE("test invalid arguments in vmem stack frame allocator", "[VMemStackFrameAllocator]") +{ + using namespace gin; + + SECTION("test initialization") + { + VMemStackFrameAllocator alloc; + + REQUIRE(!alloc.IsInitialized()); + + alloc.Initialize(0); + + REQUIRE(!alloc.IsInitialized()); + } +} +