AddressSanitizer: Heap buffer overflow

[hrydgard]

Found browsing the menus in Gran Turismo with address sanitizer on, with OpenGL. Might be an explanation for #16399, whatever it is.

Seems we're just ending up allocating too small a buffer in some cases. Although the logic seems sound...

==77607==ERROR: AddressSanitizer: heap-buffer-overflow on address 0x00010f0f7cf0 at pc 0x00010496e618 bp 0x00016df95e90 sp 0x00016df95e88
WRITE of size 4 at 0x00010f0f7cf0 thread T33
    #0 0x10496e614 in void DeIndexTexture4<unsigned int>(unsigned int*, unsigned char const*, int, unsigned int const*, unsigned int*) TextureDecoder.h:171
    #1 0x10496cfd0 in TextureCacheCommon::DecodeTextureLevel(unsigned char*, int, GETextureFormat, GEPaletteFormat, unsigned int, int, int, TexDecodeFlags) TextureCacheCommon.cpp:1726
    #2 0x1049802e0 in TextureCacheCommon::LoadTextureLevel(TexCacheEntry&, unsigned char*, int, ReplacedTexture&, int, int, Draw::DataFormat, TexDecodeFlags) TextureCacheCommon.cpp:2835
    #3 0x104631f88 in TextureCacheGLES::BuildTexture(TexCacheEntry*) TextureCacheGLES.cpp:330
    #4 0x104973b58 in TextureCacheCommon::ApplyTexture() TextureCacheCommon.cpp:2040
    #5 0x1046417bc in DrawEngineGLES::DoFlush() DrawEngineGLES.cpp:317
    #6 0x1045b9a24 in DrawEngineGLES::Flush() DrawEngineGLES.h:91
    #7 0x10464a994 in DrawEngineGLES::DispatchFlush() DrawEngineGLES.h:102
    #8 0x104a8f76c in GPUCommon::FastRunLoop(DisplayList&) GPUCommon.cpp:1214
    #9 0x104a8d720 in GPUCommon::InterpretList(DisplayList&) GPUCommon.cpp:1156
    #10 0x104a89dc8 in GPUCommon::ProcessDLQueue() GPUCommon.cpp:1378
    #11 0x104a88f0c in GPUCommon::EnqueueList(unsigned int, unsigned int, int, PSPPointer<PspGeListArgs>, bool) GPUCommon.cpp:949
    #12 0x103d15ac4 in sceGeListEnQueue(unsigned int, unsigned int, int, unsigned int) sceGe.cpp:342
    #13 0x103d1b628 in void WrapU_UUIU<&(sceGeListEnQueue(unsigned int, unsigned int, int, unsigned int))>() FunctionWrappers.h:683
    #14 0x103c36a24 in CallSyscallWithoutFlags(HLEFunction const*) HLE.cpp:657
    #15 0x13cd6ed6c  (<unknown module>)
    #16 0x1036f1d1c in MIPSComp::Arm64Jit::RunLoopUntil(unsigned long long) Arm64Jit.cpp:274
    #17 0x10431afb0 in MIPSState::RunLoopUntil(unsigned long long) MIPS.cpp:341
    #18 0x1044aae40 in PSP_RunLoopUntil(unsigned long long) System.cpp:601
    #19 0x1044aadb0 in PSP_RunLoopFor(int) System.cpp:605
    #20 0x1044aad68 in PSP_RunLoopWhileState() System.cpp:584
    #21 0x102fc3a44 in EmuScreen::render() EmuScreen.cpp:1422
    #22 0x105129c2c in ScreenManager::render() Screen.cpp:182
    #23 0x102e5aa6c in NativeRender(GraphicsContext*) NativeApp.cpp:1140
    #24 0x1037e49a8 in UpdateRunLoop() Core.cpp:216
    #25 0x1052ad6e0 in EmuThreadFunc(GraphicsContext*) SDLMain.cpp:494
    #26 0x1052af6d0 in decltype(static_cast<void (*>(fp)(static_cast<GraphicsContext*>(fp0))) std::__1::__invoke<void (*)(GraphicsContext*), GraphicsContext*>(void (*&&)(GraphicsContext*), GraphicsContext*&&) type_traits:3918
    #27 0x1052af5f0 in void std::__1::__thread_execute<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct> >, void (*)(GraphicsContext*), GraphicsContext*, 2ul>(std::__1::tuple<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct> >, void (*)(GraphicsContext*), GraphicsContext*>&, std::__1::__tuple_indices<2ul>) thread:287
    #28 0x1052ae328 in void* std::__1::__thread_proxy<std::__1::tuple<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct> >, void (*)(GraphicsContext*), GraphicsContext*> >(void*) thread:298
    #29 0x186842068 in _pthread_start+0x90 (libsystem_pthread.dylib:arm64e+0x7068)
    #30 0x18683ce28 in thread_start+0x4 (libsystem_pthread.dylib:arm64e+0x1e28)

0x00010f0f7cf0 is located 0 bytes to the right of 128-byte region [0x00010f0f7c70,0x00010f0f7cf0)
allocated by thread T33 here:
    #0 0x10c66740c in wrap_posix_memalign+0xa4 (libclang_rt.asan_osx_dynamic.dylib:arm64e+0x3f40c)
    #1 0x1052852a4 in AllocateAlignedMemory(unsigned long, unsigned long) MemoryUtil.cpp:245
    #2 0x104631da0 in TextureCacheGLES::BuildTexture(TexCacheEntry*) TextureCacheGLES.cpp:323
    #3 0x104973b58 in TextureCacheCommon::ApplyTexture() TextureCacheCommon.cpp:2040
    #4 0x1046417bc in DrawEngineGLES::DoFlush() DrawEngineGLES.cpp:317
    #5 0x1045b9a24 in DrawEngineGLES::Flush() DrawEngineGLES.h:91
    #6 0x10464a994 in DrawEngineGLES::DispatchFlush() DrawEngineGLES.h:102
    #7 0x104a8f76c in GPUCommon::FastRunLoop(DisplayList&) GPUCommon.cpp:1214
    #8 0x104a8d720 in GPUCommon::InterpretList(DisplayList&) GPUCommon.cpp:1156
    #9 0x104a89dc8 in GPUCommon::ProcessDLQueue() GPUCommon.cpp:1378
    #10 0x104a88f0c in GPUCommon::EnqueueList(unsigned int, unsigned int, int, PSPPointer<PspGeListArgs>, bool) GPUCommon.cpp:949
    #11 0x103d15ac4 in sceGeListEnQueue(unsigned int, unsigned int, int, unsigned int) sceGe.cpp:342
    #12 0x103d1b628 in void WrapU_UUIU<&(sceGeListEnQueue(unsigned int, unsigned int, int, unsigned int))>() FunctionWrappers.h:683
    #13 0x103c36a24 in CallSyscallWithoutFlags(HLEFunction const*) HLE.cpp:657
    #14 0x13cd6ed6c  (<unknown module>)
    #15 0x1036f1d1c in MIPSComp::Arm64Jit::RunLoopUntil(unsigned long long) Arm64Jit.cpp:274
    #16 0x10431afb0 in MIPSState::RunLoopUntil(unsigned long long) MIPS.cpp:341
    #17 0x1044aae40 in PSP_RunLoopUntil(unsigned long long) System.cpp:601
    #18 0x1044aadb0 in PSP_RunLoopFor(int) System.cpp:605
    #19 0x1044aad68 in PSP_RunLoopWhileState() System.cpp:584
    #20 0x102fc3a44 in EmuScreen::render() EmuScreen.cpp:1422
    #21 0x105129c2c in ScreenManager::render() Screen.cpp:182
    #22 0x102e5aa6c in NativeRender(GraphicsContext*) NativeApp.cpp:1140
    #23 0x1037e49a8 in UpdateRunLoop() Core.cpp:216
    #24 0x1052ad6e0 in EmuThreadFunc(GraphicsContext*) SDLMain.cpp:494
    #25 0x1052af6d0 in decltype(static_cast<void (*>(fp)(static_cast<GraphicsContext*>(fp0))) std::__1::__invoke<void (*)(GraphicsContext*), GraphicsContext*>(void (*&&)(GraphicsContext*), GraphicsContext*&&) type_traits:3918
    #26 0x1052af5f0 in void std::__1::__thread_execute<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct> >, void (*)(GraphicsContext*), GraphicsContext*, 2ul>(std::__1::tuple<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct> >, void (*)(GraphicsContext*), GraphicsContext*>&, std::__1::__tuple_indices<2ul>) thread:287
    #27 0x1052ae328 in void* std::__1::__thread_proxy<std::__1::tuple<std::__1::unique_ptr<std::__1::__thread_struct, std::__1::default_delete<std::__1::__thread_struct> >, void (*)(GraphicsContext*), GraphicsContext*> >(void*) thread:298
    #28 0x186842068 in _pthread_start+0x90 (libsystem_pthread.dylib:arm64e+0x7068)
    #29 0x18683ce28 in thread_start+0x4 (libsystem_pthread.dylib:arm64e+0x1e28)

Thread T33 created by T0 here:
    #0 0x10c660c5c in wrap_pthread_create+0x54 (libclang_rt.asan_osx_dynamic.dylib:arm64e+0x38c5c)
    #1 0x1052ae1c8 in std::__1::__libcpp_thread_create(_opaque_pthread_t**, void* (*)(void*), void*) __threading_support:421
    #2 0x1052add4c in std::__1::thread::thread<void (*)(GraphicsContext*), GraphicsContext*&, void>(void (*&&)(GraphicsContext*), GraphicsContext*&) thread:314
    #3 0x1052ad7a0 in std::__1::thread::thread<void (*)(GraphicsContext*), GraphicsContext*&, void>(void (*&&)(GraphicsContext*), GraphicsContext*&) thread:306
    #4 0x1052ac9c4 in EmuThreadStart(GraphicsContext*) SDLMain.cpp:504
    #5 0x1052a7b28 in main SDLMain.cpp:825
    #6 0x186517e4c  (<unknown module>)

SUMMARY: AddressSanitizer: heap-buffer-overflow TextureDecoder.h:171 in void DeIndexTexture4<unsigned int>(unsigned int*, unsigned char const*, int, unsigned int const*, unsigned int*)