Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Finish the other part of the GCTotalPhysicalMemory config implementation #36152

Merged
merged 3 commits into from
May 9, 2020
Merged

Finish the other part of the GCTotalPhysicalMemory config implementation #36152

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7b9915e
Finish the other part of the GCTotalPhysicalMemory config implementation
Maoni0 May 6, 2020
bc272f1
Delete unnecessary delay-loading of kernel32.dll
jkotas May 8, 2020
cce81b6
Avoid mutable restricted memory limit inside OS PAL
jkotas May 8, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 3 additions & 4 deletions src/coreclr/src/gc/env/gcenv.os.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -433,20 +433,19 @@ class GCToOSInterface
// Remarks:
// If a process runs with a restricted memory limit, it returns the limit. If there's no limit
// specified, it returns amount of actual physical memory.
//
// PERF TODO: Requires more work to not treat the restricted case to be special.
// To be removed before 3.0 ships.
static uint64_t GetPhysicalMemoryLimit(bool* is_restricted=NULL);

// Get memory status
// Parameters:
// restricted_limit - The amount of physical memory in bytes that the current process is being restricted to. If non-zero, it used to calculate
// memory_load and available_physical. If zero, memory_load and available_physical is calculate based on all available memory.
// memory_load - A number between 0 and 100 that specifies the approximate percentage of physical memory
// that is in use (0 indicates no memory use and 100 indicates full memory use).
// available_physical - The amount of physical memory currently available, in bytes.
// available_page_file - The maximum amount of memory the current process can commit, in bytes.
// Remarks:
// Any parameter can be null.
static void GetMemoryStatus(uint32_t* memory_load, uint64_t* available_physical, uint64_t* available_page_file);
static void GetMemoryStatus(uint64_t restricted_limit, uint32_t* memory_load, uint64_t* available_physical, uint64_t* available_page_file);

// Get size of an OS memory page
static size_t GetPageSize();
Expand Down
15 changes: 10 additions & 5 deletions src/coreclr/src/gc/gc.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2175,6 +2175,8 @@ uint32_t gc_heap::m_high_memory_load_th;

uint32_t gc_heap::v_high_memory_load_th;

bool gc_heap::is_restricted_physical_mem;

uint64_t gc_heap::total_physical_mem = 0;

uint64_t gc_heap::entry_available_physical_mem = 0;
Expand Down Expand Up @@ -19628,7 +19630,7 @@ void gc_heap::get_memory_info (uint32_t* memory_load,
uint64_t* available_physical,
uint64_t* available_page_file)
{
GCToOSInterface::GetMemoryStatus(memory_load, available_physical, available_page_file);
GCToOSInterface::GetMemoryStatus(is_restricted_physical_mem ? total_physical_mem : 0, memory_load, available_physical, available_page_file);
}

void fire_mark_event (int heap_num, int root_type, size_t bytes_marked)
Expand Down Expand Up @@ -34925,11 +34927,14 @@ HRESULT GCHeap::Initialize()
g_num_processors = GCToOSInterface::GetTotalProcessorCount();
assert(g_num_processors != 0);

bool is_restricted;
gc_heap::total_physical_mem = (size_t)GCConfig::GetGCTotalPhysicalMemory();
if (!(gc_heap::total_physical_mem))
if (gc_heap::total_physical_mem != 0)
{
gc_heap::is_restricted_physical_mem = true;
}
else
{
gc_heap::total_physical_mem = GCToOSInterface::GetPhysicalMemoryLimit (&is_restricted);
gc_heap::total_physical_mem = GCToOSInterface::GetPhysicalMemoryLimit (&gc_heap::is_restricted_physical_mem);
}

#ifdef HOST_64BIT
Expand All @@ -34948,7 +34953,7 @@ HRESULT GCHeap::Initialize()
// running in a container, use this limit for the GC heap.
if (!(gc_heap::heap_hard_limit))
{
if (is_restricted)
if (gc_heap::is_restricted_physical_mem)
{
uint64_t physical_mem_for_gc = gc_heap::total_physical_mem * (uint64_t)75 / (uint64_t)100;
gc_heap::heap_hard_limit = (size_t)max ((20 * 1024 * 1024), physical_mem_for_gc);
Expand Down
3 changes: 3 additions & 0 deletions src/coreclr/src/gc/gcpriv.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3289,6 +3289,9 @@ class gc_heap
PER_HEAP_ISOLATED
uint32_t v_high_memory_load_th;

PER_HEAP_ISOLATED
bool is_restricted_physical_mem;

PER_HEAP_ISOLATED
uint64_t mem_one_percent;

Expand Down
19 changes: 10 additions & 9 deletions src/coreclr/src/gc/unix/gcenv.unix.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -957,7 +957,7 @@ uint32_t GCToOSInterface::GetCurrentProcessCpuCount()

// Return the size of the user-mode portion of the virtual address space of this process.
// Return:
// non zero if it has succeeded, 0 if it has failed
// non zero if it has succeeded, (size_t)-1 if not available
size_t GCToOSInterface::GetVirtualMemoryLimit()
{
#ifdef HOST_64BIT
Expand Down Expand Up @@ -1143,28 +1143,28 @@ uint64_t GetAvailablePageFile()

// Get memory status
// Parameters:
// restricted_limit - The amount of physical memory in bytes that the current process is being restricted to. If non-zero, it used to calculate
// memory_load and available_physical. If zero, memory_load and available_physical is calculate based on all available memory.
// memory_load - A number between 0 and 100 that specifies the approximate percentage of physical memory
// that is in use (0 indicates no memory use and 100 indicates full memory use).
// available_physical - The amount of physical memory currently available, in bytes.
// available_page_file - The maximum amount of memory the current process can commit, in bytes.
void GCToOSInterface::GetMemoryStatus(uint32_t* memory_load, uint64_t* available_physical, uint64_t* available_page_file)
void GCToOSInterface::GetMemoryStatus(uint64_t restricted_limit, uint32_t* memory_load, uint64_t* available_physical, uint64_t* available_page_file)
{
uint64_t available = 0;
uint32_t load = 0;

if (memory_load != nullptr || available_physical != nullptr)
{
size_t used;
bool isRestricted;
uint64_t total = GetPhysicalMemoryLimit(&isRestricted);
if (isRestricted)
if (restricted_limit != 0)
{
// Get the physical memory in use - from it, we can get the physical memory available.
// We do this only when we have the total physical memory available.
if (GetPhysicalMemoryUsed(&used))
{
available = total > used ? total-used : 0;
load = (uint32_t)(((float)used * 100) / (float)total);
available = restricted_limit > used ? restricted_limit - used : 0;
load = (uint32_t)(((float)used * 100) / (float)restricted_limit);
}
}
else
Expand All @@ -1173,7 +1173,9 @@ void GCToOSInterface::GetMemoryStatus(uint32_t* memory_load, uint64_t* available

if (memory_load != NULL)
{
uint32_t load = 0;
bool isRestricted;
uint64_t total = GetPhysicalMemoryLimit(&isRestricted);

if (total > available)
{
used = total - available;
Expand All @@ -1191,7 +1193,6 @@ void GCToOSInterface::GetMemoryStatus(uint32_t* memory_load, uint64_t* available

if (available_page_file != nullptr)
*available_page_file = GetAvailablePageFile();

}

// Get a high precision performance counter
Expand Down
78 changes: 16 additions & 62 deletions src/coreclr/src/gc/windows/gcenv.windows.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,22 +18,12 @@

GCSystemInfo g_SystemInfo;

typedef BOOL (WINAPI *PGET_PROCESS_MEMORY_INFO)(HANDLE handle, PROCESS_MEMORY_COUNTERS* memCounters, uint32_t cb);
static PGET_PROCESS_MEMORY_INFO GCGetProcessMemoryInfo = 0;

static size_t g_RestrictedPhysicalMemoryLimit = (size_t)UINTPTR_MAX;

// For 32-bit processes the virtual address range could be smaller than the amount of physical
// memory on the machine/in the container, we need to restrict by the VM.
static bool g_UseRestrictedVirtualMemory = false;

static bool g_SeLockMemoryPrivilegeAcquired = false;

static AffinitySet g_processAffinitySet;

typedef BOOL (WINAPI *PIS_PROCESS_IN_JOB)(HANDLE processHandle, HANDLE jobHandle, BOOL* result);
typedef BOOL (WINAPI *PQUERY_INFORMATION_JOB_OBJECT)(HANDLE jobHandle, JOBOBJECTINFOCLASS jobObjectInfoClass, void* lpJobObjectInfo, DWORD cbJobObjectInfoLength, LPDWORD lpReturnLength);

namespace {

static bool g_fEnableGCNumaAware;
Expand Down Expand Up @@ -297,36 +287,14 @@ static size_t GetRestrictedPhysicalMemoryLimit()
uint64_t total_virtual = 0;
uint64_t total_physical = 0;
BOOL in_job_p = FALSE;
HINSTANCE hinstKernel32 = 0;

PIS_PROCESS_IN_JOB GCIsProcessInJob = 0;
PQUERY_INFORMATION_JOB_OBJECT GCQueryInformationJobObject = 0;

hinstKernel32 = LoadLibraryEx(L"kernel32.dll", nullptr, LOAD_LIBRARY_SEARCH_SYSTEM32);
if (!hinstKernel32)
goto exit;

GCIsProcessInJob = (PIS_PROCESS_IN_JOB)GetProcAddress(hinstKernel32, "IsProcessInJob");
if (!GCIsProcessInJob)
goto exit;

if (!GCIsProcessInJob(GetCurrentProcess(), NULL, &in_job_p))
if (!IsProcessInJob(GetCurrentProcess(), NULL, &in_job_p))
goto exit;

if (in_job_p)
{
GCGetProcessMemoryInfo = (PGET_PROCESS_MEMORY_INFO)GetProcAddress(hinstKernel32, "K32GetProcessMemoryInfo");

if (!GCGetProcessMemoryInfo)
goto exit;

GCQueryInformationJobObject = (PQUERY_INFORMATION_JOB_OBJECT)GetProcAddress(hinstKernel32, "QueryInformationJobObject");

if (!GCQueryInformationJobObject)
goto exit;

JOBOBJECT_EXTENDED_LIMIT_INFORMATION limit_info;
if (GCQueryInformationJobObject (NULL, JobObjectExtendedLimitInformation, &limit_info,
if (QueryInformationJobObject (NULL, JobObjectExtendedLimitInformation, &limit_info,
sizeof(limit_info), NULL))
{
size_t job_memory_limit = (size_t)UINTPTR_MAX;
Expand Down Expand Up @@ -373,13 +341,6 @@ static size_t GetRestrictedPhysicalMemoryLimit()
if (job_physical_memory_limit == (size_t)UINTPTR_MAX)
{
job_physical_memory_limit = 0;

if (hinstKernel32 != 0)
{
FreeLibrary(hinstKernel32);
hinstKernel32 = 0;
GCGetProcessMemoryInfo = 0;
}
}

// Check to see if we are limited by VM.
Expand All @@ -399,15 +360,8 @@ static size_t GetRestrictedPhysicalMemoryLimit()

if (total_virtual < total_physical)
{
if (hinstKernel32 != 0)
{
// We can also free the lib here - if we are limited by VM we will not be calling
// GetProcessMemoryInfo.
FreeLibrary(hinstKernel32);
GCGetProcessMemoryInfo = 0;
}
g_UseRestrictedVirtualMemory = true;
job_physical_memory_limit = (size_t)total_virtual;
// Limited by virtual address space
job_physical_memory_limit = 0;
}

VolatileStore(&g_RestrictedPhysicalMemoryLimit, job_physical_memory_limit);
Expand Down Expand Up @@ -1044,7 +998,7 @@ uint32_t GCToOSInterface::GetCurrentProcessCpuCount()

// Return the size of the user-mode portion of the virtual address space of this process.
// Return:
// non zero if it has succeeded, 0 if it has failed
// non zero if it has succeeded, (size_t)-1 if not available
size_t GCToOSInterface::GetVirtualMemoryLimit()
{
MEMORYSTATUSEX memStatus;
Expand All @@ -1067,7 +1021,7 @@ uint64_t GCToOSInterface::GetPhysicalMemoryLimit(bool* is_restricted)
size_t restricted_limit = GetRestrictedPhysicalMemoryLimit();
if (restricted_limit != 0)
{
if (is_restricted && !g_UseRestrictedVirtualMemory)
if (is_restricted)
*is_restricted = true;

return restricted_limit;
Expand All @@ -1081,23 +1035,22 @@ uint64_t GCToOSInterface::GetPhysicalMemoryLimit(bool* is_restricted)

// Get memory status
// Parameters:
// restricted_limit - The amount of physical memory in bytes that the current process is being restricted to. If non-zero, it used to calculate
// memory_load and available_physical. If zero, memory_load and available_physical is calculate based on all available memory.
// memory_load - A number between 0 and 100 that specifies the approximate percentage of physical memory
// that is in use (0 indicates no memory use and 100 indicates full memory use).
// available_physical - The amount of physical memory currently available, in bytes.
// available_page_file - The maximum amount of memory the current process can commit, in bytes.
void GCToOSInterface::GetMemoryStatus(uint32_t* memory_load, uint64_t* available_physical, uint64_t* available_page_file)
void GCToOSInterface::GetMemoryStatus(uint64_t restricted_limit, uint32_t* memory_load, uint64_t* available_physical, uint64_t* available_page_file)
{
uint64_t restricted_limit = GetRestrictedPhysicalMemoryLimit();
if (restricted_limit != 0)
{
size_t workingSetSize;
BOOL status = FALSE;
if (!g_UseRestrictedVirtualMemory)
{
PROCESS_MEMORY_COUNTERS pmc;
status = GCGetProcessMemoryInfo(GetCurrentProcess(), &pmc, sizeof(pmc));
workingSetSize = pmc.WorkingSetSize;
}

PROCESS_MEMORY_COUNTERS pmc;
status = GetProcessMemoryInfo(GetCurrentProcess(), &pmc, sizeof(pmc));
workingSetSize = pmc.WorkingSetSize;

if(status)
{
Expand All @@ -1123,9 +1076,10 @@ void GCToOSInterface::GetMemoryStatus(uint32_t* memory_load, uint64_t* available
MEMORYSTATUSEX ms;
::GetProcessMemoryLoad(&ms);

if (g_UseRestrictedVirtualMemory)
// For 32-bit processes the virtual address range could be smaller than the amount of physical
// memory on the machine/in the container, we need to restrict by the VM.
if (ms.ullTotalVirtual < ms.ullTotalPhys)
{
_ASSERTE (ms.ullTotalVirtual == restricted_limit);
if (memory_load != NULL)
*memory_load = (uint32_t)((float)(ms.ullTotalVirtual - ms.ullAvailVirtual) * 100.0 / (float)ms.ullTotalVirtual);
if (available_physical != NULL)
Expand Down
Loading