Handling multiple reserved region lists for numa off-heap case

In PR #22116
and #22546,
_arrayReservedRegionList and _sharedArrayReservedRegionsBytesUsed has
been introduced to share fraction of reserved regions.
_arrayReservedRegionList is set in commonAllocationContext(single
global list), For numa case, the reserved regions for large array could
be allocated from different allocation context(allocation context per
numa node and in case there is no free region for the numa note, it
could borrow from neighbor numa node). during recycling, there are no
extra information indicate which reserved regions are for recycling
off-heap array, so potentially it might cause imbalance free memory
among numa notes, then affect runtime performance in some cases.

Undate to manage reserved region list per allocation context(numa note)
to avoid imbalance free memory caused by reserved regions.

1, for reducing complexity, shared fraction reserved regions still be
managed in _arrayReservedRegionList of commom context.
set _sharedReserved in AllocateDescriptor for shared fraction reserved
region allocation.

2, new _allocationContextArray in SparseVirtualMemory(omr PR
eclipse-omr/omr#7956) to store/retrieve
allocation context for the reserved regions of large array.

3, handle allocation failure during getSparseAddressAndDecommitLeaves().

Signed-off-by: lhu <[email protected]>

Author: LinHu2016

runtime/gc_base/IndexableObjectAllocationModel.cpp

@@ -27,6 +27,7 @@
 #include "MemorySpace.hpp"
 #if defined(J9VM_GC_SPARSE_HEAP_ALLOCATION)
 #include "AllocationContextBalanced.hpp"
+#include "HeapRegionDataForAllocate.hpp"
 #include "EnvironmentVLHGC.hpp"
 #endif /* defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) */
 #if defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) || defined(J9VM_GC_ENABLE_DOUBLE_MAP)
@@ -345,12 +346,19 @@ MM_IndexableObjectAllocationModel::getSparseAddressAndDecommitLeaves(MM_Environm
 	MM_GCExtensions *extensions = MM_GCExtensions::getExtensions(envBase);
 	GC_ArrayObjectModel *indexableObjectModel = &extensions->indexableObjectModel;
 	const uintptr_t regionSize = extensions->heapRegionManager->getRegionSize();
-
 	uintptr_t byteAmount = 0;
 
 	/* Determine how many bytes to allocate outside of the spine (in arraylet leaves). */
 	Assert_MM_true(_allocateDescription.getBytesRequested() >= _allocateDescription.getContiguousBytes());
 	uintptr_t bytesRemaining = _allocateDescription.getBytesRequested() - _allocateDescription.getContiguousBytes();
+#define REGION_RESERVE_THRESHOLD 64
+	void *allocationContexts[REGION_RESERVE_THRESHOLD];
+	void **reservedRegionAllocationContexts = allocationContexts;
+	uintptr_t reservedRegionCount = bytesRemaining / regionSize;
+
+	if (reservedRegionCount > REGION_RESERVE_THRESHOLD) {
+		reservedRegionAllocationContexts = (void **)envBase->getForge()->allocate(reservedRegionCount * sizeof(uintptr_t), MM_AllocationCategory::GC_HEAP, J9_GET_CALLSITE());
+	}
 
 	MM_EnvironmentVLHGC *env = MM_EnvironmentVLHGC::getEnvironment(envBase);
 	MM_AllocationContextBalanced *commonContext = (MM_AllocationContextBalanced *)env->getCommonAllocationContext();
@@ -359,26 +367,44 @@ MM_IndexableObjectAllocationModel::getSparseAddressAndDecommitLeaves(MM_Environm
 	uintptr_t arrayReservedRegionCount = 0;
 	uintptr_t fraction = 0;
 	while (0 < bytesRemaining) {
-		/* Allocate the next arraylet leaf - leaves are allocated solely for the purpose of
+		/* Allocate the next reserved region - reserved regions are allocated solely for the purpose of
 		   decommitting the memory later on in this function. */
 		void *reservedAddressLow = NULL;
 		bool shouldAllocateReservedRegion = true;
+
 		if (regionSize > bytesRemaining) {
 			fraction = bytesRemaining;
 			/* For code simplicity and lower fragmentation, we always use Common Context. */
 			shouldAllocateReservedRegion = commonContext->allocateFromSharedArrayReservedRegion(envBase, fraction);
 		}
 		if (shouldAllocateReservedRegion) {
+			_allocateDescription.setSharedReserved(0 != fraction);
+
 			reservedAddressLow = envBase->_objectAllocationInterface->allocateArrayletLeaf(
 					envBase, &_allocateDescription, _allocateDescription.getMemorySpace(), true);
 
 			/* If reservedRegion allocation failed set the result to NULL and return. */
 			if (NULL == reservedAddressLow) {
 				Trc_MM_allocateAndConnectNonContiguousArraylet_leafFailure(envBase->getLanguageVMThread());
 				_allocateDescription.setSpine(NULL);
+				if (0 != fraction) {
+					commonContext->recycleToSharedArrayReservedRegion(envBase, fraction);
+					fraction = 0;
+				}
 				spine = NULL;
 				break;
 			}
+
+			if (0 == fraction) {
+				MM_HeapRegionDescriptorVLHGC *reservedRegion = (MM_HeapRegionDescriptorVLHGC *)extensions->heapRegionManager->regionDescriptorForAddress(reservedAddressLow);
+				MM_HeapRegionDataForAllocate *allocateData = &reservedRegion->_allocateData;
+				if (NULL != allocateData->_originalOwningContext) {
+					reservedRegionAllocationContexts[arrayReservedRegionCount] = allocateData->_originalOwningContext;
+				} else {
+					reservedRegionAllocationContexts[arrayReservedRegionCount] = allocateData->_owningContext;
+				}
+			}
+
 			/* Disable region for reads and writes, since accessing virtualLargeObjectHeapAddress through DataAddrForContiguous */
 			void *reservedAddressHigh = (void *)((uintptr_t)reservedAddressLow + regionSize);
 			bool ret = extensions->heap->decommitMemory(reservedAddressLow, regionSize, reservedAddressLow, reservedAddressHigh);
@@ -391,17 +417,22 @@ MM_IndexableObjectAllocationModel::getSparseAddressAndDecommitLeaves(MM_Environm
 		}
 
 		bytesRemaining -= OMR_MIN(bytesRemaining, regionSize);
-		arrayReservedRegionCount += 1;
+		if (0 == fraction) {
+			arrayReservedRegionCount += 1;
+		}
 	}
 
-
 	if (NULL != spine) {
 		Assert_MM_true(_layout == GC_ArrayletObjectModel::InlineContiguous);
 		Assert_MM_true(indexableObjectModel->isVirtualLargeObjectHeapEnabled());
 
 		byteAmount = _dataSize;
 		void *virtualLargeObjectHeapAddress = extensions->largeObjectVirtualMemory->allocateSparseFreeEntryAndMapToHeapObject(spine, byteAmount);
 
+		for (uintptr_t idx = 0; idx < arrayReservedRegionCount; idx++) {
+			extensions->largeObjectVirtualMemory->setAllocationContextForAddress(virtualLargeObjectHeapAddress, reservedRegionAllocationContexts[idx], idx);
+		}
+
 		if (NULL != virtualLargeObjectHeapAddress) {
 			indexableObjectModel->setDataAddrForContiguous((J9IndexableObject *)spine, virtualLargeObjectHeapAddress);
 		} else {
@@ -414,13 +445,21 @@ MM_IndexableObjectAllocationModel::getSparseAddressAndDecommitLeaves(MM_Environm
 		/* fail to reserve regions or allocateSparseFreeEntry, clean up reserved regions */
 		if (0 != fraction) {
 			/* rollback fraction */
-			commonContext->recycleToSharedArrayReservedRegion(envBase, fraction);
+			if (commonContext->recycleToSharedArrayReservedRegion(envBase, fraction)) {
+				commonContext->recycleReservedRegionsForVirtualLargeObjectHeap(env, 1, true);
+			}
 		}
 		if (0 < arrayReservedRegionCount) {
-			((MM_HeapRegionManagerVLHGC *)extensions->heapRegionManager)->recycleReservedRegionsForVirtualLargeObjectHeap(envBase, arrayReservedRegionCount);
+			for (uintptr_t idx = 0; idx < arrayReservedRegionCount; idx++) {
+				((MM_AllocationContextBalanced *)reservedRegionAllocationContexts[idx])->recycleReservedRegionsForVirtualLargeObjectHeap(env, 1, true);
+			}
 		}
 	}
 
+	if (reservedRegionCount > REGION_RESERVE_THRESHOLD) {
+		env->getForge()->free((void *)reservedRegionAllocationContexts);
+	}
+
 	Trc_MM_getSparseAddressAndDecommitLeaves_Exit(envBase->getLanguageVMThread(), spine, (void *)bytesRemaining);
 
 	return spine;

runtime/gc_vlhgc/AllocationContextBalanced.cpp

@@ -391,18 +391,21 @@ MM_AllocationContextBalanced::lockedAllocateArrayletLeaf(MM_EnvironmentBase *env
 		 * In future, allocations should remember (somewhere in Off-heap meta structures) how many regions came from each AC
 		 * \and release exact same number back to each AC.
 		 */
-		MM_AllocationContextTarok *commonContext = (MM_AllocationContextTarok *)env->getCommonAllocationContext();
-		if (this != commonContext) {
+		MM_AllocationContextTarok *context = this;
+		if (allocateDescription->getSharedReserved()) {
+			context = (MM_AllocationContextTarok *)env->getCommonAllocationContext();
+		}
+		if (this != context) {
 			/* The common allocation context is always an instance of AllocationContextBalanced */
-			((MM_AllocationContextBalanced *)commonContext)->lockCommon();
+			((MM_AllocationContextBalanced *)context)->lockCommon();
 		}
 
-		leafAllocateData->pushRegionToArrayReservedRegionList(env, ((MM_AllocationContextBalanced *)commonContext)->getArrayReservedRegionListAddress());
-		((MM_AllocationContextBalanced *)commonContext)->incrementArrayReservedRegionCount();
+		leafAllocateData->pushRegionToArrayReservedRegionList(env, ((MM_AllocationContextBalanced *)context)->getArrayReservedRegionListAddress());
+		((MM_AllocationContextBalanced *)context)->incrementArrayReservedRegionCount();
 
-		if (this != commonContext) {
+		if (this != context) {
 			/* The common allocation context is always an instance of AllocationContextBalanced */
-			((MM_AllocationContextBalanced *)commonContext)->unlockCommon();
+			((MM_AllocationContextBalanced *)context)->unlockCommon();
 		}
 	}
 #endif /* defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) */

runtime/gc_vlhgc/CopyForwardScheme.cpp

@@ -4174,17 +4174,29 @@ class MM_CopyForwardSchemeRootClearer : public MM_RootScanner
 				uintptr_t reservedRegionCount = dataSize / regionSize;
 				uintptr_t fraction = dataSize % regionSize;
 
-				MM_AllocationContextBalanced *commonContext = (MM_AllocationContextBalanced *)env->getCommonAllocationContext();
-				if ((0 != fraction) && commonContext->recycleToSharedArrayReservedRegion(env, fraction)) {
-					reservedRegionCount += 1;
-				}
+				PORT_ACCESS_FROM_ENVIRONMENT(env);
+				j9tty_printf(PORTLIB, "MM_CopyForwardSchemeRootClearer::doObjectInVirtualLargeObjectHeap reservedRegionCount=%zu, fraction=%zu, dataSize=%zu, regionSize=%zu\n",
+						reservedRegionCount, fraction, dataSize, regionSize);
 
 				Assert_MM_mustBeClass(_extensions->objectModel.getPreservedClass(&forwardedHeader));
 				env->_copyForwardStats._offHeapRegionsCleared += 1;
 				void *dataAddr = _extensions->indexableObjectModel.getDataAddrForContiguous((J9IndexableObject *)objectPtr);
-				_extensions->largeObjectVirtualMemory->freeSparseRegionAndUnmapFromHeapObject(_env, dataAddr, objectPtr, dataSize, sparseDataEntryIterator);
+
+				MM_SparseVirtualMemory *largeObjectVirtualMemory = _extensions->largeObjectVirtualMemory;
 				/* recycleLeafRegions for off-heap case */
-				commonContext->recycleReservedRegionsForVirtualLargeObjectHeap(env, reservedRegionCount);
+				MM_AllocationContextBalanced *context = NULL;
+				for (uintptr_t index = 0; index < reservedRegionCount; index++) {
+					context = (MM_AllocationContextBalanced *) largeObjectVirtualMemory->getAllocationContextForAddress(dataAddr, index);
+					context->recycleReservedRegionsForVirtualLargeObjectHeap(env, 1);
+				}
+
+				/* commonContext for fraction region */
+				context = (MM_AllocationContextBalanced *)env->getCommonAllocationContext();
+				if ((0 != fraction) && context->recycleToSharedArrayReservedRegion(env, fraction)) {
+					context->recycleReservedRegionsForVirtualLargeObjectHeap(env, 1);
+				}
+
+				_extensions->largeObjectVirtualMemory->freeSparseRegionAndUnmapFromHeapObject(_env, dataAddr, objectPtr, dataSize, sparseDataEntryIterator);
 			} else {
 				void *dataAddr = _extensions->indexableObjectModel.getDataAddrForContiguous((J9IndexableObject *)fwdOjectPtr);
 				if (NULL != dataAddr) {

runtime/gc_vlhgc/GlobalMarkingScheme.cpp

@@ -1431,14 +1431,22 @@ class MM_GlobalMarkingSchemeRootClearer : public MM_RootScanner
 
 				uintptr_t reservedRegionCount = dataSize / regionSize;
 				uintptr_t fraction = dataSize % regionSize;
-				MM_AllocationContextBalanced *commonContext = (MM_AllocationContextBalanced *)env->getCommonAllocationContext();
 
-				if ((0 != fraction) && commonContext->recycleToSharedArrayReservedRegion(env, fraction)) {
-					reservedRegionCount += 1;
+				MM_SparseVirtualMemory *largeObjectVirtualMemory = _extensions->largeObjectVirtualMemory;
+				/* recycleLeafRegions for off-heap case */
+				MM_AllocationContextBalanced *context = NULL;
+				for (uintptr_t index = 0; index < reservedRegionCount; index++) {
+					context = (MM_AllocationContextBalanced *) largeObjectVirtualMemory->getAllocationContextForAddress(dataAddr, index);
+					context->recycleReservedRegionsForVirtualLargeObjectHeap(env, 1);
+				}
+
+				/* commonContext for fraction region */
+				context = (MM_AllocationContextBalanced *)env->getCommonAllocationContext();
+				if ((0 != fraction) && context->recycleToSharedArrayReservedRegion(env, fraction)) {
+					context->recycleReservedRegionsForVirtualLargeObjectHeap(env, 1);
 				}
 
 				_extensions->largeObjectVirtualMemory->freeSparseRegionAndUnmapFromHeapObject(_env, dataAddr, objectPtr, dataSize, sparseDataEntryIterator);
-				commonContext->recycleReservedRegionsForVirtualLargeObjectHeap(env, reservedRegionCount);
 			}
 		}
 	}

runtime/gc_vlhgc/HeapRegionManagerVLHGC.cpp

@@ -222,14 +222,3 @@ MM_HeapRegionManagerVLHGC::getHeapMemorySnapshot(MM_GCExtensionsBase *extensions
 
 	return snapshot;
 }
-
-#if defined(J9VM_GC_SPARSE_HEAP_ALLOCATION)
-void
-MM_HeapRegionManagerVLHGC::recycleReservedRegionsForVirtualLargeObjectHeap(MM_EnvironmentBase *envBase, uintptr_t reservedRegionCount)
-{
-	MM_EnvironmentVLHGC *env = MM_EnvironmentVLHGC::getEnvironment(envBase);
-	MM_AllocationContextBalanced *commonContext = (MM_AllocationContextBalanced *)env->getCommonAllocationContext();
-
-	commonContext->recycleReservedRegionsForVirtualLargeObjectHeap(env, reservedRegionCount, true);
-}
-#endif /* defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) */

runtime/gc_vlhgc/HeapRegionManagerVLHGC.hpp

@@ -42,10 +42,6 @@ class MM_HeapRegionManagerVLHGC : public MM_HeapRegionManagerTarok
 
 	virtual MM_HeapMemorySnapshot *getHeapMemorySnapshot(MM_GCExtensionsBase *extensions, MM_HeapMemorySnapshot *snapshot, bool gcEnd);
 
-#if defined(J9VM_GC_SPARSE_HEAP_ALLOCATION)
-	void recycleReservedRegionsForVirtualLargeObjectHeap(MM_EnvironmentBase *envBase, uintptr_t reservedRegionCount);
-#endif /* defined(J9VM_GC_SPARSE_HEAP_ALLOCATION) */
-
 	static MM_HeapRegionManagerVLHGC *newInstance(MM_EnvironmentBase *env, uintptr_t regionSize, uintptr_t tableDescriptorSize, MM_RegionDescriptorInitializer regionDescriptorInitializer, MM_RegionDescriptorDestructor regionDescriptorDestructor);
 	MM_HeapRegionManagerVLHGC(MM_EnvironmentBase *env, uintptr_t regionSize, uintptr_t tableDescriptorSize, MM_RegionDescriptorInitializer regionDescriptorInitializer, MM_RegionDescriptorDestructor regionDescriptorDestructor);