- * Note that even though we use long for indexing, the map can support up to 2^31 keys because - * we use 32 bit MurmurHash. In either case, if the key cardinality is so high, you should probably - * be using sorting instead of hashing for better cache locality. + * The map can support up to 2^31 keys because we use 32 bit MurmurHash. If the key cardinality is + * higher than this, you should probably be using sorting instead of hashing for better cache + * locality. *
* This class is not thread safe. */ @@ -114,6 +114,8 @@ public final class BytesToBytesMap { /** * Mask for truncating hashcodes so that they do not exceed the long array's size. + * This is a strength reduction optimization; we're essentially performing a modulus operation, + * but doing so with a bitmask because this is a power-of-2-sized hash map. */ private int mask; @@ -278,10 +280,14 @@ public final class Location { private void updateAddressesAndSizes(long fullKeyAddress) { final Object page = memoryManager.getPage(fullKeyAddress); final long keyOffsetInPage = memoryManager.getOffsetInPage(fullKeyAddress); - keyMemoryLocation.setObjAndOffset(page, keyOffsetInPage + 8); - keyLength = (int) PlatformDependent.UNSAFE.getLong(page, keyOffsetInPage); - valueMemoryLocation.setObjAndOffset(page, keyOffsetInPage + 8 + keyLength + 8); - valueLength = (int) PlatformDependent.UNSAFE.getLong(page, keyOffsetInPage + 8 + keyLength); + long position = keyOffsetInPage; + keyLength = (int) PlatformDependent.UNSAFE.getLong(page, position); + position += 8; // word used to store the key size + keyMemoryLocation.setObjAndOffset(page, position); + position += keyLength; + valueLength = (int) PlatformDependent.UNSAFE.getLong(page, position); + position += 8; // word used to store the key size + valueMemoryLocation.setObjAndOffset(page, position); } Location with(int pos, int keyHashcode, boolean isDefined) { @@ -377,7 +383,8 @@ public void putNewKey( // Here, we'll copy the data into our data pages. Because we only store a relative offset from // the key address instead of storing the absolute address of the value, the key and value // must be stored in the same memory page. - final long requiredSize = 8 + 8 + keyLengthBytes + valueLengthBytes; + // (8 byte key length) (key) (8 byte value length) (value) + final long requiredSize = 8 + keyLengthBytes + 8 + valueLengthBytes; assert(requiredSize <= PAGE_SIZE_BYTES); size++; bitset.set(pos); @@ -394,11 +401,11 @@ public void putNewKey( final Object pageBaseObject = currentDataPage.getBaseObject(); final long pageBaseOffset = currentDataPage.getBaseOffset(); final long keySizeOffsetInPage = pageBaseOffset + pageCursor; - pageCursor += 8; + pageCursor += 8; // word used to store the key size final long keyDataOffsetInPage = pageBaseOffset + pageCursor; pageCursor += keyLengthBytes; final long valueSizeOffsetInPage = pageBaseOffset + pageCursor; - pageCursor += 8; + pageCursor += 8; // word used to store the value size final long valueDataOffsetInPage = pageBaseOffset + pageCursor; pageCursor += valueLengthBytes; diff --git a/unsafe/src/main/java/org/apache/spark/unsafe/memory/MemoryBlock.java b/unsafe/src/main/java/org/apache/spark/unsafe/memory/MemoryBlock.java index 49963cc099b29..0beb743e5644e 100644 --- a/unsafe/src/main/java/org/apache/spark/unsafe/memory/MemoryBlock.java +++ b/unsafe/src/main/java/org/apache/spark/unsafe/memory/MemoryBlock.java @@ -34,10 +34,6 @@ public class MemoryBlock extends MemoryLocation { */ int pageNumber = -1; - public int getPageNumber() { - return pageNumber; - } - MemoryBlock(@Nullable Object obj, long offset, long length) { super(obj, offset); this.length = length; @@ -58,13 +54,6 @@ public MemoryBlock zero() { return this; } - /** - * Creates a memory block pointing to the memory used by the byte array. - */ - public static MemoryBlock fromByteArray(final byte[] array) { - return new MemoryBlock(array, PlatformDependent.BYTE_ARRAY_OFFSET, array.length); - } - /** * Creates a memory block pointing to the memory used by the long array. */ diff --git a/unsafe/src/main/java/org/apache/spark/unsafe/memory/MemoryManager.java b/unsafe/src/main/java/org/apache/spark/unsafe/memory/MemoryManager.java index 3b6c8b09f50e8..e3b3da52e19ee 100644 --- a/unsafe/src/main/java/org/apache/spark/unsafe/memory/MemoryManager.java +++ b/unsafe/src/main/java/org/apache/spark/unsafe/memory/MemoryManager.java @@ -35,13 +35,17 @@ * store a "page number" and the lower 51 bits to store an offset within this page. These page * numbers are used to index into a "page table" array inside of the MemoryManager in order to * retrieve the base object. + *
+ * This allows us to address 8192 pages. In on-heap mode, the maximum page size is limited by the + * maximum size of a long[] array, allowing us to address 8192 * 2^32 * 8 bytes, which is + * approximately 35 terabytes of memory. */ public final class MemoryManager { /** * The number of entries in the page table. */ - private static final int PAGE_TABLE_SIZE = (int) 1L << 13; + private static final int PAGE_TABLE_SIZE = 1 << 13; /** Bit mask for the lower 51 bits of a long. */ private static final long MASK_LONG_LOWER_51_BITS = 0x7FFFFFFFFFFFFL; diff --git a/unsafe/src/test/java/org/apache/spark/unsafe/array/TestLongArray.java b/unsafe/src/test/java/org/apache/spark/unsafe/array/TestLongArray.java index e49e344041ad7..53492226a43d5 100644 --- a/unsafe/src/test/java/org/apache/spark/unsafe/array/TestLongArray.java +++ b/unsafe/src/test/java/org/apache/spark/unsafe/array/TestLongArray.java @@ -24,18 +24,13 @@ public class TestLongArray { - private static LongArray createTestData() { - byte[] bytes = new byte[16]; - LongArray arr = new LongArray(MemoryBlock.fromByteArray(bytes)); + @Test + public void basicTest() { + long[] bytes = new long[2]; + LongArray arr = new LongArray(MemoryBlock.fromLongArray(bytes)); arr.set(0, 1L); arr.set(1, 2L); arr.set(1, 3L); - return arr; - } - - @Test - public void basicTest() { - LongArray arr = createTestData(); Assert.assertEquals(2, arr.size()); Assert.assertEquals(1L, arr.get(0)); Assert.assertEquals(3L, arr.get(1));