diff --git a/hollow/src/main/java/com/netflix/hollow/core/memory/FixedLengthDataFactory.java b/hollow/src/main/java/com/netflix/hollow/core/memory/FixedLengthDataFactory.java index 1f34e47a21..0dd70c4b2b 100644 --- a/hollow/src/main/java/com/netflix/hollow/core/memory/FixedLengthDataFactory.java +++ b/hollow/src/main/java/com/netflix/hollow/core/memory/FixedLengthDataFactory.java @@ -40,6 +40,7 @@ public static FixedLengthData allocate(HollowBlobInput in, public static FixedLengthData allocate(long numBits, MemoryMode memoryMode, ArraySegmentRecycler memoryRecycler, String fileName) throws IOException { long numLongs = ((numBits - 1) >>> 6) + 1; + numLongs ++; // accommodate for reading a long starting at bit index within numLongs-1 long numBytes = numLongs << 3; if (memoryMode.equals(MemoryMode.ON_HEAP)) { return new FixedLengthElementArray(memoryRecycler, numBits); diff --git a/hollow/src/main/java/com/netflix/hollow/core/memory/VariableLengthDataFactory.java b/hollow/src/main/java/com/netflix/hollow/core/memory/VariableLengthDataFactory.java index d03e481f2e..19550bd6e3 100644 --- a/hollow/src/main/java/com/netflix/hollow/core/memory/VariableLengthDataFactory.java +++ b/hollow/src/main/java/com/netflix/hollow/core/memory/VariableLengthDataFactory.java @@ -99,6 +99,7 @@ private void writeRaf(int b) throws IOException { raf.write(b); } + // unused public void resize(long sizeInBytes) throws IOException { if (memoryMode.equals(MemoryMode.ON_HEAP)) { // TODO: NOP because array is resized dynamically diff --git a/hollow/src/main/java/com/netflix/hollow/core/memory/encoding/BlobByteBuffer.java b/hollow/src/main/java/com/netflix/hollow/core/memory/encoding/BlobByteBuffer.java index 386fed6775..f5fe6b8ba3 100644 --- a/hollow/src/main/java/com/netflix/hollow/core/memory/encoding/BlobByteBuffer.java +++ b/hollow/src/main/java/com/netflix/hollow/core/memory/encoding/BlobByteBuffer.java @@ -154,6 +154,7 @@ public byte getByte(long index) throws BufferUnderflowException { } else { assert(index < capacity + Long.BYTES); + LOG.warning("SNAP: This is happening, not necessarily bad but test using unit test readUsingVariableLengthDataModes"); // this situation occurs when read for bits near the end of the buffer requires reading a long value that // extends past the buffer capacity by upto Long.BYTES bytes. To handle this case, // return 0 for (index >= capacity - Long.BYTES && index < capacity ) @@ -212,29 +213,16 @@ public int putBytes(long index, long len, byte[] bytes, boolean restorePos) { } public void putByte(long index, byte value) { - if (index < 0 || index >= (this.capacity+1) << 6) { + if (index < 0 || index >= (this.capacity+1) << 6) { // SNAP: can test using testIncrement or testSimpleParity throw new IllegalStateException("Attempting to write a byte out of bounds"); } - if (index < capacity) { - int spineIndex = (int)(index >>> (shift)); - int bufferIndex = (int)(index & mask); + int spineIndex = (int)(index >>> (shift)); + int bufferIndex = (int)(index & mask); + try { spine[spineIndex].put(bufferIndex, value); - } - else { - assert(index < capacity + Long.BYTES); - // this situation occurs when write for bits near the end of the buffer requires writing a long value that - // extends past the buffer capacity by upto Long.BYTES bytes. To handle this case, ignore writes to - // (index >= capacity - Long.BYTES && index < capacity ) - // these zero bytes will be discarded anyway when the returned long value is shifted to get the queried bits - // these bytes should not hold a value - if (value != 0) { - if (index > capacity + Long.BYTES) { // SNAP: can make check more strict - throw new IllegalStateException("Attempting to write a byte beyond the max buffer capacity"); - // SNAP: TODO: move the inner check, and validate that value should be 0 or else those writes will be lost - // Just that that'll fail the testCopyBitRange unit test, but probably the right thing to do. - } - } + } catch (IndexOutOfBoundsException e) { + System.out.println("here"); } } diff --git a/hollow/src/main/java/com/netflix/hollow/core/write/HollowWriteStateEngine.java b/hollow/src/main/java/com/netflix/hollow/core/write/HollowWriteStateEngine.java index a77139183c..9b491dffbe 100644 --- a/hollow/src/main/java/com/netflix/hollow/core/write/HollowWriteStateEngine.java +++ b/hollow/src/main/java/com/netflix/hollow/core/write/HollowWriteStateEngine.java @@ -221,6 +221,7 @@ public void prepareForNextCycle() { overridePreviousHeaderTags(headerTags); try { + // SNAP: TODO: creates 1 thread per processor SimultaneousExecutor executor = new SimultaneousExecutor(getClass(), "prepare-for-next-cycle"); for(final Map.Entry typeStateEntry : writeStates.entrySet()) { diff --git a/hollow/src/test/java/com/netflix/hollow/core/memory/encoding/EncodedLongBufferTest.java b/hollow/src/test/java/com/netflix/hollow/core/memory/encoding/EncodedLongBufferTest.java index 6a8f992a03..64f2d2b072 100644 --- a/hollow/src/test/java/com/netflix/hollow/core/memory/encoding/EncodedLongBufferTest.java +++ b/hollow/src/test/java/com/netflix/hollow/core/memory/encoding/EncodedLongBufferTest.java @@ -1,5 +1,6 @@ package com.netflix.hollow.core.memory.encoding; +import com.netflix.hollow.core.memory.pool.WastefulRecycler; import com.netflix.hollow.core.read.HollowBlobInput; import java.io.File; import java.io.IOException; @@ -37,14 +38,18 @@ public void writeThenRead() throws IOException { } } - EncodedLongBuffer setupEncodedLongBuffer(int fileSizeInBytes, int singleBufferCapacity) throws IOException { + // SNAP: TODO: here + EncodedLongBuffer setupEncodedLongBuffer(long numBits, int singleBufferCapacity) throws IOException { + long numLongs = ((numBits - 1) >>> 6) + 1; + numLongs ++; + long numBytes = numLongs * Long.BYTES; File targetFile = new File("test-EncodedLongBuffer-" + System.currentTimeMillis() + "-" + RandomUtils.nextInt()); targetFile.deleteOnExit(); RandomAccessFile raf = new RandomAccessFile(targetFile, "rw"); - raf.setLength(fileSizeInBytes); + raf.setLength(numBytes); raf.close(); HollowBlobInput hbi = HollowBlobInput.randomAccess(targetFile, singleBufferCapacity); - EncodedLongBuffer buf = EncodedLongBuffer.newFrom(hbi, (fileSizeInBytes >> 3)); + EncodedLongBuffer buf = EncodedLongBuffer.newFrom(hbi, numLongs); return buf; } @@ -107,8 +112,8 @@ public void testCopyBitRange() throws IOException { int copyFromRangeStartBit = rand.nextInt(totalBitsInArray - totalBitsInCopyRange); int copyToRangeStartBit = rand.nextInt(100000); - EncodedLongBuffer source = setupEncodedLongBuffer((totalBitsInArray >> 3) + 1, singleBufferCapacity); - EncodedLongBuffer dest = setupEncodedLongBuffer((totalBitsInArray + copyToRangeStartBit >> 3) + 1, singleBufferCapacity); + EncodedLongBuffer source = setupEncodedLongBuffer(totalBitsInArray, singleBufferCapacity); + EncodedLongBuffer dest = setupEncodedLongBuffer(totalBitsInArray + copyToRangeStartBit, singleBufferCapacity); int numLongs = (totalBitsInArray >>> 6); @@ -140,8 +145,8 @@ public void testCopyBitRange() throws IOException { @Test public void testCopySmallBitRange() throws IOException { int singleBufferCapacity = 1024; - EncodedLongBuffer bufFrom = setupEncodedLongBuffer((64 >> 3) + 1, singleBufferCapacity); - EncodedLongBuffer bufTo = setupEncodedLongBuffer((128 >> 3) + 1, singleBufferCapacity); + EncodedLongBuffer bufFrom = setupEncodedLongBuffer(64, singleBufferCapacity); + EncodedLongBuffer bufTo = setupEncodedLongBuffer(128, singleBufferCapacity); bufFrom.setElementValue(0, 64, -1L); @@ -154,32 +159,47 @@ public void testCopySmallBitRange() throws IOException { } + @Test + public void testSimpleParity() throws Exception { + FixedLengthElementArray arr = new FixedLengthElementArray(WastefulRecycler.SMALL_ARRAY_RECYCLER, 1000000); + EncodedLongBuffer buf = setupEncodedLongBuffer(1000000, 1024); + + arr.setElementValue(999960, 60, 1700037421l); + buf.setElementValue(999960, 60, 1700037421l); + + long l1 = arr.getElementValue(999960, 60); + long l2 = buf.getElementValue(999960, 60); + + assert (l1 == l2); + + } + @Test public void testIncrement() throws IOException { int singleBufferCapacity = 1024; int numBits = 1000000; - EncodedLongBuffer buf = setupEncodedLongBuffer((numBits >> 3) + 1, singleBufferCapacity); + EncodedLongBuffer buf = setupEncodedLongBuffer(numBits, singleBufferCapacity); Random rand = new Random(); long startVal = rand.nextInt(Integer.MAX_VALUE); int elementCount = 0; - for(int i=0;i<1000000-64;i+=65) { + for(int i=0;i<1000000;i+=65) { buf.setElementValue(i, 60, startVal+i); elementCount++; } buf.incrementMany(0, 1000, 65, elementCount); - for(int i=0;i<1000000-64;i+=65) { + for(int i=0;i<1000000;i+=65) { long val = buf.getElementValue(i, 60); Assert.assertEquals(startVal + i + 1000, val); } buf.incrementMany(0, -2000, 65, elementCount); - for(int i=0;i<1000000-64;i+=65) { + for(int i=0;i<1000000;i+=65) { long val = buf.getElementValue(i, 60); Assert.assertEquals(startVal + i - 1000, val); } diff --git a/hollow/src/test/java/com/netflix/hollow/core/memory/encoding/FixedLengthElementArrayTest.java b/hollow/src/test/java/com/netflix/hollow/core/memory/encoding/FixedLengthElementArrayTest.java index 4f47512d4a..37ae3646b2 100644 --- a/hollow/src/test/java/com/netflix/hollow/core/memory/encoding/FixedLengthElementArrayTest.java +++ b/hollow/src/test/java/com/netflix/hollow/core/memory/encoding/FixedLengthElementArrayTest.java @@ -257,28 +257,29 @@ public void testCopySmallBitRange() { @Test public void testIncrement() { - FixedLengthElementArray arr = new FixedLengthElementArray(WastefulRecycler.SMALL_ARRAY_RECYCLER, 1000000); + FixedLengthElementArray arr = new FixedLengthElementArray(WastefulRecycler.SMALL_ARRAY_RECYCLER, 1000192); Random rand = new Random(); long startVal = rand.nextInt(Integer.MAX_VALUE); int elementCount = 0; - for(int i=0;i<1000000-64;i+=65) { + // for(int i=0;i<1000192;i+=65) { // SNAP: TODO: This will attempt to write past the long array capacity + for(int i=0;i<1000000;i+=65) { arr.setElementValue(i, 60, startVal+i); elementCount++; } arr.incrementMany(0, 1000, 65, elementCount); - for(int i=0;i<1000000-64;i+=65) { + for(int i=0;i<1000000;i+=65) { long val = arr.getElementValue(i, 60); Assert.assertEquals(startVal + i + 1000, val); } arr.incrementMany(0, -2000, 65, elementCount); - for(int i=0;i<1000000-64;i+=65) { + for(int i=0;i<1000000;i+=65) { long val = arr.getElementValue(i, 60); Assert.assertEquals(startVal + i - 1000, val); } diff --git a/hollow/src/test/java/com/netflix/hollow/core/memory/encoding/OnHeapArrayVsOffHeapBufferAcceptanceTest.java b/hollow/src/test/java/com/netflix/hollow/core/memory/encoding/OnHeapArrayVsOffHeapBufferAcceptanceTest.java index 8237bf0f78..6e0b3128dc 100644 --- a/hollow/src/test/java/com/netflix/hollow/core/memory/encoding/OnHeapArrayVsOffHeapBufferAcceptanceTest.java +++ b/hollow/src/test/java/com/netflix/hollow/core/memory/encoding/OnHeapArrayVsOffHeapBufferAcceptanceTest.java @@ -137,7 +137,7 @@ private void readUsingVariableLengthDataModes(File testFile, int padding) throws assertEquals(testByteArray.get(13 + padding), testByteBuffer.get(13 + padding)); assertEquals(testByteArray.get(127 + padding), testByteBuffer.get(127 + padding)); - // SNAP: TODO: // out of bounds read + // uncomment to test BlobByteBufferTest::getByte() // try { // testByteBuffer.get(testFile.length()); // Assert.fail(); diff --git a/hollow/test-EncodedLongBuffer-1687437102761-437783410 b/hollow/test-EncodedLongBuffer-1687437102761-437783410 new file mode 100644 index 0000000000..d0ca08f0e7 Binary files /dev/null and b/hollow/test-EncodedLongBuffer-1687437102761-437783410 differ diff --git a/hollow/test-EncodedLongBuffer-1687438735947-1120024470 b/hollow/test-EncodedLongBuffer-1687438735947-1120024470 new file mode 100644 index 0000000000..402f70da28 Binary files /dev/null and b/hollow/test-EncodedLongBuffer-1687438735947-1120024470 differ diff --git a/hollow/test-EncodedLongBuffer-1687439593282-2106337630 b/hollow/test-EncodedLongBuffer-1687439593282-2106337630 new file mode 100644 index 0000000000..e6452d6034 Binary files /dev/null and b/hollow/test-EncodedLongBuffer-1687439593282-2106337630 differ diff --git a/hollow/test-EncodedLongBuffer-1687439649329-969764013 b/hollow/test-EncodedLongBuffer-1687439649329-969764013 new file mode 100644 index 0000000000..e6452d6034 Binary files /dev/null and b/hollow/test-EncodedLongBuffer-1687439649329-969764013 differ diff --git a/hollow/test-EncodedLongBuffer-1687439788955-2090512045 b/hollow/test-EncodedLongBuffer-1687439788955-2090512045 new file mode 100644 index 0000000000..e6452d6034 Binary files /dev/null and b/hollow/test-EncodedLongBuffer-1687439788955-2090512045 differ diff --git a/hollow/test-EncodedLongBuffer-1687439965044-265339152 b/hollow/test-EncodedLongBuffer-1687439965044-265339152 new file mode 100644 index 0000000000..e6452d6034 Binary files /dev/null and b/hollow/test-EncodedLongBuffer-1687439965044-265339152 differ diff --git a/hollow/test-EncodedLongBuffer-1687440026051-1992638976 b/hollow/test-EncodedLongBuffer-1687440026051-1992638976 new file mode 100644 index 0000000000..394387234e Binary files /dev/null and b/hollow/test-EncodedLongBuffer-1687440026051-1992638976 differ diff --git a/hollow/test-EncodedLongBuffer-1687440055603-383576950 b/hollow/test-EncodedLongBuffer-1687440055603-383576950 new file mode 100644 index 0000000000..394387234e Binary files /dev/null and b/hollow/test-EncodedLongBuffer-1687440055603-383576950 differ diff --git a/hollow/test-EncodedLongBuffer-1687440069594-272269472 b/hollow/test-EncodedLongBuffer-1687440069594-272269472 new file mode 100644 index 0000000000..394387234e Binary files /dev/null and b/hollow/test-EncodedLongBuffer-1687440069594-272269472 differ diff --git a/hollow/test-EncodedLongBuffer-1687440173082-941114219 b/hollow/test-EncodedLongBuffer-1687440173082-941114219 new file mode 100644 index 0000000000..dacfcd8aa5 Binary files /dev/null and b/hollow/test-EncodedLongBuffer-1687440173082-941114219 differ diff --git a/hollow/test-EncodedLongBuffer-1687440567458-2019720491 b/hollow/test-EncodedLongBuffer-1687440567458-2019720491 new file mode 100644 index 0000000000..dacfcd8aa5 Binary files /dev/null and b/hollow/test-EncodedLongBuffer-1687440567458-2019720491 differ diff --git a/hollow/test-EncodedLongBuffer-1687441265265-166181696 b/hollow/test-EncodedLongBuffer-1687441265265-166181696 new file mode 100644 index 0000000000..8d168421ad Binary files /dev/null and b/hollow/test-EncodedLongBuffer-1687441265265-166181696 differ diff --git a/hollow/test-EncodedLongBuffer-1687441342392-1972517264 b/hollow/test-EncodedLongBuffer-1687441342392-1972517264 new file mode 100644 index 0000000000..95893983f1 Binary files /dev/null and b/hollow/test-EncodedLongBuffer-1687441342392-1972517264 differ diff --git a/hollow/test-EncodedLongBuffer-1687441356558-1229632638 b/hollow/test-EncodedLongBuffer-1687441356558-1229632638 new file mode 100644 index 0000000000..94f208d223 Binary files /dev/null and b/hollow/test-EncodedLongBuffer-1687441356558-1229632638 differ diff --git a/hollow/test-EncodedLongBuffer-1687441537101-1509461061 b/hollow/test-EncodedLongBuffer-1687441537101-1509461061 new file mode 100644 index 0000000000..8d168421ad Binary files /dev/null and b/hollow/test-EncodedLongBuffer-1687441537101-1509461061 differ diff --git a/hollow/test-EncodedLongBuffer-1687441637440-1831748355 b/hollow/test-EncodedLongBuffer-1687441637440-1831748355 new file mode 100644 index 0000000000..8d168421ad Binary files /dev/null and b/hollow/test-EncodedLongBuffer-1687441637440-1831748355 differ diff --git a/hollow/test-EncodedLongBuffer-1687441820596-615554462 b/hollow/test-EncodedLongBuffer-1687441820596-615554462 new file mode 100644 index 0000000000..94f208d223 Binary files /dev/null and b/hollow/test-EncodedLongBuffer-1687441820596-615554462 differ