Skip to content

Commit

Permalink
Save the extra object allocation in read path
Browse files Browse the repository at this point in the history
  • Loading branch information
@Sunjeet
Sunjeet committed Oct 18, 2023
1 parent fee0eaa commit 93b298a
Show file tree
Hide file tree
Showing 2 changed files with 63 additions and 60 deletions.
60 changes: 48 additions & 12 deletions ...w/src/main/java/com/netflix/hollow/core/read/engine/object/HollowObjectTypeReadState.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -511,17 +511,26 @@ public byte[] readBytes(int ordinal, int fieldIndex) {

HollowObjectTypeReadState.ShardsHolder shardsHolder;
HollowObjectTypeReadStateShard shard;
HollowObjectTypeReadStateShard.VarLenStats stats;
byte[] result;
int numBitsForField;
long currentBitOffset;
long endByte;
long startByte;
int shardOrdinal;

do {
do {
shardsHolder = this.shardsVolatile;
shard = shardsHolder.shards[ordinal & shardsHolder.shardNumberMask];
stats = shard.readVarLenStats(ordinal >> shard.shardOrdinalShift, fieldIndex);
shardOrdinal = ordinal >> shard.shardOrdinalShift;

numBitsForField = shard.dataElements.bitsPerField[fieldIndex];
currentBitOffset = shard.fieldOffset(shardOrdinal, fieldIndex);
endByte = shard.dataElements.fixedLengthData.getElementValue(currentBitOffset, numBitsForField);
startByte = shardOrdinal != 0 ? shard.dataElements.fixedLengthData.getElementValue(currentBitOffset - shard.dataElements.bitsPerRecord, numBitsForField) : 0;
} while (readWasUnsafe(shardsHolder));

result = shard.readBytes(stats, fieldIndex);
result = shard.readBytes(startByte, endByte, numBitsForField, fieldIndex);
} while (readWasUnsafe(shardsHolder));

return result;
Expand All @@ -533,17 +542,26 @@ public String readString(int ordinal, int fieldIndex) {

HollowObjectTypeReadState.ShardsHolder shardsHolder;
HollowObjectTypeReadStateShard shard;
HollowObjectTypeReadStateShard.VarLenStats stats;
String result;
int numBitsForField;
long currentBitOffset;
long endByte;
long startByte;
int shardOrdinal;

do {
do {
shardsHolder = this.shardsVolatile;
shard = shardsHolder.shards[ordinal & shardsHolder.shardNumberMask];
stats = shard.readVarLenStats(ordinal >> shard.shardOrdinalShift, fieldIndex);
shardOrdinal = ordinal >> shard.shardOrdinalShift;

numBitsForField = shard.dataElements.bitsPerField[fieldIndex];
currentBitOffset = shard.fieldOffset(shardOrdinal, fieldIndex);
endByte = shard.dataElements.fixedLengthData.getElementValue(currentBitOffset, numBitsForField);
startByte = shardOrdinal != 0 ? shard.dataElements.fixedLengthData.getElementValue(currentBitOffset - shard.dataElements.bitsPerRecord, numBitsForField) : 0;
} while(readWasUnsafe(shardsHolder));

result = shard.readString(stats, fieldIndex);
result = shard.readString(startByte, endByte, numBitsForField, fieldIndex);
} while(readWasUnsafe(shardsHolder));

return result;
Expand All @@ -555,17 +573,26 @@ public boolean isStringFieldEqual(int ordinal, int fieldIndex, String testValue)

HollowObjectTypeReadState.ShardsHolder shardsHolder;
HollowObjectTypeReadStateShard shard;
HollowObjectTypeReadStateShard.VarLenStats stats;
boolean result;
int numBitsForField;
long currentBitOffset;
long endByte;
long startByte;
int shardOrdinal;

do {
do {
shardsHolder = this.shardsVolatile;
shard = shardsHolder.shards[ordinal & shardsHolder.shardNumberMask];
stats = shard.readVarLenStats(ordinal >> shard.shardOrdinalShift, fieldIndex);
shardOrdinal = ordinal >> shard.shardOrdinalShift;

numBitsForField = shard.dataElements.bitsPerField[fieldIndex];
currentBitOffset = shard.fieldOffset(shardOrdinal, fieldIndex);
endByte = shard.dataElements.fixedLengthData.getElementValue(currentBitOffset, numBitsForField);
startByte = shardOrdinal != 0 ? shard.dataElements.fixedLengthData.getElementValue(currentBitOffset - shard.dataElements.bitsPerRecord, numBitsForField) : 0;
} while(readWasUnsafe(shardsHolder));

result = shard.isStringFieldEqual(stats, fieldIndex, testValue);
result = shard.isStringFieldEqual(startByte, endByte, numBitsForField, fieldIndex, testValue);
} while(readWasUnsafe(shardsHolder));

return result;
Expand All @@ -577,17 +604,26 @@ public int findVarLengthFieldHashCode(int ordinal, int fieldIndex) {

HollowObjectTypeReadState.ShardsHolder shardsHolder;
HollowObjectTypeReadStateShard shard;
HollowObjectTypeReadStateShard.VarLenStats stats;
int hashCode;
int numBitsForField;
long currentBitOffset;
long endByte;
long startByte;
int shardOrdinal;

do {
do {
shardsHolder = this.shardsVolatile;
shard = shardsHolder.shards[ordinal & shardsHolder.shardNumberMask];
stats = shard.readVarLenStats(ordinal >> shard.shardOrdinalShift, fieldIndex);
shardOrdinal = ordinal >> shard.shardOrdinalShift;

numBitsForField = shard.dataElements.bitsPerField[fieldIndex];
currentBitOffset = shard.fieldOffset(shardOrdinal, fieldIndex);
endByte = shard.dataElements.fixedLengthData.getElementValue(currentBitOffset, numBitsForField);
startByte = shardOrdinal != 0 ? shard.dataElements.fixedLengthData.getElementValue(currentBitOffset - shard.dataElements.bitsPerRecord, numBitsForField) : 0;
} while(readWasUnsafe(shardsHolder));

hashCode = shard.findVarLengthFieldHashCode(stats, fieldIndex);
hashCode = shard.findVarLengthFieldHashCode(startByte, endByte, numBitsForField, fieldIndex);
} while(readWasUnsafe(shardsHolder));

return hashCode;
Expand Down
63 changes: 15 additions & 48 deletions .../main/java/com/netflix/hollow/core/read/engine/object/HollowObjectTypeReadStateShard.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,35 +86,9 @@ private long readFixedLengthFieldValue(int ordinal, int fieldIndex) {
return value;
}

static class VarLenStats {
final int numBitsForField;
final long startByte;
final long endByte;

public VarLenStats(int numBitsForField, long startByte, long endByte) {
this.numBitsForField = numBitsForField;
this.startByte = startByte;
this.endByte = endByte;
}
}

VarLenStats readVarLenStats(int ordinal, int fieldIndex) {
int numBitsForField = dataElements.bitsPerField[fieldIndex];
long currentBitOffset = fieldOffset(ordinal, fieldIndex);

long endByte = dataElements.fixedLengthData.getElementValue(currentBitOffset, numBitsForField);
long startByte = ordinal != 0 ? dataElements.fixedLengthData.getElementValue(currentBitOffset - dataElements.bitsPerRecord, numBitsForField) : 0;

return new VarLenStats(numBitsForField, startByte, endByte);
}

public byte[] readBytes(VarLenStats stats, int fieldIndex) {
public byte[] readBytes(long startByte, long endByte, int numBitsForField, int fieldIndex) {
byte[] result;

int numBitsForField = stats.numBitsForField;
long startByte = stats.startByte;
long endByte = stats.endByte;

if((endByte & (1L << numBitsForField - 1)) != 0)
return null;

Expand All @@ -129,11 +103,7 @@ public byte[] readBytes(VarLenStats stats, int fieldIndex) {
return result;
}

public String readString(VarLenStats stats, int fieldIndex) {
int numBitsForField = stats.numBitsForField;
long startByte = stats.startByte;
long endByte = stats.endByte;

public String readString(long startByte, long endByte, int numBitsForField, int fieldIndex) {
if((endByte & (1L << numBitsForField - 1)) != 0)
return null;

Expand All @@ -144,11 +114,7 @@ public String readString(VarLenStats stats, int fieldIndex) {
return readString(dataElements.varLengthData[fieldIndex], startByte, length);
}

public boolean isStringFieldEqual(VarLenStats stats, int fieldIndex, String testValue) {
int numBitsForField = stats.numBitsForField;
long startByte = stats.startByte;
long endByte = stats.endByte;

public boolean isStringFieldEqual(long startByte, long endByte, int numBitsForField, int fieldIndex, String testValue) {
if((endByte & (1L << numBitsForField - 1)) != 0)
return testValue == null;
if(testValue == null)
Expand All @@ -161,11 +127,7 @@ public boolean isStringFieldEqual(VarLenStats stats, int fieldIndex, String test
return testStringEquality(dataElements.varLengthData[fieldIndex], startByte, length, testValue);
}

public int findVarLengthFieldHashCode(VarLenStats stats, int fieldIndex) {
int numBitsForField = stats.numBitsForField;
long startByte = stats.startByte;
long endByte = stats.endByte;

public int findVarLengthFieldHashCode(long startByte, long endByte, int numBitsForField, int fieldIndex) {
if((endByte & (1L << numBitsForField - 1)) != 0)
return -1;

Expand All @@ -184,7 +146,7 @@ public int bitsRequiredForField(String fieldName) {
return fieldIndex == -1 ? 0 : dataElements.bitsPerField[fieldIndex];
}

private long fieldOffset(int ordinal, int fieldIndex) {
long fieldOffset(int ordinal, int fieldIndex) {
return ((long)dataElements.bitsPerRecord * ordinal) + dataElements.bitOffsetPerField[fieldIndex];
}

Expand Down Expand Up @@ -227,11 +189,15 @@ private boolean testStringEquality(ByteData data, long position, int length, Str
}

protected void applyShardToChecksum(HollowChecksum checksum, HollowSchema withSchema, BitSet populatedOrdinals, int shardNumber, int shardNumberMask) {
int numBitsForField;
long bitOffset;
long endByte;
long startByte;

if(!(withSchema instanceof HollowObjectSchema))
throw new IllegalArgumentException("HollowObjectTypeReadState can only calculate checksum with a HollowObjectSchema: " + schema.getName());

HollowObjectSchema commonSchema = schema.findCommonSchema((HollowObjectSchema)withSchema);
VarLenStats stats;

List<String> commonFieldNames = new ArrayList<String>();
for(int i=0;i<commonSchema.numFields();i++)
Expand All @@ -250,9 +216,9 @@ protected void applyShardToChecksum(HollowChecksum checksum, HollowSchema withSc
checksum.applyInt(ordinal);
for(int i=0;i<fieldIndexes.length;i++) {
int fieldIdx = fieldIndexes[i];
bitOffset = fieldOffset(shardOrdinal, fieldIdx);
numBitsForField = dataElements.bitsPerField[fieldIdx];
if(!schema.getFieldType(fieldIdx).isVariableLength()) {
long bitOffset = fieldOffset(shardOrdinal, fieldIdx);
int numBitsForField = dataElements.bitsPerField[fieldIdx];
long fixedLengthValue = numBitsForField <= 56 ?
dataElements.fixedLengthData.getElementValue(bitOffset, numBitsForField)
: dataElements.fixedLengthData.getLargeElementValue(bitOffset, numBitsForField);
Expand All @@ -262,8 +228,9 @@ protected void applyShardToChecksum(HollowChecksum checksum, HollowSchema withSc
else
checksum.applyLong(fixedLengthValue);
} else {
stats = readVarLenStats(shardOrdinal, fieldIdx);
checksum.applyInt(findVarLengthFieldHashCode(stats, fieldIdx));
endByte = dataElements.fixedLengthData.getElementValue(bitOffset, numBitsForField);
startByte = shardOrdinal != 0 ? dataElements.fixedLengthData.getElementValue(bitOffset - dataElements.bitsPerRecord, numBitsForField) : 0;
checksum.applyInt(findVarLengthFieldHashCode(startByte, endByte, numBitsForField, fieldIdx));
}
}
}
Expand Down

0 comments on commit 93b298a

Please sign in to comment.