Skip to content

Commit

Permalink
Merge pull request #20 from Netflix/primarykey-auto-expansion
Browse files Browse the repository at this point in the history 
Primarykey auto expansion
  • Loading branch information
Tim Taylor authored Feb 17, 2017
2 parents d507a34 + fa2ab4e commit 3aac2a4
Show file tree
Hide file tree
Showing 7 changed files with 257 additions and 60 deletions.
84 changes: 74 additions & 10 deletions hollow/src/main/java/com/netflix/hollow/core/index/key/PrimaryKey.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,11 @@
*/
package com.netflix.hollow.core.index.key;

import com.netflix.hollow.core.HollowDataset;
import com.netflix.hollow.core.schema.HollowObjectSchema;
import com.netflix.hollow.core.schema.HollowObjectSchema.FieldType;

import com.netflix.hollow.core.HollowDataset;
import com.netflix.hollow.core.schema.HollowSchema;
import com.netflix.hollow.core.schema.HollowSchema.SchemaType;
import java.util.Arrays;

/**
Expand Down Expand Up @@ -90,25 +91,88 @@ public static FieldType getFieldType(HollowDataset dataAccess, String type, Stri
return schema.getFieldType(pathIndexes[pathIndexes.length - 1]);
}

/**
* Returns a separated field path, which has been auto-expanded if necessary based on the provided primary key field path.
*/
public static String[] getCompleteFieldPathParts(HollowDataset dataset, String type, String fieldPath) {
int fieldPathIdx[] = getFieldPathIndex(dataset, type, fieldPath);
String fieldPathParts[] = new String[fieldPathIdx.length];

HollowObjectSchema schema = (HollowObjectSchema) dataset.getSchema(type);
for(int i=0;i<fieldPathParts.length;i++) {
fieldPathParts[i] = schema.getFieldName(fieldPathIdx[i]);
schema = (HollowObjectSchema) dataset.getSchema(schema.getReferencedType(fieldPathIdx[i]));
}

return fieldPathParts;
}

/**
* The field path index is the object schemas' field positions for a particular field path.
*/
public static int[] getFieldPathIndex(HollowDataset dataset, String type, String fieldPath) {
String paths[] = fieldPath.split("\\.");
boolean isReferenceFieldPath = fieldPath.endsWith("!");

String paths[] = isReferenceFieldPath ? fieldPath.substring(0, fieldPath.length() - 1).split("\\.") : fieldPath.split("\\.");
int pathIndexes[] = new int[paths.length];

String refType = type;

for(int i=0;i<paths.length;i++) {
HollowObjectSchema schema = (HollowObjectSchema)dataset.getSchema(refType);
try {
pathIndexes[i] = schema.getPosition(paths[i]);
refType = schema.getReferencedType(pathIndexes[i]);
} catch (Exception ex) {
throw new RuntimeException("Failed create path index for fieldPath=" + fieldPath + ", fieldName=" + paths[i] + " schema=" + schema.getName());
HollowSchema schema = dataset.getSchema(refType);

if(schema == null)
throw new IllegalArgumentException("Invalid field path declaration for type " + type + ": " + fieldPath +". The type " + refType + " is unavailable.");

if(schema.getSchemaType() != SchemaType.OBJECT)
throw new IllegalArgumentException("Invalid field path declaration for type " + type + ": " + fieldPath + ". " +
"Field paths may only traverse through OBJECT types, but this declaration passes through a " + schema.getSchemaType().toString() + " type (" + refType + ").");


HollowObjectSchema objectSchema = (HollowObjectSchema)dataset.getSchema(refType);
pathIndexes[i] = objectSchema.getPosition(paths[i]);

if(pathIndexes[i] == -1)
throw new IllegalArgumentException("Invalid field path declaration for type " + type + ": " + fieldPath + ". " +
"At element " + i + ", the field " + paths[i] + " was not found in type " + refType + ".");

refType = objectSchema.getReferencedType(pathIndexes[i]);

if(i < paths.length - 1 && refType == null)
throw new IllegalArgumentException("Invalid field path declaration for type " + type + ": " + fieldPath + ". " +
"No available traversal after element " + i + ": " + paths[i] + ".");


}

if(!isReferenceFieldPath) {
while(refType != null) {
HollowSchema schema = dataset.getSchema(refType);

if(schema.getSchemaType() == SchemaType.OBJECT) {
HollowObjectSchema objectSchema = (HollowObjectSchema)schema;

if(objectSchema.numFields() == 1) {
pathIndexes = Arrays.copyOf(pathIndexes, pathIndexes.length + 1);
pathIndexes[pathIndexes.length - 1] = 0;
refType = ((HollowObjectSchema)schema).getReferencedType(0);
continue;
}

if(objectSchema.getPrimaryKey() != null && objectSchema.getPrimaryKey().numFields() == 1) {
int[] trailingFieldPathIndex = objectSchema.getPrimaryKey().getFieldPathIndex(dataset, 0);
int priorLength = pathIndexes.length;
pathIndexes = Arrays.copyOf(pathIndexes, pathIndexes.length + trailingFieldPathIndex.length);
System.arraycopy(trailingFieldPathIndex, 0, pathIndexes, priorLength, trailingFieldPathIndex.length);
return pathIndexes;
}
}

throw new IllegalArgumentException("Invalid field path declaration for type " + type + ": " + fieldPath + ". This path ends in a REFERENCE field which is not auto-traversable. " +
"If this is intended to actually indicate a REFERENCE field, specify the field path as \"" + fieldPath + "!\".");
}
}

return pathIndexes;
}

Expand Down
6 changes: 2 additions & 4 deletions hollow/src/main/java/com/netflix/hollow/tools/history/HollowHistory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ public HollowHistory(HollowReadStateEngine initialHollowStateEngine, long initia
* @param maxHistoricalStatesToKeep The number of historical states to keep in memory
*/
public HollowHistory(HollowReadStateEngine initialHollowStateEngine, long initialVersion, int maxHistoricalStatesToKeep, boolean isAutoDiscoverTypeIndex) {
this.keyIndex = new HollowHistoryKeyIndex();
this.keyIndex = new HollowHistoryKeyIndex(this);
this.creator = new HollowHistoricalStateCreator(this);
this.latestHollowReadStateEngine = initialHollowStateEngine;
this.latestHeaderEntries = latestHollowReadStateEngine.getHeaderTags();
Expand All @@ -98,9 +98,7 @@ public HollowHistory(HollowReadStateEngine initialHollowStateEngine, long initia
PrimaryKey pKey = ((HollowObjectSchema) schema).getPrimaryKey();
if (pKey == null) continue;

String type = schema.getName();
String[] keyFieldPaths = pKey.getFieldPaths();
keyIndex.addTypeIndex(type, keyFieldPaths );
keyIndex.addTypeIndex(pKey);
}
}
}
Expand Down
23 changes: 15 additions & 8 deletions hollow/src/main/java/com/netflix/hollow/tools/history/keyindex/HollowHistoryKeyIndex.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,14 @@
*/
package com.netflix.hollow.tools.history.keyindex;

import com.netflix.hollow.core.util.SimultaneousExecutor;

import com.netflix.hollow.core.write.HollowBlobWriter;
import com.netflix.hollow.core.write.HollowWriteStateEngine;
import com.netflix.hollow.core.index.key.PrimaryKey;
import com.netflix.hollow.core.read.engine.HollowBlobReader;
import com.netflix.hollow.core.read.engine.HollowReadStateEngine;
import com.netflix.hollow.core.read.engine.object.HollowObjectTypeReadState;
import com.netflix.hollow.core.util.SimultaneousExecutor;
import com.netflix.hollow.core.write.HollowBlobWriter;
import com.netflix.hollow.core.write.HollowWriteStateEngine;
import com.netflix.hollow.tools.history.HollowHistory;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
Expand All @@ -32,11 +33,13 @@

public class HollowHistoryKeyIndex {

private final HollowHistory history;
private final Map<String, HollowHistoryTypeKeyIndex> typeKeyIndexes;
private final HollowWriteStateEngine writeStateEngine;
private final HollowReadStateEngine readStateEngine;

public HollowHistoryKeyIndex() {
public HollowHistoryKeyIndex(HollowHistory history) {
this.history = history;
this.typeKeyIndexes = new HashMap<String, HollowHistoryTypeKeyIndex>();
this.writeStateEngine = new HollowWriteStateEngine();
this.readStateEngine = new HollowReadStateEngine();
Expand All @@ -55,12 +58,16 @@ public int getRecordKeyOrdinal(HollowObjectTypeReadState typeState, int ordinal)
}

public void addTypeIndex(String type, String... keyFieldPaths) {
HollowHistoryTypeKeyIndex keyIdx = new HollowHistoryTypeKeyIndex(type, writeStateEngine, readStateEngine, keyFieldPaths);
typeKeyIndexes.put(type, keyIdx);
addTypeIndex(new PrimaryKey(type, keyFieldPaths));
}

public void addTypeIndex(PrimaryKey primaryKey) {
HollowHistoryTypeKeyIndex keyIdx = new HollowHistoryTypeKeyIndex(primaryKey, history.getLatestState(), writeStateEngine, readStateEngine);
typeKeyIndexes.put(primaryKey.getType(), keyIdx);
}

public void indexTypeField(String type, String keyFieldPath) {
typeKeyIndexes.get(type).addFieldIndex(keyFieldPath);
typeKeyIndexes.get(type).addFieldIndex(keyFieldPath, history.getLatestState());
}

public Map<String, HollowHistoryTypeKeyIndex> getTypeKeyIndexes() {
Expand Down
71 changes: 38 additions & 33 deletions ...ow/src/main/java/com/netflix/hollow/tools/history/keyindex/HollowHistoryTypeKeyIndex.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,10 @@
*/
package com.netflix.hollow.tools.history.keyindex;

import com.netflix.hollow.core.memory.encoding.HashCodes;
import com.netflix.hollow.core.index.key.PrimaryKey;

import com.netflix.hollow.core.HollowDataset;
import com.netflix.hollow.core.memory.encoding.HashCodes;
import com.netflix.hollow.core.util.RemovedOrdinalIterator;
import com.netflix.hollow.core.util.IntList;
import com.netflix.hollow.core.util.LongList;
Expand All @@ -36,8 +38,7 @@

public class HollowHistoryTypeKeyIndex {

private final String type;
private final String keyFields[];
private final PrimaryKey primaryKey;
private final String keyFieldParts[][];
private final boolean keyFieldIsIndexed[];
private HollowObjectSchema keySchema;
Expand All @@ -51,19 +52,23 @@ public class HollowHistoryTypeKeyIndex {
private final HollowWriteStateEngine writeStateEngine;
private final HollowReadStateEngine readStateEngine;

public HollowHistoryTypeKeyIndex(String type, HollowWriteStateEngine writeEngine, HollowReadStateEngine readEngine, String... keyFields) {
this.type = type;
public HollowHistoryTypeKeyIndex(PrimaryKey primaryKey, HollowDataset dataModel, HollowWriteStateEngine writeEngine, HollowReadStateEngine readEngine) {
this.primaryKey = primaryKey;
this.writeStateEngine = writeEngine;
this.readStateEngine = readEngine;
this.keyFields = keyFields;
this.keyFieldParts = getKeyFieldParts();
this.keyFieldIsIndexed = new boolean[keyFields.length];
this.keyFieldParts = getKeyFieldParts(dataModel);
this.keyFieldIsIndexed = new boolean[primaryKey.numFields()];
}

public void addFieldIndex(String fieldName) {
for(int i=0;i<keyFields.length;i++) {
if(fieldName.equals(keyFields[i]))
public void addFieldIndex(String fieldName, HollowDataset dataModel) {
String fieldPathParts[] = PrimaryKey.getCompleteFieldPathParts(dataModel, primaryKey.getType(), fieldName);

for(int i=0;i<primaryKey.numFields();i++) {
String pkFieldPathParts[] = PrimaryKey.getCompleteFieldPathParts(dataModel, primaryKey.getType(), primaryKey.getFieldPath(i));
if(Arrays.equals(pkFieldPathParts, fieldPathParts)) {
keyFieldIsIndexed[i] = true;
break;
}
}
}

Expand All @@ -81,7 +86,7 @@ public void update(HollowObjectTypeReadState latestTypeState, boolean isDelta) {
}

public void hashRecordKeys() {
HollowObjectTypeReadState keyTypeState = (HollowObjectTypeReadState) readStateEngine.getTypeState(type);
HollowObjectTypeReadState keyTypeState = (HollowObjectTypeReadState) readStateEngine.getTypeState(primaryKey.getType());

int hashTableSize = HashCodes.hashTableSize(keyTypeState.maxOrdinal() + 1);

Expand All @@ -100,10 +105,10 @@ private void hashNewRecordKeys(HollowObjectTypeReadState keyTypeState) {

private void rehashAllRecordKeys(HollowObjectTypeReadState keyTypeState, int hashTableSize) {
int[] hashedRecordKeys = initializeHashedKeyArray(hashTableSize);
int[][] hashedFieldKeys = new int[keyFields.length][];
int[][] hashedFieldKeys = new int[primaryKey.numFields()][];
LongList hashedFieldKeyChains = new LongList();

for(int i=0;i<keyFields.length;i++)
for(int i=0;i<primaryKey.numFields();i++)
if(keyFieldIsIndexed[i])
hashedFieldKeys[i] = initializeHashedKeyArray(hashTableSize);

Expand All @@ -125,7 +130,7 @@ private void indexOrdinal(HollowObjectTypeReadState keyTypeState, int ordinal, i
hashedRecordKeys[bucket] = ordinal;

indexFields:
for(int j=0;j<keyFields.length;j++) {
for(int j=0;j<primaryKey.numFields();j++) {
if(keyFieldIsIndexed[j]) {
int fieldBucket = HashCodes.hashInt(HollowReadFieldUtils.fieldHashCode(keyTypeState, ordinal, j)) & bucketMask;
int chainStartIndex = hashedFieldKeys[j][fieldBucket];
Expand Down Expand Up @@ -154,15 +159,15 @@ private int[] initializeHashedKeyArray(int hashTableSize) {

private int hashKeyRecord(HollowObjectTypeReadState typeState, int ordinal) {
int hashCode = 0;
for(int i=0;i<keyFields.length;i++) {
for(int i=0;i<primaryKey.numFields();i++) {
int fieldHashCode = HollowReadFieldUtils.fieldHashCode(typeState, ordinal, i);
hashCode = (hashCode * 31) ^ fieldHashCode;
}
return HashCodes.hashInt(hashCode);
}

public int findKeyIndexOrdinal(HollowObjectTypeReadState typeState, int ordinal) {
HollowObjectTypeReadState keyTypeState = (HollowObjectTypeReadState) readStateEngine.getTypeState(type);
HollowObjectTypeReadState keyTypeState = (HollowObjectTypeReadState) readStateEngine.getTypeState(primaryKey.getType());

int bucketMask = hashedRecordKeys.length - 1;

Expand All @@ -189,10 +194,10 @@ private int findKeyHashCode(HollowObjectTypeReadState typeState, int ordinal) {
}

public IntList queryIndexedFields(final String query) {
final HollowObjectTypeReadState keyTypeState = (HollowObjectTypeReadState) readStateEngine.getTypeState(type);
final HollowObjectTypeReadState keyTypeState = (HollowObjectTypeReadState) readStateEngine.getTypeState(primaryKey.getType());
IntList matchingKeys = new IntList();

for(int i=0;i<keyFields.length;i++) {
for(int i=0;i<primaryKey.numFields();i++) {
final int fieldIndex = i;
try {
int hashCode;
Expand Down Expand Up @@ -307,7 +312,7 @@ private boolean recordFieldMatchesKey(HollowObjectTypeReadState typeState, int o
}

private void copyExistingKeys() {
HollowTypeWriteState typeState = writeStateEngine.getTypeState(type);
HollowTypeWriteState typeState = writeStateEngine.getTypeState(primaryKey.getType());
typeState.addAllObjectsFromPreviousCycle();
}

Expand Down Expand Up @@ -340,20 +345,20 @@ private void populateNewCurrentRecordKeys(HollowObjectTypeReadState typeState) {
}

public String[] getKeyFields() {
return keyFields;
return primaryKey.getFieldPaths();
}

public Object getKeyFieldValue(int keyFieldIdx, int keyOrdinal) {
return HollowReadFieldUtils.fieldValueObject((HollowObjectTypeReadState)readStateEngine.getTypeState(type), keyOrdinal, keyFieldIdx);
return HollowReadFieldUtils.fieldValueObject((HollowObjectTypeReadState)readStateEngine.getTypeState(primaryKey.getType()), keyOrdinal, keyFieldIdx);
}

public String getKeyDisplayString(int keyOrdinal) {
HollowObjectTypeReadState typeState = (HollowObjectTypeReadState) readStateEngine.getTypeState(type);
HollowObjectTypeReadState typeState = (HollowObjectTypeReadState) readStateEngine.getTypeState(primaryKey.getType());

StringBuilder builder = new StringBuilder();
for(int i=0;i<keyFields.length;i++) {
for(int i=0;i<primaryKey.numFields();i++) {
builder.append(HollowReadFieldUtils.displayString(typeState, keyOrdinal, i));
if(i < keyFields.length - 1)
if(i < primaryKey.numFields() - 1)
builder.append(':');
}
return builder.toString();
Expand All @@ -362,9 +367,9 @@ public String getKeyDisplayString(int keyOrdinal) {
private void writeKeyObject(HollowObjectTypeReadState typeState, int ordinal, HollowObjectWriteRecord rec) {
rec.reset();
for(int i=0;i<keyFieldParts.length;i++) {
writeKeyField(typeState, ordinal, rec, keyFields[i], keyFieldParts[i], 0);
writeKeyField(typeState, ordinal, rec, primaryKey.getFieldPath(i), keyFieldParts[i], 0);
}
writeStateEngine.add(type, rec);
writeStateEngine.add(primaryKey.getType(), rec);
}

private void writeKeyField(HollowObjectTypeReadState typeState, int ordinal, HollowObjectWriteRecord rec, String keyField, String keyFieldParts[], int keyFieldPartPosition) {
Expand Down Expand Up @@ -416,9 +421,9 @@ private void writeKeyField(HollowObjectTypeReadState typeState, int ordinal, Hol

private void initKeySchema(HollowObjectSchema entireObjectSchema) {
if(keySchema == null) {
keySchema = new HollowObjectSchema(type, keyFields.length);
keySchema = new HollowObjectSchema(primaryKey.getType(), primaryKey.numFields());
for(int i=0;i<keyFieldParts.length;i++)
addSchemaField(entireObjectSchema, keySchema, keyFields[i], keyFieldParts[i], 0);
addSchemaField(entireObjectSchema, keySchema, primaryKey.getFieldPath(i), keyFieldParts[i], 0);
}
}

Expand All @@ -437,10 +442,10 @@ private void initializeTypeWriteState() {
writeStateEngine.addTypeState(writeState);
}

private String[][] getKeyFieldParts() {
String keyFieldParts[][] = new String[keyFields.length][];
for(int i=0;i<keyFields.length;i++)
keyFieldParts[i] = keyFields[i].split("\\.");
private String[][] getKeyFieldParts(HollowDataset dataModel) {
String keyFieldParts[][] = new String[primaryKey.numFields()][];
for(int i=0;i<primaryKey.numFields();i++)
keyFieldParts[i] = PrimaryKey.getCompleteFieldPathParts(dataModel, primaryKey.getType(), primaryKey.getFieldPath(i));
return keyFieldParts;
}

Expand Down
2 changes: 1 addition & 1 deletion hollow/src/test/java/com/netflix/hollow/core/index/HollowPrimaryKeyIndexTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -304,7 +304,7 @@ private static void assertEquals(Object[] actual, Object... expected) {
}


@HollowPrimaryKey(fields = { "a1", "a2", "ab.b1.value" })
@HollowPrimaryKey(fields = { "a1", "a2", "ab.b1" })
private static class TypeA {
private final int a1;
private final double a2;
Expand Down
Loading

0 comments on commit 3aac2a4

Please sign in to comment.