-
Notifications
You must be signed in to change notification settings - Fork 217
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #75 from Netflix/cardinalityvalidator
Add record count variance validator and test
- Loading branch information
Showing
2 changed files
with
201 additions
and
0 deletions.
There are no files selected for viewing
70 changes: 70 additions & 0 deletions
70
...rc/main/java/com/netflix/hollow/api/producer/validation/RecordCountVarianceValidator.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,70 @@ | ||
| /* | ||
| * | ||
| * Copyright 2017 Netflix, Inc. | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| * | ||
| */ | ||
| package com.netflix.hollow.api.producer.validation; | ||
|
|
||
| import com.netflix.hollow.api.producer.HollowProducer.ReadState; | ||
| import com.netflix.hollow.api.producer.HollowProducer.Validator; | ||
| import com.netflix.hollow.core.read.engine.HollowTypeReadState; | ||
|
|
||
| /** | ||
| * @author lkanchanapalli | ||
| * | ||
| */ | ||
| public class RecordCountVarianceValidator implements Validator { | ||
| private final String typeName; | ||
| private final float allowableVariancePercent; | ||
|
|
||
| /** | ||
| * | ||
| * @param typeName | ||
| * @param allowableVariancePercent: Used to validate if the cardinality change in current cycle is with in the allowed percent. | ||
| * Ex: 0% allowableVariancePercent ensures type cardinality does not vary at all for cycle to cycle. Ex: Number of state in United States. | ||
| * 10% allowableVariancePercent: from previous cycle any addition or removal within 10% cardinality is valid. Anything more results in failure of validation. | ||
| */ | ||
| public RecordCountVarianceValidator(String typeName, float allowableVariancePercent) { | ||
| this.typeName = typeName; | ||
| if(allowableVariancePercent < 0) | ||
| throw new IllegalArgumentException("RecordCountVarianceValidator for type "+typeName+": cannot have allowableVariancePercent less than 0. Value provided: "+allowableVariancePercent); | ||
| this.allowableVariancePercent = allowableVariancePercent; | ||
| } | ||
|
|
||
| /* (non-Javadoc) | ||
| * @see com.netflix.hollow.api.producer.HollowProducer.Validator#validate(com.netflix.hollow.api.producer.HollowProducer.ReadState) | ||
| */ | ||
| @Override | ||
| public void validate(ReadState readState) { | ||
| HollowTypeReadState typeState = readState.getStateEngine().getTypeState(typeName); | ||
| int latestCardinality = typeState.getPopulatedOrdinals().cardinality(); | ||
| int previousCardinality = typeState.getPreviousOrdinals().cardinality(); | ||
|
|
||
| // TODO: log message indicating previous state is 0. Can happen for new name space. And also can happen | ||
| // when a type gets to zero count then the validation will be skipped. | ||
| if(previousCardinality == 0) | ||
| return; | ||
|
|
||
| float actualChangePercent = (float)(100*Math.abs(latestCardinality - previousCardinality))/previousCardinality; | ||
| if (Float.compare(actualChangePercent, allowableVariancePercent) > 0) { | ||
| throw new ValidationException("RecordCountVarianceValidator for type " + typeName | ||
| + " failed. Actual variance: " + actualChangePercent + "%; Allowed variance: " | ||
| + allowableVariancePercent + "%; previous cycle record count: " + previousCardinality | ||
| + "; current cycle record count: " + latestCardinality); | ||
|
|
||
| } | ||
| } | ||
| } | ||
|
|
131 changes: 131 additions & 0 deletions
131
...st/java/com/netflix/hollow/api/producer/validation/RecordCountVarianceValidatorTests.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,131 @@ | ||
| /* | ||
| * | ||
| * Copyright 2017 Netflix, Inc. | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| * | ||
| */ | ||
| package com.netflix.hollow.api.producer.validation; | ||
|
|
||
| import org.junit.Assert; | ||
| import org.junit.Before; | ||
| import org.junit.Test; | ||
|
|
||
| import com.netflix.hollow.api.consumer.HollowConsumer; | ||
| import com.netflix.hollow.api.consumer.InMemoryBlobStore; | ||
| import com.netflix.hollow.api.producer.HollowProducer; | ||
| import com.netflix.hollow.api.producer.HollowProducer.Populator; | ||
| import com.netflix.hollow.api.producer.HollowProducer.Validator.ValidationException; | ||
| import com.netflix.hollow.api.producer.HollowProducer.WriteState; | ||
| import com.netflix.hollow.api.producer.fs.HollowInMemoryBlobStager; | ||
| import com.netflix.hollow.api.producer.validation.ProducerValidationTests.TypeWithPrimaryKey; | ||
|
|
||
| public class RecordCountVarianceValidatorTests { | ||
| private InMemoryBlobStore blobStore; | ||
|
|
||
| @Before | ||
| public void setUp() { | ||
| blobStore = new InMemoryBlobStore(); | ||
| } | ||
|
|
||
| @Test | ||
| public void failTestTooManyAdded() { | ||
| try { | ||
| HollowProducer producer = HollowProducer.withPublisher(blobStore) | ||
| .withBlobStager(new HollowInMemoryBlobStager()) | ||
| .withValidator(new RecordCountVarianceValidator("TypeWithPrimaryKey", 1f)).build(); | ||
|
|
||
| producer.runCycle(new Populator() { | ||
| public void populate(WriteState newState) throws Exception { | ||
| newState.add(new TypeWithPrimaryKey(1, "Brad Pitt", "klsdjfla;sdjkf")); | ||
| newState.add(new TypeWithPrimaryKey(1, "Angelina Jolie", "as;dlkfjasd;l")); | ||
| } | ||
| }); | ||
|
|
||
| producer.runCycle(new Populator() { | ||
| public void populate(WriteState newState) throws Exception { | ||
| newState.add(new TypeWithPrimaryKey(1, "Brad Pitt", "klsdjfla;sdjkf")); | ||
| newState.add(new TypeWithPrimaryKey(2, "Angelina Jolie", "as;dlkfjasd;l")); | ||
| newState.add(new TypeWithPrimaryKey(3, "Angelina Jolie1", "as;dlkfjasd;l")); | ||
| newState.add(new TypeWithPrimaryKey(4, "Angelina Jolie2", "as;dlkfjasd;l")); | ||
| newState.add(new TypeWithPrimaryKey(5, "Angelina Jolie3", "as;dlkfjasd;l")); | ||
| newState.add(new TypeWithPrimaryKey(6, "Angelina Jolie4", "as;dlkfjasd;l")); | ||
| newState.add(new TypeWithPrimaryKey(7, "Angelina Jolie5", "as;dlkfjasd;l")); | ||
| } | ||
| }); | ||
| Assert.fail(); | ||
| } catch (ValidationException expected) { | ||
| Assert.assertEquals(1, expected.getIndividualFailures().size()); | ||
| //System.out.println("Message:"+expected.getIndividualFailures().get(0).getMessage()); | ||
| Assert.assertTrue(expected.getIndividualFailures().get(0).getMessage() | ||
| .startsWith("RecordCountVarianceValidator for type TypeWithPrimaryKey failed.")); | ||
| } | ||
| } | ||
|
|
||
| @Test | ||
| public void failTestTooManyRemoved() { | ||
| try { | ||
| HollowProducer producer = HollowProducer.withPublisher(blobStore) | ||
| .withBlobStager(new HollowInMemoryBlobStager()) | ||
| .withValidator(new RecordCountVarianceValidator("TypeWithPrimaryKey", 1f)).build(); | ||
|
|
||
| producer.runCycle(new Populator() { | ||
| public void populate(WriteState newState) throws Exception { | ||
| newState.add(new TypeWithPrimaryKey(1, "Brad Pitt", "klsdjfla;sdjkf")); | ||
| newState.add(new TypeWithPrimaryKey(1, "Angelina Jolie", "as;dlkfjasd;l")); | ||
| } | ||
| }); | ||
|
|
||
| producer.runCycle(new Populator() { | ||
| public void populate(WriteState newState) throws Exception { | ||
| newState.add(new TypeWithPrimaryKey(1, "Brad Pitt", "klsdjfla;sdjkf")); | ||
| newState.add(new TypeWithPrimaryKey(1, "Angelina Jolie", "as;dlkfjasd;l")); | ||
| newState.add(new TypeWithPrimaryKey(1, "Bruce Willis", "as;dlkfjasd;l")); | ||
| } | ||
| }); | ||
| Assert.fail(); | ||
| } catch (ValidationException expected) { | ||
| //System.out.println("Message:"+expected.getIndividualFailures().get(0).getMessage()); | ||
| Assert.assertEquals(1, expected.getIndividualFailures().size()); | ||
| Assert.assertTrue(expected.getIndividualFailures().get(0).getMessage() | ||
| .startsWith("RecordCountVarianceValidator for type TypeWithPrimaryKey failed.")); | ||
| } | ||
| } | ||
|
|
||
| @Test | ||
| public void passTestNoMoreChangeThanExpected() { | ||
| HollowProducer producer = HollowProducer.withPublisher(blobStore).withBlobStager(new HollowInMemoryBlobStager()) | ||
| .withValidator(new RecordCountVarianceValidator("TypeWithPrimaryKey", 50f)).build(); | ||
|
|
||
| // runCycle(producer, 1); | ||
| producer.runCycle(new Populator() { | ||
|
|
||
| public void populate(WriteState newState) throws Exception { | ||
| newState.add(new TypeWithPrimaryKey(1, "Brad Pitt", "klsdjfla;sdjkf")); | ||
| newState.add(new TypeWithPrimaryKey(1, "Angelina Jolie", "as;dlkfjasd;l")); | ||
| } | ||
| }); | ||
|
|
||
| producer.runCycle(new Populator() { | ||
|
|
||
| public void populate(WriteState newState) throws Exception { | ||
| newState.add(new TypeWithPrimaryKey(1, "Brad Pitt", "klsdjfla;sdjkf")); | ||
| newState.add(new TypeWithPrimaryKey(2, "Angelina Jolie", "as;dlkfjasd;l")); | ||
| newState.add(new TypeWithPrimaryKey(7, "Bruce Willis", "as;dlkfjasd;l")); | ||
| } | ||
| }); | ||
| HollowConsumer consumer = HollowConsumer.withBlobRetriever(blobStore).build(); | ||
| consumer.triggerRefresh(); | ||
| Assert.assertEquals(3,consumer.getStateEngine().getTypeState("TypeWithPrimaryKey").getPopulatedOrdinals().cardinality()); | ||
| } | ||
| } |