From 11757f037a04211570c99091f9e4955fbbea7fdc Mon Sep 17 00:00:00 2001 From: Tim Taylor Date: Tue, 7 Feb 2017 14:07:22 -0800 Subject: [PATCH 01/30] producer api: new WriteState class --- .../hollow/api/producer/WriteState.java | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 hollow/src/main/java/com/netflix/hollow/api/producer/WriteState.java diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/WriteState.java b/hollow/src/main/java/com/netflix/hollow/api/producer/WriteState.java new file mode 100644 index 0000000000..fd389614dd --- /dev/null +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/WriteState.java @@ -0,0 +1,30 @@ +package com.netflix.hollow.api.producer; + +import com.netflix.hollow.core.write.HollowWriteStateEngine; +import com.netflix.hollow.core.write.objectmapper.HollowObjectMapper; + +public class WriteState { + private final long version; + private final HollowObjectMapper objectMapper; + + WriteState(HollowWriteStateEngine writeEngine, long version) { + this.objectMapper = new HollowObjectMapper(writeEngine); + this.version = version; + } + + public long getVersion() { + return version; + } + + public int add(Object o) { + return objectMapper.add(o); + } + + public HollowObjectMapper getObjectMapper() { + return objectMapper; + } + + public HollowWriteStateEngine getStateEngine() { + return objectMapper.getStateEngine(); + } +} From 688caee03bc4c27e1e0ac02fa4a27674d1b372b1 Mon Sep 17 00:00:00 2001 From: Tim Taylor Date: Tue, 7 Feb 2017 14:10:23 -0800 Subject: [PATCH 02/30] producer api: model from/to version as a StateTransition --- .../netflix/hollow/api/StateTransition.java | 133 ++++++++++++++++++ .../hollow/api/producer/WriteState.java | 21 ++- .../hollow/api/StateTransitionTest.java | 66 +++++++++ 3 files changed, 213 insertions(+), 7 deletions(-) create mode 100644 hollow/src/main/java/com/netflix/hollow/api/StateTransition.java create mode 100644 hollow/src/test/java/com/netflix/hollow/api/StateTransitionTest.java diff --git a/hollow/src/main/java/com/netflix/hollow/api/StateTransition.java b/hollow/src/main/java/com/netflix/hollow/api/StateTransition.java new file mode 100644 index 0000000000..d2d2bf48aa --- /dev/null +++ b/hollow/src/main/java/com/netflix/hollow/api/StateTransition.java @@ -0,0 +1,133 @@ +package com.netflix.hollow.api; + +/** + * Immutable class representing a single point along the delta chain. + * + * @author Tim Taylor {@literal} + */ +public final class StateTransition { + + private final long fromVersion; + private final long toVersion; + + /** + * Creates a transition representing a brand new delta chain or a break + * in an existing chain. This transition doesn't have a from nor a to version; + * calling {@link #advance(long)} on this transition will return new a transition representing + * the first state produced on this chain, i.e. the first snapshot. + * + * @return a state transition with neither a {@code fromVersion} nor a {@code toVersion} + */ + public StateTransition() { + this(Long.MIN_VALUE, Long.MIN_VALUE); + } + + /** + * Creates a transition capable of being used to restore from a delta chain at + * the specified version, a.k.a. a snapshot.

+ * + * Consumers can initialize their read state from a snapshot corresponding to + * this transition; an already initialized consumer can only utilize + * this by performing a double snapshot.

+ * + * A producer would use this transition to restore from a previous announced state in order + * to resume producing on that delta chain by calling {@link #advance(long)} when ready to + * produce the next state. + * + * @return a state transition with no {@code fromVersion} and the specified version as the {@code toVersion} + * + * @see Double Snapshot + + */ + public StateTransition(long toVersion) { + this(Long.MIN_VALUE, toVersion); + } + + /** + * Creates a transition fully representing a transition within the delta chain, a.k.a. a delta. + * + * @return a state transition with the specified fromVersion and toVersion + */ + public StateTransition(long fromVersion, long toVersion) { + this.fromVersion = fromVersion; + this.toVersion = toVersion; + } + + /** + * Returns a new transition representing the transition from this state's {@code toVersion} to the specified version; + * equivalent to calling {@code new StateTransition(this.toVersion, nextVersion)}. + * + * 

+     * 
+     * [13,45].advance(72) == [45,72]
+     * 
+     *
+ * + * @param nextVersion the next version to transition to + * + * @return a new state transition with its {@fromVersion} and {@toVersion} assigned our {@toVersion} and + * the specified {@code nextVersion} respectively + */ + public StateTransition advance(long nextVersion) { + return new StateTransition(toVersion, nextVersion); + } + + public long getFromVersion() { + return fromVersion; + } + + public long getToVersion() { + return toVersion; + } + + /** + * Determines whether this transition represents a new or broken delta chain. + * + * @return true if this has neither a {@code fromVersion} nor a {@code toVersion}; false otherwise. + */ + public boolean isDiscontinous() { + return fromVersion == Long.MIN_VALUE && toVersion == Long.MIN_VALUE; + } + + /** + * Determines whether this state represents a delta, e.g. a transition between two state versions. + * + * @return true if this has a {@code fromVersion} and {@code toVersion}; + */ + public boolean isDelta() { + return fromVersion != Long.MIN_VALUE && toVersion != Long.MIN_VALUE; + } + + public boolean isForwardDelta() { + return isDelta() && fromVersion < toVersion; + } + + public boolean isReverseDelta() { + return isDelta() && fromVersion > toVersion; + } + + public boolean isSnapshot() { + return !isDiscontinous() && !isDelta(); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + if(isDiscontinous()) { + sb.append("new/broken delta chain"); + } else if(isDelta()) { + if(isReverseDelta()) sb.append("reverse"); + sb.append("delta ["); + sb.append(fromVersion); + sb.append(" <-> "); + sb.append(toVersion); + sb.append("]"); + } else { + sb.append("snapshot ["); + sb.append(toVersion); + sb.append("]"); + } + return sb.toString(); + } + +} diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/WriteState.java b/hollow/src/main/java/com/netflix/hollow/api/producer/WriteState.java index fd389614dd..6ef3179291 100644 --- a/hollow/src/main/java/com/netflix/hollow/api/producer/WriteState.java +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/WriteState.java @@ -1,19 +1,17 @@ package com.netflix.hollow.api.producer; +import com.netflix.hollow.api.StateTransition; import com.netflix.hollow.core.write.HollowWriteStateEngine; import com.netflix.hollow.core.write.objectmapper.HollowObjectMapper; public class WriteState { - private final long version; + + private final StateTransition transition; private final HollowObjectMapper objectMapper; - WriteState(HollowWriteStateEngine writeEngine, long version) { + WriteState(HollowWriteStateEngine writeEngine, StateTransition transition) { + this.transition = transition; this.objectMapper = new HollowObjectMapper(writeEngine); - this.version = version; - } - - public long getVersion() { - return version; } public int add(Object o) { @@ -27,4 +25,13 @@ public HollowObjectMapper getObjectMapper() { public HollowWriteStateEngine getStateEngine() { return objectMapper.getStateEngine(); } + + long getVersion() { + return transition.getToVersion(); + } + + StateTransition getTransition() { + return transition; + } + } diff --git a/hollow/src/test/java/com/netflix/hollow/api/StateTransitionTest.java b/hollow/src/test/java/com/netflix/hollow/api/StateTransitionTest.java new file mode 100644 index 0000000000..3192291879 --- /dev/null +++ b/hollow/src/test/java/com/netflix/hollow/api/StateTransitionTest.java @@ -0,0 +1,66 @@ +package com.netflix.hollow.api; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; + +public final class StateTransitionTest { + + StateTransition subject; + + @Test + public void discontinuous(){ + subject = new StateTransition(); + + assertThat(subject.getFromVersion(), equalTo(Long.MIN_VALUE)); + assertThat(subject.getToVersion(), equalTo(Long.MIN_VALUE)); + + assertTrue(subject.isDiscontinous()); + assertFalse(subject.isDelta()); + } + + @Test + public void resumed() { + subject = new StateTransition(13L); + + assertThat(subject.getFromVersion(), equalTo(Long.MIN_VALUE)); + assertThat(subject.getToVersion(), equalTo(13L)); + + assertFalse(subject.isDiscontinous()); + assertFalse(subject.isDelta()); + } + + @Test + public void bidirectional() { + subject = new StateTransition(2L, 3L); + + assertThat(subject.getFromVersion(), equalTo(2L)); + assertThat(subject.getToVersion(), equalTo(3L)); + + assertFalse(subject.isDiscontinous()); + assertTrue(subject.isDelta()); + } + + @Test + public void advancing() { + subject = new StateTransition(); + + subject = subject.advance(6L); + + assertThat(subject.getFromVersion(), equalTo(Long.MIN_VALUE)); + assertThat(subject.getToVersion(), equalTo(6L)); + assertFalse(subject.isDiscontinous()); + assertFalse(subject.isDelta()); + + subject = subject.advance(7L); + + assertThat(subject.getFromVersion(), equalTo(6L)); + assertThat(subject.getToVersion(), equalTo(7L)); + assertFalse(subject.isDiscontinous()); + assertTrue(subject.isDelta()); + } + +} From 6b94478052a4f4910cfe38fdd3e162ceedab7ff4 Mon Sep 17 00:00:00 2001 From: Tim Taylor Date: Tue, 7 Feb 2017 14:25:23 -0800 Subject: [PATCH 03/30] producer api: added reverse() to StateTransition --- .../netflix/hollow/api/StateTransition.java | 41 +++++++++++++------ .../hollow/api/StateTransitionTest.java | 32 ++++++++++++++- 2 files changed, 58 insertions(+), 15 deletions(-) diff --git a/hollow/src/main/java/com/netflix/hollow/api/StateTransition.java b/hollow/src/main/java/com/netflix/hollow/api/StateTransition.java index d2d2bf48aa..fed095c763 100644 --- a/hollow/src/main/java/com/netflix/hollow/api/StateTransition.java +++ b/hollow/src/main/java/com/netflix/hollow/api/StateTransition.java @@ -12,11 +12,11 @@ public final class StateTransition { /** * Creates a transition representing a brand new delta chain or a break - * in an existing chain. This transition doesn't have a from nor a to version; - * calling {@link #advance(long)} on this transition will return new a transition representing - * the first state produced on this chain, i.e. the first snapshot. + * in an existing chain, e.g. a state transition with neither a {@code fromVersion} nor a + * {@code toVersion}.

* - * @return a state transition with neither a {@code fromVersion} nor a {@code toVersion} + * Calling {@link #advance(long)} on this transition will return new a transition representing + * the first state produced on this chain, i.e. the first snapshot. */ public StateTransition() { this(Long.MIN_VALUE, Long.MIN_VALUE); @@ -34,19 +34,15 @@ public StateTransition() { * to resume producing on that delta chain by calling {@link #advance(long)} when ready to * produce the next state. * - * @return a state transition with no {@code fromVersion} and the specified version as the {@code toVersion} - * * @see Double Snapshot - */ public StateTransition(long toVersion) { this(Long.MIN_VALUE, toVersion); } /** - * Creates a transition fully representing a transition within the delta chain, a.k.a. a delta. - * - * @return a state transition with the specified fromVersion and toVersion + * Creates a transition fully representing a transition within the delta chain, a.k.a. a delta, between + * {@code fromVersion} and {@code toVersion}. */ public StateTransition(long fromVersion, long toVersion) { this.fromVersion = fromVersion; @@ -65,13 +61,31 @@ public StateTransition(long fromVersion, long toVersion) { * * @param nextVersion the next version to transition to * - * @return a new state transition with its {@fromVersion} and {@toVersion} assigned our {@toVersion} and + * @return a new state transition with its {@code fromVersion} and {@code toVersion} assigned our {@code toVersion} and * the specified {@code nextVersion} respectively */ public StateTransition advance(long nextVersion) { return new StateTransition(toVersion, nextVersion); } + /** + * Returns a new transition with versions swapped. Only valid on deltas. + + * 

+     * 
+     * [13,45].reverse() == [45,13]
+     * 
+     *
+ + * @return + * + * @throws IllegalStateException if this transition isn't a delta + */ + public StateTransition reverse() { + if(isDiscontinous() || isSnapshot()) throw new IllegalStateException("must be a delta"); + return new StateTransition(this.toVersion, this.fromVersion); + } + public long getFromVersion() { return fromVersion; } @@ -92,7 +106,7 @@ public boolean isDiscontinous() { /** * Determines whether this state represents a delta, e.g. a transition between two state versions. * - * @return true if this has a {@code fromVersion} and {@code toVersion}; + * @return true if this has a {@code fromVersion} and {@code toVersion}; false otherwise */ public boolean isDelta() { return fromVersion != Long.MIN_VALUE && toVersion != Long.MIN_VALUE; @@ -119,7 +133,8 @@ public String toString() { if(isReverseDelta()) sb.append("reverse"); sb.append("delta ["); sb.append(fromVersion); - sb.append(" <-> "); + if(isForwardDelta()) sb.append(" -> "); + else sb.append(" <- "); sb.append(toVersion); sb.append("]"); } else { diff --git a/hollow/src/test/java/com/netflix/hollow/api/StateTransitionTest.java b/hollow/src/test/java/com/netflix/hollow/api/StateTransitionTest.java index 3192291879..792bee6c5b 100644 --- a/hollow/src/test/java/com/netflix/hollow/api/StateTransitionTest.java +++ b/hollow/src/test/java/com/netflix/hollow/api/StateTransitionTest.java @@ -4,6 +4,7 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import org.junit.Test; @@ -19,28 +20,31 @@ public void discontinuous(){ assertThat(subject.getToVersion(), equalTo(Long.MIN_VALUE)); assertTrue(subject.isDiscontinous()); + assertFalse(subject.isSnapshot()); assertFalse(subject.isDelta()); } @Test - public void resumed() { + public void snapshot() { subject = new StateTransition(13L); assertThat(subject.getFromVersion(), equalTo(Long.MIN_VALUE)); assertThat(subject.getToVersion(), equalTo(13L)); assertFalse(subject.isDiscontinous()); + assertTrue(subject.isSnapshot()); assertFalse(subject.isDelta()); } @Test - public void bidirectional() { + public void delta() { subject = new StateTransition(2L, 3L); assertThat(subject.getFromVersion(), equalTo(2L)); assertThat(subject.getToVersion(), equalTo(3L)); assertFalse(subject.isDiscontinous()); + assertFalse(subject.isSnapshot()); assertTrue(subject.isDelta()); } @@ -63,4 +67,28 @@ public void advancing() { assertTrue(subject.isDelta()); } + @Test + public void reversing() { + subject = new StateTransition(21L, 22L); + + assertTrue(subject.isForwardDelta()); + assertFalse(subject.isReverseDelta()); + + subject = subject.reverse(); + + assertThat(subject.getFromVersion(), equalTo(22L)); + assertThat(subject.getToVersion(), equalTo(21L)); + assertFalse(subject.isForwardDelta()); + assertTrue(subject.isReverseDelta()); + + try { + new StateTransition().reverse(); + fail("expected exception"); + } catch(IllegalStateException expected){} + + try { + new StateTransition(1L).reverse(); + fail("expected exception"); + } catch(IllegalStateException expected){} + } } From 488a8f9b1ac2803cf8694de3a0de4ef8408efc86 Mon Sep 17 00:00:00 2001 From: Tim Taylor Date: Tue, 7 Feb 2017 14:12:42 -0800 Subject: [PATCH 04/30] producer api: lift high level API from reference impl --- .../hollow/api/producer/HollowAnnouncer.java | 24 ++++ .../hollow/api/producer/HollowProducer.java | 128 ++++++++++++++++++ .../hollow/api/producer/HollowPublisher.java | 32 +++++ .../hollow/api/producer/VersionMinter.java | 31 +++++ 4 files changed, 215 insertions(+) create mode 100644 hollow/src/main/java/com/netflix/hollow/api/producer/HollowAnnouncer.java create mode 100644 hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java create mode 100644 hollow/src/main/java/com/netflix/hollow/api/producer/HollowPublisher.java create mode 100644 hollow/src/main/java/com/netflix/hollow/api/producer/VersionMinter.java diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowAnnouncer.java b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowAnnouncer.java new file mode 100644 index 0000000000..92c975d10e --- /dev/null +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowAnnouncer.java @@ -0,0 +1,24 @@ +/* + * + * Copyright 2016 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.hollow.api.producer; + +public interface HollowAnnouncer { + + public void announce(long stateVersion); + +} diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java new file mode 100644 index 0000000000..1fa794c09f --- /dev/null +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java @@ -0,0 +1,128 @@ +/* + * + * Copyright 2017 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.hollow.api.producer; + + +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; + +import com.netflix.hollow.api.StateTransition; +import com.netflix.hollow.core.read.engine.HollowReadStateEngine; +import com.netflix.hollow.core.write.HollowBlobWriter; +import com.netflix.hollow.core.write.HollowWriteStateEngine; + +public class HollowProducer { + + private final VersionMinter versionMinter; + private final HollowPublisher publisher; + private final HollowAnnouncer announcer; + private final HollowWriteStateEngine writeEngine; + + private StateTransition announced; + + public HollowProducer(VersionMinter versionMinter, HollowPublisher publisher, HollowAnnouncer announcer) { + this.versionMinter = versionMinter; + this.publisher = publisher; + this.announcer = announcer; + + writeEngine = new HollowWriteStateEngine(); + announced = new StateTransition(); + } + + public void restoreFrom(HollowReadStateEngine priorAnnouncedState, long priorAnnouncedVersion) { + writeEngine.restoreFrom(priorAnnouncedState); + announced = new StateTransition(priorAnnouncedVersion); + } + + /** + * Each cycle produces a single state. + */ + public void produce(Task task) { + try { + WriteState writeState = beginCycle(announced.advance(versionMinter.mint())); + task.populate(writeState); + publish(writeState); + announced = announce(writeState); + } catch(Throwable th) { + th.printStackTrace(); + rollback(); + } + } + + private WriteState beginCycle(StateTransition transition) { + writeEngine.prepareForNextCycle(); + WriteState writeState = new WriteState(writeEngine, transition); + System.out.println("Beginning cycle " + transition); + return writeState; + } + + private void publish(WriteState writeState) throws IOException { + HollowBlobWriter writer = new HollowBlobWriter(writeEngine); + + + File snapshotFile = publisher.openSnapshot(announced.getToVersion()); + File deltaFile = publisher.openDelta(announced.getFromVersion(), announced.getToVersion()); + File reverseDeltaFile = publisher.openReverseDelta(announced.getFromVersion(), announced.getToVersion()); + + try(OutputStream os = new BufferedOutputStream(new FileOutputStream(snapshotFile))) { + writer.writeSnapshot(os); + } + + if(announced.isDelta()) { + + try(OutputStream os = new BufferedOutputStream(new FileOutputStream(deltaFile))) { + writer.writeDelta(os); + } + + try(OutputStream os = new BufferedOutputStream(new FileOutputStream(reverseDeltaFile))) { + writer.writeReverseDelta(os); + } + + publisher.publishDelta(deltaFile, announced.getFromVersion(), announced.getToVersion()); + publisher.publishReverseDelta(reverseDeltaFile, announced.getFromVersion(), announced.getToVersion()); + + deltaFile.delete(); + reverseDeltaFile.delete(); + } + + try { + /// it's ok to fail to publish a snapshot, as long as you don't miss too many in a row. + /// you can add a timeout or even do this in a separate thread. + publisher.publishSnapshot(snapshotFile, announced.getToVersion()); + snapshotFile.delete(); + } catch(Throwable ignore) { } + } + + private void rollback() { + writeEngine.resetToLastPrepareForNextCycle(); + } + + private StateTransition announce(WriteState writeState) { + announcer.announce(writeState.getVersion()); + StateTransition transition = writeState.getTransition(); + return transition; + } + + public static interface Task { + void populate(WriteState newState); + } + +} diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowPublisher.java b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowPublisher.java new file mode 100644 index 0000000000..86874e6ca6 --- /dev/null +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowPublisher.java @@ -0,0 +1,32 @@ +/* + * + * Copyright 2016 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.hollow.api.producer; + +import java.io.File; + +public interface HollowPublisher { + + public File openSnapshot(long version); + public File openDelta(long previousVersion, long currentVersion); + public File openReverseDelta(long previousVersion, long currentVersion); + + public void publishSnapshot(File blob, long version); + public void publishDelta(File blob, long previousVersion, long currentVersion); + public void publishReverseDelta(File blob, long previousVersion, long currentVersion); + +} diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/VersionMinter.java b/hollow/src/main/java/com/netflix/hollow/api/producer/VersionMinter.java new file mode 100644 index 0000000000..1d8749eb9e --- /dev/null +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/VersionMinter.java @@ -0,0 +1,31 @@ +/* + * + * Copyright 2016 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.hollow.api.producer; + +public interface VersionMinter { + + /** + * Create a new state version.

+ * + * State versions should be ascending -- later states have greater versions.

+ * + * @return a new state version + */ + long mint(); + +} From 6b14251cf2b583b933b6e280f356715e1b166599 Mon Sep 17 00:00:00 2001 From: Tim Taylor Date: Tue, 7 Feb 2017 14:14:09 -0800 Subject: [PATCH 05/30] producer api: new HollowBlob type --- .../hollow/api/producer/HollowBlob.java | 15 +++++ .../hollow/api/producer/HollowProducer.java | 58 ++++++++++--------- .../hollow/api/producer/HollowPublisher.java | 12 ++-- 3 files changed, 50 insertions(+), 35 deletions(-) create mode 100644 hollow/src/main/java/com/netflix/hollow/api/producer/HollowBlob.java diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowBlob.java b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowBlob.java new file mode 100644 index 0000000000..217bc13db9 --- /dev/null +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowBlob.java @@ -0,0 +1,15 @@ +package com.netflix.hollow.api.producer; + +import java.io.Closeable; +import java.io.OutputStream; + +public interface HollowBlob extends Closeable { + + OutputStream getOutputStream(); + + void finish(); + + @Override + void close(); + +} diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java index 1fa794c09f..4202ee1639 100644 --- a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java @@ -76,39 +76,41 @@ private WriteState beginCycle(StateTransition transition) { private void publish(WriteState writeState) throws IOException { HollowBlobWriter writer = new HollowBlobWriter(writeEngine); + StateTransition transition = writeState.getTransition(); - - File snapshotFile = publisher.openSnapshot(announced.getToVersion()); - File deltaFile = publisher.openDelta(announced.getFromVersion(), announced.getToVersion()); - File reverseDeltaFile = publisher.openReverseDelta(announced.getFromVersion(), announced.getToVersion()); - - try(OutputStream os = new BufferedOutputStream(new FileOutputStream(snapshotFile))) { - writer.writeSnapshot(os); - } - - if(announced.isDelta()) { - - try(OutputStream os = new BufferedOutputStream(new FileOutputStream(deltaFile))) { - writer.writeDelta(os); - } - - try(OutputStream os = new BufferedOutputStream(new FileOutputStream(reverseDeltaFile))) { - writer.writeReverseDelta(os); + HollowBlob snapshot = publisher.openSnapshot(transition); + try { + writer.writeSnapshot(snapshot.getOutputStream()); + snapshot.finish(); + + if(transition.isDelta()) { + HollowBlob delta = publisher.openDelta(transition); + HollowBlob reverseDelta = publisher.openReverseDelta(transition); + try { + writer.writeDelta(delta.getOutputStream()); + delta.finish(); + + writer.writeReverseDelta(reverseDelta.getOutputStream()); + reverseDelta.finish(); + + publisher.publish(delta); + publisher.publish(reverseDelta); + } finally { + delta.close(); + reverseDelta.close(); + } } - publisher.publishDelta(deltaFile, announced.getFromVersion(), announced.getToVersion()); - publisher.publishReverseDelta(reverseDeltaFile, announced.getFromVersion(), announced.getToVersion()); - - deltaFile.delete(); - reverseDeltaFile.delete(); - } - - try { /// it's ok to fail to publish a snapshot, as long as you don't miss too many in a row. /// you can add a timeout or even do this in a separate thread. - publisher.publishSnapshot(snapshotFile, announced.getToVersion()); - snapshotFile.delete(); - } catch(Throwable ignore) { } + try { + publisher.publish(snapshot); + } catch(Throwable ignored) { + ignored.printStackTrace(); // TODO: timt: log and notify listerners + } + } finally { + snapshot.close(); + } } private void rollback() { diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowPublisher.java b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowPublisher.java index 86874e6ca6..eeb74684d5 100644 --- a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowPublisher.java +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowPublisher.java @@ -17,16 +17,14 @@ */ package com.netflix.hollow.api.producer; -import java.io.File; +import com.netflix.hollow.api.StateTransition; public interface HollowPublisher { - public File openSnapshot(long version); - public File openDelta(long previousVersion, long currentVersion); - public File openReverseDelta(long previousVersion, long currentVersion); + public HollowBlob openSnapshot(StateTransition transition); + public HollowBlob openDelta(StateTransition transition); + public HollowBlob openReverseDelta(StateTransition transition); - public void publishSnapshot(File blob, long version); - public void publishDelta(File blob, long previousVersion, long currentVersion); - public void publishReverseDelta(File blob, long previousVersion, long currentVersion); + public void publish(HollowBlob blob); } From bd30dcb738889a6e90d9f89fde2b2fe985d0eca4 Mon Sep 17 00:00:00 2001 From: Kinesh Satiya Date: Wed, 8 Feb 2017 17:21:04 -0800 Subject: [PATCH 06/30] Adding HollowProducerListener interface to perform actions on different stages in a cycle of a HollowProducer --- .../hollow/api/producer/CycleStatus.java | 77 +++++++++++++++++++ .../api/producer/HollowProducerListener.java | 74 ++++++++++++++++++ 2 files changed, 151 insertions(+) create mode 100644 hollow/src/main/java/com/netflix/hollow/api/producer/CycleStatus.java create mode 100644 hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducerListener.java diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/CycleStatus.java b/hollow/src/main/java/com/netflix/hollow/api/producer/CycleStatus.java new file mode 100644 index 0000000000..20f885c928 --- /dev/null +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/CycleStatus.java @@ -0,0 +1,77 @@ +package com.netflix.hollow.api.producer; + +import com.netflix.hollow.core.read.engine.HollowReadStateEngine; + +/** + * This class represents information on details when {@link HollowProducer} has finished executing a particular stage. + * An instance of this class is provided on different events of {@link HollowProducerListener}. + * + * @author Kinesh Satiya {@literal kineshsatiya@gmail.com} + */ +public class CycleStatus { + + public enum Status { + SUCCESS, FAIL + } + + private long version; + private Status status; + private Throwable throwable; + private HollowReadStateEngine readStateEngine; + + public static CycleStatus getSuccessInstance(long version) { + return new CycleStatus(version, Status.SUCCESS, null, null); + } + + public static CycleStatus getSuccessInstance(long version, HollowReadStateEngine readStateEngine) { + return new CycleStatus(version, Status.SUCCESS, null, readStateEngine); + } + + public static CycleStatus getFailInstance(long version, Throwable th) { + return new CycleStatus(version, Status.FAIL, th, null); + } + + CycleStatus(long version, Status status, Throwable throwable, HollowReadStateEngine readStateEngine) { + this.version = version; + this.status = status; + this.throwable = throwable; + this.readStateEngine = readStateEngine; + } + + /** + * This version is currently under process by {@code HollowProducer}. + * + * @return Current version of the {@code HollowProducer}. + */ + public long getVersion() { + return version; + } + + /** + * Status of the latest stage completed by {@code HollowProducer}. + * + * @return SUCCESS or FAIL. + */ + public Status getStatus() { + return status; + } + + /** + * This method returns the exception if the latest state completed by {@code HollowProducer} failed because of an exception. + * + * @return Throwable if {@code Status.equals(FAIL)} else null. + */ + public Throwable getThrowable() { + return throwable; + } + + /** + * This method returns the resulting read state engine after adding new data into write state engine. + * + * @return Resulting read state engine only if data is added successfully else null. + */ + public HollowReadStateEngine getReadStateEngine() { + return readStateEngine; + } + +} \ No newline at end of file diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducerListener.java b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducerListener.java new file mode 100644 index 0000000000..03a0272246 --- /dev/null +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducerListener.java @@ -0,0 +1,74 @@ +package com.netflix.hollow.api.producer; + +/** + * Beta API subject to change. + * + * A class should implement {@code HollowProducerListener}, if it wants to be notified on start/completion of various stages of the {@link HollowProducer}. + * + * @author Kinesh Satiya {@literal kineshsatiya@gmail.com}. + */ +public interface HollowProducerListener { + + /** + * This method is called once {@code HollowProducer} has been initialized with the expected schemas. + */ + public void onProducerInit(); + + /** + * This method is called once {@code HollowProducer} has restored from previously produced state. + * If previous state is not available or its the first state, then this callback will not be called. + * + * @param restoreVersion Version from which the state for {@code HollowProducer} was restored. + */ + public void onProducerRestore(long restoreVersion); + + /** + * This method is called before starting to populating the data into {@code HollowWriteStateEngine}. + * + * @param version Version produced by the {@code HollowProducer} for new cycle about to start. + */ + public void onCycleStart(long version); + + /** + * This method is called when there is no change observed in write state engine, in the current cycle. This would also mean, no delta can be produced for publishing. + * + * @param version Current version of the cycle. + */ + public void onNoDeltaAvailable(long version); + + /** + * This method is called before starting to published the {@code HollowBlob} produced by the {@code HollowProducer}. + * + * @param version Version to be published. + */ + public void onPublishStart(long version); + + /** + * This method is called upon successful publishing of the resulting {@code HollowBlob} produced by the {@code HollowProducer.} + * + * @param status CycleStatus of the publish stage. + */ + public void onPublishComplete(CycleStatus status); + + /** + * This method is called before announcing the availability of new {@code HollowBlob}. + * + * @param version of {@code HollowBlob} that was announced. + */ + public void onAnnouncementStart(long version); + + /** + * This method is called when availability of new {@code HollowBlob} is announced. + * + * @param status CycleStatus of the announcement stage. + */ + public void onAnnouncementComplete(CycleStatus status); + + /** + * This method is called when {@code HollowProducer} has successfully completed a cycle. + * + * @param status CycleStatus when cycle completed. + */ + public void onCycleComplete(CycleStatus status); + +} \ No newline at end of file From 93e27e8947eb79bf86d5f23c36b27a1f10595a40 Mon Sep 17 00:00:00 2001 From: Tim Taylor Date: Tue, 14 Feb 2017 20:38:44 -0800 Subject: [PATCH 07/30] make listening to multiple producers easier to implement --- .../hollow/api/producer/CycleStatus.java | 44 +++++++++++--- .../api/producer/HollowProducerListener.java | 60 ++++++++++++------- 2 files changed, 76 insertions(+), 28 deletions(-) diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/CycleStatus.java b/hollow/src/main/java/com/netflix/hollow/api/producer/CycleStatus.java index 20f885c928..e7cc148920 100644 --- a/hollow/src/main/java/com/netflix/hollow/api/producer/CycleStatus.java +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/CycleStatus.java @@ -1,8 +1,27 @@ +/* + * + * Copyright 2016 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ package com.netflix.hollow.api.producer; import com.netflix.hollow.core.read.engine.HollowReadStateEngine; /** + * Beta API subject to change. + * * This class represents information on details when {@link HollowProducer} has finished executing a particular stage. * An instance of this class is provided on different events of {@link HollowProducerListener}. * @@ -14,30 +33,41 @@ public enum Status { SUCCESS, FAIL } + private final HollowProducer producer; private long version; private Status status; private Throwable throwable; private HollowReadStateEngine readStateEngine; - public static CycleStatus getSuccessInstance(long version) { - return new CycleStatus(version, Status.SUCCESS, null, null); + public static CycleStatus success(HollowProducer producer, long version) { + return new CycleStatus(producer, version, Status.SUCCESS, null, null); } - public static CycleStatus getSuccessInstance(long version, HollowReadStateEngine readStateEngine) { - return new CycleStatus(version, Status.SUCCESS, null, readStateEngine); + public static CycleStatus success(HollowProducer producer, long version, HollowReadStateEngine readStateEngine) { + return new CycleStatus(producer, version, Status.SUCCESS, null, readStateEngine); } - public static CycleStatus getFailInstance(long version, Throwable th) { - return new CycleStatus(version, Status.FAIL, th, null); + public static CycleStatus fail(HollowProducer producer, long version, Throwable th) { + return new CycleStatus(producer, version, Status.FAIL, th, null); } - CycleStatus(long version, Status status, Throwable throwable, HollowReadStateEngine readStateEngine) { + CycleStatus(HollowProducer producer, long version, Status status, Throwable throwable, HollowReadStateEngine readStateEngine) { + this.producer = producer; this.version = version; this.status = status; this.throwable = throwable; this.readStateEngine = readStateEngine; } + /** + * The producer whose status is being described. + * + * @return the producer + */ + public HollowProducer getProducer() { + return producer; + } + /** * This version is currently under process by {@code HollowProducer}. * diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducerListener.java b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducerListener.java index 03a0272246..a9ac0089a1 100644 --- a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducerListener.java +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducerListener.java @@ -1,5 +1,7 @@ package com.netflix.hollow.api.producer; +import java.util.EventListener; + /** * Beta API subject to change. * @@ -7,67 +9,83 @@ * * @author Kinesh Satiya {@literal kineshsatiya@gmail.com}. */ -public interface HollowProducerListener { +public interface HollowProducerListener extends EventListener { /** - * This method is called once {@code HollowProducer} has been initialized with the expected schemas. + * Called after the {@code HollowProducer} has initialized its data model. + * + * @param producer the producer which initialized */ - public void onProducerInit(); + public void onProducerInit(HollowProducer producer); /** - * This method is called once {@code HollowProducer} has restored from previously produced state. - * If previous state is not available or its the first state, then this callback will not be called. + * Called after the {@code HollowProducer} has restored its data state to the indicated version. + * If previous state is not available to restore from, then this callback will not be called. * + * @param producer the producer which restored * @param restoreVersion Version from which the state for {@code HollowProducer} was restored. */ - public void onProducerRestore(long restoreVersion); + public void onProducerRestore(HollowProducer producer, long restoreVersion); /** - * This method is called before starting to populating the data into {@code HollowWriteStateEngine}. + * Called when the {@code HollowProducer} has begun a new cycle. * + * @param producer the producer which started a cycle * @param version Version produced by the {@code HollowProducer} for new cycle about to start. */ - public void onCycleStart(long version); + public void onCycleStart(HollowProducer producer, long version); /** - * This method is called when there is no change observed in write state engine, in the current cycle. This would also mean, no delta can be produced for publishing. + * Called after the new state has been populated if the {@code HollowProducer} detects that no data has changed, thus no snapshot nor delta should be produced.

+ * + * This is a terminal cycle stage; no other stages notifications will be sent for this cycle; the {@link #onCycleComplete(CycleStatus)} will be + * notified with @{code SUCCESS}. * + * @param producer the producer * @param version Current version of the cycle. */ - public void onNoDeltaAvailable(long version); + public void onNoDeltaAvailable(HollowProducer producer, long version); /** - * This method is called before starting to published the {@code HollowBlob} produced by the {@code HollowProducer}. + * Called when the {@code HollowProducer} has begun publishing the {@code HollowBlob} produced this cycle. * + * @param producer the producer beginning to publish * @param version Version to be published. */ - public void onPublishStart(long version); + public void onPublishStart(HollowProducer producer, long version); /** - * This method is called upon successful publishing of the resulting {@code HollowBlob} produced by the {@code HollowProducer.} + * Called after the publish stage finishes normally or abnormally. On successful completion this indicates that + * the {@code HollowBlob} produced this cycle has been published to the blob store. * - * @param status CycleStatus of the publish stage. + * @param status CycleStatus of the publish stage. {@link com.netflix.hollow.api.producer.CycleStatus#getStatus()} will return {@code SUCCESS} + * when the publish was successful; @{code FAIL} otherwise. */ public void onPublishComplete(CycleStatus status); /** - * This method is called before announcing the availability of new {@code HollowBlob}. + * Called when the {@code HollowProducer} has begun announcing the {@code HollowBlob} published this cycle. * - * @param version of {@code HollowBlob} that was announced. + * @param producer the proucer beginning the announcement + * @param version of {@code HollowBlob} that will be announced. */ - public void onAnnouncementStart(long version); + public void onAnnouncementStart(HollowProducer producer, long version); /** - * This method is called when availability of new {@code HollowBlob} is announced. + * Called after the announcement stage finishes normally or abnormally. On successful completion this indicates + * that the {@code HollowBlob} published this cycle has been announced to consumers. * - * @param status CycleStatus of the announcement stage. + * @param status CycleStatus of the announcement stage. {@link com.netflix.hollow.api.producer.CycleStatus#getStatus()} will return {@code SUCCESS} + * when the announce was successful; @{code FAIL} otherwise. */ public void onAnnouncementComplete(CycleStatus status); /** - * This method is called when {@code HollowProducer} has successfully completed a cycle. + * Called after {@code HollowProducer} has completed a cycle normally or abnormally. * - * @param status CycleStatus when cycle completed. + * @param status CycleStatus of this cycle. {@link com.netflix.hollow.api.producer.CycleStatus#getStatus()} will return {@code SUCCESS} + * when the a new state has been announced to consumers or when there were no changes to the data; it will return @{code FAIL} + * when any stage fails or any other failure occurs during cycle processing. */ public void onCycleComplete(CycleStatus status); From bf396a7df55b178e8ab143c82152050555969fd0 Mon Sep 17 00:00:00 2001 From: Tim Taylor Date: Tue, 14 Feb 2017 20:56:10 -0800 Subject: [PATCH 08/30] support notifying integrity check and validation cycle stages --- .../api/producer/HollowProducerListener.java | 49 ++++++++++++++++--- 1 file changed, 42 insertions(+), 7 deletions(-) diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducerListener.java b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducerListener.java index a9ac0089a1..4756f5ae3c 100644 --- a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducerListener.java +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducerListener.java @@ -47,7 +47,7 @@ public interface HollowProducerListener extends EventListener { public void onNoDeltaAvailable(HollowProducer producer, long version); /** - * Called when the {@code HollowProducer} has begun publishing the {@code HollowBlob} produced this cycle. + * Called when the {@code HollowProducer} has begun publishing the {@code HollowBlob}s produced this cycle. * * @param producer the producer beginning to publish * @param version Version to be published. @@ -55,14 +55,48 @@ public interface HollowProducerListener extends EventListener { public void onPublishStart(HollowProducer producer, long version); /** - * Called after the publish stage finishes normally or abnormally. On successful completion this indicates that - * the {@code HollowBlob} produced this cycle has been published to the blob store. + * Called after the publish stage finishes normally or abnormally. A {@code SUCCESS} status indicates that + * the {@code HollowBlob}s produced this cycle has been published to the blob store. * * @param status CycleStatus of the publish stage. {@link com.netflix.hollow.api.producer.CycleStatus#getStatus()} will return {@code SUCCESS} * when the publish was successful; @{code FAIL} otherwise. */ public void onPublishComplete(CycleStatus status); + /** + * Called when the {@code HollowProducer} has begun checking the integrity of the {@code HollowBlob}s produced this cycle. + * + * @param producer the producer beginning the integrity checks + * @param version Version to be checked + */ + public void onIntegrityCheckStart(HollowProducer producer, long version); + + /** + * Called after the integrity check stage finishes normally or abnormally. A {@code SUCCESS} status indicates that + * the previous snapshot, current snapshot, delta, and reverse-delta {@code HollowBlob}s are all internally consistent. + * + * @param status CycleStatus of the integrity check stage. {@link com.netflix.hollow.api.producer.CycleStatus#getStatus()} will return {@code SUCCESS} + * when the blobs are internally consistent; @{code FAIL} otherwise. + */ + public void onIntegrityCheckComplete(CycleStatus status); + + /** + * Called when the {@code HollowProducer} has begun validating the new data state produced this cycle. + * + * @param producer the producer beginning the validation checks + * @param version Version to be validated + */ + public void onValidationStart(HollowProducer producer, long version); + + /** + * Called after the validation stage finishes normally or abnormally. A {@code SUCCESS} status indicates that + * the newly published data state is considered valid. + * + * @param status CycleStatus of the publish stage. {@link com.netflix.hollow.api.producer.CycleStatus#getStatus()} will return {@code SUCCESS} + * when the publish was successful; @{code FAIL} otherwise. + */ + public void onValidationComplete(CycleStatus status); + /** * Called when the {@code HollowProducer} has begun announcing the {@code HollowBlob} published this cycle. * @@ -72,7 +106,7 @@ public interface HollowProducerListener extends EventListener { public void onAnnouncementStart(HollowProducer producer, long version); /** - * Called after the announcement stage finishes normally or abnormally. On successful completion this indicates + * Called after the announcement stage finishes normally or abnormally. A {@code SUCCESS} status indicates * that the {@code HollowBlob} published this cycle has been announced to consumers. * * @param status CycleStatus of the announcement stage. {@link com.netflix.hollow.api.producer.CycleStatus#getStatus()} will return {@code SUCCESS} @@ -81,11 +115,12 @@ public interface HollowProducerListener extends EventListener { public void onAnnouncementComplete(CycleStatus status); /** - * Called after {@code HollowProducer} has completed a cycle normally or abnormally. + * Called after {@code HollowProducer} has completed a cycle normally or abnormally. A {@code SUCCESS} status indicates that the + * entire cycle was successful and the producer is available to begin another cycle. * * @param status CycleStatus of this cycle. {@link com.netflix.hollow.api.producer.CycleStatus#getStatus()} will return {@code SUCCESS} - * when the a new state has been announced to consumers or when there were no changes to the data; it will return @{code FAIL} - * when any stage fails or any other failure occurs during cycle processing. + * when the a new data state has been announced to consumers or when there were no changes to the data; it will return @{code FAIL} + * when any stage failed or any other failure occured during the cycle. */ public void onCycleComplete(CycleStatus status); From 87181846e0def47aafc24c4461f55d70a7b4e210 Mon Sep 17 00:00:00 2001 From: Tim Taylor Date: Tue, 14 Feb 2017 21:12:00 -0800 Subject: [PATCH 09/30] complete producer cycle with working restore and other stages stubbed --- .../hollow/api/producer/HollowProducer.java | 175 ++++++++++++++---- 1 file changed, 142 insertions(+), 33 deletions(-) diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java index 4202ee1639..89d95dc232 100644 --- a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java @@ -17,86 +17,147 @@ */ package com.netflix.hollow.api.producer; - -import java.io.BufferedOutputStream; -import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; -import java.io.OutputStream; - -import com.netflix.hollow.api.StateTransition; +import java.util.EventListener; + +import com.netflix.hollow.api.HollowStateTransition; +import com.netflix.hollow.api.client.HollowBlobRetriever; +import com.netflix.hollow.api.client.HollowClient; +import com.netflix.hollow.api.consumer.HollowAnnouncementRetriever; +import com.netflix.hollow.api.consumer.HollowConsumer; +import com.netflix.hollow.api.consumer.HollowConsumer.ReadState; import com.netflix.hollow.core.read.engine.HollowReadStateEngine; import com.netflix.hollow.core.write.HollowBlobWriter; import com.netflix.hollow.core.write.HollowWriteStateEngine; +import com.netflix.hollow.core.write.objectmapper.HollowObjectMapper; +/** + * Beta API subject to change. + * + * @author Tim Taylor {@literal} + */ public class HollowProducer { - private final VersionMinter versionMinter; private final HollowPublisher publisher; + private final HollowProducer.Validator validator; private final HollowAnnouncer announcer; + private final HollowAnnouncementRetriever announcementRetriever; + private final HollowBlobRetriever blobRetriever; private final HollowWriteStateEngine writeEngine; + private final HollowObjectMapper objectMapper; - private StateTransition announced; + private HollowStateTransition announced; public HollowProducer(VersionMinter versionMinter, HollowPublisher publisher, HollowAnnouncer announcer) { + this(versionMinter, publisher, announcer, HollowAnnouncementRetriever.NO_ANNOUNCEMENTS, null); + } + + public HollowProducer(VersionMinter versionMinter, + HollowPublisher publisher, + HollowAnnouncer announcer, + HollowAnnouncementRetriever announcementRetriever, + HollowBlobRetriever blobRetriever) { this.versionMinter = versionMinter; this.publisher = publisher; + this.validator = Validator.NO_VALIDATIONS; this.announcer = announcer; + this.announcementRetriever = announcementRetriever; + this.blobRetriever = blobRetriever; writeEngine = new HollowWriteStateEngine(); - announced = new StateTransition(); + objectMapper = new HollowObjectMapper(writeEngine); + announced = new HollowStateTransition(); + } + + public void initializeDataModel(Class...classes) { + for(Class c : classes) + objectMapper.initializeTypeState(c); + } + + public HollowProducer restore() { + try { + System.out.println("RESTORE PRIOR STATE..."); + long stateVersion = announcementRetriever.get(); + if(stateVersion != Long.MIN_VALUE) { + // TODO: timt: use HollowConsumer + HollowClient client = new HollowClient(blobRetriever); + client.triggerRefreshTo(stateVersion); + restoreFrom(client.getStateEngine(), client.getCurrentVersionId()); + System.out.format("RESUMING DELTA CHAIN AT %s\n", client.getCurrentVersionId()); + } else { + System.out.println("RESTORE UNAVAILABLE; PRODUCING NEW DELTA CHAIN"); + } + } catch(Exception ex) { + ex.printStackTrace(); + System.out.println("RESTORE UNAVAILABLE; PRODUCING NEW DELTA CHAIN"); + } + return this; } - public void restoreFrom(HollowReadStateEngine priorAnnouncedState, long priorAnnouncedVersion) { + public HollowProducer restoreFrom(HollowReadStateEngine priorAnnouncedState, long priorAnnouncedVersion) { writeEngine.restoreFrom(priorAnnouncedState); - announced = new StateTransition(priorAnnouncedVersion); + announced = new HollowStateTransition(priorAnnouncedVersion); + return this; } /** * Each cycle produces a single state. */ - public void produce(Task task) { + public void runCycle(Populator task) { try { WriteState writeState = beginCycle(announced.advance(versionMinter.mint())); task.populate(writeState); - publish(writeState); - announced = announce(writeState); + if(writeEngine.hasChangedSinceLastCycle()) { + publish(writeState); + + integrityCheck(); + + HollowReadStateEngine foo = null; + + // TODO: timt: provide a no-op Validator implementation for now + validate(); + + announced = announce(writeState); + } else { + // TODO: timt: replace with listener notification + System.out.println("BALLOONS!"); + writeEngine.resetToLastPrepareForNextCycle(); + } + } catch(Throwable th) { th.printStackTrace(); rollback(); } } - private WriteState beginCycle(StateTransition transition) { + private WriteState beginCycle(HollowStateTransition transition) { writeEngine.prepareForNextCycle(); - WriteState writeState = new WriteState(writeEngine, transition); - System.out.println("Beginning cycle " + transition); + WriteState writeState = new WriteStateImpl(objectMapper, transition); + System.out.format("PRODUCING %s\n", transition); return writeState; } private void publish(WriteState writeState) throws IOException { HollowBlobWriter writer = new HollowBlobWriter(writeEngine); - StateTransition transition = writeState.getTransition(); + HollowStateTransition transition = writeState.getTransition(); HollowBlob snapshot = publisher.openSnapshot(transition); try { writer.writeSnapshot(snapshot.getOutputStream()); - snapshot.finish(); if(transition.isDelta()) { HollowBlob delta = publisher.openDelta(transition); - HollowBlob reverseDelta = publisher.openReverseDelta(transition); try { writer.writeDelta(delta.getOutputStream()); - delta.finish(); - - writer.writeReverseDelta(reverseDelta.getOutputStream()); - reverseDelta.finish(); - publisher.publish(delta); - publisher.publish(reverseDelta); } finally { delta.close(); + } + HollowBlob reverseDelta = publisher.openReverseDelta(transition); + try { + writer.writeReverseDelta(reverseDelta.getOutputStream()); + publisher.publish(reverseDelta); + } finally { reverseDelta.close(); } } @@ -113,18 +174,66 @@ private void publish(WriteState writeState) throws IOException { } } + private void integrityCheck() { + /// Given + /// + /// 1. read state (S1) at the previous announced version + /// 2. read state (S2) from the currently produced snapshot + /// + /// Ensure: + /// + /// S1.apply(forward delta).checksum == S2.checksum + /// S2.apply(reverse delta).checksum == S1.checksum + } + + private void validate() { + validator.validate(null); + } + + private HollowStateTransition announce(WriteState writeState) { + HollowStateTransition transition = writeState.getTransition(); + announcer.announce(transition.getToVersion()); + return transition; + } + private void rollback() { writeEngine.resetToLastPrepareForNextCycle(); + System.out.format("ROLLED BACK\n"); } - private StateTransition announce(WriteState writeState) { - announcer.announce(writeState.getVersion()); - StateTransition transition = writeState.getTransition(); - return transition; + public static interface WriteState { + int add(Object o); + + HollowObjectMapper getObjectMapper(); + + HollowWriteStateEngine getStateEngine(); + + // TODO: timt: change to getVersion:long + HollowStateTransition getTransition(); } - public static interface Task { - void populate(WriteState newState); + public static interface VersionMinter { + /** + * Create a new state version.

+ * + * State versions should be ascending -- later states have greater versions.

+ * + * @return a new state version + */ + long mint(); + } + + public static interface Populator { + void populate(HollowProducer.WriteState newState); + } + + public static interface Validator { + static final Validator NO_VALIDATIONS = new Validator(){ + @Override + public void validate(ReadState readState) {} + }; + + void validate(HollowConsumer.ReadState readState); } } From ad6d05dccc7d0c9b9e8a5ac0aaa68536d27e60cb Mon Sep 17 00:00:00 2001 From: Tim Taylor Date: Tue, 14 Feb 2017 23:34:28 -0800 Subject: [PATCH 10/30] re-organize Producer and Consumer interfaces --- ...sition.java => HollowStateTransition.java} | 18 +-- .../hollow/api/consumer/HollowConsumer.java | 48 ++++++++ .../hollow/api/producer/HollowAnnouncer.java | 24 ---- .../hollow/api/producer/HollowBlob.java | 15 --- .../hollow/api/producer/HollowProducer.java | 105 ++++++++++-------- .../hollow/api/producer/HollowPublisher.java | 30 ----- ...ter.java => VersionMinterWithCounter.java} | 26 ++++- .../hollow/api/producer/WriteState.java | 37 ------ .../hollow/api/producer/WriteStateImpl.java | 59 ++++++++++ .../hollow/api/StateTransitionTest.java | 16 +-- 10 files changed, 209 insertions(+), 169 deletions(-) rename hollow/src/main/java/com/netflix/hollow/api/{StateTransition.java => HollowStateTransition.java} (88%) create mode 100644 hollow/src/main/java/com/netflix/hollow/api/consumer/HollowConsumer.java delete mode 100644 hollow/src/main/java/com/netflix/hollow/api/producer/HollowAnnouncer.java delete mode 100644 hollow/src/main/java/com/netflix/hollow/api/producer/HollowBlob.java delete mode 100644 hollow/src/main/java/com/netflix/hollow/api/producer/HollowPublisher.java rename hollow/src/main/java/com/netflix/hollow/api/producer/{VersionMinter.java => VersionMinterWithCounter.java} (51%) delete mode 100644 hollow/src/main/java/com/netflix/hollow/api/producer/WriteState.java create mode 100644 hollow/src/main/java/com/netflix/hollow/api/producer/WriteStateImpl.java diff --git a/hollow/src/main/java/com/netflix/hollow/api/StateTransition.java b/hollow/src/main/java/com/netflix/hollow/api/HollowStateTransition.java similarity index 88% rename from hollow/src/main/java/com/netflix/hollow/api/StateTransition.java rename to hollow/src/main/java/com/netflix/hollow/api/HollowStateTransition.java index fed095c763..262c68cc1d 100644 --- a/hollow/src/main/java/com/netflix/hollow/api/StateTransition.java +++ b/hollow/src/main/java/com/netflix/hollow/api/HollowStateTransition.java @@ -5,7 +5,8 @@ * * @author Tim Taylor {@literal} */ -public final class StateTransition { +// TODO: timt: remove from public API +public final class HollowStateTransition { private final long fromVersion; private final long toVersion; @@ -18,7 +19,8 @@ public final class StateTransition { * Calling {@link #advance(long)} on this transition will return new a transition representing * the first state produced on this chain, i.e. the first snapshot. */ - public StateTransition() { + // TODO: timt: don't need discontinous states, remove + public HollowStateTransition() { this(Long.MIN_VALUE, Long.MIN_VALUE); } @@ -36,7 +38,7 @@ public StateTransition() { * * @see Double Snapshot */ - public StateTransition(long toVersion) { + public HollowStateTransition(long toVersion) { this(Long.MIN_VALUE, toVersion); } @@ -44,7 +46,7 @@ public StateTransition(long toVersion) { * Creates a transition fully representing a transition within the delta chain, a.k.a. a delta, between * {@code fromVersion} and {@code toVersion}. */ - public StateTransition(long fromVersion, long toVersion) { + public HollowStateTransition(long fromVersion, long toVersion) { this.fromVersion = fromVersion; this.toVersion = toVersion; } @@ -64,8 +66,8 @@ public StateTransition(long fromVersion, long toVersion) { * @return a new state transition with its {@code fromVersion} and {@code toVersion} assigned our {@code toVersion} and * the specified {@code nextVersion} respectively */ - public StateTransition advance(long nextVersion) { - return new StateTransition(toVersion, nextVersion); + public HollowStateTransition advance(long nextVersion) { + return new HollowStateTransition(toVersion, nextVersion); } /** @@ -81,9 +83,9 @@ public StateTransition advance(long nextVersion) { * * @throws IllegalStateException if this transition isn't a delta */ - public StateTransition reverse() { + public HollowStateTransition reverse() { if(isDiscontinous() || isSnapshot()) throw new IllegalStateException("must be a delta"); - return new StateTransition(this.toVersion, this.fromVersion); + return new HollowStateTransition(this.toVersion, this.fromVersion); } public long getFromVersion() { diff --git a/hollow/src/main/java/com/netflix/hollow/api/consumer/HollowConsumer.java b/hollow/src/main/java/com/netflix/hollow/api/consumer/HollowConsumer.java new file mode 100644 index 0000000000..8e5cfc1592 --- /dev/null +++ b/hollow/src/main/java/com/netflix/hollow/api/consumer/HollowConsumer.java @@ -0,0 +1,48 @@ +/* + * + * Copyright 2017 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.hollow.api.consumer; + + +/** + * Alpha API subject to change. + * + * @author Tim Taylor {@literal} + */ +public class HollowConsumer { + + // TODO: timt: is this needed, or do we just use a HollowReadStateEngine in place of this? Created for now + // to have symmetry with HollowProducer.WriteState + public static interface ReadState { + long getVersion(); + HollowReadStateEngine getStateEngine(); + } + + public static interface StateRetriever { + ReadState retrieveLatestAnnounced(); + long latestAnnouncedVersion(); + } + + // TODO: timt: don't use HollowBlobRetriever or HollowClient; this is temporary bridge code + public static class BlobStoreStateRetriever implements StateRetriever { + private final HollowAnnouncementWatcher announcementWatcher; + private final HollowBlobRetriever blobRetriever; + + long get(); + } + +} diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowAnnouncer.java b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowAnnouncer.java deleted file mode 100644 index 92c975d10e..0000000000 --- a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowAnnouncer.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * - * Copyright 2016 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package com.netflix.hollow.api.producer; - -public interface HollowAnnouncer { - - public void announce(long stateVersion); - -} diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowBlob.java b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowBlob.java deleted file mode 100644 index 217bc13db9..0000000000 --- a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowBlob.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.netflix.hollow.api.producer; - -import java.io.Closeable; -import java.io.OutputStream; - -public interface HollowBlob extends Closeable { - - OutputStream getOutputStream(); - - void finish(); - - @Override - void close(); - -} diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java index 89d95dc232..bd8f351aef 100644 --- a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java @@ -17,13 +17,13 @@ */ package com.netflix.hollow.api.producer; +import java.io.Closeable; import java.io.IOException; -import java.util.EventListener; +import java.io.OutputStream; import com.netflix.hollow.api.HollowStateTransition; import com.netflix.hollow.api.client.HollowBlobRetriever; import com.netflix.hollow.api.client.HollowClient; -import com.netflix.hollow.api.consumer.HollowAnnouncementRetriever; import com.netflix.hollow.api.consumer.HollowConsumer; import com.netflix.hollow.api.consumer.HollowConsumer.ReadState; import com.netflix.hollow.core.read.engine.HollowReadStateEngine; @@ -37,32 +37,39 @@ * @author Tim Taylor {@literal} */ public class HollowProducer { + public static final Validator NO_VALIDATIONS = new Validator(){ + @Override + public void validate(ReadState readState) {} + }; + private final VersionMinter versionMinter; - private final HollowPublisher publisher; - private final HollowProducer.Validator validator; - private final HollowAnnouncer announcer; - private final HollowAnnouncementRetriever announcementRetriever; - private final HollowBlobRetriever blobRetriever; + private final Publisher publisher; + private final Validator validator; + private final Announcer announcer; private final HollowWriteStateEngine writeEngine; private final HollowObjectMapper objectMapper; private HollowStateTransition announced; - public HollowProducer(VersionMinter versionMinter, HollowPublisher publisher, HollowAnnouncer announcer) { - this(versionMinter, publisher, announcer, HollowAnnouncementRetriever.NO_ANNOUNCEMENTS, null); + public HollowProducer(HollowProducer.Publisher publisher, + HollowProducer.Announcer announcer ) { + this(new VersionMinterWithCounter(), publisher, NO_VALIDATIONS, announcer); } - public HollowProducer(VersionMinter versionMinter, - HollowPublisher publisher, - HollowAnnouncer announcer, - HollowAnnouncementRetriever announcementRetriever, - HollowBlobRetriever blobRetriever) { + public HollowProducer(HollowProducer.Publisher publisher, + HollowProducer.Validator validator, + HollowProducer.Announcer announcer ) { + this(new VersionMinterWithCounter(), publisher, validator, announcer); + } + + private HollowProducer(HollowProducer.VersionMinter versionMinter, + HollowProducer.Publisher publisher, + HollowProducer.Validator validator, + HollowProducer.Announcer announcer) { this.versionMinter = versionMinter; this.publisher = publisher; - this.validator = Validator.NO_VALIDATIONS; + this.validator = validator; this.announcer = announcer; - this.announcementRetriever = announcementRetriever; - this.blobRetriever = blobRetriever; writeEngine = new HollowWriteStateEngine(); objectMapper = new HollowObjectMapper(writeEngine); @@ -74,7 +81,8 @@ public void initializeDataModel(Class...classes) { objectMapper.initializeTypeState(c); } - public HollowProducer restore() { + public HollowProducer restore(HollowConsumer.AnnouncementRetriever announcementRetriever, + HollowBlobRetriever blobRetriever) { try { System.out.println("RESTORE PRIOR STATE..."); long stateVersion = announcementRetriever.get(); @@ -109,21 +117,14 @@ public void runCycle(Populator task) { task.populate(writeState); if(writeEngine.hasChangedSinceLastCycle()) { publish(writeState); - integrityCheck(); - - HollowReadStateEngine foo = null; - - // TODO: timt: provide a no-op Validator implementation for now validate(); - announced = announce(writeState); } else { // TODO: timt: replace with listener notification - System.out.println("BALLOONS!"); + System.out.println("NO SOURCE DATA CHANGES. NOTHING TO DO THIS CYCLE."); writeEngine.resetToLastPrepareForNextCycle(); } - } catch(Throwable th) { th.printStackTrace(); rollback(); @@ -141,19 +142,19 @@ private void publish(WriteState writeState) throws IOException { HollowBlobWriter writer = new HollowBlobWriter(writeEngine); HollowStateTransition transition = writeState.getTransition(); - HollowBlob snapshot = publisher.openSnapshot(transition); + Blob snapshot = publisher.openSnapshot(transition); try { writer.writeSnapshot(snapshot.getOutputStream()); if(transition.isDelta()) { - HollowBlob delta = publisher.openDelta(transition); + Blob delta = publisher.openDelta(transition); try { writer.writeDelta(delta.getOutputStream()); publisher.publish(delta); } finally { delta.close(); } - HollowBlob reverseDelta = publisher.openReverseDelta(transition); + Blob reverseDelta = publisher.openReverseDelta(transition); try { writer.writeReverseDelta(reverseDelta.getOutputStream()); publisher.publish(reverseDelta); @@ -201,17 +202,6 @@ private void rollback() { System.out.format("ROLLED BACK\n"); } - public static interface WriteState { - int add(Object o); - - HollowObjectMapper getObjectMapper(); - - HollowWriteStateEngine getStateEngine(); - - // TODO: timt: change to getVersion:long - HollowStateTransition getTransition(); - } - public static interface VersionMinter { /** * Create a new state version.

@@ -227,13 +217,38 @@ public static interface Populator { void populate(HollowProducer.WriteState newState); } - public static interface Validator { - static final Validator NO_VALIDATIONS = new Validator(){ - @Override - public void validate(ReadState readState) {} - }; + public static interface WriteState { + int add(Object o); + + HollowObjectMapper getObjectMapper(); + + HollowWriteStateEngine getStateEngine(); + + // TODO: timt: change to getVersion:long + HollowStateTransition getTransition(); + } + + public static interface Publisher { + public HollowProducer.Blob openSnapshot(HollowStateTransition transition); + public HollowProducer.Blob openDelta(HollowStateTransition transition); + public HollowProducer.Blob openReverseDelta(HollowStateTransition transition); + + public void publish(HollowProducer.Blob blob); + } + + public static interface Blob extends Closeable { + OutputStream getOutputStream(); + @Override + void close(); + } + + public static interface Validator { void validate(HollowConsumer.ReadState readState); } + public static interface Announcer { + public void announce(long stateVersion); + } + } diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowPublisher.java b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowPublisher.java deleted file mode 100644 index eeb74684d5..0000000000 --- a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowPublisher.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * - * Copyright 2016 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package com.netflix.hollow.api.producer; - -import com.netflix.hollow.api.StateTransition; - -public interface HollowPublisher { - - public HollowBlob openSnapshot(StateTransition transition); - public HollowBlob openDelta(StateTransition transition); - public HollowBlob openReverseDelta(StateTransition transition); - - public void publish(HollowBlob blob); - -} diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/VersionMinter.java b/hollow/src/main/java/com/netflix/hollow/api/producer/VersionMinterWithCounter.java similarity index 51% rename from hollow/src/main/java/com/netflix/hollow/api/producer/VersionMinter.java rename to hollow/src/main/java/com/netflix/hollow/api/producer/VersionMinterWithCounter.java index 1d8749eb9e..6c33a8c7d3 100644 --- a/hollow/src/main/java/com/netflix/hollow/api/producer/VersionMinter.java +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/VersionMinterWithCounter.java @@ -17,15 +17,37 @@ */ package com.netflix.hollow.api.producer; -public interface VersionMinter { +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.TimeZone; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Beta API subject to change. + * + * @author Tim Taylor {@literal} + */ +class VersionMinterWithCounter implements HollowProducer.VersionMinter { + + private static AtomicInteger versionCounter = new AtomicInteger(); /** * Create a new state version.

* * State versions should be ascending -- later states have greater versions.

* + * Here we use an easily readable timestamp format. + * * @return a new state version */ - long mint(); + public long mint() { + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMddHHmmss"); + dateFormat.setTimeZone(TimeZone.getTimeZone("UTC")); + String formattedDate = dateFormat.format(new Date()); + + String versionStr = formattedDate + String.format("%03d", versionCounter.incrementAndGet() % 1000); + + return Long.parseLong(versionStr); + } } diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/WriteState.java b/hollow/src/main/java/com/netflix/hollow/api/producer/WriteState.java deleted file mode 100644 index 6ef3179291..0000000000 --- a/hollow/src/main/java/com/netflix/hollow/api/producer/WriteState.java +++ /dev/null @@ -1,37 +0,0 @@ -package com.netflix.hollow.api.producer; - -import com.netflix.hollow.api.StateTransition; -import com.netflix.hollow.core.write.HollowWriteStateEngine; -import com.netflix.hollow.core.write.objectmapper.HollowObjectMapper; - -public class WriteState { - - private final StateTransition transition; - private final HollowObjectMapper objectMapper; - - WriteState(HollowWriteStateEngine writeEngine, StateTransition transition) { - this.transition = transition; - this.objectMapper = new HollowObjectMapper(writeEngine); - } - - public int add(Object o) { - return objectMapper.add(o); - } - - public HollowObjectMapper getObjectMapper() { - return objectMapper; - } - - public HollowWriteStateEngine getStateEngine() { - return objectMapper.getStateEngine(); - } - - long getVersion() { - return transition.getToVersion(); - } - - StateTransition getTransition() { - return transition; - } - -} diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/WriteStateImpl.java b/hollow/src/main/java/com/netflix/hollow/api/producer/WriteStateImpl.java new file mode 100644 index 0000000000..8e6413c7e5 --- /dev/null +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/WriteStateImpl.java @@ -0,0 +1,59 @@ +/* + * + * Copyright 2017 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.hollow.api.producer; + +import com.netflix.hollow.api.HollowStateTransition; +import com.netflix.hollow.core.write.HollowWriteStateEngine; +import com.netflix.hollow.core.write.objectmapper.HollowObjectMapper; + +/** + * Beta API subject to change. + * + * @author Tim Taylor {@literal} + */ +final class WriteStateImpl implements HollowProducer.WriteState { + + private final HollowObjectMapper objectMapper; + private final HollowStateTransition transition; + + protected WriteStateImpl(HollowObjectMapper objectMapper, HollowStateTransition transition) { + this.objectMapper = objectMapper; + this.transition = transition; + } + + @Override + public int add(Object o) { + return objectMapper.add(o); + } + + @Override + public HollowObjectMapper getObjectMapper() { + return objectMapper; + } + + @Override + public HollowWriteStateEngine getStateEngine() { + return objectMapper.getStateEngine(); + } + + @Override + public HollowStateTransition getTransition() { + return transition; + } + +} diff --git a/hollow/src/test/java/com/netflix/hollow/api/StateTransitionTest.java b/hollow/src/test/java/com/netflix/hollow/api/StateTransitionTest.java index 792bee6c5b..527921c23a 100644 --- a/hollow/src/test/java/com/netflix/hollow/api/StateTransitionTest.java +++ b/hollow/src/test/java/com/netflix/hollow/api/StateTransitionTest.java @@ -10,11 +10,11 @@ public final class StateTransitionTest { - StateTransition subject; + HollowStateTransition subject; @Test public void discontinuous(){ - subject = new StateTransition(); + subject = new HollowStateTransition(); assertThat(subject.getFromVersion(), equalTo(Long.MIN_VALUE)); assertThat(subject.getToVersion(), equalTo(Long.MIN_VALUE)); @@ -26,7 +26,7 @@ public void discontinuous(){ @Test public void snapshot() { - subject = new StateTransition(13L); + subject = new HollowStateTransition(13L); assertThat(subject.getFromVersion(), equalTo(Long.MIN_VALUE)); assertThat(subject.getToVersion(), equalTo(13L)); @@ -38,7 +38,7 @@ public void snapshot() { @Test public void delta() { - subject = new StateTransition(2L, 3L); + subject = new HollowStateTransition(2L, 3L); assertThat(subject.getFromVersion(), equalTo(2L)); assertThat(subject.getToVersion(), equalTo(3L)); @@ -50,7 +50,7 @@ public void delta() { @Test public void advancing() { - subject = new StateTransition(); + subject = new HollowStateTransition(); subject = subject.advance(6L); @@ -69,7 +69,7 @@ public void advancing() { @Test public void reversing() { - subject = new StateTransition(21L, 22L); + subject = new HollowStateTransition(21L, 22L); assertTrue(subject.isForwardDelta()); assertFalse(subject.isReverseDelta()); @@ -82,12 +82,12 @@ public void reversing() { assertTrue(subject.isReverseDelta()); try { - new StateTransition().reverse(); + new HollowStateTransition().reverse(); fail("expected exception"); } catch(IllegalStateException expected){} try { - new StateTransition(1L).reverse(); + new HollowStateTransition(1L).reverse(); fail("expected exception"); } catch(IllegalStateException expected){} } From 8ad039878c6c070c97ff6fe479b73908012bebcb Mon Sep 17 00:00:00 2001 From: Tim Taylor Date: Wed, 15 Feb 2017 02:56:42 -0800 Subject: [PATCH 11/30] notify producer listeners of producer events and cycle stages --- .../AbstractHollowProducerListener.java | 39 ++++++ .../hollow/api/producer/CycleStatus.java | 107 --------------- .../hollow/api/producer/HollowProducer.java | 99 +++++++++++--- .../api/producer/HollowProducerListener.java | 125 +++++++++++++++--- .../hollow/api/producer/ListenerSupport.java | 97 ++++++++++++++ 5 files changed, 320 insertions(+), 147 deletions(-) create mode 100644 hollow/src/main/java/com/netflix/hollow/api/producer/AbstractHollowProducerListener.java delete mode 100644 hollow/src/main/java/com/netflix/hollow/api/producer/CycleStatus.java create mode 100644 hollow/src/main/java/com/netflix/hollow/api/producer/ListenerSupport.java diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/AbstractHollowProducerListener.java b/hollow/src/main/java/com/netflix/hollow/api/producer/AbstractHollowProducerListener.java new file mode 100644 index 0000000000..deb839dfd0 --- /dev/null +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/AbstractHollowProducerListener.java @@ -0,0 +1,39 @@ +/* + * + * Copyright 2017 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.hollow.api.producer; + +/** + * Beta API subject to change. + * + * @author Tim Taylor {@literal} + */ +public class AbstractHollowProducerListener implements HollowProducerListener { + @Override public void onProducerInit() {} + @Override public void onProducerRestore(long restoreVersion) {} + @Override public void onCycleStart(long version) {} + @Override public void onNoDeltaAvailable(long version) {} + @Override public void onPublishStart(long version) {} + @Override public void onPublishComplete(ProducerStatus status) {} + @Override public void onIntegrityCheckStart(long version) {} + @Override public void onIntegrityCheckComplete(ProducerStatus status) {} + @Override public void onValidationStart(long version) {} + @Override public void onValidationComplete(ProducerStatus status) {} + @Override public void onAnnouncementStart(long version) {} + @Override public void onAnnouncementComplete(ProducerStatus status) {} + @Override public void onCycleComplete(ProducerStatus status) {} +} diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/CycleStatus.java b/hollow/src/main/java/com/netflix/hollow/api/producer/CycleStatus.java deleted file mode 100644 index e7cc148920..0000000000 --- a/hollow/src/main/java/com/netflix/hollow/api/producer/CycleStatus.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * - * Copyright 2016 Netflix, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package com.netflix.hollow.api.producer; - -import com.netflix.hollow.core.read.engine.HollowReadStateEngine; - -/** - * Beta API subject to change. - * - * This class represents information on details when {@link HollowProducer} has finished executing a particular stage. - * An instance of this class is provided on different events of {@link HollowProducerListener}. - * - * @author Kinesh Satiya {@literal kineshsatiya@gmail.com} - */ -public class CycleStatus { - - public enum Status { - SUCCESS, FAIL - } - - private final HollowProducer producer; - private long version; - private Status status; - private Throwable throwable; - private HollowReadStateEngine readStateEngine; - - public static CycleStatus success(HollowProducer producer, long version) { - return new CycleStatus(producer, version, Status.SUCCESS, null, null); - } - - public static CycleStatus success(HollowProducer producer, long version, HollowReadStateEngine readStateEngine) { - return new CycleStatus(producer, version, Status.SUCCESS, null, readStateEngine); - } - - public static CycleStatus fail(HollowProducer producer, long version, Throwable th) { - return new CycleStatus(producer, version, Status.FAIL, th, null); - } - - CycleStatus(HollowProducer producer, long version, Status status, Throwable throwable, HollowReadStateEngine readStateEngine) { - this.producer = producer; - this.version = version; - this.status = status; - this.throwable = throwable; - this.readStateEngine = readStateEngine; - } - - /** - * The producer whose status is being described. - * - * @return the producer - */ - public HollowProducer getProducer() { - return producer; - } - - /** - * This version is currently under process by {@code HollowProducer}. - * - * @return Current version of the {@code HollowProducer}. - */ - public long getVersion() { - return version; - } - - /** - * Status of the latest stage completed by {@code HollowProducer}. - * - * @return SUCCESS or FAIL. - */ - public Status getStatus() { - return status; - } - - /** - * This method returns the exception if the latest state completed by {@code HollowProducer} failed because of an exception. - * - * @return Throwable if {@code Status.equals(FAIL)} else null. - */ - public Throwable getThrowable() { - return throwable; - } - - /** - * This method returns the resulting read state engine after adding new data into write state engine. - * - * @return Resulting read state engine only if data is added successfully else null. - */ - public HollowReadStateEngine getReadStateEngine() { - return readStateEngine; - } - -} \ No newline at end of file diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java index bd8f351aef..ca2994cc63 100644 --- a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java @@ -26,6 +26,7 @@ import com.netflix.hollow.api.client.HollowClient; import com.netflix.hollow.api.consumer.HollowConsumer; import com.netflix.hollow.api.consumer.HollowConsumer.ReadState; +import com.netflix.hollow.api.producer.HollowProducerListener.ProducerStatus; import com.netflix.hollow.core.read.engine.HollowReadStateEngine; import com.netflix.hollow.core.write.HollowBlobWriter; import com.netflix.hollow.core.write.HollowWriteStateEngine; @@ -48,6 +49,7 @@ public void validate(ReadState readState) {} private final Announcer announcer; private final HollowWriteStateEngine writeEngine; private final HollowObjectMapper objectMapper; + private final ListenerSupport listeners; private HollowStateTransition announced; @@ -74,35 +76,39 @@ private HollowProducer(HollowProducer.VersionMinter versionMinter, writeEngine = new HollowWriteStateEngine(); objectMapper = new HollowObjectMapper(writeEngine); announced = new HollowStateTransition(); + listeners = new ListenerSupport(); } public void initializeDataModel(Class...classes) { for(Class c : classes) objectMapper.initializeTypeState(c); + listeners.fireProducerInit(); } public HollowProducer restore(HollowConsumer.AnnouncementRetriever announcementRetriever, HollowBlobRetriever blobRetriever) { try { - System.out.println("RESTORE PRIOR STATE..."); long stateVersion = announcementRetriever.get(); if(stateVersion != Long.MIN_VALUE) { // TODO: timt: use HollowConsumer HollowClient client = new HollowClient(blobRetriever); client.triggerRefreshTo(stateVersion); + // FIXME: timt: should fail if we didn't make it to the announced version restoreFrom(client.getStateEngine(), client.getCurrentVersionId()); - System.out.format("RESUMING DELTA CHAIN AT %s\n", client.getCurrentVersionId()); + listeners.fireProducerRestore(announced); } else { + // TODO: timt: notify listeners System.out.println("RESTORE UNAVAILABLE; PRODUCING NEW DELTA CHAIN"); } } catch(Exception ex) { + // TODO: timt: notify listeners ex.printStackTrace(); System.out.println("RESTORE UNAVAILABLE; PRODUCING NEW DELTA CHAIN"); } return this; } - public HollowProducer restoreFrom(HollowReadStateEngine priorAnnouncedState, long priorAnnouncedVersion) { + private HollowProducer restoreFrom(HollowReadStateEngine priorAnnouncedState, long priorAnnouncedVersion) { writeEngine.restoreFrom(priorAnnouncedState); announced = new HollowStateTransition(priorAnnouncedVersion); return this; @@ -112,37 +118,53 @@ public HollowProducer restoreFrom(HollowReadStateEngine priorAnnouncedState, lon * Each cycle produces a single state. */ public void runCycle(Populator task) { + WriteState writeState = null; + ProducerStatus cycleStatus = ProducerStatus.unknownFailure(); try { - WriteState writeState = beginCycle(announced.advance(versionMinter.mint())); + HollowConsumer.ReadState readState = null; + writeState = beginCycle(announced.advance(versionMinter.mint())); task.populate(writeState); if(writeEngine.hasChangedSinceLastCycle()) { publish(writeState); - integrityCheck(); - validate(); - announced = announce(writeState); + readState = integrityCheck(writeState); + validate(writeState.getTransition().getToVersion(), readState); + announced = announce(writeState, readState); } else { - // TODO: timt: replace with listener notification - System.out.println("NO SOURCE DATA CHANGES. NOTHING TO DO THIS CYCLE."); writeEngine.resetToLastPrepareForNextCycle(); + listeners.fireNoDelta(writeState.getTransition()); } + cycleStatus = ProducerStatus.success(writeState.getTransition(), readState); } catch(Throwable th) { th.printStackTrace(); rollback(); + if(writeState != null) cycleStatus = ProducerStatus.fail(writeState.getTransition(), th); + } finally { + listeners.fireCycleComplete(cycleStatus); } } + public void addListener(HollowProducerListener listener) { + listeners.add(listener); + } + + public void removeListener(HollowProducerListener listener) { + listeners.remove(listener); + } + private WriteState beginCycle(HollowStateTransition transition) { + listeners.fireCycleStart(transition); writeEngine.prepareForNextCycle(); WriteState writeState = new WriteStateImpl(objectMapper, transition); - System.out.format("PRODUCING %s\n", transition); return writeState; } private void publish(WriteState writeState) throws IOException { + listeners.firePublishStart(writeState.getTransition()); HollowBlobWriter writer = new HollowBlobWriter(writeEngine); HollowStateTransition transition = writeState.getTransition(); Blob snapshot = publisher.openSnapshot(transition); + ProducerStatus publishStatus = ProducerStatus.unknownFailure(); try { writer.writeSnapshot(snapshot.getOutputStream()); @@ -170,12 +192,17 @@ private void publish(WriteState writeState) throws IOException { } catch(Throwable ignored) { ignored.printStackTrace(); // TODO: timt: log and notify listerners } + publishStatus = ProducerStatus.success(writeState.getTransition()); + } catch(Throwable th) { + publishStatus = ProducerStatus.fail(writeState.getTransition(), th); + throw th; } finally { + listeners.firePublishComplete(publishStatus); snapshot.close(); } } - private void integrityCheck() { + private ReadState integrityCheck(WriteState writeState) { /// Given /// /// 1. read state (S1) at the previous announced version @@ -185,21 +212,57 @@ private void integrityCheck() { /// /// S1.apply(forward delta).checksum == S2.checksum /// S2.apply(reverse delta).checksum == S1.checksum + listeners.fireIntegrityCheckStart(writeState.getTransition()); + ProducerStatus integrityCheckStatus = ProducerStatus.unknownFailure(); + try { + ReadState fromReadState = null; + ReadState toReadState = null; + + // FIXME: timt: do the integrity checks, leaving (S2) assigned to `toReadState` + + integrityCheckStatus = ProducerStatus.success(writeState.getTransition(), toReadState); + return null; + } catch(Throwable th) { + integrityCheckStatus = ProducerStatus.fail(writeState.getTransition(), th); + throw th; + } finally { + listeners.fireIntegrityCheckComplete(integrityCheckStatus); + } + } - private void validate() { - validator.validate(null); + private void validate(long version, ReadState readState) { + listeners.fireValidationStart(version); + ProducerStatus validationStatus = ProducerStatus.unknownFailure(); + try { + validator.validate(null); + validationStatus = ProducerStatus.success(version, readState); + } catch (Throwable th) { + validationStatus = ProducerStatus.fail(version, th); + throw th; + } finally { + listeners.fireValidationComplete(validationStatus); + } } - private HollowStateTransition announce(WriteState writeState) { - HollowStateTransition transition = writeState.getTransition(); - announcer.announce(transition.getToVersion()); - return transition; + private HollowStateTransition announce(WriteState writeState, ReadState readState) { + ProducerStatus announcementStatus = ProducerStatus.unknownFailure(); + try { + HollowStateTransition transition = writeState.getTransition(); + listeners.fireAnnouncementStart(transition); + announcer.announce(transition.getToVersion()); + announcementStatus = ProducerStatus.success(transition, readState); + return transition; + } catch(Throwable th) { + announcementStatus = ProducerStatus.fail(writeState.getTransition(), th); + throw th; + } finally { + listeners.fireAnnouncementComplete(announcementStatus); + } } private void rollback() { writeEngine.resetToLastPrepareForNextCycle(); - System.out.format("ROLLED BACK\n"); } public static interface VersionMinter { diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducerListener.java b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducerListener.java index 4756f5ae3c..fdaede72ff 100644 --- a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducerListener.java +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducerListener.java @@ -2,6 +2,9 @@ import java.util.EventListener; +import com.netflix.hollow.api.HollowStateTransition; +import com.netflix.hollow.api.consumer.HollowConsumer; + /** * Beta API subject to change. * @@ -13,27 +16,23 @@ public interface HollowProducerListener extends EventListener { /** * Called after the {@code HollowProducer} has initialized its data model. - * - * @param producer the producer which initialized */ - public void onProducerInit(HollowProducer producer); + public void onProducerInit(); /** * Called after the {@code HollowProducer} has restored its data state to the indicated version. * If previous state is not available to restore from, then this callback will not be called. * - * @param producer the producer which restored * @param restoreVersion Version from which the state for {@code HollowProducer} was restored. */ - public void onProducerRestore(HollowProducer producer, long restoreVersion); + public void onProducerRestore(long restoreVersion); /** * Called when the {@code HollowProducer} has begun a new cycle. * - * @param producer the producer which started a cycle * @param version Version produced by the {@code HollowProducer} for new cycle about to start. */ - public void onCycleStart(HollowProducer producer, long version); + public void onCycleStart(long version); /** * Called after the new state has been populated if the {@code HollowProducer} detects that no data has changed, thus no snapshot nor delta should be produced.

@@ -41,18 +40,16 @@ public interface HollowProducerListener extends EventListener { * This is a terminal cycle stage; no other stages notifications will be sent for this cycle; the {@link #onCycleComplete(CycleStatus)} will be * notified with @{code SUCCESS}. * - * @param producer the producer * @param version Current version of the cycle. */ - public void onNoDeltaAvailable(HollowProducer producer, long version); + public void onNoDeltaAvailable(long version); /** * Called when the {@code HollowProducer} has begun publishing the {@code HollowBlob}s produced this cycle. * - * @param producer the producer beginning to publish * @param version Version to be published. */ - public void onPublishStart(HollowProducer producer, long version); + public void onPublishStart(long version); /** * Called after the publish stage finishes normally or abnormally. A {@code SUCCESS} status indicates that @@ -61,15 +58,14 @@ public interface HollowProducerListener extends EventListener { * @param status CycleStatus of the publish stage. {@link com.netflix.hollow.api.producer.CycleStatus#getStatus()} will return {@code SUCCESS} * when the publish was successful; @{code FAIL} otherwise. */ - public void onPublishComplete(CycleStatus status); + public void onPublishComplete(ProducerStatus status); /** * Called when the {@code HollowProducer} has begun checking the integrity of the {@code HollowBlob}s produced this cycle. * - * @param producer the producer beginning the integrity checks * @param version Version to be checked */ - public void onIntegrityCheckStart(HollowProducer producer, long version); + public void onIntegrityCheckStart(long version); /** * Called after the integrity check stage finishes normally or abnormally. A {@code SUCCESS} status indicates that @@ -78,15 +74,14 @@ public interface HollowProducerListener extends EventListener { * @param status CycleStatus of the integrity check stage. {@link com.netflix.hollow.api.producer.CycleStatus#getStatus()} will return {@code SUCCESS} * when the blobs are internally consistent; @{code FAIL} otherwise. */ - public void onIntegrityCheckComplete(CycleStatus status); + public void onIntegrityCheckComplete(ProducerStatus status); /** * Called when the {@code HollowProducer} has begun validating the new data state produced this cycle. * - * @param producer the producer beginning the validation checks * @param version Version to be validated */ - public void onValidationStart(HollowProducer producer, long version); + public void onValidationStart(long version); /** * Called after the validation stage finishes normally or abnormally. A {@code SUCCESS} status indicates that @@ -95,15 +90,14 @@ public interface HollowProducerListener extends EventListener { * @param status CycleStatus of the publish stage. {@link com.netflix.hollow.api.producer.CycleStatus#getStatus()} will return {@code SUCCESS} * when the publish was successful; @{code FAIL} otherwise. */ - public void onValidationComplete(CycleStatus status); + public void onValidationComplete(ProducerStatus status); /** * Called when the {@code HollowProducer} has begun announcing the {@code HollowBlob} published this cycle. * - * @param producer the proucer beginning the announcement * @param version of {@code HollowBlob} that will be announced. */ - public void onAnnouncementStart(HollowProducer producer, long version); + public void onAnnouncementStart(long version); /** * Called after the announcement stage finishes normally or abnormally. A {@code SUCCESS} status indicates @@ -112,7 +106,7 @@ public interface HollowProducerListener extends EventListener { * @param status CycleStatus of the announcement stage. {@link com.netflix.hollow.api.producer.CycleStatus#getStatus()} will return {@code SUCCESS} * when the announce was successful; @{code FAIL} otherwise. */ - public void onAnnouncementComplete(CycleStatus status); + public void onAnnouncementComplete(ProducerStatus status); /** * Called after {@code HollowProducer} has completed a cycle normally or abnormally. A {@code SUCCESS} status indicates that the @@ -122,6 +116,93 @@ public interface HollowProducerListener extends EventListener { * when the a new data state has been announced to consumers or when there were no changes to the data; it will return @{code FAIL} * when any stage failed or any other failure occured during the cycle. */ - public void onCycleComplete(CycleStatus status); + public void onCycleComplete(ProducerStatus status); + /** + * Beta API subject to change. + * + * This class represents information on details when {@link HollowProducer} has finished executing a particular stage. + * An instance of this class is provided on different events of {@link HollowProducerListener}. + * + * @author Kinesh Satiya {@literal kineshsatiya@gmail.com} + */ + public class ProducerStatus { + + public enum Status { + SUCCESS, FAIL + } + + private final long version; + private final Status status; + private final Throwable throwable; + private final HollowConsumer.ReadState readState; + + static ProducerStatus success(HollowStateTransition transition) { + return success(transition, null); + } + + static ProducerStatus success(HollowStateTransition transition, HollowConsumer.ReadState readState) { + return success(transition.getToVersion(), readState); + } + + static ProducerStatus success(long version, HollowConsumer.ReadState readState) { + return new ProducerStatus(version, Status.SUCCESS, null, readState); + } + + static ProducerStatus unknownFailure() { + return fail(Long.MIN_VALUE, null); + } + + static ProducerStatus fail(HollowStateTransition transition, Throwable th) { + return fail(transition.getToVersion(), th); + } + + static ProducerStatus fail(long version, Throwable th) { + return new ProducerStatus(version, Status.FAIL, th, null); + } + + private ProducerStatus(long version, Status status, Throwable throwable, HollowConsumer.ReadState readState) { + this.version = version; + this.status = status; + this.throwable = throwable; + this.readState = readState; + } + + /** + * This version is currently under process by {@code HollowProducer}. + * + * @return Current version of the {@code HollowProducer}. + */ + public long getVersion() { + return version; + } + + /** + * Status of the latest stage completed by {@code HollowProducer}. + * + * @return SUCCESS or FAIL. + */ + public Status getStatus() { + return status; + } + + /** + * Returns the failure cause if this status represents a {@code HollowProducer} failure that was caused by an exception. + * + * @return Throwable if {@code Status.equals(FAIL)} else null. + */ + public Throwable getThrowable() { + return throwable; + } + + /** + * Returns the resulting read state engine after adding new data into write state engine. + * + * @return Resulting read state engine only if data is added successfully else null. + */ + public HollowConsumer.ReadState getReadState() { + return readState; + } + + } } \ No newline at end of file diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/ListenerSupport.java b/hollow/src/main/java/com/netflix/hollow/api/producer/ListenerSupport.java new file mode 100644 index 0000000000..298d6f7dcc --- /dev/null +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/ListenerSupport.java @@ -0,0 +1,97 @@ +/* + * + * Copyright 2017 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.hollow.api.producer; + +import java.util.Set; +import java.util.concurrent.CopyOnWriteArraySet; + +import com.netflix.hollow.api.HollowStateTransition; +import com.netflix.hollow.api.producer.HollowProducerListener.ProducerStatus; + +/** + * Beta API subject to change. + * + * @author Tim Taylor {@literal} + */ +final class ListenerSupport { + private final Set listeners; + + ListenerSupport() { + listeners = new CopyOnWriteArraySet<>(); + } + + void add(HollowProducerListener listener) { + listeners.add(listener); + } + + void remove(HollowProducerListener listener) { + listeners.remove(listener); + } + + void fireProducerInit() { + for(final HollowProducerListener l : listeners) l.onProducerInit(); + } + + void fireProducerRestore(HollowStateTransition transition) { + for(final HollowProducerListener l : listeners) l.onProducerRestore(transition.getToVersion()); + } + + void fireCycleStart(HollowStateTransition transition) { + for(final HollowProducerListener l : listeners) l.onCycleStart(transition.getToVersion()); + } + + void fireCycleComplete(ProducerStatus cycleStatus) { + for(final HollowProducerListener l : listeners) l.onCycleComplete(cycleStatus); + } + + void fireNoDelta(HollowStateTransition transition) { + for(final HollowProducerListener l : listeners) l.onNoDeltaAvailable(transition.getToVersion()); + } + + void firePublishStart(HollowStateTransition transition) { + for(final HollowProducerListener l : listeners) l.onPublishStart(transition.getToVersion()); + } + + void firePublishComplete(ProducerStatus publishStatus) { + for(final HollowProducerListener l : listeners) l.onPublishComplete(publishStatus); + } + + void fireIntegrityCheckStart(HollowStateTransition transition) { + for(final HollowProducerListener l : listeners) l.onIntegrityCheckStart(transition.getToVersion()); + } + + void fireIntegrityCheckComplete(ProducerStatus integrityCheckStatus) { + for(final HollowProducerListener l : listeners) l.onIntegrityCheckComplete(integrityCheckStatus); + } + + void fireValidationStart(long version) { + for(final HollowProducerListener l : listeners) l.onValidationStart(version); + } + + void fireValidationComplete(ProducerStatus validationStatus) { + for(final HollowProducerListener l : listeners) l.onValidationComplete(validationStatus); + } + + void fireAnnouncementStart(HollowStateTransition transition) { + for(final HollowProducerListener l : listeners) l.onAnnouncementStart(transition.getToVersion()); + } + + void fireAnnouncementComplete(ProducerStatus announcementStatus) { + for(final HollowProducerListener l : listeners) l.onAnnouncementComplete(announcementStatus); + } +} From 07f5a5e3a7d35737ae964207216b7ea33d9af354 Mon Sep 17 00:00:00 2001 From: Tim Taylor Date: Wed, 15 Feb 2017 03:31:31 -0800 Subject: [PATCH 12/30] reduce HollowStateTransition API surface --- .../hollow/api/HollowStateTransition.java | 150 ------------- .../hollow/api/producer/HollowProducer.java | 202 +++++++++++++++--- .../api/producer/HollowProducerListener.java | 12 +- .../hollow/api/producer/ListenerSupport.java | 25 ++- .../hollow/api/producer/WriteStateImpl.java | 9 +- ...java => HollowProducerTransitionTest.java} | 33 +-- 6 files changed, 195 insertions(+), 236 deletions(-) delete mode 100644 hollow/src/main/java/com/netflix/hollow/api/HollowStateTransition.java rename hollow/src/test/java/com/netflix/hollow/api/{StateTransitionTest.java => HollowProducerTransitionTest.java} (67%) diff --git a/hollow/src/main/java/com/netflix/hollow/api/HollowStateTransition.java b/hollow/src/main/java/com/netflix/hollow/api/HollowStateTransition.java deleted file mode 100644 index 262c68cc1d..0000000000 --- a/hollow/src/main/java/com/netflix/hollow/api/HollowStateTransition.java +++ /dev/null @@ -1,150 +0,0 @@ -package com.netflix.hollow.api; - -/** - * Immutable class representing a single point along the delta chain. - * - * @author Tim Taylor {@literal} - */ -// TODO: timt: remove from public API -public final class HollowStateTransition { - - private final long fromVersion; - private final long toVersion; - - /** - * Creates a transition representing a brand new delta chain or a break - * in an existing chain, e.g. a state transition with neither a {@code fromVersion} nor a - * {@code toVersion}.

- * - * Calling {@link #advance(long)} on this transition will return new a transition representing - * the first state produced on this chain, i.e. the first snapshot. - */ - // TODO: timt: don't need discontinous states, remove - public HollowStateTransition() { - this(Long.MIN_VALUE, Long.MIN_VALUE); - } - - /** - * Creates a transition capable of being used to restore from a delta chain at - * the specified version, a.k.a. a snapshot.

- * - * Consumers can initialize their read state from a snapshot corresponding to - * this transition; an already initialized consumer can only utilize - * this by performing a double snapshot.

- * - * A producer would use this transition to restore from a previous announced state in order - * to resume producing on that delta chain by calling {@link #advance(long)} when ready to - * produce the next state. - * - * @see Double Snapshot - */ - public HollowStateTransition(long toVersion) { - this(Long.MIN_VALUE, toVersion); - } - - /** - * Creates a transition fully representing a transition within the delta chain, a.k.a. a delta, between - * {@code fromVersion} and {@code toVersion}. - */ - public HollowStateTransition(long fromVersion, long toVersion) { - this.fromVersion = fromVersion; - this.toVersion = toVersion; - } - - /** - * Returns a new transition representing the transition from this state's {@code toVersion} to the specified version; - * equivalent to calling {@code new StateTransition(this.toVersion, nextVersion)}. - * - * 

-     * 
-     * [13,45].advance(72) == [45,72]
-     * 
-     *
- * - * @param nextVersion the next version to transition to - * - * @return a new state transition with its {@code fromVersion} and {@code toVersion} assigned our {@code toVersion} and - * the specified {@code nextVersion} respectively - */ - public HollowStateTransition advance(long nextVersion) { - return new HollowStateTransition(toVersion, nextVersion); - } - - /** - * Returns a new transition with versions swapped. Only valid on deltas. - - * 
-     * 
-     * [13,45].reverse() == [45,13]
-     * 
-     *
- - * @return - * - * @throws IllegalStateException if this transition isn't a delta - */ - public HollowStateTransition reverse() { - if(isDiscontinous() || isSnapshot()) throw new IllegalStateException("must be a delta"); - return new HollowStateTransition(this.toVersion, this.fromVersion); - } - - public long getFromVersion() { - return fromVersion; - } - - public long getToVersion() { - return toVersion; - } - - /** - * Determines whether this transition represents a new or broken delta chain. - * - * @return true if this has neither a {@code fromVersion} nor a {@code toVersion}; false otherwise. - */ - public boolean isDiscontinous() { - return fromVersion == Long.MIN_VALUE && toVersion == Long.MIN_VALUE; - } - - /** - * Determines whether this state represents a delta, e.g. a transition between two state versions. - * - * @return true if this has a {@code fromVersion} and {@code toVersion}; false otherwise - */ - public boolean isDelta() { - return fromVersion != Long.MIN_VALUE && toVersion != Long.MIN_VALUE; - } - - public boolean isForwardDelta() { - return isDelta() && fromVersion < toVersion; - } - - public boolean isReverseDelta() { - return isDelta() && fromVersion > toVersion; - } - - public boolean isSnapshot() { - return !isDiscontinous() && !isDelta(); - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - if(isDiscontinous()) { - sb.append("new/broken delta chain"); - } else if(isDelta()) { - if(isReverseDelta()) sb.append("reverse"); - sb.append("delta ["); - sb.append(fromVersion); - if(isForwardDelta()) sb.append(" -> "); - else sb.append(" <- "); - sb.append(toVersion); - sb.append("]"); - } else { - sb.append("snapshot ["); - sb.append(toVersion); - sb.append("]"); - } - return sb.toString(); - } - -} diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java index ca2994cc63..bd3ae09d77 100644 --- a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java @@ -21,7 +21,6 @@ import java.io.IOException; import java.io.OutputStream; -import com.netflix.hollow.api.HollowStateTransition; import com.netflix.hollow.api.client.HollowBlobRetriever; import com.netflix.hollow.api.client.HollowClient; import com.netflix.hollow.api.consumer.HollowConsumer; @@ -51,7 +50,7 @@ public void validate(ReadState readState) {} private final HollowObjectMapper objectMapper; private final ListenerSupport listeners; - private HollowStateTransition announced; + private Transition announced = null; public HollowProducer(HollowProducer.Publisher publisher, HollowProducer.Announcer announcer ) { @@ -75,7 +74,6 @@ private HollowProducer(HollowProducer.VersionMinter versionMinter, writeEngine = new HollowWriteStateEngine(); objectMapper = new HollowObjectMapper(writeEngine); - announced = new HollowStateTransition(); listeners = new ListenerSupport(); } @@ -95,7 +93,7 @@ public HollowProducer restore(HollowConsumer.AnnouncementRetriever announcementR client.triggerRefreshTo(stateVersion); // FIXME: timt: should fail if we didn't make it to the announced version restoreFrom(client.getStateEngine(), client.getCurrentVersionId()); - listeners.fireProducerRestore(announced); + listeners.fireProducerRestore(announced.getToVersion()); } else { // TODO: timt: notify listeners System.out.println("RESTORE UNAVAILABLE; PRODUCING NEW DELTA CHAIN"); @@ -110,7 +108,7 @@ public HollowProducer restore(HollowConsumer.AnnouncementRetriever announcementR private HollowProducer restoreFrom(HollowReadStateEngine priorAnnouncedState, long priorAnnouncedVersion) { writeEngine.restoreFrom(priorAnnouncedState); - announced = new HollowStateTransition(priorAnnouncedVersion); + announced = new Transition(priorAnnouncedVersion); return this; } @@ -122,22 +120,24 @@ public void runCycle(Populator task) { ProducerStatus cycleStatus = ProducerStatus.unknownFailure(); try { HollowConsumer.ReadState readState = null; - writeState = beginCycle(announced.advance(versionMinter.mint())); + Transition transition = announced.advance(versionMinter.mint()); + writeState = beginCycle(transition); task.populate(writeState); if(writeEngine.hasChangedSinceLastCycle()) { - publish(writeState); + publish(transition, writeState); readState = integrityCheck(writeState); - validate(writeState.getTransition().getToVersion(), readState); - announced = announce(writeState, readState); + validate(writeState.getVersion(), readState); + announce(writeState, readState); + announced = transition; } else { writeEngine.resetToLastPrepareForNextCycle(); - listeners.fireNoDelta(writeState.getTransition()); + listeners.fireNoDelta(writeState.getVersion()); } - cycleStatus = ProducerStatus.success(writeState.getTransition(), readState); + cycleStatus = ProducerStatus.success(writeState.getVersion(), readState); } catch(Throwable th) { th.printStackTrace(); rollback(); - if(writeState != null) cycleStatus = ProducerStatus.fail(writeState.getTransition(), th); + if(writeState != null) cycleStatus = ProducerStatus.fail(writeState.getVersion(), th); } finally { listeners.fireCycleComplete(cycleStatus); } @@ -151,17 +151,17 @@ public void removeListener(HollowProducerListener listener) { listeners.remove(listener); } - private WriteState beginCycle(HollowStateTransition transition) { - listeners.fireCycleStart(transition); + private WriteState beginCycle(Transition transition) { + listeners.fireCycleStart(transition.getToVersion()); writeEngine.prepareForNextCycle(); WriteState writeState = new WriteStateImpl(objectMapper, transition); return writeState; } - private void publish(WriteState writeState) throws IOException { - listeners.firePublishStart(writeState.getTransition()); + private void publish(Transition transition, WriteState writeState) throws IOException { + long version = writeState.getVersion(); + listeners.firePublishStart(version); HollowBlobWriter writer = new HollowBlobWriter(writeEngine); - HollowStateTransition transition = writeState.getTransition(); Blob snapshot = publisher.openSnapshot(transition); ProducerStatus publishStatus = ProducerStatus.unknownFailure(); @@ -192,9 +192,9 @@ private void publish(WriteState writeState) throws IOException { } catch(Throwable ignored) { ignored.printStackTrace(); // TODO: timt: log and notify listerners } - publishStatus = ProducerStatus.success(writeState.getTransition()); + publishStatus = ProducerStatus.success(version); } catch(Throwable th) { - publishStatus = ProducerStatus.fail(writeState.getTransition(), th); + publishStatus = ProducerStatus.fail(version, th); throw th; } finally { listeners.firePublishComplete(publishStatus); @@ -212,7 +212,8 @@ private ReadState integrityCheck(WriteState writeState) { /// /// S1.apply(forward delta).checksum == S2.checksum /// S2.apply(reverse delta).checksum == S1.checksum - listeners.fireIntegrityCheckStart(writeState.getTransition()); + long version = writeState.getVersion(); + listeners.fireIntegrityCheckStart(version); ProducerStatus integrityCheckStatus = ProducerStatus.unknownFailure(); try { ReadState fromReadState = null; @@ -220,10 +221,10 @@ private ReadState integrityCheck(WriteState writeState) { // FIXME: timt: do the integrity checks, leaving (S2) assigned to `toReadState` - integrityCheckStatus = ProducerStatus.success(writeState.getTransition(), toReadState); + integrityCheckStatus = ProducerStatus.success(version, toReadState); return null; } catch(Throwable th) { - integrityCheckStatus = ProducerStatus.fail(writeState.getTransition(), th); + integrityCheckStatus = ProducerStatus.fail(version, th); throw th; } finally { listeners.fireIntegrityCheckComplete(integrityCheckStatus); @@ -245,16 +246,15 @@ private void validate(long version, ReadState readState) { } } - private HollowStateTransition announce(WriteState writeState, ReadState readState) { + private void announce(WriteState writeState, ReadState readState) { + long version = writeState.getVersion(); ProducerStatus announcementStatus = ProducerStatus.unknownFailure(); try { - HollowStateTransition transition = writeState.getTransition(); - listeners.fireAnnouncementStart(transition); - announcer.announce(transition.getToVersion()); - announcementStatus = ProducerStatus.success(transition, readState); - return transition; + listeners.fireAnnouncementStart(version); + announcer.announce(version); + announcementStatus = ProducerStatus.success(version, readState); } catch(Throwable th) { - announcementStatus = ProducerStatus.fail(writeState.getTransition(), th); + announcementStatus = ProducerStatus.fail(writeState.getVersion(), th); throw th; } finally { listeners.fireAnnouncementComplete(announcementStatus); @@ -287,14 +287,13 @@ public static interface WriteState { HollowWriteStateEngine getStateEngine(); - // TODO: timt: change to getVersion:long - HollowStateTransition getTransition(); + long getVersion(); } public static interface Publisher { - public HollowProducer.Blob openSnapshot(HollowStateTransition transition); - public HollowProducer.Blob openDelta(HollowStateTransition transition); - public HollowProducer.Blob openReverseDelta(HollowStateTransition transition); + public HollowProducer.Blob openSnapshot(Transition transition); + public HollowProducer.Blob openDelta(Transition transition); + public HollowProducer.Blob openReverseDelta(Transition transition); public void publish(HollowProducer.Blob blob); } @@ -314,4 +313,139 @@ public static interface Announcer { public void announce(long stateVersion); } + /** + * Immutable class representing a single point along the delta chain. + * + * @author Tim Taylor {@literal} + */ + public static final class Transition { + + private final long fromVersion; + private final long toVersion; + + /** + * Creates a transition capable of being used to restore from a delta chain at + * the specified version, a.k.a. a snapshot.

+ * + * Consumers can initialize their read state from a snapshot corresponding to + * this transition; an already initialized consumer can only utilize + * this by performing a double snapshot.

+ * + * A producer would use this transition to restore from a previous announced state in order + * to resume producing on that delta chain by calling {@link #advance(long)} when ready to + * produce the next state. + * + * @see Double Snapshot + */ + public Transition(long toVersion) { + this(Long.MIN_VALUE, toVersion); + } + + /** + * Creates a transition fully representing a transition within the delta chain, a.k.a. a delta, between + * {@code fromVersion} and {@code toVersion}. + */ + public Transition(long fromVersion, long toVersion) { + this.fromVersion = fromVersion; + this.toVersion = toVersion; + } + + /** + * Returns a new transition representing the transition from this state's {@code toVersion} to the specified version; + * equivalent to calling {@code new StateTransition(this.toVersion, nextVersion)}. + * + * 

+         * 
+         * [13,45].advance(72) == [45,72]
+         * 
+         *
+ * + * @param nextVersion the next version to transition to + * + * @return a new state transition with its {@code fromVersion} and {@code toVersion} assigned our {@code toVersion} and + * the specified {@code nextVersion} respectively + */ + public Transition advance(long nextVersion) { + return new Transition(toVersion, nextVersion); + } + + /** + * Returns a new transition with versions swapped. Only valid on deltas. + + * 
+         * 
+         * [13,45].reverse() == [45,13]
+         * 
+         *
+ + * @return + * + * @throws IllegalStateException if this transition isn't a delta + */ + public Transition reverse() { + if(isDiscontinous() || isSnapshot()) throw new IllegalStateException("must be a delta"); + return new Transition(this.toVersion, this.fromVersion); + } + + public long getFromVersion() { + return fromVersion; + } + + public long getToVersion() { + return toVersion; + } + + /** + * Determines whether this transition represents a new or broken delta chain. + * + * @return true if this has neither a {@code fromVersion} nor a {@code toVersion}; false otherwise. + */ + public boolean isDiscontinous() { + return fromVersion == Long.MIN_VALUE && toVersion == Long.MIN_VALUE; + } + + /** + * Determines whether this state represents a delta, e.g. a transition between two state versions. + * + * @return true if this has a {@code fromVersion} and {@code toVersion}; false otherwise + */ + public boolean isDelta() { + return fromVersion != Long.MIN_VALUE && toVersion != Long.MIN_VALUE; + } + + public boolean isForwardDelta() { + return isDelta() && fromVersion < toVersion; + } + + public boolean isReverseDelta() { + return isDelta() && fromVersion > toVersion; + } + + public boolean isSnapshot() { + return !isDiscontinous() && !isDelta(); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + if(isDiscontinous()) { + sb.append("new/broken delta chain"); + } else if(isDelta()) { + if(isReverseDelta()) sb.append("reverse"); + sb.append("delta ["); + sb.append(fromVersion); + if(isForwardDelta()) sb.append(" -> "); + else sb.append(" <- "); + sb.append(toVersion); + sb.append("]"); + } else { + sb.append("snapshot ["); + sb.append(toVersion); + sb.append("]"); + } + return sb.toString(); + } + + } + } diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducerListener.java b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducerListener.java index fdaede72ff..1b65b72f27 100644 --- a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducerListener.java +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducerListener.java @@ -2,7 +2,6 @@ import java.util.EventListener; -import com.netflix.hollow.api.HollowStateTransition; import com.netflix.hollow.api.consumer.HollowConsumer; /** @@ -137,12 +136,9 @@ public enum Status { private final Throwable throwable; private final HollowConsumer.ReadState readState; - static ProducerStatus success(HollowStateTransition transition) { - return success(transition, null); - } - static ProducerStatus success(HollowStateTransition transition, HollowConsumer.ReadState readState) { - return success(transition.getToVersion(), readState); + static ProducerStatus success(long version) { + return success(version, null); } static ProducerStatus success(long version, HollowConsumer.ReadState readState) { @@ -153,10 +149,6 @@ static ProducerStatus unknownFailure() { return fail(Long.MIN_VALUE, null); } - static ProducerStatus fail(HollowStateTransition transition, Throwable th) { - return fail(transition.getToVersion(), th); - } - static ProducerStatus fail(long version, Throwable th) { return new ProducerStatus(version, Status.FAIL, th, null); } diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/ListenerSupport.java b/hollow/src/main/java/com/netflix/hollow/api/producer/ListenerSupport.java index 298d6f7dcc..37e2162e8b 100644 --- a/hollow/src/main/java/com/netflix/hollow/api/producer/ListenerSupport.java +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/ListenerSupport.java @@ -20,7 +20,6 @@ import java.util.Set; import java.util.concurrent.CopyOnWriteArraySet; -import com.netflix.hollow.api.HollowStateTransition; import com.netflix.hollow.api.producer.HollowProducerListener.ProducerStatus; /** @@ -47,32 +46,32 @@ void fireProducerInit() { for(final HollowProducerListener l : listeners) l.onProducerInit(); } - void fireProducerRestore(HollowStateTransition transition) { - for(final HollowProducerListener l : listeners) l.onProducerRestore(transition.getToVersion()); + void fireProducerRestore(long version) { + for(final HollowProducerListener l : listeners) l.onProducerRestore(version); } - void fireCycleStart(HollowStateTransition transition) { - for(final HollowProducerListener l : listeners) l.onCycleStart(transition.getToVersion()); + void fireCycleStart(long version) { + for(final HollowProducerListener l : listeners) l.onCycleStart(version); } void fireCycleComplete(ProducerStatus cycleStatus) { for(final HollowProducerListener l : listeners) l.onCycleComplete(cycleStatus); } - void fireNoDelta(HollowStateTransition transition) { - for(final HollowProducerListener l : listeners) l.onNoDeltaAvailable(transition.getToVersion()); + void fireNoDelta(long version) { + for(final HollowProducerListener l : listeners) l.onNoDeltaAvailable(version); } - void firePublishStart(HollowStateTransition transition) { - for(final HollowProducerListener l : listeners) l.onPublishStart(transition.getToVersion()); + void firePublishStart(long version) { + for(final HollowProducerListener l : listeners) l.onPublishStart(version); } void firePublishComplete(ProducerStatus publishStatus) { for(final HollowProducerListener l : listeners) l.onPublishComplete(publishStatus); } - void fireIntegrityCheckStart(HollowStateTransition transition) { - for(final HollowProducerListener l : listeners) l.onIntegrityCheckStart(transition.getToVersion()); + void fireIntegrityCheckStart(long version) { + for(final HollowProducerListener l : listeners) l.onIntegrityCheckStart(version); } void fireIntegrityCheckComplete(ProducerStatus integrityCheckStatus) { @@ -87,8 +86,8 @@ void fireValidationComplete(ProducerStatus validationStatus) { for(final HollowProducerListener l : listeners) l.onValidationComplete(validationStatus); } - void fireAnnouncementStart(HollowStateTransition transition) { - for(final HollowProducerListener l : listeners) l.onAnnouncementStart(transition.getToVersion()); + void fireAnnouncementStart(long version) { + for(final HollowProducerListener l : listeners) l.onAnnouncementStart(version); } void fireAnnouncementComplete(ProducerStatus announcementStatus) { diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/WriteStateImpl.java b/hollow/src/main/java/com/netflix/hollow/api/producer/WriteStateImpl.java index 8e6413c7e5..21d56b317b 100644 --- a/hollow/src/main/java/com/netflix/hollow/api/producer/WriteStateImpl.java +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/WriteStateImpl.java @@ -17,7 +17,6 @@ */ package com.netflix.hollow.api.producer; -import com.netflix.hollow.api.HollowStateTransition; import com.netflix.hollow.core.write.HollowWriteStateEngine; import com.netflix.hollow.core.write.objectmapper.HollowObjectMapper; @@ -29,9 +28,9 @@ final class WriteStateImpl implements HollowProducer.WriteState { private final HollowObjectMapper objectMapper; - private final HollowStateTransition transition; + private final HollowProducer.Transition transition; - protected WriteStateImpl(HollowObjectMapper objectMapper, HollowStateTransition transition) { + protected WriteStateImpl(HollowObjectMapper objectMapper, HollowProducer.Transition transition) { this.objectMapper = objectMapper; this.transition = transition; } @@ -52,8 +51,8 @@ public HollowWriteStateEngine getStateEngine() { } @Override - public HollowStateTransition getTransition() { - return transition; + public long getVersion() { + return transition.getToVersion(); } } diff --git a/hollow/src/test/java/com/netflix/hollow/api/StateTransitionTest.java b/hollow/src/test/java/com/netflix/hollow/api/HollowProducerTransitionTest.java similarity index 67% rename from hollow/src/test/java/com/netflix/hollow/api/StateTransitionTest.java rename to hollow/src/test/java/com/netflix/hollow/api/HollowProducerTransitionTest.java index 527921c23a..f27b8a6d6e 100644 --- a/hollow/src/test/java/com/netflix/hollow/api/StateTransitionTest.java +++ b/hollow/src/test/java/com/netflix/hollow/api/HollowProducerTransitionTest.java @@ -8,25 +8,17 @@ import org.junit.Test; -public final class StateTransitionTest { +import com.netflix.hollow.api.producer.HollowProducer; - HollowStateTransition subject; +import com.netflix.hollow.api.producer.HollowProducer.Transition; - @Test - public void discontinuous(){ - subject = new HollowStateTransition(); - - assertThat(subject.getFromVersion(), equalTo(Long.MIN_VALUE)); - assertThat(subject.getToVersion(), equalTo(Long.MIN_VALUE)); +public final class HollowProducerTransitionTest { - assertTrue(subject.isDiscontinous()); - assertFalse(subject.isSnapshot()); - assertFalse(subject.isDelta()); - } + HollowProducer.Transition subject; @Test public void snapshot() { - subject = new HollowStateTransition(13L); + subject = new Transition(13L); assertThat(subject.getFromVersion(), equalTo(Long.MIN_VALUE)); assertThat(subject.getToVersion(), equalTo(13L)); @@ -38,7 +30,7 @@ public void snapshot() { @Test public void delta() { - subject = new HollowStateTransition(2L, 3L); + subject = new HollowProducer.Transition(2L, 3L); assertThat(subject.getFromVersion(), equalTo(2L)); assertThat(subject.getToVersion(), equalTo(3L)); @@ -50,9 +42,7 @@ public void delta() { @Test public void advancing() { - subject = new HollowStateTransition(); - - subject = subject.advance(6L); + subject = new Transition(6L); assertThat(subject.getFromVersion(), equalTo(Long.MIN_VALUE)); assertThat(subject.getToVersion(), equalTo(6L)); @@ -69,7 +59,7 @@ public void advancing() { @Test public void reversing() { - subject = new HollowStateTransition(21L, 22L); + subject = new Transition(21L, 22L); assertTrue(subject.isForwardDelta()); assertFalse(subject.isReverseDelta()); @@ -82,12 +72,7 @@ public void reversing() { assertTrue(subject.isReverseDelta()); try { - new HollowStateTransition().reverse(); - fail("expected exception"); - } catch(IllegalStateException expected){} - - try { - new HollowStateTransition(1L).reverse(); + new Transition(1L).reverse(); fail("expected exception"); } catch(IllegalStateException expected){} } From 33bd0cec487f15826d6bf24e03b3f0d0324f2421 Mon Sep 17 00:00:00 2001 From: Tim Taylor Date: Wed, 15 Feb 2017 23:40:52 -0800 Subject: [PATCH 13/30] integrity check stage WIP; elapsed time in stage completion notifications --- .../hollow/api/consumer/HollowConsumer.java | 17 +- .../hollow/api/consumer/ReadStateImpl.java | 53 ++++ .../AbstractHollowProducerListener.java | 22 +- .../hollow/api/producer/HollowProducer.java | 233 +++++++++++------- .../api/producer/HollowProducerListener.java | 70 +++--- .../hollow/api/producer/ListenerSupport.java | 63 +++-- 6 files changed, 303 insertions(+), 155 deletions(-) create mode 100644 hollow/src/main/java/com/netflix/hollow/api/consumer/ReadStateImpl.java diff --git a/hollow/src/main/java/com/netflix/hollow/api/consumer/HollowConsumer.java b/hollow/src/main/java/com/netflix/hollow/api/consumer/HollowConsumer.java index 8e5cfc1592..6bdd001d83 100644 --- a/hollow/src/main/java/com/netflix/hollow/api/consumer/HollowConsumer.java +++ b/hollow/src/main/java/com/netflix/hollow/api/consumer/HollowConsumer.java @@ -17,6 +17,7 @@ */ package com.netflix.hollow.api.consumer; +import com.netflix.hollow.core.read.engine.HollowReadStateEngine; /** * Alpha API subject to change. @@ -32,15 +33,13 @@ public static interface ReadState { HollowReadStateEngine getStateEngine(); } - public static interface StateRetriever { - ReadState retrieveLatestAnnounced(); - long latestAnnouncedVersion(); - } - - // TODO: timt: don't use HollowBlobRetriever or HollowClient; this is temporary bridge code - public static class BlobStoreStateRetriever implements StateRetriever { - private final HollowAnnouncementWatcher announcementWatcher; - private final HollowBlobRetriever blobRetriever; + public static interface AnnouncementRetriever { + static final AnnouncementRetriever NO_ANNOUNCEMENTS = new AnnouncementRetriever(){ + @Override + public long get() { + return Long.MIN_VALUE; + } + }; long get(); } diff --git a/hollow/src/main/java/com/netflix/hollow/api/consumer/ReadStateImpl.java b/hollow/src/main/java/com/netflix/hollow/api/consumer/ReadStateImpl.java new file mode 100644 index 0000000000..673feb28bf --- /dev/null +++ b/hollow/src/main/java/com/netflix/hollow/api/consumer/ReadStateImpl.java @@ -0,0 +1,53 @@ +/* + * + * Copyright 2017 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.hollow.api.consumer; + +import com.netflix.hollow.core.read.engine.HollowReadStateEngine; + +// TODO: timt: should be package protected +/** + * Alpha API subject to change. + * + * @author Tim Taylor {@literal} + */ +public class ReadStateImpl implements HollowConsumer.ReadState { + private final long version; + private final HollowReadStateEngine stateEngine; + + // TODO: timt: temporary until we stop using HollowClient in HollowProducer + public ReadStateImpl(HollowClient client) { + this(client.getCurrentVersionId(), client.getStateEngine()); + } + + // TODO: timt: should be package protected + public ReadStateImpl(long version, HollowReadStateEngine stateEngine) { + this.version = version; + this.stateEngine = stateEngine; + } + + + @Override + public long getVersion() { + return version; + } + + @Override + public HollowReadStateEngine getStateEngine() { + return stateEngine; + } +} diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/AbstractHollowProducerListener.java b/hollow/src/main/java/com/netflix/hollow/api/producer/AbstractHollowProducerListener.java index deb839dfd0..eeabe4b3e3 100644 --- a/hollow/src/main/java/com/netflix/hollow/api/producer/AbstractHollowProducerListener.java +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/AbstractHollowProducerListener.java @@ -17,23 +17,31 @@ */ package com.netflix.hollow.api.producer; +import java.util.concurrent.TimeUnit; + /** * Beta API subject to change. * * @author Tim Taylor {@literal} */ public class AbstractHollowProducerListener implements HollowProducerListener { - @Override public void onProducerInit() {} - @Override public void onProducerRestore(long restoreVersion) {} + @Override public void onProducerInit(long elapsed, TimeUnit unit) {} + @Override public void onProducerRestore(long restoreVersion, long elapsed, TimeUnit unit) {} + @Override public void onCycleStart(long version) {} + @Override public void onCycleComplete(ProducerStatus status, long elapsed, TimeUnit unit) {} + @Override public void onNoDeltaAvailable(long version) {} + @Override public void onPublishStart(long version) {} - @Override public void onPublishComplete(ProducerStatus status) {} + @Override public void onPublishComplete(ProducerStatus status, long elapsed, TimeUnit unit) {} + @Override public void onIntegrityCheckStart(long version) {} - @Override public void onIntegrityCheckComplete(ProducerStatus status) {} + @Override public void onIntegrityCheckComplete(ProducerStatus status, long elapsed, TimeUnit unit) {} + @Override public void onValidationStart(long version) {} - @Override public void onValidationComplete(ProducerStatus status) {} + @Override public void onValidationComplete(ProducerStatus status, long elapsed, TimeUnit unit) {} + @Override public void onAnnouncementStart(long version) {} - @Override public void onAnnouncementComplete(ProducerStatus status) {} - @Override public void onCycleComplete(ProducerStatus status) {} + @Override public void onAnnouncementComplete(ProducerStatus status, long elapsed, TimeUnit unit) {} } diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java index bd3ae09d77..db78fc6364 100644 --- a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java @@ -17,15 +17,20 @@ */ package com.netflix.hollow.api.producer; +import static java.lang.System.currentTimeMillis; + import java.io.Closeable; import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; import com.netflix.hollow.api.client.HollowBlobRetriever; import com.netflix.hollow.api.client.HollowClient; import com.netflix.hollow.api.consumer.HollowConsumer; -import com.netflix.hollow.api.consumer.HollowConsumer.ReadState; +import com.netflix.hollow.api.consumer.ReadStateImpl; import com.netflix.hollow.api.producer.HollowProducerListener.ProducerStatus; +import com.netflix.hollow.core.read.engine.HollowBlobHeaderReader; +import com.netflix.hollow.core.read.engine.HollowBlobReader; import com.netflix.hollow.core.read.engine.HollowReadStateEngine; import com.netflix.hollow.core.write.HollowBlobWriter; import com.netflix.hollow.core.write.HollowWriteStateEngine; @@ -39,61 +44,67 @@ public class HollowProducer { public static final Validator NO_VALIDATIONS = new Validator(){ @Override - public void validate(ReadState readState) {} + public void validate(HollowConsumer.ReadState readState) {} }; private final VersionMinter versionMinter; private final Publisher publisher; private final Validator validator; private final Announcer announcer; + // TODO: timt: use HollowConsumer API + private final HollowBlobRetriever blobRetriever; private final HollowWriteStateEngine writeEngine; private final HollowObjectMapper objectMapper; private final ListenerSupport listeners; - private Transition announced = null; + private Transition announced; public HollowProducer(HollowProducer.Publisher publisher, - HollowProducer.Announcer announcer ) { - this(new VersionMinterWithCounter(), publisher, NO_VALIDATIONS, announcer); + HollowProducer.Announcer announcer, + HollowBlobRetriever blobRetriever) { + this(new VersionMinterWithCounter(), publisher, NO_VALIDATIONS, announcer, blobRetriever); } public HollowProducer(HollowProducer.Publisher publisher, HollowProducer.Validator validator, - HollowProducer.Announcer announcer ) { - this(new VersionMinterWithCounter(), publisher, validator, announcer); + HollowProducer.Announcer announcer, + HollowBlobRetriever blobRetriever) { + this(new VersionMinterWithCounter(), publisher, validator, announcer, blobRetriever); } - private HollowProducer(HollowProducer.VersionMinter versionMinter, - HollowProducer.Publisher publisher, - HollowProducer.Validator validator, - HollowProducer.Announcer announcer) { + private HollowProducer(VersionMinter versionMinter, + Publisher publisher, + Validator validator, + Announcer announcer, + HollowBlobRetriever blobRetriever) { this.versionMinter = versionMinter; this.publisher = publisher; this.validator = validator; this.announcer = announcer; + this.blobRetriever = blobRetriever; + announced = new Transition(); writeEngine = new HollowWriteStateEngine(); objectMapper = new HollowObjectMapper(writeEngine); listeners = new ListenerSupport(); } public void initializeDataModel(Class...classes) { + long start = currentTimeMillis(); for(Class c : classes) objectMapper.initializeTypeState(c); - listeners.fireProducerInit(); + listeners.fireProducerInit(currentTimeMillis() - start); } - public HollowProducer restore(HollowConsumer.AnnouncementRetriever announcementRetriever, - HollowBlobRetriever blobRetriever) { + public HollowProducer restore(HollowConsumer.AnnouncementRetriever announcementRetriever) { + long start = currentTimeMillis(); try { - long stateVersion = announcementRetriever.get(); - if(stateVersion != Long.MIN_VALUE) { - // TODO: timt: use HollowConsumer - HollowClient client = new HollowClient(blobRetriever); - client.triggerRefreshTo(stateVersion); - // FIXME: timt: should fail if we didn't make it to the announced version - restoreFrom(client.getStateEngine(), client.getCurrentVersionId()); - listeners.fireProducerRestore(announced.getToVersion()); + long version = announcementRetriever.get(); + if(version != Long.MIN_VALUE) { + HollowConsumer.ReadState readState = toReadState(version); + writeEngine.restoreFrom(readState.getStateEngine()); + announced = new Transition(readState.getVersion()); + listeners.fireProducerRestore(announced.getToVersion(), currentTimeMillis() - start); } else { // TODO: timt: notify listeners System.out.println("RESTORE UNAVAILABLE; PRODUCING NEW DELTA CHAIN"); @@ -106,40 +117,38 @@ public HollowProducer restore(HollowConsumer.AnnouncementRetriever announcementR return this; } - private HollowProducer restoreFrom(HollowReadStateEngine priorAnnouncedState, long priorAnnouncedVersion) { - writeEngine.restoreFrom(priorAnnouncedState); - announced = new Transition(priorAnnouncedVersion); - return this; - } - /** - * Each cycle produces a single state. + * Each cycle produces a single data state. */ public void runCycle(Populator task) { WriteState writeState = null; + long start = currentTimeMillis(); ProducerStatus cycleStatus = ProducerStatus.unknownFailure(); + long mintedVersion = Long.MIN_VALUE; try { - HollowConsumer.ReadState readState = null; - Transition transition = announced.advance(versionMinter.mint()); + mintedVersion = versionMinter.mint(); + listeners.fireCycleStart(mintedVersion); + Transition transition = announced.advance(mintedVersion); writeState = beginCycle(transition); task.populate(writeState); if(writeEngine.hasChangedSinceLastCycle()) { publish(transition, writeState); - readState = integrityCheck(writeState); - validate(writeState.getVersion(), readState); - announce(writeState, readState); + HollowConsumer.ReadState readState = checkIntegrity(transition, writeState); + validate(readState); + announce(readState); announced = transition; + cycleStatus = ProducerStatus.success(readState); } else { writeEngine.resetToLastPrepareForNextCycle(); listeners.fireNoDelta(writeState.getVersion()); + cycleStatus = ProducerStatus.success(writeState.getVersion()); } - cycleStatus = ProducerStatus.success(writeState.getVersion(), readState); } catch(Throwable th) { th.printStackTrace(); - rollback(); - if(writeState != null) cycleStatus = ProducerStatus.fail(writeState.getVersion(), th); + writeEngine.resetToLastPrepareForNextCycle(); + cycleStatus = ProducerStatus.fail(mintedVersion, th); } finally { - listeners.fireCycleComplete(cycleStatus); + listeners.fireCycleComplete(cycleStatus, start); } } @@ -152,19 +161,17 @@ public void removeListener(HollowProducerListener listener) { } private WriteState beginCycle(Transition transition) { - listeners.fireCycleStart(transition.getToVersion()); writeEngine.prepareForNextCycle(); WriteState writeState = new WriteStateImpl(objectMapper, transition); return writeState; } private void publish(Transition transition, WriteState writeState) throws IOException { - long version = writeState.getVersion(); - listeners.firePublishStart(version); + long start = listeners.firePublishStart(writeState.getVersion()); HollowBlobWriter writer = new HollowBlobWriter(writeEngine); Blob snapshot = publisher.openSnapshot(transition); - ProducerStatus publishStatus = ProducerStatus.unknownFailure(); + ProducerStatus status = ProducerStatus.unknownFailure(); try { writer.writeSnapshot(snapshot.getOutputStream()); @@ -192,79 +199,90 @@ private void publish(Transition transition, WriteState writeState) throws IOExce } catch(Throwable ignored) { ignored.printStackTrace(); // TODO: timt: log and notify listerners } - publishStatus = ProducerStatus.success(version); + status = ProducerStatus.success(writeState.getVersion()); } catch(Throwable th) { - publishStatus = ProducerStatus.fail(version, th); + status = ProducerStatus.fail(writeState.getVersion(), th); throw th; } finally { - listeners.firePublishComplete(publishStatus); + listeners.firePublishComplete(status, start); snapshot.close(); } } - private ReadState integrityCheck(WriteState writeState) { - /// Given - /// - /// 1. read state (S1) at the previous announced version - /// 2. read state (S2) from the currently produced snapshot - /// - /// Ensure: - /// - /// S1.apply(forward delta).checksum == S2.checksum - /// S2.apply(reverse delta).checksum == S1.checksum - long version = writeState.getVersion(); - listeners.fireIntegrityCheckStart(version); - ProducerStatus integrityCheckStatus = ProducerStatus.unknownFailure(); + /** + * Given + * + * 1. read state (S1) at the previous announced version + * 2. read state (S2) from the currently produced snapshot + * + * Ensure: + * + * S1.apply(forward delta).checksum == S2.checksum + * S2.apply(reverse delta).checksum == S1.checksum + * + * @param transition + * @param writeState + * @return + */ + private HollowConsumer.ReadState checkIntegrity(Transition transition, WriteState writeState) { + long start = listeners.fireIntegrityCheckStart(writeState); + ProducerStatus status = ProducerStatus.unknownFailure(); try { - ReadState fromReadState = null; - ReadState toReadState = null; + final HollowConsumer.ReadState result; - // FIXME: timt: do the integrity checks, leaving (S2) assigned to `toReadState` + // TODO: timt: use HollowConsumer + if(transition.isDelta()) { + long desiredVersion = transition.getFromVersion(); + @SuppressWarnings("unused") + HollowConsumer.ReadState fromReadState = toReadState(desiredVersion); + // FIXME: timt: do the integrity checks, leaving (S2) assigned to `toReadState` + + result = readSnapshot(transition); + } else if(transition.isSnapshot()) { + result = readSnapshot(transition); + } else { + throw new IllegalStateException("no blobs to check"); + } - integrityCheckStatus = ProducerStatus.success(version, toReadState); - return null; + status = ProducerStatus.success(result); + return result; } catch(Throwable th) { - integrityCheckStatus = ProducerStatus.fail(version, th); + status = ProducerStatus.fail(writeState.getVersion(), th); throw th; } finally { - listeners.fireIntegrityCheckComplete(integrityCheckStatus); + listeners.fireIntegrityCheckComplete(status, start); } - } - private void validate(long version, ReadState readState) { - listeners.fireValidationStart(version); - ProducerStatus validationStatus = ProducerStatus.unknownFailure(); + /// TODO: timt: validator API TBD + private void validate(HollowConsumer.ReadState readState) { + long start = listeners.fireValidationStart(readState); + ProducerStatus status = ProducerStatus.unknownFailure(); try { - validator.validate(null); - validationStatus = ProducerStatus.success(version, readState); + validator.validate(readState); + status = ProducerStatus.success(readState); } catch (Throwable th) { - validationStatus = ProducerStatus.fail(version, th); + status = ProducerStatus.fail(readState, th); throw th; } finally { - listeners.fireValidationComplete(validationStatus); + listeners.fireValidationComplete(status, start); } } - private void announce(WriteState writeState, ReadState readState) { - long version = writeState.getVersion(); - ProducerStatus announcementStatus = ProducerStatus.unknownFailure(); + private void announce(HollowConsumer.ReadState readState) { + long start = listeners.fireAnnouncementStart(readState); + ProducerStatus status = ProducerStatus.unknownFailure(); try { - listeners.fireAnnouncementStart(version); - announcer.announce(version); - announcementStatus = ProducerStatus.success(version, readState); + announcer.announce(readState.getVersion()); + status = ProducerStatus.success(readState); } catch(Throwable th) { - announcementStatus = ProducerStatus.fail(writeState.getVersion(), th); + status = ProducerStatus.fail(readState, th); throw th; } finally { - listeners.fireAnnouncementComplete(announcementStatus); + listeners.fireAnnouncementComplete(status, start); } } - private void rollback() { - writeEngine.resetToLastPrepareForNextCycle(); - } - public static interface VersionMinter { /** * Create a new state version.

@@ -301,6 +319,9 @@ public static interface Publisher { public static interface Blob extends Closeable { OutputStream getOutputStream(); + // TODO: timt: belongs in HollowConsumer.Blob + InputStream getInputStream(); + @Override void close(); } @@ -323,6 +344,21 @@ public static final class Transition { private final long fromVersion; private final long toVersion; + /** + * Creates a null transition. + * + * A producer would use this to avoid null checks when beginning a new delta chain; calling + * {@link #advance(long)} will return a transition representing the first snapshot in + * a chain.

+ * + * Consumers cannot initialize their read state from this transition.

+ * + * @see Double Snapshot + */ + public Transition() { + this(Long.MIN_VALUE, Long.MIN_VALUE); + } + /** * Creates a transition capable of being used to restore from a delta chain at * the specified version, a.k.a. a snapshot.

@@ -448,4 +484,31 @@ public String toString() { } + ///// TODO: timt: move to HollowConsumer API //// + private HollowConsumer.ReadState toReadState(long desiredVersion) { + HollowClient client = new HollowClient(blobRetriever); + client.triggerRefreshTo(desiredVersion); + long actualVersion = client.getCurrentVersionId(); + if(desiredVersion != actualVersion) throw new IllegalStateException(String.format("desiredVersion=%d actualVersion=%d", desiredVersion, actualVersion)); + HollowConsumer.ReadState readState = new ReadStateImpl(client); + return readState; + } + + private HollowConsumer.ReadState readSnapshot(Transition transition) { + final HollowConsumer.ReadState readState; + HollowReadStateEngine readEngine = new HollowReadStateEngine(); + HollowBlobReader reader = new HollowBlobReader(readEngine, new HollowBlobHeaderReader()); + Blob snapshot = publisher.openSnapshot(transition); + try { + reader.readSnapshot(snapshot.getInputStream()); + readState = new ReadStateImpl(transition.getToVersion(), readEngine); + } catch(IOException ex) { + throw new RuntimeException(ex); + } finally { + snapshot.close(); + } + return readState; + } + ///// END TODO //// + } diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducerListener.java b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducerListener.java index 1b65b72f27..9affdcfa63 100644 --- a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducerListener.java +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducerListener.java @@ -1,6 +1,7 @@ package com.netflix.hollow.api.producer; import java.util.EventListener; +import java.util.concurrent.TimeUnit; import com.netflix.hollow.api.consumer.HollowConsumer; @@ -16,7 +17,7 @@ public interface HollowProducerListener extends EventListener { /** * Called after the {@code HollowProducer} has initialized its data model. */ - public void onProducerInit(); + public void onProducerInit(long elapsed, TimeUnit unit); /** * Called after the {@code HollowProducer} has restored its data state to the indicated version. @@ -24,7 +25,7 @@ public interface HollowProducerListener extends EventListener { * * @param restoreVersion Version from which the state for {@code HollowProducer} was restored. */ - public void onProducerRestore(long restoreVersion); + public void onProducerRestore(long restoreVersion, long elapsed, TimeUnit unit); /** * Called when the {@code HollowProducer} has begun a new cycle. @@ -34,10 +35,17 @@ public interface HollowProducerListener extends EventListener { public void onCycleStart(long version); /** - * Called after the new state has been populated if the {@code HollowProducer} detects that no data has changed, thus no snapshot nor delta should be produced.

+ * Called after {@code HollowProducer} has completed a cycle normally or abnormally. A {@code SUCCESS} status indicates that the + * entire cycle was successful and the producer is available to begin another cycle. * - * This is a terminal cycle stage; no other stages notifications will be sent for this cycle; the {@link #onCycleComplete(CycleStatus)} will be - * notified with @{code SUCCESS}. + * @param status CycleStatus of this cycle. {@link ProducerStatus#getStatus()} will return {@code SUCCESS} + * when the a new data state has been announced to consumers or when there were no changes to the data; it will return @{code FAIL} + * when any stage failed or any other failure occured during the cycle. + */ + public void onCycleComplete(ProducerStatus status, long elapsed, TimeUnit unit); + + /** + * Called after the new state has been populated if the {@code HollowProducer} detects that no data has changed, thus no snapshot nor delta should be produced.

* * @param version Current version of the cycle. */ @@ -54,10 +62,10 @@ public interface HollowProducerListener extends EventListener { * Called after the publish stage finishes normally or abnormally. A {@code SUCCESS} status indicates that * the {@code HollowBlob}s produced this cycle has been published to the blob store. * - * @param status CycleStatus of the publish stage. {@link com.netflix.hollow.api.producer.CycleStatus#getStatus()} will return {@code SUCCESS} + * @param status CycleStatus of the publish stage. {@link ProducerStatus#getStatus()} will return {@code SUCCESS} * when the publish was successful; @{code FAIL} otherwise. */ - public void onPublishComplete(ProducerStatus status); + public void onPublishComplete(ProducerStatus status, long elapsed, TimeUnit unit); /** * Called when the {@code HollowProducer} has begun checking the integrity of the {@code HollowBlob}s produced this cycle. @@ -70,10 +78,10 @@ public interface HollowProducerListener extends EventListener { * Called after the integrity check stage finishes normally or abnormally. A {@code SUCCESS} status indicates that * the previous snapshot, current snapshot, delta, and reverse-delta {@code HollowBlob}s are all internally consistent. * - * @param status CycleStatus of the integrity check stage. {@link com.netflix.hollow.api.producer.CycleStatus#getStatus()} will return {@code SUCCESS} + * @param status CycleStatus of the integrity check stage. {@link ProducerStatus#getStatus()} will return {@code SUCCESS} * when the blobs are internally consistent; @{code FAIL} otherwise. */ - public void onIntegrityCheckComplete(ProducerStatus status); + public void onIntegrityCheckComplete(ProducerStatus status, long elapsed, TimeUnit unit); /** * Called when the {@code HollowProducer} has begun validating the new data state produced this cycle. @@ -86,10 +94,10 @@ public interface HollowProducerListener extends EventListener { * Called after the validation stage finishes normally or abnormally. A {@code SUCCESS} status indicates that * the newly published data state is considered valid. * - * @param status CycleStatus of the publish stage. {@link com.netflix.hollow.api.producer.CycleStatus#getStatus()} will return {@code SUCCESS} + * @param status CycleStatus of the publish stage. {@link ProducerStatus#getStatus()} will return {@code SUCCESS} * when the publish was successful; @{code FAIL} otherwise. */ - public void onValidationComplete(ProducerStatus status); + public void onValidationComplete(ProducerStatus status, long elapsed, TimeUnit unit); /** * Called when the {@code HollowProducer} has begun announcing the {@code HollowBlob} published this cycle. @@ -102,20 +110,10 @@ public interface HollowProducerListener extends EventListener { * Called after the announcement stage finishes normally or abnormally. A {@code SUCCESS} status indicates * that the {@code HollowBlob} published this cycle has been announced to consumers. * - * @param status CycleStatus of the announcement stage. {@link com.netflix.hollow.api.producer.CycleStatus#getStatus()} will return {@code SUCCESS} + * @param status CycleStatus of the announcement stage. {@link ProducerStatus#getStatus()} will return {@code SUCCESS} * when the announce was successful; @{code FAIL} otherwise. */ - public void onAnnouncementComplete(ProducerStatus status); - - /** - * Called after {@code HollowProducer} has completed a cycle normally or abnormally. A {@code SUCCESS} status indicates that the - * entire cycle was successful and the producer is available to begin another cycle. - * - * @param status CycleStatus of this cycle. {@link com.netflix.hollow.api.producer.CycleStatus#getStatus()} will return {@code SUCCESS} - * when the a new data state has been announced to consumers or when there were no changes to the data; it will return @{code FAIL} - * when any stage failed or any other failure occured during the cycle. - */ - public void onCycleComplete(ProducerStatus status); + public void onAnnouncementComplete(ProducerStatus status, long elapsed, TimeUnit unit); /** * Beta API subject to change. @@ -138,26 +136,34 @@ public enum Status { static ProducerStatus success(long version) { - return success(version, null); + return new ProducerStatus(Status.SUCCESS, version, null, null); } - static ProducerStatus success(long version, HollowConsumer.ReadState readState) { - return new ProducerStatus(version, Status.SUCCESS, null, readState); + static ProducerStatus success(HollowConsumer.ReadState readState) { + return new ProducerStatus(Status.SUCCESS, readState.getVersion(), readState, null); } static ProducerStatus unknownFailure() { - return fail(Long.MIN_VALUE, null); + return new ProducerStatus(Status.FAIL, Long.MIN_VALUE, null, null); + } + + static ProducerStatus fail(long version) { + return new ProducerStatus(Status.FAIL, version, null, null); } static ProducerStatus fail(long version, Throwable th) { - return new ProducerStatus(version, Status.FAIL, th, null); + return new ProducerStatus(Status.FAIL, version, null, th); } - private ProducerStatus(long version, Status status, Throwable throwable, HollowConsumer.ReadState readState) { - this.version = version; + static ProducerStatus fail(HollowConsumer.ReadState readState, Throwable th) { + return new ProducerStatus(Status.FAIL, readState.getVersion(), readState, th); + } + + private ProducerStatus(Status status, long version, HollowConsumer.ReadState readState, Throwable throwable) { this.status = status; - this.throwable = throwable; + this.version = version; this.readState = readState; + this.throwable = throwable; } /** @@ -183,7 +189,7 @@ public Status getStatus() { * * @return Throwable if {@code Status.equals(FAIL)} else null. */ - public Throwable getThrowable() { + public Throwable getCause() { return throwable; } diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/ListenerSupport.java b/hollow/src/main/java/com/netflix/hollow/api/producer/ListenerSupport.java index 37e2162e8b..be88ce433a 100644 --- a/hollow/src/main/java/com/netflix/hollow/api/producer/ListenerSupport.java +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/ListenerSupport.java @@ -17,9 +17,13 @@ */ package com.netflix.hollow.api.producer; +import static java.lang.System.*; +import static java.util.concurrent.TimeUnit.MILLISECONDS; import java.util.Set; import java.util.concurrent.CopyOnWriteArraySet; +import com.netflix.hollow.api.consumer.HollowConsumer; +import com.netflix.hollow.api.consumer.HollowConsumer.ReadState; import com.netflix.hollow.api.producer.HollowProducerListener.ProducerStatus; /** @@ -42,55 +46,70 @@ void remove(HollowProducerListener listener) { listeners.remove(listener); } - void fireProducerInit() { - for(final HollowProducerListener l : listeners) l.onProducerInit(); + void fireProducerInit(long elapsedMillis) { + for(final HollowProducerListener l : listeners) l.onProducerInit(elapsedMillis, MILLISECONDS); } - void fireProducerRestore(long version) { - for(final HollowProducerListener l : listeners) l.onProducerRestore(version); + void fireProducerRestore(long version, long elapsedMillis) { + for(final HollowProducerListener l : listeners) l.onProducerRestore(version, elapsedMillis, MILLISECONDS); } - void fireCycleStart(long version) { + long fireCycleStart(long version) { + long start = currentTimeMillis(); for(final HollowProducerListener l : listeners) l.onCycleStart(version); + return start; } - void fireCycleComplete(ProducerStatus cycleStatus) { - for(final HollowProducerListener l : listeners) l.onCycleComplete(cycleStatus); + void fireCycleComplete(ProducerStatus cycleStatus, long startMillis) { + long elapsedMillis = currentTimeMillis() - startMillis; + for(final HollowProducerListener l : listeners) l.onCycleComplete(cycleStatus, elapsedMillis, MILLISECONDS); } void fireNoDelta(long version) { for(final HollowProducerListener l : listeners) l.onNoDeltaAvailable(version); } - void firePublishStart(long version) { + long firePublishStart(long version) { + long start = currentTimeMillis(); for(final HollowProducerListener l : listeners) l.onPublishStart(version); + return start; } - void firePublishComplete(ProducerStatus publishStatus) { - for(final HollowProducerListener l : listeners) l.onPublishComplete(publishStatus); + void firePublishComplete(ProducerStatus publishStatus, long startMillis) { + long elapsedMillis = currentTimeMillis() - startMillis; + for(final HollowProducerListener l : listeners) l.onPublishComplete(publishStatus, elapsedMillis, MILLISECONDS); } - void fireIntegrityCheckStart(long version) { - for(final HollowProducerListener l : listeners) l.onIntegrityCheckStart(version); + long fireIntegrityCheckStart(HollowProducer.WriteState writeState) { + long start = currentTimeMillis(); + for(final HollowProducerListener l : listeners) l.onIntegrityCheckStart(writeState.getVersion()); + return start; } - void fireIntegrityCheckComplete(ProducerStatus integrityCheckStatus) { - for(final HollowProducerListener l : listeners) l.onIntegrityCheckComplete(integrityCheckStatus); + void fireIntegrityCheckComplete(ProducerStatus integrityCheckStatus, long startMillis) { + long elapsedMillis = currentTimeMillis() - startMillis; + for(final HollowProducerListener l : listeners) l.onIntegrityCheckComplete(integrityCheckStatus, elapsedMillis, MILLISECONDS); } - void fireValidationStart(long version) { - for(final HollowProducerListener l : listeners) l.onValidationStart(version); + long fireValidationStart(HollowConsumer.ReadState readState) { + long start = currentTimeMillis(); + for(final HollowProducerListener l : listeners) l.onValidationStart(readState.getVersion()); + return start; } - void fireValidationComplete(ProducerStatus validationStatus) { - for(final HollowProducerListener l : listeners) l.onValidationComplete(validationStatus); + void fireValidationComplete(ProducerStatus validationStatus, long startMillis) { + long elapsedMillis = currentTimeMillis() - startMillis; + for(final HollowProducerListener l : listeners) l.onValidationComplete(validationStatus, elapsedMillis, MILLISECONDS); } - void fireAnnouncementStart(long version) { - for(final HollowProducerListener l : listeners) l.onAnnouncementStart(version); + long fireAnnouncementStart(HollowConsumer.ReadState readState) { + long start = currentTimeMillis(); + for(final HollowProducerListener l : listeners) l.onAnnouncementStart(readState.getVersion()); + return start; } - void fireAnnouncementComplete(ProducerStatus announcementStatus) { - for(final HollowProducerListener l : listeners) l.onAnnouncementComplete(announcementStatus); + void fireAnnouncementComplete(ProducerStatus announcementStatus, long startMillis) { + long elapsedMillis = currentTimeMillis() - startMillis; + for(final HollowProducerListener l : listeners) l.onAnnouncementComplete(announcementStatus, elapsedMillis, MILLISECONDS); } } From 13c9e55dd85c56cff2f703cbb3064dbaed74cc6a Mon Sep 17 00:00:00 2001 From: Tim Taylor Date: Wed, 22 Feb 2017 19:33:19 -0800 Subject: [PATCH 14/30] changes from team code review --- .../hollow/api/producer/HollowProducer.java | 3 +-- .../api/producer/HollowProducerListener.java | 21 ++++++++++++++++--- .../HollowProducerTransitionTest.java | 2 +- 3 files changed, 20 insertions(+), 6 deletions(-) rename hollow/src/test/java/com/netflix/hollow/api/{ => producer}/HollowProducerTransitionTest.java (98%) diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java index db78fc6364..d6a0862da1 100644 --- a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java @@ -144,7 +144,6 @@ public void runCycle(Populator task) { cycleStatus = ProducerStatus.success(writeState.getVersion()); } } catch(Throwable th) { - th.printStackTrace(); writeEngine.resetToLastPrepareForNextCycle(); cycleStatus = ProducerStatus.fail(mintedVersion, th); } finally { @@ -295,7 +294,7 @@ public static interface VersionMinter { } public static interface Populator { - void populate(HollowProducer.WriteState newState); + void populate(HollowProducer.WriteState newState) throws Exception; } public static interface WriteState { diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducerListener.java b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducerListener.java index 9affdcfa63..570db1fc49 100644 --- a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducerListener.java +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducerListener.java @@ -1,3 +1,20 @@ +/* + * + * Copyright 2017 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ package com.netflix.hollow.api.producer; import java.util.EventListener; @@ -116,8 +133,6 @@ public interface HollowProducerListener extends EventListener { public void onAnnouncementComplete(ProducerStatus status, long elapsed, TimeUnit unit); /** - * Beta API subject to change. - * * This class represents information on details when {@link HollowProducer} has finished executing a particular stage. * An instance of this class is provided on different events of {@link HollowProducerListener}. * @@ -203,4 +218,4 @@ public HollowConsumer.ReadState getReadState() { } } -} \ No newline at end of file +} diff --git a/hollow/src/test/java/com/netflix/hollow/api/HollowProducerTransitionTest.java b/hollow/src/test/java/com/netflix/hollow/api/producer/HollowProducerTransitionTest.java similarity index 98% rename from hollow/src/test/java/com/netflix/hollow/api/HollowProducerTransitionTest.java rename to hollow/src/test/java/com/netflix/hollow/api/producer/HollowProducerTransitionTest.java index f27b8a6d6e..61e2db9a69 100644 --- a/hollow/src/test/java/com/netflix/hollow/api/HollowProducerTransitionTest.java +++ b/hollow/src/test/java/com/netflix/hollow/api/producer/HollowProducerTransitionTest.java @@ -1,4 +1,4 @@ -package com.netflix.hollow.api; +package com.netflix.hollow.api.producer; import static org.hamcrest.CoreMatchers.equalTo; import static org.junit.Assert.assertFalse; From 59162b5651b0d92702a03682083508568ccf20f9 Mon Sep 17 00:00:00 2001 From: Tim Taylor Date: Mon, 27 Feb 2017 17:38:57 -0800 Subject: [PATCH 15/30] signal unsuccesful restores and new delta chains --- .../AbstractHollowProducerListener.java | 5 +- .../hollow/api/producer/HollowProducer.java | 37 +++--- .../api/producer/HollowProducerListener.java | 117 +++++++++++++++++- .../hollow/api/producer/ListenerSupport.java | 17 ++- 4 files changed, 150 insertions(+), 26 deletions(-) diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/AbstractHollowProducerListener.java b/hollow/src/main/java/com/netflix/hollow/api/producer/AbstractHollowProducerListener.java index eeabe4b3e3..8219594b5e 100644 --- a/hollow/src/main/java/com/netflix/hollow/api/producer/AbstractHollowProducerListener.java +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/AbstractHollowProducerListener.java @@ -26,7 +26,10 @@ */ public class AbstractHollowProducerListener implements HollowProducerListener { @Override public void onProducerInit(long elapsed, TimeUnit unit) {} - @Override public void onProducerRestore(long restoreVersion, long elapsed, TimeUnit unit) {} + + @Override public void onProducerRestoreStart(long restoreVersion) {} + @Override public void onProducerRestoreComplete(RestoreStatus status, long elapsed, TimeUnit unit) {} + @Override public void onNewDeltaChain(long version) {} @Override public void onCycleStart(long version) {} @Override public void onCycleComplete(ProducerStatus status, long elapsed, TimeUnit unit) {} diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java index d6a0862da1..a59c10d829 100644 --- a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java @@ -29,6 +29,7 @@ import com.netflix.hollow.api.consumer.HollowConsumer; import com.netflix.hollow.api.consumer.ReadStateImpl; import com.netflix.hollow.api.producer.HollowProducerListener.ProducerStatus; +import com.netflix.hollow.api.producer.HollowProducerListener.RestoreStatus; import com.netflix.hollow.core.read.engine.HollowBlobHeaderReader; import com.netflix.hollow.core.read.engine.HollowBlobReader; import com.netflix.hollow.core.read.engine.HollowReadStateEngine; @@ -39,7 +40,7 @@ /** * Beta API subject to change. * - * @author Tim Taylor {@literal} + * @author Tim Taylor {@literal} */ public class HollowProducer { public static final Validator NO_VALIDATIONS = new Validator(){ @@ -98,22 +99,27 @@ public void initializeDataModel(Class...classes) { public HollowProducer restore(HollowConsumer.AnnouncementRetriever announcementRetriever) { long start = currentTimeMillis(); + RestoreStatus status = RestoreStatus.unknownFailure(); + long versionDesired = Long.MIN_VALUE; + long versionReached = Long.MIN_VALUE; + try { - long version = announcementRetriever.get(); - if(version != Long.MIN_VALUE) { - HollowConsumer.ReadState readState = toReadState(version); - writeEngine.restoreFrom(readState.getStateEngine()); - announced = new Transition(readState.getVersion()); - listeners.fireProducerRestore(announced.getToVersion(), currentTimeMillis() - start); - } else { - // TODO: timt: notify listeners - System.out.println("RESTORE UNAVAILABLE; PRODUCING NEW DELTA CHAIN"); + versionDesired = announcementRetriever.get(); + listeners.fireProducerRestoreStart(versionDesired); + + if(versionDesired != Long.MIN_VALUE) { + HollowConsumer.ReadState readState = toReadState(versionDesired); + versionReached = readState.getVersion(); + if(versionReached == versionDesired) { + writeEngine.restoreFrom(readState.getStateEngine()); + announced = new Transition(versionReached); + status = RestoreStatus.success(versionDesired, versionReached); + } } - } catch(Exception ex) { - // TODO: timt: notify listeners - ex.printStackTrace(); - System.out.println("RESTORE UNAVAILABLE; PRODUCING NEW DELTA CHAIN"); + } catch(Throwable th) { + status = RestoreStatus.fail(versionDesired, versionReached, th); } + listeners.fireProducerRestoreComplete(status, currentTimeMillis() - start); return this; } @@ -127,8 +133,9 @@ public void runCycle(Populator task) { long mintedVersion = Long.MIN_VALUE; try { mintedVersion = versionMinter.mint(); - listeners.fireCycleStart(mintedVersion); Transition transition = announced.advance(mintedVersion); + if(transition.isSnapshot()) listeners.fireNewDeltaChain(mintedVersion); + listeners.fireCycleStart(mintedVersion); writeState = beginCycle(transition); task.populate(writeState); if(writeEngine.hasChangedSinceLastCycle()) { diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducerListener.java b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducerListener.java index 570db1fc49..1c6c6769b9 100644 --- a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducerListener.java +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducerListener.java @@ -42,7 +42,28 @@ public interface HollowProducerListener extends EventListener { * * @param restoreVersion Version from which the state for {@code HollowProducer} was restored. */ - public void onProducerRestore(long restoreVersion, long elapsed, TimeUnit unit); + public void onProducerRestoreStart(long restoreVersion); + + /** + * Called after the {@code HollowProducer} has restored its data state to the indicated version. + * If previous state is not available to restore from, then this callback will not be called. + * + * @param status of the restore. {@link RestoreStatus#getStatus()} will return {@code SUCCESS} when + * the desired version was reached during restore, otheriwse {@code FAIL} will be returned. + * @param elapsed duration of the restore in {@code unit} units + * @param unit units of the {@code elapsed} duration + */ + public void onProducerRestoreComplete(RestoreStatus status, long elapsed, TimeUnit unit); + + /** + * Indicates that the next state produced will begin a new delta chain. + * Thiis will be called prior to the next state being produced either if + * {@link HollowProducer#restore(com.netflix.hollow.api.consumer.HollowConsumer.AnnouncementRetriever)} + * hasn't been called or the restore failed. + * + * @param version the version of the state that will become the first of a new delta chain + */ + public void onNewDeltaChain(long version); /** * Called when the {@code HollowProducer} has begun a new cycle. @@ -55,9 +76,11 @@ public interface HollowProducerListener extends EventListener { * Called after {@code HollowProducer} has completed a cycle normally or abnormally. A {@code SUCCESS} status indicates that the * entire cycle was successful and the producer is available to begin another cycle. * - * @param status CycleStatus of this cycle. {@link ProducerStatus#getStatus()} will return {@code SUCCESS} + * @param status ProducerStatus of this cycle. {@link ProducerStatus#getStatus()} will return {@code SUCCESS} * when the a new data state has been announced to consumers or when there were no changes to the data; it will return @{code FAIL} * when any stage failed or any other failure occured during the cycle. + * @param elapsed duration of the cycle in {@code unit} units + * @param unit units of the {@code elapsed} duration */ public void onCycleComplete(ProducerStatus status, long elapsed, TimeUnit unit); @@ -81,6 +104,8 @@ public interface HollowProducerListener extends EventListener { * * @param status CycleStatus of the publish stage. {@link ProducerStatus#getStatus()} will return {@code SUCCESS} * when the publish was successful; @{code FAIL} otherwise. + * @param elapsed duration of the publish stage in {@code unit} units + * @param unit units of the {@code elapsed} duration */ public void onPublishComplete(ProducerStatus status, long elapsed, TimeUnit unit); @@ -97,6 +122,8 @@ public interface HollowProducerListener extends EventListener { * * @param status CycleStatus of the integrity check stage. {@link ProducerStatus#getStatus()} will return {@code SUCCESS} * when the blobs are internally consistent; @{code FAIL} otherwise. + * @param elapsed duration of the integrity check stage in {@code unit} units + * @param unit units of the {@code elapsed} duration */ public void onIntegrityCheckComplete(ProducerStatus status, long elapsed, TimeUnit unit); @@ -113,6 +140,8 @@ public interface HollowProducerListener extends EventListener { * * @param status CycleStatus of the publish stage. {@link ProducerStatus#getStatus()} will return {@code SUCCESS} * when the publish was successful; @{code FAIL} otherwise. + * @param elapsed duration of the validation stage in {@code unit} units + * @param unit units of the {@code elapsed} duration */ public void onValidationComplete(ProducerStatus status, long elapsed, TimeUnit unit); @@ -129,6 +158,8 @@ public interface HollowProducerListener extends EventListener { * * @param status CycleStatus of the announcement stage. {@link ProducerStatus#getStatus()} will return {@code SUCCESS} * when the announce was successful; @{code FAIL} otherwise. + * @param elapsed duration of the announcement stage in {@code unit} units + * @param unit units of the {@code elapsed} duration */ public void onAnnouncementComplete(ProducerStatus status, long elapsed, TimeUnit unit); @@ -140,10 +171,6 @@ public interface HollowProducerListener extends EventListener { */ public class ProducerStatus { - public enum Status { - SUCCESS, FAIL - } - private final long version; private final Status status; private final Throwable throwable; @@ -216,6 +243,84 @@ public Throwable getCause() { public HollowConsumer.ReadState getReadState() { return readState; } + } + + /** + * This class represents information on details when {@link HollowProducer} has finished executing a particular stage. + * An instance of this class is provided on different events of {@link HollowProducerListener}. + * + * @author Tim Taylor {@literal tim@toolbear.io} + */ + public class RestoreStatus { + private final Status status; + private final long versionDesired; + private final long versionReached; + private final Throwable throwable; + + static RestoreStatus success(long versionDesired, long versionReached) { + return new RestoreStatus(Status.SUCCESS, versionDesired, versionReached, null); + } + + static RestoreStatus unknownFailure() { + return new RestoreStatus(Status.FAIL, Long.MIN_VALUE, Long.MIN_VALUE, null); + } + + static RestoreStatus fail(long versionDesired, long versionReached, Throwable cause) { + return new RestoreStatus(Status.FAIL, versionDesired, versionReached, cause); + } + + private RestoreStatus(Status status, long versionDesired, long versionReached, Throwable throwable) { + this.status = status; + this.versionDesired = versionDesired; + this.versionReached = versionReached; + this.throwable = throwable; + } + + /** + * The version desired to restore to when calling + * {@link HollowProducer#restore(com.netflix.hollow.api.consumer.HollowConsumer.AnnouncementRetriever) + * + * @return the latest announced version or {@code Long.MIN_VALUE} if latest announced version couldn't be + * retrieved + */ + public long getDesiredVersion() { + return versionDesired; + } + + /** + * The version reached when restoring. + * When {@link HollowProducer#restore(com.netflix.hollow.api.consumer.HollowConsumer.AnnouncementRetriever)} + * succeeds then {@code versionDesired == versionReached} is always true. Can be {@code Long.MIN_VALUE} + * indicating restore failed to reach any state, or the version of an intermediate state reached. + * + * @return the version restored to when successful, otherwise {@code Long.MIN_VALUE} if no version was + * reached or the version of an intermediate state reached before restore completed unsuccessfully. + */ + public long getVersionReached() { + return versionReached; + } + + /** + * Status of the restore + * + * @return SUCCESS or FAIL. + */ + public Status getStatus() { + return status; + } + + /** + * Returns the failure cause if this status represents a {@code HollowProducer} failure that was caused by an exception. + * + * @return the {@code Throwable} cause of a failure, otherwise {@code null} if restore succeeded or it failed + * without an exception. + */ + public Throwable getCause() { + return throwable; + } + } + public enum Status { + SUCCESS, FAIL } } diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/ListenerSupport.java b/hollow/src/main/java/com/netflix/hollow/api/producer/ListenerSupport.java index be88ce433a..7de58768d0 100644 --- a/hollow/src/main/java/com/netflix/hollow/api/producer/ListenerSupport.java +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/ListenerSupport.java @@ -17,14 +17,15 @@ */ package com.netflix.hollow.api.producer; -import static java.lang.System.*; +import static java.lang.System.currentTimeMillis; import static java.util.concurrent.TimeUnit.MILLISECONDS; + import java.util.Set; import java.util.concurrent.CopyOnWriteArraySet; import com.netflix.hollow.api.consumer.HollowConsumer; -import com.netflix.hollow.api.consumer.HollowConsumer.ReadState; import com.netflix.hollow.api.producer.HollowProducerListener.ProducerStatus; +import com.netflix.hollow.api.producer.HollowProducerListener.RestoreStatus; /** * Beta API subject to change. @@ -50,8 +51,16 @@ void fireProducerInit(long elapsedMillis) { for(final HollowProducerListener l : listeners) l.onProducerInit(elapsedMillis, MILLISECONDS); } - void fireProducerRestore(long version, long elapsedMillis) { - for(final HollowProducerListener l : listeners) l.onProducerRestore(version, elapsedMillis, MILLISECONDS); + void fireProducerRestoreStart(long version) { + for(final HollowProducerListener l : listeners) l.onProducerRestoreStart(version); + } + + void fireProducerRestoreComplete(RestoreStatus status, long elapsedMillis) { + for(final HollowProducerListener l : listeners) l.onProducerRestoreComplete(status, elapsedMillis, MILLISECONDS); + } + + void fireNewDeltaChain(long version) { + for(final HollowProducerListener l : listeners) l.onNewDeltaChain(version); } long fireCycleStart(long version) { From b976cb9380456d610e04fe9b8af181c8f1237b41 Mon Sep 17 00:00:00 2001 From: Tim Taylor Date: Mon, 27 Feb 2017 18:07:00 -0800 Subject: [PATCH 16/30] fix javadoc error --- .../com/netflix/hollow/api/producer/HollowProducerListener.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducerListener.java b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducerListener.java index 1c6c6769b9..56f199c722 100644 --- a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducerListener.java +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducerListener.java @@ -278,7 +278,7 @@ private RestoreStatus(Status status, long versionDesired, long versionReached, T /** * The version desired to restore to when calling - * {@link HollowProducer#restore(com.netflix.hollow.api.consumer.HollowConsumer.AnnouncementRetriever) + * {@link HollowProducer#restore(com.netflix.hollow.api.consumer.HollowConsumer.AnnouncementRetriever)} * * @return the latest announced version or {@code Long.MIN_VALUE} if latest announced version couldn't be * retrieved From 150b4e939b04b8d497d00d68e8aec11b9bf1a6b5 Mon Sep 17 00:00:00 2001 From: Tim Taylor Date: Mon, 27 Feb 2017 23:42:16 -0800 Subject: [PATCH 17/30] HollowBlobWriter: explicitly flush output stream --- .../java/com/netflix/hollow/core/write/HollowBlobWriter.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hollow/src/main/java/com/netflix/hollow/core/write/HollowBlobWriter.java b/hollow/src/main/java/com/netflix/hollow/core/write/HollowBlobWriter.java index d7080c83d6..b86b8b8e99 100644 --- a/hollow/src/main/java/com/netflix/hollow/core/write/HollowBlobWriter.java +++ b/hollow/src/main/java/com/netflix/hollow/core/write/HollowBlobWriter.java @@ -77,6 +77,7 @@ public void run() { typeState.writeSnapshot(dos); } + os.flush(); } /** @@ -123,6 +124,7 @@ public void run() { typeState.writeDelta(dos); } } + os.flush(); } /** @@ -169,6 +171,7 @@ public void run() { typeState.writeReverseDelta(dos); } } + os.flush(); } private List changedTypes() { From d701d42fe59678e2c575916c641b3d63fee6d5ed Mon Sep 17 00:00:00 2001 From: Tim Taylor Date: Mon, 27 Feb 2017 23:52:53 -0800 Subject: [PATCH 18/30] remove HollowProducer.Transition from public API --- .../hollow/api/producer/HollowProducer.java | 110 +++++++++++------- .../hollow/api/producer/WriteStateImpl.java | 10 +- .../HollowProducerTransitionTest.java | 79 ------------- 3 files changed, 70 insertions(+), 129 deletions(-) delete mode 100644 hollow/src/test/java/com/netflix/hollow/api/producer/HollowProducerTransitionTest.java diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java index a59c10d829..6d5e0e71ba 100644 --- a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java @@ -168,7 +168,7 @@ public void removeListener(HollowProducerListener listener) { private WriteState beginCycle(Transition transition) { writeEngine.prepareForNextCycle(); - WriteState writeState = new WriteStateImpl(objectMapper, transition); + WriteState writeState = new WriteStateImpl(objectMapper, transition.toVersion); return writeState; } @@ -176,20 +176,20 @@ private void publish(Transition transition, WriteState writeState) throws IOExce long start = listeners.firePublishStart(writeState.getVersion()); HollowBlobWriter writer = new HollowBlobWriter(writeEngine); - Blob snapshot = publisher.openSnapshot(transition); + Blob snapshot = publisher.openSnapshot(transition.toVersion); ProducerStatus status = ProducerStatus.unknownFailure(); try { writer.writeSnapshot(snapshot.getOutputStream()); if(transition.isDelta()) { - Blob delta = publisher.openDelta(transition); + Blob delta = publisher.openDelta(transition.fromVersion, transition.toVersion); try { writer.writeDelta(delta.getOutputStream()); publisher.publish(delta); } finally { delta.close(); } - Blob reverseDelta = publisher.openReverseDelta(transition); + Blob reverseDelta = publisher.openReverseDelta(transition.fromVersion, transition.toVersion); try { writer.writeReverseDelta(reverseDelta.getOutputStream()); publisher.publish(reverseDelta); @@ -238,7 +238,7 @@ private HollowConsumer.ReadState checkIntegrity(Transition transition, WriteStat // TODO: timt: use HollowConsumer if(transition.isDelta()) { - long desiredVersion = transition.getFromVersion(); + long desiredVersion = transition.fromVersion; @SuppressWarnings("unused") HollowConsumer.ReadState fromReadState = toReadState(desiredVersion); // FIXME: timt: do the integrity checks, leaving (S2) assigned to `toReadState` @@ -315,10 +315,56 @@ public static interface WriteState { } public static interface Publisher { - public HollowProducer.Blob openSnapshot(Transition transition); - public HollowProducer.Blob openDelta(Transition transition); - public HollowProducer.Blob openReverseDelta(Transition transition); + /** + * Returns a blob with which a {@code HollowProducer} will write a snapshot for the version specified.

+ * + * The producer will pass the returned blob back to this publisher when calling {@link Publisher#publish(Blob)}. + * + * @param version the blob version + * + * @return a {@link HollowProducer.Blob} representing a snapshot for the {@code version} + */ + public HollowProducer.Blob openSnapshot(long version); + + /** + * Returns a blob with which a {@code HollowProducer} will write a forward delta from the version specified to + * the version specified, i.e. {@code fromVersion => toVersion}.

+ * + * The producer will pass the returned blob back to this publisher when calling {@link Publisher#publish(Blob)}. + * + * In the delta chain {@code fromVersion} is the older version such that {@code fromVersion < toVersion}. + * + * @param fromVersion the data state this delta will transition from + * @param toVersion the data state this delta will transition to + * + * @return a {@link HollowProducer.Blob} representing a snapshot for the {@code version} + */ + public HollowProducer.Blob openDelta(long fromVersion, long toVersion); + + /** + * Returns a blob with which a {@code HollowProducer} will write a reverse delta from the version specified to + * the version specified, i.e. {@code fromVersion <= toVersion}.

+ * + * The producer will pass the returned blob back to this publisher when calling {@link Publisher#publish(Blob)}. + * + * In the delta chain {@code fromVersion} is the older version such that {@code fromVersion < toVersion}. + * + * @param fromVersion version in the delta chain immediately before {@code toVersion} + * @param toVersion version in the delta chain immediately after {@code fromVersion} + * + * @return a {@link HollowProducer.Blob} representing a snapshot for the {@code version} + */ + public HollowProducer.Blob openReverseDelta(long fromVersion, long toVersion); + /** + * Publish the blob specified to this publisher's blobstore.

+ * + * It is guaranteed that {@code blob} was created by calling one of + * {@link Publisher#openSnapshot(long)}, {@link Publisher#openDelta(long,long)}, or + * {@link Publisher#openReverseDelta(long,long)} on this publisher. + * + * @param blob the blob to publish + */ public void publish(HollowProducer.Blob blob); } @@ -345,7 +391,7 @@ public static interface Announcer { * * @author Tim Taylor {@literal} */ - public static final class Transition { + private static final class Transition { private final long fromVersion; private final long toVersion; @@ -361,7 +407,7 @@ public static final class Transition { * * @see Double Snapshot */ - public Transition() { + private Transition() { this(Long.MIN_VALUE, Long.MIN_VALUE); } @@ -379,7 +425,7 @@ public Transition() { * * @see Double Snapshot */ - public Transition(long toVersion) { + private Transition(long toVersion) { this(Long.MIN_VALUE, toVersion); } @@ -387,7 +433,7 @@ public Transition(long toVersion) { * Creates a transition fully representing a transition within the delta chain, a.k.a. a delta, between * {@code fromVersion} and {@code toVersion}. */ - public Transition(long fromVersion, long toVersion) { + private Transition(long fromVersion, long toVersion) { this.fromVersion = fromVersion; this.toVersion = toVersion; } @@ -407,42 +453,18 @@ public Transition(long fromVersion, long toVersion) { * @return a new state transition with its {@code fromVersion} and {@code toVersion} assigned our {@code toVersion} and * the specified {@code nextVersion} respectively */ - public Transition advance(long nextVersion) { + private Transition advance(long nextVersion) { return new Transition(toVersion, nextVersion); } - /** - * Returns a new transition with versions swapped. Only valid on deltas. - - * 

-         * 
-         * [13,45].reverse() == [45,13]
-         * 
-         *
- - * @return - * - * @throws IllegalStateException if this transition isn't a delta - */ - public Transition reverse() { - if(isDiscontinous() || isSnapshot()) throw new IllegalStateException("must be a delta"); - return new Transition(this.toVersion, this.fromVersion); - } - - public long getFromVersion() { - return fromVersion; - } - public long getToVersion() { - return toVersion; - } /** * Determines whether this transition represents a new or broken delta chain. * * @return true if this has neither a {@code fromVersion} nor a {@code toVersion}; false otherwise. */ - public boolean isDiscontinous() { + private boolean isDiscontinous() { return fromVersion == Long.MIN_VALUE && toVersion == Long.MIN_VALUE; } @@ -451,19 +473,19 @@ public boolean isDiscontinous() { * * @return true if this has a {@code fromVersion} and {@code toVersion}; false otherwise */ - public boolean isDelta() { + private boolean isDelta() { return fromVersion != Long.MIN_VALUE && toVersion != Long.MIN_VALUE; } - public boolean isForwardDelta() { + private boolean isForwardDelta() { return isDelta() && fromVersion < toVersion; } - public boolean isReverseDelta() { + private boolean isReverseDelta() { return isDelta() && fromVersion > toVersion; } - public boolean isSnapshot() { + private boolean isSnapshot() { return !isDiscontinous() && !isDelta(); } @@ -504,10 +526,10 @@ private HollowConsumer.ReadState readSnapshot(Transition transition) { final HollowConsumer.ReadState readState; HollowReadStateEngine readEngine = new HollowReadStateEngine(); HollowBlobReader reader = new HollowBlobReader(readEngine, new HollowBlobHeaderReader()); - Blob snapshot = publisher.openSnapshot(transition); + Blob snapshot = publisher.openSnapshot(transition.toVersion); try { reader.readSnapshot(snapshot.getInputStream()); - readState = new ReadStateImpl(transition.getToVersion(), readEngine); + readState = new ReadStateImpl(transition.toVersion, readEngine); } catch(IOException ex) { throw new RuntimeException(ex); } finally { diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/WriteStateImpl.java b/hollow/src/main/java/com/netflix/hollow/api/producer/WriteStateImpl.java index 21d56b317b..e5e05b5f5c 100644 --- a/hollow/src/main/java/com/netflix/hollow/api/producer/WriteStateImpl.java +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/WriteStateImpl.java @@ -26,13 +26,12 @@ * @author Tim Taylor {@literal} */ final class WriteStateImpl implements HollowProducer.WriteState { - private final HollowObjectMapper objectMapper; - private final HollowProducer.Transition transition; + private final long version; - protected WriteStateImpl(HollowObjectMapper objectMapper, HollowProducer.Transition transition) { + protected WriteStateImpl(HollowObjectMapper objectMapper, long version) { this.objectMapper = objectMapper; - this.transition = transition; + this.version = version; } @Override @@ -52,7 +51,6 @@ public HollowWriteStateEngine getStateEngine() { @Override public long getVersion() { - return transition.getToVersion(); + return version; } - } diff --git a/hollow/src/test/java/com/netflix/hollow/api/producer/HollowProducerTransitionTest.java b/hollow/src/test/java/com/netflix/hollow/api/producer/HollowProducerTransitionTest.java deleted file mode 100644 index 61e2db9a69..0000000000 --- a/hollow/src/test/java/com/netflix/hollow/api/producer/HollowProducerTransitionTest.java +++ /dev/null @@ -1,79 +0,0 @@ -package com.netflix.hollow.api.producer; - -import static org.hamcrest.CoreMatchers.equalTo; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import org.junit.Test; - -import com.netflix.hollow.api.producer.HollowProducer; - -import com.netflix.hollow.api.producer.HollowProducer.Transition; - -public final class HollowProducerTransitionTest { - - HollowProducer.Transition subject; - - @Test - public void snapshot() { - subject = new Transition(13L); - - assertThat(subject.getFromVersion(), equalTo(Long.MIN_VALUE)); - assertThat(subject.getToVersion(), equalTo(13L)); - - assertFalse(subject.isDiscontinous()); - assertTrue(subject.isSnapshot()); - assertFalse(subject.isDelta()); - } - - @Test - public void delta() { - subject = new HollowProducer.Transition(2L, 3L); - - assertThat(subject.getFromVersion(), equalTo(2L)); - assertThat(subject.getToVersion(), equalTo(3L)); - - assertFalse(subject.isDiscontinous()); - assertFalse(subject.isSnapshot()); - assertTrue(subject.isDelta()); - } - - @Test - public void advancing() { - subject = new Transition(6L); - - assertThat(subject.getFromVersion(), equalTo(Long.MIN_VALUE)); - assertThat(subject.getToVersion(), equalTo(6L)); - assertFalse(subject.isDiscontinous()); - assertFalse(subject.isDelta()); - - subject = subject.advance(7L); - - assertThat(subject.getFromVersion(), equalTo(6L)); - assertThat(subject.getToVersion(), equalTo(7L)); - assertFalse(subject.isDiscontinous()); - assertTrue(subject.isDelta()); - } - - @Test - public void reversing() { - subject = new Transition(21L, 22L); - - assertTrue(subject.isForwardDelta()); - assertFalse(subject.isReverseDelta()); - - subject = subject.reverse(); - - assertThat(subject.getFromVersion(), equalTo(22L)); - assertThat(subject.getToVersion(), equalTo(21L)); - assertFalse(subject.isForwardDelta()); - assertTrue(subject.isReverseDelta()); - - try { - new Transition(1L).reverse(); - fail("expected exception"); - } catch(IllegalStateException expected){} - } -} From 3d4d7a84644107d9ba31c645b4e332acee23145f Mon Sep 17 00:00:00 2001 From: Tim Taylor Date: Tue, 28 Feb 2017 03:03:24 -0800 Subject: [PATCH 19/30] keep current read state around for integrity check --- .../hollow/api/producer/HollowProducer.java | 296 ++++-------------- .../api/producer/HollowProducerListener.java | 6 +- .../hollow/api/producer/ListenerSupport.java | 4 +- .../hollow/api/producer/ReadStateHelper.java | 74 +++++ 4 files changed, 140 insertions(+), 240 deletions(-) create mode 100644 hollow/src/main/java/com/netflix/hollow/api/producer/ReadStateHelper.java diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java index 6d5e0e71ba..b0420b4de4 100644 --- a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java @@ -32,7 +32,6 @@ import com.netflix.hollow.api.producer.HollowProducerListener.RestoreStatus; import com.netflix.hollow.core.read.engine.HollowBlobHeaderReader; import com.netflix.hollow.core.read.engine.HollowBlobReader; -import com.netflix.hollow.core.read.engine.HollowReadStateEngine; import com.netflix.hollow.core.write.HollowBlobWriter; import com.netflix.hollow.core.write.HollowWriteStateEngine; import com.netflix.hollow.core.write.objectmapper.HollowObjectMapper; @@ -48,46 +47,33 @@ public class HollowProducer { public void validate(HollowConsumer.ReadState readState) {} }; - private final VersionMinter versionMinter; private final Publisher publisher; private final Validator validator; private final Announcer announcer; - // TODO: timt: use HollowConsumer API - private final HollowBlobRetriever blobRetriever; private final HollowWriteStateEngine writeEngine; private final HollowObjectMapper objectMapper; + private final VersionMinter versionMinter; private final ListenerSupport listeners; - - private Transition announced; - - public HollowProducer(HollowProducer.Publisher publisher, - HollowProducer.Announcer announcer, - HollowBlobRetriever blobRetriever) { - this(new VersionMinterWithCounter(), publisher, NO_VALIDATIONS, announcer, blobRetriever); - } + private ReadStateHelper readStates; public HollowProducer(HollowProducer.Publisher publisher, - HollowProducer.Validator validator, - HollowProducer.Announcer announcer, - HollowBlobRetriever blobRetriever) { - this(new VersionMinterWithCounter(), publisher, validator, announcer, blobRetriever); + HollowProducer.Announcer announcer) { + this(publisher, NO_VALIDATIONS, announcer); } - private HollowProducer(VersionMinter versionMinter, + public HollowProducer( Publisher publisher, Validator validator, - Announcer announcer, - HollowBlobRetriever blobRetriever) { - this.versionMinter = versionMinter; + Announcer announcer) { this.publisher = publisher; this.validator = validator; this.announcer = announcer; - this.blobRetriever = blobRetriever; - announced = new Transition(); writeEngine = new HollowWriteStateEngine(); objectMapper = new HollowObjectMapper(writeEngine); + versionMinter = new VersionMinterWithCounter(); listeners = new ListenerSupport(); + readStates = ReadStateHelper.newDeltaChain(); } public void initializeDataModel(Class...classes) { @@ -97,7 +83,7 @@ public void initializeDataModel(Class...classes) { listeners.fireProducerInit(currentTimeMillis() - start); } - public HollowProducer restore(HollowConsumer.AnnouncementRetriever announcementRetriever) { + public HollowProducer restore(HollowConsumer.AnnouncementRetriever announcementRetriever, HollowBlobRetriever blobRetriever) { long start = currentTimeMillis(); RestoreStatus status = RestoreStatus.unknownFailure(); long versionDesired = Long.MIN_VALUE; @@ -106,20 +92,23 @@ public HollowProducer restore(HollowConsumer.AnnouncementRetriever announcementR try { versionDesired = announcementRetriever.get(); listeners.fireProducerRestoreStart(versionDesired); - if(versionDesired != Long.MIN_VALUE) { - HollowConsumer.ReadState readState = toReadState(versionDesired); - versionReached = readState.getVersion(); + HollowClient client = new HollowClient(blobRetriever); + client.triggerRefreshTo(versionDesired); + versionReached = client.getCurrentVersionId(); if(versionReached == versionDesired) { - writeEngine.restoreFrom(readState.getStateEngine()); - announced = new Transition(versionReached); + readStates = ReadStateHelper.restored(new ReadStateImpl(client)); + writeEngine.restoreFrom(readStates.current().getStateEngine()); status = RestoreStatus.success(versionDesired, versionReached); + } else { + status = RestoreStatus.fail(versionDesired, versionReached, null); } } } catch(Throwable th) { status = RestoreStatus.fail(versionDesired, versionReached, th); + } finally { + listeners.fireProducerRestoreComplete(status, currentTimeMillis() - start); } - listeners.fireProducerRestoreComplete(status, currentTimeMillis() - start); return this; } @@ -127,24 +116,20 @@ public HollowProducer restore(HollowConsumer.AnnouncementRetriever announcementR * Each cycle produces a single data state. */ public void runCycle(Populator task) { - WriteState writeState = null; long start = currentTimeMillis(); ProducerStatus cycleStatus = ProducerStatus.unknownFailure(); - long mintedVersion = Long.MIN_VALUE; + WriteState writeState = null; try { - mintedVersion = versionMinter.mint(); - Transition transition = announced.advance(mintedVersion); - if(transition.isSnapshot()) listeners.fireNewDeltaChain(mintedVersion); - listeners.fireCycleStart(mintedVersion); - writeState = beginCycle(transition); + writeState = beginCycle(); task.populate(writeState); if(writeEngine.hasChangedSinceLastCycle()) { - publish(transition, writeState); - HollowConsumer.ReadState readState = checkIntegrity(transition, writeState); - validate(readState); - announce(readState); - announced = transition; - cycleStatus = ProducerStatus.success(readState); + publish(writeState); + ReadStateHelper candidate = readStates.roundtrip(writeState); + checkIntegrity(candidate); + validate(candidate.pending()); + announce(candidate.pending()); + readStates = candidate.commit(); + cycleStatus = ProducerStatus.success(readStates.current()); } else { writeEngine.resetToLastPrepareForNextCycle(); listeners.fireNoDelta(writeState.getVersion()); @@ -152,7 +137,7 @@ public void runCycle(Populator task) { } } catch(Throwable th) { writeEngine.resetToLastPrepareForNextCycle(); - cycleStatus = ProducerStatus.fail(mintedVersion, th); + cycleStatus = ProducerStatus.fail(writeState != null ? writeState.getVersion() : Long.MIN_VALUE, th); } finally { listeners.fireCycleComplete(cycleStatus, start); } @@ -166,30 +151,33 @@ public void removeListener(HollowProducerListener listener) { listeners.remove(listener); } - private WriteState beginCycle(Transition transition) { + private WriteState beginCycle() { + long toVersion = versionMinter.mint(); + if(!readStates.hasCurrent()) listeners.fireNewDeltaChain(toVersion); + listeners.fireCycleStart(toVersion); writeEngine.prepareForNextCycle(); - WriteState writeState = new WriteStateImpl(objectMapper, transition.toVersion); - return writeState; + return new WriteStateImpl(objectMapper, toVersion); } - private void publish(Transition transition, WriteState writeState) throws IOException { + private void publish(WriteState writeState) throws IOException { long start = listeners.firePublishStart(writeState.getVersion()); HollowBlobWriter writer = new HollowBlobWriter(writeEngine); - Blob snapshot = publisher.openSnapshot(transition.toVersion); + Blob snapshot = null; ProducerStatus status = ProducerStatus.unknownFailure(); try { + snapshot = publisher.openSnapshot(writeState.getVersion()); writer.writeSnapshot(snapshot.getOutputStream()); - if(transition.isDelta()) { - Blob delta = publisher.openDelta(transition.fromVersion, transition.toVersion); + if(readStates.hasCurrent()) { + Blob delta = publisher.openDelta(readStates.current().getVersion(), writeState.getVersion()); try { writer.writeDelta(delta.getOutputStream()); publisher.publish(delta); } finally { delta.close(); } - Blob reverseDelta = publisher.openReverseDelta(transition.fromVersion, transition.toVersion); + Blob reverseDelta = publisher.openReverseDelta(readStates.current().getVersion(), writeState.getVersion()); try { writer.writeReverseDelta(reverseDelta.getOutputStream()); publisher.publish(reverseDelta); @@ -211,52 +199,44 @@ private void publish(Transition transition, WriteState writeState) throws IOExce throw th; } finally { listeners.firePublishComplete(status, start); - snapshot.close(); + if(snapshot != null) snapshot.close(); } } /** - * Given + * Given these read states * - * 1. read state (S1) at the previous announced version - * 2. read state (S2) from the currently produced snapshot + * * S(cur) at the currently announced version + * * S(pnd) at the pending version * - * Ensure: + * Ensure that: * - * S1.apply(forward delta).checksum == S2.checksum - * S2.apply(reverse delta).checksum == S1.checksum + * S(cur).apply(forwardDelta).checksum == S(pnd).checksum + * S(pnd).apply(reverseDelta).checksum == S(cur).checksum * - * @param transition - * @param writeState - * @return + * @param readStates */ - private HollowConsumer.ReadState checkIntegrity(Transition transition, WriteState writeState) { - long start = listeners.fireIntegrityCheckStart(writeState); + private void checkIntegrity(ReadStateHelper readStates) throws Exception { + long start = listeners.fireIntegrityCheckStart(readStates.pendingVersion()); ProducerStatus status = ProducerStatus.unknownFailure(); + Blob snapshot = null; try { - final HollowConsumer.ReadState result; - - // TODO: timt: use HollowConsumer - if(transition.isDelta()) { - long desiredVersion = transition.fromVersion; - @SuppressWarnings("unused") - HollowConsumer.ReadState fromReadState = toReadState(desiredVersion); - // FIXME: timt: do the integrity checks, leaving (S2) assigned to `toReadState` - - result = readSnapshot(transition); - } else if(transition.isSnapshot()) { - result = readSnapshot(transition); - } else { - throw new IllegalStateException("no blobs to check"); + snapshot = publisher.openSnapshot(readStates.pendingVersion()); + HollowBlobReader reader = new HollowBlobReader(readStates.pending().getStateEngine(), new HollowBlobHeaderReader()); + reader.readSnapshot(snapshot.getInputStream()); + + if(readStates.hasCurrent()) { + // FIXME: timt: do delta integrity checks; we only compare checksums for + // schemas that are common and unchanged between S(curr) and S(prev) } - status = ProducerStatus.success(result); - return result; + status = ProducerStatus.success(readStates.pending()); } catch(Throwable th) { - status = ProducerStatus.fail(writeState.getVersion(), th); + status = ProducerStatus.fail(readStates.pendingVersion(), th); throw th; } finally { listeners.fireIntegrityCheckComplete(status, start); + if(snapshot != null) snapshot.close(); } } @@ -289,7 +269,7 @@ private void announce(HollowConsumer.ReadState readState) { } } - public static interface VersionMinter { + static interface VersionMinter { /** * Create a new state version.

* @@ -385,158 +365,4 @@ public static interface Validator { public static interface Announcer { public void announce(long stateVersion); } - - /** - * Immutable class representing a single point along the delta chain. - * - * @author Tim Taylor {@literal} - */ - private static final class Transition { - - private final long fromVersion; - private final long toVersion; - - /** - * Creates a null transition. - * - * A producer would use this to avoid null checks when beginning a new delta chain; calling - * {@link #advance(long)} will return a transition representing the first snapshot in - * a chain.

- * - * Consumers cannot initialize their read state from this transition.

- * - * @see Double Snapshot - */ - private Transition() { - this(Long.MIN_VALUE, Long.MIN_VALUE); - } - - /** - * Creates a transition capable of being used to restore from a delta chain at - * the specified version, a.k.a. a snapshot.

- * - * Consumers can initialize their read state from a snapshot corresponding to - * this transition; an already initialized consumer can only utilize - * this by performing a double snapshot.

- * - * A producer would use this transition to restore from a previous announced state in order - * to resume producing on that delta chain by calling {@link #advance(long)} when ready to - * produce the next state. - * - * @see Double Snapshot - */ - private Transition(long toVersion) { - this(Long.MIN_VALUE, toVersion); - } - - /** - * Creates a transition fully representing a transition within the delta chain, a.k.a. a delta, between - * {@code fromVersion} and {@code toVersion}. - */ - private Transition(long fromVersion, long toVersion) { - this.fromVersion = fromVersion; - this.toVersion = toVersion; - } - - /** - * Returns a new transition representing the transition from this state's {@code toVersion} to the specified version; - * equivalent to calling {@code new StateTransition(this.toVersion, nextVersion)}. - * - * 

-         * 
-         * [13,45].advance(72) == [45,72]
-         * 
-         *
- * - * @param nextVersion the next version to transition to - * - * @return a new state transition with its {@code fromVersion} and {@code toVersion} assigned our {@code toVersion} and - * the specified {@code nextVersion} respectively - */ - private Transition advance(long nextVersion) { - return new Transition(toVersion, nextVersion); - } - - - - /** - * Determines whether this transition represents a new or broken delta chain. - * - * @return true if this has neither a {@code fromVersion} nor a {@code toVersion}; false otherwise. - */ - private boolean isDiscontinous() { - return fromVersion == Long.MIN_VALUE && toVersion == Long.MIN_VALUE; - } - - /** - * Determines whether this state represents a delta, e.g. a transition between two state versions. - * - * @return true if this has a {@code fromVersion} and {@code toVersion}; false otherwise - */ - private boolean isDelta() { - return fromVersion != Long.MIN_VALUE && toVersion != Long.MIN_VALUE; - } - - private boolean isForwardDelta() { - return isDelta() && fromVersion < toVersion; - } - - private boolean isReverseDelta() { - return isDelta() && fromVersion > toVersion; - } - - private boolean isSnapshot() { - return !isDiscontinous() && !isDelta(); - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - if(isDiscontinous()) { - sb.append("new/broken delta chain"); - } else if(isDelta()) { - if(isReverseDelta()) sb.append("reverse"); - sb.append("delta ["); - sb.append(fromVersion); - if(isForwardDelta()) sb.append(" -> "); - else sb.append(" <- "); - sb.append(toVersion); - sb.append("]"); - } else { - sb.append("snapshot ["); - sb.append(toVersion); - sb.append("]"); - } - return sb.toString(); - } - - } - - ///// TODO: timt: move to HollowConsumer API //// - private HollowConsumer.ReadState toReadState(long desiredVersion) { - HollowClient client = new HollowClient(blobRetriever); - client.triggerRefreshTo(desiredVersion); - long actualVersion = client.getCurrentVersionId(); - if(desiredVersion != actualVersion) throw new IllegalStateException(String.format("desiredVersion=%d actualVersion=%d", desiredVersion, actualVersion)); - HollowConsumer.ReadState readState = new ReadStateImpl(client); - return readState; - } - - private HollowConsumer.ReadState readSnapshot(Transition transition) { - final HollowConsumer.ReadState readState; - HollowReadStateEngine readEngine = new HollowReadStateEngine(); - HollowBlobReader reader = new HollowBlobReader(readEngine, new HollowBlobHeaderReader()); - Blob snapshot = publisher.openSnapshot(transition.toVersion); - try { - reader.readSnapshot(snapshot.getInputStream()); - readState = new ReadStateImpl(transition.toVersion, readEngine); - } catch(IOException ex) { - throw new RuntimeException(ex); - } finally { - snapshot.close(); - } - return readState; - } - ///// END TODO //// - } diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducerListener.java b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducerListener.java index 56f199c722..9f01b5e095 100644 --- a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducerListener.java +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducerListener.java @@ -58,7 +58,7 @@ public interface HollowProducerListener extends EventListener { /** * Indicates that the next state produced will begin a new delta chain. * Thiis will be called prior to the next state being produced either if - * {@link HollowProducer#restore(com.netflix.hollow.api.consumer.HollowConsumer.AnnouncementRetriever)} + * {@link HollowProducer#restore(com.netflix.hollow.api.consumer.HollowConsumer.AnnouncementRetriever, com.netflix.hollow.api.client.HollowBlobRetriever)} * hasn't been called or the restore failed. * * @param version the version of the state that will become the first of a new delta chain @@ -278,7 +278,7 @@ private RestoreStatus(Status status, long versionDesired, long versionReached, T /** * The version desired to restore to when calling - * {@link HollowProducer#restore(com.netflix.hollow.api.consumer.HollowConsumer.AnnouncementRetriever)} + * {@link HollowProducer#restore(com.netflix.hollow.api.consumer.HollowConsumer.AnnouncementRetriever, com.netflix.hollow.api.client.HollowBlobRetriever)} * * @return the latest announced version or {@code Long.MIN_VALUE} if latest announced version couldn't be * retrieved @@ -289,7 +289,7 @@ public long getDesiredVersion() { /** * The version reached when restoring. - * When {@link HollowProducer#restore(com.netflix.hollow.api.consumer.HollowConsumer.AnnouncementRetriever)} + * When {@link HollowProducer#restore(com.netflix.hollow.api.consumer.HollowConsumer.AnnouncementRetriever, com.netflix.hollow.api.client.HollowBlobRetriever)} * succeeds then {@code versionDesired == versionReached} is always true. Can be {@code Long.MIN_VALUE} * indicating restore failed to reach any state, or the version of an intermediate state reached. * diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/ListenerSupport.java b/hollow/src/main/java/com/netflix/hollow/api/producer/ListenerSupport.java index 7de58768d0..540c751176 100644 --- a/hollow/src/main/java/com/netflix/hollow/api/producer/ListenerSupport.java +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/ListenerSupport.java @@ -89,9 +89,9 @@ void firePublishComplete(ProducerStatus publishStatus, long startMillis) { for(final HollowProducerListener l : listeners) l.onPublishComplete(publishStatus, elapsedMillis, MILLISECONDS); } - long fireIntegrityCheckStart(HollowProducer.WriteState writeState) { + long fireIntegrityCheckStart(long version) { long start = currentTimeMillis(); - for(final HollowProducerListener l : listeners) l.onIntegrityCheckStart(writeState.getVersion()); + for(final HollowProducerListener l : listeners) l.onIntegrityCheckStart(version); return start; } diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/ReadStateHelper.java b/hollow/src/main/java/com/netflix/hollow/api/producer/ReadStateHelper.java new file mode 100644 index 0000000000..aca468e767 --- /dev/null +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/ReadStateHelper.java @@ -0,0 +1,74 @@ +/* + * + * Copyright 2017 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.hollow.api.producer; + +import com.netflix.hollow.api.consumer.HollowConsumer; +import com.netflix.hollow.api.consumer.ReadStateImpl; +import com.netflix.hollow.core.read.engine.HollowReadStateEngine; +/** + * Beta API subject to change. + * + * Helper for {@link HollowProducer} to manage current and pending read states and the + * transition that occurs within a cycle. + * + * @author Tim Taylor {@literal} + * + */ +final class ReadStateHelper { + static ReadStateHelper newDeltaChain() { + return new ReadStateHelper(null, null); + } + + static ReadStateHelper restored(HollowConsumer.ReadState state) { + return new ReadStateHelper(state, null); + } + + private final HollowConsumer.ReadState current; + private final HollowConsumer.ReadState pending; + + private ReadStateHelper(HollowConsumer.ReadState current, HollowConsumer.ReadState pending) { + this.current = current; + this.pending = pending; + } + + ReadStateHelper roundtrip(HollowProducer.WriteState writeState) { + if(pending != null) throw new IllegalStateException(); + return new ReadStateHelper(this.current, new ReadStateImpl(writeState.getVersion(), new HollowReadStateEngine())); + } + + ReadStateHelper commit() { + if(pending == null) throw new IllegalStateException(); + return new ReadStateHelper(this.pending, null); + } + + HollowConsumer.ReadState current() { + return current; + } + + boolean hasCurrent() { + return current != null; + } + + HollowConsumer.ReadState pending() { + return pending; + } + + long pendingVersion() { + return pending != null ? pending.getVersion() : Long.MIN_VALUE; + } +} From 8bb3610d19b81a3612d6330c6fd59c733868c594 Mon Sep 17 00:00:00 2001 From: Tim Taylor Date: Tue, 28 Feb 2017 05:13:07 -0800 Subject: [PATCH 20/30] producer.restore simplified --- .../hollow/api/consumer/HollowConsumer.java | 30 +++++++++++++++++++ .../hollow/api/consumer/ReadStateImpl.java | 5 +--- .../hollow/api/producer/HollowProducer.java | 23 ++++++-------- .../api/producer/HollowProducerListener.java | 6 ++-- 4 files changed, 43 insertions(+), 21 deletions(-) diff --git a/hollow/src/main/java/com/netflix/hollow/api/consumer/HollowConsumer.java b/hollow/src/main/java/com/netflix/hollow/api/consumer/HollowConsumer.java index 6bdd001d83..d1cc4696a0 100644 --- a/hollow/src/main/java/com/netflix/hollow/api/consumer/HollowConsumer.java +++ b/hollow/src/main/java/com/netflix/hollow/api/consumer/HollowConsumer.java @@ -17,6 +17,9 @@ */ package com.netflix.hollow.api.consumer; +import com.netflix.hollow.api.client.HollowAnnouncementWatcher; +import com.netflix.hollow.api.client.HollowBlobRetriever; +import com.netflix.hollow.api.client.HollowClient; import com.netflix.hollow.core.read.engine.HollowReadStateEngine; /** @@ -44,4 +47,31 @@ public long get() { long get(); } + public static interface StateRetriever { + ReadState retrieveLatestAnnounced(); + long latestAnnouncedVersion(); + } + + // TODO: timt: don't use HollowBlobRetriever or HollowClient; this is temporary bridge code + public static class BlobStoreStateRetriever implements StateRetriever { + private final HollowAnnouncementWatcher announcementWatcher; + private final HollowBlobRetriever blobRetriever; + + public BlobStoreStateRetriever(HollowAnnouncementWatcher announcementWatcher, HollowBlobRetriever blobRetriever) { + this.announcementWatcher = announcementWatcher; + this.blobRetriever = blobRetriever; + } + + @Override + public long latestAnnouncedVersion() { + return announcementWatcher.getLatestVersion(); + } + + @Override + public ReadState retrieveLatestAnnounced() { + HollowClient client = new HollowClient(blobRetriever); + client.triggerRefreshTo(latestAnnouncedVersion()); + return new ReadStateImpl(client.getCurrentVersionId(), client.getStateEngine()); + } + } } diff --git a/hollow/src/main/java/com/netflix/hollow/api/consumer/ReadStateImpl.java b/hollow/src/main/java/com/netflix/hollow/api/consumer/ReadStateImpl.java index 673feb28bf..9da69d1f47 100644 --- a/hollow/src/main/java/com/netflix/hollow/api/consumer/ReadStateImpl.java +++ b/hollow/src/main/java/com/netflix/hollow/api/consumer/ReadStateImpl.java @@ -29,10 +29,7 @@ public class ReadStateImpl implements HollowConsumer.ReadState { private final long version; private final HollowReadStateEngine stateEngine; - // TODO: timt: temporary until we stop using HollowClient in HollowProducer - public ReadStateImpl(HollowClient client) { - this(client.getCurrentVersionId(), client.getStateEngine()); - } + // TODO: timt: should be package protected public ReadStateImpl(long version, HollowReadStateEngine stateEngine) { diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java index b0420b4de4..484462ca46 100644 --- a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java @@ -24,10 +24,7 @@ import java.io.InputStream; import java.io.OutputStream; -import com.netflix.hollow.api.client.HollowBlobRetriever; -import com.netflix.hollow.api.client.HollowClient; import com.netflix.hollow.api.consumer.HollowConsumer; -import com.netflix.hollow.api.consumer.ReadStateImpl; import com.netflix.hollow.api.producer.HollowProducerListener.ProducerStatus; import com.netflix.hollow.api.producer.HollowProducerListener.RestoreStatus; import com.netflix.hollow.core.read.engine.HollowBlobHeaderReader; @@ -83,29 +80,27 @@ public void initializeDataModel(Class...classes) { listeners.fireProducerInit(currentTimeMillis() - start); } - public HollowProducer restore(HollowConsumer.AnnouncementRetriever announcementRetriever, HollowBlobRetriever blobRetriever) { + public HollowProducer restore(HollowConsumer.StateRetriever retriever) { long start = currentTimeMillis(); RestoreStatus status = RestoreStatus.unknownFailure(); long versionDesired = Long.MIN_VALUE; - long versionReached = Long.MIN_VALUE; + HollowConsumer.ReadState readState = null; try { - versionDesired = announcementRetriever.get(); + versionDesired = retriever.latestAnnouncedVersion(); listeners.fireProducerRestoreStart(versionDesired); if(versionDesired != Long.MIN_VALUE) { - HollowClient client = new HollowClient(blobRetriever); - client.triggerRefreshTo(versionDesired); - versionReached = client.getCurrentVersionId(); - if(versionReached == versionDesired) { - readStates = ReadStateHelper.restored(new ReadStateImpl(client)); + readState = retriever.retrieveLatestAnnounced(); + if(readState.getVersion() == versionDesired) { + readStates = ReadStateHelper.restored(readState); writeEngine.restoreFrom(readStates.current().getStateEngine()); - status = RestoreStatus.success(versionDesired, versionReached); + status = RestoreStatus.success(versionDesired, readState.getVersion()); } else { - status = RestoreStatus.fail(versionDesired, versionReached, null); + status = RestoreStatus.fail(versionDesired, readState.getVersion(), null); } } } catch(Throwable th) { - status = RestoreStatus.fail(versionDesired, versionReached, th); + status = RestoreStatus.fail(versionDesired, readState != null ? readState.getVersion() : Long.MIN_VALUE, th); } finally { listeners.fireProducerRestoreComplete(status, currentTimeMillis() - start); } diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducerListener.java b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducerListener.java index 9f01b5e095..7337777ba5 100644 --- a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducerListener.java +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducerListener.java @@ -58,7 +58,7 @@ public interface HollowProducerListener extends EventListener { /** * Indicates that the next state produced will begin a new delta chain. * Thiis will be called prior to the next state being produced either if - * {@link HollowProducer#restore(com.netflix.hollow.api.consumer.HollowConsumer.AnnouncementRetriever, com.netflix.hollow.api.client.HollowBlobRetriever)} + * {@link HollowProducer#restore(com.netflix.hollow.api.consumer.HollowConsumer.StateRetriever)} * hasn't been called or the restore failed. * * @param version the version of the state that will become the first of a new delta chain @@ -278,7 +278,7 @@ private RestoreStatus(Status status, long versionDesired, long versionReached, T /** * The version desired to restore to when calling - * {@link HollowProducer#restore(com.netflix.hollow.api.consumer.HollowConsumer.AnnouncementRetriever, com.netflix.hollow.api.client.HollowBlobRetriever)} + * {@link HollowProducer#restore(com.netflix.hollow.api.consumer.HollowConsumer.StateRetriever)} * * @return the latest announced version or {@code Long.MIN_VALUE} if latest announced version couldn't be * retrieved @@ -289,7 +289,7 @@ public long getDesiredVersion() { /** * The version reached when restoring. - * When {@link HollowProducer#restore(com.netflix.hollow.api.consumer.HollowConsumer.AnnouncementRetriever, com.netflix.hollow.api.client.HollowBlobRetriever)} + * When {@link HollowProducer#restore(com.netflix.hollow.api.consumer.HollowConsumer.StateRetriever)} * succeeds then {@code versionDesired == versionReached} is always true. Can be {@code Long.MIN_VALUE} * indicating restore failed to reach any state, or the version of an intermediate state reached. * From fa68738d1fb9effec5f26731f4b7ba86c4247446 Mon Sep 17 00:00:00 2001 From: Tim Taylor Date: Tue, 28 Feb 2017 05:21:53 -0800 Subject: [PATCH 21/30] always fail cycle if snapshot publish fails --- .../com/netflix/hollow/api/producer/HollowProducer.java | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java index 484462ca46..cc842e67c6 100644 --- a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java @@ -181,13 +181,8 @@ private void publish(WriteState writeState) throws IOException { } } - /// it's ok to fail to publish a snapshot, as long as you don't miss too many in a row. - /// you can add a timeout or even do this in a separate thread. - try { - publisher.publish(snapshot); - } catch(Throwable ignored) { - ignored.printStackTrace(); // TODO: timt: log and notify listerners - } + // TODO: timt: allow some failed snapshot publishes + publisher.publish(snapshot); status = ProducerStatus.success(writeState.getVersion()); } catch(Throwable th) { status = ProducerStatus.fail(writeState.getVersion(), th); From df9e17195cf7b83e1bd385beea85e12c72810d04 Mon Sep 17 00:00:00 2001 From: Tim Taylor Date: Wed, 1 Mar 2017 17:10:40 -0800 Subject: [PATCH 22/30] pare back HollowConsumer API to just ReadState --- .../hollow/api/consumer/HollowConsumer.java | 47 ++----------------- .../hollow/api/consumer/ReadStateImpl.java | 3 +- .../hollow/api/producer/HollowProducer.java | 12 +++-- .../hollow/api/producer/ReadStateHelper.java | 6 ++- 4 files changed, 16 insertions(+), 52 deletions(-) diff --git a/hollow/src/main/java/com/netflix/hollow/api/consumer/HollowConsumer.java b/hollow/src/main/java/com/netflix/hollow/api/consumer/HollowConsumer.java index d1cc4696a0..017ff6c26d 100644 --- a/hollow/src/main/java/com/netflix/hollow/api/consumer/HollowConsumer.java +++ b/hollow/src/main/java/com/netflix/hollow/api/consumer/HollowConsumer.java @@ -17,9 +17,6 @@ */ package com.netflix.hollow.api.consumer; -import com.netflix.hollow.api.client.HollowAnnouncementWatcher; -import com.netflix.hollow.api.client.HollowBlobRetriever; -import com.netflix.hollow.api.client.HollowClient; import com.netflix.hollow.core.read.engine.HollowReadStateEngine; /** @@ -28,50 +25,12 @@ * @author Tim Taylor {@literal} */ public class HollowConsumer { + public static ReadState newReadState(long version, HollowReadStateEngine stateEngine) { + return new ReadStateImpl(version, stateEngine); + } - // TODO: timt: is this needed, or do we just use a HollowReadStateEngine in place of this? Created for now - // to have symmetry with HollowProducer.WriteState public static interface ReadState { long getVersion(); HollowReadStateEngine getStateEngine(); } - - public static interface AnnouncementRetriever { - static final AnnouncementRetriever NO_ANNOUNCEMENTS = new AnnouncementRetriever(){ - @Override - public long get() { - return Long.MIN_VALUE; - } - }; - - long get(); - } - - public static interface StateRetriever { - ReadState retrieveLatestAnnounced(); - long latestAnnouncedVersion(); - } - - // TODO: timt: don't use HollowBlobRetriever or HollowClient; this is temporary bridge code - public static class BlobStoreStateRetriever implements StateRetriever { - private final HollowAnnouncementWatcher announcementWatcher; - private final HollowBlobRetriever blobRetriever; - - public BlobStoreStateRetriever(HollowAnnouncementWatcher announcementWatcher, HollowBlobRetriever blobRetriever) { - this.announcementWatcher = announcementWatcher; - this.blobRetriever = blobRetriever; - } - - @Override - public long latestAnnouncedVersion() { - return announcementWatcher.getLatestVersion(); - } - - @Override - public ReadState retrieveLatestAnnounced() { - HollowClient client = new HollowClient(blobRetriever); - client.triggerRefreshTo(latestAnnouncedVersion()); - return new ReadStateImpl(client.getCurrentVersionId(), client.getStateEngine()); - } - } } diff --git a/hollow/src/main/java/com/netflix/hollow/api/consumer/ReadStateImpl.java b/hollow/src/main/java/com/netflix/hollow/api/consumer/ReadStateImpl.java index 9da69d1f47..b3e172ad4c 100644 --- a/hollow/src/main/java/com/netflix/hollow/api/consumer/ReadStateImpl.java +++ b/hollow/src/main/java/com/netflix/hollow/api/consumer/ReadStateImpl.java @@ -19,13 +19,12 @@ import com.netflix.hollow.core.read.engine.HollowReadStateEngine; -// TODO: timt: should be package protected /** * Alpha API subject to change. * * @author Tim Taylor {@literal} */ -public class ReadStateImpl implements HollowConsumer.ReadState { +class ReadStateImpl implements HollowConsumer.ReadState { private final long version; private final HollowReadStateEngine stateEngine; diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java index cc842e67c6..010908cde7 100644 --- a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java @@ -17,6 +17,7 @@ */ package com.netflix.hollow.api.producer; +import static com.netflix.hollow.api.consumer.HollowConsumer.newReadState; import static java.lang.System.currentTimeMillis; import java.io.Closeable; @@ -24,6 +25,8 @@ import java.io.InputStream; import java.io.OutputStream; +import com.netflix.hollow.api.client.HollowBlobRetriever; +import com.netflix.hollow.api.client.HollowClient; import com.netflix.hollow.api.consumer.HollowConsumer; import com.netflix.hollow.api.producer.HollowProducerListener.ProducerStatus; import com.netflix.hollow.api.producer.HollowProducerListener.RestoreStatus; @@ -80,17 +83,18 @@ public void initializeDataModel(Class...classes) { listeners.fireProducerInit(currentTimeMillis() - start); } - public HollowProducer restore(HollowConsumer.StateRetriever retriever) { + public HollowProducer restore(long versionDesired, HollowBlobRetriever blobRetriever) { long start = currentTimeMillis(); RestoreStatus status = RestoreStatus.unknownFailure(); - long versionDesired = Long.MIN_VALUE; HollowConsumer.ReadState readState = null; try { - versionDesired = retriever.latestAnnouncedVersion(); listeners.fireProducerRestoreStart(versionDesired); if(versionDesired != Long.MIN_VALUE) { - readState = retriever.retrieveLatestAnnounced(); + + HollowClient client = new HollowClient(blobRetriever); + client.triggerRefreshTo(versionDesired); + readState = newReadState(client.getCurrentVersionId(), client.getStateEngine()); if(readState.getVersion() == versionDesired) { readStates = ReadStateHelper.restored(readState); writeEngine.restoreFrom(readStates.current().getStateEngine()); diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/ReadStateHelper.java b/hollow/src/main/java/com/netflix/hollow/api/producer/ReadStateHelper.java index aca468e767..0f7a156e37 100644 --- a/hollow/src/main/java/com/netflix/hollow/api/producer/ReadStateHelper.java +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/ReadStateHelper.java @@ -17,9 +17,11 @@ */ package com.netflix.hollow.api.producer; +import static com.netflix.hollow.api.consumer.HollowConsumer.newReadState; + import com.netflix.hollow.api.consumer.HollowConsumer; -import com.netflix.hollow.api.consumer.ReadStateImpl; import com.netflix.hollow.core.read.engine.HollowReadStateEngine; + /** * Beta API subject to change. * @@ -48,7 +50,7 @@ private ReadStateHelper(HollowConsumer.ReadState current, HollowConsumer.ReadSta ReadStateHelper roundtrip(HollowProducer.WriteState writeState) { if(pending != null) throw new IllegalStateException(); - return new ReadStateHelper(this.current, new ReadStateImpl(writeState.getVersion(), new HollowReadStateEngine())); + return new ReadStateHelper(this.current, newReadState(writeState.getVersion(), new HollowReadStateEngine())); } ReadStateHelper commit() { From 8d3dfe56fff83483e24cccd37df418456714b194 Mon Sep 17 00:00:00 2001 From: Tim Taylor Date: Wed, 1 Mar 2017 21:39:16 -0800 Subject: [PATCH 23/30] AbstractHollowPublisher for publishers that stage artifacts on local FS --- .../hollow/api/producer/HollowProducer.java | 19 ++- .../producer/fs/AbstractHollowPublisher.java | 157 ++++++++++++++++++ 2 files changed, 175 insertions(+), 1 deletion(-) create mode 100644 hollow/src/main/java/com/netflix/hollow/api/producer/fs/AbstractHollowPublisher.java diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java index 010908cde7..a05c6733e5 100644 --- a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java @@ -345,11 +345,28 @@ public static interface Publisher { public static interface Blob extends Closeable { OutputStream getOutputStream(); - // TODO: timt: belongs in HollowConsumer.Blob InputStream getInputStream(); @Override void close(); + + long getFromVersion(); + + long getToVersion(); + + Type getType(); + + public static enum Type { + SNAPSHOT("snapshot"), + DELTA("delta"), + REVERSE_DELTA("reversedelta"); + + public final String prefix; + + Type(String prefix) { + this.prefix = prefix; + } + } } public static interface Validator { diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/fs/AbstractHollowPublisher.java b/hollow/src/main/java/com/netflix/hollow/api/producer/fs/AbstractHollowPublisher.java new file mode 100644 index 0000000000..26c50d19a8 --- /dev/null +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/fs/AbstractHollowPublisher.java @@ -0,0 +1,157 @@ +/* + * + * Copyright 2017 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.hollow.api.producer.fs; + +import static com.netflix.hollow.api.producer.HollowProducer.Blob.Type.*; +import static java.nio.file.Files.createDirectories; +import static java.nio.file.Files.deleteIfExists; +import static java.nio.file.Files.newInputStream; +import static java.nio.file.Files.newOutputStream; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Path; +import java.nio.file.Paths; + +import com.netflix.hollow.api.producer.HollowProducer; +import com.netflix.hollow.api.producer.HollowProducer.Blob; +import com.netflix.hollow.api.producer.HollowProducer.Publisher; +import com.netflix.hollow.api.producer.HollowProducer.Blob.Type; + +public abstract class AbstractHollowPublisher implements HollowProducer.Publisher { + protected final String namespace; + protected final Path scratchPath; + protected final Path stagingPath; + + protected AbstractHollowPublisher(String namespace) { + this(namespace, Paths.get(System.getProperty("java.io.tmpdir"))); + } + + protected AbstractHollowPublisher(String namespace, Path scratchPath) { + this(namespace, scratchPath, scratchPath.resolve(Paths.get(namespace, "staged"))); + } + + protected AbstractHollowPublisher(String namespace, Path scratchPath, Path stagingPath) { + this.namespace = namespace; + this.scratchPath = scratchPath; + this.stagingPath = stagingPath; + } + + @Override + public Blob openSnapshot(long version) { + return new StagedBlob(SNAPSHOT, namespace, stagingPath, Long.MIN_VALUE, version); + } + + @Override + public Blob openDelta(long fromVersion, long toVersion) { + return new StagedBlob(DELTA, namespace, stagingPath, fromVersion, toVersion); + } + + @Override + public Blob openReverseDelta(long fromVersion, long toVersion) { + return new StagedBlob(REVERSE_DELTA, namespace, stagingPath, fromVersion, toVersion); + } + + public static class StagedBlob implements Blob { + protected final Blob.Type type; + protected final long fromVersion; + protected final long toVersion; + protected final Path stagedArtifactPath; + private BufferedOutputStream out; + private BufferedInputStream in; + + protected StagedBlob(Blob.Type type, String namespace, Path stagingPath, long fromVersion, long toVersion) { + this.type = type; + this.fromVersion = fromVersion; + this.toVersion = toVersion; + + final Path p; + switch(type) { + case SNAPSHOT: + p = stagingPath.resolve(String.format("%s-%s-%d", namespace, type.prefix, toVersion)); + break; + case DELTA: + p = stagingPath.resolve(String.format("%s-%s-%d-%d", namespace, type.prefix, fromVersion, toVersion)); + break; + case REVERSE_DELTA: + p = stagingPath.resolve(String.format("%s-%s-%d-%d", namespace, type.prefix, toVersion, fromVersion)); + break; + default: + throw new IllegalStateException("unknown blob type, type=" + type); + } + this.stagedArtifactPath = p; + } + + @Override + public OutputStream getOutputStream() { + try { + createDirectories(stagedArtifactPath.getParent()); + out = new BufferedOutputStream(newOutputStream(stagedArtifactPath)); + return out; + } catch(IOException ex) { + throw new RuntimeException(ex); + } + } + + @Override + public InputStream getInputStream() { + try { + in = new BufferedInputStream(newInputStream(stagedArtifactPath)); + return in; + } catch(IOException ex) { + throw new RuntimeException(ex); + } + } + + @Override + public void close() { + if(out != null) { + try { + if(out != null) out.close(); + out = null; + if(in != null) in.close(); + in = null; + // deleteIfExists(stagedPath); + } catch(IOException ex) { + throw new RuntimeException(ex); + } + } + } + + public Blob.Type getType() { + return type; + } + + public Path getStagedArtifactPath() { + return stagedArtifactPath; + } + + @Override + public long getFromVersion() { + return fromVersion; + } + + @Override + public long getToVersion() { + return toVersion; + } + } +} From 4cce4f85b163c78b8a9fd5bc213e0f69066f45de Mon Sep 17 00:00:00 2001 From: Tim Taylor Date: Wed, 1 Mar 2017 21:40:01 -0800 Subject: [PATCH 24/30] provide a default publisher that publishes & announces to local FS --- .../fs/HollowFilesystemAnnouncer.java | 69 +++++++++++++++++ .../fs/HollowFilesystemPublisher.java | 76 +++++++++++++++++++ 2 files changed, 145 insertions(+) create mode 100644 hollow/src/main/java/com/netflix/hollow/api/producer/fs/HollowFilesystemAnnouncer.java create mode 100644 hollow/src/main/java/com/netflix/hollow/api/producer/fs/HollowFilesystemPublisher.java diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/fs/HollowFilesystemAnnouncer.java b/hollow/src/main/java/com/netflix/hollow/api/producer/fs/HollowFilesystemAnnouncer.java new file mode 100644 index 0000000000..e36983a41b --- /dev/null +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/fs/HollowFilesystemAnnouncer.java @@ -0,0 +1,69 @@ +/* + * + * Copyright 2017 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.hollow.api.producer.fs; + +import static java.nio.file.Files.createDirectories; +import static java.nio.file.Files.newBufferedWriter; +import static java.nio.file.StandardOpenOption.CREATE; +import static java.nio.file.StandardOpenOption.DSYNC; +import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING; +import static java.nio.file.StandardOpenOption.WRITE; + +import java.io.BufferedWriter; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; + +import com.netflix.hollow.api.producer.HollowProducer; + +public class HollowFilesystemAnnouncer implements HollowProducer.Announcer { + private final Path publishPath; + private final Path annnouncementPath; + + public HollowFilesystemAnnouncer(String namespace) { + this(namespace, "announced.version"); + } + + public HollowFilesystemAnnouncer(String namespace, String announcementFilename) { + this(Paths.get(System.getProperty("java.io.tmpdir"), namespace, "published"), + announcementFilename); + } + + public HollowFilesystemAnnouncer(Path publishDir, String announcementFilename) { + this.publishPath = publishDir; + annnouncementPath = publishPath.resolve(announcementFilename); + } + + @Override + public void announce(long stateVersion) { + BufferedWriter writer = null; + try { + createDirectories(publishPath); + writer = newBufferedWriter(annnouncementPath, CREATE, WRITE, TRUNCATE_EXISTING, DSYNC); + writer.write(String.valueOf(stateVersion)); + } catch(IOException ex) { + throw new RuntimeException("Unable to write to announcement file", ex); + } finally { + try { + if(writer != null) writer.close(); + } catch(IOException ex) { + ex.printStackTrace(); + } + } + } +} diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/fs/HollowFilesystemPublisher.java b/hollow/src/main/java/com/netflix/hollow/api/producer/fs/HollowFilesystemPublisher.java new file mode 100644 index 0000000000..69bb51ea66 --- /dev/null +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/fs/HollowFilesystemPublisher.java @@ -0,0 +1,76 @@ +/* + * + * Copyright 2017 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.netflix.hollow.api.producer.fs; + +import static java.nio.file.Files.createDirectories; +import static java.nio.file.StandardCopyOption.ATOMIC_MOVE; +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import com.netflix.hollow.api.producer.HollowProducer; + +public class HollowFilesystemPublisher extends AbstractHollowPublisher { + private final Path publishPath; + + public HollowFilesystemPublisher(String namespace) { + super(namespace); + this.publishPath = scratchPath.resolve(Paths.get(namespace, "published")); + } + + public HollowFilesystemPublisher(String namespace, Path scratchPath) { + super(namespace, scratchPath); + this.publishPath = scratchPath.resolve(Paths.get(namespace, "published")); + } + + public HollowFilesystemPublisher(String namespace, Path scratchPath, Path stagingPath, Path publishPath) { + super(namespace, scratchPath, stagingPath); + this.publishPath = publishPath; + } + + @Override + public void publish(HollowProducer.Blob blob) { + publishBlob((StagedBlob)blob); + } + + public Path getStagingDir() { + return publishPath; + } + + public Path getPublishDir() { + return publishPath; + } + + private void publishBlob(StagedBlob blob) { + try { + createDirectories(publishPath); + + Path source = blob.getStagedArtifactPath(); + Path filename = source.getFileName(); + Path destination = publishPath.resolve(filename); + Path intermediate = destination.resolveSibling(filename + ".incomplete"); + Files.copy(source, intermediate, REPLACE_EXISTING); + Files.move(intermediate, destination, ATOMIC_MOVE); + } catch(IOException ex) { + throw new RuntimeException("Unable to publish file!", ex); + } + } +} From 859578b04b9b8734d4738033a6719fb258d34aa2 Mon Sep 17 00:00:00 2001 From: Tim Taylor Date: Thu, 2 Mar 2017 01:09:10 -0800 Subject: [PATCH 25/30] keep staged artifacts between stages; cleanup when cycle complete --- .../hollow/api/producer/HollowProducer.java | 90 ++++++++++++------- .../producer/fs/AbstractHollowPublisher.java | 37 +++----- 2 files changed, 70 insertions(+), 57 deletions(-) diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java index a05c6733e5..339442770b 100644 --- a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java @@ -20,7 +20,6 @@ import static com.netflix.hollow.api.consumer.HollowConsumer.newReadState; import static java.lang.System.currentTimeMillis; -import java.io.Closeable; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -118,11 +117,12 @@ public void runCycle(Populator task) { long start = currentTimeMillis(); ProducerStatus cycleStatus = ProducerStatus.unknownFailure(); WriteState writeState = null; + Artifacts artifacts = new Artifacts(); try { writeState = beginCycle(); task.populate(writeState); if(writeEngine.hasChangedSinceLastCycle()) { - publish(writeState); + publish(writeState, artifacts); ReadStateHelper candidate = readStates.roundtrip(writeState); checkIntegrity(candidate); validate(candidate.pending()); @@ -138,6 +138,7 @@ public void runCycle(Populator task) { writeEngine.resetToLastPrepareForNextCycle(); cycleStatus = ProducerStatus.fail(writeState != null ? writeState.getVersion() : Long.MIN_VALUE, th); } finally { + artifacts.cleanup(); listeners.fireCycleComplete(cycleStatus, start); } } @@ -158,42 +159,38 @@ private WriteState beginCycle() { return new WriteStateImpl(objectMapper, toVersion); } - private void publish(WriteState writeState) throws IOException { + private void publish(WriteState writeState, Artifacts artifacts) throws IOException { long start = listeners.firePublishStart(writeState.getVersion()); HollowBlobWriter writer = new HollowBlobWriter(writeEngine); - Blob snapshot = null; ProducerStatus status = ProducerStatus.unknownFailure(); try { - snapshot = publisher.openSnapshot(writeState.getVersion()); - writer.writeSnapshot(snapshot.getOutputStream()); + // 1. Write the snapshot + artifacts.snapshot = publisher.openSnapshot(writeState.getVersion()); + OutputStream ssos = artifacts.snapshot.newOutputStream(); + try { writer.writeSnapshot(ssos); } finally { ssos.close(); } + // 2. Write & publish the deltas; active canaries and tooling receive it sooner if(readStates.hasCurrent()) { - Blob delta = publisher.openDelta(readStates.current().getVersion(), writeState.getVersion()); - try { - writer.writeDelta(delta.getOutputStream()); - publisher.publish(delta); - } finally { - delta.close(); - } - Blob reverseDelta = publisher.openReverseDelta(readStates.current().getVersion(), writeState.getVersion()); - try { - writer.writeReverseDelta(reverseDelta.getOutputStream()); - publisher.publish(reverseDelta); - } finally { - reverseDelta.close(); - } + artifacts.delta = publisher.openDelta(readStates.current().getVersion(), writeState.getVersion()); + OutputStream fdos = artifacts.delta.newOutputStream(); + try { writer.writeDelta(fdos); } finally { fdos.close(); } + publisher.publish(artifacts.delta); + + artifacts.reverseDelta = publisher.openReverseDelta(readStates.current().getVersion(), writeState.getVersion()); + OutputStream rdos = artifacts.reverseDelta.newOutputStream(); + try { writer.writeReverseDelta(rdos); } finally { rdos.close(); } + publisher.publish(artifacts.reverseDelta); } - // TODO: timt: allow some failed snapshot publishes - publisher.publish(snapshot); + // 3. Publish the snapshot + publisher.publish(artifacts.snapshot); status = ProducerStatus.success(writeState.getVersion()); } catch(Throwable th) { status = ProducerStatus.fail(writeState.getVersion(), th); throw th; } finally { listeners.firePublishComplete(status, start); - if(snapshot != null) snapshot.close(); } } @@ -342,19 +339,13 @@ public static interface Publisher { public void publish(HollowProducer.Blob blob); } - public static interface Blob extends Closeable { - OutputStream getOutputStream(); - - InputStream getInputStream(); - - @Override - void close(); - + public static interface Blob { + OutputStream newOutputStream(); + InputStream newInputStream(); long getFromVersion(); - long getToVersion(); - Type getType(); + void cleanup(); public static enum Type { SNAPSHOT("snapshot"), @@ -376,4 +367,37 @@ public static interface Validator { public static interface Announcer { public void announce(long stateVersion); } + + private static final class Artifacts { + Blob snapshot = null; + Blob delta = null; + Blob reverseDelta = null; + + void cleanup() { + if(snapshot != null) { + snapshot.cleanup(); + snapshot = null; + } + if(delta != null) { + delta.cleanup(); + delta = null; + } + if(reverseDelta != null) { + reverseDelta.cleanup(); + reverseDelta = null; + } + } + + public boolean hasSnapshot() { + return snapshot != null; + } + + boolean hasDelta() { + return delta != null; + } + + public boolean hasReverseDelta() { + return reverseDelta != null; + } + } } diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/fs/AbstractHollowPublisher.java b/hollow/src/main/java/com/netflix/hollow/api/producer/fs/AbstractHollowPublisher.java index 26c50d19a8..949efffcba 100644 --- a/hollow/src/main/java/com/netflix/hollow/api/producer/fs/AbstractHollowPublisher.java +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/fs/AbstractHollowPublisher.java @@ -17,24 +17,23 @@ */ package com.netflix.hollow.api.producer.fs; -import static com.netflix.hollow.api.producer.HollowProducer.Blob.Type.*; +import static com.netflix.hollow.api.producer.HollowProducer.Blob.Type.DELTA; +import static com.netflix.hollow.api.producer.HollowProducer.Blob.Type.REVERSE_DELTA; +import static com.netflix.hollow.api.producer.HollowProducer.Blob.Type.SNAPSHOT; import static java.nio.file.Files.createDirectories; import static java.nio.file.Files.deleteIfExists; -import static java.nio.file.Files.newInputStream; -import static java.nio.file.Files.newOutputStream; import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import com.netflix.hollow.api.producer.HollowProducer; import com.netflix.hollow.api.producer.HollowProducer.Blob; -import com.netflix.hollow.api.producer.HollowProducer.Publisher; -import com.netflix.hollow.api.producer.HollowProducer.Blob.Type; public abstract class AbstractHollowPublisher implements HollowProducer.Publisher { protected final String namespace; @@ -75,8 +74,6 @@ public static class StagedBlob implements Blob { protected final long fromVersion; protected final long toVersion; protected final Path stagedArtifactPath; - private BufferedOutputStream out; - private BufferedInputStream in; protected StagedBlob(Blob.Type type, String namespace, Path stagingPath, long fromVersion, long toVersion) { this.type = type; @@ -101,38 +98,30 @@ protected StagedBlob(Blob.Type type, String namespace, Path stagingPath, long fr } @Override - public OutputStream getOutputStream() { + public OutputStream newOutputStream() { try { createDirectories(stagedArtifactPath.getParent()); - out = new BufferedOutputStream(newOutputStream(stagedArtifactPath)); - return out; + return new BufferedOutputStream(Files.newOutputStream(stagedArtifactPath)); } catch(IOException ex) { throw new RuntimeException(ex); } } @Override - public InputStream getInputStream() { + public InputStream newInputStream() { try { - in = new BufferedInputStream(newInputStream(stagedArtifactPath)); - return in; + return new BufferedInputStream(Files.newInputStream(stagedArtifactPath)); } catch(IOException ex) { throw new RuntimeException(ex); } } @Override - public void close() { - if(out != null) { - try { - if(out != null) out.close(); - out = null; - if(in != null) in.close(); - in = null; - // deleteIfExists(stagedPath); - } catch(IOException ex) { - throw new RuntimeException(ex); - } + public void cleanup() { + try { + deleteIfExists(stagedArtifactPath); + } catch(IOException ex) { + ex.printStackTrace(); } } From 4098f338f516d464bed118664c3d5b9e1ea10218 Mon Sep 17 00:00:00 2001 From: Tim Taylor Date: Thu, 2 Mar 2017 01:11:46 -0800 Subject: [PATCH 26/30] validate forward and reverse delta checksums --- .../hollow/api/producer/HollowProducer.java | 72 ++++++++++++++++--- .../hollow/api/producer/ReadStateHelper.java | 11 +++ 2 files changed, 73 insertions(+), 10 deletions(-) diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java index 339442770b..bb4e10aba9 100644 --- a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java @@ -18,7 +18,10 @@ package com.netflix.hollow.api.producer; import static com.netflix.hollow.api.consumer.HollowConsumer.newReadState; +import static com.netflix.hollow.api.producer.HollowProducer.Blob.Type.DELTA; +import static com.netflix.hollow.api.producer.HollowProducer.Blob.Type.REVERSE_DELTA; import static java.lang.System.currentTimeMillis; +import static java.lang.System.out; import java.io.IOException; import java.io.InputStream; @@ -31,9 +34,11 @@ import com.netflix.hollow.api.producer.HollowProducerListener.RestoreStatus; import com.netflix.hollow.core.read.engine.HollowBlobHeaderReader; import com.netflix.hollow.core.read.engine.HollowBlobReader; +import com.netflix.hollow.core.read.engine.HollowReadStateEngine; import com.netflix.hollow.core.write.HollowBlobWriter; import com.netflix.hollow.core.write.HollowWriteStateEngine; import com.netflix.hollow.core.write.objectmapper.HollowObjectMapper; +import com.netflix.hollow.tools.checksum.HollowChecksum; /** * Beta API subject to change. @@ -124,7 +129,7 @@ public void runCycle(Populator task) { if(writeEngine.hasChangedSinceLastCycle()) { publish(writeState, artifacts); ReadStateHelper candidate = readStates.roundtrip(writeState); - checkIntegrity(candidate); + candidate = checkIntegrity(candidate, artifacts); validate(candidate.pending()); announce(candidate.pending()); readStates = candidate.commit(); @@ -206,28 +211,75 @@ private void publish(WriteState writeState, Artifacts artifacts) throws IOExcept * S(pnd).apply(reverseDelta).checksum == S(cur).checksum * * @param readStates + * @return updated read states */ - private void checkIntegrity(ReadStateHelper readStates) throws Exception { + private ReadStateHelper checkIntegrity(ReadStateHelper readStates, Artifacts artifacts) throws Exception { + ReadStateHelper result = readStates; long start = listeners.fireIntegrityCheckStart(readStates.pendingVersion()); ProducerStatus status = ProducerStatus.unknownFailure(); - Blob snapshot = null; try { - snapshot = publisher.openSnapshot(readStates.pendingVersion()); - HollowBlobReader reader = new HollowBlobReader(readStates.pending().getStateEngine(), new HollowBlobHeaderReader()); - reader.readSnapshot(snapshot.getInputStream()); + HollowReadStateEngine current = readStates.hasCurrent() ? readStates.current().getStateEngine() : null; + HollowReadStateEngine pending = readStates.pending().getStateEngine(); + readSnapshot(artifacts.snapshot, pending); if(readStates.hasCurrent()) { - // FIXME: timt: do delta integrity checks; we only compare checksums for - // schemas that are common and unchanged between S(curr) and S(prev) - } + System.out.println("CHECKSUMS"); + HollowChecksum currentChecksum = HollowChecksum.forStateEngineWithCommonSchemas(current, pending); + out.format(" CUR %s\n", currentChecksum); + + HollowChecksum pendingChecksum = HollowChecksum.forStateEngineWithCommonSchemas(pending, current); + out.format(" PND %s\n", pendingChecksum); + + if(artifacts.hasDelta()) { + // FIXME: timt: future cycles will fail unless this delta validates *and* we have a reverse + // delta *and* it also validates + applyDelta(artifacts.delta, current); + HollowChecksum forwardChecksum = HollowChecksum.forStateEngineWithCommonSchemas(current, pending); + out.format(" CUR => PND %s\n", forwardChecksum); + if(!forwardChecksum.equals(pendingChecksum)) throw new ChecksumValidationException(DELTA); + } + if(artifacts.hasReverseDelta()) { + applyDelta(artifacts.reverseDelta, pending); + HollowChecksum reverseChecksum = HollowChecksum.forStateEngineWithCommonSchemas(pending, current); + out.format(" CUR <= PND %s\n", reverseChecksum); + if(!reverseChecksum.equals(currentChecksum)) throw new ChecksumValidationException(REVERSE_DELTA); + result = readStates.swap(); + } + } status = ProducerStatus.success(readStates.pending()); } catch(Throwable th) { status = ProducerStatus.fail(readStates.pendingVersion(), th); throw th; } finally { listeners.fireIntegrityCheckComplete(status, start); - if(snapshot != null) snapshot.close(); + } + return result; + } + + public static final class ChecksumValidationException extends IllegalStateException { + private static final long serialVersionUID = -4399719849669674206L; + + ChecksumValidationException(Blob.Type type) { + super(type.name() + " checksum invalid"); + } + } + + private void readSnapshot(Blob blob, HollowReadStateEngine stateEngine) throws IOException { + InputStream is = blob.newInputStream(); + try { + new HollowBlobReader(stateEngine, new HollowBlobHeaderReader()).readSnapshot(is); + } finally { + is.close(); + } + } + + private void applyDelta(Blob blob, HollowReadStateEngine stateEngine) throws IOException { + InputStream is = blob.newInputStream(); + try { + new HollowBlobReader(stateEngine, new HollowBlobHeaderReader()).applyDelta(is); + } finally { + is.close(); } } diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/ReadStateHelper.java b/hollow/src/main/java/com/netflix/hollow/api/producer/ReadStateHelper.java index 0f7a156e37..4d4f995543 100644 --- a/hollow/src/main/java/com/netflix/hollow/api/producer/ReadStateHelper.java +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/ReadStateHelper.java @@ -53,6 +53,17 @@ ReadStateHelper roundtrip(HollowProducer.WriteState writeState) { return new ReadStateHelper(this.current, newReadState(writeState.getVersion(), new HollowReadStateEngine())); } + /** + * Swap underlying state engines between current and pending while keeping the versions consistent; + * used after delta integrity checks have altered the underlying state engines. + * + * @return + */ + ReadStateHelper swap() { + return new ReadStateHelper(newReadState(current.getVersion(), pending.getStateEngine()), + newReadState(pending.getVersion(), current.getStateEngine())); + } + ReadStateHelper commit() { if(pending == null) throw new IllegalStateException(); return new ReadStateHelper(this.pending, null); From 0355dabaf22201059280490eb575417f04d053c6 Mon Sep 17 00:00:00 2001 From: Tim Taylor Date: Thu, 2 Mar 2017 03:15:19 -0800 Subject: [PATCH 27/30] reduce producer listener notification boilerplate --- .../hollow/api/producer/HollowProducer.java | 77 ++++++++----------- .../api/producer/HollowProducerListener.java | 61 +++++++++++++++ .../hollow/api/producer/ListenerSupport.java | 75 +++++++++--------- 3 files changed, 132 insertions(+), 81 deletions(-) diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java index bb4e10aba9..86be3e5340 100644 --- a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java @@ -119,32 +119,34 @@ public HollowProducer restore(long versionDesired, HollowBlobRetriever blobRetri * Each cycle produces a single data state. */ public void runCycle(Populator task) { - long start = currentTimeMillis(); - ProducerStatus cycleStatus = ProducerStatus.unknownFailure(); - WriteState writeState = null; + long toVersion = versionMinter.mint(); + if(!readStates.hasCurrent()) listeners.fireNewDeltaChain(toVersion); + ProducerStatus.Builder cycleStatus = listeners.fireCycleStart(toVersion); Artifacts artifacts = new Artifacts(); try { - writeState = beginCycle(); + writeEngine.prepareForNextCycle(); + WriteState writeState = new WriteStateImpl(objectMapper, toVersion); task.populate(writeState); if(writeEngine.hasChangedSinceLastCycle()) { publish(writeState, artifacts); ReadStateHelper candidate = readStates.roundtrip(writeState); + cycleStatus.version(candidate.pending()); candidate = checkIntegrity(candidate, artifacts); validate(candidate.pending()); announce(candidate.pending()); readStates = candidate.commit(); - cycleStatus = ProducerStatus.success(readStates.current()); + cycleStatus.version(readStates.current()).success(); } else { writeEngine.resetToLastPrepareForNextCycle(); - listeners.fireNoDelta(writeState.getVersion()); - cycleStatus = ProducerStatus.success(writeState.getVersion()); + listeners.fireNoDelta(cycleStatus); + cycleStatus.success(); } } catch(Throwable th) { writeEngine.resetToLastPrepareForNextCycle(); - cycleStatus = ProducerStatus.fail(writeState != null ? writeState.getVersion() : Long.MIN_VALUE, th); + cycleStatus.fail(th); } finally { artifacts.cleanup(); - listeners.fireCycleComplete(cycleStatus, start); + listeners.fireCycleComplete(cycleStatus); } } @@ -156,19 +158,10 @@ public void removeListener(HollowProducerListener listener) { listeners.remove(listener); } - private WriteState beginCycle() { - long toVersion = versionMinter.mint(); - if(!readStates.hasCurrent()) listeners.fireNewDeltaChain(toVersion); - listeners.fireCycleStart(toVersion); - writeEngine.prepareForNextCycle(); - return new WriteStateImpl(objectMapper, toVersion); - } - private void publish(WriteState writeState, Artifacts artifacts) throws IOException { - long start = listeners.firePublishStart(writeState.getVersion()); + ProducerStatus.Builder status = listeners.firePublishStart(writeState); HollowBlobWriter writer = new HollowBlobWriter(writeEngine); - ProducerStatus status = ProducerStatus.unknownFailure(); try { // 1. Write the snapshot artifacts.snapshot = publisher.openSnapshot(writeState.getVersion()); @@ -190,12 +183,12 @@ private void publish(WriteState writeState, Artifacts artifacts) throws IOExcept // 3. Publish the snapshot publisher.publish(artifacts.snapshot); - status = ProducerStatus.success(writeState.getVersion()); + status.success(); } catch(Throwable th) { - status = ProducerStatus.fail(writeState.getVersion(), th); + status.fail(th); throw th; } finally { - listeners.firePublishComplete(status, start); + listeners.firePublishComplete(status); } } @@ -214,10 +207,9 @@ private void publish(WriteState writeState, Artifacts artifacts) throws IOExcept * @return updated read states */ private ReadStateHelper checkIntegrity(ReadStateHelper readStates, Artifacts artifacts) throws Exception { - ReadStateHelper result = readStates; - long start = listeners.fireIntegrityCheckStart(readStates.pendingVersion()); - ProducerStatus status = ProducerStatus.unknownFailure(); + ProducerStatus.Builder status = listeners.fireIntegrityCheckStart(readStates.pending()); try { + ReadStateHelper result = readStates; HollowReadStateEngine current = readStates.hasCurrent() ? readStates.current().getStateEngine() : null; HollowReadStateEngine pending = readStates.pending().getStateEngine(); readSnapshot(artifacts.snapshot, pending); @@ -225,36 +217,36 @@ private ReadStateHelper checkIntegrity(ReadStateHelper readStates, Artifacts art if(readStates.hasCurrent()) { System.out.println("CHECKSUMS"); HollowChecksum currentChecksum = HollowChecksum.forStateEngineWithCommonSchemas(current, pending); - out.format(" CUR %s\n", currentChecksum); + //out.format(" CUR %s\n", currentChecksum); HollowChecksum pendingChecksum = HollowChecksum.forStateEngineWithCommonSchemas(pending, current); - out.format(" PND %s\n", pendingChecksum); + //out.format(" PND %s\n", pendingChecksum); if(artifacts.hasDelta()) { // FIXME: timt: future cycles will fail unless this delta validates *and* we have a reverse // delta *and* it also validates applyDelta(artifacts.delta, current); HollowChecksum forwardChecksum = HollowChecksum.forStateEngineWithCommonSchemas(current, pending); - out.format(" CUR => PND %s\n", forwardChecksum); + //out.format(" CUR => PND %s\n", forwardChecksum); if(!forwardChecksum.equals(pendingChecksum)) throw new ChecksumValidationException(DELTA); } if(artifacts.hasReverseDelta()) { applyDelta(artifacts.reverseDelta, pending); HollowChecksum reverseChecksum = HollowChecksum.forStateEngineWithCommonSchemas(pending, current); - out.format(" CUR <= PND %s\n", reverseChecksum); + //out.format(" CUR <= PND %s\n", reverseChecksum); if(!reverseChecksum.equals(currentChecksum)) throw new ChecksumValidationException(REVERSE_DELTA); result = readStates.swap(); } } - status = ProducerStatus.success(readStates.pending()); + status.success(); + return result; } catch(Throwable th) { - status = ProducerStatus.fail(readStates.pendingVersion(), th); + status.fail(th); throw th; } finally { - listeners.fireIntegrityCheckComplete(status, start); + listeners.fireIntegrityCheckComplete(status); } - return result; } public static final class ChecksumValidationException extends IllegalStateException { @@ -283,32 +275,29 @@ private void applyDelta(Blob blob, HollowReadStateEngine stateEngine) throws IOE } } - /// TODO: timt: validator API TBD private void validate(HollowConsumer.ReadState readState) { - long start = listeners.fireValidationStart(readState); - ProducerStatus status = ProducerStatus.unknownFailure(); + ProducerStatus.Builder status = listeners.fireValidationStart(readState); try { validator.validate(readState); - status = ProducerStatus.success(readState); + status.success(); } catch (Throwable th) { - status = ProducerStatus.fail(readState, th); + status.fail(th); throw th; } finally { - listeners.fireValidationComplete(status, start); + listeners.fireValidationComplete(status); } } private void announce(HollowConsumer.ReadState readState) { - long start = listeners.fireAnnouncementStart(readState); - ProducerStatus status = ProducerStatus.unknownFailure(); + ProducerStatus.Builder status = listeners.fireAnnouncementStart(readState); try { announcer.announce(readState.getVersion()); - status = ProducerStatus.success(readState); + status.success(); } catch(Throwable th) { - status = ProducerStatus.fail(readState, th); + status.fail(th); throw th; } finally { - listeners.fireAnnouncementComplete(status, start); + listeners.fireAnnouncementComplete(status); } } diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducerListener.java b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducerListener.java index 7337777ba5..1af53d68b6 100644 --- a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducerListener.java +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducerListener.java @@ -17,10 +17,15 @@ */ package com.netflix.hollow.api.producer; +import static com.netflix.hollow.api.producer.HollowProducerListener.Status.FAIL; +import static com.netflix.hollow.api.producer.HollowProducerListener.Status.SUCCESS; + import java.util.EventListener; import java.util.concurrent.TimeUnit; import com.netflix.hollow.api.consumer.HollowConsumer; +import com.netflix.hollow.api.consumer.HollowConsumer.ReadState; +import com.netflix.hollow.api.producer.HollowProducer.WriteState; /** * Beta API subject to change. @@ -243,6 +248,62 @@ public Throwable getCause() { public HollowConsumer.ReadState getReadState() { return readState; } + + static final class Builder { + private final long start; + private long end; + + private long version = Long.MIN_VALUE; + private Status status = FAIL; + private Throwable cause = null; + private HollowConsumer.ReadState readState = null; + + Builder() { + start = System.currentTimeMillis(); + } + + ProducerStatus.Builder version(long version) { + this.version = version; + return this; + } + + ProducerStatus.Builder version(WriteState writeState) { + return version(writeState.getVersion()); + } + + ProducerStatus.Builder version(ReadState readState) { + this.readState = readState; + return version(readState.getVersion()); + } + + ProducerStatus.Builder success() { + this.status = SUCCESS; + return this; + } + + ProducerStatus.Builder fail() { + return this; + } + + ProducerStatus.Builder fail(Throwable cause) { + this.status = FAIL; + this.cause = cause; + return this; + } + + ProducerStatus build() { + end = System.currentTimeMillis(); + return new ProducerStatus(status, version, readState, cause); + } + + long elapsed() { + return end - start; + } + + long version() { + return version; + } + } } /** diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/ListenerSupport.java b/hollow/src/main/java/com/netflix/hollow/api/producer/ListenerSupport.java index 540c751176..9845aff5e9 100644 --- a/hollow/src/main/java/com/netflix/hollow/api/producer/ListenerSupport.java +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/ListenerSupport.java @@ -17,13 +17,14 @@ */ package com.netflix.hollow.api.producer; -import static java.lang.System.currentTimeMillis; import static java.util.concurrent.TimeUnit.MILLISECONDS; import java.util.Set; import java.util.concurrent.CopyOnWriteArraySet; import com.netflix.hollow.api.consumer.HollowConsumer; +import com.netflix.hollow.api.consumer.HollowConsumer.ReadState; +import com.netflix.hollow.api.producer.HollowProducer.WriteState; import com.netflix.hollow.api.producer.HollowProducerListener.ProducerStatus; import com.netflix.hollow.api.producer.HollowProducerListener.RestoreStatus; @@ -63,62 +64,62 @@ void fireNewDeltaChain(long version) { for(final HollowProducerListener l : listeners) l.onNewDeltaChain(version); } - long fireCycleStart(long version) { - long start = currentTimeMillis(); + ProducerStatus.Builder fireCycleStart(long version) { + ProducerStatus.Builder psb = new ProducerStatus.Builder().version(version); for(final HollowProducerListener l : listeners) l.onCycleStart(version); - return start; + return psb; } - void fireCycleComplete(ProducerStatus cycleStatus, long startMillis) { - long elapsedMillis = currentTimeMillis() - startMillis; - for(final HollowProducerListener l : listeners) l.onCycleComplete(cycleStatus, elapsedMillis, MILLISECONDS); + void fireCycleComplete(ProducerStatus.Builder psb) { + ProducerStatus st = psb.build(); + for(final HollowProducerListener l : listeners) l.onCycleComplete(st, psb.elapsed(), MILLISECONDS); } - void fireNoDelta(long version) { - for(final HollowProducerListener l : listeners) l.onNoDeltaAvailable(version); + void fireNoDelta(ProducerStatus.Builder psb) { + for(final HollowProducerListener l : listeners) l.onNoDeltaAvailable(psb.version()); } - long firePublishStart(long version) { - long start = currentTimeMillis(); - for(final HollowProducerListener l : listeners) l.onPublishStart(version); - return start; + ProducerStatus.Builder firePublishStart(WriteState writeState) { + ProducerStatus.Builder psb = new ProducerStatus.Builder().version(writeState); + for(final HollowProducerListener l : listeners) l.onPublishStart(psb.version()); + return psb; } - void firePublishComplete(ProducerStatus publishStatus, long startMillis) { - long elapsedMillis = currentTimeMillis() - startMillis; - for(final HollowProducerListener l : listeners) l.onPublishComplete(publishStatus, elapsedMillis, MILLISECONDS); + void firePublishComplete(ProducerStatus.Builder psb) { + ProducerStatus st = psb.build(); + for(final HollowProducerListener l : listeners) l.onPublishComplete(st, psb.elapsed(), MILLISECONDS); } - long fireIntegrityCheckStart(long version) { - long start = currentTimeMillis(); - for(final HollowProducerListener l : listeners) l.onIntegrityCheckStart(version); - return start; + ProducerStatus.Builder fireIntegrityCheckStart(ReadState readState) { + ProducerStatus.Builder psb = new ProducerStatus.Builder().version(readState); + for(final HollowProducerListener l : listeners) l.onIntegrityCheckStart(psb.version()); + return psb; } - void fireIntegrityCheckComplete(ProducerStatus integrityCheckStatus, long startMillis) { - long elapsedMillis = currentTimeMillis() - startMillis; - for(final HollowProducerListener l : listeners) l.onIntegrityCheckComplete(integrityCheckStatus, elapsedMillis, MILLISECONDS); + void fireIntegrityCheckComplete(ProducerStatus.Builder psb) { + ProducerStatus st = psb.build(); + for(final HollowProducerListener l : listeners) l.onIntegrityCheckComplete(st, psb.elapsed(), MILLISECONDS); } - long fireValidationStart(HollowConsumer.ReadState readState) { - long start = currentTimeMillis(); - for(final HollowProducerListener l : listeners) l.onValidationStart(readState.getVersion()); - return start; + ProducerStatus.Builder fireValidationStart(HollowConsumer.ReadState readState) { + ProducerStatus.Builder psb = new ProducerStatus.Builder().version(readState); + for(final HollowProducerListener l : listeners) l.onValidationStart(psb.version()); + return psb; } - void fireValidationComplete(ProducerStatus validationStatus, long startMillis) { - long elapsedMillis = currentTimeMillis() - startMillis; - for(final HollowProducerListener l : listeners) l.onValidationComplete(validationStatus, elapsedMillis, MILLISECONDS); + void fireValidationComplete(ProducerStatus.Builder psb) { + ProducerStatus st = psb.build(); + for(final HollowProducerListener l : listeners) l.onValidationComplete(st, psb.elapsed(), MILLISECONDS); } - long fireAnnouncementStart(HollowConsumer.ReadState readState) { - long start = currentTimeMillis(); - for(final HollowProducerListener l : listeners) l.onAnnouncementStart(readState.getVersion()); - return start; + ProducerStatus.Builder fireAnnouncementStart(HollowConsumer.ReadState readState) { + ProducerStatus.Builder psb = new ProducerStatus.Builder().version(readState); + for(final HollowProducerListener l : listeners) l.onAnnouncementStart(psb.version()); + return psb; } - void fireAnnouncementComplete(ProducerStatus announcementStatus, long startMillis) { - long elapsedMillis = currentTimeMillis() - startMillis; - for(final HollowProducerListener l : listeners) l.onAnnouncementComplete(announcementStatus, elapsedMillis, MILLISECONDS); + void fireAnnouncementComplete(ProducerStatus.Builder psb) { + ProducerStatus st = psb.build(); + for(final HollowProducerListener l : listeners) l.onAnnouncementComplete(st, psb.elapsed(), MILLISECONDS); } } From 356c648e3fa96fa773b220fca695a004e492acc8 Mon Sep 17 00:00:00 2001 From: Tim Taylor Date: Thu, 2 Mar 2017 03:21:29 -0800 Subject: [PATCH 28/30] tweak WriteState creation; document runCycle with comments --- .../hollow/api/producer/HollowProducer.java | 22 +++++++++++++++---- .../hollow/api/producer/WriteStateImpl.java | 6 ++--- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java index 86be3e5340..1048ec8e71 100644 --- a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducer.java @@ -21,7 +21,6 @@ import static com.netflix.hollow.api.producer.HollowProducer.Blob.Type.DELTA; import static com.netflix.hollow.api.producer.HollowProducer.Blob.Type.REVERSE_DELTA; import static java.lang.System.currentTimeMillis; -import static java.lang.System.out; import java.io.IOException; import java.io.InputStream; @@ -51,6 +50,10 @@ public class HollowProducer { public void validate(HollowConsumer.ReadState readState) {} }; + public static WriteState newWriteState(long version, HollowObjectMapper objectMapper) { + return new WriteStateImpl(version, objectMapper); + } + private final Publisher publisher; private final Validator validator; private final Announcer announcer; @@ -119,27 +122,38 @@ public HollowProducer restore(long versionDesired, HollowBlobRetriever blobRetri * Each cycle produces a single data state. */ public void runCycle(Populator task) { + // 1. Begin a new cycle long toVersion = versionMinter.mint(); if(!readStates.hasCurrent()) listeners.fireNewDeltaChain(toVersion); ProducerStatus.Builder cycleStatus = listeners.fireCycleStart(toVersion); Artifacts artifacts = new Artifacts(); try { + // 1a. Prepare the write state writeEngine.prepareForNextCycle(); - WriteState writeState = new WriteStateImpl(objectMapper, toVersion); + HollowObjectMapper objectMapper = this.objectMapper; + WriteState writeState = newWriteState(toVersion, objectMapper); + + // 2. Populate the state task.populate(writeState); + + // 3. Produce a new state if there's work to do if(writeEngine.hasChangedSinceLastCycle()) { + // 3a. Publish, run checks & validation, then announce new state consumers publish(writeState, artifacts); + ReadStateHelper candidate = readStates.roundtrip(writeState); cycleStatus.version(candidate.pending()); candidate = checkIntegrity(candidate, artifacts); + validate(candidate.pending()); + announce(candidate.pending()); readStates = candidate.commit(); cycleStatus.version(readStates.current()).success(); } else { + // 3b. Nothing to do; reset the effects of Step 2 writeEngine.resetToLastPrepareForNextCycle(); - listeners.fireNoDelta(cycleStatus); - cycleStatus.success(); + listeners.fireNoDelta(cycleStatus.success()); } } catch(Throwable th) { writeEngine.resetToLastPrepareForNextCycle(); diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/WriteStateImpl.java b/hollow/src/main/java/com/netflix/hollow/api/producer/WriteStateImpl.java index e5e05b5f5c..fdeb5ce9a7 100644 --- a/hollow/src/main/java/com/netflix/hollow/api/producer/WriteStateImpl.java +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/WriteStateImpl.java @@ -26,12 +26,12 @@ * @author Tim Taylor {@literal} */ final class WriteStateImpl implements HollowProducer.WriteState { - private final HollowObjectMapper objectMapper; private final long version; + private final HollowObjectMapper objectMapper; - protected WriteStateImpl(HollowObjectMapper objectMapper, long version) { - this.objectMapper = objectMapper; + protected WriteStateImpl(long version, HollowObjectMapper objectMapper) { this.version = version; + this.objectMapper = objectMapper; } @Override From 5505121019bd11771529777308ea4a99bf1834af Mon Sep 17 00:00:00 2001 From: Tim Taylor Date: Thu, 2 Mar 2017 03:29:07 -0800 Subject: [PATCH 29/30] fix jdk1.7 compilation error --- .../hollow/api/producer/fs/HollowFilesystemAnnouncer.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/fs/HollowFilesystemAnnouncer.java b/hollow/src/main/java/com/netflix/hollow/api/producer/fs/HollowFilesystemAnnouncer.java index e36983a41b..b3b8cd4e77 100644 --- a/hollow/src/main/java/com/netflix/hollow/api/producer/fs/HollowFilesystemAnnouncer.java +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/fs/HollowFilesystemAnnouncer.java @@ -17,12 +17,9 @@ */ package com.netflix.hollow.api.producer.fs; +import static java.nio.charset.StandardCharsets.UTF_8; import static java.nio.file.Files.createDirectories; import static java.nio.file.Files.newBufferedWriter; -import static java.nio.file.StandardOpenOption.CREATE; -import static java.nio.file.StandardOpenOption.DSYNC; -import static java.nio.file.StandardOpenOption.TRUNCATE_EXISTING; -import static java.nio.file.StandardOpenOption.WRITE; import java.io.BufferedWriter; import java.io.IOException; @@ -54,7 +51,7 @@ public void announce(long stateVersion) { BufferedWriter writer = null; try { createDirectories(publishPath); - writer = newBufferedWriter(annnouncementPath, CREATE, WRITE, TRUNCATE_EXISTING, DSYNC); + writer = newBufferedWriter(annnouncementPath, UTF_8); writer.write(String.valueOf(stateVersion)); } catch(IOException ex) { throw new RuntimeException("Unable to write to announcement file", ex); From f94b4dc330bef0b2e1db425235e533dd75e1aeb2 Mon Sep 17 00:00:00 2001 From: Tim Taylor Date: Thu, 2 Mar 2017 03:31:30 -0800 Subject: [PATCH 30/30] fix JavaDoc error --- .../netflix/hollow/api/producer/HollowProducerListener.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducerListener.java b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducerListener.java index 1af53d68b6..88ca4400a3 100644 --- a/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducerListener.java +++ b/hollow/src/main/java/com/netflix/hollow/api/producer/HollowProducerListener.java @@ -63,7 +63,7 @@ public interface HollowProducerListener extends EventListener { /** * Indicates that the next state produced will begin a new delta chain. * Thiis will be called prior to the next state being produced either if - * {@link HollowProducer#restore(com.netflix.hollow.api.consumer.HollowConsumer.StateRetriever)} + * {@link HollowProducer#restore(long, com.netflix.hollow.api.client.HollowBlobRetriever)} * hasn't been called or the restore failed. * * @param version the version of the state that will become the first of a new delta chain @@ -339,7 +339,7 @@ private RestoreStatus(Status status, long versionDesired, long versionReached, T /** * The version desired to restore to when calling - * {@link HollowProducer#restore(com.netflix.hollow.api.consumer.HollowConsumer.StateRetriever)} + * {@link HollowProducer#restore(long, com.netflix.hollow.api.client.HollowBlobRetriever)} * * @return the latest announced version or {@code Long.MIN_VALUE} if latest announced version couldn't be * retrieved @@ -350,7 +350,7 @@ public long getDesiredVersion() { /** * The version reached when restoring. - * When {@link HollowProducer#restore(com.netflix.hollow.api.consumer.HollowConsumer.StateRetriever)} + * When {@link HollowProducer#restore(long, com.netflix.hollow.api.client.HollowBlobRetriever)} * succeeds then {@code versionDesired == versionReached} is always true. Can be {@code Long.MIN_VALUE} * indicating restore failed to reach any state, or the version of an intermediate state reached. *