diff --git a/README.md b/README.md index 3332453..00437df 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ https://github.com/ess-dmsc/streaming-data-types | ev42 | Event data (deprecated in favour of ev44) | | ev43 | Event data from multiple pulses | | ev44 | Event data with signed data types | +| an44 | ev44 with ANSTO specific changes | | x5f2 | Status messages | | tdct | Timestamps | | ep00 | EPICS connection info (deprecated in favour of ep01) | diff --git a/streaming_data_types/__init__.py b/streaming_data_types/__init__.py index 9f33376..b2624ae 100644 --- a/streaming_data_types/__init__.py +++ b/streaming_data_types/__init__.py @@ -11,18 +11,19 @@ deserialise_ep00, serialise_ep00, ) +from streaming_data_types.eventdata_an44 import deserialise_an44, serialise_an44 from streaming_data_types.eventdata_ev42 import deserialise_ev42, serialise_ev42 from streaming_data_types.eventdata_ev43 import deserialise_ev43, serialise_ev43 from streaming_data_types.eventdata_ev44 import deserialise_ev44, serialise_ev44 from streaming_data_types.finished_writing_wrdn import deserialise_wrdn, serialise_wrdn -from streaming_data_types.forwarder_config_update_rf5k import ( - deserialise_rf5k, - serialise_rf5k, -) from streaming_data_types.forwarder_config_update_fc00 import ( deserialise_fc00, serialise_fc00, ) +from streaming_data_types.forwarder_config_update_rf5k import ( + deserialise_rf5k, + serialise_rf5k, +) from streaming_data_types.histogram_hs00 import deserialise_hs00, serialise_hs00 from streaming_data_types.histogram_hs01 import deserialise_hs01, serialise_hs01 from streaming_data_types.json_json import deserialise_json, serialise_json @@ -41,6 +42,7 @@ __version__ = version SERIALISERS = { + "an44": serialise_an44, "ev42": serialise_ev42, "ev43": serialise_ev43, "ev44": serialise_ev44, @@ -71,6 +73,7 @@ DESERIALISERS = { + "an44": deserialise_an44, "ev42": deserialise_ev42, "ev43": deserialise_ev43, "ev44": deserialise_ev44, diff --git a/streaming_data_types/_version.py b/streaming_data_types/_version.py index c012f41..83ee1a0 100644 --- a/streaming_data_types/_version.py +++ b/streaming_data_types/_version.py @@ -1,4 +1,4 @@ # Version is not directly defined in __init__ because that causes all # run time dependencies to become build-time dependencies when it is # imported in setup.py -version = "0.26.1" +version = "0.27.0" diff --git a/streaming_data_types/eventdata_an44.py b/streaming_data_types/eventdata_an44.py new file mode 100644 index 0000000..0e41e8d --- /dev/null +++ b/streaming_data_types/eventdata_an44.py @@ -0,0 +1,95 @@ +from collections import namedtuple + +import flatbuffers +import numpy as np + +import streaming_data_types.fbschemas.eventdata_an44.AN44EventMessage as AN44EventMessage +from streaming_data_types.utils import check_schema_identifier + +FILE_IDENTIFIER = b"an44" + + +EventData = namedtuple( + "EventData", + ( + "source_name", + "message_id", + "reference_time", + "reference_time_index", + "time_of_flight", + "pixel_id", + "weight", + ), +) + + +def deserialise_an44(buffer): + """ + Deserialise FlatBuffer an44. + + :param buffer: The FlatBuffers buffer. + :return: The deserialised data. + """ + check_schema_identifier(buffer, FILE_IDENTIFIER) + + event = AN44EventMessage.AN44EventMessage.GetRootAs(buffer, 0) + + return EventData( + event.SourceName().decode("utf-8"), + event.MessageId(), + event.ReferenceTimeAsNumpy(), + event.ReferenceTimeIndexAsNumpy(), + event.TimeOfFlightAsNumpy(), + event.PixelIdAsNumpy(), + event.WeightAsNumpy(), + ) + + +def serialise_an44( + source_name, + message_id, + reference_time, + reference_time_index, + time_of_flight, + pixel_id, + weight, +): + """ + Serialise event data as an an44 FlatBuffers message. + + :param source_name: + :param message_id: + :param reference_time: + :param reference_time_index: + :param time_of_flight: + :param pixel_id: + :param weight: + :return: + """ + builder = flatbuffers.Builder(1024) + builder.ForceDefaults(True) + + source = builder.CreateString(source_name) + ref_time_data = builder.CreateNumpyVector( + np.asarray(reference_time).astype(np.int64) + ) + ref_time_index_data = builder.CreateNumpyVector( + np.asarray(reference_time_index).astype(np.int32) + ) + tof_data = builder.CreateNumpyVector(np.asarray(time_of_flight).astype(np.int32)) + pixel_id_data = builder.CreateNumpyVector(np.asarray(pixel_id).astype(np.int32)) + weight_data = builder.CreateNumpyVector(np.asarray(weight).astype(np.int16)) + + AN44EventMessage.AN44EventMessageStart(builder) + AN44EventMessage.AN44EventMessageAddReferenceTime(builder, ref_time_data) + AN44EventMessage.AN44EventMessageAddReferenceTimeIndex(builder, ref_time_index_data) + AN44EventMessage.AN44EventMessageAddTimeOfFlight(builder, tof_data) + AN44EventMessage.AN44EventMessageAddPixelId(builder, pixel_id_data) + AN44EventMessage.AN44EventMessageAddWeight(builder, weight_data) + AN44EventMessage.AN44EventMessageAddMessageId(builder, message_id) + AN44EventMessage.AN44EventMessageAddSourceName(builder, source) + + data = AN44EventMessage.AN44EventMessageEnd(builder) + builder.Finish(data, file_identifier=FILE_IDENTIFIER) + + return bytes(builder.Output()) diff --git a/streaming_data_types/fbschemas/eventdata_an44/AN44EventMessage.py b/streaming_data_types/fbschemas/eventdata_an44/AN44EventMessage.py new file mode 100644 index 0000000..a107bca --- /dev/null +++ b/streaming_data_types/fbschemas/eventdata_an44/AN44EventMessage.py @@ -0,0 +1,322 @@ +# automatically generated by the FlatBuffers compiler, do not modify + +# namespace: + +import flatbuffers +from flatbuffers.compat import import_numpy + +np = import_numpy() + + +class AN44EventMessage(object): + __slots__ = ["_tab"] + + @classmethod + def GetRootAs(cls, buf, offset=0): + n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset) + x = AN44EventMessage() + x.Init(buf, n + offset) + return x + + @classmethod + def GetRootAsAN44EventMessage(cls, buf, offset=0): + """This method is deprecated. Please switch to GetRootAs.""" + return cls.GetRootAs(buf, offset) + + @classmethod + def AN44EventMessageBufferHasIdentifier(cls, buf, offset, size_prefixed=False): + return flatbuffers.util.BufferHasIdentifier( + buf, offset, b"\x61\x6E\x34\x34", size_prefixed=size_prefixed + ) + + # AN44EventMessage + def Init(self, buf, pos): + self._tab = flatbuffers.table.Table(buf, pos) + + # AN44EventMessage + def SourceName(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4)) + if o != 0: + return self._tab.String(o + self._tab.Pos) + return None + + # AN44EventMessage + def MessageId(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6)) + if o != 0: + return self._tab.Get(flatbuffers.number_types.Int64Flags, o + self._tab.Pos) + return 0 + + # AN44EventMessage + def ReferenceTime(self, j): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(8)) + if o != 0: + a = self._tab.Vector(o) + return self._tab.Get( + flatbuffers.number_types.Int64Flags, + a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 8), + ) + return 0 + + # AN44EventMessage + def ReferenceTimeAsNumpy(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(8)) + if o != 0: + return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Int64Flags, o) + return 0 + + # AN44EventMessage + def ReferenceTimeLength(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(8)) + if o != 0: + return self._tab.VectorLen(o) + return 0 + + # AN44EventMessage + def ReferenceTimeIsNone(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(8)) + return o == 0 + + # AN44EventMessage + def ReferenceTimeIndex(self, j): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(10)) + if o != 0: + a = self._tab.Vector(o) + return self._tab.Get( + flatbuffers.number_types.Int32Flags, + a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4), + ) + return 0 + + # AN44EventMessage + def ReferenceTimeIndexAsNumpy(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(10)) + if o != 0: + return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Int32Flags, o) + return 0 + + # AN44EventMessage + def ReferenceTimeIndexLength(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(10)) + if o != 0: + return self._tab.VectorLen(o) + return 0 + + # AN44EventMessage + def ReferenceTimeIndexIsNone(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(10)) + return o == 0 + + # AN44EventMessage + def TimeOfFlight(self, j): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(12)) + if o != 0: + a = self._tab.Vector(o) + return self._tab.Get( + flatbuffers.number_types.Int32Flags, + a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4), + ) + return 0 + + # AN44EventMessage + def TimeOfFlightAsNumpy(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(12)) + if o != 0: + return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Int32Flags, o) + return 0 + + # AN44EventMessage + def TimeOfFlightLength(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(12)) + if o != 0: + return self._tab.VectorLen(o) + return 0 + + # AN44EventMessage + def TimeOfFlightIsNone(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(12)) + return o == 0 + + # AN44EventMessage + def PixelId(self, j): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(14)) + if o != 0: + a = self._tab.Vector(o) + return self._tab.Get( + flatbuffers.number_types.Int32Flags, + a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4), + ) + return 0 + + # AN44EventMessage + def PixelIdAsNumpy(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(14)) + if o != 0: + return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Int32Flags, o) + return 0 + + # AN44EventMessage + def PixelIdLength(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(14)) + if o != 0: + return self._tab.VectorLen(o) + return 0 + + # AN44EventMessage + def PixelIdIsNone(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(14)) + return o == 0 + + # AN44EventMessage + def Weight(self, j): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(16)) + if o != 0: + a = self._tab.Vector(o) + return self._tab.Get( + flatbuffers.number_types.Int16Flags, + a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 2), + ) + return 0 + + # AN44EventMessage + def WeightAsNumpy(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(16)) + if o != 0: + return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Int16Flags, o) + return 0 + + # AN44EventMessage + def WeightLength(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(16)) + if o != 0: + return self._tab.VectorLen(o) + return 0 + + # AN44EventMessage + def WeightIsNone(self): + o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(16)) + return o == 0 + + +def AN44EventMessageStart(builder): + builder.StartObject(7) + + +def Start(builder): + AN44EventMessageStart(builder) + + +def AN44EventMessageAddSourceName(builder, sourceName): + builder.PrependUOffsetTRelativeSlot( + 0, flatbuffers.number_types.UOffsetTFlags.py_type(sourceName), 0 + ) + + +def AddSourceName(builder, sourceName): + AN44EventMessageAddSourceName(builder, sourceName) + + +def AN44EventMessageAddMessageId(builder, messageId): + builder.PrependInt64Slot(1, messageId, 0) + + +def AddMessageId(builder, messageId): + AN44EventMessageAddMessageId(builder, messageId) + + +def AN44EventMessageAddReferenceTime(builder, referenceTime): + builder.PrependUOffsetTRelativeSlot( + 2, flatbuffers.number_types.UOffsetTFlags.py_type(referenceTime), 0 + ) + + +def AddReferenceTime(builder, referenceTime): + AN44EventMessageAddReferenceTime(builder, referenceTime) + + +def AN44EventMessageStartReferenceTimeVector(builder, numElems): + return builder.StartVector(8, numElems, 8) + + +def StartReferenceTimeVector(builder, numElems): + return AN44EventMessageStartReferenceTimeVector(builder, numElems) + + +def AN44EventMessageAddReferenceTimeIndex(builder, referenceTimeIndex): + builder.PrependUOffsetTRelativeSlot( + 3, flatbuffers.number_types.UOffsetTFlags.py_type(referenceTimeIndex), 0 + ) + + +def AddReferenceTimeIndex(builder, referenceTimeIndex): + AN44EventMessageAddReferenceTimeIndex(builder, referenceTimeIndex) + + +def AN44EventMessageStartReferenceTimeIndexVector(builder, numElems): + return builder.StartVector(4, numElems, 4) + + +def StartReferenceTimeIndexVector(builder, numElems): + return AN44EventMessageStartReferenceTimeIndexVector(builder, numElems) + + +def AN44EventMessageAddTimeOfFlight(builder, timeOfFlight): + builder.PrependUOffsetTRelativeSlot( + 4, flatbuffers.number_types.UOffsetTFlags.py_type(timeOfFlight), 0 + ) + + +def AddTimeOfFlight(builder, timeOfFlight): + AN44EventMessageAddTimeOfFlight(builder, timeOfFlight) + + +def AN44EventMessageStartTimeOfFlightVector(builder, numElems): + return builder.StartVector(4, numElems, 4) + + +def StartTimeOfFlightVector(builder, numElems): + return AN44EventMessageStartTimeOfFlightVector(builder, numElems) + + +def AN44EventMessageAddPixelId(builder, pixelId): + builder.PrependUOffsetTRelativeSlot( + 5, flatbuffers.number_types.UOffsetTFlags.py_type(pixelId), 0 + ) + + +def AddPixelId(builder, pixelId): + AN44EventMessageAddPixelId(builder, pixelId) + + +def AN44EventMessageStartPixelIdVector(builder, numElems): + return builder.StartVector(4, numElems, 4) + + +def StartPixelIdVector(builder, numElems): + return AN44EventMessageStartPixelIdVector(builder, numElems) + + +def AN44EventMessageAddWeight(builder, weight): + builder.PrependUOffsetTRelativeSlot( + 6, flatbuffers.number_types.UOffsetTFlags.py_type(weight), 0 + ) + + +def AddWeight(builder, weight): + AN44EventMessageAddWeight(builder, weight) + + +def AN44EventMessageStartWeightVector(builder, numElems): + return builder.StartVector(2, numElems, 2) + + +def StartWeightVector(builder, numElems): + return AN44EventMessageStartWeightVector(builder, numElems) + + +def AN44EventMessageEnd(builder): + return builder.EndObject() + + +def End(builder): + return AN44EventMessageEnd(builder) diff --git a/streaming_data_types/fbschemas/eventdata_an44/__init__.py b/streaming_data_types/fbschemas/eventdata_an44/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_an44.py b/tests/test_an44.py new file mode 100644 index 0000000..cdd8cf8 --- /dev/null +++ b/tests/test_an44.py @@ -0,0 +1,118 @@ +import numpy as np +import pytest + +from streaming_data_types import DESERIALISERS, SERIALISERS +from streaming_data_types.eventdata_an44 import deserialise_an44, serialise_an44 +from streaming_data_types.exceptions import WrongSchemaException + + +class TestSerialisationEv44: + def test_serialises_and_deserialises_an44_message_correctly(self): + """ + Round-trip to check what we serialise is what we get back. + """ + original_entry = { + "source_name": "some_source", + "message_id": 123456, + "reference_time": [ + 1618573589123781958, + 1618573590133830371, + 1618573593677164112, + 1618573594185190549, + 1618573596217316066, + 1618573596725363109, + 1618573601295720976, + 1618573601799761445, + 1618573607354064836, + ], + "reference_time_index": [2, 4, 5, 7], + "time_of_flight": [100, 200, 300, 400, 500, 600, 700, 800, 900], + "pixel_id": [10, 20, 30, 40, 50, 60, 70, 80, 90], + "weight": [51, 52, 53, 54, 55, 56, 57, 58, 59], + } + + buf = serialise_an44(**original_entry) + entry = deserialise_an44(buf) + + assert entry.source_name == original_entry["source_name"] + assert entry.message_id == original_entry["message_id"] + assert np.array_equal(entry.reference_time, original_entry["reference_time"]) + assert np.array_equal( + entry.reference_time_index, original_entry["reference_time_index"] + ) + assert np.array_equal(entry.time_of_flight, original_entry["time_of_flight"]) + assert np.array_equal(entry.pixel_id, original_entry["pixel_id"]) + assert np.array_equal(entry.weight, original_entry["weight"]) + + def test_serialises_and_deserialises_an44_message_correctly_for_numpy_arrays(self): + """ + Round-trip to check what we serialise is what we get back. + """ + original_entry = { + "source_name": "some_source", + "message_id": 123456, + "reference_time": np.array( + [ + 1618573589123781958, + 1618573590133830371, + 1618573593677164112, + 1618573594185190549, + 1618573596217316066, + 1618573596725363109, + 1618573601295720976, + 1618573601799761445, + 1618573607354064836, + ] + ), + "reference_time_index": np.array([2, 4, 5, 7]), + "time_of_flight": np.array([100, 200, 300, 400, 500, 600, 700, 800, 900]), + "pixel_id": np.array([10, 20, 30, 40, 50, 60, 70, 80, 90]), + "weight": np.array([51, 52, 53, 54, 55, 56, 57, 58, 59]), + } + + buf = serialise_an44(**original_entry) + entry = deserialise_an44(buf) + + assert entry.source_name == original_entry["source_name"] + assert entry.message_id == original_entry["message_id"] + assert np.array_equal(entry.reference_time, original_entry["reference_time"]) + assert np.array_equal( + entry.reference_time_index, original_entry["reference_time_index"] + ) + assert np.array_equal(entry.time_of_flight, original_entry["time_of_flight"]) + assert np.array_equal(entry.pixel_id, original_entry["pixel_id"]) + + def test_if_buffer_has_wrong_id_then_throws(self): + original_entry = { + "source_name": "some_source", + "message_id": 123456, + "reference_time": np.array( + [ + 1618573589123781958, + 1618573590133830371, + 1618573593677164112, + 1618573594185190549, + 1618573596217316066, + 1618573596725363109, + 1618573601295720976, + 1618573601799761445, + 1618573607354064836, + ] + ), + "reference_time_index": np.array([2, 4, 5, 7]), + "time_of_flight": np.array([100, 200, 300, 400, 500, 600, 700, 800, 900]), + "pixel_id": np.array([10, 20, 30, 40, 50, 60, 70, 80, 90]), + "weight": np.array([51, 52, 53, 54, 55, 56, 57, 58, 59]), + } + buf = serialise_an44(**original_entry) + + # Manually introduce error in id. + buf = bytearray(buf) + buf[4:8] = b"1234" + + with pytest.raises(WrongSchemaException): + deserialise_an44(buf) + + def test_schema_type_is_in_global_serialisers_list(self): + assert "an44" in SERIALISERS + assert "an44" in DESERIALISERS