Merge pull request #53 from geishm-ansto/main

added ev43 and fix int8 f142 serialise

.pre-commit-config.yaml

@@ -3,7 +3,7 @@ repos:
     rev: stable
     hooks:
     - id: black
-      language_version: python3.7
+      language_version: python3.6
 -   repo: https://github.com/pycqa/flake8
     rev: 3.8.3
     hooks:

README.md

@@ -14,6 +14,7 @@ https://github.com/ess-dmsc/streaming-data-types
 |6s4t|Run stop|
 |f142|Log data|
 |ev42|Event data|
+|ev43|Event data from multiple pulses|
 |x5f2|Status messages|
 |tdct|Timestamps|
 |ep00|EPICS connection info|

streaming_data_types/__init__.py

@@ -1,4 +1,5 @@
 from streaming_data_types.eventdata_ev42 import deserialise_ev42, serialise_ev42
+from streaming_data_types.eventdata_ev43 import deserialise_ev43, serialise_ev43
 from streaming_data_types.histogram_hs00 import deserialise_hs00, serialise_hs00
 from streaming_data_types.logdata_f142 import deserialise_f142, serialise_f142
 from streaming_data_types.nicos_cache_ns10 import deserialise_ns10, serialise_ns10
@@ -28,6 +29,7 @@
 
 SERIALISERS = {
     "ev42": serialise_ev42,
+    "ev43": serialise_ev43,
     "hs00": serialise_hs00,
     "f142": serialise_f142,
     "ns10": serialise_ns10,
@@ -47,6 +49,7 @@
 
 DESERIALISERS = {
     "ev42": deserialise_ev42,
+    "ev43": deserialise_ev43,
     "hs00": deserialise_hs00,
     "f142": deserialise_f142,
     "ns10": deserialise_ns10,

streaming_data_types/eventdata_ev43.py

@@ -0,0 +1,81 @@
+from collections import namedtuple
+import flatbuffers
+import streaming_data_types.fbschemas.eventdata_ev43.Event43Message as Event43Message
+from streaming_data_types.utils import check_schema_identifier
+import numpy as np
+
+
+FILE_IDENTIFIER = b"ev43"
+
+
+EventData = namedtuple(
+    "EventData",
+    (
+        "source_name",
+        "message_id",
+        "pulse_time",
+        "pulse_index",
+        "time_of_flight",
+        "detector_id",
+    ),
+)
+
+
+def deserialise_ev43(buffer):
+    """
+    Deserialise FlatBuffer ev43.
+
+    :param buffer: The FlatBuffers buffer.
+    :return: The deserialised data.
+    """
+    check_schema_identifier(buffer, FILE_IDENTIFIER)
+
+    event = Event43Message.Event43Message.GetRootAsEvent43Message(buffer, 0)
+
+    return EventData(
+        event.SourceName().decode("utf-8"),
+        event.MessageId(),
+        event.PulseTimeAsNumpy(),
+        event.PulseIndexAsNumpy(),
+        event.TimeOfFlightAsNumpy(),
+        event.DetectorIdAsNumpy(),
+    )
+
+
+def serialise_ev43(
+    source_name, message_id, pulse_time, pulse_index, time_of_flight, detector_id
+):
+    """
+    Serialise event data as an ev43 FlatBuffers message.
+
+    :param source_name:
+    :param message_id:
+    :param pulse_time:
+    :param pulse_index:
+    :param time_of_flight:
+    :param detector_id:
+    :return:
+    """
+    builder = flatbuffers.Builder(1024)
+    builder.ForceDefaults(True)
+
+    source = builder.CreateString(source_name)
+
+    pulse_ts_data = builder.CreateNumpyVector(np.asarray(pulse_time).astype(np.uint64))
+    pulse_ix_data = builder.CreateNumpyVector(np.asarray(pulse_index).astype(np.uint32))
+    tof_data = builder.CreateNumpyVector(np.asarray(time_of_flight).astype(np.uint32))
+    det_data = builder.CreateNumpyVector(np.asarray(detector_id).astype(np.uint32))
+
+    # Build the actual buffer
+    Event43Message.Event43MessageStart(builder)
+    Event43Message.Event43MessageAddPulseTime(builder, pulse_ts_data)
+    Event43Message.Event43MessageAddPulseIndex(builder, pulse_ix_data)
+    Event43Message.Event43MessageAddDetectorId(builder, det_data)
+    Event43Message.Event43MessageAddTimeOfFlight(builder, tof_data)
+    Event43Message.Event43MessageAddMessageId(builder, message_id)
+    Event43Message.Event43MessageAddSourceName(builder, source)
+
+    data = Event43Message.Event43MessageEnd(builder)
+
+    builder.Finish(data, file_identifier=FILE_IDENTIFIER)
+    return bytes(builder.Output())

streaming_data_types/fbschemas/eventdata_ev43/Event43Message.py

@@ -0,0 +1,223 @@
+# automatically generated by the FlatBuffers compiler, do not modify
+
+# namespace:
+
+import flatbuffers
+from flatbuffers.compat import import_numpy
+
+np = import_numpy()
+
+
+class Event43Message(object):
+    __slots__ = ["_tab"]
+
+    @classmethod
+    def GetRootAsEvent43Message(cls, buf, offset):
+        n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset)
+        x = Event43Message()
+        x.Init(buf, n + offset)
+        return x
+
+    @classmethod
+    def Event43MessageBufferHasIdentifier(cls, buf, offset, size_prefixed=False):
+        return flatbuffers.util.BufferHasIdentifier(
+            buf, offset, b"\x65\x76\x34\x33", size_prefixed=size_prefixed
+        )
+
+    # Event43Message
+    def Init(self, buf, pos):
+        self._tab = flatbuffers.table.Table(buf, pos)
+
+    # Event43Message
+    def SourceName(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
+        if o != 0:
+            return self._tab.String(o + self._tab.Pos)
+        return None
+
+    # Event43Message
+    def MessageId(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6))
+        if o != 0:
+            return self._tab.Get(
+                flatbuffers.number_types.Uint64Flags, o + self._tab.Pos
+            )
+        return 0
+
+    # Event43Message
+    def PulseTime(self, j):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(8))
+        if o != 0:
+            a = self._tab.Vector(o)
+            return self._tab.Get(
+                flatbuffers.number_types.Uint64Flags,
+                a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 8),
+            )
+        return 0
+
+    # Event43Message
+    def PulseTimeAsNumpy(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(8))
+        if o != 0:
+            return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Uint64Flags, o)
+        return 0
+
+    # Event43Message
+    def PulseTimeLength(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(8))
+        if o != 0:
+            return self._tab.VectorLen(o)
+        return 0
+
+    # Event43Message
+    def PulseTimeIsNone(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(8))
+        return o == 0
+
+    # Event43Message
+    def PulseIndex(self, j):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(10))
+        if o != 0:
+            a = self._tab.Vector(o)
+            return self._tab.Get(
+                flatbuffers.number_types.Uint32Flags,
+                a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4),
+            )
+        return 0
+
+    # Event43Message
+    def PulseIndexAsNumpy(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(10))
+        if o != 0:
+            return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Uint32Flags, o)
+        return 0
+
+    # Event43Message
+    def PulseIndexLength(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(10))
+        if o != 0:
+            return self._tab.VectorLen(o)
+        return 0
+
+    # Event43Message
+    def PulseIndexIsNone(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(10))
+        return o == 0
+
+    # Event43Message
+    def TimeOfFlight(self, j):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(12))
+        if o != 0:
+            a = self._tab.Vector(o)
+            return self._tab.Get(
+                flatbuffers.number_types.Uint32Flags,
+                a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4),
+            )
+        return 0
+
+    # Event43Message
+    def TimeOfFlightAsNumpy(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(12))
+        if o != 0:
+            return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Uint32Flags, o)
+        return 0
+
+    # Event43Message
+    def TimeOfFlightLength(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(12))
+        if o != 0:
+            return self._tab.VectorLen(o)
+        return 0
+
+    # Event43Message
+    def TimeOfFlightIsNone(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(12))
+        return o == 0
+
+    # Event43Message
+    def DetectorId(self, j):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(14))
+        if o != 0:
+            a = self._tab.Vector(o)
+            return self._tab.Get(
+                flatbuffers.number_types.Uint32Flags,
+                a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4),
+            )
+        return 0
+
+    # Event43Message
+    def DetectorIdAsNumpy(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(14))
+        if o != 0:
+            return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Uint32Flags, o)
+        return 0
+
+    # Event43Message
+    def DetectorIdLength(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(14))
+        if o != 0:
+            return self._tab.VectorLen(o)
+        return 0
+
+    # Event43Message
+    def DetectorIdIsNone(self):
+        o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(14))
+        return o == 0
+
+
+def Event43MessageStart(builder):
+    builder.StartObject(6)
+
+
+def Event43MessageAddSourceName(builder, sourceName):
+    builder.PrependUOffsetTRelativeSlot(
+        0, flatbuffers.number_types.UOffsetTFlags.py_type(sourceName), 0
+    )
+
+
+def Event43MessageAddMessageId(builder, messageId):
+    builder.PrependUint64Slot(1, messageId, 0)
+
+
+def Event43MessageAddPulseTime(builder, pulseTime):
+    builder.PrependUOffsetTRelativeSlot(
+        2, flatbuffers.number_types.UOffsetTFlags.py_type(pulseTime), 0
+    )
+
+
+def Event43MessageStartPulseTimeVector(builder, numElems):
+    return builder.StartVector(8, numElems, 8)
+
+
+def Event43MessageAddPulseIndex(builder, pulseIndex):
+    builder.PrependUOffsetTRelativeSlot(
+        3, flatbuffers.number_types.UOffsetTFlags.py_type(pulseIndex), 0
+    )
+
+
+def Event43MessageStartPulseIndexVector(builder, numElems):
+    return builder.StartVector(4, numElems, 4)
+
+
+def Event43MessageAddTimeOfFlight(builder, timeOfFlight):
+    builder.PrependUOffsetTRelativeSlot(
+        4, flatbuffers.number_types.UOffsetTFlags.py_type(timeOfFlight), 0
+    )
+
+
+def Event43MessageStartTimeOfFlightVector(builder, numElems):
+    return builder.StartVector(4, numElems, 4)
+
+
+def Event43MessageAddDetectorId(builder, detectorId):
+    builder.PrependUOffsetTRelativeSlot(
+        5, flatbuffers.number_types.UOffsetTFlags.py_type(detectorId), 0
+    )
+
+
+def Event43MessageStartDetectorIdVector(builder, numElems):
+    return builder.StartVector(4, numElems, 4)
+
+
+def Event43MessageEnd(builder):
+    return builder.EndObject()

streaming_data_types/logdata_f142.py

@@ -451,11 +451,11 @@ def _serialise_stringarray(builder: flatbuffers.Builder, data: np.ndarray, sourc
 _map_scalar_type_to_serialiser = {
     np.dtype("byte"): _serialise_byte,
     np.dtype("ubyte"): _serialise_ubyte,
-    np.dtype("int8"): _serialise_short,
+    np.dtype("int8"): _serialise_byte,
     np.dtype("int16"): _serialise_short,
     np.dtype("int32"): _serialise_int,
     np.dtype("int64"): _serialise_long,
-    np.dtype("uint8"): _serialise_ushort,
+    np.dtype("uint8"): _serialise_ubyte,
     np.dtype("uint16"): _serialise_ushort,
     np.dtype("uint32"): _serialise_uint,
     np.dtype("uint64"): _serialise_ulong,
@@ -466,11 +466,11 @@ def _serialise_stringarray(builder: flatbuffers.Builder, data: np.ndarray, sourc
 _map_array_type_to_serialiser = {
     np.dtype("byte"): _serialise_bytearray,
     np.dtype("ubyte"): _serialise_ubytearray,
-    np.dtype("int8"): _serialise_shortarray,
+    np.dtype("int8"): _serialise_bytearray,
     np.dtype("int16"): _serialise_shortarray,
     np.dtype("int32"): _serialise_intarray,
     np.dtype("int64"): _serialise_longarray,
-    np.dtype("uint8"): _serialise_ushortarray,
+    np.dtype("uint8"): _serialise_ubytearray,
     np.dtype("uint16"): _serialise_ushortarray,
     np.dtype("uint32"): _serialise_uintarray,
     np.dtype("uint64"): _serialise_ulongarray,