diff --git a/benchmarks/NexusMods.MnemonicDB.Benchmarks/Benchmarks/IndexSegmentABenchmarks.cs b/benchmarks/NexusMods.MnemonicDB.Benchmarks/Benchmarks/IndexSegmentABenchmarks.cs new file mode 100644 index 00000000..204de990 --- /dev/null +++ b/benchmarks/NexusMods.MnemonicDB.Benchmarks/Benchmarks/IndexSegmentABenchmarks.cs @@ -0,0 +1,48 @@ +using System; +using BenchmarkDotNet.Attributes; +using NexusMods.MnemonicDB.Abstractions; +using NexusMods.MnemonicDB.Abstractions.DatomIterators; +using NexusMods.MnemonicDB.Abstractions.ElementComparers; +using NexusMods.MnemonicDB.Abstractions.IndexSegments; +using NexusMods.MnemonicDB.Abstractions.Internals; +using NexusMods.MnemonicDB.Storage; + +namespace NexusMods.MnemonicDB.Benchmarks.Benchmarks; + +public class IndexSegmentABenchmarks +{ + private readonly IndexSegment _index; + + public IndexSegmentABenchmarks() + { + var registry = new AttributeRegistry([]); + using var builder = new IndexSegmentBuilder(registry); + + for (int a = 1; a < 100; a++) + { + var prefix = new KeyPrefix(EntityId.From(42), AttributeId.From((ushort)a), TxId.From(42), false, + ValueTags.Null); + builder.Add(new Datom(prefix, ReadOnlyMemory.Empty, registry)); + } + + _index = builder.Build(); + } + + [Params(1, 2, 3, 4, 5, 20, 30, 50, 99)] + public int ToFind { get; set; } + + [Benchmark] + public int FindLinear() + { + var find = AttributeId.From((ushort)ToFind); + for (var i = 0; i < _index.Count; i++) + { + var datom = _index[i]; + if (datom.A == find) + return i; + } + return -1; + } + + +} diff --git a/benchmarks/NexusMods.MnemonicDB.Benchmarks/Benchmarks/IndexSegmentEBenchmarks.cs b/benchmarks/NexusMods.MnemonicDB.Benchmarks/Benchmarks/IndexSegmentEBenchmarks.cs new file mode 100644 index 00000000..8470cbba --- /dev/null +++ b/benchmarks/NexusMods.MnemonicDB.Benchmarks/Benchmarks/IndexSegmentEBenchmarks.cs @@ -0,0 +1,95 @@ +using System; +using BenchmarkDotNet.Attributes; +using NexusMods.MnemonicDB.Abstractions; +using NexusMods.MnemonicDB.Abstractions.DatomIterators; +using NexusMods.MnemonicDB.Abstractions.ElementComparers; +using NexusMods.MnemonicDB.Abstractions.IndexSegments; +using NexusMods.MnemonicDB.Abstractions.Internals; +using NexusMods.MnemonicDB.Storage; + +namespace NexusMods.MnemonicDB.Benchmarks.Benchmarks; + +[MemoryDiagnoser] +public class IndexSegmentEBenchmarks +{ + private readonly IndexSegment _index; + + public IndexSegmentEBenchmarks() + { + var registry = new AttributeRegistry([]); + using var builder = new IndexSegmentBuilder(registry); + + for (var e = 1; e < 100; e++) + { + for (var a = 0; a < 20; a++) + { + builder.Add(new Datom(new KeyPrefix(EntityId.From((ulong)e), AttributeId.From((ushort)a), TxId.From((ulong)(e + a)), false, + ValueTags.Null), ReadOnlyMemory.Empty, registry)); + } + } + + _index = builder.Build(); + } + + [Params(0, 1, 2, 3, 10, 71, 99)] public int ToFind { get; set; } + + + [Benchmark] + public int FindLinear() + { + var find = EntityId.From((ulong)ToFind); + for (var i = 0; i < _index.Count; i++) + { + var datom = _index[i]; + if (datom.E == find) + return i; + } + return -1; + } + + [Benchmark] + public int FindBinarySearch() + { + var find = EntityId.From((ulong)ToFind); + + var left = 0; + var right = _index.Count - 1; + var result = -1; + while (left <= right) + { + var mid = left + (right - left) / 2; + var datom = _index[mid]; + var comparison = datom.E.CompareTo(find); + if (comparison == 0) + { + result = mid; // Don't return, but continue searching to the left + right = mid - 1; + } + else if (comparison < 0) + { + left = mid + 1; + } + else + { + right = mid - 1; + } + } + return result; // Return the first occurrence found, or -1 if not found + } + + [Benchmark] + public int FindBinarySearchRework() + { + var find = EntityId.From((ulong)ToFind); + return _index.FindFirst(find); // Return the first occurrence found, or -1 if not found + } + + [Benchmark] + public int FindBinarySearchReworkAVX2() + { + var find = EntityId.From((ulong)ToFind); + return _index.FindFirstAVX2(find.Value); // Return the first occurrence found, or -1 if not found + } + + +} diff --git a/benchmarks/NexusMods.MnemonicDB.Benchmarks/Program.cs b/benchmarks/NexusMods.MnemonicDB.Benchmarks/Program.cs index 6357e958..8c20ac7e 100644 --- a/benchmarks/NexusMods.MnemonicDB.Benchmarks/Program.cs +++ b/benchmarks/NexusMods.MnemonicDB.Benchmarks/Program.cs @@ -2,13 +2,14 @@ using System; using System.Diagnostics; +using BenchmarkDotNet.Configs; using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Running; using JetBrains.Profiler.Api; using NexusMods.MnemonicDB.Benchmarks.Benchmarks; -//#if DEBUG +#if DEBUG var benchmark = new ReadTests { @@ -29,12 +30,9 @@ MeasureProfiler.SaveData(); Console.WriteLine("Elapsed: " + sw.Elapsed + " Result: " + result); -/* #else -BenchmarkRunner.Run(); - +BenchmarkRunner.Run(config: DefaultConfig.Instance.WithOption(ConfigOptions.DisableOptimizationsValidator, true)); #endif -*/ diff --git a/src/NexusMods.MnemonicDB.Abstractions/Attribute.cs b/src/NexusMods.MnemonicDB.Abstractions/Attribute.cs index 7ed97007..f0203abb 100644 --- a/src/NexusMods.MnemonicDB.Abstractions/Attribute.cs +++ b/src/NexusMods.MnemonicDB.Abstractions/Attribute.cs @@ -2,14 +2,11 @@ using System.Buffers; using System.Collections.Generic; using System.Diagnostics; -using System.Linq; -using System.Numerics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Text; using NexusMods.MnemonicDB.Abstractions.ElementComparers; using NexusMods.MnemonicDB.Abstractions.Exceptions; -using NexusMods.MnemonicDB.Abstractions.IndexSegments; using NexusMods.MnemonicDB.Abstractions.Internals; using NexusMods.MnemonicDB.Abstractions.Models; using Reloaded.Memory.Extensions; @@ -198,13 +195,13 @@ public AttributeId GetDbId(RegistryId id) /// public IReadDatom Resolve(EntityId entityId, AttributeId attributeId, ReadOnlySpan value, TxId tx, - bool isRetract) + bool isRetract, ValueTags valueTag) { return new ReadDatom { E = entityId, A = this, - V = ReadValue(value), + V = ReadValue(value, valueTag), T = tx, IsRetract = isRetract }; @@ -310,54 +307,47 @@ private void WriteValueLowLevel(TLowLevelType value, TWriter writer) private void WriteNull(TWriter writer) where TWriter : IBufferWriter { - var span = writer.GetSpan(1); - span[0] = (byte)ValueTags.Null; - writer.Advance(1); + // Do Nothing } private void WriteAscii(string s, TWriter writer) where TWriter : IBufferWriter { var size = s.Length; - var span = writer.GetSpan(size + 1); - span[0] = (byte)LowLevelType; - AsciiEncoding.GetBytes(s, span.SliceFast(1)); - writer.Advance(size + 1); + var span = writer.GetSpan(size); + AsciiEncoding.GetBytes(s, span); + writer.Advance(size); } private void WriteUtf8(string s, TWriter writer) where TWriter : IBufferWriter { var size = Utf8Encoding.GetByteCount(s); - var span = writer.GetSpan(size + 1); - span[0] = (byte)LowLevelType; - Utf8Encoding.GetBytes(s, span.SliceFast(1)); - writer.Advance(size + 1); + var span = writer.GetSpan(size); + Utf8Encoding.GetBytes(s, span); + writer.Advance(size); } - public TValueType ReadValue(ReadOnlySpan span) + public TValueType ReadValue(ReadOnlySpan span, ValueTags tag) { - var tag = (ValueTags)span[0]; - var rest = span.SliceFast(1); - Debug.Assert(tag == LowLevelType, "Tag mismatch"); - return tag switch + return LowLevelType switch { ValueTags.Null => NullFromLowLevel(), - ValueTags.UInt8 => FromLowLevel(ReadUnmanaged(rest), tag), - ValueTags.UInt16 => FromLowLevel(ReadUnmanaged(rest), tag), - ValueTags.UInt32 => FromLowLevel(ReadUnmanaged(rest), tag), - ValueTags.UInt64 => FromLowLevel(ReadUnmanaged(rest), tag), - ValueTags.UInt128 => FromLowLevel(ReadUnmanaged(rest), tag), - ValueTags.Int16 => FromLowLevel(ReadUnmanaged(rest), tag), - ValueTags.Int32 => FromLowLevel(ReadUnmanaged(rest), tag), - ValueTags.Int64 => FromLowLevel(ReadUnmanaged(rest), tag), - ValueTags.Int128 => FromLowLevel(ReadUnmanaged(rest), tag), - ValueTags.Float32 => FromLowLevel(ReadUnmanaged(rest), tag), - ValueTags.Float64 => FromLowLevel(ReadUnmanaged(rest), tag), - ValueTags.Reference => FromLowLevel(ReadUnmanaged(rest), tag), - ValueTags.Ascii => FromLowLevel(ReadAscii(rest), tag), - ValueTags.Utf8 => FromLowLevel(ReadUtf8(rest), tag), - ValueTags.Utf8Insensitive => FromLowLevel(ReadUtf8(rest), tag), - ValueTags.Blob => FromLowLevel(rest, tag), - ValueTags.HashedBlob => FromLowLevel(rest.SliceFast(sizeof(ulong)), tag), + ValueTags.UInt8 => FromLowLevel(ReadUnmanaged(span), tag), + ValueTags.UInt16 => FromLowLevel(ReadUnmanaged(span), tag), + ValueTags.UInt32 => FromLowLevel(ReadUnmanaged(span), tag), + ValueTags.UInt64 => FromLowLevel(ReadUnmanaged(span), tag), + ValueTags.UInt128 => FromLowLevel(ReadUnmanaged(span), tag), + ValueTags.Int16 => FromLowLevel(ReadUnmanaged(span), tag), + ValueTags.Int32 => FromLowLevel(ReadUnmanaged(span), tag), + ValueTags.Int64 => FromLowLevel(ReadUnmanaged(span), tag), + ValueTags.Int128 => FromLowLevel(ReadUnmanaged(span), tag), + ValueTags.Float32 => FromLowLevel(ReadUnmanaged(span), tag), + ValueTags.Float64 => FromLowLevel(ReadUnmanaged(span), tag), + ValueTags.Reference => FromLowLevel(ReadUnmanaged(span), tag), + ValueTags.Ascii => FromLowLevel(ReadAscii(span), tag), + ValueTags.Utf8 => FromLowLevel(ReadUtf8(span), tag), + ValueTags.Utf8Insensitive => FromLowLevel(ReadUtf8(span), tag), + ValueTags.Blob => FromLowLevel(span, tag), + ValueTags.HashedBlob => FromLowLevel(span.SliceFast(sizeof(ulong)), tag), _ => throw new UnsupportedLowLevelReadType(tag) }; } @@ -382,10 +372,9 @@ private unsafe void WriteUnmanaged(TValue value, TWriter writer where TWriter : IBufferWriter where TValue : unmanaged { - var span = writer.GetSpan(sizeof(TValue) + 1); - span[0] = (byte) LowLevelType; - MemoryMarshal.Write(span.SliceFast(1), value); - writer.Advance(sizeof(TValue) + 1); + var span = writer.GetSpan(sizeof(TValue)); + MemoryMarshal.Write(span, value); + writer.Advance(sizeof(TValue)); } private TValue ReadUnmanaged(ReadOnlySpan span) @@ -401,7 +390,7 @@ public virtual void Write(EntityId entityId, RegistryId registryId, TVa where TWriter : IBufferWriter { Debug.Assert(LowLevelType != ValueTags.Blob, "Blobs should overwrite this method and throw when ToLowLevel is called"); - var prefix = new KeyPrefix().Set(entityId, GetDbId(registryId), txId, isRetract); + var prefix = new KeyPrefix(entityId, GetDbId(registryId), txId, isRetract, LowLevelType); var span = writer.GetSpan(KeyPrefix.Size); MemoryMarshal.Write(span, prefix); writer.Advance(KeyPrefix.Size); @@ -414,7 +403,7 @@ public virtual void Write(EntityId entityId, RegistryId registryId, TVa protected void WritePrefix(EntityId entityId, RegistryId registryId, TxId txId, bool isRetract, TWriter writer) where TWriter : IBufferWriter { - var prefix = new KeyPrefix().Set(entityId, GetDbId(registryId), txId, isRetract); + var prefix = new KeyPrefix(entityId, GetDbId(registryId), txId, isRetract, LowLevelType); var span = writer.GetSpan(KeyPrefix.Size); MemoryMarshal.Write(span, prefix); writer.Advance(KeyPrefix.Size); diff --git a/src/NexusMods.MnemonicDB.Abstractions/Attributes/BlobAttribute.cs b/src/NexusMods.MnemonicDB.Abstractions/Attributes/BlobAttribute.cs index 91b5e23c..57cc286c 100644 --- a/src/NexusMods.MnemonicDB.Abstractions/Attributes/BlobAttribute.cs +++ b/src/NexusMods.MnemonicDB.Abstractions/Attributes/BlobAttribute.cs @@ -15,9 +15,6 @@ public abstract class BlobAttribute(string ns, string name) : ScalarAttr public override void Write(EntityId entityId, RegistryId registryId, TValue value, TxId txId, bool isRetract, TWriter writer) { WritePrefix(entityId, registryId, txId, isRetract, writer); - var span = writer.GetSpan(1); - span[0] = (byte)ValueTags.Blob; - writer.Advance(1); WriteValue(value, writer); } diff --git a/src/NexusMods.MnemonicDB.Abstractions/Attributes/HashedBlobAttribute.cs b/src/NexusMods.MnemonicDB.Abstractions/Attributes/HashedBlobAttribute.cs index 50ec4b38..8db188ce 100644 --- a/src/NexusMods.MnemonicDB.Abstractions/Attributes/HashedBlobAttribute.cs +++ b/src/NexusMods.MnemonicDB.Abstractions/Attributes/HashedBlobAttribute.cs @@ -19,10 +19,9 @@ public override void Write(EntityId entityId, RegistryId registryId, TV var hash = XxHash3.HashToUInt64(valueSpan); - var writerSpan = writer.GetSpan(sizeof(ulong) + 1); - writerSpan[0] = (byte)ValueTags.HashedBlob; - MemoryMarshal.Write(writerSpan.SliceFast(1), hash); - writer.Advance(sizeof(ulong) + 1); + var writerSpan = writer.GetSpan(sizeof(ulong)); + MemoryMarshal.Write(writerSpan, hash); + writer.Advance(sizeof(ulong)); writer.Write(valueSpan); } diff --git a/src/NexusMods.MnemonicDB.Abstractions/Attributes/ScalarAttribute.cs b/src/NexusMods.MnemonicDB.Abstractions/Attributes/ScalarAttribute.cs index 71db80d6..cd67dad3 100644 --- a/src/NexusMods.MnemonicDB.Abstractions/Attributes/ScalarAttribute.cs +++ b/src/NexusMods.MnemonicDB.Abstractions/Attributes/ScalarAttribute.cs @@ -30,7 +30,7 @@ public TValue Get(IHasEntityIdAndDb entity) { var datom = segment[i]; if (datom.A != dbId) continue; - return ReadValue(datom.ValueSpan); + return ReadValue(datom.ValueSpan, datom.Prefix.ValueTag); } if (DefaultValue is not null) @@ -64,7 +64,7 @@ public bool TryGet(IHasEntityIdAndDb entity, out TValue value) { var datom = segment[i]; if (datom.A != dbId) continue; - value = ReadValue(datom.ValueSpan); + value = ReadValue(datom.ValueSpan, datom.Prefix.ValueTag); return true; } diff --git a/src/NexusMods.MnemonicDB.Abstractions/Datom.cs b/src/NexusMods.MnemonicDB.Abstractions/Datom.cs index 86643dcc..bf550656 100644 --- a/src/NexusMods.MnemonicDB.Abstractions/Datom.cs +++ b/src/NexusMods.MnemonicDB.Abstractions/Datom.cs @@ -9,38 +9,63 @@ namespace NexusMods.MnemonicDB.Abstractions.DatomIterators; /// Represents a raw (unparsed) datom from an index. Most of the time this datom is only valid for the /// lifetime of the current iteration. It is not safe to store this datom for later use. /// -public struct Datom(ReadOnlyMemory memory, IAttributeRegistry registry) +public readonly struct Datom { + private readonly KeyPrefix _prefix; + private readonly ReadOnlyMemory _valueBlob; + private readonly IAttributeRegistry _registry; + /// - /// A span of the raw datom + /// Create a new datom from the given prefix and value /// - public ReadOnlySpan RawSpan => memory.Span; + public Datom(in KeyPrefix prefix, ReadOnlyMemory value, IAttributeRegistry registry) + { + _registry = registry; + _prefix = prefix; + _valueBlob = value; + } /// - /// The KeyPrefix of the datom + /// Create a new datom from the given datom memory span and registry /// - public KeyPrefix Prefix => MemoryMarshal.Read(RawSpan); + public Datom(ReadOnlyMemory datom, IAttributeRegistry registry) + { + _registry = registry; + _prefix = KeyPrefix.Read(datom.Span); + _valueBlob = datom[KeyPrefix.Size..]; + } /// - /// The valuespan of the datom + /// Converts the entire datom into a byte array /// - public ReadOnlySpan ValueSpan => RawSpan.SliceFast(KeyPrefix.Size); + public byte[] ToArray() + { + var array = new byte[KeyPrefix.Size + _valueBlob.Length]; + MemoryMarshal.Write(array, _prefix); + _valueBlob.Span.CopyTo(array.AsSpan(KeyPrefix.Size)); + return array; + } /// - /// The attribute registry for this datom + /// The KeyPrefix of the datom + /// + public KeyPrefix Prefix => _prefix; + + /// + /// The valuespan of the datom /// - public IAttributeRegistry Registry => registry; + public ReadOnlySpan ValueSpan => _valueBlob.Span; /// /// Resolves this datom into the IReadDatom form /// - public IReadDatom Resolved => registry.Resolve(RawSpan); + public IReadDatom Resolved => _registry.Resolve(_prefix, _valueBlob.Span); /// /// Resolves the value of the datom into the given type /// public TValue Resolve(Attribute attribute) => - attribute.ReadValue(ValueSpan); + attribute.ReadValue(ValueSpan, Prefix.ValueTag); /// /// EntityId of the datom @@ -67,15 +92,13 @@ public TValue Resolve(Attribute attribute) /// public Datom Clone() { - var copy = new byte[memory.Length]; - memory.Span.CopyTo(copy); - return new Datom(copy, registry); + return new Datom(_prefix, _valueBlob.ToArray(), _registry); } /// /// Returns true if the datom is valid /// - public bool Valid => memory.Length > 0; + public bool Valid => _prefix.IsValid; /// public override string ToString() @@ -92,19 +115,19 @@ public int Compare(Datom other, IndexType indexType) switch (indexType) { case IndexType.TxLog: - return DatomComparators.TxLogComparator.Compare(RawSpan, other.RawSpan); + return DatomComparators.TxLogComparator.Compare(this, other); case IndexType.EAVTCurrent: case IndexType.EAVTHistory: - return DatomComparators.EAVTComparator.Compare(RawSpan, other.RawSpan); + return DatomComparators.EAVTComparator.Compare(this, other); case IndexType.AEVTCurrent: case IndexType.AEVTHistory: - return DatomComparators.AEVTComparator.Compare(RawSpan, other.RawSpan); + return DatomComparators.AEVTComparator.Compare(this, other); case IndexType.AVETCurrent: case IndexType.AVETHistory: - return DatomComparators.AVETComparator.Compare(RawSpan, other.RawSpan); + return DatomComparators.AVETComparator.Compare(this, other); case IndexType.VAETCurrent: case IndexType.VAETHistory: - return DatomComparators.VAETComparator.Compare(RawSpan, other.RawSpan); + return DatomComparators.VAETComparator.Compare(this, other); default: throw new ArgumentOutOfRangeException(nameof(indexType), indexType, "Unknown index type"); } @@ -116,12 +139,6 @@ public int Compare(Datom other, IndexType indexType) /// public Datom Retract() { - var data = GC.AllocateUninitializedArray(RawSpan.Length); - var dataSpan = data.AsSpan(); - RawSpan.CopyTo(dataSpan); - var prefix = MemoryMarshal.Read(dataSpan); - var newPrefix = new KeyPrefix().Set(prefix.E, prefix.A, TxId.Tmp, true); - MemoryMarshal.Write(dataSpan, newPrefix); - return new Datom(data, registry); + return new Datom(_prefix with {IsRetract = true, T = TxId.Tmp}, _valueBlob, _registry); } } diff --git a/src/NexusMods.MnemonicDB.Abstractions/DatomComparators/ADatomComparator.cs b/src/NexusMods.MnemonicDB.Abstractions/DatomComparators/ADatomComparator.cs index bf6a307a..829c039d 100644 --- a/src/NexusMods.MnemonicDB.Abstractions/DatomComparators/ADatomComparator.cs +++ b/src/NexusMods.MnemonicDB.Abstractions/DatomComparators/ADatomComparator.cs @@ -8,13 +8,28 @@ namespace NexusMods.MnemonicDB.Abstractions.DatomComparators; /// /// Abstract datom comparator, compares the A, B, C, D and E parts of the datom, in that order /// -public abstract unsafe class ADatomComparator : IDatomComparator +public abstract unsafe class ADatomComparator : IDatomComparator where TA : IElementComparer where TB : IElementComparer where TC : IElementComparer where TD : IElementComparer - where TE : IElementComparer { + /// + public static int Compare(KeyPrefix* aPrefix, byte* aPtr, int aLen, KeyPrefix* bPrefix, byte* bPtr, int bLen) + { + var cmp = TA.Compare(aPrefix, aPtr, aLen, bPrefix, bPtr, bLen); + if (cmp != 0) return cmp; + + cmp = TB.Compare(aPrefix, aPtr, aLen, bPrefix, bPtr, bLen); + if (cmp != 0) return cmp; + + cmp = TC.Compare(aPrefix, aPtr, aLen, bPrefix, bPtr, bLen); + if (cmp != 0) return cmp; + + return TD.Compare(aPrefix, aPtr, aLen, bPrefix, bPtr, bLen); + } + + /// public static int Compare(byte* aPtr, int aLen, byte* bPtr, int bLen) { var cmp = TA.Compare(aPtr, aLen, bPtr, bLen); @@ -26,33 +41,96 @@ public static int Compare(byte* aPtr, int aLen, byte* bPtr, int bLen) cmp = TC.Compare(aPtr, aLen, bPtr, bLen); if (cmp != 0) return cmp; - cmp = TD.Compare(aPtr, aLen, bPtr, bLen); + return TD.Compare(aPtr, aLen, bPtr, bLen); + } + + /// + public static int Compare(in Datom a, in Datom b) + { + var cmp = TA.Compare(a, b); + if (cmp != 0) return cmp; + + cmp = TB.Compare(a, b); + if (cmp != 0) return cmp; + + cmp = TC.Compare(a, b); if (cmp != 0) return cmp; - return TE.Compare(aPtr, aLen, bPtr, bLen); + return TD.Compare(a, b); } - /// - /// Compare two datom spans - /// + /// public static int Compare(ReadOnlySpan a, ReadOnlySpan b) { - fixed(byte* aPtr = a) - fixed(byte* bPtr = b) - return Compare(aPtr, a.Length, bPtr, b.Length); + var cmp = TA.Compare(a, b); + if (cmp != 0) return cmp; + + cmp = TB.Compare(a, b); + if (cmp != 0) return cmp; + + cmp = TC.Compare(a, b); + if (cmp != 0) return cmp; + + return TD.Compare(a, b); } - /// - /// Compare two datoms - /// - public static int Compare(in Datom a, in Datom b) + /// + public int CompareInstance(KeyPrefix* aPrefix, byte* aPtr, int aLen, KeyPrefix* bPrefix, byte* bPtr, int bLen) { - return Compare(a.RawSpan, b.RawSpan); + var cmp = TA.Compare(aPrefix, aPtr, aLen, bPrefix, bPtr, bLen); + if (cmp != 0) return cmp; + + cmp = TB.Compare(aPrefix, aPtr, aLen, bPrefix, bPtr, bLen); + if (cmp != 0) return cmp; + + cmp = TC.Compare(aPrefix, aPtr, aLen, bPrefix, bPtr, bLen); + if (cmp != 0) return cmp; + + return TD.Compare(aPrefix, aPtr, aLen, bPrefix, bPtr, bLen); } /// public int CompareInstance(byte* aPtr, int aLen, byte* bPtr, int bLen) { - return Compare(aPtr, aLen, bPtr, bLen); + var cmp = TA.Compare(aPtr, aLen, bPtr, bLen); + if (cmp != 0) return cmp; + + cmp = TB.Compare(aPtr, aLen, bPtr, bLen); + if (cmp != 0) return cmp; + + cmp = TC.Compare(aPtr, aLen, bPtr, bLen); + if (cmp != 0) return cmp; + + return TD.Compare(aPtr, aLen, bPtr, bLen); + } + + /// + public int CompareInstance(in Datom a, in Datom b) + { + var cmp = TA.Compare(a, b); + if (cmp != 0) return cmp; + + cmp = TB.Compare(a, b); + if (cmp != 0) return cmp; + + cmp = TC.Compare(a, b); + if (cmp != 0) return cmp; + + return TD.Compare(a, b); + } + + /// + public int CompareInstance(ReadOnlySpan a, ReadOnlySpan b) + { + var cmp = TA.Compare(a, b); + if (cmp != 0) return cmp; + + cmp = TB.Compare(a, b); + if (cmp != 0) return cmp; + + cmp = TC.Compare(a, b); + if (cmp != 0) return cmp; + + return TD.Compare(a, b); } } diff --git a/src/NexusMods.MnemonicDB.Abstractions/DatomComparators/AEVTComparator.cs b/src/NexusMods.MnemonicDB.Abstractions/DatomComparators/AEVTComparator.cs index d8b72ee4..75e8f72f 100644 --- a/src/NexusMods.MnemonicDB.Abstractions/DatomComparators/AEVTComparator.cs +++ b/src/NexusMods.MnemonicDB.Abstractions/DatomComparators/AEVTComparator.cs @@ -12,5 +12,4 @@ public class AEVTComparator : ADatomComparator< AComparer, EComparer, ValueComparer, - TxComparer, - AssertComparer>; + TxComparer>; diff --git a/src/NexusMods.MnemonicDB.Abstractions/DatomComparators/APartialDatomComparator.cs b/src/NexusMods.MnemonicDB.Abstractions/DatomComparators/APartialDatomComparator.cs index 3fd5d68c..e42b49f6 100644 --- a/src/NexusMods.MnemonicDB.Abstractions/DatomComparators/APartialDatomComparator.cs +++ b/src/NexusMods.MnemonicDB.Abstractions/DatomComparators/APartialDatomComparator.cs @@ -1,6 +1,7 @@ using System; using NexusMods.MnemonicDB.Abstractions.DatomIterators; using NexusMods.MnemonicDB.Abstractions.ElementComparers; +using NexusMods.MnemonicDB.Abstractions.Internals; namespace NexusMods.MnemonicDB.Abstractions.DatomComparators; @@ -13,6 +14,19 @@ public abstract unsafe class APartialDatomComparator : IDatomCompara where TB : IElementComparer where TC : IElementComparer { + /// + public static int Compare(KeyPrefix* aPrefix, byte* aPtr, int aLen, KeyPrefix* bPrefix, byte* bPtr, int bLen) + { + var cmp = TA.Compare(aPrefix, aPtr, aLen, bPrefix, bPtr, bLen); + if (cmp != 0) return cmp; + + cmp = TB.Compare(aPrefix, aPtr, aLen, bPrefix, bPtr, bLen); + if (cmp != 0) return cmp; + + return TC.Compare(aPrefix, aPtr, aLen, bPrefix, bPtr, bLen); + } + + /// public static int Compare(byte* aPtr, int aLen, byte* bPtr, int bLen) { var cmp = TA.Compare(aPtr, aLen, bPtr, bLen); @@ -24,27 +38,75 @@ public static int Compare(byte* aPtr, int aLen, byte* bPtr, int bLen) return TC.Compare(aPtr, aLen, bPtr, bLen); } - /// - /// Compare two datom spans - /// + /// + public static int Compare(in Datom a, in Datom b) + { + var cmp = TA.Compare(a, b); + if (cmp != 0) return cmp; + + cmp = TB.Compare(a, b); + if (cmp != 0) return cmp; + + return TC.Compare(a, b); + } + + /// public static int Compare(ReadOnlySpan a, ReadOnlySpan b) { - fixed(byte* aPtr = a) - fixed(byte* bPtr = b) - return Compare(aPtr, a.Length, bPtr, b.Length); + var cmp = TA.Compare(a, b); + if (cmp != 0) return cmp; + + cmp = TB.Compare(a, b); + if (cmp != 0) return cmp; + + return TC.Compare(a, b); } - /// - /// Compare two datoms - /// - public static int Compare(in Datom a, in Datom b) + /// + public int CompareInstance(KeyPrefix* aPrefix, byte* aPtr, int aLen, KeyPrefix* bPrefix, byte* bPtr, int bLen) { - return Compare(a.RawSpan, b.RawSpan); + var cmp = TA.Compare(aPrefix, aPtr, aLen, bPrefix, bPtr, bLen); + if (cmp != 0) return cmp; + + cmp = TB.Compare(aPrefix, aPtr, aLen, bPrefix, bPtr, bLen); + if (cmp != 0) return cmp; + + return TC.Compare(aPrefix, aPtr, aLen, bPrefix, bPtr, bLen); } /// public int CompareInstance(byte* aPtr, int aLen, byte* bPtr, int bLen) { - return Compare(aPtr, aLen, bPtr, bLen); + var cmp = TA.Compare(aPtr, aLen, bPtr, bLen); + if (cmp != 0) return cmp; + + cmp = TB.Compare(aPtr, aLen, bPtr, bLen); + if (cmp != 0) return cmp; + + return TC.Compare(aPtr, aLen, bPtr, bLen); + } + + /// + public int CompareInstance(in Datom a, in Datom b) + { + var cmp = TA.Compare(a, b); + if (cmp != 0) return cmp; + + cmp = TB.Compare(a, b); + if (cmp != 0) return cmp; + + return TC.Compare(a, b); + } + + /// + public int CompareInstance(ReadOnlySpan a, ReadOnlySpan b) + { + var cmp = TA.Compare(a, b); + if (cmp != 0) return cmp; + + cmp = TB.Compare(a, b); + if (cmp != 0) return cmp; + + return TC.Compare(a, b); } } diff --git a/src/NexusMods.MnemonicDB.Abstractions/DatomComparators/AVETComparator.cs b/src/NexusMods.MnemonicDB.Abstractions/DatomComparators/AVETComparator.cs index b3e8a9d2..f2d49838 100644 --- a/src/NexusMods.MnemonicDB.Abstractions/DatomComparators/AVETComparator.cs +++ b/src/NexusMods.MnemonicDB.Abstractions/DatomComparators/AVETComparator.cs @@ -11,5 +11,4 @@ public class AVETComparator : ADatomComparator< AComparer, ValueComparer, EComparer, - TxComparer, - AssertComparer>; + TxComparer>; diff --git a/src/NexusMods.MnemonicDB.Abstractions/DatomComparators/EAVTComparator.cs b/src/NexusMods.MnemonicDB.Abstractions/DatomComparators/EAVTComparator.cs index 1a51123d..04a21fbc 100644 --- a/src/NexusMods.MnemonicDB.Abstractions/DatomComparators/EAVTComparator.cs +++ b/src/NexusMods.MnemonicDB.Abstractions/DatomComparators/EAVTComparator.cs @@ -12,6 +12,5 @@ public class EAVTComparator : ADatomComparator< EComparer, AComparer, ValueComparer, - TxComparer, - AssertComparer>; + TxComparer>; diff --git a/src/NexusMods.MnemonicDB.Abstractions/DatomComparators/IDatomComparator.cs b/src/NexusMods.MnemonicDB.Abstractions/DatomComparators/IDatomComparator.cs index 1560d20c..0d8e8d8f 100644 --- a/src/NexusMods.MnemonicDB.Abstractions/DatomComparators/IDatomComparator.cs +++ b/src/NexusMods.MnemonicDB.Abstractions/DatomComparators/IDatomComparator.cs @@ -1,4 +1,6 @@ using System; +using NexusMods.MnemonicDB.Abstractions.DatomIterators; +using NexusMods.MnemonicDB.Abstractions.Internals; namespace NexusMods.MnemonicDB.Abstractions.DatomComparators; @@ -7,23 +9,44 @@ namespace NexusMods.MnemonicDB.Abstractions.DatomComparators; /// public unsafe interface IDatomComparator { + + /// + /// Compares two elements of a datom from the given pointers + /// + public static abstract int Compare(KeyPrefix* aPrefix, byte* aPtr, int aLen, KeyPrefix* bPrefix, byte* bPtr, int bLen); + /// - /// Compare two datoms + /// Compares two elements of a datom from the given pointers /// public static abstract int Compare(byte* aPtr, int aLen, byte* bPtr, int bLen); /// - /// Compare two datoms + /// Compares the elements from two datoms. + /// + public static abstract int Compare(in Datom a, in Datom b); + + /// + /// Compare two datom spans + /// + public static abstract int Compare(ReadOnlySpan a, ReadOnlySpan b); + + /// + /// Compares two elements of a datom from the given pointers + /// + public int CompareInstance(KeyPrefix* aPrefix, byte* aPtr, int aLen, KeyPrefix* bPrefix, byte* bPtr, int bLen); + + /// + /// Compares two elements of a datom from the given pointers /// public int CompareInstance(byte* aPtr, int aLen, byte* bPtr, int bLen); /// - /// Compare two datoms + /// Compares the elements from two datoms. + /// + public int CompareInstance(in Datom a, in Datom b); + + /// + /// Compare two datom spans /// - public int CompareInstance(ReadOnlySpan a, ReadOnlySpan b) - { - fixed(byte* aPtr = a) - fixed(byte* bPtr = b) - return CompareInstance(aPtr, a.Length, bPtr, b.Length); - } + public int CompareInstance(ReadOnlySpan a, ReadOnlySpan b); } diff --git a/src/NexusMods.MnemonicDB.Abstractions/DatomComparators/TxLogComparator.cs b/src/NexusMods.MnemonicDB.Abstractions/DatomComparators/TxLogComparator.cs index 857dedb1..55f543d0 100644 --- a/src/NexusMods.MnemonicDB.Abstractions/DatomComparators/TxLogComparator.cs +++ b/src/NexusMods.MnemonicDB.Abstractions/DatomComparators/TxLogComparator.cs @@ -11,5 +11,4 @@ public class TxLogComparator : ADatomComparator< TxComparer, EComparer, AComparer, - ValueComparer, - AssertComparer>; + ValueComparer>; diff --git a/src/NexusMods.MnemonicDB.Abstractions/DatomComparators/VAETComparator.cs b/src/NexusMods.MnemonicDB.Abstractions/DatomComparators/VAETComparator.cs index 8db84975..73b5b17d 100644 --- a/src/NexusMods.MnemonicDB.Abstractions/DatomComparators/VAETComparator.cs +++ b/src/NexusMods.MnemonicDB.Abstractions/DatomComparators/VAETComparator.cs @@ -11,6 +11,5 @@ public class VAETComparator : ADatomComparator< ValueComparer, AComparer, EComparer, - TxComparer, - AssertComparer>; + TxComparer>; diff --git a/src/NexusMods.MnemonicDB.Abstractions/ElementComparers/AComparer.cs b/src/NexusMods.MnemonicDB.Abstractions/ElementComparers/AComparer.cs index c1b35f88..db35ffa7 100644 --- a/src/NexusMods.MnemonicDB.Abstractions/ElementComparers/AComparer.cs +++ b/src/NexusMods.MnemonicDB.Abstractions/ElementComparers/AComparer.cs @@ -1,13 +1,37 @@ -namespace NexusMods.MnemonicDB.Abstractions.ElementComparers; +using System; +using NexusMods.MnemonicDB.Abstractions.DatomIterators; +using NexusMods.MnemonicDB.Abstractions.Internals; + +namespace NexusMods.MnemonicDB.Abstractions.ElementComparers; /// /// Compares the A part of the key. /// public class AComparer : IElementComparer { + /// + public static unsafe int Compare(KeyPrefix* aPrefix, byte* aPtr, int aLen, KeyPrefix* bPrefix, byte* bPtr, int bLen) + { + return aPrefix->A.CompareTo(bPrefix->A); + } + /// public static unsafe int Compare(byte* aPtr, int aLen, byte* bPtr, int bLen) { - return IElementComparer.KeyPrefix(aPtr)->A.CompareTo(IElementComparer.KeyPrefix(bPtr)->A); + return ((KeyPrefix*)aPtr)->A.CompareTo(((KeyPrefix*)bPtr)->A); + } + + /// + public static int Compare(in Datom a, in Datom b) + { + return a.A.CompareTo(b.A); + } + + /// + public static int Compare(ReadOnlySpan a, ReadOnlySpan b) + { + var keyA = KeyPrefix.Read(a); + var keyB = KeyPrefix.Read(b); + return keyA.A.CompareTo(keyB.A); } } diff --git a/src/NexusMods.MnemonicDB.Abstractions/ElementComparers/EComparer.cs b/src/NexusMods.MnemonicDB.Abstractions/ElementComparers/EComparer.cs index f7f41f59..371ddfab 100644 --- a/src/NexusMods.MnemonicDB.Abstractions/ElementComparers/EComparer.cs +++ b/src/NexusMods.MnemonicDB.Abstractions/ElementComparers/EComparer.cs @@ -1,5 +1,6 @@ using System; using System.Runtime.InteropServices; +using NexusMods.MnemonicDB.Abstractions.DatomIterators; using NexusMods.MnemonicDB.Abstractions.Internals; namespace NexusMods.MnemonicDB.Abstractions.ElementComparers; @@ -9,8 +10,29 @@ namespace NexusMods.MnemonicDB.Abstractions.ElementComparers; /// public class EComparer : IElementComparer { + /// + public static unsafe int Compare(KeyPrefix* aPrefix, byte* aPtr, int aLen, KeyPrefix* bPrefix, byte* bPtr, int bLen) + { + return aPrefix->E.CompareTo(bPrefix->E); + } + + /// public static unsafe int Compare(byte* aPtr, int aLen, byte* bPtr, int bLen) { - return IElementComparer.KeyPrefix(aPtr)->E.CompareTo(IElementComparer.KeyPrefix(bPtr)->E); + return ((KeyPrefix*)aPtr)->E.CompareTo(((KeyPrefix*)bPtr)->E); + } + + /// + public static int Compare(in Datom a, in Datom b) + { + return a.E.CompareTo(b.E); + } + + /// + public static int Compare(ReadOnlySpan a, ReadOnlySpan b) + { + var keyA = KeyPrefix.Read(a); + var keyB = KeyPrefix.Read(b); + return keyA.E.CompareTo(keyB.E); } } diff --git a/src/NexusMods.MnemonicDB.Abstractions/ElementComparers/IElementComparer.cs b/src/NexusMods.MnemonicDB.Abstractions/ElementComparers/IElementComparer.cs index 90b660c7..659ba9af 100644 --- a/src/NexusMods.MnemonicDB.Abstractions/ElementComparers/IElementComparer.cs +++ b/src/NexusMods.MnemonicDB.Abstractions/ElementComparers/IElementComparer.cs @@ -1,4 +1,5 @@ using System; +using NexusMods.MnemonicDB.Abstractions.DatomIterators; using NexusMods.MnemonicDB.Abstractions.Internals; namespace NexusMods.MnemonicDB.Abstractions.ElementComparers; @@ -10,12 +11,22 @@ namespace NexusMods.MnemonicDB.Abstractions.ElementComparers; public interface IElementComparer { /// - /// Compares two elements of a datom. + /// Compares two elements of a datom from the given pointers + /// + public static abstract unsafe int Compare(KeyPrefix* aPrefix, byte* aPtr, int aLen, KeyPrefix* bPrefix, byte* bPtr, int bLen); + + /// + /// Compares two elements of a datom from the given pointers /// public static abstract unsafe int Compare(byte* aPtr, int aLen, byte* bPtr, int bLen); /// - /// Get the key prefix from a pointer. + /// Compares the elements from two datoms. + /// + public static abstract int Compare(in Datom a, in Datom b); + + /// + /// Compare two datom spans /// - protected static unsafe KeyPrefix* KeyPrefix(byte *ptr) => (KeyPrefix*) ptr; + public static abstract int Compare(ReadOnlySpan a, ReadOnlySpan b); } diff --git a/src/NexusMods.MnemonicDB.Abstractions/ElementComparers/RetractComparer.cs b/src/NexusMods.MnemonicDB.Abstractions/ElementComparers/RetractComparer.cs deleted file mode 100644 index 68900b5b..00000000 --- a/src/NexusMods.MnemonicDB.Abstractions/ElementComparers/RetractComparer.cs +++ /dev/null @@ -1,12 +0,0 @@ -namespace NexusMods.MnemonicDB.Abstractions.ElementComparers; - -/// -/// Compares the assert part of the datom -/// -public class AssertComparer: IElementComparer -{ - public static unsafe int Compare(byte* aPtr, int aLen, byte* bPtr, int bLen) - { - return IElementComparer.KeyPrefix(aPtr)->IsRetract.CompareTo(IElementComparer.KeyPrefix(bPtr)->IsRetract); - } -} diff --git a/src/NexusMods.MnemonicDB.Abstractions/ElementComparers/TxComparer.cs b/src/NexusMods.MnemonicDB.Abstractions/ElementComparers/TxComparer.cs index 4a9ba022..6c943b3f 100644 --- a/src/NexusMods.MnemonicDB.Abstractions/ElementComparers/TxComparer.cs +++ b/src/NexusMods.MnemonicDB.Abstractions/ElementComparers/TxComparer.cs @@ -1,5 +1,6 @@ using System; using System.Runtime.InteropServices; +using NexusMods.MnemonicDB.Abstractions.DatomIterators; using NexusMods.MnemonicDB.Abstractions.ElementComparers; using NexusMods.MnemonicDB.Abstractions.Internals; @@ -10,8 +11,29 @@ namespace NexusMods.MnemonicDB.Storage.Abstractions.ElementComparers; /// public class TxComparer : IElementComparer { + /// + public static unsafe int Compare(KeyPrefix* aPrefix, byte* aPtr, int aLen, KeyPrefix* bPrefix, byte* bPtr, int bLen) + { + return aPrefix->T.CompareTo(bPrefix->T); + } + + /// public static unsafe int Compare(byte* aPtr, int aLen, byte* bPtr, int bLen) { - return IElementComparer.KeyPrefix(aPtr)->T.CompareTo(IElementComparer.KeyPrefix(bPtr)->T); + return ((KeyPrefix*)aPtr)->T.CompareTo(((KeyPrefix*)bPtr)->T); + } + + /// + public static int Compare(in Datom a, in Datom b) + { + return a.T.CompareTo(b.T); + } + + /// + public static int Compare(ReadOnlySpan a, ReadOnlySpan b) + { + var keyA = KeyPrefix.Read(a); + var keyB = KeyPrefix.Read(b); + return keyA.T.CompareTo(keyB.T); } } diff --git a/src/NexusMods.MnemonicDB.Abstractions/ElementComparers/ValueComparer.cs b/src/NexusMods.MnemonicDB.Abstractions/ElementComparers/ValueComparer.cs index fac81ebe..652aac08 100644 --- a/src/NexusMods.MnemonicDB.Abstractions/ElementComparers/ValueComparer.cs +++ b/src/NexusMods.MnemonicDB.Abstractions/ElementComparers/ValueComparer.cs @@ -1,6 +1,8 @@ using System; using System.Text; +using NexusMods.MnemonicDB.Abstractions.DatomIterators; using NexusMods.MnemonicDB.Abstractions.Internals; +using Reloaded.Memory.Extensions; namespace NexusMods.MnemonicDB.Abstractions.ElementComparers; @@ -10,44 +12,77 @@ namespace NexusMods.MnemonicDB.Abstractions.ElementComparers; public class ValueComparer : IElementComparer { #region Constants - private const int MaxStackAlloc = 128; - private static readonly Encoding AsciiEncoding = Encoding.ASCII; private static readonly Encoding Utf8Encoding = Encoding.UTF8; #endregion + /// + public static unsafe int Compare(KeyPrefix* aPrefix, byte* aPtr, int aLen, KeyPrefix* bPrefix, byte* bPtr, int bLen) + { + var prefixA = *(KeyPrefix*)aPtr; + var prefixB = *(KeyPrefix*)bPtr; + + var typeA = prefixA.ValueTag; + var typeB = prefixB.ValueTag; + + return CompareValues(typeA, aPtr, aLen, typeB, bPtr, bLen); + } /// public static unsafe int Compare(byte* aPtr, int aLen, byte* bPtr, int bLen) { - var ptrA = aPtr + sizeof(KeyPrefix); - var ptrB = bPtr + sizeof(KeyPrefix); - var aSize = aLen - sizeof(KeyPrefix); - var bSize = bLen - sizeof(KeyPrefix); + var typeA = ((KeyPrefix*)aPtr)->ValueTag; + var typeB = ((KeyPrefix*)bPtr)->ValueTag; + + return CompareValues(typeA, aPtr + KeyPrefix.Size, aLen - KeyPrefix.Size, typeB, bPtr + KeyPrefix.Size, bLen - KeyPrefix.Size); + } + + /// + public static int Compare(in Datom a, in Datom b) + { + var typeA = a.Prefix.ValueTag; + var typeB = b.Prefix.ValueTag; - return CompareValues(ptrA, aSize, ptrB, bSize); + unsafe + { + fixed (byte* aPtr = a.ValueSpan) + { + fixed (byte* bPtr = b.ValueSpan) + { + return CompareValues(typeA, aPtr, a.ValueSpan.Length, typeB, bPtr, b.ValueSpan.Length); + } + } + } + } + + /// + public static int Compare(ReadOnlySpan a, ReadOnlySpan b) + { + var typeA = KeyPrefix.Read(a).ValueTag; + var typeB = KeyPrefix.Read(b).ValueTag; + + unsafe + { + fixed (byte* aPtr = a.SliceFast(KeyPrefix.Size)) + { + fixed (byte* bPtr = b.SliceFast(KeyPrefix.Size)) + { + return CompareValues(typeA, aPtr + KeyPrefix.Size, a.Length - KeyPrefix.Size, typeB, bPtr + KeyPrefix.Size, b.Length - KeyPrefix.Size); + } + } + } } /// /// Performs a highly optimized, sort between two value pointers. /// - public static unsafe int CompareValues(byte* a, int alen, byte* b, int blen) + public static unsafe int CompareValues(ValueTags typeA, byte* aVal, int aLen, ValueTags typeB, byte* bVal, int bLen) { - if (alen == 0 || blen == 0) - return alen.CompareTo(blen); - - var typeA = a[0]; - var typeB = b[0]; - + if (aLen == 0 || bLen == 0) + return aLen.CompareTo(bLen); if (typeA != typeB) return typeA.CompareTo(typeB); - var aVal = a + 1; - var bVal = b + 1; - - alen -= 1; - blen -= 1; - - return (ValueTags)typeA switch + return typeA switch { ValueTags.Null => 0, ValueTags.UInt8 => CompareInternal(aVal, bVal), @@ -61,10 +96,10 @@ public static unsafe int CompareValues(byte* a, int alen, byte* b, int blen) ValueTags.Int128 => CompareInternal(aVal, bVal), ValueTags.Float32 => CompareInternal(aVal, bVal), ValueTags.Float64 => CompareInternal(aVal, bVal), - ValueTags.Ascii => CompareBlobInternal(aVal, alen, bVal, blen), - ValueTags.Utf8 => CompareBlobInternal(aVal, alen, bVal, blen), - ValueTags.Utf8Insensitive => CompareUtf8Insensitive(aVal, alen, bVal, blen), - ValueTags.Blob => CompareBlobInternal(aVal, alen, bVal, blen), + ValueTags.Ascii => CompareBlobInternal(aVal, aLen, bVal, bLen), + ValueTags.Utf8 => CompareBlobInternal(aVal, aLen, bVal, bLen), + ValueTags.Utf8Insensitive => CompareUtf8Insensitive(aVal, aLen, bVal, bLen), + ValueTags.Blob => CompareBlobInternal(aVal, aLen, bVal, bLen), // HashedBlob is a special case, we compare the hashes not the blobs ValueTags.HashedBlob => CompareInternal(aVal, bVal), ValueTags.Reference => CompareInternal(aVal, bVal), @@ -97,6 +132,4 @@ private static unsafe int CompareInternal(byte* aVal, byte* bVal) return ((T*)aVal)[0].CompareTo(((T*)bVal)[0]); } - - } diff --git a/src/NexusMods.MnemonicDB.Abstractions/IAttribute.cs b/src/NexusMods.MnemonicDB.Abstractions/IAttribute.cs index 6924bd2a..bb902382 100644 --- a/src/NexusMods.MnemonicDB.Abstractions/IAttribute.cs +++ b/src/NexusMods.MnemonicDB.Abstractions/IAttribute.cs @@ -68,7 +68,7 @@ public interface IAttribute /// /// Converts the given values into a typed datom /// - IReadDatom Resolve(EntityId entityId, AttributeId attributeId, ReadOnlySpan value, TxId tx, bool isRetract); + IReadDatom Resolve(EntityId entityId, AttributeId attributeId, ReadOnlySpan value, TxId tx, bool isRetract, ValueTags valueTag); /// diff --git a/src/NexusMods.MnemonicDB.Abstractions/ICommitResult.cs b/src/NexusMods.MnemonicDB.Abstractions/ICommitResult.cs index 31a471fd..c97674a9 100644 --- a/src/NexusMods.MnemonicDB.Abstractions/ICommitResult.cs +++ b/src/NexusMods.MnemonicDB.Abstractions/ICommitResult.cs @@ -15,13 +15,6 @@ public interface ICommitResult /// public EntityId this[EntityId id] { get; } - - /// - /// Remaps a ReadModel to a new instance with the new ids, if the entity is not new, it - /// updates the entity anyway to be current as of the commit - /// - public T Remap(T model) where T : IHasEntityId; - /// /// Gets the new TxId after the commit /// diff --git a/src/NexusMods.MnemonicDB.Abstractions/IndexSegments/IndexSegment.cs b/src/NexusMods.MnemonicDB.Abstractions/IndexSegments/IndexSegment.cs index e4064b98..98799a5c 100644 --- a/src/NexusMods.MnemonicDB.Abstractions/IndexSegments/IndexSegment.cs +++ b/src/NexusMods.MnemonicDB.Abstractions/IndexSegments/IndexSegment.cs @@ -1,10 +1,15 @@ using System; using System.Collections; using System.Collections.Generic; +using System.Numerics; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.X86; using DynamicData; using NexusMods.MnemonicDB.Abstractions.DatomIterators; using NexusMods.MnemonicDB.Abstractions.Internals; using NexusMods.Paths; +using Reloaded.Memory.Extensions; namespace NexusMods.MnemonicDB.Abstractions.IndexSegments; @@ -14,10 +19,10 @@ namespace NexusMods.MnemonicDB.Abstractions.IndexSegments; /// public readonly struct IndexSegment : IEnumerable { - private readonly Memory _data; - private readonly Memory _offsets; private readonly IAttributeRegistry _registry; - private readonly RegistryId _registryId; + private readonly int _rowCount; + private readonly ReadOnlyMemory _data; + /// /// Construct a new index segment from the given data and offsets @@ -25,11 +30,75 @@ namespace NexusMods.MnemonicDB.Abstractions.IndexSegments; public IndexSegment(ReadOnlySpan data, ReadOnlySpan offsets, IAttributeRegistry registry) { _registry = registry; - _registryId = registry.Id; - _data = data.ToArray(); - _offsets = offsets.ToArray(); + if (data.Length == 0) + { + _rowCount = 0; + _data = ReadOnlyMemory.Empty; + return; + } + _rowCount = offsets.Length - 1; + var memory = new Memory(GC.AllocateUninitializedArray(data.Length + (_rowCount + 1) * sizeof(int))); + _data = memory; + ReprocessData(data, offsets, memory.Span); } + /// + /// All the upper values + /// + private ReadOnlySpan _uppers => _data.Span.SliceFast(0, _rowCount * sizeof(ulong)).CastFast(); + + /// + /// All the lower values + /// + private ReadOnlySpan _lowers => _data.Span.SliceFast(_rowCount * sizeof(ulong), _rowCount * sizeof(ulong)).CastFast(); + + /// + /// All the offsets + /// + private ReadOnlySpan _offsets => _data.Span.SliceFast(_rowCount * sizeof(ulong) * 2, (_rowCount + 1) * sizeof(int)).CastFast(); + + /// + /// Pivots all the data into 4 columns: + /// - (ulong) upper part of the key prefix + /// - (ulong) lower part of the key prefix + /// - (int) offsets for each row's value into the value blob + /// - (byte[]) value blob + /// + private void ReprocessData(ReadOnlySpan data, ReadOnlySpan offsets, Span dataSpan) + { + var uppers = dataSpan.SliceFast(0, _rowCount * sizeof(ulong)).CastFast(); + var lowers = dataSpan.SliceFast(_rowCount * sizeof(ulong), _rowCount * sizeof(ulong)).CastFast(); + + // Extra space for one int in the offsets so we can calculate the size of the last row + var valueOffsets = dataSpan.SliceFast(_rowCount * sizeof(ulong) * 2, (_rowCount + 1) * sizeof(int)).CastFast(); + var values = dataSpan.SliceFast((_rowCount * (sizeof(ulong) * 2 + sizeof(int))) + sizeof(int)); + + var relativeValueOffset = 0; + + // The first row starts at the beginning of the value blob + var absoluteValueOffset = _rowCount * (sizeof(ulong) * 2 + sizeof(int)) + sizeof(int); + + for (var i = 0; i < _rowCount; i++) + { + var rowSegment = data.Slice(offsets[i], offsets[i + 1] - offsets[i]); + var prefix = MemoryMarshal.Read(rowSegment); + uppers[i] = prefix.Upper; + lowers[i] = prefix.Lower; + valueOffsets[i] = absoluteValueOffset; + + var valueSpan = rowSegment.SliceFast(KeyPrefix.Size); + valueSpan.CopyTo(values.SliceFast(relativeValueOffset)); + + relativeValueOffset += valueSpan.Length; + absoluteValueOffset += valueSpan.Length; + } + + // The last row's offset is the size of the value blob + valueOffsets[_rowCount] = absoluteValueOffset; + } + + + /// /// Returns true if this segment is valid (contains data) /// @@ -43,12 +112,12 @@ public IndexSegment(ReadOnlySpan data, ReadOnlySpan offsets, IAttribu /// /// The number of datoms in this segment /// - public int Count => _offsets.Length - 1; + public int Count => _rowCount; /// /// The assigned registry id /// - public RegistryId RegistryId => _registryId; + public RegistryId RegistryId => _registry.Id; /// /// Get the datom of the given index @@ -57,8 +126,13 @@ public Datom this[int idx] { get { - var fromOffset = _offsets.Span[idx]; - return new Datom(_data.Slice(fromOffset, _offsets.Span[idx + 1] - fromOffset), _registry); + var offsets = _offsets; + var fromOffset = offsets[idx]; + var toOffset = offsets[idx + 1]; + + var valueSlice = _data.Slice(fromOffset, toOffset - fromOffset); + + return new Datom(new KeyPrefix(_uppers[idx], _lowers[idx]), valueSlice, _registry); } } @@ -67,7 +141,7 @@ public Datom this[int idx] /// public bool Contains(IAttribute attribute) { - var id = attribute.GetDbId(_registryId); + var id = attribute.GetDbId(_registry.Id); foreach (var datom in this) if (datom.A == id) return true; @@ -77,10 +151,9 @@ public bool Contains(IAttribute attribute) /// public IEnumerator GetEnumerator() { - for (var i = 0; i < _offsets.Length - 1; i++) + for (var i = 0; i < _rowCount; i++) { - var fromOffset = _offsets.Span[i]; - yield return new Datom(_data.Slice(fromOffset, _offsets.Span[i + 1] - fromOffset), _registry); + yield return this[i]; } } @@ -98,4 +171,78 @@ public static IndexSegment From(IAttributeRegistry registry, IReadOnlyCollection builder.Add(datoms); return builder.Build(); } + + /// + /// Finds the first index of the given entity id + /// + /// + public int FindFirst(EntityId find) + { + var left = 0; + var right = _rowCount - 1; + var result = -1; + while (left <= right) + { + var mid = left + (right - left) / 2; + + + var lower = _lowers[mid]; + var e = EntityId.From((lower & 0xFF00000000000000) | ((lower >> 8) & 0x0000FFFFFFFFFFFF)); + + var comparison = e.CompareTo(find); + if (comparison == 0) + { + result = mid; // Don't return, but continue searching to the left + right = mid - 1; + } + else if (comparison < 0) + { + left = mid + 1; + } + else + { + right = mid - 1; + } + } + return result; // Return the first occurrence found, or -1 if not found + } + + public int FindFirstAVX2(ulong find) + { + var targetVector = Vector256.Zero.WithElement(0, find) + .WithElement(1, find) + .WithElement(2, find) + .WithElement(3, find); + + var casted = MemoryMarshal.Cast>(_lowers); + + for (var idx = 0; idx < casted.Length; idx += 1) + { + var lowers = casted[idx]; + + var maskHigh = Avx2.And(lowers, Vector256.Create(0x7F00000000000000UL)); + var maskLow = Avx2.And(Avx2.ShiftRightLogical(lowers, 8), Vector256.Create(0x0000FFFFFFFFFFFFUL)); + var ored = Avx2.Or(maskHigh, maskLow); + + var comparison = Avx2.CompareEqual(ored, targetVector); + + var mask = Avx2.MoveMask(comparison.As()); + if (mask != 0) + { + var index = BitOperations.TrailingZeroCount(mask) / 8; + return idx * Vector256.Count + index; + } + } + + // Handle remaining elements + for (int i = (casted.Length * Vector256.Count); i < _lowers.Length; i++) + { + if (_lowers[i] >= find) + { + return i; + } + } + + return -1; // No value found that is greater than or equal to the target + } } diff --git a/src/NexusMods.MnemonicDB.Abstractions/IndexSegments/IndexSegmentBuilder.cs b/src/NexusMods.MnemonicDB.Abstractions/IndexSegments/IndexSegmentBuilder.cs index 6602ca64..65f793a4 100644 --- a/src/NexusMods.MnemonicDB.Abstractions/IndexSegments/IndexSegmentBuilder.cs +++ b/src/NexusMods.MnemonicDB.Abstractions/IndexSegments/IndexSegmentBuilder.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Diagnostics; +using System.Runtime.InteropServices; using NexusMods.MnemonicDB.Abstractions.DatomIterators; using NexusMods.MnemonicDB.Abstractions.Internals; @@ -42,17 +43,29 @@ public void Reset() } /// - /// Add a datom to the segment + /// Add the datoms to the segment /// public void Add(IEnumerable datoms) { foreach (var datom in datoms) { - _offsets.Add(_data.Length); - _data.Write(datom.RawSpan); + Add(datom); } } + /// + /// Add a datom to the segment + /// + /// + public void Add(in Datom datom) + { + _offsets.Add(_data.Length); + var span = _data.GetSpan(KeyPrefix.Size); + MemoryMarshal.Write(span, datom.Prefix); + _data.Advance(KeyPrefix.Size); + _data.Write(datom.ValueSpan); + } + /// /// Add a datom to the segment /// @@ -85,7 +98,7 @@ public readonly void Add(EntityId entityId, Attribute public readonly void Add(ReadOnlySpan rawData) { - Debug.Assert(rawData.Length > KeyPrefix.Size, "Raw data must be at least the size of a KeyPrefix"); + Debug.Assert(rawData.Length >= KeyPrefix.Size, "Raw data must be at least the size of a KeyPrefix"); _offsets.Add(_data.Length); _data.Write(rawData); } diff --git a/src/NexusMods.MnemonicDB.Abstractions/IndexSegments/Values.cs b/src/NexusMods.MnemonicDB.Abstractions/IndexSegments/Values.cs index 03fe5108..142aa2c6 100644 --- a/src/NexusMods.MnemonicDB.Abstractions/IndexSegments/Values.cs +++ b/src/NexusMods.MnemonicDB.Abstractions/IndexSegments/Values.cs @@ -13,7 +13,14 @@ public struct Values(IndexSegment segment, int start, /// /// Gets the value at the given location /// - public TValueType this[int idx] => attribute.ReadValue(segment[idx + start].ValueSpan); + public TValueType this[int idx] + { + get + { + var datom = segment[idx + start]; + return attribute.ReadValue(datom.ValueSpan, datom.Prefix.ValueTag); + } + } /// /// Returns the number of items in the collection diff --git a/src/NexusMods.MnemonicDB.Abstractions/Internals/IAttributeRegistry.cs b/src/NexusMods.MnemonicDB.Abstractions/Internals/IAttributeRegistry.cs index b668d7d3..e7b3732d 100644 --- a/src/NexusMods.MnemonicDB.Abstractions/Internals/IAttributeRegistry.cs +++ b/src/NexusMods.MnemonicDB.Abstractions/Internals/IAttributeRegistry.cs @@ -15,7 +15,7 @@ public interface IAttributeRegistry /// /// /// - public IReadDatom Resolve(ReadOnlySpan datom); + public IReadDatom Resolve(in KeyPrefix prefix, ReadOnlySpan datom); /// /// Populates the registry with the given attributes, mostly used for diff --git a/src/NexusMods.MnemonicDB.Abstractions/Internals/KeyPrefix.cs b/src/NexusMods.MnemonicDB.Abstractions/Internals/KeyPrefix.cs index b3c51e4f..4ff56310 100644 --- a/src/NexusMods.MnemonicDB.Abstractions/Internals/KeyPrefix.cs +++ b/src/NexusMods.MnemonicDB.Abstractions/Internals/KeyPrefix.cs @@ -1,34 +1,34 @@ using System; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using NexusMods.MnemonicDB.Abstractions.ElementComparers; namespace NexusMods.MnemonicDB.Abstractions.Internals; /// /// The system encodes keys as a 16 byte prefix followed by the actual key data, the format /// of the value is defined by the IValueSerializer -/// /// and the length is maintained by the datastore. /// This KeyPrefix then contains the other parts of the Datom: EntityId, AttributeId, TxId, and Flags. /// Encodes and decodes the prefix of a key, the format is: /// [AttributeId: 2bytes] /// [TxId: 6bytes] /// [EntityID + PartitionID: 7bytes] -/// [IsRetract: 1byte] +/// [IsRetract: 1 bit] [ValueTag: 7 bits] /// The Entity Id is created by taking the last 6 bytes of the id and combining it with /// the partition id. So the encoding logic looks like this: /// packed = (e & 0x00FFFFFFFFFFFFFF) >> 8 | (e & 0xFFFFFFFFFFFF) << 8 /// [StructLayout(LayoutKind.Explicit, Size = Size)] -public struct KeyPrefix +public readonly record struct KeyPrefix { /// /// Fixed size of the KeyPrefix /// public const int Size = 16; - [FieldOffset(0)] private ulong _upper; - [FieldOffset(8)] private ulong _lower; + [FieldOffset(0)] private readonly ulong _upper; + [FieldOffset(8)] private readonly ulong _lower; /// /// The upper 8 bytes of the key @@ -40,33 +40,78 @@ public struct KeyPrefix /// public ulong Lower => _lower; - public KeyPrefix Set(EntityId id, AttributeId attributeId, TxId txId, bool isRetract) + /// + /// Sets the key prefix to the given values + /// + public KeyPrefix(ulong upper, ulong lower) + { + _upper = upper; + _lower = lower; + } + + /// + /// Disallow the default constructor + /// + /// + public KeyPrefix() + { + throw new InvalidOperationException("This constructor should not be called, use the Create method instead"); + } + + /// + /// Creates a new KeyPrefix with the given values + /// + public KeyPrefix(EntityId id, AttributeId attributeId, TxId txId, bool isRetract, ValueTags tags) { _upper = ((ulong)attributeId << 48) | ((ulong)txId & 0x0000FFFFFFFFFFFF); - _lower = ((ulong)id & 0xFF00000000000000) | (((ulong)id & 0x0000FFFFFFFFFFFF) << 8) | (isRetract ? 1UL : 0UL); - return this; + _lower = ((ulong)id & 0xFF00000000000000) | (((ulong)id & 0x0000FFFFFFFFFFFF) << 8) | (isRetract ? 1UL : 0UL) | ((ulong)tags << 1); } /// /// The EntityId /// - public EntityId E => (EntityId)((_lower & 0xFF00000000000000) | ((_lower >> 8) & 0x0000FFFFFFFFFFFF)); + public EntityId E + { + get => (EntityId)((_lower & 0xFF00000000000000) | ((_lower >> 8) & 0x0000FFFFFFFFFFFF)); + init => _lower = (_lower & 0xFF) | ((ulong)value & 0xFF00000000000000) | (((ulong)value & 0x0000FFFFFFFFFFFF) << 8); + } /// /// True if this is a retraction /// - public bool IsRetract => (_lower & 1) == 1; + public bool IsRetract + { + get => (_lower & 1) == 1; + init => _lower = value ? _lower | 1UL : _lower & ~1UL; + } + + /// + /// The value tag of the datom, which defines the actual value type of the datom + /// + public ValueTags ValueTag + { + get => (ValueTags)((_lower >> 1) & 0x7F); + init => _lower = (_lower & 0xFF00000000000001) | ((ulong)value << 1); + } /// /// The attribute id, maximum of 2^16 attributes are supported in the system /// - public AttributeId A => (AttributeId)(_upper >> 48); + public AttributeId A + { + get => (AttributeId)(_upper >> 48); + init => _upper = (_upper & 0x0000FFFFFFFFFFFF) | ((ulong)value << 48); + } /// /// The transaction id, maximum of 2^63 transactions are supported in the system, but really /// it's 2^56 as the upper 8 bits are used for the partition id. /// - public TxId T => TxId.From(PartitionId.Transactions.MakeEntityId(_upper & 0x0000FFFFFFFFFFFF).Value); + public TxId T + { + get => TxId.From(PartitionId.Transactions.MakeEntityId(_upper & 0x0000FFFFFFFFFFFF).Value); + init => _upper = (_upper & 0xFFFF000000000000) | ((ulong)value & 0x0000FFFFFFFFFFFF); + } /// public override string ToString() @@ -74,17 +119,6 @@ public override string ToString() return $"E: {E}, A: {A}, T: {T}, Retract: {IsRetract}"; } - /// - /// Deconstructs the key into its parts - /// - public void Deconstruct(out EntityId entityId, out AttributeId attributeId, out TxId txId, out bool isRetract) - { - entityId = E; - attributeId = A; - txId = T; - isRetract = IsRetract; - } - /// /// Gets the KeyPrefix from the given bytes @@ -97,50 +131,14 @@ public static KeyPrefix Read(ReadOnlySpan bytes) #region Constants - public static KeyPrefix Min => new KeyPrefix - { - _upper = 0, - _lower = 0 - }; - - public static KeyPrefix Max => new KeyPrefix - { - _upper = ulong.MaxValue, - _lower = ulong.MaxValue - }; - #endregion + public static KeyPrefix Min => new(0, 0); - #region Constructors + public static KeyPrefix Max => new(0, 0); /// - /// Creates a new KeyPrefix with the given values, and everything else set to the minimum value + /// Returns true if this key is valid /// - public static implicit operator KeyPrefix(TxId id) - { - var prefix = new KeyPrefix(); - prefix.Set(EntityId.MinValueNoPartition, AttributeId.Min, id, false); - return prefix; - } - - public static implicit operator KeyPrefix(EntityId id) - { - var prefix = new KeyPrefix(); - prefix.Set(id, AttributeId.Min, TxId.MinValue, false); - return prefix; - } - - - public static implicit operator KeyPrefix(AttributeId id) - { - var prefix = new KeyPrefix(); - prefix.Set(EntityId.MinValueNoPartition, id, TxId.MinValue, false); - return prefix; - } - - - - - + public bool IsValid => _upper != 0; #endregion } diff --git a/src/NexusMods.MnemonicDB.Abstractions/KeyBuilder.cs b/src/NexusMods.MnemonicDB.Abstractions/KeyBuilder.cs deleted file mode 100644 index 40688e27..00000000 --- a/src/NexusMods.MnemonicDB.Abstractions/KeyBuilder.cs +++ /dev/null @@ -1,54 +0,0 @@ -using System; -using NexusMods.MnemonicDB.Abstractions.ElementComparers; -using NexusMods.MnemonicDB.Abstractions.Internals; - -namespace NexusMods.MnemonicDB.Abstractions; - -/// -/// Assists in building keys for the datastore, packing multiple values into a single memory buffer. -/// -public class KeyBuilder -{ - private readonly RegistryId _registryId; - - /// - /// Primary constructor, requires a registry id for resolving attribute ids. - /// - public KeyBuilder(RegistryId registryId) - { - _registryId = registryId; - } - - /// - /// Write a lower bound key for the given entity id, with other values set to their minimum. - /// - /// - /// - public Memory From(EntityId e) - { - var writer = new PooledMemoryBufferWriter(32); - var prefix = new KeyPrefix().Set(e, AttributeId.Min, TxId.MinValue, false); - writer.WriteMarshal(prefix); - writer.WriteMarshal((byte)ValueTags.Null); - var output = GC.AllocateUninitializedArray(writer.Length); - writer.WrittenMemory.Span.CopyTo(output); - return output; - } - - /// - /// Write an upper bound key for the given entity id, with other values set to their maximum. - /// - /// - /// - public Memory To(EntityId e) - { - var writer = new PooledMemoryBufferWriter(32); - var prefix = new KeyPrefix().Set(e, AttributeId.Max, TxId.MaxValue, false); - writer.WriteMarshal(prefix); - writer.WriteMarshal((byte)ValueTags.Null); - var output = GC.AllocateUninitializedArray(writer.Length); - writer.WrittenMemory.Span.CopyTo(output); - return output; - } - -} diff --git a/src/NexusMods.MnemonicDB.Abstractions/PooledMemoryBufferWriter.cs b/src/NexusMods.MnemonicDB.Abstractions/PooledMemoryBufferWriter.cs index 163aec99..99a7e4fd 100644 --- a/src/NexusMods.MnemonicDB.Abstractions/PooledMemoryBufferWriter.cs +++ b/src/NexusMods.MnemonicDB.Abstractions/PooledMemoryBufferWriter.cs @@ -3,6 +3,8 @@ using System.Diagnostics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +using NexusMods.MnemonicDB.Abstractions.DatomIterators; +using NexusMods.MnemonicDB.Abstractions.Internals; using Reloaded.Memory.Extensions; namespace NexusMods.MnemonicDB.Abstractions; @@ -99,6 +101,14 @@ public void Write(ReadOnlySpan span) Advance(span.Length); } + public void Write(in Datom datom) + { + var span = GetSpan(KeyPrefix.Size + datom.ValueSpan.Length); + MemoryMarshal.Write(span, datom.Prefix); + datom.ValueSpan.CopyTo(span.SliceFast(KeyPrefix.Size)); + Advance(KeyPrefix.Size + datom.ValueSpan.Length); + } + /// /// Writes the value to the buffer as-is, via MemoryMarshal.Write. /// diff --git a/src/NexusMods.MnemonicDB.Abstractions/Query/ObservableDatoms.cs b/src/NexusMods.MnemonicDB.Abstractions/Query/ObservableDatoms.cs index 226b800c..d3366929 100644 --- a/src/NexusMods.MnemonicDB.Abstractions/Query/ObservableDatoms.cs +++ b/src/NexusMods.MnemonicDB.Abstractions/Query/ObservableDatoms.cs @@ -142,13 +142,9 @@ private static ChangeSet Setup(SortedSet set, IDb db, SliceDescrip private struct Comparer : IComparer, IEqualityComparer where TInner : IDatomComparator { - public unsafe int Compare(Datom a, Datom b) + public int Compare(Datom a, Datom b) { - var aSpan = a.RawSpan; - var bSpan = b.RawSpan; - fixed(byte* aPtr = aSpan) - fixed(byte* bPtr = bSpan) - return TInner.Compare(aPtr, aSpan.Length, bPtr, bSpan.Length); + return TInner.Compare(a, b); } public bool Equals(Datom x, Datom y) diff --git a/src/NexusMods.MnemonicDB.Abstractions/Query/SliceDescriptor.cs b/src/NexusMods.MnemonicDB.Abstractions/Query/SliceDescriptor.cs index dc7299c9..07114b5c 100644 --- a/src/NexusMods.MnemonicDB.Abstractions/Query/SliceDescriptor.cs +++ b/src/NexusMods.MnemonicDB.Abstractions/Query/SliceDescriptor.cs @@ -177,8 +177,6 @@ public static SliceDescriptor Exact(IndexType index, ReadOnlySpan span, IA { var from = span.ToArray(); var to = span.ToArray(); - var prefix = MemoryMarshal.Read(to.AsSpan()); - prefix.Set(prefix.E, prefix.A, TxId.From(prefix.T.Value + 1), prefix.IsRetract); return new SliceDescriptor { Index = index, @@ -226,12 +224,19 @@ public static SliceDescriptor Create(IndexType index, IAttributeRegistry registr { // VAET has a special case where we need to include the reference type and an actual reference // in the slice - var from = GC.AllocateUninitializedArray(KeyPrefix.Size + 9); + var from = GC.AllocateUninitializedArray(KeyPrefix.Size + sizeof(ulong)); from.AsSpan().Clear(); - from[KeyPrefix.Size + 1] = (byte)ValueTags.Reference; - var to = GC.AllocateUninitializedArray(KeyPrefix.Size + 9); + + var fromPrefix = new KeyPrefix(EntityId.MinValueNoPartition, AttributeId.Min, TxId.MinValue, false, ValueTags.Reference); + MemoryMarshal.Write(from, fromPrefix); + + + var to = GC.AllocateUninitializedArray(KeyPrefix.Size + sizeof(ulong)); to.AsSpan().Fill(byte.MaxValue); - from[KeyPrefix.Size + 1] = (byte)ValueTags.Reference; + + var toPrefix = new KeyPrefix(EntityId.MaxValueNoPartition, AttributeId.Max, TxId.MaxValue, true, ValueTags.Reference); + MemoryMarshal.Write(to, toPrefix); + return new SliceDescriptor { Index = index, @@ -261,7 +266,7 @@ public static SliceDescriptor Create(IndexType index, IAttributeRegistry registr public static Datom Datom(EntityId e, AttributeId a, TxId id, bool isRetract, IAttributeRegistry registry) { var data = GC.AllocateUninitializedArray(KeyPrefix.Size); - var prefix = new KeyPrefix().Set(e, a, id, isRetract); + var prefix = new KeyPrefix(e, a, id, isRetract, ValueTags.Null); MemoryMarshal.Write(data, prefix); return new Datom(data, registry); } @@ -294,12 +299,11 @@ public static SliceDescriptor Create(EntityId from, EntityId to, IAttributeRegis /// public static Datom Datom(EntityId e, AttributeId a, EntityId value, TxId id, bool isRetract, IAttributeRegistry registry) { - var data = new Memory(GC.AllocateUninitializedArray(KeyPrefix.Size + 1 + sizeof(ulong))); + var data = new Memory(GC.AllocateUninitializedArray(KeyPrefix.Size + sizeof(ulong))); var span = data.Span; - var prefix = new KeyPrefix().Set(e, a, id, isRetract); + var prefix = new KeyPrefix(e, a, id, isRetract, ValueTags.Reference); MemoryMarshal.Write(span, prefix); - span[KeyPrefix.Size] = (byte)ValueTags.Reference; - MemoryMarshal.Write(span.SliceFast(KeyPrefix.Size + 1), value); + MemoryMarshal.Write(span.SliceFast(KeyPrefix.Size), value); return new Datom(data, registry); } diff --git a/src/NexusMods.MnemonicDB.SourceGenerator/Template.weave b/src/NexusMods.MnemonicDB.SourceGenerator/Template.weave index b3c152c0..26884641 100644 --- a/src/NexusMods.MnemonicDB.SourceGenerator/Template.weave +++ b/src/NexusMods.MnemonicDB.SourceGenerator/Template.weave @@ -22,6 +22,7 @@ using DynamicData; using Microsoft.Extensions.DependencyInjection; using NexusMods.MnemonicDB.Abstractions; using System.Diagnostics.CodeAnalysis; +using System.Diagnostics.Contracts; {{= model.Comments}} public partial class {{= model.Name}} : __MODELS__.IModelFactory<{{= model.Name}}, {{= model.Name}}.ReadOnly> @@ -223,17 +224,6 @@ public partial class {{= model.Name}} : __MODELS__.IModelFactory<{{= model.Name} {{/if}} {{/each}} #endregion - - #region Methods - /// - /// Assumes that this model has been commited to the database - /// in the commit result. Loads this entity from the commited database - /// and returns a ReadOnly model. - /// - public {{= model.Name}}.ReadOnly Remap(__ABSTRACTIONS__.ICommitResult result) { - return new {{= model.Name}}.ReadOnly(result.Db, result[Id]); - } - #endregion } {{= model.Comments}} @@ -328,7 +318,7 @@ public partial class {{= model.Name}} : __MODELS__.IModelFactory<{{= model.Name} } {{each include in model.Includes}} - public {{= include.ToDisplayString()}}.ReadOnly {{= include.Name}} => new {{= include.ToDisplayString()}}.ReadOnly(Db, Id); + public {{= include.ToDisplayString()}}.ReadOnly As{{= include.Name}}() => new {{= include.ToDisplayString()}}.ReadOnly(Db, Id); {{/each}} {{each attr in model.Attributes}} @@ -359,6 +349,7 @@ public partial class {{= model.Name}} : __MODELS__.IModelFactory<{{= model.Name} /// Reloads the entity from the given database, essentially /// refreshing the entity. /// + [Pure] public ReadOnly Rebase(__ABSTRACTIONS__.IDb db) => new ReadOnly(db, Id); /// @@ -485,4 +476,13 @@ public static class {{= model.Name}}Extensions { } {{/each}} + /// + /// Assumes that this model has been commited to the database + /// in the commit result. Loads this entity from the commited database + /// and returns a ReadOnly model. + /// + public static {{= model.Name}}.ReadOnly Remap(this __ABSTRACTIONS__.ICommitResult result, {{= model.Name}}.New model) { + return new {{= model.Name}}.ReadOnly(result.Db, result[model.Id]); + } + } diff --git a/src/NexusMods.MnemonicDB.Storage/Abstractions/AIndex.cs b/src/NexusMods.MnemonicDB.Storage/Abstractions/AIndex.cs index 280e27a5..9d52b709 100644 --- a/src/NexusMods.MnemonicDB.Storage/Abstractions/AIndex.cs +++ b/src/NexusMods.MnemonicDB.Storage/Abstractions/AIndex.cs @@ -8,11 +8,25 @@ public abstract class AIndex(TIndexStore store) : IInd where TComparator : IDatomComparator where TIndexStore : class, IIndexStore { + /// + public void Put(IWriteBatch batch, in Datom datom) + { + batch.Add(store, datom); + } + + /// public void Put(IWriteBatch batch, ReadOnlySpan datom) { batch.Add(store, datom); } + /// + public void Delete(IWriteBatch batch, in Datom datom) + { + batch.Delete(store, datom); + } + + /// public void Delete(IWriteBatch batch, ReadOnlySpan datom) { batch.Delete(store, datom); diff --git a/src/NexusMods.MnemonicDB.Storage/Abstractions/IIndex.cs b/src/NexusMods.MnemonicDB.Storage/Abstractions/IIndex.cs index 292ac118..3c76b97a 100644 --- a/src/NexusMods.MnemonicDB.Storage/Abstractions/IIndex.cs +++ b/src/NexusMods.MnemonicDB.Storage/Abstractions/IIndex.cs @@ -3,8 +3,28 @@ namespace NexusMods.MnemonicDB.Storage.Abstractions; +/// +/// A index definition for a backing datom storage +/// public interface IIndex { + /// + /// Add the delete to the batch + /// + void Delete(IWriteBatch batch, in Datom datom); + + /// + /// Add the delete to the batch + /// void Delete(IWriteBatch batch, ReadOnlySpan span); + + /// + /// Add a put to the batch + /// + void Put(IWriteBatch batch, in Datom datom); + + /// + /// Add a put to the batch + /// void Put(IWriteBatch batch, ReadOnlySpan span); } diff --git a/src/NexusMods.MnemonicDB.Storage/Abstractions/IWriteBatch.cs b/src/NexusMods.MnemonicDB.Storage/Abstractions/IWriteBatch.cs index 1a4f454f..df8ffafa 100644 --- a/src/NexusMods.MnemonicDB.Storage/Abstractions/IWriteBatch.cs +++ b/src/NexusMods.MnemonicDB.Storage/Abstractions/IWriteBatch.cs @@ -1,11 +1,37 @@ using System; +using NexusMods.MnemonicDB.Abstractions.DatomIterators; namespace NexusMods.MnemonicDB.Storage.Abstractions; +/// +/// A write batch for writing multiple operations to the storage +/// public interface IWriteBatch : IDisposable { + /// + /// Commit the batch to the storage + /// public void Commit(); - public void Add(IIndexStore store, ReadOnlySpan key); + /// + /// Add a datom to the batch + /// + /// + /// + public void Add(IIndexStore store, ReadOnlySpan datom); + + /// + /// Add a datom to the batch + /// + public void Add(IIndexStore store, in Datom datom); + + /// + /// Add a delete to the batch + /// public void Delete(IIndexStore store, ReadOnlySpan key); + + /// + /// Add a delete to the batch + /// + public void Delete(IIndexStore store, in Datom datom); } diff --git a/src/NexusMods.MnemonicDB.Storage/AttributeRegistry.cs b/src/NexusMods.MnemonicDB.Storage/AttributeRegistry.cs index bcd368d8..b7e2e6cf 100644 --- a/src/NexusMods.MnemonicDB.Storage/AttributeRegistry.cs +++ b/src/NexusMods.MnemonicDB.Storage/AttributeRegistry.cs @@ -70,7 +70,13 @@ public IReadDatom Resolve(ReadOnlySpan datom) var c = MemoryMarshal.Read(datom); var attr = _attributes[c.A.Value]; - return attr.Resolve(c.E, c.A, datom.SliceFast(KeyPrefix.Size), c.T, c.IsRetract); + return attr.Resolve(c.E, c.A, datom.SliceFast(KeyPrefix.Size), c.T, c.IsRetract, c.ValueTag); + } + + public IReadDatom Resolve(in KeyPrefix prefix, ReadOnlySpan datom) + { + var attr = _attributes[prefix.A.Value]; + return attr.Resolve(prefix.E, prefix.A, datom, prefix.T, prefix.IsRetract, prefix.ValueTag); } /// diff --git a/src/NexusMods.MnemonicDB.Storage/DatomStore.cs b/src/NexusMods.MnemonicDB.Storage/DatomStore.cs index af4b5097..6f02fac4 100644 --- a/src/NexusMods.MnemonicDB.Storage/DatomStore.cs +++ b/src/NexusMods.MnemonicDB.Storage/DatomStore.cs @@ -349,14 +349,13 @@ private void Log(PendingTransaction pendingTransaction, out StoreResult result) var currentPrefix = datom.Prefix; var newE = isRemapped ? remapFn(currentPrefix.E) : currentPrefix.E; - var keyPrefix = new KeyPrefix().Set(newE, currentPrefix.A, thisTx, currentPrefix.IsRetract); + var keyPrefix = currentPrefix with {E = newE, T = thisTx}; { if (attr.IsReference) { - var newV = remapFn(MemoryMarshal.Read(datom.ValueSpan.SliceFast(1))); + var newV = remapFn(MemoryMarshal.Read(datom.ValueSpan)); _writer.WriteMarshal(keyPrefix); - _writer.WriteMarshal((byte)ValueTags.Reference); _writer.WriteMarshal(newV); } else @@ -421,8 +420,7 @@ private void Log(PendingTransaction pendingTransaction, out StoreResult result) private void SwitchPrevToRetraction(TxId thisTx) { var prevKey = MemoryMarshal.Read(_retractWriter.GetWrittenSpan()); - var (e, a, _, _) = prevKey; - prevKey.Set(e, a, thisTx, true); + prevKey = prevKey with {T = thisTx, IsRetract = true}; MemoryMarshal.Write(_retractWriter.GetWrittenSpanWritable(), prevKey); } @@ -431,14 +429,16 @@ private void ProcessRetract(IWriteBatch batch, IAttribute attribute, ReadOnlySpa _prevWriter.Reset(); _prevWriter.Write(datom); var prevKey = MemoryMarshal.Read(_prevWriter.GetWrittenSpan()); - var (e, a, _, _) = prevKey; - prevKey.Set(e, a, TxId.MinValue, false); + + prevKey = prevKey with {T = TxId.MinValue, IsRetract = false}; MemoryMarshal.Write(_prevWriter.GetWrittenSpanWritable(), prevKey); var low = new Datom(_prevWriter.GetWrittenSpan().ToArray(), Registry); - prevKey.Set(e, a, TxId.MaxValue, false); + + prevKey = prevKey with {T = TxId.MaxValue, IsRetract = false}; MemoryMarshal.Write(_prevWriter.GetWrittenSpanWritable(), prevKey); var high = new Datom(_prevWriter.GetWrittenSpan().ToArray(), Registry); + var sliceDescriptor = new SliceDescriptor { Index = IndexType.EAVTCurrent, @@ -455,45 +455,48 @@ private void ProcessRetract(IWriteBatch batch, IAttribute attribute, ReadOnlySpa { Debug.Assert(prevDatom.Valid, "Previous datom should exist"); var debugKey = prevDatom.Prefix; - Debug.Assert(debugKey.E == MemoryMarshal.Read(datom).E, "Entity should match"); - Debug.Assert(debugKey.A == MemoryMarshal.Read(datom).A, "Attribute should match"); + + var otherPrefix = MemoryMarshal.Read(datom); + Debug.Assert(debugKey.E == otherPrefix.E, "Entity should match"); + Debug.Assert(debugKey.A == otherPrefix.A, "Attribute should match"); fixed (byte* aTmp = prevDatom.ValueSpan) fixed (byte* bTmp = datom.SliceFast(sizeof(KeyPrefix))) { - var cmp = ValueComparer.CompareValues(aTmp, prevDatom.ValueSpan.Length, bTmp, datom.Length - sizeof(KeyPrefix)); + var valueTag = prevDatom.Prefix.ValueTag; + var cmp = ValueComparer.CompareValues(prevDatom.Prefix.ValueTag, aTmp, prevDatom.ValueSpan.Length, otherPrefix.ValueTag, bTmp, datom.Length - sizeof(KeyPrefix)); Debug.Assert(cmp == 0, "Values should match"); } } #endif - _eavtCurrent.Delete(batch, prevDatom.RawSpan); - _aevtCurrent.Delete(batch, prevDatom.RawSpan); + _eavtCurrent.Delete(batch, prevDatom); + _aevtCurrent.Delete(batch, prevDatom); if (attribute.IsReference) - _vaetCurrent.Delete(batch, prevDatom.RawSpan); + _vaetCurrent.Delete(batch, prevDatom); if (attribute.IsIndexed) - _avetCurrent.Delete(batch, prevDatom.RawSpan); + _avetCurrent.Delete(batch, prevDatom); _txLog.Put(batch, datom); if (attribute.NoHistory) return; // Move the datom to the history index and also record the retraction - _eavtHistory.Put(batch, prevDatom.RawSpan); + _eavtHistory.Put(batch, prevDatom); _eavtHistory.Put(batch, datom); // Move the datom to the history index and also record the retraction - _aevtHistory.Put(batch, prevDatom.RawSpan); + _aevtHistory.Put(batch, prevDatom); _aevtHistory.Put(batch, datom); if (attribute.IsReference) { - _vaetHistory.Put(batch, prevDatom.RawSpan); + _vaetHistory.Put(batch, prevDatom); _vaetHistory.Put(batch, datom); } if (attribute.IsIndexed) { - _avetHistory.Put(batch, prevDatom.RawSpan); + _avetHistory.Put(batch, prevDatom); _avetHistory.Put(batch, datom); } } @@ -536,7 +539,7 @@ private unsafe PrevState GetPreviousState(bool isRemapped, IAttribute attribute, fixed (byte* a = aSpan) fixed (byte* b = bSpan) { - var cmp = ValueComparer.CompareValues(a, aSpan.Length, b, bSpan.Length); + var cmp = ValueComparer.CompareValues(found.Prefix.ValueTag, a, aSpan.Length, keyPrefix.ValueTag, b, bSpan.Length); return cmp == 0 ? PrevState.Duplicate : PrevState.NotExists; } } @@ -555,15 +558,16 @@ private unsafe PrevState GetPreviousState(bool isRemapped, IAttribute attribute, var aSpan = datom.ValueSpan; var bSpan = span.SliceFast(sizeof(KeyPrefix)); + var bPrefix = MemoryMarshal.Read(span); fixed (byte* a = aSpan) fixed (byte* b = bSpan) { - var cmp = ValueComparer.CompareValues(a, aSpan.Length, b, bSpan.Length); + var cmp = ValueComparer.CompareValues(datom.Prefix.ValueTag, a, aSpan.Length, bPrefix.ValueTag, b, bSpan.Length); if (cmp == 0) return PrevState.Duplicate; } _retractWriter.Reset(); - _retractWriter.Write(datom.RawSpan); + _retractWriter.Write(datom); return PrevState.Exists; } diff --git a/src/NexusMods.MnemonicDB.Storage/InMemoryBackend/Batch.cs b/src/NexusMods.MnemonicDB.Storage/InMemoryBackend/Batch.cs index bf74dbc7..e8c5843a 100644 --- a/src/NexusMods.MnemonicDB.Storage/InMemoryBackend/Batch.cs +++ b/src/NexusMods.MnemonicDB.Storage/InMemoryBackend/Batch.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using NexusMods.MnemonicDB.Abstractions; +using NexusMods.MnemonicDB.Abstractions.DatomIterators; using NexusMods.MnemonicDB.Storage.Abstractions; using IWriteBatch = NexusMods.MnemonicDB.Storage.Abstractions.IWriteBatch; @@ -10,8 +11,10 @@ public class Batch(IndexStore[] stores) : IWriteBatch { private readonly Dictionary> _datoms = new(); + /// public void Dispose() { } + /// public void Commit() { foreach (var (index, datoms) in _datoms) @@ -21,6 +24,7 @@ public void Commit() } } + /// public void Add(IIndexStore store, ReadOnlySpan key) { if (store is not IndexStore indexStore) @@ -35,6 +39,23 @@ public void Add(IIndexStore store, ReadOnlySpan key) datoms.Add((false, key.ToArray())); } + /// + public void Add(IIndexStore store, in Datom datom) + { + if (store is not IndexStore indexStore) + throw new ArgumentException("Invalid store type", nameof(store)); + + if (!_datoms.TryGetValue(indexStore.Type, out var datoms)) + { + datoms = new List<(bool IsDelete, byte[] Data)>(); + _datoms.Add(indexStore.Type, datoms); + } + + datoms.Add((false, datom.ToArray())); + } + + + /// public void Delete(IIndexStore store, ReadOnlySpan key) { if (store is not IndexStore indexStore) @@ -48,4 +69,19 @@ public void Delete(IIndexStore store, ReadOnlySpan key) datoms.Add((true, key.ToArray())); } + + /// + public void Delete(IIndexStore store, in Datom datom) + { + if (store is not IndexStore indexStore) + throw new ArgumentException("Invalid store type", nameof(store)); + + if (!_datoms.TryGetValue(indexStore.Type, out var datoms)) + { + datoms = new List<(bool IsDelete, byte[] Data)>(); + _datoms.Add(indexStore.Type, datoms); + } + + datoms.Add((true, datom.ToArray())); + } } diff --git a/src/NexusMods.MnemonicDB.Storage/InMemoryBackend/Snapshot.cs b/src/NexusMods.MnemonicDB.Storage/InMemoryBackend/Snapshot.cs index f5c74316..7657b7d1 100644 --- a/src/NexusMods.MnemonicDB.Storage/InMemoryBackend/Snapshot.cs +++ b/src/NexusMods.MnemonicDB.Storage/InMemoryBackend/Snapshot.cs @@ -27,8 +27,8 @@ public IndexSegment Datoms(SliceDescriptor descriptor) if (thisIndex.Count == 0) return new IndexSegment(); - var idxLower = thisIndex.IndexOf(descriptor.From.RawSpan.ToArray()); - var idxUpper = thisIndex.IndexOf(descriptor.To.RawSpan.ToArray()); + var idxLower = thisIndex.IndexOf(descriptor.From.ToArray()); + var idxUpper = thisIndex.IndexOf(descriptor.To.ToArray()); bool upperExact = true; bool lowerExact = true; @@ -81,8 +81,8 @@ public IndexSegment Datoms(SliceDescriptor descriptor) /// public IEnumerable DatomsChunked(SliceDescriptor descriptor, int chunkSize) { - var idxLower = _indexes[(int)descriptor.Index].IndexOf(descriptor.From.RawSpan.ToArray()); - var idxUpper = _indexes[(int)descriptor.Index].IndexOf(descriptor.To.RawSpan.ToArray()); + var idxLower = _indexes[(int)descriptor.Index].IndexOf(descriptor.From.ToArray()); + var idxUpper = _indexes[(int)descriptor.Index].IndexOf(descriptor.To.ToArray()); if (idxLower < 0) idxLower = ~idxLower; diff --git a/src/NexusMods.MnemonicDB.Storage/InternalTransaction.cs b/src/NexusMods.MnemonicDB.Storage/InternalTransaction.cs index f8e4f77f..a67900d6 100644 --- a/src/NexusMods.MnemonicDB.Storage/InternalTransaction.cs +++ b/src/NexusMods.MnemonicDB.Storage/InternalTransaction.cs @@ -60,7 +60,7 @@ public void Attach(ITemporaryEntity entity) public void Add(Datom datom) { - datoms.Add(datom.RawSpan); + datoms.Add(datom); } /// diff --git a/src/NexusMods.MnemonicDB.Storage/RocksDbBackend/Batch.cs b/src/NexusMods.MnemonicDB.Storage/RocksDbBackend/Batch.cs index 26700054..b1e26b9d 100644 --- a/src/NexusMods.MnemonicDB.Storage/RocksDbBackend/Batch.cs +++ b/src/NexusMods.MnemonicDB.Storage/RocksDbBackend/Batch.cs @@ -1,4 +1,6 @@ using System; +using System.Runtime.InteropServices; +using NexusMods.MnemonicDB.Abstractions.DatomIterators; using NexusMods.MnemonicDB.Abstractions.ElementComparers; using NexusMods.MnemonicDB.Abstractions.Internals; using NexusMods.MnemonicDB.Storage.Abstractions; @@ -12,6 +14,7 @@ public class Batch(RocksDb db) : IWriteBatch { private readonly WriteBatch _batch = new(); + /// public void Dispose() { _batch.Dispose(); @@ -19,33 +22,99 @@ public void Dispose() private ValueTags Tag(ReadOnlySpan key) { - if (key.Length < KeyPrefix.Size + 1) - return ValueTags.Null; - return (ValueTags)key[KeyPrefix.Size]; + var prefix = MemoryMarshal.Read(key); + return prefix.ValueTag; } + /// public void Add(IIndexStore store, ReadOnlySpan key) { var outOfBandData = ReadOnlySpan.Empty; if (Tag(key) == ValueTags.HashedBlob) { - outOfBandData = key.SliceFast(KeyPrefix.Size + 1 + sizeof(ulong)); - key = key.SliceFast(0, KeyPrefix.Size + 1 + sizeof(ulong)); + outOfBandData = key.SliceFast(KeyPrefix.Size + sizeof(ulong)); + key = key.SliceFast(0, KeyPrefix.Size + sizeof(ulong)); } _batch.Put(key, outOfBandData, ((IRocksDBIndexStore)store).Handle); } + /// + public void Add(IIndexStore store, in Datom datom) + { + if (datom.Prefix.ValueTag == ValueTags.HashedBlob) + { + var outOfBandData = datom.ValueSpan.SliceFast(sizeof(ulong)); + Span keySpan = stackalloc byte[KeyPrefix.Size + sizeof(ulong)]; + + MemoryMarshal.Write(keySpan, datom.Prefix); + datom.ValueSpan.SliceFast(0, sizeof(ulong)).CopyTo(keySpan.SliceFast(KeyPrefix.Size)); + _batch.Put(keySpan, outOfBandData, ((IRocksDBIndexStore)store).Handle); + } + else if (datom.ValueSpan.Length < 256) + { + Span keySpan = stackalloc byte[KeyPrefix.Size + datom.ValueSpan.Length]; + + MemoryMarshal.Write(keySpan, datom.Prefix); + datom.ValueSpan.CopyTo(keySpan.SliceFast(KeyPrefix.Size)); + + _batch.Put(keySpan, ReadOnlySpan.Empty, ((IRocksDBIndexStore)store).Handle); + } + else + { + var keySpan = GC.AllocateUninitializedArray(KeyPrefix.Size + datom.ValueSpan.Length).AsSpan(); + + MemoryMarshal.Write(keySpan, datom.Prefix); + datom.ValueSpan.CopyTo(keySpan[KeyPrefix.Size..]); + + _batch.Put(keySpan, ReadOnlySpan.Empty, ((IRocksDBIndexStore)store).Handle); + } + } + + /// public void Delete(IIndexStore store, ReadOnlySpan key) { if (Tag(key) == ValueTags.HashedBlob) { - key = key.SliceFast(0, KeyPrefix.Size + 1 + sizeof(ulong)); + key = key.SliceFast(0, KeyPrefix.Size + sizeof(ulong)); } _batch.Delete(key, ((IRocksDBIndexStore)store).Handle); } + /// + public void Delete(IIndexStore store, in Datom datom) + { + if (datom.Prefix.ValueTag == ValueTags.HashedBlob) + { + Span keySpan = stackalloc byte[KeyPrefix.Size + sizeof(ulong)]; + + MemoryMarshal.Write(keySpan, datom.Prefix); + datom.ValueSpan.SliceFast(0, sizeof(ulong)).CopyTo(keySpan.SliceFast(KeyPrefix.Size)); + _batch.Delete(keySpan, ((IRocksDBIndexStore)store).Handle); + } + else if (datom.ValueSpan.Length < 256) + { + Span keySpan = stackalloc byte[KeyPrefix.Size + datom.ValueSpan.Length]; + + MemoryMarshal.Write(keySpan, datom.Prefix); + datom.ValueSpan.CopyTo(keySpan.SliceFast(KeyPrefix.Size)); + + _batch.Delete(keySpan, ((IRocksDBIndexStore)store).Handle); + } + else + { + var keySpan = GC.AllocateUninitializedArray(KeyPrefix.Size + datom.ValueSpan.Length).AsSpan(); + + MemoryMarshal.Write(keySpan, datom.Prefix); + datom.ValueSpan.CopyTo(keySpan[KeyPrefix.Size..]); + + _batch.Delete(keySpan, ((IRocksDBIndexStore)store).Handle); + } + + } + + /// public void Commit() { db.Write(_batch); diff --git a/src/NexusMods.MnemonicDB.Storage/RocksDbBackend/Snapshot.cs b/src/NexusMods.MnemonicDB.Storage/RocksDbBackend/Snapshot.cs index fc74e8ae..592ebbb5 100644 --- a/src/NexusMods.MnemonicDB.Storage/RocksDbBackend/Snapshot.cs +++ b/src/NexusMods.MnemonicDB.Storage/RocksDbBackend/Snapshot.cs @@ -20,8 +20,8 @@ public IndexSegment Datoms(SliceDescriptor descriptor) var options = new ReadOptions() .SetSnapshot(_snapshot) - .SetIterateLowerBound(from.RawSpan.ToArray()) - .SetIterateUpperBound(to.RawSpan.ToArray()); + .SetIterateLowerBound(from.ToArray()) + .SetIterateUpperBound(to.ToArray()); using var builder = new IndexSegmentBuilder(registry); @@ -38,10 +38,10 @@ public IndexSegment Datoms(SliceDescriptor descriptor) writer.Reset(); writer.Write(iterator.GetKeySpan()); - if (writer.Length >= KeyPrefix.Size + 1) + if (writer.Length >= KeyPrefix.Size) { - var tag = (ValueTags)writer.GetWrittenSpan()[KeyPrefix.Size]; - if (tag == ValueTags.HashedBlob) + var prefix = KeyPrefix.Read(writer.GetWrittenSpan()); + if (prefix.ValueTag == ValueTags.HashedBlob) { writer.Write(iterator.GetValueSpan()); } @@ -65,8 +65,8 @@ public IEnumerable DatomsChunked(SliceDescriptor descriptor, int c var options = new ReadOptions() .SetSnapshot(_snapshot) - .SetIterateLowerBound(from.RawSpan.ToArray()) - .SetIterateUpperBound(to.RawSpan.ToArray()); + .SetIterateLowerBound(from.ToArray()) + .SetIterateUpperBound(to.ToArray()); using var builder = new IndexSegmentBuilder(registry); @@ -83,10 +83,10 @@ public IEnumerable DatomsChunked(SliceDescriptor descriptor, int c writer.Reset(); writer.Write(iterator.GetKeySpan()); - if (writer.Length >= KeyPrefix.Size + 1) + if (writer.Length >= KeyPrefix.Size) { - var tag = (ValueTags)writer.GetWrittenSpan()[KeyPrefix.Size]; - if (tag == ValueTags.HashedBlob) + var prefix = KeyPrefix.Read(writer.GetWrittenSpan()); + if (prefix.ValueTag == ValueTags.HashedBlob) { writer.Write(iterator.GetValueSpan()); } diff --git a/src/NexusMods.MnemonicDB/AsOfSnapshot.cs b/src/NexusMods.MnemonicDB/AsOfSnapshot.cs index 187bc6c1..d757dc6d 100644 --- a/src/NexusMods.MnemonicDB/AsOfSnapshot.cs +++ b/src/NexusMods.MnemonicDB/AsOfSnapshot.cs @@ -29,14 +29,14 @@ public IndexSegment Datoms(SliceDescriptor descriptor) using var builder = new IndexSegmentBuilder(registry); var merged = current.Merge(history, - (dCurrent, dHistory) => comparatorFn.CompareInstance(dCurrent.RawSpan, dHistory.RawSpan)); + (dCurrent, dHistory) => comparatorFn.CompareInstance(dCurrent, dHistory)); var filtered = merged.Where(d => d.T <= asOfTxId); var withoutRetracts = ApplyRetracts(filtered); foreach (var datom in withoutRetracts) { - builder.Add(datom.RawSpan); + builder.Add(datom); } return builder.Build(); @@ -52,14 +52,14 @@ public IEnumerable DatomsChunked(SliceDescriptor descriptor, int c using var builder = new IndexSegmentBuilder(registry); var merged = current.Merge(history, - (dCurrent, dHistory) => comparatorFn.CompareInstance(dCurrent.RawSpan, dHistory.RawSpan)); + (dCurrent, dHistory) => comparatorFn.CompareInstance(dCurrent, dHistory)); var filtered = merged.Where(d => d.T <= asOfTxId); var withoutRetracts = ApplyRetracts(filtered); foreach (var datom in withoutRetracts) { - builder.Add(datom.RawSpan); + builder.Add(datom); if (builder.Count % chunkSize == 0) { yield return builder.Build(); @@ -95,12 +95,12 @@ public IEnumerable ApplyRetracts(IEnumerable src) if (!havePrevious) { lastDatom.Reset(); - lastDatom.Write(entry.RawSpan); + lastDatom.Write(entry); havePrevious = true; continue; } - var isRetract = IsRetractionFor(lastDatom.GetWrittenSpan(), entry.RawSpan); + var isRetract = IsRetractionFor(lastDatom.GetWrittenSpan(), entry); if (isRetract) { @@ -111,7 +111,7 @@ public IEnumerable ApplyRetracts(IEnumerable src) yield return new Datom(lastDatom.WrittenMemory, registry); lastDatom.Reset(); - lastDatom.Write(entry.RawSpan); + lastDatom.Write(entry); } if (havePrevious) { @@ -119,31 +119,35 @@ public IEnumerable ApplyRetracts(IEnumerable src) } } - private bool IsRetractionFor(ReadOnlySpan aSpan, ReadOnlySpan bSpan) + private bool IsRetractionFor(ReadOnlySpan aSpan, Datom bDatom) { var spanA = MemoryMarshal.Read(aSpan); - var spanB = MemoryMarshal.Read(bSpan); // The lower bit of the lower 8 bytes is the retraction bit, the rest is the Entity ID // so if this XOR returns 1, we know they are the same Entity ID and one of them is a retraction - if ((spanA.Lower ^ spanB.Lower) != 1) + if ((spanA.Lower ^ bDatom.Prefix.Lower) != 1) { return false; } // If the attribute is different, then it's not a retraction - if (spanA.A != spanB.A) + if (spanA.A != bDatom.A) { return false; } // Retracts have to come after the asserts - if (spanA.IsRetract && spanA.T < spanB.T) + if (spanA.IsRetract && spanA.T < bDatom.T) { return true; } - return aSpan.SliceFast(KeyPrefix.Size).SequenceEqual(bSpan.SliceFast(KeyPrefix.Size)); + if (spanA.ValueTag != bDatom.Prefix.ValueTag) + { + return false; + } + + return aSpan.SliceFast(KeyPrefix.Size).SequenceEqual(bDatom.ValueSpan); } } diff --git a/src/NexusMods.MnemonicDB/CommitResult.cs b/src/NexusMods.MnemonicDB/CommitResult.cs index dfa12d6e..78c55e66 100644 --- a/src/NexusMods.MnemonicDB/CommitResult.cs +++ b/src/NexusMods.MnemonicDB/CommitResult.cs @@ -12,15 +12,6 @@ public class CommitResult(IDb db, IDictionary remaps) : ICom public EntityId this[EntityId id] => remaps.TryGetValue(id, out var found) ? found : id; - /// - public T Remap(T model) where T : IHasEntityId - { - var id = model.Id; - if (remaps.TryGetValue(id, out var found)) - id = found; - throw new NotImplementedException(); - } - /// public TxId NewTx => db.BasisTxId; diff --git a/src/NexusMods.MnemonicDB/Db.cs b/src/NexusMods.MnemonicDB/Db.cs index 49286665..a26a9654 100644 --- a/src/NexusMods.MnemonicDB/Db.cs +++ b/src/NexusMods.MnemonicDB/Db.cs @@ -102,13 +102,6 @@ public IndexSegment ReferencesTo(EntityId id) return _referencesCache.Get(this, id); } - public IndexSegment GetSegment(EntityId id) - { - var a = KeyPrefix.Min; - var b = new KeyPrefix().Set(EntityId.MaxValueNoPartition, AttributeId.Max, TxId.MaxValue, false); - return _entityCache.Get(this, id); - } - public IEnumerable GetAll(EntityId id, Attribute attribute) { var attrId = attribute.GetDbId(_registry.Id); diff --git a/src/NexusMods.MnemonicDB/Transaction.cs b/src/NexusMods.MnemonicDB/Transaction.cs index f2a0fded..7df0d3f8 100644 --- a/src/NexusMods.MnemonicDB/Transaction.cs +++ b/src/NexusMods.MnemonicDB/Transaction.cs @@ -56,7 +56,7 @@ public void Add(EntityId entityId, ReferencesAttribute attribute, IEnumerable public void Add(Datom datom) { - _datoms.Add(datom.RawSpan); + _datoms.Add(datom); } public void Add(ITxFunction fn) diff --git a/tests/NexusMods.MnemonicDB.Storage.Tests/ABackendTest.cs b/tests/NexusMods.MnemonicDB.Storage.Tests/ABackendTest.cs index a5dd723a..d49df73e 100644 --- a/tests/NexusMods.MnemonicDB.Storage.Tests/ABackendTest.cs +++ b/tests/NexusMods.MnemonicDB.Storage.Tests/ABackendTest.cs @@ -136,14 +136,9 @@ await Verify(merged.ToTable(Registry)) .UseParameters(type); } - private static unsafe Func CompareDatoms(IDatomComparator comparer) + private static Func CompareDatoms(IDatomComparator comparer) { - return (a, b) => - { - fixed (byte* aPtr = a.RawSpan) - fixed(byte* bPtr = b.RawSpan) - return comparer.CompareInstance(aPtr, a.RawSpan.Length, bPtr, b.RawSpan.Length); - }; + return (a, b) => comparer.CompareInstance(a, b); } diff --git a/tests/NexusMods.MnemonicDB.Tests/AMnemonicDBTest.cs b/tests/NexusMods.MnemonicDB.Tests/AMnemonicDBTest.cs index eb9d45ca..8ce45d1b 100644 --- a/tests/NexusMods.MnemonicDB.Tests/AMnemonicDBTest.cs +++ b/tests/NexusMods.MnemonicDB.Tests/AMnemonicDBTest.cs @@ -99,7 +99,7 @@ protected SettingsTask VerifyTable(IEnumerable datoms) } var txResult = await tx.Commit(); - var loadoutWritten = loadout.Remap(txResult); + var loadoutWritten = txResult.Remap(loadout); var tx2 = Connection.BeginTransaction(); foreach (var mod in loadoutWritten.Mods) diff --git a/tests/NexusMods.MnemonicDB.Tests/ComplexModelTests.cs b/tests/NexusMods.MnemonicDB.Tests/ComplexModelTests.cs index c9bacbb8..12edf409 100644 --- a/tests/NexusMods.MnemonicDB.Tests/ComplexModelTests.cs +++ b/tests/NexusMods.MnemonicDB.Tests/ComplexModelTests.cs @@ -1,6 +1,7 @@ using System.Diagnostics; using Microsoft.Extensions.Logging; using NexusMods.Hashing.xxHash64; +using NexusMods.MnemonicDB.Abstractions.BuiltInEntities; using NexusMods.MnemonicDB.TestModel; using NexusMods.Paths; using File = NexusMods.MnemonicDB.TestModel.File; @@ -77,15 +78,11 @@ public async Task CanCreateLoadout(int modCount, int filesPerMod) Logger.LogInformation($"Commit took {sw.ElapsedMilliseconds}ms"); - var db = Connection.Db; + var loadoutRO = result.Remap(loadout); - var loadoutRO = loadout.Remap(result); + loadoutRO.Mods.Count.Should().Be(modCount, "all mods should be loaded"); - var totalSize = Size.Zero; - - loadoutRO.Mods.Count().Should().Be(modCount, "all mods should be loaded"); - - loadoutRO.Collections.Count().Should().Be(2, "all collections should be loaded"); + loadoutRO.Collections.Count.Should().Be(2, "all collections should be loaded"); loadoutRO.Collections.SelectMany(c => c.ModIds) .Count().Should().Be(loadoutRO.Mods.Count(), "all mods should be in a collection"); @@ -155,9 +152,9 @@ public async Task CanRestartStorage(int modCount, int filesPerMod, int extraFile var result = await tx.Commit(); var extraTx = Connection.BeginTransaction(); - var loadout = newLoadout.Remap(result); + var loadout = result.Remap(newLoadout); - var firstMod = mods[0].Remap(result); + var firstMod = result.Remap(mods[0]); for (var idx = 0; idx < extraFiles; idx++) { var name = $"Extra File {idx}"; @@ -202,7 +199,7 @@ public async Task CanRestartStorage(int modCount, int filesPerMod, int extraFile }; var result2 = await tx2.Commit(); - var newNewLoadOut = newNewLoadOutNew.Remap(result2); + var newNewLoadOut = result2.Remap(newNewLoadOutNew); newNewLoadOut.Id.Should().NotBe(loadout.Id, "new loadout should have a different id because the connection re-detected the max EntityId"); diff --git a/tests/NexusMods.MnemonicDB.Tests/DbTests.cs b/tests/NexusMods.MnemonicDB.Tests/DbTests.cs index 96c9ac40..2dbd059f 100644 --- a/tests/NexusMods.MnemonicDB.Tests/DbTests.cs +++ b/tests/NexusMods.MnemonicDB.Tests/DbTests.cs @@ -37,7 +37,6 @@ public async Task ReadDatomsForEntity() var oldTx = Connection.TxId; var result = await tx.Commit(); - await Task.Delay(1000); result.NewTx.Should().NotBe(oldTx, "transaction id should be incremented"); result.NewTx.Value.Should().Be(oldTx.Value + 1, "transaction id should be incremented by 1"); @@ -170,7 +169,7 @@ public async Task ReadModelsCanHaveExtraAttributes() readModel.Id.Should().Be(archiveReadModel.Id, "both models are the same entity"); - archiveReadModel.File.ToArray().Should().BeEquivalentTo(readModel.ToArray(), "archive file should have the same base data as the file"); + archiveReadModel.AsFile().ToArray().Should().BeEquivalentTo(readModel.ToArray(), "archive file should have the same base data as the file"); readModel.TryGetAsArchiveFile(out var castedDown).Should().BeTrue(); (castedDown is ArchiveFile.ReadOnly).Should().BeTrue(); @@ -253,7 +252,7 @@ public async Task CanGetChildEntities() var newDb = Connection.Db; - var loadoutWritten = loadout.Remap(result); + var loadoutWritten = result.Remap(loadout); loadoutWritten.Mods.Count.Should().Be(2); loadoutWritten.Mods.Select(m => m.Name).Should().BeEquivalentTo(["Test Mod 1", "Test Mod 2"]); @@ -331,9 +330,9 @@ public async Task CanPutEntitiesInDifferentPartitions() (file3.Id.Value >> 40 & 0xFF).Should().Be(200); var result = await tx.Commit(); - var file1RO = file1.Remap(result); - var file2RO = file2.Remap(result); - var file3RO = file3.Remap(result); + var file1RO = result.Remap(file1); + var file2RO = result.Remap(file2); + var file3RO = result.Remap(file3); var allDatoms = file1RO.Concat(file2RO).Concat(file3RO); @@ -458,7 +457,7 @@ public async Task NonRecursiveDeleteDeletesOnlyOneEntity() tx.Delete(firstMod.Id, false); var result = await tx.Commit(); - loadout = loadout.Rebase(result.Db); + loadout = loadout.Rebase(); loadout.Mods.Count.Should().Be(2); @@ -541,7 +540,7 @@ public async Task CanReadAndWriteOptionalAttributes() }; var result = await tx.Commit(); - var remapped = mod.Remap(result); + var remapped = result.Remap(mod); remapped.Description.Should().Be("Test Description"); }