Skip to content

Commit

Permalink
Merge pull request #66 from Nexus-Mods/rework-value-serialization
Browse files Browse the repository at this point in the history
Rework value serialization
  • Loading branch information
halgari authored Jun 23, 2024
2 parents 1378b92 + a77fd0b commit 27da3d9
Show file tree
Hide file tree
Showing 52 changed files with 1,122 additions and 453 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
using System;
using BenchmarkDotNet.Attributes;
using NexusMods.MnemonicDB.Abstractions;
using NexusMods.MnemonicDB.Abstractions.DatomIterators;
using NexusMods.MnemonicDB.Abstractions.ElementComparers;
using NexusMods.MnemonicDB.Abstractions.IndexSegments;
using NexusMods.MnemonicDB.Abstractions.Internals;
using NexusMods.MnemonicDB.Storage;

namespace NexusMods.MnemonicDB.Benchmarks.Benchmarks;

public class IndexSegmentABenchmarks
{
private readonly IndexSegment _index;

public IndexSegmentABenchmarks()
{
var registry = new AttributeRegistry([]);
using var builder = new IndexSegmentBuilder(registry);

for (int a = 1; a < 100; a++)
{
var prefix = new KeyPrefix(EntityId.From(42), AttributeId.From((ushort)a), TxId.From(42), false,
ValueTags.Null);
builder.Add(new Datom(prefix, ReadOnlyMemory<byte>.Empty, registry));
}

_index = builder.Build();
}

[Params(1, 2, 3, 4, 5, 20, 30, 50, 99)]
public int ToFind { get; set; }

[Benchmark]
public int FindLinear()
{
var find = AttributeId.From((ushort)ToFind);
for (var i = 0; i < _index.Count; i++)
{
var datom = _index[i];
if (datom.A == find)
return i;
}
return -1;
}


}
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
using System;
using BenchmarkDotNet.Attributes;
using NexusMods.MnemonicDB.Abstractions;
using NexusMods.MnemonicDB.Abstractions.DatomIterators;
using NexusMods.MnemonicDB.Abstractions.ElementComparers;
using NexusMods.MnemonicDB.Abstractions.IndexSegments;
using NexusMods.MnemonicDB.Abstractions.Internals;
using NexusMods.MnemonicDB.Storage;

namespace NexusMods.MnemonicDB.Benchmarks.Benchmarks;

[MemoryDiagnoser]
public class IndexSegmentEBenchmarks
{
private readonly IndexSegment _index;

public IndexSegmentEBenchmarks()
{
var registry = new AttributeRegistry([]);
using var builder = new IndexSegmentBuilder(registry);

for (var e = 1; e < 100; e++)
{
for (var a = 0; a < 20; a++)
{
builder.Add(new Datom(new KeyPrefix(EntityId.From((ulong)e), AttributeId.From((ushort)a), TxId.From((ulong)(e + a)), false,
ValueTags.Null), ReadOnlyMemory<byte>.Empty, registry));
}
}

_index = builder.Build();
}

[Params(0, 1, 2, 3, 10, 71, 99)] public int ToFind { get; set; }


[Benchmark]
public int FindLinear()
{
var find = EntityId.From((ulong)ToFind);
for (var i = 0; i < _index.Count; i++)
{
var datom = _index[i];
if (datom.E == find)
return i;
}
return -1;
}

[Benchmark]
public int FindBinarySearch()
{
var find = EntityId.From((ulong)ToFind);

var left = 0;
var right = _index.Count - 1;
var result = -1;
while (left <= right)
{
var mid = left + (right - left) / 2;
var datom = _index[mid];
var comparison = datom.E.CompareTo(find);
if (comparison == 0)
{
result = mid; // Don't return, but continue searching to the left
right = mid - 1;
}
else if (comparison < 0)
{
left = mid + 1;
}
else
{
right = mid - 1;
}
}
return result; // Return the first occurrence found, or -1 if not found
}

[Benchmark]
public int FindBinarySearchRework()
{
var find = EntityId.From((ulong)ToFind);
return _index.FindFirst(find); // Return the first occurrence found, or -1 if not found
}

[Benchmark]
public int FindBinarySearchReworkAVX2()
{
var find = EntityId.From((ulong)ToFind);
return _index.FindFirstAVX2(find.Value); // Return the first occurrence found, or -1 if not found
}


}
8 changes: 3 additions & 5 deletions benchmarks/NexusMods.MnemonicDB.Benchmarks/Program.cs
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,14 @@

using System;
using System.Diagnostics;
using BenchmarkDotNet.Configs;
using BenchmarkDotNet.Diagnosers;
using BenchmarkDotNet.Running;
using JetBrains.Profiler.Api;
using NexusMods.MnemonicDB.Benchmarks.Benchmarks;


//#if DEBUG
#if DEBUG

var benchmark = new ReadTests
{
Expand All @@ -29,12 +30,9 @@
MeasureProfiler.SaveData();
Console.WriteLine("Elapsed: " + sw.Elapsed + " Result: " + result);

/*
#else

BenchmarkRunner.Run<ReadTests>();
BenchmarkRunner.Run<IndexSegmentEBenchmarks>(config: DefaultConfig.Instance.WithOption(ConfigOptions.DisableOptimizationsValidator, true));
#endif

*/

77 changes: 33 additions & 44 deletions src/NexusMods.MnemonicDB.Abstractions/Attribute.cs
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,11 @@
using System.Buffers;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Numerics;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Text;
using NexusMods.MnemonicDB.Abstractions.ElementComparers;
using NexusMods.MnemonicDB.Abstractions.Exceptions;
using NexusMods.MnemonicDB.Abstractions.IndexSegments;
using NexusMods.MnemonicDB.Abstractions.Internals;
using NexusMods.MnemonicDB.Abstractions.Models;
using Reloaded.Memory.Extensions;
Expand Down Expand Up @@ -198,13 +195,13 @@ public AttributeId GetDbId(RegistryId id)

/// <inheritdoc />
public IReadDatom Resolve(EntityId entityId, AttributeId attributeId, ReadOnlySpan<byte> value, TxId tx,
bool isRetract)
bool isRetract, ValueTags valueTag)
{
return new ReadDatom
{
E = entityId,
A = this,
V = ReadValue(value),
V = ReadValue(value, valueTag),
T = tx,
IsRetract = isRetract
};
Expand Down Expand Up @@ -310,54 +307,47 @@ private void WriteValueLowLevel<TWriter>(TLowLevelType value, TWriter writer)

private void WriteNull<TWriter>(TWriter writer) where TWriter : IBufferWriter<byte>
{
var span = writer.GetSpan(1);
span[0] = (byte)ValueTags.Null;
writer.Advance(1);
// Do Nothing
}

private void WriteAscii<TWriter>(string s, TWriter writer) where TWriter : IBufferWriter<byte>
{
var size = s.Length;
var span = writer.GetSpan(size + 1);
span[0] = (byte)LowLevelType;
AsciiEncoding.GetBytes(s, span.SliceFast(1));
writer.Advance(size + 1);
var span = writer.GetSpan(size);
AsciiEncoding.GetBytes(s, span);
writer.Advance(size);
}

private void WriteUtf8<TWriter>(string s, TWriter writer) where TWriter : IBufferWriter<byte>
{
var size = Utf8Encoding.GetByteCount(s);
var span = writer.GetSpan(size + 1);
span[0] = (byte)LowLevelType;
Utf8Encoding.GetBytes(s, span.SliceFast(1));
writer.Advance(size + 1);
var span = writer.GetSpan(size);
Utf8Encoding.GetBytes(s, span);
writer.Advance(size);
}

public TValueType ReadValue(ReadOnlySpan<byte> span)
public TValueType ReadValue(ReadOnlySpan<byte> span, ValueTags tag)
{
var tag = (ValueTags)span[0];
var rest = span.SliceFast(1);
Debug.Assert(tag == LowLevelType, "Tag mismatch");
return tag switch
return LowLevelType switch
{
ValueTags.Null => NullFromLowLevel(),
ValueTags.UInt8 => FromLowLevel(ReadUnmanaged<byte>(rest), tag),
ValueTags.UInt16 => FromLowLevel(ReadUnmanaged<ushort>(rest), tag),
ValueTags.UInt32 => FromLowLevel(ReadUnmanaged<uint>(rest), tag),
ValueTags.UInt64 => FromLowLevel(ReadUnmanaged<ulong>(rest), tag),
ValueTags.UInt128 => FromLowLevel(ReadUnmanaged<UInt128>(rest), tag),
ValueTags.Int16 => FromLowLevel(ReadUnmanaged<short>(rest), tag),
ValueTags.Int32 => FromLowLevel(ReadUnmanaged<int>(rest), tag),
ValueTags.Int64 => FromLowLevel(ReadUnmanaged<long>(rest), tag),
ValueTags.Int128 => FromLowLevel(ReadUnmanaged<Int128>(rest), tag),
ValueTags.Float32 => FromLowLevel(ReadUnmanaged<float>(rest), tag),
ValueTags.Float64 => FromLowLevel(ReadUnmanaged<double>(rest), tag),
ValueTags.Reference => FromLowLevel(ReadUnmanaged<ulong>(rest), tag),
ValueTags.Ascii => FromLowLevel(ReadAscii(rest), tag),
ValueTags.Utf8 => FromLowLevel(ReadUtf8(rest), tag),
ValueTags.Utf8Insensitive => FromLowLevel(ReadUtf8(rest), tag),
ValueTags.Blob => FromLowLevel(rest, tag),
ValueTags.HashedBlob => FromLowLevel(rest.SliceFast(sizeof(ulong)), tag),
ValueTags.UInt8 => FromLowLevel(ReadUnmanaged<byte>(span), tag),
ValueTags.UInt16 => FromLowLevel(ReadUnmanaged<ushort>(span), tag),
ValueTags.UInt32 => FromLowLevel(ReadUnmanaged<uint>(span), tag),
ValueTags.UInt64 => FromLowLevel(ReadUnmanaged<ulong>(span), tag),
ValueTags.UInt128 => FromLowLevel(ReadUnmanaged<UInt128>(span), tag),
ValueTags.Int16 => FromLowLevel(ReadUnmanaged<short>(span), tag),
ValueTags.Int32 => FromLowLevel(ReadUnmanaged<int>(span), tag),
ValueTags.Int64 => FromLowLevel(ReadUnmanaged<long>(span), tag),
ValueTags.Int128 => FromLowLevel(ReadUnmanaged<Int128>(span), tag),
ValueTags.Float32 => FromLowLevel(ReadUnmanaged<float>(span), tag),
ValueTags.Float64 => FromLowLevel(ReadUnmanaged<double>(span), tag),
ValueTags.Reference => FromLowLevel(ReadUnmanaged<ulong>(span), tag),
ValueTags.Ascii => FromLowLevel(ReadAscii(span), tag),
ValueTags.Utf8 => FromLowLevel(ReadUtf8(span), tag),
ValueTags.Utf8Insensitive => FromLowLevel(ReadUtf8(span), tag),
ValueTags.Blob => FromLowLevel(span, tag),
ValueTags.HashedBlob => FromLowLevel(span.SliceFast(sizeof(ulong)), tag),
_ => throw new UnsupportedLowLevelReadType(tag)
};
}
Expand All @@ -382,10 +372,9 @@ private unsafe void WriteUnmanaged<TWriter, TValue>(TValue value, TWriter writer
where TWriter : IBufferWriter<byte>
where TValue : unmanaged
{
var span = writer.GetSpan(sizeof(TValue) + 1);
span[0] = (byte) LowLevelType;
MemoryMarshal.Write(span.SliceFast(1), value);
writer.Advance(sizeof(TValue) + 1);
var span = writer.GetSpan(sizeof(TValue));
MemoryMarshal.Write(span, value);
writer.Advance(sizeof(TValue));
}

private TValue ReadUnmanaged<TValue>(ReadOnlySpan<byte> span)
Expand All @@ -401,7 +390,7 @@ public virtual void Write<TWriter>(EntityId entityId, RegistryId registryId, TVa
where TWriter : IBufferWriter<byte>
{
Debug.Assert(LowLevelType != ValueTags.Blob, "Blobs should overwrite this method and throw when ToLowLevel is called");
var prefix = new KeyPrefix().Set(entityId, GetDbId(registryId), txId, isRetract);
var prefix = new KeyPrefix(entityId, GetDbId(registryId), txId, isRetract, LowLevelType);
var span = writer.GetSpan(KeyPrefix.Size);
MemoryMarshal.Write(span, prefix);
writer.Advance(KeyPrefix.Size);
Expand All @@ -414,7 +403,7 @@ public virtual void Write<TWriter>(EntityId entityId, RegistryId registryId, TVa
protected void WritePrefix<TWriter>(EntityId entityId, RegistryId registryId, TxId txId, bool isRetract, TWriter writer)
where TWriter : IBufferWriter<byte>
{
var prefix = new KeyPrefix().Set(entityId, GetDbId(registryId), txId, isRetract);
var prefix = new KeyPrefix(entityId, GetDbId(registryId), txId, isRetract, LowLevelType);
var span = writer.GetSpan(KeyPrefix.Size);
MemoryMarshal.Write(span, prefix);
writer.Advance(KeyPrefix.Size);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,6 @@ public abstract class BlobAttribute<TValue>(string ns, string name) : ScalarAttr
public override void Write<TWriter>(EntityId entityId, RegistryId registryId, TValue value, TxId txId, bool isRetract, TWriter writer)
{
WritePrefix(entityId, registryId, txId, isRetract, writer);
var span = writer.GetSpan(1);
span[0] = (byte)ValueTags.Blob;
writer.Advance(1);
WriteValue(value, writer);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,9 @@ public override void Write<TWriter>(EntityId entityId, RegistryId registryId, TV

var hash = XxHash3.HashToUInt64(valueSpan);

var writerSpan = writer.GetSpan(sizeof(ulong) + 1);
writerSpan[0] = (byte)ValueTags.HashedBlob;
MemoryMarshal.Write(writerSpan.SliceFast(1), hash);
writer.Advance(sizeof(ulong) + 1);
var writerSpan = writer.GetSpan(sizeof(ulong));
MemoryMarshal.Write(writerSpan, hash);
writer.Advance(sizeof(ulong));
writer.Write(valueSpan);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ public TValue Get(IHasEntityIdAndDb entity)
{
var datom = segment[i];
if (datom.A != dbId) continue;
return ReadValue(datom.ValueSpan);
return ReadValue(datom.ValueSpan, datom.Prefix.ValueTag);
}

if (DefaultValue is not null)
Expand Down Expand Up @@ -64,7 +64,7 @@ public bool TryGet(IHasEntityIdAndDb entity, out TValue value)
{
var datom = segment[i];
if (datom.A != dbId) continue;
value = ReadValue(datom.ValueSpan);
value = ReadValue(datom.ValueSpan, datom.Prefix.ValueTag);
return true;
}

Expand Down
Loading

0 comments on commit 27da3d9

Please sign in to comment.