Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix tuple 3 handling, and add an API for value remapping from user code #100

Merged
merged 2 commits into from
Sep 21, 2024
Merged

Fix tuple 3 handling, and add an API for value remapping from user code #100

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5d9a464
Fix tuple 3 handling, and add an API for value remapping from user code
halgari Sep 21, 2024
3b4edec
Update changelog
halgari Sep 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
## Changelog

### 0.9.84 - 20/09/2024
* Fixed a bug with Tuple3 values that had a reference in the first position.
* Added a user accessible remap function for values

### 0.9.83 - 20/09/2024
* Optimized the interface with RocksDB used all throughout the library. Results in a 30% speedup on search operations
inside RocksDB.
Expand Down
15 changes: 14 additions & 1 deletion src/NexusMods.MnemonicDB.Abstractions/PartitionId.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,20 @@ public static PartitionId User(byte id)
public EntityId MaxValue => EntityId.From(((ulong)Value << 56) | 0x00FFFFFFFFFFFFFF);

/// <inheritdoc />
public override string ToString() => $"PartId:{Value:x}";
public override string ToString()
{
if (Value == Attribute)
return $"PartId:({Attribute.Value:x})Attribute";
if (Value == Transactions)
return $"PartId:({Transactions.Value:x})Transactions";
if (Value == Entity)
return $"PartId:({Entity.Value:x})Entity";
if (Value == Temp)
return $"PartId:({Temp.Value:x})Temp";
if (Value > Temp)
return $"PartId:({Value:x})User";
return $"PartId:{Value:x}";
}

/// <summary>
/// Encode a partition id and entity id pair
Expand Down
66 changes: 66 additions & 0 deletions src/NexusMods.MnemonicDB.Abstractions/ValueHelpers.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
using System;
using System.Runtime.InteropServices;
using NexusMods.MnemonicDB.Abstractions.ElementComparers;
using NexusMods.MnemonicDB.Abstractions.Internals;
using Reloaded.Memory.Extensions;

namespace NexusMods.MnemonicDB.Abstractions;

/// <summary>
/// Static methods that help with reading, writing and formatting values
/// </summary>
public static class ValueHelpers
{
/// <summary>
/// Remaps the values of this attribute, if required.
/// </summary>
public static void Remap(Func<EntityId, EntityId> remapFn, in KeyPrefix prefix, Span<byte> valueSpan)
{
switch (prefix.ValueTag)
{
case ValueTags.Reference:
var oldId = MemoryMarshal.Read<EntityId>(valueSpan);
var newId = remapFn(oldId);
MemoryMarshal.Write(valueSpan, newId);
break;
case ValueTags.Tuple2:
{
var tag1 = (ValueTags)valueSpan[0];
var tag2 = (ValueTags)valueSpan[1];
if (tag1 == ValueTags.Reference)
{
var entityId = MemoryMarshal.Read<EntityId>(valueSpan.SliceFast(2));
var newEntityId = remapFn(entityId);
MemoryMarshal.Write(valueSpan.SliceFast(2), newEntityId);
}
if (tag2 == ValueTags.Reference)
{
throw new NotSupportedException("This attribute does not support remapping of the second element.");
}
break;
}
case ValueTags.Tuple3:
{
var tag1 = (ValueTags)valueSpan[0];
var tag2 = (ValueTags)valueSpan[1];
var tag3 = (ValueTags)valueSpan[2];
if (tag1 == ValueTags.Reference)
{
var entityId = MemoryMarshal.Read<EntityId>(valueSpan.SliceFast(3));
var newEntityId = remapFn(entityId);
MemoryMarshal.Write(valueSpan.SliceFast(3), newEntityId);
}
if (tag2 == ValueTags.Reference)
{
throw new NotSupportedException("This attribute does not support remapping of the second element.");
}
if (tag3 == ValueTags.Reference)
{
throw new NotSupportedException("This attribute does not support remapping of the third element.");
}
break;
}
}
}

}
37 changes: 7 additions & 30 deletions src/NexusMods.MnemonicDB/Storage/DatomStore.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,11 @@ public class DatomStore : IDatomStore
private IDb? _currentDb = null;

private static readonly TimeSpan TransactionTimeout = TimeSpan.FromMinutes(120);

/// <summary>
/// Cached function to remap temporary entity ids to real entity ids
/// </summary>
private readonly Func<EntityId, EntityId> _remapFunc;

/// <summary>
/// Used to remap temporary entity ids to real entity ids, this is cleared after each transaction
Expand All @@ -76,6 +81,7 @@ public class DatomStore : IDatomStore
/// </summary>
public DatomStore(ILogger<DatomStore> logger, DatomStoreSettings settings, IStoreBackend backend)
{
_remapFunc = Remap;
_attributeCache = backend.AttributeCache;
_pendingTransactions = new BlockingCollection<PendingTransaction>(new ConcurrentQueue<PendingTransaction>());

Expand Down Expand Up @@ -400,7 +406,7 @@ private void Log(PendingTransaction pendingTransaction, out StoreResult result)
var valueSpan = datom.ValueSpan;
var span = _writer.GetSpan(valueSpan.Length);
valueSpan.CopyTo(span);
Remap(in keyPrefix, span);
ValueHelpers.Remap(_remapFunc, in keyPrefix, span);
_writer.Advance(valueSpan.Length);
}

Expand Down Expand Up @@ -450,35 +456,6 @@ private void Log(PendingTransaction pendingTransaction, out StoreResult result)
};
}

private void Remap(in KeyPrefix prefix, Span<byte> valueSpan)
{
switch (prefix.ValueTag)
{
case ValueTags.Reference:
var oldId = MemoryMarshal.Read<EntityId>(valueSpan);
var newId = Remap(oldId);
MemoryMarshal.Write(valueSpan, newId);
break;
case ValueTags.Tuple2:
{
var tag1 = (ValueTags)valueSpan[0];
var tag2 = (ValueTags)valueSpan[1];
if (tag1 == ValueTags.Reference)
{
var entityId = MemoryMarshal.Read<EntityId>(valueSpan.SliceFast(2));
var newEntityId = Remap(entityId);
MemoryMarshal.Write(valueSpan.SliceFast(2), newEntityId);
}
if (tag2 == ValueTags.Reference)
{
throw new NotSupportedException("This attribute does not support remapping of the second element.");
}
break;
}
}

}

/// <summary>
/// Updates the data in _prevWriter to be a retraction of the data in that write.
/// </summary>
Expand Down
18 changes: 0 additions & 18 deletions tests/NexusMods.MnemonicDB.TestModel/Attributes/Int3Attribute.cs
View file
Open in desktop

This file was deleted.

18 changes: 18 additions & 0 deletions tests/NexusMods.MnemonicDB.TestModel/Attributes/Tuple3TestAttribute.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
using NexusMods.MnemonicDB.Abstractions;
using NexusMods.MnemonicDB.Abstractions.Attributes;
using NexusMods.MnemonicDB.Abstractions.ElementComparers;

namespace NexusMods.MnemonicDB.TestModel.Attributes;

public class Tuple3TestAttribute(string ns, string name) : TupleAttribute<EntityId, ulong, int, int, string, string>(ValueTags.Reference, ValueTags.Int32, ValueTags.Ascii, ns, name)
{
protected override (EntityId, int, string) FromLowLevel((ulong, int, string) value)
{
return (EntityId.From(value.Item1), value.Item2, value.Item3);
}

protected override (ulong, int, string) ToLowLevel((EntityId, int, string) value)
{
return (value.Item1.Value, value.Item2, value.Item3);
}
}
2 changes: 1 addition & 1 deletion tests/NexusMods.MnemonicDB.TestModel/File.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,5 +20,5 @@ public partial class File : IModelDefinition
/// <summary>
/// Tuple3 test
/// </summary>
public static readonly Int3Attribute TupleTest = new(Namespace, nameof(TupleTest)) { IsIndexed = true, IsOptional = true};
public static readonly Tuple3TestAttribute TupleTest = new(Namespace, nameof(TupleTest)) { IsIndexed = true, IsOptional = true};
}
54 changes: 27 additions & 27 deletions tests/NexusMods.MnemonicDB.Tests/DbTests.CanUseTuple3Attributes.verified.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,27 +1,27 @@
+ | 0200000000000001 | TupleTest | (1, 1, 1) | 0100000000000003
+ | 0200000000000002 | TupleTest | (1, 1, 2) | 0100000000000003
+ | 0200000000000003 | TupleTest | (1, 1, 3) | 0100000000000003
+ | 0200000000000004 | TupleTest | (1, 2, 1) | 0100000000000003
+ | 0200000000000005 | TupleTest | (1, 2, 2) | 0100000000000003
+ | 0200000000000006 | TupleTest | (1, 2, 3) | 0100000000000003
+ | 0200000000000007 | TupleTest | (1, 3, 1) | 0100000000000003
+ | 0200000000000008 | TupleTest | (1, 3, 2) | 0100000000000003
+ | 0200000000000009 | TupleTest | (1, 3, 3) | 0100000000000003
+ | 020000000000000A | TupleTest | (2, 1, 1) | 0100000000000003
+ | 020000000000000B | TupleTest | (2, 1, 2) | 0100000000000003
+ | 020000000000000C | TupleTest | (2, 1, 3) | 0100000000000003
+ | 020000000000000D | TupleTest | (2, 2, 1) | 0100000000000003
+ | 020000000000000E | TupleTest | (2, 2, 2) | 0100000000000003
+ | 020000000000000F | TupleTest | (2, 2, 3) | 0100000000000003
+ | 0200000000000010 | TupleTest | (2, 3, 1) | 0100000000000003
+ | 0200000000000011 | TupleTest | (2, 3, 2) | 0100000000000003
+ | 0200000000000012 | TupleTest | (2, 3, 3) | 0100000000000003
+ | 0200000000000013 | TupleTest | (3, 1, 1) | 0100000000000003
+ | 0200000000000014 | TupleTest | (3, 1, 2) | 0100000000000003
+ | 0200000000000015 | TupleTest | (3, 1, 3) | 0100000000000003
+ | 0200000000000016 | TupleTest | (3, 2, 1) | 0100000000000003
+ | 0200000000000017 | TupleTest | (3, 2, 2) | 0100000000000003
+ | 0200000000000018 | TupleTest | (3, 2, 3) | 0100000000000003
+ | 0200000000000019 | TupleTest | (3, 3, 1) | 0100000000000003
+ | 020000000000001A | TupleTest | (3, 3, 2) | 0100000000000003
+ | 020000000000001B | TupleTest | (3, 3, 3) | 0100000000000003
+ | 0200000000000001 | TupleTest | (EId:200000000000001, 1, 1) | 0100000000000003
+ | 0200000000000002 | TupleTest | (EId:200000000000002, 1, 2) | 0100000000000003
+ | 0200000000000003 | TupleTest | (EId:200000000000003, 1, 3) | 0100000000000003
+ | 0200000000000004 | TupleTest | (EId:200000000000004, 2, 1) | 0100000000000003
+ | 0200000000000005 | TupleTest | (EId:200000000000005, 2, 2) | 0100000000000003
+ | 0200000000000006 | TupleTest | (EId:200000000000006, 2, 3) | 0100000000000003
+ | 0200000000000007 | TupleTest | (EId:200000000000007, 3, 1) | 0100000000000003
+ | 0200000000000008 | TupleTest | (EId:200000000000008, 3, 2) | 0100000000000003
+ | 0200000000000009 | TupleTest | (EId:200000000000009, 3, 3) | 0100000000000003
+ | 020000000000000A | TupleTest | (EId:20000000000000A, 1, 1) | 0100000000000003
+ | 020000000000000B | TupleTest | (EId:20000000000000B, 1, 2) | 0100000000000003
+ | 020000000000000C | TupleTest | (EId:20000000000000C, 1, 3) | 0100000000000003
+ | 020000000000000D | TupleTest | (EId:20000000000000D, 2, 1) | 0100000000000003
+ | 020000000000000E | TupleTest | (EId:20000000000000E, 2, 2) | 0100000000000003
+ | 020000000000000F | TupleTest | (EId:20000000000000F, 2, 3) | 0100000000000003
+ | 0200000000000010 | TupleTest | (EId:200000000000010, 3, 1) | 0100000000000003
+ | 0200000000000011 | TupleTest | (EId:200000000000011, 3, 2) | 0100000000000003
+ | 0200000000000012 | TupleTest | (EId:200000000000012, 3, 3) | 0100000000000003
+ | 0200000000000013 | TupleTest | (EId:200000000000013, 1, 1) | 0100000000000003
+ | 0200000000000014 | TupleTest | (EId:200000000000014, 1, 2) | 0100000000000003
+ | 0200000000000015 | TupleTest | (EId:200000000000015, 1, 3) | 0100000000000003
+ | 0200000000000016 | TupleTest | (EId:200000000000016, 2, 1) | 0100000000000003
+ | 0200000000000017 | TupleTest | (EId:200000000000017, 2, 2) | 0100000000000003
+ | 0200000000000018 | TupleTest | (EId:200000000000018, 2, 3) | 0100000000000003
+ | 0200000000000019 | TupleTest | (EId:200000000000019, 3, 1) | 0100000000000003
+ | 020000000000001A | TupleTest | (EId:20000000000001A, 3, 2) | 0100000000000003
+ | 020000000000001B | TupleTest | (EId:20000000000001B, 3, 3) | 0100000000000003
10 changes: 8 additions & 2 deletions tests/NexusMods.MnemonicDB.Tests/DbTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
using NexusMods.MnemonicDB.Abstractions.TxFunctions;
using NexusMods.MnemonicDB.TestModel;
using NexusMods.MnemonicDB.TestModel.Analyzers;
using NexusMods.MnemonicDB.TestModel.Attributes;
using NexusMods.Paths;
using File = NexusMods.MnemonicDB.TestModel.File;

Expand Down Expand Up @@ -1002,12 +1003,17 @@ public async Task CanUseTuple3Attributes()
foreach (var c in new[]{1, 2, 3})
{
var tmpId = tx.TempId();
tx.Add(tmpId, File.TupleTest, (a, b, c.ToString()));
tx.Add(tmpId, File.TupleTest, (tmpId, b, c.ToString()));
}

var results = await tx.Commit();

var resolved = results.Db.Datoms(File.TupleTest).Resolved(Connection).ToArray();

resolved.Select(v => ((Tuple3TestAttribute.ReadDatom)v).V.Item1)
.Should().AllSatisfy(id => id.Partition.Should().NotBe(PartitionId.Temp));

await VerifyTable(results.Db.Datoms(File.TupleTest).Resolved(Connection));
await VerifyTable(resolved);
}

[Fact]
Expand Down
Loading