Implement support for KV1 binary serialization with string tables (#100)

* fix typo in existing test case * Bump C# lang version to latest (12 with SDK8) * Implement binary string table support * add constructors to apisurface test
ValveResourceFormat · Jul 3, 2024 · 36bd1d2 · 36bd1d2
1 parent f7d32f2
commit 36bd1d2
Show file tree

Hide file tree

Showing 11 changed files with 306 additions and 8 deletions.
diff --git a/Directory.Build.props b/Directory.Build.props
@@ -7,6 +7,7 @@
     <AssemblyVersion>$(ProjectBaseVersion)</AssemblyVersion>
     <FileVersion>$(ProjectVersion)</FileVersion>
     <Version>$(ProjectVersion)</Version>
+	<LangVersion>latest</LangVersion>
   </PropertyGroup>
 
   <PropertyGroup>

diff --git a/ValveKeyValue/ValveKeyValue.Test/ApiSurfaceTestCase.cs b/ValveKeyValue/ValveKeyValue.Test/ApiSurfaceTestCase.cs
@@ -83,6 +83,37 @@ static void GenerateTypeApiSurface(StringBuilder sb, Type type)
                 sb.Append('\n');
             }
 
+            var constructors = type
+                .GetConstructors(BindingFlags.Static | BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic)
+                .Where(t => !t.IsPrivate && !t.IsAssembly && !t.IsFamilyAndAssembly)
+                .OrderBy(t => t.Name, StringComparer.InvariantCulture)
+                .ThenBy(t => string.Join(", ", t.GetParameters().Select(GetParameterAsString)), StringComparer.InvariantCulture);
+
+            foreach (var constructor in constructors)
+            {
+                sb.Append("    ");
+
+                if (constructor.IsPublic)
+                {
+                    sb.Append("public");
+                }
+                else
+                {
+                    sb.Append("protected");
+                }
+
+                if (constructor.IsStatic)
+                {
+                    sb.Append(" static");
+                }
+
+                sb.Append(' ');
+                sb.Append(constructor.Name);
+                sb.Append('(');
+                sb.Append(string.Join(", ", constructor.GetParameters().Select(GetParameterAsString)));
+                sb.Append(");\n");
+            }
+
             var methods = type
                 .GetMethods(BindingFlags.Static | BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic)
                 .Where(t => !t.IsPrivate && !t.IsAssembly && !t.IsFamilyAndAssembly)

diff --git a/ValveKeyValue/ValveKeyValue.Test/Binary/SimpleBinaryTestCase.cs b/ValveKeyValue/ValveKeyValue.Test/Binary/SimpleBinaryTestCase.cs
@@ -56,7 +56,7 @@ public void SetUp()
                     0x06, // pointer: ptr = 0x11223344
                         0x70, 0x74, 0x72, 0x00,
                         0x44, 0x33, 0x22, 0x11,
-                    0x07, // uint64: long = 0x1122334455667788
+                    0x07, // uint64: lng = 0x1122334455667788
                         0x6C, 0x6E, 0x67, 0x00,
                         0x88, 0x77, 0x66, 0x55, 0x44, 0x33, 0x22, 0x11,
                     0x0A, // int64, i64 = 0x0102030405070809

diff --git a/ValveKeyValue/ValveKeyValue.Test/Binary/StringTableFromScratchTestCase.cs b/ValveKeyValue/ValveKeyValue.Test/Binary/StringTableFromScratchTestCase.cs
@@ -0,0 +1,29 @@
+using System.Linq;
+
+namespace ValveKeyValue.Test
+{
+    class StringTableFromScratchTestCase
+    {
+        [Test]
+        public void PopulatesStringTableDuringSerialization()
+        {
+            var kv = new KVObject("root",
+            [
+                new KVObject("key", "value"),
+                new KVObject("child", [
+                    new KVObject("key", 123),
+                ]),
+            ]);
+
+            var stringTable = new StringTable();
+
+            var serializer = KVSerializer.Create(KVSerializationFormat.KeyValues1Binary);
+
+            using var ms = new MemoryStream();
+            serializer.Serialize(ms, kv, new KVSerializerOptions { StringTable = stringTable });
+
+            var strings = stringTable.ToArray();
+            Assert.That(strings, Is.EqualTo(new[] { "root", "key", "child" }));
+        }
+    }
+}
diff --git a/ValveKeyValue/ValveKeyValue.Test/Binary/StringTableTestCase.cs b/ValveKeyValue/ValveKeyValue.Test/Binary/StringTableTestCase.cs
@@ -0,0 +1,87 @@
+using System.Linq;
+
+namespace ValveKeyValue.Test
+{
+    class StringTableTestCase
+    {
+        [Test]
+        public void IsNotNull()
+            => Assert.That(obj, Is.Not.Null);
+
+        [Test]
+        public void HasName()
+            => Assert.That(obj.Name, Is.EqualTo("TestObject"));
+
+        [Test]
+        public void IsObjectWithChildren()
+            => Assert.That(obj.Value.ValueType, Is.EqualTo(KVValueType.Collection));
+
+        [TestCase(ExpectedResult = 5)]
+        public int HasChildren()
+            => obj.Children.Count();
+
+        [TestCase("key", "value", typeof(string))]
+        [TestCase("int", 0x01020304, typeof(int))]
+        [TestCase("flt", 1234.5678f, typeof(float))]
+        [TestCase("lng", 0x1122334455667788, typeof(ulong))]
+        [TestCase("i64", 0x0102030405060708, typeof(long))]
+        public void HasNamedChildWithValue(string name, object value, Type valueType)
+        {
+            Assert.That(Convert.ChangeType(obj[name], valueType), Is.EqualTo(value));
+        }
+
+        [Test]
+        public void SymmetricStringTableSerialization()
+        {
+            var serializer = KVSerializer.Create(KVSerializationFormat.KeyValues1Binary);
+
+            using var ms = new MemoryStream();
+            serializer.Serialize(ms, obj, new KVSerializerOptions { StringTable = new(TestStringTable) });
+
+            Assert.That(ms.ToArray(), Is.EqualTo(TestData.ToArray()));
+        }
+
+        KVObject obj;
+
+        [OneTimeSetUp]
+        public void SetUp()
+        {
+            obj = KVSerializer.Create(KVSerializationFormat.KeyValues1Binary)
+                .Deserialize(
+                    TestData.ToArray(),
+                    new KVSerializerOptions { StringTable = new(TestStringTable) });
+        }
+
+        static string[] TestStringTable => [
+            "flt",
+            "i64",
+            "int",
+            "key",
+            "lng",
+            "TestObject"
+        ];
+
+        static ReadOnlySpan<byte> TestData =>
+        [
+            0x00, // object: TestObject
+                0x05, 0x00, 0x00, 0x00, // stringTable[5] = "TestObject",
+                0x01, // string: key = value
+                    0x03, 0x00, 0x00, 0x00, // stringTable[3] = "key",
+                    0x76, 0x61, 0x6C, 0x75, 0x65, 0x00,
+                0x02, // int32: int = 0x01020304
+                    0x02, 0x00, 0x00, 0x00, // stringTable[2] = "int"
+                    0x04, 0x03, 0x02, 0x01,
+                0x03, // float32: flt = 1234.5678f
+                    0x00, 0x00, 0x00, 0x00, // stringTable[0] = "flt"
+                    0x2B, 0x52, 0x9A, 0x44,
+                0x07, // uint64: lng = 0x1122334455667788
+                    0x04, 0x00, 0x00, 0x00, // stringTable[4] = "lng"
+                    0x88, 0x77, 0x66, 0x55, 0x44, 0x33, 0x22, 0x11,
+                0x0A, // int64, i64 = 0x0102030405070809
+                    0x01, 0x00, 0x00, 0x00, // stringTable[1] = "i64"
+                    0x08, 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01,
+                0x08, // end object
+            0x08, // end document
+        ];
+    }
+}
diff --git a/ValveKeyValue/ValveKeyValue.Test/Test Data/apisurface.txt b/ValveKeyValue/ValveKeyValue.Test/Test Data/apisurface.txt
@@ -5,6 +5,9 @@ public interface ValveKeyValue.IIncludedFileLoader
 
 public class ValveKeyValue.KeyValueException
 {
+    public .ctor();
+    public .ctor(string message);
+    public .ctor(string message, Exception inner);
     protected void add_SerializeObjectState(EventHandler`1[[System.Runtime.Serialization.SafeSerializationEventArgs]] value);
     public bool Equals(object obj);
     protected void Finalize();
@@ -31,6 +34,7 @@ public class ValveKeyValue.KeyValueException
 
 public class ValveKeyValue.KVArrayValue
 {
+    public .ctor();
     public void Add(ValveKeyValue.KVValue value);
     public void AddRange(System.Collections.Generic.IEnumerable`1[[ValveKeyValue.KVValue]] values);
     public void Clear();
@@ -74,6 +78,8 @@ public class ValveKeyValue.KVArrayValue
 
 public class ValveKeyValue.KVBinaryBlob
 {
+    public .ctor(byte[] value);
+    public .ctor(Memory`1[[byte]] value);
     public bool Equals(object obj);
     protected void Finalize();
     public Memory`1[[byte]] get_Bytes();
@@ -104,6 +110,7 @@ public class ValveKeyValue.KVBinaryBlob
 
 public class ValveKeyValue.KVDocument
 {
+    public .ctor(string name, ValveKeyValue.KVValue value);
     public void Add(ValveKeyValue.KVObject value);
     public bool Equals(object obj);
     protected void Finalize();
@@ -121,6 +128,7 @@ public class ValveKeyValue.KVDocument
 
 public sealed class ValveKeyValue.KVIgnoreAttribute
 {
+    public .ctor();
     public bool Equals(object obj);
     protected void Finalize();
     public object get_TypeId();
@@ -134,6 +142,8 @@ public sealed class ValveKeyValue.KVIgnoreAttribute
 
 public class ValveKeyValue.KVObject
 {
+    public .ctor(string name, System.Collections.Generic.IEnumerable`1[[ValveKeyValue.KVObject]] items);
+    public .ctor(string name, ValveKeyValue.KVValue value);
     public void Add(ValveKeyValue.KVObject value);
     public bool Equals(object obj);
     protected void Finalize();
@@ -151,6 +161,7 @@ public class ValveKeyValue.KVObject
 
 public sealed class ValveKeyValue.KVPropertyAttribute
 {
+    public .ctor(string propertyName);
     public bool Equals(object obj);
     protected void Finalize();
     public string get_PropertyName();
@@ -200,24 +211,28 @@ public class ValveKeyValue.KVSerializer
 
 public sealed class ValveKeyValue.KVSerializerOptions
 {
+    public .ctor();
     public bool Equals(object obj);
     protected void Finalize();
     public System.Collections.Generic.IList`1[[string]] get_Conditions();
     public static ValveKeyValue.KVSerializerOptions get_DefaultOptions();
     public bool get_EnableValveNullByteBugBehavior();
     public ValveKeyValue.IIncludedFileLoader get_FileLoader();
     public bool get_HasEscapeSequences();
+    public ValveKeyValue.StringTable get_StringTable();
     public int GetHashCode();
     public Type GetType();
     protected object MemberwiseClone();
     public void set_EnableValveNullByteBugBehavior(bool value);
     public void set_FileLoader(ValveKeyValue.IIncludedFileLoader value);
     public void set_HasEscapeSequences(bool value);
+    public void set_StringTable(ValveKeyValue.StringTable value);
     public string ToString();
 }
 
 public class ValveKeyValue.KVValue
 {
+    protected .ctor();
     public bool Equals(object obj);
     protected void Finalize();
     public ValveKeyValue.KVValue get_Item(string key);
@@ -295,3 +310,20 @@ public sealed enum ValveKeyValue.KVValueType
     public string ToString(string format, IFormatProvider provider);
 }
 
+public sealed class ValveKeyValue.StringTable
+{
+    public .ctor();
+    public .ctor(int capacity);
+    public .ctor(System.Collections.Generic.IList`1[[string]] values);
+    public void Add(string value);
+    public bool Equals(object obj);
+    protected void Finalize();
+    public string get_Item(int index);
+    public int GetHashCode();
+    public int GetOrAdd(string value);
+    public Type GetType();
+    protected object MemberwiseClone();
+    public string[] ToArray();
+    public string ToString();
+}
+
diff --git a/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues1/KV1BinaryReader.cs b/ValveKeyValue/ValveKeyValue/Deserialization/KeyValues1/KV1BinaryReader.cs
@@ -8,7 +8,7 @@ class KV1BinaryReader : IVisitingReader
     {
         public const int BinaryMagicHeader = 0x564B4256; // VBKV
 
-        public KV1BinaryReader(Stream stream, IVisitationListener listener)
+        public KV1BinaryReader(Stream stream, IVisitationListener listener, StringTable stringTable)
         {
             Require.NotNull(stream, nameof(stream));
             Require.NotNull(listener, nameof(listener));
@@ -20,12 +20,14 @@ public KV1BinaryReader(Stream stream, IVisitationListener listener)
 
             this.stream = stream;
             this.listener = listener;
+            this.stringTable = stringTable;
             reader = new BinaryReader(stream, Encoding.UTF8, leaveOpen: true);
         }
 
         readonly Stream stream;
         readonly BinaryReader reader;
         readonly IVisitationListener listener;
+        readonly StringTable stringTable;
         bool disposed;
         KV1BinaryNodeType endMarker = KV1BinaryNodeType.End;
 
@@ -74,9 +76,20 @@ void ReadObjectCore()
             }
         }
 
+        string ReadKeyForNextValue()
+        {
+            if (stringTable is not null)
+            {
+                var index = reader.ReadInt32();
+                return stringTable[index];
+            }
+
+            return Encoding.UTF8.GetString(ReadNullTerminatedBytes());
+        }
+
         void ReadValue(KV1BinaryNodeType type)
         {
-            var name = Encoding.UTF8.GetString(ReadNullTerminatedBytes());
+            var name = ReadKeyForNextValue();
             KVValue value;
 
             switch (type)

diff --git a/ValveKeyValue/ValveKeyValue/KVSerializer.cs b/ValveKeyValue/ValveKeyValue/KVSerializer.cs
@@ -115,7 +115,7 @@ IVisitingReader MakeReader(Stream stream, IParsingVisitationListener listener, K
             return format switch
             {
                 KVSerializationFormat.KeyValues1Text => new KV1TextReader(new StreamReader(stream), listener, options),
-                KVSerializationFormat.KeyValues1Binary => new KV1BinaryReader(stream, listener),
+                KVSerializationFormat.KeyValues1Binary => new KV1BinaryReader(stream, listener, options.StringTable),
                 _ => throw new ArgumentOutOfRangeException(nameof(format), format, "Invalid serialization format."),
             };
         }
@@ -128,9 +128,10 @@ IVisitationListener MakeSerializer(Stream stream, KVSerializerOptions options)
             return format switch
             {
                 KVSerializationFormat.KeyValues1Text => new KV1TextSerializer(stream, options),
-                KVSerializationFormat.KeyValues1Binary => new KV1BinarySerializer(stream),
+                KVSerializationFormat.KeyValues1Binary => new KV1BinarySerializer(stream, options.StringTable),
                 _ => throw new ArgumentOutOfRangeException(nameof(format), format, "Invalid serialization format."),
             };
+            ;
         }
     }
 }
diff --git a/ValveKeyValue/ValveKeyValue/KVSerializerOptions.cs b/ValveKeyValue/ValveKeyValue/KVSerializerOptions.cs
@@ -27,6 +27,12 @@ public sealed class KVSerializerOptions
         /// </summary>
         public IIncludedFileLoader FileLoader { get; set; }
 
+
+        /// <summary>
+        /// Gets or sets the string table used for smaller binary serialization.
+        /// </summary>
+        public StringTable StringTable { get; set; }
+
         /// <summary>
         /// Gets the default options (used when none are specified).
         /// </summary>