Various improvements (#331)

* Support equality semantics for concrete grammar objects.

The original plan was to do it only for handles, but since we almost everywhere use concrete objects, they should support equality as well.

* Improve documentation.

* Remove `GrammarExtensions.WriteGrammarToFile`.

It can be done with one line since .NET 9 and with a few more lines in earlier frameworks.

* Make `IProduction` public and have `IProduction<T>` inherit it.

This allows us to accept params immutable array when setting typed productions.

* Fail when setting an empty array of typed productions.

* Refactor `DefaultParserImplementation` to reduce generic instantiations.

It no longer depends on the result type at all, and the APIs that don't even depend on the character type are moved to a non-generic class.

* Fix some Sonar warnings.

* Exclude several diagnostics-only APIs from code coverage.

* Fix F# API.

* Update grammar creation APIs.

We use `Load` instead of `Create` and `Convert` for GOLD Parser grammar conversions.
The read-only span overload was removed because it always had to copy the span if the grammar is valid. Better make memory management explicit to the user.
Also added an API to convert a GOLD Parser grammar from a file.

* Add grammar object equality test.

* Add documentation.

performance/Farkle.Benchmarks.CSharp/GrammarReaderBenchmark.cs

@@ -36,17 +36,17 @@ public object ReadFarkle6() =>
 
     [BenchmarkCategory("Read"), Benchmark]
     public object ReadFarkle7() =>
-        Grammar.Create(Farkle7Grammar);
+        Grammar.Load(Farkle7Grammar);
 
     [BenchmarkCategory("Read"), Benchmark]
     public object ReadFarkle7NoValidation() =>
-        Grammar.CreateUnsafe(Farkle7Grammar);
+        Grammar.LoadUnsafe(Farkle7Grammar);
 
     [BenchmarkCategory("Convert"), Benchmark(Baseline = true)]
     public object ConvertFarkle6() =>
         Farkle6.Grammar.EGT.ReadFromStream(new MemoryStream(Egt, false));
 
     [BenchmarkCategory("Convert"), Benchmark]
     public object ConvertFarkle7() =>
-        Grammar.CreateFromGoldParserGrammar(new MemoryStream(Egt, false));
+        Grammar.ConvertFromGoldParser(new MemoryStream(Egt, false));
 }

performance/Farkle.Benchmarks.CSharp/JsonBenchmark.cs

@@ -37,7 +37,7 @@ public void GlobalSetup()
         _jsonText = File.ReadAllText($"resources/{FileName}");
         _farkle6Runtime = Farkle6.RuntimeFarkle<object>.Create(Farkle6.Grammar.EGT.ReadFromFile("resources/JSON.egt"), Farkle6.PostProcessors.SyntaxChecker);
         _farkle6Tokenizer = new Farkle6.Parser.DefaultTokenizer(_farkle6Runtime.GetGrammar());
-        _farkle7Parser = CharParser.CreateSyntaxChecker(Grammar.Create(File.ReadAllBytes("resources/JSON.grammar.dat")));
+        _farkle7Parser = CharParser.CreateSyntaxChecker(Grammar.Load("resources/JSON.grammar.dat"));
         _farkle7Tokenizer = Parser.Tokenizers.Tokenizer.Create<char>(_farkle7Parser.GetGrammar());
     }
 

performance/Farkle.Performance.Profiling/Program.cs

@@ -23,7 +23,7 @@ internal static class Program
     private static readonly Farkle6.RuntimeFarkle<object> _syntaxCheck =
         Farkle6.RuntimeFarkle<object>.Create(Farkle6.Grammar.EGT.ReadFromFile(Farkle6GrammarPath), Farkle6.PostProcessors.SyntaxChecker);
     private static readonly CharParser<object> _syntaxCheck7 =
-        CharParser.CreateSyntaxChecker(Grammar.CreateFromFile(Farkle7GrammarPath));
+        CharParser.CreateSyntaxChecker(Grammar.Load(Farkle7GrammarPath));
     private static readonly Farkle6.Parser.Tokenizer _tokenizer =
         new Farkle6.Parser.DefaultTokenizer(_syntaxCheck.GetGrammar());
     private static readonly Farkle.Parser.Tokenizers.Tokenizer<char> _tokenizer7 =

sample/Farkle.Samples.CSharp/JSON.cs

@@ -36,14 +36,14 @@ private static JsonValue ToDecimal(ReadOnlySpan<char> data)
         static JSON()
         {
             var number = Terminal.Create("Number",
-                Join([
+                Join(
                     Literal('-').Optional(),
                     Literal('0') | OneOf("123456789") + OneOf("0123456789").ZeroOrMore(),
                     (Literal('.') + OneOf("0123456789").AtLeast(1)).Optional(),
-                    Join([
+                    Join(
                         OneOf("eE"),
                         OneOf("+-").Optional(),
-                        OneOf("0123456789").AtLeast(1)]).Optional()]),
+                        OneOf("0123456789").AtLeast(1)).Optional()),
                  (ref ParserState _, ReadOnlySpan<char> data) => ToDecimal(data));
             var jsonString = Terminals.String("String", '"', "/bfnrtu", false);
             var jsonObject = Nonterminal.Create<JsonObject>("Object");

src/FarkleNeo/Builder/Dfa/DfaBuild.cs

@@ -720,7 +720,7 @@ public bool IsTransitionSpaceFull()
 
     private abstract class RegexLeaf
     {
-        public static RegexLeaf Any { get; } = new Chars([], true);
+        public static Chars Any { get; } = new Chars([], true);
 
         public sealed class Chars(ImmutableArray<(TChar Start, TChar End)> ranges, bool isInverted) : RegexLeaf
         {

src/FarkleNeo/Builder/GrammarBuild.cs

@@ -290,7 +290,7 @@ private static Grammar Build(GrammarDefinition grammarDefinition, BuilderArtifac
         NonterminalHandle startSymbol = (NonterminalHandle)symbolMap[grammarDefinition.StartSymbol];
         writer.SetGrammarInfo(writer.GetOrAddString(grammarDefinition.GrammarName), startSymbol, attributes);
 
-        return Grammar.Create(writer.ToImmutableArray());
+        return Grammar.Load(writer.ToImmutableArray());
 
         // Gets the handle to a group end literal symbol, creating it if it does not exist.
         // Multiple groups can end with the same symbol without causing a conflict, because

src/FarkleNeo/Builder/Nonterminal.cs

@@ -18,7 +18,7 @@ public static class Nonterminal
     /// <typeparam name="T">The type of values the nonterminal will produce.</typeparam>
     /// <param name="name">The nonterminal's name.</param>
     /// <seealso cref="Nonterminal{T}.SetProductions(IProduction{T}[])"/>
-    /// <seealso cref="Nonterminal{T}.SetProductions(ReadOnlySpan{IProduction{T}})"/>
+    /// <seealso cref="Nonterminal{T}.SetProductions(ImmutableArray{IProduction{T}})"/>
     public static Nonterminal<T> Create<T>(string name)
     {
         ArgumentNullExceptionCompat.ThrowIfNull(name);
@@ -32,25 +32,24 @@ public static Nonterminal<T> Create<T>(string name)
     /// <param name="name">The nonterminal's name.</param>
     /// <param name="productions">The nonterminal's productions.</param>
     /// <exception cref="ArgumentException"><paramref name="productions"/> is empty.</exception>
-    public static IGrammarSymbol<T> Create<T>(string name, params ReadOnlySpan<IProduction<T>> productions)
+    public static IGrammarSymbol<T> Create<T>(string name, params ImmutableArray<IProduction<T>> productions)
     {
         ArgumentNullExceptionCompat.ThrowIfNull(name);
-        if (productions.IsEmpty)
+        if (productions.IsDefault)
         {
-            ThrowHelpers.ThrowArgumentExceptionLocalized(nameof(Resources.Builder_Nonterminal_EmptyProductions), nameof(productions));
+            ThrowHelpers.ThrowArgumentNullException(nameof(productions));
         }
-        var builder = ImmutableArray.CreateBuilder<IProduction>(productions.Length);
-        foreach (var production in productions)
+        if (productions.IsEmpty)
         {
-            builder.Add(production.Production);
+            ThrowHelpers.ThrowArgumentExceptionLocalized(nameof(Resources.Builder_Nonterminal_EmptyProductions), nameof(productions));
         }
-        return new Nonterminal<T>(name, builder.MoveToImmutable());
+        return new Nonterminal<T>(name, ImmutableArray<IProduction>.CastUp(productions));
     }
 
-    /// <inheritdoc cref="Create{T}(string, ReadOnlySpan{IProduction{T}})"/>
-    [ExcludeFromCodeCoverage]
+    /// <inheritdoc cref="Create{T}(string, ImmutableArray{IProduction{T}})"/>
+    [ExcludeFromCodeCoverage, OverloadResolutionPriority(-1)]
     public static IGrammarSymbol<T> Create<T>(string name, params IProduction<T>[] productions) =>
-        Create<T>(name, productions.AsSpanChecked());
+        Create(name, productions.ToImmutableArrayChecked());
 
     /// <summary>
     /// Creates a nonterminal that does not produce a value and whose productions
@@ -87,8 +86,7 @@ public static IGrammarSymbol CreateUntyped(string name, params ImmutableArray<Pr
     }
 
     /// <inheritdoc cref="CreateUntyped(string, ImmutableArray{ProductionBuilder})"/>
-    [ExcludeFromCodeCoverage]
-    [OverloadResolutionPriority(-1)]
+    [ExcludeFromCodeCoverage, OverloadResolutionPriority(-1)]
     public static IGrammarSymbol CreateUntyped(string name, params ProductionBuilder[] productions) =>
         CreateUntyped(name, productions.ToImmutableArrayChecked());
 }
@@ -128,11 +126,11 @@ internal Nonterminal(string name, ImmutableArray<IProduction> productions = defa
     /// <exception cref="InvalidOperationException">The productions have already been successfully set.</exception>
     /// <remarks>This function and its overloads must be called exactly once, and before the
     /// nonterminal is used in building a grammar.</remarks>
-    [ExcludeFromCodeCoverage]
+    [ExcludeFromCodeCoverage, OverloadResolutionPriority(-1)]
     public void SetProductions(params IProduction<T>[] productions)
     {
         ArgumentNullExceptionCompat.ThrowIfNull(productions);
-        SetProductions(productions.AsSpan());
+        SetProductions(productions.ToImmutableArrayChecked());
     }
 
     /// <summary>
@@ -143,14 +141,17 @@ public void SetProductions(params IProduction<T>[] productions)
     /// <exception cref="InvalidOperationException">The productions have already been successfully set.</exception>
     /// <remarks>This function and its overloads must be called exactly once, and before the
     /// nonterminal is used in building a grammar.</remarks>
-    public void SetProductions(params ReadOnlySpan<IProduction<T>> productions)
+    public void SetProductions(params ImmutableArray<IProduction<T>> productions)
     {
-        var builder = ImmutableArray.CreateBuilder<IProduction>(productions.Length);
-        foreach (var production in productions)
+        if (productions.IsDefault)
+        {
+            ThrowHelpers.ThrowArgumentNullException(nameof(productions));
+        }
+        if (productions.IsEmpty)
         {
-            builder.Add(production.Production);
+            ThrowHelpers.ThrowArgumentExceptionLocalized(nameof(Resources.Builder_Nonterminal_EmptyProductions), nameof(productions));
         }
-        _innerNonterminal.SetProductions(builder.MoveToImmutable());
+        _innerNonterminal.SetProductions(ImmutableArray<IProduction>.CastUp(productions));
     }
 
     ImmutableArray<IProduction> INonterminal.FreezeAndGetProductions() =>

src/FarkleNeo/Builder/ObjectModel.cs

@@ -184,13 +184,21 @@ internal class BlockGroup(string name, string groupStart, string groupEnd, Trans
     public string GroupEnd { get; } = groupEnd;
 }
 
-internal interface IProduction
+/// <summary>
+/// Represents a production in a grammar to be built that produces a value.
+/// </summary>
+/// <remarks>
+/// This interface cannot be implemented by user code and is not directly accepted by any API.
+/// </remarks>
+/// <seealso cref="IProduction{T}"/>
+/// <seealso cref="ProductionBuilder"/>
+public interface IProduction
 {
-    ImmutableArray<IGrammarSymbol> Members { get; }
+    internal ImmutableArray<IGrammarSymbol> Members { get; }
 
-    Fuser<object?> Fuser { get; }
+    internal Fuser<object?> Fuser { get; }
 
-    object? PrecedenceToken { get; }
+    internal object? PrecedenceToken { get; }
 }
 
 // This is an interface because both Nonterminal and Nonterminal<T>
@@ -214,23 +222,16 @@ internal sealed class BlockGroup<T>(string name, string groupStart, string group
 /// This interface cannot be implemented by user code.
 /// </remarks>
 /// <seealso cref="Nonterminal.Create{T}(string, IProduction{T}[])"/>
-/// <seealso cref="Nonterminal.Create{T}(string, ReadOnlySpan{IProduction{T}})"/>
+/// <seealso cref="Nonterminal.Create{T}(string, ImmutableArray{IProduction{T}})"/>
 /// <seealso cref="Nonterminal{T}.SetProductions(IProduction{T}[])"/>
-/// <seealso cref="Nonterminal{T}.SetProductions(ReadOnlySpan{IProduction{T}})"/>
-public interface IProduction<out T>
-{
-    // We cannot inherit IProduction because we want the generic interface to be public.
-    // Instead, we expose the IProduction through this property.
-    internal IProduction Production { get; }
-}
+/// <seealso cref="Nonterminal{T}.SetProductions(ImmutableArray{IProduction{T}})"/>
+public interface IProduction<out T> : IProduction;
 
-internal class Production<T>(ImmutableArray<IGrammarSymbol> symbols, Fuser<object?> fuser, object? precedenceToken) : IProduction, IProduction<T>
+internal class Production<T>(ImmutableArray<IGrammarSymbol> symbols, Fuser<object?> fuser, object? precedenceToken) : IProduction<T>
 {
     public ImmutableArray<IGrammarSymbol> Members { get; } = symbols;
 
     public Fuser<object?> Fuser { get; } = fuser;
 
     public object? PrecedenceToken { get; } = precedenceToken;
-
-    IProduction IProduction<T>.Production => this;
 }

src/FarkleNeo/Builder/OperatorPrecedence/OperatorScope.cs

@@ -44,7 +44,7 @@ public OperatorScope(bool canResolveReduceReduceConflicts, params ImmutableArray
             ArgumentNullExceptionCompat.ThrowIfNull(associativityGroups[i]);
         }
         CanResolveReduceReduceConflicts = canResolveReduceReduceConflicts;
-        AssociativityGroups = associativityGroups.ToImmutableArray();
+        AssociativityGroups = associativityGroups;
     }
 
     /// <inheritdoc cref="OperatorScope(bool, ImmutableArray{AssociativityGroup})"/>

src/FarkleNeo/Builder/Untyped.cs

@@ -51,8 +51,7 @@ internal Nonterminal(string name, ImmutableArray<IProduction> productions = defa
     /// <exception cref="InvalidOperationException">The productions have already been successfully set.</exception>
     /// <remarks>This function and its overloads must be called exactly once, and before the
     /// nonterminal is used in building a grammar.</remarks>
-    [ExcludeFromCodeCoverage]
-    [OverloadResolutionPriority(-1)]
+    [ExcludeFromCodeCoverage, OverloadResolutionPriority(-1)]
     public void SetProductions(params ProductionBuilder[] productions)
     {
         ArgumentNullExceptionCompat.ThrowIfNull(productions);

src/FarkleNeo/DebuggerTypeProxies.cs

@@ -5,28 +5,35 @@
 using Farkle.Grammars;
 using Farkle.Grammars.StateMachines;
 using System.Diagnostics;
+using System.Diagnostics.CodeAnalysis;
 
 namespace Farkle.DebuggerTypeProxies;
 
+[ExcludeFromCodeCoverage]
 internal class FlatCollectionProxy<T, TCollection>(TCollection list) where TCollection : IEnumerable<T>
 {
     [DebuggerBrowsable(DebuggerBrowsableState.RootHidden)]
     public readonly T[] _items = list.ToArray();
 }
 
+[ExcludeFromCodeCoverage]
 internal class DfaProxy<TChar>(Dfa<TChar> dfa) : FlatCollectionProxy<DfaState<TChar>, Dfa<TChar>>(dfa);
 
+[ExcludeFromCodeCoverage]
 internal class DfaAcceptSymbolsProxy<TChar>(DfaState<TChar>.AcceptSymbolCollection collection) : FlatCollectionProxy<TokenSymbol, DfaState<TChar>.AcceptSymbolCollection>(collection);
 
+[ExcludeFromCodeCoverage]
 internal class DfaEdgesProxy<TChar>(DfaState<TChar>.EdgeCollection collection) : FlatCollectionProxy<DfaEdge<TChar>, DfaState<TChar>.EdgeCollection>(collection);
 
 [DebuggerDisplay("{Value,nq}", Name = "{Name,nq}")]
+[ExcludeFromCodeCoverage]
 internal readonly struct NameValuePair(string name, string value)
 {
     [DebuggerBrowsable(DebuggerBrowsableState.Never)]
     public readonly string Name = name, Value = value;
 }
 
+[ExcludeFromCodeCoverage]
 internal sealed class LrStateProxy
 {
     [DebuggerBrowsable(DebuggerBrowsableState.RootHidden)]
@@ -54,6 +61,7 @@ public LrStateProxy(LrState state)
     }
 }
 
+[ExcludeFromCodeCoverage]
 internal sealed class DfaStateProxy<TChar>
 {
     [DebuggerBrowsable(DebuggerBrowsableState.RootHidden)]

src/FarkleNeo/Farkle.fs

@@ -17,17 +17,17 @@ module internal Grammar =
     open System.Collections.Immutable
     open System.IO
 
-    /// Creates a grammar from a read-only span of bytes.
-    let inline ofSpan (x: ReadOnlySpan<byte>) = Grammar.Create x
-
     /// Creates a grammar from an immutable array of bytes.
-    /// Should be preferred over ofSpan when an immutable array is available.
-    let inline ofBytes (x: ImmutableArray<byte>) = Grammar.Create x
+    let inline ofBytes (x: ImmutableArray<byte>) = Grammar.Load x
+
+    /// Loads a grammar from a file.
+    let inline ofFile (path: string) = Grammar.Load path
 
-    let inline ofFile path = Grammar.CreateFromFile path
+    /// Converts a GOLD Parser grammar to a Farkle grammar.
+    let inline ofGoldParserStream (x: Stream) = Grammar.ConvertFromGoldParser x
 
     /// Converts a GOLD Parser grammar to a Farkle grammar.
-    let inline ofGoldParserStream (x: Stream) = Grammar.CreateFromGoldParserGrammar x
+    let inline ofGoldParserFile (path: string) = Grammar.ConvertFromGoldParser path
 
 namespace Farkle.Grammars.StateMachines
 
@@ -527,7 +527,7 @@ module internal GrammarBuilderOperators =
     /// Creates a `IGrammarSymbol&lt;'T&gt;` that represents
     /// a nonterminal with the given name and productions.
     let inline (||=) name (productions: #seq<_>) =
-        Nonterminal.Create(name, (Internal.makeImmutableArray productions).AsSpan())
+        Nonterminal.Create(name, Internal.makeImmutableArray productions)
 
     /// Creates an `IGrammarSymbol&lt;'T&gt;` that represents
     /// a nonterminal with the given name and productions.

src/FarkleNeo/Grammars/EntityHandle.cs

@@ -83,7 +83,7 @@ internal void TypeCheck(TableKind expectedKind)
     public static bool operator ==(EntityHandle left, EntityHandle right) => left.Equals(right);
 
     /// <summary>
-    /// Checks if two <see cref="EntityHandle"/>s are pointing to the same table row.
+    /// Checks if two <see cref="EntityHandle"/>s are not pointing to the same table row.
     /// </summary>
     /// <param name="left">The first handle.</param>
     /// <param name="right">The second handle.</param>

src/FarkleNeo/Grammars/GoldParser/GoldGrammarConverter.cs

@@ -17,7 +17,6 @@ public static ImmutableArray<byte> Convert(GoldGrammar grammar)
         Symbol[] symbols = grammar.Symbols;
         GoldGrammar.Production[] productions = grammar.Productions;
         GoldGrammar.Group[] groups = grammar.Groups;
-        ImmutableArray<LalrAction>[] lalrStates = grammar.LalrStates;
 
         EntityHandle[] symbolMapping = new EntityHandle[symbols.Length];
         ProductionHandle[] productionMapping = new ProductionHandle[productions.Length];

src/FarkleNeo/Grammars/GoldParser/GrammarBinaryReader.cs

@@ -87,7 +87,10 @@ private string ReadNullTerminatedString(bool isHeader = false)
 
     private void SkipNullTerminatedString()
     {
-        while (_reader.ReadUInt16() != 0) { }
+        while (_reader.ReadUInt16() != 0)
+        {
+            // Ignore the characters.
+        }
     }
 
     public byte ReadByte()