Fixes 1.3.2025 (#37)

* Fix test

* Move regex logic up to Clang AST travsesal

* Fix macro object explorer possibly returning invalid Clang type

* Fix Windows default system include directories

* Make types visible but not function externs

* Improve logging

* Re-order modifiers

* Update README.md

* Fix re-order modifiers

README.md

@@ -198,7 +198,7 @@ Note that the internals of the C library is irrelevant and to which this list do
 |❌|Function-like macros <sup>5</sup>|
 |✅|Object-like macros <sup>2, 6, 7</sup>|
 
-<sup>1</sup>: When declaring your external functions or variables, do set the default visibility explictly. This is necessary because `c2ffi` is configured to have the visbiity set to hidden so that only the strict subset of functions and variables intended for FFI are extracted. Most C libraries will have an `API_DECL` macro object defined which can be redefined to also set the visibility. See [ffi_helper.h](src/c/production/ffi_helper/include/ffi_helper.h) for an example.
+<sup>1</sup>: When declaring your external functions or variables, do set the default visibility explictly. This is necessary because `c2ffi` is configured to have the visibiity set to hidden when using `libclang` via the flag `-fvisibility=hidden` so that only the strict subset of functions and variables intended for FFI are extracted. Most C libraries will have an `API_DECL` macro object defined which can be redefined to also set the visibility. See [ffi_helper.h](src/c/production/ffi_helper/include/ffi_helper.h) for an example in C. You can also use the config `.json` file to define tyour `API_DECL` macro object.
 
 Bad
 ```c

src/c/tests/functions/function_internal/config.json

@@ -4,11 +4,11 @@
     "../../../production/ffi_helper/include"
   ],
   "defines": {
-    "BAD_CUSTOM_API_DECL": "extern",
+    "BAD_CUSTOM_API_DECL": "",
     "GOOD_CUSTOM_API_DECL": "extern __attribute__ ((visibility(\"default\")))"
   },
   "ignoredIncludeFiles": [
-    "../../../production/ffi_helper/include/ffi_helper.h"
+    "./include.h"
   ],
     "targetPlatforms": {
     "windows": {

src/c/tests/functions/function_internal/include.h

@@ -0,0 +1,6 @@
+#include <stdio.h>
+#include "ffi_helper.h"
+
+void function_internal_1()
+{
+}

src/c/tests/functions/function_internal/main.c

@@ -1,9 +1,6 @@
 #include <stdio.h>
 #include "ffi_helper.h"
-
-void function_internal_1()
-{
-}
+#include "include.h"
 
 FFI_API_DECL void function_internal_2()
 {

src/cs/production/c2ffi.Data/CLocation.cs

@@ -75,6 +75,7 @@ public readonly int CompareTo(CLocation other)
     }
 
     /// <inheritdoc />
+    // ReSharper disable once ArrangeModifiersOrder
     public override readonly string ToString()
     {
         if (LineNumber == 0 && LineColumn == 0)

src/cs/production/c2ffi.Tool/Clang/ClangExtensions.cs

@@ -3,6 +3,7 @@
 
 using System.Collections.Immutable;
 using System.Runtime.InteropServices;
+using c2ffi.Extract.Parse;
 using static bottlenoselabs.clang;
 
 #pragma warning disable CA1806
@@ -11,7 +12,8 @@ namespace c2ffi.Clang
 {
     internal static class ClangExtensions
     {
-        internal delegate bool VisitChildPredicate(CXCursor child, CXCursor parent);
+        internal delegate bool VisitChildPredicate(
+            ParseContext parseContext, CXCursor child, CXCursor parent);
 
         private static VisitChildInstance[] _visitChildInstances = new VisitChildInstance[512];
         private static int _visitChildCount;
@@ -20,7 +22,7 @@ internal static class ClangExtensions
 
         private static readonly CXCursorVisitor VisitorChild;
         private static readonly CXFieldVisitor VisitorField;
-        private static readonly VisitChildPredicate EmptyVisitChildPredicate = static (_, _) => true;
+        private static readonly VisitChildPredicate EmptyVisitChildPredicate = static (_, _, _) => true;
 
         static ClangExtensions()
         {
@@ -93,18 +95,14 @@ public static bool IsSignedPrimitive(this CXType clangType)
             };
         }
 
-        public static bool IsFromMainFile(this CXCursor clangCursor)
-        {
-            var location = clang_getCursorLocation(clangCursor);
-            var isFromMainFile = clang_Location_isFromMainFile(location) > 0;
-            return isFromMainFile;
-        }
-
         public static ImmutableArray<CXCursor> GetDescendents(
-            this CXCursor cursor, VisitChildPredicate? predicate = null)
+            this CXCursor cursor,
+            ParseContext parseContext,
+            VisitChildPredicate? predicate = null,
+            bool isRecurse = false)
         {
             var predicate2 = predicate ?? EmptyVisitChildPredicate;
-            var visitData = new VisitChildInstance(predicate2);
+            var visitData = new VisitChildInstance(parseContext, predicate2, isRecurse);
             var visitsCount = Interlocked.Increment(ref _visitChildCount);
             if (visitsCount > _visitChildInstances.Length)
             {
@@ -164,15 +162,20 @@ private static CXChildVisitResult VisitChild(CXCursor child, CXCursor parent, CX
             }
 
             var data = _visitChildInstances[index - 1];
-            var result = data.Predicate(child, parent);
+            var result = data.Predicate(data.ParseContext, child, parent);
+
             if (!result)
             {
-                return CXChildVisitResult.CXChildVisit_Continue;
+                return data.IsRecurse
+                    ? CXChildVisitResult.CXChildVisit_Recurse
+                    : CXChildVisitResult.CXChildVisit_Continue;
             }
 
             data.CursorBuilder.Add(child);
 
-            return CXChildVisitResult.CXChildVisit_Continue;
+            return data.IsRecurse
+                ? CXChildVisitResult.CXChildVisit_Break
+                : CXChildVisitResult.CXChildVisit_Continue;
         }
 
         [UnmanagedCallersOnly]
@@ -189,8 +192,11 @@ private static CXVisitorResult VisitField(CXCursor cursor, CXClientData clientDa
             return CXVisitorResult.CXVisit_Continue;
         }
 
-        private readonly struct VisitChildInstance(VisitChildPredicate predicate)
+        private readonly struct VisitChildInstance(
+            ParseContext parseContext, VisitChildPredicate predicate, bool isRecurse)
         {
+            public readonly bool IsRecurse = isRecurse;
+            public readonly ParseContext ParseContext = parseContext;
             public readonly VisitChildPredicate Predicate = predicate;
             public readonly ImmutableArray<CXCursor>.Builder CursorBuilder = ImmutableArray.CreateBuilder<CXCursor>();
         }

src/cs/production/c2ffi.Tool/Extract/Explore/ExploreContext.cs

@@ -30,7 +30,7 @@ public CFfiTargetPlatform GetFfi()
     public void TryEnqueueNode(NodeInfo info)
     {
         var handler = GetHandler(info.NodeKind);
-        if (!handler.CanVisitInternal(this, info))
+        if (!handler.CanVisitInternal(info))
         {
             return;
         }
@@ -140,7 +140,7 @@ private CType VisitTypeInternal(
         NodeInfo? rootNode)
     {
         var clangCursorLocation = clang.clang_getTypeDeclaration(clangType);
-        var location = ParseContext.Location(clangCursorLocation);
+        var location = ParseContext.Location(clangCursorLocation, out _);
 
         int? sizeOf;
         int? alignOf;
@@ -281,7 +281,7 @@ private NodeInfo CreateNodeInfo(
         clang.CXType clangType,
         NodeInfo? parentInfo)
     {
-        var location = ParseContext.Location(clangCursor);
+        var location = ParseContext.Location(clangCursor, out _);
         var sizeOf = ParseContext.SizeOf(kind, clangType);
         var alignOf = ParseContext.AlignOf(kind, clangType);
 

src/cs/production/c2ffi.Tool/Extract/Explore/Explorer.cs

@@ -73,7 +73,7 @@ private void VisitTranslationUnit(ExploreContext exploreContext, ParseContext pa
 
     private void VisitFunctions(ExploreContext exploreContext, ParseContext parseContext)
     {
-        var functionCursors = parseContext.GetExternalFunctions();
+        var functionCursors = parseContext.GetExternalFunctions(parseContext);
         foreach (var cursor in functionCursors)
         {
             VisitFunction(exploreContext, cursor);
@@ -168,6 +168,7 @@ private void VisitInclude(ExploreContext exploreContext, clang.CXCursor clangCur
         {
             if (filePath.Contains(systemIncludeDirectory, StringComparison.InvariantCulture))
             {
+                LogSkippedSystemInclude(filePath);
                 return;
             }
         }
@@ -257,4 +258,7 @@ private void LogFoundFunctions(CFfiTargetPlatform ffi)
 
     [LoggerMessage(8, LogLevel.Debug, "- Skipping already visited include file header: {FilePath}")]
     private partial void LogAlreadyVisitedInclude(string filePath);
+
+    [LoggerMessage(9, LogLevel.Debug, "- Skipping system include file header: {FilePath}")]
+    private partial void LogSkippedSystemInclude(string filePath);
 }

src/cs/production/c2ffi.Tool/Extract/Explore/NodeExplorer.cs

@@ -44,7 +44,7 @@ internal abstract partial class NodeExplorer(
         return result;
     }
 
-    internal bool CanVisitInternal(ExploreContext exploreContext, NodeInfo info)
+    internal bool CanVisitInternal(NodeInfo info)
     {
         if (!IsExpectedCursor(info))
         {
@@ -73,23 +73,12 @@ internal bool CanVisitInternal(ExploreContext exploreContext, NodeInfo info)
             return false;
         }
 
-        if (!IsIgnored(exploreContext, info))
-        {
-            LogIgnored(info.NodeKind.ToString(), info.Name);
-            return false;
-        }
-
         MarkAsVisited(info);
         return true;
     }
 
     protected abstract CNode? GetNode(ExploreContext exploreContext, NodeInfo info);
 
-    protected virtual bool IsIgnored(ExploreContext exploreContext, NodeInfo info)
-    {
-        return true;
-    }
-
     private bool IsAlreadyVisited(NodeInfo info, out CLocation? firstLocation)
     {
         var result = _visitedNodeNames.TryGetValue(info.Name, out firstLocation);
@@ -130,6 +119,6 @@ private bool IsExpectedType(NodeInfo info)
     [LoggerMessage(5, LogLevel.Debug, "- Explored {Kind} '{Name}' ({Location})'")]
     private partial void LogExplored(string kind, string name, CLocation? location);
 
-    [LoggerMessage(6, LogLevel.Debug, "- Ignored {Kind} '{Name}'")]
-    private partial void LogIgnored(string kind, string name);
+    [LoggerMessage(6, LogLevel.Information, "- Tried to explore {Kind} '{Name}' ({Location})' but failed.")]
+    private partial void LogNotExplored(string kind, string name, CLocation? location);
 }

src/cs/production/c2ffi.Tool/Extract/Explore/NodeExplorers/EnumExplorer.cs

@@ -5,6 +5,7 @@
 using c2ffi.Clang;
 using c2ffi.Data;
 using c2ffi.Data.Nodes;
+using c2ffi.Extract.Parse;
 using JetBrains.Annotations;
 using Microsoft.Extensions.Logging;
 using static bottlenoselabs.clang;
@@ -30,7 +31,7 @@ protected override CNode GetNode(ExploreContext exploreContext, NodeInfo info)
     private CEnum Enum(ExploreContext exploreContext, NodeInfo info)
     {
         var integerType = IntegerTypeInfo(exploreContext, info);
-        var enumValues = EnumValues(info.ClangCursor);
+        var enumValues = EnumValues(exploreContext.ParseContext, info.ClangCursor);
         var comment = exploreContext.Comment(info.ClangCursor);
 
         var result = new CEnum
@@ -51,12 +52,13 @@ private static CType IntegerTypeInfo(ExploreContext exploreContext, NodeInfo inf
         return exploreContext.VisitType(clangType, info);
     }
 
-    private ImmutableArray<CEnumValue> EnumValues(CXCursor clangCursor)
+    private ImmutableArray<CEnumValue> EnumValues(ParseContext parseContext, CXCursor clangCursor)
     {
         var builder = ImmutableArray.CreateBuilder<CEnumValue>();
 
         var enumValuesCursors = clangCursor.GetDescendents(
-            static (child, _) => child.kind == CXCursorKind.CXCursor_EnumConstantDecl);
+            parseContext,
+            static (_, child, _) => child.kind == CXCursorKind.CXCursor_EnumConstantDecl);
 
         foreach (var enumValueCursor in enumValuesCursors)
         {

src/cs/production/c2ffi.Tool/Extract/Explore/NodeExplorers/FunctionExplorer.cs

@@ -29,20 +29,6 @@ internal sealed class FunctionExplorer(ILogger<FunctionExplorer> logger)
     protected override KindTypes ExpectedTypes { get; } = KindTypes.Either(
         CXTypeKind.CXType_FunctionProto, CXTypeKind.CXType_FunctionNoProto);
 
-    protected override bool IsIgnored(ExploreContext exploreContext, NodeInfo info)
-    {
-        var regexes = exploreContext.ParseContext.InputSanitized.IgnoredFunctionRegexes;
-        foreach (var regex in regexes)
-        {
-            if (regex.IsMatch(info.Name))
-            {
-                return false;
-            }
-        }
-
-        return true;
-    }
-
     protected override CNode GetNode(ExploreContext exploreContext, NodeInfo info)
     {
         var function = Function(exploreContext, info);

src/cs/production/c2ffi.Tool/Extract/Explore/NodeExplorers/MacroObjectExplorer.cs

@@ -30,20 +30,6 @@ internal sealed class MacroObjectExplorer(
 
     protected override KindTypes ExpectedTypes => KindTypes.Any;
 
-    protected override bool IsIgnored(ExploreContext exploreContext, NodeInfo info)
-    {
-        var ignoredMacroObjectRegexes = exploreContext.ParseContext.InputSanitized.IgnoredMacroObjectsRegexes;
-        foreach (var regex in ignoredMacroObjectRegexes)
-        {
-            if (regex.IsMatch(info.Name))
-            {
-                return false;
-            }
-        }
-
-        return true;
-    }
-
     protected override CNode? GetNode(ExploreContext exploreContext, NodeInfo info)
     {
         return MacroObject(exploreContext, info);
@@ -124,7 +110,7 @@ int main(void)
 
         var translationUnitCursor = clang.clang_getTranslationUnitCursor(parseContext.TranslationUnit);
         var functionCursor = translationUnitCursor
-            .GetDescendents(static (cursor, _) =>
+            .GetDescendents(parseContext, static (_, cursor, _) =>
             {
                 var sourceLocation = clang.clang_getCursorLocation(cursor);
                 var isFromMainFile = clang.clang_Location_isFromMainFile(sourceLocation) > 0;
@@ -141,20 +127,20 @@ int main(void)
                 "Failed to parse C++ file to determine types of macro objects. Please ensure your libclang version is up-to-date.");
         }
 
-        var compoundStatement = functionCursor.GetDescendents(static (cursor, _) =>
+        var compoundStatement = functionCursor.GetDescendents(parseContext, static (_, cursor, _) =>
                 cursor.kind == clang.CXCursorKind.CXCursor_CompoundStmt)
             .FirstOrDefault();
         var declarationStatement =
-            compoundStatement.GetDescendents(static (cursor, _) =>
+            compoundStatement.GetDescendents(parseContext, static (_, cursor, _) =>
                 cursor.kind == clang.CXCursorKind.CXCursor_DeclStmt).FirstOrDefault();
 
-        var variable = declarationStatement.GetDescendents(static (cursor, _) =>
+        var variable = declarationStatement.GetDescendents(parseContext, static (_, cursor, _) =>
                 cursor.kind == clang.CXCursorKind.CXCursor_VarDecl)
             .FirstOrDefault();
         var variableName = variable.Spelling();
         var macroName =
             variableName.Replace("variable_", string.Empty, StringComparison.InvariantCultureIgnoreCase);
-        var variableDescendents = variable.GetDescendents();
+        var variableDescendents = variable.GetDescendents(parseContext);
         if (variableDescendents.IsDefaultOrEmpty)
         {
             return null;
@@ -173,7 +159,36 @@ int main(void)
             return null;
         }
 
-        var type = exploreContext.VisitType(clangType, info);
+        var typeName = clangType.Spelling();
+        var mainTranslationUnit = clang.clang_Cursor_getTranslationUnit(info.ClangCursor);
+        var mainTranslationUnitCursor = clang.clang_getTranslationUnitCursor(mainTranslationUnit);
+
+        var clangCursorsInMainTranslationUnit = mainTranslationUnitCursor.GetDescendents(
+            exploreContext.ParseContext,
+            (_, b, _) => FindTypeName(typeName, b),
+            isRecurse: true);
+
+        static bool FindTypeName(string typeName, clang.CXCursor clangCursor)
+        {
+            var clangType = clang.clang_getCursorType(clangCursor);
+            if (clangType.kind == clang.CXTypeKind.CXType_Invalid)
+            {
+                return false;
+            }
+
+            var clangTypeName = clangType.Spelling();
+            return clangTypeName == typeName;
+        }
+
+        if (clangCursorsInMainTranslationUnit.IsDefaultOrEmpty)
+        {
+            throw new ToolException(
+                $"Failed to find matching type in main translation unit for macro object '{macroName}'.");
+        }
+
+        var clangCursorInMainTranslationUnit = clangCursorsInMainTranslationUnit.First();
+        var clangTypeInMainTranslationUnit = clang.clang_getCursorType(clangCursorInMainTranslationUnit);
+        var type = exploreContext.VisitType(clangTypeInMainTranslationUnit, info);
         var macroObject = new CMacroObject
         {
             Name = macroName,
@@ -250,7 +265,7 @@ private sealed class MacroObjectCandidate
             clang.CXCursor clangCursor)
         {
             var name = clangCursor.Spelling();
-            var location = parseContext.Location(clangCursor);
+            var location = parseContext.Location(clangCursor, out _);
 
             // clang doesn't have a thing where we can easily get a value of a macro
             // we need to:

src/cs/production/c2ffi.Tool/Extract/Explore/NodeExplorers/StructExplorer.cs

@@ -72,7 +72,7 @@ private CRecordField StructField(
     {
         var fieldName = exploreContext.GetFieldName(clangCursor);
         var clangType = clang_getCursorType(clangCursor);
-        var location = exploreContext.ParseContext.Location(clangCursor);
+        var location = exploreContext.ParseContext.Location(clangCursor, out _);
         var type = exploreContext.VisitType(clangType, structInfo);
         var offsetOf = (int)clang_Cursor_getOffsetOfField(clangCursor) / 8;
         var comment = exploreContext.Comment(clangCursor);