(#538) Preprocessor: properly parse the multi-line blocks

ForNeVeR · Feb 10, 2024 · 3f249e0 · 3f249e0
1 parent 134bf14
commit 3f249e0
Show file tree

Hide file tree

Showing 5 changed files with 61 additions and 21 deletions.
diff --git a/Cesium.Parser.Tests/PreprocessorTests/PreprocessorTests.cs b/Cesium.Parser.Tests/PreprocessorTests/PreprocessorTests.cs
@@ -1046,5 +1046,13 @@ int main(void)
 
    return 42;
 }
+""");
+
+    [Fact]
+    public Task MultiLineMacro() => DoTest("""
+#define MACRO(x) x
+MACRO(1
+2
+3)
 """);
 }
diff --git a/Cesium.Parser.Tests/PreprocessorTests/verified/PreprocessorTests.MultiLineMacro.verified.txt b/Cesium.Parser.Tests/PreprocessorTests/verified/PreprocessorTests.MultiLineMacro.verified.txt
@@ -0,0 +1,3 @@
+1
+2
+3
diff --git a/Cesium.Preprocessor/CPreprocessor.cs b/Cesium.Preprocessor/CPreprocessor.cs
@@ -17,7 +17,7 @@ public record CPreprocessor(
     IMacroContext MacroContext,
     IWarningProcessor WarningProcessor)
 {
-    private MacroExpansionEngine _macroExpansion = new(WarningProcessor, MacroContext);
+    private readonly MacroExpansionEngine _macroExpansion = new(WarningProcessor, MacroContext);
 
     public async Task<string> ProcessSource()
     {
@@ -176,8 +176,8 @@ private async IAsyncEnumerable<IToken<CPreprocessorTokenType>> ProcessGroupPart(
             }
             case EmptyDirective:
                 break;
-            case TextLine textLine:
-                foreach (var token in _macroExpansion.ExpandMacros(textLine.Tokens ?? []))
+            case TextLineBlock textLine:
+                foreach (var token in _macroExpansion.ExpandMacros(textLine.Tokens))
                 {
                     yield return token;
                 }

diff --git a/Cesium.Preprocessor/CPreprocessorParser.cs b/Cesium.Preprocessor/CPreprocessorParser.cs
@@ -48,7 +48,7 @@ private ParseResult<IGroupPart> ParseGroupPart()
         var controlLine = ParseControlLine();
         if (controlLine.IsOk) return transaction.End(controlLine);
 
-        var textLine = ParseTextLine();
+        var textLine = ParseTextLineBlock();
         if (textLine.IsOk) return transaction.End(textLine);
 
         if (Peek() is var token and not { Kind: CPreprocessorTokenType.Hash })
@@ -473,7 +473,47 @@ private ParseResult<IGroupPart> ParsePragma(Location location)
                 new PragmaDirective(location, pragma, tokens.IsOk ? tokens.Ok.Value.ToImmutableArray() : null)));
     }
 
-    private ParseResult<IGroupPart> ParseTextLine()
+    private ParseResult<IGroupPart> ParseTextLineBlock()
+    {
+        using var transaction = lexer.BeginTransaction();
+
+        var lines = new List<TextLineBlock>();
+        while (!lexer.IsEnd)
+        {
+            var nextLine = ParseTextLine();
+            if (nextLine.IsOk)
+            {
+                lines.Add(nextLine.Ok.Value);
+            }
+            else
+            {
+                return transaction.End(GetResultBlock(nextLine.Error));
+            }
+        }
+
+        return transaction.End(GetResultBlock(null));
+
+        ParseResult<IGroupPart> GetResultBlock(ParseError? error)
+        {
+            // No lines: error.
+            if (lines.Count == 0)
+            {
+                if (error == null) throw new AssertException("Error should not be null at this point.");
+                return error;
+            }
+
+            // Strip the terminating new-line (will be added back by the preprocessing engine):
+            var allTokens = lines.SelectMany(l => l.Tokens).ToList();
+            var tokenBlock = allTokens[^1] is { Kind: CPreprocessorTokenType.NewLine }
+                ? allTokens.Take(allTokens.Count - 1)
+                : allTokens;
+
+            var block = new TextLineBlock(lines[0].Location, tokenBlock.ToImmutableArray());
+            return Ok<IGroupPart>(block);
+        }
+    }
+
+    private ParseResult<TextLineBlock> ParseTextLine()
     {
         using var transaction = lexer.BeginTransaction();
 
@@ -484,9 +524,9 @@ private ParseResult<IGroupPart> ParseTextLine()
         var newLine = ParseNewLine();
         if (!newLine.IsOk) return transaction.End(newLine.Error);
 
-        ImmutableArray<ICPreprocessorToken>? allTokens = tokens.IsOk ? tokens.Ok.Value.ToImmutableArray() : null;
-        var location = allTokens?.FirstOrDefault()?.Location ?? newLine.Ok.Value.Location;
-        return transaction.End(Ok<IGroupPart>(new TextLine(location, allTokens)));
+        var allTokens = tokens.IsOk ? tokens.Ok.Value.ToImmutableArray().Add(newLine.Ok.Value) : [];
+        var location = allTokens.FirstOrDefault()?.Location ?? newLine.Ok.Value.Location;
+        return transaction.End(Ok(new TextLineBlock(location, allTokens)));
     }
 
     private ParseResult<IGroupPart> ParseNonDirective(Location location)

diff --git a/Cesium.Preprocessor/Directives.cs b/Cesium.Preprocessor/Directives.cs
@@ -1,5 +1,4 @@
 using System.Collections.Immutable;
-using Cesium.Core;
 using Yoakke.SynKit.Lexer;
 using Yoakke.SynKit.Text;
 
@@ -44,17 +43,7 @@ internal record EmbedDirective(Location Location, ICPreprocessorToken Keyword, T
 public record MacroParameters(
     Tokens Parameters,
     bool HasEllipsis
-)
-{
-    internal string GetName(int index)
-    {
-        var token = Parameters[index];
-        if (token.Kind != CPreprocessorTokenType.PreprocessingToken)
-            throw new PreprocessorException(token.Location, $"Parameter {token.Text} is not an identifier, but an {token.Kind}.");
-
-        return token.Text;
-    }
-}
+);
 
 /// <param name="Parameters">
 /// If <c>null</c> then the macro is not a function-like. If empty then it is function-like and requires parens to be
@@ -87,7 +76,7 @@ internal record EmptyDirective(Location Location) : IGroupPart
     public ICPreprocessorToken? Keyword => null;
 }
 
-internal record TextLine(Location Location, Tokens? Tokens) : IGroupPart
+internal record TextLineBlock(Location Location, Tokens Tokens) : IGroupPart
 {
     public ICPreprocessorToken? Keyword => null;
 }