From 3f249e0b59b9516162139b4d5a4aa35c39701d86 Mon Sep 17 00:00:00 2001 From: Friedrich von Never Date: Sat, 10 Feb 2024 22:53:27 +0100 Subject: [PATCH] (#538) Preprocessor: properly parse the multi-line blocks --- .../PreprocessorTests/PreprocessorTests.cs | 8 +++ ...processorTests.MultiLineMacro.verified.txt | 3 ++ Cesium.Preprocessor/CPreprocessor.cs | 6 +-- Cesium.Preprocessor/CPreprocessorParser.cs | 50 +++++++++++++++++-- Cesium.Preprocessor/Directives.cs | 15 +----- 5 files changed, 61 insertions(+), 21 deletions(-) create mode 100644 Cesium.Parser.Tests/PreprocessorTests/verified/PreprocessorTests.MultiLineMacro.verified.txt diff --git a/Cesium.Parser.Tests/PreprocessorTests/PreprocessorTests.cs b/Cesium.Parser.Tests/PreprocessorTests/PreprocessorTests.cs index 03167984..d05cae49 100644 --- a/Cesium.Parser.Tests/PreprocessorTests/PreprocessorTests.cs +++ b/Cesium.Parser.Tests/PreprocessorTests/PreprocessorTests.cs @@ -1046,5 +1046,13 @@ int main(void) return 42; } +"""); + + [Fact] + public Task MultiLineMacro() => DoTest(""" +#define MACRO(x) x +MACRO(1 +2 +3) """); } diff --git a/Cesium.Parser.Tests/PreprocessorTests/verified/PreprocessorTests.MultiLineMacro.verified.txt b/Cesium.Parser.Tests/PreprocessorTests/verified/PreprocessorTests.MultiLineMacro.verified.txt new file mode 100644 index 00000000..5adb0eff --- /dev/null +++ b/Cesium.Parser.Tests/PreprocessorTests/verified/PreprocessorTests.MultiLineMacro.verified.txt @@ -0,0 +1,3 @@ +1 +2 +3 diff --git a/Cesium.Preprocessor/CPreprocessor.cs b/Cesium.Preprocessor/CPreprocessor.cs index 868e6e3f..789582aa 100644 --- a/Cesium.Preprocessor/CPreprocessor.cs +++ b/Cesium.Preprocessor/CPreprocessor.cs @@ -17,7 +17,7 @@ public record CPreprocessor( IMacroContext MacroContext, IWarningProcessor WarningProcessor) { - private MacroExpansionEngine _macroExpansion = new(WarningProcessor, MacroContext); + private readonly MacroExpansionEngine _macroExpansion = new(WarningProcessor, MacroContext); public async Task ProcessSource() { @@ -176,8 +176,8 @@ private async IAsyncEnumerable> ProcessGroupPart( } case EmptyDirective: break; - case TextLine textLine: - foreach (var token in _macroExpansion.ExpandMacros(textLine.Tokens ?? [])) + case TextLineBlock textLine: + foreach (var token in _macroExpansion.ExpandMacros(textLine.Tokens)) { yield return token; } diff --git a/Cesium.Preprocessor/CPreprocessorParser.cs b/Cesium.Preprocessor/CPreprocessorParser.cs index 1c6c9acc..66cfd719 100644 --- a/Cesium.Preprocessor/CPreprocessorParser.cs +++ b/Cesium.Preprocessor/CPreprocessorParser.cs @@ -48,7 +48,7 @@ private ParseResult ParseGroupPart() var controlLine = ParseControlLine(); if (controlLine.IsOk) return transaction.End(controlLine); - var textLine = ParseTextLine(); + var textLine = ParseTextLineBlock(); if (textLine.IsOk) return transaction.End(textLine); if (Peek() is var token and not { Kind: CPreprocessorTokenType.Hash }) @@ -473,7 +473,47 @@ private ParseResult ParsePragma(Location location) new PragmaDirective(location, pragma, tokens.IsOk ? tokens.Ok.Value.ToImmutableArray() : null))); } - private ParseResult ParseTextLine() + private ParseResult ParseTextLineBlock() + { + using var transaction = lexer.BeginTransaction(); + + var lines = new List(); + while (!lexer.IsEnd) + { + var nextLine = ParseTextLine(); + if (nextLine.IsOk) + { + lines.Add(nextLine.Ok.Value); + } + else + { + return transaction.End(GetResultBlock(nextLine.Error)); + } + } + + return transaction.End(GetResultBlock(null)); + + ParseResult GetResultBlock(ParseError? error) + { + // No lines: error. + if (lines.Count == 0) + { + if (error == null) throw new AssertException("Error should not be null at this point."); + return error; + } + + // Strip the terminating new-line (will be added back by the preprocessing engine): + var allTokens = lines.SelectMany(l => l.Tokens).ToList(); + var tokenBlock = allTokens[^1] is { Kind: CPreprocessorTokenType.NewLine } + ? allTokens.Take(allTokens.Count - 1) + : allTokens; + + var block = new TextLineBlock(lines[0].Location, tokenBlock.ToImmutableArray()); + return Ok(block); + } + } + + private ParseResult ParseTextLine() { using var transaction = lexer.BeginTransaction(); @@ -484,9 +524,9 @@ private ParseResult ParseTextLine() var newLine = ParseNewLine(); if (!newLine.IsOk) return transaction.End(newLine.Error); - ImmutableArray? allTokens = tokens.IsOk ? tokens.Ok.Value.ToImmutableArray() : null; - var location = allTokens?.FirstOrDefault()?.Location ?? newLine.Ok.Value.Location; - return transaction.End(Ok(new TextLine(location, allTokens))); + var allTokens = tokens.IsOk ? tokens.Ok.Value.ToImmutableArray().Add(newLine.Ok.Value) : []; + var location = allTokens.FirstOrDefault()?.Location ?? newLine.Ok.Value.Location; + return transaction.End(Ok(new TextLineBlock(location, allTokens))); } private ParseResult ParseNonDirective(Location location) diff --git a/Cesium.Preprocessor/Directives.cs b/Cesium.Preprocessor/Directives.cs index b5cc05c1..422c6a22 100644 --- a/Cesium.Preprocessor/Directives.cs +++ b/Cesium.Preprocessor/Directives.cs @@ -1,5 +1,4 @@ using System.Collections.Immutable; -using Cesium.Core; using Yoakke.SynKit.Lexer; using Yoakke.SynKit.Text; @@ -44,17 +43,7 @@ internal record EmbedDirective(Location Location, ICPreprocessorToken Keyword, T public record MacroParameters( Tokens Parameters, bool HasEllipsis -) -{ - internal string GetName(int index) - { - var token = Parameters[index]; - if (token.Kind != CPreprocessorTokenType.PreprocessingToken) - throw new PreprocessorException(token.Location, $"Parameter {token.Text} is not an identifier, but an {token.Kind}."); - - return token.Text; - } -} +); /// /// If null then the macro is not a function-like. If empty then it is function-like and requires parens to be @@ -87,7 +76,7 @@ internal record EmptyDirective(Location Location) : IGroupPart public ICPreprocessorToken? Keyword => null; } -internal record TextLine(Location Location, Tokens? Tokens) : IGroupPart +internal record TextLineBlock(Location Location, Tokens Tokens) : IGroupPart { public ICPreprocessorToken? Keyword => null; }