From f3712e4e30cf5c4d279512882954e170155609c9 Mon Sep 17 00:00:00 2001 From: Stan H Date: Sun, 31 Jan 2021 11:30:05 +0100 Subject: [PATCH 01/59] port to .NET 5 and enable Nullable --- ColorzCore/App.config | 6 - ColorzCore/ColorzCore.csproj | 123 +----------------- ColorzCore/DataTypes/CaseInsensitiveString.cs | 6 +- ColorzCore/DataTypes/Extension.cs | 57 ++++---- ColorzCore/DataTypes/Maybe.cs | 6 +- ColorzCore/ExecTimer.cs | 14 +- ColorzCore/IO/IncludeFileSearcher.cs | 6 +- ColorzCore/Lexer/Tokenizer.cs | 14 +- ColorzCore/Parser/AST/IParamNode.cs | 2 +- ColorzCore/Parser/EAParser.cs | 44 +++---- ColorzCore/Preprocessor/DirectiveHandler.cs | 10 +- .../Directives/IncludeBinaryDirective.cs | 12 +- .../Directives/IncludeDirective.cs | 12 +- .../Directives/IncludeExternalDirective.cs | 8 +- .../Directives/IncludeToolEventDirective.cs | 8 +- .../Directives/UndefineDirective.cs | 4 +- ColorzCore/Program.cs | 2 +- ColorzCore/Properties/AssemblyInfo.cs | 36 ----- ColorzCore/Raws/Raw.cs | 44 +++---- ColorzCore/Raws/RawReader.cs | 12 +- 20 files changed, 137 insertions(+), 289 deletions(-) delete mode 100644 ColorzCore/App.config delete mode 100644 ColorzCore/Properties/AssemblyInfo.cs diff --git a/ColorzCore/App.config b/ColorzCore/App.config deleted file mode 100644 index d740e88..0000000 --- a/ColorzCore/App.config +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/ColorzCore/ColorzCore.csproj b/ColorzCore/ColorzCore.csproj index 842e13f..67b7429 100644 --- a/ColorzCore/ColorzCore.csproj +++ b/ColorzCore/ColorzCore.csproj @@ -1,124 +1,7 @@ - - - + - Debug - AnyCPU - {B98F7CCF-9CAA-406E-88D7-2040FA99F631} + net5.0 Exe - ColorzCore - ColorzCore - v4.5.2 - 512 - true + enable - - AnyCPU - true - full - false - bin\Debug\ - DEBUG;TRACE - prompt - 4 - - - AnyCPU - pdbonly - true - bin\Release\ - TRACE - prompt - 4 - - - bin\32bit\ - x86 - true - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/ColorzCore/DataTypes/CaseInsensitiveString.cs b/ColorzCore/DataTypes/CaseInsensitiveString.cs index 8fd8ade..cbc65e7 100644 --- a/ColorzCore/DataTypes/CaseInsensitiveString.cs +++ b/ColorzCore/DataTypes/CaseInsensitiveString.cs @@ -18,15 +18,15 @@ public string String { } public CaseInsensitiveString(string input) { - String = input; + data = input.ToUpper(); } public override int GetHashCode() { return String.GetHashCode(); } - public override bool Equals(object obj) + public override bool Equals(object? obj) { - return String == obj.ToString(); + return obj != null && String == obj.ToString(); } public override string ToString() { diff --git a/ColorzCore/DataTypes/Extension.cs b/ColorzCore/DataTypes/Extension.cs index abb0c0b..41d6848 100644 --- a/ColorzCore/DataTypes/Extension.cs +++ b/ColorzCore/DataTypes/Extension.cs @@ -50,6 +50,7 @@ public static int ToInt(this string numString) } } public static Dictionary> AddTo(this Dictionary> self, K key, V val) + where K: notnull { if (self.ContainsKey(key)) self[key].Add(val); @@ -58,8 +59,8 @@ public static Dictionary> AddTo(this Dictionary> se self[key] = new List { val }; } return self; - } - + } + public static void SetBits(this byte[] array, int bitOffset, int bitSize, int value) { array.SetBits(0, bitOffset, bitSize, value); @@ -67,10 +68,10 @@ public static void SetBits(this byte[] array, int bitOffset, int bitSize, int va public static void SetBits(this byte[] array, int byteOffset, int bitOffset, int bitSize, int value) { - if (bitOffset >= 8) + if (bitOffset >= 8) { array.SetBits(byteOffset + bitOffset / 8, bitOffset % 8, bitSize, value); - return; + return; } long bytes = 0; @@ -95,23 +96,23 @@ public static void SetBits(this byte[] array, int byteOffset, int bitOffset, int array[byteOffset + i] = (byte)(bytes >> i * 8); } - public static void SetBits(this byte[] array, int bitOffset, int bitSize, byte[] data) - { - array.SetBits(0, bitOffset, bitSize, data); + public static void SetBits(this byte[] array, int bitOffset, int bitSize, byte[] data) + { + array.SetBits(0, bitOffset, bitSize, data); } - public static void SetBits(this byte[] array, int byteOffset, int bitOffset, int bitSize, byte[] data) - { - if (bitOffset >= 8) + public static void SetBits(this byte[] array, int byteOffset, int bitOffset, int bitSize, byte[] data) + { + if (bitOffset >= 8) { array.SetBits(byteOffset + bitOffset / 8, bitOffset % 8, bitSize, data); - return; + return; } int byteSize = (bitOffset + bitSize + 7) / 8; - if (bitOffset == 0) - { + if (bitOffset == 0) + { for (int i = 0; i < byteSize - 1; ++i) array[byteOffset + i] = data[i]; @@ -119,24 +120,24 @@ public static void SetBits(this byte[] array, int byteOffset, int bitOffset, int uint endMask = (uint)((1 << shift) - 1); array[byteOffset + byteSize - 1] &= (byte)~endMask; - array[byteOffset + byteSize - 1] |= (byte)(data[byteSize - 1] & endMask); + array[byteOffset + byteSize - 1] |= (byte)(data[byteSize - 1] & endMask); } else - { - for (int i = 0; i < byteSize; ++i) - { - int mask = ((1 << Math.Min(bitSize - i * 8, 8)) - 1) << bitOffset; - - byte loMask = (byte)mask; - byte hiMask = (byte)(mask >> 8); - - array[byteOffset + i] &= (byte)~loMask; - array[byteOffset + i] |= (byte)((data[i] << bitOffset) & loMask); - - array[byteOffset + i + 1] &= (byte)~hiMask; - array[byteOffset + i + 1] |= (byte)((data[i] >> (8 - bitOffset)) & hiMask); + { + for (int i = 0; i < byteSize; ++i) + { + int mask = ((1 << Math.Min(bitSize - i * 8, 8)) - 1) << bitOffset; + + byte loMask = (byte)mask; + byte hiMask = (byte)(mask >> 8); + + array[byteOffset + i] &= (byte)~loMask; + array[byteOffset + i] |= (byte)((data[i] << bitOffset) & loMask); + + array[byteOffset + i + 1] &= (byte)~hiMask; + array[byteOffset + i + 1] |= (byte)((data[i] >> (8 - bitOffset)) & hiMask); } - } + } } } } diff --git a/ColorzCore/DataTypes/Maybe.cs b/ColorzCore/DataTypes/Maybe.cs index 130583c..6ffcff3 100644 --- a/ColorzCore/DataTypes/Maybe.cs +++ b/ColorzCore/DataTypes/Maybe.cs @@ -21,7 +21,7 @@ public interface Maybe Maybe Fmap(UnaryFunction f); Maybe Bind(MaybeAction f); R IfJust(UnaryFunction just, RConst nothing); - void IfJust(TAction just, NullaryAction nothing = null); + void IfJust(TAction just, NullaryAction? nothing = null); } public class Just : Maybe { @@ -45,7 +45,7 @@ public R IfJust(UnaryFunction just, RConst nothing) { return just(FromJust); } - public void IfJust(TAction just, NullaryAction nothing) + public void IfJust(TAction just, NullaryAction? nothing) { just(FromJust); } @@ -69,7 +69,7 @@ public R IfJust(UnaryFunction just, RConst nothing) { return nothing(); } - public void IfJust(TAction just, NullaryAction nothing) + public void IfJust(TAction just, NullaryAction? nothing) { nothing?.Invoke(); } diff --git a/ColorzCore/ExecTimer.cs b/ColorzCore/ExecTimer.cs index 5c603ad..45277b9 100644 --- a/ColorzCore/ExecTimer.cs +++ b/ColorzCore/ExecTimer.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Diagnostics; namespace ColorzCore { @@ -14,8 +15,8 @@ public class ExecTimer private List> timingPoints; - private Dictionary times; - private Dictionary counts; + private Dictionary? times; + private Dictionary? counts; private TimeSpan totalTime; @@ -37,8 +38,9 @@ public SortedList SortedTimes SortedList sortedTimes = new SortedList(); - foreach (KeyValuePair time in this.times) - sortedTimes.Add(time.Value, time.Key); + if (this.times != null) + foreach (KeyValuePair time in this.times) + sortedTimes.Add(time.Value, time.Key); return sortedTimes; } @@ -51,7 +53,7 @@ public Dictionary Times if (this.times == null) ComputeTimes(); - return this.times; + return this.times!; } } @@ -62,7 +64,7 @@ public Dictionary Counts if (this.counts == null) ComputeTimes(); - return this.counts; + return this.counts!; } } diff --git a/ColorzCore/IO/IncludeFileSearcher.cs b/ColorzCore/IO/IncludeFileSearcher.cs index 4fe5f1b..2e1c63d 100644 --- a/ColorzCore/IO/IncludeFileSearcher.cs +++ b/ColorzCore/IO/IncludeFileSearcher.cs @@ -15,7 +15,7 @@ public Maybe FindFile(string name) return FindFile(null, name); } - public Maybe FindFile(string cwd, string name) + public Maybe FindFile(string? cwd, string name) { // Find the first valid file in the list of possible file paths @@ -33,7 +33,7 @@ public Maybe FindDirectory(string name) return FindDirectory(null, name); } - public Maybe FindDirectory(string cwd, string name) + public Maybe FindDirectory(string? cwd, string name) { // Find the first valid directory in the list of possible file paths @@ -46,7 +46,7 @@ public Maybe FindDirectory(string cwd, string name) return new Nothing(); } - protected IEnumerable EnumeratePossibleAccessPaths(string cwd, string name) + protected IEnumerable EnumeratePossibleAccessPaths(string? cwd, string name) { if (AllowRelativeInclude) { diff --git a/ColorzCore/Lexer/Tokenizer.cs b/ColorzCore/Lexer/Tokenizer.cs index 0d7dc06..f9cf5fd 100644 --- a/ColorzCore/Lexer/Tokenizer.cs +++ b/ColorzCore/Lexer/Tokenizer.cs @@ -305,13 +305,17 @@ public IEnumerable Tokenize(Stream input, string fileName) int curLine = 1; while (!sin.EndOfStream) { - string line = sin.ReadLine(); - foreach (Token t in TokenizeLine(line, fileName, curLine)) + string? line = sin.ReadLine(); + + if (line != null) { - yield return t; + foreach (Token t in TokenizeLine(line, fileName, curLine)) + { + yield return t; + } + yield return new Token(TokenType.NEWLINE, fileName, curLine, line.Length); + curLine++; } - yield return new Token(TokenType.NEWLINE, fileName, curLine, line.Length); - curLine++; } } diff --git a/ColorzCore/Parser/AST/IParamNode.cs b/ColorzCore/Parser/AST/IParamNode.cs index 7c70c0b..73a6ac0 100644 --- a/ColorzCore/Parser/AST/IParamNode.cs +++ b/ColorzCore/Parser/AST/IParamNode.cs @@ -10,7 +10,7 @@ namespace ColorzCore.Parser.AST { public interface IParamNode { - string ToString(); //For use in other programs. + string? ToString(); //For use in other programs. ParamType Type { get; } string PrettyPrint(); Location MyLocation { get; } diff --git a/ColorzCore/Parser/EAParser.cs b/ColorzCore/Parser/EAParser.cs index 3ff8431..21ab4bb 100644 --- a/ColorzCore/Parser/EAParser.cs +++ b/ColorzCore/Parser/EAParser.cs @@ -65,7 +65,7 @@ public bool IsIncluding { get private bool validOffset; private bool offsetInitialized; // false until first ORG, used to warn about writing before first org private int currentOffset; - private Token head; //TODO: Make this make sense + private Token? head; //TODO: Make this make sense public EAParser(Dictionary> raws, Log log, DirectiveHandler directiveHandler) { @@ -278,46 +278,46 @@ private Maybe ParseStatement(MergeableGenerator tokens, Immuta }), () => { Error(parameters[0].MyLocation, "Expected atomic param to ALIGN"); } ); - break; + break; case "FILL": - if (parameters.Count > 2 || parameters.Count == 0) - { - Error(head.Location, "Incorrect number of parameters in FILL: " + parameters.Count); + if (parameters.Count > 2 || parameters.Count == 0) + { + Error(head.Location, "Incorrect number of parameters in FILL: " + parameters.Count); } - else + else { // FILL [value] int amount = 0; - int value = 0; - + int value = 0; + if (parameters.Count == 2) { // param 2 (if given) is fill value - + parameters[1].AsAtom().IfJust( - (IAtomNode atom) => atom.TryEvaluate((Exception e) => { Error(parameters[0].MyLocation, e.Message); }).IfJust( + (IAtomNode atom) => atom.TryEvaluate((Exception e) => { Error(parameters[0].MyLocation, e.Message); }).IfJust( (int val) => { value = val; }), () => { Error(parameters[0].MyLocation, "Expected atomic param to FILL"); }); - } + } - // param 1 is amount of bytes to fill + // param 1 is amount of bytes to fill parameters[0].AsAtom().IfJust( - (IAtomNode atom) => atom.TryEvaluate((Exception e) => { Error(parameters[0].MyLocation, e.Message); }).IfJust( - (int val) => { amount = val; }), - () => { Error(parameters[0].MyLocation, "Expected atomic param to FILL"); }); + (IAtomNode atom) => atom.TryEvaluate((Exception e) => { Error(parameters[0].MyLocation, e.Message); }).IfJust( + (int val) => { amount = val; }), + () => { Error(parameters[0].MyLocation, "Expected atomic param to FILL"); }); var data = new byte[amount]; for (int i = 0; i < amount; ++i) - data[i] = (byte) value; + data[i] = (byte) value; + + var node = new DataNode(CurrentOffset, data); - var node = new DataNode(CurrentOffset, data); - CheckDataWrite(amount); CurrentOffset += amount; - - return new Just(node); + + return new Just(node); } break; @@ -877,7 +877,7 @@ private void CheckDataWrite(int length) // TODO: maybe make this warning optional? if (!offsetInitialized) { - Warning(head.Location, "Writing before initializing offset. You may be breaking the ROM! (use `ORG offset` to set write offset)."); + Warning(head?.Location, "Writing before initializing offset. You may be breaking the ROM! (use `ORG offset` to set write offset)."); offsetInitialized = false; // only warn once } @@ -888,7 +888,7 @@ private void CheckDataWrite(int length) if (!prot.IsNothing) { Location l = prot.FromJust; - Error(head.Location, System.String.Format("Trying to write data to area protected in file {0} at line {1}, column {2}.", Path.GetFileName(l.file), l.lineNum, l.colNum)); + Error(head?.Location, System.String.Format("Trying to write data to area protected in file {0} at line {1}, column {2}.", Path.GetFileName(l.file), l.lineNum, l.colNum)); } } } diff --git a/ColorzCore/Preprocessor/DirectiveHandler.cs b/ColorzCore/Preprocessor/DirectiveHandler.cs index 3cf8ff2..bc07de9 100644 --- a/ColorzCore/Preprocessor/DirectiveHandler.cs +++ b/ColorzCore/Preprocessor/DirectiveHandler.cs @@ -1,5 +1,5 @@ using ColorzCore.DataTypes; -using ColorzCore.IO; +using ColorzCore.IO; using ColorzCore.Lexer; using ColorzCore.Parser; using ColorzCore.Parser.AST; @@ -16,8 +16,8 @@ class DirectiveHandler { private Dictionary directives; - public DirectiveHandler(IncludeFileSearcher includeSearcher, IncludeFileSearcher toolSearcher) - { + public DirectiveHandler(IncludeFileSearcher includeSearcher, IncludeFileSearcher toolSearcher) + { directives = new Dictionary { { "include", new IncludeDirective { FileSearcher = includeSearcher } }, @@ -32,14 +32,14 @@ public DirectiveHandler(IncludeFileSearcher includeSearcher, IncludeFileSearcher { "define", new DefineDirective() }, { "pool", new PoolDirective() }, { "undef", new UndefineDirective() }, - }; + }; } public Maybe HandleDirective(EAParser p, Token directive, IList parameters, MergeableGenerator tokens) { string directiveName = directive.Content.Substring(1); - if (directives.TryGetValue(directiveName, out IDirective toExec)) + if (directives.TryGetValue(directiveName, out IDirective? toExec)) { if (!toExec.RequireInclusion || p.IsIncluding) { diff --git a/ColorzCore/Preprocessor/Directives/IncludeBinaryDirective.cs b/ColorzCore/Preprocessor/Directives/IncludeBinaryDirective.cs index c590ed3..628f6f2 100644 --- a/ColorzCore/Preprocessor/Directives/IncludeBinaryDirective.cs +++ b/ColorzCore/Preprocessor/Directives/IncludeBinaryDirective.cs @@ -8,8 +8,8 @@ using ColorzCore.Parser.AST; using System.IO; using ColorzCore.Parser; -using ColorzCore.IO; - +using ColorzCore.IO; + namespace ColorzCore.Preprocessor.Directives { class IncludeBinaryDirective : IDirective @@ -18,13 +18,13 @@ class IncludeBinaryDirective : IDirective public int? MaxParams { get { return 1; } } - public bool RequireInclusion { get { return true; } } - - public IncludeFileSearcher FileSearcher { get; set; } + public bool RequireInclusion { get { return true; } } + + public IncludeFileSearcher FileSearcher { get; set; } = new IncludeFileSearcher(); public Maybe Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) { - Maybe existantFile = FileSearcher.FindFile(Path.GetDirectoryName(self.FileName), parameters[0].ToString()); + Maybe existantFile = FileSearcher.FindFile(Path.GetDirectoryName(self.FileName), parameters[0].ToString()!); if (!existantFile.IsNothing) { diff --git a/ColorzCore/Preprocessor/Directives/IncludeDirective.cs b/ColorzCore/Preprocessor/Directives/IncludeDirective.cs index e70f6c3..24bb81f 100644 --- a/ColorzCore/Preprocessor/Directives/IncludeDirective.cs +++ b/ColorzCore/Preprocessor/Directives/IncludeDirective.cs @@ -8,8 +8,8 @@ using ColorzCore.Parser.AST; using System.IO; using ColorzCore.Parser; -using ColorzCore.IO; - +using ColorzCore.IO; + namespace ColorzCore.Preprocessor.Directives { class IncludeDirective : IDirective @@ -18,13 +18,13 @@ class IncludeDirective : IDirective public int? MaxParams { get { return 1; } } - public bool RequireInclusion { get { return true; } } - - public IncludeFileSearcher FileSearcher { get; set; } + public bool RequireInclusion { get { return true; } } + + public IncludeFileSearcher FileSearcher { get; set; } = new IncludeFileSearcher(); public Maybe Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) { - Maybe existantFile = FileSearcher.FindFile(Path.GetDirectoryName(self.FileName), parameters[0].ToString()); + Maybe existantFile = FileSearcher.FindFile(Path.GetDirectoryName(self.FileName), parameters[0].ToString()!); if (!existantFile.IsNothing) { diff --git a/ColorzCore/Preprocessor/Directives/IncludeExternalDirective.cs b/ColorzCore/Preprocessor/Directives/IncludeExternalDirective.cs index cb82495..fa663c7 100644 --- a/ColorzCore/Preprocessor/Directives/IncludeExternalDirective.cs +++ b/ColorzCore/Preprocessor/Directives/IncludeExternalDirective.cs @@ -18,13 +18,13 @@ class IncludeExternalDirective : IDirective public int? MaxParams { get { return null; } } public bool RequireInclusion { get { return true; } } - public IncludeFileSearcher FileSearcher { get; set; } + public IncludeFileSearcher FileSearcher { get; set; } = new IncludeFileSearcher(); public Maybe Execute(EAParser parse, Token self, IList parameters, MergeableGenerator tokens) { ExecTimer.Timer.AddTimingPoint(ExecTimer.KEY_GENERIC); - Maybe validFile = FileSearcher.FindFile(Path.GetDirectoryName(self.FileName), IOUtility.GetToolFileName(parameters[0].ToString())); + Maybe validFile = FileSearcher.FindFile(Path.GetDirectoryName(self.FileName), IOUtility.GetToolFileName(parameters[0].ToString()!)); if (validFile.IsNothing) { @@ -38,7 +38,7 @@ public Maybe Execute(EAParser parse, Token self, IList pa System.Diagnostics.Process p = new System.Diagnostics.Process(); p.StartInfo.RedirectStandardError = true; // Redirect the output stream of the child process. - p.StartInfo.WorkingDirectory = Path.GetDirectoryName(self.FileName); + p.StartInfo.WorkingDirectory = Path.GetDirectoryName(self.FileName)!; p.StartInfo.UseShellExecute = false; p.StartInfo.RedirectStandardOutput = true; p.StartInfo.CreateNoWindow = true; @@ -76,7 +76,7 @@ public Maybe Execute(EAParser parse, Token self, IList pa parse.Error(self.Location, Encoding.ASCII.GetString(output.Skip(7).ToArray())); } - ExecTimer.Timer.AddTimingPoint(parameters[0].ToString().ToLower()); + ExecTimer.Timer.AddTimingPoint(parameters[0].ToString()!.ToLower()); return new Just(new DataNode(parse.CurrentOffset, output)); } diff --git a/ColorzCore/Preprocessor/Directives/IncludeToolEventDirective.cs b/ColorzCore/Preprocessor/Directives/IncludeToolEventDirective.cs index 14111a4..7311d42 100644 --- a/ColorzCore/Preprocessor/Directives/IncludeToolEventDirective.cs +++ b/ColorzCore/Preprocessor/Directives/IncludeToolEventDirective.cs @@ -18,13 +18,13 @@ class IncludeToolEventDirective : IDirective public int? MaxParams { get { return null; } } public bool RequireInclusion { get { return true; } } - public IncludeFileSearcher FileSearcher { get; set; } + public IncludeFileSearcher FileSearcher { get; set; } = new IncludeFileSearcher(); public Maybe Execute(EAParser parse, Token self, IList parameters, MergeableGenerator tokens) { ExecTimer.Timer.AddTimingPoint(ExecTimer.KEY_GENERIC); - Maybe validFile = FileSearcher.FindFile(Path.GetDirectoryName(self.FileName), IOUtility.GetToolFileName(parameters[0].ToString())); + Maybe validFile = FileSearcher.FindFile(Path.GetDirectoryName(self.FileName), IOUtility.GetToolFileName(parameters[0].ToString()!)); if (validFile.IsNothing) { @@ -37,7 +37,7 @@ public Maybe Execute(EAParser parse, Token self, IList pa System.Diagnostics.Process p = new System.Diagnostics.Process(); p.StartInfo.RedirectStandardError = true; // Redirect the output stream of the child process. - p.StartInfo.WorkingDirectory = Path.GetDirectoryName(self.FileName); + p.StartInfo.WorkingDirectory = Path.GetDirectoryName(self.FileName)!; p.StartInfo.UseShellExecute = false; p.StartInfo.RedirectStandardOutput = true; p.StartInfo.CreateNoWindow = true; @@ -78,7 +78,7 @@ public Maybe Execute(EAParser parse, Token self, IList pa tokens.PrependEnumerator(t.Tokenize(outputBytes, Path.GetFileName(self.FileName) + ", Line " + self.LineNumber + "; " + parameters[0].ToString()).GetEnumerator()); } - ExecTimer.Timer.AddTimingPoint(parameters[0].ToString().ToLower()); + ExecTimer.Timer.AddTimingPoint(parameters[0].ToString()!.ToLower()); return new Nothing(); } diff --git a/ColorzCore/Preprocessor/Directives/UndefineDirective.cs b/ColorzCore/Preprocessor/Directives/UndefineDirective.cs index 287fa26..6eb93ef 100644 --- a/ColorzCore/Preprocessor/Directives/UndefineDirective.cs +++ b/ColorzCore/Preprocessor/Directives/UndefineDirective.cs @@ -22,9 +22,9 @@ public Maybe Execute(EAParser p, Token self, IList parame { foreach (IParamNode parm in parameters) { - string s = parm.ToString(); + string s = parm.ToString()!; if (p.Definitions.ContainsKey(s)) - p.Definitions.Remove(parm.ToString()); + p.Definitions.Remove(s); else p.Warning(parm.MyLocation, "Undefining non-existant definition: " + s); } diff --git a/ColorzCore/Program.cs b/ColorzCore/Program.cs index 000c8af..f0aa8a5 100644 --- a/ColorzCore/Program.cs +++ b/ColorzCore/Program.cs @@ -77,7 +77,7 @@ static int Main(string[] args) Stream inStream = Console.OpenStandardInput(); string inFileName = "stdin"; - IOutput output = null; + IOutput? output = null; string outFileName = "none"; string ldsFileName = "none"; diff --git a/ColorzCore/Properties/AssemblyInfo.cs b/ColorzCore/Properties/AssemblyInfo.cs deleted file mode 100644 index 4694981..0000000 --- a/ColorzCore/Properties/AssemblyInfo.cs +++ /dev/null @@ -1,36 +0,0 @@ -using System.Reflection; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; - -// General Information about an assembly is controlled through the following -// set of attributes. Change these attribute values to modify the information -// associated with an assembly. -[assembly: AssemblyTitle("ColorzCore")] -[assembly: AssemblyDescription("")] -[assembly: AssemblyConfiguration("")] -[assembly: AssemblyCompany("")] -[assembly: AssemblyProduct("ColorzCore")] -[assembly: AssemblyCopyright("Copyright © 2017")] -[assembly: AssemblyTrademark("")] -[assembly: AssemblyCulture("")] - -// Setting ComVisible to false makes the types in this assembly not visible -// to COM components. If you need to access a type in this assembly from -// COM, set the ComVisible attribute to true on that type. -[assembly: ComVisible(false)] - -// The following GUID is for the ID of the typelib if this project is exposed to COM -[assembly: Guid("b98f7ccf-9caa-406e-88d7-2040fa99f631")] - -// Version information for an assembly consists of the following four values: -// -// Major Version -// Minor Version -// Build Number -// Revision -// -// You can specify all the values or you can default the Build and Revision Numbers -// by using the '*' as shown below: -// [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("0.0.2.*")] -[assembly: AssemblyFileVersion("0.0.1.0")] diff --git a/ColorzCore/Raws/Raw.cs b/ColorzCore/Raws/Raw.cs index d8069d0..7f53fb6 100644 --- a/ColorzCore/Raws/Raw.cs +++ b/ColorzCore/Raws/Raw.cs @@ -21,7 +21,7 @@ class Raw private readonly int unitSize; private readonly byte[] baseUnit; - private readonly byte[] endUnit; // Note: nullable + private readonly byte[]? endUnit; // TODO: fixed mask? @@ -34,7 +34,7 @@ public Raw(string name, int length, short code, int offsetMod, HashSet g this.parameters = varParams; this.repeatable = repeatable; - + unitSize = length; baseUnit = new byte[(unitSize + 7) / 8]; @@ -43,32 +43,32 @@ public Raw(string name, int length, short code, int offsetMod, HashSet g if (code != 0) baseUnit.SetBits(0, 16, code); - foreach (var fp in fixedParams) + foreach (var fp in fixedParams) baseUnit.SetBits(fp.position, fp.size, fp.value); // Build end unit, if needed - if (!terminatingList.IsNothing) + if (!terminatingList.IsNothing) { int terminator = terminatingList.FromJust; if (parameters.Count == 0) return; - + endUnit = (byte[])baseUnit.Clone(); endUnit.SetBits(parameters[0].Position, parameters[0].Length, terminator); // force repeatable to be true if this is terminating list - this.repeatable = true; + this.repeatable = true; } } - public int UnitCount(int paramCount) + public int UnitCount(int paramCount) { if (parameters.Count == 0) return 1; - return paramCount / parameters.Count; + return paramCount / parameters.Count; } public int LengthBits(int paramCount) @@ -77,7 +77,7 @@ public int LengthBits(int paramCount) if (endUnit != null) count++; - + return count * unitSize; } @@ -98,13 +98,13 @@ public bool Fits(IList arguments) if (arguments.Count != parameters.Count * unitCount) return false; - for (int i = 0; i < unitCount; ++i) - { - for (int j = 0; j < parameters.Count; ++j) - { + for (int i = 0; i < unitCount; ++i) + { + for (int j = 0; j < parameters.Count; ++j) + { if (!parameters[j].Fits(arguments[i * parameters.Count + j])) - return false; - } + return false; + } } return true; @@ -117,24 +117,24 @@ public byte[] GetBytes(IList arguments) var count = UnitCount(arguments.Count); - for (int i = 0; i < count; ++i) - { - var unit = (byte[])baseUnit.Clone(); - + for (int i = 0; i < count; ++i) + { + var unit = (byte[])baseUnit.Clone(); + for (int j = 0; j < parameters.Count; ++j) { parameters[j].Set(unit, arguments[i * parameters.Count + j]); } - result.SetBits(i * unitSize, unitSize, unit); + result.SetBits(i * unitSize, unitSize, unit); } if (endUnit != null) result.SetBits(count * unitSize, unitSize, endUnit); return result; - } - + } + public struct FixedParam { public int position; diff --git a/ColorzCore/Raws/RawReader.cs b/ColorzCore/Raws/RawReader.cs index 10cf0a7..2454237 100644 --- a/ColorzCore/Raws/RawReader.cs +++ b/ColorzCore/Raws/RawReader.cs @@ -82,7 +82,7 @@ public static IEnumerable ParseAllRaws(FileStream fs) { while (!reader.EndOfStream) { - Raw raw = null; + Raw? raw = null; try { @@ -102,7 +102,7 @@ private static Raw ParseRaw(FileLineReader source) { // Since the writer of the raws is expected to know what they're doing, I'm going to be a lot more lax with error messages and graceful failure. - string rawLine; + string? rawLine; do { @@ -179,7 +179,7 @@ private static Raw ParseRaw(FileLineReader source) if (!char.IsWhiteSpace((char)next)) break; - string line = source.ReadLine(); + string? line = source.ReadLine(); if (string.IsNullOrEmpty(line) || line.Trim().Length == 0) continue; @@ -347,7 +347,7 @@ private static Dictionary ParseFlags(string flagStr) name = withoutDash; } - if (FLAG_ALIAS_MAP.TryGetValue(name, out string realName)) + if (FLAG_ALIAS_MAP.TryGetValue(name, out string? realName)) { name = realName; } @@ -379,9 +379,9 @@ public FileLineReader(FileStream input) LineNumber = 0; } - public string ReadLine() + public string? ReadLine() { - string line = reader.ReadLine(); + string? line = reader.ReadLine(); if (line != null) LineNumber = LineNumber + 1; From 54bb84deb62f6ab2ac61ee077dda76b7487ee4ab Mon Sep 17 00:00:00 2001 From: Stan H Date: Sun, 31 Jan 2021 12:10:08 +0100 Subject: [PATCH 02/59] remove build output --- .../EA Standard library/Convo Helpers.txt | 36 -- .../EA Standard library/FE8 Definitions.txt | 476 ------------------ ColorzCore/bin/Debug/ColorzCore.exe.config | 6 - ColorzCore/bin/Release/ColorzCore.exe | Bin 117760 -> 0 bytes 4 files changed, 518 deletions(-) delete mode 100644 ColorzCore/EA Standard library/Convo Helpers.txt delete mode 100644 ColorzCore/EA Standard library/FE8 Definitions.txt delete mode 100644 ColorzCore/bin/Debug/ColorzCore.exe.config delete mode 100644 ColorzCore/bin/Release/ColorzCore.exe diff --git a/ColorzCore/EA Standard library/Convo Helpers.txt b/ColorzCore/EA Standard library/Convo Helpers.txt deleted file mode 100644 index fa836a1..0000000 --- a/ColorzCore/EA Standard library/Convo Helpers.txt +++ /dev/null @@ -1,36 +0,0 @@ - -//Show text with background and return to map -#ifdef _FE6_ -#define Text(text) "TEX1 text; REMA" -#define Text(background,text) "FADI 0x10; HIDEMAP; BACG background; FADU 0x10; SHOWMAP; TEX1 text; REMA" -#endif - -#ifdef _FE7_ -#define Text(text) "TEX1 text; REMA" -#define Text(background,text) "FADI 0x10; HIDEMAP; BACG background; FADU 0x10; SHOWMAP; TEX1 text; REMA" -#define ClearBrownBox "_ASM0x42 0x83181" -#endif - -#ifdef _FE8_ -#define Text(text) "TEXTSTART; TEXTSHOW text; TEXTEND; REMA" -#define Text(background,text) "_SETVAL 2 background; _SETVAL 3 text; CALL $9EE310" -#define SetBackground(background) "SVAL 2 background; CALL $9EE2E8" //EVBIT 0x8 = fade in? -#define ClearBackground "CALL 0x9EE2C4" -#define ClearBackgroundSmooth "FADI 0x10; ClearBackground" -//#define SetBackground(background) "SETVAL 0x2 background; EVBIT_CHECK 0x8; REMOVEPORTRAITS; BACG 0xFFFF 0x0 0x0" -#define CenterTutorialTextBox "_SETVAL 0xB 0xFFFFFFFF" -#define FlashWhite "FAWI 0x20; STAL 0x10; FAWU 0x20" -#define FlashBlack "FADI 0x20; STAL 0x10; FADU 0x20" -#define ShowCG(BGIndex) "SETVAL 0x2 BGIndex; REMOVEPORTRAITS; BACG 0xFFFF;" -#endif - - -//Smooth changing to CG -#ifdef _FE7_ -#define ChangeToCutScene(cutscene) "FADICG 0x10; HIDEMAP; SHCG cutscene; FADUCG 0x10; SHOWMAP" -#endif - -//Shows text on a scroll -#ifdef _FE7_ -#define ScrollText(textID) "TEX6 7 [0,0] textID; _ASM0x42 $83181; _0x89" -#endif diff --git a/ColorzCore/EA Standard library/FE8 Definitions.txt b/ColorzCore/EA Standard library/FE8 Definitions.txt deleted file mode 100644 index 01faeac..0000000 --- a/ColorzCore/EA Standard library/FE8 Definitions.txt +++ /dev/null @@ -1,476 +0,0 @@ -//Characters -#define Eirika 0x01 -#define Seth 0x02 -#define Gilliam 0x03 -#define Franz 0x04 -#define Moulder 0x05 -#define Vanessa 0x06 -#define Ross 0x07 -#define Neimi 0x08 -#define Colm 0x09 -#define Garcia 0x0A -#define Innes 0x0B -#define Lute 0x0C -#define Natasha 0x0D -#define Cormag 0x0E -#define Ephraim 0x0F -#define Forde 0x10 -#define Kyle 0x11 -#define Amelia 0x12 -#define Artur 0x13 -#define Gerik 0x14 -#define Tethys 0x15 -#define Marisa 0x16 -#define Saleh 0x17 -#define Ewan 0x18 -#define LArachel 0x19 -#define Dozla 0x1A -#define Rennac 0x1C -#define Duessel 0x1D -#define Myrrh 0x1E -#define Knoll 0x1F -#define Joshua 0x20 -#define Syrene 0x21 -#define Tana 0x22 -#define LyonCC 0x23 -#define OrsonCC 0x24 -#define GlenCC 0x25 -#define SelenaCC 0x26 -#define ValterCC 0x27 -#define RievCC 0x28 -#define CaellachCC 0x29 -#define FadoCC 0x2A -#define IsmaireCC 0x2B -#define HaydenCC 0x2C -#define LyonSummon 0x3B -#define KnollSummon 0x3E -#define EvanSummon 0x3F -#define Lyon_Ch17 0x40 -#define Morva_Ch20 0x41 -#define Orson_Ch5x 0x42 -#define Valter_Ch15_ 0x43 -#define Valter_Ch15 0x43 -#define Selena_Ch10B_and_13B 0x44 -#define Valter_Prologue 0x45 -#define Breguet 0x46 -#define Bone 0x47 -#define Bazba 0x48 -#define Entombed_boss 0x49 -#define Saar 0x4A -#define Novala 0x4B -#define Murray 0x4C -#define Tirado 0x4D -#define Binks 0x4E -#define Pablo 0x4F -#define Macdaire 0x50 -#define Maelduin_boss 0x50 -#define Aias 0x51 -#define Carlyle 0x52 -#define Caellach 0x53 -#define Pablo_2 0x54 -#define Gorgon_boss 0x56 -#define Riev 0x57 -#define Gheb 0x5A -#define Beran 0x5B -#define Cyclops_boss 0x5C -#define Wight_boss 0x5D -#define Deathgoyle_boss 0x5E -#define Bandit 0x66 -#define ONeill 0x68 -#define Glen 0x69 -#define Zonta 0x6A -#define Vigarde 0x6B -#define Lyon_Final 0x6C -#define Orson_Boss 0x6D -#define Fomortiis 0xBE -#define Fado 0xC5 -#define Hayden 0xC7 -#define Mansel 0xC8 -#define Klimt 0xC9 -#define Dara 0xCA -#define Ismaire 0xCB -#define PegasusMessenger 0xCC -#define RiverFolkChild_F 0xF4 -#define RiverFolk_F 0xF5 -#define RiverFolk 0xF6 -#define RiverFolk_F_2 0xF7 -#define RiverFolkChild 0xF8 -#define RenaisCivilianChild_F 0xF9 -#define RenaisCivilian 0xFA -#define RenaisCivilian_F 0xFB -#define OldCivilian 0xFC -#define Wall 0xFE -#define Snag 0xFF - -//Classes -#define EphraimLord 0x01 -#define EirikaLord 0x02 -#define EphraimMasterLord 0x03 -#define EirikaMasterLord 0x04 -#define Cavalier 0x05 -#define Cavalier_F 0x06 -#define Paladin 0x07 -#define Paladin_F 0x08 -#define Knight 0x09 -#define Knight_F 0x0A -#define General 0x0B -#define General_F 0x0C -#define Thief 0x0D -#define Manakete 0x0E -#define Mercenary 0x0F -#define Mercenary_F 0x10 -#define Hero 0x11 -#define Hero_F 0x12 -#define Myrmidon 0x13 -#define Myrmidon_F 0x14 -#define Swordmaster 0x15 -#define Swordmaster_F 0x16 -#define Assassin 0x17 -#define Assassin_F 0x18 -#define Archer 0x19 -#define Archer_F 0x1A -#define Sniper 0x1B -#define Sniper_F 0x1C -#define Ranger 0x1D -#define Ranger_F 0x1E -#define WyvernRider 0x1F -#define WyvernRider_F 0x20 -#define WyvernLord 0x21 -#define WyvernLord_F 0x22 -#define WyvernKnight 0x23 -#define WyvernKnight_F 0x24 -#define Mage 0x25 -#define Mage_F 0x26 -#define Sage 0x27 -#define Sage_F 0x28 -#define MageKnight 0x29 -#define MageKnight_F 0x2A -#define Bishop 0x2B -#define Bishop_F 0x2C -#define Shaman 0x2D -#define Shaman_F 0x2E -#define Druid 0x2F -#define Druid_F 0x30 -#define Summoner 0x31 -#define Summoner_F 0x32 -#define Rogue 0x33 -#define GorgonEgg 0x34 -#define GreatKnight 0x35 -#define GreatKnight_F 0x36 -#define Recruit_2 0x37 -#define Journeyman_3 0x38 -#define Pupil_3 0x39 -#define Recruit_3 0x3A -#define Manakete_2 0x3B -#define Manakete_2_F 0x3C -#define Journeyman_1 0x3D -#define Pupil_1 0x3E -#define Fighter 0x3F -#define Warrior 0x40 -#define Brigand 0x41 -#define Pirate 0x42 -#define Berserker 0x43 -#define Monk 0x44 -#define Priest 0x45 -#define Bard 0x46 -#define Recruit_1 0x47 -#define PegasusKnight 0x48 -#define FalcoKnight 0x49 -#define Cleric 0x4A -#define Troubadour 0x4B -#define Valkyrie 0x4C -#define Dancer 0x4D -#define Soldier 0x4E -#define Necromancer 0x4F -#define Fleet 0x50 -#define GhostFighter 0x51 -#define Revenant 0x52 -#define Entombed 0x53 -#define Bonewalker 0x54 -#define Bonewalker_Bow 0x55 -#define Wight 0x56 -#define Wight_Bow 0x57 -#define Bael 0x58 -#define ElderBael 0x59 -#define Cyclops 0x5A -#define Mauthedoog 0x5B -#define Gwyllgi 0x5C -#define Tarvos 0x5D -#define Maelduin 0x5E -#define Mogall 0x5F -#define ArchMogall 0x60 -#define Gorgon 0x61 -#define GorgonEgg_2 0x62 -#define Gargoyle 0x63 -#define Deathgoyle 0x64 -#define DracoZombie 0x65 -#define DemonKing 0x66 -#define ArcheronBallista 0x67 -#define ArcheronIronBallista 0x68 -#define ArcheronKillerBallista 0x69 -#define EmptyBallista 0x6A -#define EmptyIronBallista 0x6B -#define EmptyKillerBallista 0x6C -#define Civilian 0x6D -#define Civilian_F 0x6E -#define Civilian_2 0x6F -#define Civilian_F_2 0x70 -#define Civilian_3 0x71 -#define Civilian_F_3 0x72 -#define Peer 0x73 -#define Queen 0x74 -#define Prince 0x75 -#define Queen_2 0x76 -#define FallenPrince 0x78 -#define Tent 0x79 -#define Pontifex 0x7A -#define FallenPeer 0x7B -#define Cyclops_2 0x7C -#define ElderBael_2 0x7D -#define Journeyman_2 0x7E -#define Pupil_2 0x7F - -//Items -#define IronSword 0x01 -#define SlimSword 0x02 -#define SteelSword 0x03 -#define SilverSword 0x04 -#define IronBlade 0x05 -#define SteelBlade 0x06 -#define SilverBlade 0x07 -#define PoisonSword 0x08 -#define Rapier 0x09 -#define ManiKatti 0x0A -#define BraveSword 0x0B -#define Shamshir 0x0C -#define KillingEdge 0x0D -#define Armourslayer 0x0E -#define Armorslayer 0x0E -#define Wyrmslayer 0x0F -#define LightBrand 0x10 -#define Lightbrand 0x10 -#define Runesword 0x11 -#define Lancereaver 0x12 -#define Longsword 0x13 -#define Zanbato 0x13 -#define Zanbatou 0x13 -#define IronLance 0x14 -#define SlimLance 0x15 -#define SteelLance 0x16 -#define SilverLance 0x17 -#define PoisonLance 0x18 -#define ToxicLance 0x18 -#define ToxinLance 0x18 -#define BraveLance 0x19 -#define KillerLance 0x1A -#define Horseslayer 0x1B -#define Ridersbane 0x1B -#define Horsekiller 0x1B -#define Javelin 0x1C -#define Spear 0x1D -#define Axereaver 0x1E -#define IronAxe 0x1F -#define SteelAxe 0x20 -#define SilverAxe 0x21 -#define PoisonAxe 0x22 -#define BraveAxe 0x23 -#define KillerAxe 0x24 -#define Halberd 0x25 -#define Hammer 0x26 -#define DevilAxe 0x27 -#define HandAxe 0x28 -#define Tomahawk 0x29 -#define Swordreaver 0x2A -#define Swordslayer 0x2B -#define Hatchet 0x2C -#define IronBow 0x2D -#define SteelBow 0x2E -#define SilverBow 0x2F -#define PoisonBow 0x30 -#define KillerBow 0x31 -#define BraveBow 0x32 -#define ShortBow 0x33 -#define Longbow 0x34 -#define Ballista 0x35 -#define IronBallista 0x36 -#define KillerBallista 0x37 -#define Fire 0x38 -#define Thunder 0x39 -#define Elfire 0x3A -#define Bolting 0x3B -#define Fimbulvetr 0x3C -#define Forblaze 0x3D -#define Excalibur 0x3E -#define Lightning 0x3F -#define Shine 0x40 -#define Divine 0x41 -#define Purge 0x42 -#define Aura 0x43 -#define Luce 0x44 -#define Flux 0x45 -#define Luna 0x46 -#define Nosferatu 0x47 -#define Eclipse 0x48 -#define Fenrir 0x49 -#define Gleipnir 0x4A -#define Heal 0x4B -#define Mend 0x4C -#define Recover 0x4D -#define Physic 0x4E -#define Fortify 0x4F -#define Restore 0x50 -#define Silence 0x51 -#define Sleep 0x52 -#define Berserk 0x53 -#define Warp 0x54 -#define Rescue 0x55 -#define TorchStaff 0x56 -#define Hammerne 0x57 -#define Unlock 0x58 -#define Barrier 0x59 -#define DragonAxe 0x5A -#define AngelicRobe 0x5B -#define EnergyRing 0x5C -#define SecretBook 0x5D -#define Speedwings 0x5E -#define GoddessIcon 0x5F -#define Dragonshield 0x60 -#define Dracoshield 0x60 -#define Talisman 0x61 -#define Boots 0x62 -#define BodyRing 0x63 -#define HeroCrest 0x64 -#define HerosCrest 0x64 -#define KnightCrest 0x65 -#define KnightsCrest 0x65 -#define OrionBolt 0x66 -#define OrionsBolt 0x66 -#define ElysianWhip 0x67 -#define GuidingRing 0x68 -#define ChestKey 0x69 -#define DoorKey 0x6A -#define Lockpick 0x6B -#define Vulnerary 0x6C -#define Elixir 0x6D -#define PureWater 0x6E -#define Antidote 0x6F -#define Antitoxin 0x6F -#define Torch 0x70 -#define TorchItem 0x70 -#define DelphiShield 0x71 -#define MemberCard 0x72 -#define SilverCard 0x73 -#define WhiteGem 0x74 -#define BlueGem 0x75 -#define RedGem 0x76 -#define Gold 0x77 -#define Reginleif 0x78 -#define ChestKey_5 0x79 -#define Mine 0x7A -#define LightRune 0x7B -#define HoplonShield 0x7C -#define FillasMight 0x7D -#define NinissGrace 0x7E -#define ThorsIre 0x7F -#define SetsLitany 0x80 -#define ShadowKiller 0x81 -#define Shadowkiller 0x81 -#define BrightLance 0x82 -#define FiendCleaver 0x83 -#define Fiendcleaver 0x83 -#define BeaconBow 0x84 -#define Sieglind 0x85 -#define Sieglinde 0x85 -#define BattleAxe 0x86 -#define Ivaldi 0x87 -#define MasterProof 0x88 -#define MasterSeal 0x88 -#define MetissTome 0x89 -#define HeavenSeal 0x8A -#define SharpClaw 0x8B -#define Latona 0x8C -#define DragonSpear 0x8D -#define Vidofnir 0x8E -#define Naglfar 0x8F -#define WretchedAir 0x90 -#define Audomra 0x91 -#define Audhulma 0x91 -#define Siegmund 0x92 -#define Garm 0x93 -#define Nidhogg 0x94 -#define HeavySpear 0x95 -#define ShortSpear 0x96 -#define ConquerorsProof 0x97 -#define OceanSeal 0x97 -#define MoonBracelet 0x98 -#define LunarBracelet 0x98 -#define SunBracelet 0x99 -#define SolarBracelet 0x99 -#define _1G 0x9A -#define _5G 0x9B -#define _10G 0x9C -#define _50G 0x9D -#define _100G 0x9E -#define _3000G 0x9F -#define _5000G 0xA0 -#define WindSword 0xA1 -#define Vulnerary_2 0xA2 -#define Greennote 0xA3 -#define Rednote 0xA4 -#define Dance 0xA5 -#define Nightmare 0xA6 -#define NightmareStaff 0xA6 -#define DemonShard 0xA7 -#define DemonStone 0xA7 -#define DemonLight 0xA8 -#define Ravager 0xA9 -#define HolyDragonStone 0xAA -#define DivineDragonStone 0xAA -#define DragonStone 0xAA -#define Dragonstone 0xAA -#define DemonSurge 0xAB -#define Shadowshot 0xAC -#define RottenClaw 0xAD -#define FetidClaw 0xAE -#define PoisonClaw 0xAF -#define LongPoisonClaw 0xB0 -#define LethalTalon 0xB0 -#define FireFang 0xB1 -#define HellFang 0xB2 -#define EvilEye 0xB3 -#define BloodyEye 0xB4 -#define CrimsonEye 0xB4 -#define Stone 0xB5 -#define Aircalibur 0xB6 -#define JunaFruit 0xB7 -#define _150G 0xB8 -#define _200G 0xB9 -#define BlackGem 0xBA -#define GoldGem 0xBB - -#define r0 0x0 -#define r1 0x1 -#define r2 0x2 -#define r3 0x3 -#define r4 0x4 -#define r5 0x5 -#define r6 0x6 -#define r7 0x7 -#define r8 0x8 -#define r9 0x9 -#define rA 0xA -#define r10 0xA -#define rB 0xB -#define r11 0xB -#define rC 0xC -#define r12 0xC -#define rD 0xD -#define r13 0xD -#define QP 0xD //QueuePointer - -#define ActiveUnit (-1) // 0xFFFF -#define UnitAtCoordsSlotB (-2) // 0xFFFE -#define UnitInSlot2 (-3) // 0xFFFD - -#define CurrentUnit ActiveUnit -#define UnitAtCoords UnitAtCoordsSlotB diff --git a/ColorzCore/bin/Debug/ColorzCore.exe.config b/ColorzCore/bin/Debug/ColorzCore.exe.config deleted file mode 100644 index d740e88..0000000 --- a/ColorzCore/bin/Debug/ColorzCore.exe.config +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/ColorzCore/bin/Release/ColorzCore.exe b/ColorzCore/bin/Release/ColorzCore.exe deleted file mode 100644 index e10082fc3022ea2530d4007cfe4b84c402bbc796..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 117760 zcmce<37i~7^#@+lGut!Mdt_%Pnc2;;vpII@%&|#!N!VmL5=gkheQpv$5^e&dp(i3_ z=w%TQL~ihk0*V(P0t)y8R1j2D6cGeOz!gCd6ckXr3IFf+RrgHKBw#-O|L611C(~8+ z>eZ`PuU@@+b#(9kE3Z^hrBn>R_ufv& zwa`y$6!o3oM$~~4H`EWBd|)`}F;olE7mP>L)G+*i#e1r(3uXf8y+dXCHy(f1Mx-Zx z&uhm3-p~%{Px9lDZfaVPdZ0HYw8~V3csm1;7@uxaKB(DlQ3#~`C!rMJmOrme1 z{rpGX4ArDwe?qBGH%A~lLs=oQ3et8wCXb?2EU(gY4Yk_GH_9`Wvb;Dl{3tpVQRsl> zB^kAQ5j8R%c!?nZ`wXC$4A3hK&~OIm@R-p9$_cL-(R_cILK)MwDm7(plZXeDQa;(3 z*TynYuY*CXsV!!se$z(HrU!Y&##gs=#I{UY3JG+?63%}jfMXoQnD{Rf_evt`-G?w0 zYi)~XlI1>d?4p*_olQWgf)dUS$Pm~h+eUO;8Es7-lrUzr#l4h-rZ)j$`9c)5Ob4wt z?b$4q=c9m!4Z>!H0&|K;rke_xjwZXrsHqF2OhE*+AJEe8#@jlYwq!tlii2ia&kOP1 zjB_rCH^&?5aQ6Y<_8j1aYGa>lRa@MzE&jo^8AgSIY>S)m@*=Qa{xoIV3XoCZ)43dJ z_i)r2lc$ZrZIv$p(3yu=7-%z#)qaqTY zGS^i7TJ2Ni6%?$EicR-y_D~uzEmCTmGk_{v3%hkh?LoxSK98pRMc}4l&W-@OckB3k z#O(zHKnG6GWSmtX?OX?XeJH*YGD`PA^wBLD)ZvR>x5hLo(&-g;wR2QZtUQD=&EQgP=_19eUTR4d(sY|_`k!ni^?)N7R<7;i~6 z72`-s=O>(S?Uw`E1$!I0(<2a& zCS>mh62b+lSd4h-TF4=>CDT=vgYE#()*WC}sQiwDx=I_XO`jJu-5NzpV#SFXZF0)W zIGB!1=WO7opdlQ*j$*=Z^TYLR_AyQdhF{Z}h`LCh?koX+*?buM-2e=|QW5ZPs9v?- z8CtKD`nIxusCqYvzTKggy`#;lp^}l)&(`{8sINtKeJvv9cFNqC;^YN;mC{mkt|aR) zwwCiT#-<@=6$++OG7gzrcvvFdt@lPtZa!*u|*A_~U$; zD(?lL6?L)*nhSH|&O}675)Leh>FkYou}voo2ZjPEC?(90ncx^I^u;x6KOVQ@MKi8h zGg*h`tyS&3kLGR4+_?HKMjQ-y>Vu5kzU0H52AQ->Y;|WNOviN4WG_b^Y@Ry^sPhpt zG#z&jBu2ux{N&C#E$#{=Y)B6soJl8*%x3o*LiQ))P1$s^Fom$s5Ox4zE$(%-5}oO` zp$9Or7^$|+83%rVl`Hj6lSdXmCR_1wSqo!s!VI`iO*u9`l)8C?N5Ne=%R@S(41#4BU3ogkzy z9NS}LctB%)`im19bcJo$)&@XDapyP8(;g(aEm5GgyEZ5bdoNbe!Ka>;nh0x($N(^96nLU zNpkoTM0}l!nd&*@Ge3DzNbk|y^7!yqHOoIa=}%7&Kc%Z7!thQkai1=6zrRC*Gx@*Hob!iF>_<-dTliAoOAleV_`Z>onwQ=&!2K(+E9EL*J@G*<(X{ zYUn#v=)r`-w6GQNSitY0gihAb<|>qWJ`~f?t}1jlLaD)^kgGyr1k}(o8roZh?n>xG z8alfQwF$jdL+4kaLgAAdy08jO6PMydZA+@qcd04sHT1wLbPjP3!^2EX$ZqzH&(N_% zLpq%p8VRDYRI)fb6?2be*+Xavno_Y$`B21CO*$;EfkkK?>Sd9beHe0zeTcv$TbII^ zM6&XLG*_f%VbVs4!qnDZyfd@obR$Hi3P@SIJ(&^tEP6g{IpCZ-V&w&Fb zTllb7pj7kFEk4X%iDY>dDQTxDD<1aHCCJZ?aS|Sa)2PH>%8ExmggHA%qx)h*XiJbr zdtyVFC9A%gIdgVYpS)lx28n}DejDM6*c{FWgiPF=}f96m2r@3 z_NH4gTADL%Gcd$egj1M-Bb?E0iKauvS~wYP0>5+$o#V)hj=O7EatZn8dclFWZaT-a zmh8}ZRE(^10#ltn{B#bVNTPeZvId$S##_KC>9vA5GL3#1E>lUSmwqiaiM2Ov8q z3&LdgY(bc!D*_qQFYK%bf_Q*8_D*%rC6;{(K*AOnfOxD$<@eru4~@&BaarBCZMEin z6xPN3=8;WZAzVnafvK84#hp`8Nzz#l@jw+$Ln75$GT>9?Q)&8kOU}Uq|89WT=OcU) zAt)*v4J7;zNW-60TYNaBMEEQC?FZNZhG#G0_(h;RYJUu4c*SZx!WM-5NBEscUyE=r z47vc`trzn7H2my8300*sVySWim%({tcU&d1&ARaB&@G*AGS0BeZI*DdyF|h%?(Gsz zb^jpYG#{_qy*NSqT++QwM=bYY9cgy2NlI3W`wbmwbyr#fX>+mdz)}<3pXo@u`=*Yh z+*6w+tHZrgN7C+Ki$Es2`|HRgcU(s@Zbz$RW!=x}NT>U#j&!+uv`N-v*U^zF?&CT# z)%}f*Omp{{AUNIbY8}bB*Xc-)`wbnL?!KiXGu-!dWTv~kT}sVzPt%cJ_uo3w=Ppf2 z)(-Apb;Ncf9Riu{o}wdp_aYrBxXjL}f%Cy#bA_@lp_Q$8Kp9&`umc0TPhkao`p-Li|A zQ^v!X=yeleqL*V)=o8mJm9OQvC?X0yBz1+c&*%=r-F&4sC0iJgg9i`ik= zA4UYhSqPl78IRal7+R(0u-=Vq$;_^E+C&IQ0o z+85&Kd=!srzUk83%i64TLP~GW=+p>O+$qqheYHT+@oLF9DB~xVGCSg21geEuVXl%= zlP;%AWd#d6Q$1+E=0-%J8^> z`pbx!W1^MVM41lyd4x0ka3;g6eQk|;%r;^kbB!jCnMPa8n?L?QG}T-5wZab8-woX zEmhfh<#w9o3H*S7tq|;l02V8M9EjyhtJt8{M`{Q$kx6JXtTm3=XdH9VIA*eO7!w?H z)Kd%_ZsDh58~Y(p#{!=Q_eHTuL#$O+rrEg+6}A+565J)UTG2v~Gvgy>QcHwwjGH6P z3BB*|%(#^(Vn+hAkM(?96YI}bOKCuSMMld;?@3Ho#r^f3#3fo**}-g7bJRhOgqJM_ zC75Ar!MwB2bE5G~)RM^x_c5`}GN5u?rYN)jeLG@BB_W@`+($Y9z? z_o~a_=tsP{NHrBF7!^h`x%MXExh?Kofwn=xP3|LE%{o6``5Ca{l}F|Ib36sx-za)A zyitVho2yW;v<{K05os|hWXHLNz(^i^utMirBsXGnbBZ?)Nu#2l#LPIKK%N7gg=|6q zXfJkw*daz%T9()HJyy-VM$7lia8>XhqM`IE$cpCzu-ZgUVUG%Yn6SO9xfnCEBNpnT zj8^6ckPEO$X-T=5f+ zN=nCCVYZr4?P-k?Z_T7*g*lo!wwx*LWHemc%2dqf2jkXE#o*@S=poF`LniP=Rys)9 zSEFBq>-fMGzDy^ZXk!RTg9X#f?~1*{EhvM%!)GAS{FJ##bxBOARa9McVUTrkivSU+4&kGC^5fECOTP!Er5CwVPUtR4D6HNLCxh`Nj;rP z7xF>I`<5+#-N)sEQd`4JDh^#-IH^GwK?>51p^O6SriR zR{fBALVE^;GFaZyG-n3Vehh&SOj+}K+!+~c7-Ssy9k3hrU7%d2^g(Uk*t#}p-`(UX zTR5O2Yk!N;&Qzz&JgiN}5IhP=jG#8#-Nf2_cyg7jVz*gq*8V@X@n+PT-I7^WZDT6N z?SYf9q;J;E)nZPyrrOFdTy&}-HPd7Irbp~es~Cys%5*Fd#F~<$ivm3>4#u@PiQ~v@(h*?2Baw=EoZt~@ z@+e+J;@-~@NhQkng0*~dbhujnY(MdNWQd`X6TbV9D&=rO_ici|gGUhmF5;qX*iTG~ zQ5*d&i1n7TeWT-uY%#HQZ-Img#j07Q)ipy zu4*=A4JS`c;2iQG3PAZdX=HZ&JA0@q-yYxYV|E<+C3514%{Z+-5STcDX?7p1id&zx zw2FOC$I9KP2{u2s$Zi6I#WRytii>=Tu=#PE!o*T>91soemWpFDuB5RCn5Y&g1qBkR z1h(KfU!)S);bWxSBlQ%zP{KA*Q~3vAt(?=)Xfhoe-5Fv?#X3yqhsY_$Qf4}a@rKjM zhmqFPFgE38!J1Nsxp#r#B>!?V$2fDk|3M?iI03oMoruxMx|0~AS#)PJwt%s{7^97H zS2IS-?4H6Hd%~?SM$K|BVvIUc{t;N{b#FjgPjEk$^qowL$<^r}G0hqro&E#UqGL>t zv4QTpfQ`|ZOB>?q9ls61`r;=-xv2}IV*`ZK8|6ll={zFfNd%v$!N=FYGYLM2;FzX7ah?jA#pWm-OLHOS1Aw67W|R1f zb9asN9MNpS?gB=>Rh*&WH7Lx0H7}Ye!~=|$c$D{7oC8oSkF_I~%=%Kmlwe!(vtS$B zlA4mgr`YNKTA*F#jlGlIe=%;K0@BXcL8dS@Ovil}xu)J1WNsM-$jakvH9a>X4t@&l z?oeAMEr$W8UUN<4pkW<=H#v_1+bZm5I=?^!*G>%Z*AkW69|`Aiz={)fN@qDw0MrK# z;!ac05_fR2Iz6E|z;4XEo7B>H*~31NWB(bs6KR?-n!;p zG$q8nl24QC0cH_HvCxR@6h5kTV3A@V#F zPq~9s#w2Y?|K?GWy6ki!uLf=zU)`rOT8(7r_RwBa)agml`r0TTFZ!Dy6Y5Chqq^(f zglKAn^W_&%bv}`o@u$>Ls&k_%48j+?8tcgn^akrmm`L75`2&NP_XdL|?@tKwD@Rcr zS~rne*NVp&l{0T)-zB4mFKleozmrM5yM`SD#k7i@b(ZQBG(?Z5Sy3PBMPOkM$4|XP z3H=@qZglvT?CtHLmkQ{SLa17-R^b1%*gvjaA4*lOenKsi3|7KG>DveXuAfQ&5fV^=)KQ zZS=H_0OSxh~dfqR@1 zK^r)}?n0PBKN$9LXnsrvi~TZ086K_9dFToqPs*LQ04TqWM{5~(M!1m`H$6DtSk`2{zacD~k&0Ickai+Yp!Rim zqL7MjG5cs@Aq8uqgG?*=z6EgU!gN5(>PzN&oWC>T6T~DM$D(LNs`O!DGUCYY=3iaBVo_I`)py}%Q>{N!l^`|C31 zv>JKAb3OL^{vcGWbGfVN%>g$xvIX2D&a9{Lwqz<#C);pGC>0CbbYDG8p^X?sWp4yp}Y?$B&q`jrx(1xtv&F8 zTN0u1pleUm#zQiIX-RGWMA&`a)!Fr=v{dS zvG-+6=cWg%6vH9T$x$Abn9kjJ#GNF1h0AA$mJZuaI7V=q5%d-2N0Wf#Dx=9wZ|t4J z3|05eaf!<;w*ap<9rKuxicNG{kxn{ocox4eCq1E>95}|-zy)DQJ0NC-T#4U+!+Rch zn{ZE-0+x59P6`>*29duj;kOR)Whz+*( z_QQSraJe5I#*jAQH1rLOPAb}ih%xU#qjCtcibvs0@o+{ubXeP&C?siSF#At2-3K6y z2@&^qlt~)xlNE@K?l$pa59#E=DXYTiB9jOXB@~S9$B@pSL;!$6)&Pk>Zwy8KnF`*r zmSJB5qRoYN=*Wu@z#13;I`gn5TuH#gHrJG1())5zbqUJxpD!$<{1Ftku@_*KC*k2y z2u3->=z$W=GI|iBG4EgoQI7&MGWOMMi_d&Um^Tb=gPI;CY?QG^vpfpfu)7eouO)ty zhy5I6{vWITBp&(AhJ6J>twUU_S*6tIzCiLJsLg^NZ7}T11?_R9OU+QQ&mgH!v0cNy zjCnEHI%vq$(TWMv%*UFj#yNYhi>YVu7bNLcr!ujm2ux( zK&Qu~j@}k$nn*(W5;p*_FJU=3JfcFl)aDc6=FF7AAN$axPp*(M{1<3mrv6du;=D`K zq~1#?m7^SX7C9Im=4JG0%)SLUrN84am(BO6-#)ESMa5hwk?l{x0>rZR7f{l}dLxfvHe3i&*madJb#xiPf7Gl z%YJEa_+p4mgNDC~=+GZcrf+?3_l?7tDiNJ~&a0p3-=s?qW1y>{9|_8T3ctRaRPI0j z3%ARC`a};w!=ERWFA2)o@qdo@PfGO6yL-lO`_D}DJa)M8m;Mz)7a=q5#;5s?(4vP} z!WH0zDsU!iS}nk?D)0;f_ZHyvD)4Uv4hnEy6=)g&b_;Mx71&8&Qh@tcfq4S|)KT@V5dyr3(BHRt`gt2=K#I;88IE?*a=GWBNC-z9GkZEZ zQYRWY;s1KYK^T7@$U6iYo@O9saS3T&|^Rg31geReDeoOgA>Ny^NPU>S`Sv)I8J^SQa9%cRhj5dIFt`;r6LA-nGCPQz~Y9n3WJ_Jj|gI zVyco5J_EuEvqiN-6{fJ|c1Tbo6NlGT87`%_iRE4z5swI$!rN9MeMTu(x#%Q}J;XH| z6>_zLyXKjBvyBR;N#qHuP2v8xp&HI?Jci)JY~7jed(<)M?Bq7{Z^?b27C#9a^S`>@ z==nerS5--b1c^btPaqM*_p1}g(9zW*GQd0MwuX2pZDY6}@xd&>9aZI(sDn!j)5AQ3 zgT{=9nOZ`41_&zz*KAqm;BXig?oKegDq!Bn*@6nFB}sw`}6IyuC74X51`nX*#w*2UZp0&n4k$nvYOoDv|bi-oQ1 ziF*?|M`LN*UjT8j{d^&n?tsqJgv)p<4ev)7xbCtGVs*{XAl(eS0PzQM)bblnI?~+i_Vl@Xff!XelfV`99MI&3JZ2bD<*yn>rZdQYztt zN}%Brh?k~@@TT=o38lLGTSKYRXW{DgOUbGiI<3Dwgt6#E2rrbLhC6A+X};SQn$Hh` z2t-r`oH>b$Y08@B5wvkS+qeemj2nm^95!6&61NTRWh~oVIJLYEc`XUk!z3H?+v8!@ zrJvYTGUKvopS8b;9!OM(Q-PxZhq$LtC-rlSxt1el4qDUO>JcNE*kT@29Yx%np^U2y zEp~+x*?wekfx2)*eS@=u)dp3_a9;cOBWTwQvwrvO)bC%P8;RcAl04bJj;2z2@rqBj86Ln}d(YsaFo8!4jE0)3G;rqYp z`}hG*or2#2{D$tw!<5dUrwvC|R^+Dawv^|gvV71-<)x@!gPQPuxbK%2v8simO@RlP zSpR$;$ri&*#JF;+7TQ6tlS~;l{yB6VohD5OuqOLUATbJGMjw&F1Hv45JbP#WO!ba2 z$HDL@V67Tp!{Z5KpHRbQj>blFd3WGvQKe>8<<(c-5k%1~@@8eG#dPK)U2HX-1qdBj z6bmOR{%A;R=T{Y%aMXXU!iNJwhf~0Xsva&}o=;z=Z(XUq;KaEkm!P z%d*WwFC&OA`YoOzf>T#4gSnLBODMOZ-@!2agPT^D4(G9_8`FdhVd17wHCF?Ew@U9( z)@dwrt4cGie|YP0dkMyE2?TSRNaBJG1rE3^a*LN&Ufu@nGC* zEWY6MF{peQX4-Og#lvT5=n1g86?@ou-zL@m8NM|MsA6kg)5W^ftPwAqThGl|K{=?a z7;Is^4ky?qcK-}qY!Le$Bn=O}X$J6NsH*-_N8MZ?q!SBYL%Vz$xz53rO~uwg`xpIO zXOT|BhE;JIt~2Qgt_(Lm)S@Bma`0|pto#p(zADCo7gW+Uh|LJSe(o!52%r-OC!$mt zV?62Es7tl@8G{1kULND*u(2Mbfcm4Oh_Xp_*B z`{&T8DlRho)M6H|>B@1aGn~VbCUUPW{QT@CC|KSd51B*C5J@kjTsSf6>_L*uwqy)O z^3Q0{r#D8hW!bZh>EGm&b?CaVVsBhG0&&U zcuyO1SgzSapJBrRwi7Qj?TfcvmLXD21tt z@T9XB5}>pET-eJHF&TsOsJ4CaO;bGCw|x^P%u$s zv%-c^UX;(ru*R_=AH&9MY%ZLQWXR+d!&_U~K}-lDAm7*1h_XNByM0tM%j{Yqw@oma zDr`2YkKiZklib7=%dn-QFmNSf9YX$Ssl{3!g+wrZi%G_A0#m^>8{(^TF`V_lWX7D` zVO2v8DqqPmi>0j{+4~@a5T5o$xzsDH3y^7$Y+O_@O9KI@CPU1i;qncRaAE3JoM_aR!qRr^?1GpB_jAAw)1`wOanDedDY$!kG zLyIR@b!(y!Sg(uXVn<=kxsRX9pcPfGAlJ1RgB}aE%Rt13%Z&9f(hEN(DoA9=LB>VZ z5tIV@9#z8Gk8C`I62&6k79**;9C)a>(-w2Si?DV@qEM_Gy|G^0xszE})6?t1YPaBh zr!d!pvvo2UuhCqm{@9ZKhEf?Xahjrn_&AZjlP_(SwsB9tIA0S;2V2Kr{BrQ;&x+3e zXcFR=*0G$;jk_7-**O4I^fI015YgHRh<|V*`w<@XMteP(bB>wlc+xw?-N$-!%|zHL>r7u4=TtwbF~>bGoE@%j)h zu+BAk5)XBXjl`Wp5u4YIE2@l0-&-Sd@Q&W$NU{fUWoWX0Nfn<&s(r1gd?fOCP0pVW zThz{b>w38kpwgD4-b2w(r5RhBjl-a7vj2s%G2-}JB2ixBhjDjCK)$rM>8wE5ymj#% z`k<+mdr;6eff29Vi>Rn;ymDUvE?*h;xHK zh;v^*h;utXh;tV|h;#Ekh;z?Ah;yqxh;xTt$H(>sgXN=8E#AoQ;-w`-tRT`d*@v_S zk+vy5WI_;W4EZvVN;I*1uU-p)Uim#9H@y|nrYbI6O7}Ckq%PF3&sg};*D9n5J{R#L2nLq`hU@E zpmHLq1cg|&W#)gbD~dU+wgi%7B33>LDd%K7f^7296yg%Sx>} zD=Or`FHgc?Z$BLVGRI%2MIB@$*f??(Y(CK8kLLN1JB`%_-J6|&qS5@WD}3;;dt@z9ZLk(j44kPVGXf( z#4QmV-IEDSWsi~IAY;crt`{8FOD3{2ia@7>g zUX+_?xSh}nnR;|`YBh-wur}a9A~hbKA`iUZd#UCDc9J~6B$5YMY~JsMMa1_wquKtZ zXeY#&%}*|$%C110?7xygx1kqIsAwH4y3Md(0(=yf63#kaMS?lSZVSHJWR_1u$=)&M z@D53#c2{BwGCV2HcMnF6Ww7erDm&uJs$W+ zn9UR>FMr;6Jv?s&^HhuKoVC5QBHK;t2w#z--ZAI_=;pEPh(4pj@bZ)ME8QOGgJJ&> z9M?9z^9R6TmI8w3U>KV4GHx?QG5hGQ!*qnM!*}+~^6Aob%;8llQ(i%b$%KuF%60bv z_Sa@$a<3JWOAO071|DW3ECv~kN=ckKe3k^y0Ktj;9uXF>#Gyu&ZSv5TYsRpzz)RH| zKKQbR{UYkbC?X8)JD%;+HV?C?8f1S|2($_d^d}nq;^K74iY2=>{36iIn@$-lVCvn7 zp!kq3SH2z`Zo)kRL-m0Ej?&-%1iHL|YK^xy!*%!&3dq~5_#CY1-}}Y;h;6FnZ zZvdyTKA1dWRiUvAuA7X)O6q>ZRDyoInPS)d$Q%bQ1cz_s1WFU(i41GHL@;D6P~Fdf zaG$q2E?4RCf(_{T@CPlBi}Qn1`lA=lS%~%FJ_+IcjFXjJF(+x}in@?&JBKm{zH8=I zH;g~(*7nvJ@*Dp@)Jcinq9uxH5QfiPOG^rOoBhYV@tq+9i!$J0NgpPR8M+a^0OX5X zmNtz}1;zCD4{Aaf_=Ukattu4Q(_vDF{IQK5{tU-STK*?9X!4lJPf@HVlGO!YTHpEK zsn#Dxg}8b@+&6r7wTD*V!#L7G=b$!O3bhm_CY;G=MXRZoF2OzDG+?FQhcEO)qAV<{ zt!?#UHFMxIW^Q&}FX}u0JE@){wcY!-QDMVm3JWnDs=)a^#)Qt&P@}>|%NmXwpvubuviuzFMD884-Q4hN!Z6YBtP5BMluAD z%OK_vgZd^Vp!4(v|BY}6`esy!D>+T>3*}8vQ78CT57WC-d{_#nC?T>6x5Gm2-y@<`vjEzBU<*b33?8SDJZmpIY^bC zL>A{ij4^!HgF@0;(~C*(DQZe+($iJwd=DCgNm8C$%%>YLx0t^oWQ+rM0`-gQiF5>& z;$QhARK8qt6xqVJi-&=9MsN-{kDD*Z5{X~4avD=0`u4s%24dD)fr++viFob z=Z1_N=K`o>wE#^Z%;mR&)r)39ku{iDGy z!)CK_B52rHqM0|^7f}G9N1Ob0gjjp{KAFK!FI)~#JaJ8HGPm3O#mU^P70X%xfuo=n zhki_le?eXVL+Sp3!op;3l|FIE0zdpq<|-?eMFBKIzXBY*qx|_lsl)156Hey2iZ-Kx z_GUcT{aF5OSk|)CjHqbXtvdHMg0(!?qc!~J9)iBo+H?VUbZ@FA zFJyAOn*1n}#%%2CU5swAv1Br3+g+E%8YAZBk*kw9tcbbCRFfY=GURc86-n5vn+T=v zi<{W=H&gCeDA9vNBC0v9XZKMkHA{PfwZfiycl?1=gU`C!COo%13Q?_-=87C#N>t`snzrrpODMEyH!A_zNAK#v6zEQc7;sn-C52Rc)TbMQ~0Rj>~{RG;&2n?ywZ-5&RbzniPh1UUM z*w_XzOSukp&i}y}*q=g15iir})TfcMKZA#1!{s*JkDxcr%C~?hqr@-VWN!w_1jGI; z3-9XVUJhKqJ_ndgJwco^k8H*S0M0%FaIUM?)Z{Fo=+89D*8}EJn1*}{+M^h??r42_ z-j@YuD62O^fnAJdVQYaOt@p@lhsw2-KTkSrB0hsg%xnnr7O)y%wr+*l8N$4;hA9%- zGU*b&45Ud)Z~B%0poW-BZbfo(h$O$FA5?BNVve!Hs7BxP!#f#rUsWP~#>A+79(soN zCZe)uy_k9ds{+5nbTBY+0@P_b-H0^D^{3jCaoLxto%FmA-m%lWoceuk16Lkq4e0f& zR0ZEU@E4at?jKkeCLpW$|E@0X4>#+H>f7isOk>|=5c3{DkUzoyyGs-aUQePRe1(9V z?xK9~2Seokk0+^M(v8x8FzEroS2CS1f)_4j!E@2}?#S>y(Jo$`k}YI)DF+}BwyV`TJ)s_`=w*x zad)&i);-VxJ?maI66;>gKSL!Lne|qzX@sA-Y|gY6CSqPoj$E}N*}XEi+nQt6TFLB4 zYcf9YYIc1S0onB{Vhpi!eZm^cY>TazWMa))i=ULt7*TvU66eEG{8-$%_P-n%A9y+7 z3zl8;)frK0k8SZK7(P>0th54r;l@~Uq%D@b>T|*n=Bj}=nz~mPoA4+^x|_4{Tn^l= zz!y2qk?e{q=oa+ul{n#vHBTH;2WrD-K380oSo6f%Cl0}1Y>5qB2I*w+?rQhS72P&$ zkEIF481z1oZt6bvi8O?QX8A-@O+M8c2ya)%u96a|CZR0&*=h;1R-y`>pHisOQ*} zDWw8onY*0=2Na{mck5$iSV60N4<6pVcwpJ+9|zqRfQ)l6>@59VBz&>@JH*12NNfiq z(aP`XC+;T%gR%kgJwhM}`f?m2DUK1L$zvS+*&09bAn`D3Jw!6!#{&zEUjm`m^2h); zh;6}HlqvrJ01U++BFH28uulGn$%hQP9oqx;PmmaWgc&@3`6(h>%%9;|ew45==Gb7R zfIqYL7~*5nt}}6w1h&k8Nxg@bBqo0#4Qu9LQS=L-hE{lz+!t3D9tYfc0*~@9iLoF6 zJz19`LM`tprY4tv45i&Px%{+FNzfg0e#K-yR>a%r7{%P7wx1ygR_)geY4s&?dC+0c z0_;79hf(1(>P(05oac#*9wNy34Ie!K=U)j!(twnDeaJX)vh&*f)NhgM!yS&S{Q_b< z553W_e}^c#P16(%9gj+F*vVrdPS-CXg72pN9)a^R9+%#T`O$s_F(?GrZF@r>_|K;$ z{B6z6VA8L@FezMVA&%El$(H7bFNZ=G{r~_}<5gCI{+ zk(Gg&=kja-Z|7FWpZ7hPu-*hU?;*}sJRdQ5?kD@}Muqvf^!VQu*#$5!gVXSFuK)I% zX13oLPpR=X9A|;|=M5X}G@s8$rb3o@JW6Jw1-XoIxU&-ZPXJcBw+Y-aIS?@t+XC*gl zjUQv=G^URy>he=92YeM@Xo)nCkZKL#2p{?&V>+Q(#WxK1hA{*l8P8bb*cyWNn9rN} zC6HqbZQ{Qy_yVH%0SqwUN>YW=dn=!;8MLDH!s9gJd>Q|1n3Y zJ*)kZ+cMZ`RHz0v6#_SnGg@L+A)b-Hg~pFa{{Zud8E4;!fq-I~FlXpCbL^)8 z!qmu8FQ)gDpzpsR6yf6xX+k6dYmWW9nDY)QZUeyZsy;mSP`9u;=l7aMqksMmo@cLtTt=dsWLp{}b_S|8jKw zMs%vwWK@0yie+L@q z@MiyqA*}{NzaY(g!ik$#y_GHPV-c4l;dbcC( z?(WK1s{7GUBg$AA#P?H^)&=pCm2rxW^IiSxqDt*6{Yajhp2V77|i<1po(Qfgf=?2>qVxTpX8-SD69d6r1yh0WLRa z7WRXvU&cV0%&yaFfG7Y$V(Q{Jp+d4Cq*AMwaMmYv1_^B@%!TPntEZ1v5FrGIy029 zKZbUqcEi_i=f8D3;RaMY42z_csoOclX9@;8r!m8ggd4y?=r^*kW6+eTL8)E+fl}xS zvZe)DGySaaAlT(wc;7}F>?{X2w>#wTr_fIG-?dTw3;PBlCt;p8TN(lDX5o`2dXQ?_ zGfz5O^58dU9nb~p3&Pm5k6AbE{VIgZt9C-$dm6r)y%!F|-esyp2p7AG5G*m;uX%<05{*I{?e4+6?(p zTeS9BDeS1ocZB$b8f>fNIG5CB_J5+BZ@B-BDBDNhw>GyEpvKzKNw^RG}6S=S) z#QPrvEqaPyicWMPf2lGd-x_ifXoL4(;Be$KqtIuZ!?c}54jfqTU(C!(9_IFe0MEy= zfK__bu*s4JZJBgCnxOS1Pew&wc(zG>x!?kA>rV*ykjl1XSf)Cwvr0Nl3@?Qwrsy@s zPAI>BwQai3DYIIiOUtlrlPS+e&FphFUp}8C{RzQ_mmTjS)0h2wh-%$xQV*e?1Ed}z zpvw*Hu{HDNceZ~tlLzNg5P!fN@S!nacVJ*EB;@mye)JH`Dt`1(MC}`xj#r>Bm^4_j z_NC#()?m&_a*5M}y`}tFC6=Im-9iAp1BafZG(k;8uBGm_9zMPNsT&qY!va71z-qg+ z{MhcA{G?qXKYm7v@?+CIJ`j&E&Sr`%hbT*UNHG9>t%fAHmydC&4njRRzD^*M2g`=B z#oQGI^ue;_5pK#IK%YEY0%^u-#eJv2{?FtMq+8M{w|7<}MMbrnQ64ie^ z@#`Fk;WO4~4;V4#4Si5q7vt!CzFC#m<)ZR&0-OPV00(Y+!GBLk>rkBxtWf)Q56OV= zKnBeroIOG^AUu%4v(e0)c^}Bde&8aiZ*@+GmC+afKbY~;LyR}{T}z0kJwtpG9`M~7 z!r3dtH{k)_e&5Lp7}3eP0s8H%1KEQ%IF;{&{0|X-v&v}$pKT!?_6_k#c);g`5YF-t zpM(c|itg?KmEpfY&eS^jibZraE}4r8ND_Z`x|ZOHSqO2(_GrxCs?GV}4jL#{HLIcB znNXYX7kR5=i}0Y`Sh82gAO6&Bb!-tHwEGo9e^Evcut~`IU!x4>r~_!Sc1|<$;`Whn zt9=+yXtWpDvaa%bWT;Jq2W{#K;owj7R@+2)&?c_xCn7E9(<{-RzCD@>SfyPE^z<3> zVHN${V(uTRju=68(*#?au%klN5gx8?GSZxE#V$ct1MBf7O|5D0#s-gaRh=QnZCka1LmaI{|te6_90wUbdA%PqXcQ z5*&qdNLwzt=oda0RZSRtzd2-?62=>R_1*iGc66M*+YBNUkP+0in$EQaFe0s2zbMvUGnAruD=;K+0KFi0n!xlo%s?C{ebz%_b3I# zcVe6^6hGjsdU>PQ0(tL@DCJFmb^xIzbrQkf%Y{RQD-m1o=8l3SHWfSuAfWeirz>M- zuxBtyP09y*1~CPVQioP)!{^-vRkXyVOSvYw6+d`3+$QMLD^OuFQ8^mUk6vBwPOjkr zGh+xlOtAk1>6LOa*g5z(G7|Dmkbg@WR?sS*g%PqVkn3{Rqq)-mn*JU_|Gz@NP{7xC z1LA4f+#P0iR&ucrKacfeDI{EKR9T~9^-9U0Pg%S$AG^yOGWc>jsc~F|-aGY#W~Wbc zd~VP>zhNY}D9eps5i@AAz1nN9Q10;B!~4H`LU8LR;W-myY_3FB-(;=3`C;3E`WM>0OIM>^( zx~m>fYyf!@8$h|^3>Rl0WJ}APa;eE(8Y*(Xz?VBk>S9vB8M%%{V0@a=)Zr z?)w4m%l!aEb-%~d3h2_2Qa=$O_k$5&jg>UB!?0Ld*3807c+l*yut&=6u!t>j8+%8l z{#x)oh`~D}qU^MF4j!DK**=5-4h)wsqKXl9N*J@)G9zv_Y@r374VQT)F!j1j9l>OX zSY1WW*B7eeJ_WuLnQ#@ngI1#yZqnUol2L&+#1iE*AW{b#&6cqZ9=!d513ja#M9&zJ z&Spk1&Q_tnN2>DH{muTU^*4(d(%*askmHP9;_-nvmas98;zS%qC+1Oph#$^4xs<-Y zntk72{se-tBLIYZAs+?+?9+se5N1U;(c4&sxcvExsQowU_DaDwDn~KoOfQj5hKr>P zJ`W`00yn%YeNYm$uVY=ceU#NG=IhxSQb2oYUpRhONV|yuJzI;wU(0vPfR=)BdBe&( znnbc5K_W^J&_~d$D$lrrM#6TGWs(4?l0d&d8{MZ zS{sf<(r?4@%t2lCdqG6-_kue8bDpn*&Nlag;_5CKF!VckWjR>k6A)ltYyw@{@n7Iq z#8`V4Kg{BsAJ0dCJqBS!J_+ZYh%yQNjuQ5k_=l5vuqBHPO#I;_yeEcSe1HEi^(T!p4do!&H}&ahjQ%&`_Rfla`&q7ApS&azTczcM@PmZcI~jRH?F8v+@a7O8q4{~e)bp;C=KRdG zO#XV;RcLnxw4n^LQ7%%_sKNZq-oRWc8x*}yT7lOL|4G=9q8pi62ekCVaT6+pa`@eLQL-VEdE}$k;@q8Q1rr$ zAHA2XXlRWqzZdbhRk5&d=)gt>U#(|wvd`dnhtHjMKE6hM#@X|Z;2NA`ACBKqP!;@I z@Z-N=-x;WizpzhI>l3;i(v5-J!A`7>;iIbKLXTmNx7rvMDl&#|0XrPS96BB!GKRka zC=YK>AxYXQiJV|WYj30lW(tEaY^F{{34e%hVBz}l&D@6ZeJ|y)?Y&;~h*~Y}U^z6` zK@jqLdy&5$KaA@u5kSmRdqUoO^G*XY#tT8%*X6#jpL-f4EF)UGuBQ_Zt}$~9eMW`p z>Isq%=?sX-!LLx!STZWGIm|X-RIsG(h^{2xI=d%&j)H0*oRVI(1)zA+J|u8 zVcUqU`;Y>y^`TIaKI9vG>_dvj<3sxJyMV$De-FXf835+nqQ{49aTy@Lqdvr32$1VW z>f;)zmk5|o&P3qPCv#;!k%GC2ac3MxF)nZ4*%PZvxQ!&6^@t=h3?nkbFawov&f*-i zwpxd78+oX;P3n-gJ><8IY{)x+(zYM7ZMAvj!^rg8a<;BU=9T%V$C7%8K)svW%!@T$ zyh&i?vD@8*lip?=|KeKpqagXb+${JxW0c3%>#~9}O1dAZ*E3)z*H!7GD>tDU=qJZ7 z$1Hu&i}8C5KjBep$Hz;WOaY4+G7>e>DR*m#Z?MS8zh6D>Id&Z6cHW^p32YY3cYh#SJWGo z8ayVpn7>Azyg94inf>PI>7alY%aALq)LtodaQu%!$I?-H4m?LUp{n!9qVqY_;XH%k!W%FA zfE9J0|8PxU;+Y5}o>9?$WYq@V`6%oUybH--%_lux`ic3dfDD1QSF;;@crRT=g zpC`zhy0E!K0N0*<$&LRptFk*IUB!w@cSozL)cgQFn-^{?;rRLVep@Y-$FQ{ zoA67Sn4|Ff8h)?iSAx>tg5R`+QWxU)7=FhkmAVtZck$cBg5QeYZOux(h~MYn6nv@` z-%)5&>Rl1N^pe$xvSU`uUrD<%j|kP z8kET82lv*QS#0rN0i-!Y;Ud`){U?p*3{H>hg4q@F9vdnm$<7g7)tA9pA~$FIh~;a< zMz%$41jICXeE=wp*@HF6U%f6)PvoqAESm|hVL*i_fuYHomH7t%Yxq}?ZX!2h#WDcZ z!|0Bu0pRaSbY0X>KaOr~M;aT}>Irn=;|SB}LgJ;^rH>1hMfzjF82tPQhq@M>c^5MF zL}w;~7Y8#S^;rP1HdINFPXsRxo{tjq0eA@E(0RZxuoAxt{2G;O3hiKaDYEKzX*a__T)NpJMT8+g-f7q4)@k=aSo9K9_7D zkQD-$A-9wNoYg?!Y1kG6p8fWgZ*D06NtTzPSG5>$7HUSOW-(H*5ws}ULL9wyxoujS z|79G8Nhb%}yuA%*X=uPxFtSDtw>-C(Kx+emYe}FY*0w^^yGCBB8~vC}=`}(0#9#xo z1hl+X$cd7J%R1O(z!n1T8d=5TdH<^uG4FFAW#qPL6_{mXZDxjyLawczE#Dv;6(Z|p zvlhH1?ZxKd_O^FIJ>j7*(OTs&DQ+);_67pS3W1L8Cy;6&FhdBWx1T^q1A*VcMmNqE zqaSVPu)4m|-Nc)B+toYaj!D;7{(c>!`O%g3!~2I)!R8iGN(^Weco4ljXhZ_iGuucuL@Lua$zZF1aJs^>UUPibhktvKHK zcbF=-#q_R=e)@$>VvT?-SWg2ewT z^VKk~zwao~UatK0rO^u;*vu`i8?zz%%5UBakC)L1BT?f~=B~8$z7i~64&(+(p_iZ}XvI-j`q%?YQg9LI*>SPd7<0Sr;jtJs95kxX1h&mMvbfr-Zd$x}| zEie7vvD?&zSBi1J+A2X~NzjK+3gE>!RwT0Y%*xVp=q57fj%}Z^=_Y&@bhkNC7!_uP zm23L@w1IdbKc~KPJ&S+T?^#?2xW8v{1EOHJ;p~hE{@!}tg3#Xj z$3g!E{IG91k-9s{69$aR=YgkR8v8P0P=&7`7C+t_4jNV=bF$Tmn?{U-q~=}P^bXpza0>4n%L{~fImh4M5ASR42tn}|hcNh$ zd;KCcX&Lu5rG>%1COMJ)tpkC(zSs4#gCGhf{=amP7MyMS#2b&3N5z1d4+t z5fw*jg(`d4Yyvi+gFE`yU$47}{Ugjth3p@xIOrc%Ed8?w+5u`<0ufvH59L?uAE6@s z!;dYoe<%u%55#X{oUNBWNdhC(C-(sE_sM;TZrvwD@cX38KND|bgSK^M7F9{;9}c*{ zE6brt-$oERlh_+=IRU?K;D;`H9s#64+=Pu?XR~;uQTYy1{L3>|sfaPnlu@~#usMM7 z0|M+17Qw*$Y>fbLiKP)beS~q(CvPNdj~!kljqi~LI-Eob^Ne$twoAyte%@Afbp@p4 z-3%7-3-pskdi#BeXg8u; z>jM!Q#%KVI;4Pe5 zU*m)ZV~C}7U$ZZ2eJxbw{^$;Xv#)K09%Y6&-p#Q*@`reyaW+r-x`0UN!xDI@DELMK zd=7Pn)JY&b_8&q6{4Zfu1 zV-+8i^A#3j`x)x?*FJ|Zk9pL4*b<*FffJwj4_D6`VdGr~fAoyyH z&mQ5E@WMhG2M3CDL2bp2X3!vKK^<_Hsv)HCr5VS;#Q=7sdGm0cM9=)6~8xe&_46fjqb@CPAn7SnQtj|DXi?w!w*%*!7#J~Hgm7T_gRW? zBhs^WKzjF6onwfMe^{J!w?>QuPf;|D->KdfMrrd*u zo6El7{d4sQelOs6BYqfTQl2J=_ts|OhpSNdci&XqDpQ>^`H9)4dR3l>PhtF*T_2vE zRyd_srut7;dWNZfI+@Q~<+<||#uM_KF3-6F|3lZEDEmWsULla>lUGkO)tWiWdrb9> zrUNIK>O*nHZ|Pz!kHrbOL*NhRzTCQ8EuGC8&lkv^eP3=h)t3b4x1B82AsjA}+@~^x ze0Yb&B~w*q67#+cnvAO~pQ~r_`S%QQZqM`iiAj8(+|Or1aL$;&yr)e)lG$sPsVQ;(u{b!_RK&Slk9p^vE+xqsEJX7(_ZRu=Nx)h~OQ>Qt>L-L4+Zz16== zP3M3>s&yJuy=t2JJksf&xBB_F?50az5~*cK^-1b03z*8|*1#Aht4C%rHNbD=gUSuP z`|q#}-&9;AsTorfg=P5SAIC|%>hM!bB(iuL|Y~_$#C2`Y}IcL|E=wDR+WOL|ASS$}NN6P4B#2{a)%jNpWG_uKtPCwXogY zg4rZBY*4dAMW@VqqwOhmy4+%IS2@W$UCoUERsEv#jkb63Rq>sK${xsjMr}fh@+?Yy zo*EXK_ep)DYKf#`(=Y5>2B{w?D0B6BqaDN)z08^pdT~gnJQs1}LeUdszQr}SzO6nd^xmSHh{a8|0`FTGPDxU>S zrS4b1l)O8UT87j!lDbb)52$fzZDuB^d{6yedh>ADo@GeA9EqZR$3Tiqy%LE|hlWgG z>Zj_>h=sgg=M<#!GxgU<6f*nbEap9il!4R>GnBelJ%QUOq;i(YyeHKbNv%h%%ha`m+-3fy#R-aRQ42JUQ+n{9#Tt%%3BH-Kat|By-fX8&61QA zW9o0JS5i#|Q-4=GNNR5!PA^mcRI??;>l4fHUebW1)(U1UvPe=K2g{Th*;!IOLbD@s zu%x~zEt(v`H-k|7m|*52Yb14)V9toF3$kZ0t7;V2> z&54{Qd8fbuS*GSiTuHqyd@hP?lGIX*c}pV~gi`xOE)CUpMC95~Y9#V0!ORJtDT05N zAB;?tyg%S*dYSrJWSXQ-5y}4|GBa9h?URw&l84D3m`_KFlG;UT|8-;_YO3|KKLNjO z#Y{dg?&tHcJfCgdeBRf?=dFcLrA>9OJdZ3~pPr{yx8DND{c~Pvji@}$cszZ?*ez{ynvNLT)WYXS1rQ&uIl^5gz0Kx`knN)a&MkZOkL{8iHzUd9-lZ}+5Hpn{9T4+pPfqh>2tGy zoP_5B^#E0eiOMyHfPg@{YOZ+zy|8{FP%32fHibA1pVoLo! zHwRPKypA2W`Fwrax54@HG~4+6+=bxb)9KyutfcqCvu)yS=>=-bZ1S8*9Rv!uq*ma$ zF0~GjJ9_^g;@$+ls^a_~erE2yA%UzAvOp3N2p}Yc1QHfSnveuc4GAPc)KZd5as#(y zP};*!TN9bM8iB-?smE zfzSP(d6s!*=9%@(oO9wr_^LcN@cG=;iCAlj|Nm$5^>K=l;~D=Zwf0!t*2Dt(Y{GWr zZRB*$+nJ%)BUabAu(7E1V+Kt+3}eFij^Id7=(VSj_o z-%a}`TS#C#zm~jBO|TEoqAL=)y_LkAh?z!1r~lK^oJJ86K8je==o7(Lf$(8^Fb^L^ z+lOD&ePs;eHG;zh_Y40{!5;``wV+QpCnOvf%dw71_+`Q02%ZweCt@M}TEcG#ekAy@ zV3J6GEO?b*q43is{BsExA*|?zQBlBq1Roj|i||;fsgr2uib|wad^n8TMlLXm9vwOi zI*%u&4KJh(v6=ACn?r9vjbA64xzduCW!J|ly0~a-Yy!Pqv@;g>mgw~n^!-eDhAszY z(VoN)VE@Y8y=)nAvnU&!EIL2&BsiC)a_VOaxu&j0Z7TXWn=O0*ROHU$a1&6`D~?>` z=foh^`BBsqgntz^efTtriT>8`2wFLsQ{=KlNc&NAIJ*pa{v$>PMZ<@3PB@$;;epca z%$Wi^(}0Td22~(@PF@u-C-)-YUAi84wS9%8=oy|wyVV*AcO!gKZ2>+PaW&%lOT_Kj zl<48yI}mq25N#OkKZ3>s6+JrX=iomGR8*Do7{V1mMKOc8N1Pb+48loKzXRXI+L-$m z!sq1u4L$ny3@+(Aj$`2L%Rd3UOMeJ`f9jtj%&AAvv79pqzv57MtkN$H<}GdbR#;0G zJeD&UIu3X^g0@cl75HqEEpVPqx<4Hfn-oD=(J8>8#NqWe4*$c!;kZZ+?}*@VjfB^V z{&k{xCXVNkl|^j-PZ)JM|C!LdRCHb*!f`uA^Y>zt#{t|wi~gsBSo1HUb5f))7qK5q z-tN+4A?4AGZB9<&R#BLgM88)vfgbxzNau(*?$UD+zS_P3czN!jfQ5&1F9pXhZGs>E z8PDTt<1qJtelux3gcKD#^CIkw0kWTY^fRsWY;SXJJ+Kz4A8O|n2iLf%_p{@EQ!KW* zX22WL;&|C+s z-4g3&E7rg|y-0nGwCe82gRpaL6!+p^MR9p=9eT$A`De>_gU>$U9R3{kcj*1!p!xVW z@Sn0Qu(hX9znv4F1%3x3JhzhC>J>-%JrS>4v72I;adD-Vw=mE{|dh#^-0-VgGQRL&re&ynlu*iv1|*qVC)n z#(2TsMsxT)gfR|a6&FFTkNYgdOPrQ}{NWYOu;Hu|3&io2c=#w(7~A+dO-Fk_}8 z{ayH$fEWX_KR`GEsA!#xd#izp&Xc*|#Gt03LC>(}v7DWlmBtKW^p~_}Xeggs)W@+& zG(x`~qGRGd8!{`ISkJ~S$iy<9c1KPfZcD~|C)B2t>Ir;9dQhlqQWi~k6nT0{sI8Di zLi4%NgnwIU?c_{~q_VMwipot%j-&%ZT|XCz0_>lQU;@&r7c{Jup^pIn$6LrcbS zylW^Yt0Fm$_VO(SaSFUAvQH?JI_`0&uxxka^JA-%DPz>h`H&*iRPz>iRNO8^rl!Ma? zQk=64^fiuX%9lxFEsCvV(j<#wE16UjNd3{|Oj=@OQod}gAjOK}6vojoevOZ|Msm)^ zV-0Son1TpBp2k>IZ1U5|<0;orDA5a`7R(S$sw(<8IiET#>gK$UlMCo3i+W+=$H`OZ zPK&xX@2|;IX@;C~Y^9GTew|!Mr-j;0Ns|Ys6j3y9nxdWEJS{0@8Xcc)s82>^q)ev; z#fG{iXHv>cT2UgDwE1%>cMelrn=?|*rTkK%@P5FYlvy;kjHwe*b8{nfF`Z*k9l6ew z;(+Y7tV}AVVk1-EA2|oI&{kd+&}1uR^g@W0pp31Q(cus)Wwc}-;omj%k5S)CxqvD! zFw{RMZA|%gSPE6N%E+WdRrEulOs=bFyG3!XgR7e2L~(kT22x-v zmr|jTNf|B;v$8bI%F-|^OT(-zrB6bvEDf`=EX>NXFe^bB=Vck)`EA0#t&zjV?nzlj z_gPf-*t zt*vA$#-BmiZCRPvgWf4JQ-&7WZBd*;3-y|KQij&B6jp_$u!=sZ4yF*4aSE#_Vgbuc z3acpAqBwTVgC{J7)qxu0Hn=*hWvvdR&MB;>tEIsjE33n-Tt*eOk~;ePeJPjG z#X=pTFEXaZtfNk$Hbs7xvod)-Z3t*8T2EISO~sn)=~jz!jCwF-1MM@K^l`?cDI4ig zi#n5YFr}M*Ybd0rsGHsmP``)lBTKVn^qVQ0$X+Koi@a&liIg8uv_;iUIteP#qSWNS zfl9Ney(#w8A5gYMy)rpM|A2CZ+7x+wvI8eFcUZD5S+S{`=^l%^E-M+-gBCSsR90#a zJz>SWaMU6x^J`13(#jLc2lLc0w`naR^)en^j7)V-j#(VG_aOmcDR6;`cm zrHzp7pf4@i%u#bwucY-daBQMyGZv@rln><)ZKC({TT-v4v<69!9xvLOdab-*&9Zxo zZb`k4-f*&P6Wv(!VCrs~waieTjCwKkMrvs^)Z)xPr`}9u%MG<|xMRev^o&I{3?DI~ zmzq`>*~X&05w}r~o2jidH>D+Y52dvjnQMGY>K(L0sI9bO?7R{GL$9|Q&8Sw7-Ex>%^KRGY1^w|=7~Tbk<~ zaX+25s3GwiNBn|(eEf#eUW0AZ{S@87lv%O-k_NA0YHOrt;`R~0q@hA>id;YOdQc;c zOj_N8G{aDI73S#&X{Auc%7bhI{|-5(=YAFwr2a9ccf>2>j~GR0vg{FBV{At5 zo%H00$LaePHNN0^P?rm}2`%~epsqGD>2Z(KDMQf*V~SIspbI=~b}M~0=A97-=thfr zcFge+Ptgk&#a{X~y=hUeqAmTJz7XmVab6Blte34EA}+)8lqi&0WB!KH0@TS7zoG95 zbwY8x7wCsZ7D?ltAMqkxE!3vSgmL$!yht|%;@y|>65T13N$=&b^bXS@k@e7vanoX6 zrH_T$Mek>RJ>oSwV^ME{dYz(uQd&vx4N4Tsr1u7;8<}|NP0AI@r1usrv?wm$J8S~~ z4#gA?kI?T1M#V=`2911|zH2nq*0C|5%)dj9Nkwl{kos9}QcxE2a#m)La%@Q*2dYV| zoKU%wrjI;Itrq3XEFaky)+YL>Q)F8shi5H-Y=hB^dOj^e_t9NOCatHB9_#MYp{4zMpciUxE~SB1m!B->37%%5Hif-#79DYO$!+>FY-RnI5#L$MbiLJV{Rq zbtsbS;bVH*qS&7w(@R2e8FqtuU#Ly=>--4)30<&;vvY_zy-%peqBv)t(v23yHTEgJ zZ&94jPw7jGdM2G^=HH`G|aO(+B(q00Tw2%Ph z;5HGYxJ`UcjaR3o2Kr6H)Amzfqx3TO)hN-2=+MqUQAeMzf3zUU~>rolu8} zbM_@YXRM&6nL4+NQ`k!Ei7)Bf7PTey*Pt#I%Cy!msn^INZ=MjLzoL6A8RzpW+Hc7c z^Iin?n4ysRJD`pk%_z><89HvoR*ohI;<^`8X>Ynk?cWI;|)ce7gDn=iakN9%A&X}k?J;-#8c|Y zkoKw64@Jf;Ql}|(Pk@R_)9SeZm5^pv?;0!O>j?F+MR5ud>K_)xc^QOrKfFRO_fCa2 zKXphwkd&Inr@JgWr0z``l{O?m?M=!_i&I!!;h*U_@oJPsacSdKu27uMDQWSl&d8+f ziR#h~5>IlLs9G$FbC#(3gfg?*aP`lPtjSi2(}t@Iex(EHabHPQGYkcrYd}3_QQTKX zsJDgM8kv~4Fm0rIPbi-Mz5|MF@VLuXtcTSw8?u!k#a7bQDT#MNagWPT4c#1XYvkWD zo6|w?6t}ujYOPSFjz+1=4Ml&P*pim1ZW7A$j!gAa6Hj_arh44cbHP#24p}iwZeI_!~kFwO~LK%OKRuP*B|F%Xxh&mdrVlC>C zluc=)Rgy(b#_D3U8e>uJ>gVB0Mt8zoZyO+aCYk;~oZM@nclqu~* z^}bNo;CZ$mr%hD1{Xp`9csPwz_Zf;(Q}?9hsE;h_(v%0%3Y6L`n)F`U!L%YZDnR`{ zZMvEvlquf~b+3`Z=G$pA)H6bDqLneNBhOV0J#1wYEsAx}EOoz7CsfOr){*C{r!2~u zujpIqEsHuQm8nxg?WXI~4-Wd4x&X^)9wTcr4h||-K8vbIJ~*gEu>>v4!8Y-Dr$w>N zvH-<4%T%w#;}&wiE>m9#bwaUbxysne5lzm@RfR=yym{&(p|(aIjmreJRH#jnAIA*` zwcL_T&C8?hEcl>UeWB(?_H)QY!*fcKSu?IiY&!nViY# z7poISCV9D3$s=s&0pG{$bg7EG!sO+O^mtH7hJt21eMgNI>JV+ix?qW#Vo@LF%uHXR z$}NiLxbLd@LhYu8F|*UZt6D7Tc;dqJ2K6h8`gOi7c9|NtozvS*&11fs-l*y<>Zf@d z(wkL7fZCe=J#{ibU7g;pz6?-5O82U?D<#*+=gvWE)ij~Z4(nxVkx;uM$Bgo; zcGIDxN8&G2=j`OXn7Uo33Wef&zaxE}w0Zx$veBY=UdgcWvq1iNWncQH0JS&iSLr=r z@wNnH4ytDlIgVdQ?D5+vLQATPb9r7)TYQ4vBN?ADWJJd-LHmR z#V*_$`DtMe{X(T1idIj{1hv?rHjn>&&@WWCMctY4Dm|cjh1x{9DVaEVJSEgI6?e{g zplnyOP3{3@pg7Dm7N5_WekwPN8kwIHBAb!ltOA;j#MJeU{c5P#gw^WSb8K z>W^(cq_RxvQEc-eHNm3T=0hsq${E{yNX-t|tgt^Eu);PU38cU_9|>4tn~$jZAvPaT zi$ZKZ5>^k7sV0$KM+F(D(;ri}1k!WRf&pe+C(4W zwCxGyvnUnG)HOmKi@KsvWjvv7GL-tkYAD!D%XmsX z9-tibYxRau#-E2&{4UO?@#i6RKq%wSL+Xf;i3bmbc`ztr4<1s-Ei3H7Lu&l>#wL64 zP?*mSsmUVSO!IAHpn1MU4R%b(cvf8~l=1a*Vey_*i!IGw^|EualQaBuz!eO=2O5tIJ^xl%`QB2cBHK()+pmFU=xF-(SyZ#7 zc|_ly5iUETdIGY$GLEPlgfgCZN9`BNc;a1k(xTWC@2VSb33}pP^$VenMa4|LFXP>? z^1Z7bv1DAnchz%-QrC}v7@9+V94z0vYM4+a^>>xo(=n+ZRr!`Cr+!p@Z7A{Q(XjM_ zGEVQPioDgN!2Ud{7F!hi^Jtitj;eZ*ZKgl#yPz5B&py>*$%ff(&*)RnTGTy54rLru zpIFp6(GL29s`v?~x0znD<4I5Dv#3i3-JbDYAl?W%lyO}B){^z$OLTu!y4S=zh_;0L zFpJs~h3BOL@oW)?GXA7CTe9Rx2feTMS=8jgiat<>E$X$ww`Y71h_?&z{;YCuBmCP; zmm=PWYMn(*#SQw&fMz#JbW-iHWT(+~KMKfhMy@{!$Q0sz9Ok8u)eBa<$;kD`YRv5> zy=AcZ7xi6>dUJ>aPq{Z+)XPZWQ`Kuxi;(M21MxN>z0cGamh3fn>92u!AELCU!csq_ zXpc#+9I2mD%@*|q()(O>SkzCD!r#6y{(tFtsSMYb7l1su-!O8Kev_QM%}57YdII>*wy4}J3=s>Gshuko zs6QDQM!qu{q`wGI4t&lbwEx+3;vHOCGq2cm!Sh6K|{hk5v@;)%(S{_U3O2<6VYLw2+G(K(Yn%Tisz&CRTjmbh}Jg< zWjqlbR=#K}_08CIK9muy_lTycqZoa_vcl~%R)1+xTt~6``N$Ug6m`Ha=I7RC9D)!z|{J(z^Av^5H~Dbg`M4OEMf@p=&7B-c4}=7;IOS`>S5 znEr=QCcR-gWxvTa*I%MuYf+qDqJHL=LB9kk9$yl5pU8}t62tNmlyP1X_4`Isyp*VO z9t?UZG0ZQCdaB4a({Ho`@h-Bc`|S?=0qFG>HE+-u{Go!-o|>%pTe6==ew>`Fj|pWw zpAx2-qCc`UFIA~&Df%;usJ)6(>5i|U9>P0P^gA@5W$Flv z8ag;NZM2THs9%E0)=3tXJR~)3oX)VQ-Js6VV=O8mIyG&gKF6XIrKaVEHKpz##q*tI%TQY8uQ*`0; zCTE=96n)EYm@*^E6#XlqOdFgMmU>XesZY@djHaYMMUVWgvB{}V(PM-%sZY_xmW)%M zs@GZ+r#@BBdLfwlRDH2fCiSUdsRw18`c&OuG$r+^`sWtKsZZ4p31w2Bst;Q-PQ6e^ z|IXx-Q!muTzYnHfs4o@jSk$9a--ax7tr3)Q>V>+=XsUVVOd3_FpR_1Wy-+_Zlu5l% zAGc(jdXY|hF_=22mx8Gm=@~*Di`p_Z2b!U&7wKY4#;F(S1%{Hnpd$SH0IFOzJbkQa@MQUkUp2T>V``Nr}!4 zOZ{B!vSgh4x!NO?v3YKo&GU3mh|Tl#AFOy>qVvLR24!rMze&OG@A20x{+i7D2p?zT z`+ImlhrgJy0Eo8-!5@S-&HS4DCJ*bgPI#CtXuL7V{Qt>ebWdlDNMZiU(QN;6*s;^X zam>$+V}3eP_lF-E$zj7$Vj(G=bzV&j@k`mUnj@Af^4T0;cz^d#~g#5l0)Bc=4_ENcI9)pUGiWX%Q>$A^4mci zzAKGWKL-}<^f1!$^BpMTf5K1C48~+)L&a?-?pwd9$@16%l4{8iHLc1jEX zrZA^#Eb|*m9F9WU*3y6R&WVf$##axD2OblftZB2#-k&q2fbE!`81$PaD=dA4r8YlD zyxrf9Ns-gicxRPu_SZ3aGb#3$8Vjb@`lJmQ{C8p} zQWIGMpu7IU4W&F&_t|2%GlvO{s+1c^au;6qQ;qeA6Rdmv(DLn^0?mMgIrV zevH2H#%`3t-!Dw7Fy1ga&C(YJNvz3I*E6Jy!LW^b#K+q$KF=&BMSjDIbAE%=<2^DC z@O-Cbj#D!C8T4~ZeHk?EFjT)GhOLEiOnLjG>BV1SF4Ghm=2^*RaZD_e$}sTpkxcPh ze@>{>#4@wzUhyH1%o=ZMi4Reyc*X?1Pt!@DqDN%BH99=++vv{{>)WF999o5qUcrcG zx5i$>e?a&wwTsjdCJl{c`qVeYGSVE>yQV_%zo})eH7)G{YY*uop)J|ukWowPwS~3b zP)%;z`WvPH-`I&14;%anZCKMH;e1sTkH64j!X_u*tQm~;@93B|aHsg~enIvyz5yX) z$Qc>mOp5=SGZ*#C@9Z%KM-2WV0CP+m8zmmrXql)fX#GdqrRZhD7bCxS9@Cy|M-q7_ptesTEW zZA!dfUyLuO6;mR9!|_YPFB!iS{4(%9&1n3_QZIh@;mc_I@c!UFItSrA{3hc!4fJ$e z&%;y7v+*m%H`7XK8(oO6rB&ir1-c6K0%$J8uMS^ns{>ufcRKhoSsnIG7UOye_?Lix z3E$-jy@{40cT2%riX1NmZz*&e@i);L!D~cL8^LP?uL*ih;59+73A`ro+QDlFuN}O0 z@Y=!a0IvhQ4)8j_>i};Rc&os}8&3Fbz;6@2h&U9#ZTQ9Fw*kLRvjJL}ye{y%!0Q5U9eC@&TL<1c@YaF10lW?1Z2)hB{NF?y!P^MlM({R*w-I`q!P^Yp zX7DzHw;8-1@Or@O0j~$V9`Lq+w*|Z{;B5hK3wT?>+X~)R@V0`t6})Yzy>0S81IRhp zgvaA1;4 zP^yHpfP)EN*1>{treKL+h2TQLO9Y*QZs2hEn)SN`HwoSb#99?Ng1#bt)fK1(R2*TO`&#q_s}%mvkSKTs+uOUlQZz{EpDBkrC}-N1X}7<&!M})CP*LB>&0WY>ANFd(5H!?F0(84 z;c4Qh*ShrfX~#5|NKrIu^D~`@`puy<#pOyr{RwPxT`!Sb%~D*0vlQ26g}P|6wwI~zP9LnxByTfS(dZ%IaP4OcE)kneic8u8 zn`k*ev<+Z|y$6=h!?TNA-dz&DMZ$X(dugx4+NZb%AJ$i-vR`AfD_LplHz)i_K0+c=JM{+To*fF^HsV5Y+Z>gt#d;P28`L%MYI`p631giUYMzjqhr5EjaZv=|>|E6F={K7(U!N=oI2AIxV?%jzt z?&)dJ*%_5BaZi#Nf2t?n8I>z`_RGjtDAHNr6xzycT%t;ZU!^+XG|6+gOGJN(ja$wV z+bbDpbvAx#v&F_`Y_a)Aw%Z2#M6*dofc0W$iH&pEgK&kt$;Q2-$;SP6hb84aFR_(O zun*Z~<9>Gw_=UD68~5B@Htv0U!C{~9Q?ZXpNed+>yQDW?Po$0$7kZ=EZY7hmFqH_nTx&@WA@jN#es65!rc^j08;FUY`m^Hj$0#+aFU zP1dTI4|HqhnwV6?yXV4Y`K*-O)&XBdnDhLp%;{Jk4o8~=-$D3hC%%QV z#Li#7Sz_mJ;Y^U3WrALr;h+iV#h-yQGxLiWUXx4%?i74I{Y(tc@3z<`JGbX1crU}& zf>bcyNxD7>H^H~D>UljrYmZ|+__3--^0ULv-+0?4nmg?LrMDe+{?gkmc5Yp_*tvDB zw{x#tZ|7dQM04qO*tuqRNZsv4TnFv7^H+KHNqC=x@6y$&kH8v_5Bu%>C7%6u{wmLY zJD-d_22Hba$c)<|sXt@qFUdV4DZXImHyI9t|0*3p*d2Eqi1|)5Pa&LxRU-TLw4J{> zXLpz|e}&HO;67`2a8I{8xOLeb+)q#2`T6$Kb{_E%3(uuTORQ*zj7|<-X`Ht6XrBhY zgVG$wa-NOLmT4J%I@lm$h z@Jw`q^p`HhPd#@b&-devI{q3=s*Rs@PQ{3XH|}h_-aaYorBi}sNVm|oPjmaPlyIHA zx3I*q1O8tk?h!o>?h!o>et%%Eoh|RR^Y@>2*m)k` zYiBz<>^#ow6X`yY?gPJP=zeK4+{SjIW;yO@yR330_0x9tOt$c|!RZ+qNf*=2z$Mf> zl<#Tp0Is0@z!rK3*g=1X^bN&2M^rDcPYoLui8WmcFj}Vr^EFEgHA`n`md?^FE!Lxf zWqLgD0-X!2)CIr=dK$1!pEnHqclvsSFV@AtCAu8gpuY{oS_!y9*FncEIxYGVa60r- zpieh~-zogHdIdP^bt|x2dw@N94e*D0J#dHK1iV^r1@6+@fj8=_hsDxOc0TnyXy=^2 z3>-?^A~^hv;BygNmKP$pEH6cHSq?{x9+rvwP1U7wjpBk5M)7Sk0w4J)LO7N0m@6cYs1DeBqf@I^c z66_VsvNNYd&?$JE-~qvdf_;L=1j!*51jjhI6eSXN3U&)_66_V+BRU5RCt^F@ouwk# zLY82OU}+@VaZ30$!2^-o>7A@&5>`>7FX$B9BzT)(<6xF<7ThDKhA?N0V2NO zaFgI3!KVd}3BD(&qS=B?FiUWZV5wlEV7K6A!Ct{Vf(Hek7Ca{So}ev8awS+Q*dXW> zY!vJk+$^|9@R;Cxg0@((AXqF|D%c>{D7Z;*v)~@V_XKTm95+*Nj9{@~qu^%2+XVLr zJ}r1m@I65lFFp~>5*#B~B3LTeDA+ByS#Xcw(}I}^+o@pmm;|;{D%dF44gRvxnoW$WiLCWKB zreLvPgJ8E{uiyc}KEs*J`UeF21iSOM)1tB61>30?$ffHQJRsO7NK>TLg2jRj2B&VP zSH^Y&hmLEQE;*UWSS*-%?si%=t{8Z9T*G<7pCzf!`Ym+&1RKuhaJOKu-~qut!G_ta z-!0fH*zhgk2p;&B*bywoLv5%j!EV7`!2<^8h^Am~DTfaT_6brMb21Hw_vYepCBz1O~GQp2ElH@Uco*=su4}WV!;N%?pn@muV7y->rfqIv0#H> zw_vZ}0m0%$tkWQPK(J4cE|OFPiv=4lk~$SUAe=rUT`U#^y9Ij%4+s`tEKpiPmzk%uFXM-Gdc z5LFs=Z`3cOuJes}OWL&}EK4si|HFyzW1e;i_u9vfW}eMxjnbXWA|=-tsb zM?V$)Vf3l!te8153uEeIR>u4&=GmB!VzOg%W50}3@gw3hxu zI7_=37JrO;i=R*%-G=kAJ-Dm50~Y@W=UsQf>d$b-wGUSB#owyA4=dsOar*NBQurm- z+YiC|!#H<(6e&E8{rM-6%0b#rPvbP?*GTPIoD)4y_?u(Hi*R!N`q*=U7fk#XuzBKz zz$1AV0aH@yfgdNkfXha^f!!I5f1lVoAiuw+;UpArtf9%>z;EYr*q6Hvct_T?|G#r) zjom$99{!)=>s+Oeg`}+oqh7diLU_f zoOA@3Q*Z+K4ya%p4hvpL<} zg0V&1>OPBOZ_LkQ+?gXdVK($vOuGP>F>yYyHHrBzrL+F{QH#JiF`YSg#nmG`c~ldy zrid+HD8Buqi22_XZi(f%pBJ)DVm^oS1xMvGXI6gOf5E?fiWjj?Oj!#IE$Q}AY(JC} zs+m58J!WFfELa~1hw|^0R&-D97Dz++|4uk+Jlp>!&8YEL{R`;7Ou6nU zV6XNMH;J$Q6`#j{8^-?_nFZo72k>7DiUKBJCeRq~V}MDR12o3@1Yiore?0q$(O*** zcp9hgslYMVjnEk9(}Ck@6!09#@$4NW8Z*UMU>+nIv&018`IrMVo~4)sEWsS0X%1!p zjS+q-umZDy#z;RMcp>H%jZywQ-~z|h@mkzTmmn+fEqK-cYtoh(D)13^}v;gfjxep#_Z7u z?0~Mu46+>Pfv&Z2Cycuz?1>(t6%t81&FhEV$)8z>725QW4+kn5sjHK`-&Fu(3 z0ehOBge6S}U_;|9{W?g02h=!AzaHUNff|2z?grpnm?Jg5#&|REJYK z8h>$a5Abu$ml}Ul>wkb>W6soMQ$GVb)IGo`^>g50bw6;3dH^_7{Sugn*S64d)Wg6L z>QUfG^*C^pdJ>qa4g#~))4;Ln*T8)BEU-X551gTX3p`K#4tTzL5m=&L2F_9c3oKQy z0;|>Qzy<0};6n8_utvQDtW`&Wb?O+fUA>3$b^uXd>W>I}fSSDOPYC;f*vC^J09UII z!C3=DO{tH78`WRH`94t7R`n^u_{&oW?^b^W-k?4Q-l)C+-lV<){z&};xL=(C{!;xD zh`-{dG(C#HgNA30l>_*c8U%bEe@#x&Z-A&BytJW9{5)^0eDK~Ag#}Vs6jOe_zyK1 zcv=+zwVn#J>mp#fo(>$PX96eb^MI4}`M@ImE#P!r0&LZ#u+|2IZ*@7s9YEd-od;Z_ zD}dkE7Xmlw`M}F{HE^3=2)s(y0UFYfe&aW zboK+`N!j-ByY*==XR5rH)>*jR(GFn*e;n#w*>oY?FXTY?FcSLjMx_8k(2j0o?IG zyL|#M!kz<+v`+#Kwoe8|+Y2ye<=Mvq^P#zv&b1c-XF+o*&4%VuDz;At&VgnlPRYgs zFM?(xU22~RTmsETS_;iZYOtRNTn5c1YKCSLt$=0|eGfL9s0Er$)CSEa>VVBA{H2!y zjA19B*-r06vz`77&32p%njQ2vXm-$-(CnbEq1k~)x^sZqF$rjQ zOb13d3NRK9c8mo^JH`WJ9Yw%+XyOSqXs)6}Xs)6pXnJS_G(9*S8V}5XriU`2>7mik z^x!06I&d5`*HAGu*U%hjuAwq$uA#ZmT!X(wI|=x0Xs)43Xm-&>(Cng%q1lDAog&~; zXm(KpG`naSG`pw?n(L?yn(L?on(N31&2_XIn(JsSG}qB(&|F9Bp^2x^pt*s52u(a` z22DIy22DIs2F(q0H8eNSwb0y1yP>&}ZiMDWx)qumsTZ0X>2_#tq`lDGh`+a6fKmT2 zG&j?0(8MXlb*QCF(#)sNKy^_+T5{Z;)_y`taL?`vg? zwI$nfY>R9wY&&gxY`?O7ZcDaj+6(M6>`U#<_VxB2`wshc_M7du+J9+(!al`uuH!<- zLdV692FD7=D#uR8^^RK{_c`8meBiK0%!^nYu`sAr;Hiuy~`mrLkX z8{)RaT^V;{-2HLCihDBdrMNfZ{t)*`+|>Am@vit?@js4#ApSS;@5R>*Z5e9sCi#c4 z(`!PX<7sFdh6VmzJG3ByC^uc;=9G;MzfEC>Yw3dk&VJ2eT$D>Fd)_jvOI^?4b<)iO&&!395 z-Bj#aOvUPMDt0cWp>=W3oQXa+6TN9BU5-`l5AoZE-Ht1;8s3hT?GF5|#BV3&n5*#P zf9Rd!qbqP#ps%1>Jjso}q;6ezTJmcx`Lz~*m&M<0U2n9mw^-L(X*8aIBD$5ng*Sr; zPeU)oU*X}av46Wo-)_-~%H$)-x~5py5!N-$x@K6{RGZNsWxGV|j2f`cKg|+n=JZvHd3Y$F`YqXKeH0mE9A+MeW6Jp=~_Ep4rRl>kFpV7v$5d*)tKub%E34b=CTuK3A)&%_j=r*0ry6wYe{IEm+}kdR?=d zjBfeb4rg0aNlQyb6Lj6pZkNaFmsPeqSyJO%6JT)?QwB_exh$PJpp?0@8|&-K+}@5B zXIE*9)9WqBuOEP0WVo;-fzo!Dr_ohmiXog@cWXzBySaJm zh}P5rwFcxC8ZOsCA;*$h061Hv2872BlE8A8uf7(Q!J?p~sDHdd;c>iaEG%E^YV7n$ zG@;M-sbO*+rnCB7PkU>5TW71w<6PDvu?#<0LQy;0$_$QsHgoFCfw+Z+ixeeL=jke6 z?QH2p+vB1Igs?Zy<*RI83GB7dU+@OhQhn(6?3m$@vRI-r!fXV(-u)&N{|ivhR= zQdiDc;b_9tY$cg2N0|X_6Sf5J6)Zk(C%B|Zd=as*49oR>tYCPTnR>9 zNuv*~j76pGEiJA9<*(>=4`lOQp5-o%HqX_D{)V9$%nO{}6}2uON99uBT2qSl%7Jfp z%7sUQ3XIgq`0VNR^=mxNjshyJ&7L)TMtyxtd!w_(%c9o$`nm2lXG=>LQ{4Qf4Cpd& z{R6!W)IjXtl@dxZ59{B z?y!O$m$S*`v53a@PE069`>a+8o6@+$3qf-`+Zr2Cl+F$`p(&CUEoyUmx>!S6 z$rLJg`&PI-P`l98brA+b*8-=T$x2tV50@JE@)dloZTI+GP0}o;P?=j|!w@9?Hi69e z+r*T!+60S&ZGtIo6H^DuJ#)j_1nB-X4W`uMfF`DZ8&W${>p{16mA5sq6A^Cg^i;ar zq*6dhJ(amJ%=_G{{mirc47L7#22+OGGFP*^&CTBAM(F1T2H>ftI;L9HF*V>XX!`xd z!JvOY^-ry!a3JoKLPB$$23c5vK})d~uWQ?htLO?4^F ztt^>WKeyuI@-kY`;`BAQds-{XsI;=Ww!D5$O-X4vRWB&7vVx+#sH(PPZaK{_udOYa zS8fc{lrJbRsVkXNY0>ja=9g1d`Qplos&Xp7xT3CpZb?PuqMC9^y9}y}YbxrXQVm#u z*vJa(RoB#)Rn(N1)>T|oPK!%wsw%4HQEgofU&_koESg92OD?XjD66k5ubNkPfeaxf zwWSpmIHGBDt*NBeuA1ukR8j|6>_MNeYcHvtZvPHJPcGL-R{6nNwarzofLL+LW`TvZACGj0IG?Xik0g0xBshlS`Fc=2x4` zqDp37jN;%@S$jdn+&T``R?Mp^FRQN!Fv=<}Vt&=4`E$x^sG_XAs;**g1%l$}1=ST* zbs)-XYN{<(WkqdWJ$Umgs!+%pE_98*q?Kj0;*j#{TJmCj!dG#9?V?h4E?roGT$WYT zE~qTI1lQ8KkZQiLye?SLOmKUtDX&FH@-nBQZc$YQVq8#CQ_n35Sq->kUR97Rt*)*q zE3IAxafnxCT=C?U)YcIf0lO^RLh4JZE+M`zaMjnFQj|G;PE?{?nk4LPZ1=Rdm!T^) zlC!aqTHS4YD?+%7X!O@q$Ra7vS|mwxXm2E}#l$J4ogNQvmFGL#oXcHJbt`yfQPBje zE%o&>YdKp2OrFRufSjs3Tx~TRm^FK9efdsF~OsBWp<3Uqu zl3Ku8(oedWUEPQQo`n_O1)g>vW?O{Yo10N%6+C-7TX2KfL~bOC3Y@*NzJ3nYvsmNK zb-P-?4BG22tH2u7$#-b|Vujqj^^;*$*uJL5h0c$8-@j1nr!~*nI)L4(0l9PCEv|tj zwT8cir7PSmO=ubYCE^Lb zPwg+_$w$^X1EllKGTc|1)lt7#yb7!8Y+Z(>Za*3OvexDFG-7euUs~tJE2&?2SSh(YZfrvJORUt{>2)c-nWV^@h8bF83IRIN$ z+XL{c+GPYmcXciwKy1za{R&~-qW9wm*QWy*lV$1v{5hTO7Ie&S62ryl0dVI4Q5QQs zZRoe&5|2wfI)I97()G(Imic_s(ci}q$Zbe}NlA-)d7JDT_OoW%LjNeZqxJDdW4{Qz z{lvGq{Uz0%zK+iRe2n0@sh~QK8{1YGB0Df_Hqjik6PoW`&h3RTfcZx`tXc>zTY-xV ztBq@{fUmvAg>A-07luYzu-Ec}T}E&Jz$Qd+EF+&2ckdxvQ^c-2YrBfP?VZ?+q#9RK zCob+bue-^`ihP$}+Kw=HG33fcJnd9*0Jr_YZS>cdd2MXVy#JUCh!|h() zMIn!?!v!yL5IY}D>{@aQHgk}-QQC2pTXM-g&npt(2}34SugkOA)g&ZmmLm(nYnlLo z`m4i8AE4cBfgnVIu(Ysdzl91fh8lO13>;!|KFYfQW`Z0^5xY)SZbUygaD;NDnuWnV z$%%F)1XA~pfnCB_&gGJPv~h?tBLOrv7Z2Z3__}r<79dRd0wJvGLG${9TO)BkdE#H{zgHlH8LDgKHcc3%?XAl=rsSiAL!~Y4KrXIKyV77xhK%d9mtFR4S;fiwq)7)aUM$!uqb&s@D|5Mk#WoI5c63-Mt4 z$}>S6tFjB=oYfe5VG7GRpNr;qw)otQPOq=b)xsUlMJ3CZdrWB7?1K7wERVnp+0>xY z6|TmW-0#Jcvf?u#c&r0e<+1h!T}>rEi~!59jZgDB-62$&Ygy;=<=os48|5rOQiiN1MWSW{9CcBo8*iTAiMi!8Gb*?3>H;&zkm7fyt1| zbAh`Fqf0PC`C5$kyr2oP{N}m8Tm(-JC`9vab}#QlE(a8sxx9@YcSpF;cu!*R%5trQ zy!{i#5*VYhufL$Qy`#&+TW$l1T5h_f<`Vr;RM$0Z(h(L{NXR)vAtWZYWdqU5+!g`#&knkB{) zOoa1!3k+^=#gfa)1g}_VK__5t3nNrk1ai*@RZ>86XCwu*f->sM#9}cp1>xq2XL|2q zYu(GP81J^L>$-XJwJbJ9pF~=wCQph!zYcPd`;|bFIS;7Fq~EQpAEr#N4)} zd=fw^b?p+G)7jkY!YvAF#pP_3b)6MtqdfP7kSKh0vqd|-T)7;_$McXCZfWvbq1G-d zFdv=4<0m-d?napck=9yQlL?x`c@vgwo1nj67>bv3CRBk1oe6dD%>-dcT+UZaVXabH zK4+5BO28};(Rxg4msPjr9b^X;#n{xA5IwkV9m`I0l1eaYj5Qc+psoqEDm;pFvhfD3g@+JV0JSl_2BV`Ad4ec z*IEODaVn3?Jl`451}N67!Nh>!FR%e15&ZY{gf)Z*HyLIJKt%Jq0%VJykF_s4x=edi zY0jLe#=6R>rl-rI{G*nu**c>{$#G_lk-<;5chsYe%iaMy#M*8$H0Rg9v0!xxv&%qO zksB4Wdr&PiG*wiWuWfXhFvic?&W;W_&Ayve?$n;Ny*rtOg9=-U*%@4(DNMzY0rb><-B=jE>;JJss~OnI?-F2OXM8UIg)tfJO3LWCgkSXltFV zZ5D&qf~?9n9*AiMK8vWsXkZ0V(^kk6P(+om6F8cbi#y@mcA?=0Gdg3`kfUJ#o|fd4 z(+n{D_jv(^wRj0r@?i=HDwzAaU_;?s6RfjB7&y)jG$WpXLMV4o5Lo}1Rhowa%OF1k zV-1#X#=kredXSW>(S+*S7h&te6S`Ew=!)A}Dlf4D>}>?mpe+U$HjL5QWl73QWS#l~~KMz&vDT ziY3Wndk{5+SR9ZV!JXW4G`_YbS+k&!awl&HEaFT6x127Y335?o-%DyZWM@&#h0=|| zT}RPH=WA4$QCEqQKbJCJ$IK z$LdWmXN|aC|0zPAKO*k#tS2(^22u-sWWcXyhNxW2IGj3l%Iw)N9P+TnzngO1e;Ln8 z+vMcGv=QQDX)JltSpLtAVf^ss0sak3A(e2J|MK`*N;0Rn6kK{cJsv_s_fRwLFi?8G zpUd1%>|tZ=-RPBu7P_M?3=Dy#?Ksx<5?Y59vYhC@tB~H&-cgH9DmP{dziGdirzGAs z2*}ZYt>@;T+u>@&0S|9Fdd)f?Yvtv<^=ReC42}LN2Ycu|RPnfHt+)7krvDAaEkJa>&};WjKShyGy?0+a;{B=cb|+JFo~+0$m-4dA1Vh(y z{>FsH7NX3HE$I>1ALrWPW@31aBEdvy_JV>d4diWr=eDH);ypP>GVbnRZyK7LZoi`> zX}cf?3?-t3 z{RX*O*_N<}&7d9P28`z7X%$?s{o?aD8+}y0?0Yy+=N$~b)Z!LeE%_}DfjQM(X7h*chkh`>_3cg*1GIJjQve3vj5PGl?l^j{^KHqWP*F8S;_s29QfC8 z+{gZf+W%{H$}Q?YOc(X~U&K1AZvP7lA+8NuasL;l&Z^`8!a_)9g7y4=V<)t(Lse(Z zo>E`mXdc9p5sg<9+_@XAEAQI`XBECv^Dk#Cb<8YgpHk);Y<;_Nv&l@EbjaB%1V$K- z&+OCjM1dnhA5~(eGoupEFW67_#YN`KRDwRH#RaHiEDISu_;KY`wXcySfKjh%UyPki zu;x40V*41YHrHCh@uNS)i&EbTKHoMXOu6jRz_Q+Oy|OSfVXQzcRjjaazRgDhs0gfn zG0&8EJkBm$d0}P3SeKXp?(ra6*51@5*~IA1buH)c9A1f^e*pL z-uIhVf;T=h@n)z8|J(7)1UBM-C&DhgHJXXHJy+mO(H2mdcq`NiK7R!SJKmr@py7X_ zA$5q?4lRUj^+coU@fK<+zD81sAHQjAC_F5R5abiWElm=OSQo%zhotPY(ohLwkPAGQ zAT;NJZ1@erla}Dqm_^8m3vZ*cy*AP0JmZ7`W#AI!;S))$(_!V$l&J}FAAXtmc1trP z*jWPlK#vJ%ehY8A(j<->nm${13#SeGZpjzt`dsiS)o+*I#pf@k1WHj`1^c0X_9ADQ z_{t0W)d{b0$r_&?{MdEm84s*R{Y z_ReyN&oyfD#{O<&OT+mM-gEIz@eI7fn~C>%`CWe+;zj=Ps3&wNiY>9HX=J5ks}onY zn2GweT0vAEVywlFX4XKm3Z-x(E#q@j<9K`)w5e&%HBNa@CVz`UUK}O(+6~$8yRaH5 zw%{$}OxR}~JP`zs7!JqE#IFm8SXb88NWzbm;-*LThO;N)C)!zBX)^(ou z{XO@RyobLo$wMiUY*M*qO!a2O3e*D)}N~T%b1jWNaf!FWbmO z=aij)J|z9lRq-%v^0qvA46KW^xyEScL$YPcaLa1dQ(EX4lj^KdA*y-!82UtKQau&I zpxQ+nPP%9L#(C&I#T$k~^&+}31$-ViW2{N_DE8+z5eCb)5k{qr^ZZ}nSMq|!nrOf) z?0UAuw}#n``xKX?WHV>fO3z$s2@lS5;}5=E;V2%u?A>BDBFRs0@+kWUX{<|H<9?mG z>5v?m1?uZyxJ;RPX*4(aEYsRRTHZrG(XmLodC*=u^uV2U=mKzM4W>Jrq$%fs&wUZ# zX*^bb#C!8g&D1fhzd_`Py8lLgC5g&oo(7xy9a_&$ z^Y(#qTb-+AgLiKo=A8sb`IkRAXeFPgr>N1@OX!WXibRa}CM0Y{F1)Ub>&NRl(14?b zZb=uzyr8%DYnSg@8H!tKv0!p2xFsK_NW&f?;CPL9GDvo6tvGhcC=Gj$ws1)TlD_yR zFCibJ-W|PP^PfGH58XqfT&hBgBh{5$z0gSD3LMwXK%`Pq7FW?|P7MNhWaTf=R%WHX zbHH?yR#*^Y#jwIOMV>IU4@!YB+<(}hNO1}%E*>|aYf=u}T~zVM5wHv6U`U$3FU@N3 zzO#t7Am1ZXyX2%J8nnwfw)Oa?cpxiy9a2TSh@PZGzE3%W`-j-KlkPoedV16bK4-S- ztZ-}-WJDf|*T>HZv0}U!q(>YLRle#vdYd$V{Z2#mCJK5ArW%Ypw?f4vj~mpsJDJGQ zIHaG%4jJK{1{cG>Br5YN0;L8K9m8udz|An>eJ6$B3ayJRBTX@3agAM@3W%{_{FEn= z9*pmR?@M=wPwq$t==$mq+qG6)m72_dsIZF!%IT9yyNkU8ea&BKCvXlD)-pr7SoT9Q zY4_;v7q9UqLV0V?C)}jHEP9xN45@^(@6JN?L3kgAL2hw}tfTq&o%m-#ajIqNCk1qP_TxRGFk^6y8>njk6J?H5#p= zl-oby_)lUEnxnWX3rpECM%`I!!|gDmH76elX6)r=V}X_|znDKn_j@ca^ZYoDQe1Y) z$asr!mCGUqIDcQk0hw-xnM?EUIYW+89g|m&NA4GjE=qzlO_QO?sflS?h7f7(>{CE? zpWr>z(g_z4iiINOJZWOP8%t)VNCA}|dWI4|T_w+&;(ARHtqxeup?t;}r75~X@^ARC z4uhi53fF7bD&_VK-lw&8B||BUz!fNzw|ZF_#e@xCy6FK>IvAaw278rj&0~sE4WkHK zO6}x_6s$DQ69%7~CYPM{V@5-C$3?wj%N1*{u>TC_!YycG?>xT}iQ2mtlgpMBc_!<^ z)p`+l@0ETCYlm6sNvGljy+yfj6fHHXIE#?&l}8^j`RI#2;Cn!M)Y{po{Dsz5WVzxe z;>$TucnMb_DVuXO&Scl}IL--N#!(*Scu~g0_Tjnri%}rpRE$8T!t!5;Rl#b#HVr z51-3LlnR-Px^ckZ=u)fHdBwU815xr(Gq?tfz{>pQJ(dMOy?Gj` zv=pJiN7dGnzzu4(02IaMi;Kw@J{)&6biIc3uZ-(tRH^4J%As6AS6S|zpk^MqP0-g z>3ZXoK8*S*nN~^Nzf(INbKSLiFYgmJ%6`|XaTt{7wOVt5+AyRMxp$Z4g#JPS!X~M* zAbE-}V-UN;sM^a4Z?XdC_83OXD|TKQKIw{mCsPR*p&oJoMo}b1m zpI*nd!e{H;M&UW5H|B?06TV;BW$lv>niQo64=D;|D*>c0Om*{R!Ps^7&6Pbd0riWl_(WK7=8~ zT}4pEXyA2gAu`;11Rte#yp4x>CNH|-_as-RLbz|1wq+YB`v&8B_2+3NjV<`@(s>1r zw48ZOG(3eRo&eX$@HBTVlzpARHyj7kNq!Gge~jxFIC~CCl$p_^zZgRpJZvph0&PPl z_vO^}{V)ylNN+pK>04_TzvT zxnLapSSWOE8oWJzUSvqF4bthgmJ#VdhfaSS<0{L?XlcHsDTf?~J3&_sxjv=w9(JQc zEUVchE#w4UIK>1ZwpGaUkhDH=jjer|*quT6DhZ>+r`teCqc)aCB2V;?$!Y*iZ-hJB3VHwPr`sFSlapnu1d+$=KeRel5o6zJVt= zK6458-%VS8K%sDfF0DduVz?4?n>HWQZ6qZ{m!V7ai8HbhO~74ub?eX}h<3LXjZC%q zt?CD6IWK!QP2nTdI2}gvJ35acP72oY3SK|exrdKXn(t?qOimg@C(Q)Tp3ZPOaa#75 zKf1Dg?c^-ZBHeE*J(@(POe=W5o-Y(Af!8AjSB#G_@W!6SjL>mJTW+6OLviyDUN9z0 z`8?|sLw(gpj!UIJuX7jnMyp-fdC5#(gp?b6id7P;=fF}0iJGsq=kVin+gGISqva(1 zWVj9X0r1?WJyd;;8eb+=E9D&JTWn=Ij780Vq~);>dmtb__!Plv%|S$pOKYQ9)XyUI zj69abx-NMN=$w}JN|SZRW}YES$~#$qfY*cGGne|PmXefS=&`e7tnO~3#YtdBoe!h4 zu%$?@Wo2R17uQaK;ROFS5o7C`P-Slb9%J&wsH1#pg{HxF1=5gnus>?ZXBepj?aMt* zQRvn(TG+GRV+xMEoHW0a4l%0g6c5=v#OEXYZzmDn&$4CcKFD%qsOiXO^=J5hng2KW zClTNCGD3d|3a40LI?Pdz+?`!hGDG=3D}M;x-xiY3J_#M3{(p&0LH?F`xgrJkv55ai zj4uWM^?;blT@r42yC@5%&yeDZk{KQ%JImH*udrw+UcAH>71EL^OhNQLX|-=sy6r>i zUIoH~A=wk5ew6VgpYmR!=4J3qfKmALI{{SjAUqHuzszXPZ}%x+ zeIyz-7VyuYd*TRKN2kp?e!=iP4tL<(Ufl(alX1^IBmXDBc@vr+W^*~WCp~&2C&D7h zrfGCfI;n?o}@XStK_a2htj z_!9jd0@@X1?Bx$aJJffd0eYzR+SzNI_1W1O&iJR6C7%91?BIx5E7}R^xk;x z_WU>)WapMw4aHkc?)3bDM2#m5wJ$Per{GY`rpS3KxeUU{mOBSo=o_yurAGnRSsQ9H4UvluA(z5=3d51GB z8{lF7@!$EUxBt!Wyz}%o|4Z+``}6<#XTPQAoRh9blBC@nMmFlSu0uuF*)LLE>Dic_ zPqN9b276UxkliKELpRPBK%vb6ur?>LcZk5(K-b(5Q zcQhC{pr~ey-iwpK?B#QRWi*T?l+Px0K+Gl^v*VR|eRD0%7K8nLZe($ozl|evNxfcA zH;y#_ZNjxkH&W|eK_g?E-^KexF71R!&AS7j>OG;#u@2VLm3nr$)~L{6G&xY~&zkRV z%wA+5%fiff)XNEoN0`!F*5Bd>s-%tmE%y~o8JkEJ>yldN5-uSVz~x@36dy)IUJ_Zi z%(cO;$&IzD4a^2IX+S5l$^KFq z4V%B2)RC&>55K$n;KSsTwR^8>pn?^?GmC{3@SW^B-o- zZ$zp_n%}I85PkCB1>?`^a7l6y%Z(RFHATbfDZ?7Ck5|E5N$M4vRFDVqJ6!L9maO^b z1EC5QMt*E$ah57wKxUGi9U88{isO~7VQZrQF31-Aw4O2~G|0I;a(m3WeWQ*(HQzG2 z>aMmTQ=%Ry(~br?@c=;~Qs@|!#)PCkK9&M_i_}H*j&1(LZ~}S>P)BN@gWqajQU|6h z>qzt0pepb|=i{LNwfdN3l9PhI{sw|=e$#}SL`YSyPxC)+4h%jsGQ z*r;*B5HV4ynj9j<8c|md#RKK$X(REL{XzWkKaF(~elHDkatKzxm>Aqt>^)vdQ-nqM@F}- zA@Jyt6^}}R#$cZ~Fq4=HKdcWUNNNq=<*CN-k51bCwSEn;HNZ*=Km%Qpl^icpGGyZF z@1g)1Y6~&0wUlfP*6NVA+E~L-jA!HZ6w4oDv%2y60DTy#`j~=Ohbf3KR4R(;RZnqL zqhT0d=THvy*Q)jGkojc|2lp&lFh=}eS|cW?B-aY$$e;Ktosn#%Gb;ycxSF>asU*Eh z=Si5BpA%r>X;CY#6piq9#2#*6Z*1BLaz!{Hm z0%nZJfK=pTgo`MsJYO>2ZR-6nLt3uKRya)Jn4+4Ts5t+ahU}3?p}SG*%NCmsGHd>) ztocV-^S|;p-kmM}?UcTX?TeSx6MJbc4Hhb!EjDnE^pLmaqiSiucL{a){6-k>CaCn` z4ZG2vWITFGykWLDk3rYRyOn12ZK@$?ji?)PNlcFFg3;TAY zZCNMsS<8L7Vvz|b&eVHr(q7|qh@3_hn*Rei;L9rTQtrC>zwj-}CTi7e@t4=48?IHU z%bH)0SU_hAwER$QFkAe(B8-4#i~of5WC>SfQX}4G%^$=*Ml)hm@?lY0JuuRUx|i@R zmp@t=GU!E(dep}(puZT^2Mm^(Fc#4twHT`Mi;-Gaw)m@%^eqUG{8s2voQPQ~eab1+ zbtN5T;LX*M<_9XsL8yatY6T>V8_;;%`pW2xG8dR_k#7Sgzm>#6t+vvTj4@mMZBAlm)IXQz66OY_6?10j$ViK$ zv^9m?RO=QGN^Iqdzaf$OM}>7ycz7(NUOod9z|mE?v#2oU8j4HoZdBeKkG zv=<{Zw`F!I9)71AFpGaEIAe*|Tk6`MQ-|67K$QhV}q>PRqt7;on?aI8(lo^vW?aP|Khj6po7PNc} z4S8DJigHy*X}Nv+YPe1FV5J`*b>h@2He zZ_7Gs@KY0vN>5tt84Y5qQtyhT+$mE$5`vTnk}73AZRzZbL^`_TLveCK4dn;Fsl)_qC~(4CZ_Ym-qrB2nL5a;dDo77Nb?bujS;{Oo zbY`fB21ac6g8h7oltYlB>4WS2I=+OqUa~g+;TH z6Fs#m9%0fhUbW+!qk!0u2Mv7MExE_uK9dD!vDS+7AQE$=qNyx;mCz4t^MrXrzrdE8 zf1m~oirI6T3b;-T|4j{bAgK=pF;wDp9ak_I$QurjI;Gh~n^xgKY?|f0zk<1@*=4yw zgZ8yy^FC(V0KhTHNJvdEWjxM!_juLXy={ETnqPJjt?YZf%to^2S17XUcQpISnqSMq zvJ5_3{5{+lh2OG^AKqK*)z!Db?+%LysURtO`cddsnxtszP~Y#f!8E@Y3;w%9e8(XE zD5vZD)W!z?M>V&D)&Ia7|ImsCw$=u~{ytPMl2*0-sVHrb`H)9yEr5x+^%?G$g3t-* z#U?9A4$ zbl9lFCv@1PgLdEU+M>ff9Fn95ma*#9Zy%=o^=eO+$`e?a#(pL#xXS<8t{zrpbof!V zXI#xT47R#yJ)qY+F|n5*=$?-7Nj4;-BVcGcuNo}`=@ZxuAaz%Is(A-nR8dB72sDdV zbuf=(RxHfTd;*wVHb^}#=8V3b9m8_w`al#~B4UJ7U9oNGx-#2nu!co0!;dYk@@d>^f{7{GY z5%ldbDq$d9oV?1CbyVMt45|j11ouYA^Zcu5<46z^;cauep z8>|glyo*bGTC!7Htk2fVImNrOBKS>9-BK2#nCLRyZmwv?OBP7fA@5zu)Un-2S*atw zVYA9Dl{J$4X*JByVZ8S>dMW5p_~gpSYUt=ILvW`;?ng8QfmJf8Q-q7>il=DZq|}?W z)=Biau90fqq~$8s@zsP8uq?!q7{N`e@LL;Qw6S?kcD1FmE0oz?Ht{z z+OBNF5xS;lSE1U1K8=X(cFd!`mLLLb5klmWWLGr~<3c?h(CC>GnL;x(=|MFli6{BS`uD@Um#&lbfu^BFBWCGcB!+L#`G`%Evt%c7Z3M4u#WN3kI z78uSAngwa8z<6MT$u4ebI0vj*Tg^uApR*=*+tMI`71S0pb3m2Ng4C(z9(6Hctm{)$ zEky(o9MnR8`(!*Fg~w|PTX@%k$Y8@7s9MX)Ep51tqJhE3jF4P8n=7=T)Fz|MEW~_s z`z}U1-Kqf@OLNsKc%&%P66sCJ>8h>mBVkF2c`Fu`y{Y>QRpzgRJsYOjwwgmpbx%Rb zmzm_dE`H`u2G+2uVsY5y3!TbXEL9xTl9yO)S7zkK^(E^m6w9Wt%^&&5-4ofG7OUF0 z3L?{RKu1Xa#7hIyYXu5tT4pmUF$J_zRIc#X0&eSYH1uIOQp_~1n_{;D;5~SVmQAi@ zoM8aF)48feLQul23J!zpxPgOB`x{KI-%BC)-;dFd&-Vre)RI_6G_Ce2WD^G4ET9T7xsrf6&+H<}SnsxU+g6(&7{vw5&-QI@GX81`ZaO`GJ63 zC3I~nWTL`xmyQ0b=AxS4AgmkTETl_IE}1o7vVqdS0sA|ot%jcRyrlCt7AJu2#Zzr62 z(Rg=XEJI-+k)igJo;TlP@`^BOx--QU1aXWdRK4f&e9u?``PORVjNhxuXQ$An>1J0* zdKpJ(-O443u_C}-Q90S>R}HPg6<1p8RUp}%JD6OX%!?aGMjOPi_lj$gG?KG0Ha6l| z=v{63OAOIuXA-g;#;U9=@8!hgYPxLUM7fLk{dj}f+kD4AR)~{kFH5e0re?VcfIB)D zPV*ZuQbWtzm~onpFPpx;?cm;yilB&#AX#N0{m)VzHuO(YlHbki(>cyZp*}xNyC)Uu ztCOM5Kl5l>tcmFcK=-CNu{97HR_LZ!$C0R&!y;r*Iz17{EqmlNmj*G+AQs%qdI$JVqEmzViUoomyt@j63>V^e|OEqDT>7-5&D^8O< zGhnTt&o2`ZGs!5&Y{-&j4hD%B@VC<>q)dYXCK8fxs`7~!7Ur%#bLF+x))(wjd*zGQ z65465#?v`T=qp~*$n!s%HHJ+|$XZ)yHU9NmjmFLycHa&UCSltv`yRWn_uRP)Q~S=% z%}wo|dG!3$V~@SMcWUR%?wJd7`}SRUbQe$O4J4s9Gd(lCW17eGjwfMk`k7;=TDy%s zobRP_;Pu_pd!~0nX?;VhGT+2WyWee4uSTn;VXbG9u)QF8`dE3RgYE3>J@;z<{$K4Q zzXSBZpDJ~TFQWAQC${^h<>f9rLpYg)XG&f2S3B~~W8Od7dK=-jX?t;WvAfpx{hi*> z9LZQix4kftx0(8Ob|Sp>dT;fB+k2C+vrL4)ak_X}^>*LNy(mz>T_NmA!p!=9{_^Q* zf2(!g)ymWhF6~ajBX{g&4cB%8)nf^-yef9T&c~FaYHf79w&R|k(9t$v(UwddI_^ut z?)4bi8G5>SQ?@p^+}^g``3_WaR}!@QLfol4r&T@m+Re^=9!`uHC zD&E4*hAQik?Yon>)hWK0dwn;JtFyS(&t=`A7aG2%SLs!;?7Oz05&5PbCoV4Vp3A7i zk0jxt5hhapgUrxe{ciG?H ztShe+_Y9eeuf1ijkg4_H9}QtX32Zpq(%)i0$Ir8kt#**iH=De&AQC&E$X}0KMz#Oc z!4BAv+>s<4{0P|AQ;an2=b!(B|IBmH!Q#yr(PimBBw^`$NqFw$(UYJ3xefboZ~os; z|N7qF|Lxhg{_@ibK1Uuu!+whk*UlVSn0w>aIUDevGe@tS^B2dT(e@V{A|J}9ytw@g z`$e8dVN1o+S6_X_DEa^A@cw_aFl+?>h0YtDY5$W_yAarji8r&Kx1G=C*_UJC{Q2ou z+4D|6e|djs9I~^&IsA3xKqY8Fy%9oF3+5;I`@wtGhXSKAhwW4$t!t@N=Ba@>P&$xYoCIpX6LW-|hP2zhEKX&dZ-q z|S%_ z!u6?hSLPpQN7L!ocT6b0+aD9Qd+G}6LyQF95g!KdWwe#!)z{}I~Yf2V)vpRILiR!|o*V*yv z^XISKCDdIJYJ;Bwj`rodp+!(E1#MiK v56XZ%(t(;t`Xg<$?|-Dl1RCsrBua>HIsBD>Rui-@gkc%{-~9Zw Date: Thu, 8 Sep 2022 14:19:28 +0200 Subject: [PATCH 03/59] set target framework to .NET 6 (latest LTS) --- ColorzCore/ColorzCore.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ColorzCore/ColorzCore.csproj b/ColorzCore/ColorzCore.csproj index 67b7429..dc24dde 100644 --- a/ColorzCore/ColorzCore.csproj +++ b/ColorzCore/ColorzCore.csproj @@ -1,6 +1,6 @@  - net5.0 + net6.0 Exe enable From 4b726867acbe5718c3729f711f37be6f4bc2b4f6 Mon Sep 17 00:00:00 2001 From: Nat_776 Date: Thu, 4 May 2023 23:23:06 +0200 Subject: [PATCH 04/59] fix bad merge --- ColorzCore/Lexer/Tokenizer.cs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/ColorzCore/Lexer/Tokenizer.cs b/ColorzCore/Lexer/Tokenizer.cs index bb06e33..f48c264 100644 --- a/ColorzCore/Lexer/Tokenizer.cs +++ b/ColorzCore/Lexer/Tokenizer.cs @@ -313,16 +313,13 @@ public IEnumerable Tokenize(Stream input, string fileName) curLine++; line = line.Substring(0, line.Length - 1) + " " + sin.ReadLine(); } - + foreach (Token t in TokenizeLine(line, fileName, curLine)) { - foreach (Token t in TokenizeLine(line, fileName, curLine)) - { - yield return t; - } - yield return new Token(TokenType.NEWLINE, fileName, curLine, line.Length); - curLine++; + yield return t; } + yield return new Token(TokenType.NEWLINE, fileName, curLine, line.Length); + curLine++; } } From 67af0b27643a8ea8a01918742383065f5755d25b Mon Sep 17 00:00:00 2001 From: Nat_776 Date: Thu, 25 May 2023 23:09:53 +0200 Subject: [PATCH 05/59] update gitignore because I think sme broke it --- .gitignore | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 23616f6..68f5431 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,4 @@ -################################################################################ -# This .gitignore file was automatically created by Microsoft(R) Visual Studio. -################################################################################ -ColorzCore/obj -ColorzCore/.vs +obj/ +bin/ +.vs/ *.cache From 7a5df38a3a2c0f76b6d8de61ab0ad9ac4952176f Mon Sep 17 00:00:00 2001 From: Nat_776 Date: Mon, 15 Apr 2024 19:33:20 +0200 Subject: [PATCH 06/59] ORG allows addresses, POIN allows anything + extra options --- ColorzCore/EAOptions.cs | 6 +- ColorzCore/IO/ROM.cs | 6 +- ColorzCore/Parser/EAParser.cs | 77 +++++++++++++++------ ColorzCore/Program.cs | 123 ++++++++++++++++++++------------- ColorzCore/Raws/AtomicParam.cs | 14 ++-- 5 files changed, 143 insertions(+), 83 deletions(-) diff --git a/ColorzCore/EAOptions.cs b/ColorzCore/EAOptions.cs index f3d934e..a2cc6c4 100644 --- a/ColorzCore/EAOptions.cs +++ b/ColorzCore/EAOptions.cs @@ -21,7 +21,8 @@ class EAOptions public List> defs = new List>(); public static EAOptions Instance { get; } = new EAOptions(); - public int romOffset; + public int romBaseAddress; + public int maximumRomSize; private EAOptions() { @@ -31,7 +32,8 @@ private EAOptions() noColoredLog = false; nocashSym = false; buildTimes = false; - romOffset = 0x8000000; + romBaseAddress = 0x8000000; + maximumRomSize = 0x2000000; } } } diff --git a/ColorzCore/IO/ROM.cs b/ColorzCore/IO/ROM.cs index 32b03de..133311e 100644 --- a/ColorzCore/IO/ROM.cs +++ b/ColorzCore/IO/ROM.cs @@ -13,11 +13,11 @@ class ROM : IOutput private byte[] myData; private int size; - public ROM(Stream myROM) + public ROM(Stream myROM, int maximumSize) { myStream = new BufferedStream(myROM); - myData = new byte[0x2000000]; - size = myStream.Read(myData, 0, 0x2000000); + myData = new byte[maximumSize]; + size = myStream.Read(myData, 0, maximumSize); myStream.Position = 0; } diff --git a/ColorzCore/Parser/EAParser.cs b/ColorzCore/Parser/EAParser.cs index 21ab4bb..b3e0020 100644 --- a/ColorzCore/Parser/EAParser.cs +++ b/ColorzCore/Parser/EAParser.cs @@ -25,9 +25,12 @@ class EAParser //TODO: Built in macros. //public static readonly Dictionary BuiltInMacros; public ImmutableStack GlobalScope { get; } - public int CurrentOffset { get { return currentOffset; } private set + public int CurrentOffset + { + get { return currentOffset; } + private set { - if (value > 0x2000000) + if (value > EAOptions.Instance.maximumRomSize) { if (validOffset) //Error only the first time. { @@ -55,13 +58,16 @@ class EAParser public Log log; - public bool IsIncluding { get + public bool IsIncluding + { + get { bool acc = true; for (ImmutableStack temp = Inclusion; !temp.IsEmpty && acc; temp = temp.Tail) acc &= temp.Head; return acc; - } } + } + } private bool validOffset; private bool offsetInitialized; // false until first ORG, used to warn about writing before first org private int currentOffset; @@ -114,7 +120,7 @@ public IList ParseAll(IEnumerable tokenStream) if (tokens.Current.Type != TokenType.NEWLINE || tokens.MoveNext()) { Maybe retVal = ParseLine(tokens, GlobalScope); - retVal.IfJust( (ILineNode n) => myLines.Add(n)); + retVal.IfJust((ILineNode n) => myLines.Add(n)); } } return myLines; @@ -137,6 +143,35 @@ private BlockNode ParseBlock(MergeableGenerator tokens, ImmutableStack 0 && value < EAOptions.Instance.maximumRomSize) + { + value += EAOptions.Instance.romBaseAddress; + } + + return value; + } + + public static int ConvertToOffset(int value) + { + if (value >= EAOptions.Instance.romBaseAddress && value <= EAOptions.Instance.romBaseAddress + EAOptions.Instance.maximumRomSize) + { + value -= EAOptions.Instance.romBaseAddress; + } + + return value; + } + private Maybe ParseStatement(MergeableGenerator tokens, ImmutableStack scopes) { while (ExpandIdentifier(tokens, scopes)) { } @@ -171,10 +206,7 @@ private Maybe ParseStatement(MergeableGenerator tokens, Immuta (IAtomNode atom) => atom.TryEvaluate((Exception e) => { Error(parameters[0].MyLocation, e.Message); }).IfJust( (int temp) => { - if (temp > 0x2000000) - Error(parameters[0].MyLocation, "Tried to set offset to 0x" + temp.ToString("X")); - else - CurrentOffset = temp; + CurrentOffset = ConvertToOffset(temp); }), () => { Error(parameters[0].MyLocation, "Expected atomic param to ORG."); } ); @@ -191,7 +223,8 @@ private Maybe ParseStatement(MergeableGenerator tokens, Immuta Error(head.Location, "Incorrect number of parameters in POP: " + parameters.Count); else if (pastOffsets.Count == 0) Error(head.Location, "POP without matching PUSH."); - else { + else + { Tuple tuple = pastOffsets.Pop(); CurrentOffset = tuple.Item1; @@ -310,7 +343,7 @@ private Maybe ParseStatement(MergeableGenerator tokens, Immuta var data = new byte[amount]; for (int i = 0; i < amount; ++i) - data[i] = (byte) value; + data[i] = (byte)value; var node = new DataNode(CurrentOffset, data); @@ -327,7 +360,7 @@ private Maybe ParseStatement(MergeableGenerator tokens, Immuta else if (Raws.ContainsKey(upperCodeIdentifier)) { //TODO: Check for matches. Currently should type error. - foreach(Raw r in Raws[upperCodeIdentifier]) + foreach (Raw r in Raws[upperCodeIdentifier]) { if (r.Fits(parameters)) { @@ -377,7 +410,7 @@ public IList> ParseMacroParamList(MergeableGenerator tokens) } parameters.Add(currentParam); } while (tokens.Current.Type != TokenType.CLOSE_PAREN && tokens.Current.Type != TokenType.NEWLINE); - if(tokens.Current.Type != TokenType.CLOSE_PAREN || parenNestings != 0) + if (tokens.Current.Type != TokenType.CLOSE_PAREN || parenNestings != 0) { Error(tokens.Current.Location, "Unmatched open parenthesis."); } @@ -408,9 +441,9 @@ private IList ParseParamList(MergeableGenerator tokens, Immut private IList ParsePreprocParamList(MergeableGenerator tokens, ImmutableStack scopes) { IList temp = ParseParamList(tokens, scopes, false); - for(int i=0; i ParseParam(MergeableGenerator tokens, Immutable if (expandDefs && Definitions.ContainsKey(head.Content) && ExpandIdentifier(tokens, scopes)) return ParseParam(tokens, scopes, expandDefs); else - return ParseAtom(tokens,scopes,expandDefs).Fmap((IAtomNode x) => (IParamNode)x.Simplify()); + return ParseAtom(tokens, scopes, expandDefs).Fmap((IAtomNode x) => (IParamNode)x.Simplify()); default: return ParseAtom(tokens, scopes, expandDefs).Fmap((IAtomNode x) => (IParamNode)x.Simplify()); } @@ -541,12 +574,12 @@ private Maybe ParseAtom(MergeableGenerator tokens, ImmutableSt //Assume unary negation. tokens.MoveNext(); Maybe interior = ParseAtom(tokens, scopes); - if(interior.IsNothing) + if (interior.IsNothing) { Error(lookAhead.Location, "Expected expression after negation. "); return new Nothing(); } - grammarSymbols.Push(new Left(new NegationNode(lookAhead, interior.FromJust))); + grammarSymbols.Push(new Left(new NegationNode(lookAhead, interior.FromJust))); break; } case TokenType.COMMA: @@ -628,7 +661,7 @@ private void Reduce(Stack> grammarSymbols, int targetPr //These shouldn't error... IAtomNode r = grammarSymbols.Pop().GetLeft; - if(precedences[grammarSymbols.Peek().GetRight.Type] > targetPrecedence) + if (precedences[grammarSymbols.Peek().GetRight.Type] > targetPrecedence) { grammarSymbols.Push(new Left(r)); break; @@ -701,7 +734,7 @@ public Maybe ParseLine(MergeableGenerator tokens, ImmutableSta { Warning(head.Location, "Label already in scope, ignoring: " + head.Content);//replacing: " + head.Content); } - else if(!IsValidLabelName(head.Content)) + else if (!IsValidLabelName(head.Content)) { Error(head.Location, "Invalid label name " + head.Content + '.'); } @@ -794,7 +827,7 @@ public bool ExpandIdentifier(MergeableGenerator tokens, ImmutableStack parameters) { StringBuilder sb = new StringBuilder(); - foreach(IParamNode parameter in parameters) + foreach (IParamNode parameter in parameters) { sb.Append(parameter.PrettyPrint()); sb.Append(' '); diff --git a/ColorzCore/Program.cs b/ColorzCore/Program.cs index a0286c8..2adb831 100644 --- a/ColorzCore/Program.cs +++ b/ColorzCore/Program.cs @@ -54,10 +54,14 @@ class Program " Enable debug mode. Not recommended for end users.", "--build-times", " Print build times at the end of build.", + "--base-address:", + " Treats the base load address of the binary as the given (hexadecimal) number,", + " for the purposes of POIN, ORG and CURRENTOFFSET. Defaults to 0x08000000.", + " Addresses are added to offsets from 0 to the maximum binary size.", + "--maximum-size:", + " Sets the maximum size of the binary. Defaults to 0x02000000.", "-romoffset:", - " Treats the offset of the ROM as the given number,", - " for the purposes of POIN. Addresses are or'd.", - " Hex literals only. Defaults to 0x08000000.", + " Compatibility alias for --base-address:", "-h|--help", " Display this message and exit.", "" @@ -77,7 +81,6 @@ static int Main(string[] args) Stream inStream = Console.OpenStandardInput(); string inFileName = "stdin"; - IOutput? output = null; string outFileName = "none"; string ldsFileName = "none"; @@ -88,7 +91,7 @@ static int Main(string[] args) if (args.Length < 2) { - Console.WriteLine("Required parameters missing."); + Console.WriteLine(helpstring); return EXIT_FAILURE; } @@ -126,27 +129,6 @@ static int Main(string[] args) case "output": outFileName = flag[1]; - if(outputASM) - { - ldsFileName = Path.ChangeExtension(outFileName, "lds"); - output = new ASM(new StreamWriter(outFileName, false), - new StreamWriter(ldsFileName, false)); - } else - { - FileStream outStream; - if(File.Exists(outFileName) && !File.GetAttributes(outFileName).HasFlag(FileAttributes.ReadOnly)) - { - outStream = File.Open(outFileName, FileMode.Open, FileAccess.ReadWrite); - } else if(!File.Exists(outFileName)) - { - outStream = File.Create(outFileName); - } else - { - Console.Error.WriteLine("Output file is read-only."); - return EXIT_FAILURE; - } - output = new ROM(outStream); - } break; case "input": @@ -216,21 +198,37 @@ static int Main(string[] args) case "D": case "def": case "define": - try { + try + { string[] def_args = flag[1].Split(new char[] { '=' }, 2); EAOptions.Instance.defs.Add(Tuple.Create(def_args[0], def_args[1])); - } catch (IndexOutOfRangeException) + } + catch (IndexOutOfRangeException) { Console.Error.WriteLine("Improperly formed -define directive."); } break; case "romoffset": + case "-base-address": + try + { + EAOptions.Instance.romBaseAddress = Convert.ToInt32(flag[1], 16); + } + catch + { + Console.Error.WriteLine("Invalid hex base address given for binary."); + } + break; + + case "-maximum-size": try { - EAOptions.Instance.romOffset = Convert.ToInt32(flag[1], 16); - } catch { - Console.Error.WriteLine("Invalid hex offset given for ROM."); + EAOptions.Instance.maximumRomSize = Convert.ToInt32(flag[1], 16); + } + catch + { + Console.Error.WriteLine("Invalid hex size given for binary."); } break; @@ -246,8 +244,8 @@ static int Main(string[] args) } } } - - if (output == null) + + if (outFileName == null) { Console.Error.WriteLine("No output specified for assembly."); return EXIT_FAILURE; @@ -259,11 +257,41 @@ static int Main(string[] args) return EXIT_FAILURE; } + IOutput output; + + if (outputASM) + { + ldsFileName = Path.ChangeExtension(outFileName, "lds"); + output = new ASM(new StreamWriter(outFileName, false), + new StreamWriter(ldsFileName, false)); + } + else + { + FileStream outStream; + + if (File.Exists(outFileName) && !File.GetAttributes(outFileName).HasFlag(FileAttributes.ReadOnly)) + { + outStream = File.Open(outFileName, FileMode.Open, FileAccess.ReadWrite); + } + else if (!File.Exists(outFileName)) + { + outStream = File.Create(outFileName); + } + else + { + Console.Error.WriteLine("Output file is read-only."); + return EXIT_FAILURE; + } + + output = new ROM(outStream, EAOptions.Instance.maximumRomSize); + } + string game = args[1]; //FirstPass(Tokenizer.Tokenize(inputStream)); - Log log = new Log { + Log log = new Log + { Output = errorStream, WarningsAreErrors = EAOptions.Instance.werr, NoColoredTags = EAOptions.Instance.noColoredLog @@ -292,25 +320,24 @@ static int Main(string[] args) } } - if (EAOptions.Instance.buildTimes) { - - // Print times - - log.Output.WriteLine(); - log.Output.WriteLine("Times:"); - - foreach (KeyValuePair time in ExecTimer.Timer.SortedTimes) + if (EAOptions.Instance.buildTimes) { - log.Output.WriteLine(" " + time.Value + ": " + time.Key.ToString() + " (" + ExecTimer.Timer.Counts[time.Value] + ")"); - } + // Print times + + log.Output.WriteLine(); + log.Output.WriteLine("Times:"); - // Print total time + foreach (KeyValuePair time in ExecTimer.Timer.SortedTimes) + { + log.Output.WriteLine(" " + time.Value + ": " + time.Key.ToString() + " (" + ExecTimer.Timer.Counts[time.Value] + ")"); + } - log.Output.WriteLine(); - log.Output.WriteLine("Total:"); + // Print total time - log.Output.WriteLine(" " + ExecTimer.Timer.TotalTime.ToString()); + log.Output.WriteLine(); + log.Output.WriteLine("Total:"); + log.Output.WriteLine(" " + ExecTimer.Timer.TotalTime.ToString()); } inStream.Close(); diff --git a/ColorzCore/Raws/AtomicParam.cs b/ColorzCore/Raws/AtomicParam.cs index c328067..34268c2 100644 --- a/ColorzCore/Raws/AtomicParam.cs +++ b/ColorzCore/Raws/AtomicParam.cs @@ -1,11 +1,8 @@ using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; +using System.Collections; +using ColorzCore.Parser; using ColorzCore.Parser.AST; using ColorzCore.DataTypes; -using System.Collections; namespace ColorzCore.Raws { @@ -34,9 +31,10 @@ public void Set(byte[] data, IParamNode input) public void Set(byte[] data, int value) { - if (pointer && value != 0) - value |= EAOptions.Instance.romOffset; - //TODO: Perhaps additional EAOption to add for ROM offsets that aren't address spaces, e.g. program being loaded at 0x100? + if (pointer) + { + value = EAParser.ConvertToAddress(value); + } data.SetBits(Position, Length, value); } From 3f87c6cf41b029745bf2dcab7240a50262b70463 Mon Sep 17 00:00:00 2001 From: Nat_776 Date: Mon, 15 Apr 2024 19:34:29 +0200 Subject: [PATCH 07/59] fix some warnings --- ColorzCore/ExecTimer.cs | 6 +++--- ColorzCore/Lexer/Tokenizer.cs | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ColorzCore/ExecTimer.cs b/ColorzCore/ExecTimer.cs index 45277b9..d8ffbb5 100644 --- a/ColorzCore/ExecTimer.cs +++ b/ColorzCore/ExecTimer.cs @@ -18,8 +18,8 @@ public class ExecTimer private Dictionary? times; private Dictionary? counts; - private TimeSpan totalTime; - + private TimeSpan totalTime = TimeSpan.Zero; + private ExecTimer() { timingPoints = new List>(); @@ -72,7 +72,7 @@ public TimeSpan TotalTime { get { - if (this.totalTime == null) + if (this.totalTime == TimeSpan.Zero) ComputeTimes(); return this.totalTime; diff --git a/ColorzCore/Lexer/Tokenizer.cs b/ColorzCore/Lexer/Tokenizer.cs index f48c264..a81388a 100644 --- a/ColorzCore/Lexer/Tokenizer.cs +++ b/ColorzCore/Lexer/Tokenizer.cs @@ -305,8 +305,8 @@ public IEnumerable Tokenize(Stream input, string fileName) int curLine = 1; while (!sin.EndOfStream) { - string? line = sin.ReadLine(); - + string line = sin.ReadLine()!; + //allow escaping newlines while (line.Length > 0 && line.Substring(line.Length-1) == "\\") { From 840086df8fc0296a49ef09295507744a0b19ad13 Mon Sep 17 00:00:00 2001 From: Nat_776 Date: Tue, 16 Apr 2024 18:21:33 +0200 Subject: [PATCH 08/59] refactor use of Maybe --- ColorzCore/DataTypes/ImmutableStack.cs | 45 +++--- ColorzCore/DataTypes/Maybe.cs | 100 ++++++------ ColorzCore/IO/IncludeFileSearcher.cs | 16 +- ColorzCore/Parser/AST/AtomNodeKernel.cs | 10 +- ColorzCore/Parser/AST/IAtomNode.cs | 20 +-- ColorzCore/Parser/AST/IParamNode.cs | 2 +- ColorzCore/Parser/AST/IdentifierNode.cs | 31 ++-- ColorzCore/Parser/AST/ListNode.cs | 10 +- ColorzCore/Parser/AST/MacroInvocationNode.cs | 4 +- ColorzCore/Parser/AST/NegationNode.cs | 6 +- ColorzCore/Parser/AST/NumberNode.cs | 14 +- ColorzCore/Parser/AST/OperatorNode.cs | 38 +++-- ColorzCore/Parser/AST/StringNode.cs | 2 +- ColorzCore/Parser/EAParser.cs | 147 +++++++++--------- ColorzCore/Preprocessor/DirectiveHandler.cs | 4 +- .../Directives/DefineDirective.cs | 77 +++++---- .../Preprocessor/Directives/ElseDirective.cs | 4 +- .../Preprocessor/Directives/EndIfDirective.cs | 4 +- .../Preprocessor/Directives/IDirective.cs | 2 +- .../Directives/IfDefinedDirective.cs | 10 +- .../Directives/IfNotDefinedDirective.cs | 10 +- .../Directives/IncludeBinaryDirective.cs | 12 +- .../Directives/IncludeDirective.cs | 12 +- .../Directives/IncludeExternalDirective.cs | 14 +- .../Directives/IncludeToolEventDirective.cs | 14 +- .../Preprocessor/Directives/PoolDirective.cs | 4 +- .../Directives/UndefineDirective.cs | 4 +- ColorzCore/Program.cs | 6 +- ColorzCore/Raws/AtomicParam.cs | 2 +- ColorzCore/Raws/Raw.cs | 10 +- ColorzCore/Raws/RawReader.cs | 12 +- 31 files changed, 317 insertions(+), 329 deletions(-) diff --git a/ColorzCore/DataTypes/ImmutableStack.cs b/ColorzCore/DataTypes/ImmutableStack.cs index 51e5aef..ea4a2bb 100644 --- a/ColorzCore/DataTypes/ImmutableStack.cs +++ b/ColorzCore/DataTypes/ImmutableStack.cs @@ -9,42 +9,32 @@ namespace ColorzCore.DataTypes { public class ImmutableStack : IEnumerable { - private Maybe>> member; - int? count; + private readonly Tuple>? member; + private int? count; public ImmutableStack(T elem, ImmutableStack tail) - { member = new Just>>(new Tuple>(elem, tail)); } - private ImmutableStack() { - member = new Nothing>>(); - count = null; + member = new Tuple>(elem, tail); } - private static ImmutableStack emptyList = new ImmutableStack(); - public static ImmutableStack Nil { get { return emptyList;} } - - public bool IsEmpty { get { return member.IsNothing; } } - public T Head { get { return member.FromJust.Item1; } } - public ImmutableStack Tail { get { return member.FromJust.Item2; } } - public int Count { get - { - if (count.HasValue) - return count.Value; - else - return (count = Tail.Count + 1).Value; - } } - /* - public bool Contains(T toLookFor) + private ImmutableStack() { - bool acc = false; - for(ImmutableStack temp = this; !acc && !temp.IsEmpty; temp = temp.Tail) acc |= temp.Head.Equals(toLookFor); - return acc; + member = null; + count = 0; } - */ + + public static ImmutableStack Nil { get; } = new ImmutableStack(); + + public bool IsEmpty => member == null; + public T Head => member!.Item1; + public ImmutableStack Tail => member!.Item2; + public int Count => count ?? (count = Tail.Count + 1).Value; + public IEnumerator GetEnumerator() { ImmutableStack temp = this; - while(!temp.IsEmpty) + + while (!temp.IsEmpty) { yield return temp.Head; temp = temp.Tail; @@ -54,6 +44,7 @@ public IEnumerator GetEnumerator() IEnumerator IEnumerable.GetEnumerator() { ImmutableStack temp = this; + while (!temp.IsEmpty) { yield return temp.Head; @@ -63,7 +54,7 @@ IEnumerator IEnumerable.GetEnumerator() public static ImmutableStack FromEnumerable(IEnumerable content) { - return content.Reverse().Aggregate(Nil, (ImmutableStack acc, T elem) => new ImmutableStack(elem, acc)); + return content.Reverse().Aggregate(Nil, (acc, elem) => new ImmutableStack(elem, acc)); } } } diff --git a/ColorzCore/DataTypes/Maybe.cs b/ColorzCore/DataTypes/Maybe.cs index 6ffcff3..9a86c1f 100644 --- a/ColorzCore/DataTypes/Maybe.cs +++ b/ColorzCore/DataTypes/Maybe.cs @@ -6,74 +6,70 @@ namespace ColorzCore.DataTypes { - public delegate R UnaryFunction(T val); + public delegate R UnaryFunction(T val); public delegate R RConst(); public delegate void TAction(T val); public delegate void NullaryAction(); - public delegate Maybe MaybeAction(T val); + public delegate R? MaybeAction(T val); -#pragma warning disable IDE1006 // Naming Styles - public interface Maybe -#pragma warning restore IDE1006 // Naming Styles + public static class MaybeExtensions { - bool IsNothing { get; } - T FromJust { get; } - Maybe Fmap(UnaryFunction f); - Maybe Bind(MaybeAction f); - R IfJust(UnaryFunction just, RConst nothing); - void IfJust(TAction just, NullaryAction? nothing = null); - } - public class Just : Maybe - { - public bool IsNothing { get { return false; } } - public T FromJust { get; } - - public Just(T val) + public static R? Fmap(this T? self, UnaryFunction f) + where T : class { - FromJust = val; + return self != null ? f(self) : default; } - public Maybe Fmap(UnaryFunction f) - { - return new Just(f(FromJust)); - } - public Maybe Bind(MaybeAction f) - { - return f(FromJust); - } - public R IfJust(UnaryFunction just, RConst nothing) - { - return just(FromJust); - } - public void IfJust(TAction just, NullaryAction? nothing) + public static R IfJust(this T? self, UnaryFunction just, RConst nothing) + where T : class { - just(FromJust); + if (self != null) + { + return just(self); + } + else + { + return nothing(); + } } - } - public class Nothing : Maybe - { - public bool IsNothing { get { return true; } } - public T FromJust { get { throw new MaybeException(); } } - public Nothing() { } - - public Maybe Fmap(UnaryFunction f) - { - return new Nothing(); - } - public Maybe Bind(MaybeAction f) + public static R IfJust(this T? self, UnaryFunction just, RConst nothing) + where T : struct { - return new Nothing(); + if (self.HasValue) + { + return just(self.Value); + } + else + { + return nothing(); + } } - public R IfJust(UnaryFunction just, RConst nothing) + + public static void IfJust(this T? self, TAction just, NullaryAction? nothing = null) + where T : class { - return nothing(); + if (self != null) + { + just(self); + } + else + { + nothing?.Invoke(); + } } - public void IfJust(TAction just, NullaryAction? nothing) + + public static void IfJust(this T? self, TAction just, NullaryAction? nothing = null) + where T : struct { - nothing?.Invoke(); + if (self.HasValue) + { + just(self.Value); + } + else + { + nothing?.Invoke(); + } } } - - public class MaybeException : Exception { } } diff --git a/ColorzCore/IO/IncludeFileSearcher.cs b/ColorzCore/IO/IncludeFileSearcher.cs index 2e1c63d..2195ed3 100644 --- a/ColorzCore/IO/IncludeFileSearcher.cs +++ b/ColorzCore/IO/IncludeFileSearcher.cs @@ -10,40 +10,40 @@ public class IncludeFileSearcher public List IncludeDirectories { get; } = new List(); public bool AllowRelativeInclude { get; set; } = true; - public Maybe FindFile(string name) + public string? FindFile(string name) { return FindFile(null, name); } - public Maybe FindFile(string? cwd, string name) + public string? FindFile(string? cwd, string name) { // Find the first valid file in the list of possible file paths foreach (string path in EnumeratePossibleAccessPaths(cwd, name)) { if (File.Exists(path)) - return new Just(path); + return path; } - return new Nothing(); + return null; } - public Maybe FindDirectory(string name) + public string? FindDirectory(string name) { return FindDirectory(null, name); } - public Maybe FindDirectory(string? cwd, string name) + public string? FindDirectory(string? cwd, string name) { // Find the first valid directory in the list of possible file paths foreach (string path in EnumeratePossibleAccessPaths(cwd, name)) { if (Directory.Exists(path)) - return new Just(path); + return path; } - return new Nothing(); + return null; } protected IEnumerable EnumeratePossibleAccessPaths(string? cwd, string name) diff --git a/ColorzCore/Parser/AST/AtomNodeKernel.cs b/ColorzCore/Parser/AST/AtomNodeKernel.cs index 8998bb6..a8f3c83 100644 --- a/ColorzCore/Parser/AST/AtomNodeKernel.cs +++ b/ColorzCore/Parser/AST/AtomNodeKernel.cs @@ -14,25 +14,25 @@ public abstract class AtomNodeKernel : IAtomNode public ParamType Type { get { return ParamType.ATOM; } } - public virtual Maybe GetIdentifier() + public virtual string? GetIdentifier() { - return new Nothing(); + return null; } public abstract string PrettyPrint(); public abstract IEnumerable ToTokens(); public abstract Location MyLocation { get; } - public abstract Maybe TryEvaluate(TAction handler); + public abstract int? TryEvaluate(TAction handler); public IParamNode SimplifyExpressions(TAction handler) { return this.Simplify(handler); } - public Maybe AsAtom() + public IAtomNode? AsAtom() { - return new Just(this); + return this; } } } diff --git a/ColorzCore/Parser/AST/IAtomNode.cs b/ColorzCore/Parser/AST/IAtomNode.cs index 9311846..071126b 100644 --- a/ColorzCore/Parser/AST/IAtomNode.cs +++ b/ColorzCore/Parser/AST/IAtomNode.cs @@ -11,27 +11,27 @@ namespace ColorzCore.Parser.AST public interface IAtomNode : IParamNode { //TODO: Simplify() partial evaluation as much as is defined, to save on memory space. - int Precedence { get; } - Maybe GetIdentifier(); + int Precedence { get; } + string? GetIdentifier(); IEnumerable ToTokens(); - Maybe TryEvaluate(TAction handler); //Simplifies the AST as much as possible. + int? TryEvaluate(TAction handler); //Simplifies the AST as much as possible. } public static class AtomExtensions { public static int CoerceInt(this IAtomNode n) { - return n.TryEvaluate((Exception e) => { throw e; }).FromJust; + return n.TryEvaluate(e => throw e)!.Value; } - public static IAtomNode Simplify(this IAtomNode n, TAction handler) + + public static IAtomNode Simplify(this IAtomNode self, TAction handler) { - IAtomNode ret = n; - n.TryEvaluate(handler).IfJust((int i) => { ret = FromInt(n.MyLocation, i); }); - return ret; + return self.TryEvaluate(handler).IfJust(intValue => FromInt(self.MyLocation, intValue), () => self); } - public static IAtomNode FromInt(Location l, int i) + + public static IAtomNode FromInt(Location location, int intValue) { - return new NumberNode(l, i); + return new NumberNode(location, intValue); } } } diff --git a/ColorzCore/Parser/AST/IParamNode.cs b/ColorzCore/Parser/AST/IParamNode.cs index 73a6ac0..74c0465 100644 --- a/ColorzCore/Parser/AST/IParamNode.cs +++ b/ColorzCore/Parser/AST/IParamNode.cs @@ -15,7 +15,7 @@ public interface IParamNode string PrettyPrint(); Location MyLocation { get; } IParamNode SimplifyExpressions(TAction handler); //TODO: Abstract this into a general traverse method. - Maybe AsAtom(); + IAtomNode? AsAtom(); } public static class ParamExtensions diff --git a/ColorzCore/Parser/AST/IdentifierNode.cs b/ColorzCore/Parser/AST/IdentifierNode.cs index c0db903..e911d93 100644 --- a/ColorzCore/Parser/AST/IdentifierNode.cs +++ b/ColorzCore/Parser/AST/IdentifierNode.cs @@ -11,21 +11,21 @@ public class IdentifierNode : AtomNodeKernel private Token identifier; readonly ImmutableStack scope; - public override int Precedence { get { return 11; } } + public override int Precedence { get { return 11; } } public override Location MyLocation { get { return identifier.Location; } } public IdentifierNode(Token id, ImmutableStack scopes) - { + { identifier = id; scope = scopes; - } - - private int ToInt() + } + + private int ToInt() { ImmutableStack temp = scope; - while(!temp.IsEmpty) + while (!temp.IsEmpty) { - if(temp.Head.HasLocalLabel(identifier.Content)) + if (temp.Head.HasLocalLabel(identifier.Content)) return temp.Head.GetLabel(identifier.Content); else temp = temp.Tail; @@ -33,28 +33,29 @@ private int ToInt() throw new UndefinedIdentifierException(identifier); } - public override Maybe TryEvaluate(TAction handler) + public override int? TryEvaluate(TAction handler) { try { - return new Just(ToInt()); - } catch(UndefinedIdentifierException e) + return ToInt(); + } + catch (UndefinedIdentifierException e) { handler(e); - return new Nothing(); + return null; } } - - public override Maybe GetIdentifier() + + public override string? GetIdentifier() { - return new Just(identifier.Content); + return identifier.Content; } public override string PrettyPrint() { try { - return "0x"+ToInt().ToString("X"); + return "0x" + ToInt().ToString("X"); } catch (UndefinedIdentifierException) { diff --git a/ColorzCore/Parser/AST/ListNode.cs b/ColorzCore/Parser/AST/ListNode.cs index 92ef528..4255f3c 100644 --- a/ColorzCore/Parser/AST/ListNode.cs +++ b/ColorzCore/Parser/AST/ListNode.cs @@ -25,10 +25,10 @@ public override string ToString() { StringBuilder sb = new StringBuilder(); sb.Append('['); - for(int i=0; i ToTokens() { //Similar code to ParenthesizedAtom IList> temp = new List>(); - foreach(IAtomNode n in Interior) + foreach (IAtomNode n in Interior) { temp.Add(new List(n.ToTokens())); } @@ -82,7 +82,7 @@ public Either TryEvaluate() public IParamNode SimplifyExpressions(TAction handler) { IEnumerable acc = new List(); - for(int i=0; i handler) public int NumCoords { get { return Interior.Count; } } - public Maybe AsAtom() { return new Nothing(); } + public IAtomNode? AsAtom() { return null; } } } diff --git a/ColorzCore/Parser/AST/MacroInvocationNode.cs b/ColorzCore/Parser/AST/MacroInvocationNode.cs index 65e42b2..43814f7 100644 --- a/ColorzCore/Parser/AST/MacroInvocationNode.cs +++ b/ColorzCore/Parser/AST/MacroInvocationNode.cs @@ -39,7 +39,7 @@ public string PrettyPrint() StringBuilder sb = new StringBuilder(); sb.Append(invokeToken.Content); sb.Append('('); - for(int i=0; i TryEvaluate() public Location MyLocation { get { return invokeToken.Location; } } - public Maybe AsAtom() { return new Nothing(); } + public IAtomNode? AsAtom() { return null; } public IParamNode SimplifyExpressions(TAction handler) { diff --git a/ColorzCore/Parser/AST/NegationNode.cs b/ColorzCore/Parser/AST/NegationNode.cs index 43ab565..50565ad 100644 --- a/ColorzCore/Parser/AST/NegationNode.cs +++ b/ColorzCore/Parser/AST/NegationNode.cs @@ -30,13 +30,13 @@ public override string PrettyPrint() public override IEnumerable ToTokens() { yield return myToken; - foreach(Token t in interior.ToTokens()) + foreach (Token t in interior.ToTokens()) yield return t; } - public override Maybe TryEvaluate(TAction handler) + public override int? TryEvaluate(TAction handler) { - return interior.TryEvaluate(handler).Fmap((int x) => -x); + return -interior.TryEvaluate(handler); } } } diff --git a/ColorzCore/Parser/AST/NumberNode.cs b/ColorzCore/Parser/AST/NumberNode.cs index d8bcca4..f2e02e8 100644 --- a/ColorzCore/Parser/AST/NumberNode.cs +++ b/ColorzCore/Parser/AST/NumberNode.cs @@ -15,10 +15,10 @@ public class NumberNode : AtomNodeKernel public override Location MyLocation { get; } public override int Precedence { get { return 11; } } - public NumberNode(Token num) - { + public NumberNode(Token num) + { MyLocation = num.Location; - value = num.Content.ToInt(); + value = num.Content.ToInt(); } public NumberNode(Token text, int value) { @@ -30,12 +30,12 @@ public NumberNode(Location loc, int value) MyLocation = loc; this.value = value; } - - public override IEnumerable ToTokens () { yield return new Token(TokenType.NUMBER, MyLocation, value.ToString()); } - public override Maybe TryEvaluate(TAction handler) + public override IEnumerable ToTokens() { yield return new Token(TokenType.NUMBER, MyLocation, value.ToString()); } + + public override int? TryEvaluate(TAction handler) { - return new Just(value); + return value; } public override string PrettyPrint() diff --git a/ColorzCore/Parser/AST/OperatorNode.cs b/ColorzCore/Parser/AST/OperatorNode.cs index 65b751c..6ac4acc 100644 --- a/ColorzCore/Parser/AST/OperatorNode.cs +++ b/ColorzCore/Parser/AST/OperatorNode.cs @@ -10,7 +10,7 @@ namespace ColorzCore.Parser.AST { delegate int BinaryIntOp(int a, int b); - + class OperatorNode : AtomNodeKernel { public static readonly Dictionary Operators = new Dictionary { @@ -24,27 +24,27 @@ class OperatorNode : AtomNodeKernel { TokenType.SIGNED_RSHIFT_OP , (x, y) => x>>y }, { TokenType.AND_OP , (x, y) => x&y }, { TokenType.XOR_OP , (x, y) => x^y }, - { TokenType.OR_OP , (x, y) => x|y } + { TokenType.OR_OP , (x, y) => x|y } }; - + private IAtomNode left, right; private Token op; - public override int Precedence { get; } + public override int Precedence { get; } public override Location MyLocation { get { return op.Location; } } - - public OperatorNode(IAtomNode l, Token op, IAtomNode r, int prec) - { + + public OperatorNode(IAtomNode l, Token op, IAtomNode r, int prec) + { left = l; right = r; this.op = op; Precedence = prec; - } + } public override string PrettyPrint() { StringBuilder sb = new StringBuilder(left.PrettyPrint()); - switch(op.Type) + switch (op.Type) { case TokenType.MUL_OP: sb.Append("*"); @@ -84,24 +84,28 @@ public override string PrettyPrint() } public override IEnumerable ToTokens() { - foreach(Token t in left.ToTokens()) + foreach (Token t in left.ToTokens()) { yield return t; } yield return op; - foreach(Token t in right.ToTokens()) + foreach (Token t in right.ToTokens()) { yield return t; } } - public override Maybe TryEvaluate(TAction handler) + public override int? TryEvaluate(TAction handler) { - Maybe l = left.TryEvaluate(handler); - l.IfJust((int i) => { this.left = new NumberNode(left.MyLocation, i); }); - Maybe r = right.TryEvaluate(handler); - r.IfJust((int i) => { this.right = new NumberNode(right.MyLocation, i); }); - return l.Bind((int newL) => r.Fmap((int newR) => Operators[op.Type](newL, newR))); + int? l = left.TryEvaluate(handler); + l.IfJust(i => left = new NumberNode(left.MyLocation, i)); + int? r = right.TryEvaluate(handler); + r.IfJust(i => right = new NumberNode(right.MyLocation, i)); + + if (l is int li && r is int ri) + return Operators[op.Type](li, ri); + + return null; } } } diff --git a/ColorzCore/Parser/AST/StringNode.cs b/ColorzCore/Parser/AST/StringNode.cs index e9effae..9664025 100644 --- a/ColorzCore/Parser/AST/StringNode.cs +++ b/ColorzCore/Parser/AST/StringNode.cs @@ -51,7 +51,7 @@ public IdentifierNode ToIdentifier(ImmutableStack scope) return new IdentifierNode(MyToken, scope); } - public Maybe AsAtom() { return new Nothing(); } + public IAtomNode? AsAtom() { return null; } public IParamNode SimplifyExpressions(TAction handler) { return this; } } diff --git a/ColorzCore/Parser/EAParser.cs b/ColorzCore/Parser/EAParser.cs index b3e0020..d8cf4dd 100644 --- a/ColorzCore/Parser/EAParser.cs +++ b/ColorzCore/Parser/EAParser.cs @@ -119,8 +119,8 @@ public IList ParseAll(IEnumerable tokenStream) { if (tokens.Current.Type != TokenType.NEWLINE || tokens.MoveNext()) { - Maybe retVal = ParseLine(tokens, GlobalScope); - retVal.IfJust((ILineNode n) => myLines.Add(n)); + ILineNode? retVal = ParseLine(tokens, GlobalScope); + retVal.IfJust(n => myLines.Add(n)); } } return myLines; @@ -131,11 +131,12 @@ private BlockNode ParseBlock(MergeableGenerator tokens, ImmutableStack x; + while (!tokens.EOS && tokens.Current.Type != TokenType.CLOSE_BRACE) { - if (!(x = ParseLine(tokens, scopes)).IsNothing) - temp.Children.Add(x.FromJust); + ILineNode? x = ParseLine(tokens, scopes); + if (x != null) + temp.Children.Add(x); } if (!tokens.EOS) tokens.MoveNext(); @@ -172,7 +173,7 @@ public static int ConvertToOffset(int value) return value; } - private Maybe ParseStatement(MergeableGenerator tokens, ImmutableStack scopes) + private ILineNode? ParseStatement(MergeableGenerator tokens, ImmutableStack scopes) { while (ExpandIdentifier(tokens, scopes)) { } head = tokens.Current; @@ -203,8 +204,8 @@ private Maybe ParseStatement(MergeableGenerator tokens, Immuta else { parameters[0].AsAtom().IfJust( - (IAtomNode atom) => atom.TryEvaluate((Exception e) => { Error(parameters[0].MyLocation, e.Message); }).IfJust( - (int temp) => + atom => atom.TryEvaluate(e => { Error(parameters[0].MyLocation, e.Message); }).IfJust( + temp => { CurrentOffset = ConvertToOffset(temp); }), @@ -247,10 +248,10 @@ private Maybe ParseStatement(MergeableGenerator tokens, Immuta { parameters[0].AsAtom().IfJust( - (IAtomNode atom) => + atom => { - atom.TryEvaluate((Exception e) => { Error(parameters[0].MyLocation, e.Message); }).IfJust( - (int temp) => + atom.TryEvaluate(e => { Error(parameters[0].MyLocation, e.Message); }).IfJust( + temp => { if (temp < 0) Error(parameters[0].MyLocation, "Assertion error: " + temp); @@ -263,8 +264,8 @@ private Maybe ParseStatement(MergeableGenerator tokens, Immuta case "PROTECT": if (parameters.Count == 1) parameters[0].AsAtom().IfJust( - (IAtomNode atom) => atom.TryEvaluate((Exception e) => { Error(parameters[0].MyLocation, e.Message); }).IfJust( - (int temp) => + atom => atom.TryEvaluate(e => { Error(parameters[0].MyLocation, e.Message); }).IfJust( + temp => { protectedRegions.Add(new Tuple(temp, 4, head.Location)); }), @@ -274,15 +275,15 @@ private Maybe ParseStatement(MergeableGenerator tokens, Immuta int start = 0, end = 0; bool errorOccurred = false; parameters[0].AsAtom().IfJust( - (IAtomNode atom) => atom.TryEvaluate((Exception e) => { Error(parameters[0].MyLocation, e.Message); errorOccurred = true; }).IfJust( - (int temp) => + atom => atom.TryEvaluate(e => { Error(parameters[0].MyLocation, e.Message); errorOccurred = true; }).IfJust( + temp => { start = temp; }), () => { Error(parameters[0].MyLocation, "Expected atomic param to PROTECT"); errorOccurred = true; }); parameters[1].AsAtom().IfJust( - (IAtomNode atom) => atom.TryEvaluate((Exception e) => { Error(parameters[0].MyLocation, e.Message); errorOccurred = true; }).IfJust( - (int temp) => + atom => atom.TryEvaluate(e => { Error(parameters[0].MyLocation, e.Message); errorOccurred = true; }).IfJust( + temp => { end = temp; }), @@ -304,8 +305,8 @@ private Maybe ParseStatement(MergeableGenerator tokens, Immuta Error(head.Location, "Incorrect number of parameters in ALIGN: " + parameters.Count); else parameters[0].AsAtom().IfJust( - (IAtomNode atom) => atom.TryEvaluate((Exception e) => { Error(parameters[0].MyLocation, e.Message); }).IfJust( - (int temp) => + atom => atom.TryEvaluate(e => { Error(parameters[0].MyLocation, e.Message); }).IfJust( + temp => { CurrentOffset = CurrentOffset % temp != 0 ? CurrentOffset + temp - CurrentOffset % temp : CurrentOffset; }), @@ -329,15 +330,15 @@ private Maybe ParseStatement(MergeableGenerator tokens, Immuta // param 2 (if given) is fill value parameters[1].AsAtom().IfJust( - (IAtomNode atom) => atom.TryEvaluate((Exception e) => { Error(parameters[0].MyLocation, e.Message); }).IfJust( - (int val) => { value = val; }), + atom => atom.TryEvaluate(e => { Error(parameters[0].MyLocation, e.Message); }).IfJust( + val => { value = val; }), () => { Error(parameters[0].MyLocation, "Expected atomic param to FILL"); }); } // param 1 is amount of bytes to fill parameters[0].AsAtom().IfJust( - (IAtomNode atom) => atom.TryEvaluate((Exception e) => { Error(parameters[0].MyLocation, e.Message); }).IfJust( - (int val) => { amount = val; }), + atom => atom.TryEvaluate(e => { Error(parameters[0].MyLocation, e.Message); }).IfJust( + val => { amount = val; }), () => { Error(parameters[0].MyLocation, "Expected atomic param to FILL"); }); var data = new byte[amount]; @@ -350,12 +351,12 @@ private Maybe ParseStatement(MergeableGenerator tokens, Immuta CheckDataWrite(amount); CurrentOffset += amount; - return new Just(node); + return node; } break; } - return new Nothing(); + return null; } else if (Raws.ContainsKey(upperCodeIdentifier)) { @@ -374,18 +375,18 @@ private Maybe ParseStatement(MergeableGenerator tokens, Immuta CheckDataWrite(temp.Size); CurrentOffset += temp.Size; //TODO: more efficient spacewise to just have contiguous writing and not an offset with every line? - return new Just(temp); + return temp; } } //TODO: Better error message (a la EA's ATOM ATOM [ATOM,ATOM]) Error(head.Location, "Incorrect parameters in raw " + head.Content + '.'); IgnoreRestOfStatement(tokens); - return new Nothing(); + return null; } else //TODO: Move outside of this else. { Error(head.Location, "Unrecognized code: " + head.Content); - return new Nothing(); + return null; } } @@ -429,7 +430,7 @@ private IList ParseParamList(MergeableGenerator tokens, Immut { Token head = tokens.Current; ParseParam(tokens, scopes, expandFirstDef || !first).IfJust( - (IParamNode n) => paramList.Add(n), + n => paramList.Add(n), () => Error(head.Location, "Expected parameter.")); first = false; } @@ -451,16 +452,16 @@ private IList ParsePreprocParamList(MergeableGenerator tokens return temp; } - private Maybe ParseParam(MergeableGenerator tokens, ImmutableStack scopes, bool expandDefs = true) + private IParamNode? ParseParam(MergeableGenerator tokens, ImmutableStack scopes, bool expandDefs = true) { Token head = tokens.Current; switch (tokens.Current.Type) { case TokenType.OPEN_BRACKET: - return new Just(new ListNode(head.Location, ParseList(tokens, scopes)).Simplify()); + return new ListNode(head.Location, ParseList(tokens, scopes)).Simplify(); case TokenType.STRING: tokens.MoveNext(); - return new Just(new StringNode(head)); + return new StringNode(head); case TokenType.MAYBE_MACRO: //TODO: Move this and the one in ExpandId to a separate ParseMacroNode that may return an Invocation. if (expandDefs && ExpandIdentifier(tokens, scopes)) @@ -472,15 +473,15 @@ private Maybe ParseParam(MergeableGenerator tokens, Immutable tokens.MoveNext(); IList> param = ParseMacroParamList(tokens); //TODO: Smart errors if trying to redefine a macro with the same num of params. - return new Just(new MacroInvocationNode(this, head, param, scopes)); + return new MacroInvocationNode(this, head, param, scopes); } case TokenType.IDENTIFIER: if (expandDefs && Definitions.ContainsKey(head.Content) && ExpandIdentifier(tokens, scopes)) return ParseParam(tokens, scopes, expandDefs); else - return ParseAtom(tokens, scopes, expandDefs).Fmap((IAtomNode x) => (IParamNode)x.Simplify()); + return ParseAtom(tokens, scopes, expandDefs).Fmap(x => (IParamNode)x.Simplify()); default: - return ParseAtom(tokens, scopes, expandDefs).Fmap((IAtomNode x) => (IParamNode)x.Simplify()); + return ParseAtom(tokens, scopes, expandDefs).Fmap(x => (IParamNode)x.Simplify()); } } @@ -500,7 +501,7 @@ private Maybe ParseParam(MergeableGenerator tokens, Immutable - private Maybe ParseAtom(MergeableGenerator tokens, ImmutableStack scopes, bool expandDefs = true) + private IAtomNode? ParseAtom(MergeableGenerator tokens, ImmutableStack scopes, bool expandDefs = true) { //Use Shift Reduce Parsing Token head = tokens.Current; @@ -551,20 +552,20 @@ private Maybe ParseAtom(MergeableGenerator tokens, ImmutableSt case TokenType.OPEN_PAREN: { tokens.MoveNext(); - Maybe interior = ParseAtom(tokens, scopes); + IAtomNode? interior = ParseAtom(tokens, scopes); if (tokens.Current.Type != TokenType.CLOSE_PAREN) { Error(tokens.Current.Location, "Unmatched open parenthesis (currently at " + tokens.Current.Type + ")."); - return new Nothing(); + return null; } - else if (interior.IsNothing) + else if (interior == null) { Error(lookAhead.Location, "Expected expression inside paretheses. "); - return new Nothing(); + return null; } else { - grammarSymbols.Push(new Left(interior.FromJust)); + grammarSymbols.Push(new Left(interior)); tokens.MoveNext(); break; } @@ -573,19 +574,19 @@ private Maybe ParseAtom(MergeableGenerator tokens, ImmutableSt { //Assume unary negation. tokens.MoveNext(); - Maybe interior = ParseAtom(tokens, scopes); - if (interior.IsNothing) + IAtomNode? interior = ParseAtom(tokens, scopes); + if (interior == null) { Error(lookAhead.Location, "Expected expression after negation. "); - return new Nothing(); + return null; } - grammarSymbols.Push(new Left(new NegationNode(lookAhead, interior.FromJust))); + grammarSymbols.Push(new Left(new NegationNode(lookAhead, interior))); break; } case TokenType.COMMA: Error(lookAhead.Location, "Unexpected comma (perhaps unrecognized macro invocation?)."); IgnoreRestOfStatement(tokens); - return new Nothing(); + return null; case TokenType.MUL_OP: case TokenType.DIV_OP: case TokenType.MOD_OP: @@ -599,7 +600,7 @@ private Maybe ParseAtom(MergeableGenerator tokens, ImmutableSt default: Error(lookAhead.Location, "Expected identifier or literal, got " + lookAhead.Type + ": " + lookAhead.Content + '.'); IgnoreRestOfStatement(tokens); - return new Nothing(); + return null; } } @@ -625,9 +626,9 @@ private Maybe ParseAtom(MergeableGenerator tokens, ImmutableSt } else if (lookAhead.Type == TokenType.ERROR) { - Error(lookAhead.Location, System.String.Format("Unexpected token: {0}", lookAhead.Content)); + Error(lookAhead.Location, $"Unexpected token: {lookAhead.Content}"); tokens.MoveNext(); - return new Nothing(); + return null; } else { @@ -645,7 +646,7 @@ private Maybe ParseAtom(MergeableGenerator tokens, ImmutableSt { Error(grammarSymbols.Peek().GetRight.Location, "Unexpected token: " + grammarSymbols.Peek().GetRight.Type); } - return new Just(grammarSymbols.Peek().GetLeft); + return grammarSymbols.Peek().GetLeft; } /*** @@ -680,8 +681,8 @@ private int GetLowestPrecedence(Stack> grammarSymbols) { int minPrec = 11; //TODO: Note that this is the largest possible value. foreach (Either e in grammarSymbols) - e.Case((IAtomNode n) => { minPrec = Math.Min(minPrec, n.Precedence); }, - (Token t) => { minPrec = Math.Min(minPrec, precedences[t.Type]); }); + e.Case(n => { minPrec = Math.Min(minPrec, n.Precedence); }, + t => { minPrec = Math.Min(minPrec, precedences[t.Type]); }); return minPrec; } @@ -692,9 +693,9 @@ private IList ParseList(MergeableGenerator tokens, ImmutableSt IList atoms = new List(); while (tokens.Current.Type != TokenType.NEWLINE && tokens.Current.Type != TokenType.CLOSE_BRACKET) { - Maybe res = ParseAtom(tokens, scopes); + IAtomNode? res = ParseAtom(tokens, scopes); res.IfJust( - (IAtomNode n) => atoms.Add(n), + n => atoms.Add(n), () => Error(tokens.Current.Location, "Expected atomic value, got " + tokens.Current.Type + ".")); if (tokens.Current.Type == TokenType.COMMA) tokens.MoveNext(); @@ -706,14 +707,14 @@ private IList ParseList(MergeableGenerator tokens, ImmutableSt return atoms; } - public Maybe ParseLine(MergeableGenerator tokens, ImmutableStack scopes) + public ILineNode? ParseLine(MergeableGenerator tokens, ImmutableStack scopes) { if (IsIncluding) { if (tokens.Current.Type == TokenType.NEWLINE || tokens.Current.Type == TokenType.SEMICOLON) { tokens.MoveNext(); - return new Nothing(); + return null; } head = tokens.Current; switch (head.Type) @@ -743,7 +744,7 @@ public Maybe ParseLine(MergeableGenerator tokens, ImmutableSta scopes.Head.AddLabel(head.Content, CurrentOffset); } - return new Nothing(); + return null; } else { @@ -752,7 +753,7 @@ public Maybe ParseLine(MergeableGenerator tokens, ImmutableSta } } case TokenType.OPEN_BRACE: - return new Just(ParseBlock(tokens, new ImmutableStack(new Closure(), scopes))); + return ParseBlock(tokens, new ImmutableStack(new Closure(), scopes)); case TokenType.PREPROCESSOR_DIRECTIVE: return ParsePreprocessor(tokens, scopes); case TokenType.OPEN_BRACKET: @@ -766,11 +767,11 @@ public Maybe ParseLine(MergeableGenerator tokens, ImmutableSta break; default: tokens.MoveNext(); - Error(head.Location, System.String.Format("Unexpected token: {0}: {1}", head.Type, head.Content)); + Error(head.Location, $"Unexpected token: {head.Type}: {head.Content}"); IgnoreRestOfLine(tokens); break; } - return new Nothing(); + return null; } else { @@ -782,23 +783,23 @@ public Maybe ParseLine(MergeableGenerator tokens, ImmutableSta } else { - Error(null, System.String.Format("Missing {0} endif(s).", Inclusion.Count)); - return new Nothing(); + Error(null, $"Missing {Inclusion.Count} endif(s)."); + return null; } } } - private Maybe ParsePreprocessor(MergeableGenerator tokens, ImmutableStack scopes) + private ILineNode? ParsePreprocessor(MergeableGenerator tokens, ImmutableStack scopes) { head = tokens.Current; tokens.MoveNext(); //Note: Not a ParseParamList because no commas. IList paramList = ParsePreprocParamList(tokens, scopes); - Maybe retVal = directiveHandler.HandleDirective(this, head, paramList, tokens); - if (!retVal.IsNothing) + ILineNode? retVal = directiveHandler.HandleDirective(this, head, paramList, tokens); + if (retVal != null) { - CheckDataWrite(retVal.FromJust.Size); - CurrentOffset += retVal.FromJust.Size; + CheckDataWrite(retVal.Size); + CurrentOffset += retVal.Size; } return retVal; } @@ -893,16 +894,16 @@ private string PrettyPrintParams(IList parameters) } // Return value: Location where protection occurred. Nothing if location was not protected. - private Maybe IsProtected(int offset, int length) + private Location? IsProtected(int offset, int length) { foreach (Tuple protectedRegion in protectedRegions) { //They intersect if the last offset in the given region is after the start of this one //and the first offset in the given region is before the last of this one if (offset + length > protectedRegion.Item1 && offset < protectedRegion.Item1 + protectedRegion.Item2) - return new Just(protectedRegion.Item3); + return protectedRegion.Item3; } - return new Nothing(); + return null; } private void CheckDataWrite(int length) @@ -917,11 +918,9 @@ private void CheckDataWrite(int length) // TODO (maybe?): save Location of PROTECT statement, for better diagnosis // We would then print something like "Trying to write data to area protected at " - Maybe prot = IsProtected(CurrentOffset, length); - if (!prot.IsNothing) + if (IsProtected(CurrentOffset, length) is Location prot) { - Location l = prot.FromJust; - Error(head?.Location, System.String.Format("Trying to write data to area protected in file {0} at line {1}, column {2}.", Path.GetFileName(l.file), l.lineNum, l.colNum)); + Error(head?.Location, $"Trying to write data to area protected in file {Path.GetFileName(prot.file)} at line {prot.lineNum}, column {prot.colNum}."); } } } diff --git a/ColorzCore/Preprocessor/DirectiveHandler.cs b/ColorzCore/Preprocessor/DirectiveHandler.cs index bc07de9..373bb7c 100644 --- a/ColorzCore/Preprocessor/DirectiveHandler.cs +++ b/ColorzCore/Preprocessor/DirectiveHandler.cs @@ -35,7 +35,7 @@ public DirectiveHandler(IncludeFileSearcher includeSearcher, IncludeFileSearcher }; } - public Maybe HandleDirective(EAParser p, Token directive, IList parameters, MergeableGenerator tokens) + public ILineNode? HandleDirective(EAParser p, Token directive, IList parameters, MergeableGenerator tokens) { string directiveName = directive.Content.Substring(1); @@ -58,7 +58,7 @@ public Maybe HandleDirective(EAParser p, Token directive, IList(); + return null; } } } diff --git a/ColorzCore/Preprocessor/Directives/DefineDirective.cs b/ColorzCore/Preprocessor/Directives/DefineDirective.cs index 83e86f9..3c0e602 100644 --- a/ColorzCore/Preprocessor/Directives/DefineDirective.cs +++ b/ColorzCore/Preprocessor/Directives/DefineDirective.cs @@ -19,18 +19,17 @@ class DefineDirective : IDirective public bool RequireInclusion => true; - public Maybe Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) + public ILineNode? Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) { - if (parameters[0].Type == ParamType.MACRO) + if (parameters[0] is MacroInvocationNode signature) { - MacroInvocationNode signature = (MacroInvocationNode)(parameters[0]); string name = signature.Name; IList myParams = new List(); foreach (IList l1 in signature.Parameters) { if (l1.Count != 1 || l1[0].Type != TokenType.IDENTIFIER) { - p.Error(l1[0].Location, "Macro parameters must be identifiers (got " + l1[0].Content + ")."); + p.Error(l1[0].Location, $"Macro parameters must be identifiers (got {l1[0].Content})."); } else { @@ -46,35 +45,34 @@ public Maybe Execute(EAParser p, Token self, IList parame else p.Warning(signature.MyLocation, "Redefining " + name + '.'); }*/ - if(p.Macros.HasMacro(name, myParams.Count)) - p.Warning(signature.MyLocation, "Redefining " + name + '.'); - Maybe> toRepl; + if (p.Macros.HasMacro(name, myParams.Count)) + p.Warning(signature.MyLocation, $"Redefining {name}."); + IList? toRepl; if (parameters.Count != 2) { - toRepl = new Just>(new List()); + toRepl = new List(); } else toRepl = ExpandParam(p, parameters[1], myParams.Select((Token t) => t.Content)); - if (!toRepl.IsNothing) + if (toRepl != null) { - p.Macros.AddMacro(new Macro(myParams, toRepl.FromJust), name, myParams.Count); + p.Macros.AddMacro(new Macro(myParams, toRepl), name, myParams.Count); } } else { //Note [mutually] recursive definitions are handled by Parser expansion. - Maybe maybeIdentifier; - if (parameters[0].Type == ParamType.ATOM && !(maybeIdentifier = ((IAtomNode)parameters[0]).GetIdentifier()).IsNothing) + string? name; + if (parameters[0].Type == ParamType.ATOM && (name = ((IAtomNode)parameters[0]).GetIdentifier()) != null) { - string name = maybeIdentifier.FromJust; - if(p.Definitions.ContainsKey(name)) + if (p.Definitions.ContainsKey(name)) p.Warning(parameters[0].MyLocation, "Redefining " + name + '.'); if (parameters.Count == 2) { - Maybe> toRepl = ExpandParam(p, parameters[1], Enumerable.Empty()); - if (!toRepl.IsNothing) + IList? toRepl = ExpandParam(p, parameters[1], Enumerable.Empty()); + if (toRepl != null) { - p.Definitions[name] = new Definition(toRepl.FromJust); + p.Definitions[name] = new Definition(toRepl); } } else @@ -84,30 +82,32 @@ public Maybe Execute(EAParser p, Token self, IList parame } else { - p.Error(parameters[0].MyLocation, "Definition names must be identifiers (got " + parameters[0].ToString() + ")."); + p.Error(parameters[0].MyLocation, $"Definition names must be identifiers (got {parameters[0].ToString()})."); } } - return new Nothing(); + return null; } - delegate Maybe> ExpParamType(EAParser p, IParamNode param, IEnumerable myParams); - private static ExpParamType ExpandParam = (EAParser p, IParamNode param, IEnumerable myParams) => - TokenizeParam(p, param).Fmap>( (IList l) => - ExpandAllIdentifiers(p, new Queue(l), ImmutableStack.FromEnumerable(myParams), ImmutableStack>.Nil)).Fmap( - (IEnumerable x) => (IList)new List(x)); - private static Maybe> TokenizeParam(EAParser p, IParamNode param) + + private static IList? ExpandParam(EAParser p, IParamNode param, IEnumerable myParams) { + return TokenizeParam(p, param).Fmap(tokens => ExpandAllIdentifiers(p, + new Queue(tokens), ImmutableStack.FromEnumerable(myParams), + ImmutableStack>.Nil)).Fmap(x => new List(x)); + } + private static IList? TokenizeParam(EAParser p, IParamNode param) + { switch (param.Type) { case ParamType.STRING: Token input = ((StringNode)param).MyToken; Tokenizer t = new Tokenizer(); - return new Just>(new List(t.TokenizeLine(input.Content, input.FileName, input.LineNumber, input.ColumnNumber))); + return new List(t.TokenizeLine(input.Content, input.FileName, input.LineNumber, input.ColumnNumber)); case ParamType.MACRO: try { IList myBody = new List(((MacroInvocationNode)param).ExpandMacro()); - return new Just>(myBody); + return myBody; } catch (KeyNotFoundException) { @@ -117,31 +117,30 @@ private static Maybe> TokenizeParam(EAParser p, IParamNode param) break; case ParamType.LIST: ListNode n = (ListNode)param; - return new Just>(new List(n.ToTokens())); + return new List(n.ToTokens()); case ParamType.ATOM: - return new Just>(new List(((IAtomNode)param).ToTokens())); + return new List(((IAtomNode)param).ToTokens()); } - return new Nothing>(); + return null; } private static IEnumerable ExpandAllIdentifiers(EAParser p, Queue tokens, ImmutableStack seenDefs, ImmutableStack> seenMacros) { - IEnumerable output = new List(); - while(tokens.Count > 0) + while (tokens.Count > 0) { Token current = tokens.Dequeue(); - if(current.Type == TokenType.IDENTIFIER) + if (current.Type == TokenType.IDENTIFIER) { - if(p.Macros.ContainsName(current.Content) && tokens.Count > 0 && tokens.Peek().Type == TokenType.OPEN_PAREN) + if (p.Macros.ContainsName(current.Content) && tokens.Count > 0 && tokens.Peek().Type == TokenType.OPEN_PAREN) { IList> param = p.ParseMacroParamList(new MergeableGenerator(tokens)); //TODO: I don't like wrapping this in a mergeable generator..... Maybe interface the original better? if (!seenMacros.Contains(new Tuple(current.Content, param.Count)) && p.Macros.HasMacro(current.Content, param.Count)) { - foreach(Token t in p.Macros.GetMacro(current.Content, param.Count).ApplyMacro(current, param, p.GlobalScope)) + foreach (Token t in p.Macros.GetMacro(current.Content, param.Count).ApplyMacro(current, param, p.GlobalScope)) { - yield return t; + yield return t; } } - else if(seenMacros.Contains(new Tuple(current.Content, param.Count))) + else if (seenMacros.Contains(new Tuple(current.Content, param.Count))) { yield return current; foreach (IList l in param) @@ -153,7 +152,7 @@ private static IEnumerable ExpandAllIdentifiers(EAParser p, Queue yield return current; } } - else if(!seenDefs.Contains(current.Content) && p.Definitions.ContainsKey(current.Content)) + else if (!seenDefs.Contains(current.Content) && p.Definitions.ContainsKey(current.Content)) { foreach (Token t in p.Definitions[current.Content].ApplyDefinition(current)) yield return t; @@ -162,7 +161,7 @@ private static IEnumerable ExpandAllIdentifiers(EAParser p, Queue { yield return current; } - } + } else { yield return current; diff --git a/ColorzCore/Preprocessor/Directives/ElseDirective.cs b/ColorzCore/Preprocessor/Directives/ElseDirective.cs index 7e169d2..d3b3e2d 100644 --- a/ColorzCore/Preprocessor/Directives/ElseDirective.cs +++ b/ColorzCore/Preprocessor/Directives/ElseDirective.cs @@ -18,13 +18,13 @@ class ElseDirective : IDirective public bool RequireInclusion => false; - public Maybe Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) + public ILineNode? Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) { if (p.Inclusion.IsEmpty) p.Error(self.Location, "No matching if[n]def."); else p.Inclusion = new ImmutableStack(!p.Inclusion.Head, p.Inclusion.Tail); - return new Nothing(); + return null; } } } diff --git a/ColorzCore/Preprocessor/Directives/EndIfDirective.cs b/ColorzCore/Preprocessor/Directives/EndIfDirective.cs index 09212a4..9ec2cce 100644 --- a/ColorzCore/Preprocessor/Directives/EndIfDirective.cs +++ b/ColorzCore/Preprocessor/Directives/EndIfDirective.cs @@ -18,13 +18,13 @@ class EndIfDirective : IDirective public bool RequireInclusion => false; - public Maybe Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) + public ILineNode? Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) { if (p.Inclusion.IsEmpty) p.Error(self.Location, "No matching if[n]def."); else p.Inclusion = p.Inclusion.Tail; - return new Nothing(); + return null; } } } diff --git a/ColorzCore/Preprocessor/Directives/IDirective.cs b/ColorzCore/Preprocessor/Directives/IDirective.cs index 2079dff..c1e44e4 100644 --- a/ColorzCore/Preprocessor/Directives/IDirective.cs +++ b/ColorzCore/Preprocessor/Directives/IDirective.cs @@ -18,7 +18,7 @@ interface IDirective * * Return: If a string is returned, it is interpreted as an error. */ - Maybe Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens); + ILineNode? Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens); /*** * Minimum number of parameters, inclusive. */ diff --git a/ColorzCore/Preprocessor/Directives/IfDefinedDirective.cs b/ColorzCore/Preprocessor/Directives/IfDefinedDirective.cs index b4e9144..0447a79 100644 --- a/ColorzCore/Preprocessor/Directives/IfDefinedDirective.cs +++ b/ColorzCore/Preprocessor/Directives/IfDefinedDirective.cs @@ -18,15 +18,15 @@ class IfDefinedDirective : IDirective public bool RequireInclusion => false; - public Maybe Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) + public ILineNode? Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) { bool flag = true; - Maybe identifier; + string? identifier; foreach (IParamNode parameter in parameters) { - if(parameter.Type==ParamType.ATOM && !(identifier = ((IAtomNode)parameter).GetIdentifier()).IsNothing) + if (parameter.Type == ParamType.ATOM && (identifier = ((IAtomNode)parameter).GetIdentifier()) != null) { - flag &= p.Macros.ContainsName(identifier.FromJust) || p.Definitions.ContainsKey(identifier.FromJust); //TODO: Built in definitions? + flag &= p.Macros.ContainsName(identifier) || p.Definitions.ContainsKey(identifier); //TODO: Built in definitions? } else { @@ -34,7 +34,7 @@ public Maybe Execute(EAParser p, Token self, IList parame } } p.Inclusion = new ImmutableStack(flag, p.Inclusion); - return new Nothing(); + return null; } } } diff --git a/ColorzCore/Preprocessor/Directives/IfNotDefinedDirective.cs b/ColorzCore/Preprocessor/Directives/IfNotDefinedDirective.cs index 02476b1..ae48c0e 100644 --- a/ColorzCore/Preprocessor/Directives/IfNotDefinedDirective.cs +++ b/ColorzCore/Preprocessor/Directives/IfNotDefinedDirective.cs @@ -18,15 +18,15 @@ class IfNotDefinedDirective : IDirective public bool RequireInclusion => false; - public Maybe Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) + public ILineNode? Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) { bool flag = true; - Maybe identifier; + string? identifier; foreach (IParamNode parameter in parameters) { - if(parameter.Type==ParamType.ATOM && !(identifier = ((IAtomNode)parameter).GetIdentifier()).IsNothing) + if (parameter.Type == ParamType.ATOM && (identifier = ((IAtomNode)parameter).GetIdentifier()) != null) { - flag &= !p.Macros.ContainsName(identifier.FromJust) && !p.Definitions.ContainsKey(identifier.FromJust); //TODO: Built in definitions? + flag &= !p.Macros.ContainsName(identifier) && !p.Definitions.ContainsKey(identifier); //TODO: Built in definitions? } else { @@ -34,7 +34,7 @@ public Maybe Execute(EAParser p, Token self, IList parame } } p.Inclusion = new ImmutableStack(flag, p.Inclusion); - return new Nothing(); + return null; } } } diff --git a/ColorzCore/Preprocessor/Directives/IncludeBinaryDirective.cs b/ColorzCore/Preprocessor/Directives/IncludeBinaryDirective.cs index 628f6f2..98bdc0c 100644 --- a/ColorzCore/Preprocessor/Directives/IncludeBinaryDirective.cs +++ b/ColorzCore/Preprocessor/Directives/IncludeBinaryDirective.cs @@ -22,16 +22,16 @@ class IncludeBinaryDirective : IDirective public IncludeFileSearcher FileSearcher { get; set; } = new IncludeFileSearcher(); - public Maybe Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) + public ILineNode? Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) { - Maybe existantFile = FileSearcher.FindFile(Path.GetDirectoryName(self.FileName), parameters[0].ToString()!); + string? existantFile = FileSearcher.FindFile(Path.GetDirectoryName(self.FileName), parameters[0].ToString()!); - if (!existantFile.IsNothing) + if (existantFile != null) { try { - string pathname = existantFile.FromJust; - return new Just(new DataNode(p.CurrentOffset, File.ReadAllBytes(pathname))); + string pathname = existantFile; + return new DataNode(p.CurrentOffset, File.ReadAllBytes(pathname)); } catch (Exception) { @@ -42,7 +42,7 @@ public Maybe Execute(EAParser p, Token self, IList parame { p.Error(parameters[0].MyLocation, "Could not find file \"" + parameters[0].ToString() + "\"."); } - return new Nothing(); + return null; } } } diff --git a/ColorzCore/Preprocessor/Directives/IncludeDirective.cs b/ColorzCore/Preprocessor/Directives/IncludeDirective.cs index 24bb81f..4a0cbcd 100644 --- a/ColorzCore/Preprocessor/Directives/IncludeDirective.cs +++ b/ColorzCore/Preprocessor/Directives/IncludeDirective.cs @@ -22,21 +22,21 @@ class IncludeDirective : IDirective public IncludeFileSearcher FileSearcher { get; set; } = new IncludeFileSearcher(); - public Maybe Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) + public ILineNode? Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) { - Maybe existantFile = FileSearcher.FindFile(Path.GetDirectoryName(self.FileName), parameters[0].ToString()!); + string? existantFile = FileSearcher.FindFile(Path.GetDirectoryName(self.FileName), parameters[0].ToString()!); - if (!existantFile.IsNothing) + if (existantFile != null) { try { - string pathname = existantFile.FromJust; + string pathname = existantFile; FileStream inputFile = new FileStream(pathname, FileMode.Open); Tokenizer newFileTokenizer = new Tokenizer(); tokens.PrependEnumerator(newFileTokenizer.Tokenize(inputFile).GetEnumerator()); } - catch(Exception) + catch (Exception) { p.Error(self.Location, "Error reading file \"" + parameters[0].ToString() + "\"."); } @@ -45,7 +45,7 @@ public Maybe Execute(EAParser p, Token self, IList parame { p.Error(parameters[0].MyLocation, "Could not find file \"" + parameters[0].ToString() + "\"."); } - return new Nothing(); + return null; } } } diff --git a/ColorzCore/Preprocessor/Directives/IncludeExternalDirective.cs b/ColorzCore/Preprocessor/Directives/IncludeExternalDirective.cs index fa663c7..94656d5 100644 --- a/ColorzCore/Preprocessor/Directives/IncludeExternalDirective.cs +++ b/ColorzCore/Preprocessor/Directives/IncludeExternalDirective.cs @@ -20,16 +20,16 @@ class IncludeExternalDirective : IDirective public IncludeFileSearcher FileSearcher { get; set; } = new IncludeFileSearcher(); - public Maybe Execute(EAParser parse, Token self, IList parameters, MergeableGenerator tokens) + public ILineNode? Execute(EAParser parse, Token self, IList parameters, MergeableGenerator tokens) { ExecTimer.Timer.AddTimingPoint(ExecTimer.KEY_GENERIC); - Maybe validFile = FileSearcher.FindFile(Path.GetDirectoryName(self.FileName), IOUtility.GetToolFileName(parameters[0].ToString()!)); + string? validFile = FileSearcher.FindFile(Path.GetDirectoryName(self.FileName), IOUtility.GetToolFileName(parameters[0].ToString()!)); - if (validFile.IsNothing) + if (validFile == null) { parse.Error(parameters[0].MyLocation, "Tool " + parameters[0].ToString() + " not found."); - return new Nothing(); + return null; } //TODO: abstract out all this running stuff into a method so I don't have code duplication with inctext @@ -42,11 +42,11 @@ public Maybe Execute(EAParser parse, Token self, IList pa p.StartInfo.UseShellExecute = false; p.StartInfo.RedirectStandardOutput = true; p.StartInfo.CreateNoWindow = true; - p.StartInfo.FileName = validFile.FromJust; + p.StartInfo.FileName = validFile; StringBuilder argumentBuilder = new StringBuilder(); for (int i = 1; i < parameters.Count; i++) { - if(parameters[i].Type == ParamType.ATOM) + if (parameters[i].Type == ParamType.ATOM) { parameters[i] = ((IAtomNode)parameters[i]).Simplify(); } @@ -78,7 +78,7 @@ public Maybe Execute(EAParser parse, Token self, IList pa ExecTimer.Timer.AddTimingPoint(parameters[0].ToString()!.ToLower()); - return new Just(new DataNode(parse.CurrentOffset, output)); + return new DataNode(parse.CurrentOffset, output); } } } diff --git a/ColorzCore/Preprocessor/Directives/IncludeToolEventDirective.cs b/ColorzCore/Preprocessor/Directives/IncludeToolEventDirective.cs index 7311d42..5b4120f 100644 --- a/ColorzCore/Preprocessor/Directives/IncludeToolEventDirective.cs +++ b/ColorzCore/Preprocessor/Directives/IncludeToolEventDirective.cs @@ -20,16 +20,16 @@ class IncludeToolEventDirective : IDirective public IncludeFileSearcher FileSearcher { get; set; } = new IncludeFileSearcher(); - public Maybe Execute(EAParser parse, Token self, IList parameters, MergeableGenerator tokens) + public ILineNode? Execute(EAParser parse, Token self, IList parameters, MergeableGenerator tokens) { ExecTimer.Timer.AddTimingPoint(ExecTimer.KEY_GENERIC); - Maybe validFile = FileSearcher.FindFile(Path.GetDirectoryName(self.FileName), IOUtility.GetToolFileName(parameters[0].ToString()!)); + string? validFile = FileSearcher.FindFile(Path.GetDirectoryName(self.FileName), IOUtility.GetToolFileName(parameters[0].ToString()!)); - if (validFile.IsNothing) + if (validFile == null) { parse.Error(parameters[0].MyLocation, "Tool " + parameters[0].ToString() + " not found."); - return new Nothing(); + return null; } //from http://stackoverflow.com/a/206347/1644720 @@ -41,7 +41,7 @@ public Maybe Execute(EAParser parse, Token self, IList pa p.StartInfo.UseShellExecute = false; p.StartInfo.RedirectStandardOutput = true; p.StartInfo.CreateNoWindow = true; - p.StartInfo.FileName = validFile.FromJust; + p.StartInfo.FileName = validFile; StringBuilder argumentBuilder = new StringBuilder(); for (int i = 1; i < parameters.Count; i++) { @@ -63,7 +63,7 @@ public Maybe Execute(EAParser parse, Token self, IList pa p.WaitForExit(); byte[] output = outputBytes.GetBuffer().Take((int)outputBytes.Length).ToArray(); - if(errorStream.Length > 0) + if (errorStream.Length > 0) { parse.Error(self.Location, Encoding.ASCII.GetString(errorStream.GetBuffer().Take((int)errorStream.Length).ToArray())); } @@ -80,7 +80,7 @@ public Maybe Execute(EAParser parse, Token self, IList pa ExecTimer.Timer.AddTimingPoint(parameters[0].ToString()!.ToLower()); - return new Nothing(); + return null; } } } diff --git a/ColorzCore/Preprocessor/Directives/PoolDirective.cs b/ColorzCore/Preprocessor/Directives/PoolDirective.cs index fbd057c..5d0240b 100644 --- a/ColorzCore/Preprocessor/Directives/PoolDirective.cs +++ b/ColorzCore/Preprocessor/Directives/PoolDirective.cs @@ -13,7 +13,7 @@ class PoolDirective : IDirective public int? MaxParams => 0; public bool RequireInclusion => true; - public Maybe Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) + public ILineNode? Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) { BlockNode result = new BlockNode(); @@ -36,7 +36,7 @@ public Maybe Execute(EAParser p, Token self, IList parame p.Pool.Lines.Clear(); - return new Just(result); + return result; } } } diff --git a/ColorzCore/Preprocessor/Directives/UndefineDirective.cs b/ColorzCore/Preprocessor/Directives/UndefineDirective.cs index 6eb93ef..0583d30 100644 --- a/ColorzCore/Preprocessor/Directives/UndefineDirective.cs +++ b/ColorzCore/Preprocessor/Directives/UndefineDirective.cs @@ -18,7 +18,7 @@ class UndefineDirective : IDirective public bool RequireInclusion => true; - public Maybe Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) + public ILineNode? Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) { foreach (IParamNode parm in parameters) { @@ -28,7 +28,7 @@ public Maybe Execute(EAParser p, Token self, IList parame else p.Warning(parm.MyLocation, "Undefining non-existant definition: " + s); } - return new Nothing(); + return null; } } } diff --git a/ColorzCore/Program.cs b/ColorzCore/Program.cs index 2adb831..1004209 100644 --- a/ColorzCore/Program.cs +++ b/ColorzCore/Program.cs @@ -86,7 +86,7 @@ static int Main(string[] args) TextWriter errorStream = Console.Error; - Maybe rawsFolder = rawSearcher.FindDirectory("Language Raws"); + string? rawsFolder = rawSearcher.FindDirectory("Language Raws"); string rawsExtension = ".txt"; if (args.Length < 2) @@ -251,7 +251,7 @@ static int Main(string[] args) return EXIT_FAILURE; } - if (rawsFolder.IsNothing) + if (rawsFolder == null) { Console.Error.WriteLine("Couldn't find raws folder"); return EXIT_FAILURE; @@ -303,7 +303,7 @@ static int Main(string[] args) if (EAOptions.Instance.nomess) log.IgnoredKinds.Add(Log.MsgKind.MESSAGE); - EAInterpreter myInterpreter = new EAInterpreter(output, game, rawsFolder.FromJust, rawsExtension, inStream, inFileName, log); + EAInterpreter myInterpreter = new EAInterpreter(output, game, rawsFolder, rawsExtension, inStream, inFileName, log); ExecTimer.Timer.AddTimingPoint(ExecTimer.KEY_RAWPROC); diff --git a/ColorzCore/Raws/AtomicParam.cs b/ColorzCore/Raws/AtomicParam.cs index 34268c2..c60a8d9 100644 --- a/ColorzCore/Raws/AtomicParam.cs +++ b/ColorzCore/Raws/AtomicParam.cs @@ -26,7 +26,7 @@ public AtomicParam(string name, int position, int length, bool isPointer) public void Set(byte[] data, IParamNode input) { - Set(data, input.AsAtom().FromJust.CoerceInt()); + Set(data, input.AsAtom()!.CoerceInt()); } public void Set(byte[] data, int value) diff --git a/ColorzCore/Raws/Raw.cs b/ColorzCore/Raws/Raw.cs index 7f53fb6..688c90b 100644 --- a/ColorzCore/Raws/Raw.cs +++ b/ColorzCore/Raws/Raw.cs @@ -25,8 +25,8 @@ class Raw // TODO: fixed mask? - public Raw(string name, int length, short code, int offsetMod, HashSet game, IList varParams, - IList fixedParams, Maybe terminatingList, bool repeatable) + public Raw(string name, int length, short code, int offsetMod, HashSet game, IList varParams, + IList fixedParams, int? terminatingList, bool repeatable) { Name = name; Game = game; @@ -48,10 +48,8 @@ public Raw(string name, int length, short code, int offsetMod, HashSet g // Build end unit, if needed - if (!terminatingList.IsNothing) + if (terminatingList is int terminator) { - int terminator = terminatingList.FromJust; - if (parameters.Count == 0) return; @@ -109,7 +107,7 @@ public bool Fits(IList arguments) return true; } - + /* Precondition: params fits the shape of this raw's params. */ public byte[] GetBytes(IList arguments) { diff --git a/ColorzCore/Raws/RawReader.cs b/ColorzCore/Raws/RawReader.cs index 1e8bdef..4f4306f 100644 --- a/ColorzCore/Raws/RawReader.cs +++ b/ColorzCore/Raws/RawReader.cs @@ -215,25 +215,25 @@ private static Raw ParseRaw(FileLineReader source) ? flags[FLAG_ALIGNMENT].Values.GetLeft[0].ToInt() : 4; - Maybe listTerminator = flags.ContainsKey(FLAG_LIST_TERMINATOR) - ? (Maybe)new Just(flags[FLAG_LIST_TERMINATOR].Values.GetLeft[0].ToInt()) - : new Nothing(); + int? listTerminator = flags.ContainsKey(FLAG_LIST_TERMINATOR) + ? flags[FLAG_LIST_TERMINATOR].Values.GetLeft[0].ToInt() + : null; - if (!listTerminator.IsNothing && code != 0) + if (listTerminator != null && code != 0) { throw new RawParseException("TerminatingList with code nonzero.", source.FileName, lineNumber); } bool isRepeatable = flags.ContainsKey(FLAG_REPEATABLE); - if ((isRepeatable || !listTerminator.IsNothing) && (parameters.Count > 1) && fixedParams.Count > 0) + if ((isRepeatable || listTerminator != null) && (parameters.Count > 1) && fixedParams.Count > 0) { throw new RawParseException("Repeatable or terminatingList code with multiple parameters or fixed parameters.", source.FileName, lineNumber); } // HACK: support terminating lists that have a code size of 0 (ugh) - if (!listTerminator.IsNothing) + if (listTerminator != null) { foreach (var param in parameters) size = Math.Max(size, param.Position + param.Length); From f1c96560bbd628a8da81fb302ac144966f32fa90 Mon Sep 17 00:00:00 2001 From: Nat_776 Date: Wed, 17 Apr 2024 18:05:38 +0200 Subject: [PATCH 09/59] add conditional operators, fallback raws, some qol --- ColorzCore/DataTypes/MergeableGenerator.cs | 38 +- ColorzCore/EAInterpreter.cs | 64 ++- ColorzCore/IO/ASM.cs | 17 +- ColorzCore/IO/Log.cs | 28 +- ColorzCore/Lexer/TokenType.cs | 9 + ColorzCore/Lexer/Tokenizer.cs | 115 ++-- ColorzCore/Parser/AST/NegationNode.cs | 42 -- ColorzCore/Parser/AST/OperatorNode.cs | 93 ++-- ColorzCore/Parser/AST/UnaryOperatorNode.cs | 63 +++ ColorzCore/Parser/EAParser.cs | 612 +++++++++++++-------- ColorzCore/Program.cs | 27 +- ColorzCore/Raws/Raw.cs | 46 +- ColorzCore/Raws/RawReader.cs | 8 +- 13 files changed, 736 insertions(+), 426 deletions(-) delete mode 100644 ColorzCore/Parser/AST/NegationNode.cs create mode 100644 ColorzCore/Parser/AST/UnaryOperatorNode.cs diff --git a/ColorzCore/DataTypes/MergeableGenerator.cs b/ColorzCore/DataTypes/MergeableGenerator.cs index a483226..ae7fe97 100644 --- a/ColorzCore/DataTypes/MergeableGenerator.cs +++ b/ColorzCore/DataTypes/MergeableGenerator.cs @@ -7,22 +7,24 @@ namespace ColorzCore.DataTypes { - public class MergeableGenerator: IEnumerable + public class MergeableGenerator : IEnumerable { - private Stack> myEnums; + private readonly Stack> myEnums; public bool EOS { get; private set; } + public MergeableGenerator(IEnumerable baseEnum) { myEnums = new Stack>(); myEnums.Push(baseEnum.GetEnumerator()); } - public T Current { get { return myEnums.Peek().Current; } } + public T Current => myEnums.Peek().Current; + public bool MoveNext() { - if(!myEnums.Peek().MoveNext()) + if (!myEnums.Peek().MoveNext()) { - if(myEnums.Count > 1) + if (myEnums.Count > 1) { myEnums.Pop(); return true; @@ -38,43 +40,55 @@ public bool MoveNext() return true; } } + public void PrependEnumerator(IEnumerator nextEnum) { if (EOS) + { myEnums.Pop(); + } + myEnums.Push(nextEnum); Prime(); } + public void PutBack(T elem) { - this.PrependEnumerator(new List { elem }.GetEnumerator()); + PrependEnumerator(new List { elem }.GetEnumerator()); } + private bool Prime() { if (!myEnums.Peek().MoveNext()) { if (myEnums.Count == 1) EOS = true; - else + else myEnums.Pop(); } else + { EOS = false; + } + return !EOS; } + IEnumerator IEnumerable.GetEnumerator() { - while(!EOS) { - yield return this.Current; - this.MoveNext(); + while (!EOS) + { + yield return Current; + MoveNext(); } } + public IEnumerator GetEnumerator() { while (!EOS) { - yield return this.Current; - this.MoveNext(); + yield return Current; + MoveNext(); } } } diff --git a/ColorzCore/EAInterpreter.cs b/ColorzCore/EAInterpreter.cs index c01a7d8..b686f3c 100644 --- a/ColorzCore/EAInterpreter.cs +++ b/ColorzCore/EAInterpreter.cs @@ -21,14 +21,14 @@ class EAInterpreter private Log log; private IOutput output; - public EAInterpreter(IOutput output, string game, string rawsFolder, string rawsExtension, Stream sin, string inFileName, Log log) + public EAInterpreter(IOutput output, string game, string? rawsFolder, string rawsExtension, Stream sin, string inFileName, Log log) { this.game = game; this.output = output; try { - allRaws = ProcessRaws(game, ListAllRaws(rawsFolder, rawsExtension)); + allRaws = SelectRaws(game, ListAllRaws(rawsFolder, rawsExtension)); } catch (RawReader.RawParseException e) { @@ -39,8 +39,8 @@ public EAInterpreter(IOutput output, string game, string rawsFolder, string raws colNum = 1 }; - log.Message(Log.MsgKind.ERROR, loc, "An error occured while parsing raws"); - log.Message(Log.MsgKind.ERROR, loc, e.Message); + log.Message(Log.MessageKind.ERROR, loc, "An error occured while parsing raws"); + log.Message(Log.MessageKind.ERROR, loc, e.Message); Environment.Exit(-1); // ew? } @@ -96,7 +96,8 @@ public bool Interpret() try { line.EvaluateExpressions(undefinedIds); - } catch (MacroInvocationNode.MacroException e) + } + catch (MacroInvocationNode.MacroException e) { myParser.Error(e.CausedError.MyLocation, "Unexpanded macro."); } @@ -124,7 +125,7 @@ public bool Interpret() { if (Program.Debug) { - log.Message(Log.MsgKind.DEBUG, line.PrettyPrint(0)); + log.Message(Log.MessageKind.DEBUG, line.PrettyPrint(0)); } line.WriteData(output); @@ -146,32 +147,63 @@ public bool WriteNocashSymbols(TextWriter output) { foreach (var label in myParser.GlobalScope.Head.LocalLabels()) { - // TODO: more elegant offset to address mapping - output.WriteLine("{0:X8} {1}", label.Value + 0x8000000, label.Key); + output.WriteLine("{0:X8} {1}", EAParser.ConvertToAddress(label.Value), label.Key); } return true; } - private static IEnumerable LoadAllRaws(string rawsFolder, string rawsExtension) + private static IEnumerable LoadAllRaws(string rawsFolder, string rawsExtension) { var directoryInfo = new DirectoryInfo(rawsFolder); var files = directoryInfo.GetFiles("*" + rawsExtension, SearchOption.AllDirectories); foreach (FileInfo fileInfo in files) { - using (var fs = new FileStream(fileInfo.FullName, FileMode.Open)) - foreach (var raw in RawReader.ParseAllRaws(fs)) - yield return raw; - } + using var fs = new FileStream(fileInfo.FullName, FileMode.Open); + + foreach (var raw in RawReader.ParseAllRaws(fs)) + yield return raw; + } + } + + private static IList ListAllRaws(string? rawsFolder, string rawsExtension) + { + if (rawsFolder != null) + { + return new List(LoadAllRaws(rawsFolder, rawsExtension)); + } + else + { + return GetFallbackRaws(); + } } - private static IList ListAllRaws(string rawsFolder, string rawsExtension) + private static IList GetFallbackRaws() { - return new List(LoadAllRaws(rawsFolder, rawsExtension)); + static List CreateParams(int bitSize, bool isPointer) + { + return new() { new AtomicParam("Data", 0, bitSize, isPointer) }; + } + + static Raw CreateRaw(string name, int byteSize, int alignment, bool isPointer) + { + return new(name, byteSize * 8, 0, alignment, CreateParams(byteSize * 8, isPointer), true); + } + + return new List() + { + CreateRaw("BYTE", 1, 1, false), + CreateRaw("SHORT", 2, 2, false), + CreateRaw("WORD", 4, 4, false), + CreateRaw("POIN", 4, 4, true), + CreateRaw("SHORT2", 2, 1, false), + CreateRaw("WORD2", 4, 1, false), + CreateRaw("POIN2", 4, 1, true), + }; } - private static Dictionary> ProcessRaws(string game, IList allRaws) + private static Dictionary> SelectRaws(string game, IList allRaws) { Dictionary> result = new Dictionary>(); diff --git a/ColorzCore/IO/ASM.cs b/ColorzCore/IO/ASM.cs index 4fb8dfb..14c02f8 100644 --- a/ColorzCore/IO/ASM.cs +++ b/ColorzCore/IO/ASM.cs @@ -4,6 +4,7 @@ using System.Linq; using System.Text; using System.Threading.Tasks; +using ColorzCore.Parser; namespace ColorzCore.IO { @@ -19,21 +20,21 @@ public ASM(StreamWriter asmStream, StreamWriter ldsStream) private void WriteToASM(int position, byte[] data) { - string sectionName = String.Format(".ea_{0:x}", 0x8000000 + position); - asmStream.WriteLine(".section {0},\"ax\",%progbits", sectionName); - asmStream.WriteLine(".global {0}", sectionName); - asmStream.WriteLine("{0}:", sectionName); + string sectionName = $".ea_{EAParser.ConvertToAddress(position):x}"; + asmStream.WriteLine($".section {sectionName},\"ax\",%progbits"); + asmStream.WriteLine($".global {sectionName}"); + asmStream.WriteLine($"{sectionName}:"); asmStream.Write("\t.byte "); foreach (byte value in data) - asmStream.Write("0x{0:x}, ", value); + asmStream.Write($"0x{value:x}, "); asmStream.WriteLine(); } private void WriteToLDS(int position) { - string sectionName = String.Format(".ea_{0:x}", 0x8000000 + position); - ldsStream.WriteLine(". = 0x{0:x};", 0x8000000 + position); - ldsStream.WriteLine("{0} : {{*.o({0})}}", sectionName); + string sectionName = $".ea_{EAParser.ConvertToAddress(position):x}"; + ldsStream.WriteLine($". = 0x{EAParser.ConvertToAddress(position):x};"); + ldsStream.WriteLine($"{sectionName} : {{*.o({sectionName})}}"); } public void WriteTo(int position, byte[] data) diff --git a/ColorzCore/IO/Log.cs b/ColorzCore/IO/Log.cs index 2cb74af..e1f80ff 100644 --- a/ColorzCore/IO/Log.cs +++ b/ColorzCore/IO/Log.cs @@ -7,7 +7,7 @@ namespace ColorzCore.IO { public class Log { - public enum MsgKind + public enum MessageKind { ERROR, WARNING, @@ -21,7 +21,7 @@ public enum MsgKind public bool NoColoredTags { get; set; } = false; - public List IgnoredKinds { get; } = new List(); + public List IgnoredKinds { get; } = new List(); public TextWriter Output { get; set; } = Console.Error; @@ -31,32 +31,32 @@ protected struct LogDisplayConfig public ConsoleColor? tagColor; } - protected static readonly Dictionary KIND_DISPLAY_DICT = new Dictionary { - { MsgKind.ERROR, new LogDisplayConfig { tag = "error", tagColor = ConsoleColor.Red } }, - { MsgKind.WARNING, new LogDisplayConfig { tag = "warning", tagColor = ConsoleColor.Magenta } }, - { MsgKind.NOTE, new LogDisplayConfig { tag = "note", tagColor = null } }, - { MsgKind.MESSAGE, new LogDisplayConfig { tag = "message", tagColor = ConsoleColor.Blue } }, - { MsgKind.DEBUG, new LogDisplayConfig { tag = "debug", tagColor = ConsoleColor.Green } } + protected static readonly Dictionary KIND_DISPLAY_DICT = new Dictionary { + { MessageKind.ERROR, new LogDisplayConfig { tag = "error", tagColor = ConsoleColor.Red } }, + { MessageKind.WARNING, new LogDisplayConfig { tag = "warning", tagColor = ConsoleColor.Magenta } }, + { MessageKind.NOTE, new LogDisplayConfig { tag = "note", tagColor = null } }, + { MessageKind.MESSAGE, new LogDisplayConfig { tag = "message", tagColor = ConsoleColor.Blue } }, + { MessageKind.DEBUG, new LogDisplayConfig { tag = "debug", tagColor = ConsoleColor.Green } } }; public void Message(string message) { - Message(MsgKind.MESSAGE, null, message); + Message(MessageKind.MESSAGE, null, message); } - public void Message(MsgKind kind, string message) + public void Message(MessageKind kind, string message) { Message(kind, null, message); } - public void Message(MsgKind kind, Location? source, string message) + public void Message(MessageKind kind, Location? source, string message) { - if (WarningsAreErrors && (kind == MsgKind.WARNING)) + if (WarningsAreErrors && (kind == MessageKind.WARNING)) { - kind = MsgKind.ERROR; + kind = MessageKind.ERROR; } - HasErrored |= (kind == MsgKind.ERROR); + HasErrored |= (kind == MessageKind.ERROR); if (!IgnoredKinds.Contains(kind)) { diff --git a/ColorzCore/Lexer/TokenType.cs b/ColorzCore/Lexer/TokenType.cs index 96e8c4d..62694e8 100644 --- a/ColorzCore/Lexer/TokenType.cs +++ b/ColorzCore/Lexer/TokenType.cs @@ -28,6 +28,15 @@ public enum TokenType AND_OP, XOR_OP, OR_OP, + LOGNOT_OP, + LOGAND_OP, + LOGOR_OP, + COMPARE_EQ, + COMPARE_NE, + COMPARE_LT, + COMPARE_LE, + COMPARE_GT, + COMPARE_GE, NUMBER, OPEN_BRACKET, CLOSE_BRACKET, diff --git a/ColorzCore/Lexer/Tokenizer.cs b/ColorzCore/Lexer/Tokenizer.cs index a81388a..6442587 100644 --- a/ColorzCore/Lexer/Tokenizer.cs +++ b/ColorzCore/Lexer/Tokenizer.cs @@ -15,16 +15,16 @@ class Tokenizer public static readonly Regex numRegex = new Regex("\\G([01]+b|0x[\\da-fA-F]+|\\$[\\da-fA-F]+|\\d+)"); public static readonly Regex idRegex = new Regex("\\G([a-zA-Z_][a-zA-Z0-9_]*)"); public static readonly Regex stringRegex = new Regex("\\G(([^\\\"]|\\\\\\\")*)"); //"\\G(([^\\\\\\\"]|\\\\[rnt\\\\\\\"])*)"); - public static readonly Regex winPathnameRegex = new Regex(String.Format("\\G([^ \\{0}]|\\ |\\\\)+", Process(Path.GetInvalidPathChars()))); + public static readonly Regex winPathnameRegex = new Regex(string.Format("\\G([^ \\{0}]|\\ |\\\\)+", Process(Path.GetInvalidPathChars()))); public static readonly Regex preprocDirectiveRegex = new Regex("\\G(#[a-zA-Z_][a-zA-Z0-9_]*)"); public static readonly Regex wordRegex = new Regex("\\G([^\\s]+)"); private static string Process(char[] chars) { StringBuilder sb = new StringBuilder(); - foreach(char c in chars) + foreach (char c in chars) { - switch(c) + switch (c) { case '.': case '\\': @@ -57,7 +57,7 @@ public Tokenizer() { multilineCommentNesting = 0; } - + public IEnumerable TokenizePhrase(string line, string fileName, int lineNum, int startOffs, int endOffs, int offset = 0) { bool afterInclude = false, afterDirective = false, afterWhitespace = false; @@ -73,7 +73,8 @@ public IEnumerable TokenizePhrase(string line, string fileName, int lineN multilineCommentNesting -= 1; curCol += 2; continue; - } else if (nextChar == '/' && curCol + 1 < endOffs && line[curCol + 1] == '*') + } + else if (nextChar == '/' && curCol + 1 < endOffs && line[curCol + 1] == '*') { multilineCommentNesting += 1; curCol += 2; @@ -85,7 +86,7 @@ public IEnumerable TokenizePhrase(string line, string fileName, int lineN continue; } } - if (Char.IsWhiteSpace(nextChar) && nextChar != '\n') + if (char.IsWhiteSpace(nextChar) && nextChar != '\n') { curCol++; afterWhitespace = true; @@ -95,37 +96,37 @@ public IEnumerable TokenizePhrase(string line, string fileName, int lineN switch (nextChar) { case ';': - yield return new Token(TokenType.SEMICOLON, fileName, lineNum, curCol+offset); + yield return new Token(TokenType.SEMICOLON, fileName, lineNum, curCol + offset); break; case ':': - yield return new Token(TokenType.COLON, fileName, lineNum, curCol+offset); + yield return new Token(TokenType.COLON, fileName, lineNum, curCol + offset); break; case '{': - yield return new Token(TokenType.OPEN_BRACE, fileName, lineNum, curCol+offset); + yield return new Token(TokenType.OPEN_BRACE, fileName, lineNum, curCol + offset); break; case '}': - yield return new Token(TokenType.CLOSE_BRACE, fileName, lineNum, curCol+offset); + yield return new Token(TokenType.CLOSE_BRACE, fileName, lineNum, curCol + offset); break; case '[': - yield return new Token(TokenType.OPEN_BRACKET, fileName, lineNum, curCol+offset); + yield return new Token(TokenType.OPEN_BRACKET, fileName, lineNum, curCol + offset); break; case ']': - yield return new Token(TokenType.CLOSE_BRACKET, fileName, lineNum, curCol+offset); + yield return new Token(TokenType.CLOSE_BRACKET, fileName, lineNum, curCol + offset); break; case '(': - yield return new Token(TokenType.OPEN_PAREN, fileName, lineNum, curCol+offset); + yield return new Token(TokenType.OPEN_PAREN, fileName, lineNum, curCol + offset); break; case ')': - yield return new Token(TokenType.CLOSE_PAREN, fileName, lineNum, curCol+offset); + yield return new Token(TokenType.CLOSE_PAREN, fileName, lineNum, curCol + offset); break; case '*': - yield return new Token(TokenType.MUL_OP, fileName, lineNum, curCol+offset); + yield return new Token(TokenType.MUL_OP, fileName, lineNum, curCol + offset); break; case '%': yield return new Token(TokenType.MOD_OP, fileName, lineNum, curCol + offset); break; case ',': - yield return new Token(TokenType.COMMA, fileName, lineNum, curCol+offset); + yield return new Token(TokenType.COMMA, fileName, lineNum, curCol + offset); break; case '/': if (curCol + 1 < endOffs && line[curCol + 1] == '/') @@ -141,11 +142,11 @@ public IEnumerable TokenizePhrase(string line, string fileName, int lineN } else { - yield return new Token(TokenType.DIV_OP, fileName, lineNum, curCol+offset); + yield return new Token(TokenType.DIV_OP, fileName, lineNum, curCol + offset); } break; case '+': - yield return new Token(TokenType.ADD_OP, fileName, lineNum, curCol+offset); + yield return new Token(TokenType.ADD_OP, fileName, lineNum, curCol + offset); break; case '-': if (afterWhitespace && afterDirective) @@ -162,13 +163,27 @@ public IEnumerable TokenizePhrase(string line, string fileName, int lineN yield return new Token(TokenType.SUB_OP, fileName, lineNum, curCol + offset); break; case '&': - yield return new Token(TokenType.AND_OP, fileName, lineNum, curCol+offset); + if (curCol + 1 < endOffs && line[curCol + 1] == '&') + { + yield return new Token(TokenType.LOGAND_OP, fileName, lineNum, curCol + offset); + curCol++; + break; + } + + yield return new Token(TokenType.AND_OP, fileName, lineNum, curCol + offset); break; case '^': - yield return new Token(TokenType.XOR_OP, fileName, lineNum, curCol+offset); + yield return new Token(TokenType.XOR_OP, fileName, lineNum, curCol + offset); break; case '|': - yield return new Token(TokenType.OR_OP, fileName, lineNum, curCol+offset); + if (curCol + 1 < endOffs && line[curCol + 1] == '|') + { + yield return new Token(TokenType.LOGOR_OP, fileName, lineNum, curCol + offset); + curCol++; + break; + } + + yield return new Token(TokenType.OR_OP, fileName, lineNum, curCol + offset); break; case '\"': { @@ -186,13 +201,19 @@ public IEnumerable TokenizePhrase(string line, string fileName, int lineN case '<': if (curCol + 1 < endOffs && line[curCol + 1] == '<') { - yield return new Token(TokenType.LSHIFT_OP, fileName, lineNum, curCol+offset); + yield return new Token(TokenType.LSHIFT_OP, fileName, lineNum, curCol + offset); + curCol++; + break; + } + else if (curCol + 1 < endOffs && line[curCol + 1] == '=') + { + yield return new Token(TokenType.COMPARE_LE, fileName, lineNum, curCol + offset); curCol++; break; } else { - yield return new Token(TokenType.ERROR, fileName, lineNum, curCol, "<"); + yield return new Token(TokenType.COMPARE_LT, fileName, lineNum, curCol + offset); break; } case '>': @@ -200,23 +221,53 @@ public IEnumerable TokenizePhrase(string line, string fileName, int lineN { if (curCol + 2 < endOffs && line[curCol + 2] == '>') { - yield return new Token(TokenType.SIGNED_RSHIFT_OP, fileName, lineNum, curCol+offset); + yield return new Token(TokenType.SIGNED_RSHIFT_OP, fileName, lineNum, curCol + offset); curCol += 2; } else { - yield return new Token(TokenType.RSHIFT_OP, fileName, lineNum, curCol+offset); + yield return new Token(TokenType.RSHIFT_OP, fileName, lineNum, curCol + offset); curCol++; } break; } + else if (curCol + 1 < endOffs && line[curCol + 1] == '=') + { + yield return new Token(TokenType.COMPARE_GE, fileName, lineNum, curCol + offset); + curCol++; + break; + } + else + { + yield return new Token(TokenType.COMPARE_GT, fileName, lineNum, curCol + offset); + break; + } + case '=': + if (curCol + 1 < endOffs && line[curCol + 1] == '=') + { + yield return new Token(TokenType.COMPARE_EQ, fileName, lineNum, curCol + offset); + curCol++; + break; + } + else + { + yield return new Token(TokenType.ERROR, fileName, lineNum, curCol, "="); + break; + } + case '!': + if (curCol + 1 < endOffs && line[curCol + 1] == '=') + { + yield return new Token(TokenType.COMPARE_NE, fileName, lineNum, curCol + offset); + curCol++; + break; + } else { - yield return new Token(TokenType.ERROR, fileName, lineNum, curCol, ">"); + yield return new Token(TokenType.LOGNOT_OP, fileName, lineNum, curCol + offset); break; } case '\n': - yield return new Token(TokenType.NEWLINE, fileName, lineNum, curCol+offset); + yield return new Token(TokenType.NEWLINE, fileName, lineNum, curCol + offset); break; default: if (afterInclude) @@ -243,13 +294,13 @@ public IEnumerable TokenizePhrase(string line, string fileName, int lineN curCol += match.Length; if (curCol < endOffs && line[curCol] == '(') yield return new Token(TokenType.MAYBE_MACRO, fileName, lineNum, idCol, match); - else + else yield return new Token(TokenType.IDENTIFIER, fileName, lineNum, idCol, match); - if (curCol < endOffs && (Char.IsLetterOrDigit(line[curCol]) | line[curCol] == '_')) + if (curCol < endOffs && (char.IsLetterOrDigit(line[curCol]) | line[curCol] == '_')) { Match idMatch2 = new Regex("[a-zA-Z0-9_]+").Match(line, curCol, endOffs - curCol); match = idMatch2.Value; - yield return new Token(TokenType.ERROR, fileName, lineNum, curCol, String.Format("Identifier longer than {0} characters.", MAX_ID_LENGTH)); + yield return new Token(TokenType.ERROR, fileName, lineNum, curCol, $"Identifier longer than {MAX_ID_LENGTH} characters."); curCol += match.Length; } continue; @@ -259,7 +310,7 @@ public IEnumerable TokenizePhrase(string line, string fileName, int lineN { string match = numMatch.Value; //Verify that next token isn't start of an identifier - if (curCol + match.Length >= endOffs || (!Char.IsLetter(line[curCol + match.Length]) && line[curCol + match.Length] != '_')) + if (curCol + match.Length >= endOffs || (!char.IsLetter(line[curCol + match.Length]) && line[curCol + match.Length] != '_')) { yield return new Token(TokenType.NUMBER, fileName, lineNum, curCol, match.TrimEnd()); curCol += match.Length; @@ -308,7 +359,7 @@ public IEnumerable Tokenize(Stream input, string fileName) string line = sin.ReadLine()!; //allow escaping newlines - while (line.Length > 0 && line.Substring(line.Length-1) == "\\") + while (line.Length > 0 && line.Substring(line.Length - 1) == "\\") { curLine++; line = line.Substring(0, line.Length - 1) + " " + sin.ReadLine(); diff --git a/ColorzCore/Parser/AST/NegationNode.cs b/ColorzCore/Parser/AST/NegationNode.cs deleted file mode 100644 index 50565ad..0000000 --- a/ColorzCore/Parser/AST/NegationNode.cs +++ /dev/null @@ -1,42 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using ColorzCore.DataTypes; -using ColorzCore.Lexer; - -namespace ColorzCore.Parser.AST -{ - class NegationNode : AtomNodeKernel - { - private Token myToken; - private IAtomNode interior; - - public NegationNode(Token token, IAtomNode inside) - { - myToken = token; - interior = inside; - } - - public override int Precedence => 11; - public override Location MyLocation => myToken.Location; - - public override string PrettyPrint() - { - return '-' + interior.PrettyPrint(); - } - - public override IEnumerable ToTokens() - { - yield return myToken; - foreach (Token t in interior.ToTokens()) - yield return t; - } - - public override int? TryEvaluate(TAction handler) - { - return -interior.TryEvaluate(handler); - } - } -} diff --git a/ColorzCore/Parser/AST/OperatorNode.cs b/ColorzCore/Parser/AST/OperatorNode.cs index 6ac4acc..67bae7d 100644 --- a/ColorzCore/Parser/AST/OperatorNode.cs +++ b/ColorzCore/Parser/AST/OperatorNode.cs @@ -14,17 +14,25 @@ namespace ColorzCore.Parser.AST class OperatorNode : AtomNodeKernel { public static readonly Dictionary Operators = new Dictionary { - { TokenType.MUL_OP , (x, y) => x*y }, - { TokenType.DIV_OP , (x, y) => x/y }, - { TokenType.MOD_OP , (x, y) => x%y }, - { TokenType.ADD_OP , (x, y) => x+y }, - { TokenType.SUB_OP , (x, y) => x-y }, - { TokenType.LSHIFT_OP , (x, y) => x< (int)(((uint)x)>>y) }, - { TokenType.SIGNED_RSHIFT_OP , (x, y) => x>>y }, - { TokenType.AND_OP , (x, y) => x&y }, - { TokenType.XOR_OP , (x, y) => x^y }, - { TokenType.OR_OP , (x, y) => x|y } + { TokenType.MUL_OP , (lhs, rhs) => lhs * rhs }, + { TokenType.DIV_OP , (lhs, rhs) => lhs / rhs }, + { TokenType.MOD_OP , (lhs, rhs) => lhs % rhs }, + { TokenType.ADD_OP , (lhs, rhs) => lhs + rhs }, + { TokenType.SUB_OP , (lhs, rhs) => lhs - rhs }, + { TokenType.LSHIFT_OP , (lhs, rhs) => lhs << rhs }, + { TokenType.RSHIFT_OP , (lhs, rhs) => (int)(((uint)lhs) >> rhs) }, + { TokenType.SIGNED_RSHIFT_OP , (lhs, rhs) => lhs >> rhs }, + { TokenType.AND_OP , (lhs, rhs) => lhs & rhs }, + { TokenType.XOR_OP , (lhs, rhs) => lhs ^ rhs }, + { TokenType.OR_OP , (lhs, rhs) => lhs | rhs }, + { TokenType.LOGAND_OP, (lhs, rhs) => lhs != 0 ? rhs : 0 }, + { TokenType.LOGOR_OP, (lhs, rhs) => lhs != 0 ? lhs : rhs }, + { TokenType.COMPARE_EQ, (lhs, rhs) => lhs == rhs ? 1 : 0 }, + { TokenType.COMPARE_NE, (lhs, rhs) => lhs != rhs ? 1 : 0 }, + { TokenType.COMPARE_LT, (lhs, rhs) => lhs < rhs ? 1 : 0 }, + { TokenType.COMPARE_LE, (lhs, rhs) => lhs <= rhs ? 1 : 0 }, + { TokenType.COMPARE_GE, (lhs, rhs) => lhs >= rhs ? 1 : 0 }, + { TokenType.COMPARE_GT, (lhs, rhs) => lhs > rhs ? 1 : 0 }, }; private IAtomNode left, right; @@ -43,45 +51,36 @@ public OperatorNode(IAtomNode l, Token op, IAtomNode r, int prec) public override string PrettyPrint() { - StringBuilder sb = new StringBuilder(left.PrettyPrint()); - switch (op.Type) + static string GetOperatorString(TokenType tokenType) { - case TokenType.MUL_OP: - sb.Append("*"); - break; - case TokenType.DIV_OP: - sb.Append("/"); - break; - case TokenType.ADD_OP: - sb.Append("+"); - break; - case TokenType.SUB_OP: - sb.Append("-"); - break; - case TokenType.LSHIFT_OP: - sb.Append("<<"); - break; - case TokenType.RSHIFT_OP: - sb.Append(">>"); - break; - case TokenType.SIGNED_RSHIFT_OP: - sb.Append(">>>"); - break; - case TokenType.AND_OP: - sb.Append("&"); - break; - case TokenType.XOR_OP: - sb.Append("^"); - break; - case TokenType.OR_OP: - sb.Append("|"); - break; - default: - break; + return tokenType switch + { + TokenType.MUL_OP => "*", + TokenType.DIV_OP => "/", + TokenType.MOD_OP => "%", + TokenType.ADD_OP => "+", + TokenType.SUB_OP => "-", + TokenType.LSHIFT_OP => "<<", + TokenType.RSHIFT_OP => ">>", + TokenType.SIGNED_RSHIFT_OP => ">>>", + TokenType.AND_OP => "&", + TokenType.XOR_OP => "^", + TokenType.OR_OP => "|", + TokenType.LOGAND_OP => "&&", + TokenType.LOGOR_OP => "||", + TokenType.COMPARE_EQ => "==", + TokenType.COMPARE_NE => "!=", + TokenType.COMPARE_LT => "<", + TokenType.COMPARE_LE => "<=", + TokenType.COMPARE_GT => ">", + TokenType.COMPARE_GE => ">=", + _ => "" + }; } - sb.Append(right.PrettyPrint()); - return sb.ToString(); + + return $"({left.PrettyPrint()} {GetOperatorString(op.Type)} {right.PrettyPrint()})"; } + public override IEnumerable ToTokens() { foreach (Token t in left.ToTokens()) diff --git a/ColorzCore/Parser/AST/UnaryOperatorNode.cs b/ColorzCore/Parser/AST/UnaryOperatorNode.cs new file mode 100644 index 0000000..ca21cef --- /dev/null +++ b/ColorzCore/Parser/AST/UnaryOperatorNode.cs @@ -0,0 +1,63 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using ColorzCore.DataTypes; +using ColorzCore.Lexer; + +namespace ColorzCore.Parser.AST +{ + class UnaryOperatorNode : AtomNodeKernel + { + private readonly Token myToken; + private readonly IAtomNode interior; + + public UnaryOperatorNode(Token token, IAtomNode inside) + { + myToken = token; + interior = inside; + } + + public override int Precedence => 11; + public override Location MyLocation => myToken.Location; + + public override string PrettyPrint() + { + string operatorString = myToken.Type switch + { + TokenType.SUB_OP => "-", + TokenType.LOGNOT_OP => "!", + _ => "", + }; + + return operatorString + interior.PrettyPrint(); + } + + public override IEnumerable ToTokens() + { + yield return myToken; + foreach (Token t in interior.ToTokens()) + yield return t; + } + + public override int? TryEvaluate(TAction handler) + { + int? inner = interior.TryEvaluate(handler); + + if (inner != null) + { + return myToken.Type switch + { + TokenType.SUB_OP => -inner, + TokenType.LOGNOT_OP => inner != 0 ? 0 : 1, + _ => null, + }; + } + else + { + return null; + } + } + } +} diff --git a/ColorzCore/Parser/EAParser.cs b/ColorzCore/Parser/EAParser.cs index d8cf4dd..ce9948f 100644 --- a/ColorzCore/Parser/EAParser.cs +++ b/ColorzCore/Parser/EAParser.cs @@ -53,8 +53,8 @@ private set private readonly DirectiveHandler directiveHandler; - private Stack> pastOffsets; // currentOffset, offsetInitialized - private IList> protectedRegions; + private readonly Stack> pastOffsets; // currentOffset, offsetInitialized + private readonly IList> protectedRegions; public Log log; @@ -63,11 +63,16 @@ public bool IsIncluding get { bool acc = true; + for (ImmutableStack temp = Inclusion; !temp.IsEmpty && acc; temp = temp.Tail) + { acc &= temp.Head; + } + return acc; } } + private bool validOffset; private bool offsetInitialized; // false until first ORG, used to warn about writing before first org private int currentOffset; @@ -135,13 +140,22 @@ private BlockNode ParseBlock(MergeableGenerator tokens, ImmutableStack tokens, ImmutableStack scopes) { while (ExpandIdentifier(tokens, scopes)) { } + head = tokens.Current; tokens.MoveNext(); + //TODO: Replace with real raw information, and error if not valid. IList parameters; //TODO: Make intelligent to reject malformed parameters. @@ -196,198 +212,272 @@ public static int ConvertToOffset(int value) if (SpecialCodes.Contains(upperCodeIdentifier)) { - switch (upperCodeIdentifier) + return upperCodeIdentifier switch { - case "ORG": - if (parameters.Count != 1) - Error(head.Location, "Incorrect number of parameters in ORG: " + parameters.Count); - else + "ORG" => ParseOrgStatement(parameters), + "PUSH" => ParsePushStatement(parameters), + "POP" => ParsePopStatement(parameters), + "ASSERT" => ParseAssertStatement(parameters), + "PROTECT" => ParseProtectStatement(parameters), + "ALIGN" => ParseAlignStatement(parameters), + "FILL" => ParseFillStatement(parameters), + "MESSAGE" => ParseMessageStatement(parameters), + "WARNING" => ParseWarningStatement(parameters), + "ERROR" => ParseErrorStatement(parameters), + _ => null, // TODO: this is an error + }; + } + else if (Raws.TryGetValue(upperCodeIdentifier, out IList? raws)) + { + //TODO: Check for matches. Currently should type error. + foreach (Raw raw in raws) + { + if (raw.Fits(parameters)) + { + if ((CurrentOffset % raw.Alignment) != 0) { - parameters[0].AsAtom().IfJust( - atom => atom.TryEvaluate(e => { Error(parameters[0].MyLocation, e.Message); }).IfJust( - temp => - { - CurrentOffset = ConvertToOffset(temp); - }), - () => { Error(parameters[0].MyLocation, "Expected atomic param to ORG."); } - ); + Error($"Bad code alignment (offset: {CurrentOffset:X8})"); } - break; - case "PUSH": - if (parameters.Count != 0) - Error(head.Location, "Incorrect number of parameters in PUSH: " + parameters.Count); - else - pastOffsets.Push(new Tuple(CurrentOffset, offsetInitialized)); - break; - case "POP": - if (parameters.Count != 0) - Error(head.Location, "Incorrect number of parameters in POP: " + parameters.Count); - else if (pastOffsets.Count == 0) - Error(head.Location, "POP without matching PUSH."); - else - { - Tuple tuple = pastOffsets.Pop(); - CurrentOffset = tuple.Item1; - offsetInitialized = tuple.Item2; - } - break; - case "MESSAGE": - Message(head.Location, PrettyPrintParams(parameters)); - break; - case "WARNING": - Warning(head.Location, PrettyPrintParams(parameters)); - break; - case "ERROR": - Error(head.Location, PrettyPrintParams(parameters)); - break; - case "ASSERT": - if (parameters.Count != 1) - Error(head.Location, "Incorrect number of parameters in ASSERT: " + parameters.Count); - else - { + StatementNode temp = new RawNode(raw, head, CurrentOffset, parameters); - parameters[0].AsAtom().IfJust( - atom => - { - atom.TryEvaluate(e => { Error(parameters[0].MyLocation, e.Message); }).IfJust( - temp => - { - if (temp < 0) - Error(parameters[0].MyLocation, "Assertion error: " + temp); - }); - }, - () => { Error(parameters[0].MyLocation, "Expected atomic param to ASSERT."); } - ); - } - break; - case "PROTECT": - if (parameters.Count == 1) - parameters[0].AsAtom().IfJust( - atom => atom.TryEvaluate(e => { Error(parameters[0].MyLocation, e.Message); }).IfJust( - temp => - { - protectedRegions.Add(new Tuple(temp, 4, head.Location)); - }), - () => { Error(parameters[0].MyLocation, "Expected atomic param to PROTECT"); }); - else if (parameters.Count == 2) - { - int start = 0, end = 0; - bool errorOccurred = false; - parameters[0].AsAtom().IfJust( - atom => atom.TryEvaluate(e => { Error(parameters[0].MyLocation, e.Message); errorOccurred = true; }).IfJust( - temp => - { - start = temp; - }), - () => { Error(parameters[0].MyLocation, "Expected atomic param to PROTECT"); errorOccurred = true; }); - parameters[1].AsAtom().IfJust( - atom => atom.TryEvaluate(e => { Error(parameters[0].MyLocation, e.Message); errorOccurred = true; }).IfJust( - temp => - { - end = temp; - }), - () => { Error(parameters[0].MyLocation, "Expected atomic param to PROTECT"); errorOccurred = true; }); - if (!errorOccurred) - { - int length = end - start; - if (length > 0) - protectedRegions.Add(new Tuple(start, length, head.Location)); - else - Warning(head.Location, "Protected region not valid (end offset not after start offset). No region protected."); - } - } - else - Error(head.Location, "Incorrect number of parameters in PROTECT: " + parameters.Count); - break; - case "ALIGN": - if (parameters.Count != 1) - Error(head.Location, "Incorrect number of parameters in ALIGN: " + parameters.Count); - else - parameters[0].AsAtom().IfJust( - atom => atom.TryEvaluate(e => { Error(parameters[0].MyLocation, e.Message); }).IfJust( - temp => - { - CurrentOffset = CurrentOffset % temp != 0 ? CurrentOffset + temp - CurrentOffset % temp : CurrentOffset; - }), - () => { Error(parameters[0].MyLocation, "Expected atomic param to ALIGN"); } - ); - break; - case "FILL": - if (parameters.Count > 2 || parameters.Count == 0) - { - Error(head.Location, "Incorrect number of parameters in FILL: " + parameters.Count); - } - else - { - // FILL [value] + // TODO: more efficient spacewise to just have contiguous writing and not an offset with every line? + CheckDataWrite(temp.Size); + CurrentOffset += temp.Size; - int amount = 0; - int value = 0; + return temp; + } + } - if (parameters.Count == 2) - { - // param 2 (if given) is fill value + if (raws.Count == 1) + { + Error($"Incorrect parameters in raw `{raws[0].ToPrettyString()}`"); + } + else + { + Error($"Couldn't find suitable variant of raw `{head.Content}`."); - parameters[1].AsAtom().IfJust( - atom => atom.TryEvaluate(e => { Error(parameters[0].MyLocation, e.Message); }).IfJust( - val => { value = val; }), - () => { Error(parameters[0].MyLocation, "Expected atomic param to FILL"); }); - } + for (int i = 0; i < raws.Count; i++) + { + Error($"Variant {i + 1}: `{raws[i].ToPrettyString()}`"); + } + } + + IgnoreRestOfStatement(tokens); + return null; + } + else + { + Error("Unrecognized code: " + head.Content); + return null; + } + } + + private ILineNode? ParseOrgStatement(IList parameters) + { + if (parameters.Count != 1) + { + Error($"Incorrect number of parameters in ORG: {parameters.Count}"); + return null; + } - // param 1 is amount of bytes to fill - parameters[0].AsAtom().IfJust( - atom => atom.TryEvaluate(e => { Error(parameters[0].MyLocation, e.Message); }).IfJust( - val => { amount = val; }), - () => { Error(parameters[0].MyLocation, "Expected atomic param to FILL"); }); + parameters[0].AsAtom().IfJust( + atom => atom.TryEvaluate(e => Error(parameters[0].MyLocation, e.Message)).IfJust( + offsetValue => { CurrentOffset = ConvertToOffset(offsetValue); }, + () => Error(parameters[0].MyLocation, "Expected atomic param to ORG."))); + + return null; + } - var data = new byte[amount]; + private ILineNode? ParsePushStatement(IList parameters) + { + if (parameters.Count != 0) + { + Error("Incorrect number of parameters in PUSH: " + parameters.Count); + } + else + { + pastOffsets.Push(new Tuple(CurrentOffset, offsetInitialized)); + } - for (int i = 0; i < amount; ++i) - data[i] = (byte)value; + return null; + } - var node = new DataNode(CurrentOffset, data); + private ILineNode? ParsePopStatement(IList parameters) + { + if (parameters.Count != 0) + { + Error($"Incorrect number of parameters in POP: {parameters.Count}"); + } + else if (pastOffsets.Count == 0) + { + Error("POP without matching PUSH."); + } + else + { + Tuple tuple = pastOffsets.Pop(); - CheckDataWrite(amount); - CurrentOffset += amount; + CurrentOffset = tuple.Item1; + offsetInitialized = tuple.Item2; + } - return node; - } + return null; + } - break; - } + private ILineNode? ParseAssertStatement(IList parameters) + { + if (parameters.Count != 1) + { + Error($"Incorrect number of parameters in ASSERT: {parameters.Count}"); return null; } - else if (Raws.ContainsKey(upperCodeIdentifier)) - { - //TODO: Check for matches. Currently should type error. - foreach (Raw r in Raws[upperCodeIdentifier]) - { - if (r.Fits(parameters)) + + parameters[0].AsAtom().IfJust( + atom => atom.TryEvaluate(e => Error(parameters[0].MyLocation, e.Message)).IfJust( + temp => { - if ((CurrentOffset % r.Alignment) != 0) + if (temp < 0) { - Error(head.Location, string.Format("Bad code alignment (offset: {0:X8})", CurrentOffset)); - + Error(parameters[0].MyLocation, "Assertion error: " + temp); } - StatementNode temp = new RawNode(r, head, CurrentOffset, parameters); + }), + () => Error(parameters[0].MyLocation, "Expected atomic param to ASSERT.")); - CheckDataWrite(temp.Size); - CurrentOffset += temp.Size; //TODO: more efficient spacewise to just have contiguous writing and not an offset with every line? + return null; + } - return temp; + private ILineNode? ParseProtectStatement(IList parameters) + { + if (parameters.Count == 1) + { + parameters[0].AsAtom().IfJust( + atom => atom.TryEvaluate(e => { Error(parameters[0].MyLocation, e.Message); }).IfJust( + temp => + { + protectedRegions.Add(new Tuple(temp, 4, head!.Location)); + }), + () => { Error(parameters[0].MyLocation, "Expected atomic param to PROTECT"); }); + } + else if (parameters.Count == 2) + { + int start = 0, end = 0; + bool errorOccurred = false; + parameters[0].AsAtom().IfJust( + atom => atom.TryEvaluate(e => { Error(parameters[0].MyLocation, e.Message); errorOccurred = true; }).IfJust( + temp => + { + start = temp; + }), + () => { Error(parameters[0].MyLocation, "Expected atomic param to PROTECT"); errorOccurred = true; }); + parameters[1].AsAtom().IfJust( + atom => atom.TryEvaluate(e => { Error(parameters[0].MyLocation, e.Message); errorOccurred = true; }).IfJust( + temp => + { + end = temp; + }), + () => { Error(parameters[0].MyLocation, "Expected atomic param to PROTECT"); errorOccurred = true; }); + if (!errorOccurred) + { + int length = end - start; + if (length > 0) + { + protectedRegions.Add(new Tuple(start, length, head!.Location)); + } + else + { + Warning("Protected region not valid (end offset not after start offset). No region protected."); } } - //TODO: Better error message (a la EA's ATOM ATOM [ATOM,ATOM]) - Error(head.Location, "Incorrect parameters in raw " + head.Content + '.'); - IgnoreRestOfStatement(tokens); + } + else + { + Error("Incorrect number of parameters in PROTECT: " + parameters.Count); + } + + return null; + } + + private ILineNode? ParseAlignStatement(IList parameters) + { + if (parameters.Count != 1) + { + Error("Incorrect number of parameters in ALIGN: " + parameters.Count); return null; } - else //TODO: Move outside of this else. + + parameters[0].AsAtom().IfJust( + atom => atom.TryEvaluate(e => Error(parameters[0].MyLocation, e.Message)).IfJust( + temp => CurrentOffset = CurrentOffset % temp != 0 ? CurrentOffset + temp - CurrentOffset % temp : CurrentOffset), + () => Error(parameters[0].MyLocation, "Expected atomic param to ALIGN")); + + return null; + } + + private ILineNode? ParseFillStatement(IList parameters) + { + if (parameters.Count > 2 || parameters.Count == 0) { - Error(head.Location, "Unrecognized code: " + head.Content); + Error("Incorrect number of parameters in FILL: " + parameters.Count); return null; } + + // FILL amount [value] + + int amount = 0; + int value = 0; + + if (parameters.Count == 2) + { + // param 2 (if given) is fill value + + parameters[1].AsAtom().IfJust( + atom => atom.TryEvaluate(e => Error(parameters[0].MyLocation, e.Message)).IfJust( + val => { value = val; }), + () => Error(parameters[0].MyLocation, "Expected atomic param to FILL")); + } + + // param 1 is amount of bytes to fill + parameters[0].AsAtom().IfJust( + atom => atom.TryEvaluate(e => Error(parameters[0].MyLocation, e.Message)).IfJust( + val => { amount = val; }), + () => Error(parameters[0].MyLocation, "Expected atomic param to FILL")); + + if (amount > 0) + { + var data = new byte[amount]; + + for (int i = 0; i < amount; ++i) + { + data[i] = (byte)value; + } + + var node = new DataNode(CurrentOffset, data); + + CheckDataWrite(amount); + CurrentOffset += amount; + + return node; + } + + return null; + } + + private ILineNode? ParseMessageStatement(IList parameters) + { + Message(PrettyPrintParams(parameters)); + return null; + } + + private ILineNode? ParseWarningStatement(IList parameters) + { + Warning(PrettyPrintParams(parameters)); + return null; + } + + private ILineNode? ParseErrorStatement(IList parameters) + { + Error(PrettyPrintParams(parameters)); + return null; } public IList> ParseMacroParamList(MergeableGenerator tokens) @@ -403,9 +493,14 @@ public IList> ParseMacroParamList(MergeableGenerator tokens) && tokens.Current.Type != TokenType.NEWLINE) { if (tokens.Current.Type == TokenType.CLOSE_PAREN) + { parenNestings--; + } else if (tokens.Current.Type == TokenType.OPEN_PAREN) + { parenNestings++; + } + currentParam.Add(tokens.Current); tokens.MoveNext(); } @@ -426,16 +521,21 @@ private IList ParseParamList(MergeableGenerator tokens, Immut { IList paramList = new List(); bool first = true; + while (tokens.Current.Type != TokenType.NEWLINE && tokens.Current.Type != TokenType.SEMICOLON && !tokens.EOS) { - Token head = tokens.Current; + Token localHead = tokens.Current; ParseParam(tokens, scopes, expandFirstDef || !first).IfJust( n => paramList.Add(n), - () => Error(head.Location, "Expected parameter.")); + () => Error(localHead.Location, "Expected parameter.")); first = false; } + if (tokens.Current.Type == TokenType.SEMICOLON) + { tokens.MoveNext(); + } + return paramList; } @@ -454,14 +554,14 @@ private IList ParsePreprocParamList(MergeableGenerator tokens private IParamNode? ParseParam(MergeableGenerator tokens, ImmutableStack scopes, bool expandDefs = true) { - Token head = tokens.Current; + Token localHead = tokens.Current; switch (tokens.Current.Type) { case TokenType.OPEN_BRACKET: - return new ListNode(head.Location, ParseList(tokens, scopes)).Simplify(); + return new ListNode(localHead.Location, ParseList(tokens, scopes)).Simplify(); case TokenType.STRING: tokens.MoveNext(); - return new StringNode(head); + return new StringNode(localHead); case TokenType.MAYBE_MACRO: //TODO: Move this and the one in ExpandId to a separate ParseMacroNode that may return an Invocation. if (expandDefs && ExpandIdentifier(tokens, scopes)) @@ -473,38 +573,49 @@ private IList ParsePreprocParamList(MergeableGenerator tokens tokens.MoveNext(); IList> param = ParseMacroParamList(tokens); //TODO: Smart errors if trying to redefine a macro with the same num of params. - return new MacroInvocationNode(this, head, param, scopes); + return new MacroInvocationNode(this, localHead, param, scopes); } case TokenType.IDENTIFIER: - if (expandDefs && Definitions.ContainsKey(head.Content) && ExpandIdentifier(tokens, scopes)) + if (expandDefs && Definitions.ContainsKey(localHead.Content) && ExpandIdentifier(tokens, scopes)) + { return ParseParam(tokens, scopes, expandDefs); + } else - return ParseAtom(tokens, scopes, expandDefs).Fmap(x => (IParamNode)x.Simplify()); + { + return ParseAtom(tokens, scopes, expandDefs)?.Simplify(); + } + default: - return ParseAtom(tokens, scopes, expandDefs).Fmap(x => (IParamNode)x.Simplify()); + return ParseAtom(tokens, scopes, expandDefs)?.Simplify(); } } private static readonly Dictionary precedences = new Dictionary { - { TokenType.MUL_OP , 3 }, - { TokenType.DIV_OP , 3 }, - { TokenType.ADD_OP , 4 }, - { TokenType.SUB_OP , 4 }, - { TokenType.LSHIFT_OP , 5 }, - { TokenType.RSHIFT_OP , 5 }, - { TokenType.SIGNED_RSHIFT_OP , 5 }, - { TokenType.AND_OP , 8 }, - { TokenType.XOR_OP , 9 }, - { TokenType.OR_OP , 10 }, - { TokenType.MOD_OP , 3 } + { TokenType.MUL_OP, 3 }, + { TokenType.DIV_OP, 3 }, + { TokenType.MOD_OP, 3 }, + { TokenType.ADD_OP, 4 }, + { TokenType.SUB_OP, 4 }, + { TokenType.LSHIFT_OP, 5 }, + { TokenType.RSHIFT_OP, 5 }, + { TokenType.SIGNED_RSHIFT_OP, 5 }, + { TokenType.COMPARE_GE, 6 }, + { TokenType.COMPARE_GT, 6 }, + { TokenType.COMPARE_LT, 6 }, + { TokenType.COMPARE_LE, 6 }, + { TokenType.COMPARE_EQ, 7 }, + { TokenType.COMPARE_NE, 7 }, + { TokenType.AND_OP, 8 }, + { TokenType.XOR_OP, 9 }, + { TokenType.OR_OP, 10 }, + { TokenType.LOGAND_OP, 11 }, + { TokenType.LOGOR_OP, 12 }, }; - - private IAtomNode? ParseAtom(MergeableGenerator tokens, ImmutableStack scopes, bool expandDefs = true) { //Use Shift Reduce Parsing - Token head = tokens.Current; + Token localHead = tokens.Current; Stack> grammarSymbols = new Stack>(); bool ended = false; while (!ended) @@ -514,7 +625,7 @@ private IList ParsePreprocParamList(MergeableGenerator tokens if (!ended && !lookingForAtom) //Is already a complete node. Needs an operator of matching precedence and a node of matching prec to reduce. { - //Verify next symbol to be an operator. + //Verify next symbol to be a binary operator. switch (lookAhead.Type) { case TokenType.MUL_OP: @@ -528,9 +639,17 @@ private IList ParsePreprocParamList(MergeableGenerator tokens case TokenType.AND_OP: case TokenType.XOR_OP: case TokenType.OR_OP: - if (precedences.ContainsKey(lookAhead.Type)) + case TokenType.LOGAND_OP: + case TokenType.LOGOR_OP: + case TokenType.COMPARE_LT: + case TokenType.COMPARE_LE: + case TokenType.COMPARE_EQ: + case TokenType.COMPARE_NE: + case TokenType.COMPARE_GE: + case TokenType.COMPARE_GT: + if (precedences.TryGetValue(lookAhead.Type, out int precedence)) { - Reduce(grammarSymbols, precedences[lookAhead.Type]); + Reduce(grammarSymbols, precedence); } shift = true; break; @@ -571,16 +690,17 @@ private IList ParsePreprocParamList(MergeableGenerator tokens } } case TokenType.SUB_OP: + case TokenType.LOGNOT_OP: { //Assume unary negation. tokens.MoveNext(); IAtomNode? interior = ParseAtom(tokens, scopes); if (interior == null) { - Error(lookAhead.Location, "Expected expression after negation. "); + Error(lookAhead.Location, "Expected expression after unary operator."); return null; } - grammarSymbols.Push(new Left(new NegationNode(lookAhead, interior))); + grammarSymbols.Push(new Left(new UnaryOperatorNode(lookAhead, interior))); break; } case TokenType.COMMA: @@ -597,6 +717,14 @@ private IList ParsePreprocParamList(MergeableGenerator tokens case TokenType.AND_OP: case TokenType.XOR_OP: case TokenType.OR_OP: + case TokenType.LOGAND_OP: + case TokenType.LOGOR_OP: + case TokenType.COMPARE_LT: + case TokenType.COMPARE_LE: + case TokenType.COMPARE_EQ: + case TokenType.COMPARE_NE: + case TokenType.COMPARE_GE: + case TokenType.COMPARE_GT: default: Error(lookAhead.Location, "Expected identifier or literal, got " + lookAhead.Type + ": " + lookAhead.Content + '.'); IgnoreRestOfStatement(tokens); @@ -609,11 +737,18 @@ private IList ParsePreprocParamList(MergeableGenerator tokens if (lookAhead.Type == TokenType.IDENTIFIER) { if (expandDefs && ExpandIdentifier(tokens, scopes)) + { continue; + } + if (lookAhead.Content.ToUpper() == "CURRENTOFFSET") + { grammarSymbols.Push(new Left(new NumberNode(lookAhead, CurrentOffset))); + } else + { grammarSymbols.Push(new Left(new IdentifierNode(lookAhead, scopes))); + } } else if (lookAhead.Type == TokenType.MAYBE_MACRO) { @@ -644,7 +779,7 @@ private IList ParsePreprocParamList(MergeableGenerator tokens } if (grammarSymbols.Peek().IsRight) { - Error(grammarSymbols.Peek().GetRight.Location, "Unexpected token: " + grammarSymbols.Peek().GetRight.Type); + Error(grammarSymbols.Peek().GetRight.Location, $"Unexpected token: {grammarSymbols.Peek().GetRight.Type}"); } return grammarSymbols.Peek().GetLeft; } @@ -655,7 +790,7 @@ private IList ParsePreprocParamList(MergeableGenerator tokens * Postcondition: Either grammarSymbols.Count == 1, or everything in grammarSymbols will have precedence <= targetPrecedence. * */ - private void Reduce(Stack> grammarSymbols, int targetPrecedence) + private static void Reduce(Stack> grammarSymbols, int targetPrecedence) { while (grammarSymbols.Count > 1)// && grammarSymbols.Peek().GetLeft.Precedence > targetPrecedence) { @@ -677,19 +812,11 @@ private void Reduce(Stack> grammarSymbols, int targetPr } } - private int GetLowestPrecedence(Stack> grammarSymbols) - { - int minPrec = 11; //TODO: Note that this is the largest possible value. - foreach (Either e in grammarSymbols) - e.Case(n => { minPrec = Math.Min(minPrec, n.Precedence); }, - t => { minPrec = Math.Min(minPrec, precedences[t.Type]); }); - return minPrec; - } - private IList ParseList(MergeableGenerator tokens, ImmutableStack scopes) { - Token head = tokens.Current; + Token localHead = tokens.Current; tokens.MoveNext(); + IList atoms = new List(); while (tokens.Current.Type != TokenType.NEWLINE && tokens.Current.Type != TokenType.CLOSE_BRACKET) { @@ -698,12 +825,19 @@ private IList ParseList(MergeableGenerator tokens, ImmutableSt n => atoms.Add(n), () => Error(tokens.Current.Location, "Expected atomic value, got " + tokens.Current.Type + ".")); if (tokens.Current.Type == TokenType.COMMA) + { tokens.MoveNext(); + } } if (tokens.Current.Type == TokenType.CLOSE_BRACKET) + { tokens.MoveNext(); + } else - Error(head.Location, "Unmatched open bracket."); + { + Error(localHead.Location, "Unmatched open bracket."); + } + return atoms; } @@ -733,11 +867,11 @@ private IList ParseList(MergeableGenerator tokens, ImmutableSt tokens.MoveNext(); if (scopes.Head.HasLocalLabel(head.Content)) { - Warning(head.Location, "Label already in scope, ignoring: " + head.Content);//replacing: " + head.Content); + Warning("Label already in scope, ignoring: " + head.Content);//replacing: " + head.Content); } else if (!IsValidLabelName(head.Content)) { - Error(head.Location, "Invalid label name " + head.Content + '.'); + Error("Invalid label name " + head.Content + '.'); } else { @@ -757,17 +891,17 @@ private IList ParseList(MergeableGenerator tokens, ImmutableSt case TokenType.PREPROCESSOR_DIRECTIVE: return ParsePreprocessor(tokens, scopes); case TokenType.OPEN_BRACKET: - Error(head.Location, "Unexpected list literal."); + Error("Unexpected list literal."); IgnoreRestOfLine(tokens); break; case TokenType.NUMBER: case TokenType.OPEN_PAREN: - Error(head.Location, "Unexpected mathematical expression."); + Error("Unexpected mathematical expression."); IgnoreRestOfLine(tokens); break; default: tokens.MoveNext(); - Error(head.Location, $"Unexpected token: {head.Type}: {head.Content}"); + Error($"Unexpected token: {head.Type}: {head.Content}"); IgnoreRestOfLine(tokens); break; } @@ -776,7 +910,11 @@ private IList ParseList(MergeableGenerator tokens, ImmutableSt else { bool hasNext = true; - while (tokens.Current.Type != TokenType.PREPROCESSOR_DIRECTIVE && (hasNext = tokens.MoveNext())) ; + while (tokens.Current.Type != TokenType.PREPROCESSOR_DIRECTIVE && (hasNext = tokens.MoveNext())) + { + ; + } + if (hasNext) { return ParsePreprocessor(tokens, scopes); @@ -793,14 +931,17 @@ private IList ParseList(MergeableGenerator tokens, ImmutableSt { head = tokens.Current; tokens.MoveNext(); + //Note: Not a ParseParamList because no commas. IList paramList = ParsePreprocParamList(tokens, scopes); ILineNode? retVal = directiveHandler.HandleDirective(this, head, paramList, tokens); + if (retVal != null) { CheckDataWrite(retVal.Size); CurrentOffset += retVal.Size; } + return retVal; } @@ -815,31 +956,31 @@ public bool ExpandIdentifier(MergeableGenerator tokens, ImmutableStack> parameters = ParseMacroParamList(tokens); - if (Macros.HasMacro(head.Content, parameters.Count)) + if (Macros.HasMacro(localHead.Content, parameters.Count)) { - tokens.PrependEnumerator(Macros.GetMacro(head.Content, parameters.Count).ApplyMacro(head, parameters, scopes).GetEnumerator()); + tokens.PrependEnumerator(Macros.GetMacro(localHead.Content, parameters.Count).ApplyMacro(localHead, parameters, scopes).GetEnumerator()); } else { - Error(head.Location, System.String.Format("No overload of {0} with {1} parameters.", head.Content, parameters.Count)); + Error(System.String.Format("No overload of {0} with {1} parameters.", localHead.Content, parameters.Count)); } return true; } else if (tokens.Current.Type == TokenType.MAYBE_MACRO) { - Token head = tokens.Current; + Token localHead = tokens.Current; tokens.MoveNext(); - tokens.PutBack(new Token(TokenType.IDENTIFIER, head.Location, head.Content)); + tokens.PutBack(new Token(TokenType.IDENTIFIER, localHead.Location, localHead.Content)); return true; } else if (Definitions.ContainsKey(tokens.Current.Content)) { - Token head = tokens.Current; + Token localHead = tokens.Current; tokens.MoveNext(); - tokens.PrependEnumerator(Definitions[head.Content].ApplyDefinition(head).GetEnumerator()); + tokens.PrependEnumerator(Definitions[localHead.Content].ApplyDefinition(localHead).GetEnumerator()); return true; } @@ -848,28 +989,37 @@ public bool ExpandIdentifier(MergeableGenerator tokens, ImmutableStack Message(head?.Location, message); + public void Warning(string message) => Warning(head?.Location, message); + public void Error(string message) => Error(head?.Location, message); + private void IgnoreRestOfStatement(MergeableGenerator tokens) { - while (tokens.Current.Type != TokenType.NEWLINE && tokens.Current.Type != TokenType.SEMICOLON && tokens.MoveNext()) ; - if (tokens.Current.Type == TokenType.SEMICOLON) tokens.MoveNext(); + while (tokens.Current.Type != TokenType.NEWLINE && tokens.Current.Type != TokenType.SEMICOLON && tokens.MoveNext()) { } + if (tokens.Current.Type == TokenType.SEMICOLON) + { + tokens.MoveNext(); + } } private void IgnoreRestOfLine(MergeableGenerator tokens) { - while (tokens.Current.Type != TokenType.NEWLINE && tokens.MoveNext()) ; + while (tokens.Current.Type != TokenType.NEWLINE && tokens.MoveNext()) { } } public void Clear() @@ -887,8 +1037,7 @@ private string PrettyPrintParams(IList parameters) StringBuilder sb = new StringBuilder(); foreach (IParamNode parameter in parameters) { - sb.Append(parameter.PrettyPrint()); - sb.Append(' '); + sb.Append(parameter.PrettyPrint()).Append(' '); } return sb.ToString(); } @@ -901,8 +1050,11 @@ private string PrettyPrintParams(IList parameters) //They intersect if the last offset in the given region is after the start of this one //and the first offset in the given region is before the last of this one if (offset + length > protectedRegion.Item1 && offset < protectedRegion.Item1 + protectedRegion.Item2) + { return protectedRegion.Item3; + } } + return null; } @@ -911,7 +1063,7 @@ private void CheckDataWrite(int length) // TODO: maybe make this warning optional? if (!offsetInitialized) { - Warning(head?.Location, "Writing before initializing offset. You may be breaking the ROM! (use `ORG offset` to set write offset)."); + Warning("Writing before initializing offset. You may be breaking the ROM! (use `ORG offset` to set write offset)."); offsetInitialized = false; // only warn once } @@ -920,7 +1072,7 @@ private void CheckDataWrite(int length) if (IsProtected(CurrentOffset, length) is Location prot) { - Error(head?.Location, $"Trying to write data to area protected in file {Path.GetFileName(prot.file)} at line {prot.lineNum}, column {prot.colNum}."); + Error($"Trying to write data to area protected in file {Path.GetFileName(prot.file)} at line {prot.lineNum}, column {prot.colNum}."); } } } diff --git a/ColorzCore/Program.cs b/ColorzCore/Program.cs index 1004209..8b04632 100644 --- a/ColorzCore/Program.cs +++ b/ColorzCore/Program.cs @@ -121,6 +121,13 @@ static int Main(string[] args) { case "raws": rawsFolder = rawSearcher.FindDirectory(flag[1]); + + if (rawsFolder == null) + { + Console.Error.WriteLine($"No such folder: {flag[1]}"); + return EXIT_FAILURE; + } + break; case "rawsExt": @@ -251,12 +258,6 @@ static int Main(string[] args) return EXIT_FAILURE; } - if (rawsFolder == null) - { - Console.Error.WriteLine("Couldn't find raws folder"); - return EXIT_FAILURE; - } - IOutput output; if (outputASM) @@ -298,10 +299,10 @@ static int Main(string[] args) }; if (EAOptions.Instance.nowarn) - log.IgnoredKinds.Add(Log.MsgKind.WARNING); + log.IgnoredKinds.Add(Log.MessageKind.WARNING); if (EAOptions.Instance.nomess) - log.IgnoredKinds.Add(Log.MsgKind.MESSAGE); + log.IgnoredKinds.Add(Log.MessageKind.MESSAGE); EAInterpreter myInterpreter = new EAInterpreter(output, game, rawsFolder, rawsExtension, inStream, inFileName, log); @@ -311,12 +312,11 @@ static int Main(string[] args) if (success && EAOptions.Instance.nocashSym) { - using (var symOut = File.CreateText(Path.ChangeExtension(outFileName, "sym"))) + using StreamWriter symOut = File.CreateText(Path.ChangeExtension(outFileName, "sym")); + + if (!(success = myInterpreter.WriteNocashSymbols(symOut))) { - if (!(success = myInterpreter.WriteNocashSymbols(symOut))) - { - log.Message(Log.MsgKind.ERROR, "Error trying to write no$gba symbol file."); - } + log.Message(Log.MessageKind.ERROR, "Error trying to write no$gba symbol file."); } } @@ -345,7 +345,6 @@ static int Main(string[] args) errorStream.Close(); return success ? EXIT_SUCCESS : EXIT_FAILURE; - } } } diff --git a/ColorzCore/Raws/Raw.cs b/ColorzCore/Raws/Raw.cs index 688c90b..5788334 100644 --- a/ColorzCore/Raws/Raw.cs +++ b/ColorzCore/Raws/Raw.cs @@ -16,6 +16,9 @@ class Raw public int Alignment { get; } public HashSet Game { get; } + // symbolic helper + public static HashSet AnyGame { get; } = new(); + private readonly IList parameters; private readonly bool repeatable; @@ -25,8 +28,15 @@ class Raw // TODO: fixed mask? + public struct FixedParam + { + public int position; + public int size; + public int value; + } + public Raw(string name, int length, short code, int offsetMod, HashSet game, IList varParams, - IList fixedParams, int? terminatingList, bool repeatable) + IList? fixedParams, int? terminatingList, bool repeatable) { Name = name; Game = game; @@ -41,10 +51,15 @@ public Raw(string name, int length, short code, int offsetMod, HashSet g // Build base unit if (code != 0) + { baseUnit.SetBits(0, 16, code); + } - foreach (var fp in fixedParams) - baseUnit.SetBits(fp.position, fp.size, fp.value); + if (fixedParams != null) + { + foreach (var fp in fixedParams) + baseUnit.SetBits(fp.position, fp.size, fp.value); + } // Build end unit, if needed @@ -61,6 +76,11 @@ public Raw(string name, int length, short code, int offsetMod, HashSet g } } + public Raw(string name, int length, short code, int offsetMod, IList varParams, bool repeatable) + : this(name, length, code, offsetMod, AnyGame, varParams, null, null, repeatable) + { + } + public int UnitCount(int paramCount) { if (parameters.Count == 0) @@ -133,11 +153,23 @@ public byte[] GetBytes(IList arguments) return result; } - public struct FixedParam + public string ToPrettyString() { - public int position; - public int size; - public int value; + StringBuilder sb = new(); + + sb.Append(Name); + + foreach (IRawParam param in parameters) + { + sb.Append(' ').Append((param is ListParam) ? $"[{param.Name}...]" : param.Name); + } + + if (repeatable) + { + sb.Append("..."); + } + + return sb.ToString(); } } } diff --git a/ColorzCore/Raws/RawReader.cs b/ColorzCore/Raws/RawReader.cs index 4f4306f..ee30b72 100644 --- a/ColorzCore/Raws/RawReader.cs +++ b/ColorzCore/Raws/RawReader.cs @@ -321,9 +321,9 @@ private static Dictionary ParseFlags(string flagStr) if (flag[0] != '-') throw new Exception("Flag does not start with '-'"); - var withoutDash = flag.Substring(1); + var withoutDash = flag[1..]; - var name = string.Empty; + string name; var value = new Flag(); if (withoutDash.Contains(':')) @@ -365,7 +365,7 @@ private class FileLineReader : IDisposable // Helper wrapper class for reading lines from file while counting them // That way we can print line number in error messages - private StreamReader reader; + private readonly StreamReader reader; public string FileName { get; } public int LineNumber { get; private set; } @@ -385,7 +385,7 @@ public FileLineReader(FileStream input) string? line = reader.ReadLine(); if (line != null) - LineNumber = LineNumber + 1; + LineNumber++; return line; } From 2d56cd72033bb2975e5e09c817c675a37ca68c23 Mon Sep 17 00:00:00 2001 From: Nat_776 Date: Wed, 17 Apr 2024 18:37:24 +0200 Subject: [PATCH 10/59] add #if directive --- ColorzCore/EAInterpreter.cs | 2 +- ColorzCore/Parser/AST/IdentifierNode.cs | 2 +- ColorzCore/Parser/EAParser.cs | 21 +++++---- ColorzCore/Preprocessor/DirectiveHandler.cs | 7 +-- .../Preprocessor/Directives/IfDirective.cs | 44 +++++++++++++++++++ 5 files changed, 62 insertions(+), 14 deletions(-) create mode 100644 ColorzCore/Preprocessor/Directives/IfDirective.cs diff --git a/ColorzCore/EAInterpreter.cs b/ColorzCore/EAInterpreter.cs index b686f3c..406c201 100644 --- a/ColorzCore/EAInterpreter.cs +++ b/ColorzCore/EAInterpreter.cs @@ -111,7 +111,7 @@ public bool Interpret() } else { - myParser.Error(errCause.Location, "Undefined identifier: " + errCause.Content); + myParser.Error(errCause.Location, $"Undefined identifier `{errCause.Content}`"); } } diff --git a/ColorzCore/Parser/AST/IdentifierNode.cs b/ColorzCore/Parser/AST/IdentifierNode.cs index e911d93..d90d264 100644 --- a/ColorzCore/Parser/AST/IdentifierNode.cs +++ b/ColorzCore/Parser/AST/IdentifierNode.cs @@ -68,7 +68,7 @@ public override string PrettyPrint() public class UndefinedIdentifierException : Exception { public Token CausedError { get; set; } - public UndefinedIdentifierException(Token causedError) : base("Undefined identifier: " + causedError.Content) + public UndefinedIdentifierException(Token causedError) : base($"Undefined identifier `{causedError.Content}`") { this.CausedError = causedError; } diff --git a/ColorzCore/Parser/EAParser.cs b/ColorzCore/Parser/EAParser.cs index ce9948f..923b719 100644 --- a/ColorzCore/Parser/EAParser.cs +++ b/ColorzCore/Parser/EAParser.cs @@ -539,9 +539,10 @@ private IList ParseParamList(MergeableGenerator tokens, Immut return paramList; } - private IList ParsePreprocParamList(MergeableGenerator tokens, ImmutableStack scopes) + private IList ParsePreprocParamList(MergeableGenerator tokens, ImmutableStack scopes, bool allowsFirstExpanded) { - IList temp = ParseParamList(tokens, scopes, false); + IList temp = ParseParamList(tokens, scopes, allowsFirstExpanded); + for (int i = 0; i < temp.Count; i++) { if (temp[i].Type == ParamType.STRING && ((StringNode)temp[i]).IsValidIdentifier()) @@ -549,6 +550,7 @@ private IList ParsePreprocParamList(MergeableGenerator tokens temp[i] = ((StringNode)temp[i]).ToIdentifier(scopes); } } + return temp; } @@ -932,17 +934,18 @@ private IList ParseList(MergeableGenerator tokens, ImmutableSt head = tokens.Current; tokens.MoveNext(); - //Note: Not a ParseParamList because no commas. - IList paramList = ParsePreprocParamList(tokens, scopes); - ILineNode? retVal = directiveHandler.HandleDirective(this, head, paramList, tokens); + // Note: Not a ParseParamList because no commas. + // HACK: #if wants its parameters to be expanded, but other directives (define, ifdef, undef, etc) do not + IList paramList = ParsePreprocParamList(tokens, scopes, head.Content == "#if"); + ILineNode? result = directiveHandler.HandleDirective(this, head, paramList, tokens); - if (retVal != null) + if (result != null) { - CheckDataWrite(retVal.Size); - CurrentOffset += retVal.Size; + CheckDataWrite(result.Size); + CurrentOffset += result.Size; } - return retVal; + return result; } /*** diff --git a/ColorzCore/Preprocessor/DirectiveHandler.cs b/ColorzCore/Preprocessor/DirectiveHandler.cs index 373bb7c..0556787 100644 --- a/ColorzCore/Preprocessor/DirectiveHandler.cs +++ b/ColorzCore/Preprocessor/DirectiveHandler.cs @@ -27,6 +27,7 @@ public DirectiveHandler(IncludeFileSearcher includeSearcher, IncludeFileSearcher { "inctevent", new IncludeToolEventDirective { FileSearcher = toolSearcher } }, { "ifdef", new IfDefinedDirective() }, { "ifndef", new IfNotDefinedDirective() }, + { "if", new IfDirective() }, { "else", new ElseDirective() }, { "endif", new EndIfDirective() }, { "define", new DefineDirective() }, @@ -37,7 +38,7 @@ public DirectiveHandler(IncludeFileSearcher includeSearcher, IncludeFileSearcher public ILineNode? HandleDirective(EAParser p, Token directive, IList parameters, MergeableGenerator tokens) { - string directiveName = directive.Content.Substring(1); + string directiveName = directive.Content[1..]; if (directives.TryGetValue(directiveName, out IDirective? toExec)) { @@ -49,13 +50,13 @@ public DirectiveHandler(IncludeFileSearcher includeSearcher, IncludeFileSearcher } else { - p.Error(directive.Location, "Invalid number of parameters (" + parameters.Count + ") to directive " + directiveName + "."); + p.Error(directive.Location, $"Invalid number of parameters ({parameters.Count}) to directive {directiveName}."); } } } else { - p.Error(directive.Location, "Directive not recognized: " + directiveName); + p.Error(directive.Location, $"Directive not recognized: {directiveName}"); } return null; diff --git a/ColorzCore/Preprocessor/Directives/IfDirective.cs b/ColorzCore/Preprocessor/Directives/IfDirective.cs new file mode 100644 index 0000000..90b2756 --- /dev/null +++ b/ColorzCore/Preprocessor/Directives/IfDirective.cs @@ -0,0 +1,44 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using ColorzCore.DataTypes; +using ColorzCore.Lexer; +using ColorzCore.Parser; +using ColorzCore.Parser.AST; + +namespace ColorzCore.Preprocessor.Directives +{ + class IfDirective : IDirective + { + public int MinParams => 1; + + public int? MaxParams => 1; + + public bool RequireInclusion => false; + + public ILineNode? Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) + { + bool flag = true; + + foreach (IParamNode parameter in parameters) + { + if (parameter is IAtomNode atomNode) + { + if (atomNode.TryEvaluate(e => p.Error(self.Location, $"Error while evaluating expression: {e.Message}")) is int value) + { + flag = value != 0; + } + } + else + { + p.Error(self.Location, "Expected an expression."); + } + } + + p.Inclusion = new ImmutableStack(flag, p.Inclusion); + return null; + } + } +} From f425f9f884dddf39d965edbeccac3a88e17ae5a3 Mon Sep 17 00:00:00 2001 From: Nat_776 Date: Wed, 17 Apr 2024 18:58:00 +0200 Subject: [PATCH 11/59] add alternative projet targetting traditional .NET framework --- ColorzCore/ColorzCore.Framework.csproj | 11 +++++++++++ ColorzCore/ColorzCore.csproj | 4 ++++ ColorzCore/EAInterpreter.cs | 4 ++-- ColorzCore/Preprocessor/DirectiveHandler.cs | 2 +- ColorzCore/Raws/Raw.cs | 4 ++-- ColorzCore/Raws/RawReader.cs | 2 +- 6 files changed, 21 insertions(+), 6 deletions(-) create mode 100644 ColorzCore/ColorzCore.Framework.csproj diff --git a/ColorzCore/ColorzCore.Framework.csproj b/ColorzCore/ColorzCore.Framework.csproj new file mode 100644 index 0000000..6918bae --- /dev/null +++ b/ColorzCore/ColorzCore.Framework.csproj @@ -0,0 +1,11 @@ + + + net48 + Exe + annotations + + + + 9 + + diff --git a/ColorzCore/ColorzCore.csproj b/ColorzCore/ColorzCore.csproj index dc24dde..f1236eb 100644 --- a/ColorzCore/ColorzCore.csproj +++ b/ColorzCore/ColorzCore.csproj @@ -4,4 +4,8 @@ Exe enable + + + 9 + \ No newline at end of file diff --git a/ColorzCore/EAInterpreter.cs b/ColorzCore/EAInterpreter.cs index 406c201..7055327 100644 --- a/ColorzCore/EAInterpreter.cs +++ b/ColorzCore/EAInterpreter.cs @@ -183,12 +183,12 @@ private static IList GetFallbackRaws() { static List CreateParams(int bitSize, bool isPointer) { - return new() { new AtomicParam("Data", 0, bitSize, isPointer) }; + return new List() { new AtomicParam("Data", 0, bitSize, isPointer) }; } static Raw CreateRaw(string name, int byteSize, int alignment, bool isPointer) { - return new(name, byteSize * 8, 0, alignment, CreateParams(byteSize * 8, isPointer), true); + return new Raw(name, byteSize * 8, 0, alignment, CreateParams(byteSize * 8, isPointer), true); } return new List() diff --git a/ColorzCore/Preprocessor/DirectiveHandler.cs b/ColorzCore/Preprocessor/DirectiveHandler.cs index 0556787..b3b37d2 100644 --- a/ColorzCore/Preprocessor/DirectiveHandler.cs +++ b/ColorzCore/Preprocessor/DirectiveHandler.cs @@ -38,7 +38,7 @@ public DirectiveHandler(IncludeFileSearcher includeSearcher, IncludeFileSearcher public ILineNode? HandleDirective(EAParser p, Token directive, IList parameters, MergeableGenerator tokens) { - string directiveName = directive.Content[1..]; + string directiveName = directive.Content.Substring(1); if (directives.TryGetValue(directiveName, out IDirective? toExec)) { diff --git a/ColorzCore/Raws/Raw.cs b/ColorzCore/Raws/Raw.cs index 5788334..9ee0551 100644 --- a/ColorzCore/Raws/Raw.cs +++ b/ColorzCore/Raws/Raw.cs @@ -17,7 +17,7 @@ class Raw public HashSet Game { get; } // symbolic helper - public static HashSet AnyGame { get; } = new(); + public static HashSet AnyGame { get; } = new HashSet(); private readonly IList parameters; private readonly bool repeatable; @@ -155,7 +155,7 @@ public byte[] GetBytes(IList arguments) public string ToPrettyString() { - StringBuilder sb = new(); + StringBuilder sb = new StringBuilder(); sb.Append(Name); diff --git a/ColorzCore/Raws/RawReader.cs b/ColorzCore/Raws/RawReader.cs index ee30b72..f646bd9 100644 --- a/ColorzCore/Raws/RawReader.cs +++ b/ColorzCore/Raws/RawReader.cs @@ -321,7 +321,7 @@ private static Dictionary ParseFlags(string flagStr) if (flag[0] != '-') throw new Exception("Flag does not start with '-'"); - var withoutDash = flag[1..]; + var withoutDash = flag.Substring(1); string name; var value = new Flag(); From 4d820ac2471bdf9d12f55f8901e92b43239649ac Mon Sep 17 00:00:00 2001 From: Nat_776 Date: Wed, 17 Apr 2024 21:43:22 +0200 Subject: [PATCH 12/59] basic symbol assignment --- ColorzCore/Lexer/TokenType.cs | 1 + ColorzCore/Lexer/Tokenizer.cs | 7 +++++ ColorzCore/Parser/EAParser.cs | 55 ++++++++++++++++++++++------------- 3 files changed, 43 insertions(+), 20 deletions(-) diff --git a/ColorzCore/Lexer/TokenType.cs b/ColorzCore/Lexer/TokenType.cs index 62694e8..ecaf7f3 100644 --- a/ColorzCore/Lexer/TokenType.cs +++ b/ColorzCore/Lexer/TokenType.cs @@ -37,6 +37,7 @@ public enum TokenType COMPARE_LE, COMPARE_GT, COMPARE_GE, + ASSIGN, NUMBER, OPEN_BRACKET, CLOSE_BRACKET, diff --git a/ColorzCore/Lexer/Tokenizer.cs b/ColorzCore/Lexer/Tokenizer.cs index 6442587..179bbb9 100644 --- a/ColorzCore/Lexer/Tokenizer.cs +++ b/ColorzCore/Lexer/Tokenizer.cs @@ -99,6 +99,13 @@ public IEnumerable TokenizePhrase(string line, string fileName, int lineN yield return new Token(TokenType.SEMICOLON, fileName, lineNum, curCol + offset); break; case ':': + if (curCol + 1 < endOffs && line[curCol + 1] == '=') // ':=' + { + yield return new Token(TokenType.ASSIGN, fileName, lineNum, curCol + offset); + curCol++; + break; + } + yield return new Token(TokenType.COLON, fileName, lineNum, curCol + offset); break; case '{': diff --git a/ColorzCore/Parser/EAParser.cs b/ColorzCore/Parser/EAParser.cs index 923b719..3103c4f 100644 --- a/ColorzCore/Parser/EAParser.cs +++ b/ColorzCore/Parser/EAParser.cs @@ -864,28 +864,26 @@ private IList ParseList(MergeableGenerator tokens, ImmutableSt else { tokens.MoveNext(); - if (tokens.Current.Type == TokenType.COLON) + switch (tokens.Current.Type) { - tokens.MoveNext(); - if (scopes.Head.HasLocalLabel(head.Content)) - { - Warning("Label already in scope, ignoring: " + head.Content);//replacing: " + head.Content); - } - else if (!IsValidLabelName(head.Content)) - { - Error("Invalid label name " + head.Content + '.'); - } - else - { - scopes.Head.AddLabel(head.Content, CurrentOffset); - } + case TokenType.COLON: + tokens.MoveNext(); + TryDefineSymbol(scopes, head.Content, CurrentOffset); + return null; + case TokenType.ASSIGN: + tokens.MoveNext(); - return null; - } - else - { - tokens.PutBack(head); - return ParseStatement(tokens, scopes); + ParseAtom(tokens, scopes, true).IfJust( + atom => atom.TryEvaluate( + e => Error($"Couldn't define symbol `{head.Content}`: can't resolve value.")).IfJust( + value => TryDefineSymbol(scopes, head.Content, value)), + () => Error($"Couldn't define symbol `{head.Content}`: exprected expression.")); + + return null; + + default: + tokens.PutBack(head); + return ParseStatement(tokens, scopes); } } case TokenType.OPEN_BRACE: @@ -929,6 +927,23 @@ private IList ParseList(MergeableGenerator tokens, ImmutableSt } } + private void TryDefineSymbol(ImmutableStack scopes, string name, int value) + { + if (scopes.Head.HasLocalLabel(name)) + { + Warning($"Symbol already in scope, ignoring: {name}"); + } + else if (!IsValidLabelName(name)) + { + // NOTE: IsValidLabelName returns true always. This is dead code + Error($"Invalid symbol name {name}."); + } + else + { + scopes.Head.AddLabel(name, value); + } + } + private ILineNode? ParsePreprocessor(MergeableGenerator tokens, ImmutableStack scopes) { head = tokens.Current; From 5a732ca5765c313f840a05d007778ad13f9da333 Mon Sep 17 00:00:00 2001 From: Nat_776 Date: Thu, 18 Apr 2024 18:09:34 +0200 Subject: [PATCH 13/59] add binary negation (bit flip) operator --- ColorzCore/Lexer/TokenType.cs | 1 + ColorzCore/Lexer/Tokenizer.cs | 3 +++ ColorzCore/Parser/AST/UnaryOperatorNode.cs | 4 ++++ ColorzCore/Parser/EAParser.cs | 1 + 4 files changed, 9 insertions(+) diff --git a/ColorzCore/Lexer/TokenType.cs b/ColorzCore/Lexer/TokenType.cs index ecaf7f3..cc968e1 100644 --- a/ColorzCore/Lexer/TokenType.cs +++ b/ColorzCore/Lexer/TokenType.cs @@ -25,6 +25,7 @@ public enum TokenType LSHIFT_OP, RSHIFT_OP, SIGNED_RSHIFT_OP, + NOT_OP, AND_OP, XOR_OP, OR_OP, diff --git a/ColorzCore/Lexer/Tokenizer.cs b/ColorzCore/Lexer/Tokenizer.cs index 179bbb9..f3acfc6 100644 --- a/ColorzCore/Lexer/Tokenizer.cs +++ b/ColorzCore/Lexer/Tokenizer.cs @@ -273,6 +273,9 @@ public IEnumerable TokenizePhrase(string line, string fileName, int lineN yield return new Token(TokenType.LOGNOT_OP, fileName, lineNum, curCol + offset); break; } + case '~': + yield return new Token(TokenType.NOT_OP, fileName, lineNum, curCol + offset); + break; case '\n': yield return new Token(TokenType.NEWLINE, fileName, lineNum, curCol + offset); break; diff --git a/ColorzCore/Parser/AST/UnaryOperatorNode.cs b/ColorzCore/Parser/AST/UnaryOperatorNode.cs index ca21cef..31c83ca 100644 --- a/ColorzCore/Parser/AST/UnaryOperatorNode.cs +++ b/ColorzCore/Parser/AST/UnaryOperatorNode.cs @@ -27,6 +27,7 @@ public override string PrettyPrint() string operatorString = myToken.Type switch { TokenType.SUB_OP => "-", + TokenType.NOT_OP => "~", TokenType.LOGNOT_OP => "!", _ => "", }; @@ -47,9 +48,12 @@ public override IEnumerable ToTokens() if (inner != null) { + // int? is magic and all of these operations conveniently propagate nulls + return myToken.Type switch { TokenType.SUB_OP => -inner, + TokenType.NOT_OP => ~inner, TokenType.LOGNOT_OP => inner != 0 ? 0 : 1, _ => null, }; diff --git a/ColorzCore/Parser/EAParser.cs b/ColorzCore/Parser/EAParser.cs index 3103c4f..5a23146 100644 --- a/ColorzCore/Parser/EAParser.cs +++ b/ColorzCore/Parser/EAParser.cs @@ -693,6 +693,7 @@ private IList ParsePreprocParamList(MergeableGenerator tokens } case TokenType.SUB_OP: case TokenType.LOGNOT_OP: + case TokenType.NOT_OP: { //Assume unary negation. tokens.MoveNext(); From d42135702c4a74b09d648b556593791d17cc21af Mon Sep 17 00:00:00 2001 From: Nat_776 Date: Thu, 18 Apr 2024 19:11:44 +0200 Subject: [PATCH 14/59] symbol assignment can be lazy --- ColorzCore/DataTypes/Location.cs | 2 + ColorzCore/EAInterpreter.cs | 15 +++--- ColorzCore/IO/Log.cs | 7 +-- ColorzCore/Parser/AST/BlockNode.cs | 16 +++---- ColorzCore/Parser/AST/DataNode.cs | 9 ++-- ColorzCore/Parser/AST/ILineNode.cs | 7 ++- ColorzCore/Parser/AST/IdentifierNode.cs | 11 +++-- ColorzCore/Parser/AST/StatementNode.cs | 26 +++++------ ColorzCore/Parser/BaseClosure.cs | 4 +- ColorzCore/Parser/Closure.cs | 62 +++++++++++++++++++------ ColorzCore/Parser/EAParser.cs | 25 ++++++++-- 11 files changed, 118 insertions(+), 66 deletions(-) diff --git a/ColorzCore/DataTypes/Location.cs b/ColorzCore/DataTypes/Location.cs index 7699204..563406d 100644 --- a/ColorzCore/DataTypes/Location.cs +++ b/ColorzCore/DataTypes/Location.cs @@ -17,5 +17,7 @@ public Location(string fileName, int lineNum, int colNum) : this() this.lineNum = lineNum; this.colNum = colNum; } + + public override readonly string ToString() => $"{file}:{lineNum}:{colNum}"; } } diff --git a/ColorzCore/EAInterpreter.cs b/ColorzCore/EAInterpreter.cs index 7055327..6e50c4a 100644 --- a/ColorzCore/EAInterpreter.cs +++ b/ColorzCore/EAInterpreter.cs @@ -90,12 +90,12 @@ public bool Interpret() * At parse time, myLabel did not exist for the POIN. * It is at this point we want to make sure all references to identifiers are valid, before assembling. */ - List undefinedIds = new List(); + List<(Location, Exception)> evaluationErrors = new List<(Location, Exception)>(); foreach (ILineNode line in lines) { try { - line.EvaluateExpressions(undefinedIds); + line.EvaluateExpressions(evaluationErrors); } catch (MacroInvocationNode.MacroException e) { @@ -103,15 +103,16 @@ public bool Interpret() } } - foreach (Token errCause in undefinedIds) + foreach ((Location location, Exception e) in evaluationErrors) { - if (errCause.Content.StartsWith(Pool.pooledLabelPrefix, StringComparison.Ordinal)) + if (e is IdentifierNode.UndefinedIdentifierException uie + && uie.CausedError.Content.StartsWith(Pool.pooledLabelPrefix, StringComparison.Ordinal)) { - myParser.Error(errCause.Location, "Unpooled data (forgot #pool?)"); + myParser.Error(location, "Unpooled data (forgot #pool?)"); } else { - myParser.Error(errCause.Location, $"Undefined identifier `{errCause.Content}`"); + myParser.Error(location, e.Message); } } @@ -145,7 +146,7 @@ public bool Interpret() public bool WriteNocashSymbols(TextWriter output) { - foreach (var label in myParser.GlobalScope.Head.LocalLabels()) + foreach (var label in myParser.GlobalScope.Head.LocalSymbols()) { output.WriteLine("{0:X8} {1}", EAParser.ConvertToAddress(label.Value), label.Key); } diff --git a/ColorzCore/IO/Log.cs b/ColorzCore/IO/Log.cs index e1f80ff..950327c 100644 --- a/ColorzCore/IO/Log.cs +++ b/ColorzCore/IO/Log.cs @@ -31,7 +31,8 @@ protected struct LogDisplayConfig public ConsoleColor? tagColor; } - protected static readonly Dictionary KIND_DISPLAY_DICT = new Dictionary { + protected static readonly Dictionary KIND_DISPLAY_DICT = new Dictionary + { { MessageKind.ERROR, new LogDisplayConfig { tag = "error", tagColor = ConsoleColor.Red } }, { MessageKind.WARNING, new LogDisplayConfig { tag = "warning", tagColor = ConsoleColor.Magenta } }, { MessageKind.NOTE, new LogDisplayConfig { tag = "note", tagColor = null } }, @@ -65,14 +66,14 @@ public void Message(MessageKind kind, Location? source, string message) if (!NoColoredTags && config.tagColor.HasValue) Console.ForegroundColor = config.tagColor.Value; - Output.Write("{0}: ", config.tag); + Output.Write($"{config.tag}: "); if (!NoColoredTags) Console.ResetColor(); if (source.HasValue) { - Output.Write("{0}:{1}:{2}: ", source.Value.file, source.Value.lineNum, source.Value.colNum); + Output.Write($"{source.Value}: "); } Output.WriteLine(message); diff --git a/ColorzCore/Parser/AST/BlockNode.cs b/ColorzCore/Parser/AST/BlockNode.cs index f122600..c4db0e3 100644 --- a/ColorzCore/Parser/AST/BlockNode.cs +++ b/ColorzCore/Parser/AST/BlockNode.cs @@ -13,11 +13,7 @@ class BlockNode : ILineNode { public IList Children { get; } - public int Size { - get - { - return Children.Sum((ILineNode n) => n.Size); - } } + public int Size => Children.Sum(n => n.Size); public BlockNode() { @@ -30,7 +26,7 @@ public string PrettyPrint(int indentation) sb.Append(' ', indentation); sb.Append('{'); sb.Append('\n'); - foreach(ILineNode i in Children) + foreach (ILineNode i in Children) { sb.Append(i.PrettyPrint(indentation + 4)); sb.Append('\n'); @@ -42,16 +38,18 @@ public string PrettyPrint(int indentation) public void WriteData(IOutput output) { - foreach(ILineNode child in Children) + foreach (ILineNode child in Children) { child.WriteData(output); } } - public void EvaluateExpressions(ICollection undefinedIdentifiers) + public void EvaluateExpressions(ICollection<(Location, Exception)> evaluationErrors) { foreach (ILineNode line in Children) - line.EvaluateExpressions(undefinedIdentifiers); + { + line.EvaluateExpressions(evaluationErrors); + } } } } diff --git a/ColorzCore/Parser/AST/DataNode.cs b/ColorzCore/Parser/AST/DataNode.cs index ca85cd0..446d56b 100644 --- a/ColorzCore/Parser/AST/DataNode.cs +++ b/ColorzCore/Parser/AST/DataNode.cs @@ -1,4 +1,5 @@ -using ColorzCore.IO; +using ColorzCore.DataTypes; +using ColorzCore.IO; using ColorzCore.Lexer; using System; using System.Collections.Generic; @@ -25,14 +26,14 @@ public string PrettyPrint(int indentation) { return String.Format("Raw Data Block of Length {0}", Size); } - + public void WriteData(IOutput output) { output.WriteTo(offset, data); } - public void EvaluateExpressions(ICollection undefinedIdentifiers) - { + public void EvaluateExpressions(ICollection<(Location, Exception)> evaluationErrors) + { // Nothing to be done because we contain no expressions. } } diff --git a/ColorzCore/Parser/AST/ILineNode.cs b/ColorzCore/Parser/AST/ILineNode.cs index c92f331..3e15cc8 100644 --- a/ColorzCore/Parser/AST/ILineNode.cs +++ b/ColorzCore/Parser/AST/ILineNode.cs @@ -1,10 +1,9 @@ -using ColorzCore.IO; +using ColorzCore.DataTypes; +using ColorzCore.IO; using ColorzCore.Lexer; using System; using System.Collections.Generic; using System.Linq; -using System.Text; -using System.Threading.Tasks; namespace ColorzCore.Parser.AST { @@ -13,6 +12,6 @@ interface ILineNode int Size { get; } string PrettyPrint(int indentation); void WriteData(IOutput output); - void EvaluateExpressions(ICollection undefinedIdentifiers); //Return: undefined identifiers + void EvaluateExpressions(ICollection<(Location, Exception)> evaluationErrors); //Return: undefined identifiers } } diff --git a/ColorzCore/Parser/AST/IdentifierNode.cs b/ColorzCore/Parser/AST/IdentifierNode.cs index d90d264..4640c1c 100644 --- a/ColorzCore/Parser/AST/IdentifierNode.cs +++ b/ColorzCore/Parser/AST/IdentifierNode.cs @@ -25,8 +25,8 @@ private int ToInt() ImmutableStack temp = scope; while (!temp.IsEmpty) { - if (temp.Head.HasLocalLabel(identifier.Content)) - return temp.Head.GetLabel(identifier.Content); + if (temp.Head.HasLocalSymbol(identifier.Content)) + return temp.Head.GetSymbol(identifier.Content); else temp = temp.Tail; } @@ -44,6 +44,11 @@ private int ToInt() handler(e); return null; } + catch (Closure.SymbolComputeException e) + { + handler(e); + return null; + } } public override string? GetIdentifier() @@ -55,7 +60,7 @@ public override string PrettyPrint() { try { - return "0x" + ToInt().ToString("X"); + return $"0x{ToInt():X}"; } catch (UndefinedIdentifierException) { diff --git a/ColorzCore/Parser/AST/StatementNode.cs b/ColorzCore/Parser/AST/StatementNode.cs index abbf331..426431f 100644 --- a/ColorzCore/Parser/AST/StatementNode.cs +++ b/ColorzCore/Parser/AST/StatementNode.cs @@ -1,4 +1,5 @@ -using ColorzCore.IO; +using ColorzCore.DataTypes; +using ColorzCore.IO; using ColorzCore.Lexer; using ColorzCore.Raws; using System; @@ -23,22 +24,17 @@ protected StatementNode(IList parameters) public abstract string PrettyPrint(int indentation); public abstract void WriteData(IOutput output); - public void EvaluateExpressions(ICollection undefinedIdentifiers) + public void EvaluateExpressions(ICollection<(Location, Exception)> evaluationErrors) { for (int i = 0; i < Parameters.Count; i++) - Parameters[i] = Parameters[i].SimplifyExpressions((Exception e) => - { - try - { - throw e; - } - catch (IdentifierNode.UndefinedIdentifierException uie) - { - undefinedIdentifiers.Add(uie.CausedError); - } - catch (Exception) - { } - }); + { + Parameters[i] = Parameters[i].SimplifyExpressions(e => evaluationErrors.Add(e switch + { + IdentifierNode.UndefinedIdentifierException uie => (uie.CausedError.Location, uie), + Closure.SymbolComputeException sce => (sce.Expression.MyLocation, sce), + _ => (Parameters[i].MyLocation, e), + })); + } } } } diff --git a/ColorzCore/Parser/BaseClosure.cs b/ColorzCore/Parser/BaseClosure.cs index d950c8f..b5bc1a6 100644 --- a/ColorzCore/Parser/BaseClosure.cs +++ b/ColorzCore/Parser/BaseClosure.cs @@ -7,9 +7,9 @@ public BaseClosure(EAParser enclosing) { this.enclosing = enclosing; } - public override bool HasLocalLabel(string label) + public override bool HasLocalSymbol(string label) { - return label.ToUpper() == "CURRENTOFFSET" || base.HasLocalLabel(label); + return label.ToUpper() == "CURRENTOFFSET" || base.HasLocalSymbol(label); } } } diff --git a/ColorzCore/Parser/Closure.cs b/ColorzCore/Parser/Closure.cs index 452f8ea..f6afcd6 100644 --- a/ColorzCore/Parser/Closure.cs +++ b/ColorzCore/Parser/Closure.cs @@ -1,34 +1,66 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; +using ColorzCore.Parser.AST; namespace ColorzCore.Parser { public class Closure { - private Dictionary Labels { get; } + private Dictionary Symbols { get; } + private Dictionary NonComputedSymbols { get; } public Closure() { - Labels = new Dictionary(); + Symbols = new Dictionary(); + NonComputedSymbols = new Dictionary(); } - public virtual bool HasLocalLabel(string label) + + public virtual bool HasLocalSymbol(string label) { - return Labels.ContainsKey(label); + return Symbols.ContainsKey(label) || NonComputedSymbols.ContainsKey(label); } - public virtual int GetLabel(string label) + + public virtual int GetSymbol(string label) { - return Labels[label]; + if (Symbols.TryGetValue(label, out int value)) + { + return value; + } + + // Try to evaluate assigned symbol + + IAtomNode node = NonComputedSymbols[label]; + NonComputedSymbols.Remove(label); + + return node.TryEvaluate(e => + { + NonComputedSymbols.Add(label, node); + throw new SymbolComputeException(label, node, e); + })!.Value; } - public void AddLabel(string label, int value) + + public void AddSymbol(string label, int value) => Symbols[label] = value; + public void AddSymbol(string label, IAtomNode node) => NonComputedSymbols[label] = node.Simplify(e => { }); + + public IEnumerable> LocalSymbols() { - Labels[label] = value; + foreach (KeyValuePair label in Symbols) + { + yield return label; + } } - public IEnumerable> LocalLabels() - { - foreach (KeyValuePair label in Labels) - { - yield return label; - } + public class SymbolComputeException : Exception + { + public string SymbolName { get; } + public IAtomNode Expression { get; } + + public SymbolComputeException(string name, IAtomNode node, Exception e) + : base($"Couldn't evaluate value of symbol `{name}` ({e.Message})") + { + SymbolName = name; + Expression = node; + } } } } \ No newline at end of file diff --git a/ColorzCore/Parser/EAParser.cs b/ColorzCore/Parser/EAParser.cs index 5a23146..4380918 100644 --- a/ColorzCore/Parser/EAParser.cs +++ b/ColorzCore/Parser/EAParser.cs @@ -876,7 +876,7 @@ private IList ParseList(MergeableGenerator tokens, ImmutableSt ParseAtom(tokens, scopes, true).IfJust( atom => atom.TryEvaluate( - e => Error($"Couldn't define symbol `{head.Content}`: can't resolve value.")).IfJust( + e => TryDefineSymbol(scopes, head.Content, atom)).IfJust( value => TryDefineSymbol(scopes, head.Content, value)), () => Error($"Couldn't define symbol `{head.Content}`: exprected expression.")); @@ -930,7 +930,7 @@ private IList ParseList(MergeableGenerator tokens, ImmutableSt private void TryDefineSymbol(ImmutableStack scopes, string name, int value) { - if (scopes.Head.HasLocalLabel(name)) + if (scopes.Head.HasLocalSymbol(name)) { Warning($"Symbol already in scope, ignoring: {name}"); } @@ -941,7 +941,24 @@ private void TryDefineSymbol(ImmutableStack scopes, string name, int va } else { - scopes.Head.AddLabel(name, value); + scopes.Head.AddSymbol(name, value); + } + } + + private void TryDefineSymbol(ImmutableStack scopes, string name, IAtomNode expression) + { + if (scopes.Head.HasLocalSymbol(name)) + { + Warning($"Symbol already in scope, ignoring: {name}"); + } + else if (!IsValidLabelName(name)) + { + // NOTE: IsValidLabelName returns true always. This is dead code + Error($"Invalid symbol name {name}."); + } + else + { + scopes.Head.AddSymbol(name, expression); } } @@ -1091,7 +1108,7 @@ private void CheckDataWrite(int length) if (IsProtected(CurrentOffset, length) is Location prot) { - Error($"Trying to write data to area protected in file {Path.GetFileName(prot.file)} at line {prot.lineNum}, column {prot.colNum}."); + Error($"Trying to write data to area protected by {prot}"); } } } From 2c6f67fbb391d147dbb4b15a70c09b1412e0deb4 Mon Sep 17 00:00:00 2001 From: Nat_776 Date: Fri, 19 Apr 2024 17:56:57 +0200 Subject: [PATCH 15/59] '??' operator + ASSERT changes + fixes --- ColorzCore/EAInterpreter.cs | 2 +- ColorzCore/Lexer/TokenType.cs | 1 + ColorzCore/Lexer/Tokenizer.cs | 12 ++ ColorzCore/Parser/AST/AtomNodeKernel.cs | 8 +- ColorzCore/Parser/AST/BlockNode.cs | 4 +- ColorzCore/Parser/AST/DataNode.cs | 2 +- ColorzCore/Parser/AST/IAtomNode.cs | 8 +- ColorzCore/Parser/AST/ILineNode.cs | 2 +- ColorzCore/Parser/AST/IParamNode.cs | 6 +- ColorzCore/Parser/AST/IdentifierNode.cs | 14 +- ColorzCore/Parser/AST/ListNode.cs | 4 +- ColorzCore/Parser/AST/MacroInvocationNode.cs | 2 +- ColorzCore/Parser/AST/NumberNode.cs | 2 +- ColorzCore/Parser/AST/OperatorNode.cs | 132 +++++++++++++----- ColorzCore/Parser/AST/StatementNode.cs | 4 +- ColorzCore/Parser/AST/StringNode.cs | 11 +- ColorzCore/Parser/AST/UnaryOperatorNode.cs | 17 +-- ColorzCore/Parser/Closure.cs | 7 +- ColorzCore/Parser/EAParser.cs | 78 ++++++++--- ColorzCore/Parser/EvaluationPhase.cs | 16 +++ .../Preprocessor/Directives/IfDirective.cs | 2 +- .../Directives/IncludeExternalDirective.cs | 2 +- .../Directives/IncludeToolEventDirective.cs | 2 +- 23 files changed, 235 insertions(+), 103 deletions(-) create mode 100644 ColorzCore/Parser/EvaluationPhase.cs diff --git a/ColorzCore/EAInterpreter.cs b/ColorzCore/EAInterpreter.cs index 6e50c4a..77233c1 100644 --- a/ColorzCore/EAInterpreter.cs +++ b/ColorzCore/EAInterpreter.cs @@ -95,7 +95,7 @@ public bool Interpret() { try { - line.EvaluateExpressions(evaluationErrors); + line.EvaluateExpressions(evaluationErrors, EvaluationPhase.Final); } catch (MacroInvocationNode.MacroException e) { diff --git a/ColorzCore/Lexer/TokenType.cs b/ColorzCore/Lexer/TokenType.cs index cc968e1..8a0911e 100644 --- a/ColorzCore/Lexer/TokenType.cs +++ b/ColorzCore/Lexer/TokenType.cs @@ -25,6 +25,7 @@ public enum TokenType LSHIFT_OP, RSHIFT_OP, SIGNED_RSHIFT_OP, + UNDEFINED_COALESCE_OP, NOT_OP, AND_OP, XOR_OP, diff --git a/ColorzCore/Lexer/Tokenizer.cs b/ColorzCore/Lexer/Tokenizer.cs index f3acfc6..a55b58c 100644 --- a/ColorzCore/Lexer/Tokenizer.cs +++ b/ColorzCore/Lexer/Tokenizer.cs @@ -276,6 +276,18 @@ public IEnumerable TokenizePhrase(string line, string fileName, int lineN case '~': yield return new Token(TokenType.NOT_OP, fileName, lineNum, curCol + offset); break; + case '?': + if (curCol + 1 < endOffs && line[curCol + 1] == '?') + { + yield return new Token(TokenType.UNDEFINED_COALESCE_OP, fileName, lineNum, curCol + offset); + curCol++; + break; + } + else + { + yield return new Token(TokenType.ERROR, fileName, lineNum, curCol, "?"); + break; + } case '\n': yield return new Token(TokenType.NEWLINE, fileName, lineNum, curCol + offset); break; diff --git a/ColorzCore/Parser/AST/AtomNodeKernel.cs b/ColorzCore/Parser/AST/AtomNodeKernel.cs index a8f3c83..e5f7b50 100644 --- a/ColorzCore/Parser/AST/AtomNodeKernel.cs +++ b/ColorzCore/Parser/AST/AtomNodeKernel.cs @@ -12,7 +12,7 @@ public abstract class AtomNodeKernel : IAtomNode { public abstract int Precedence { get; } - public ParamType Type { get { return ParamType.ATOM; } } + public ParamType Type => ParamType.ATOM; public virtual string? GetIdentifier() { @@ -23,11 +23,11 @@ public abstract class AtomNodeKernel : IAtomNode public abstract IEnumerable ToTokens(); public abstract Location MyLocation { get; } - public abstract int? TryEvaluate(TAction handler); + public abstract int? TryEvaluate(TAction handler, EvaluationPhase evaluationPhase); - public IParamNode SimplifyExpressions(TAction handler) + public IParamNode SimplifyExpressions(TAction handler, EvaluationPhase evaluationPhase) { - return this.Simplify(handler); + return this.Simplify(handler, evaluationPhase); } public IAtomNode? AsAtom() diff --git a/ColorzCore/Parser/AST/BlockNode.cs b/ColorzCore/Parser/AST/BlockNode.cs index c4db0e3..d407fde 100644 --- a/ColorzCore/Parser/AST/BlockNode.cs +++ b/ColorzCore/Parser/AST/BlockNode.cs @@ -44,11 +44,11 @@ public void WriteData(IOutput output) } } - public void EvaluateExpressions(ICollection<(Location, Exception)> evaluationErrors) + public void EvaluateExpressions(ICollection<(Location, Exception)> evaluationErrors, EvaluationPhase evaluationPhase) { foreach (ILineNode line in Children) { - line.EvaluateExpressions(evaluationErrors); + line.EvaluateExpressions(evaluationErrors, evaluationPhase); } } } diff --git a/ColorzCore/Parser/AST/DataNode.cs b/ColorzCore/Parser/AST/DataNode.cs index 446d56b..f0dad5c 100644 --- a/ColorzCore/Parser/AST/DataNode.cs +++ b/ColorzCore/Parser/AST/DataNode.cs @@ -32,7 +32,7 @@ public void WriteData(IOutput output) output.WriteTo(offset, data); } - public void EvaluateExpressions(ICollection<(Location, Exception)> evaluationErrors) + public void EvaluateExpressions(ICollection<(Location, Exception)> evaluationErrors, EvaluationPhase evaluationPhase) { // Nothing to be done because we contain no expressions. } diff --git a/ColorzCore/Parser/AST/IAtomNode.cs b/ColorzCore/Parser/AST/IAtomNode.cs index 071126b..39115b2 100644 --- a/ColorzCore/Parser/AST/IAtomNode.cs +++ b/ColorzCore/Parser/AST/IAtomNode.cs @@ -14,19 +14,19 @@ public interface IAtomNode : IParamNode int Precedence { get; } string? GetIdentifier(); IEnumerable ToTokens(); - int? TryEvaluate(TAction handler); //Simplifies the AST as much as possible. + int? TryEvaluate(TAction handler, EvaluationPhase evaluationPhase); //Simplifies the AST as much as possible. } public static class AtomExtensions { public static int CoerceInt(this IAtomNode n) { - return n.TryEvaluate(e => throw e)!.Value; + return n.TryEvaluate(e => throw e, EvaluationPhase.Final)!.Value; } - public static IAtomNode Simplify(this IAtomNode self, TAction handler) + public static IAtomNode Simplify(this IAtomNode self, TAction handler, EvaluationPhase evaluationPhase) { - return self.TryEvaluate(handler).IfJust(intValue => FromInt(self.MyLocation, intValue), () => self); + return self.TryEvaluate(handler, evaluationPhase).IfJust(intValue => FromInt(self.MyLocation, intValue), () => self); } public static IAtomNode FromInt(Location location, int intValue) diff --git a/ColorzCore/Parser/AST/ILineNode.cs b/ColorzCore/Parser/AST/ILineNode.cs index 3e15cc8..3397809 100644 --- a/ColorzCore/Parser/AST/ILineNode.cs +++ b/ColorzCore/Parser/AST/ILineNode.cs @@ -12,6 +12,6 @@ interface ILineNode int Size { get; } string PrettyPrint(int indentation); void WriteData(IOutput output); - void EvaluateExpressions(ICollection<(Location, Exception)> evaluationErrors); //Return: undefined identifiers + void EvaluateExpressions(ICollection<(Location, Exception)> evaluationErrors, EvaluationPhase evaluationPhase); // outputs errors into evaluationErrors } } diff --git a/ColorzCore/Parser/AST/IParamNode.cs b/ColorzCore/Parser/AST/IParamNode.cs index 74c0465..0f06daa 100644 --- a/ColorzCore/Parser/AST/IParamNode.cs +++ b/ColorzCore/Parser/AST/IParamNode.cs @@ -14,15 +14,15 @@ public interface IParamNode ParamType Type { get; } string PrettyPrint(); Location MyLocation { get; } - IParamNode SimplifyExpressions(TAction handler); //TODO: Abstract this into a general traverse method. + IParamNode SimplifyExpressions(TAction handler, EvaluationPhase evaluationPhase); //TODO: Abstract this into a general traverse method. IAtomNode? AsAtom(); } public static class ParamExtensions { - public static IParamNode Simplify(this IParamNode n) + public static IParamNode Simplify(this IParamNode n, EvaluationPhase evaluationPhase) { - return n.SimplifyExpressions((Exception e) => { }); + return n.SimplifyExpressions(e => { }, evaluationPhase); } } } diff --git a/ColorzCore/Parser/AST/IdentifierNode.cs b/ColorzCore/Parser/AST/IdentifierNode.cs index 4640c1c..cb8b48f 100644 --- a/ColorzCore/Parser/AST/IdentifierNode.cs +++ b/ColorzCore/Parser/AST/IdentifierNode.cs @@ -20,24 +20,24 @@ public IdentifierNode(Token id, ImmutableStack scopes) scope = scopes; } - private int ToInt() + private int ToInt(EvaluationPhase evaluationPhase) { ImmutableStack temp = scope; while (!temp.IsEmpty) { if (temp.Head.HasLocalSymbol(identifier.Content)) - return temp.Head.GetSymbol(identifier.Content); + return temp.Head.GetSymbol(identifier.Content, evaluationPhase); else temp = temp.Tail; } throw new UndefinedIdentifierException(identifier); } - public override int? TryEvaluate(TAction handler) + public override int? TryEvaluate(TAction handler, EvaluationPhase evaluationPhase) { try { - return ToInt(); + return ToInt(evaluationPhase); } catch (UndefinedIdentifierException e) { @@ -60,12 +60,16 @@ public override string PrettyPrint() { try { - return $"0x{ToInt():X}"; + return $"0x{ToInt(EvaluationPhase.Immediate):X}"; } catch (UndefinedIdentifierException) { return identifier.Content; } + catch (Closure.SymbolComputeException) + { + return identifier.Content; + } } public override IEnumerable ToTokens() { yield return identifier; } diff --git a/ColorzCore/Parser/AST/ListNode.cs b/ColorzCore/Parser/AST/ListNode.cs index 4255f3c..74b5687 100644 --- a/ColorzCore/Parser/AST/ListNode.cs +++ b/ColorzCore/Parser/AST/ListNode.cs @@ -79,12 +79,12 @@ public Either TryEvaluate() return new Right("Expected atomic parameter."); } - public IParamNode SimplifyExpressions(TAction handler) + public IParamNode SimplifyExpressions(TAction handler, EvaluationPhase evaluationPhase) { IEnumerable acc = new List(); for (int i = 0; i < Interior.Count; i++) { - Interior[i] = Interior[i].Simplify(handler); + Interior[i] = Interior[i].Simplify(handler, evaluationPhase); } return this; } diff --git a/ColorzCore/Parser/AST/MacroInvocationNode.cs b/ColorzCore/Parser/AST/MacroInvocationNode.cs index 43814f7..c57e33e 100644 --- a/ColorzCore/Parser/AST/MacroInvocationNode.cs +++ b/ColorzCore/Parser/AST/MacroInvocationNode.cs @@ -68,7 +68,7 @@ public Either TryEvaluate() public IAtomNode? AsAtom() { return null; } - public IParamNode SimplifyExpressions(TAction handler) + public IParamNode SimplifyExpressions(TAction handler, EvaluationPhase evaluationPhase) { handler(new MacroException(this)); return this; diff --git a/ColorzCore/Parser/AST/NumberNode.cs b/ColorzCore/Parser/AST/NumberNode.cs index f2e02e8..542bfa0 100644 --- a/ColorzCore/Parser/AST/NumberNode.cs +++ b/ColorzCore/Parser/AST/NumberNode.cs @@ -33,7 +33,7 @@ public NumberNode(Location loc, int value) public override IEnumerable ToTokens() { yield return new Token(TokenType.NUMBER, MyLocation, value.ToString()); } - public override int? TryEvaluate(TAction handler) + public override int? TryEvaluate(TAction handler, EvaluationPhase evaluationPhase) { return value; } diff --git a/ColorzCore/Parser/AST/OperatorNode.cs b/ColorzCore/Parser/AST/OperatorNode.cs index 67bae7d..6f855e5 100644 --- a/ColorzCore/Parser/AST/OperatorNode.cs +++ b/ColorzCore/Parser/AST/OperatorNode.cs @@ -9,43 +9,20 @@ namespace ColorzCore.Parser.AST { - delegate int BinaryIntOp(int a, int b); - class OperatorNode : AtomNodeKernel { - public static readonly Dictionary Operators = new Dictionary { - { TokenType.MUL_OP , (lhs, rhs) => lhs * rhs }, - { TokenType.DIV_OP , (lhs, rhs) => lhs / rhs }, - { TokenType.MOD_OP , (lhs, rhs) => lhs % rhs }, - { TokenType.ADD_OP , (lhs, rhs) => lhs + rhs }, - { TokenType.SUB_OP , (lhs, rhs) => lhs - rhs }, - { TokenType.LSHIFT_OP , (lhs, rhs) => lhs << rhs }, - { TokenType.RSHIFT_OP , (lhs, rhs) => (int)(((uint)lhs) >> rhs) }, - { TokenType.SIGNED_RSHIFT_OP , (lhs, rhs) => lhs >> rhs }, - { TokenType.AND_OP , (lhs, rhs) => lhs & rhs }, - { TokenType.XOR_OP , (lhs, rhs) => lhs ^ rhs }, - { TokenType.OR_OP , (lhs, rhs) => lhs | rhs }, - { TokenType.LOGAND_OP, (lhs, rhs) => lhs != 0 ? rhs : 0 }, - { TokenType.LOGOR_OP, (lhs, rhs) => lhs != 0 ? lhs : rhs }, - { TokenType.COMPARE_EQ, (lhs, rhs) => lhs == rhs ? 1 : 0 }, - { TokenType.COMPARE_NE, (lhs, rhs) => lhs != rhs ? 1 : 0 }, - { TokenType.COMPARE_LT, (lhs, rhs) => lhs < rhs ? 1 : 0 }, - { TokenType.COMPARE_LE, (lhs, rhs) => lhs <= rhs ? 1 : 0 }, - { TokenType.COMPARE_GE, (lhs, rhs) => lhs >= rhs ? 1 : 0 }, - { TokenType.COMPARE_GT, (lhs, rhs) => lhs > rhs ? 1 : 0 }, - }; - private IAtomNode left, right; - private Token op; - public override int Precedence { get; } - public override Location MyLocation { get { return op.Location; } } + public Token OperatorToken { get; } + + public override int Precedence { get; } + public override Location MyLocation => OperatorToken.Location; public OperatorNode(IAtomNode l, Token op, IAtomNode r, int prec) { left = l; right = r; - this.op = op; + OperatorToken = op; Precedence = prec; } @@ -63,6 +40,7 @@ static string GetOperatorString(TokenType tokenType) TokenType.LSHIFT_OP => "<<", TokenType.RSHIFT_OP => ">>", TokenType.SIGNED_RSHIFT_OP => ">>>", + TokenType.UNDEFINED_COALESCE_OP => "??", TokenType.AND_OP => "&", TokenType.XOR_OP => "^", TokenType.OR_OP => "|", @@ -78,7 +56,7 @@ static string GetOperatorString(TokenType tokenType) }; } - return $"({left.PrettyPrint()} {GetOperatorString(op.Type)} {right.PrettyPrint()})"; + return $"({left.PrettyPrint()} {GetOperatorString(OperatorToken.Type)} {right.PrettyPrint()})"; } public override IEnumerable ToTokens() @@ -87,22 +65,102 @@ public override IEnumerable ToTokens() { yield return t; } - yield return op; + yield return OperatorToken; foreach (Token t in right.ToTokens()) { yield return t; } } - public override int? TryEvaluate(TAction handler) + private int? TryCoalesceUndefined(TAction handler) { - int? l = left.TryEvaluate(handler); - l.IfJust(i => left = new NumberNode(left.MyLocation, i)); - int? r = right.TryEvaluate(handler); - r.IfJust(i => right = new NumberNode(right.MyLocation, i)); + List? leftExceptions = null; + + // the left side of an undefined coalescing operation is allowed to raise exactly UndefinedIdentifierException + // we need to catch that, so don't forward all exceptions raised by left just yet - if (l is int li && r is int ri) - return Operators[op.Type](li, ri); + int? leftValue = left.TryEvaluate(e => (leftExceptions ??= new List()).Add(e), EvaluationPhase.Final); + + if (leftExceptions == null) + { + // left evaluated properly => result is left + return leftValue; + } + else if (leftExceptions.All(e => e is IdentifierNode.UndefinedIdentifierException)) + { + // left did not evalute due to undefined identifier => result is right + return right.TryEvaluate(handler, EvaluationPhase.Final); + } + else + { + // left did evaluate + foreach (Exception e in leftExceptions.Where(e => e is not IdentifierNode.UndefinedIdentifierException)) + { + handler(e); + } + + return null; + } + } + + public override int? TryEvaluate(TAction handler, EvaluationPhase evaluationPhase) + { + /* undefined-coalescing operator is special because + * 1. it should only be evaluated at final evaluation. + * 2. it is legal for its left operand to fail evaluation. */ + if (OperatorToken.Type == TokenType.UNDEFINED_COALESCE_OP) + { + // TODO: better exception types here? + + switch (evaluationPhase) + { + case EvaluationPhase.Immediate: + handler(new Exception("Invalid use of '??'.")); + return null; + case EvaluationPhase.Early: + /* NOTE: you'd think one could optimize this by reducing this if left can be evaluated early + * but that would allow simplifying expressions even in contexts where '??' makes no sense + * (for example: 'ORG SomePossiblyUndefinedLabel ?? SomeOtherLabel') + * I don't think that's desirable */ + handler(new Exception("The value of a '??' expression cannot be resolved early.")); + return null; + case EvaluationPhase.Final: + return TryCoalesceUndefined(handler); + } + } + + int? leftValue = left.TryEvaluate(handler, evaluationPhase); + leftValue.IfJust(i => left = new NumberNode(left.MyLocation, i)); + + int? rightValue = right.TryEvaluate(handler, evaluationPhase); + rightValue.IfJust(i => right = new NumberNode(right.MyLocation, i)); + + if (leftValue is int lhs && rightValue is int rhs) + { + return OperatorToken.Type switch + { + TokenType.MUL_OP => lhs * rhs, + TokenType.DIV_OP => lhs / rhs, + TokenType.MOD_OP => lhs % rhs, + TokenType.ADD_OP => lhs + rhs, + TokenType.SUB_OP => lhs - rhs, + TokenType.LSHIFT_OP => lhs << rhs, + TokenType.RSHIFT_OP => (int)(((uint)lhs) >> rhs), + TokenType.SIGNED_RSHIFT_OP => lhs >> rhs, + TokenType.AND_OP => lhs & rhs, + TokenType.XOR_OP => lhs ^ rhs, + TokenType.OR_OP => lhs | rhs, + TokenType.LOGAND_OP => lhs != 0 ? rhs : 0, + TokenType.LOGOR_OP => lhs != 0 ? lhs : rhs, + TokenType.COMPARE_EQ => lhs == rhs ? 1 : 0, + TokenType.COMPARE_NE => lhs != rhs ? 1 : 0, + TokenType.COMPARE_LT => lhs < rhs ? 1 : 0, + TokenType.COMPARE_LE => lhs <= rhs ? 1 : 0, + TokenType.COMPARE_GE => lhs >= rhs ? 1 : 0, + TokenType.COMPARE_GT => lhs > rhs ? 1 : 0, + _ => null, + }; + } return null; } diff --git a/ColorzCore/Parser/AST/StatementNode.cs b/ColorzCore/Parser/AST/StatementNode.cs index 426431f..ee7af84 100644 --- a/ColorzCore/Parser/AST/StatementNode.cs +++ b/ColorzCore/Parser/AST/StatementNode.cs @@ -24,7 +24,7 @@ protected StatementNode(IList parameters) public abstract string PrettyPrint(int indentation); public abstract void WriteData(IOutput output); - public void EvaluateExpressions(ICollection<(Location, Exception)> evaluationErrors) + public void EvaluateExpressions(ICollection<(Location, Exception)> evaluationErrors, EvaluationPhase evaluationPhase) { for (int i = 0; i < Parameters.Count; i++) { @@ -33,7 +33,7 @@ public void EvaluateExpressions(ICollection<(Location, Exception)> evaluationErr IdentifierNode.UndefinedIdentifierException uie => (uie.CausedError.Location, uie), Closure.SymbolComputeException sce => (sce.Expression.MyLocation, sce), _ => (Parameters[i].MyLocation, e), - })); + }), evaluationPhase); } } } diff --git a/ColorzCore/Parser/AST/StringNode.cs b/ColorzCore/Parser/AST/StringNode.cs index 9664025..530697b 100644 --- a/ColorzCore/Parser/AST/StringNode.cs +++ b/ColorzCore/Parser/AST/StringNode.cs @@ -37,22 +37,17 @@ public string PrettyPrint() } public IEnumerable ToTokens() { yield return MyToken; } - public Either TryEvaluate() - { - return new Right("Expected atomic parameter."); - } - public bool IsValidIdentifier() { return idRegex.IsMatch(MyToken.Content); } + public IdentifierNode ToIdentifier(ImmutableStack scope) { return new IdentifierNode(MyToken, scope); } - public IAtomNode? AsAtom() { return null; } - - public IParamNode SimplifyExpressions(TAction handler) { return this; } + public IAtomNode? AsAtom() => null; + public IParamNode SimplifyExpressions(TAction handler, EvaluationPhase evaluationPhase) => this; } } diff --git a/ColorzCore/Parser/AST/UnaryOperatorNode.cs b/ColorzCore/Parser/AST/UnaryOperatorNode.cs index 31c83ca..e66086b 100644 --- a/ColorzCore/Parser/AST/UnaryOperatorNode.cs +++ b/ColorzCore/Parser/AST/UnaryOperatorNode.cs @@ -10,21 +10,22 @@ namespace ColorzCore.Parser.AST { class UnaryOperatorNode : AtomNodeKernel { - private readonly Token myToken; private readonly IAtomNode interior; + public Token OperatorToken { get; } + public UnaryOperatorNode(Token token, IAtomNode inside) { - myToken = token; + OperatorToken = token; interior = inside; } public override int Precedence => 11; - public override Location MyLocation => myToken.Location; + public override Location MyLocation => OperatorToken.Location; public override string PrettyPrint() { - string operatorString = myToken.Type switch + string operatorString = OperatorToken.Type switch { TokenType.SUB_OP => "-", TokenType.NOT_OP => "~", @@ -37,20 +38,20 @@ public override string PrettyPrint() public override IEnumerable ToTokens() { - yield return myToken; + yield return OperatorToken; foreach (Token t in interior.ToTokens()) yield return t; } - public override int? TryEvaluate(TAction handler) + public override int? TryEvaluate(TAction handler, EvaluationPhase evaluationPhase) { - int? inner = interior.TryEvaluate(handler); + int? inner = interior.TryEvaluate(handler, evaluationPhase); if (inner != null) { // int? is magic and all of these operations conveniently propagate nulls - return myToken.Type switch + return OperatorToken.Type switch { TokenType.SUB_OP => -inner, TokenType.NOT_OP => ~inner, diff --git a/ColorzCore/Parser/Closure.cs b/ColorzCore/Parser/Closure.cs index f6afcd6..268a12c 100644 --- a/ColorzCore/Parser/Closure.cs +++ b/ColorzCore/Parser/Closure.cs @@ -20,7 +20,8 @@ public virtual bool HasLocalSymbol(string label) return Symbols.ContainsKey(label) || NonComputedSymbols.ContainsKey(label); } - public virtual int GetSymbol(string label) + // HACK: having evaluationPhase being passed here is a bit suspect. + public virtual int GetSymbol(string label, EvaluationPhase evaluationPhase) { if (Symbols.TryGetValue(label, out int value)) { @@ -36,11 +37,11 @@ public virtual int GetSymbol(string label) { NonComputedSymbols.Add(label, node); throw new SymbolComputeException(label, node, e); - })!.Value; + }, evaluationPhase)!.Value; } public void AddSymbol(string label, int value) => Symbols[label] = value; - public void AddSymbol(string label, IAtomNode node) => NonComputedSymbols[label] = node.Simplify(e => { }); + public void AddSymbol(string label, IAtomNode node) => NonComputedSymbols[label] = node.Simplify(e => { }, EvaluationPhase.Early); public IEnumerable> LocalSymbols() { diff --git a/ColorzCore/Parser/EAParser.cs b/ColorzCore/Parser/EAParser.cs index 4380918..cb956f8 100644 --- a/ColorzCore/Parser/EAParser.cs +++ b/ColorzCore/Parser/EAParser.cs @@ -282,7 +282,7 @@ public static int ConvertToOffset(int value) } parameters[0].AsAtom().IfJust( - atom => atom.TryEvaluate(e => Error(parameters[0].MyLocation, e.Message)).IfJust( + atom => atom.TryEvaluate(e => Error(parameters[0].MyLocation, e.Message), EvaluationPhase.Immediate).IfJust( offsetValue => { CurrentOffset = ConvertToOffset(offsetValue); }, () => Error(parameters[0].MyLocation, "Expected atomic param to ORG."))); @@ -332,16 +332,54 @@ public static int ConvertToOffset(int value) return null; } - parameters[0].AsAtom().IfJust( - atom => atom.TryEvaluate(e => Error(parameters[0].MyLocation, e.Message)).IfJust( + // helper for distinguishing boolean expressions and other expressions + static bool IsConditionalOperatorHelper(IAtomNode node) + { + return node switch + { + UnaryOperatorNode uon => uon.OperatorToken.Type switch + { + TokenType.LOGNOT_OP => true, + _ => false, + }, + + OperatorNode on => on.OperatorToken.Type switch + { + TokenType.LOGAND_OP => true, + TokenType.LOGOR_OP => true, + TokenType.COMPARE_EQ => true, + TokenType.COMPARE_NE => true, + TokenType.COMPARE_GT => true, + TokenType.COMPARE_GE => true, + TokenType.COMPARE_LE => true, + TokenType.COMPARE_LT => true, + _ => false, + }, + + _ => false, + }; + } + + IAtomNode? atom = parameters[0].AsAtom(); + + if (atom != null) + { + bool isBoolean = IsConditionalOperatorHelper(atom); + + atom.TryEvaluate(e => Error(parameters[0].MyLocation, e.Message), EvaluationPhase.Immediate).IfJust( temp => { - if (temp < 0) + // if boolean expession => fail if 0, else (legacy behavoir) fail if negative + if (isBoolean && temp == 0 || !isBoolean && temp < 0) { Error(parameters[0].MyLocation, "Assertion error: " + temp); } - }), - () => Error(parameters[0].MyLocation, "Expected atomic param to ASSERT.")); + }); + } + else + { + Error(parameters[0].MyLocation, "Expected atomic param to ASSERT."); + } return null; } @@ -351,7 +389,7 @@ public static int ConvertToOffset(int value) if (parameters.Count == 1) { parameters[0].AsAtom().IfJust( - atom => atom.TryEvaluate(e => { Error(parameters[0].MyLocation, e.Message); }).IfJust( + atom => atom.TryEvaluate(e => { Error(parameters[0].MyLocation, e.Message); }, EvaluationPhase.Immediate).IfJust( temp => { protectedRegions.Add(new Tuple(temp, 4, head!.Location)); @@ -363,14 +401,14 @@ public static int ConvertToOffset(int value) int start = 0, end = 0; bool errorOccurred = false; parameters[0].AsAtom().IfJust( - atom => atom.TryEvaluate(e => { Error(parameters[0].MyLocation, e.Message); errorOccurred = true; }).IfJust( + atom => atom.TryEvaluate(e => { Error(parameters[0].MyLocation, e.Message); errorOccurred = true; }, EvaluationPhase.Immediate).IfJust( temp => { start = temp; }), () => { Error(parameters[0].MyLocation, "Expected atomic param to PROTECT"); errorOccurred = true; }); parameters[1].AsAtom().IfJust( - atom => atom.TryEvaluate(e => { Error(parameters[0].MyLocation, e.Message); errorOccurred = true; }).IfJust( + atom => atom.TryEvaluate(e => { Error(parameters[0].MyLocation, e.Message); errorOccurred = true; }, EvaluationPhase.Immediate).IfJust( temp => { end = temp; @@ -406,7 +444,7 @@ public static int ConvertToOffset(int value) } parameters[0].AsAtom().IfJust( - atom => atom.TryEvaluate(e => Error(parameters[0].MyLocation, e.Message)).IfJust( + atom => atom.TryEvaluate(e => Error(parameters[0].MyLocation, e.Message), EvaluationPhase.Immediate).IfJust( temp => CurrentOffset = CurrentOffset % temp != 0 ? CurrentOffset + temp - CurrentOffset % temp : CurrentOffset), () => Error(parameters[0].MyLocation, "Expected atomic param to ALIGN")); @@ -431,14 +469,14 @@ public static int ConvertToOffset(int value) // param 2 (if given) is fill value parameters[1].AsAtom().IfJust( - atom => atom.TryEvaluate(e => Error(parameters[0].MyLocation, e.Message)).IfJust( + atom => atom.TryEvaluate(e => Error(parameters[0].MyLocation, e.Message), EvaluationPhase.Immediate).IfJust( val => { value = val; }), () => Error(parameters[0].MyLocation, "Expected atomic param to FILL")); } // param 1 is amount of bytes to fill parameters[0].AsAtom().IfJust( - atom => atom.TryEvaluate(e => Error(parameters[0].MyLocation, e.Message)).IfJust( + atom => atom.TryEvaluate(e => Error(parameters[0].MyLocation, e.Message), EvaluationPhase.Immediate).IfJust( val => { amount = val; }), () => Error(parameters[0].MyLocation, "Expected atomic param to FILL")); @@ -560,7 +598,7 @@ private IList ParsePreprocParamList(MergeableGenerator tokens switch (tokens.Current.Type) { case TokenType.OPEN_BRACKET: - return new ListNode(localHead.Location, ParseList(tokens, scopes)).Simplify(); + return new ListNode(localHead.Location, ParseList(tokens, scopes)).Simplify(EvaluationPhase.Early); case TokenType.STRING: tokens.MoveNext(); return new StringNode(localHead); @@ -584,11 +622,11 @@ private IList ParsePreprocParamList(MergeableGenerator tokens } else { - return ParseAtom(tokens, scopes, expandDefs)?.Simplify(); + return ParseAtom(tokens, scopes, expandDefs)?.Simplify(EvaluationPhase.Early); } default: - return ParseAtom(tokens, scopes, expandDefs)?.Simplify(); + return ParseAtom(tokens, scopes, expandDefs)?.Simplify(EvaluationPhase.Early); } } @@ -612,6 +650,7 @@ private IList ParsePreprocParamList(MergeableGenerator tokens { TokenType.OR_OP, 10 }, { TokenType.LOGAND_OP, 11 }, { TokenType.LOGOR_OP, 12 }, + { TokenType.UNDEFINED_COALESCE_OP, 13 }, }; private IAtomNode? ParseAtom(MergeableGenerator tokens, ImmutableStack scopes, bool expandDefs = true) @@ -655,6 +694,10 @@ private IList ParsePreprocParamList(MergeableGenerator tokens } shift = true; break; + case TokenType.UNDEFINED_COALESCE_OP: + // '??' is right-associative, so don't reduce here + shift = true; + break; default: ended = true; break; @@ -728,6 +771,7 @@ private IList ParsePreprocParamList(MergeableGenerator tokens case TokenType.COMPARE_NE: case TokenType.COMPARE_GE: case TokenType.COMPARE_GT: + case TokenType.UNDEFINED_COALESCE_OP: default: Error(lookAhead.Location, "Expected identifier or literal, got " + lookAhead.Type + ": " + lookAhead.Content + '.'); IgnoreRestOfStatement(tokens); @@ -778,7 +822,7 @@ private IList ParsePreprocParamList(MergeableGenerator tokens } while (grammarSymbols.Count > 1) { - Reduce(grammarSymbols, 11); + Reduce(grammarSymbols, int.MaxValue); } if (grammarSymbols.Peek().IsRight) { @@ -876,7 +920,7 @@ private IList ParseList(MergeableGenerator tokens, ImmutableSt ParseAtom(tokens, scopes, true).IfJust( atom => atom.TryEvaluate( - e => TryDefineSymbol(scopes, head.Content, atom)).IfJust( + e => TryDefineSymbol(scopes, head.Content, atom), EvaluationPhase.Early).IfJust( value => TryDefineSymbol(scopes, head.Content, value)), () => Error($"Couldn't define symbol `{head.Content}`: exprected expression.")); diff --git a/ColorzCore/Parser/EvaluationPhase.cs b/ColorzCore/Parser/EvaluationPhase.cs new file mode 100644 index 0000000..e96fb8e --- /dev/null +++ b/ColorzCore/Parser/EvaluationPhase.cs @@ -0,0 +1,16 @@ +using System; + +namespace ColorzCore.Parser +{ + public enum EvaluationPhase + { + // Early-evaluation: we are not done with parsing and obtaining a value now would just be an optimization + Early, + + // Final-evaluation: we are done with parsing and need to freeze this value now + Final, + + // Immediate-evaluation: we are not done with parsing but need this value now + Immediate, + } +} diff --git a/ColorzCore/Preprocessor/Directives/IfDirective.cs b/ColorzCore/Preprocessor/Directives/IfDirective.cs index 90b2756..2700b5f 100644 --- a/ColorzCore/Preprocessor/Directives/IfDirective.cs +++ b/ColorzCore/Preprocessor/Directives/IfDirective.cs @@ -26,7 +26,7 @@ class IfDirective : IDirective { if (parameter is IAtomNode atomNode) { - if (atomNode.TryEvaluate(e => p.Error(self.Location, $"Error while evaluating expression: {e.Message}")) is int value) + if (atomNode.TryEvaluate(e => p.Error(self.Location, $"Error while evaluating expression: {e.Message}"), EvaluationPhase.Immediate) is int value) { flag = value != 0; } diff --git a/ColorzCore/Preprocessor/Directives/IncludeExternalDirective.cs b/ColorzCore/Preprocessor/Directives/IncludeExternalDirective.cs index 94656d5..e8249bc 100644 --- a/ColorzCore/Preprocessor/Directives/IncludeExternalDirective.cs +++ b/ColorzCore/Preprocessor/Directives/IncludeExternalDirective.cs @@ -48,7 +48,7 @@ class IncludeExternalDirective : IDirective { if (parameters[i].Type == ParamType.ATOM) { - parameters[i] = ((IAtomNode)parameters[i]).Simplify(); + parameters[i] = ((IAtomNode)parameters[i]).Simplify(EvaluationPhase.Early); } argumentBuilder.Append(parameters[i].PrettyPrint()); argumentBuilder.Append(' '); diff --git a/ColorzCore/Preprocessor/Directives/IncludeToolEventDirective.cs b/ColorzCore/Preprocessor/Directives/IncludeToolEventDirective.cs index 5b4120f..df5e05f 100644 --- a/ColorzCore/Preprocessor/Directives/IncludeToolEventDirective.cs +++ b/ColorzCore/Preprocessor/Directives/IncludeToolEventDirective.cs @@ -45,7 +45,7 @@ class IncludeToolEventDirective : IDirective StringBuilder argumentBuilder = new StringBuilder(); for (int i = 1; i < parameters.Count; i++) { - parameters[i].AsAtom().IfJust((IAtomNode n) => { parameters[i] = n.Simplify(); }); + parameters[i].AsAtom().IfJust((IAtomNode n) => { parameters[i] = n.Simplify(EvaluationPhase.Early); }); argumentBuilder.Append(parameters[i].PrettyPrint()); argumentBuilder.Append(' '); } From 4187197399d29b8ec466c41d1178da8b166c4552 Mon Sep 17 00:00:00 2001 From: Nat_776 Date: Fri, 19 Apr 2024 18:27:42 +0200 Subject: [PATCH 16/59] purge remains of Maybe --- ColorzCore/DataTypes/Either.cs | 8 ++++---- .../{Maybe.cs => NullableExtensions.cs} | 18 ++++++------------ ColorzCore/Parser/AST/AtomNodeKernel.cs | 4 ++-- ColorzCore/Parser/AST/IAtomNode.cs | 4 ++-- ColorzCore/Parser/AST/IParamNode.cs | 2 +- ColorzCore/Parser/AST/IdentifierNode.cs | 2 +- ColorzCore/Parser/AST/ListNode.cs | 2 +- ColorzCore/Parser/AST/MacroInvocationNode.cs | 2 +- ColorzCore/Parser/AST/NumberNode.cs | 2 +- ColorzCore/Parser/AST/OperatorNode.cs | 6 +++--- ColorzCore/Parser/AST/StringNode.cs | 2 +- ColorzCore/Parser/AST/UnaryOperatorNode.cs | 2 +- 12 files changed, 24 insertions(+), 30 deletions(-) rename ColorzCore/DataTypes/{Maybe.cs => NullableExtensions.cs} (59%) diff --git a/ColorzCore/DataTypes/Either.cs b/ColorzCore/DataTypes/Either.cs index 5d4896f..77c2476 100644 --- a/ColorzCore/DataTypes/Either.cs +++ b/ColorzCore/DataTypes/Either.cs @@ -14,9 +14,9 @@ public interface Either bool IsRight { get; } Left GetLeft { get; } Right GetRight { get; } - void Case(TAction LAction, TAction RAction); + void Case(Action LAction, Action RAction); } - public class Left : Either + public class Left : Either { public Left(L val) { @@ -27,7 +27,7 @@ public Left(L val) public bool IsRight { get { return false; } } public L GetLeft { get; } public R GetRight { get { throw new WrongEitherException(); } } - public void Case(TAction LAction, TAction RAction) { LAction(GetLeft); } + public void Case(Action LAction, Action RAction) { LAction(GetLeft); } } public class Right : Either { @@ -40,7 +40,7 @@ public Right(R val) public bool IsRight { get { return true; } } public L GetLeft { get { throw new WrongEitherException(); } } public R GetRight { get; } - public void Case(TAction LAction, TAction RAction) { RAction(GetRight); } + public void Case(Action LAction, Action RAction) { RAction(GetRight); } } class WrongEitherException : Exception { } } diff --git a/ColorzCore/DataTypes/Maybe.cs b/ColorzCore/DataTypes/NullableExtensions.cs similarity index 59% rename from ColorzCore/DataTypes/Maybe.cs rename to ColorzCore/DataTypes/NullableExtensions.cs index 9a86c1f..1a0fef4 100644 --- a/ColorzCore/DataTypes/Maybe.cs +++ b/ColorzCore/DataTypes/NullableExtensions.cs @@ -6,21 +6,15 @@ namespace ColorzCore.DataTypes { - public delegate R UnaryFunction(T val); - public delegate R RConst(); - public delegate void TAction(T val); - public delegate void NullaryAction(); - public delegate R? MaybeAction(T val); - - public static class MaybeExtensions + public static class NullableExtensions { - public static R? Fmap(this T? self, UnaryFunction f) + public static R? Fmap(this T? self, Func f) where T : class { return self != null ? f(self) : default; } - public static R IfJust(this T? self, UnaryFunction just, RConst nothing) + public static R IfJust(this T? self, Func just, Func nothing) where T : class { if (self != null) @@ -33,7 +27,7 @@ public static R IfJust(this T? self, UnaryFunction just, RConst n } } - public static R IfJust(this T? self, UnaryFunction just, RConst nothing) + public static R IfJust(this T? self, Func just, Func nothing) where T : struct { if (self.HasValue) @@ -46,7 +40,7 @@ public static R IfJust(this T? self, UnaryFunction just, RConst n } } - public static void IfJust(this T? self, TAction just, NullaryAction? nothing = null) + public static void IfJust(this T? self, Action just, Action? nothing = null) where T : class { if (self != null) @@ -59,7 +53,7 @@ public static void IfJust(this T? self, TAction just, NullaryAction? nothi } } - public static void IfJust(this T? self, TAction just, NullaryAction? nothing = null) + public static void IfJust(this T? self, Action just, Action? nothing = null) where T : struct { if (self.HasValue) diff --git a/ColorzCore/Parser/AST/AtomNodeKernel.cs b/ColorzCore/Parser/AST/AtomNodeKernel.cs index e5f7b50..fb5ff9e 100644 --- a/ColorzCore/Parser/AST/AtomNodeKernel.cs +++ b/ColorzCore/Parser/AST/AtomNodeKernel.cs @@ -23,9 +23,9 @@ public abstract class AtomNodeKernel : IAtomNode public abstract IEnumerable ToTokens(); public abstract Location MyLocation { get; } - public abstract int? TryEvaluate(TAction handler, EvaluationPhase evaluationPhase); + public abstract int? TryEvaluate(Action handler, EvaluationPhase evaluationPhase); - public IParamNode SimplifyExpressions(TAction handler, EvaluationPhase evaluationPhase) + public IParamNode SimplifyExpressions(Action handler, EvaluationPhase evaluationPhase) { return this.Simplify(handler, evaluationPhase); } diff --git a/ColorzCore/Parser/AST/IAtomNode.cs b/ColorzCore/Parser/AST/IAtomNode.cs index 39115b2..31a6294 100644 --- a/ColorzCore/Parser/AST/IAtomNode.cs +++ b/ColorzCore/Parser/AST/IAtomNode.cs @@ -14,7 +14,7 @@ public interface IAtomNode : IParamNode int Precedence { get; } string? GetIdentifier(); IEnumerable ToTokens(); - int? TryEvaluate(TAction handler, EvaluationPhase evaluationPhase); //Simplifies the AST as much as possible. + int? TryEvaluate(Action handler, EvaluationPhase evaluationPhase); //Simplifies the AST as much as possible. } public static class AtomExtensions @@ -24,7 +24,7 @@ public static int CoerceInt(this IAtomNode n) return n.TryEvaluate(e => throw e, EvaluationPhase.Final)!.Value; } - public static IAtomNode Simplify(this IAtomNode self, TAction handler, EvaluationPhase evaluationPhase) + public static IAtomNode Simplify(this IAtomNode self, Action handler, EvaluationPhase evaluationPhase) { return self.TryEvaluate(handler, evaluationPhase).IfJust(intValue => FromInt(self.MyLocation, intValue), () => self); } diff --git a/ColorzCore/Parser/AST/IParamNode.cs b/ColorzCore/Parser/AST/IParamNode.cs index 0f06daa..9f0e7aa 100644 --- a/ColorzCore/Parser/AST/IParamNode.cs +++ b/ColorzCore/Parser/AST/IParamNode.cs @@ -14,7 +14,7 @@ public interface IParamNode ParamType Type { get; } string PrettyPrint(); Location MyLocation { get; } - IParamNode SimplifyExpressions(TAction handler, EvaluationPhase evaluationPhase); //TODO: Abstract this into a general traverse method. + IParamNode SimplifyExpressions(Action handler, EvaluationPhase evaluationPhase); //TODO: Abstract this into a general traverse method. IAtomNode? AsAtom(); } diff --git a/ColorzCore/Parser/AST/IdentifierNode.cs b/ColorzCore/Parser/AST/IdentifierNode.cs index cb8b48f..bd571f9 100644 --- a/ColorzCore/Parser/AST/IdentifierNode.cs +++ b/ColorzCore/Parser/AST/IdentifierNode.cs @@ -33,7 +33,7 @@ private int ToInt(EvaluationPhase evaluationPhase) throw new UndefinedIdentifierException(identifier); } - public override int? TryEvaluate(TAction handler, EvaluationPhase evaluationPhase) + public override int? TryEvaluate(Action handler, EvaluationPhase evaluationPhase) { try { diff --git a/ColorzCore/Parser/AST/ListNode.cs b/ColorzCore/Parser/AST/ListNode.cs index 74b5687..4ce011f 100644 --- a/ColorzCore/Parser/AST/ListNode.cs +++ b/ColorzCore/Parser/AST/ListNode.cs @@ -79,7 +79,7 @@ public Either TryEvaluate() return new Right("Expected atomic parameter."); } - public IParamNode SimplifyExpressions(TAction handler, EvaluationPhase evaluationPhase) + public IParamNode SimplifyExpressions(Action handler, EvaluationPhase evaluationPhase) { IEnumerable acc = new List(); for (int i = 0; i < Interior.Count; i++) diff --git a/ColorzCore/Parser/AST/MacroInvocationNode.cs b/ColorzCore/Parser/AST/MacroInvocationNode.cs index c57e33e..cec08a4 100644 --- a/ColorzCore/Parser/AST/MacroInvocationNode.cs +++ b/ColorzCore/Parser/AST/MacroInvocationNode.cs @@ -68,7 +68,7 @@ public Either TryEvaluate() public IAtomNode? AsAtom() { return null; } - public IParamNode SimplifyExpressions(TAction handler, EvaluationPhase evaluationPhase) + public IParamNode SimplifyExpressions(Action handler, EvaluationPhase evaluationPhase) { handler(new MacroException(this)); return this; diff --git a/ColorzCore/Parser/AST/NumberNode.cs b/ColorzCore/Parser/AST/NumberNode.cs index 542bfa0..f03b8e0 100644 --- a/ColorzCore/Parser/AST/NumberNode.cs +++ b/ColorzCore/Parser/AST/NumberNode.cs @@ -33,7 +33,7 @@ public NumberNode(Location loc, int value) public override IEnumerable ToTokens() { yield return new Token(TokenType.NUMBER, MyLocation, value.ToString()); } - public override int? TryEvaluate(TAction handler, EvaluationPhase evaluationPhase) + public override int? TryEvaluate(Action handler, EvaluationPhase evaluationPhase) { return value; } diff --git a/ColorzCore/Parser/AST/OperatorNode.cs b/ColorzCore/Parser/AST/OperatorNode.cs index 6f855e5..80ab03c 100644 --- a/ColorzCore/Parser/AST/OperatorNode.cs +++ b/ColorzCore/Parser/AST/OperatorNode.cs @@ -72,7 +72,7 @@ public override IEnumerable ToTokens() } } - private int? TryCoalesceUndefined(TAction handler) + private int? TryCoalesceUndefined(Action handler) { List? leftExceptions = null; @@ -93,7 +93,7 @@ public override IEnumerable ToTokens() } else { - // left did evaluate + // left failed to evaluate for some other reason foreach (Exception e in leftExceptions.Where(e => e is not IdentifierNode.UndefinedIdentifierException)) { handler(e); @@ -103,7 +103,7 @@ public override IEnumerable ToTokens() } } - public override int? TryEvaluate(TAction handler, EvaluationPhase evaluationPhase) + public override int? TryEvaluate(Action handler, EvaluationPhase evaluationPhase) { /* undefined-coalescing operator is special because * 1. it should only be evaluated at final evaluation. diff --git a/ColorzCore/Parser/AST/StringNode.cs b/ColorzCore/Parser/AST/StringNode.cs index 530697b..2caa7ed 100644 --- a/ColorzCore/Parser/AST/StringNode.cs +++ b/ColorzCore/Parser/AST/StringNode.cs @@ -48,6 +48,6 @@ public IdentifierNode ToIdentifier(ImmutableStack scope) } public IAtomNode? AsAtom() => null; - public IParamNode SimplifyExpressions(TAction handler, EvaluationPhase evaluationPhase) => this; + public IParamNode SimplifyExpressions(Action handler, EvaluationPhase evaluationPhase) => this; } } diff --git a/ColorzCore/Parser/AST/UnaryOperatorNode.cs b/ColorzCore/Parser/AST/UnaryOperatorNode.cs index e66086b..768447f 100644 --- a/ColorzCore/Parser/AST/UnaryOperatorNode.cs +++ b/ColorzCore/Parser/AST/UnaryOperatorNode.cs @@ -43,7 +43,7 @@ public override IEnumerable ToTokens() yield return t; } - public override int? TryEvaluate(TAction handler, EvaluationPhase evaluationPhase) + public override int? TryEvaluate(Action handler, EvaluationPhase evaluationPhase) { int? inner = interior.TryEvaluate(handler, evaluationPhase); From f11127ae14b084910bdc67a292db0dacd4559ff4 Mon Sep 17 00:00:00 2001 From: Nat_776 Date: Fri, 19 Apr 2024 20:37:01 +0200 Subject: [PATCH 17/59] fix macros treating vectors as multiple parameters --- ColorzCore/Parser/EAParser.cs | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/ColorzCore/Parser/EAParser.cs b/ColorzCore/Parser/EAParser.cs index cb956f8..54792fc 100644 --- a/ColorzCore/Parser/EAParser.cs +++ b/ColorzCore/Parser/EAParser.cs @@ -522,21 +522,32 @@ public IList> ParseMacroParamList(MergeableGenerator tokens) { IList> parameters = new List>(); int parenNestings = 0; + + // HACK: this allows macro([1, 2, 3]) from expanding into a single parameter + int bracketBalance = 0; do { tokens.MoveNext(); List currentParam = new List(); while ( - !(parenNestings == 0 && (tokens.Current.Type == TokenType.CLOSE_PAREN || tokens.Current.Type == TokenType.COMMA)) + !(parenNestings == 0 + && (tokens.Current.Type == TokenType.CLOSE_PAREN || (bracketBalance == 0 && tokens.Current.Type == TokenType.COMMA))) && tokens.Current.Type != TokenType.NEWLINE) { - if (tokens.Current.Type == TokenType.CLOSE_PAREN) - { - parenNestings--; - } - else if (tokens.Current.Type == TokenType.OPEN_PAREN) + switch (tokens.Current.Type) { - parenNestings++; + case TokenType.CLOSE_PAREN: + parenNestings--; + break; + case TokenType.OPEN_PAREN: + parenNestings++; + break; + case TokenType.OPEN_BRACKET: + bracketBalance++; + break; + case TokenType.CLOSE_BRACKET: + bracketBalance--; + break; } currentParam.Add(tokens.Current); From b2dca340ca3c8aa046ad7a7fe34ef8b879d4f455 Mon Sep 17 00:00:00 2001 From: Nat_776 Date: Fri, 19 Apr 2024 21:31:52 +0200 Subject: [PATCH 18/59] IsSymbolDefined built-in macro --- ColorzCore/Parser/Macros/BuiltInMacro.cs | 4 +- ColorzCore/Parser/Macros/IsDefined.cs | 4 +- ColorzCore/Parser/Macros/IsSymbolDefined.cs | 61 +++++++++++++++++++++ ColorzCore/Parser/Macros/MacroCollection.cs | 9 ++- 4 files changed, 70 insertions(+), 8 deletions(-) create mode 100644 ColorzCore/Parser/Macros/IsSymbolDefined.cs diff --git a/ColorzCore/Parser/Macros/BuiltInMacro.cs b/ColorzCore/Parser/Macros/BuiltInMacro.cs index a38209d..0c38a85 100644 --- a/ColorzCore/Parser/Macros/BuiltInMacro.cs +++ b/ColorzCore/Parser/Macros/BuiltInMacro.cs @@ -1,14 +1,12 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Text; -using System.Threading.Tasks; using ColorzCore.DataTypes; using ColorzCore.Lexer; namespace ColorzCore.Parser.Macros { - abstract class BuiltInMacro : IMacro + public abstract class BuiltInMacro : IMacro { public abstract bool ValidNumParams(int num); public abstract IEnumerable ApplyMacro(Token head, IList> parameters, ImmutableStack scopes); diff --git a/ColorzCore/Parser/Macros/IsDefined.cs b/ColorzCore/Parser/Macros/IsDefined.cs index 135c840..c4ff891 100644 --- a/ColorzCore/Parser/Macros/IsDefined.cs +++ b/ColorzCore/Parser/Macros/IsDefined.cs @@ -46,12 +46,12 @@ protected bool IsReallyDefined(string name) return ParentParser.Definitions.ContainsKey(name) || ParentParser.Macros.ContainsName(name); } - protected static Token MakeTrueToken(DataTypes.Location location) + protected static Token MakeTrueToken(Location location) { return new Token(TokenType.NUMBER, location, "1"); } - protected static Token MakeFalseToken(DataTypes.Location location) + protected static Token MakeFalseToken(Location location) { return new Token(TokenType.NUMBER, location, "0"); } diff --git a/ColorzCore/Parser/Macros/IsSymbolDefined.cs b/ColorzCore/Parser/Macros/IsSymbolDefined.cs new file mode 100644 index 0000000..13b9a38 --- /dev/null +++ b/ColorzCore/Parser/Macros/IsSymbolDefined.cs @@ -0,0 +1,61 @@ +using System; +using System.Collections.Generic; +using ColorzCore.DataTypes; +using ColorzCore.Lexer; + +namespace ColorzCore.Parser.Macros +{ + public class IsSymbolDefined : BuiltInMacro + { + + public override IEnumerable ApplyMacro(Token head, IList> parameters, ImmutableStack scopes) + { + if (parameters[0].Count != 1) + { + // TODO: err somehow + yield return MakeFalseToken(head.Location); + } + else + { + Token token = parameters[0][0]; + + if ((token.Type == TokenType.IDENTIFIER) && IsReallyDefined(scopes, token.Content)) + { + yield return MakeTrueToken(head.Location); + } + else + { + yield return MakeFalseToken(head.Location); + } + } + } + + public override bool ValidNumParams(int num) + { + return num == 1; + } + + protected static bool IsReallyDefined(ImmutableStack scopes, string name) + { + for (ImmutableStack it = scopes; it != ImmutableStack.Nil; it = it.Tail) + { + if (it.Head.HasLocalSymbol(name)) + { + return true; + } + } + + return false; + } + + protected static Token MakeTrueToken(Location location) + { + return new Token(TokenType.NUMBER, location, "1"); + } + + protected static Token MakeFalseToken(Location location) + { + return new Token(TokenType.NUMBER, location, "0"); + } + } +} diff --git a/ColorzCore/Parser/Macros/MacroCollection.cs b/ColorzCore/Parser/Macros/MacroCollection.cs index 40e78d0..142946e 100644 --- a/ColorzCore/Parser/Macros/MacroCollection.cs +++ b/ColorzCore/Parser/Macros/MacroCollection.cs @@ -1,8 +1,6 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Text; -using System.Threading.Tasks; namespace ColorzCore.Parser.Macros { @@ -18,9 +16,12 @@ public MacroCollection(EAParser parent) Macros = new Dictionary>(); Parent = parent; - BuiltInMacros = new Dictionary { + BuiltInMacros = new Dictionary + { { "String", new String() }, { "IsDefined", new IsDefined(parent) }, + { "IsSymbolDefined", new IsSymbolDefined() }, + { "IsLabelDefined", new IsSymbolDefined() }, // alias { "AddToPool", new AddToPool(parent) }, }; } @@ -33,12 +34,14 @@ public IMacro GetMacro(string name, int paramNum) { return BuiltInMacros.ContainsKey(name) && BuiltInMacros[name].ValidNumParams(paramNum) ? BuiltInMacros[name] : Macros[name][paramNum]; } + public void AddMacro(IMacro macro, string name, int paramNum) { if (!Macros.ContainsKey(name)) Macros[name] = new Dictionary(); Macros[name][paramNum] = macro; } + public void Clear() { Macros.Clear(); From 2afb563f47caec0ebe36c8f0d51e2ccc38aee26a Mon Sep 17 00:00:00 2001 From: Nat_776 Date: Fri, 19 Apr 2024 21:45:55 +0200 Subject: [PATCH 19/59] better error message when macro expansion doesn't result in valid statement --- ColorzCore/Parser/EAParser.cs | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/ColorzCore/Parser/EAParser.cs b/ColorzCore/Parser/EAParser.cs index 54792fc..91d1f76 100644 --- a/ColorzCore/Parser/EAParser.cs +++ b/ColorzCore/Parser/EAParser.cs @@ -915,7 +915,23 @@ private IList ParseList(MergeableGenerator tokens, ImmutableSt case TokenType.MAYBE_MACRO: if (ExpandIdentifier(tokens, scopes)) { - return ParseLine(tokens, scopes); + // NOTE: we check here if we didn't end up with something that can't be a statement + + switch (tokens.Current.Type) + { + case TokenType.IDENTIFIER: + case TokenType.MAYBE_MACRO: + case TokenType.OPEN_BRACE: + case TokenType.PREPROCESSOR_DIRECTIVE: + return ParseLine(tokens, scopes); + + default: + // it is common for users to do '#define Foo 0xABCD' and then later 'Foo:' + Error($"Expansion of macro `{head.Content}` did not result in a valid statement. Did you perhaps attempt to define a label or symbol with that name?"); + IgnoreRestOfLine(tokens); + + return null; + } } else { @@ -957,7 +973,16 @@ private IList ParseList(MergeableGenerator tokens, ImmutableSt break; default: tokens.MoveNext(); - Error($"Unexpected token: {head.Type}: {head.Content}"); + + if (string.IsNullOrEmpty(head.Content)) + { + Error($"Unexpected token: {head.Type}."); + } + else + { + Error($"Unexpected token: {head.Type}: {head.Content}."); + } + IgnoreRestOfLine(tokens); break; } @@ -1056,7 +1081,7 @@ public bool ExpandIdentifier(MergeableGenerator tokens, ImmutableStack Date: Fri, 19 Apr 2024 22:49:49 +0200 Subject: [PATCH 20/59] print better messages using format strings --- ColorzCore/Parser/EAParser.cs | 71 +++++++++++++++++++++++++++++------ 1 file changed, 59 insertions(+), 12 deletions(-) diff --git a/ColorzCore/Parser/EAParser.cs b/ColorzCore/Parser/EAParser.cs index 91d1f76..1f30622 100644 --- a/ColorzCore/Parser/EAParser.cs +++ b/ColorzCore/Parser/EAParser.cs @@ -10,6 +10,7 @@ using System.IO; using System.Linq; using System.Text; +using System.Text.RegularExpressions; //TODO: Make errors less redundant (due to recursive nature, many paths will give several redundant errors). @@ -221,9 +222,9 @@ public static int ConvertToOffset(int value) "PROTECT" => ParseProtectStatement(parameters), "ALIGN" => ParseAlignStatement(parameters), "FILL" => ParseFillStatement(parameters), - "MESSAGE" => ParseMessageStatement(parameters), - "WARNING" => ParseWarningStatement(parameters), - "ERROR" => ParseErrorStatement(parameters), + "MESSAGE" => ParseMessageStatement(parameters, scopes), + "WARNING" => ParseWarningStatement(parameters, scopes), + "ERROR" => ParseErrorStatement(parameters, scopes), _ => null, // TODO: this is an error }; } @@ -500,21 +501,21 @@ static bool IsConditionalOperatorHelper(IAtomNode node) return null; } - private ILineNode? ParseMessageStatement(IList parameters) + private ILineNode? ParseMessageStatement(IList parameters, ImmutableStack scopes) { - Message(PrettyPrintParams(parameters)); + Message(PrettyPrintParamsForMessage(parameters, scopes)); return null; } - private ILineNode? ParseWarningStatement(IList parameters) + private ILineNode? ParseWarningStatement(IList parameters, ImmutableStack scopes) { - Warning(PrettyPrintParams(parameters)); + Warning(PrettyPrintParamsForMessage(parameters, scopes)); return null; } - private ILineNode? ParseErrorStatement(IList parameters) + private ILineNode? ParseErrorStatement(IList parameters, ImmutableStack scopes) { - Error(PrettyPrintParams(parameters)); + Error(PrettyPrintParamsForMessage(parameters, scopes)); return null; } @@ -926,7 +927,7 @@ private IList ParseList(MergeableGenerator tokens, ImmutableSt return ParseLine(tokens, scopes); default: - // it is common for users to do '#define Foo 0xABCD' and then later 'Foo:' + // it is somewhat common for users to do '#define Foo 0xABCD' and then later 'Foo:' Error($"Expansion of macro `{head.Content}` did not result in a valid statement. Did you perhaps attempt to define a label or symbol with that name?"); IgnoreRestOfLine(tokens); @@ -1148,12 +1149,58 @@ public void Clear() pastOffsets.Clear(); } - private string PrettyPrintParams(IList parameters) + private static readonly Regex formatItemRegex = new Regex(@"\{(?[^:]+)(?:\:(?\w*))?\}"); + + private string PrettyPrintParamsForMessage(IList parameters, ImmutableStack scopes) { StringBuilder sb = new StringBuilder(); + foreach (IParamNode parameter in parameters) { - sb.Append(parameter.PrettyPrint()).Append(' '); + if (parameter is StringNode node) + { + string theString = formatItemRegex.Replace(node.MyToken.Content, match => + { + string expr = match.Groups["expr"].Value!; + string? format = match.Groups["format"].Value; + + MergeableGenerator tokens = new MergeableGenerator( + new Tokenizer().TokenizeLine( + $"{expr} \n", parameter.MyLocation.file, parameter.MyLocation.lineNum, parameter.MyLocation.colNum + match.Index + 1)); + tokens.MoveNext(); + + IAtomNode? node = ParseAtom(tokens, scopes); + + if (node == null || tokens.Current.Type != TokenType.NEWLINE) + { + return $""; + } + + if (node.TryEvaluate(e => Error(node.MyLocation, e.Message), + EvaluationPhase.Early) is int value) + { + try + { + return value.ToString(format); + } + catch (FormatException e) + { + // TODO: this seems to never happen + return $""; + } + } + else + { + return $""; + } + }); + + sb.Append(theString).Append(' '); + } + else + { + sb.Append(parameter.PrettyPrint()).Append(' '); + } } return sb.ToString(); } From 69e5025453a0d7c797a94f436a627029885e3a5c Mon Sep 17 00:00:00 2001 From: Nat_776 Date: Sat, 20 Apr 2024 12:36:31 +0200 Subject: [PATCH 21/59] fixes to format string --- ColorzCore/DataTypes/Location.cs | 2 + ColorzCore/Parser/AST/StringNode.cs | 14 +++-- ColorzCore/Parser/EAParser.cs | 85 +++++++++++++++-------------- 3 files changed, 55 insertions(+), 46 deletions(-) diff --git a/ColorzCore/DataTypes/Location.cs b/ColorzCore/DataTypes/Location.cs index 563406d..554dfbc 100644 --- a/ColorzCore/DataTypes/Location.cs +++ b/ColorzCore/DataTypes/Location.cs @@ -18,6 +18,8 @@ public Location(string fileName, int lineNum, int colNum) : this() this.colNum = colNum; } + public readonly Location OffsetBy(int columns) => new Location(file, lineNum, colNum + columns); + public override readonly string ToString() => $"{file}:{lineNum}:{colNum}"; } } diff --git a/ColorzCore/Parser/AST/StringNode.cs b/ColorzCore/Parser/AST/StringNode.cs index 2caa7ed..4b30e33 100644 --- a/ColorzCore/Parser/AST/StringNode.cs +++ b/ColorzCore/Parser/AST/StringNode.cs @@ -14,8 +14,10 @@ public class StringNode : IParamNode public static readonly Regex idRegex = new Regex("^([a-zA-Z_][a-zA-Z0-9_]*)$"); public Token MyToken { get; } - public Location MyLocation { get { return MyToken.Location; } } - public ParamType Type { get { return ParamType.STRING; } } + public Location MyLocation => MyToken.Location; + public ParamType Type => ParamType.STRING; + + public string Value => MyToken.Content; public StringNode(Token value) { @@ -29,17 +31,19 @@ public IEnumerable ToBytes() public override string ToString() { - return MyToken.Content; + return Value; } + public string PrettyPrint() { - return '"' + ToString() + '"'; + return $"\"{Value}\""; } + public IEnumerable ToTokens() { yield return MyToken; } public bool IsValidIdentifier() { - return idRegex.IsMatch(MyToken.Content); + return idRegex.IsMatch(Value); } public IdentifierNode ToIdentifier(ImmutableStack scope) diff --git a/ColorzCore/Parser/EAParser.cs b/ColorzCore/Parser/EAParser.cs index 1f30622..413c86b 100644 --- a/ColorzCore/Parser/EAParser.cs +++ b/ColorzCore/Parser/EAParser.cs @@ -7,6 +7,7 @@ using ColorzCore.Raws; using System; using System.Collections.Generic; +using System.Globalization; using System.IO; using System.Linq; using System.Text; @@ -1149,60 +1150,62 @@ public void Clear() pastOffsets.Clear(); } - private static readonly Regex formatItemRegex = new Regex(@"\{(?[^:]+)(?:\:(?\w*))?\}"); - private string PrettyPrintParamsForMessage(IList parameters, ImmutableStack scopes) { - StringBuilder sb = new StringBuilder(); + return string.Join(" ", parameters.Select(parameter => parameter switch + { + StringNode node => ExpandUserFormatString(scopes, parameter.MyLocation.OffsetBy(1), node.Value), + _ => parameter.PrettyPrint(), + })); + } + + private static readonly Regex formatItemRegex = new Regex(@"\{(?[^:}]+)(?:\:(?[^:}]*))?\}"); - foreach (IParamNode parameter in parameters) + private string ExpandUserFormatString(ImmutableStack scopes, Location baseLocation, string stringValue) + { + string UserFormatStringError(string message, string details) { - if (parameter is StringNode node) - { - string theString = formatItemRegex.Replace(node.MyToken.Content, match => - { - string expr = match.Groups["expr"].Value!; - string? format = match.Groups["format"].Value; + Error($"An error occurred while expanding format string ({message})."); + return $"<{message}: {details}>"; + } - MergeableGenerator tokens = new MergeableGenerator( - new Tokenizer().TokenizeLine( - $"{expr} \n", parameter.MyLocation.file, parameter.MyLocation.lineNum, parameter.MyLocation.colNum + match.Index + 1)); - tokens.MoveNext(); + return formatItemRegex.Replace(stringValue, match => + { + string expr = match.Groups["expr"].Value!; + string? format = match.Groups["format"].Value; - IAtomNode? node = ParseAtom(tokens, scopes); + MergeableGenerator tokens = new MergeableGenerator( + new Tokenizer().TokenizeLine( + $"{expr} \n", baseLocation.file, baseLocation.lineNum, baseLocation.colNum + match.Index)); - if (node == null || tokens.Current.Type != TokenType.NEWLINE) - { - return $""; - } + tokens.MoveNext(); - if (node.TryEvaluate(e => Error(node.MyLocation, e.Message), - EvaluationPhase.Early) is int value) - { - try - { - return value.ToString(format); - } - catch (FormatException e) - { - // TODO: this seems to never happen - return $""; - } - } - else - { - return $""; - } - }); + IAtomNode? node = ParseAtom(tokens, scopes); + + if (node == null || tokens.Current.Type != TokenType.NEWLINE) + { + return UserFormatStringError("bad expression", $"'{expr}'"); + } - sb.Append(theString).Append(' '); + if (node.TryEvaluate(e => Error(node.MyLocation, e.Message), + EvaluationPhase.Early) is int value) + { + try + { + // TODO: do we need to raise an error when result == format? + // this happens (I think) when a custom format specifier with no substitution + return value.ToString(format, CultureInfo.InvariantCulture); + } + catch (FormatException e) + { + return UserFormatStringError("bad format specifier", $"'{format}' ({e.Message})"); + } } else { - sb.Append(parameter.PrettyPrint()).Append(' '); + return UserFormatStringError("failed to evaluate expression", $"'{expr}'"); } - } - return sb.ToString(); + }); } // Return value: Location where protection occurred. Nothing if location was not protected. From ee90e8f68305131cafdcf8a449ef408a960ac663 Mon Sep 17 00:00:00 2001 From: Nat_776 Date: Sat, 20 Apr 2024 16:10:00 +0200 Subject: [PATCH 22/59] don't simplify parameters too early (broke ASSERT hack) --- ColorzCore/Parser/EAParser.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ColorzCore/Parser/EAParser.cs b/ColorzCore/Parser/EAParser.cs index 413c86b..f6fe0d3 100644 --- a/ColorzCore/Parser/EAParser.cs +++ b/ColorzCore/Parser/EAParser.cs @@ -611,7 +611,7 @@ private IList ParsePreprocParamList(MergeableGenerator tokens switch (tokens.Current.Type) { case TokenType.OPEN_BRACKET: - return new ListNode(localHead.Location, ParseList(tokens, scopes)).Simplify(EvaluationPhase.Early); + return new ListNode(localHead.Location, ParseList(tokens, scopes)); case TokenType.STRING: tokens.MoveNext(); return new StringNode(localHead); @@ -635,11 +635,11 @@ private IList ParsePreprocParamList(MergeableGenerator tokens } else { - return ParseAtom(tokens, scopes, expandDefs)?.Simplify(EvaluationPhase.Early); + return ParseAtom(tokens, scopes, expandDefs); } default: - return ParseAtom(tokens, scopes, expandDefs)?.Simplify(EvaluationPhase.Early); + return ParseAtom(tokens, scopes, expandDefs); } } From c289b6434386953c7ae0d34c5ac04aedcb45b985 Mon Sep 17 00:00:00 2001 From: Nat_776 Date: Tue, 23 Apr 2024 16:54:31 +0200 Subject: [PATCH 23/59] refactor directive input --- ColorzCore/Parser/AST/ILineNode.cs | 2 +- ColorzCore/Parser/EAParser.cs | 9 ++-- ColorzCore/Parser/Macros/MacroCollection.cs | 2 +- ColorzCore/Parser/Pool.cs | 2 +- ColorzCore/Preprocessor/DirectiveHandler.cs | 13 ++--- .../Directives/DefineDirective.cs | 12 ++--- .../Preprocessor/Directives/ElseDirective.cs | 12 ++--- .../Preprocessor/Directives/EndIfDirective.cs | 13 +++-- .../Preprocessor/Directives/IDirective.cs | 15 ++---- .../Directives/IfDefinedDirective.cs | 12 ++--- .../Preprocessor/Directives/IfDirective.cs | 12 ++--- .../Directives/IfNotDefinedDirective.cs | 12 ++--- .../Directives/IncludeBinaryDirective.cs | 13 +++-- .../Directives/IncludeDirective.cs | 12 ++--- .../Directives/IncludeExternalDirective.cs | 11 ++--- .../Directives/IncludeToolEventDirective.cs | 11 ++--- .../Preprocessor/Directives/PoolDirective.cs | 10 ++-- .../Directives/SimpleDirective.cs | 47 +++++++++++++++++++ .../Directives/UndefineDirective.cs | 12 ++--- ColorzCore/Raws/IRawParam.cs | 2 +- ColorzCore/Raws/Raw.cs | 2 +- 21 files changed, 123 insertions(+), 113 deletions(-) create mode 100644 ColorzCore/Preprocessor/Directives/SimpleDirective.cs diff --git a/ColorzCore/Parser/AST/ILineNode.cs b/ColorzCore/Parser/AST/ILineNode.cs index 3397809..8a59918 100644 --- a/ColorzCore/Parser/AST/ILineNode.cs +++ b/ColorzCore/Parser/AST/ILineNode.cs @@ -7,7 +7,7 @@ namespace ColorzCore.Parser.AST { - interface ILineNode + public interface ILineNode { int Size { get; } string PrettyPrint(int indentation); diff --git a/ColorzCore/Parser/EAParser.cs b/ColorzCore/Parser/EAParser.cs index f6fe0d3..393d62f 100644 --- a/ColorzCore/Parser/EAParser.cs +++ b/ColorzCore/Parser/EAParser.cs @@ -17,7 +17,7 @@ namespace ColorzCore.Parser { - class EAParser + public class EAParser { public MacroCollection Macros { get; } public Dictionary Definitions { get; } @@ -590,7 +590,7 @@ private IList ParseParamList(MergeableGenerator tokens, Immut return paramList; } - private IList ParsePreprocParamList(MergeableGenerator tokens, ImmutableStack scopes, bool allowsFirstExpanded) + public IList ParsePreprocParamList(MergeableGenerator tokens, ImmutableStack scopes, bool allowsFirstExpanded) { IList temp = ParseParamList(tokens, scopes, allowsFirstExpanded); @@ -1049,10 +1049,7 @@ private void TryDefineSymbol(ImmutableStack scopes, string name, IAtomN head = tokens.Current; tokens.MoveNext(); - // Note: Not a ParseParamList because no commas. - // HACK: #if wants its parameters to be expanded, but other directives (define, ifdef, undef, etc) do not - IList paramList = ParsePreprocParamList(tokens, scopes, head.Content == "#if"); - ILineNode? result = directiveHandler.HandleDirective(this, head, paramList, tokens); + ILineNode? result = directiveHandler.HandleDirective(this, head, tokens, scopes); if (result != null) { diff --git a/ColorzCore/Parser/Macros/MacroCollection.cs b/ColorzCore/Parser/Macros/MacroCollection.cs index 142946e..2293bf3 100644 --- a/ColorzCore/Parser/Macros/MacroCollection.cs +++ b/ColorzCore/Parser/Macros/MacroCollection.cs @@ -4,7 +4,7 @@ namespace ColorzCore.Parser.Macros { - class MacroCollection + public class MacroCollection { public Dictionary BuiltInMacros { get; } public EAParser Parent { get; } diff --git a/ColorzCore/Parser/Pool.cs b/ColorzCore/Parser/Pool.cs index b8b73c5..cc9eb41 100644 --- a/ColorzCore/Parser/Pool.cs +++ b/ColorzCore/Parser/Pool.cs @@ -5,7 +5,7 @@ namespace ColorzCore.Parser { - class Pool + public class Pool { public struct PooledLine { diff --git a/ColorzCore/Preprocessor/DirectiveHandler.cs b/ColorzCore/Preprocessor/DirectiveHandler.cs index b3b37d2..255fb65 100644 --- a/ColorzCore/Preprocessor/DirectiveHandler.cs +++ b/ColorzCore/Preprocessor/DirectiveHandler.cs @@ -12,7 +12,7 @@ namespace ColorzCore.Preprocessor { - class DirectiveHandler + public class DirectiveHandler { private Dictionary directives; @@ -36,7 +36,7 @@ public DirectiveHandler(IncludeFileSearcher includeSearcher, IncludeFileSearcher }; } - public ILineNode? HandleDirective(EAParser p, Token directive, IList parameters, MergeableGenerator tokens) + public ILineNode? HandleDirective(EAParser p, Token directive, MergeableGenerator tokens, ImmutableStack scopes) { string directiveName = directive.Content.Substring(1); @@ -44,14 +44,7 @@ public DirectiveHandler(IncludeFileSearcher includeSearcher, IncludeFileSearcher { if (!toExec.RequireInclusion || p.IsIncluding) { - if (toExec.MinParams <= parameters.Count && (!toExec.MaxParams.HasValue || parameters.Count <= toExec.MaxParams)) - { - return toExec.Execute(p, directive, parameters, tokens); - } - else - { - p.Error(directive.Location, $"Invalid number of parameters ({parameters.Count}) to directive {directiveName}."); - } + return toExec.Execute(p, directive, tokens, scopes); } } else diff --git a/ColorzCore/Preprocessor/Directives/DefineDirective.cs b/ColorzCore/Preprocessor/Directives/DefineDirective.cs index 3c0e602..7ffb7fa 100644 --- a/ColorzCore/Preprocessor/Directives/DefineDirective.cs +++ b/ColorzCore/Preprocessor/Directives/DefineDirective.cs @@ -1,8 +1,6 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Text; -using System.Threading.Tasks; using ColorzCore.DataTypes; using ColorzCore.Lexer; using ColorzCore.Parser; @@ -11,15 +9,15 @@ namespace ColorzCore.Preprocessor.Directives { - class DefineDirective : IDirective + class DefineDirective : SimpleDirective { - public int MinParams => 1; + public override int MinParams => 1; - public int? MaxParams => 2; + public override int? MaxParams => 2; - public bool RequireInclusion => true; + public override bool RequireInclusion => true; - public ILineNode? Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) + public override ILineNode? Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) { if (parameters[0] is MacroInvocationNode signature) { diff --git a/ColorzCore/Preprocessor/Directives/ElseDirective.cs b/ColorzCore/Preprocessor/Directives/ElseDirective.cs index d3b3e2d..c48fc69 100644 --- a/ColorzCore/Preprocessor/Directives/ElseDirective.cs +++ b/ColorzCore/Preprocessor/Directives/ElseDirective.cs @@ -1,8 +1,6 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Text; -using System.Threading.Tasks; using ColorzCore.DataTypes; using ColorzCore.Lexer; using ColorzCore.Parser; @@ -10,15 +8,15 @@ namespace ColorzCore.Preprocessor.Directives { - class ElseDirective : IDirective + class ElseDirective : SimpleDirective { - public int MinParams => 0; + public override int MinParams => 0; - public int? MaxParams => 0; + public override int? MaxParams => 0; - public bool RequireInclusion => false; + public override bool RequireInclusion => false; - public ILineNode? Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) + public override ILineNode? Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) { if (p.Inclusion.IsEmpty) p.Error(self.Location, "No matching if[n]def."); diff --git a/ColorzCore/Preprocessor/Directives/EndIfDirective.cs b/ColorzCore/Preprocessor/Directives/EndIfDirective.cs index 9ec2cce..74a2371 100644 --- a/ColorzCore/Preprocessor/Directives/EndIfDirective.cs +++ b/ColorzCore/Preprocessor/Directives/EndIfDirective.cs @@ -1,8 +1,6 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Text; -using System.Threading.Tasks; using ColorzCore.DataTypes; using ColorzCore.Lexer; using ColorzCore.Parser; @@ -10,20 +8,21 @@ namespace ColorzCore.Preprocessor.Directives { - class EndIfDirective : IDirective + class EndIfDirective : SimpleDirective { - public int MinParams => 0; + public override int MinParams => 0; - public int? MaxParams => 0; + public override int? MaxParams => 0; - public bool RequireInclusion => false; + public override bool RequireInclusion => false; - public ILineNode? Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) + public override ILineNode? Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) { if (p.Inclusion.IsEmpty) p.Error(self.Location, "No matching if[n]def."); else p.Inclusion = p.Inclusion.Tail; + return null; } } diff --git a/ColorzCore/Preprocessor/Directives/IDirective.cs b/ColorzCore/Preprocessor/Directives/IDirective.cs index c1e44e4..742e2f6 100644 --- a/ColorzCore/Preprocessor/Directives/IDirective.cs +++ b/ColorzCore/Preprocessor/Directives/IDirective.cs @@ -5,12 +5,10 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Text; -using System.Threading.Tasks; namespace ColorzCore.Preprocessor.Directives { - interface IDirective + public interface IDirective { /*** * Perform the directive's action, be it altering tokens, for just emitting a special ILineNode. @@ -18,15 +16,8 @@ interface IDirective * * Return: If a string is returned, it is interpreted as an error. */ - ILineNode? Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens); - /*** - * Minimum number of parameters, inclusive. - */ - int MinParams { get; } - /*** - * Maximum number of parameters, inclusive. Null for no limit. - */ - int? MaxParams { get; } + ILineNode? Execute(EAParser p, Token self, MergeableGenerator tokens, ImmutableStack scopes); + /*** * Whether requires the parser to be taking in tokens. * This may not hold when the parser is skipping, e.g. from an #ifdef. diff --git a/ColorzCore/Preprocessor/Directives/IfDefinedDirective.cs b/ColorzCore/Preprocessor/Directives/IfDefinedDirective.cs index 0447a79..669b554 100644 --- a/ColorzCore/Preprocessor/Directives/IfDefinedDirective.cs +++ b/ColorzCore/Preprocessor/Directives/IfDefinedDirective.cs @@ -1,8 +1,6 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Text; -using System.Threading.Tasks; using ColorzCore.DataTypes; using ColorzCore.Lexer; using ColorzCore.Parser; @@ -10,15 +8,15 @@ namespace ColorzCore.Preprocessor.Directives { - class IfDefinedDirective : IDirective + class IfDefinedDirective : SimpleDirective { - public int MinParams => 1; + public override int MinParams => 1; - public int? MaxParams => 1; + public override int? MaxParams => 1; - public bool RequireInclusion => false; + public override bool RequireInclusion => false; - public ILineNode? Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) + public override ILineNode? Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) { bool flag = true; string? identifier; diff --git a/ColorzCore/Preprocessor/Directives/IfDirective.cs b/ColorzCore/Preprocessor/Directives/IfDirective.cs index 2700b5f..a582ed9 100644 --- a/ColorzCore/Preprocessor/Directives/IfDirective.cs +++ b/ColorzCore/Preprocessor/Directives/IfDirective.cs @@ -1,8 +1,6 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Text; -using System.Threading.Tasks; using ColorzCore.DataTypes; using ColorzCore.Lexer; using ColorzCore.Parser; @@ -10,15 +8,15 @@ namespace ColorzCore.Preprocessor.Directives { - class IfDirective : IDirective + class IfDirective : SimpleDirective { - public int MinParams => 1; + public override int MinParams => 1; - public int? MaxParams => 1; + public override int? MaxParams => 1; - public bool RequireInclusion => false; + public override bool RequireInclusion => false; - public ILineNode? Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) + public override ILineNode? Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) { bool flag = true; diff --git a/ColorzCore/Preprocessor/Directives/IfNotDefinedDirective.cs b/ColorzCore/Preprocessor/Directives/IfNotDefinedDirective.cs index ae48c0e..d08d8f6 100644 --- a/ColorzCore/Preprocessor/Directives/IfNotDefinedDirective.cs +++ b/ColorzCore/Preprocessor/Directives/IfNotDefinedDirective.cs @@ -1,8 +1,6 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Text; -using System.Threading.Tasks; using ColorzCore.DataTypes; using ColorzCore.Lexer; using ColorzCore.Parser; @@ -10,15 +8,15 @@ namespace ColorzCore.Preprocessor.Directives { - class IfNotDefinedDirective : IDirective + class IfNotDefinedDirective : SimpleDirective { - public int MinParams => 1; + public override int MinParams => 1; - public int? MaxParams => 1; + public override int? MaxParams => 1; - public bool RequireInclusion => false; + public override bool RequireInclusion => false; - public ILineNode? Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) + public override ILineNode? Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) { bool flag = true; string? identifier; diff --git a/ColorzCore/Preprocessor/Directives/IncludeBinaryDirective.cs b/ColorzCore/Preprocessor/Directives/IncludeBinaryDirective.cs index 98bdc0c..8cc0842 100644 --- a/ColorzCore/Preprocessor/Directives/IncludeBinaryDirective.cs +++ b/ColorzCore/Preprocessor/Directives/IncludeBinaryDirective.cs @@ -1,8 +1,6 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Text; -using System.Threading.Tasks; using ColorzCore.DataTypes; using ColorzCore.Lexer; using ColorzCore.Parser.AST; @@ -12,17 +10,17 @@ namespace ColorzCore.Preprocessor.Directives { - class IncludeBinaryDirective : IDirective + class IncludeBinaryDirective : SimpleDirective { - public int MinParams { get { return 1; } } + public override int MinParams => 1; - public int? MaxParams { get { return 1; } } + public override int? MaxParams => 1; - public bool RequireInclusion { get { return true; } } + public override bool RequireInclusion => true; public IncludeFileSearcher FileSearcher { get; set; } = new IncludeFileSearcher(); - public ILineNode? Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) + public override ILineNode? Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) { string? existantFile = FileSearcher.FindFile(Path.GetDirectoryName(self.FileName), parameters[0].ToString()!); @@ -42,6 +40,7 @@ class IncludeBinaryDirective : IDirective { p.Error(parameters[0].MyLocation, "Could not find file \"" + parameters[0].ToString() + "\"."); } + return null; } } diff --git a/ColorzCore/Preprocessor/Directives/IncludeDirective.cs b/ColorzCore/Preprocessor/Directives/IncludeDirective.cs index 4a0cbcd..5c144d0 100644 --- a/ColorzCore/Preprocessor/Directives/IncludeDirective.cs +++ b/ColorzCore/Preprocessor/Directives/IncludeDirective.cs @@ -1,8 +1,6 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Text; -using System.Threading.Tasks; using ColorzCore.DataTypes; using ColorzCore.Lexer; using ColorzCore.Parser.AST; @@ -12,17 +10,17 @@ namespace ColorzCore.Preprocessor.Directives { - class IncludeDirective : IDirective + class IncludeDirective : SimpleDirective { - public int MinParams { get { return 1; } } + public override int MinParams => 1; - public int? MaxParams { get { return 1; } } + public override int? MaxParams => 1; - public bool RequireInclusion { get { return true; } } + public override bool RequireInclusion => true; public IncludeFileSearcher FileSearcher { get; set; } = new IncludeFileSearcher(); - public ILineNode? Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) + public override ILineNode? Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) { string? existantFile = FileSearcher.FindFile(Path.GetDirectoryName(self.FileName), parameters[0].ToString()!); diff --git a/ColorzCore/Preprocessor/Directives/IncludeExternalDirective.cs b/ColorzCore/Preprocessor/Directives/IncludeExternalDirective.cs index e8249bc..4e2deee 100644 --- a/ColorzCore/Preprocessor/Directives/IncludeExternalDirective.cs +++ b/ColorzCore/Preprocessor/Directives/IncludeExternalDirective.cs @@ -2,7 +2,6 @@ using System.Collections.Generic; using System.Linq; using System.Text; -using System.Threading.Tasks; using ColorzCore.DataTypes; using ColorzCore.Lexer; using ColorzCore.Parser; @@ -12,15 +11,15 @@ namespace ColorzCore.Preprocessor.Directives { - class IncludeExternalDirective : IDirective + class IncludeExternalDirective : SimpleDirective { - public int MinParams { get { return 1; } } - public int? MaxParams { get { return null; } } - public bool RequireInclusion { get { return true; } } + public override int MinParams => 1; + public override int? MaxParams => null; + public override bool RequireInclusion => true; public IncludeFileSearcher FileSearcher { get; set; } = new IncludeFileSearcher(); - public ILineNode? Execute(EAParser parse, Token self, IList parameters, MergeableGenerator tokens) + public override ILineNode? Execute(EAParser parse, Token self, IList parameters, MergeableGenerator tokens) { ExecTimer.Timer.AddTimingPoint(ExecTimer.KEY_GENERIC); diff --git a/ColorzCore/Preprocessor/Directives/IncludeToolEventDirective.cs b/ColorzCore/Preprocessor/Directives/IncludeToolEventDirective.cs index df5e05f..cc5c22c 100644 --- a/ColorzCore/Preprocessor/Directives/IncludeToolEventDirective.cs +++ b/ColorzCore/Preprocessor/Directives/IncludeToolEventDirective.cs @@ -2,7 +2,6 @@ using System.Collections.Generic; using System.Linq; using System.Text; -using System.Threading.Tasks; using ColorzCore.DataTypes; using ColorzCore.Lexer; using ColorzCore.Parser; @@ -12,15 +11,15 @@ namespace ColorzCore.Preprocessor.Directives { - class IncludeToolEventDirective : IDirective + class IncludeToolEventDirective : SimpleDirective { - public int MinParams { get { return 1; } } - public int? MaxParams { get { return null; } } - public bool RequireInclusion { get { return true; } } + public override int MinParams => 1; + public override int? MaxParams => null; + public override bool RequireInclusion => true; public IncludeFileSearcher FileSearcher { get; set; } = new IncludeFileSearcher(); - public ILineNode? Execute(EAParser parse, Token self, IList parameters, MergeableGenerator tokens) + public override ILineNode? Execute(EAParser parse, Token self, IList parameters, MergeableGenerator tokens) { ExecTimer.Timer.AddTimingPoint(ExecTimer.KEY_GENERIC); diff --git a/ColorzCore/Preprocessor/Directives/PoolDirective.cs b/ColorzCore/Preprocessor/Directives/PoolDirective.cs index 5d0240b..68b271c 100644 --- a/ColorzCore/Preprocessor/Directives/PoolDirective.cs +++ b/ColorzCore/Preprocessor/Directives/PoolDirective.cs @@ -7,13 +7,13 @@ namespace ColorzCore.Preprocessor.Directives { - class PoolDirective : IDirective + class PoolDirective : SimpleDirective { - public int MinParams => 0; - public int? MaxParams => 0; - public bool RequireInclusion => true; + public override int MinParams => 0; + public override int? MaxParams => 0; + public override bool RequireInclusion => true; - public ILineNode? Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) + public override ILineNode? Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) { BlockNode result = new BlockNode(); diff --git a/ColorzCore/Preprocessor/Directives/SimpleDirective.cs b/ColorzCore/Preprocessor/Directives/SimpleDirective.cs new file mode 100644 index 0000000..e714cc0 --- /dev/null +++ b/ColorzCore/Preprocessor/Directives/SimpleDirective.cs @@ -0,0 +1,47 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using ColorzCore.DataTypes; +using ColorzCore.Lexer; +using ColorzCore.Parser; +using ColorzCore.Parser.AST; + +namespace ColorzCore.Preprocessor.Directives +{ + /*** + * Simple abstract base class for directives that don't care about the details of their parameters + */ + public abstract class SimpleDirective : IDirective + { + public abstract bool RequireInclusion { get; } + + /*** + * Minimum number of parameters, inclusive. + */ + public abstract int MinParams { get; } + + /*** + * Maximum number of parameters, inclusive. Null for no limit. + */ + public abstract int? MaxParams { get; } + + public ILineNode? Execute(EAParser p, Token self, MergeableGenerator tokens, ImmutableStack scopes) + { + // Note: Not a ParseParamList because no commas. + // HACK: #if wants its parameters to be expanded, but other directives (define, ifdef, undef, etc) do not + IList parameters = p.ParsePreprocParamList(tokens, scopes, self.Content == "#if"); + + if (MinParams <= parameters.Count && (!MaxParams.HasValue || parameters.Count <= MaxParams)) + { + return Execute(p, self, parameters, tokens); + } + else + { + p.Error(self.Location, $"Invalid number of parameters ({parameters.Count}) to directive {self}."); + return null; + } + } + + public abstract ILineNode? Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens); + } +} diff --git a/ColorzCore/Preprocessor/Directives/UndefineDirective.cs b/ColorzCore/Preprocessor/Directives/UndefineDirective.cs index 0583d30..40fe263 100644 --- a/ColorzCore/Preprocessor/Directives/UndefineDirective.cs +++ b/ColorzCore/Preprocessor/Directives/UndefineDirective.cs @@ -1,8 +1,6 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Text; -using System.Threading.Tasks; using ColorzCore.DataTypes; using ColorzCore.Lexer; using ColorzCore.Parser; @@ -10,15 +8,15 @@ namespace ColorzCore.Preprocessor.Directives { - class UndefineDirective : IDirective + class UndefineDirective : SimpleDirective { - public int MinParams => 1; + public override int MinParams => 1; - public int? MaxParams => null; + public override int? MaxParams => null; - public bool RequireInclusion => true; + public override bool RequireInclusion => true; - public ILineNode? Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) + public override ILineNode? Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) { foreach (IParamNode parm in parameters) { diff --git a/ColorzCore/Raws/IRawParam.cs b/ColorzCore/Raws/IRawParam.cs index a9fdaf2..9198a72 100644 --- a/ColorzCore/Raws/IRawParam.cs +++ b/ColorzCore/Raws/IRawParam.cs @@ -8,7 +8,7 @@ namespace ColorzCore.Raws { - interface IRawParam + public interface IRawParam { string Name { get; } int Position { get; } //Length and position in bits diff --git a/ColorzCore/Raws/Raw.cs b/ColorzCore/Raws/Raw.cs index 9ee0551..c998bd7 100644 --- a/ColorzCore/Raws/Raw.cs +++ b/ColorzCore/Raws/Raw.cs @@ -10,7 +10,7 @@ namespace ColorzCore.Raws { - class Raw + public class Raw { public string Name { get; } public int Alignment { get; } From 2ba17702dfd52a34b974732dc0235f2cb8778996 Mon Sep 17 00:00:00 2001 From: Nat_776 Date: Tue, 23 Apr 2024 17:01:41 +0200 Subject: [PATCH 24/59] merge both ifdef with ifndef into one class --- ColorzCore/Preprocessor/DirectiveHandler.cs | 4 +- .../Directives/IfDefinedDirective.cs | 11 +++++- .../Directives/IfNotDefinedDirective.cs | 38 ------------------- 3 files changed, 12 insertions(+), 41 deletions(-) delete mode 100644 ColorzCore/Preprocessor/Directives/IfNotDefinedDirective.cs diff --git a/ColorzCore/Preprocessor/DirectiveHandler.cs b/ColorzCore/Preprocessor/DirectiveHandler.cs index 255fb65..6451dd5 100644 --- a/ColorzCore/Preprocessor/DirectiveHandler.cs +++ b/ColorzCore/Preprocessor/DirectiveHandler.cs @@ -25,8 +25,8 @@ public DirectiveHandler(IncludeFileSearcher includeSearcher, IncludeFileSearcher { "incext", new IncludeExternalDirective { FileSearcher = toolSearcher } }, { "inctext", new IncludeToolEventDirective { FileSearcher = toolSearcher } }, { "inctevent", new IncludeToolEventDirective { FileSearcher = toolSearcher } }, - { "ifdef", new IfDefinedDirective() }, - { "ifndef", new IfNotDefinedDirective() }, + { "ifdef", new IfDefinedDirective(false) }, + { "ifndef", new IfDefinedDirective(true) }, { "if", new IfDirective() }, { "else", new ElseDirective() }, { "endif", new EndIfDirective() }, diff --git a/ColorzCore/Preprocessor/Directives/IfDefinedDirective.cs b/ColorzCore/Preprocessor/Directives/IfDefinedDirective.cs index 669b554..93e92f4 100644 --- a/ColorzCore/Preprocessor/Directives/IfDefinedDirective.cs +++ b/ColorzCore/Preprocessor/Directives/IfDefinedDirective.cs @@ -16,6 +16,13 @@ class IfDefinedDirective : SimpleDirective public override bool RequireInclusion => false; + public bool Inverted { get; } + + public IfDefinedDirective(bool invert) + { + Inverted = invert; + } + public override ILineNode? Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) { bool flag = true; @@ -24,7 +31,9 @@ class IfDefinedDirective : SimpleDirective { if (parameter.Type == ParamType.ATOM && (identifier = ((IAtomNode)parameter).GetIdentifier()) != null) { - flag &= p.Macros.ContainsName(identifier) || p.Definitions.ContainsKey(identifier); //TODO: Built in definitions? + // TODO: Built in definitions? + bool isDefined = p.Macros.ContainsName(identifier) || p.Definitions.ContainsKey(identifier); + flag &= Inverted ? !isDefined : isDefined; } else { diff --git a/ColorzCore/Preprocessor/Directives/IfNotDefinedDirective.cs b/ColorzCore/Preprocessor/Directives/IfNotDefinedDirective.cs deleted file mode 100644 index d08d8f6..0000000 --- a/ColorzCore/Preprocessor/Directives/IfNotDefinedDirective.cs +++ /dev/null @@ -1,38 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using ColorzCore.DataTypes; -using ColorzCore.Lexer; -using ColorzCore.Parser; -using ColorzCore.Parser.AST; - -namespace ColorzCore.Preprocessor.Directives -{ - class IfNotDefinedDirective : SimpleDirective - { - public override int MinParams => 1; - - public override int? MaxParams => 1; - - public override bool RequireInclusion => false; - - public override ILineNode? Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) - { - bool flag = true; - string? identifier; - foreach (IParamNode parameter in parameters) - { - if (parameter.Type == ParamType.ATOM && (identifier = ((IAtomNode)parameter).GetIdentifier()) != null) - { - flag &= !p.Macros.ContainsName(identifier) && !p.Definitions.ContainsKey(identifier); //TODO: Built in definitions? - } - else - { - p.Error(parameter.MyLocation, "Definition name must be an identifier."); - } - } - p.Inclusion = new ImmutableStack(flag, p.Inclusion); - return null; - } - } -} From bd103bba7ec66cedcf23e1792d32c22ac8ce041f Mon Sep 17 00:00:00 2001 From: Nat_776 Date: Tue, 23 Apr 2024 17:34:16 +0200 Subject: [PATCH 25/59] add ReadByteAt (and friends) built-in macro --- ColorzCore/EAInterpreter.cs | 20 ++++++- ColorzCore/EAOptions.cs | 2 + ColorzCore/IO/ROM.cs | 10 ++-- ColorzCore/Parser/EAParser.cs | 2 +- ColorzCore/Parser/Macros/ErrorMacro.cs | 34 ++++++++++++ ColorzCore/Parser/Macros/ReadDataAt.cs | 72 ++++++++++++++++++++++++++ 6 files changed, 133 insertions(+), 7 deletions(-) create mode 100644 ColorzCore/Parser/Macros/ErrorMacro.cs create mode 100644 ColorzCore/Parser/Macros/ReadDataAt.cs diff --git a/ColorzCore/EAInterpreter.cs b/ColorzCore/EAInterpreter.cs index 77233c1..2c37ed2 100644 --- a/ColorzCore/EAInterpreter.cs +++ b/ColorzCore/EAInterpreter.cs @@ -3,6 +3,7 @@ using ColorzCore.Lexer; using ColorzCore.Parser; using ColorzCore.Parser.AST; +using ColorzCore.Parser.Macros; using ColorzCore.Raws; using System; using System.Collections.Generic; @@ -63,8 +64,25 @@ public EAInterpreter(IOutput output, string game, string? rawsFolder, string raw myParser = new EAParser(allRaws, log, new Preprocessor.DirectiveHandler(includeSearcher, toolSearcher)); - myParser.Definitions['_' + game + '_'] = new Definition(); + myParser.Definitions[$"_{game}_"] = new Definition(); myParser.Definitions["__COLORZ_CORE__"] = new Definition(); + + if (EAOptions.Instance.readDataMacros && output is ROM rom) + { + myParser.Definitions["__has_read_data_macros"] = new Definition(); + + myParser.Macros.BuiltInMacros.Add("ReadByteAt", new ReadDataAt(myParser, rom, 1)); + myParser.Macros.BuiltInMacros.Add("ReadShortAt", new ReadDataAt(myParser, rom, 2)); + myParser.Macros.BuiltInMacros.Add("ReadWordAt", new ReadDataAt(myParser, rom, 4)); + } + else + { + BuiltInMacro unsupportedMacro = new ErrorMacro(myParser, "Macro unsupported in this configuration.", i => i == 1); + + myParser.Macros.BuiltInMacros.Add("ReadByteAt", unsupportedMacro); + myParser.Macros.BuiltInMacros.Add("ReadShortAt", unsupportedMacro); + myParser.Macros.BuiltInMacros.Add("ReadWordAt", unsupportedMacro); + } } public bool Interpret() diff --git a/ColorzCore/EAOptions.cs b/ColorzCore/EAOptions.cs index a2cc6c4..da11199 100644 --- a/ColorzCore/EAOptions.cs +++ b/ColorzCore/EAOptions.cs @@ -15,6 +15,7 @@ class EAOptions public bool noColoredLog; public bool nocashSym; + public bool readDataMacros; public List includePaths = new List(); public List toolsPaths = new List(); @@ -31,6 +32,7 @@ private EAOptions() nomess = false; noColoredLog = false; nocashSym = false; + readDataMacros = true; buildTimes = false; romBaseAddress = 0x8000000; maximumRomSize = 0x2000000; diff --git a/ColorzCore/IO/ROM.cs b/ColorzCore/IO/ROM.cs index 133311e..eb81761 100644 --- a/ColorzCore/IO/ROM.cs +++ b/ColorzCore/IO/ROM.cs @@ -2,17 +2,17 @@ using System.Collections.Generic; using System.IO; using System.Linq; -using System.Text; -using System.Threading.Tasks; namespace ColorzCore.IO { - class ROM : IOutput + public class ROM : IOutput { - private BufferedStream myStream; - private byte[] myData; + private readonly BufferedStream myStream; + private readonly byte[] myData; private int size; + public byte this[int offset] => myData[offset]; + public ROM(Stream myROM, int maximumSize) { myStream = new BufferedStream(myROM); diff --git a/ColorzCore/Parser/EAParser.cs b/ColorzCore/Parser/EAParser.cs index 393d62f..921bd21 100644 --- a/ColorzCore/Parser/EAParser.cs +++ b/ColorzCore/Parser/EAParser.cs @@ -666,7 +666,7 @@ public IList ParsePreprocParamList(MergeableGenerator tokens, { TokenType.UNDEFINED_COALESCE_OP, 13 }, }; - private IAtomNode? ParseAtom(MergeableGenerator tokens, ImmutableStack scopes, bool expandDefs = true) + public IAtomNode? ParseAtom(MergeableGenerator tokens, ImmutableStack scopes, bool expandDefs = true) { //Use Shift Reduce Parsing Token localHead = tokens.Current; diff --git a/ColorzCore/Parser/Macros/ErrorMacro.cs b/ColorzCore/Parser/Macros/ErrorMacro.cs new file mode 100644 index 0000000..42fb9f1 --- /dev/null +++ b/ColorzCore/Parser/Macros/ErrorMacro.cs @@ -0,0 +1,34 @@ +using ColorzCore.DataTypes; +using ColorzCore.IO; +using ColorzCore.Lexer; +using ColorzCore.Parser.AST; +using System; +using System.Collections.Generic; +using System.Linq; + +namespace ColorzCore.Parser.Macros +{ + class ErrorMacro : BuiltInMacro + { + public delegate bool ValidateNumParamsIndirect(int num); + + private readonly EAParser parser; + private readonly string message; + private readonly ValidateNumParamsIndirect validateNumParamsIndirect; + + public ErrorMacro(EAParser parser, string message, ValidateNumParamsIndirect validateNumParamsIndirect) + { + this.parser = parser; + this.message = message; + this.validateNumParamsIndirect = validateNumParamsIndirect; + } + + public override IEnumerable ApplyMacro(Token head, IList> parameters, ImmutableStack scopes) + { + parser.Error(head.Location, message); + yield return new Token(TokenType.NUMBER, head.Location, "0"); + } + + public override bool ValidNumParams(int num) => validateNumParamsIndirect(num); + } +} diff --git a/ColorzCore/Parser/Macros/ReadDataAt.cs b/ColorzCore/Parser/Macros/ReadDataAt.cs new file mode 100644 index 0000000..f5cea99 --- /dev/null +++ b/ColorzCore/Parser/Macros/ReadDataAt.cs @@ -0,0 +1,72 @@ +using ColorzCore.DataTypes; +using ColorzCore.IO; +using ColorzCore.Lexer; +using ColorzCore.Parser.AST; +using System; +using System.Collections.Generic; +using System.Linq; + +namespace ColorzCore.Parser.Macros +{ + public class ReadDataAt : BuiltInMacro + { + private readonly EAParser parser; + private readonly ROM rom; + private readonly int readLength; + + public ReadDataAt(EAParser parser, ROM rom, int readLength) + { + this.parser = parser; + this.rom = rom; + this.readLength = readLength; + } + + public override IEnumerable ApplyMacro(Token head, IList> parameters, ImmutableStack scopes) + { + // HACK: hack + MergeableGenerator tokens = new MergeableGenerator( + Enumerable.Repeat(new Token(TokenType.NEWLINE, head.Location, "\n"), 1)); + tokens.PrependEnumerator(parameters[0].GetEnumerator()); + + IAtomNode? atom = parser.ParseAtom(tokens, scopes, true); + + if (tokens.Current.Type != TokenType.NEWLINE) + { + parser.Error(head.Location, "Garbage at the end of macro parameter."); + yield return new Token(TokenType.NUMBER, head.Location, "0"); + } + else if (atom?.TryEvaluate(e => parser.Error(atom.MyLocation, e.Message), EvaluationPhase.Immediate) is int offset) + { + offset = EAParser.ConvertToOffset(offset); + + if (offset >= 0 && offset <= EAOptions.Instance.maximumRomSize - readLength) + { + int data = 0; + + // little endian!!! + for (int i = 0; i < readLength; i++) + { + data |= rom[offset + i] << (i * 8); + } + + yield return new Token(TokenType.NUMBER, head.Location, $"0x{data:X}"); + } + else + { + parser.Error(head.Location, $"Read offset out of bounds: {offset:08X}"); + yield return new Token(TokenType.NUMBER, head.Location, "0"); + } + } + else + { + parser.Error(head.Location, "Could not read data from base binary."); + yield return new Token(TokenType.NUMBER, head.Location, "0"); + } + } + + public override bool ValidNumParams(int num) + { + return num == 1; + } + } +} From 07eb1a64d1ef0ee1f511ebcd52971592a0ea2a4e Mon Sep 17 00:00:00 2001 From: Nat_776 Date: Fri, 26 Apr 2024 18:16:37 +0200 Subject: [PATCH 26/59] relocate macros to preprocessor --- ColorzCore/EAInterpreter.cs | 2 +- ColorzCore/Parser/EAParser.cs | 1 - .../Directives/DefineDirective.cs | 2 +- .../MacroCollection.cs | 6 +++-- .../Macros/AddToPool.cs | 3 ++- .../Macros/BuiltInMacro.cs | 3 ++- .../Macros/ErrorMacro.cs | 5 ++--- .../{Parser => Preprocessor}/Macros/IMacro.cs | 3 ++- .../Macros/IsDefined.cs | 3 ++- .../Macros/IsSymbolDefined.cs | 4 ++-- .../Macros/ReadDataAt.cs | 3 ++- .../Macros/StringMacro.cs} | 6 ++--- .../Macros/UserMacro.cs} | 22 +++++++++---------- 13 files changed, 33 insertions(+), 30 deletions(-) rename ColorzCore/{Parser/Macros => Preprocessor}/MacroCollection.cs (92%) rename ColorzCore/{Parser => Preprocessor}/Macros/AddToPool.cs (96%) rename ColorzCore/{Parser => Preprocessor}/Macros/BuiltInMacro.cs (84%) rename ColorzCore/{Parser => Preprocessor}/Macros/ErrorMacro.cs (92%) rename ColorzCore/{Parser => Preprocessor}/Macros/IMacro.cs (83%) rename ColorzCore/{Parser => Preprocessor}/Macros/IsDefined.cs (96%) rename ColorzCore/{Parser => Preprocessor}/Macros/IsSymbolDefined.cs (96%) rename ColorzCore/{Parser => Preprocessor}/Macros/ReadDataAt.cs (97%) rename ColorzCore/{Parser/Macros/String.cs => Preprocessor/Macros/StringMacro.cs} (87%) rename ColorzCore/{Parser/Macros/Macro.cs => Preprocessor/Macros/UserMacro.cs} (61%) diff --git a/ColorzCore/EAInterpreter.cs b/ColorzCore/EAInterpreter.cs index 2c37ed2..c23490d 100644 --- a/ColorzCore/EAInterpreter.cs +++ b/ColorzCore/EAInterpreter.cs @@ -3,7 +3,7 @@ using ColorzCore.Lexer; using ColorzCore.Parser; using ColorzCore.Parser.AST; -using ColorzCore.Parser.Macros; +using ColorzCore.Preprocessor.Macros; using ColorzCore.Raws; using System; using System.Collections.Generic; diff --git a/ColorzCore/Parser/EAParser.cs b/ColorzCore/Parser/EAParser.cs index 921bd21..4f93626 100644 --- a/ColorzCore/Parser/EAParser.cs +++ b/ColorzCore/Parser/EAParser.cs @@ -2,7 +2,6 @@ using ColorzCore.IO; using ColorzCore.Lexer; using ColorzCore.Parser.AST; -using ColorzCore.Parser.Macros; using ColorzCore.Preprocessor; using ColorzCore.Raws; using System; diff --git a/ColorzCore/Preprocessor/Directives/DefineDirective.cs b/ColorzCore/Preprocessor/Directives/DefineDirective.cs index 7ffb7fa..9937769 100644 --- a/ColorzCore/Preprocessor/Directives/DefineDirective.cs +++ b/ColorzCore/Preprocessor/Directives/DefineDirective.cs @@ -5,7 +5,7 @@ using ColorzCore.Lexer; using ColorzCore.Parser; using ColorzCore.Parser.AST; -using ColorzCore.Parser.Macros; +using ColorzCore.Preprocessor.Macros; namespace ColorzCore.Preprocessor.Directives { diff --git a/ColorzCore/Parser/Macros/MacroCollection.cs b/ColorzCore/Preprocessor/MacroCollection.cs similarity index 92% rename from ColorzCore/Parser/Macros/MacroCollection.cs rename to ColorzCore/Preprocessor/MacroCollection.cs index 2293bf3..040565d 100644 --- a/ColorzCore/Parser/Macros/MacroCollection.cs +++ b/ColorzCore/Preprocessor/MacroCollection.cs @@ -1,8 +1,10 @@ using System; using System.Collections.Generic; using System.Linq; +using ColorzCore.Parser; +using ColorzCore.Preprocessor.Macros; -namespace ColorzCore.Parser.Macros +namespace ColorzCore.Preprocessor { public class MacroCollection { @@ -18,7 +20,7 @@ public MacroCollection(EAParser parent) BuiltInMacros = new Dictionary { - { "String", new String() }, + { "String", new StringMacro() }, { "IsDefined", new IsDefined(parent) }, { "IsSymbolDefined", new IsSymbolDefined() }, { "IsLabelDefined", new IsSymbolDefined() }, // alias diff --git a/ColorzCore/Parser/Macros/AddToPool.cs b/ColorzCore/Preprocessor/Macros/AddToPool.cs similarity index 96% rename from ColorzCore/Parser/Macros/AddToPool.cs rename to ColorzCore/Preprocessor/Macros/AddToPool.cs index 565b265..97285c2 100644 --- a/ColorzCore/Parser/Macros/AddToPool.cs +++ b/ColorzCore/Preprocessor/Macros/AddToPool.cs @@ -2,8 +2,9 @@ using System.Collections.Generic; using ColorzCore.DataTypes; using ColorzCore.Lexer; +using ColorzCore.Parser; -namespace ColorzCore.Parser.Macros +namespace ColorzCore.Preprocessor.Macros { class AddToPool : BuiltInMacro { diff --git a/ColorzCore/Parser/Macros/BuiltInMacro.cs b/ColorzCore/Preprocessor/Macros/BuiltInMacro.cs similarity index 84% rename from ColorzCore/Parser/Macros/BuiltInMacro.cs rename to ColorzCore/Preprocessor/Macros/BuiltInMacro.cs index 0c38a85..c4f0c16 100644 --- a/ColorzCore/Parser/Macros/BuiltInMacro.cs +++ b/ColorzCore/Preprocessor/Macros/BuiltInMacro.cs @@ -3,8 +3,9 @@ using System.Linq; using ColorzCore.DataTypes; using ColorzCore.Lexer; +using ColorzCore.Parser; -namespace ColorzCore.Parser.Macros +namespace ColorzCore.Preprocessor.Macros { public abstract class BuiltInMacro : IMacro { diff --git a/ColorzCore/Parser/Macros/ErrorMacro.cs b/ColorzCore/Preprocessor/Macros/ErrorMacro.cs similarity index 92% rename from ColorzCore/Parser/Macros/ErrorMacro.cs rename to ColorzCore/Preprocessor/Macros/ErrorMacro.cs index 42fb9f1..bfa9fda 100644 --- a/ColorzCore/Parser/Macros/ErrorMacro.cs +++ b/ColorzCore/Preprocessor/Macros/ErrorMacro.cs @@ -1,12 +1,11 @@ using ColorzCore.DataTypes; -using ColorzCore.IO; using ColorzCore.Lexer; -using ColorzCore.Parser.AST; +using ColorzCore.Parser; using System; using System.Collections.Generic; using System.Linq; -namespace ColorzCore.Parser.Macros +namespace ColorzCore.Preprocessor.Macros { class ErrorMacro : BuiltInMacro { diff --git a/ColorzCore/Parser/Macros/IMacro.cs b/ColorzCore/Preprocessor/Macros/IMacro.cs similarity index 83% rename from ColorzCore/Parser/Macros/IMacro.cs rename to ColorzCore/Preprocessor/Macros/IMacro.cs index 0cbd03f..a631559 100644 --- a/ColorzCore/Parser/Macros/IMacro.cs +++ b/ColorzCore/Preprocessor/Macros/IMacro.cs @@ -1,12 +1,13 @@ using ColorzCore.DataTypes; using ColorzCore.Lexer; +using ColorzCore.Parser; using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading.Tasks; -namespace ColorzCore.Parser.Macros +namespace ColorzCore.Preprocessor.Macros { public interface IMacro { diff --git a/ColorzCore/Parser/Macros/IsDefined.cs b/ColorzCore/Preprocessor/Macros/IsDefined.cs similarity index 96% rename from ColorzCore/Parser/Macros/IsDefined.cs rename to ColorzCore/Preprocessor/Macros/IsDefined.cs index c4ff891..f75c1ac 100644 --- a/ColorzCore/Parser/Macros/IsDefined.cs +++ b/ColorzCore/Preprocessor/Macros/IsDefined.cs @@ -2,8 +2,9 @@ using System.Collections.Generic; using ColorzCore.DataTypes; using ColorzCore.Lexer; +using ColorzCore.Parser; -namespace ColorzCore.Parser.Macros +namespace ColorzCore.Preprocessor.Macros { class IsDefined : BuiltInMacro { diff --git a/ColorzCore/Parser/Macros/IsSymbolDefined.cs b/ColorzCore/Preprocessor/Macros/IsSymbolDefined.cs similarity index 96% rename from ColorzCore/Parser/Macros/IsSymbolDefined.cs rename to ColorzCore/Preprocessor/Macros/IsSymbolDefined.cs index 13b9a38..bbb2949 100644 --- a/ColorzCore/Parser/Macros/IsSymbolDefined.cs +++ b/ColorzCore/Preprocessor/Macros/IsSymbolDefined.cs @@ -2,12 +2,12 @@ using System.Collections.Generic; using ColorzCore.DataTypes; using ColorzCore.Lexer; +using ColorzCore.Parser; -namespace ColorzCore.Parser.Macros +namespace ColorzCore.Preprocessor.Macros { public class IsSymbolDefined : BuiltInMacro { - public override IEnumerable ApplyMacro(Token head, IList> parameters, ImmutableStack scopes) { if (parameters[0].Count != 1) diff --git a/ColorzCore/Parser/Macros/ReadDataAt.cs b/ColorzCore/Preprocessor/Macros/ReadDataAt.cs similarity index 97% rename from ColorzCore/Parser/Macros/ReadDataAt.cs rename to ColorzCore/Preprocessor/Macros/ReadDataAt.cs index f5cea99..52b393a 100644 --- a/ColorzCore/Parser/Macros/ReadDataAt.cs +++ b/ColorzCore/Preprocessor/Macros/ReadDataAt.cs @@ -1,12 +1,13 @@ using ColorzCore.DataTypes; using ColorzCore.IO; using ColorzCore.Lexer; +using ColorzCore.Parser; using ColorzCore.Parser.AST; using System; using System.Collections.Generic; using System.Linq; -namespace ColorzCore.Parser.Macros +namespace ColorzCore.Preprocessor.Macros { public class ReadDataAt : BuiltInMacro { diff --git a/ColorzCore/Parser/Macros/String.cs b/ColorzCore/Preprocessor/Macros/StringMacro.cs similarity index 87% rename from ColorzCore/Parser/Macros/String.cs rename to ColorzCore/Preprocessor/Macros/StringMacro.cs index 4366673..4e88686 100644 --- a/ColorzCore/Parser/Macros/String.cs +++ b/ColorzCore/Preprocessor/Macros/StringMacro.cs @@ -1,14 +1,14 @@ using ColorzCore.DataTypes; using ColorzCore.Lexer; +using ColorzCore.Parser; using System; using System.Collections.Generic; using System.Linq; using System.Text; -using System.Threading.Tasks; -namespace ColorzCore.Parser.Macros +namespace ColorzCore.Preprocessor.Macros { - class String : BuiltInMacro + class StringMacro : BuiltInMacro { public override IEnumerable ApplyMacro(Token head, IList> parameters, ImmutableStack scopes) { diff --git a/ColorzCore/Parser/Macros/Macro.cs b/ColorzCore/Preprocessor/Macros/UserMacro.cs similarity index 61% rename from ColorzCore/Parser/Macros/Macro.cs rename to ColorzCore/Preprocessor/Macros/UserMacro.cs index 78581ae..d3fd36d 100644 --- a/ColorzCore/Parser/Macros/Macro.cs +++ b/ColorzCore/Preprocessor/Macros/UserMacro.cs @@ -1,23 +1,21 @@ using ColorzCore.DataTypes; using ColorzCore.Lexer; -using ColorzCore.Parser.AST; +using ColorzCore.Parser; using System; using System.Collections.Generic; using System.Linq; -using System.Text; -using System.Threading.Tasks; -namespace ColorzCore.Parser.Macros +namespace ColorzCore.Preprocessor.Macros { - class Macro : IMacro + class UserMacro : IMacro { - Dictionary idToParamNum; - IList body; + readonly Dictionary idToParamNum; + readonly IList body; - public Macro(IList parameters, IList body) + public UserMacro(IList parameters, IList body) { idToParamNum = new Dictionary(); - for(int i=0; i parameters, IList body) */ public IEnumerable ApplyMacro(Token head, IList> parameters, ImmutableStack scopes) { - foreach(Token t in body) + foreach (Token t in body) { - if(t.Type == TokenType.IDENTIFIER && idToParamNum.ContainsKey(t.Content)) + if (t.Type == TokenType.IDENTIFIER && idToParamNum.ContainsKey(t.Content)) { - foreach(Token t2 in parameters[idToParamNum[t.Content]]) + foreach (Token t2 in parameters[idToParamNum[t.Content]]) { yield return t2; } From 1fbbe8a765320a7a43f2274702bcf8ae4f806eb0 Mon Sep 17 00:00:00 2001 From: Nat_776 Date: Fri, 26 Apr 2024 18:16:57 +0200 Subject: [PATCH 27/59] fix define --- ColorzCore/Preprocessor/Directives/DefineDirective.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ColorzCore/Preprocessor/Directives/DefineDirective.cs b/ColorzCore/Preprocessor/Directives/DefineDirective.cs index 9937769..3e14593 100644 --- a/ColorzCore/Preprocessor/Directives/DefineDirective.cs +++ b/ColorzCore/Preprocessor/Directives/DefineDirective.cs @@ -54,7 +54,7 @@ class DefineDirective : SimpleDirective toRepl = ExpandParam(p, parameters[1], myParams.Select((Token t) => t.Content)); if (toRepl != null) { - p.Macros.AddMacro(new Macro(myParams, toRepl), name, myParams.Count); + p.Macros.AddMacro(new UserMacro(myParams, toRepl), name, myParams.Count); } } else From c8e0bdab712336fe454bd862f3dd6e556645e2b3 Mon Sep 17 00:00:00 2001 From: Nat_776 Date: Fri, 26 Apr 2024 19:47:08 +0200 Subject: [PATCH 28/59] refactor define (allow any token sequence) + non-productive definitions --- ColorzCore/Parser/Definition.cs | 20 +- ColorzCore/Parser/EAParser.cs | 42 +++- .../Directives/DefineDirective.cs | 217 +++++++----------- ColorzCore/Preprocessor/Macros/UserMacro.cs | 12 +- 4 files changed, 147 insertions(+), 144 deletions(-) diff --git a/ColorzCore/Parser/Definition.cs b/ColorzCore/Parser/Definition.cs index 2658b6f..c6ec58f 100644 --- a/ColorzCore/Parser/Definition.cs +++ b/ColorzCore/Parser/Definition.cs @@ -10,11 +10,17 @@ namespace ColorzCore.Parser { public class Definition { - private IList replacement; + private readonly IList? replacement; + + /// + /// A non-productive definition is a definition that doesn't participate in macro expansion. + /// (it is still visible by ifdef and other such constructs). + /// + public bool NonProductive => replacement == null; public Definition() { - replacement = new List(); + replacement = null; } public Definition(IList defn) @@ -22,11 +28,15 @@ public Definition(IList defn) replacement = defn; } - public IEnumerable ApplyDefinition(Token toReplace) + public IEnumerable ApplyDefinition(Token token) { - for(int i=0; i replacement = this.replacement!; + + for (int i = 0; i < replacement.Count; i++) { - Location newLoc = new Location(toReplace.FileName, toReplace.LineNumber, toReplace.ColumnNumber); + Location newLoc = new Location(token.FileName, token.LineNumber, token.ColumnNumber); yield return new Token(replacement[i].Type, newLoc, replacement[i].Content); } } diff --git a/ColorzCore/Parser/EAParser.cs b/ColorzCore/Parser/EAParser.cs index 4f93626..db9bc33 100644 --- a/ColorzCore/Parser/EAParser.cs +++ b/ColorzCore/Parser/EAParser.cs @@ -31,7 +31,7 @@ public int CurrentOffset get { return currentOffset; } private set { - if (value > EAOptions.Instance.maximumRomSize) + if (value < 0 || value > EAOptions.Instance.maximumRomSize) { if (validOffset) //Error only the first time. { @@ -446,7 +446,17 @@ static bool IsConditionalOperatorHelper(IAtomNode node) parameters[0].AsAtom().IfJust( atom => atom.TryEvaluate(e => Error(parameters[0].MyLocation, e.Message), EvaluationPhase.Immediate).IfJust( - temp => CurrentOffset = CurrentOffset % temp != 0 ? CurrentOffset + temp - CurrentOffset % temp : CurrentOffset), + temp => + { + if (temp <= 0) + { + Error($"Cannot align address to {temp}"); + } + else if (CurrentOffset % temp != 0) + { + CurrentOffset += temp - CurrentOffset % temp; + } + }), () => Error(parameters[0].MyLocation, "Expected atomic param to ALIGN")); return null; @@ -1090,11 +1100,11 @@ public bool ExpandIdentifier(MergeableGenerator tokens, ImmutableStack tokens) } } - private void IgnoreRestOfLine(MergeableGenerator tokens) + public void IgnoreRestOfLine(MergeableGenerator tokens) { while (tokens.Current.Type != TokenType.NEWLINE && tokens.MoveNext()) { } } + /// + /// Consumes incoming tokens util end of line. + /// + /// token stream + /// If non-null, will expand any macros as they are encountered using this scope + /// The resulting list of tokens + public IList GetRestOfLine(MergeableGenerator tokens, ImmutableStack? scopesForMacros) + { + IList result = new List(); + + while (tokens.Current.Type != TokenType.NEWLINE) + { + if (scopesForMacros == null || !ExpandIdentifier(tokens, scopesForMacros)) + { + result.Add(tokens.Current); + tokens.MoveNext(); + } + } + + return result; + } + public void Clear() { Macros.Clear(); diff --git a/ColorzCore/Preprocessor/Directives/DefineDirective.cs b/ColorzCore/Preprocessor/Directives/DefineDirective.cs index 3e14593..a92cfcc 100644 --- a/ColorzCore/Preprocessor/Directives/DefineDirective.cs +++ b/ColorzCore/Preprocessor/Directives/DefineDirective.cs @@ -9,162 +9,121 @@ namespace ColorzCore.Preprocessor.Directives { - class DefineDirective : SimpleDirective + class DefineDirective : IDirective { - public override int MinParams => 1; + public bool RequireInclusion => true; - public override int? MaxParams => 2; + public ILineNode? Execute(EAParser p, Token self, MergeableGenerator tokens, ImmutableStack scopes) + { + Token nextToken = tokens.Current; + IList? parameters; + + switch (nextToken.Type) + { + case TokenType.IDENTIFIER: + tokens.MoveNext(); + parameters = null; + break; - public override bool RequireInclusion => true; + case TokenType.MAYBE_MACRO: + tokens.MoveNext(); + parameters = FlattenParameters(p, p.ParseMacroParamList(tokens)); + break; - public override ILineNode? Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) - { - if (parameters[0] is MacroInvocationNode signature) + case TokenType.NEWLINE: + p.Error(self.Location, "Invalid use of directive '#define': missing macro name."); + return null; + + default: + p.Error(self.Location, $"Invalid use of directive '#define': expected macro name, got {nextToken}"); + p.IgnoreRestOfLine(tokens); + return null; + } + + IList? macroBody = ExpandMacroBody(p, p.GetRestOfLine(tokens, scopes)); + + if (parameters != null) { - string name = signature.Name; - IList myParams = new List(); - foreach (IList l1 in signature.Parameters) - { - if (l1.Count != 1 || l1[0].Type != TokenType.IDENTIFIER) - { - p.Error(l1[0].Location, $"Macro parameters must be identifiers (got {l1[0].Content})."); - } - else - { - myParams.Add(l1[0]); - } - } - /* if (!p.IsValidMacroName(name, myParams.Count)) - { - if (p.IsReservedName(name)) - { - p.Error(signature.MyLocation, "Invalid redefinition: " + name); - } - else - p.Warning(signature.MyLocation, "Redefining " + name + '.'); - }*/ - if (p.Macros.HasMacro(name, myParams.Count)) - p.Warning(signature.MyLocation, $"Redefining {name}."); - IList? toRepl; - if (parameters.Count != 2) - { - toRepl = new List(); - } - else - toRepl = ExpandParam(p, parameters[1], myParams.Select((Token t) => t.Content)); - if (toRepl != null) - { - p.Macros.AddMacro(new UserMacro(myParams, toRepl), name, myParams.Count); - } + // function-like macro + DefineFunctionMacro(p, nextToken, parameters, macroBody); } else { - //Note [mutually] recursive definitions are handled by Parser expansion. - string? name; - if (parameters[0].Type == ParamType.ATOM && (name = ((IAtomNode)parameters[0]).GetIdentifier()) != null) - { - if (p.Definitions.ContainsKey(name)) - p.Warning(parameters[0].MyLocation, "Redefining " + name + '.'); - if (parameters.Count == 2) - { - IList? toRepl = ExpandParam(p, parameters[1], Enumerable.Empty()); - if (toRepl != null) - { - p.Definitions[name] = new Definition(toRepl); - } - } - else - { - p.Definitions[name] = new Definition(); - } - } - else - { - p.Error(parameters[0].MyLocation, $"Definition names must be identifiers (got {parameters[0].ToString()})."); - } + // object-like macro + DefineObjectMacro(p, nextToken, macroBody); } + return null; } - private static IList? ExpandParam(EAParser p, IParamNode param, IEnumerable myParams) + private static void DefineObjectMacro(EAParser p, Token nameToken, IList macroBody) { - return TokenizeParam(p, param).Fmap(tokens => ExpandAllIdentifiers(p, - new Queue(tokens), ImmutableStack.FromEnumerable(myParams), - ImmutableStack>.Nil)).Fmap(x => new List(x)); + string name = nameToken.Content; + + if (p.Definitions.ContainsKey(name)) + { + // TODO: stricter error (opt-in?) + p.Warning(nameToken.Location, $"Redefining {name}."); + } + + if (macroBody.Count == 1 && macroBody[0].Type == TokenType.IDENTIFIER && macroBody[0].Content == name) + { + /* an object-like macro whose only inner token is a reference to itself is non-productive + * this is: it doesn't participate in macro expansion. */ + + p.Definitions[name] = new Definition(); + } + else + { + p.Definitions[name] = new Definition(macroBody); + } } - private static IList? TokenizeParam(EAParser p, IParamNode param) + private static void DefineFunctionMacro(EAParser p, Token nameToken, IList parameters, IList macroBody) { - switch (param.Type) + string name = nameToken.Content; + + if (p.Macros.HasMacro(name, parameters.Count)) { - case ParamType.STRING: - Token input = ((StringNode)param).MyToken; - Tokenizer t = new Tokenizer(); - return new List(t.TokenizeLine(input.Content, input.FileName, input.LineNumber, input.ColumnNumber)); - case ParamType.MACRO: - try - { - IList myBody = new List(((MacroInvocationNode)param).ExpandMacro()); - return myBody; - } - catch (KeyNotFoundException) - { - MacroInvocationNode asMacro = (MacroInvocationNode)param; - p.Error(asMacro.MyLocation, "Undefined macro: " + asMacro.Name); - } - break; - case ParamType.LIST: - ListNode n = (ListNode)param; - return new List(n.ToTokens()); - case ParamType.ATOM: - return new List(((IAtomNode)param).ToTokens()); + // TODO: stricter error (opt-in?) + p.Warning(nameToken.Location, $"Redefining {name}(...) with {parameters.Count} parameters."); } - return null; + + p.Macros.AddMacro(new UserMacro(parameters, macroBody), name, parameters.Count); } - private static IEnumerable ExpandAllIdentifiers(EAParser p, Queue tokens, ImmutableStack seenDefs, ImmutableStack> seenMacros) + + private static IList FlattenParameters(EAParser p, IList> rawParameters) { - while (tokens.Count > 0) + IList result = new List(); + + foreach (IList parameter in rawParameters) { - Token current = tokens.Dequeue(); - if (current.Type == TokenType.IDENTIFIER) + if (parameter.Count != 1 || parameter[0].Type != TokenType.IDENTIFIER) { - if (p.Macros.ContainsName(current.Content) && tokens.Count > 0 && tokens.Peek().Type == TokenType.OPEN_PAREN) - { - IList> param = p.ParseMacroParamList(new MergeableGenerator(tokens)); //TODO: I don't like wrapping this in a mergeable generator..... Maybe interface the original better? - if (!seenMacros.Contains(new Tuple(current.Content, param.Count)) && p.Macros.HasMacro(current.Content, param.Count)) - { - foreach (Token t in p.Macros.GetMacro(current.Content, param.Count).ApplyMacro(current, param, p.GlobalScope)) - { - yield return t; - } - } - else if (seenMacros.Contains(new Tuple(current.Content, param.Count))) - { - yield return current; - foreach (IList l in param) - foreach (Token t in l) - yield return t; - } - else - { - yield return current; - } - } - else if (!seenDefs.Contains(current.Content) && p.Definitions.ContainsKey(current.Content)) - { - foreach (Token t in p.Definitions[current.Content].ApplyDefinition(current)) - yield return t; - } - else - { - yield return current; - } + p.Error(parameter[0].Location, $"Macro parameters must be single identifiers (got {parameter[0].Content})."); + result.Add($"${result.Count}"); } else { - yield return current; + result.Add(parameter[0].Content); } } + + return result; + } + + private static IList ExpandMacroBody(EAParser _, IList body) + { + if (body.Count == 1 && body[0].Type == TokenType.STRING) + { + Token token = body[0]; + + // TODO: does this need to be column number + 1? + return new List(new Tokenizer().TokenizeLine( + token.Content, token.FileName, token.LineNumber, token.ColumnNumber)); + } + + return body; } } } diff --git a/ColorzCore/Preprocessor/Macros/UserMacro.cs b/ColorzCore/Preprocessor/Macros/UserMacro.cs index d3fd36d..7bdd190 100644 --- a/ColorzCore/Preprocessor/Macros/UserMacro.cs +++ b/ColorzCore/Preprocessor/Macros/UserMacro.cs @@ -12,14 +12,16 @@ class UserMacro : IMacro readonly Dictionary idToParamNum; readonly IList body; - public UserMacro(IList parameters, IList body) + public UserMacro(IList parameters, IList macroBody) { idToParamNum = new Dictionary(); + for (int i = 0; i < parameters.Count; i++) { - idToParamNum[parameters[i].Content] = i; + idToParamNum[parameters[i]] = i; } - this.body = body; + + body = macroBody; } /*** @@ -29,9 +31,9 @@ public IEnumerable ApplyMacro(Token head, IList> parameters, { foreach (Token t in body) { - if (t.Type == TokenType.IDENTIFIER && idToParamNum.ContainsKey(t.Content)) + if (t.Type == TokenType.IDENTIFIER && idToParamNum.TryGetValue(t.Content, out int paramNum)) { - foreach (Token t2 in parameters[idToParamNum[t.Content]]) + foreach (Token t2 in parameters[paramNum]) { yield return t2; } From 95e2dd58e3e30858cb4147fb7209f8c1bf10d6bd Mon Sep 17 00:00:00 2001 From: Nat_776 Date: Fri, 26 Apr 2024 19:47:41 +0200 Subject: [PATCH 29/59] add very basic and bad and incomplete testing script --- Tests/README.md | 1 + Tests/run_tests.py | 186 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 187 insertions(+) create mode 100644 Tests/README.md create mode 100644 Tests/run_tests.py diff --git a/Tests/README.md b/Tests/README.md new file mode 100644 index 0000000..e3bca9c --- /dev/null +++ b/Tests/README.md @@ -0,0 +1 @@ +Run using `python run_tests.py path/to/ColorzCore.exe`. diff --git a/Tests/run_tests.py b/Tests/run_tests.py new file mode 100644 index 0000000..9bfebfa --- /dev/null +++ b/Tests/run_tests.py @@ -0,0 +1,186 @@ +import sys, os +import subprocess, tempfile + + +class Config: + command : list[str] + extra_params : list[str] + + def __init__(self, command : str, extra_params : str | None) -> None: + self.command = command.split() + self.extra_params = extra_params.split() if extra_params is not None else [] + + +class Test: + name : str + script : str + expected : bytes | None + + def __init__(self, name : str, script : str, expected : bytes | None) -> None: + self.name = name + self.script = script + self.expected = expected + + def run_test(self, config : Config) -> bool: + success = False + + with tempfile.NamedTemporaryFile(delete = False) as f: + f.close() + + completed = subprocess.run(config.command + ["A", "FE6", f"-output:{f.name}"] + config.extra_params, + text = True, input = self.script, stdout = subprocess.DEVNULL, stderr = subprocess.DEVNULL) + + if self.expected is None: + # success on error + success = completed.returncode != 0 + + else: + # success on resulting bytes matching + with open(f.name, 'rb') as f2: + result_bytes = f2.read() + + success = result_bytes == self.expected + + os.remove(f.name) + + return success + + +class bcolors: + HEADER = '\033[95m' + OKBLUE = '\033[94m' + OKCYAN = '\033[96m' + OKGREEN = '\033[92m' + WARNING = '\033[93m' + FAIL = '\033[91m' + ENDC = '\033[0m' + BOLD = '\033[1m' + UNDERLINE = '\033[4m' + + +def run_tests(config : Config, test_cases : list[Test]) -> None: + success_count = 0 + test_count = len(test_cases) + + success_message = f"{bcolors.OKBLUE}SUCCESS{bcolors.ENDC}" + failure_message = f"{bcolors.FAIL}FAILURE{bcolors.ENDC}" + + for i, test_case in enumerate(test_cases): + success = test_case.run_test(config) + + message = success_message if success else failure_message + print(f"[{i + 1}/{test_count}] {test_case.name}: {message}") + + if success: + success_count = success_count + 1 + + if success_count == test_count: + print(f"{success_count}/{test_count} tests passed {success_message}") + + else: + print(f"{success_count}/{test_count} tests passed {failure_message}") + + +BASIC_TESTS = [ + Test("Basic", "ORG 0 ; BYTE 1", b"\x01"), + Test("Addition", "ORG 0 ; BYTE 1 + 2", b"\x03"), + Test("Precedence 1", "ORG 0 ; BYTE 1 + 2 * 10", b"\x15"), + + # POIN + Test("POIN 1", "ORG 0 ; POIN 4", b"\x04\x00\x00\x08"), + Test("POIN 2", "ORG 0 ; POIN 0", b"\x00\x00\x00\x00"), + Test("POIN 3", "ORG 0 ; POIN 0x08000000", b"\x00\x00\x00\x08"), + Test("POIN 4", "ORG 0 ; POIN 0x02000000", b"\x00\x00\x00\x02"), + + # ORG + Test("ORG 1", "ORG 1 ; BYTE 1 ; ORG 10 ; BYTE 10", b"\x00\x01" + b"\x00" * 8 + b"\x0A"), + Test("ORG 2", "ORG 0x08000001 ; BYTE 1 ; ORG 0x0800000A ; BYTE 10", b"\x00\x01" + b"\x00" * 8 + b"\x0A"), + Test("ORG 3", "ORG 0x10000000 ; BYTE 1", None), + Test("ORG 4", "ORG -1 ; BYTE 1", None), + + # ALIGN + Test("ALIGN 1", "ORG 1 ; ALIGN 4 ; WORD CURRENTOFFSET", b"\x00\x00\x00\x00\x04\x00\x00\x00"), + Test("ALIGN 2", "ORG 4 ; ALIGN 4 ; WORD CURRENTOFFSET", b"\x00\x00\x00\x00\x04\x00\x00\x00"), + Test("ALIGN 3", "ORG 1 ; ALIGN 0 ; WORD CURRENTOFFSET", None), + Test("ALIGN 4", "ORG 1 ; ALIGN -1 ; WORD CURRENTOFFSET", None), + + # FILL + Test("FILL 1", "ORG 0 ; FILL 0x10", b"\x00" * 0x10), + Test("FILL 2", "ORG 4 ; FILL 0x10 0xFF", b"\x00\x00\x00\x00" + b"\xFF" * 0x10), + + # ASSERT + Test("ASSERT 1", "ASSERT 0", b""), + Test("ASSERT 2", "ASSERT -1", None), + Test("ASSERT 3", "ASSERT 1 < 0", None), + Test("ASSERT 4", "ASSERT 1 - 2", None) +] + + +EXPRESSION_TESTS = [ + Test("UNDCOERCE 1", 'A := 0 ; ORG 0 ; BYTE (A || 1) ?? 0', b"\x01"), + Test("UNDCOERCE 2", 'ORG 0 ; BYTE (A || 1) ?? 0', b"\x00"), +] + + +PREPROC_TESTS = [ + # '#define' traditional nominal behavior + Test("Define 1", '#define Value 0xFA \n ORG 0 ; BYTE Value', b"\xFA"), + Test("Define 2", '#define Macro(a) "0xFA + (a)" \n ORG 0 ; BYTE Macro(2)', b"\xFC"), + Test("Define 3", '#define Value \n #ifdef Value \n ORG 0 ; BYTE 1 \n #endif', b"\x01"), + + # '#define' a second time overrides the first definition + Test("Define override", '#define Value 1 \n #define Value 2 \n ORG 0 ; BYTE Value', b"\x02"), + + # '#define' using a vector as argument (extra commas) + Test("Define vector argument", '#define Macro(a) "BYTE 1" \n ORG 0 ; Macro([1, 2, 3])', b"\x01"), + + # '#define ... "..."' with escaped newlines inside string + Test("Multi-line string define", '#define SomeLongMacro(A, B, C) "\\\n ALIGN 4 ; \\\n WORD C ; \\\n SHORT B ; \\\n BYTE A" \n ORG 0 ; SomeLongMacro(0xAA, 0xBB, 0xCC)', b"\xCC\x00\x00\x00\xBB\x00\xAA"), + + # '#define ...' multi-token without quotes + Test("Multi-token define 1", '#define Value (1 + 2) \n ORG 0 ; BYTE Value', b"\x03"), + Test("Multi-token define 2", '#define Macro(a, b) (a + b) \n ORG 0 ; BYTE Macro(1, 2)', b"\x03"), + Test("Multi-token define 2", '#define Macro(a, b) BYTE a a + b b \n ORG 0 ; Macro(1, 2)', b"\x01\x03\x02"), + + # '#ifdef' + Test("Ifdef", 'ORG 0 \n #define Value \n #ifdef Value \n BYTE 1 \n #else \n BYTE 0 \n #endif', b"\x01"), + + # '#ifndef' + Test("Ifndef", 'ORG 0 \n #define Value \n #ifndef Value \n BYTE 1 \n #else \n BYTE 0 \n #endif', b"\x00"), + + # '#define MyMacro MyMacro' (MyMacro shouldn't expand) + Test("Non-productive macros 1", '#define MyMacro MyMacro \n ORG 0 ; MyMacro: ; BYTE 1', b'\x01'), + Test("Non-productive macros 2", '#define MyMacro MyMacro \n ORG 0 ; BYTE IsDefined(MyMacro)', b'\x01'), + Test("Non-productive macros 3", '#define MyMacro MyMacro \n ORG 0 ; #ifdef MyMacro \n BYTE 1 \n #else \n BYTE 0 \n #endif', b'\x01'), + + # Test("IFDEF 2", 'ORG 0 \n #define A \n #define B \n #ifdef A B \n BYTE 1 \n #else \n BYTE 0 \n #endif', b"\x01"), + + # '#undef' + Test("Undef 1", '#define Value 1 \n #undef Value \n ORG 0 ; BYTE Value', None), + Test("Undef 2", '#define Value 1 \n #undef Value \n #ifndef Value \n ORG 0 ; BYTE 1 \n #endif', b"\x01"), +] + + +ALL_TEST_CASES = BASIC_TESTS + EXPRESSION_TESTS + PREPROC_TESTS + +def main(args): + import argparse + + arg_parse = argparse.ArgumentParser() + + arg_parse.add_argument("command") + arg_parse.add_argument("--extra-params") + + args = arg_parse.parse_args(args[1:]) + + command : str = args.command + extra_params : str = args.extra_params + + test_cases = ALL_TEST_CASES + + config = Config(command, extra_params) + run_tests(config, test_cases) + + +if __name__ == '__main__': + sys.exit(main(sys.argv)) From f4b7370d668d736cff087eb0b6bb2dd0742f4a61 Mon Sep 17 00:00:00 2001 From: Nat_776 Date: Fri, 26 Apr 2024 22:03:11 +0200 Subject: [PATCH 30/59] move Definition to Preprocessor namespace --- ColorzCore/EAInterpreter.cs | 5 +++-- ColorzCore/{Parser => Preprocessor}/Definition.cs | 4 +--- 2 files changed, 4 insertions(+), 5 deletions(-) rename ColorzCore/{Parser => Preprocessor}/Definition.cs (90%) diff --git a/ColorzCore/EAInterpreter.cs b/ColorzCore/EAInterpreter.cs index c23490d..9934429 100644 --- a/ColorzCore/EAInterpreter.cs +++ b/ColorzCore/EAInterpreter.cs @@ -3,6 +3,7 @@ using ColorzCore.Lexer; using ColorzCore.Parser; using ColorzCore.Parser.AST; +using ColorzCore.Preprocessor; using ColorzCore.Preprocessor.Macros; using ColorzCore.Raws; using System; @@ -91,9 +92,9 @@ public bool Interpret() ExecTimer.Timer.AddTimingPoint(ExecTimer.KEY_GENERIC); - foreach (Tuple defpair in EAOptions.Instance.defs) + foreach ((string name, string body) in EAOptions.Instance.defs) { - myParser.ParseAll(t.TokenizeLine("#define " + defpair.Item1 + " " + defpair.Item2, "cmd", 0)); + myParser.ParseAll(t.TokenizeLine($"#define {name} {body}", "cmd", 0)); } IList lines = new List(myParser.ParseAll(t.Tokenize(sin, iFile))); diff --git a/ColorzCore/Parser/Definition.cs b/ColorzCore/Preprocessor/Definition.cs similarity index 90% rename from ColorzCore/Parser/Definition.cs rename to ColorzCore/Preprocessor/Definition.cs index c6ec58f..0be8705 100644 --- a/ColorzCore/Parser/Definition.cs +++ b/ColorzCore/Preprocessor/Definition.cs @@ -3,10 +3,8 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Text; -using System.Threading.Tasks; -namespace ColorzCore.Parser +namespace ColorzCore.Preprocessor { public class Definition { From b74b504b3f513e6e5ee73b6c623c9cca720a6e47 Mon Sep 17 00:00:00 2001 From: Nat_776 Date: Fri, 26 Apr 2024 22:03:39 +0200 Subject: [PATCH 31/59] Add rudimentary __LINE__ support --- ColorzCore/Parser/EAParser.cs | 63 +++++++++++++++++------------------ 1 file changed, 30 insertions(+), 33 deletions(-) diff --git a/ColorzCore/Parser/EAParser.cs b/ColorzCore/Parser/EAParser.cs index db9bc33..4662b1e 100644 --- a/ColorzCore/Parser/EAParser.cs +++ b/ColorzCore/Parser/EAParser.cs @@ -803,40 +803,37 @@ public IList ParsePreprocParamList(MergeableGenerator tokens, if (shift) { - if (lookAhead.Type == TokenType.IDENTIFIER) + switch (lookAhead.Type) { - if (expandDefs && ExpandIdentifier(tokens, scopes)) - { - continue; - } + case TokenType.IDENTIFIER: + if (expandDefs && ExpandIdentifier(tokens, scopes)) + { + continue; + } - if (lookAhead.Content.ToUpper() == "CURRENTOFFSET") - { - grammarSymbols.Push(new Left(new NumberNode(lookAhead, CurrentOffset))); - } - else - { - grammarSymbols.Push(new Left(new IdentifierNode(lookAhead, scopes))); - } - } - else if (lookAhead.Type == TokenType.MAYBE_MACRO) - { - ExpandIdentifier(tokens, scopes); - continue; - } - else if (lookAhead.Type == TokenType.NUMBER) - { - grammarSymbols.Push(new Left(new NumberNode(lookAhead))); - } - else if (lookAhead.Type == TokenType.ERROR) - { - Error(lookAhead.Location, $"Unexpected token: {lookAhead.Content}"); - tokens.MoveNext(); - return null; - } - else - { - grammarSymbols.Push(new Right(lookAhead)); + grammarSymbols.Push(new Left(lookAhead.Content.ToUpperInvariant() switch + { + "CURRENTOFFSET" => new NumberNode(lookAhead, CurrentOffset), + // TODO: __LINE__ within macro expansion should expand to line of caller + "__LINE__" => new NumberNode(lookAhead, lookAhead.Location.lineNum), + _ => new IdentifierNode(lookAhead, scopes), + })); + + break; + + case TokenType.MAYBE_MACRO: + ExpandIdentifier(tokens, scopes); + continue; + case TokenType.NUMBER: + grammarSymbols.Push(new Left(new NumberNode(lookAhead))); + break; + case TokenType.ERROR: + Error(lookAhead.Location, $"Unexpected token: {lookAhead.Content}"); + tokens.MoveNext(); + return null; + default: + grammarSymbols.Push(new Right(lookAhead)); + break; } tokens.MoveNext(); continue; @@ -1182,7 +1179,7 @@ private string PrettyPrintParamsForMessage(IList parameters, Immutab { return string.Join(" ", parameters.Select(parameter => parameter switch { - StringNode node => ExpandUserFormatString(scopes, parameter.MyLocation.OffsetBy(1), node.Value), + StringNode node => ExpandUserFormatString(scopes, parameter.MyLocation, node.Value), _ => parameter.PrettyPrint(), })); } From 58f6f745ce3ffc8208a0bd157b7c485bc76046e6 Mon Sep 17 00:00:00 2001 From: Nat_776 Date: Fri, 26 Apr 2024 23:11:37 +0200 Subject: [PATCH 32/59] Remember macro expansion location chain + working __FILE__ --- ColorzCore/DataTypes/Location.cs | 15 +++--- ColorzCore/DataTypes/MacroLocation.cs | 14 ++++++ ColorzCore/EAInterpreter.cs | 4 +- ColorzCore/Lexer/Token.cs | 35 ++++++++----- ColorzCore/Lexer/Tokenizer.cs | 7 +++ ColorzCore/Parser/AST/ListNode.cs | 6 +-- ColorzCore/Parser/EAParser.cs | 54 ++++++++++++--------- ColorzCore/Preprocessor/Definition.cs | 4 +- ColorzCore/Preprocessor/Macros/UserMacro.cs | 12 +++-- 9 files changed, 98 insertions(+), 53 deletions(-) create mode 100644 ColorzCore/DataTypes/MacroLocation.cs diff --git a/ColorzCore/DataTypes/Location.cs b/ColorzCore/DataTypes/Location.cs index 554dfbc..4ea0e52 100644 --- a/ColorzCore/DataTypes/Location.cs +++ b/ColorzCore/DataTypes/Location.cs @@ -9,17 +9,20 @@ namespace ColorzCore.DataTypes public struct Location { public string file; - public int lineNum, colNum; + public int line, column; + public MacroLocation? macroLocation; - public Location(string fileName, int lineNum, int colNum) : this() + public Location(string fileName, int lineNum, int colNum, MacroLocation? macro = null) : this() { file = fileName; - this.lineNum = lineNum; - this.colNum = colNum; + line = lineNum; + column = colNum; + macroLocation = macro; } - public readonly Location OffsetBy(int columns) => new Location(file, lineNum, colNum + columns); + public readonly Location OffsetBy(int columns) => new Location(file, line, column + columns, macroLocation); + public readonly Location MacroClone(MacroLocation macro) => new Location(file, line, column, macro); - public override readonly string ToString() => $"{file}:{lineNum}:{colNum}"; + public override readonly string ToString() => $"{file}:{line}:{column}"; } } diff --git a/ColorzCore/DataTypes/MacroLocation.cs b/ColorzCore/DataTypes/MacroLocation.cs new file mode 100644 index 0000000..fd50152 --- /dev/null +++ b/ColorzCore/DataTypes/MacroLocation.cs @@ -0,0 +1,14 @@ +namespace ColorzCore.DataTypes +{ + public class MacroLocation + { + public string MacroName { get; } + public Location Location { get; } + + public MacroLocation(string macroName, Location location) + { + MacroName = macroName; + Location = location; + } + } +} diff --git a/ColorzCore/EAInterpreter.cs b/ColorzCore/EAInterpreter.cs index 9934429..885081e 100644 --- a/ColorzCore/EAInterpreter.cs +++ b/ColorzCore/EAInterpreter.cs @@ -37,8 +37,8 @@ public EAInterpreter(IOutput output, string game, string? rawsFolder, string raw Location loc = new Location { file = e.FileName, - lineNum = e.LineNumber, - colNum = 1 + line = e.LineNumber, + column = 1 }; log.Message(Log.MessageKind.ERROR, loc, "An error occured while parsing raws"); diff --git a/ColorzCore/Lexer/Token.cs b/ColorzCore/Lexer/Token.cs index ba40c15..2d96884 100644 --- a/ColorzCore/Lexer/Token.cs +++ b/ColorzCore/Lexer/Token.cs @@ -1,39 +1,48 @@ using ColorzCore.DataTypes; using System; using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; namespace ColorzCore.Lexer { public class Token { - public Location Location { get; } - public TokenType Type { get; } - public string FileName { get { return Location.file; } } - public int LineNumber { get { return Location.lineNum; } } - public int ColumnNumber { get { return Location.colNum; } } public string Content { get; } + public Location Location { get; } + + public string FileName => Location.file; + public int LineNumber => Location.line; + public int ColumnNumber => Location.column; public Token(TokenType type, string fileName, int lineNum, int colNum, string original = "") { Type = type; - Location = new Location(fileName, lineNum, colNum+1); + Location = new Location(fileName, lineNum, colNum + 1); Content = original; } - public Token(TokenType type, Location newLoc, string content) + public Token(TokenType type, Location location, string content) { Type = type; - this.Location = newLoc; + Location = location; Content = content; } - public override string ToString() + public override string ToString() => $"{Location}, {Type}: {Content}"; + + public Token MacroClone(MacroLocation macroLocation) => new Token(Type, Location.MacroClone(macroLocation), Content); + + // used for __LINE__ and __FILE__ + public Location GetSourceLocation() { - return String.Format("File {4}, Line {0}, Column {1}, {2}: {3}", LineNumber, ColumnNumber, Type, Content, FileName); + Location location = Location; + + while (location.macroLocation != null) + { + location = location.macroLocation.Location; + } + + return location; } } } diff --git a/ColorzCore/Lexer/Tokenizer.cs b/ColorzCore/Lexer/Tokenizer.cs index a55b58c..79663f7 100644 --- a/ColorzCore/Lexer/Tokenizer.cs +++ b/ColorzCore/Lexer/Tokenizer.cs @@ -5,6 +5,7 @@ using System.Text; using System.Text.RegularExpressions; using System.Threading.Tasks; +using ColorzCore.DataTypes; using ColorzCore.IO; namespace ColorzCore.Lexer @@ -363,11 +364,17 @@ public IEnumerable TokenizePhrase(string line, string fileName, int lineN afterWhitespace = false; } } + public IEnumerable TokenizeLine(string line, string fileName, int lineNum, int offset = 0) { return TokenizePhrase(line, fileName, lineNum, 0, line.Length, offset); } + public IEnumerable TokenizeLine(string line, Location location) + { + return TokenizeLine(line, location.file, location.line, location.column); + } + /*** * All Token streams end in a NEWLINE. * diff --git a/ColorzCore/Parser/AST/ListNode.cs b/ColorzCore/Parser/AST/ListNode.cs index 4ce011f..41d3cf1 100644 --- a/ColorzCore/Parser/AST/ListNode.cs +++ b/ColorzCore/Parser/AST/ListNode.cs @@ -58,7 +58,7 @@ public IEnumerable ToTokens() } Location myStart = MyLocation; Location myEnd = temp.Count > 0 ? temp.Last().Last().Location : MyLocation; - yield return new Token(TokenType.OPEN_BRACKET, new Location(myStart.file, myStart.lineNum, myStart.colNum - 1), "["); + yield return new Token(TokenType.OPEN_BRACKET, new Location(myStart.file, myStart.line, myStart.column - 1), "["); for (int i = 0; i < temp.Count; i++) { foreach (Token t in temp[i]) @@ -68,10 +68,10 @@ public IEnumerable ToTokens() if (i < temp.Count - 1) { Location tempEnd = temp[i].Last().Location; - yield return new Token(TokenType.COMMA, new Location(tempEnd.file, tempEnd.lineNum, tempEnd.colNum + temp[i].Last().Content.Length), ","); + yield return new Token(TokenType.COMMA, new Location(tempEnd.file, tempEnd.line, tempEnd.column + temp[i].Last().Content.Length), ","); } } - yield return new Token(TokenType.CLOSE_BRACKET, new Location(myEnd.file, myEnd.lineNum, myEnd.colNum + (temp.Count > 0 ? temp.Last().Last().Content.Length : 1)), "]"); + yield return new Token(TokenType.CLOSE_BRACKET, new Location(myEnd.file, myEnd.line, myEnd.column + (temp.Count > 0 ? temp.Last().Last().Content.Length : 1)), "]"); } public Either TryEvaluate() diff --git a/ColorzCore/Parser/EAParser.cs b/ColorzCore/Parser/EAParser.cs index 4662b1e..e8a17ce 100644 --- a/ColorzCore/Parser/EAParser.cs +++ b/ColorzCore/Parser/EAParser.cs @@ -35,7 +35,7 @@ private set { if (validOffset) //Error only the first time. { - Error(head == null ? new Location?() : head.Location, "Invalid offset: " + value.ToString("X")); + Error($"Invalid offset: {value:X}"); validOffset = false; } } @@ -617,7 +617,7 @@ public IList ParsePreprocParamList(MergeableGenerator tokens, private IParamNode? ParseParam(MergeableGenerator tokens, ImmutableStack scopes, bool expandDefs = true) { Token localHead = tokens.Current; - switch (tokens.Current.Type) + switch (localHead.Type) { case TokenType.OPEN_BRACKET: return new ListNode(localHead.Location, ParseList(tokens, scopes)); @@ -644,7 +644,15 @@ public IList ParsePreprocParamList(MergeableGenerator tokens, } else { - return ParseAtom(tokens, scopes, expandDefs); + switch (localHead.Content.ToUpperInvariant()) + { + case "__FILE__": + tokens.MoveNext(); + return new StringNode(new Token(TokenType.STRING, localHead.Location, localHead.GetSourceLocation().file)); + + default: + return ParseAtom(tokens, scopes, expandDefs); + } } default: @@ -795,7 +803,7 @@ public IList ParsePreprocParamList(MergeableGenerator tokens, case TokenType.COMPARE_GT: case TokenType.UNDEFINED_COALESCE_OP: default: - Error(lookAhead.Location, "Expected identifier or literal, got " + lookAhead.Type + ": " + lookAhead.Content + '.'); + Error(lookAhead.Location, $"Expected identifier or literal, got {lookAhead.Type}: {lookAhead.Content}."); IgnoreRestOfStatement(tokens); return null; } @@ -814,8 +822,7 @@ public IList ParsePreprocParamList(MergeableGenerator tokens, grammarSymbols.Push(new Left(lookAhead.Content.ToUpperInvariant() switch { "CURRENTOFFSET" => new NumberNode(lookAhead, CurrentOffset), - // TODO: __LINE__ within macro expansion should expand to line of caller - "__LINE__" => new NumberNode(lookAhead, lookAhead.Location.lineNum), + "__LINE__" => new NumberNode(lookAhead, lookAhead.GetSourceLocation().line), _ => new IdentifierNode(lookAhead, scopes), })); @@ -1108,26 +1115,28 @@ public bool ExpandIdentifier(MergeableGenerator tokens, ImmutableStack Message(head?.Location, message); - public void Warning(string message) => Warning(head?.Location, message); - public void Error(string message) => Error(head?.Location, message); + public void Message(Location? location, string message) => MessageTrace(Log.MessageKind.MESSAGE, location, message); + public void Warning(Location? location, string message) => MessageTrace(Log.MessageKind.WARNING, location, message); + public void Error(Location? location, string message) => MessageTrace(Log.MessageKind.ERROR, location, message); + + public void Message(string message) => MessageTrace(Log.MessageKind.MESSAGE, head?.Location, message); + public void Warning(string message) => MessageTrace(Log.MessageKind.WARNING, head?.Location, message); + public void Error(string message) => MessageTrace(Log.MessageKind.ERROR, head?.Location, message); private void IgnoreRestOfStatement(MergeableGenerator tokens) { @@ -1199,9 +1208,10 @@ string UserFormatStringError(string message, string details) string expr = match.Groups["expr"].Value!; string? format = match.Groups["format"].Value; + Location itemLocation = baseLocation.OffsetBy(match.Index); + MergeableGenerator tokens = new MergeableGenerator( - new Tokenizer().TokenizeLine( - $"{expr} \n", baseLocation.file, baseLocation.lineNum, baseLocation.colNum + match.Index)); + new Tokenizer().TokenizeLine($"{expr} \n", itemLocation)); tokens.MoveNext(); diff --git a/ColorzCore/Preprocessor/Definition.cs b/ColorzCore/Preprocessor/Definition.cs index 0be8705..905648b 100644 --- a/ColorzCore/Preprocessor/Definition.cs +++ b/ColorzCore/Preprocessor/Definition.cs @@ -31,11 +31,11 @@ public IEnumerable ApplyDefinition(Token token) // assumes !NonProductive IList replacement = this.replacement!; + MacroLocation macroLocation = new MacroLocation(token.Content, token.Location); for (int i = 0; i < replacement.Count; i++) { - Location newLoc = new Location(token.FileName, token.LineNumber, token.ColumnNumber); - yield return new Token(replacement[i].Type, newLoc, replacement[i].Content); + yield return replacement[i].MacroClone(macroLocation); } } } diff --git a/ColorzCore/Preprocessor/Macros/UserMacro.cs b/ColorzCore/Preprocessor/Macros/UserMacro.cs index 7bdd190..f970153 100644 --- a/ColorzCore/Preprocessor/Macros/UserMacro.cs +++ b/ColorzCore/Preprocessor/Macros/UserMacro.cs @@ -29,18 +29,20 @@ public UserMacro(IList parameters, IList macroBody) */ public IEnumerable ApplyMacro(Token head, IList> parameters, ImmutableStack scopes) { - foreach (Token t in body) + MacroLocation macroLocation = new MacroLocation(head.Content, head.Location); + + foreach (Token bodyToken in body) { - if (t.Type == TokenType.IDENTIFIER && idToParamNum.TryGetValue(t.Content, out int paramNum)) + if (bodyToken.Type == TokenType.IDENTIFIER && idToParamNum.TryGetValue(bodyToken.Content, out int paramNum)) { - foreach (Token t2 in parameters[paramNum]) + foreach (Token paramToken in parameters[paramNum]) { - yield return t2; + yield return paramToken; } } else { - yield return new Token(t.Type, head.Location, t.Content); + yield return bodyToken.MacroClone(macroLocation); } } } From 17918ffccaaf8c310f529ec2f6242778fc1bc149 Mon Sep 17 00:00:00 2001 From: Nat_776 Date: Sat, 27 Apr 2024 13:55:47 +0200 Subject: [PATCH 33/59] warn on non-portable paths on windows + update vs solution --- ColorzCore/ColorzCore.Framework.csproj | 3 +- ColorzCore/ColorzCore.sln | 14 +++- ColorzCore/EAOptions.cs | 16 +++- ColorzCore/IO/IOUtility.cs | 82 ++++++++++++++++--- .../Directives/IncludeBinaryDirective.cs | 22 ++++- .../Directives/IncludeDirective.cs | 35 ++++++-- 6 files changed, 144 insertions(+), 28 deletions(-) diff --git a/ColorzCore/ColorzCore.Framework.csproj b/ColorzCore/ColorzCore.Framework.csproj index 6918bae..2035bb8 100644 --- a/ColorzCore/ColorzCore.Framework.csproj +++ b/ColorzCore/ColorzCore.Framework.csproj @@ -1,8 +1,9 @@ - + net48 Exe annotations + ColorzCore diff --git a/ColorzCore/ColorzCore.sln b/ColorzCore/ColorzCore.sln index f0a2e0f..805d6ec 100644 --- a/ColorzCore/ColorzCore.sln +++ b/ColorzCore/ColorzCore.sln @@ -1,9 +1,11 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio 15 -VisualStudioVersion = 15.0.26430.6 +# Visual Studio Version 17 +VisualStudioVersion = 17.9.34714.143 MinimumVisualStudioVersion = 10.0.40219.1 -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ColorzCore", "ColorzCore.csproj", "{B98F7CCF-9CAA-406E-88D7-2040FA99F631}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ColorzCore", "ColorzCore.csproj", "{B98F7CCF-9CAA-406E-88D7-2040FA99F631}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ColorzCore.Framework", "ColorzCore.Framework.csproj", "{33DC9CB0-82AB-47EC-9D7F-AD9FD5776BCF}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -18,6 +20,12 @@ Global {B98F7CCF-9CAA-406E-88D7-2040FA99F631}.Debug|Any CPU.Build.0 = Debug|Any CPU {B98F7CCF-9CAA-406E-88D7-2040FA99F631}.Release|Any CPU.ActiveCfg = Release|Any CPU {B98F7CCF-9CAA-406E-88D7-2040FA99F631}.Release|Any CPU.Build.0 = Release|Any CPU + {33DC9CB0-82AB-47EC-9D7F-AD9FD5776BCF}.32bit|Any CPU.ActiveCfg = Debug|Any CPU + {33DC9CB0-82AB-47EC-9D7F-AD9FD5776BCF}.32bit|Any CPU.Build.0 = Debug|Any CPU + {33DC9CB0-82AB-47EC-9D7F-AD9FD5776BCF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {33DC9CB0-82AB-47EC-9D7F-AD9FD5776BCF}.Debug|Any CPU.Build.0 = Debug|Any CPU + {33DC9CB0-82AB-47EC-9D7F-AD9FD5776BCF}.Release|Any CPU.ActiveCfg = Release|Any CPU + {33DC9CB0-82AB-47EC-9D7F-AD9FD5776BCF}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/ColorzCore/EAOptions.cs b/ColorzCore/EAOptions.cs index da11199..5bf0824 100644 --- a/ColorzCore/EAOptions.cs +++ b/ColorzCore/EAOptions.cs @@ -1,21 +1,26 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Text; -using System.Threading.Tasks; namespace ColorzCore { class EAOptions { + // TODO: clean up + public bool werr; public bool nowarn, nomess; public bool buildTimes; + // TODO: better warning flags + + public bool warnPortablePath; + public bool noColoredLog; public bool nocashSym; public bool readDataMacros; + public bool translateBackslashesInPath; public List includePaths = new List(); public List toolsPaths = new List(); @@ -30,6 +35,13 @@ private EAOptions() werr = false; nowarn = false; nomess = false; + + warnPortablePath = true; + + // this allows some non-portable paths to be made portable automatically + // also prevents the portable path warning from being emitted for some genereated files + translateBackslashesInPath = true; + noColoredLog = false; nocashSym = false; readDataMacros = true; diff --git a/ColorzCore/IO/IOUtility.cs b/ColorzCore/IO/IOUtility.cs index 73c7c78..a0a6559 100644 --- a/ColorzCore/IO/IOUtility.cs +++ b/ColorzCore/IO/IOUtility.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Runtime.InteropServices; using System.Text; using System.Threading.Tasks; @@ -22,17 +23,78 @@ public static string UnescapePath(string param) return sb.Replace("\\ ", " ").Replace("\\\\", "\\").ToString(); } - public static string GetToolFileName(string name) - { - switch (Environment.OSVersion.Platform) + public static string GetToolFileName(string name) + { + return RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? $"{name}.exe" : name; + } + + private static readonly char[] pathSeparators = new char[] { Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar }; + private static readonly char[] invalidFileCharacters = Path.GetInvalidFileNameChars(); + + /// + /// Convert a user-given path expression to a 'portable' one. That is, it would work on other systems. + /// This is used for warning emission from '#include' and '#incbin' directives that refer to non-portable paths + /// + /// The full real path corresponding to the given expression + /// The user given expression + /// The portable version of the expression, possibly the same as the input expression + public static string GetPortablePathExpression(string fullPath, string pathExpression) + { + if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + // this method is only meaningful on Windows, which has case-insensitive paths and '\' path separators. + // I believe that on other systems (Linux and friends for sure, probably macOS also?), valid paths are necessarily portable. + return pathExpression; + } + + if (Path.IsPathRooted(pathExpression)) { - case PlatformID.Win32Windows: // Who knows, maybe someone runs EA on win 95 - case PlatformID.Win32NT: - return name + ".exe"; - - default: - return name; - } + // HACK: rooted paths (like absolute paths) don't really make sense to make portable anyway + // those are most likely generated paths from tools so this doesn't really matter + + return pathExpression; + } + + IList inputComponents = pathExpression.Split(pathSeparators).ToList(); + IList outputComponents = new List(); // will be in reverse + + int upwind = 0; + + foreach (string component in inputComponents.Reverse()) + { + if (component.IndexOfAny(invalidFileCharacters) != -1) + { + // fallback just in case + outputComponents.Add(component); + } + if (component == "..") + { + outputComponents.Add(component); + upwind++; + } + else if (component == ".") + { + outputComponents.Add(component); + } + else if (upwind > 0) + { + // this was a "DirName/..", get the real name of this DirName + string? correctComponent = Path.GetFileName(Directory.GetFileSystemEntries(fullPath, component).FirstOrDefault()); + outputComponents.Add(correctComponent ?? component); + + upwind--; + } + else + { + string? reducedPath = Path.GetDirectoryName(fullPath); + fullPath = string.IsNullOrEmpty(reducedPath) ? "." : reducedPath; + + string? correctComponent = Path.GetFileName(Directory.GetFileSystemEntries(fullPath, component).FirstOrDefault()); + outputComponents.Add(correctComponent ?? component); + } + } + + return string.Join("/", outputComponents.Reverse()); } } } diff --git a/ColorzCore/Preprocessor/Directives/IncludeBinaryDirective.cs b/ColorzCore/Preprocessor/Directives/IncludeBinaryDirective.cs index 8cc0842..6f952f5 100644 --- a/ColorzCore/Preprocessor/Directives/IncludeBinaryDirective.cs +++ b/ColorzCore/Preprocessor/Directives/IncludeBinaryDirective.cs @@ -22,14 +22,30 @@ class IncludeBinaryDirective : SimpleDirective public override ILineNode? Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) { - string? existantFile = FileSearcher.FindFile(Path.GetDirectoryName(self.FileName), parameters[0].ToString()!); + string pathExpression = parameters[0].ToString()!; + + if (EAOptions.Instance.translateBackslashesInPath) + { + pathExpression = pathExpression.Replace('\\', '/'); + } + + string? existantFile = FileSearcher.FindFile(Path.GetDirectoryName(self.FileName), pathExpression); if (existantFile != null) { + if (EAOptions.Instance.warnPortablePath) + { + string portablePathExpression = IOUtility.GetPortablePathExpression(existantFile, pathExpression); + + if (pathExpression != portablePathExpression) + { + p.Warning(self.Location, $"Path is not portable (should be \"{portablePathExpression}\")."); + } + } + try { - string pathname = existantFile; - return new DataNode(p.CurrentOffset, File.ReadAllBytes(pathname)); + return new DataNode(p.CurrentOffset, File.ReadAllBytes(existantFile)); } catch (Exception) { diff --git a/ColorzCore/Preprocessor/Directives/IncludeDirective.cs b/ColorzCore/Preprocessor/Directives/IncludeDirective.cs index 5c144d0..43f676d 100644 --- a/ColorzCore/Preprocessor/Directives/IncludeDirective.cs +++ b/ColorzCore/Preprocessor/Directives/IncludeDirective.cs @@ -12,6 +12,8 @@ namespace ColorzCore.Preprocessor.Directives { class IncludeDirective : SimpleDirective { + // TODO: merge include and incbin into a shared base class + public override int MinParams => 1; public override int? MaxParams => 1; @@ -22,27 +24,42 @@ class IncludeDirective : SimpleDirective public override ILineNode? Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) { - string? existantFile = FileSearcher.FindFile(Path.GetDirectoryName(self.FileName), parameters[0].ToString()!); + string pathExpression = parameters[0].ToString()!; + + if (EAOptions.Instance.translateBackslashesInPath) + { + pathExpression = pathExpression.Replace('\\', '/'); + } + + string? existantFile = FileSearcher.FindFile(Path.GetDirectoryName(self.FileName), pathExpression); if (existantFile != null) { - try + if (EAOptions.Instance.warnPortablePath) { - string pathname = existantFile; + string portablePathExpression = IOUtility.GetPortablePathExpression(existantFile, pathExpression); - FileStream inputFile = new FileStream(pathname, FileMode.Open); - Tokenizer newFileTokenizer = new Tokenizer(); - tokens.PrependEnumerator(newFileTokenizer.Tokenize(inputFile).GetEnumerator()); + if (pathExpression != portablePathExpression) + { + p.Warning(self.Location, $"Path is not portable (should be \"{portablePathExpression}\")."); + } } - catch (Exception) + + try { - p.Error(self.Location, "Error reading file \"" + parameters[0].ToString() + "\"."); + FileStream inputFile = new FileStream(existantFile, FileMode.Open); + tokens.PrependEnumerator(new Tokenizer().Tokenize(inputFile).GetEnumerator()); + } + catch (Exception e) + { + p.Error(self.Location, $"Error reading file \"{parameters[0].ToString()}\": {e.Message}."); } } else { - p.Error(parameters[0].MyLocation, "Could not find file \"" + parameters[0].ToString() + "\"."); + p.Error(parameters[0].MyLocation, $"Could not find file \"{parameters[0].ToString()}\"."); } + return null; } } From 38a66cd2862c8f7ce50d74c84a1669ec0e5aace9 Mon Sep 17 00:00:00 2001 From: Nat_776 Date: Sat, 27 Apr 2024 13:56:34 +0200 Subject: [PATCH 34/59] labels are now addresses --- ColorzCore/Parser/EAParser.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ColorzCore/Parser/EAParser.cs b/ColorzCore/Parser/EAParser.cs index e8a17ce..518f132 100644 --- a/ColorzCore/Parser/EAParser.cs +++ b/ColorzCore/Parser/EAParser.cs @@ -955,7 +955,7 @@ private IList ParseList(MergeableGenerator tokens, ImmutableSt { case TokenType.COLON: tokens.MoveNext(); - TryDefineSymbol(scopes, head.Content, CurrentOffset); + TryDefineSymbol(scopes, head.Content, ConvertToAddress(CurrentOffset)); return null; case TokenType.ASSIGN: tokens.MoveNext(); From 32ff5c9615a564003a96b7bdde104b6473e8d3cd Mon Sep 17 00:00:00 2001 From: Nat_776 Date: Sat, 27 Apr 2024 15:05:43 +0200 Subject: [PATCH 35/59] refactor EAOptions --- ColorzCore/EAInterpreter.cs | 8 +-- ColorzCore/EAOptions.cs | 67 ++++++++----------- ColorzCore/Lexer/Tokenizer.cs | 9 +++ ColorzCore/Parser/BaseClosure.cs | 11 ++- ColorzCore/Parser/EAParser.cs | 12 ++-- .../Directives/IncludeBinaryDirective.cs | 4 +- .../Directives/IncludeDirective.cs | 4 +- ColorzCore/Preprocessor/Macros/ReadDataAt.cs | 2 +- ColorzCore/Program.cs | 46 ++++++------- 9 files changed, 81 insertions(+), 82 deletions(-) diff --git a/ColorzCore/EAInterpreter.cs b/ColorzCore/EAInterpreter.cs index 885081e..ee20d97 100644 --- a/ColorzCore/EAInterpreter.cs +++ b/ColorzCore/EAInterpreter.cs @@ -54,13 +54,13 @@ public EAInterpreter(IOutput output, string game, string? rawsFolder, string raw IncludeFileSearcher includeSearcher = new IncludeFileSearcher(); includeSearcher.IncludeDirectories.Add(AppDomain.CurrentDomain.BaseDirectory); - foreach (string path in EAOptions.Instance.includePaths) + foreach (string path in EAOptions.IncludePaths) includeSearcher.IncludeDirectories.Add(path); IncludeFileSearcher toolSearcher = new IncludeFileSearcher { AllowRelativeInclude = false }; toolSearcher.IncludeDirectories.Add(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Tools")); - foreach (string path in EAOptions.Instance.toolsPaths) + foreach (string path in EAOptions.ToolsPaths) includeSearcher.IncludeDirectories.Add(path); myParser = new EAParser(allRaws, log, new Preprocessor.DirectiveHandler(includeSearcher, toolSearcher)); @@ -68,7 +68,7 @@ public EAInterpreter(IOutput output, string game, string? rawsFolder, string raw myParser.Definitions[$"_{game}_"] = new Definition(); myParser.Definitions["__COLORZ_CORE__"] = new Definition(); - if (EAOptions.Instance.readDataMacros && output is ROM rom) + if (EAOptions.IsExtensionEnabled(EAOptions.Extensions.ReadDataMacros) && output is ROM rom) { myParser.Definitions["__has_read_data_macros"] = new Definition(); @@ -92,7 +92,7 @@ public bool Interpret() ExecTimer.Timer.AddTimingPoint(ExecTimer.KEY_GENERIC); - foreach ((string name, string body) in EAOptions.Instance.defs) + foreach ((string name, string body) in EAOptions.PreDefintions) { myParser.ParseAll(t.TokenizeLine($"#define {name} {body}", "cmd", 0)); } diff --git a/ColorzCore/EAOptions.cs b/ColorzCore/EAOptions.cs index 5bf0824..029b0cb 100644 --- a/ColorzCore/EAOptions.cs +++ b/ColorzCore/EAOptions.cs @@ -4,50 +4,41 @@ namespace ColorzCore { - class EAOptions + public static class EAOptions { - // TODO: clean up - - public bool werr; - public bool nowarn, nomess; - public bool buildTimes; - - // TODO: better warning flags - - public bool warnPortablePath; - - public bool noColoredLog; - - public bool nocashSym; - public bool readDataMacros; - public bool translateBackslashesInPath; + [Flags] + public enum Warnings + { + None = 0, + NonPortablePath = 1, + } - public List includePaths = new List(); - public List toolsPaths = new List(); - public List> defs = new List>(); - public static EAOptions Instance { get; } = new EAOptions(); + [Flags] + public enum Extensions + { + None = 0, + ReadDataMacros = 1, + } - public int romBaseAddress; - public int maximumRomSize; + public static bool WarningsAreErrors { get; set; } + public static bool QuietWarnings { get; set; } + public static bool QuietMessages { get; set; } + public static bool MonochromeLog { get; set; } + public static bool BenchmarkBuildTimes { get; set; } + public static bool ProduceNocashSym { get; set; } + public static bool TranslateBackslashesInPaths { get; set; } = true; - private EAOptions() - { - werr = false; - nowarn = false; - nomess = false; + public static int BaseAddress { get; set; } = 0x8000000; + public static int MaximumBinarySize { get; set; } = 0x2000000; - warnPortablePath = true; + public static List IncludePaths { get; } = new List(); + public static List ToolsPaths { get; } = new List(); + public static List> PreDefintions { get; } = new List>(); - // this allows some non-portable paths to be made portable automatically - // also prevents the portable path warning from being emitted for some genereated files - translateBackslashesInPath = true; + public static Warnings EnabledWarnings { get; set; } = Warnings.NonPortablePath; + public static Extensions EnabledExtensions { get; set; } = Extensions.ReadDataMacros; - noColoredLog = false; - nocashSym = false; - readDataMacros = true; - buildTimes = false; - romBaseAddress = 0x8000000; - maximumRomSize = 0x2000000; - } + public static bool IsWarningEnabled(Warnings warning) => (EnabledWarnings & warning) != 0; + public static bool IsExtensionEnabled(Extensions extension) => (EnabledExtensions & extension) != 0; } } diff --git a/ColorzCore/Lexer/Tokenizer.cs b/ColorzCore/Lexer/Tokenizer.cs index 79663f7..0e8c8a1 100644 --- a/ColorzCore/Lexer/Tokenizer.cs +++ b/ColorzCore/Lexer/Tokenizer.cs @@ -292,6 +292,15 @@ public IEnumerable TokenizePhrase(string line, string fileName, int lineN case '\n': yield return new Token(TokenType.NEWLINE, fileName, lineNum, curCol + offset); break; + case '\\': + if (curCol + 1 < endOffs && line[curCol + 1] == '\n') + { + curCol++; + continue; + } + yield return new Token(TokenType.ERROR, fileName, lineNum, curCol, "\\"); + break; + default: if (afterInclude) { diff --git a/ColorzCore/Parser/BaseClosure.cs b/ColorzCore/Parser/BaseClosure.cs index b5bc1a6..0f69b04 100644 --- a/ColorzCore/Parser/BaseClosure.cs +++ b/ColorzCore/Parser/BaseClosure.cs @@ -2,14 +2,13 @@ { class BaseClosure : Closure { - private EAParser enclosing; - public BaseClosure(EAParser enclosing) - { - this.enclosing = enclosing; - } public override bool HasLocalSymbol(string label) { - return label.ToUpper() == "CURRENTOFFSET" || base.HasLocalSymbol(label); + return label.ToUpperInvariant() switch + { + "CURRENTOFFSET" or "__LINE__" or "__FILE__" => true, + _ => base.HasLocalSymbol(label), + }; } } } diff --git a/ColorzCore/Parser/EAParser.cs b/ColorzCore/Parser/EAParser.cs index 518f132..36fdee9 100644 --- a/ColorzCore/Parser/EAParser.cs +++ b/ColorzCore/Parser/EAParser.cs @@ -31,7 +31,7 @@ public int CurrentOffset get { return currentOffset; } private set { - if (value < 0 || value > EAOptions.Instance.maximumRomSize) + if (value < 0 || value > EAOptions.MaximumBinarySize) { if (validOffset) //Error only the first time. { @@ -81,7 +81,7 @@ public bool IsIncluding public EAParser(Dictionary> raws, Log log, DirectiveHandler directiveHandler) { - GlobalScope = new ImmutableStack(new BaseClosure(this), ImmutableStack.Nil); + GlobalScope = new ImmutableStack(new BaseClosure(), ImmutableStack.Nil); pastOffsets = new Stack>(); protectedRegions = new List>(); this.log = log; @@ -170,9 +170,9 @@ If one wants to instead refer to ROM offset 0 they would want to use the address If ROM offset 0 is already address 0 then this is a moot point. */ - if (value > 0 && value < EAOptions.Instance.maximumRomSize) + if (value > 0 && value < EAOptions.MaximumBinarySize) { - value += EAOptions.Instance.romBaseAddress; + value += EAOptions.BaseAddress; } return value; @@ -180,9 +180,9 @@ If ROM offset 0 is already address 0 then this is a moot point. public static int ConvertToOffset(int value) { - if (value >= EAOptions.Instance.romBaseAddress && value <= EAOptions.Instance.romBaseAddress + EAOptions.Instance.maximumRomSize) + if (value >= EAOptions.BaseAddress && value <= EAOptions.BaseAddress + EAOptions.MaximumBinarySize) { - value -= EAOptions.Instance.romBaseAddress; + value -= EAOptions.BaseAddress; } return value; diff --git a/ColorzCore/Preprocessor/Directives/IncludeBinaryDirective.cs b/ColorzCore/Preprocessor/Directives/IncludeBinaryDirective.cs index 6f952f5..371fd1d 100644 --- a/ColorzCore/Preprocessor/Directives/IncludeBinaryDirective.cs +++ b/ColorzCore/Preprocessor/Directives/IncludeBinaryDirective.cs @@ -24,7 +24,7 @@ class IncludeBinaryDirective : SimpleDirective { string pathExpression = parameters[0].ToString()!; - if (EAOptions.Instance.translateBackslashesInPath) + if (EAOptions.TranslateBackslashesInPaths) { pathExpression = pathExpression.Replace('\\', '/'); } @@ -33,7 +33,7 @@ class IncludeBinaryDirective : SimpleDirective if (existantFile != null) { - if (EAOptions.Instance.warnPortablePath) + if (EAOptions.IsWarningEnabled(EAOptions.Warnings.NonPortablePath)) { string portablePathExpression = IOUtility.GetPortablePathExpression(existantFile, pathExpression); diff --git a/ColorzCore/Preprocessor/Directives/IncludeDirective.cs b/ColorzCore/Preprocessor/Directives/IncludeDirective.cs index 43f676d..4ec546d 100644 --- a/ColorzCore/Preprocessor/Directives/IncludeDirective.cs +++ b/ColorzCore/Preprocessor/Directives/IncludeDirective.cs @@ -26,7 +26,7 @@ class IncludeDirective : SimpleDirective { string pathExpression = parameters[0].ToString()!; - if (EAOptions.Instance.translateBackslashesInPath) + if (EAOptions.TranslateBackslashesInPaths) { pathExpression = pathExpression.Replace('\\', '/'); } @@ -35,7 +35,7 @@ class IncludeDirective : SimpleDirective if (existantFile != null) { - if (EAOptions.Instance.warnPortablePath) + if (EAOptions.IsWarningEnabled(EAOptions.Warnings.NonPortablePath)) { string portablePathExpression = IOUtility.GetPortablePathExpression(existantFile, pathExpression); diff --git a/ColorzCore/Preprocessor/Macros/ReadDataAt.cs b/ColorzCore/Preprocessor/Macros/ReadDataAt.cs index 52b393a..9ef9727 100644 --- a/ColorzCore/Preprocessor/Macros/ReadDataAt.cs +++ b/ColorzCore/Preprocessor/Macros/ReadDataAt.cs @@ -40,7 +40,7 @@ public override IEnumerable ApplyMacro(Token head, IList> pa { offset = EAParser.ConvertToOffset(offset); - if (offset >= 0 && offset <= EAOptions.Instance.maximumRomSize - readLength) + if (offset >= 0 && offset <= EAOptions.MaximumBinarySize - readLength) { int data = 0; diff --git a/ColorzCore/Program.cs b/ColorzCore/Program.cs index 8b04632..45d6c7d 100644 --- a/ColorzCore/Program.cs +++ b/ColorzCore/Program.cs @@ -145,7 +145,7 @@ static int Main(string[] args) case "error": errorStream = new StreamWriter(File.OpenWrite(flag[1])); - EAOptions.Instance.noColoredLog = true; + EAOptions.MonochromeLog = true; break; case "debug": @@ -153,48 +153,48 @@ static int Main(string[] args) break; case "werr": - EAOptions.Instance.werr = true; + EAOptions.WarningsAreErrors = true; break; case "-no-mess": - EAOptions.Instance.nomess = true; + EAOptions.QuietMessages = true; break; case "-no-warn": - EAOptions.Instance.nowarn = true; + EAOptions.QuietWarnings = true; break; case "-no-colored-log": - EAOptions.Instance.noColoredLog = true; + EAOptions.MonochromeLog = true; break; case "quiet": - EAOptions.Instance.nomess = true; - EAOptions.Instance.nowarn = true; + EAOptions.QuietMessages = true; + EAOptions.QuietWarnings = true; break; case "-nocash-sym": - EAOptions.Instance.nocashSym = true; + EAOptions.ProduceNocashSym = true; break; case "-build-times": - EAOptions.Instance.buildTimes = true; + EAOptions.BenchmarkBuildTimes = true; break; case "I": case "-include": - EAOptions.Instance.includePaths.Add(flag[1]); + EAOptions.IncludePaths.Add(flag[1]); break; case "T": case "-tools": - EAOptions.Instance.toolsPaths.Add(flag[1]); + EAOptions.ToolsPaths.Add(flag[1]); break; case "IT": case "TI": - EAOptions.Instance.includePaths.Add(flag[1]); - EAOptions.Instance.toolsPaths.Add(flag[1]); + EAOptions.IncludePaths.Add(flag[1]); + EAOptions.ToolsPaths.Add(flag[1]); break; case "h": @@ -208,7 +208,7 @@ static int Main(string[] args) try { string[] def_args = flag[1].Split(new char[] { '=' }, 2); - EAOptions.Instance.defs.Add(Tuple.Create(def_args[0], def_args[1])); + EAOptions.PreDefintions.Add(Tuple.Create(def_args[0], def_args[1])); } catch (IndexOutOfRangeException) { @@ -220,7 +220,7 @@ static int Main(string[] args) case "-base-address": try { - EAOptions.Instance.romBaseAddress = Convert.ToInt32(flag[1], 16); + EAOptions.BaseAddress = Convert.ToInt32(flag[1], 16); } catch { @@ -231,7 +231,7 @@ static int Main(string[] args) case "-maximum-size": try { - EAOptions.Instance.maximumRomSize = Convert.ToInt32(flag[1], 16); + EAOptions.MaximumBinarySize = Convert.ToInt32(flag[1], 16); } catch { @@ -284,7 +284,7 @@ static int Main(string[] args) return EXIT_FAILURE; } - output = new ROM(outStream, EAOptions.Instance.maximumRomSize); + output = new ROM(outStream, EAOptions.MaximumBinarySize); } string game = args[1]; @@ -294,14 +294,14 @@ static int Main(string[] args) Log log = new Log { Output = errorStream, - WarningsAreErrors = EAOptions.Instance.werr, - NoColoredTags = EAOptions.Instance.noColoredLog + WarningsAreErrors = EAOptions.WarningsAreErrors, + NoColoredTags = EAOptions.MonochromeLog }; - if (EAOptions.Instance.nowarn) + if (EAOptions.QuietWarnings) log.IgnoredKinds.Add(Log.MessageKind.WARNING); - if (EAOptions.Instance.nomess) + if (EAOptions.QuietMessages) log.IgnoredKinds.Add(Log.MessageKind.MESSAGE); EAInterpreter myInterpreter = new EAInterpreter(output, game, rawsFolder, rawsExtension, inStream, inFileName, log); @@ -310,7 +310,7 @@ static int Main(string[] args) bool success = myInterpreter.Interpret(); - if (success && EAOptions.Instance.nocashSym) + if (success && EAOptions.ProduceNocashSym) { using StreamWriter symOut = File.CreateText(Path.ChangeExtension(outFileName, "sym")); @@ -320,7 +320,7 @@ static int Main(string[] args) } } - if (EAOptions.Instance.buildTimes) + if (EAOptions.BenchmarkBuildTimes) { // Print times From 9376334f3ab0ae285c6b1466f076a816553378ec Mon Sep 17 00:00:00 2001 From: Nat_776 Date: Sat, 27 Apr 2024 16:28:13 +0200 Subject: [PATCH 36/59] warn on suspicious expansion of macro featuring unguarded operators --- ColorzCore/EAOptions.cs | 29 +++- ColorzCore/Lexer/Tokenizer.cs | 4 +- ColorzCore/Parser/EAParser.cs | 133 +++++++++++++----- .../Directives/DefineDirective.cs | 11 +- ColorzCore/Preprocessor/MacroCollection.cs | 19 +++ ColorzCore/Program.cs | 2 +- 6 files changed, 148 insertions(+), 50 deletions(-) diff --git a/ColorzCore/EAOptions.cs b/ColorzCore/EAOptions.cs index 029b0cb..06240a5 100644 --- a/ColorzCore/EAOptions.cs +++ b/ColorzCore/EAOptions.cs @@ -7,17 +7,34 @@ namespace ColorzCore public static class EAOptions { [Flags] - public enum Warnings + public enum Warnings : long { None = 0, + + // warn on non-portable include paths on Windows NonPortablePath = 1, + + // warn on #define on an existing definition + ReDefine = 2, + + // warn on write before ORG + UninitializedOffset = 4, + + // warn on expansion of unguarded expression within macro + UnguardedExpressionMacros = 8, + + All = long.MaxValue, } [Flags] - public enum Extensions + public enum Extensions : long { None = 0, + + // enable ReadByteAt and friends ReadDataMacros = 1, + + All = long.MaxValue, } public static bool WarningsAreErrors { get; set; } @@ -33,12 +50,12 @@ public enum Extensions public static List IncludePaths { get; } = new List(); public static List ToolsPaths { get; } = new List(); - public static List> PreDefintions { get; } = new List>(); + public static List<(string, string)> PreDefintions { get; } = new List<(string, string)>(); - public static Warnings EnabledWarnings { get; set; } = Warnings.NonPortablePath; + public static Warnings EnabledWarnings { get; set; } = Warnings.All; public static Extensions EnabledExtensions { get; set; } = Extensions.ReadDataMacros; - public static bool IsWarningEnabled(Warnings warning) => (EnabledWarnings & warning) != 0; - public static bool IsExtensionEnabled(Extensions extension) => (EnabledExtensions & extension) != 0; + public static bool IsWarningEnabled(Warnings warning) => EnabledWarnings.HasFlag(warning); + public static bool IsExtensionEnabled(Extensions extension) => EnabledExtensions.HasFlag(extension); } } diff --git a/ColorzCore/Lexer/Tokenizer.cs b/ColorzCore/Lexer/Tokenizer.cs index 0e8c8a1..35664d9 100644 --- a/ColorzCore/Lexer/Tokenizer.cs +++ b/ColorzCore/Lexer/Tokenizer.cs @@ -379,9 +379,9 @@ public IEnumerable TokenizeLine(string line, string fileName, int lineNum return TokenizePhrase(line, fileName, lineNum, 0, line.Length, offset); } - public IEnumerable TokenizeLine(string line, Location location) + public static IEnumerable TokenizeLine(string line, Location location) { - return TokenizeLine(line, location.file, location.line, location.column); + return new Tokenizer().TokenizeLine(line, location.file, location.line, location.column); } /*** diff --git a/ColorzCore/Parser/EAParser.cs b/ColorzCore/Parser/EAParser.cs index 36fdee9..e68a20a 100644 --- a/ColorzCore/Parser/EAParser.cs +++ b/ColorzCore/Parser/EAParser.cs @@ -3,6 +3,7 @@ using ColorzCore.Lexer; using ColorzCore.Parser.AST; using ColorzCore.Preprocessor; +using ColorzCore.Preprocessor.Macros; using ColorzCore.Raws; using System; using System.Collections.Generic; @@ -28,7 +29,8 @@ public class EAParser public ImmutableStack GlobalScope { get; } public int CurrentOffset { - get { return currentOffset; } + get => currentOffset; + private set { if (value < 0 || value > EAOptions.MaximumBinarySize) @@ -99,7 +101,7 @@ public EAParser(Dictionary> raws, Log log, DirectiveHandler d public bool IsReservedName(string name) { - return Raws.ContainsKey(name.ToUpper()) || SpecialCodes.Contains(name.ToUpper()); + return Raws.ContainsKey(name.ToUpperInvariant()) || SpecialCodes.Contains(name.ToUpperInvariant()); } public bool IsValidDefinitionName(string name) { @@ -107,7 +109,7 @@ public bool IsValidDefinitionName(string name) } public bool IsValidMacroName(string name, int paramNum) { - return !(Macros.HasMacro(name, paramNum)) && !IsReservedName(name); + return !Macros.HasMacro(name, paramNum) && !IsReservedName(name); } public bool IsValidLabelName(string name) { @@ -418,6 +420,7 @@ static bool IsConditionalOperatorHelper(IAtomNode node) if (!errorOccurred) { int length = end - start; + if (length > 0) { protectedRegions.Add(new Tuple(start, length, head!.Location)); @@ -626,7 +629,7 @@ public IList ParsePreprocParamList(MergeableGenerator tokens, return new StringNode(localHead); case TokenType.MAYBE_MACRO: //TODO: Move this and the one in ExpandId to a separate ParseMacroNode that may return an Invocation. - if (expandDefs && ExpandIdentifier(tokens, scopes)) + if (expandDefs && ExpandIdentifier(tokens, scopes, true)) { return ParseParam(tokens, scopes); } @@ -638,7 +641,7 @@ public IList ParsePreprocParamList(MergeableGenerator tokens, return new MacroInvocationNode(this, localHead, param, scopes); } case TokenType.IDENTIFIER: - if (expandDefs && Definitions.ContainsKey(localHead.Content) && ExpandIdentifier(tokens, scopes)) + if (expandDefs && ExpandIdentifier(tokens, scopes, true)) { return ParseParam(tokens, scopes, expandDefs); } @@ -814,7 +817,7 @@ public IList ParsePreprocParamList(MergeableGenerator tokens, switch (lookAhead.Type) { case TokenType.IDENTIFIER: - if (expandDefs && ExpandIdentifier(tokens, scopes)) + if (expandDefs && ExpandIdentifier(tokens, scopes, true)) { continue; } @@ -829,7 +832,7 @@ public IList ParsePreprocParamList(MergeableGenerator tokens, break; case TokenType.MAYBE_MACRO: - ExpandIdentifier(tokens, scopes); + ExpandIdentifier(tokens, scopes, true); continue; case TokenType.NUMBER: grammarSymbols.Push(new Left(new NumberNode(lookAhead))); @@ -1078,41 +1081,105 @@ private void TryDefineSymbol(ImmutableStack scopes, string name, IAtomN * Postcondition: tokens.Current is fully reduced (i.e. not a macro, and not a definition) * Returns: true iff tokens was actually expanded. */ - public bool ExpandIdentifier(MergeableGenerator tokens, ImmutableStack scopes) + public bool ExpandIdentifier(MergeableGenerator tokens, ImmutableStack scopes, bool insideExpression = false) { - bool ret = false; - //Macros and Definitions. - if (tokens.Current.Type == TokenType.MAYBE_MACRO && Macros.ContainsName(tokens.Current.Content)) + // function-like macros + if (tokens.Current.Type == TokenType.MAYBE_MACRO) { - Token localHead = tokens.Current; - tokens.MoveNext(); - IList> parameters = ParseMacroParamList(tokens); - if (Macros.HasMacro(localHead.Content, parameters.Count)) + if (Macros.ContainsName(tokens.Current.Content)) { - tokens.PrependEnumerator(Macros.GetMacro(localHead.Content, parameters.Count).ApplyMacro(localHead, parameters, scopes).GetEnumerator()); + Token localHead = tokens.Current; + tokens.MoveNext(); + + IList> parameters = ParseMacroParamList(tokens); + + if (Macros.TryGetMacro(localHead.Content, parameters.Count, out IMacro? macro)) + { + /* macro is 100% not null here, but because we can't use NotNullWhen on TryGetMacro, + * since the attribute is unavailable in .NET Framework (which we still target), + * the compiler will still diagnose a nullable dereference if we don't use '!' also */ + + ApplyMacroExpansion(tokens, macro!.ApplyMacro(localHead, parameters, scopes), insideExpression); + } + else + { + Error($"No overload of {localHead.Content} with {parameters.Count} parameters."); + } + return true; } else { - Error($"No overload of {localHead.Content} with {parameters.Count} parameters."); + Token localHead = tokens.Current; + tokens.MoveNext(); + + tokens.PutBack(new Token(TokenType.IDENTIFIER, localHead.Location, localHead.Content)); + return true; } - return true; } - else if (tokens.Current.Type == TokenType.MAYBE_MACRO) + + // object-like macros (aka "Definitions") + if (Definitions.TryGetValue(tokens.Current.Content, out Definition? definition) && !definition.NonProductive) { Token localHead = tokens.Current; tokens.MoveNext(); - tokens.PutBack(new Token(TokenType.IDENTIFIER, localHead.Location, localHead.Content)); + + ApplyMacroExpansion(tokens, definition.ApplyDefinition(localHead), insideExpression); return true; } - else if (Definitions.TryGetValue(tokens.Current.Content, out Definition? definition) && !definition.NonProductive) + + return false; + } + + private void ApplyMacroExpansion(MergeableGenerator tokens, IEnumerable expandedTokens, bool insideExpression = false) + { + if (insideExpression && EAOptions.IsWarningEnabled(EAOptions.Warnings.UnguardedExpressionMacros)) { - Token localHead = tokens.Current; - tokens.MoveNext(); - tokens.PrependEnumerator(definition.ApplyDefinition(localHead).GetEnumerator()); - return true; - } + // here we check for any operator that isn't enclosed in parenthesises + + IList expandedList = expandedTokens.ToList(); + + if (expandedList.Count > 1) + { + int paren = 0; + int bracket = 0; + + foreach (Token token in expandedList) + { + switch (token.Type) + { + case TokenType.OPEN_PAREN: + paren++; + break; + + case TokenType.CLOSE_PAREN: + paren--; + break; + + case TokenType.OPEN_BRACKET: + bracket++; + break; + + case TokenType.CLOSE_BRACKET: + bracket--; + break; + + default: + if (paren == 0 && bracket == 0 && precedences.ContainsKey(token.Type)) + { + Warning(token.Location, $"Unguarded expansion of mathematical operator. Consider adding guarding parenthesises around definition."); + } - return ret; + break; + } + } + } + + tokens.PrependEnumerator(expandedList.GetEnumerator()); + } + else + { + tokens.PrependEnumerator(expandedTokens.GetEnumerator()); + } } private void MessageTrace(Log.MessageKind kind, Location? location, string message) @@ -1211,7 +1278,7 @@ string UserFormatStringError(string message, string details) Location itemLocation = baseLocation.OffsetBy(match.Index); MergeableGenerator tokens = new MergeableGenerator( - new Tokenizer().TokenizeLine($"{expr} \n", itemLocation)); + Tokenizer.TokenizeLine($"{expr} \n", itemLocation)); tokens.MoveNext(); @@ -1261,16 +1328,16 @@ string UserFormatStringError(string message, string details) private void CheckDataWrite(int length) { - // TODO: maybe make this warning optional? if (!offsetInitialized) { - Warning("Writing before initializing offset. You may be breaking the ROM! (use `ORG offset` to set write offset)."); + if (EAOptions.IsWarningEnabled(EAOptions.Warnings.UninitializedOffset)) + { + Warning("Writing before initializing offset. You may be breaking the ROM! (use `ORG offset` to set write offset)."); + } + offsetInitialized = false; // only warn once } - // TODO (maybe?): save Location of PROTECT statement, for better diagnosis - // We would then print something like "Trying to write data to area protected at " - if (IsProtected(CurrentOffset, length) is Location prot) { Error($"Trying to write data to area protected by {prot}"); diff --git a/ColorzCore/Preprocessor/Directives/DefineDirective.cs b/ColorzCore/Preprocessor/Directives/DefineDirective.cs index a92cfcc..576fd7c 100644 --- a/ColorzCore/Preprocessor/Directives/DefineDirective.cs +++ b/ColorzCore/Preprocessor/Directives/DefineDirective.cs @@ -60,9 +60,8 @@ private static void DefineObjectMacro(EAParser p, Token nameToken, IList { string name = nameToken.Content; - if (p.Definitions.ContainsKey(name)) + if (p.Definitions.ContainsKey(name) && EAOptions.IsWarningEnabled(EAOptions.Warnings.ReDefine)) { - // TODO: stricter error (opt-in?) p.Warning(nameToken.Location, $"Redefining {name}."); } @@ -83,9 +82,8 @@ private static void DefineFunctionMacro(EAParser p, Token nameToken, IList ExpandMacroBody(EAParser _, IList body) if (body.Count == 1 && body[0].Type == TokenType.STRING) { Token token = body[0]; - - // TODO: does this need to be column number + 1? - return new List(new Tokenizer().TokenizeLine( - token.Content, token.FileName, token.LineNumber, token.ColumnNumber)); + return new List(Tokenizer.TokenizeLine(token.Content, token.Location)); } return body; diff --git a/ColorzCore/Preprocessor/MacroCollection.cs b/ColorzCore/Preprocessor/MacroCollection.cs index 040565d..75c0f13 100644 --- a/ColorzCore/Preprocessor/MacroCollection.cs +++ b/ColorzCore/Preprocessor/MacroCollection.cs @@ -32,6 +32,25 @@ public bool HasMacro(string name, int paramNum) { return BuiltInMacros.ContainsKey(name) && BuiltInMacros[name].ValidNumParams(paramNum) || Macros.ContainsKey(name) && Macros[name].ContainsKey(paramNum); } + + // NOTE: NotNullWhen(true) is not available on .NET Framework 4.x, one of our targets + public bool TryGetMacro(string name, int paramNum, out IMacro? macro) + { + if (BuiltInMacros.TryGetValue(name, out BuiltInMacro? builtinMacro) && builtinMacro.ValidNumParams(paramNum)) + { + macro = builtinMacro; + return true; + } + + if (Macros.TryGetValue(name, out Dictionary? macros)) + { + return macros.TryGetValue(paramNum, out macro); + } + + macro = null; + return false; + } + public IMacro GetMacro(string name, int paramNum) { return BuiltInMacros.ContainsKey(name) && BuiltInMacros[name].ValidNumParams(paramNum) ? BuiltInMacros[name] : Macros[name][paramNum]; diff --git a/ColorzCore/Program.cs b/ColorzCore/Program.cs index 45d6c7d..f1374fb 100644 --- a/ColorzCore/Program.cs +++ b/ColorzCore/Program.cs @@ -208,7 +208,7 @@ static int Main(string[] args) try { string[] def_args = flag[1].Split(new char[] { '=' }, 2); - EAOptions.PreDefintions.Add(Tuple.Create(def_args[0], def_args[1])); + EAOptions.PreDefintions.Add((def_args[0], def_args[1])); } catch (IndexOutOfRangeException) { From 7a4d069391f7c3c2774339911cccdb4a7754386d Mon Sep 17 00:00:00 2001 From: Nat_776 Date: Sat, 27 Apr 2024 16:49:52 +0200 Subject: [PATCH 37/59] allow tuning of warnings and extensions via commandline --- ColorzCore/EAInterpreter.cs | 2 +- ColorzCore/EAOptions.cs | 2 +- ColorzCore/Program.cs | 356 +++++++++++++++++++++++------------- 3 files changed, 229 insertions(+), 131 deletions(-) diff --git a/ColorzCore/EAInterpreter.cs b/ColorzCore/EAInterpreter.cs index ee20d97..31f25b1 100644 --- a/ColorzCore/EAInterpreter.cs +++ b/ColorzCore/EAInterpreter.cs @@ -63,7 +63,7 @@ public EAInterpreter(IOutput output, string game, string? rawsFolder, string raw foreach (string path in EAOptions.ToolsPaths) includeSearcher.IncludeDirectories.Add(path); - myParser = new EAParser(allRaws, log, new Preprocessor.DirectiveHandler(includeSearcher, toolSearcher)); + myParser = new EAParser(allRaws, log, new DirectiveHandler(includeSearcher, toolSearcher)); myParser.Definitions[$"_{game}_"] = new Definition(); myParser.Definitions["__COLORZ_CORE__"] = new Definition(); diff --git a/ColorzCore/EAOptions.cs b/ColorzCore/EAOptions.cs index 06240a5..12fe9c7 100644 --- a/ColorzCore/EAOptions.cs +++ b/ColorzCore/EAOptions.cs @@ -53,7 +53,7 @@ public enum Extensions : long public static List<(string, string)> PreDefintions { get; } = new List<(string, string)>(); public static Warnings EnabledWarnings { get; set; } = Warnings.All; - public static Extensions EnabledExtensions { get; set; } = Extensions.ReadDataMacros; + public static Extensions EnabledExtensions { get; set; } = Extensions.None; public static bool IsWarningEnabled(Warnings warning) => EnabledWarnings.HasFlag(warning); public static bool IsExtensionEnabled(Extensions extension) => EnabledExtensions.HasFlag(extension); diff --git a/ColorzCore/Program.cs b/ColorzCore/Program.cs index f1374fb..24724d0 100644 --- a/ColorzCore/Program.cs +++ b/ColorzCore/Program.cs @@ -10,7 +10,20 @@ class Program { public static bool Debug = false; - private static string[] helpstringarr = { + private static readonly IDictionary warningNames = new Dictionary() + { + { "nonportable-pathnames", EAOptions.Warnings.NonPortablePath }, + { "unguarded-expression-macros", EAOptions.Warnings.UnguardedExpressionMacros }, + { "redefine", EAOptions.Warnings.ReDefine }, + { "all", EAOptions.Warnings.All }, + }; + + private static readonly IDictionary extensionNames = new Dictionary() + { + { "read-data-macros", EAOptions.Extensions.ReadDataMacros } + }; + + private static readonly string[] helpstringarr = { "EA Colorz Core. Usage:", "./ColorzCore [-opts]", "", @@ -38,6 +51,11 @@ class Program " Add given path to list of paths to search for tools in.", "-IT:|-TI:", " Combines --include: and --tools:.", + "-W:[no-]:...|--warnings:[no-]:...", + " Enable or disable warnings. By default, all warnings are enabled.", + " Multiple warnings can be enabled/disabled at once.", + " Example: '--warnings:no-nonportable-pathnames:no-redefine'.", + " Possible values: " + string.Join(", ", warningNames.Keys), "-werr", " Treat all warnings as errors and prevent assembly.", "--no-mess", @@ -62,6 +80,9 @@ class Program " Sets the maximum size of the binary. Defaults to 0x02000000.", "-romoffset:", " Compatibility alias for --base-address:", + "--extensions:[no-]:...", + " Enable or disable extensions. By default, no extension is enabled.", + " Possible values: " + string.Join(", ", extensionNames.Keys), "-h|--help", " Display this message and exit.", "" @@ -110,146 +131,223 @@ static int Main(string[] args) if (args[i][0] != '-') { Console.Error.WriteLine("Unrecognized paramter: " + args[i]); + continue; } - else - { - string[] flag = args[i].Substring(1).Split(new char[] { ':' }, 2); - try - { - switch (flag[0]) - { - case "raws": - rawsFolder = rawSearcher.FindDirectory(flag[1]); - - if (rawsFolder == null) - { - Console.Error.WriteLine($"No such folder: {flag[1]}"); - return EXIT_FAILURE; - } + string[] flag = args[i].Split(':'); - break; - - case "rawsExt": - rawsExtension = flag[1]; - break; - - case "output": - outFileName = flag[1]; - break; - - case "input": - inFileName = flag[1]; - inStream = File.OpenRead(flag[1]); - break; - - case "error": - errorStream = new StreamWriter(File.OpenWrite(flag[1])); - EAOptions.MonochromeLog = true; - break; - - case "debug": - Debug = true; - break; - - case "werr": - EAOptions.WarningsAreErrors = true; - break; - - case "-no-mess": - EAOptions.QuietMessages = true; - break; - - case "-no-warn": - EAOptions.QuietWarnings = true; - break; - - case "-no-colored-log": - EAOptions.MonochromeLog = true; - break; - - case "quiet": - EAOptions.QuietMessages = true; - EAOptions.QuietWarnings = true; - break; - - case "-nocash-sym": - EAOptions.ProduceNocashSym = true; - break; - - case "-build-times": - EAOptions.BenchmarkBuildTimes = true; - break; - - case "I": - case "-include": - EAOptions.IncludePaths.Add(flag[1]); - break; - - case "T": - case "-tools": - EAOptions.ToolsPaths.Add(flag[1]); - break; - - case "IT": - case "TI": - EAOptions.IncludePaths.Add(flag[1]); - EAOptions.ToolsPaths.Add(flag[1]); - break; - - case "h": - case "-help": - Console.Out.WriteLine(helpstring); - return EXIT_SUCCESS; - - case "D": - case "def": - case "define": - try - { - string[] def_args = flag[1].Split(new char[] { '=' }, 2); - EAOptions.PreDefintions.Add((def_args[0], def_args[1])); - } - catch (IndexOutOfRangeException) - { - Console.Error.WriteLine("Improperly formed -define directive."); - } - break; + try + { + switch (flag[0]) + { + case "-raws": + rawsFolder = rawSearcher.FindDirectory(flag[1]); - case "romoffset": - case "-base-address": - try + if (rawsFolder == null) + { + Console.Error.WriteLine($"No such folder: {flag[1]}"); + return EXIT_FAILURE; + } + + break; + + case "-rawsExt": + rawsExtension = flag[1]; + break; + + case "-output": + outFileName = flag[1]; + break; + + case "-input": + inFileName = flag[1]; + inStream = File.OpenRead(flag[1]); + break; + + case "-error": + errorStream = new StreamWriter(File.OpenWrite(flag[1])); + EAOptions.MonochromeLog = true; + break; + + case "-debug": + Debug = true; + break; + + case "-werr": + EAOptions.WarningsAreErrors = true; + break; + + case "--no-mess": + EAOptions.QuietMessages = true; + break; + + case "--no-warn": + EAOptions.QuietWarnings = true; + break; + + case "--no-colored-log": + EAOptions.MonochromeLog = true; + break; + + case "-quiet": + case "--quiet": + EAOptions.QuietMessages = true; + EAOptions.QuietWarnings = true; + break; + + case "--nocash-sym": + EAOptions.ProduceNocashSym = true; + break; + + case "--build-times": + EAOptions.BenchmarkBuildTimes = true; + break; + + case "-I": + case "--include": + EAOptions.IncludePaths.Add(flag[1]); + break; + + case "-T": + case "--tools": + EAOptions.ToolsPaths.Add(flag[1]); + break; + + case "-IT": + case "-TI": + EAOptions.IncludePaths.Add(flag[1]); + EAOptions.ToolsPaths.Add(flag[1]); + break; + + case "-h": + case "--help": + Console.Out.WriteLine(helpstring); + return EXIT_SUCCESS; + + case "-D": + case "-def": + case "-define": + try + { + string[] def_args = flag[1].Split(new char[] { '=' }, 2); + EAOptions.PreDefintions.Add((def_args[0], def_args[1])); + } + catch (IndexOutOfRangeException) + { + Console.Error.WriteLine("Improperly formed -define directive."); + } + break; + + case "-romoffset": + case "--base-address": + try + { + EAOptions.BaseAddress = Convert.ToInt32(flag[1], 16); + } + catch + { + Console.Error.WriteLine("Invalid hex base address given for binary."); + } + break; + + case "--maximum-size": + try + { + EAOptions.MaximumBinarySize = Convert.ToInt32(flag[1], 16); + } + catch + { + Console.Error.WriteLine("Invalid hex size given for binary."); + } + break; + + case "-W": + case "--warnings": + if (flag.Length == 1) + { + EAOptions.EnabledWarnings |= EAOptions.Warnings.All; + } + else + { + for (int j = 1; j < flag.Length; j++) { - EAOptions.BaseAddress = Convert.ToInt32(flag[1], 16); + string name = flag[j]; + bool invert = false; + + if (name.StartsWith("no-")) + { + name = name.Substring(3); + invert = true; + } + + if (warningNames.TryGetValue(name, out EAOptions.Warnings warn)) + { + if (invert) + { + EAOptions.EnabledWarnings &= ~warn; + } + else + { + EAOptions.EnabledWarnings |= warn; + } + } + else + { + Console.Error.WriteLine($"Unrecognized warning: {name}"); + } } - catch + } + + break; + + case "--extensions": + if (flag.Length == 1) + { + EAOptions.EnabledWarnings |= EAOptions.Warnings.All; + } + else + { + for (int j = 1; j < flag.Length; j++) { - Console.Error.WriteLine("Invalid hex base address given for binary."); + string name = flag[j]; + bool invert = false; + + if (name.StartsWith("no-")) + { + name = name.Substring(3); + invert = true; + } + + if (extensionNames.TryGetValue(name, out EAOptions.Extensions ext)) + { + if (invert) + { + EAOptions.EnabledExtensions &= ~ext; + } + else + { + EAOptions.EnabledExtensions |= ext; + } + } + else + { + Console.Error.WriteLine($"Unrecognized extension: {name}"); + } } - break; + } - case "-maximum-size": - try - { - EAOptions.MaximumBinarySize = Convert.ToInt32(flag[1], 16); - } - catch - { - Console.Error.WriteLine("Invalid hex size given for binary."); - } - break; + break; - default: - Console.Error.WriteLine("Unrecognized flag: " + flag[0]); - return EXIT_FAILURE; - } - } - catch (IOException e) - { - Console.Error.WriteLine("Exception: " + e.Message); - return EXIT_FAILURE; + default: + Console.Error.WriteLine($"Unrecognized flag: {flag[0]}"); + return EXIT_FAILURE; } } + catch (IOException e) + { + Console.Error.WriteLine("Exception: " + e.Message); + return EXIT_FAILURE; + } } if (outFileName == null) From 3608375e388ac03a8368840ccc84ef645d298154 Mon Sep 17 00:00:00 2001 From: Nat_776 Date: Sun, 28 Apr 2024 14:07:34 +0200 Subject: [PATCH 38/59] attempt at a better diagnostic reguarding unintuitive expressions from macros --- ColorzCore/EAOptions.cs | 10 +- ColorzCore/IO/Log.cs | 25 ++- ColorzCore/Parser/AST/NumberNode.cs | 20 +- ColorzCore/Parser/AST/OperatorNode.cs | 83 ++++---- ColorzCore/Parser/AST/UnaryOperatorNode.cs | 28 +-- ColorzCore/Parser/AtomVisitor.cs | 32 +++ ColorzCore/Parser/DiagnosticHelpers.cs | 204 ++++++++++++++++++++ ColorzCore/Parser/EAParser.cs | 71 +++---- ColorzCore/Preprocessor/Macros/UserMacro.cs | 2 +- ColorzCore/Program.cs | 5 +- 10 files changed, 361 insertions(+), 119 deletions(-) create mode 100644 ColorzCore/Parser/AtomVisitor.cs create mode 100644 ColorzCore/Parser/DiagnosticHelpers.cs diff --git a/ColorzCore/EAOptions.cs b/ColorzCore/EAOptions.cs index 12fe9c7..e5fdc6d 100644 --- a/ColorzCore/EAOptions.cs +++ b/ColorzCore/EAOptions.cs @@ -20,10 +20,14 @@ public enum Warnings : long // warn on write before ORG UninitializedOffset = 4, - // warn on expansion of unguarded expression within macro - UnguardedExpressionMacros = 8, + // warn on unintuitive macro expansions (#define A 1 + 2 ... BYTE A * 2 ) + UnintuitiveExpressionMacros = 8, - All = long.MaxValue, + // warn on expansion of unguarded expression within macro () + UnguardedExpressionMacros = 16, + + Extra = UnguardedExpressionMacros, + All = long.MaxValue & ~Extra, } [Flags] diff --git a/ColorzCore/IO/Log.cs b/ColorzCore/IO/Log.cs index 950327c..705523b 100644 --- a/ColorzCore/IO/Log.cs +++ b/ColorzCore/IO/Log.cs @@ -14,6 +14,7 @@ public enum MessageKind NOTE, MESSAGE, DEBUG, + CONTINUE, } public bool HasErrored { get; private set; } = false; @@ -25,6 +26,9 @@ public enum MessageKind public TextWriter Output { get; set; } = Console.Error; + private MessageKind continueKind = MessageKind.CONTINUE; + private int continueLength = 0; + protected struct LogDisplayConfig { public string tag; @@ -57,26 +61,41 @@ public void Message(MessageKind kind, Location? source, string message) kind = MessageKind.ERROR; } - HasErrored |= (kind == MessageKind.ERROR); + HasErrored |= kind == MessageKind.ERROR; - if (!IgnoredKinds.Contains(kind)) + if (kind == MessageKind.CONTINUE) + { + if (continueKind != MessageKind.CONTINUE && !IgnoredKinds.Contains(continueKind)) + { + Console.Write("".PadLeft(continueLength)); + Console.WriteLine(message); + } + } + else if (!IgnoredKinds.Contains(kind)) { if (KIND_DISPLAY_DICT.TryGetValue(kind, out LogDisplayConfig config)) { if (!NoColoredTags && config.tagColor.HasValue) + { Console.ForegroundColor = config.tagColor.Value; + } Output.Write($"{config.tag}: "); + continueLength = config.tag.Length + 2; if (!NoColoredTags) Console.ResetColor(); if (source.HasValue) { - Output.Write($"{source.Value}: "); + string locString = source.Value.ToString(); + Output.Write($"{locString}: "); + continueLength += locString.Length + 2; } Output.WriteLine(message); + + continueKind = kind; } } } diff --git a/ColorzCore/Parser/AST/NumberNode.cs b/ColorzCore/Parser/AST/NumberNode.cs index f03b8e0..a6ca3d0 100644 --- a/ColorzCore/Parser/AST/NumberNode.cs +++ b/ColorzCore/Parser/AST/NumberNode.cs @@ -10,7 +10,7 @@ namespace ColorzCore.Parser.AST { public class NumberNode : AtomNodeKernel { - private int value; + public int Value { get; } public override Location MyLocation { get; } public override int Precedence { get { return 11; } } @@ -18,29 +18,29 @@ public class NumberNode : AtomNodeKernel public NumberNode(Token num) { MyLocation = num.Location; - value = num.Content.ToInt(); + Value = num.Content.ToInt(); } public NumberNode(Token text, int value) { MyLocation = text.Location; - this.value = value; + Value = value; } public NumberNode(Location loc, int value) { MyLocation = loc; - this.value = value; + Value = value; } - public override IEnumerable ToTokens() { yield return new Token(TokenType.NUMBER, MyLocation, value.ToString()); } - - public override int? TryEvaluate(Action handler, EvaluationPhase evaluationPhase) + public override IEnumerable ToTokens() { - return value; + yield return new Token(TokenType.NUMBER, MyLocation, Value.ToString()); } - public override string PrettyPrint() + public override int? TryEvaluate(Action handler, EvaluationPhase evaluationPhase) { - return "0x" + value.ToString("X"); + return Value; } + + public override string PrettyPrint() => Value >= 16 ? $"0x{Value:X}" : $"{Value}"; } } diff --git a/ColorzCore/Parser/AST/OperatorNode.cs b/ColorzCore/Parser/AST/OperatorNode.cs index 80ab03c..393946b 100644 --- a/ColorzCore/Parser/AST/OperatorNode.cs +++ b/ColorzCore/Parser/AST/OperatorNode.cs @@ -4,15 +4,13 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Text; -using System.Threading.Tasks; namespace ColorzCore.Parser.AST { - class OperatorNode : AtomNodeKernel + public class OperatorNode : AtomNodeKernel { - private IAtomNode left, right; - + public IAtomNode Left { get; private set; } + public IAtomNode Right { get; private set; } public Token OperatorToken { get; } public override int Precedence { get; } @@ -20,53 +18,50 @@ class OperatorNode : AtomNodeKernel public OperatorNode(IAtomNode l, Token op, IAtomNode r, int prec) { - left = l; - right = r; + Left = l; + Right = r; OperatorToken = op; Precedence = prec; } - public override string PrettyPrint() + public string OperatorString => OperatorToken.Type switch { - static string GetOperatorString(TokenType tokenType) - { - return tokenType switch - { - TokenType.MUL_OP => "*", - TokenType.DIV_OP => "/", - TokenType.MOD_OP => "%", - TokenType.ADD_OP => "+", - TokenType.SUB_OP => "-", - TokenType.LSHIFT_OP => "<<", - TokenType.RSHIFT_OP => ">>", - TokenType.SIGNED_RSHIFT_OP => ">>>", - TokenType.UNDEFINED_COALESCE_OP => "??", - TokenType.AND_OP => "&", - TokenType.XOR_OP => "^", - TokenType.OR_OP => "|", - TokenType.LOGAND_OP => "&&", - TokenType.LOGOR_OP => "||", - TokenType.COMPARE_EQ => "==", - TokenType.COMPARE_NE => "!=", - TokenType.COMPARE_LT => "<", - TokenType.COMPARE_LE => "<=", - TokenType.COMPARE_GT => ">", - TokenType.COMPARE_GE => ">=", - _ => "" - }; - } + TokenType.MUL_OP => "*", + TokenType.DIV_OP => "/", + TokenType.MOD_OP => "%", + TokenType.ADD_OP => "+", + TokenType.SUB_OP => "-", + TokenType.LSHIFT_OP => "<<", + TokenType.RSHIFT_OP => ">>", + TokenType.SIGNED_RSHIFT_OP => ">>>", + TokenType.UNDEFINED_COALESCE_OP => "??", + TokenType.AND_OP => "&", + TokenType.XOR_OP => "^", + TokenType.OR_OP => "|", + TokenType.LOGAND_OP => "&&", + TokenType.LOGOR_OP => "||", + TokenType.COMPARE_EQ => "==", + TokenType.COMPARE_NE => "!=", + TokenType.COMPARE_LT => "<", + TokenType.COMPARE_LE => "<=", + TokenType.COMPARE_GT => ">", + TokenType.COMPARE_GE => ">=", + _ => "" + }; - return $"({left.PrettyPrint()} {GetOperatorString(OperatorToken.Type)} {right.PrettyPrint()})"; + public override string PrettyPrint() + { + return $"({Left.PrettyPrint()} {OperatorString} {Right.PrettyPrint()})"; } public override IEnumerable ToTokens() { - foreach (Token t in left.ToTokens()) + foreach (Token t in Left.ToTokens()) { yield return t; } yield return OperatorToken; - foreach (Token t in right.ToTokens()) + foreach (Token t in Right.ToTokens()) { yield return t; } @@ -79,7 +74,7 @@ public override IEnumerable ToTokens() // the left side of an undefined coalescing operation is allowed to raise exactly UndefinedIdentifierException // we need to catch that, so don't forward all exceptions raised by left just yet - int? leftValue = left.TryEvaluate(e => (leftExceptions ??= new List()).Add(e), EvaluationPhase.Final); + int? leftValue = Left.TryEvaluate(e => (leftExceptions ??= new List()).Add(e), EvaluationPhase.Final); if (leftExceptions == null) { @@ -89,7 +84,7 @@ public override IEnumerable ToTokens() else if (leftExceptions.All(e => e is IdentifierNode.UndefinedIdentifierException)) { // left did not evalute due to undefined identifier => result is right - return right.TryEvaluate(handler, EvaluationPhase.Final); + return Right.TryEvaluate(handler, EvaluationPhase.Final); } else { @@ -129,11 +124,11 @@ public override IEnumerable ToTokens() } } - int? leftValue = left.TryEvaluate(handler, evaluationPhase); - leftValue.IfJust(i => left = new NumberNode(left.MyLocation, i)); + int? leftValue = Left.TryEvaluate(handler, evaluationPhase); + leftValue.IfJust(i => Left = new NumberNode(Left.MyLocation, i)); - int? rightValue = right.TryEvaluate(handler, evaluationPhase); - rightValue.IfJust(i => right = new NumberNode(right.MyLocation, i)); + int? rightValue = Right.TryEvaluate(handler, evaluationPhase); + rightValue.IfJust(i => Right = new NumberNode(Right.MyLocation, i)); if (leftValue is int lhs && rightValue is int rhs) { diff --git a/ColorzCore/Parser/AST/UnaryOperatorNode.cs b/ColorzCore/Parser/AST/UnaryOperatorNode.cs index 768447f..06a140c 100644 --- a/ColorzCore/Parser/AST/UnaryOperatorNode.cs +++ b/ColorzCore/Parser/AST/UnaryOperatorNode.cs @@ -8,44 +8,44 @@ namespace ColorzCore.Parser.AST { - class UnaryOperatorNode : AtomNodeKernel + public class UnaryOperatorNode : AtomNodeKernel { - private readonly IAtomNode interior; + public IAtomNode Inner { get; private set; } public Token OperatorToken { get; } public UnaryOperatorNode(Token token, IAtomNode inside) { OperatorToken = token; - interior = inside; + Inner = inside; } public override int Precedence => 11; public override Location MyLocation => OperatorToken.Location; - public override string PrettyPrint() + public string OperatorString => OperatorToken.Type switch { - string operatorString = OperatorToken.Type switch - { - TokenType.SUB_OP => "-", - TokenType.NOT_OP => "~", - TokenType.LOGNOT_OP => "!", - _ => "", - }; + TokenType.SUB_OP => "-", + TokenType.NOT_OP => "~", + TokenType.LOGNOT_OP => "!", + _ => "", + }; - return operatorString + interior.PrettyPrint(); + public override string PrettyPrint() + { + return OperatorString + Inner.PrettyPrint(); } public override IEnumerable ToTokens() { yield return OperatorToken; - foreach (Token t in interior.ToTokens()) + foreach (Token t in Inner.ToTokens()) yield return t; } public override int? TryEvaluate(Action handler, EvaluationPhase evaluationPhase) { - int? inner = interior.TryEvaluate(handler, evaluationPhase); + int? inner = Inner.TryEvaluate(handler, evaluationPhase); if (inner != null) { diff --git a/ColorzCore/Parser/AtomVisitor.cs b/ColorzCore/Parser/AtomVisitor.cs new file mode 100644 index 0000000..fc1ac9f --- /dev/null +++ b/ColorzCore/Parser/AtomVisitor.cs @@ -0,0 +1,32 @@ + +using ColorzCore.Parser.AST; + +namespace ColorzCore.Parser +{ + public abstract class AtomVisitor + { + protected void Visit(IAtomNode node) + { + switch (node) + { + case OperatorNode operatorNode: + VisitNode(operatorNode); + break; + case UnaryOperatorNode unaryOperatorNode: + VisitNode(unaryOperatorNode); + break; + case IdentifierNode identifierNode: + VisitNode(identifierNode); + break; + case NumberNode numberNode: + VisitNode(numberNode); + break; + } + } + + protected abstract void VisitNode(UnaryOperatorNode node); + protected abstract void VisitNode(IdentifierNode node); + protected abstract void VisitNode(NumberNode node); + protected abstract void VisitNode(OperatorNode node); + } +} diff --git a/ColorzCore/Parser/DiagnosticHelpers.cs b/ColorzCore/Parser/DiagnosticHelpers.cs new file mode 100644 index 0000000..b0983c0 --- /dev/null +++ b/ColorzCore/Parser/DiagnosticHelpers.cs @@ -0,0 +1,204 @@ +using System; +using System.Collections.Generic; +using System.Text; +using ColorzCore.DataTypes; +using ColorzCore.Lexer; +using ColorzCore.Parser.AST; + +namespace ColorzCore.Parser +{ + public static class DiagnosticHelpers + { + // Helper class for printing expressions (IAtomNode) with some bits emphasized + private class EmphasisExpressionPrinter : AtomVisitor + { + readonly StringBuilder stringBuilder = new StringBuilder(); + readonly StringBuilder underlineBuilder = new StringBuilder(); + + readonly Func emphasisPredicate; + bool wasEmphasized = false; + + public EmphasisExpressionPrinter(Func predicate) + { + emphasisPredicate = predicate; + // targetMacroLocation = macroLocation; + } + + public string PrintExpression(IAtomNode expression) + { + stringBuilder.Clear(); + underlineBuilder.Clear(); + + Visit(expression); + + return $"{stringBuilder}\n{underlineBuilder}"; + } + + private void AppendString(bool strong, string value) + { + if (strong) + { + stringBuilder.Append("\x1B[1;37m"); + stringBuilder.Append(value); + stringBuilder.Append("\x1B[0m"); + } + else + { + stringBuilder.Append(value); + } + + underlineBuilder.Append(strong ? '~' : ' ', value.Length); + + wasEmphasized = strong; + } + + protected override void VisitNode(OperatorNode node) + { + AppendString(emphasisPredicate(node.Left.MyLocation), "("); + + Visit(node.Left); + + AppendString(emphasisPredicate(node.OperatorToken.Location), $" {node.OperatorString} "); + + Visit(node.Right); + + AppendString(wasEmphasized, ")"); + } + + protected override void VisitNode(UnaryOperatorNode node) + { + AppendString(emphasisPredicate(node.OperatorToken.Location), node.OperatorString); + + Visit(node.Inner); + } + + protected override void VisitNode(IdentifierNode node) + { + AppendString(emphasisPredicate(node.MyLocation), node.GetIdentifier()!); + } + + protected override void VisitNode(NumberNode node) + { + AppendString(emphasisPredicate(node.MyLocation), node.PrettyPrint()); + } + } + + // Gets string representation of expression with emphasis on locations for which the predicate is true + public static string GetEmphasizedExpression(IAtomNode node, Func emphasisPredicate) + { + return new EmphasisExpressionPrinter(emphasisPredicate).PrintExpression(node); + } + + /* + // Print expression (unused) + public static string PrettyPrintExpression(IAtomNode node) + { + return GetEmphasizedExpression(node, _ => false); + } + */ + + // visits operators that aren't around parenthesises or brackets + public static void VisitUnguardedOperators(IList tokens, Action action) + { + if (tokens.Count > 1) + { + int paren = 0; + int bracket = 0; + + foreach (Token token in tokens) + { + switch (token.Type) + { + case TokenType.OPEN_PAREN: + paren++; + break; + + case TokenType.CLOSE_PAREN: + paren--; + break; + + case TokenType.OPEN_BRACKET: + bracket++; + break; + + case TokenType.CLOSE_BRACKET: + bracket--; + break; + + default: + if (paren == 0 && bracket == 0 && EAParser.IsInfixOperator(token)) + { + action.Invoke(token); + } + + break; + } + } + } + } + + // helper for DoesOperationSpanMultipleMacrosUnintuitively + private static IAtomNode GetLeftmostInnerNode(IAtomNode node) + { + if (node is OperatorNode operatorNode) + { + return GetLeftmostInnerNode(operatorNode.Left); + } + + return node; + } + + // helper for DoesOperationSpanMultipleMacrosUnintuitively + private static IAtomNode GetRightmostInnerNode(IAtomNode node) + { + if (node is OperatorNode operatorNode) + { + return GetRightmostInnerNode(operatorNode.Right); + } + + if (node is UnaryOperatorNode unaryOperatorNode) + { + return GetRightmostInnerNode(unaryOperatorNode.Inner); + } + + return node; + } + + // returns true if node spans multiple macros unintuitively + public static bool DoesOperationSpanMultipleMacrosUnintuitively(OperatorNode operatorNode) + { + /* The condition for this diagnostic are as follows: + * 1. The operator node is from the same macro expansion as the closest node on either side + * 2. The operator node is not from the same macro expansion as the furthest node on that same side */ + + MacroLocation? macroLocation = operatorNode.MyLocation.macroLocation; + + // do not check for non-macros expressions + if (macroLocation != null) + { + IAtomNode left = operatorNode.Left; + IAtomNode right = operatorNode.Right; + + // TODO: verify operator token rather than furthest node? + + if (macroLocation == GetRightmostInnerNode(left).MyLocation.macroLocation) + { + if (macroLocation != GetLeftmostInnerNode(left).MyLocation.macroLocation) + { + return true; + } + } + + if (macroLocation == GetLeftmostInnerNode(right).MyLocation.macroLocation) + { + if (macroLocation != GetRightmostInnerNode(right).MyLocation.macroLocation) + { + return true; + } + } + } + + return false; + } + } +} diff --git a/ColorzCore/Parser/EAParser.cs b/ColorzCore/Parser/EAParser.cs index e68a20a..876c542 100644 --- a/ColorzCore/Parser/EAParser.cs +++ b/ColorzCore/Parser/EAParser.cs @@ -686,6 +686,8 @@ public IList ParsePreprocParamList(MergeableGenerator tokens, { TokenType.UNDEFINED_COALESCE_OP, 13 }, }; + public static bool IsInfixOperator(Token token) => precedences.ContainsKey(token.Type); + public IAtomNode? ParseAtom(MergeableGenerator tokens, ImmutableStack scopes, bool expandDefs = true) { //Use Shift Reduce Parsing @@ -866,11 +868,11 @@ public IList ParsePreprocParamList(MergeableGenerator tokens, * Postcondition: Either grammarSymbols.Count == 1, or everything in grammarSymbols will have precedence <= targetPrecedence. * */ - private static void Reduce(Stack> grammarSymbols, int targetPrecedence) + private void Reduce(Stack> grammarSymbols, int targetPrecedence) { while (grammarSymbols.Count > 1)// && grammarSymbols.Peek().GetLeft.Precedence > targetPrecedence) { - //These shouldn't error... + // These shouldn't error... IAtomNode r = grammarSymbols.Pop().GetLeft; if (precedences[grammarSymbols.Peek().GetRight.Type] > targetPrecedence) @@ -883,7 +885,16 @@ private static void Reduce(Stack> grammarSymbols, int t Token op = grammarSymbols.Pop().GetRight; IAtomNode l = grammarSymbols.Pop().GetLeft; - grammarSymbols.Push(new Left(new OperatorNode(l, op, r, l.Precedence))); + OperatorNode operatorNode = new OperatorNode(l, op, r, l.Precedence); + + if (DiagnosticHelpers.DoesOperationSpanMultipleMacrosUnintuitively(operatorNode)) + { + MacroLocation? mloc = operatorNode.MyLocation.macroLocation; + string expr = DiagnosticHelpers.GetEmphasizedExpression(operatorNode, l => l.macroLocation == mloc); + Warning(operatorNode.MyLocation, $"{expr}\nUnintuitive macro expansion within expression.\nThis may be a mistake. Consider guarding your expressions using parenthesis."); + } + + grammarSymbols.Push(new Left(operatorNode)); } } } @@ -1138,41 +1149,8 @@ private void ApplyMacroExpansion(MergeableGenerator tokens, IEnumerable expandedList = expandedTokens.ToList(); - if (expandedList.Count > 1) - { - int paren = 0; - int bracket = 0; - - foreach (Token token in expandedList) - { - switch (token.Type) - { - case TokenType.OPEN_PAREN: - paren++; - break; - - case TokenType.CLOSE_PAREN: - paren--; - break; - - case TokenType.OPEN_BRACKET: - bracket++; - break; - - case TokenType.CLOSE_BRACKET: - bracket--; - break; - - default: - if (paren == 0 && bracket == 0 && precedences.ContainsKey(token.Type)) - { - Warning(token.Location, $"Unguarded expansion of mathematical operator. Consider adding guarding parenthesises around definition."); - } - - break; - } - } - } + DiagnosticHelpers.VisitUnguardedOperators(expandedList, + token => Warning(token.Location, $"Unguarded expansion of mathematical operator. Consider adding guarding parenthesises around definition.")); tokens.PrependEnumerator(expandedList.GetEnumerator()); } @@ -1184,13 +1162,20 @@ private void ApplyMacroExpansion(MergeableGenerator tokens, IEnumerable ApplyMacro(Token head, IList> parameters, { foreach (Token paramToken in parameters[paramNum]) { - yield return paramToken; + yield return paramToken.MacroClone(macroLocation); } } else diff --git a/ColorzCore/Program.cs b/ColorzCore/Program.cs index 24724d0..0613a55 100644 --- a/ColorzCore/Program.cs +++ b/ColorzCore/Program.cs @@ -13,9 +13,11 @@ class Program private static readonly IDictionary warningNames = new Dictionary() { { "nonportable-pathnames", EAOptions.Warnings.NonPortablePath }, + { "unintuitive-expression-macros" , EAOptions.Warnings.UnintuitiveExpressionMacros }, { "unguarded-expression-macros", EAOptions.Warnings.UnguardedExpressionMacros }, { "redefine", EAOptions.Warnings.ReDefine }, { "all", EAOptions.Warnings.All }, + { "extra", EAOptions.Warnings.Extra }, }; private static readonly IDictionary extensionNames = new Dictionary() @@ -52,7 +54,8 @@ class Program "-IT:|-TI:", " Combines --include: and --tools:.", "-W:[no-]:...|--warnings:[no-]:...", - " Enable or disable warnings. By default, all warnings are enabled.", + " Enable or disable warnings.", + " By default, all warnings but 'unguarded-expression-macros' are enabled.", " Multiple warnings can be enabled/disabled at once.", " Example: '--warnings:no-nonportable-pathnames:no-redefine'.", " Possible values: " + string.Join(", ", warningNames.Keys), From 55fdf6472ba0606b5765a8a24baa7ee08a160640 Mon Sep 17 00:00:00 2001 From: Nat_776 Date: Sun, 28 Apr 2024 15:12:07 +0200 Subject: [PATCH 39/59] adjustments --- ColorzCore/ColorzCore.Framework.csproj | 6 ++---- ColorzCore/ColorzCore.csproj | 4 +--- ColorzCore/Lexer/Tokenizer.cs | 19 +++++++++++------- ColorzCore/Parser/DiagnosticHelpers.cs | 20 +++++++++---------- ColorzCore/Parser/EAParser.cs | 16 +++++++++++++-- .../Directives/DefineDirective.cs | 2 ++ .../Directives/IncludeDirective.cs | 2 +- 7 files changed, 42 insertions(+), 27 deletions(-) diff --git a/ColorzCore/ColorzCore.Framework.csproj b/ColorzCore/ColorzCore.Framework.csproj index 2035bb8..0184d24 100644 --- a/ColorzCore/ColorzCore.Framework.csproj +++ b/ColorzCore/ColorzCore.Framework.csproj @@ -3,10 +3,8 @@ net48 Exe annotations - ColorzCore - - - + ColorzCore 9 + bin/Framework diff --git a/ColorzCore/ColorzCore.csproj b/ColorzCore/ColorzCore.csproj index f1236eb..11a245f 100644 --- a/ColorzCore/ColorzCore.csproj +++ b/ColorzCore/ColorzCore.csproj @@ -3,9 +3,7 @@ net6.0 Exe enable - - - 9 + bin/Core \ No newline at end of file diff --git a/ColorzCore/Lexer/Tokenizer.cs b/ColorzCore/Lexer/Tokenizer.cs index 35664d9..3de3d35 100644 --- a/ColorzCore/Lexer/Tokenizer.cs +++ b/ColorzCore/Lexer/Tokenizer.cs @@ -15,7 +15,7 @@ class Tokenizer public const int MAX_ID_LENGTH = 64; public static readonly Regex numRegex = new Regex("\\G([01]+b|0x[\\da-fA-F]+|\\$[\\da-fA-F]+|\\d+)"); public static readonly Regex idRegex = new Regex("\\G([a-zA-Z_][a-zA-Z0-9_]*)"); - public static readonly Regex stringRegex = new Regex("\\G(([^\\\"]|\\\\\\\")*)"); //"\\G(([^\\\\\\\"]|\\\\[rnt\\\\\\\"])*)"); + public static readonly Regex stringRegex = new Regex(@"\G(([^\""]|\\\"")*)"); //"\\G(([^\\\\\\\"]|\\\\[rnt\\\\\\\"])*)"); public static readonly Regex winPathnameRegex = new Regex(string.Format("\\G([^ \\{0}]|\\ |\\\\)+", Process(Path.GetInvalidPathChars()))); public static readonly Regex preprocDirectiveRegex = new Regex("\\G(#[a-zA-Z_][a-zA-Z0-9_]*)"); public static readonly Regex wordRegex = new Regex("\\G([^\\s]+)"); @@ -198,7 +198,7 @@ public IEnumerable TokenizePhrase(string line, string fileName, int lineN curCol++; Match quoteInterior = stringRegex.Match(line, curCol, endOffs - curCol); string match = quoteInterior.Value; - yield return new Token(TokenType.STRING, fileName, lineNum, curCol, /*IOUtility.UnescapeString(*/match/*)*/); + yield return new Token(TokenType.STRING, fileName, lineNum, curCol + offset, /*IOUtility.UnescapeString(*/match/*)*/); curCol += match.Length; if (curCol == endOffs || line[curCol] != '\"') { @@ -322,7 +322,7 @@ public IEnumerable TokenizePhrase(string line, string fileName, int lineN if (idMatch.Success) { string match = idMatch.Value; - int idCol = curCol; + int idCol = curCol + offset; curCol += match.Length; if (curCol < endOffs && line[curCol] == '(') yield return new Token(TokenType.MAYBE_MACRO, fileName, lineNum, idCol, match); @@ -344,7 +344,7 @@ public IEnumerable TokenizePhrase(string line, string fileName, int lineN //Verify that next token isn't start of an identifier if (curCol + match.Length >= endOffs || (!char.IsLetter(line[curCol + match.Length]) && line[curCol + match.Length] != '_')) { - yield return new Token(TokenType.NUMBER, fileName, lineNum, curCol, match.TrimEnd()); + yield return new Token(TokenType.NUMBER, fileName, lineNum, curCol + offset, match.TrimEnd()); curCol += match.Length; continue; } @@ -353,7 +353,7 @@ public IEnumerable TokenizePhrase(string line, string fileName, int lineN if (directiveMatch.Success) { string match = directiveMatch.Value; - yield return new Token(TokenType.PREPROCESSOR_DIRECTIVE, fileName, lineNum, curCol, match); + yield return new Token(TokenType.PREPROCESSOR_DIRECTIVE, fileName, lineNum, curCol + offset, match); curCol += match.Length; if (match.Substring(1).Equals("include") || match.Substring(1).Equals("incbin")) { @@ -412,11 +412,16 @@ public IEnumerable Tokenize(Stream input, string fileName) } } - public IEnumerable Tokenize(FileStream fs) + public IEnumerable TokenizeFile(FileStream fs, string filename) { - foreach (Token t in Tokenize(fs, fs.Name)) + foreach (Token t in Tokenize(fs, filename)) yield return t; fs.Close(); } + + public IEnumerable Tokenize(FileStream fs) + { + return TokenizeFile(fs, fs.Name); + } } } diff --git a/ColorzCore/Parser/DiagnosticHelpers.cs b/ColorzCore/Parser/DiagnosticHelpers.cs index b0983c0..1f71633 100644 --- a/ColorzCore/Parser/DiagnosticHelpers.cs +++ b/ColorzCore/Parser/DiagnosticHelpers.cs @@ -169,29 +169,29 @@ public static bool DoesOperationSpanMultipleMacrosUnintuitively(OperatorNode ope { /* The condition for this diagnostic are as follows: * 1. The operator node is from the same macro expansion as the closest node on either side - * 2. The operator node is not from the same macro expansion as the furthest node on that same side */ + * 2. The operator node is not from the same macro expansion as the operator token on that same side */ MacroLocation? macroLocation = operatorNode.MyLocation.macroLocation; - // do not check for non-macros expressions - if (macroLocation != null) - { - IAtomNode left = operatorNode.Left; - IAtomNode right = operatorNode.Right; - - // TODO: verify operator token rather than furthest node? + IAtomNode left = operatorNode.Left; + IAtomNode right = operatorNode.Right; + if (left is OperatorNode leftNode) + { if (macroLocation == GetRightmostInnerNode(left).MyLocation.macroLocation) { - if (macroLocation != GetLeftmostInnerNode(left).MyLocation.macroLocation) + if (macroLocation != leftNode.OperatorToken.Location.macroLocation) { return true; } } + } + if (right is OperatorNode rightNode) + { if (macroLocation == GetLeftmostInnerNode(right).MyLocation.macroLocation) { - if (macroLocation != GetRightmostInnerNode(right).MyLocation.macroLocation) + if (macroLocation != rightNode.OperatorToken.Location.macroLocation) { return true; } diff --git a/ColorzCore/Parser/EAParser.cs b/ColorzCore/Parser/EAParser.cs index 876c542..d3c81af 100644 --- a/ColorzCore/Parser/EAParser.cs +++ b/ColorzCore/Parser/EAParser.cs @@ -890,8 +890,20 @@ private void Reduce(Stack> grammarSymbols, int targetPr if (DiagnosticHelpers.DoesOperationSpanMultipleMacrosUnintuitively(operatorNode)) { MacroLocation? mloc = operatorNode.MyLocation.macroLocation; - string expr = DiagnosticHelpers.GetEmphasizedExpression(operatorNode, l => l.macroLocation == mloc); - Warning(operatorNode.MyLocation, $"{expr}\nUnintuitive macro expansion within expression.\nThis may be a mistake. Consider guarding your expressions using parenthesis."); + string message = DiagnosticHelpers.GetEmphasizedExpression(operatorNode, l => l.macroLocation == mloc); + + if (mloc != null) + { + message += $"\nUnintuitive expression resulting from expansion of macro `{mloc.MacroName}`."; + } + else + { + message += "\nUnintuitive expression resulting from expansion of macro."; + } + + message += "\nConsider guarding your expressions using parenthesis."; + + Warning(operatorNode.MyLocation, message); } grammarSymbols.Push(new Left(operatorNode)); diff --git a/ColorzCore/Preprocessor/Directives/DefineDirective.cs b/ColorzCore/Preprocessor/Directives/DefineDirective.cs index 576fd7c..5de7695 100644 --- a/ColorzCore/Preprocessor/Directives/DefineDirective.cs +++ b/ColorzCore/Preprocessor/Directives/DefineDirective.cs @@ -114,6 +114,8 @@ private static IList ExpandMacroBody(EAParser _, IList body) { if (body.Count == 1 && body[0].Type == TokenType.STRING) { + // FIXME: for some reason, locations of tokens in this are offset by 1 + Token token = body[0]; return new List(Tokenizer.TokenizeLine(token.Content, token.Location)); } diff --git a/ColorzCore/Preprocessor/Directives/IncludeDirective.cs b/ColorzCore/Preprocessor/Directives/IncludeDirective.cs index 4ec546d..5cb8613 100644 --- a/ColorzCore/Preprocessor/Directives/IncludeDirective.cs +++ b/ColorzCore/Preprocessor/Directives/IncludeDirective.cs @@ -48,7 +48,7 @@ class IncludeDirective : SimpleDirective try { FileStream inputFile = new FileStream(existantFile, FileMode.Open); - tokens.PrependEnumerator(new Tokenizer().Tokenize(inputFile).GetEnumerator()); + tokens.PrependEnumerator(new Tokenizer().TokenizeFile(inputFile, existantFile.Replace('\\', '/')).GetEnumerator()); } catch (Exception e) { From b7596c7c17d734e3b3456aea91592c323b0d7093 Mon Sep 17 00:00:00 2001 From: Nat_776 Date: Sun, 28 Apr 2024 18:32:22 +0200 Subject: [PATCH 40/59] fixes --- ColorzCore/ColorzCore.Framework.csproj | 2 +- ColorzCore/IO/IOUtility.cs | 15 + ColorzCore/IO/Log.cs | 8 + ColorzCore/Parser/EAParser.cs | 2695 ++++++++++++------------ ColorzCore/Program.cs | 30 +- 5 files changed, 1390 insertions(+), 1360 deletions(-) diff --git a/ColorzCore/ColorzCore.Framework.csproj b/ColorzCore/ColorzCore.Framework.csproj index 0184d24..fc5ed33 100644 --- a/ColorzCore/ColorzCore.Framework.csproj +++ b/ColorzCore/ColorzCore.Framework.csproj @@ -3,7 +3,7 @@ net48 Exe annotations - ColorzCore + ColorzCore 9 bin/Framework diff --git a/ColorzCore/IO/IOUtility.cs b/ColorzCore/IO/IOUtility.cs index a0a6559..d7c8fe0 100644 --- a/ColorzCore/IO/IOUtility.cs +++ b/ColorzCore/IO/IOUtility.cs @@ -28,6 +28,21 @@ public static string GetToolFileName(string name) return RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? $"{name}.exe" : name; } + // HACK: this is to make Log not print the entire path every time + public static string GetPortableBasePathForPrefix(string name) + { + string? result = Path.GetDirectoryName(name); + + if (string.IsNullOrEmpty(result)) + { + return ""; + } + else + { + return (result + '/').Replace('\\', '/'); + } + } + private static readonly char[] pathSeparators = new char[] { Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar }; private static readonly char[] invalidFileCharacters = Path.GetInvalidFileNameChars(); diff --git a/ColorzCore/IO/Log.cs b/ColorzCore/IO/Log.cs index 705523b..ee81ccf 100644 --- a/ColorzCore/IO/Log.cs +++ b/ColorzCore/IO/Log.cs @@ -22,6 +22,8 @@ public enum MessageKind public bool NoColoredTags { get; set; } = false; + public string LocationBasePath { get; set; } = string.Empty; + public List IgnoredKinds { get; } = new List(); public TextWriter Output { get; set; } = Console.Error; @@ -89,6 +91,12 @@ public void Message(MessageKind kind, Location? source, string message) if (source.HasValue) { string locString = source.Value.ToString(); + + if (locString.StartsWith(LocationBasePath)) + { + locString = locString.Substring(LocationBasePath.Length); + } + Output.Write($"{locString}: "); continueLength += locString.Length + 2; } diff --git a/ColorzCore/Parser/EAParser.cs b/ColorzCore/Parser/EAParser.cs index d3c81af..8c64431 100644 --- a/ColorzCore/Parser/EAParser.cs +++ b/ColorzCore/Parser/EAParser.cs @@ -1,1344 +1,1351 @@ -using ColorzCore.DataTypes; -using ColorzCore.IO; -using ColorzCore.Lexer; -using ColorzCore.Parser.AST; -using ColorzCore.Preprocessor; -using ColorzCore.Preprocessor.Macros; -using ColorzCore.Raws; -using System; -using System.Collections.Generic; -using System.Globalization; -using System.IO; -using System.Linq; -using System.Text; -using System.Text.RegularExpressions; - -//TODO: Make errors less redundant (due to recursive nature, many paths will give several redundant errors). - -namespace ColorzCore.Parser -{ - public class EAParser - { - public MacroCollection Macros { get; } - public Dictionary Definitions { get; } - public Dictionary> Raws { get; } - public static readonly HashSet SpecialCodes = new HashSet { "ORG", "PUSH", "POP", "MESSAGE", "WARNING", "ERROR", "ASSERT", "PROTECT", "ALIGN", "FILL" }; - //public static readonly HashSet BuiltInMacros = new HashSet { "String", "AddToPool" }; - //TODO: Built in macros. - //public static readonly Dictionary BuiltInMacros; - public ImmutableStack GlobalScope { get; } - public int CurrentOffset - { - get => currentOffset; - - private set - { - if (value < 0 || value > EAOptions.MaximumBinarySize) - { - if (validOffset) //Error only the first time. - { - Error($"Invalid offset: {value:X}"); - validOffset = false; - } - } - else - { - currentOffset = value; - validOffset = true; - offsetInitialized = true; - } - } - - } - public ImmutableStack Inclusion { get; set; } - - public Pool Pool { get; private set; } - - private readonly DirectiveHandler directiveHandler; - - private readonly Stack> pastOffsets; // currentOffset, offsetInitialized - private readonly IList> protectedRegions; - - public Log log; - - public bool IsIncluding - { - get - { - bool acc = true; - - for (ImmutableStack temp = Inclusion; !temp.IsEmpty && acc; temp = temp.Tail) - { - acc &= temp.Head; - } - - return acc; - } - } - - private bool validOffset; - private bool offsetInitialized; // false until first ORG, used to warn about writing before first org - private int currentOffset; - private Token? head; //TODO: Make this make sense - - public EAParser(Dictionary> raws, Log log, DirectiveHandler directiveHandler) - { - GlobalScope = new ImmutableStack(new BaseClosure(), ImmutableStack.Nil); - pastOffsets = new Stack>(); - protectedRegions = new List>(); - this.log = log; - Raws = raws; - CurrentOffset = 0; - validOffset = true; - offsetInitialized = false; - Macros = new MacroCollection(this); - Definitions = new Dictionary(); - Inclusion = ImmutableStack.Nil; - this.directiveHandler = directiveHandler; - - Pool = new Pool(); - } - - public bool IsReservedName(string name) - { - return Raws.ContainsKey(name.ToUpperInvariant()) || SpecialCodes.Contains(name.ToUpperInvariant()); - } - public bool IsValidDefinitionName(string name) - { - return !(Definitions.ContainsKey(name) || IsReservedName(name)); - } - public bool IsValidMacroName(string name, int paramNum) - { - return !Macros.HasMacro(name, paramNum) && !IsReservedName(name); - } - public bool IsValidLabelName(string name) - { - return true;//!IsReservedName(name); - //TODO? - } - public IList ParseAll(IEnumerable tokenStream) - { - //TODO: Make BlockNode or EAProgramNode? - //Note must be strict to get all information on the closure before evaluating terms. - IList myLines = new List(); - MergeableGenerator tokens = new MergeableGenerator(tokenStream); - tokens.MoveNext(); - while (!tokens.EOS) - { - if (tokens.Current.Type != TokenType.NEWLINE || tokens.MoveNext()) - { - ILineNode? retVal = ParseLine(tokens, GlobalScope); - retVal.IfJust(n => myLines.Add(n)); - } - } - return myLines; - } - - private BlockNode ParseBlock(MergeableGenerator tokens, ImmutableStack scopes) - { - Location start = tokens.Current.Location; - tokens.MoveNext(); - BlockNode temp = new BlockNode(); - - while (!tokens.EOS && tokens.Current.Type != TokenType.CLOSE_BRACE) - { - ILineNode? x = ParseLine(tokens, scopes); - - if (x != null) - { - temp.Children.Add(x); - } - } - - if (!tokens.EOS) - { - tokens.MoveNext(); - } - else - { - Error(start, "Unmatched brace."); - } - - return temp; - } - - // TODO: these next two functions should probably be moved into their own module - - public static int ConvertToAddress(int value) - { - /* - NOTE: Offset 0 is always converted to a null address - If one wants to instead refer to ROM offset 0 they would want to use the address directly instead. - If ROM offset 0 is already address 0 then this is a moot point. - */ - - if (value > 0 && value < EAOptions.MaximumBinarySize) - { - value += EAOptions.BaseAddress; - } - - return value; - } - - public static int ConvertToOffset(int value) - { - if (value >= EAOptions.BaseAddress && value <= EAOptions.BaseAddress + EAOptions.MaximumBinarySize) - { - value -= EAOptions.BaseAddress; - } - - return value; - } - - private ILineNode? ParseStatement(MergeableGenerator tokens, ImmutableStack scopes) - { - while (ExpandIdentifier(tokens, scopes)) { } - - head = tokens.Current; - tokens.MoveNext(); - - //TODO: Replace with real raw information, and error if not valid. - IList parameters; - //TODO: Make intelligent to reject malformed parameters. - //TODO: Parse parameters after checking code validity. - if (tokens.Current.Type != TokenType.NEWLINE && tokens.Current.Type != TokenType.SEMICOLON) - { - parameters = ParseParamList(tokens, scopes); - } - else - { - parameters = new List(); - tokens.MoveNext(); - } - - string upperCodeIdentifier = head.Content.ToUpperInvariant(); - - if (SpecialCodes.Contains(upperCodeIdentifier)) - { - return upperCodeIdentifier switch - { - "ORG" => ParseOrgStatement(parameters), - "PUSH" => ParsePushStatement(parameters), - "POP" => ParsePopStatement(parameters), - "ASSERT" => ParseAssertStatement(parameters), - "PROTECT" => ParseProtectStatement(parameters), - "ALIGN" => ParseAlignStatement(parameters), - "FILL" => ParseFillStatement(parameters), - "MESSAGE" => ParseMessageStatement(parameters, scopes), - "WARNING" => ParseWarningStatement(parameters, scopes), - "ERROR" => ParseErrorStatement(parameters, scopes), - _ => null, // TODO: this is an error - }; - } - else if (Raws.TryGetValue(upperCodeIdentifier, out IList? raws)) - { - //TODO: Check for matches. Currently should type error. - foreach (Raw raw in raws) - { - if (raw.Fits(parameters)) - { - if ((CurrentOffset % raw.Alignment) != 0) - { - Error($"Bad code alignment (offset: {CurrentOffset:X8})"); - } - - StatementNode temp = new RawNode(raw, head, CurrentOffset, parameters); - - // TODO: more efficient spacewise to just have contiguous writing and not an offset with every line? - CheckDataWrite(temp.Size); - CurrentOffset += temp.Size; - - return temp; - } - } - - if (raws.Count == 1) - { - Error($"Incorrect parameters in raw `{raws[0].ToPrettyString()}`"); - } - else - { - Error($"Couldn't find suitable variant of raw `{head.Content}`."); - - for (int i = 0; i < raws.Count; i++) - { - Error($"Variant {i + 1}: `{raws[i].ToPrettyString()}`"); - } - } - - IgnoreRestOfStatement(tokens); - return null; - } - else - { - Error("Unrecognized code: " + head.Content); - return null; - } - } - - private ILineNode? ParseOrgStatement(IList parameters) - { - if (parameters.Count != 1) - { - Error($"Incorrect number of parameters in ORG: {parameters.Count}"); - return null; - } - - parameters[0].AsAtom().IfJust( - atom => atom.TryEvaluate(e => Error(parameters[0].MyLocation, e.Message), EvaluationPhase.Immediate).IfJust( - offsetValue => { CurrentOffset = ConvertToOffset(offsetValue); }, - () => Error(parameters[0].MyLocation, "Expected atomic param to ORG."))); - - return null; - } - - private ILineNode? ParsePushStatement(IList parameters) - { - if (parameters.Count != 0) - { - Error("Incorrect number of parameters in PUSH: " + parameters.Count); - } - else - { - pastOffsets.Push(new Tuple(CurrentOffset, offsetInitialized)); - } - - return null; - } - - private ILineNode? ParsePopStatement(IList parameters) - { - if (parameters.Count != 0) - { - Error($"Incorrect number of parameters in POP: {parameters.Count}"); - } - else if (pastOffsets.Count == 0) - { - Error("POP without matching PUSH."); - } - else - { - Tuple tuple = pastOffsets.Pop(); - - CurrentOffset = tuple.Item1; - offsetInitialized = tuple.Item2; - } - - return null; - } - - private ILineNode? ParseAssertStatement(IList parameters) - { - if (parameters.Count != 1) - { - Error($"Incorrect number of parameters in ASSERT: {parameters.Count}"); - return null; - } - - // helper for distinguishing boolean expressions and other expressions - static bool IsConditionalOperatorHelper(IAtomNode node) - { - return node switch - { - UnaryOperatorNode uon => uon.OperatorToken.Type switch - { - TokenType.LOGNOT_OP => true, - _ => false, - }, - - OperatorNode on => on.OperatorToken.Type switch - { - TokenType.LOGAND_OP => true, - TokenType.LOGOR_OP => true, - TokenType.COMPARE_EQ => true, - TokenType.COMPARE_NE => true, - TokenType.COMPARE_GT => true, - TokenType.COMPARE_GE => true, - TokenType.COMPARE_LE => true, - TokenType.COMPARE_LT => true, - _ => false, - }, - - _ => false, - }; - } - - IAtomNode? atom = parameters[0].AsAtom(); - - if (atom != null) - { - bool isBoolean = IsConditionalOperatorHelper(atom); - - atom.TryEvaluate(e => Error(parameters[0].MyLocation, e.Message), EvaluationPhase.Immediate).IfJust( - temp => - { - // if boolean expession => fail if 0, else (legacy behavoir) fail if negative - if (isBoolean && temp == 0 || !isBoolean && temp < 0) - { - Error(parameters[0].MyLocation, "Assertion error: " + temp); - } - }); - } - else - { - Error(parameters[0].MyLocation, "Expected atomic param to ASSERT."); - } - - return null; - } - - private ILineNode? ParseProtectStatement(IList parameters) - { - if (parameters.Count == 1) - { - parameters[0].AsAtom().IfJust( - atom => atom.TryEvaluate(e => { Error(parameters[0].MyLocation, e.Message); }, EvaluationPhase.Immediate).IfJust( - temp => - { - protectedRegions.Add(new Tuple(temp, 4, head!.Location)); - }), - () => { Error(parameters[0].MyLocation, "Expected atomic param to PROTECT"); }); - } - else if (parameters.Count == 2) - { - int start = 0, end = 0; - bool errorOccurred = false; - parameters[0].AsAtom().IfJust( - atom => atom.TryEvaluate(e => { Error(parameters[0].MyLocation, e.Message); errorOccurred = true; }, EvaluationPhase.Immediate).IfJust( - temp => - { - start = temp; - }), - () => { Error(parameters[0].MyLocation, "Expected atomic param to PROTECT"); errorOccurred = true; }); - parameters[1].AsAtom().IfJust( - atom => atom.TryEvaluate(e => { Error(parameters[0].MyLocation, e.Message); errorOccurred = true; }, EvaluationPhase.Immediate).IfJust( - temp => - { - end = temp; - }), - () => { Error(parameters[0].MyLocation, "Expected atomic param to PROTECT"); errorOccurred = true; }); - if (!errorOccurred) - { - int length = end - start; - - if (length > 0) - { - protectedRegions.Add(new Tuple(start, length, head!.Location)); - } - else - { - Warning("Protected region not valid (end offset not after start offset). No region protected."); - } - } - } - else - { - Error("Incorrect number of parameters in PROTECT: " + parameters.Count); - } - - return null; - } - - private ILineNode? ParseAlignStatement(IList parameters) - { - if (parameters.Count != 1) - { - Error("Incorrect number of parameters in ALIGN: " + parameters.Count); - return null; - } - - parameters[0].AsAtom().IfJust( - atom => atom.TryEvaluate(e => Error(parameters[0].MyLocation, e.Message), EvaluationPhase.Immediate).IfJust( - temp => - { - if (temp <= 0) - { - Error($"Cannot align address to {temp}"); - } - else if (CurrentOffset % temp != 0) - { - CurrentOffset += temp - CurrentOffset % temp; - } - }), - () => Error(parameters[0].MyLocation, "Expected atomic param to ALIGN")); - - return null; - } - - private ILineNode? ParseFillStatement(IList parameters) - { - if (parameters.Count > 2 || parameters.Count == 0) - { - Error("Incorrect number of parameters in FILL: " + parameters.Count); - return null; - } - - // FILL amount [value] - - int amount = 0; - int value = 0; - - if (parameters.Count == 2) - { - // param 2 (if given) is fill value - - parameters[1].AsAtom().IfJust( - atom => atom.TryEvaluate(e => Error(parameters[0].MyLocation, e.Message), EvaluationPhase.Immediate).IfJust( - val => { value = val; }), - () => Error(parameters[0].MyLocation, "Expected atomic param to FILL")); - } - - // param 1 is amount of bytes to fill - parameters[0].AsAtom().IfJust( - atom => atom.TryEvaluate(e => Error(parameters[0].MyLocation, e.Message), EvaluationPhase.Immediate).IfJust( - val => { amount = val; }), - () => Error(parameters[0].MyLocation, "Expected atomic param to FILL")); - - if (amount > 0) - { - var data = new byte[amount]; - - for (int i = 0; i < amount; ++i) - { - data[i] = (byte)value; - } - - var node = new DataNode(CurrentOffset, data); - - CheckDataWrite(amount); - CurrentOffset += amount; - - return node; - } - - return null; - } - - private ILineNode? ParseMessageStatement(IList parameters, ImmutableStack scopes) - { - Message(PrettyPrintParamsForMessage(parameters, scopes)); - return null; - } - - private ILineNode? ParseWarningStatement(IList parameters, ImmutableStack scopes) - { - Warning(PrettyPrintParamsForMessage(parameters, scopes)); - return null; - } - - private ILineNode? ParseErrorStatement(IList parameters, ImmutableStack scopes) - { - Error(PrettyPrintParamsForMessage(parameters, scopes)); - return null; - } - - public IList> ParseMacroParamList(MergeableGenerator tokens) - { - IList> parameters = new List>(); - int parenNestings = 0; - - // HACK: this allows macro([1, 2, 3]) from expanding into a single parameter - int bracketBalance = 0; - do - { - tokens.MoveNext(); - List currentParam = new List(); - while ( - !(parenNestings == 0 - && (tokens.Current.Type == TokenType.CLOSE_PAREN || (bracketBalance == 0 && tokens.Current.Type == TokenType.COMMA))) - && tokens.Current.Type != TokenType.NEWLINE) - { - switch (tokens.Current.Type) - { - case TokenType.CLOSE_PAREN: - parenNestings--; - break; - case TokenType.OPEN_PAREN: - parenNestings++; - break; - case TokenType.OPEN_BRACKET: - bracketBalance++; - break; - case TokenType.CLOSE_BRACKET: - bracketBalance--; - break; - } - - currentParam.Add(tokens.Current); - tokens.MoveNext(); - } - parameters.Add(currentParam); - } while (tokens.Current.Type != TokenType.CLOSE_PAREN && tokens.Current.Type != TokenType.NEWLINE); - if (tokens.Current.Type != TokenType.CLOSE_PAREN || parenNestings != 0) - { - Error(tokens.Current.Location, "Unmatched open parenthesis."); - } - else - { - tokens.MoveNext(); - } - return parameters; - } - - private IList ParseParamList(MergeableGenerator tokens, ImmutableStack scopes, bool expandFirstDef = true) - { - IList paramList = new List(); - bool first = true; - - while (tokens.Current.Type != TokenType.NEWLINE && tokens.Current.Type != TokenType.SEMICOLON && !tokens.EOS) - { - Token localHead = tokens.Current; - ParseParam(tokens, scopes, expandFirstDef || !first).IfJust( - n => paramList.Add(n), - () => Error(localHead.Location, "Expected parameter.")); - first = false; - } - - if (tokens.Current.Type == TokenType.SEMICOLON) - { - tokens.MoveNext(); - } - - return paramList; - } - - public IList ParsePreprocParamList(MergeableGenerator tokens, ImmutableStack scopes, bool allowsFirstExpanded) - { - IList temp = ParseParamList(tokens, scopes, allowsFirstExpanded); - - for (int i = 0; i < temp.Count; i++) - { - if (temp[i].Type == ParamType.STRING && ((StringNode)temp[i]).IsValidIdentifier()) - { - temp[i] = ((StringNode)temp[i]).ToIdentifier(scopes); - } - } - - return temp; - } - - private IParamNode? ParseParam(MergeableGenerator tokens, ImmutableStack scopes, bool expandDefs = true) - { - Token localHead = tokens.Current; - switch (localHead.Type) - { - case TokenType.OPEN_BRACKET: - return new ListNode(localHead.Location, ParseList(tokens, scopes)); - case TokenType.STRING: - tokens.MoveNext(); - return new StringNode(localHead); - case TokenType.MAYBE_MACRO: - //TODO: Move this and the one in ExpandId to a separate ParseMacroNode that may return an Invocation. - if (expandDefs && ExpandIdentifier(tokens, scopes, true)) - { - return ParseParam(tokens, scopes); - } - else - { - tokens.MoveNext(); - IList> param = ParseMacroParamList(tokens); - //TODO: Smart errors if trying to redefine a macro with the same num of params. - return new MacroInvocationNode(this, localHead, param, scopes); - } - case TokenType.IDENTIFIER: - if (expandDefs && ExpandIdentifier(tokens, scopes, true)) - { - return ParseParam(tokens, scopes, expandDefs); - } - else - { - switch (localHead.Content.ToUpperInvariant()) - { - case "__FILE__": - tokens.MoveNext(); - return new StringNode(new Token(TokenType.STRING, localHead.Location, localHead.GetSourceLocation().file)); - - default: - return ParseAtom(tokens, scopes, expandDefs); - } - } - - default: - return ParseAtom(tokens, scopes, expandDefs); - } - } - - private static readonly Dictionary precedences = new Dictionary { - { TokenType.MUL_OP, 3 }, - { TokenType.DIV_OP, 3 }, - { TokenType.MOD_OP, 3 }, - { TokenType.ADD_OP, 4 }, - { TokenType.SUB_OP, 4 }, - { TokenType.LSHIFT_OP, 5 }, - { TokenType.RSHIFT_OP, 5 }, - { TokenType.SIGNED_RSHIFT_OP, 5 }, - { TokenType.COMPARE_GE, 6 }, - { TokenType.COMPARE_GT, 6 }, - { TokenType.COMPARE_LT, 6 }, - { TokenType.COMPARE_LE, 6 }, - { TokenType.COMPARE_EQ, 7 }, - { TokenType.COMPARE_NE, 7 }, - { TokenType.AND_OP, 8 }, - { TokenType.XOR_OP, 9 }, - { TokenType.OR_OP, 10 }, - { TokenType.LOGAND_OP, 11 }, - { TokenType.LOGOR_OP, 12 }, - { TokenType.UNDEFINED_COALESCE_OP, 13 }, - }; - - public static bool IsInfixOperator(Token token) => precedences.ContainsKey(token.Type); - - public IAtomNode? ParseAtom(MergeableGenerator tokens, ImmutableStack scopes, bool expandDefs = true) - { - //Use Shift Reduce Parsing - Token localHead = tokens.Current; - Stack> grammarSymbols = new Stack>(); - bool ended = false; - while (!ended) - { - bool shift = false, lookingForAtom = grammarSymbols.Count == 0 || grammarSymbols.Peek().IsRight; - Token lookAhead = tokens.Current; - - if (!ended && !lookingForAtom) //Is already a complete node. Needs an operator of matching precedence and a node of matching prec to reduce. - { - //Verify next symbol to be a binary operator. - switch (lookAhead.Type) - { - case TokenType.MUL_OP: - case TokenType.DIV_OP: - case TokenType.MOD_OP: - case TokenType.ADD_OP: - case TokenType.SUB_OP: - case TokenType.LSHIFT_OP: - case TokenType.RSHIFT_OP: - case TokenType.SIGNED_RSHIFT_OP: - case TokenType.AND_OP: - case TokenType.XOR_OP: - case TokenType.OR_OP: - case TokenType.LOGAND_OP: - case TokenType.LOGOR_OP: - case TokenType.COMPARE_LT: - case TokenType.COMPARE_LE: - case TokenType.COMPARE_EQ: - case TokenType.COMPARE_NE: - case TokenType.COMPARE_GE: - case TokenType.COMPARE_GT: - if (precedences.TryGetValue(lookAhead.Type, out int precedence)) - { - Reduce(grammarSymbols, precedence); - } - shift = true; - break; - case TokenType.UNDEFINED_COALESCE_OP: - // '??' is right-associative, so don't reduce here - shift = true; - break; - default: - ended = true; - break; - } - } - else if (!ended) //Is just an operator. Error if two operators in a row. - { - //Error if two operators in a row. - switch (lookAhead.Type) - { - case TokenType.IDENTIFIER: - case TokenType.MAYBE_MACRO: - case TokenType.NUMBER: - shift = true; - break; - case TokenType.OPEN_PAREN: - { - tokens.MoveNext(); - IAtomNode? interior = ParseAtom(tokens, scopes); - if (tokens.Current.Type != TokenType.CLOSE_PAREN) - { - Error(tokens.Current.Location, "Unmatched open parenthesis (currently at " + tokens.Current.Type + ")."); - return null; - } - else if (interior == null) - { - Error(lookAhead.Location, "Expected expression inside paretheses. "); - return null; - } - else - { - grammarSymbols.Push(new Left(interior)); - tokens.MoveNext(); - break; - } - } - case TokenType.SUB_OP: - case TokenType.LOGNOT_OP: - case TokenType.NOT_OP: - { - //Assume unary negation. - tokens.MoveNext(); - IAtomNode? interior = ParseAtom(tokens, scopes); - if (interior == null) - { - Error(lookAhead.Location, "Expected expression after unary operator."); - return null; - } - grammarSymbols.Push(new Left(new UnaryOperatorNode(lookAhead, interior))); - break; - } - case TokenType.COMMA: - Error(lookAhead.Location, "Unexpected comma (perhaps unrecognized macro invocation?)."); - IgnoreRestOfStatement(tokens); - return null; - case TokenType.MUL_OP: - case TokenType.DIV_OP: - case TokenType.MOD_OP: - case TokenType.ADD_OP: - case TokenType.LSHIFT_OP: - case TokenType.RSHIFT_OP: - case TokenType.SIGNED_RSHIFT_OP: - case TokenType.AND_OP: - case TokenType.XOR_OP: - case TokenType.OR_OP: - case TokenType.LOGAND_OP: - case TokenType.LOGOR_OP: - case TokenType.COMPARE_LT: - case TokenType.COMPARE_LE: - case TokenType.COMPARE_EQ: - case TokenType.COMPARE_NE: - case TokenType.COMPARE_GE: - case TokenType.COMPARE_GT: - case TokenType.UNDEFINED_COALESCE_OP: - default: - Error(lookAhead.Location, $"Expected identifier or literal, got {lookAhead.Type}: {lookAhead.Content}."); - IgnoreRestOfStatement(tokens); - return null; - } - } - - if (shift) - { - switch (lookAhead.Type) - { - case TokenType.IDENTIFIER: - if (expandDefs && ExpandIdentifier(tokens, scopes, true)) - { - continue; - } - - grammarSymbols.Push(new Left(lookAhead.Content.ToUpperInvariant() switch - { - "CURRENTOFFSET" => new NumberNode(lookAhead, CurrentOffset), - "__LINE__" => new NumberNode(lookAhead, lookAhead.GetSourceLocation().line), - _ => new IdentifierNode(lookAhead, scopes), - })); - - break; - - case TokenType.MAYBE_MACRO: - ExpandIdentifier(tokens, scopes, true); - continue; - case TokenType.NUMBER: - grammarSymbols.Push(new Left(new NumberNode(lookAhead))); - break; - case TokenType.ERROR: - Error(lookAhead.Location, $"Unexpected token: {lookAhead.Content}"); - tokens.MoveNext(); - return null; - default: - grammarSymbols.Push(new Right(lookAhead)); - break; - } - tokens.MoveNext(); - continue; - } - } - while (grammarSymbols.Count > 1) - { - Reduce(grammarSymbols, int.MaxValue); - } - if (grammarSymbols.Peek().IsRight) - { - Error(grammarSymbols.Peek().GetRight.Location, $"Unexpected token: {grammarSymbols.Peek().GetRight.Type}"); - } - return grammarSymbols.Peek().GetLeft; - } - - /*** - * Precondition: grammarSymbols alternates between IAtomNodes, operator Tokens, .Count is odd - * the precedences of the IAtomNodes is increasing. - * Postcondition: Either grammarSymbols.Count == 1, or everything in grammarSymbols will have precedence <= targetPrecedence. - * - */ - private void Reduce(Stack> grammarSymbols, int targetPrecedence) - { - while (grammarSymbols.Count > 1)// && grammarSymbols.Peek().GetLeft.Precedence > targetPrecedence) - { - // These shouldn't error... - IAtomNode r = grammarSymbols.Pop().GetLeft; - - if (precedences[grammarSymbols.Peek().GetRight.Type] > targetPrecedence) - { - grammarSymbols.Push(new Left(r)); - break; - } - else - { - Token op = grammarSymbols.Pop().GetRight; - IAtomNode l = grammarSymbols.Pop().GetLeft; - - OperatorNode operatorNode = new OperatorNode(l, op, r, l.Precedence); - - if (DiagnosticHelpers.DoesOperationSpanMultipleMacrosUnintuitively(operatorNode)) - { - MacroLocation? mloc = operatorNode.MyLocation.macroLocation; - string message = DiagnosticHelpers.GetEmphasizedExpression(operatorNode, l => l.macroLocation == mloc); - - if (mloc != null) - { - message += $"\nUnintuitive expression resulting from expansion of macro `{mloc.MacroName}`."; - } - else - { - message += "\nUnintuitive expression resulting from expansion of macro."; - } - - message += "\nConsider guarding your expressions using parenthesis."; - - Warning(operatorNode.MyLocation, message); - } - - grammarSymbols.Push(new Left(operatorNode)); - } - } - } - - private IList ParseList(MergeableGenerator tokens, ImmutableStack scopes) - { - Token localHead = tokens.Current; - tokens.MoveNext(); - - IList atoms = new List(); - while (tokens.Current.Type != TokenType.NEWLINE && tokens.Current.Type != TokenType.CLOSE_BRACKET) - { - IAtomNode? res = ParseAtom(tokens, scopes); - res.IfJust( - n => atoms.Add(n), - () => Error(tokens.Current.Location, "Expected atomic value, got " + tokens.Current.Type + ".")); - if (tokens.Current.Type == TokenType.COMMA) - { - tokens.MoveNext(); - } - } - if (tokens.Current.Type == TokenType.CLOSE_BRACKET) - { - tokens.MoveNext(); - } - else - { - Error(localHead.Location, "Unmatched open bracket."); - } - - return atoms; - } - - public ILineNode? ParseLine(MergeableGenerator tokens, ImmutableStack scopes) - { - if (IsIncluding) - { - if (tokens.Current.Type == TokenType.NEWLINE || tokens.Current.Type == TokenType.SEMICOLON) - { - tokens.MoveNext(); - return null; - } - head = tokens.Current; - switch (head.Type) - { - case TokenType.IDENTIFIER: - case TokenType.MAYBE_MACRO: - if (ExpandIdentifier(tokens, scopes)) - { - // NOTE: we check here if we didn't end up with something that can't be a statement - - switch (tokens.Current.Type) - { - case TokenType.IDENTIFIER: - case TokenType.MAYBE_MACRO: - case TokenType.OPEN_BRACE: - case TokenType.PREPROCESSOR_DIRECTIVE: - return ParseLine(tokens, scopes); - - default: - // it is somewhat common for users to do '#define Foo 0xABCD' and then later 'Foo:' - Error($"Expansion of macro `{head.Content}` did not result in a valid statement. Did you perhaps attempt to define a label or symbol with that name?"); - IgnoreRestOfLine(tokens); - - return null; - } - } - else - { - tokens.MoveNext(); - switch (tokens.Current.Type) - { - case TokenType.COLON: - tokens.MoveNext(); - TryDefineSymbol(scopes, head.Content, ConvertToAddress(CurrentOffset)); - return null; - case TokenType.ASSIGN: - tokens.MoveNext(); - - ParseAtom(tokens, scopes, true).IfJust( - atom => atom.TryEvaluate( - e => TryDefineSymbol(scopes, head.Content, atom), EvaluationPhase.Early).IfJust( - value => TryDefineSymbol(scopes, head.Content, value)), - () => Error($"Couldn't define symbol `{head.Content}`: exprected expression.")); - - return null; - - default: - tokens.PutBack(head); - return ParseStatement(tokens, scopes); - } - } - case TokenType.OPEN_BRACE: - return ParseBlock(tokens, new ImmutableStack(new Closure(), scopes)); - case TokenType.PREPROCESSOR_DIRECTIVE: - return ParsePreprocessor(tokens, scopes); - case TokenType.OPEN_BRACKET: - Error("Unexpected list literal."); - IgnoreRestOfLine(tokens); - break; - case TokenType.NUMBER: - case TokenType.OPEN_PAREN: - Error("Unexpected mathematical expression."); - IgnoreRestOfLine(tokens); - break; - default: - tokens.MoveNext(); - - if (string.IsNullOrEmpty(head.Content)) - { - Error($"Unexpected token: {head.Type}."); - } - else - { - Error($"Unexpected token: {head.Type}: {head.Content}."); - } - - IgnoreRestOfLine(tokens); - break; - } - return null; - } - else - { - bool hasNext = true; - while (tokens.Current.Type != TokenType.PREPROCESSOR_DIRECTIVE && (hasNext = tokens.MoveNext())) - { - ; - } - - if (hasNext) - { - return ParsePreprocessor(tokens, scopes); - } - else - { - Error(null, $"Missing {Inclusion.Count} endif(s)."); - return null; - } - } - } - - private void TryDefineSymbol(ImmutableStack scopes, string name, int value) - { - if (scopes.Head.HasLocalSymbol(name)) - { - Warning($"Symbol already in scope, ignoring: {name}"); - } - else if (!IsValidLabelName(name)) - { - // NOTE: IsValidLabelName returns true always. This is dead code - Error($"Invalid symbol name {name}."); - } - else - { - scopes.Head.AddSymbol(name, value); - } - } - - private void TryDefineSymbol(ImmutableStack scopes, string name, IAtomNode expression) - { - if (scopes.Head.HasLocalSymbol(name)) - { - Warning($"Symbol already in scope, ignoring: {name}"); - } - else if (!IsValidLabelName(name)) - { - // NOTE: IsValidLabelName returns true always. This is dead code - Error($"Invalid symbol name {name}."); - } - else - { - scopes.Head.AddSymbol(name, expression); - } - } - - private ILineNode? ParsePreprocessor(MergeableGenerator tokens, ImmutableStack scopes) - { - head = tokens.Current; - tokens.MoveNext(); - - ILineNode? result = directiveHandler.HandleDirective(this, head, tokens, scopes); - - if (result != null) - { - CheckDataWrite(result.Size); - CurrentOffset += result.Size; - } - - return result; - } - - /*** - * Precondition: tokens.Current.Type == TokenType.IDENTIFIER || MAYBE_MACRO - * Postcondition: tokens.Current is fully reduced (i.e. not a macro, and not a definition) - * Returns: true iff tokens was actually expanded. - */ - public bool ExpandIdentifier(MergeableGenerator tokens, ImmutableStack scopes, bool insideExpression = false) - { - // function-like macros - if (tokens.Current.Type == TokenType.MAYBE_MACRO) - { - if (Macros.ContainsName(tokens.Current.Content)) - { - Token localHead = tokens.Current; - tokens.MoveNext(); - - IList> parameters = ParseMacroParamList(tokens); - - if (Macros.TryGetMacro(localHead.Content, parameters.Count, out IMacro? macro)) - { - /* macro is 100% not null here, but because we can't use NotNullWhen on TryGetMacro, - * since the attribute is unavailable in .NET Framework (which we still target), - * the compiler will still diagnose a nullable dereference if we don't use '!' also */ - - ApplyMacroExpansion(tokens, macro!.ApplyMacro(localHead, parameters, scopes), insideExpression); - } - else - { - Error($"No overload of {localHead.Content} with {parameters.Count} parameters."); - } - return true; - } - else - { - Token localHead = tokens.Current; - tokens.MoveNext(); - - tokens.PutBack(new Token(TokenType.IDENTIFIER, localHead.Location, localHead.Content)); - return true; - } - } - - // object-like macros (aka "Definitions") - if (Definitions.TryGetValue(tokens.Current.Content, out Definition? definition) && !definition.NonProductive) - { - Token localHead = tokens.Current; - tokens.MoveNext(); - - ApplyMacroExpansion(tokens, definition.ApplyDefinition(localHead), insideExpression); - return true; - } - - return false; - } - - private void ApplyMacroExpansion(MergeableGenerator tokens, IEnumerable expandedTokens, bool insideExpression = false) - { - if (insideExpression && EAOptions.IsWarningEnabled(EAOptions.Warnings.UnguardedExpressionMacros)) - { - // here we check for any operator that isn't enclosed in parenthesises - - IList expandedList = expandedTokens.ToList(); - - DiagnosticHelpers.VisitUnguardedOperators(expandedList, - token => Warning(token.Location, $"Unguarded expansion of mathematical operator. Consider adding guarding parenthesises around definition.")); - - tokens.PrependEnumerator(expandedList.GetEnumerator()); - } - else - { - tokens.PrependEnumerator(expandedTokens.GetEnumerator()); - } - } - - private void MessageTrace(Log.MessageKind kind, Location? location, string message) - { - if (location is Location myLocation && myLocation.macroLocation != null) - { - MacroLocation macroLocation = myLocation.macroLocation; - MessageTrace(kind, macroLocation.Location, message); - log.Message(Log.MessageKind.NOTE, location, $"From inside of macro `{macroLocation.MacroName}`."); - } - else - { - string[] messages = message.Split('\n'); - log.Message(kind, location, messages[0]); - - for (int i = 1; i < messages.Length; i++) - { - log.Message(Log.MessageKind.CONTINUE, messages[i]); - } - } - } - - // shorthand helpers - - public void Message(Location? location, string message) => MessageTrace(Log.MessageKind.MESSAGE, location, message); - public void Warning(Location? location, string message) => MessageTrace(Log.MessageKind.WARNING, location, message); - public void Error(Location? location, string message) => MessageTrace(Log.MessageKind.ERROR, location, message); - - public void Message(string message) => MessageTrace(Log.MessageKind.MESSAGE, head?.Location, message); - public void Warning(string message) => MessageTrace(Log.MessageKind.WARNING, head?.Location, message); - public void Error(string message) => MessageTrace(Log.MessageKind.ERROR, head?.Location, message); - - private void IgnoreRestOfStatement(MergeableGenerator tokens) - { - while (tokens.Current.Type != TokenType.NEWLINE && tokens.Current.Type != TokenType.SEMICOLON && tokens.MoveNext()) { } - if (tokens.Current.Type == TokenType.SEMICOLON) - { - tokens.MoveNext(); - } - } - - public void IgnoreRestOfLine(MergeableGenerator tokens) - { - while (tokens.Current.Type != TokenType.NEWLINE && tokens.MoveNext()) { } - } - - /// - /// Consumes incoming tokens util end of line. - /// - /// token stream - /// If non-null, will expand any macros as they are encountered using this scope - /// The resulting list of tokens - public IList GetRestOfLine(MergeableGenerator tokens, ImmutableStack? scopesForMacros) - { - IList result = new List(); - - while (tokens.Current.Type != TokenType.NEWLINE) - { - if (scopesForMacros == null || !ExpandIdentifier(tokens, scopesForMacros)) - { - result.Add(tokens.Current); - tokens.MoveNext(); - } - } - - return result; - } - - public void Clear() - { - Macros.Clear(); - Definitions.Clear(); - Raws.Clear(); - Inclusion = ImmutableStack.Nil; - CurrentOffset = 0; - pastOffsets.Clear(); - } - - private string PrettyPrintParamsForMessage(IList parameters, ImmutableStack scopes) - { - return string.Join(" ", parameters.Select(parameter => parameter switch - { - StringNode node => ExpandUserFormatString(scopes, parameter.MyLocation, node.Value), - _ => parameter.PrettyPrint(), - })); - } - - private static readonly Regex formatItemRegex = new Regex(@"\{(?[^:}]+)(?:\:(?[^:}]*))?\}"); - - private string ExpandUserFormatString(ImmutableStack scopes, Location baseLocation, string stringValue) - { - string UserFormatStringError(string message, string details) - { - Error($"An error occurred while expanding format string ({message})."); - return $"<{message}: {details}>"; - } - - return formatItemRegex.Replace(stringValue, match => - { - string expr = match.Groups["expr"].Value!; - string? format = match.Groups["format"].Value; - - Location itemLocation = baseLocation.OffsetBy(match.Index); - - MergeableGenerator tokens = new MergeableGenerator( - Tokenizer.TokenizeLine($"{expr} \n", itemLocation)); - - tokens.MoveNext(); - - IAtomNode? node = ParseAtom(tokens, scopes); - - if (node == null || tokens.Current.Type != TokenType.NEWLINE) - { - return UserFormatStringError("bad expression", $"'{expr}'"); - } - - if (node.TryEvaluate(e => Error(node.MyLocation, e.Message), - EvaluationPhase.Early) is int value) - { - try - { - // TODO: do we need to raise an error when result == format? - // this happens (I think) when a custom format specifier with no substitution - return value.ToString(format, CultureInfo.InvariantCulture); - } - catch (FormatException e) - { - return UserFormatStringError("bad format specifier", $"'{format}' ({e.Message})"); - } - } - else - { - return UserFormatStringError("failed to evaluate expression", $"'{expr}'"); - } - }); - } - - // Return value: Location where protection occurred. Nothing if location was not protected. - private Location? IsProtected(int offset, int length) - { - foreach (Tuple protectedRegion in protectedRegions) - { - //They intersect if the last offset in the given region is after the start of this one - //and the first offset in the given region is before the last of this one - if (offset + length > protectedRegion.Item1 && offset < protectedRegion.Item1 + protectedRegion.Item2) - { - return protectedRegion.Item3; - } - } - - return null; - } - - private void CheckDataWrite(int length) - { - if (!offsetInitialized) - { - if (EAOptions.IsWarningEnabled(EAOptions.Warnings.UninitializedOffset)) - { - Warning("Writing before initializing offset. You may be breaking the ROM! (use `ORG offset` to set write offset)."); - } - - offsetInitialized = false; // only warn once - } - - if (IsProtected(CurrentOffset, length) is Location prot) - { - Error($"Trying to write data to area protected by {prot}"); - } - } - } -} +using ColorzCore.DataTypes; +using ColorzCore.IO; +using ColorzCore.Lexer; +using ColorzCore.Parser.AST; +using ColorzCore.Preprocessor; +using ColorzCore.Preprocessor.Macros; +using ColorzCore.Raws; +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; + +//TODO: Make errors less redundant (due to recursive nature, many paths will give several redundant errors). + +namespace ColorzCore.Parser +{ + public class EAParser + { + public MacroCollection Macros { get; } + public Dictionary Definitions { get; } + public Dictionary> Raws { get; } + public static readonly HashSet SpecialCodes = new HashSet { "ORG", "PUSH", "POP", "MESSAGE", "WARNING", "ERROR", "ASSERT", "PROTECT", "ALIGN", "FILL" }; + //public static readonly HashSet BuiltInMacros = new HashSet { "String", "AddToPool" }; + //TODO: Built in macros. + //public static readonly Dictionary BuiltInMacros; + public ImmutableStack GlobalScope { get; } + public int CurrentOffset + { + get => currentOffset; + + private set + { + if (value < 0 || value > EAOptions.MaximumBinarySize) + { + if (validOffset) //Error only the first time. + { + Error($"Invalid offset: {value:X}"); + validOffset = false; + } + } + else + { + currentOffset = value; + validOffset = true; + offsetInitialized = true; + } + } + + } + public ImmutableStack Inclusion { get; set; } + + public Pool Pool { get; private set; } + + private readonly DirectiveHandler directiveHandler; + + private readonly Stack> pastOffsets; // currentOffset, offsetInitialized + private readonly IList> protectedRegions; + + public Log log; + + public bool IsIncluding + { + get + { + bool acc = true; + + for (ImmutableStack temp = Inclusion; !temp.IsEmpty && acc; temp = temp.Tail) + { + acc &= temp.Head; + } + + return acc; + } + } + + private bool validOffset; + private bool offsetInitialized; // false until first ORG, used to warn about writing before first org + private int currentOffset; + private Token? head; //TODO: Make this make sense + + public EAParser(Dictionary> raws, Log log, DirectiveHandler directiveHandler) + { + GlobalScope = new ImmutableStack(new BaseClosure(), ImmutableStack.Nil); + pastOffsets = new Stack>(); + protectedRegions = new List>(); + this.log = log; + Raws = raws; + CurrentOffset = 0; + validOffset = true; + offsetInitialized = false; + Macros = new MacroCollection(this); + Definitions = new Dictionary(); + Inclusion = ImmutableStack.Nil; + this.directiveHandler = directiveHandler; + + Pool = new Pool(); + } + + public bool IsReservedName(string name) + { + return Raws.ContainsKey(name.ToUpperInvariant()) || SpecialCodes.Contains(name.ToUpperInvariant()); + } + public bool IsValidDefinitionName(string name) + { + return !(Definitions.ContainsKey(name) || IsReservedName(name)); + } + public bool IsValidMacroName(string name, int paramNum) + { + return !Macros.HasMacro(name, paramNum) && !IsReservedName(name); + } + public bool IsValidLabelName(string name) + { + return true;//!IsReservedName(name); + //TODO? + } + public IList ParseAll(IEnumerable tokenStream) + { + //TODO: Make BlockNode or EAProgramNode? + //Note must be strict to get all information on the closure before evaluating terms. + IList myLines = new List(); + MergeableGenerator tokens = new MergeableGenerator(tokenStream); + tokens.MoveNext(); + while (!tokens.EOS) + { + if (tokens.Current.Type != TokenType.NEWLINE || tokens.MoveNext()) + { + ILineNode? retVal = ParseLine(tokens, GlobalScope); + retVal.IfJust(n => myLines.Add(n)); + } + } + return myLines; + } + + private BlockNode ParseBlock(MergeableGenerator tokens, ImmutableStack scopes) + { + Location start = tokens.Current.Location; + tokens.MoveNext(); + BlockNode temp = new BlockNode(); + + while (!tokens.EOS && tokens.Current.Type != TokenType.CLOSE_BRACE) + { + ILineNode? x = ParseLine(tokens, scopes); + + if (x != null) + { + temp.Children.Add(x); + } + } + + if (!tokens.EOS) + { + tokens.MoveNext(); + } + else + { + Error(start, "Unmatched brace."); + } + + return temp; + } + + // TODO: these next two functions should probably be moved into their own module + + public static int ConvertToAddress(int value) + { + /* + NOTE: Offset 0 is always converted to a null address + If one wants to instead refer to ROM offset 0 they would want to use the address directly instead. + If ROM offset 0 is already address 0 then this is a moot point. + */ + + if (value > 0 && value < EAOptions.MaximumBinarySize) + { + value += EAOptions.BaseAddress; + } + + return value; + } + + public static int ConvertToOffset(int value) + { + if (value >= EAOptions.BaseAddress && value <= EAOptions.BaseAddress + EAOptions.MaximumBinarySize) + { + value -= EAOptions.BaseAddress; + } + + return value; + } + + private ILineNode? ParseStatement(MergeableGenerator tokens, ImmutableStack scopes) + { + while (ExpandIdentifier(tokens, scopes)) { } + + head = tokens.Current; + tokens.MoveNext(); + + //TODO: Replace with real raw information, and error if not valid. + IList parameters; + //TODO: Make intelligent to reject malformed parameters. + //TODO: Parse parameters after checking code validity. + if (tokens.Current.Type != TokenType.NEWLINE && tokens.Current.Type != TokenType.SEMICOLON) + { + parameters = ParseParamList(tokens, scopes); + } + else + { + parameters = new List(); + tokens.MoveNext(); + } + + string upperCodeIdentifier = head.Content.ToUpperInvariant(); + + if (SpecialCodes.Contains(upperCodeIdentifier)) + { + return upperCodeIdentifier switch + { + "ORG" => ParseOrgStatement(parameters), + "PUSH" => ParsePushStatement(parameters), + "POP" => ParsePopStatement(parameters), + "ASSERT" => ParseAssertStatement(parameters), + "PROTECT" => ParseProtectStatement(parameters), + "ALIGN" => ParseAlignStatement(parameters), + "FILL" => ParseFillStatement(parameters), + "MESSAGE" => ParseMessageStatement(parameters, scopes), + "WARNING" => ParseWarningStatement(parameters, scopes), + "ERROR" => ParseErrorStatement(parameters, scopes), + _ => null, // TODO: this is an error + }; + } + else if (Raws.TryGetValue(upperCodeIdentifier, out IList? raws)) + { + //TODO: Check for matches. Currently should type error. + foreach (Raw raw in raws) + { + if (raw.Fits(parameters)) + { + if ((CurrentOffset % raw.Alignment) != 0) + { + Error($"Bad code alignment (offset: {CurrentOffset:X8})"); + } + + StatementNode temp = new RawNode(raw, head, CurrentOffset, parameters); + + // TODO: more efficient spacewise to just have contiguous writing and not an offset with every line? + CheckDataWrite(temp.Size); + CurrentOffset += temp.Size; + + return temp; + } + } + + if (raws.Count == 1) + { + Error($"Incorrect parameters in raw `{raws[0].ToPrettyString()}`"); + } + else + { + StringBuilder sb = new StringBuilder(); + + sb.Append($"Couldn't find suitable variant of raw `{head.Content}`."); + + for (int i = 0; i < raws.Count; i++) + { + sb.Append($"\nVariant {i + 1}: `{raws[i].ToPrettyString()}`"); + } + + Error(sb.ToString()); + } + + IgnoreRestOfStatement(tokens); + return null; + } + else + { + Error("Unrecognized code: " + head.Content); + return null; + } + } + + private ILineNode? ParseOrgStatement(IList parameters) + { + if (parameters.Count != 1) + { + Error($"Incorrect number of parameters in ORG: {parameters.Count}"); + return null; + } + + parameters[0].AsAtom().IfJust( + atom => atom.TryEvaluate(e => Error(parameters[0].MyLocation, e.Message), EvaluationPhase.Immediate).IfJust( + offsetValue => { CurrentOffset = ConvertToOffset(offsetValue); }, + () => Error(parameters[0].MyLocation, "Expected atomic param to ORG."))); + + return null; + } + + private ILineNode? ParsePushStatement(IList parameters) + { + if (parameters.Count != 0) + { + Error("Incorrect number of parameters in PUSH: " + parameters.Count); + } + else + { + pastOffsets.Push(new Tuple(CurrentOffset, offsetInitialized)); + } + + return null; + } + + private ILineNode? ParsePopStatement(IList parameters) + { + if (parameters.Count != 0) + { + Error($"Incorrect number of parameters in POP: {parameters.Count}"); + } + else if (pastOffsets.Count == 0) + { + Error("POP without matching PUSH."); + } + else + { + Tuple tuple = pastOffsets.Pop(); + + CurrentOffset = tuple.Item1; + offsetInitialized = tuple.Item2; + } + + return null; + } + + private ILineNode? ParseAssertStatement(IList parameters) + { + if (parameters.Count != 1) + { + Error($"Incorrect number of parameters in ASSERT: {parameters.Count}"); + return null; + } + + // helper for distinguishing boolean expressions and other expressions + static bool IsConditionalOperatorHelper(IAtomNode node) + { + return node switch + { + UnaryOperatorNode uon => uon.OperatorToken.Type switch + { + TokenType.LOGNOT_OP => true, + _ => false, + }, + + OperatorNode on => on.OperatorToken.Type switch + { + TokenType.LOGAND_OP => true, + TokenType.LOGOR_OP => true, + TokenType.COMPARE_EQ => true, + TokenType.COMPARE_NE => true, + TokenType.COMPARE_GT => true, + TokenType.COMPARE_GE => true, + TokenType.COMPARE_LE => true, + TokenType.COMPARE_LT => true, + _ => false, + }, + + _ => false, + }; + } + + IAtomNode? atom = parameters[0].AsAtom(); + + if (atom != null) + { + bool isBoolean = IsConditionalOperatorHelper(atom); + + atom.TryEvaluate(e => Error(parameters[0].MyLocation, e.Message), EvaluationPhase.Immediate).IfJust( + temp => + { + // if boolean expession => fail if 0, else (legacy behavoir) fail if negative + if (isBoolean && temp == 0 || !isBoolean && temp < 0) + { + Error(parameters[0].MyLocation, "Assertion error: " + temp); + } + }); + } + else + { + Error(parameters[0].MyLocation, "Expected atomic param to ASSERT."); + } + + return null; + } + + private ILineNode? ParseProtectStatement(IList parameters) + { + if (parameters.Count == 1) + { + parameters[0].AsAtom().IfJust( + atom => atom.TryEvaluate(e => { Error(parameters[0].MyLocation, e.Message); }, EvaluationPhase.Immediate).IfJust( + temp => + { + protectedRegions.Add(new Tuple(temp, 4, head!.Location)); + }), + () => { Error(parameters[0].MyLocation, "Expected atomic param to PROTECT"); }); + } + else if (parameters.Count == 2) + { + int start = 0, end = 0; + bool errorOccurred = false; + parameters[0].AsAtom().IfJust( + atom => atom.TryEvaluate(e => { Error(parameters[0].MyLocation, e.Message); errorOccurred = true; }, EvaluationPhase.Immediate).IfJust( + temp => + { + start = temp; + }), + () => { Error(parameters[0].MyLocation, "Expected atomic param to PROTECT"); errorOccurred = true; }); + parameters[1].AsAtom().IfJust( + atom => atom.TryEvaluate(e => { Error(parameters[0].MyLocation, e.Message); errorOccurred = true; }, EvaluationPhase.Immediate).IfJust( + temp => + { + end = temp; + }), + () => { Error(parameters[0].MyLocation, "Expected atomic param to PROTECT"); errorOccurred = true; }); + if (!errorOccurred) + { + int length = end - start; + + if (length > 0) + { + protectedRegions.Add(new Tuple(start, length, head!.Location)); + } + else + { + Warning("Protected region not valid (end offset not after start offset). No region protected."); + } + } + } + else + { + Error("Incorrect number of parameters in PROTECT: " + parameters.Count); + } + + return null; + } + + private ILineNode? ParseAlignStatement(IList parameters) + { + if (parameters.Count != 1) + { + Error("Incorrect number of parameters in ALIGN: " + parameters.Count); + return null; + } + + parameters[0].AsAtom().IfJust( + atom => atom.TryEvaluate(e => Error(parameters[0].MyLocation, e.Message), EvaluationPhase.Immediate).IfJust( + temp => + { + if (temp <= 0) + { + Error($"Cannot align address to {temp}"); + } + else if (CurrentOffset % temp != 0) + { + CurrentOffset += temp - CurrentOffset % temp; + } + }), + () => Error(parameters[0].MyLocation, "Expected atomic param to ALIGN")); + + return null; + } + + private ILineNode? ParseFillStatement(IList parameters) + { + if (parameters.Count > 2 || parameters.Count == 0) + { + Error("Incorrect number of parameters in FILL: " + parameters.Count); + return null; + } + + // FILL amount [value] + + int amount = 0; + int value = 0; + + if (parameters.Count == 2) + { + // param 2 (if given) is fill value + + parameters[1].AsAtom().IfJust( + atom => atom.TryEvaluate(e => Error(parameters[0].MyLocation, e.Message), EvaluationPhase.Immediate).IfJust( + val => { value = val; }), + () => Error(parameters[0].MyLocation, "Expected atomic param to FILL")); + } + + // param 1 is amount of bytes to fill + parameters[0].AsAtom().IfJust( + atom => atom.TryEvaluate(e => Error(parameters[0].MyLocation, e.Message), EvaluationPhase.Immediate).IfJust( + val => { amount = val; }), + () => Error(parameters[0].MyLocation, "Expected atomic param to FILL")); + + if (amount > 0) + { + var data = new byte[amount]; + + for (int i = 0; i < amount; ++i) + { + data[i] = (byte)value; + } + + var node = new DataNode(CurrentOffset, data); + + CheckDataWrite(amount); + CurrentOffset += amount; + + return node; + } + + return null; + } + + private ILineNode? ParseMessageStatement(IList parameters, ImmutableStack scopes) + { + Message(PrettyPrintParamsForMessage(parameters, scopes)); + return null; + } + + private ILineNode? ParseWarningStatement(IList parameters, ImmutableStack scopes) + { + Warning(PrettyPrintParamsForMessage(parameters, scopes)); + return null; + } + + private ILineNode? ParseErrorStatement(IList parameters, ImmutableStack scopes) + { + Error(PrettyPrintParamsForMessage(parameters, scopes)); + return null; + } + + public IList> ParseMacroParamList(MergeableGenerator tokens) + { + IList> parameters = new List>(); + int parenNestings = 0; + + // HACK: this allows macro([1, 2, 3]) from expanding into a single parameter + int bracketBalance = 0; + do + { + tokens.MoveNext(); + List currentParam = new List(); + while ( + !(parenNestings == 0 + && (tokens.Current.Type == TokenType.CLOSE_PAREN || (bracketBalance == 0 && tokens.Current.Type == TokenType.COMMA))) + && tokens.Current.Type != TokenType.NEWLINE) + { + switch (tokens.Current.Type) + { + case TokenType.CLOSE_PAREN: + parenNestings--; + break; + case TokenType.OPEN_PAREN: + parenNestings++; + break; + case TokenType.OPEN_BRACKET: + bracketBalance++; + break; + case TokenType.CLOSE_BRACKET: + bracketBalance--; + break; + } + + currentParam.Add(tokens.Current); + tokens.MoveNext(); + } + parameters.Add(currentParam); + } while (tokens.Current.Type != TokenType.CLOSE_PAREN && tokens.Current.Type != TokenType.NEWLINE); + if (tokens.Current.Type != TokenType.CLOSE_PAREN || parenNestings != 0) + { + Error(tokens.Current.Location, "Unmatched open parenthesis."); + } + else + { + tokens.MoveNext(); + } + return parameters; + } + + private IList ParseParamList(MergeableGenerator tokens, ImmutableStack scopes, bool expandFirstDef = true) + { + IList paramList = new List(); + bool first = true; + + while (tokens.Current.Type != TokenType.NEWLINE && tokens.Current.Type != TokenType.SEMICOLON && !tokens.EOS) + { + Token localHead = tokens.Current; + ParseParam(tokens, scopes, expandFirstDef || !first).IfJust( + n => paramList.Add(n), + () => Error(localHead.Location, "Expected parameter.")); + first = false; + } + + if (tokens.Current.Type == TokenType.SEMICOLON) + { + tokens.MoveNext(); + } + + return paramList; + } + + public IList ParsePreprocParamList(MergeableGenerator tokens, ImmutableStack scopes, bool allowsFirstExpanded) + { + IList temp = ParseParamList(tokens, scopes, allowsFirstExpanded); + + for (int i = 0; i < temp.Count; i++) + { + if (temp[i].Type == ParamType.STRING && ((StringNode)temp[i]).IsValidIdentifier()) + { + temp[i] = ((StringNode)temp[i]).ToIdentifier(scopes); + } + } + + return temp; + } + + private IParamNode? ParseParam(MergeableGenerator tokens, ImmutableStack scopes, bool expandDefs = true) + { + Token localHead = tokens.Current; + switch (localHead.Type) + { + case TokenType.OPEN_BRACKET: + return new ListNode(localHead.Location, ParseList(tokens, scopes)); + case TokenType.STRING: + tokens.MoveNext(); + return new StringNode(localHead); + case TokenType.MAYBE_MACRO: + //TODO: Move this and the one in ExpandId to a separate ParseMacroNode that may return an Invocation. + if (expandDefs && ExpandIdentifier(tokens, scopes, true)) + { + return ParseParam(tokens, scopes); + } + else + { + tokens.MoveNext(); + IList> param = ParseMacroParamList(tokens); + //TODO: Smart errors if trying to redefine a macro with the same num of params. + return new MacroInvocationNode(this, localHead, param, scopes); + } + case TokenType.IDENTIFIER: + if (expandDefs && ExpandIdentifier(tokens, scopes, true)) + { + return ParseParam(tokens, scopes, expandDefs); + } + else + { + switch (localHead.Content.ToUpperInvariant()) + { + case "__FILE__": + tokens.MoveNext(); + return new StringNode(new Token(TokenType.STRING, localHead.Location, localHead.GetSourceLocation().file)); + + default: + return ParseAtom(tokens, scopes, expandDefs); + } + } + + default: + return ParseAtom(tokens, scopes, expandDefs); + } + } + + private static readonly Dictionary precedences = new Dictionary { + { TokenType.MUL_OP, 3 }, + { TokenType.DIV_OP, 3 }, + { TokenType.MOD_OP, 3 }, + { TokenType.ADD_OP, 4 }, + { TokenType.SUB_OP, 4 }, + { TokenType.LSHIFT_OP, 5 }, + { TokenType.RSHIFT_OP, 5 }, + { TokenType.SIGNED_RSHIFT_OP, 5 }, + { TokenType.COMPARE_GE, 6 }, + { TokenType.COMPARE_GT, 6 }, + { TokenType.COMPARE_LT, 6 }, + { TokenType.COMPARE_LE, 6 }, + { TokenType.COMPARE_EQ, 7 }, + { TokenType.COMPARE_NE, 7 }, + { TokenType.AND_OP, 8 }, + { TokenType.XOR_OP, 9 }, + { TokenType.OR_OP, 10 }, + { TokenType.LOGAND_OP, 11 }, + { TokenType.LOGOR_OP, 12 }, + { TokenType.UNDEFINED_COALESCE_OP, 13 }, + }; + + public static bool IsInfixOperator(Token token) => precedences.ContainsKey(token.Type); + + public IAtomNode? ParseAtom(MergeableGenerator tokens, ImmutableStack scopes, bool expandDefs = true) + { + //Use Shift Reduce Parsing + Token localHead = tokens.Current; + Stack> grammarSymbols = new Stack>(); + bool ended = false; + while (!ended) + { + bool shift = false, lookingForAtom = grammarSymbols.Count == 0 || grammarSymbols.Peek().IsRight; + Token lookAhead = tokens.Current; + + if (!ended && !lookingForAtom) //Is already a complete node. Needs an operator of matching precedence and a node of matching prec to reduce. + { + //Verify next symbol to be a binary operator. + switch (lookAhead.Type) + { + case TokenType.MUL_OP: + case TokenType.DIV_OP: + case TokenType.MOD_OP: + case TokenType.ADD_OP: + case TokenType.SUB_OP: + case TokenType.LSHIFT_OP: + case TokenType.RSHIFT_OP: + case TokenType.SIGNED_RSHIFT_OP: + case TokenType.AND_OP: + case TokenType.XOR_OP: + case TokenType.OR_OP: + case TokenType.LOGAND_OP: + case TokenType.LOGOR_OP: + case TokenType.COMPARE_LT: + case TokenType.COMPARE_LE: + case TokenType.COMPARE_EQ: + case TokenType.COMPARE_NE: + case TokenType.COMPARE_GE: + case TokenType.COMPARE_GT: + if (precedences.TryGetValue(lookAhead.Type, out int precedence)) + { + Reduce(grammarSymbols, precedence); + } + shift = true; + break; + case TokenType.UNDEFINED_COALESCE_OP: + // '??' is right-associative, so don't reduce here + shift = true; + break; + default: + ended = true; + break; + } + } + else if (!ended) //Is just an operator. Error if two operators in a row. + { + //Error if two operators in a row. + switch (lookAhead.Type) + { + case TokenType.IDENTIFIER: + case TokenType.MAYBE_MACRO: + case TokenType.NUMBER: + shift = true; + break; + case TokenType.OPEN_PAREN: + { + tokens.MoveNext(); + IAtomNode? interior = ParseAtom(tokens, scopes); + if (tokens.Current.Type != TokenType.CLOSE_PAREN) + { + Error(tokens.Current.Location, "Unmatched open parenthesis (currently at " + tokens.Current.Type + ")."); + return null; + } + else if (interior == null) + { + Error(lookAhead.Location, "Expected expression inside paretheses. "); + return null; + } + else + { + grammarSymbols.Push(new Left(interior)); + tokens.MoveNext(); + break; + } + } + case TokenType.SUB_OP: + case TokenType.LOGNOT_OP: + case TokenType.NOT_OP: + { + //Assume unary negation. + tokens.MoveNext(); + IAtomNode? interior = ParseAtom(tokens, scopes); + if (interior == null) + { + Error(lookAhead.Location, "Expected expression after unary operator."); + return null; + } + grammarSymbols.Push(new Left(new UnaryOperatorNode(lookAhead, interior))); + break; + } + case TokenType.COMMA: + Error(lookAhead.Location, "Unexpected comma (perhaps unrecognized macro invocation?)."); + IgnoreRestOfStatement(tokens); + return null; + case TokenType.MUL_OP: + case TokenType.DIV_OP: + case TokenType.MOD_OP: + case TokenType.ADD_OP: + case TokenType.LSHIFT_OP: + case TokenType.RSHIFT_OP: + case TokenType.SIGNED_RSHIFT_OP: + case TokenType.AND_OP: + case TokenType.XOR_OP: + case TokenType.OR_OP: + case TokenType.LOGAND_OP: + case TokenType.LOGOR_OP: + case TokenType.COMPARE_LT: + case TokenType.COMPARE_LE: + case TokenType.COMPARE_EQ: + case TokenType.COMPARE_NE: + case TokenType.COMPARE_GE: + case TokenType.COMPARE_GT: + case TokenType.UNDEFINED_COALESCE_OP: + default: + Error(lookAhead.Location, $"Expected identifier or literal, got {lookAhead.Type}: {lookAhead.Content}."); + IgnoreRestOfStatement(tokens); + return null; + } + } + + if (shift) + { + switch (lookAhead.Type) + { + case TokenType.IDENTIFIER: + if (expandDefs && ExpandIdentifier(tokens, scopes, true)) + { + continue; + } + + grammarSymbols.Push(new Left(lookAhead.Content.ToUpperInvariant() switch + { + "CURRENTOFFSET" => new NumberNode(lookAhead, CurrentOffset), + "__LINE__" => new NumberNode(lookAhead, lookAhead.GetSourceLocation().line), + _ => new IdentifierNode(lookAhead, scopes), + })); + + break; + + case TokenType.MAYBE_MACRO: + ExpandIdentifier(tokens, scopes, true); + continue; + case TokenType.NUMBER: + grammarSymbols.Push(new Left(new NumberNode(lookAhead))); + break; + case TokenType.ERROR: + Error(lookAhead.Location, $"Unexpected token: {lookAhead.Content}"); + tokens.MoveNext(); + return null; + default: + grammarSymbols.Push(new Right(lookAhead)); + break; + } + tokens.MoveNext(); + continue; + } + } + while (grammarSymbols.Count > 1) + { + Reduce(grammarSymbols, int.MaxValue); + } + if (grammarSymbols.Peek().IsRight) + { + Error(grammarSymbols.Peek().GetRight.Location, $"Unexpected token: {grammarSymbols.Peek().GetRight.Type}"); + } + return grammarSymbols.Peek().GetLeft; + } + + /*** + * Precondition: grammarSymbols alternates between IAtomNodes, operator Tokens, .Count is odd + * the precedences of the IAtomNodes is increasing. + * Postcondition: Either grammarSymbols.Count == 1, or everything in grammarSymbols will have precedence <= targetPrecedence. + * + */ + private void Reduce(Stack> grammarSymbols, int targetPrecedence) + { + while (grammarSymbols.Count > 1)// && grammarSymbols.Peek().GetLeft.Precedence > targetPrecedence) + { + // These shouldn't error... + IAtomNode r = grammarSymbols.Pop().GetLeft; + + if (precedences[grammarSymbols.Peek().GetRight.Type] > targetPrecedence) + { + grammarSymbols.Push(new Left(r)); + break; + } + else + { + Token op = grammarSymbols.Pop().GetRight; + IAtomNode l = grammarSymbols.Pop().GetLeft; + + OperatorNode operatorNode = new OperatorNode(l, op, r, l.Precedence); + + if (EAOptions.IsWarningEnabled(EAOptions.Warnings.UnintuitiveExpressionMacros)) + { + if (DiagnosticHelpers.DoesOperationSpanMultipleMacrosUnintuitively(operatorNode)) + { + MacroLocation? mloc = operatorNode.MyLocation.macroLocation; + string message = DiagnosticHelpers.GetEmphasizedExpression(operatorNode, l => l.macroLocation == mloc); + + if (mloc != null) + { + message += $"\nUnintuitive expression resulting from expansion of macro `{mloc.MacroName}`."; + } + else + { + message += "\nUnintuitive expression resulting from expansion of macro."; + } + + message += "\nConsider guarding your expressions using parenthesis."; + + Warning(operatorNode.MyLocation, message); + } + } + + grammarSymbols.Push(new Left(operatorNode)); + } + } + } + + private IList ParseList(MergeableGenerator tokens, ImmutableStack scopes) + { + Token localHead = tokens.Current; + tokens.MoveNext(); + + IList atoms = new List(); + while (tokens.Current.Type != TokenType.NEWLINE && tokens.Current.Type != TokenType.CLOSE_BRACKET) + { + IAtomNode? res = ParseAtom(tokens, scopes); + res.IfJust( + n => atoms.Add(n), + () => Error(tokens.Current.Location, "Expected atomic value, got " + tokens.Current.Type + ".")); + if (tokens.Current.Type == TokenType.COMMA) + { + tokens.MoveNext(); + } + } + if (tokens.Current.Type == TokenType.CLOSE_BRACKET) + { + tokens.MoveNext(); + } + else + { + Error(localHead.Location, "Unmatched open bracket."); + } + + return atoms; + } + + public ILineNode? ParseLine(MergeableGenerator tokens, ImmutableStack scopes) + { + if (IsIncluding) + { + if (tokens.Current.Type == TokenType.NEWLINE || tokens.Current.Type == TokenType.SEMICOLON) + { + tokens.MoveNext(); + return null; + } + head = tokens.Current; + switch (head.Type) + { + case TokenType.IDENTIFIER: + case TokenType.MAYBE_MACRO: + if (ExpandIdentifier(tokens, scopes)) + { + // NOTE: we check here if we didn't end up with something that can't be a statement + + switch (tokens.Current.Type) + { + case TokenType.IDENTIFIER: + case TokenType.MAYBE_MACRO: + case TokenType.OPEN_BRACE: + case TokenType.PREPROCESSOR_DIRECTIVE: + return ParseLine(tokens, scopes); + + default: + // it is somewhat common for users to do '#define Foo 0xABCD' and then later 'Foo:' + Error($"Expansion of macro `{head.Content}` did not result in a valid statement. Did you perhaps attempt to define a label or symbol with that name?"); + IgnoreRestOfLine(tokens); + + return null; + } + } + else + { + tokens.MoveNext(); + switch (tokens.Current.Type) + { + case TokenType.COLON: + tokens.MoveNext(); + TryDefineSymbol(scopes, head.Content, ConvertToAddress(CurrentOffset)); + return null; + case TokenType.ASSIGN: + tokens.MoveNext(); + + ParseAtom(tokens, scopes, true).IfJust( + atom => atom.TryEvaluate( + e => TryDefineSymbol(scopes, head.Content, atom), EvaluationPhase.Early).IfJust( + value => TryDefineSymbol(scopes, head.Content, value)), + () => Error($"Couldn't define symbol `{head.Content}`: exprected expression.")); + + return null; + + default: + tokens.PutBack(head); + return ParseStatement(tokens, scopes); + } + } + case TokenType.OPEN_BRACE: + return ParseBlock(tokens, new ImmutableStack(new Closure(), scopes)); + case TokenType.PREPROCESSOR_DIRECTIVE: + return ParsePreprocessor(tokens, scopes); + case TokenType.OPEN_BRACKET: + Error("Unexpected list literal."); + IgnoreRestOfLine(tokens); + break; + case TokenType.NUMBER: + case TokenType.OPEN_PAREN: + Error("Unexpected mathematical expression."); + IgnoreRestOfLine(tokens); + break; + default: + tokens.MoveNext(); + + if (string.IsNullOrEmpty(head.Content)) + { + Error($"Unexpected token: {head.Type}."); + } + else + { + Error($"Unexpected token: {head.Type}: {head.Content}."); + } + + IgnoreRestOfLine(tokens); + break; + } + return null; + } + else + { + bool hasNext = true; + while (tokens.Current.Type != TokenType.PREPROCESSOR_DIRECTIVE && (hasNext = tokens.MoveNext())) + { + ; + } + + if (hasNext) + { + return ParsePreprocessor(tokens, scopes); + } + else + { + Error(null, $"Missing {Inclusion.Count} endif(s)."); + return null; + } + } + } + + private void TryDefineSymbol(ImmutableStack scopes, string name, int value) + { + if (scopes.Head.HasLocalSymbol(name)) + { + Warning($"Symbol already in scope, ignoring: {name}"); + } + else if (!IsValidLabelName(name)) + { + // NOTE: IsValidLabelName returns true always. This is dead code + Error($"Invalid symbol name {name}."); + } + else + { + scopes.Head.AddSymbol(name, value); + } + } + + private void TryDefineSymbol(ImmutableStack scopes, string name, IAtomNode expression) + { + if (scopes.Head.HasLocalSymbol(name)) + { + Warning($"Symbol already in scope, ignoring: {name}"); + } + else if (!IsValidLabelName(name)) + { + // NOTE: IsValidLabelName returns true always. This is dead code + Error($"Invalid symbol name {name}."); + } + else + { + scopes.Head.AddSymbol(name, expression); + } + } + + private ILineNode? ParsePreprocessor(MergeableGenerator tokens, ImmutableStack scopes) + { + head = tokens.Current; + tokens.MoveNext(); + + ILineNode? result = directiveHandler.HandleDirective(this, head, tokens, scopes); + + if (result != null) + { + CheckDataWrite(result.Size); + CurrentOffset += result.Size; + } + + return result; + } + + /*** + * Precondition: tokens.Current.Type == TokenType.IDENTIFIER || MAYBE_MACRO + * Postcondition: tokens.Current is fully reduced (i.e. not a macro, and not a definition) + * Returns: true iff tokens was actually expanded. + */ + public bool ExpandIdentifier(MergeableGenerator tokens, ImmutableStack scopes, bool insideExpression = false) + { + // function-like macros + if (tokens.Current.Type == TokenType.MAYBE_MACRO) + { + if (Macros.ContainsName(tokens.Current.Content)) + { + Token localHead = tokens.Current; + tokens.MoveNext(); + + IList> parameters = ParseMacroParamList(tokens); + + if (Macros.TryGetMacro(localHead.Content, parameters.Count, out IMacro? macro)) + { + /* macro is 100% not null here, but because we can't use NotNullWhen on TryGetMacro, + * since the attribute is unavailable in .NET Framework (which we still target), + * the compiler will still diagnose a nullable dereference if we don't use '!' also */ + + ApplyMacroExpansion(tokens, macro!.ApplyMacro(localHead, parameters, scopes), insideExpression); + } + else + { + Error($"No overload of {localHead.Content} with {parameters.Count} parameters."); + } + return true; + } + else + { + Token localHead = tokens.Current; + tokens.MoveNext(); + + tokens.PutBack(new Token(TokenType.IDENTIFIER, localHead.Location, localHead.Content)); + return true; + } + } + + // object-like macros (aka "Definitions") + if (Definitions.TryGetValue(tokens.Current.Content, out Definition? definition) && !definition.NonProductive) + { + Token localHead = tokens.Current; + tokens.MoveNext(); + + ApplyMacroExpansion(tokens, definition.ApplyDefinition(localHead), insideExpression); + return true; + } + + return false; + } + + private void ApplyMacroExpansion(MergeableGenerator tokens, IEnumerable expandedTokens, bool insideExpression = false) + { + if (insideExpression && EAOptions.IsWarningEnabled(EAOptions.Warnings.UnguardedExpressionMacros)) + { + // here we check for any operator that isn't enclosed in parenthesises + + IList expandedList = expandedTokens.ToList(); + + DiagnosticHelpers.VisitUnguardedOperators(expandedList, + token => Warning(token.Location, $"Unguarded expansion of mathematical operator. Consider adding guarding parenthesises around definition.")); + + tokens.PrependEnumerator(expandedList.GetEnumerator()); + } + else + { + tokens.PrependEnumerator(expandedTokens.GetEnumerator()); + } + } + + private void MessageTrace(Log.MessageKind kind, Location? location, string message) + { + if (location is Location myLocation && myLocation.macroLocation != null) + { + MacroLocation macroLocation = myLocation.macroLocation; + MessageTrace(kind, macroLocation.Location, message); + log.Message(Log.MessageKind.NOTE, location, $"From inside of macro `{macroLocation.MacroName}`."); + } + else + { + string[] messages = message.Split('\n'); + log.Message(kind, location, messages[0]); + + for (int i = 1; i < messages.Length; i++) + { + log.Message(Log.MessageKind.CONTINUE, messages[i]); + } + } + } + + // shorthand helpers + + public void Message(Location? location, string message) => MessageTrace(Log.MessageKind.MESSAGE, location, message); + public void Warning(Location? location, string message) => MessageTrace(Log.MessageKind.WARNING, location, message); + public void Error(Location? location, string message) => MessageTrace(Log.MessageKind.ERROR, location, message); + + public void Message(string message) => MessageTrace(Log.MessageKind.MESSAGE, head?.Location, message); + public void Warning(string message) => MessageTrace(Log.MessageKind.WARNING, head?.Location, message); + public void Error(string message) => MessageTrace(Log.MessageKind.ERROR, head?.Location, message); + + private void IgnoreRestOfStatement(MergeableGenerator tokens) + { + while (tokens.Current.Type != TokenType.NEWLINE && tokens.Current.Type != TokenType.SEMICOLON && tokens.MoveNext()) { } + if (tokens.Current.Type == TokenType.SEMICOLON) + { + tokens.MoveNext(); + } + } + + public void IgnoreRestOfLine(MergeableGenerator tokens) + { + while (tokens.Current.Type != TokenType.NEWLINE && tokens.MoveNext()) { } + } + + /// + /// Consumes incoming tokens util end of line. + /// + /// token stream + /// If non-null, will expand any macros as they are encountered using this scope + /// The resulting list of tokens + public IList GetRestOfLine(MergeableGenerator tokens, ImmutableStack? scopesForMacros) + { + IList result = new List(); + + while (tokens.Current.Type != TokenType.NEWLINE) + { + if (scopesForMacros == null || !ExpandIdentifier(tokens, scopesForMacros)) + { + result.Add(tokens.Current); + tokens.MoveNext(); + } + } + + return result; + } + + public void Clear() + { + Macros.Clear(); + Definitions.Clear(); + Raws.Clear(); + Inclusion = ImmutableStack.Nil; + CurrentOffset = 0; + pastOffsets.Clear(); + } + + private string PrettyPrintParamsForMessage(IList parameters, ImmutableStack scopes) + { + return string.Join(" ", parameters.Select(parameter => parameter switch + { + StringNode node => ExpandUserFormatString(scopes, parameter.MyLocation, node.Value), + _ => parameter.PrettyPrint(), + })); + } + + private static readonly Regex formatItemRegex = new Regex(@"\{(?[^:}]+)(?:\:(?[^:}]*))?\}"); + + private string ExpandUserFormatString(ImmutableStack scopes, Location baseLocation, string stringValue) + { + string UserFormatStringError(string message, string details) + { + Error($"An error occurred while expanding format string ({message})."); + return $"<{message}: {details}>"; + } + + return formatItemRegex.Replace(stringValue, match => + { + string expr = match.Groups["expr"].Value!; + string? format = match.Groups["format"].Value; + + Location itemLocation = baseLocation.OffsetBy(match.Index); + + MergeableGenerator tokens = new MergeableGenerator( + Tokenizer.TokenizeLine($"{expr} \n", itemLocation)); + + tokens.MoveNext(); + + IAtomNode? node = ParseAtom(tokens, scopes); + + if (node == null || tokens.Current.Type != TokenType.NEWLINE) + { + return UserFormatStringError("bad expression", $"'{expr}'"); + } + + if (node.TryEvaluate(e => Error(node.MyLocation, e.Message), + EvaluationPhase.Early) is int value) + { + try + { + // TODO: do we need to raise an error when result == format? + // this happens (I think) when a custom format specifier with no substitution + return value.ToString(format, CultureInfo.InvariantCulture); + } + catch (FormatException e) + { + return UserFormatStringError("bad format specifier", $"'{format}' ({e.Message})"); + } + } + else + { + return UserFormatStringError("failed to evaluate expression", $"'{expr}'"); + } + }); + } + + // Return value: Location where protection occurred. Nothing if location was not protected. + private Location? IsProtected(int offset, int length) + { + foreach (Tuple protectedRegion in protectedRegions) + { + //They intersect if the last offset in the given region is after the start of this one + //and the first offset in the given region is before the last of this one + if (offset + length > protectedRegion.Item1 && offset < protectedRegion.Item1 + protectedRegion.Item2) + { + return protectedRegion.Item3; + } + } + + return null; + } + + private void CheckDataWrite(int length) + { + if (!offsetInitialized) + { + if (EAOptions.IsWarningEnabled(EAOptions.Warnings.UninitializedOffset)) + { + Warning("Writing before initializing offset. You may be breaking the ROM! (use `ORG offset` to set write offset)."); + } + + offsetInitialized = false; // only warn once + } + + if (IsProtected(CurrentOffset, length) is Location prot) + { + Error($"Trying to write data to area protected by {prot}"); + } + } + } +} diff --git a/ColorzCore/Program.cs b/ColorzCore/Program.cs index 0613a55..25b3622 100644 --- a/ColorzCore/Program.cs +++ b/ColorzCore/Program.cs @@ -137,7 +137,7 @@ static int Main(string[] args) continue; } - string[] flag = args[i].Split(':'); + string[] flag = args[i].Split(new char[] { ':' }, 2); try { @@ -163,7 +163,7 @@ static int Main(string[] args) break; case "-input": - inFileName = flag[1]; + inFileName = flag[1].Replace('\\', '/'); inStream = File.OpenRead(flag[1]); break; @@ -272,9 +272,9 @@ static int Main(string[] args) } else { - for (int j = 1; j < flag.Length; j++) + foreach (string warning in flag[1].Split(':')) { - string name = flag[j]; + string name = warning; bool invert = false; if (name.StartsWith("no-")) @@ -283,15 +283,15 @@ static int Main(string[] args) invert = true; } - if (warningNames.TryGetValue(name, out EAOptions.Warnings warn)) + if (warningNames.TryGetValue(name, out EAOptions.Warnings warnFlag)) { if (invert) { - EAOptions.EnabledWarnings &= ~warn; + EAOptions.EnabledWarnings &= ~warnFlag; } else { - EAOptions.EnabledWarnings |= warn; + EAOptions.EnabledWarnings |= warnFlag; } } else @@ -306,13 +306,13 @@ static int Main(string[] args) case "--extensions": if (flag.Length == 1) { - EAOptions.EnabledWarnings |= EAOptions.Warnings.All; + EAOptions.EnabledExtensions |= EAOptions.Extensions.All; } else { - for (int j = 1; j < flag.Length; j++) + foreach (string extension in flag[1].Split(':')) { - string name = flag[j]; + string name = extension; bool invert = false; if (name.StartsWith("no-")) @@ -321,15 +321,15 @@ static int Main(string[] args) invert = true; } - if (extensionNames.TryGetValue(name, out EAOptions.Extensions ext)) + if (extensionNames.TryGetValue(name, out EAOptions.Extensions extFlag)) { if (invert) { - EAOptions.EnabledExtensions &= ~ext; + EAOptions.EnabledExtensions &= ~extFlag; } else { - EAOptions.EnabledExtensions |= ext; + EAOptions.EnabledExtensions |= extFlag; } } else @@ -396,7 +396,8 @@ static int Main(string[] args) { Output = errorStream, WarningsAreErrors = EAOptions.WarningsAreErrors, - NoColoredTags = EAOptions.MonochromeLog + NoColoredTags = EAOptions.MonochromeLog, + LocationBasePath = IOUtility.GetPortableBasePathForPrefix(inFileName), }; if (EAOptions.QuietWarnings) @@ -449,4 +450,3 @@ static int Main(string[] args) } } } - From 0f4843064745d53e81c38986c0ed832a840f426c Mon Sep 17 00:00:00 2001 From: Nat_776 Date: Mon, 29 Apr 2024 21:13:00 +0200 Subject: [PATCH 41/59] major parser restructuration --- ColorzCore/EAInterpreter.cs | 18 +- ColorzCore/IO/{Log.cs => Logger.cs} | 2 +- ColorzCore/Parser/AST/RawNode.cs | 13 +- ColorzCore/Parser/AtomParser.cs | 261 ++++ ...nosticHelpers.cs => DiagnosticsHelpers.cs} | 13 +- ColorzCore/Parser/EAParser.cs | 1247 ++++++++--------- ColorzCore/Parser/Pool.cs | 2 +- ColorzCore/Preprocessor/DirectiveHandler.cs | 12 +- .../Preprocessor/Directives/PoolDirective.cs | 13 +- ColorzCore/Preprocessor/MacroCollection.cs | 1 - ColorzCore/Preprocessor/Macros/AddToPool.cs | 10 +- ColorzCore/Program.cs | 8 +- 12 files changed, 906 insertions(+), 694 deletions(-) rename ColorzCore/IO/{Log.cs => Logger.cs} (99%) create mode 100644 ColorzCore/Parser/AtomParser.cs rename ColorzCore/Parser/{DiagnosticHelpers.cs => DiagnosticsHelpers.cs} (93%) diff --git a/ColorzCore/EAInterpreter.cs b/ColorzCore/EAInterpreter.cs index 31f25b1..70a6365 100644 --- a/ColorzCore/EAInterpreter.cs +++ b/ColorzCore/EAInterpreter.cs @@ -4,6 +4,7 @@ using ColorzCore.Parser; using ColorzCore.Parser.AST; using ColorzCore.Preprocessor; +using ColorzCore.Preprocessor.Directives; using ColorzCore.Preprocessor.Macros; using ColorzCore.Raws; using System; @@ -20,10 +21,10 @@ class EAInterpreter private EAParser myParser; private string game, iFile; private Stream sin; - private Log log; + private Logger log; private IOutput output; - public EAInterpreter(IOutput output, string game, string? rawsFolder, string rawsExtension, Stream sin, string inFileName, Log log) + public EAInterpreter(IOutput output, string game, string? rawsFolder, string rawsExtension, Stream sin, string inFileName, Logger log) { this.game = game; this.output = output; @@ -41,8 +42,8 @@ public EAInterpreter(IOutput output, string game, string? rawsFolder, string raw column = 1 }; - log.Message(Log.MessageKind.ERROR, loc, "An error occured while parsing raws"); - log.Message(Log.MessageKind.ERROR, loc, e.Message); + log.Message(Logger.MessageKind.ERROR, loc, "An error occured while parsing raws"); + log.Message(Logger.MessageKind.ERROR, loc, e.Message); Environment.Exit(-1); // ew? } @@ -84,6 +85,13 @@ public EAInterpreter(IOutput output, string game, string? rawsFolder, string raw myParser.Macros.BuiltInMacros.Add("ReadShortAt", unsupportedMacro); myParser.Macros.BuiltInMacros.Add("ReadWordAt", unsupportedMacro); } + + { + Pool pool = new Pool(); + + myParser.Macros.BuiltInMacros.Add("AddToPool", new AddToPool(pool)); + myParser.DirectiveHandler.Directives.Add("pool", new PoolDirective(pool)); + } } public bool Interpret() @@ -145,7 +153,7 @@ public bool Interpret() { if (Program.Debug) { - log.Message(Log.MessageKind.DEBUG, line.PrettyPrint(0)); + log.Message(Logger.MessageKind.DEBUG, line.PrettyPrint(0)); } line.WriteData(output); diff --git a/ColorzCore/IO/Log.cs b/ColorzCore/IO/Logger.cs similarity index 99% rename from ColorzCore/IO/Log.cs rename to ColorzCore/IO/Logger.cs index ee81ccf..80d46f9 100644 --- a/ColorzCore/IO/Log.cs +++ b/ColorzCore/IO/Logger.cs @@ -5,7 +5,7 @@ namespace ColorzCore.IO { - public class Log + public class Logger { public enum MessageKind { diff --git a/ColorzCore/Parser/AST/RawNode.cs b/ColorzCore/Parser/AST/RawNode.cs index 98e3018..d402abf 100644 --- a/ColorzCore/Parser/AST/RawNode.cs +++ b/ColorzCore/Parser/AST/RawNode.cs @@ -11,18 +11,18 @@ namespace ColorzCore.Parser.AST { class RawNode : StatementNode { - private Raw myRaw; + public Raw Raw { get; } private Token myToken; - private int offset; + private int Offset { get; } public RawNode(Raw raw, Token t, int offset, IList paramList) : base(paramList) { myToken = t; - myRaw = raw; - this.offset = offset; + Raw = raw; + Offset = offset; } - public override int Size => myRaw.LengthBytes(Parameters.Count); + public override int Size => Raw.LengthBytes(Parameters.Count); public override string PrettyPrint(int indentation) { @@ -36,9 +36,10 @@ public override string PrettyPrint(int indentation) } return sb.ToString(); } + public override void WriteData(IOutput output) { - output.WriteTo(offset, myRaw.GetBytes(Parameters)); + output.WriteTo(Offset, Raw.GetBytes(Parameters)); } } } diff --git a/ColorzCore/Parser/AtomParser.cs b/ColorzCore/Parser/AtomParser.cs new file mode 100644 index 0000000..ad6d72d --- /dev/null +++ b/ColorzCore/Parser/AtomParser.cs @@ -0,0 +1,261 @@ +using System.Collections.Generic; +using ColorzCore.DataTypes; +using ColorzCore.Lexer; +using ColorzCore.Parser.AST; + +namespace ColorzCore.Parser +{ + public static class AtomParser + { + private static readonly IDictionary precedences = new Dictionary { + { TokenType.MUL_OP, 3 }, + { TokenType.DIV_OP, 3 }, + { TokenType.MOD_OP, 3 }, + { TokenType.ADD_OP, 4 }, + { TokenType.SUB_OP, 4 }, + { TokenType.LSHIFT_OP, 5 }, + { TokenType.RSHIFT_OP, 5 }, + { TokenType.SIGNED_RSHIFT_OP, 5 }, + { TokenType.COMPARE_GE, 6 }, + { TokenType.COMPARE_GT, 6 }, + { TokenType.COMPARE_LT, 6 }, + { TokenType.COMPARE_LE, 6 }, + { TokenType.COMPARE_EQ, 7 }, + { TokenType.COMPARE_NE, 7 }, + { TokenType.AND_OP, 8 }, + { TokenType.XOR_OP, 9 }, + { TokenType.OR_OP, 10 }, + { TokenType.LOGAND_OP, 11 }, + { TokenType.LOGOR_OP, 12 }, + { TokenType.UNDEFINED_COALESCE_OP, 13 }, + }; + + public static bool IsInfixOperator(Token token) => precedences.ContainsKey(token.Type); + + public static IAtomNode? ParseAtom(this EAParser self, MergeableGenerator tokens, ImmutableStack scopes, bool expandDefs = true) + { + //Use Shift Reduce Parsing + Token localHead = tokens.Current; + Stack> grammarSymbols = new Stack>(); + bool ended = false; + while (!ended) + { + bool shift = false, lookingForAtom = grammarSymbols.Count == 0 || grammarSymbols.Peek().IsRight; + Token lookAhead = tokens.Current; + + if (!ended && !lookingForAtom) //Is already a complete node. Needs an operator of matching precedence and a node of matching prec to reduce. + { + //Verify next symbol to be a binary operator. + switch (lookAhead.Type) + { + case TokenType.MUL_OP: + case TokenType.DIV_OP: + case TokenType.MOD_OP: + case TokenType.ADD_OP: + case TokenType.SUB_OP: + case TokenType.LSHIFT_OP: + case TokenType.RSHIFT_OP: + case TokenType.SIGNED_RSHIFT_OP: + case TokenType.AND_OP: + case TokenType.XOR_OP: + case TokenType.OR_OP: + case TokenType.LOGAND_OP: + case TokenType.LOGOR_OP: + case TokenType.COMPARE_LT: + case TokenType.COMPARE_LE: + case TokenType.COMPARE_EQ: + case TokenType.COMPARE_NE: + case TokenType.COMPARE_GE: + case TokenType.COMPARE_GT: + if (precedences.TryGetValue(lookAhead.Type, out int precedence)) + { + self.Reduce(grammarSymbols, precedence); + } + shift = true; + break; + case TokenType.UNDEFINED_COALESCE_OP: + // '??' is right-associative, so don't reduce here + shift = true; + break; + default: + ended = true; + break; + } + } + else if (!ended) //Is just an operator. Error if two operators in a row. + { + //Error if two operators in a row. + switch (lookAhead.Type) + { + case TokenType.IDENTIFIER: + case TokenType.MAYBE_MACRO: + case TokenType.NUMBER: + shift = true; + break; + case TokenType.OPEN_PAREN: + { + tokens.MoveNext(); + IAtomNode? interior = self.ParseAtom(tokens, scopes); + if (tokens.Current.Type != TokenType.CLOSE_PAREN) + { + self.Error(tokens.Current.Location, "Unmatched open parenthesis (currently at " + tokens.Current.Type + ")."); + return null; + } + else if (interior == null) + { + self.Error(lookAhead.Location, "Expected expression inside paretheses. "); + return null; + } + else + { + grammarSymbols.Push(new Left(interior)); + tokens.MoveNext(); + break; + } + } + case TokenType.SUB_OP: + case TokenType.LOGNOT_OP: + case TokenType.NOT_OP: + { + //Assume unary negation. + tokens.MoveNext(); + IAtomNode? interior = self.ParseAtom(tokens, scopes); + if (interior == null) + { + self.Error(lookAhead.Location, "Expected expression after unary operator."); + return null; + } + grammarSymbols.Push(new Left(new UnaryOperatorNode(lookAhead, interior))); + break; + } + case TokenType.COMMA: + self.Error(lookAhead.Location, "Unexpected comma (perhaps unrecognized macro invocation?)."); + self.IgnoreRestOfStatement(tokens); + return null; + case TokenType.MUL_OP: + case TokenType.DIV_OP: + case TokenType.MOD_OP: + case TokenType.ADD_OP: + case TokenType.LSHIFT_OP: + case TokenType.RSHIFT_OP: + case TokenType.SIGNED_RSHIFT_OP: + case TokenType.AND_OP: + case TokenType.XOR_OP: + case TokenType.OR_OP: + case TokenType.LOGAND_OP: + case TokenType.LOGOR_OP: + case TokenType.COMPARE_LT: + case TokenType.COMPARE_LE: + case TokenType.COMPARE_EQ: + case TokenType.COMPARE_NE: + case TokenType.COMPARE_GE: + case TokenType.COMPARE_GT: + case TokenType.UNDEFINED_COALESCE_OP: + default: + self.Error(lookAhead.Location, $"Expected identifier or literal, got {lookAhead.Type}: {lookAhead.Content}."); + self.IgnoreRestOfStatement(tokens); + return null; + } + } + + if (shift) + { + switch (lookAhead.Type) + { + case TokenType.IDENTIFIER: + if (expandDefs && self.ExpandIdentifier(tokens, scopes, true)) + { + continue; + } + + grammarSymbols.Push(new Left(lookAhead.Content.ToUpperInvariant() switch + { + "CURRENTOFFSET" => new NumberNode(lookAhead, self.CurrentOffset), + "__LINE__" => new NumberNode(lookAhead, lookAhead.GetSourceLocation().line), + _ => new IdentifierNode(lookAhead, scopes), + })); + + break; + + case TokenType.MAYBE_MACRO: + self.ExpandIdentifier(tokens, scopes, true); + continue; + case TokenType.NUMBER: + grammarSymbols.Push(new Left(new NumberNode(lookAhead))); + break; + case TokenType.ERROR: + self.Error(lookAhead.Location, $"Unexpected token: {lookAhead.Content}"); + tokens.MoveNext(); + return null; + default: + grammarSymbols.Push(new Right(lookAhead)); + break; + } + tokens.MoveNext(); + continue; + } + } + while (grammarSymbols.Count > 1) + { + self.Reduce(grammarSymbols, int.MaxValue); + } + if (grammarSymbols.Peek().IsRight) + { + self.Error(grammarSymbols.Peek().GetRight.Location, $"Unexpected token: {grammarSymbols.Peek().GetRight.Type}"); + } + return grammarSymbols.Peek().GetLeft; + } + + /*** + * Precondition: grammarSymbols alternates between IAtomNodes, operator Tokens, .Count is odd + * the precedences of the IAtomNodes is increasing. + * Postcondition: Either grammarSymbols.Count == 1, or everything in grammarSymbols will have precedence <= targetPrecedence. + * + */ + private static void Reduce(this EAParser self, Stack> grammarSymbols, int targetPrecedence) + { + while (grammarSymbols.Count > 1)// && grammarSymbols.Peek().GetLeft.Precedence > targetPrecedence) + { + // These shouldn't error... + IAtomNode r = grammarSymbols.Pop().GetLeft; + + if (precedences[grammarSymbols.Peek().GetRight.Type] > targetPrecedence) + { + grammarSymbols.Push(new Left(r)); + break; + } + else + { + Token op = grammarSymbols.Pop().GetRight; + IAtomNode l = grammarSymbols.Pop().GetLeft; + + OperatorNode operatorNode = new OperatorNode(l, op, r, l.Precedence); + + if (EAOptions.IsWarningEnabled(EAOptions.Warnings.UnintuitiveExpressionMacros)) + { + if (DiagnosticsHelpers.DoesOperationSpanMultipleMacrosUnintuitively(operatorNode)) + { + MacroLocation? mloc = operatorNode.MyLocation.macroLocation; + string message = DiagnosticsHelpers.GetEmphasizedExpression(operatorNode, l => l.macroLocation == mloc); + + if (mloc != null) + { + message += $"\nUnintuitive expression resulting from expansion of macro `{mloc.MacroName}`."; + } + else + { + message += "\nUnintuitive expression resulting from expansion of macro."; + } + + message += "\nConsider guarding your expressions using parenthesis."; + + self.Warning(operatorNode.MyLocation, message); + } + } + + grammarSymbols.Push(new Left(operatorNode)); + } + } + } + } +} diff --git a/ColorzCore/Parser/DiagnosticHelpers.cs b/ColorzCore/Parser/DiagnosticsHelpers.cs similarity index 93% rename from ColorzCore/Parser/DiagnosticHelpers.cs rename to ColorzCore/Parser/DiagnosticsHelpers.cs index 1f71633..3dbcb28 100644 --- a/ColorzCore/Parser/DiagnosticHelpers.cs +++ b/ColorzCore/Parser/DiagnosticsHelpers.cs @@ -7,7 +7,7 @@ namespace ColorzCore.Parser { - public static class DiagnosticHelpers + public static class DiagnosticsHelpers { // Helper class for printing expressions (IAtomNode) with some bits emphasized private class EmphasisExpressionPrinter : AtomVisitor @@ -126,7 +126,7 @@ public static void VisitUnguardedOperators(IList tokens, Action ac break; default: - if (paren == 0 && bracket == 0 && EAParser.IsInfixOperator(token)) + if (paren == 0 && bracket == 0 && AtomParser.IsInfixOperator(token)) { action.Invoke(token); } @@ -200,5 +200,14 @@ public static bool DoesOperationSpanMultipleMacrosUnintuitively(OperatorNode ope return false; } + + public static string PrettyParamType(ParamType paramType) => paramType switch + { + ParamType.ATOM => "Atom", + ParamType.LIST => "List", + ParamType.STRING => "String", + ParamType.MACRO => "Macro", + _ => "", + }; } } diff --git a/ColorzCore/Parser/EAParser.cs b/ColorzCore/Parser/EAParser.cs index 8c64431..59657a6 100644 --- a/ColorzCore/Parser/EAParser.cs +++ b/ColorzCore/Parser/EAParser.cs @@ -1,4 +1,4 @@ -using ColorzCore.DataTypes; +using ColorzCore.DataTypes; using ColorzCore.IO; using ColorzCore.Lexer; using ColorzCore.Parser.AST; @@ -19,47 +19,15 @@ namespace ColorzCore.Parser { public class EAParser { - public MacroCollection Macros { get; } public Dictionary Definitions { get; } - public Dictionary> Raws { get; } - public static readonly HashSet SpecialCodes = new HashSet { "ORG", "PUSH", "POP", "MESSAGE", "WARNING", "ERROR", "ASSERT", "PROTECT", "ALIGN", "FILL" }; - //public static readonly HashSet BuiltInMacros = new HashSet { "String", "AddToPool" }; - //TODO: Built in macros. - //public static readonly Dictionary BuiltInMacros; - public ImmutableStack GlobalScope { get; } - public int CurrentOffset - { - get => currentOffset; + public MacroCollection Macros { get; } + public DirectiveHandler DirectiveHandler { get; } - private set - { - if (value < 0 || value > EAOptions.MaximumBinarySize) - { - if (validOffset) //Error only the first time. - { - Error($"Invalid offset: {value:X}"); - validOffset = false; - } - } - else - { - currentOffset = value; - validOffset = true; - offsetInitialized = true; - } - } + public Dictionary> Raws { get; } - } public ImmutableStack Inclusion { get; set; } - public Pool Pool { get; private set; } - - private readonly DirectiveHandler directiveHandler; - - private readonly Stack> pastOffsets; // currentOffset, offsetInitialized - private readonly IList> protectedRegions; - - public Log log; + public Logger log; public bool IsIncluding { @@ -76,16 +44,13 @@ public bool IsIncluding } } - private bool validOffset; - private bool offsetInitialized; // false until first ORG, used to warn about writing before first org - private int currentOffset; - private Token? head; //TODO: Make this make sense + private Token? head; // TODO: Make this make sense - public EAParser(Dictionary> raws, Log log, DirectiveHandler directiveHandler) + public EAParser(Dictionary> raws, Logger log, DirectiveHandler directiveHandler) { GlobalScope = new ImmutableStack(new BaseClosure(), ImmutableStack.Nil); - pastOffsets = new Stack>(); - protectedRegions = new List>(); + pastOffsets = new Stack<(int, bool)>(); + protectedRegions = new List<(int, int, Location)>(); this.log = log; Raws = raws; CurrentOffset = 0; @@ -94,35 +59,33 @@ public EAParser(Dictionary> raws, Log log, DirectiveHandler d Macros = new MacroCollection(this); Definitions = new Dictionary(); Inclusion = ImmutableStack.Nil; - this.directiveHandler = directiveHandler; - - Pool = new Pool(); + DirectiveHandler = directiveHandler; } public bool IsReservedName(string name) { return Raws.ContainsKey(name.ToUpperInvariant()) || SpecialCodes.Contains(name.ToUpperInvariant()); } + public bool IsValidDefinitionName(string name) { return !(Definitions.ContainsKey(name) || IsReservedName(name)); } + public bool IsValidMacroName(string name, int paramNum) { return !Macros.HasMacro(name, paramNum) && !IsReservedName(name); } - public bool IsValidLabelName(string name) - { - return true;//!IsReservedName(name); - //TODO? - } + public IList ParseAll(IEnumerable tokenStream) { //TODO: Make BlockNode or EAProgramNode? //Note must be strict to get all information on the closure before evaluating terms. IList myLines = new List(); + MergeableGenerator tokens = new MergeableGenerator(tokenStream); tokens.MoveNext(); + while (!tokens.EOS) { if (tokens.Current.Type != TokenType.NEWLINE || tokens.MoveNext()) @@ -131,6 +94,7 @@ public IList ParseAll(IEnumerable tokenStream) retVal.IfJust(n => myLines.Add(n)); } } + return myLines; } @@ -138,6 +102,7 @@ private BlockNode ParseBlock(MergeableGenerator tokens, ImmutableStack tokens, ImmutableStack 0 && value < EAOptions.MaximumBinarySize) - { - value += EAOptions.BaseAddress; - } - - return value; - } - - public static int ConvertToOffset(int value) - { - if (value >= EAOptions.BaseAddress && value <= EAOptions.BaseAddress + EAOptions.MaximumBinarySize) - { - value -= EAOptions.BaseAddress; - } - - return value; - } + public static readonly HashSet SpecialCodes = new HashSet { "ORG", "PUSH", "POP", "MESSAGE", "WARNING", "ERROR", "ASSERT", "PROTECT", "ALIGN", "FILL" }; private ILineNode? ParseStatement(MergeableGenerator tokens, ImmutableStack scopes) { - while (ExpandIdentifier(tokens, scopes)) { } + // NOTE: here previously lied en ExpandIdentifier loop + // though because this is only called from ParseLine after the corresponding check, this is not needed head = tokens.Current; tokens.MoveNext(); - //TODO: Replace with real raw information, and error if not valid. - IList parameters; - //TODO: Make intelligent to reject malformed parameters. - //TODO: Parse parameters after checking code validity. - if (tokens.Current.Type != TokenType.NEWLINE && tokens.Current.Type != TokenType.SEMICOLON) + switch (tokens.Current.Type) { - parameters = ParseParamList(tokens, scopes); + case TokenType.COLON: + tokens.MoveNext(); + return HandleLabel(head.Content, scopes); + + case TokenType.ASSIGN: + tokens.MoveNext(); + return ParseAssignment(head.Content, tokens, scopes); } - else + + // NOTE: those remarks are old ones from Colorz, idrk what they mean -Stan + // TODO: Replace with real raw information, and error if not valid. + // TODO: Make intelligent to reject malformed parameters. + // TODO: Parse parameters after checking code validity. + + IList parameters = tokens.Current.Type switch { - parameters = new List(); - tokens.MoveNext(); - } + TokenType.NEWLINE or TokenType.SEMICOLON => new List(), + _ => ParseParamList(tokens, scopes), + }; string upperCodeIdentifier = head.Content.ToUpperInvariant(); @@ -217,38 +165,49 @@ public static int ConvertToOffset(int value) { return upperCodeIdentifier switch { - "ORG" => ParseOrgStatement(parameters), - "PUSH" => ParsePushStatement(parameters), - "POP" => ParsePopStatement(parameters), - "ASSERT" => ParseAssertStatement(parameters), - "PROTECT" => ParseProtectStatement(parameters), - "ALIGN" => ParseAlignStatement(parameters), - "FILL" => ParseFillStatement(parameters), + "ORG" => ParseOrgStatement(parameters, scopes), + "PUSH" => ParsePushStatement(parameters, scopes), + "POP" => ParsePopStatement(parameters, scopes), + "ASSERT" => ParseAssertStatement(parameters, scopes), + "PROTECT" => ParseProtectStatement(parameters, scopes), + "ALIGN" => ParseAlignStatement(parameters, scopes), + "FILL" => ParseFillStatement(parameters, scopes), "MESSAGE" => ParseMessageStatement(parameters, scopes), "WARNING" => ParseWarningStatement(parameters, scopes), "ERROR" => ParseErrorStatement(parameters, scopes), _ => null, // TODO: this is an error }; } - else if (Raws.TryGetValue(upperCodeIdentifier, out IList? raws)) + + return ParseRawStatement(upperCodeIdentifier, tokens, parameters); + } + + private ILineNode? ParseAssignment(string name, MergeableGenerator tokens, ImmutableStack scopes) + { + IAtomNode? atom = this.ParseAtom(tokens, scopes, true); + + if (atom != null) + { + return HandleSymbolAssignment(name, atom, scopes); + } + else + { + Error($"Couldn't define symbol `{name}`: exprected expression."); + } + + return null; + } + + private ILineNode? ParseRawStatement(string upperCodeIdentifier, MergeableGenerator tokens, IList parameters) + { + if (Raws.TryGetValue(upperCodeIdentifier, out IList? raws)) { - //TODO: Check for matches. Currently should type error. + // find the raw matching with parameters foreach (Raw raw in raws) { if (raw.Fits(parameters)) { - if ((CurrentOffset % raw.Alignment) != 0) - { - Error($"Bad code alignment (offset: {CurrentOffset:X8})"); - } - - StatementNode temp = new RawNode(raw, head, CurrentOffset, parameters); - - // TODO: more efficient spacewise to just have contiguous writing and not an offset with every line? - CheckDataWrite(temp.Size); - CurrentOffset += temp.Size; - - return temp; + return HandleRawStatement(new RawNode(raw, head!, CurrentOffset, parameters)); } } @@ -260,7 +219,7 @@ public static int ConvertToOffset(int value) { StringBuilder sb = new StringBuilder(); - sb.Append($"Couldn't find suitable variant of raw `{head.Content}`."); + sb.Append($"Couldn't find suitable variant of raw `{head!.Content}`."); for (int i = 0; i < raws.Count; i++) { @@ -275,247 +234,141 @@ public static int ConvertToOffset(int value) } else { - Error("Unrecognized code: " + head.Content); + Error($"Unrecognized statement code: {head!.Content}"); return null; } } - private ILineNode? ParseOrgStatement(IList parameters) + private ILineNode? ParseOrgStatement(IList parameters, ImmutableStack _) { - if (parameters.Count != 1) - { - Error($"Incorrect number of parameters in ORG: {parameters.Count}"); - return null; - } - - parameters[0].AsAtom().IfJust( - atom => atom.TryEvaluate(e => Error(parameters[0].MyLocation, e.Message), EvaluationPhase.Immediate).IfJust( - offsetValue => { CurrentOffset = ConvertToOffset(offsetValue); }, - () => Error(parameters[0].MyLocation, "Expected atomic param to ORG."))); - - return null; + return ParseStatementOneParam("ORG", parameters, HandleOrgStatement); } - private ILineNode? ParsePushStatement(IList parameters) + private ILineNode? ParsePushStatement(IList parameters, ImmutableStack _) { - if (parameters.Count != 0) + if (parameters.Count == 0) { - Error("Incorrect number of parameters in PUSH: " + parameters.Count); + return HandlePushStatement(); } else { - pastOffsets.Push(new Tuple(CurrentOffset, offsetInitialized)); + return StatementExpectsParamCount("PUSH", parameters, 0, 0); } - - return null; } - private ILineNode? ParsePopStatement(IList parameters) + private ILineNode? ParsePopStatement(IList parameters, ImmutableStack _) { - if (parameters.Count != 0) - { - Error($"Incorrect number of parameters in POP: {parameters.Count}"); - } - else if (pastOffsets.Count == 0) + if (parameters.Count == 0) { - Error("POP without matching PUSH."); + return HandlePopStatement(); } else { - Tuple tuple = pastOffsets.Pop(); - - CurrentOffset = tuple.Item1; - offsetInitialized = tuple.Item2; + return StatementExpectsParamCount("POP", parameters, 0, 0); } + } + + private ILineNode? ParseAssertStatement(IList parameters, ImmutableStack _) + { + return ParseStatementOneParam("ASSERT", parameters, HandleAssertStatement); + } + + // Helper method for printing errors + private ILineNode? StatementExpectsAtom(string statementName, IParamNode param) + { + Error(param.MyLocation, + $"{statementName} expects an Atom (got {DiagnosticsHelpers.PrettyParamType(param.Type)})."); return null; } - private ILineNode? ParseAssertStatement(IList parameters) + // Helper method for printing errors + private ILineNode? StatementExpectsParamCount(string statementName, IList parameters, int min, int max) { - if (parameters.Count != 1) + if (min == max) { - Error($"Incorrect number of parameters in ASSERT: {parameters.Count}"); - return null; + Error($"A {statementName} statement expects {min} parameters, got {parameters.Count}."); } - - // helper for distinguishing boolean expressions and other expressions - static bool IsConditionalOperatorHelper(IAtomNode node) + else { - return node switch - { - UnaryOperatorNode uon => uon.OperatorToken.Type switch - { - TokenType.LOGNOT_OP => true, - _ => false, - }, - - OperatorNode on => on.OperatorToken.Type switch - { - TokenType.LOGAND_OP => true, - TokenType.LOGOR_OP => true, - TokenType.COMPARE_EQ => true, - TokenType.COMPARE_NE => true, - TokenType.COMPARE_GT => true, - TokenType.COMPARE_GE => true, - TokenType.COMPARE_LE => true, - TokenType.COMPARE_LT => true, - _ => false, - }, - - _ => false, - }; + Error($"A {statementName} statement expects {min} to {max} parameters, got {parameters.Count}."); } - IAtomNode? atom = parameters[0].AsAtom(); + return null; + } - if (atom != null) - { - bool isBoolean = IsConditionalOperatorHelper(atom); + private delegate ILineNode? HandleStatementOne(IAtomNode node); + private delegate ILineNode? HandleStatementTwo(IAtomNode firstNode, IAtomNode? optionalSecondNode); - atom.TryEvaluate(e => Error(parameters[0].MyLocation, e.Message), EvaluationPhase.Immediate).IfJust( - temp => - { - // if boolean expession => fail if 0, else (legacy behavoir) fail if negative - if (isBoolean && temp == 0 || !isBoolean && temp < 0) - { - Error(parameters[0].MyLocation, "Assertion error: " + temp); - } - }); + private ILineNode? ParseStatementOneParam(string name, IList parameters, HandleStatementOne handler) + { + if (parameters.Count == 1) + { + if (parameters[0] is IAtomNode expr) + { + return handler(expr); + } + else + { + return StatementExpectsAtom(name, parameters[0]); + } } else { - Error(parameters[0].MyLocation, "Expected atomic param to ASSERT."); + return StatementExpectsParamCount(name, parameters, 1, 1); } - - return null; } - private ILineNode? ParseProtectStatement(IList parameters) + private ILineNode? ParseStatementTwoParam(string name, IList parameters, HandleStatementTwo handler) { if (parameters.Count == 1) { - parameters[0].AsAtom().IfJust( - atom => atom.TryEvaluate(e => { Error(parameters[0].MyLocation, e.Message); }, EvaluationPhase.Immediate).IfJust( - temp => - { - protectedRegions.Add(new Tuple(temp, 4, head!.Location)); - }), - () => { Error(parameters[0].MyLocation, "Expected atomic param to PROTECT"); }); + if (parameters[0] is IAtomNode firstNode) + { + return handler(firstNode, null); + } + else + { + return StatementExpectsAtom(name, parameters[0]); + } } else if (parameters.Count == 2) { - int start = 0, end = 0; - bool errorOccurred = false; - parameters[0].AsAtom().IfJust( - atom => atom.TryEvaluate(e => { Error(parameters[0].MyLocation, e.Message); errorOccurred = true; }, EvaluationPhase.Immediate).IfJust( - temp => - { - start = temp; - }), - () => { Error(parameters[0].MyLocation, "Expected atomic param to PROTECT"); errorOccurred = true; }); - parameters[1].AsAtom().IfJust( - atom => atom.TryEvaluate(e => { Error(parameters[0].MyLocation, e.Message); errorOccurred = true; }, EvaluationPhase.Immediate).IfJust( - temp => - { - end = temp; - }), - () => { Error(parameters[0].MyLocation, "Expected atomic param to PROTECT"); errorOccurred = true; }); - if (!errorOccurred) + if (parameters[0] is IAtomNode firstNode) { - int length = end - start; - - if (length > 0) + if (parameters[1] is IAtomNode secondNode) { - protectedRegions.Add(new Tuple(start, length, head!.Location)); + return handler(firstNode, secondNode); } else { - Warning("Protected region not valid (end offset not after start offset). No region protected."); + return StatementExpectsAtom(name, parameters[1]); } } + else + { + return StatementExpectsAtom(name, parameters[0]); + } } else { - Error("Incorrect number of parameters in PROTECT: " + parameters.Count); + return StatementExpectsParamCount(name, parameters, 1, 2); } - - return null; } - private ILineNode? ParseAlignStatement(IList parameters) + private ILineNode? ParseProtectStatement(IList parameters, ImmutableStack _) { - if (parameters.Count != 1) - { - Error("Incorrect number of parameters in ALIGN: " + parameters.Count); - return null; - } - - parameters[0].AsAtom().IfJust( - atom => atom.TryEvaluate(e => Error(parameters[0].MyLocation, e.Message), EvaluationPhase.Immediate).IfJust( - temp => - { - if (temp <= 0) - { - Error($"Cannot align address to {temp}"); - } - else if (CurrentOffset % temp != 0) - { - CurrentOffset += temp - CurrentOffset % temp; - } - }), - () => Error(parameters[0].MyLocation, "Expected atomic param to ALIGN")); - - return null; + return ParseStatementTwoParam("PROTECT", parameters, HandleProtectStatement); } - private ILineNode? ParseFillStatement(IList parameters) + private ILineNode? ParseAlignStatement(IList parameters, ImmutableStack _) { - if (parameters.Count > 2 || parameters.Count == 0) - { - Error("Incorrect number of parameters in FILL: " + parameters.Count); - return null; - } - - // FILL amount [value] - - int amount = 0; - int value = 0; - - if (parameters.Count == 2) - { - // param 2 (if given) is fill value - - parameters[1].AsAtom().IfJust( - atom => atom.TryEvaluate(e => Error(parameters[0].MyLocation, e.Message), EvaluationPhase.Immediate).IfJust( - val => { value = val; }), - () => Error(parameters[0].MyLocation, "Expected atomic param to FILL")); - } - - // param 1 is amount of bytes to fill - parameters[0].AsAtom().IfJust( - atom => atom.TryEvaluate(e => Error(parameters[0].MyLocation, e.Message), EvaluationPhase.Immediate).IfJust( - val => { amount = val; }), - () => Error(parameters[0].MyLocation, "Expected atomic param to FILL")); - - if (amount > 0) - { - var data = new byte[amount]; - - for (int i = 0; i < amount; ++i) - { - data[i] = (byte)value; - } - - var node = new DataNode(CurrentOffset, data); - - CheckDataWrite(amount); - CurrentOffset += amount; - - return node; - } + return ParseStatementTwoParam("ALIGN", parameters, HandleAlignStatement); + } - return null; + private ILineNode? ParseFillStatement(IList parameters, ImmutableStack _) + { + return ParseStatementTwoParam("FILL", parameters, HandleFillStatement); } private ILineNode? ParseMessageStatement(IList parameters, ImmutableStack scopes) @@ -543,6 +396,7 @@ public IList> ParseMacroParamList(MergeableGenerator tokens) // HACK: this allows macro([1, 2, 3]) from expanding into a single parameter int bracketBalance = 0; + do { tokens.MoveNext(); @@ -571,8 +425,11 @@ public IList> ParseMacroParamList(MergeableGenerator tokens) currentParam.Add(tokens.Current); tokens.MoveNext(); } + parameters.Add(currentParam); - } while (tokens.Current.Type != TokenType.CLOSE_PAREN && tokens.Current.Type != TokenType.NEWLINE); + } + while (tokens.Current.Type != TokenType.CLOSE_PAREN && tokens.Current.Type != TokenType.NEWLINE); + if (tokens.Current.Type != TokenType.CLOSE_PAREN || parenNestings != 0) { Error(tokens.Current.Location, "Unmatched open parenthesis."); @@ -581,6 +438,7 @@ public IList> ParseMacroParamList(MergeableGenerator tokens) { tokens.MoveNext(); } + return parameters; } @@ -658,304 +516,54 @@ public IList ParsePreprocParamList(MergeableGenerator tokens, return new StringNode(new Token(TokenType.STRING, localHead.Location, localHead.GetSourceLocation().file)); default: - return ParseAtom(tokens, scopes, expandDefs); + return this.ParseAtom(tokens, scopes, expandDefs); } } default: - return ParseAtom(tokens, scopes, expandDefs); + return this.ParseAtom(tokens, scopes, expandDefs); } } - private static readonly Dictionary precedences = new Dictionary { - { TokenType.MUL_OP, 3 }, - { TokenType.DIV_OP, 3 }, - { TokenType.MOD_OP, 3 }, - { TokenType.ADD_OP, 4 }, - { TokenType.SUB_OP, 4 }, - { TokenType.LSHIFT_OP, 5 }, - { TokenType.RSHIFT_OP, 5 }, - { TokenType.SIGNED_RSHIFT_OP, 5 }, - { TokenType.COMPARE_GE, 6 }, - { TokenType.COMPARE_GT, 6 }, - { TokenType.COMPARE_LT, 6 }, - { TokenType.COMPARE_LE, 6 }, - { TokenType.COMPARE_EQ, 7 }, - { TokenType.COMPARE_NE, 7 }, - { TokenType.AND_OP, 8 }, - { TokenType.XOR_OP, 9 }, - { TokenType.OR_OP, 10 }, - { TokenType.LOGAND_OP, 11 }, - { TokenType.LOGOR_OP, 12 }, - { TokenType.UNDEFINED_COALESCE_OP, 13 }, - }; - - public static bool IsInfixOperator(Token token) => precedences.ContainsKey(token.Type); - - public IAtomNode? ParseAtom(MergeableGenerator tokens, ImmutableStack scopes, bool expandDefs = true) + private IList ParseList(MergeableGenerator tokens, ImmutableStack scopes) { - //Use Shift Reduce Parsing Token localHead = tokens.Current; - Stack> grammarSymbols = new Stack>(); - bool ended = false; - while (!ended) - { - bool shift = false, lookingForAtom = grammarSymbols.Count == 0 || grammarSymbols.Peek().IsRight; - Token lookAhead = tokens.Current; + tokens.MoveNext(); - if (!ended && !lookingForAtom) //Is already a complete node. Needs an operator of matching precedence and a node of matching prec to reduce. + IList atoms = new List(); + while (tokens.Current.Type != TokenType.NEWLINE && tokens.Current.Type != TokenType.CLOSE_BRACKET) + { + IAtomNode? res = this.ParseAtom(tokens, scopes); + res.IfJust( + n => atoms.Add(n), + () => Error(tokens.Current.Location, "Expected atomic value, got " + tokens.Current.Type + ".")); + if (tokens.Current.Type == TokenType.COMMA) { - //Verify next symbol to be a binary operator. - switch (lookAhead.Type) - { - case TokenType.MUL_OP: - case TokenType.DIV_OP: - case TokenType.MOD_OP: - case TokenType.ADD_OP: - case TokenType.SUB_OP: - case TokenType.LSHIFT_OP: - case TokenType.RSHIFT_OP: - case TokenType.SIGNED_RSHIFT_OP: - case TokenType.AND_OP: - case TokenType.XOR_OP: - case TokenType.OR_OP: - case TokenType.LOGAND_OP: - case TokenType.LOGOR_OP: - case TokenType.COMPARE_LT: - case TokenType.COMPARE_LE: - case TokenType.COMPARE_EQ: - case TokenType.COMPARE_NE: - case TokenType.COMPARE_GE: - case TokenType.COMPARE_GT: - if (precedences.TryGetValue(lookAhead.Type, out int precedence)) - { - Reduce(grammarSymbols, precedence); - } - shift = true; - break; - case TokenType.UNDEFINED_COALESCE_OP: - // '??' is right-associative, so don't reduce here - shift = true; - break; - default: - ended = true; - break; - } + tokens.MoveNext(); } - else if (!ended) //Is just an operator. Error if two operators in a row. - { - //Error if two operators in a row. - switch (lookAhead.Type) - { - case TokenType.IDENTIFIER: - case TokenType.MAYBE_MACRO: - case TokenType.NUMBER: - shift = true; - break; - case TokenType.OPEN_PAREN: - { - tokens.MoveNext(); - IAtomNode? interior = ParseAtom(tokens, scopes); - if (tokens.Current.Type != TokenType.CLOSE_PAREN) - { - Error(tokens.Current.Location, "Unmatched open parenthesis (currently at " + tokens.Current.Type + ")."); - return null; - } - else if (interior == null) - { - Error(lookAhead.Location, "Expected expression inside paretheses. "); - return null; - } - else - { - grammarSymbols.Push(new Left(interior)); - tokens.MoveNext(); - break; - } - } - case TokenType.SUB_OP: - case TokenType.LOGNOT_OP: - case TokenType.NOT_OP: - { - //Assume unary negation. - tokens.MoveNext(); - IAtomNode? interior = ParseAtom(tokens, scopes); - if (interior == null) - { - Error(lookAhead.Location, "Expected expression after unary operator."); - return null; - } - grammarSymbols.Push(new Left(new UnaryOperatorNode(lookAhead, interior))); - break; - } - case TokenType.COMMA: - Error(lookAhead.Location, "Unexpected comma (perhaps unrecognized macro invocation?)."); - IgnoreRestOfStatement(tokens); - return null; - case TokenType.MUL_OP: - case TokenType.DIV_OP: - case TokenType.MOD_OP: - case TokenType.ADD_OP: - case TokenType.LSHIFT_OP: - case TokenType.RSHIFT_OP: - case TokenType.SIGNED_RSHIFT_OP: - case TokenType.AND_OP: - case TokenType.XOR_OP: - case TokenType.OR_OP: - case TokenType.LOGAND_OP: - case TokenType.LOGOR_OP: - case TokenType.COMPARE_LT: - case TokenType.COMPARE_LE: - case TokenType.COMPARE_EQ: - case TokenType.COMPARE_NE: - case TokenType.COMPARE_GE: - case TokenType.COMPARE_GT: - case TokenType.UNDEFINED_COALESCE_OP: - default: - Error(lookAhead.Location, $"Expected identifier or literal, got {lookAhead.Type}: {lookAhead.Content}."); - IgnoreRestOfStatement(tokens); - return null; - } - } - - if (shift) - { - switch (lookAhead.Type) - { - case TokenType.IDENTIFIER: - if (expandDefs && ExpandIdentifier(tokens, scopes, true)) - { - continue; - } - - grammarSymbols.Push(new Left(lookAhead.Content.ToUpperInvariant() switch - { - "CURRENTOFFSET" => new NumberNode(lookAhead, CurrentOffset), - "__LINE__" => new NumberNode(lookAhead, lookAhead.GetSourceLocation().line), - _ => new IdentifierNode(lookAhead, scopes), - })); - - break; - - case TokenType.MAYBE_MACRO: - ExpandIdentifier(tokens, scopes, true); - continue; - case TokenType.NUMBER: - grammarSymbols.Push(new Left(new NumberNode(lookAhead))); - break; - case TokenType.ERROR: - Error(lookAhead.Location, $"Unexpected token: {lookAhead.Content}"); - tokens.MoveNext(); - return null; - default: - grammarSymbols.Push(new Right(lookAhead)); - break; - } - tokens.MoveNext(); - continue; - } - } - while (grammarSymbols.Count > 1) - { - Reduce(grammarSymbols, int.MaxValue); - } - if (grammarSymbols.Peek().IsRight) - { - Error(grammarSymbols.Peek().GetRight.Location, $"Unexpected token: {grammarSymbols.Peek().GetRight.Type}"); - } - return grammarSymbols.Peek().GetLeft; - } - - /*** - * Precondition: grammarSymbols alternates between IAtomNodes, operator Tokens, .Count is odd - * the precedences of the IAtomNodes is increasing. - * Postcondition: Either grammarSymbols.Count == 1, or everything in grammarSymbols will have precedence <= targetPrecedence. - * - */ - private void Reduce(Stack> grammarSymbols, int targetPrecedence) - { - while (grammarSymbols.Count > 1)// && grammarSymbols.Peek().GetLeft.Precedence > targetPrecedence) - { - // These shouldn't error... - IAtomNode r = grammarSymbols.Pop().GetLeft; - - if (precedences[grammarSymbols.Peek().GetRight.Type] > targetPrecedence) - { - grammarSymbols.Push(new Left(r)); - break; - } - else - { - Token op = grammarSymbols.Pop().GetRight; - IAtomNode l = grammarSymbols.Pop().GetLeft; - - OperatorNode operatorNode = new OperatorNode(l, op, r, l.Precedence); - - if (EAOptions.IsWarningEnabled(EAOptions.Warnings.UnintuitiveExpressionMacros)) - { - if (DiagnosticHelpers.DoesOperationSpanMultipleMacrosUnintuitively(operatorNode)) - { - MacroLocation? mloc = operatorNode.MyLocation.macroLocation; - string message = DiagnosticHelpers.GetEmphasizedExpression(operatorNode, l => l.macroLocation == mloc); - - if (mloc != null) - { - message += $"\nUnintuitive expression resulting from expansion of macro `{mloc.MacroName}`."; - } - else - { - message += "\nUnintuitive expression resulting from expansion of macro."; - } - - message += "\nConsider guarding your expressions using parenthesis."; - - Warning(operatorNode.MyLocation, message); - } - } - - grammarSymbols.Push(new Left(operatorNode)); - } - } - } - - private IList ParseList(MergeableGenerator tokens, ImmutableStack scopes) - { - Token localHead = tokens.Current; - tokens.MoveNext(); - - IList atoms = new List(); - while (tokens.Current.Type != TokenType.NEWLINE && tokens.Current.Type != TokenType.CLOSE_BRACKET) - { - IAtomNode? res = ParseAtom(tokens, scopes); - res.IfJust( - n => atoms.Add(n), - () => Error(tokens.Current.Location, "Expected atomic value, got " + tokens.Current.Type + ".")); - if (tokens.Current.Type == TokenType.COMMA) - { - tokens.MoveNext(); - } - } - if (tokens.Current.Type == TokenType.CLOSE_BRACKET) - { - tokens.MoveNext(); - } - else - { - Error(localHead.Location, "Unmatched open bracket."); - } - - return atoms; - } - - public ILineNode? ParseLine(MergeableGenerator tokens, ImmutableStack scopes) - { - if (IsIncluding) - { - if (tokens.Current.Type == TokenType.NEWLINE || tokens.Current.Type == TokenType.SEMICOLON) + } + if (tokens.Current.Type == TokenType.CLOSE_BRACKET) + { + tokens.MoveNext(); + } + else + { + Error(localHead.Location, "Unmatched open bracket."); + } + + return atoms; + } + + public ILineNode? ParseLine(MergeableGenerator tokens, ImmutableStack scopes) + { + if (IsIncluding) + { + if (tokens.Current.Type == TokenType.NEWLINE || tokens.Current.Type == TokenType.SEMICOLON) { tokens.MoveNext(); return null; } + head = tokens.Current; switch (head.Type) { @@ -981,44 +589,26 @@ private IList ParseList(MergeableGenerator tokens, ImmutableSt return null; } } - else - { - tokens.MoveNext(); - switch (tokens.Current.Type) - { - case TokenType.COLON: - tokens.MoveNext(); - TryDefineSymbol(scopes, head.Content, ConvertToAddress(CurrentOffset)); - return null; - case TokenType.ASSIGN: - tokens.MoveNext(); - ParseAtom(tokens, scopes, true).IfJust( - atom => atom.TryEvaluate( - e => TryDefineSymbol(scopes, head.Content, atom), EvaluationPhase.Early).IfJust( - value => TryDefineSymbol(scopes, head.Content, value)), - () => Error($"Couldn't define symbol `{head.Content}`: exprected expression.")); + return ParseStatement(tokens, scopes); - return null; - - default: - tokens.PutBack(head); - return ParseStatement(tokens, scopes); - } - } case TokenType.OPEN_BRACE: return ParseBlock(tokens, new ImmutableStack(new Closure(), scopes)); + case TokenType.PREPROCESSOR_DIRECTIVE: return ParsePreprocessor(tokens, scopes); + case TokenType.OPEN_BRACKET: Error("Unexpected list literal."); IgnoreRestOfLine(tokens); break; + case TokenType.NUMBER: case TokenType.OPEN_PAREN: Error("Unexpected mathematical expression."); IgnoreRestOfLine(tokens); break; + default: tokens.MoveNext(); @@ -1039,9 +629,9 @@ private IList ParseList(MergeableGenerator tokens, ImmutableSt else { bool hasNext = true; + while (tokens.Current.Type != TokenType.PREPROCESSOR_DIRECTIVE && (hasNext = tokens.MoveNext())) { - ; } if (hasNext) @@ -1056,46 +646,12 @@ private IList ParseList(MergeableGenerator tokens, ImmutableSt } } - private void TryDefineSymbol(ImmutableStack scopes, string name, int value) - { - if (scopes.Head.HasLocalSymbol(name)) - { - Warning($"Symbol already in scope, ignoring: {name}"); - } - else if (!IsValidLabelName(name)) - { - // NOTE: IsValidLabelName returns true always. This is dead code - Error($"Invalid symbol name {name}."); - } - else - { - scopes.Head.AddSymbol(name, value); - } - } - - private void TryDefineSymbol(ImmutableStack scopes, string name, IAtomNode expression) - { - if (scopes.Head.HasLocalSymbol(name)) - { - Warning($"Symbol already in scope, ignoring: {name}"); - } - else if (!IsValidLabelName(name)) - { - // NOTE: IsValidLabelName returns true always. This is dead code - Error($"Invalid symbol name {name}."); - } - else - { - scopes.Head.AddSymbol(name, expression); - } - } - private ILineNode? ParsePreprocessor(MergeableGenerator tokens, ImmutableStack scopes) { head = tokens.Current; tokens.MoveNext(); - ILineNode? result = directiveHandler.HandleDirective(this, head, tokens, scopes); + ILineNode? result = DirectiveHandler.HandleDirective(this, head, tokens, scopes); if (result != null) { @@ -1168,7 +724,7 @@ private void ApplyMacroExpansion(MergeableGenerator tokens, IEnumerable expandedList = expandedTokens.ToList(); - DiagnosticHelpers.VisitUnguardedOperators(expandedList, + DiagnosticsHelpers.VisitUnguardedOperators(expandedList, token => Warning(token.Location, $"Unguarded expansion of mathematical operator. Consider adding guarding parenthesises around definition.")); tokens.PrependEnumerator(expandedList.GetEnumerator()); @@ -1179,13 +735,13 @@ private void ApplyMacroExpansion(MergeableGenerator tokens, IEnumerable MessageTrace(Log.MessageKind.MESSAGE, location, message); - public void Warning(Location? location, string message) => MessageTrace(Log.MessageKind.WARNING, location, message); - public void Error(Location? location, string message) => MessageTrace(Log.MessageKind.ERROR, location, message); + public void Message(Location? location, string message) => MessageTrace(Logger.MessageKind.MESSAGE, location, message); + public void Warning(Location? location, string message) => MessageTrace(Logger.MessageKind.WARNING, location, message); + public void Error(Location? location, string message) => MessageTrace(Logger.MessageKind.ERROR, location, message); - public void Message(string message) => MessageTrace(Log.MessageKind.MESSAGE, head?.Location, message); - public void Warning(string message) => MessageTrace(Log.MessageKind.WARNING, head?.Location, message); - public void Error(string message) => MessageTrace(Log.MessageKind.ERROR, head?.Location, message); + public void Message(string message) => MessageTrace(Logger.MessageKind.MESSAGE, head?.Location, message); + public void Warning(string message) => MessageTrace(Logger.MessageKind.WARNING, head?.Location, message); + public void Error(string message) => MessageTrace(Logger.MessageKind.ERROR, head?.Location, message); - private void IgnoreRestOfStatement(MergeableGenerator tokens) + public void IgnoreRestOfStatement(MergeableGenerator tokens) { while (tokens.Current.Type != TokenType.NEWLINE && tokens.Current.Type != TokenType.SEMICOLON && tokens.MoveNext()) { } if (tokens.Current.Type == TokenType.SEMICOLON) @@ -1245,16 +801,6 @@ public IList GetRestOfLine(MergeableGenerator tokens, ImmutableSta return result; } - public void Clear() - { - Macros.Clear(); - Definitions.Clear(); - Raws.Clear(); - Inclusion = ImmutableStack.Nil; - CurrentOffset = 0; - pastOffsets.Clear(); - } - private string PrettyPrintParamsForMessage(IList parameters, ImmutableStack scopes) { return string.Join(" ", parameters.Select(parameter => parameter switch @@ -1286,7 +832,7 @@ string UserFormatStringError(string message, string details) tokens.MoveNext(); - IAtomNode? node = ParseAtom(tokens, scopes); + IAtomNode? node = this.ParseAtom(tokens, scopes); if (node == null || tokens.Current.Type != TokenType.NEWLINE) { @@ -1314,16 +860,397 @@ string UserFormatStringError(string message, string details) }); } + /* + * ========================================= + * = NON STRICTLY PARSE RELATED START HERE = + * ========================================= + */ + + public int CurrentOffset + { + get => currentOffset; + + private set + { + if (value < 0 || value > EAOptions.MaximumBinarySize) + { + if (validOffset) //Error only the first time. + { + Error($"Invalid offset: {value:X}"); + validOffset = false; + } + } + else + { + currentOffset = value; + validOffset = true; + offsetInitialized = true; + } + } + } + + public ImmutableStack GlobalScope { get; } + + private readonly Stack<(int, bool)> pastOffsets; // currentOffset, offsetInitialized + private readonly IList<(int, int, Location)> protectedRegions; + + private bool validOffset; + private bool offsetInitialized; // false until first ORG, used to warn about writing before first org + private int currentOffset; + + // TODO: these next two functions should probably be moved into their own module + + public static int ConvertToAddress(int value) + { + /* + NOTE: Offset 0 is always converted to a null address + If one wants to instead refer to ROM offset 0 they would want to use the address directly instead. + If ROM offset 0 is already address 0 then this is a moot point. + */ + + if (value > 0 && value < EAOptions.MaximumBinarySize) + { + value += EAOptions.BaseAddress; + } + + return value; + } + + public static int ConvertToOffset(int value) + { + if (value >= EAOptions.BaseAddress && value <= EAOptions.BaseAddress + EAOptions.MaximumBinarySize) + { + value -= EAOptions.BaseAddress; + } + + return value; + } + + // Helper method for statement handlers + private int? EvaluteAtom(IAtomNode node) + { + return node.TryEvaluate(e => Error(node.MyLocation, e.Message), EvaluationPhase.Immediate); + } + + private ILineNode? HandleRawStatement(RawNode node) + { + if ((CurrentOffset % node.Raw.Alignment) != 0) + { + Error($"Bad alignment for raw {node.Raw.Name}: offseet ({CurrentOffset:X8}) needs to be {node.Raw.Alignment}-aligned."); + return null; + } + else + { + // TODO: more efficient spacewise to just have contiguous writing and not an offset with every line? + CheckDataWrite(node.Size); + CurrentOffset += node.Size; + + return node; + } + } + + private ILineNode? HandleOrgStatement(IAtomNode offsetNode) + { + if (EvaluteAtom(offsetNode) is int offset) + { + CurrentOffset = ConvertToOffset(offset); + } + else + { + // EvaluateAtom already printed an error message + } + + return null; + } + + private ILineNode? HandlePushStatement() + { + pastOffsets.Push((CurrentOffset, offsetInitialized)); + return null; + } + + private ILineNode? HandlePopStatement() + { + if (pastOffsets.Count == 0) + { + Error("POP without matching PUSH."); + } + else + { + (CurrentOffset, offsetInitialized) = pastOffsets.Pop(); + } + + return null; + } + + private ILineNode? HandleAssertStatement(IAtomNode node) + { + // helper for distinguishing boolean expressions and other expressions + // TODO: move elsewhere perhaps + static bool IsBooleanResultHelper(IAtomNode node) + { + return node switch + { + UnaryOperatorNode uon => uon.OperatorToken.Type switch + { + TokenType.LOGNOT_OP => true, + _ => false, + }, + + OperatorNode on => on.OperatorToken.Type switch + { + TokenType.LOGAND_OP => true, + TokenType.LOGOR_OP => true, + TokenType.COMPARE_EQ => true, + TokenType.COMPARE_NE => true, + TokenType.COMPARE_GT => true, + TokenType.COMPARE_GE => true, + TokenType.COMPARE_LE => true, + TokenType.COMPARE_LT => true, + _ => false, + }, + + _ => false, + }; + } + + bool isBoolean = IsBooleanResultHelper(node); + + if (EvaluteAtom(node) is int result) + { + if (isBoolean && result == 0) + { + Error(node.MyLocation, "Assertion failed"); + } + else if (!isBoolean && result < 0) + { + Error(node.MyLocation, $"Assertion failed with value {result}."); + } + } + else + { + Error("Failed to evaluate ASSERT expression."); + } + + return null; + } + + private ILineNode? HandleProtectStatement(IAtomNode beginAtom, IAtomNode? endAtom) + { + if (EvaluteAtom(beginAtom) is int beginValue) + { + beginValue = ConvertToAddress(beginValue); + + int length = 4; + + if (endAtom != null) + { + if (EvaluteAtom(endAtom) is int endValue) + { + endValue = ConvertToAddress(endValue); + + length = endValue - beginValue; + + switch (length) + { + case < 0: + Error($"Invalid PROTECT region: end address ({endValue:X8}) is before start address ({beginValue:X8})."); + return null; + + case 0: + // NOTE: does this need to be an error? + Error($"Empty PROTECT region: end address is equal to start address ({beginValue:X8})."); + return null; + } + } + else + { + // EvaluateAtom already printed an error message + return null; + } + } + + protectedRegions.Add((beginValue, length, head!.Location)); + + return null; + } + else + { + // EvaluateAtom already printed an error message + return null; + } + } + + private ILineNode? HandleAlignStatement(IAtomNode alignNode, IAtomNode? offsetNode) + { + if (EvaluteAtom(alignNode) is int alignValue) + { + if (alignValue > 0) + { + int alignOffset = 0; + + if (offsetNode != null) + { + if (EvaluteAtom(offsetNode) is int rawOffset) + { + if (rawOffset >= 0) + { + alignOffset = ConvertToOffset(rawOffset) % alignValue; + } + else + { + Error($"ALIGN offset cannot be negative (got {rawOffset})"); + return null; + } + } + else + { + // EvaluateAtom already printed an error message + return null; + } + } + + if (CurrentOffset % alignValue != alignOffset) + { + CurrentOffset += alignValue - (CurrentOffset + alignValue - alignOffset) % alignValue; + } + + return null; + } + else + { + Error($"Invalid ALIGN value (got {alignValue})."); + return null; + } + } + else + { + // EvaluateAtom already printed an error message + return null; + } + } + + private ILineNode? HandleFillStatement(IAtomNode amountNode, IAtomNode? valueNode) + { + if (EvaluteAtom(amountNode) is int amount) + { + if (amount > 0) + { + int fillValue = 0; + + if (valueNode != null) + { + if (EvaluteAtom(valueNode) is int rawValue) + { + fillValue = rawValue; + } + else + { + // EvaluateAtom already printed an error message + return null; + } + } + + var data = new byte[amount]; + + for (int i = 0; i < amount; ++i) + { + data[i] = (byte)fillValue; + } + + var node = new DataNode(CurrentOffset, data); + + CheckDataWrite(amount); + CurrentOffset += amount; + + return node; + } + else + { + Error($"Invalid FILL amount (got {amount})."); + return null; + } + } + else + { + // EvaluateAtom already printed an error message + return null; + } + } + + private ILineNode? HandleSymbolAssignment(string name, IAtomNode atom, ImmutableStack scopes) + { + if (atom.TryEvaluate(_ => { }, EvaluationPhase.Early) is int value) + { + TryDefineSymbol(scopes, name, value); + } + else + { + TryDefineSymbol(scopes, name, atom); + } + + return null; + } + + private ILineNode? HandleLabel(string name, ImmutableStack scopes) + { + TryDefineSymbol(scopes, name, ConvertToAddress(CurrentOffset)); + return null; + } + + public bool IsValidLabelName(string name) + { + // TODO: this could be where checks for CURRENTOFFSET, __LINE__ and __FILE__ are? + return true; // !IsReservedName(name); + } + + private void TryDefineSymbol(ImmutableStack scopes, string name, int value) + { + if (scopes.Head.HasLocalSymbol(name)) + { + Warning($"Symbol already in scope, ignoring: {name}"); + } + else if (!IsValidLabelName(name)) + { + // NOTE: IsValidLabelName returns true always. This is dead code + Error($"Invalid symbol name {name}."); + } + else + { + scopes.Head.AddSymbol(name, value); + } + } + + private void TryDefineSymbol(ImmutableStack scopes, string name, IAtomNode expression) + { + if (scopes.Head.HasLocalSymbol(name)) + { + Warning($"Symbol already in scope, ignoring: {name}"); + } + else if (!IsValidLabelName(name)) + { + // NOTE: IsValidLabelName returns true always. This is dead code + Error($"Invalid symbol name {name}."); + } + else + { + scopes.Head.AddSymbol(name, expression); + } + } + // Return value: Location where protection occurred. Nothing if location was not protected. private Location? IsProtected(int offset, int length) { - foreach (Tuple protectedRegion in protectedRegions) + int address = ConvertToAddress(offset); + + foreach ((int protectedAddress, int protectedLength, Location location) in protectedRegions) { - //They intersect if the last offset in the given region is after the start of this one - //and the first offset in the given region is before the last of this one - if (offset + length > protectedRegion.Item1 && offset < protectedRegion.Item1 + protectedRegion.Item2) + /* They intersect if the last offset in the given region is after the start of this one + * and the first offset in the given region is before the last of this one. */ + + if (address + length > protectedAddress && address < protectedAddress + protectedLength) { - return protectedRegion.Item3; + return location; } } diff --git a/ColorzCore/Parser/Pool.cs b/ColorzCore/Parser/Pool.cs index cc9eb41..0e7d95e 100644 --- a/ColorzCore/Parser/Pool.cs +++ b/ColorzCore/Parser/Pool.cs @@ -34,7 +34,7 @@ public Pool() public string MakePoolLabelName() { // The presence of $ in the label name guarantees that it can't be a user label - return string.Format("{0}{1}", pooledLabelPrefix, poolLabelCounter++); + return $"{pooledLabelPrefix}{poolLabelCounter++}"; } } } diff --git a/ColorzCore/Preprocessor/DirectiveHandler.cs b/ColorzCore/Preprocessor/DirectiveHandler.cs index 6451dd5..83473e6 100644 --- a/ColorzCore/Preprocessor/DirectiveHandler.cs +++ b/ColorzCore/Preprocessor/DirectiveHandler.cs @@ -7,18 +7,19 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Text; -using System.Threading.Tasks; namespace ColorzCore.Preprocessor { public class DirectiveHandler { - private Dictionary directives; + public Dictionary Directives { get; } public DirectiveHandler(IncludeFileSearcher includeSearcher, IncludeFileSearcher toolSearcher) { - directives = new Dictionary + // TODO: move out from this directives that need external context + // (already done for pool, but could be done for includes as well) + + Directives = new Dictionary { { "include", new IncludeDirective { FileSearcher = includeSearcher } }, { "incbin", new IncludeBinaryDirective { FileSearcher = includeSearcher } }, @@ -31,7 +32,6 @@ public DirectiveHandler(IncludeFileSearcher includeSearcher, IncludeFileSearcher { "else", new ElseDirective() }, { "endif", new EndIfDirective() }, { "define", new DefineDirective() }, - { "pool", new PoolDirective() }, { "undef", new UndefineDirective() }, }; } @@ -40,7 +40,7 @@ public DirectiveHandler(IncludeFileSearcher includeSearcher, IncludeFileSearcher { string directiveName = directive.Content.Substring(1); - if (directives.TryGetValue(directiveName, out IDirective? toExec)) + if (Directives.TryGetValue(directiveName, out IDirective? toExec)) { if (!toExec.RequireInclusion || p.IsIncluding) { diff --git a/ColorzCore/Preprocessor/Directives/PoolDirective.cs b/ColorzCore/Preprocessor/Directives/PoolDirective.cs index 68b271c..745d35e 100644 --- a/ColorzCore/Preprocessor/Directives/PoolDirective.cs +++ b/ColorzCore/Preprocessor/Directives/PoolDirective.cs @@ -13,6 +13,13 @@ class PoolDirective : SimpleDirective public override int? MaxParams => 0; public override bool RequireInclusion => true; + private readonly Pool pool; + + public PoolDirective(Pool pool) + { + this.pool = pool; + } + public override ILineNode? Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) { BlockNode result = new BlockNode(); @@ -20,9 +27,9 @@ class PoolDirective : SimpleDirective // Iterating indices (and not values via foreach) // to avoid crashes occuring with AddToPool within AddToPool - for (int i = 0; i < p.Pool.Lines.Count; ++i) + for (int i = 0; i < pool.Lines.Count; ++i) { - Pool.PooledLine line = p.Pool.Lines[i]; + Pool.PooledLine line = pool.Lines[i]; MergeableGenerator tempGenerator = new MergeableGenerator(line.Tokens); tempGenerator.MoveNext(); @@ -34,7 +41,7 @@ class PoolDirective : SimpleDirective } } - p.Pool.Lines.Clear(); + pool.Lines.Clear(); return result; } diff --git a/ColorzCore/Preprocessor/MacroCollection.cs b/ColorzCore/Preprocessor/MacroCollection.cs index 75c0f13..0bafbd3 100644 --- a/ColorzCore/Preprocessor/MacroCollection.cs +++ b/ColorzCore/Preprocessor/MacroCollection.cs @@ -24,7 +24,6 @@ public MacroCollection(EAParser parent) { "IsDefined", new IsDefined(parent) }, { "IsSymbolDefined", new IsSymbolDefined() }, { "IsLabelDefined", new IsSymbolDefined() }, // alias - { "AddToPool", new AddToPool(parent) }, }; } diff --git a/ColorzCore/Preprocessor/Macros/AddToPool.cs b/ColorzCore/Preprocessor/Macros/AddToPool.cs index 97285c2..1bb4248 100644 --- a/ColorzCore/Preprocessor/Macros/AddToPool.cs +++ b/ColorzCore/Preprocessor/Macros/AddToPool.cs @@ -14,18 +14,18 @@ class AddToPool : BuiltInMacro * AddToPool(tokens..., alignment): adds token to pool and make sure pooled tokens are aligned given alignment */ - public EAParser ParentParser { get; private set; } + public Pool Pool { get; } - public AddToPool(EAParser parent) + public AddToPool(Pool pool) { - ParentParser = parent; + Pool = pool; } public override IEnumerable ApplyMacro(Token head, IList> parameters, ImmutableStack scopes) { List line = new List(6 + parameters[0].Count); - string labelName = ParentParser.Pool.MakePoolLabelName(); + string labelName = Pool.MakePoolLabelName(); if (parameters.Count == 2) { @@ -45,7 +45,7 @@ public override IEnumerable ApplyMacro(Token head, IList> pa line.AddRange(parameters[0]); line.Add(new Token(TokenType.NEWLINE, head.Location, "\n")); - ParentParser.Pool.Lines.Add(new Pool.PooledLine(scopes, line)); + Pool.Lines.Add(new Pool.PooledLine(scopes, line)); yield return new Token(TokenType.IDENTIFIER, head.Location, labelName); } diff --git a/ColorzCore/Program.cs b/ColorzCore/Program.cs index 25b3622..ca1f7c9 100644 --- a/ColorzCore/Program.cs +++ b/ColorzCore/Program.cs @@ -392,7 +392,7 @@ static int Main(string[] args) //FirstPass(Tokenizer.Tokenize(inputStream)); - Log log = new Log + Logger log = new Logger { Output = errorStream, WarningsAreErrors = EAOptions.WarningsAreErrors, @@ -401,10 +401,10 @@ static int Main(string[] args) }; if (EAOptions.QuietWarnings) - log.IgnoredKinds.Add(Log.MessageKind.WARNING); + log.IgnoredKinds.Add(Logger.MessageKind.WARNING); if (EAOptions.QuietMessages) - log.IgnoredKinds.Add(Log.MessageKind.MESSAGE); + log.IgnoredKinds.Add(Logger.MessageKind.MESSAGE); EAInterpreter myInterpreter = new EAInterpreter(output, game, rawsFolder, rawsExtension, inStream, inFileName, log); @@ -418,7 +418,7 @@ static int Main(string[] args) if (!(success = myInterpreter.WriteNocashSymbols(symOut))) { - log.Message(Log.MessageKind.ERROR, "Error trying to write no$gba symbol file."); + log.Message(Logger.MessageKind.ERROR, "Error trying to write no$gba symbol file."); } } From 2efee5bbd21535085bc0c1b0be071523b531b668 Mon Sep 17 00:00:00 2001 From: Nat_776 Date: Mon, 29 Apr 2024 22:09:41 +0200 Subject: [PATCH 42/59] decouple parse from interpretation --- ColorzCore/EAInterpreter.cs | 14 +- ColorzCore/IO/ASM.cs | 6 +- ColorzCore/IO/LoggerExtensions.cs | 36 ++ ColorzCore/Parser/AST/RawNode.cs | 8 +- ColorzCore/Parser/AST/StatementNode.cs | 2 +- ColorzCore/Parser/AtomParser.cs | 19 +- ColorzCore/Parser/EAParseConsumer.cs | 448 ++++++++++++++ ColorzCore/Parser/EAParser.cs | 556 ++---------------- ColorzCore/Preprocessor/DirectiveHandler.cs | 2 +- .../Directives/DefineDirective.cs | 11 +- .../Preprocessor/Directives/ElseDirective.cs | 3 +- .../Preprocessor/Directives/EndIfDirective.cs | 3 +- .../Directives/IfDefinedDirective.cs | 3 +- .../Preprocessor/Directives/IfDirective.cs | 5 +- .../Directives/IncludeBinaryDirective.cs | 8 +- .../Directives/IncludeDirective.cs | 6 +- .../Directives/IncludeExternalDirective.cs | 8 +- .../Directives/IncludeToolEventDirective.cs | 6 +- .../Directives/SimpleDirective.cs | 3 +- .../Directives/UndefineDirective.cs | 3 +- ColorzCore/Preprocessor/Macros/ErrorMacro.cs | 3 +- ColorzCore/Preprocessor/Macros/ReadDataAt.cs | 10 +- ColorzCore/Raws/AtomicParam.cs | 2 +- 23 files changed, 607 insertions(+), 558 deletions(-) create mode 100644 ColorzCore/IO/LoggerExtensions.cs create mode 100644 ColorzCore/Parser/EAParseConsumer.cs diff --git a/ColorzCore/EAInterpreter.cs b/ColorzCore/EAInterpreter.cs index 70a6365..59fe601 100644 --- a/ColorzCore/EAInterpreter.cs +++ b/ColorzCore/EAInterpreter.cs @@ -19,6 +19,7 @@ class EAInterpreter { private Dictionary> allRaws; private EAParser myParser; + private EAParseConsumer myParseConsumer; private string game, iFile; private Stream sin; private Logger log; @@ -64,7 +65,8 @@ public EAInterpreter(IOutput output, string game, string? rawsFolder, string raw foreach (string path in EAOptions.ToolsPaths) includeSearcher.IncludeDirectories.Add(path); - myParser = new EAParser(allRaws, log, new DirectiveHandler(includeSearcher, toolSearcher)); + myParseConsumer = new EAParseConsumer(log); + myParser = new EAParser(log, allRaws, new DirectiveHandler(includeSearcher, toolSearcher), myParseConsumer); myParser.Definitions[$"_{game}_"] = new Definition(); myParser.Definitions["__COLORZ_CORE__"] = new Definition(); @@ -126,7 +128,7 @@ public bool Interpret() } catch (MacroInvocationNode.MacroException e) { - myParser.Error(e.CausedError.MyLocation, "Unexpanded macro."); + log.Error(e.CausedError.MyLocation, "Unexpanded macro."); } } @@ -135,11 +137,11 @@ public bool Interpret() if (e is IdentifierNode.UndefinedIdentifierException uie && uie.CausedError.Content.StartsWith(Pool.pooledLabelPrefix, StringComparison.Ordinal)) { - myParser.Error(location, "Unpooled data (forgot #pool?)"); + log.Error(location, "Unpooled data (forgot #pool?)"); } else { - myParser.Error(location, e.Message); + log.Error(location, e.Message); } } @@ -173,9 +175,9 @@ public bool Interpret() public bool WriteNocashSymbols(TextWriter output) { - foreach (var label in myParser.GlobalScope.Head.LocalSymbols()) + foreach (var label in myParseConsumer.GlobalScope.Head.LocalSymbols()) { - output.WriteLine("{0:X8} {1}", EAParser.ConvertToAddress(label.Value), label.Key); + output.WriteLine("{0:X8} {1}", EAParseConsumer.ConvertToAddress(label.Value), label.Key); } return true; diff --git a/ColorzCore/IO/ASM.cs b/ColorzCore/IO/ASM.cs index 14c02f8..a0c3df0 100644 --- a/ColorzCore/IO/ASM.cs +++ b/ColorzCore/IO/ASM.cs @@ -20,7 +20,7 @@ public ASM(StreamWriter asmStream, StreamWriter ldsStream) private void WriteToASM(int position, byte[] data) { - string sectionName = $".ea_{EAParser.ConvertToAddress(position):x}"; + string sectionName = $".ea_{EAParseConsumer.ConvertToAddress(position):x}"; asmStream.WriteLine($".section {sectionName},\"ax\",%progbits"); asmStream.WriteLine($".global {sectionName}"); asmStream.WriteLine($"{sectionName}:"); @@ -32,8 +32,8 @@ private void WriteToASM(int position, byte[] data) private void WriteToLDS(int position) { - string sectionName = $".ea_{EAParser.ConvertToAddress(position):x}"; - ldsStream.WriteLine($". = 0x{EAParser.ConvertToAddress(position):x};"); + string sectionName = $".ea_{EAParseConsumer.ConvertToAddress(position):x}"; + ldsStream.WriteLine($". = 0x{EAParseConsumer.ConvertToAddress(position):x};"); ldsStream.WriteLine($"{sectionName} : {{*.o({sectionName})}}"); } diff --git a/ColorzCore/IO/LoggerExtensions.cs b/ColorzCore/IO/LoggerExtensions.cs new file mode 100644 index 0000000..8ec85c4 --- /dev/null +++ b/ColorzCore/IO/LoggerExtensions.cs @@ -0,0 +1,36 @@ +using System; +using System.Collections.Generic; +using System.IO; +using ColorzCore.DataTypes; + +namespace ColorzCore.IO +{ + public static class LoggerExtensions + { + // shorthand helpers + + public static void Message(this Logger self, Location? location, string message) => self.MessageTrace(Logger.MessageKind.MESSAGE, location, message); + public static void Warning(this Logger self, Location? location, string message) => self.MessageTrace(Logger.MessageKind.WARNING, location, message); + public static void Error(this Logger self, Location? location, string message) => self.MessageTrace(Logger.MessageKind.ERROR, location, message); + + private static void MessageTrace(this Logger self, Logger.MessageKind kind, Location? location, string message) + { + if (location is Location myLocation && myLocation.macroLocation != null) + { + MacroLocation macroLocation = myLocation.macroLocation; + self.MessageTrace(kind, macroLocation.Location, message); + self.Message(Logger.MessageKind.NOTE, location, $"From inside of macro `{macroLocation.MacroName}`."); + } + else + { + string[] messages = message.Split('\n'); + self.Message(kind, location, messages[0]); + + for (int i = 1; i < messages.Length; i++) + { + self.Message(Logger.MessageKind.CONTINUE, messages[i]); + } + } + } + } +} \ No newline at end of file diff --git a/ColorzCore/Parser/AST/RawNode.cs b/ColorzCore/Parser/AST/RawNode.cs index d402abf..c22400f 100644 --- a/ColorzCore/Parser/AST/RawNode.cs +++ b/ColorzCore/Parser/AST/RawNode.cs @@ -1,20 +1,22 @@ -using ColorzCore.IO; +using ColorzCore.DataTypes; +using ColorzCore.IO; using ColorzCore.Lexer; using ColorzCore.Raws; using System; using System.Collections.Generic; using System.Linq; using System.Text; -using System.Threading.Tasks; namespace ColorzCore.Parser.AST { - class RawNode : StatementNode + public class RawNode : StatementNode { public Raw Raw { get; } private Token myToken; private int Offset { get; } + public Location Location => myToken.Location; + public RawNode(Raw raw, Token t, int offset, IList paramList) : base(paramList) { myToken = t; diff --git a/ColorzCore/Parser/AST/StatementNode.cs b/ColorzCore/Parser/AST/StatementNode.cs index ee7af84..f2c0f21 100644 --- a/ColorzCore/Parser/AST/StatementNode.cs +++ b/ColorzCore/Parser/AST/StatementNode.cs @@ -10,7 +10,7 @@ namespace ColorzCore.Parser.AST { - abstract class StatementNode : ILineNode + public abstract class StatementNode : ILineNode { public IList Parameters { get; } diff --git a/ColorzCore/Parser/AtomParser.cs b/ColorzCore/Parser/AtomParser.cs index ad6d72d..2cde2dd 100644 --- a/ColorzCore/Parser/AtomParser.cs +++ b/ColorzCore/Parser/AtomParser.cs @@ -1,5 +1,6 @@ using System.Collections.Generic; using ColorzCore.DataTypes; +using ColorzCore.IO; using ColorzCore.Lexer; using ColorzCore.Parser.AST; @@ -98,12 +99,12 @@ public static class AtomParser IAtomNode? interior = self.ParseAtom(tokens, scopes); if (tokens.Current.Type != TokenType.CLOSE_PAREN) { - self.Error(tokens.Current.Location, "Unmatched open parenthesis (currently at " + tokens.Current.Type + ")."); + self.Logger.Error(tokens.Current.Location, "Unmatched open parenthesis (currently at " + tokens.Current.Type + ")."); return null; } else if (interior == null) { - self.Error(lookAhead.Location, "Expected expression inside paretheses. "); + self.Logger.Error(lookAhead.Location, "Expected expression inside paretheses. "); return null; } else @@ -122,14 +123,14 @@ public static class AtomParser IAtomNode? interior = self.ParseAtom(tokens, scopes); if (interior == null) { - self.Error(lookAhead.Location, "Expected expression after unary operator."); + self.Logger.Error(lookAhead.Location, "Expected expression after unary operator."); return null; } grammarSymbols.Push(new Left(new UnaryOperatorNode(lookAhead, interior))); break; } case TokenType.COMMA: - self.Error(lookAhead.Location, "Unexpected comma (perhaps unrecognized macro invocation?)."); + self.Logger.Error(lookAhead.Location, "Unexpected comma (perhaps unrecognized macro invocation?)."); self.IgnoreRestOfStatement(tokens); return null; case TokenType.MUL_OP: @@ -152,7 +153,7 @@ public static class AtomParser case TokenType.COMPARE_GT: case TokenType.UNDEFINED_COALESCE_OP: default: - self.Error(lookAhead.Location, $"Expected identifier or literal, got {lookAhead.Type}: {lookAhead.Content}."); + self.Logger.Error(lookAhead.Location, $"Expected identifier or literal, got {lookAhead.Type}: {lookAhead.Content}."); self.IgnoreRestOfStatement(tokens); return null; } @@ -170,7 +171,7 @@ public static class AtomParser grammarSymbols.Push(new Left(lookAhead.Content.ToUpperInvariant() switch { - "CURRENTOFFSET" => new NumberNode(lookAhead, self.CurrentOffset), + "CURRENTOFFSET" => new NumberNode(lookAhead, self.ParseConsumer.CurrentOffset), "__LINE__" => new NumberNode(lookAhead, lookAhead.GetSourceLocation().line), _ => new IdentifierNode(lookAhead, scopes), })); @@ -184,7 +185,7 @@ public static class AtomParser grammarSymbols.Push(new Left(new NumberNode(lookAhead))); break; case TokenType.ERROR: - self.Error(lookAhead.Location, $"Unexpected token: {lookAhead.Content}"); + self.Logger.Error(lookAhead.Location, $"Unexpected token: {lookAhead.Content}"); tokens.MoveNext(); return null; default: @@ -201,7 +202,7 @@ public static class AtomParser } if (grammarSymbols.Peek().IsRight) { - self.Error(grammarSymbols.Peek().GetRight.Location, $"Unexpected token: {grammarSymbols.Peek().GetRight.Type}"); + self.Logger.Error(grammarSymbols.Peek().GetRight.Location, $"Unexpected token: {grammarSymbols.Peek().GetRight.Type}"); } return grammarSymbols.Peek().GetLeft; } @@ -249,7 +250,7 @@ private static void Reduce(this EAParser self, Stack> g message += "\nConsider guarding your expressions using parenthesis."; - self.Warning(operatorNode.MyLocation, message); + self.Logger.Warning(operatorNode.MyLocation, message); } } diff --git a/ColorzCore/Parser/EAParseConsumer.cs b/ColorzCore/Parser/EAParseConsumer.cs new file mode 100644 index 0000000..f59f5a6 --- /dev/null +++ b/ColorzCore/Parser/EAParseConsumer.cs @@ -0,0 +1,448 @@ +using System.Collections.Generic; +using ColorzCore.DataTypes; +using ColorzCore.IO; +using ColorzCore.Lexer; +using ColorzCore.Parser.AST; + +namespace ColorzCore.Parser +{ + public class EAParseConsumer + { + public int CurrentOffset => currentOffset; + + public ImmutableStack GlobalScope { get; } + + private readonly Stack<(int, bool)> pastOffsets; // currentOffset, offsetInitialized + private readonly IList<(int, int, Location)> protectedRegions; + + private bool diagnosedOverflow; // true until first overflow diagnostic. + private bool offsetInitialized; // false until first ORG, used for diagnostics + private int currentOffset; + + Logger Logger { get; } + + public EAParseConsumer(Logger logger) + { + GlobalScope = new ImmutableStack(new BaseClosure(), ImmutableStack.Nil); + pastOffsets = new Stack<(int, bool)>(); + protectedRegions = new List<(int, int, Location)>(); + currentOffset = 0; + diagnosedOverflow = true; + offsetInitialized = false; + Logger = logger; + } + + // TODO: these next two functions should probably be moved into their own module + + public static int ConvertToAddress(int value) + { + /* + NOTE: Offset 0 is always converted to a null address + If one wants to instead refer to ROM offset 0 they would want to use the address directly instead. + If ROM offset 0 is already address 0 then this is a moot point. + */ + + if (value > 0 && value < EAOptions.MaximumBinarySize) + { + value += EAOptions.BaseAddress; + } + + return value; + } + + public static int ConvertToOffset(int value) + { + if (value >= EAOptions.BaseAddress && value <= EAOptions.BaseAddress + EAOptions.MaximumBinarySize) + { + value -= EAOptions.BaseAddress; + } + + return value; + } + + // Helper method for statement handlers + private int? EvaluteAtom(IAtomNode node) + { + return node.TryEvaluate(e => Logger.Error(node.MyLocation, e.Message), EvaluationPhase.Immediate); + } + + public ILineNode? HandleRawStatement(RawNode node) + { + if ((CurrentOffset % node.Raw.Alignment) != 0) + { + Logger.Error(node.Location, + $"Bad alignment for raw {node.Raw.Name}: offset ({CurrentOffset:X8}) needs to be {node.Raw.Alignment}-aligned."); + + return null; + } + else + { + // TODO: more efficient spacewise to just have contiguous writing and not an offset with every line? + CheckWriteBytes(node.Location, node.Size); + return node; + } + } + + public ILineNode? HandleOrgStatement(Location location, IAtomNode offsetNode) + { + if (EvaluteAtom(offsetNode) is int offset) + { + offset = ConvertToOffset(offset); + + if (!TrySetCurrentOffset(offset)) + { + Logger.Error(location, $"Invalid offset: 0x{offset:X}"); + } + else + { + diagnosedOverflow = false; + offsetInitialized = true; + } + } + else + { + // EvaluateAtom already printed an error message + } + + return null; + } + + public ILineNode? HandlePushStatement(Location _) + { + pastOffsets.Push((CurrentOffset, offsetInitialized)); + return null; + } + + public ILineNode? HandlePopStatement(Location location) + { + if (pastOffsets.Count == 0) + { + Logger.Error(location, "POP without matching PUSH."); + } + else + { + (currentOffset, offsetInitialized) = pastOffsets.Pop(); + } + + return null; + } + + public ILineNode? HandleAssertStatement(Location location, IAtomNode node) + { + // helper for distinguishing boolean expressions and other expressions + // TODO: move elsewhere perhaps + static bool IsBooleanResultHelper(IAtomNode node) + { + return node switch + { + UnaryOperatorNode uon => uon.OperatorToken.Type switch + { + TokenType.LOGNOT_OP => true, + _ => false, + }, + + OperatorNode on => on.OperatorToken.Type switch + { + TokenType.LOGAND_OP => true, + TokenType.LOGOR_OP => true, + TokenType.COMPARE_EQ => true, + TokenType.COMPARE_NE => true, + TokenType.COMPARE_GT => true, + TokenType.COMPARE_GE => true, + TokenType.COMPARE_LE => true, + TokenType.COMPARE_LT => true, + _ => false, + }, + + _ => false, + }; + } + + bool isBoolean = IsBooleanResultHelper(node); + + if (EvaluteAtom(node) is int result) + { + if (isBoolean && result == 0) + { + Logger.Error(location, "Assertion failed"); + } + else if (!isBoolean && result < 0) + { + Logger.Error(location, $"Assertion failed with value {result}."); + } + } + else + { + Logger.Error(node.MyLocation, "Failed to evaluate ASSERT expression."); + } + + return null; + } + + public ILineNode? HandleProtectStatement(Location location, IAtomNode beginAtom, IAtomNode? endAtom) + { + if (EvaluteAtom(beginAtom) is int beginValue) + { + beginValue = ConvertToAddress(beginValue); + + int length = 4; + + if (endAtom != null) + { + if (EvaluteAtom(endAtom) is int endValue) + { + endValue = ConvertToAddress(endValue); + + length = endValue - beginValue; + + switch (length) + { + case < 0: + Logger.Error(location, $"Invalid PROTECT region: end address ({endValue:X8}) is before start address ({beginValue:X8})."); + return null; + + case 0: + // NOTE: does this need to be an error? + Logger.Error(location, $"Empty PROTECT region: end address is equal to start address ({beginValue:X8})."); + return null; + } + } + else + { + // EvaluateAtom already printed an error message + return null; + } + } + + protectedRegions.Add((beginValue, length, location)); + + return null; + } + else + { + // EvaluateAtom already printed an error message + return null; + } + } + + public ILineNode? HandleAlignStatement(Location location, IAtomNode alignNode, IAtomNode? offsetNode) + { + if (EvaluteAtom(alignNode) is int alignValue) + { + if (alignValue > 0) + { + int alignOffset = 0; + + if (offsetNode != null) + { + if (EvaluteAtom(offsetNode) is int rawOffset) + { + if (rawOffset >= 0) + { + alignOffset = ConvertToOffset(rawOffset) % alignValue; + } + else + { + Logger.Error(location, $"ALIGN offset cannot be negative (got {rawOffset})"); + return null; + } + } + else + { + // EvaluateAtom already printed an error message + return null; + } + } + + if (CurrentOffset % alignValue != alignOffset) + { + int skip = alignValue - (CurrentOffset + alignValue - alignOffset) % alignValue; + + if (!TrySetCurrentOffset(CurrentOffset + skip) && !diagnosedOverflow) + { + Logger.Error(location, $"Trying to jump past end of binary (CURRENTOFFSET would surpass {EAOptions.MaximumBinarySize:X})"); + diagnosedOverflow = true; + } + } + + return null; + } + else + { + Logger.Error(location, $"Invalid ALIGN value (got {alignValue})."); + return null; + } + } + else + { + // EvaluateAtom already printed an error message + return null; + } + } + + public ILineNode? HandleFillStatement(Location location, IAtomNode amountNode, IAtomNode? valueNode) + { + if (EvaluteAtom(amountNode) is int amount) + { + if (amount > 0) + { + int fillValue = 0; + + if (valueNode != null) + { + if (EvaluteAtom(valueNode) is int rawValue) + { + fillValue = rawValue; + } + else + { + // EvaluateAtom already printed an error message + return null; + } + } + + var data = new byte[amount]; + + for (int i = 0; i < amount; ++i) + { + data[i] = (byte)fillValue; + } + + var node = new DataNode(CurrentOffset, data); + + CheckWriteBytes(location, amount); + return node; + } + else + { + Logger.Error(location, $"Invalid FILL amount (got {amount})."); + return null; + } + } + else + { + // EvaluateAtom already printed an error message + return null; + } + } + + public ILineNode? HandleSymbolAssignment(Location location, string name, IAtomNode atom, ImmutableStack scopes) + { + if (atom.TryEvaluate(_ => { }, EvaluationPhase.Early) is int value) + { + TryDefineSymbol(location, scopes, name, value); + } + else + { + TryDefineSymbol(location, scopes, name, atom); + } + + return null; + } + + public ILineNode? HandleLabel(Location location, string name, ImmutableStack scopes) + { + TryDefineSymbol(location, scopes, name, ConvertToAddress(CurrentOffset)); + return null; + } + + public ILineNode? HandlePreprocessorLineNode(Location location, ILineNode node) + { + CheckWriteBytes(location, node.Size); + return node; + } + + public bool IsValidLabelName(string name) + { + // TODO: this could be where checks for CURRENTOFFSET, __LINE__ and __FILE__ are? + return true; // !IsReservedName(name); + } + + public void TryDefineSymbol(Location location, ImmutableStack scopes, string name, int value) + { + if (scopes.Head.HasLocalSymbol(name)) + { + Logger.Warning(location, $"Symbol already in scope, ignoring: {name}"); + } + else if (!IsValidLabelName(name)) + { + // NOTE: IsValidLabelName returns true always. This is dead code + Logger.Error(location, $"Invalid symbol name {name}."); + } + else + { + scopes.Head.AddSymbol(name, value); + } + } + + public void TryDefineSymbol(Location location, ImmutableStack scopes, string name, IAtomNode expression) + { + if (scopes.Head.HasLocalSymbol(name)) + { + Logger.Warning(location, $"Symbol already in scope, ignoring: {name}"); + } + else if (!IsValidLabelName(name)) + { + // NOTE: IsValidLabelName returns true always. This is dead code + Logger.Error(location, $"Invalid symbol name {name}."); + } + else + { + scopes.Head.AddSymbol(name, expression); + } + } + + // Return value: Location where protection occurred. Nothing if location was not protected. + public Location? IsProtected(int offset, int length) + { + int address = ConvertToAddress(offset); + + foreach ((int protectedAddress, int protectedLength, Location location) in protectedRegions) + { + /* They intersect if the last offset in the given region is after the start of this one + * and the first offset in the given region is before the last of this one. */ + + if (address + length > protectedAddress && address < protectedAddress + protectedLength) + { + return location; + } + } + + return null; + } + + public void CheckWriteBytes(Location location, int length) + { + if (!offsetInitialized && EAOptions.IsWarningEnabled(EAOptions.Warnings.UninitializedOffset)) + { + Logger.Warning(location, "Writing before initializing offset. You may be breaking the ROM! (use `ORG offset` to set write offset)."); + } + + if (IsProtected(CurrentOffset, length) is Location prot) + { + Logger.Error(location, $"Trying to write data to area protected by {prot}"); + } + + if (!TrySetCurrentOffset(CurrentOffset + length) && !diagnosedOverflow) + { + Logger.Error(location, $"Trying to write past end of binary (CURRENTOFFSET would surpass {EAOptions.MaximumBinarySize:X})"); + diagnosedOverflow = true; + } + } + + private bool TrySetCurrentOffset(int value) + { + if (value < 0 || value > EAOptions.MaximumBinarySize) + { + return false; + } + else + { + currentOffset = value; + diagnosedOverflow = false; + offsetInitialized = true; + return true; + } + } + } +} diff --git a/ColorzCore/Parser/EAParser.cs b/ColorzCore/Parser/EAParser.cs index 59657a6..d653a3c 100644 --- a/ColorzCore/Parser/EAParser.cs +++ b/ColorzCore/Parser/EAParser.cs @@ -27,7 +27,7 @@ public class EAParser public ImmutableStack Inclusion { get; set; } - public Logger log; + public Logger Logger { get; } public bool IsIncluding { @@ -46,20 +46,17 @@ public bool IsIncluding private Token? head; // TODO: Make this make sense - public EAParser(Dictionary> raws, Logger log, DirectiveHandler directiveHandler) + public EAParseConsumer ParseConsumer { get; } + + public EAParser(Logger log, Dictionary> raws, DirectiveHandler directiveHandler, EAParseConsumer parseConsumer) { - GlobalScope = new ImmutableStack(new BaseClosure(), ImmutableStack.Nil); - pastOffsets = new Stack<(int, bool)>(); - protectedRegions = new List<(int, int, Location)>(); - this.log = log; + Logger = log; Raws = raws; - CurrentOffset = 0; - validOffset = true; - offsetInitialized = false; Macros = new MacroCollection(this); Definitions = new Dictionary(); Inclusion = ImmutableStack.Nil; DirectiveHandler = directiveHandler; + ParseConsumer = parseConsumer; } public bool IsReservedName(string name) @@ -90,7 +87,7 @@ public IList ParseAll(IEnumerable tokenStream) { if (tokens.Current.Type != TokenType.NEWLINE || tokens.MoveNext()) { - ILineNode? retVal = ParseLine(tokens, GlobalScope); + ILineNode? retVal = ParseLine(tokens, ParseConsumer.GlobalScope); retVal.IfJust(n => myLines.Add(n)); } } @@ -121,7 +118,7 @@ private BlockNode ParseBlock(MergeableGenerator tokens, ImmutableStack tokens, ImmutableStack tokens, ImmutableStack tokens, ImmutableStack tokens, ImmutableStack tokens, ImmutableStack parameters, ImmutableStack _) { - return ParseStatementOneParam("ORG", parameters, HandleOrgStatement); + return ParseStatementOneParam("ORG", parameters, ParseConsumer.HandleOrgStatement); } private ILineNode? ParsePushStatement(IList parameters, ImmutableStack _) { if (parameters.Count == 0) { - return HandlePushStatement(); + return ParseConsumer.HandlePushStatement(head!.Location); } else { @@ -260,7 +257,7 @@ private BlockNode ParseBlock(MergeableGenerator tokens, ImmutableStack tokens, ImmutableStack parameters, ImmutableStack _) { - return ParseStatementOneParam("ASSERT", parameters, HandleAssertStatement); + return ParseStatementOneParam("ASSERT", parameters, ParseConsumer.HandleAssertStatement); } // Helper method for printing errors private ILineNode? StatementExpectsAtom(string statementName, IParamNode param) { - Error(param.MyLocation, + Logger.Error(param.MyLocation, $"{statementName} expects an Atom (got {DiagnosticsHelpers.PrettyParamType(param.Type)})."); return null; @@ -287,18 +284,18 @@ private BlockNode ParseBlock(MergeableGenerator tokens, ImmutableStack parameters, HandleStatementOne handler) { @@ -306,7 +303,7 @@ private BlockNode ParseBlock(MergeableGenerator tokens, ImmutableStack tokens, ImmutableStack tokens, ImmutableStack tokens, ImmutableStack parameters, ImmutableStack _) { - return ParseStatementTwoParam("PROTECT", parameters, HandleProtectStatement); + return ParseStatementTwoParam("PROTECT", parameters, ParseConsumer.HandleProtectStatement); } private ILineNode? ParseAlignStatement(IList parameters, ImmutableStack _) { - return ParseStatementTwoParam("ALIGN", parameters, HandleAlignStatement); + return ParseStatementTwoParam("ALIGN", parameters, ParseConsumer.HandleAlignStatement); } private ILineNode? ParseFillStatement(IList parameters, ImmutableStack _) { - return ParseStatementTwoParam("FILL", parameters, HandleFillStatement); + return ParseStatementTwoParam("FILL", parameters, ParseConsumer.HandleFillStatement); } private ILineNode? ParseMessageStatement(IList parameters, ImmutableStack scopes) { - Message(PrettyPrintParamsForMessage(parameters, scopes)); + Logger.Message(head!.Location, PrettyPrintParamsForMessage(parameters, scopes)); return null; } private ILineNode? ParseWarningStatement(IList parameters, ImmutableStack scopes) { - Warning(PrettyPrintParamsForMessage(parameters, scopes)); + Logger.Warning(head!.Location, PrettyPrintParamsForMessage(parameters, scopes)); return null; } private ILineNode? ParseErrorStatement(IList parameters, ImmutableStack scopes) { - Error(PrettyPrintParamsForMessage(parameters, scopes)); + Logger.Error(head!.Location, PrettyPrintParamsForMessage(parameters, scopes)); return null; } @@ -432,7 +429,7 @@ public IList> ParseMacroParamList(MergeableGenerator tokens) if (tokens.Current.Type != TokenType.CLOSE_PAREN || parenNestings != 0) { - Error(tokens.Current.Location, "Unmatched open parenthesis."); + Logger.Error(tokens.Current.Location, "Unmatched open parenthesis."); } else { @@ -452,7 +449,7 @@ private IList ParseParamList(MergeableGenerator tokens, Immut Token localHead = tokens.Current; ParseParam(tokens, scopes, expandFirstDef || !first).IfJust( n => paramList.Add(n), - () => Error(localHead.Location, "Expected parameter.")); + () => Logger.Error(localHead.Location, "Expected parameter.")); first = false; } @@ -536,7 +533,7 @@ private IList ParseList(MergeableGenerator tokens, ImmutableSt IAtomNode? res = this.ParseAtom(tokens, scopes); res.IfJust( n => atoms.Add(n), - () => Error(tokens.Current.Location, "Expected atomic value, got " + tokens.Current.Type + ".")); + () => Logger.Error(tokens.Current.Location, "Expected atomic value, got " + tokens.Current.Type + ".")); if (tokens.Current.Type == TokenType.COMMA) { tokens.MoveNext(); @@ -548,7 +545,7 @@ private IList ParseList(MergeableGenerator tokens, ImmutableSt } else { - Error(localHead.Location, "Unmatched open bracket."); + Logger.Error(localHead.Location, "Unmatched open bracket."); } return atoms; @@ -583,7 +580,7 @@ private IList ParseList(MergeableGenerator tokens, ImmutableSt default: // it is somewhat common for users to do '#define Foo 0xABCD' and then later 'Foo:' - Error($"Expansion of macro `{head.Content}` did not result in a valid statement. Did you perhaps attempt to define a label or symbol with that name?"); + Logger.Error(head.Location, $"Expansion of macro `{head.Content}` did not result in a valid statement. Did you perhaps attempt to define a label or symbol with that name?"); IgnoreRestOfLine(tokens); return null; @@ -599,13 +596,13 @@ private IList ParseList(MergeableGenerator tokens, ImmutableSt return ParsePreprocessor(tokens, scopes); case TokenType.OPEN_BRACKET: - Error("Unexpected list literal."); + Logger.Error(head.Location, "Unexpected list literal."); IgnoreRestOfLine(tokens); break; case TokenType.NUMBER: case TokenType.OPEN_PAREN: - Error("Unexpected mathematical expression."); + Logger.Error(head.Location, "Unexpected mathematical expression."); IgnoreRestOfLine(tokens); break; @@ -614,11 +611,11 @@ private IList ParseList(MergeableGenerator tokens, ImmutableSt if (string.IsNullOrEmpty(head.Content)) { - Error($"Unexpected token: {head.Type}."); + Logger.Error(head.Location, $"Unexpected token: {head.Type}."); } else { - Error($"Unexpected token: {head.Type}: {head.Content}."); + Logger.Error(head.Location, $"Unexpected token: {head.Type}: {head.Content}."); } IgnoreRestOfLine(tokens); @@ -640,7 +637,7 @@ private IList ParseList(MergeableGenerator tokens, ImmutableSt } else { - Error(null, $"Missing {Inclusion.Count} endif(s)."); + Logger.Error(null, $"Missing {Inclusion.Count} endif(s)."); return null; } } @@ -655,11 +652,10 @@ private IList ParseList(MergeableGenerator tokens, ImmutableSt if (result != null) { - CheckDataWrite(result.Size); - CurrentOffset += result.Size; + ParseConsumer.HandlePreprocessorLineNode(head.Location, result); } - return result; + return null; } /*** @@ -689,8 +685,9 @@ public bool ExpandIdentifier(MergeableGenerator tokens, ImmutableStack tokens, IEnumerable expandedList = expandedTokens.ToList(); DiagnosticsHelpers.VisitUnguardedOperators(expandedList, - token => Warning(token.Location, $"Unguarded expansion of mathematical operator. Consider adding guarding parenthesises around definition.")); + token => Logger.Warning(token.Location, $"Unguarded expansion of mathematical operator. Consider adding guarding parenthesises around definition.")); tokens.PrependEnumerator(expandedList.GetEnumerator()); } @@ -735,36 +732,6 @@ private void ApplyMacroExpansion(MergeableGenerator tokens, IEnumerable MessageTrace(Logger.MessageKind.MESSAGE, location, message); - public void Warning(Location? location, string message) => MessageTrace(Logger.MessageKind.WARNING, location, message); - public void Error(Location? location, string message) => MessageTrace(Logger.MessageKind.ERROR, location, message); - - public void Message(string message) => MessageTrace(Logger.MessageKind.MESSAGE, head?.Location, message); - public void Warning(string message) => MessageTrace(Logger.MessageKind.WARNING, head?.Location, message); - public void Error(string message) => MessageTrace(Logger.MessageKind.ERROR, head?.Location, message); - public void IgnoreRestOfStatement(MergeableGenerator tokens) { while (tokens.Current.Type != TokenType.NEWLINE && tokens.Current.Type != TokenType.SEMICOLON && tokens.MoveNext()) { } @@ -814,9 +781,9 @@ private string PrettyPrintParamsForMessage(IList parameters, Immutab private string ExpandUserFormatString(ImmutableStack scopes, Location baseLocation, string stringValue) { - string UserFormatStringError(string message, string details) + string UserFormatStringError(Location loc, string message, string details) { - Error($"An error occurred while expanding format string ({message})."); + Logger.Error(loc, $"An error occurred while expanding format string ({message})."); return $"<{message}: {details}>"; } @@ -836,10 +803,10 @@ string UserFormatStringError(string message, string details) if (node == null || tokens.Current.Type != TokenType.NEWLINE) { - return UserFormatStringError("bad expression", $"'{expr}'"); + return UserFormatStringError(itemLocation, "bad expression", $"'{expr}'"); } - if (node.TryEvaluate(e => Error(node.MyLocation, e.Message), + if (node.TryEvaluate(e => Logger.Error(node.MyLocation, e.Message), EvaluationPhase.Early) is int value) { try @@ -850,429 +817,14 @@ string UserFormatStringError(string message, string details) } catch (FormatException e) { - return UserFormatStringError("bad format specifier", $"'{format}' ({e.Message})"); + return UserFormatStringError(node.MyLocation, "bad format specifier", $"'{format}' ({e.Message})"); } } else { - return UserFormatStringError("failed to evaluate expression", $"'{expr}'"); + return UserFormatStringError(node.MyLocation, "failed to evaluate expression", $"'{expr}'"); } }); } - - /* - * ========================================= - * = NON STRICTLY PARSE RELATED START HERE = - * ========================================= - */ - - public int CurrentOffset - { - get => currentOffset; - - private set - { - if (value < 0 || value > EAOptions.MaximumBinarySize) - { - if (validOffset) //Error only the first time. - { - Error($"Invalid offset: {value:X}"); - validOffset = false; - } - } - else - { - currentOffset = value; - validOffset = true; - offsetInitialized = true; - } - } - } - - public ImmutableStack GlobalScope { get; } - - private readonly Stack<(int, bool)> pastOffsets; // currentOffset, offsetInitialized - private readonly IList<(int, int, Location)> protectedRegions; - - private bool validOffset; - private bool offsetInitialized; // false until first ORG, used to warn about writing before first org - private int currentOffset; - - // TODO: these next two functions should probably be moved into their own module - - public static int ConvertToAddress(int value) - { - /* - NOTE: Offset 0 is always converted to a null address - If one wants to instead refer to ROM offset 0 they would want to use the address directly instead. - If ROM offset 0 is already address 0 then this is a moot point. - */ - - if (value > 0 && value < EAOptions.MaximumBinarySize) - { - value += EAOptions.BaseAddress; - } - - return value; - } - - public static int ConvertToOffset(int value) - { - if (value >= EAOptions.BaseAddress && value <= EAOptions.BaseAddress + EAOptions.MaximumBinarySize) - { - value -= EAOptions.BaseAddress; - } - - return value; - } - - // Helper method for statement handlers - private int? EvaluteAtom(IAtomNode node) - { - return node.TryEvaluate(e => Error(node.MyLocation, e.Message), EvaluationPhase.Immediate); - } - - private ILineNode? HandleRawStatement(RawNode node) - { - if ((CurrentOffset % node.Raw.Alignment) != 0) - { - Error($"Bad alignment for raw {node.Raw.Name}: offseet ({CurrentOffset:X8}) needs to be {node.Raw.Alignment}-aligned."); - return null; - } - else - { - // TODO: more efficient spacewise to just have contiguous writing and not an offset with every line? - CheckDataWrite(node.Size); - CurrentOffset += node.Size; - - return node; - } - } - - private ILineNode? HandleOrgStatement(IAtomNode offsetNode) - { - if (EvaluteAtom(offsetNode) is int offset) - { - CurrentOffset = ConvertToOffset(offset); - } - else - { - // EvaluateAtom already printed an error message - } - - return null; - } - - private ILineNode? HandlePushStatement() - { - pastOffsets.Push((CurrentOffset, offsetInitialized)); - return null; - } - - private ILineNode? HandlePopStatement() - { - if (pastOffsets.Count == 0) - { - Error("POP without matching PUSH."); - } - else - { - (CurrentOffset, offsetInitialized) = pastOffsets.Pop(); - } - - return null; - } - - private ILineNode? HandleAssertStatement(IAtomNode node) - { - // helper for distinguishing boolean expressions and other expressions - // TODO: move elsewhere perhaps - static bool IsBooleanResultHelper(IAtomNode node) - { - return node switch - { - UnaryOperatorNode uon => uon.OperatorToken.Type switch - { - TokenType.LOGNOT_OP => true, - _ => false, - }, - - OperatorNode on => on.OperatorToken.Type switch - { - TokenType.LOGAND_OP => true, - TokenType.LOGOR_OP => true, - TokenType.COMPARE_EQ => true, - TokenType.COMPARE_NE => true, - TokenType.COMPARE_GT => true, - TokenType.COMPARE_GE => true, - TokenType.COMPARE_LE => true, - TokenType.COMPARE_LT => true, - _ => false, - }, - - _ => false, - }; - } - - bool isBoolean = IsBooleanResultHelper(node); - - if (EvaluteAtom(node) is int result) - { - if (isBoolean && result == 0) - { - Error(node.MyLocation, "Assertion failed"); - } - else if (!isBoolean && result < 0) - { - Error(node.MyLocation, $"Assertion failed with value {result}."); - } - } - else - { - Error("Failed to evaluate ASSERT expression."); - } - - return null; - } - - private ILineNode? HandleProtectStatement(IAtomNode beginAtom, IAtomNode? endAtom) - { - if (EvaluteAtom(beginAtom) is int beginValue) - { - beginValue = ConvertToAddress(beginValue); - - int length = 4; - - if (endAtom != null) - { - if (EvaluteAtom(endAtom) is int endValue) - { - endValue = ConvertToAddress(endValue); - - length = endValue - beginValue; - - switch (length) - { - case < 0: - Error($"Invalid PROTECT region: end address ({endValue:X8}) is before start address ({beginValue:X8})."); - return null; - - case 0: - // NOTE: does this need to be an error? - Error($"Empty PROTECT region: end address is equal to start address ({beginValue:X8})."); - return null; - } - } - else - { - // EvaluateAtom already printed an error message - return null; - } - } - - protectedRegions.Add((beginValue, length, head!.Location)); - - return null; - } - else - { - // EvaluateAtom already printed an error message - return null; - } - } - - private ILineNode? HandleAlignStatement(IAtomNode alignNode, IAtomNode? offsetNode) - { - if (EvaluteAtom(alignNode) is int alignValue) - { - if (alignValue > 0) - { - int alignOffset = 0; - - if (offsetNode != null) - { - if (EvaluteAtom(offsetNode) is int rawOffset) - { - if (rawOffset >= 0) - { - alignOffset = ConvertToOffset(rawOffset) % alignValue; - } - else - { - Error($"ALIGN offset cannot be negative (got {rawOffset})"); - return null; - } - } - else - { - // EvaluateAtom already printed an error message - return null; - } - } - - if (CurrentOffset % alignValue != alignOffset) - { - CurrentOffset += alignValue - (CurrentOffset + alignValue - alignOffset) % alignValue; - } - - return null; - } - else - { - Error($"Invalid ALIGN value (got {alignValue})."); - return null; - } - } - else - { - // EvaluateAtom already printed an error message - return null; - } - } - - private ILineNode? HandleFillStatement(IAtomNode amountNode, IAtomNode? valueNode) - { - if (EvaluteAtom(amountNode) is int amount) - { - if (amount > 0) - { - int fillValue = 0; - - if (valueNode != null) - { - if (EvaluteAtom(valueNode) is int rawValue) - { - fillValue = rawValue; - } - else - { - // EvaluateAtom already printed an error message - return null; - } - } - - var data = new byte[amount]; - - for (int i = 0; i < amount; ++i) - { - data[i] = (byte)fillValue; - } - - var node = new DataNode(CurrentOffset, data); - - CheckDataWrite(amount); - CurrentOffset += amount; - - return node; - } - else - { - Error($"Invalid FILL amount (got {amount})."); - return null; - } - } - else - { - // EvaluateAtom already printed an error message - return null; - } - } - - private ILineNode? HandleSymbolAssignment(string name, IAtomNode atom, ImmutableStack scopes) - { - if (atom.TryEvaluate(_ => { }, EvaluationPhase.Early) is int value) - { - TryDefineSymbol(scopes, name, value); - } - else - { - TryDefineSymbol(scopes, name, atom); - } - - return null; - } - - private ILineNode? HandleLabel(string name, ImmutableStack scopes) - { - TryDefineSymbol(scopes, name, ConvertToAddress(CurrentOffset)); - return null; - } - - public bool IsValidLabelName(string name) - { - // TODO: this could be where checks for CURRENTOFFSET, __LINE__ and __FILE__ are? - return true; // !IsReservedName(name); - } - - private void TryDefineSymbol(ImmutableStack scopes, string name, int value) - { - if (scopes.Head.HasLocalSymbol(name)) - { - Warning($"Symbol already in scope, ignoring: {name}"); - } - else if (!IsValidLabelName(name)) - { - // NOTE: IsValidLabelName returns true always. This is dead code - Error($"Invalid symbol name {name}."); - } - else - { - scopes.Head.AddSymbol(name, value); - } - } - - private void TryDefineSymbol(ImmutableStack scopes, string name, IAtomNode expression) - { - if (scopes.Head.HasLocalSymbol(name)) - { - Warning($"Symbol already in scope, ignoring: {name}"); - } - else if (!IsValidLabelName(name)) - { - // NOTE: IsValidLabelName returns true always. This is dead code - Error($"Invalid symbol name {name}."); - } - else - { - scopes.Head.AddSymbol(name, expression); - } - } - - // Return value: Location where protection occurred. Nothing if location was not protected. - private Location? IsProtected(int offset, int length) - { - int address = ConvertToAddress(offset); - - foreach ((int protectedAddress, int protectedLength, Location location) in protectedRegions) - { - /* They intersect if the last offset in the given region is after the start of this one - * and the first offset in the given region is before the last of this one. */ - - if (address + length > protectedAddress && address < protectedAddress + protectedLength) - { - return location; - } - } - - return null; - } - - private void CheckDataWrite(int length) - { - if (!offsetInitialized) - { - if (EAOptions.IsWarningEnabled(EAOptions.Warnings.UninitializedOffset)) - { - Warning("Writing before initializing offset. You may be breaking the ROM! (use `ORG offset` to set write offset)."); - } - - offsetInitialized = false; // only warn once - } - - if (IsProtected(CurrentOffset, length) is Location prot) - { - Error($"Trying to write data to area protected by {prot}"); - } - } } } diff --git a/ColorzCore/Preprocessor/DirectiveHandler.cs b/ColorzCore/Preprocessor/DirectiveHandler.cs index 83473e6..3a4b946 100644 --- a/ColorzCore/Preprocessor/DirectiveHandler.cs +++ b/ColorzCore/Preprocessor/DirectiveHandler.cs @@ -49,7 +49,7 @@ public DirectiveHandler(IncludeFileSearcher includeSearcher, IncludeFileSearcher } else { - p.Error(directive.Location, $"Directive not recognized: {directiveName}"); + p.Logger.Error(directive.Location, $"Directive not recognized: {directiveName}"); } return null; diff --git a/ColorzCore/Preprocessor/Directives/DefineDirective.cs b/ColorzCore/Preprocessor/Directives/DefineDirective.cs index 5de7695..21353dd 100644 --- a/ColorzCore/Preprocessor/Directives/DefineDirective.cs +++ b/ColorzCore/Preprocessor/Directives/DefineDirective.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Linq; using ColorzCore.DataTypes; +using ColorzCore.IO; using ColorzCore.Lexer; using ColorzCore.Parser; using ColorzCore.Parser.AST; @@ -31,11 +32,11 @@ class DefineDirective : IDirective break; case TokenType.NEWLINE: - p.Error(self.Location, "Invalid use of directive '#define': missing macro name."); + p.Logger.Error(self.Location, "Invalid use of directive '#define': missing macro name."); return null; default: - p.Error(self.Location, $"Invalid use of directive '#define': expected macro name, got {nextToken}"); + p.Logger.Error(self.Location, $"Invalid use of directive '#define': expected macro name, got {nextToken}"); p.IgnoreRestOfLine(tokens); return null; } @@ -62,7 +63,7 @@ private static void DefineObjectMacro(EAParser p, Token nameToken, IList if (p.Definitions.ContainsKey(name) && EAOptions.IsWarningEnabled(EAOptions.Warnings.ReDefine)) { - p.Warning(nameToken.Location, $"Redefining {name}."); + p.Logger.Warning(nameToken.Location, $"Redefining {name}."); } if (macroBody.Count == 1 && macroBody[0].Type == TokenType.IDENTIFIER && macroBody[0].Content == name) @@ -84,7 +85,7 @@ private static void DefineFunctionMacro(EAParser p, Token nameToken, IList FlattenParameters(EAParser p, IList> r { if (parameter.Count != 1 || parameter[0].Type != TokenType.IDENTIFIER) { - p.Error(parameter[0].Location, $"Macro parameters must be single identifiers (got {parameter[0].Content})."); + p.Logger.Error(parameter[0].Location, $"Macro parameters must be single identifiers (got {parameter[0].Content})."); result.Add($"${result.Count}"); } else diff --git a/ColorzCore/Preprocessor/Directives/ElseDirective.cs b/ColorzCore/Preprocessor/Directives/ElseDirective.cs index c48fc69..8948d00 100644 --- a/ColorzCore/Preprocessor/Directives/ElseDirective.cs +++ b/ColorzCore/Preprocessor/Directives/ElseDirective.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Linq; using ColorzCore.DataTypes; +using ColorzCore.IO; using ColorzCore.Lexer; using ColorzCore.Parser; using ColorzCore.Parser.AST; @@ -19,7 +20,7 @@ class ElseDirective : SimpleDirective public override ILineNode? Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) { if (p.Inclusion.IsEmpty) - p.Error(self.Location, "No matching if[n]def."); + p.Logger.Error(self.Location, "No matching conditional (if, ifdef, ifndef)."); else p.Inclusion = new ImmutableStack(!p.Inclusion.Head, p.Inclusion.Tail); return null; diff --git a/ColorzCore/Preprocessor/Directives/EndIfDirective.cs b/ColorzCore/Preprocessor/Directives/EndIfDirective.cs index 74a2371..37afef6 100644 --- a/ColorzCore/Preprocessor/Directives/EndIfDirective.cs +++ b/ColorzCore/Preprocessor/Directives/EndIfDirective.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Linq; using ColorzCore.DataTypes; +using ColorzCore.IO; using ColorzCore.Lexer; using ColorzCore.Parser; using ColorzCore.Parser.AST; @@ -19,7 +20,7 @@ class EndIfDirective : SimpleDirective public override ILineNode? Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) { if (p.Inclusion.IsEmpty) - p.Error(self.Location, "No matching if[n]def."); + p.Logger.Error(self.Location, "No matching conditional (if, ifdef, ifndef)."); else p.Inclusion = p.Inclusion.Tail; diff --git a/ColorzCore/Preprocessor/Directives/IfDefinedDirective.cs b/ColorzCore/Preprocessor/Directives/IfDefinedDirective.cs index 93e92f4..82a2143 100644 --- a/ColorzCore/Preprocessor/Directives/IfDefinedDirective.cs +++ b/ColorzCore/Preprocessor/Directives/IfDefinedDirective.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Linq; using ColorzCore.DataTypes; +using ColorzCore.IO; using ColorzCore.Lexer; using ColorzCore.Parser; using ColorzCore.Parser.AST; @@ -37,7 +38,7 @@ public IfDefinedDirective(bool invert) } else { - p.Error(parameter.MyLocation, "Definition name must be an identifier."); + p.Logger.Error(parameter.MyLocation, "Definition name must be an identifier."); } } p.Inclusion = new ImmutableStack(flag, p.Inclusion); diff --git a/ColorzCore/Preprocessor/Directives/IfDirective.cs b/ColorzCore/Preprocessor/Directives/IfDirective.cs index a582ed9..0256f1c 100644 --- a/ColorzCore/Preprocessor/Directives/IfDirective.cs +++ b/ColorzCore/Preprocessor/Directives/IfDirective.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Linq; using ColorzCore.DataTypes; +using ColorzCore.IO; using ColorzCore.Lexer; using ColorzCore.Parser; using ColorzCore.Parser.AST; @@ -24,14 +25,14 @@ class IfDirective : SimpleDirective { if (parameter is IAtomNode atomNode) { - if (atomNode.TryEvaluate(e => p.Error(self.Location, $"Error while evaluating expression: {e.Message}"), EvaluationPhase.Immediate) is int value) + if (atomNode.TryEvaluate(e => p.Logger.Error(self.Location, $"Error while evaluating expression: {e.Message}"), EvaluationPhase.Immediate) is int value) { flag = value != 0; } } else { - p.Error(self.Location, "Expected an expression."); + p.Logger.Error(self.Location, "Expected an expression."); } } diff --git a/ColorzCore/Preprocessor/Directives/IncludeBinaryDirective.cs b/ColorzCore/Preprocessor/Directives/IncludeBinaryDirective.cs index 371fd1d..e13791c 100644 --- a/ColorzCore/Preprocessor/Directives/IncludeBinaryDirective.cs +++ b/ColorzCore/Preprocessor/Directives/IncludeBinaryDirective.cs @@ -39,22 +39,22 @@ class IncludeBinaryDirective : SimpleDirective if (pathExpression != portablePathExpression) { - p.Warning(self.Location, $"Path is not portable (should be \"{portablePathExpression}\")."); + p.Logger.Warning(self.Location, $"Path is not portable (should be \"{portablePathExpression}\")."); } } try { - return new DataNode(p.CurrentOffset, File.ReadAllBytes(existantFile)); + return new DataNode(p.ParseConsumer.CurrentOffset, File.ReadAllBytes(existantFile)); } catch (Exception) { - p.Error(self.Location, "Error reading file \"" + parameters[0].ToString() + "\"."); + p.Logger.Error(self.Location, "Error reading file \"" + parameters[0].ToString() + "\"."); } } else { - p.Error(parameters[0].MyLocation, "Could not find file \"" + parameters[0].ToString() + "\"."); + p.Logger.Error(parameters[0].MyLocation, "Could not find file \"" + parameters[0].ToString() + "\"."); } return null; diff --git a/ColorzCore/Preprocessor/Directives/IncludeDirective.cs b/ColorzCore/Preprocessor/Directives/IncludeDirective.cs index 5cb8613..1fb424a 100644 --- a/ColorzCore/Preprocessor/Directives/IncludeDirective.cs +++ b/ColorzCore/Preprocessor/Directives/IncludeDirective.cs @@ -41,7 +41,7 @@ class IncludeDirective : SimpleDirective if (pathExpression != portablePathExpression) { - p.Warning(self.Location, $"Path is not portable (should be \"{portablePathExpression}\")."); + p.Logger.Warning(self.Location, $"Path is not portable (should be \"{portablePathExpression}\")."); } } @@ -52,12 +52,12 @@ class IncludeDirective : SimpleDirective } catch (Exception e) { - p.Error(self.Location, $"Error reading file \"{parameters[0].ToString()}\": {e.Message}."); + p.Logger.Error(self.Location, $"Error reading file \"{parameters[0].ToString()}\": {e.Message}."); } } else { - p.Error(parameters[0].MyLocation, $"Could not find file \"{parameters[0].ToString()}\"."); + p.Logger.Error(parameters[0].MyLocation, $"Could not find file \"{parameters[0].ToString()}\"."); } return null; diff --git a/ColorzCore/Preprocessor/Directives/IncludeExternalDirective.cs b/ColorzCore/Preprocessor/Directives/IncludeExternalDirective.cs index 4e2deee..11272f9 100644 --- a/ColorzCore/Preprocessor/Directives/IncludeExternalDirective.cs +++ b/ColorzCore/Preprocessor/Directives/IncludeExternalDirective.cs @@ -27,7 +27,7 @@ class IncludeExternalDirective : SimpleDirective if (validFile == null) { - parse.Error(parameters[0].MyLocation, "Tool " + parameters[0].ToString() + " not found."); + parse.Logger.Error(parameters[0].MyLocation, "Tool " + parameters[0].ToString() + " not found."); return null; } //TODO: abstract out all this running stuff into a method so I don't have code duplication with inctext @@ -68,16 +68,16 @@ class IncludeExternalDirective : SimpleDirective byte[] output = outputBytes.GetBuffer().Take((int)outputBytes.Length).ToArray(); if (errorStream.Length > 0) { - parse.Error(self.Location, Encoding.ASCII.GetString(errorStream.GetBuffer().Take((int)errorStream.Length).ToArray())); + parse.Logger.Error(self.Location, Encoding.ASCII.GetString(errorStream.GetBuffer().Take((int)errorStream.Length).ToArray())); } else if (output.Length >= 7 && Encoding.ASCII.GetString(output.Take(7).ToArray()) == "ERROR: ") { - parse.Error(self.Location, Encoding.ASCII.GetString(output.Skip(7).ToArray())); + parse.Logger.Error(self.Location, Encoding.ASCII.GetString(output.Skip(7).ToArray())); } ExecTimer.Timer.AddTimingPoint(parameters[0].ToString()!.ToLower()); - return new DataNode(parse.CurrentOffset, output); + return new DataNode(parse.ParseConsumer.CurrentOffset, output); } } } diff --git a/ColorzCore/Preprocessor/Directives/IncludeToolEventDirective.cs b/ColorzCore/Preprocessor/Directives/IncludeToolEventDirective.cs index cc5c22c..0d98187 100644 --- a/ColorzCore/Preprocessor/Directives/IncludeToolEventDirective.cs +++ b/ColorzCore/Preprocessor/Directives/IncludeToolEventDirective.cs @@ -27,7 +27,7 @@ class IncludeToolEventDirective : SimpleDirective if (validFile == null) { - parse.Error(parameters[0].MyLocation, "Tool " + parameters[0].ToString() + " not found."); + parse.Logger.Error(parameters[0].MyLocation, "Tool " + parameters[0].ToString() + " not found."); return null; } @@ -64,11 +64,11 @@ class IncludeToolEventDirective : SimpleDirective byte[] output = outputBytes.GetBuffer().Take((int)outputBytes.Length).ToArray(); if (errorStream.Length > 0) { - parse.Error(self.Location, Encoding.ASCII.GetString(errorStream.GetBuffer().Take((int)errorStream.Length).ToArray())); + parse.Logger.Error(self.Location, Encoding.ASCII.GetString(errorStream.GetBuffer().Take((int)errorStream.Length).ToArray())); } else if (output.Length >= 7 && Encoding.ASCII.GetString(output.Take(7).ToArray()) == "ERROR: ") { - parse.Error(self.Location, Encoding.ASCII.GetString(output.Skip(7).ToArray())); + parse.Logger.Error(self.Location, Encoding.ASCII.GetString(output.Skip(7).ToArray())); } else { diff --git a/ColorzCore/Preprocessor/Directives/SimpleDirective.cs b/ColorzCore/Preprocessor/Directives/SimpleDirective.cs index e714cc0..2b591bc 100644 --- a/ColorzCore/Preprocessor/Directives/SimpleDirective.cs +++ b/ColorzCore/Preprocessor/Directives/SimpleDirective.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Linq; using ColorzCore.DataTypes; +using ColorzCore.IO; using ColorzCore.Lexer; using ColorzCore.Parser; using ColorzCore.Parser.AST; @@ -37,7 +38,7 @@ public abstract class SimpleDirective : IDirective } else { - p.Error(self.Location, $"Invalid number of parameters ({parameters.Count}) to directive {self}."); + p.Logger.Error(self.Location, $"Invalid number of parameters ({parameters.Count}) to directive {self}."); return null; } } diff --git a/ColorzCore/Preprocessor/Directives/UndefineDirective.cs b/ColorzCore/Preprocessor/Directives/UndefineDirective.cs index 40fe263..4600b97 100644 --- a/ColorzCore/Preprocessor/Directives/UndefineDirective.cs +++ b/ColorzCore/Preprocessor/Directives/UndefineDirective.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Linq; using ColorzCore.DataTypes; +using ColorzCore.IO; using ColorzCore.Lexer; using ColorzCore.Parser; using ColorzCore.Parser.AST; @@ -24,7 +25,7 @@ class UndefineDirective : SimpleDirective if (p.Definitions.ContainsKey(s)) p.Definitions.Remove(s); else - p.Warning(parm.MyLocation, "Undefining non-existant definition: " + s); + p.Logger.Warning(parm.MyLocation, "Undefining non-existant definition: " + s); } return null; } diff --git a/ColorzCore/Preprocessor/Macros/ErrorMacro.cs b/ColorzCore/Preprocessor/Macros/ErrorMacro.cs index bfa9fda..498f870 100644 --- a/ColorzCore/Preprocessor/Macros/ErrorMacro.cs +++ b/ColorzCore/Preprocessor/Macros/ErrorMacro.cs @@ -1,4 +1,5 @@ using ColorzCore.DataTypes; +using ColorzCore.IO; using ColorzCore.Lexer; using ColorzCore.Parser; using System; @@ -24,7 +25,7 @@ public ErrorMacro(EAParser parser, string message, ValidateNumParamsIndirect val public override IEnumerable ApplyMacro(Token head, IList> parameters, ImmutableStack scopes) { - parser.Error(head.Location, message); + parser.Logger.Error(head.Location, message); yield return new Token(TokenType.NUMBER, head.Location, "0"); } diff --git a/ColorzCore/Preprocessor/Macros/ReadDataAt.cs b/ColorzCore/Preprocessor/Macros/ReadDataAt.cs index 9ef9727..402d3f7 100644 --- a/ColorzCore/Preprocessor/Macros/ReadDataAt.cs +++ b/ColorzCore/Preprocessor/Macros/ReadDataAt.cs @@ -33,12 +33,12 @@ public override IEnumerable ApplyMacro(Token head, IList> pa if (tokens.Current.Type != TokenType.NEWLINE) { - parser.Error(head.Location, "Garbage at the end of macro parameter."); + parser.Logger.Error(head.Location, "Garbage at the end of macro parameter."); yield return new Token(TokenType.NUMBER, head.Location, "0"); } - else if (atom?.TryEvaluate(e => parser.Error(atom.MyLocation, e.Message), EvaluationPhase.Immediate) is int offset) + else if (atom?.TryEvaluate(e => parser.Logger.Error(atom.MyLocation, e.Message), EvaluationPhase.Immediate) is int offset) { - offset = EAParser.ConvertToOffset(offset); + offset = EAParseConsumer.ConvertToOffset(offset); if (offset >= 0 && offset <= EAOptions.MaximumBinarySize - readLength) { @@ -54,13 +54,13 @@ public override IEnumerable ApplyMacro(Token head, IList> pa } else { - parser.Error(head.Location, $"Read offset out of bounds: {offset:08X}"); + parser.Logger.Error(head.Location, $"Read offset out of bounds: {offset:08X}"); yield return new Token(TokenType.NUMBER, head.Location, "0"); } } else { - parser.Error(head.Location, "Could not read data from base binary."); + parser.Logger.Error(head.Location, "Could not read data from base binary."); yield return new Token(TokenType.NUMBER, head.Location, "0"); } } diff --git a/ColorzCore/Raws/AtomicParam.cs b/ColorzCore/Raws/AtomicParam.cs index c60a8d9..1f1e093 100644 --- a/ColorzCore/Raws/AtomicParam.cs +++ b/ColorzCore/Raws/AtomicParam.cs @@ -33,7 +33,7 @@ public void Set(byte[] data, int value) { if (pointer) { - value = EAParser.ConvertToAddress(value); + value = EAParseConsumer.ConvertToAddress(value); } data.SetBits(Position, Length, value); From 1493cf2c86f0e23bfbace0e22af64b95056ef542 Mon Sep 17 00:00:00 2001 From: Nat_776 Date: Mon, 29 Apr 2024 22:45:27 +0200 Subject: [PATCH 43/59] change AST production --- ColorzCore/Parser/AST/BlockNode.cs | 55 ---- ColorzCore/Parser/AST/RawNode.cs | 2 - ColorzCore/Parser/EAParseConsumer.cs | 124 +++++---- ColorzCore/Parser/EAParser.cs | 241 +++++++++--------- ColorzCore/Parser/IParseConsumer.cs | 23 ++ .../Preprocessor/Directives/PoolDirective.cs | 8 +- 6 files changed, 217 insertions(+), 236 deletions(-) delete mode 100644 ColorzCore/Parser/AST/BlockNode.cs create mode 100644 ColorzCore/Parser/IParseConsumer.cs diff --git a/ColorzCore/Parser/AST/BlockNode.cs b/ColorzCore/Parser/AST/BlockNode.cs deleted file mode 100644 index d407fde..0000000 --- a/ColorzCore/Parser/AST/BlockNode.cs +++ /dev/null @@ -1,55 +0,0 @@ -using ColorzCore.DataTypes; -using ColorzCore.IO; -using ColorzCore.Lexer; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace ColorzCore.Parser.AST -{ - class BlockNode : ILineNode - { - public IList Children { get; } - - public int Size => Children.Sum(n => n.Size); - - public BlockNode() - { - Children = new List(); - } - - public string PrettyPrint(int indentation) - { - StringBuilder sb = new StringBuilder(); - sb.Append(' ', indentation); - sb.Append('{'); - sb.Append('\n'); - foreach (ILineNode i in Children) - { - sb.Append(i.PrettyPrint(indentation + 4)); - sb.Append('\n'); - } - sb.Append(' ', indentation); - sb.Append('}'); - return sb.ToString(); - } - - public void WriteData(IOutput output) - { - foreach (ILineNode child in Children) - { - child.WriteData(output); - } - } - - public void EvaluateExpressions(ICollection<(Location, Exception)> evaluationErrors, EvaluationPhase evaluationPhase) - { - foreach (ILineNode line in Children) - { - line.EvaluateExpressions(evaluationErrors, evaluationPhase); - } - } - } -} diff --git a/ColorzCore/Parser/AST/RawNode.cs b/ColorzCore/Parser/AST/RawNode.cs index c22400f..5160834 100644 --- a/ColorzCore/Parser/AST/RawNode.cs +++ b/ColorzCore/Parser/AST/RawNode.cs @@ -15,8 +15,6 @@ public class RawNode : StatementNode private Token myToken; private int Offset { get; } - public Location Location => myToken.Location; - public RawNode(Raw raw, Token t, int offset, IList paramList) : base(paramList) { myToken = t; diff --git a/ColorzCore/Parser/EAParseConsumer.cs b/ColorzCore/Parser/EAParseConsumer.cs index f59f5a6..313b405 100644 --- a/ColorzCore/Parser/EAParseConsumer.cs +++ b/ColorzCore/Parser/EAParseConsumer.cs @@ -1,4 +1,5 @@ using System.Collections.Generic; +using System.Linq; using ColorzCore.DataTypes; using ColorzCore.IO; using ColorzCore.Lexer; @@ -6,11 +7,14 @@ namespace ColorzCore.Parser { - public class EAParseConsumer + public class EAParseConsumer : IParseConsumer { public int CurrentOffset => currentOffset; + public IList AllScopes { get; } + public ImmutableStack GlobalScope { get; } + public ImmutableStack CurrentScope { get; set; } private readonly Stack<(int, bool)> pastOffsets; // currentOffset, offsetInitialized private readonly IList<(int, int, Location)> protectedRegions; @@ -23,7 +27,8 @@ public class EAParseConsumer public EAParseConsumer(Logger logger) { - GlobalScope = new ImmutableStack(new BaseClosure(), ImmutableStack.Nil); + AllScopes = new List() { new BaseClosure() }; + CurrentScope = GlobalScope = new ImmutableStack(AllScopes.First()!, ImmutableStack.Nil); pastOffsets = new Stack<(int, bool)>(); protectedRegions = new List<(int, int, Location)>(); currentOffset = 0; @@ -66,24 +71,53 @@ public static int ConvertToOffset(int value) return node.TryEvaluate(e => Logger.Error(node.MyLocation, e.Message), EvaluationPhase.Immediate); } - public ILineNode? HandleRawStatement(RawNode node) + private IList LineNodes { get; } = new List(); + + public IList HandleEndOfInput() + { + if (CurrentScope != GlobalScope) + { + Logger.Error(null, "Reached end of input with an open local scope."); + } + + return LineNodes; + } + + #region AST Handlers + + public void OnOpenScope(Location _) + { + CurrentScope = new ImmutableStack(new Closure(), CurrentScope); + } + + public void OnCloseScope(Location location) + { + if (CurrentScope != GlobalScope) + { + CurrentScope = CurrentScope.Tail; + } + else + { + Logger.Error(location, "Attempted to close local scope without being within one."); + } + } + + public void OnRawStatement(Location location, RawNode node) { if ((CurrentOffset % node.Raw.Alignment) != 0) { - Logger.Error(node.Location, + Logger.Error(location, $"Bad alignment for raw {node.Raw.Name}: offset ({CurrentOffset:X8}) needs to be {node.Raw.Alignment}-aligned."); - - return null; } else { // TODO: more efficient spacewise to just have contiguous writing and not an offset with every line? - CheckWriteBytes(node.Location, node.Size); - return node; + CheckWriteBytes(location, node.Size); + LineNodes.Add(node); } } - public ILineNode? HandleOrgStatement(Location location, IAtomNode offsetNode) + public void OnOrgStatement(Location location, IAtomNode offsetNode) { if (EvaluteAtom(offsetNode) is int offset) { @@ -103,17 +137,14 @@ public static int ConvertToOffset(int value) { // EvaluateAtom already printed an error message } - - return null; } - public ILineNode? HandlePushStatement(Location _) + public void OnPushStatement(Location _) { pastOffsets.Push((CurrentOffset, offsetInitialized)); - return null; } - public ILineNode? HandlePopStatement(Location location) + public void OnPopStatement(Location location) { if (pastOffsets.Count == 0) { @@ -123,11 +154,9 @@ public static int ConvertToOffset(int value) { (currentOffset, offsetInitialized) = pastOffsets.Pop(); } - - return null; } - public ILineNode? HandleAssertStatement(Location location, IAtomNode node) + public void OnAssertStatement(Location location, IAtomNode node) { // helper for distinguishing boolean expressions and other expressions // TODO: move elsewhere perhaps @@ -175,11 +204,9 @@ static bool IsBooleanResultHelper(IAtomNode node) { Logger.Error(node.MyLocation, "Failed to evaluate ASSERT expression."); } - - return null; } - public ILineNode? HandleProtectStatement(Location location, IAtomNode beginAtom, IAtomNode? endAtom) + public void OnProtectStatement(Location location, IAtomNode beginAtom, IAtomNode? endAtom) { if (EvaluteAtom(beginAtom) is int beginValue) { @@ -199,33 +226,29 @@ static bool IsBooleanResultHelper(IAtomNode node) { case < 0: Logger.Error(location, $"Invalid PROTECT region: end address ({endValue:X8}) is before start address ({beginValue:X8})."); - return null; + return; case 0: // NOTE: does this need to be an error? Logger.Error(location, $"Empty PROTECT region: end address is equal to start address ({beginValue:X8})."); - return null; + return; } } else { // EvaluateAtom already printed an error message - return null; } } protectedRegions.Add((beginValue, length, location)); - - return null; } else { // EvaluateAtom already printed an error message - return null; } } - public ILineNode? HandleAlignStatement(Location location, IAtomNode alignNode, IAtomNode? offsetNode) + public void OnAlignStatement(Location location, IAtomNode alignNode, IAtomNode? offsetNode) { if (EvaluteAtom(alignNode) is int alignValue) { @@ -244,13 +267,13 @@ static bool IsBooleanResultHelper(IAtomNode node) else { Logger.Error(location, $"ALIGN offset cannot be negative (got {rawOffset})"); - return null; + return; } } else { // EvaluateAtom already printed an error message - return null; + return; } } @@ -264,23 +287,19 @@ static bool IsBooleanResultHelper(IAtomNode node) diagnosedOverflow = true; } } - - return null; } else { Logger.Error(location, $"Invalid ALIGN value (got {alignValue})."); - return null; } } else { // EvaluateAtom already printed an error message - return null; } } - public ILineNode? HandleFillStatement(Location location, IAtomNode amountNode, IAtomNode? valueNode) + public void OnFillStatement(Location location, IAtomNode amountNode, IAtomNode? valueNode) { if (EvaluteAtom(amountNode) is int amount) { @@ -297,7 +316,7 @@ static bool IsBooleanResultHelper(IAtomNode node) else { // EvaluateAtom already printed an error message - return null; + return; } } @@ -311,56 +330,53 @@ static bool IsBooleanResultHelper(IAtomNode node) var node = new DataNode(CurrentOffset, data); CheckWriteBytes(location, amount); - return node; + LineNodes.Add(node); } else { Logger.Error(location, $"Invalid FILL amount (got {amount})."); - return null; } } else { // EvaluateAtom already printed an error message - return null; } } - public ILineNode? HandleSymbolAssignment(Location location, string name, IAtomNode atom, ImmutableStack scopes) + public void OnSymbolAssignment(Location location, string name, IAtomNode atom) { if (atom.TryEvaluate(_ => { }, EvaluationPhase.Early) is int value) { - TryDefineSymbol(location, scopes, name, value); + TryDefineSymbol(location, name, value); } else { - TryDefineSymbol(location, scopes, name, atom); + TryDefineSymbol(location, name, atom); } - - return null; } - public ILineNode? HandleLabel(Location location, string name, ImmutableStack scopes) + public void OnLabel(Location location, string name) { - TryDefineSymbol(location, scopes, name, ConvertToAddress(CurrentOffset)); - return null; + TryDefineSymbol(location, name, ConvertToAddress(CurrentOffset)); } - public ILineNode? HandlePreprocessorLineNode(Location location, ILineNode node) + public void OnPreprocessorData(Location location, ILineNode node) { CheckWriteBytes(location, node.Size); - return node; + LineNodes.Add(node); } + #endregion + public bool IsValidLabelName(string name) { // TODO: this could be where checks for CURRENTOFFSET, __LINE__ and __FILE__ are? return true; // !IsReservedName(name); } - public void TryDefineSymbol(Location location, ImmutableStack scopes, string name, int value) + public void TryDefineSymbol(Location location, string name, int value) { - if (scopes.Head.HasLocalSymbol(name)) + if (CurrentScope.Head.HasLocalSymbol(name)) { Logger.Warning(location, $"Symbol already in scope, ignoring: {name}"); } @@ -371,13 +387,13 @@ public void TryDefineSymbol(Location location, ImmutableStack scopes, s } else { - scopes.Head.AddSymbol(name, value); + CurrentScope.Head.AddSymbol(name, value); } } - public void TryDefineSymbol(Location location, ImmutableStack scopes, string name, IAtomNode expression) + public void TryDefineSymbol(Location location, string name, IAtomNode expression) { - if (scopes.Head.HasLocalSymbol(name)) + if (CurrentScope.Head.HasLocalSymbol(name)) { Logger.Warning(location, $"Symbol already in scope, ignoring: {name}"); } @@ -388,7 +404,7 @@ public void TryDefineSymbol(Location location, ImmutableStack scopes, s } else { - scopes.Head.AddSymbol(name, expression); + CurrentScope.Head.AddSymbol(name, expression); } } diff --git a/ColorzCore/Parser/EAParser.cs b/ColorzCore/Parser/EAParser.cs index d653a3c..c6aa175 100644 --- a/ColorzCore/Parser/EAParser.cs +++ b/ColorzCore/Parser/EAParser.cs @@ -76,10 +76,6 @@ public bool IsValidMacroName(string name, int paramNum) public IList ParseAll(IEnumerable tokenStream) { - //TODO: Make BlockNode or EAProgramNode? - //Note must be strict to get all information on the closure before evaluating terms. - IList myLines = new List(); - MergeableGenerator tokens = new MergeableGenerator(tokenStream); tokens.MoveNext(); @@ -87,46 +83,16 @@ public IList ParseAll(IEnumerable tokenStream) { if (tokens.Current.Type != TokenType.NEWLINE || tokens.MoveNext()) { - ILineNode? retVal = ParseLine(tokens, ParseConsumer.GlobalScope); - retVal.IfJust(n => myLines.Add(n)); - } - } - - return myLines; - } - - private BlockNode ParseBlock(MergeableGenerator tokens, ImmutableStack scopes) - { - Location start = tokens.Current.Location; - tokens.MoveNext(); - - BlockNode temp = new BlockNode(); - - while (!tokens.EOS && tokens.Current.Type != TokenType.CLOSE_BRACE) - { - ILineNode? x = ParseLine(tokens, scopes); - - if (x != null) - { - temp.Children.Add(x); + ParseLine(tokens, ParseConsumer.GlobalScope); } } - if (!tokens.EOS) - { - tokens.MoveNext(); - } - else - { - Logger.Error(start, "Didn't find matching brace."); - } - - return temp; + return ParseConsumer.HandleEndOfInput(); } public static readonly HashSet SpecialCodes = new HashSet { "ORG", "PUSH", "POP", "MESSAGE", "WARNING", "ERROR", "ASSERT", "PROTECT", "ALIGN", "FILL" }; - private ILineNode? ParseStatement(MergeableGenerator tokens, ImmutableStack scopes) + private void ParseStatement(MergeableGenerator tokens, ImmutableStack scopes) { // NOTE: here previously lied en ExpandIdentifier loop // though because this is only called from ParseLine after the corresponding check, this is not needed @@ -138,11 +104,13 @@ private BlockNode ParseBlock(MergeableGenerator tokens, ImmutableStack tokens, ImmutableStack ParseOrgStatement(parameters, scopes), - "PUSH" => ParsePushStatement(parameters, scopes), - "POP" => ParsePopStatement(parameters, scopes), - "ASSERT" => ParseAssertStatement(parameters, scopes), - "PROTECT" => ParseProtectStatement(parameters, scopes), - "ALIGN" => ParseAlignStatement(parameters, scopes), - "FILL" => ParseFillStatement(parameters, scopes), - "MESSAGE" => ParseMessageStatement(parameters, scopes), - "WARNING" => ParseWarningStatement(parameters, scopes), - "ERROR" => ParseErrorStatement(parameters, scopes), - _ => null, // TODO: this is an error - }; - } + case "ORG": + ParseOrgStatement(parameters); + return; + + case "PUSH": + ParsePushStatement(parameters); + return; + + case "POP": + ParsePopStatement(parameters); + return; + + case "ASSERT": + ParseAssertStatement(parameters); + return; + + case "PROTECT": + ParseProtectStatement(parameters); + return; - return ParseRawStatement(upperCodeIdentifier, tokens, parameters); + case "ALIGN": + ParseAlignStatement(parameters); + return; + + case "FILL": + ParseFillStatement(parameters); + return; + + case "MESSAGE": + ParseMessageStatement(parameters); + return; + + case "WARNING": + ParseWarningStatement(parameters); + return; + + case "ERROR": + ParseErrorStatement(parameters); + return; + } + } + else + { + ParseRawStatement(upperCodeIdentifier, tokens, parameters); + } } - private ILineNode? ParseAssignment(string name, MergeableGenerator tokens, ImmutableStack scopes) + private void ParseAssignment(string name, MergeableGenerator tokens) { - IAtomNode? atom = this.ParseAtom(tokens, scopes, true); + IAtomNode? atom = this.ParseAtom(tokens, ParseConsumer.CurrentScope, true); if (atom != null) { - return ParseConsumer.HandleSymbolAssignment(head!.Location, name, atom, scopes); + ParseConsumer.OnSymbolAssignment(head!.Location, name, atom); } else { Logger.Error(head!.Location, $"Couldn't define symbol `{name}`: exprected expression."); } - - return null; } - private ILineNode? ParseRawStatement(string upperCodeIdentifier, MergeableGenerator tokens, IList parameters) + private void ParseRawStatement(string upperCodeIdentifier, MergeableGenerator tokens, IList parameters) { if (Raws.TryGetValue(upperCodeIdentifier, out IList? raws)) { @@ -204,7 +200,9 @@ private BlockNode ParseBlock(MergeableGenerator tokens, ImmutableStack tokens, ImmutableStack parameters, ImmutableStack _) + private void ParseOrgStatement(IList parameters) { - return ParseStatementOneParam("ORG", parameters, ParseConsumer.HandleOrgStatement); + ParseStatementOneParam("ORG", parameters, ParseConsumer.OnOrgStatement); } - private ILineNode? ParsePushStatement(IList parameters, ImmutableStack _) + private void ParsePushStatement(IList parameters) { if (parameters.Count == 0) { - return ParseConsumer.HandlePushStatement(head!.Location); + ParseConsumer.OnPushStatement(head!.Location); } else { - return StatementExpectsParamCount("PUSH", parameters, 0, 0); + StatementExpectsParamCount("PUSH", parameters, 0, 0); } } - private ILineNode? ParsePopStatement(IList parameters, ImmutableStack _) + private void ParsePopStatement(IList parameters) { if (parameters.Count == 0) { - return ParseConsumer.HandlePopStatement(head!.Location); + ParseConsumer.OnPopStatement(head!.Location); } else { - return StatementExpectsParamCount("POP", parameters, 0, 0); + StatementExpectsParamCount("POP", parameters, 0, 0); } } - private ILineNode? ParseAssertStatement(IList parameters, ImmutableStack _) + private void ParseAssertStatement(IList parameters) { - return ParseStatementOneParam("ASSERT", parameters, ParseConsumer.HandleAssertStatement); + ParseStatementOneParam("ASSERT", parameters, ParseConsumer.OnAssertStatement); } // Helper method for printing errors - private ILineNode? StatementExpectsAtom(string statementName, IParamNode param) + private void StatementExpectsAtom(string statementName, IParamNode param) { Logger.Error(param.MyLocation, $"{statementName} expects an Atom (got {DiagnosticsHelpers.PrettyParamType(param.Type)})."); - - return null; } // Helper method for printing errors - private ILineNode? StatementExpectsParamCount(string statementName, IList parameters, int min, int max) + private void StatementExpectsParamCount(string statementName, IList parameters, int min, int max) { if (min == max) { @@ -290,43 +284,41 @@ private BlockNode ParseBlock(MergeableGenerator tokens, ImmutableStack parameters, HandleStatementOne handler) + private void ParseStatementOneParam(string name, IList parameters, HandleStatementOne handler) { if (parameters.Count == 1) { if (parameters[0] is IAtomNode expr) { - return handler(head!.Location, expr); + handler(head!.Location, expr); } else { - return StatementExpectsAtom(name, parameters[0]); + StatementExpectsAtom(name, parameters[0]); } } else { - return StatementExpectsParamCount(name, parameters, 1, 1); + StatementExpectsParamCount(name, parameters, 1, 1); } } - private ILineNode? ParseStatementTwoParam(string name, IList parameters, HandleStatementTwo handler) + private void ParseStatementTwoParam(string name, IList parameters, HandleStatementTwo handler) { if (parameters.Count == 1) { if (parameters[0] is IAtomNode firstNode) { - return handler(head!.Location, firstNode, null); + handler(head!.Location, firstNode, null); } else { - return StatementExpectsAtom(name, parameters[0]); + StatementExpectsAtom(name, parameters[0]); } } else if (parameters.Count == 2) @@ -335,55 +327,55 @@ private BlockNode ParseBlock(MergeableGenerator tokens, ImmutableStack parameters, ImmutableStack _) + private void ParseProtectStatement(IList parameters) { - return ParseStatementTwoParam("PROTECT", parameters, ParseConsumer.HandleProtectStatement); + ParseStatementTwoParam("PROTECT", parameters, ParseConsumer.OnProtectStatement); } - private ILineNode? ParseAlignStatement(IList parameters, ImmutableStack _) + private void ParseAlignStatement(IList parameters) { - return ParseStatementTwoParam("ALIGN", parameters, ParseConsumer.HandleAlignStatement); + ParseStatementTwoParam("ALIGN", parameters, ParseConsumer.OnAlignStatement); } - private ILineNode? ParseFillStatement(IList parameters, ImmutableStack _) + private void ParseFillStatement(IList parameters) { - return ParseStatementTwoParam("FILL", parameters, ParseConsumer.HandleFillStatement); + ParseStatementTwoParam("FILL", parameters, ParseConsumer.OnFillStatement); } - private ILineNode? ParseMessageStatement(IList parameters, ImmutableStack scopes) + private void ParseMessageStatement(IList parameters) { + ImmutableStack scopes = ParseConsumer.CurrentScope; Logger.Message(head!.Location, PrettyPrintParamsForMessage(parameters, scopes)); - return null; } - private ILineNode? ParseWarningStatement(IList parameters, ImmutableStack scopes) + private void ParseWarningStatement(IList parameters) { + ImmutableStack scopes = ParseConsumer.CurrentScope; Logger.Warning(head!.Location, PrettyPrintParamsForMessage(parameters, scopes)); - return null; } - private ILineNode? ParseErrorStatement(IList parameters, ImmutableStack scopes) + private void ParseErrorStatement(IList parameters) { + ImmutableStack scopes = ParseConsumer.CurrentScope; Logger.Error(head!.Location, PrettyPrintParamsForMessage(parameters, scopes)); - return null; } public IList> ParseMacroParamList(MergeableGenerator tokens) @@ -551,14 +543,14 @@ private IList ParseList(MergeableGenerator tokens, ImmutableSt return atoms; } - public ILineNode? ParseLine(MergeableGenerator tokens, ImmutableStack scopes) + public void ParseLine(MergeableGenerator tokens, ImmutableStack scopes) { if (IsIncluding) { if (tokens.Current.Type == TokenType.NEWLINE || tokens.Current.Type == TokenType.SEMICOLON) { tokens.MoveNext(); - return null; + return; } head = tokens.Current; @@ -576,24 +568,39 @@ private IList ParseList(MergeableGenerator tokens, ImmutableSt case TokenType.MAYBE_MACRO: case TokenType.OPEN_BRACE: case TokenType.PREPROCESSOR_DIRECTIVE: - return ParseLine(tokens, scopes); + // recursion! + ParseLine(tokens, scopes); + return; default: // it is somewhat common for users to do '#define Foo 0xABCD' and then later 'Foo:' Logger.Error(head.Location, $"Expansion of macro `{head.Content}` did not result in a valid statement. Did you perhaps attempt to define a label or symbol with that name?"); IgnoreRestOfLine(tokens); - - return null; + break; } + + return; + } + else + { + ParseStatement(tokens, scopes); } - return ParseStatement(tokens, scopes); + break; case TokenType.OPEN_BRACE: - return ParseBlock(tokens, new ImmutableStack(new Closure(), scopes)); + tokens.MoveNext(); + ParseConsumer.OnOpenScope(head.Location); + break; + + case TokenType.CLOSE_BRACE: + tokens.MoveNext(); + ParseConsumer.OnCloseScope(head.Location); + break; case TokenType.PREPROCESSOR_DIRECTIVE: - return ParsePreprocessor(tokens, scopes); + ParsePreprocessor(tokens, scopes); + break; case TokenType.OPEN_BRACKET: Logger.Error(head.Location, "Unexpected list literal."); @@ -621,7 +628,6 @@ private IList ParseList(MergeableGenerator tokens, ImmutableSt IgnoreRestOfLine(tokens); break; } - return null; } else { @@ -633,17 +639,16 @@ private IList ParseList(MergeableGenerator tokens, ImmutableSt if (hasNext) { - return ParsePreprocessor(tokens, scopes); + ParsePreprocessor(tokens, scopes); } else { Logger.Error(null, $"Missing {Inclusion.Count} endif(s)."); - return null; } } } - private ILineNode? ParsePreprocessor(MergeableGenerator tokens, ImmutableStack scopes) + private void ParsePreprocessor(MergeableGenerator tokens, ImmutableStack scopes) { head = tokens.Current; tokens.MoveNext(); @@ -652,10 +657,8 @@ private IList ParseList(MergeableGenerator tokens, ImmutableSt if (result != null) { - ParseConsumer.HandlePreprocessorLineNode(head.Location, result); + ParseConsumer.OnPreprocessorData(head.Location, result); } - - return null; } /*** diff --git a/ColorzCore/Parser/IParseConsumer.cs b/ColorzCore/Parser/IParseConsumer.cs new file mode 100644 index 0000000..93d5f31 --- /dev/null +++ b/ColorzCore/Parser/IParseConsumer.cs @@ -0,0 +1,23 @@ + +using ColorzCore.DataTypes; +using ColorzCore.Parser.AST; + +namespace ColorzCore.Parser +{ + public interface IParseConsumer + { + void OnOpenScope(Location location); + void OnCloseScope(Location location); + void OnRawStatement(Location location, RawNode node); + void OnOrgStatement(Location location, IAtomNode offsetNode); + void OnPushStatement(Location location); + void OnPopStatement(Location location); + void OnAssertStatement(Location location, IAtomNode node); + void OnProtectStatement(Location location, IAtomNode beginAtom, IAtomNode? endAtom); + void OnAlignStatement(Location location, IAtomNode alignNode, IAtomNode? offsetNode); + void OnFillStatement(Location location, IAtomNode amountNode, IAtomNode? valueNode); + void OnSymbolAssignment(Location location, string name, IAtomNode atom); + void OnLabel(Location location, string name); + void OnPreprocessorData(Location location, ILineNode node); + } +} diff --git a/ColorzCore/Preprocessor/Directives/PoolDirective.cs b/ColorzCore/Preprocessor/Directives/PoolDirective.cs index 745d35e..ad5a4b0 100644 --- a/ColorzCore/Preprocessor/Directives/PoolDirective.cs +++ b/ColorzCore/Preprocessor/Directives/PoolDirective.cs @@ -22,8 +22,6 @@ public PoolDirective(Pool pool) public override ILineNode? Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) { - BlockNode result = new BlockNode(); - // Iterating indices (and not values via foreach) // to avoid crashes occuring with AddToPool within AddToPool @@ -36,14 +34,12 @@ public PoolDirective(Pool pool) while (!tempGenerator.EOS) { - p.ParseLine(tempGenerator, line.Scope).IfJust( - (lineNode) => result.Children.Add(lineNode)); + p.ParseLine(tempGenerator, line.Scope); } } pool.Lines.Clear(); - - return result; + return null; } } } From 898ceeeacda70c0b459d4e2fb8f62fd600c03e12 Mon Sep 17 00:00:00 2001 From: Nat_776 Date: Tue, 30 Apr 2024 20:24:36 +0200 Subject: [PATCH 44/59] UTF8, BASE64, directive refactor, better undef , less leaky CurrentOffset, nocash sym prints locals --- ColorzCore/EAInterpreter.cs | 94 ++++--- ColorzCore/EAOptions.cs | 9 +- ColorzCore/Parser/AST/RawNode.cs | 29 ++- ColorzCore/Parser/AST/StatementNode.cs | 40 --- ColorzCore/Parser/EAParseConsumer.cs | 13 +- ColorzCore/Parser/EAParser.cs | 234 ++++++++++++------ ColorzCore/Parser/IParseConsumer.cs | 7 +- ColorzCore/Preprocessor/DirectiveHandler.cs | 12 +- .../Directives/BaseIncludeDirective.cs | 55 ++++ .../Directives/DefineDirective.cs | 8 +- .../Preprocessor/Directives/ElseDirective.cs | 7 +- .../Preprocessor/Directives/EndIfDirective.cs | 8 +- .../Preprocessor/Directives/IDirective.cs | 3 +- .../Directives/IfDefinedDirective.cs | 8 +- .../Preprocessor/Directives/IfDirective.cs | 3 +- .../Directives/IncludeBinaryDirective.cs | 49 +--- .../Directives/IncludeDirective.cs | 52 +--- .../Directives/IncludeExternalDirective.cs | 6 +- .../Directives/IncludeToolEventDirective.cs | 6 +- .../Preprocessor/Directives/PoolDirective.cs | 3 +- .../Directives/SimpleDirective.cs | 9 +- .../Directives/UndefineDirective.cs | 43 +++- 22 files changed, 390 insertions(+), 308 deletions(-) delete mode 100644 ColorzCore/Parser/AST/StatementNode.cs create mode 100644 ColorzCore/Preprocessor/Directives/BaseIncludeDirective.cs diff --git a/ColorzCore/EAInterpreter.cs b/ColorzCore/EAInterpreter.cs index 59fe601..017c902 100644 --- a/ColorzCore/EAInterpreter.cs +++ b/ColorzCore/EAInterpreter.cs @@ -20,14 +20,13 @@ class EAInterpreter private Dictionary> allRaws; private EAParser myParser; private EAParseConsumer myParseConsumer; - private string game, iFile; + private string iFile; private Stream sin; - private Logger log; + private Logger Logger { get; } private IOutput output; - public EAInterpreter(IOutput output, string game, string? rawsFolder, string rawsExtension, Stream sin, string inFileName, Logger log) + public EAInterpreter(IOutput output, string game, string? rawsFolder, string rawsExtension, Stream sin, string inFileName, Logger logger) { - this.game = game; this.output = output; try @@ -36,21 +35,16 @@ public EAInterpreter(IOutput output, string game, string? rawsFolder, string raw } catch (RawReader.RawParseException e) { - Location loc = new Location - { - file = e.FileName, - line = e.LineNumber, - column = 1 - }; + Location loc = new Location(e.FileName, e.LineNumber, 1); - log.Message(Logger.MessageKind.ERROR, loc, "An error occured while parsing raws"); - log.Message(Logger.MessageKind.ERROR, loc, e.Message); + logger.Message(Logger.MessageKind.ERROR, loc, "An error occured while parsing raws"); + logger.Message(Logger.MessageKind.ERROR, loc, e.Message); Environment.Exit(-1); // ew? } this.sin = sin; - this.log = log; + Logger = logger; iFile = inFileName; IncludeFileSearcher includeSearcher = new IncludeFileSearcher(); @@ -59,14 +53,8 @@ public EAInterpreter(IOutput output, string game, string? rawsFolder, string raw foreach (string path in EAOptions.IncludePaths) includeSearcher.IncludeDirectories.Add(path); - IncludeFileSearcher toolSearcher = new IncludeFileSearcher { AllowRelativeInclude = false }; - toolSearcher.IncludeDirectories.Add(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Tools")); - - foreach (string path in EAOptions.ToolsPaths) - includeSearcher.IncludeDirectories.Add(path); - - myParseConsumer = new EAParseConsumer(log); - myParser = new EAParser(log, allRaws, new DirectiveHandler(includeSearcher, toolSearcher), myParseConsumer); + myParseConsumer = new EAParseConsumer(logger); + myParser = new EAParser(logger, allRaws, new DirectiveHandler(includeSearcher), myParseConsumer); myParser.Definitions[$"_{game}_"] = new Definition(); myParser.Definitions["__COLORZ_CORE__"] = new Definition(); @@ -88,6 +76,25 @@ public EAInterpreter(IOutput output, string game, string? rawsFolder, string raw myParser.Macros.BuiltInMacros.Add("ReadWordAt", unsupportedMacro); } + if (EAOptions.IsExtensionEnabled(EAOptions.Extensions.IncludeTools)) + { + IncludeFileSearcher toolSearcher = new IncludeFileSearcher { AllowRelativeInclude = false }; + toolSearcher.IncludeDirectories.Add(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Tools")); + + foreach (string path in EAOptions.ToolsPaths) + { + includeSearcher.IncludeDirectories.Add(path); + } + + IDirective incextDirective = new IncludeExternalDirective { FileSearcher = toolSearcher }; + IDirective inctextDirective = new IncludeToolEventDirective { FileSearcher = toolSearcher }; + + myParser.DirectiveHandler.Directives["incext"] = incextDirective; + myParser.DirectiveHandler.Directives["inctext"] = inctextDirective; + myParser.DirectiveHandler.Directives["inctevent"] = inctextDirective; + } + + if (EAOptions.IsExtensionEnabled(EAOptions.Extensions.AddToPool)) { Pool pool = new Pool(); @@ -98,16 +105,16 @@ public EAInterpreter(IOutput output, string game, string? rawsFolder, string raw public bool Interpret() { - Tokenizer t = new Tokenizer(); + Tokenizer tokenizer = new Tokenizer(); ExecTimer.Timer.AddTimingPoint(ExecTimer.KEY_GENERIC); foreach ((string name, string body) in EAOptions.PreDefintions) { - myParser.ParseAll(t.TokenizeLine($"#define {name} {body}", "cmd", 0)); + myParser.ParseAll(tokenizer.TokenizeLine($"#define {name} {body}", "cmd", 0)); } - IList lines = new List(myParser.ParseAll(t.Tokenize(sin, iFile))); + IList lines = new List(myParser.ParseAll(tokenizer.Tokenize(sin, iFile))); /* First pass on AST: Identifier resolution. * @@ -128,7 +135,7 @@ public bool Interpret() } catch (MacroInvocationNode.MacroException e) { - log.Error(e.CausedError.MyLocation, "Unexpanded macro."); + Logger.Error(e.CausedError.MyLocation, "Unexpanded macro."); } } @@ -137,11 +144,11 @@ public bool Interpret() if (e is IdentifierNode.UndefinedIdentifierException uie && uie.CausedError.Content.StartsWith(Pool.pooledLabelPrefix, StringComparison.Ordinal)) { - log.Error(location, "Unpooled data (forgot #pool?)"); + Logger.Error(location, "Unpooled data (forgot #pool?)"); } else { - log.Error(location, e.Message); + Logger.Error(location, e.Message); } } @@ -149,13 +156,13 @@ public bool Interpret() ExecTimer.Timer.AddTimingPoint(ExecTimer.KEY_DATAWRITE); - if (!log.HasErrored) + if (!Logger.HasErrored) { foreach (ILineNode line in lines) { if (Program.Debug) { - log.Message(Logger.MessageKind.DEBUG, line.PrettyPrint(0)); + Logger.Message(Logger.MessageKind.DEBUG, line.PrettyPrint(0)); } line.WriteData(output); @@ -163,21 +170,42 @@ public bool Interpret() output.Commit(); - log.Output.WriteLine("No errors. Please continue being awesome."); + Logger.Output.WriteLine("No errors. Please continue being awesome."); return true; } else { - log.Output.WriteLine("Errors occurred; no changes written."); + Logger.Output.WriteLine("Errors occurred; no changes written."); return false; } } public bool WriteNocashSymbols(TextWriter output) { - foreach (var label in myParseConsumer.GlobalScope.Head.LocalSymbols()) + for (int i = 0; i < myParseConsumer.AllScopes.Count; i++) { - output.WriteLine("{0:X8} {1}", EAParseConsumer.ConvertToAddress(label.Value), label.Key); + string manglePrefix = string.Empty; + + switch (i) + { + case 0: + output.WriteLine("; global scope"); + break; + default: + output.WriteLine($"; local scope {i}"); + manglePrefix = $"${i}$"; + break; + } + + Closure scope = myParseConsumer.AllScopes[i]; + + foreach (KeyValuePair pair in scope.LocalSymbols()) + { + string name = pair.Key; + int address = EAParseConsumer.ConvertToAddress(pair.Value); + + output.WriteLine($"{address:X8} {manglePrefix}{name}"); + } } return true; diff --git a/ColorzCore/EAOptions.cs b/ColorzCore/EAOptions.cs index e5fdc6d..7b4dac0 100644 --- a/ColorzCore/EAOptions.cs +++ b/ColorzCore/EAOptions.cs @@ -38,7 +38,14 @@ public enum Extensions : long // enable ReadByteAt and friends ReadDataMacros = 1, + // enable incext and inctext/inctevent + IncludeTools = 2, + + // enable AddToPool and #pool + AddToPool = 4, + All = long.MaxValue, + Default = IncludeTools | AddToPool, } public static bool WarningsAreErrors { get; set; } @@ -57,7 +64,7 @@ public enum Extensions : long public static List<(string, string)> PreDefintions { get; } = new List<(string, string)>(); public static Warnings EnabledWarnings { get; set; } = Warnings.All; - public static Extensions EnabledExtensions { get; set; } = Extensions.None; + public static Extensions EnabledExtensions { get; set; } = Extensions.Default; public static bool IsWarningEnabled(Warnings warning) => EnabledWarnings.HasFlag(warning); public static bool IsExtensionEnabled(Extensions extension) => EnabledExtensions.HasFlag(extension); diff --git a/ColorzCore/Parser/AST/RawNode.cs b/ColorzCore/Parser/AST/RawNode.cs index 5160834..5b9f3f9 100644 --- a/ColorzCore/Parser/AST/RawNode.cs +++ b/ColorzCore/Parser/AST/RawNode.cs @@ -9,26 +9,39 @@ namespace ColorzCore.Parser.AST { - public class RawNode : StatementNode + public class RawNode : ILineNode { + public IList Parameters { get; } public Raw Raw { get; } - private Token myToken; private int Offset { get; } - public RawNode(Raw raw, Token t, int offset, IList paramList) : base(paramList) + public RawNode(Raw raw, int offset, IList parameters) { - myToken = t; + Parameters = parameters; Raw = raw; Offset = offset; } - public override int Size => Raw.LengthBytes(Parameters.Count); + public void EvaluateExpressions(ICollection<(Location, Exception)> evaluationErrors, EvaluationPhase evaluationPhase) + { + for (int i = 0; i < Parameters.Count; i++) + { + Parameters[i] = Parameters[i].SimplifyExpressions(e => evaluationErrors.Add(e switch + { + IdentifierNode.UndefinedIdentifierException uie => (uie.CausedError.Location, uie), + Closure.SymbolComputeException sce => (sce.Expression.MyLocation, sce), + _ => (Parameters[i].MyLocation, e), + }), evaluationPhase); + } + } + + public int Size => Raw.LengthBytes(Parameters.Count); - public override string PrettyPrint(int indentation) + public string PrettyPrint(int indentation) { StringBuilder sb = new StringBuilder(); sb.Append(' ', indentation); - sb.Append(myToken.Content); + sb.Append(Raw.Name); foreach (IParamNode n in Parameters) { sb.Append(' '); @@ -37,7 +50,7 @@ public override string PrettyPrint(int indentation) return sb.ToString(); } - public override void WriteData(IOutput output) + public void WriteData(IOutput output) { output.WriteTo(Offset, Raw.GetBytes(Parameters)); } diff --git a/ColorzCore/Parser/AST/StatementNode.cs b/ColorzCore/Parser/AST/StatementNode.cs deleted file mode 100644 index f2c0f21..0000000 --- a/ColorzCore/Parser/AST/StatementNode.cs +++ /dev/null @@ -1,40 +0,0 @@ -using ColorzCore.DataTypes; -using ColorzCore.IO; -using ColorzCore.Lexer; -using ColorzCore.Raws; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace ColorzCore.Parser.AST -{ - public abstract class StatementNode : ILineNode - { - public IList Parameters { get; } - - protected StatementNode(IList parameters) - { - Parameters = parameters; - } - - public abstract int Size { get; } - - public abstract string PrettyPrint(int indentation); - public abstract void WriteData(IOutput output); - - public void EvaluateExpressions(ICollection<(Location, Exception)> evaluationErrors, EvaluationPhase evaluationPhase) - { - for (int i = 0; i < Parameters.Count; i++) - { - Parameters[i] = Parameters[i].SimplifyExpressions(e => evaluationErrors.Add(e switch - { - IdentifierNode.UndefinedIdentifierException uie => (uie.CausedError.Location, uie), - Closure.SymbolComputeException sce => (sce.Expression.MyLocation, sce), - _ => (Parameters[i].MyLocation, e), - }), evaluationPhase); - } - } - } -} diff --git a/ColorzCore/Parser/EAParseConsumer.cs b/ColorzCore/Parser/EAParseConsumer.cs index 313b405..11b8b21 100644 --- a/ColorzCore/Parser/EAParseConsumer.cs +++ b/ColorzCore/Parser/EAParseConsumer.cs @@ -4,6 +4,7 @@ using ColorzCore.IO; using ColorzCore.Lexer; using ColorzCore.Parser.AST; +using ColorzCore.Raws; namespace ColorzCore.Parser { @@ -87,7 +88,10 @@ public IList HandleEndOfInput() public void OnOpenScope(Location _) { - CurrentScope = new ImmutableStack(new Closure(), CurrentScope); + Closure newClosure = new Closure(); + + AllScopes.Add(newClosure); + CurrentScope = new ImmutableStack(newClosure, CurrentScope); } public void OnCloseScope(Location location) @@ -102,8 +106,10 @@ public void OnCloseScope(Location location) } } - public void OnRawStatement(Location location, RawNode node) + public void OnRawStatement(Location location, Raw raw, IList parameters) { + RawNode node = new RawNode(raw, CurrentOffset, parameters); + if ((CurrentOffset % node.Raw.Alignment) != 0) { Logger.Error(location, @@ -360,8 +366,9 @@ public void OnLabel(Location location, string name) TryDefineSymbol(location, name, ConvertToAddress(CurrentOffset)); } - public void OnPreprocessorData(Location location, ILineNode node) + public void OnData(Location location, byte[] data) { + DataNode node = new DataNode(CurrentOffset, data); CheckWriteBytes(location, node.Size); LineNodes.Add(node); } diff --git a/ColorzCore/Parser/EAParser.cs b/ColorzCore/Parser/EAParser.cs index c6aa175..f3b111b 100644 --- a/ColorzCore/Parser/EAParser.cs +++ b/ColorzCore/Parser/EAParser.cs @@ -44,8 +44,6 @@ public bool IsIncluding } } - private Token? head; // TODO: Make this make sense - public EAParseConsumer ParseConsumer { get; } public EAParser(Logger log, Dictionary> raws, DirectiveHandler directiveHandler, EAParseConsumer parseConsumer) @@ -90,14 +88,30 @@ public IList ParseAll(IEnumerable tokenStream) return ParseConsumer.HandleEndOfInput(); } - public static readonly HashSet SpecialCodes = new HashSet { "ORG", "PUSH", "POP", "MESSAGE", "WARNING", "ERROR", "ASSERT", "PROTECT", "ALIGN", "FILL" }; + public static readonly HashSet SpecialCodes = new HashSet() + { + "ORG", + "PUSH", + "POP", + "MESSAGE", + "WARNING", + "ERROR", + "ASSERT", + "PROTECT", + "ALIGN", + "FILL", + "UTF8", + "BASE64", + // "SECTION", // TODO + // "DSECTION", // TODO + }; private void ParseStatement(MergeableGenerator tokens, ImmutableStack scopes) { // NOTE: here previously lied en ExpandIdentifier loop // though because this is only called from ParseLine after the corresponding check, this is not needed - head = tokens.Current; + Token head = tokens.Current; tokens.MoveNext(); switch (tokens.Current.Type) @@ -109,7 +123,7 @@ private void ParseStatement(MergeableGenerator tokens, ImmutableStack tokens, ImmutableStack tokens) + private void ParseAssignment(Token head, string name, MergeableGenerator tokens) { IAtomNode? atom = this.ParseAtom(tokens, ParseConsumer.CurrentScope, true); if (atom != null) { - ParseConsumer.OnSymbolAssignment(head!.Location, name, atom); + ParseConsumer.OnSymbolAssignment(head.Location, name, atom); } else { - Logger.Error(head!.Location, $"Couldn't define symbol `{name}`: exprected expression."); + Logger.Error(head.Location, $"Couldn't define symbol `{name}`: exprected expression."); } } - private void ParseRawStatement(string upperCodeIdentifier, MergeableGenerator tokens, IList parameters) + private void ParseRawStatement(Token head, string upperCodeIdentifier, MergeableGenerator tokens, IList parameters) { if (Raws.TryGetValue(upperCodeIdentifier, out IList? raws)) { @@ -200,70 +222,69 @@ private void ParseRawStatement(string upperCodeIdentifier, MergeableGenerator parameters) + private void ParseOrgStatement(Token head, IList parameters) { - ParseStatementOneParam("ORG", parameters, ParseConsumer.OnOrgStatement); + ParseStatementOneParam(head, "ORG", parameters, ParseConsumer.OnOrgStatement); } - private void ParsePushStatement(IList parameters) + private void ParsePushStatement(Token head, IList parameters) { if (parameters.Count == 0) { - ParseConsumer.OnPushStatement(head!.Location); + ParseConsumer.OnPushStatement(head.Location); } else { - StatementExpectsParamCount("PUSH", parameters, 0, 0); + StatementExpectsParamCount(head, "PUSH", parameters, 0, 0); } } - private void ParsePopStatement(IList parameters) + private void ParsePopStatement(Token head, IList parameters) { if (parameters.Count == 0) { - ParseConsumer.OnPopStatement(head!.Location); + ParseConsumer.OnPopStatement(head.Location); } else { - StatementExpectsParamCount("POP", parameters, 0, 0); + StatementExpectsParamCount(head, "POP", parameters, 0, 0); } } - private void ParseAssertStatement(IList parameters) + private void ParseAssertStatement(Token head, IList parameters) { - ParseStatementOneParam("ASSERT", parameters, ParseConsumer.OnAssertStatement); + ParseStatementOneParam(head, "ASSERT", parameters, ParseConsumer.OnAssertStatement); } // Helper method for printing errors @@ -274,28 +295,28 @@ private void StatementExpectsAtom(string statementName, IParamNode param) } // Helper method for printing errors - private void StatementExpectsParamCount(string statementName, IList parameters, int min, int max) + private void StatementExpectsParamCount(Token head, string statementName, IList parameters, int min, int max) { if (min == max) { - Logger.Error(head!.Location, $"A {statementName} statement expects {min} parameters, got {parameters.Count}."); + Logger.Error(head.Location, $"A {statementName} statement expects {min} parameters, got {parameters.Count}."); } else { - Logger.Error(head!.Location, $"A {statementName} statement expects {min} to {max} parameters, got {parameters.Count}."); + Logger.Error(head.Location, $"A {statementName} statement expects {min} to {max} parameters, got {parameters.Count}."); } } private delegate void HandleStatementOne(Location location, IAtomNode node); private delegate void HandleStatementTwo(Location location, IAtomNode firstNode, IAtomNode? optionalSecondNode); - private void ParseStatementOneParam(string name, IList parameters, HandleStatementOne handler) + private void ParseStatementOneParam(Token head, string name, IList parameters, HandleStatementOne handler) { if (parameters.Count == 1) { if (parameters[0] is IAtomNode expr) { - handler(head!.Location, expr); + handler(head.Location, expr); } else { @@ -304,17 +325,17 @@ private void ParseStatementOneParam(string name, IList parameters, H } else { - StatementExpectsParamCount(name, parameters, 1, 1); + StatementExpectsParamCount(head, name, parameters, 1, 1); } } - private void ParseStatementTwoParam(string name, IList parameters, HandleStatementTwo handler) + private void ParseStatementTwoParam(Token head, string name, IList parameters, HandleStatementTwo handler) { if (parameters.Count == 1) { if (parameters[0] is IAtomNode firstNode) { - handler(head!.Location, firstNode, null); + handler(head.Location, firstNode, null); } else { @@ -327,7 +348,7 @@ private void ParseStatementTwoParam(string name, IList parameters, H { if (parameters[1] is IAtomNode secondNode) { - handler(head!.Location, firstNode, secondNode); + handler(head.Location, firstNode, secondNode); } else { @@ -341,41 +362,91 @@ private void ParseStatementTwoParam(string name, IList parameters, H } else { - StatementExpectsParamCount(name, parameters, 1, 2); + StatementExpectsParamCount(head, name, parameters, 1, 2); } } - private void ParseProtectStatement(IList parameters) + private void ParseProtectStatement(Token head, IList parameters) { - ParseStatementTwoParam("PROTECT", parameters, ParseConsumer.OnProtectStatement); + ParseStatementTwoParam(head, "PROTECT", parameters, ParseConsumer.OnProtectStatement); } - private void ParseAlignStatement(IList parameters) + private void ParseAlignStatement(Token head, IList parameters) { - ParseStatementTwoParam("ALIGN", parameters, ParseConsumer.OnAlignStatement); + ParseStatementTwoParam(head, "ALIGN", parameters, ParseConsumer.OnAlignStatement); } - private void ParseFillStatement(IList parameters) + private void ParseFillStatement(Token head, IList parameters) { - ParseStatementTwoParam("FILL", parameters, ParseConsumer.OnFillStatement); + ParseStatementTwoParam(head, "FILL", parameters, ParseConsumer.OnFillStatement); } - private void ParseMessageStatement(IList parameters) + private void ParseMessageStatement(Token head, IList parameters) { ImmutableStack scopes = ParseConsumer.CurrentScope; - Logger.Message(head!.Location, PrettyPrintParamsForMessage(parameters, scopes)); + Logger.Message(head.Location, PrettyPrintParamsForMessage(parameters, scopes)); } - private void ParseWarningStatement(IList parameters) + private void ParseWarningStatement(Token head, IList parameters) { ImmutableStack scopes = ParseConsumer.CurrentScope; - Logger.Warning(head!.Location, PrettyPrintParamsForMessage(parameters, scopes)); + Logger.Warning(head.Location, PrettyPrintParamsForMessage(parameters, scopes)); } - private void ParseErrorStatement(IList parameters) + private void ParseErrorStatement(Token head, IList parameters) { ImmutableStack scopes = ParseConsumer.CurrentScope; - Logger.Error(head!.Location, PrettyPrintParamsForMessage(parameters, scopes)); + Logger.Error(head.Location, PrettyPrintParamsForMessage(parameters, scopes)); + } + + private void ParseUtf8Statement(Token head, IList parameters) + { + using MemoryStream memoryStream = new MemoryStream(); + + foreach (IParamNode parameter in parameters) + { + if (parameter is StringNode node) + { + byte[] utf8Bytes = Encoding.UTF8.GetBytes(node.Value); + memoryStream.Write(utf8Bytes, 0, utf8Bytes.Length); + } + else + { + Logger.Error(head.Location, $"expects a String (got {DiagnosticsHelpers.PrettyParamType(parameter.Type)})."); + } + } + + byte[] bytes = memoryStream.ToArray(); + ParseConsumer.OnData(head.Location, bytes); + } + + private void ParseBase64Statement(Token head, IList parameters) + { + using MemoryStream memoryStream = new MemoryStream(); + + foreach (IParamNode parameter in parameters) + { + if (parameter is StringNode node) + { + try + { + byte[] base64Bytes = Convert.FromBase64String(node.Value); + memoryStream.Write(base64Bytes, 0, base64Bytes.Length); + } + catch (FormatException e) + { + Logger.Error(node.MyLocation, $"Failed to parse Base64 string: {e.Message}"); + return; + } + } + else + { + Logger.Error(head.Location, $"expects a String (got {DiagnosticsHelpers.PrettyParamType(parameter.Type)})."); + } + } + + byte[] bytes = memoryStream.ToArray(); + ParseConsumer.OnData(head.Location, bytes); } public IList> ParseMacroParamList(MergeableGenerator tokens) @@ -553,7 +624,7 @@ public void ParseLine(MergeableGenerator tokens, ImmutableStack return; } - head = tokens.Current; + Token head = tokens.Current; switch (head.Type) { case TokenType.IDENTIFIER: @@ -575,7 +646,7 @@ public void ParseLine(MergeableGenerator tokens, ImmutableStack default: // it is somewhat common for users to do '#define Foo 0xABCD' and then later 'Foo:' Logger.Error(head.Location, $"Expansion of macro `{head.Content}` did not result in a valid statement. Did you perhaps attempt to define a label or symbol with that name?"); - IgnoreRestOfLine(tokens); + IgnoreRestOfStatement(tokens); break; } @@ -604,13 +675,13 @@ public void ParseLine(MergeableGenerator tokens, ImmutableStack case TokenType.OPEN_BRACKET: Logger.Error(head.Location, "Unexpected list literal."); - IgnoreRestOfLine(tokens); + IgnoreRestOfStatement(tokens); break; case TokenType.NUMBER: case TokenType.OPEN_PAREN: Logger.Error(head.Location, "Unexpected mathematical expression."); - IgnoreRestOfLine(tokens); + IgnoreRestOfStatement(tokens); break; default: @@ -625,7 +696,7 @@ public void ParseLine(MergeableGenerator tokens, ImmutableStack Logger.Error(head.Location, $"Unexpected token: {head.Type}: {head.Content}."); } - IgnoreRestOfLine(tokens); + IgnoreRestOfStatement(tokens); break; } } @@ -650,15 +721,9 @@ public void ParseLine(MergeableGenerator tokens, ImmutableStack private void ParsePreprocessor(MergeableGenerator tokens, ImmutableStack scopes) { - head = tokens.Current; + Token head = tokens.Current; tokens.MoveNext(); - - ILineNode? result = DirectiveHandler.HandleDirective(this, head, tokens, scopes); - - if (result != null) - { - ParseConsumer.OnPreprocessorData(head.Location, result); - } + DirectiveHandler.HandleDirective(this, head, tokens, scopes); } /*** @@ -737,7 +802,10 @@ private void ApplyMacroExpansion(MergeableGenerator tokens, IEnumerable tokens) { + // TODO: also check for OPEN_BRACE? + while (tokens.Current.Type != TokenType.NEWLINE && tokens.Current.Type != TokenType.SEMICOLON && tokens.MoveNext()) { } + if (tokens.Current.Type == TokenType.SEMICOLON) { tokens.MoveNext(); @@ -746,7 +814,9 @@ public void IgnoreRestOfStatement(MergeableGenerator tokens) public void IgnoreRestOfLine(MergeableGenerator tokens) { - while (tokens.Current.Type != TokenType.NEWLINE && tokens.MoveNext()) { } + while (tokens.Current.Type != TokenType.NEWLINE && tokens.MoveNext()) + { + } } /// diff --git a/ColorzCore/Parser/IParseConsumer.cs b/ColorzCore/Parser/IParseConsumer.cs index 93d5f31..221f2ac 100644 --- a/ColorzCore/Parser/IParseConsumer.cs +++ b/ColorzCore/Parser/IParseConsumer.cs @@ -1,6 +1,7 @@ - +using System.Collections.Generic; using ColorzCore.DataTypes; using ColorzCore.Parser.AST; +using ColorzCore.Raws; namespace ColorzCore.Parser { @@ -8,7 +9,7 @@ public interface IParseConsumer { void OnOpenScope(Location location); void OnCloseScope(Location location); - void OnRawStatement(Location location, RawNode node); + void OnRawStatement(Location location, Raw raw, IList parameters); void OnOrgStatement(Location location, IAtomNode offsetNode); void OnPushStatement(Location location); void OnPopStatement(Location location); @@ -18,6 +19,6 @@ public interface IParseConsumer void OnFillStatement(Location location, IAtomNode amountNode, IAtomNode? valueNode); void OnSymbolAssignment(Location location, string name, IAtomNode atom); void OnLabel(Location location, string name); - void OnPreprocessorData(Location location, ILineNode node); + void OnData(Location location, byte[] data); } } diff --git a/ColorzCore/Preprocessor/DirectiveHandler.cs b/ColorzCore/Preprocessor/DirectiveHandler.cs index 3a4b946..6d4a740 100644 --- a/ColorzCore/Preprocessor/DirectiveHandler.cs +++ b/ColorzCore/Preprocessor/DirectiveHandler.cs @@ -14,7 +14,7 @@ public class DirectiveHandler { public Dictionary Directives { get; } - public DirectiveHandler(IncludeFileSearcher includeSearcher, IncludeFileSearcher toolSearcher) + public DirectiveHandler(IncludeFileSearcher includeSearcher) { // TODO: move out from this directives that need external context // (already done for pool, but could be done for includes as well) @@ -23,9 +23,6 @@ public DirectiveHandler(IncludeFileSearcher includeSearcher, IncludeFileSearcher { { "include", new IncludeDirective { FileSearcher = includeSearcher } }, { "incbin", new IncludeBinaryDirective { FileSearcher = includeSearcher } }, - { "incext", new IncludeExternalDirective { FileSearcher = toolSearcher } }, - { "inctext", new IncludeToolEventDirective { FileSearcher = toolSearcher } }, - { "inctevent", new IncludeToolEventDirective { FileSearcher = toolSearcher } }, { "ifdef", new IfDefinedDirective(false) }, { "ifndef", new IfDefinedDirective(true) }, { "if", new IfDirective() }, @@ -36,7 +33,7 @@ public DirectiveHandler(IncludeFileSearcher includeSearcher, IncludeFileSearcher }; } - public ILineNode? HandleDirective(EAParser p, Token directive, MergeableGenerator tokens, ImmutableStack scopes) + public void HandleDirective(EAParser p, Token directive, MergeableGenerator tokens, ImmutableStack scopes) { string directiveName = directive.Content.Substring(1); @@ -44,15 +41,14 @@ public DirectiveHandler(IncludeFileSearcher includeSearcher, IncludeFileSearcher { if (!toExec.RequireInclusion || p.IsIncluding) { - return toExec.Execute(p, directive, tokens, scopes); + toExec.Execute(p, directive, tokens, scopes); } } else { p.Logger.Error(directive.Location, $"Directive not recognized: {directiveName}"); + p.IgnoreRestOfLine(tokens); } - - return null; } } } diff --git a/ColorzCore/Preprocessor/Directives/BaseIncludeDirective.cs b/ColorzCore/Preprocessor/Directives/BaseIncludeDirective.cs new file mode 100644 index 0000000..7afdb96 --- /dev/null +++ b/ColorzCore/Preprocessor/Directives/BaseIncludeDirective.cs @@ -0,0 +1,55 @@ + +using System.Collections.Generic; +using System.IO; +using ColorzCore.DataTypes; +using ColorzCore.IO; +using ColorzCore.Lexer; +using ColorzCore.Parser; +using ColorzCore.Parser.AST; + +namespace ColorzCore.Preprocessor.Directives +{ + public abstract class BaseIncludeDirective : SimpleDirective + { + public override int MinParams => 1; + + public override int? MaxParams => 1; + + public override bool RequireInclusion => true; + + public IncludeFileSearcher FileSearcher { get; set; } = new IncludeFileSearcher(); + + public override void Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) + { + string pathExpression = parameters[0].ToString()!; + + if (EAOptions.TranslateBackslashesInPaths) + { + pathExpression = pathExpression.Replace('\\', '/'); + } + + string? existantFile = FileSearcher.FindFile(Path.GetDirectoryName(self.FileName), pathExpression); + + if (existantFile != null) + { + if (EAOptions.IsWarningEnabled(EAOptions.Warnings.NonPortablePath)) + { + string portablePathExpression = IOUtility.GetPortablePathExpression(existantFile, pathExpression); + + if (pathExpression != portablePathExpression) + { + p.Logger.Warning(self.Location, $"Path is not portable (should be \"{portablePathExpression}\")."); + } + } + + HandleInclude(p, self, existantFile, tokens); + } + else + { + p.Logger.Error(parameters[0].MyLocation, $"Could not find file \"{pathExpression}\"."); + } + } + + public abstract void HandleInclude(EAParser p, Token self, string path, MergeableGenerator tokens); + } +} diff --git a/ColorzCore/Preprocessor/Directives/DefineDirective.cs b/ColorzCore/Preprocessor/Directives/DefineDirective.cs index 21353dd..2584e2d 100644 --- a/ColorzCore/Preprocessor/Directives/DefineDirective.cs +++ b/ColorzCore/Preprocessor/Directives/DefineDirective.cs @@ -14,7 +14,7 @@ class DefineDirective : IDirective { public bool RequireInclusion => true; - public ILineNode? Execute(EAParser p, Token self, MergeableGenerator tokens, ImmutableStack scopes) + public void Execute(EAParser p, Token self, MergeableGenerator tokens, ImmutableStack scopes) { Token nextToken = tokens.Current; IList? parameters; @@ -33,12 +33,12 @@ class DefineDirective : IDirective case TokenType.NEWLINE: p.Logger.Error(self.Location, "Invalid use of directive '#define': missing macro name."); - return null; + return; default: p.Logger.Error(self.Location, $"Invalid use of directive '#define': expected macro name, got {nextToken}"); p.IgnoreRestOfLine(tokens); - return null; + return; } IList? macroBody = ExpandMacroBody(p, p.GetRestOfLine(tokens, scopes)); @@ -53,8 +53,6 @@ class DefineDirective : IDirective // object-like macro DefineObjectMacro(p, nextToken, macroBody); } - - return null; } private static void DefineObjectMacro(EAParser p, Token nameToken, IList macroBody) diff --git a/ColorzCore/Preprocessor/Directives/ElseDirective.cs b/ColorzCore/Preprocessor/Directives/ElseDirective.cs index 8948d00..1d03012 100644 --- a/ColorzCore/Preprocessor/Directives/ElseDirective.cs +++ b/ColorzCore/Preprocessor/Directives/ElseDirective.cs @@ -17,13 +17,16 @@ class ElseDirective : SimpleDirective public override bool RequireInclusion => false; - public override ILineNode? Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) + public override void Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) { if (p.Inclusion.IsEmpty) + { p.Logger.Error(self.Location, "No matching conditional (if, ifdef, ifndef)."); + } else + { p.Inclusion = new ImmutableStack(!p.Inclusion.Head, p.Inclusion.Tail); - return null; + } } } } diff --git a/ColorzCore/Preprocessor/Directives/EndIfDirective.cs b/ColorzCore/Preprocessor/Directives/EndIfDirective.cs index 37afef6..84ec8f9 100644 --- a/ColorzCore/Preprocessor/Directives/EndIfDirective.cs +++ b/ColorzCore/Preprocessor/Directives/EndIfDirective.cs @@ -17,14 +17,16 @@ class EndIfDirective : SimpleDirective public override bool RequireInclusion => false; - public override ILineNode? Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) + public override void Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) { if (p.Inclusion.IsEmpty) + { p.Logger.Error(self.Location, "No matching conditional (if, ifdef, ifndef)."); + } else + { p.Inclusion = p.Inclusion.Tail; - - return null; + } } } } diff --git a/ColorzCore/Preprocessor/Directives/IDirective.cs b/ColorzCore/Preprocessor/Directives/IDirective.cs index 742e2f6..de0eb36 100644 --- a/ColorzCore/Preprocessor/Directives/IDirective.cs +++ b/ColorzCore/Preprocessor/Directives/IDirective.cs @@ -1,7 +1,6 @@ using ColorzCore.DataTypes; using ColorzCore.Lexer; using ColorzCore.Parser; -using ColorzCore.Parser.AST; using System; using System.Collections.Generic; using System.Linq; @@ -16,7 +15,7 @@ public interface IDirective * * Return: If a string is returned, it is interpreted as an error. */ - ILineNode? Execute(EAParser p, Token self, MergeableGenerator tokens, ImmutableStack scopes); + void Execute(EAParser p, Token self, MergeableGenerator tokens, ImmutableStack scopes); /*** * Whether requires the parser to be taking in tokens. diff --git a/ColorzCore/Preprocessor/Directives/IfDefinedDirective.cs b/ColorzCore/Preprocessor/Directives/IfDefinedDirective.cs index 82a2143..a917668 100644 --- a/ColorzCore/Preprocessor/Directives/IfDefinedDirective.cs +++ b/ColorzCore/Preprocessor/Directives/IfDefinedDirective.cs @@ -24,12 +24,14 @@ public IfDefinedDirective(bool invert) Inverted = invert; } - public override ILineNode? Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) + public override void Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) { bool flag = true; - string? identifier; + foreach (IParamNode parameter in parameters) { + string? identifier; + if (parameter.Type == ParamType.ATOM && (identifier = ((IAtomNode)parameter).GetIdentifier()) != null) { // TODO: Built in definitions? @@ -41,8 +43,8 @@ public IfDefinedDirective(bool invert) p.Logger.Error(parameter.MyLocation, "Definition name must be an identifier."); } } + p.Inclusion = new ImmutableStack(flag, p.Inclusion); - return null; } } } diff --git a/ColorzCore/Preprocessor/Directives/IfDirective.cs b/ColorzCore/Preprocessor/Directives/IfDirective.cs index 0256f1c..c864f6a 100644 --- a/ColorzCore/Preprocessor/Directives/IfDirective.cs +++ b/ColorzCore/Preprocessor/Directives/IfDirective.cs @@ -17,7 +17,7 @@ class IfDirective : SimpleDirective public override bool RequireInclusion => false; - public override ILineNode? Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) + public override void Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) { bool flag = true; @@ -37,7 +37,6 @@ class IfDirective : SimpleDirective } p.Inclusion = new ImmutableStack(flag, p.Inclusion); - return null; } } } diff --git a/ColorzCore/Preprocessor/Directives/IncludeBinaryDirective.cs b/ColorzCore/Preprocessor/Directives/IncludeBinaryDirective.cs index e13791c..2c5b5f1 100644 --- a/ColorzCore/Preprocessor/Directives/IncludeBinaryDirective.cs +++ b/ColorzCore/Preprocessor/Directives/IncludeBinaryDirective.cs @@ -10,54 +10,19 @@ namespace ColorzCore.Preprocessor.Directives { - class IncludeBinaryDirective : SimpleDirective + public class IncludeBinaryDirective : BaseIncludeDirective { - public override int MinParams => 1; - - public override int? MaxParams => 1; - - public override bool RequireInclusion => true; - - public IncludeFileSearcher FileSearcher { get; set; } = new IncludeFileSearcher(); - - public override ILineNode? Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) + public override void HandleInclude(EAParser p, Token self, string path, MergeableGenerator _) { - string pathExpression = parameters[0].ToString()!; - - if (EAOptions.TranslateBackslashesInPaths) - { - pathExpression = pathExpression.Replace('\\', '/'); - } - - string? existantFile = FileSearcher.FindFile(Path.GetDirectoryName(self.FileName), pathExpression); - - if (existantFile != null) + try { - if (EAOptions.IsWarningEnabled(EAOptions.Warnings.NonPortablePath)) - { - string portablePathExpression = IOUtility.GetPortablePathExpression(existantFile, pathExpression); - - if (pathExpression != portablePathExpression) - { - p.Logger.Warning(self.Location, $"Path is not portable (should be \"{portablePathExpression}\")."); - } - } - - try - { - return new DataNode(p.ParseConsumer.CurrentOffset, File.ReadAllBytes(existantFile)); - } - catch (Exception) - { - p.Logger.Error(self.Location, "Error reading file \"" + parameters[0].ToString() + "\"."); - } + byte[] data = File.ReadAllBytes(path); + p.ParseConsumer.OnData(self.Location, data); } - else + catch (IOException e) { - p.Logger.Error(parameters[0].MyLocation, "Could not find file \"" + parameters[0].ToString() + "\"."); + p.Logger.Error(self.Location, $"Error reading file \"{path}\": {e.Message}."); } - - return null; } } } diff --git a/ColorzCore/Preprocessor/Directives/IncludeDirective.cs b/ColorzCore/Preprocessor/Directives/IncludeDirective.cs index 1fb424a..3b4f653 100644 --- a/ColorzCore/Preprocessor/Directives/IncludeDirective.cs +++ b/ColorzCore/Preprocessor/Directives/IncludeDirective.cs @@ -10,57 +10,19 @@ namespace ColorzCore.Preprocessor.Directives { - class IncludeDirective : SimpleDirective + class IncludeDirective : BaseIncludeDirective { - // TODO: merge include and incbin into a shared base class - - public override int MinParams => 1; - - public override int? MaxParams => 1; - - public override bool RequireInclusion => true; - - public IncludeFileSearcher FileSearcher { get; set; } = new IncludeFileSearcher(); - - public override ILineNode? Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) + public override void HandleInclude(EAParser p, Token self, string path, MergeableGenerator tokens) { - string pathExpression = parameters[0].ToString()!; - - if (EAOptions.TranslateBackslashesInPaths) + try { - pathExpression = pathExpression.Replace('\\', '/'); + FileStream inputFile = new FileStream(path, FileMode.Open); + tokens.PrependEnumerator(new Tokenizer().TokenizeFile(inputFile, path.Replace('\\', '/')).GetEnumerator()); } - - string? existantFile = FileSearcher.FindFile(Path.GetDirectoryName(self.FileName), pathExpression); - - if (existantFile != null) - { - if (EAOptions.IsWarningEnabled(EAOptions.Warnings.NonPortablePath)) - { - string portablePathExpression = IOUtility.GetPortablePathExpression(existantFile, pathExpression); - - if (pathExpression != portablePathExpression) - { - p.Logger.Warning(self.Location, $"Path is not portable (should be \"{portablePathExpression}\")."); - } - } - - try - { - FileStream inputFile = new FileStream(existantFile, FileMode.Open); - tokens.PrependEnumerator(new Tokenizer().TokenizeFile(inputFile, existantFile.Replace('\\', '/')).GetEnumerator()); - } - catch (Exception e) - { - p.Logger.Error(self.Location, $"Error reading file \"{parameters[0].ToString()}\": {e.Message}."); - } - } - else + catch (IOException e) { - p.Logger.Error(parameters[0].MyLocation, $"Could not find file \"{parameters[0].ToString()}\"."); + p.Logger.Error(self.Location, $"Error reading file \"{path}\": {e.Message}."); } - - return null; } } } diff --git a/ColorzCore/Preprocessor/Directives/IncludeExternalDirective.cs b/ColorzCore/Preprocessor/Directives/IncludeExternalDirective.cs index 11272f9..f5469ef 100644 --- a/ColorzCore/Preprocessor/Directives/IncludeExternalDirective.cs +++ b/ColorzCore/Preprocessor/Directives/IncludeExternalDirective.cs @@ -19,7 +19,7 @@ class IncludeExternalDirective : SimpleDirective public IncludeFileSearcher FileSearcher { get; set; } = new IncludeFileSearcher(); - public override ILineNode? Execute(EAParser parse, Token self, IList parameters, MergeableGenerator tokens) + public override void Execute(EAParser parse, Token self, IList parameters, MergeableGenerator tokens) { ExecTimer.Timer.AddTimingPoint(ExecTimer.KEY_GENERIC); @@ -28,7 +28,7 @@ class IncludeExternalDirective : SimpleDirective if (validFile == null) { parse.Logger.Error(parameters[0].MyLocation, "Tool " + parameters[0].ToString() + " not found."); - return null; + return; } //TODO: abstract out all this running stuff into a method so I don't have code duplication with inctext @@ -77,7 +77,7 @@ class IncludeExternalDirective : SimpleDirective ExecTimer.Timer.AddTimingPoint(parameters[0].ToString()!.ToLower()); - return new DataNode(parse.ParseConsumer.CurrentOffset, output); + parse.ParseConsumer.OnData(self.Location, output); } } } diff --git a/ColorzCore/Preprocessor/Directives/IncludeToolEventDirective.cs b/ColorzCore/Preprocessor/Directives/IncludeToolEventDirective.cs index 0d98187..7ea7e0f 100644 --- a/ColorzCore/Preprocessor/Directives/IncludeToolEventDirective.cs +++ b/ColorzCore/Preprocessor/Directives/IncludeToolEventDirective.cs @@ -19,7 +19,7 @@ class IncludeToolEventDirective : SimpleDirective public IncludeFileSearcher FileSearcher { get; set; } = new IncludeFileSearcher(); - public override ILineNode? Execute(EAParser parse, Token self, IList parameters, MergeableGenerator tokens) + public override void Execute(EAParser parse, Token self, IList parameters, MergeableGenerator tokens) { ExecTimer.Timer.AddTimingPoint(ExecTimer.KEY_GENERIC); @@ -28,7 +28,7 @@ class IncludeToolEventDirective : SimpleDirective if (validFile == null) { parse.Logger.Error(parameters[0].MyLocation, "Tool " + parameters[0].ToString() + " not found."); - return null; + return; } //from http://stackoverflow.com/a/206347/1644720 @@ -78,8 +78,6 @@ class IncludeToolEventDirective : SimpleDirective } ExecTimer.Timer.AddTimingPoint(parameters[0].ToString()!.ToLower()); - - return null; } } } diff --git a/ColorzCore/Preprocessor/Directives/PoolDirective.cs b/ColorzCore/Preprocessor/Directives/PoolDirective.cs index ad5a4b0..194d9b4 100644 --- a/ColorzCore/Preprocessor/Directives/PoolDirective.cs +++ b/ColorzCore/Preprocessor/Directives/PoolDirective.cs @@ -20,7 +20,7 @@ public PoolDirective(Pool pool) this.pool = pool; } - public override ILineNode? Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) + public override void Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) { // Iterating indices (and not values via foreach) // to avoid crashes occuring with AddToPool within AddToPool @@ -39,7 +39,6 @@ public PoolDirective(Pool pool) } pool.Lines.Clear(); - return null; } } } diff --git a/ColorzCore/Preprocessor/Directives/SimpleDirective.cs b/ColorzCore/Preprocessor/Directives/SimpleDirective.cs index 2b591bc..1930349 100644 --- a/ColorzCore/Preprocessor/Directives/SimpleDirective.cs +++ b/ColorzCore/Preprocessor/Directives/SimpleDirective.cs @@ -26,7 +26,7 @@ public abstract class SimpleDirective : IDirective */ public abstract int? MaxParams { get; } - public ILineNode? Execute(EAParser p, Token self, MergeableGenerator tokens, ImmutableStack scopes) + public void Execute(EAParser p, Token self, MergeableGenerator tokens, ImmutableStack scopes) { // Note: Not a ParseParamList because no commas. // HACK: #if wants its parameters to be expanded, but other directives (define, ifdef, undef, etc) do not @@ -34,15 +34,14 @@ public abstract class SimpleDirective : IDirective if (MinParams <= parameters.Count && (!MaxParams.HasValue || parameters.Count <= MaxParams)) { - return Execute(p, self, parameters, tokens); + Execute(p, self, parameters, tokens); } else { - p.Logger.Error(self.Location, $"Invalid number of parameters ({parameters.Count}) to directive {self}."); - return null; + p.Logger.Error(self.Location, $"Invalid number of parameters ({parameters.Count}) to directive {self.Content}."); } } - public abstract ILineNode? Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens); + public abstract void Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens); } } diff --git a/ColorzCore/Preprocessor/Directives/UndefineDirective.cs b/ColorzCore/Preprocessor/Directives/UndefineDirective.cs index 4600b97..544127b 100644 --- a/ColorzCore/Preprocessor/Directives/UndefineDirective.cs +++ b/ColorzCore/Preprocessor/Directives/UndefineDirective.cs @@ -9,25 +9,44 @@ namespace ColorzCore.Preprocessor.Directives { - class UndefineDirective : SimpleDirective + class UndefineDirective : IDirective { - public override int MinParams => 1; + public bool RequireInclusion => true; - public override int? MaxParams => null; + public void Execute(EAParser p, Token self, MergeableGenerator tokens, ImmutableStack scopes) + { + if (tokens.Current.Type == TokenType.NEWLINE) + { + p.Logger.Error(self.Location, $"Invalid use of directive '{self.Content}': expected at least one macro name."); + } + + while (tokens.Current.Type != TokenType.NEWLINE) + { + Token current = tokens.Current; + tokens.MoveNext(); - public override bool RequireInclusion => true; + switch (current.Type) + { + case TokenType.IDENTIFIER: + ApplyUndefine(p, current); + break; - public override ILineNode? Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) + default: + p.Logger.Error(self.Location, $"Invalid use of directive '{self.Content}': expected macro name, got {current}."); + p.IgnoreRestOfLine(tokens); + return; + } + } + } + + private static void ApplyUndefine(EAParser parser, Token token) { - foreach (IParamNode parm in parameters) + string name = token.Content; + + if (!parser.Definitions.Remove(name)) { - string s = parm.ToString()!; - if (p.Definitions.ContainsKey(s)) - p.Definitions.Remove(s); - else - p.Logger.Warning(parm.MyLocation, "Undefining non-existant definition: " + s); + parser.Logger.Warning(token.Location, $"Attempted to purge non existant definition '{name}'"); } - return null; } } } From 8181f3fbe02bb76ae4131a95659e5ef06478e5e1 Mon Sep 17 00:00:00 2001 From: Nat_776 Date: Tue, 30 Apr 2024 21:27:02 +0200 Subject: [PATCH 45/59] add more tests + fix PROTECT 0 --- ColorzCore/Parser/EAParseConsumer.cs | 12 +- Tests/ea_test.py | 84 +++++++++++++ Tests/run_tests.py | 162 +++++-------------------- Tests/statements.py | 169 +++++++++++++++++++++++++++ 4 files changed, 291 insertions(+), 136 deletions(-) create mode 100644 Tests/ea_test.py create mode 100644 Tests/statements.py diff --git a/ColorzCore/Parser/EAParseConsumer.cs b/ColorzCore/Parser/EAParseConsumer.cs index 11b8b21..f4bf598 100644 --- a/ColorzCore/Parser/EAParseConsumer.cs +++ b/ColorzCore/Parser/EAParseConsumer.cs @@ -216,7 +216,7 @@ public void OnProtectStatement(Location location, IAtomNode beginAtom, IAtomNode { if (EvaluteAtom(beginAtom) is int beginValue) { - beginValue = ConvertToAddress(beginValue); + beginValue = ConvertToOffset(beginValue); int length = 4; @@ -224,7 +224,7 @@ public void OnProtectStatement(Location location, IAtomNode beginAtom, IAtomNode { if (EvaluteAtom(endAtom) is int endValue) { - endValue = ConvertToAddress(endValue); + endValue = ConvertToOffset(endValue); length = endValue - beginValue; @@ -243,9 +243,11 @@ public void OnProtectStatement(Location location, IAtomNode beginAtom, IAtomNode else { // EvaluateAtom already printed an error message + return; } } + Logger.Message(location, $"{beginValue} {length}"); protectedRegions.Add((beginValue, length, location)); } else @@ -418,14 +420,14 @@ public void TryDefineSymbol(Location location, string name, IAtomNode expression // Return value: Location where protection occurred. Nothing if location was not protected. public Location? IsProtected(int offset, int length) { - int address = ConvertToAddress(offset); + offset = ConvertToOffset(offset); - foreach ((int protectedAddress, int protectedLength, Location location) in protectedRegions) + foreach ((int protectedOffset, int protectedLength, Location location) in protectedRegions) { /* They intersect if the last offset in the given region is after the start of this one * and the first offset in the given region is before the last of this one. */ - if (address + length > protectedAddress && address < protectedAddress + protectedLength) + if (offset + length > protectedOffset && offset < protectedOffset + protectedLength) { return location; } diff --git a/Tests/ea_test.py b/Tests/ea_test.py new file mode 100644 index 0000000..9ec480a --- /dev/null +++ b/Tests/ea_test.py @@ -0,0 +1,84 @@ +import os, subprocess, tempfile + + +class EATestConfig: + command : list[str] + extra_params : list[str] + + def __init__(self, command : str, extra_params : str | None) -> None: + self.command = command.split() + self.extra_params = extra_params.split() if extra_params is not None else [] + + +class EATest: + name : str + script : str + expected : bytes | None + + def __init__(self, name : str, script : str, expected : bytes | None) -> None: + self.name = name + self.script = script + self.expected = expected + + def run_test(self, config : EATestConfig) -> bool: + success = False + + # TODO: this could be better than just "success/failure" + # Failure here can be two causes: result doesn't match OR program crashed. + + with tempfile.NamedTemporaryFile(delete = False) as f: + f.close() + + completed = subprocess.run(config.command + ["A", "FE6", f"-output:{f.name}"] + config.extra_params, + text = True, input = self.script, stdout = subprocess.DEVNULL, stderr = subprocess.PIPE) + + if self.expected is None: + # success on error + success = completed.returncode != 0 and "Errors occurred; no changes written." in completed.stderr + + else: + # success on resulting bytes matching + with open(f.name, 'rb') as f2: + result_bytes = f2.read() + + success = result_bytes == self.expected + + os.remove(f.name) + + return success + + +HEADER = '\033[95m' +CC_OKBLUE = '\033[94m' +CC_OKCYAN = '\033[96m' +CC_OKGREEN = '\033[92m' +CC_WARNING = '\033[93m' +CC_FAIL = '\033[91m' +CC_ENDC = '\033[0m' +CC_BOLD = '\033[1m' +CC_UNDERLINE = '\033[4m' + + +SUCCESS_MESSAGE = f"{CC_OKBLUE}SUCCESS{CC_ENDC}" +FAILURE_MESSAGE = f"{CC_FAIL}FAILURE{CC_ENDC}" + + +def run_tests(config : EATestConfig, test_cases : list[EATest]) -> None: + success_count = 0 + test_count = len(test_cases) + + for i, test_case in enumerate(test_cases): + success = test_case.run_test(config) + + message = SUCCESS_MESSAGE if success else FAILURE_MESSAGE + print(f"[{i + 1}/{test_count}] {test_case.name}: {message}") + + if success: + success_count = success_count + 1 + + if success_count == test_count: + print(f"{success_count}/{test_count} tests passed {SUCCESS_MESSAGE}") + + else: + print(f"{success_count}/{test_count} tests passed {FAILURE_MESSAGE}") + diff --git a/Tests/run_tests.py b/Tests/run_tests.py index 9bfebfa..3b4f91a 100644 --- a/Tests/run_tests.py +++ b/Tests/run_tests.py @@ -1,167 +1,67 @@ -import sys, os -import subprocess, tempfile - - -class Config: - command : list[str] - extra_params : list[str] - - def __init__(self, command : str, extra_params : str | None) -> None: - self.command = command.split() - self.extra_params = extra_params.split() if extra_params is not None else [] - - -class Test: - name : str - script : str - expected : bytes | None - - def __init__(self, name : str, script : str, expected : bytes | None) -> None: - self.name = name - self.script = script - self.expected = expected - - def run_test(self, config : Config) -> bool: - success = False - - with tempfile.NamedTemporaryFile(delete = False) as f: - f.close() - - completed = subprocess.run(config.command + ["A", "FE6", f"-output:{f.name}"] + config.extra_params, - text = True, input = self.script, stdout = subprocess.DEVNULL, stderr = subprocess.DEVNULL) - - if self.expected is None: - # success on error - success = completed.returncode != 0 - - else: - # success on resulting bytes matching - with open(f.name, 'rb') as f2: - result_bytes = f2.read() - - success = result_bytes == self.expected - - os.remove(f.name) - - return success - - -class bcolors: - HEADER = '\033[95m' - OKBLUE = '\033[94m' - OKCYAN = '\033[96m' - OKGREEN = '\033[92m' - WARNING = '\033[93m' - FAIL = '\033[91m' - ENDC = '\033[0m' - BOLD = '\033[1m' - UNDERLINE = '\033[4m' - - -def run_tests(config : Config, test_cases : list[Test]) -> None: - success_count = 0 - test_count = len(test_cases) - - success_message = f"{bcolors.OKBLUE}SUCCESS{bcolors.ENDC}" - failure_message = f"{bcolors.FAIL}FAILURE{bcolors.ENDC}" - - for i, test_case in enumerate(test_cases): - success = test_case.run_test(config) - - message = success_message if success else failure_message - print(f"[{i + 1}/{test_count}] {test_case.name}: {message}") - - if success: - success_count = success_count + 1 - - if success_count == test_count: - print(f"{success_count}/{test_count} tests passed {success_message}") - - else: - print(f"{success_count}/{test_count} tests passed {failure_message}") - +import sys +from ea_test import EATestConfig as Config, EATest as T, run_tests BASIC_TESTS = [ - Test("Basic", "ORG 0 ; BYTE 1", b"\x01"), - Test("Addition", "ORG 0 ; BYTE 1 + 2", b"\x03"), - Test("Precedence 1", "ORG 0 ; BYTE 1 + 2 * 10", b"\x15"), + T("Basic", "ORG 0 ; BYTE 1", b"\x01"), + T("Addition", "ORG 0 ; BYTE 1 + 2", b"\x03"), + T("Precedence 1", "ORG 0 ; BYTE 1 + 2 * 10", b"\x15"), # POIN - Test("POIN 1", "ORG 0 ; POIN 4", b"\x04\x00\x00\x08"), - Test("POIN 2", "ORG 0 ; POIN 0", b"\x00\x00\x00\x00"), - Test("POIN 3", "ORG 0 ; POIN 0x08000000", b"\x00\x00\x00\x08"), - Test("POIN 4", "ORG 0 ; POIN 0x02000000", b"\x00\x00\x00\x02"), - - # ORG - Test("ORG 1", "ORG 1 ; BYTE 1 ; ORG 10 ; BYTE 10", b"\x00\x01" + b"\x00" * 8 + b"\x0A"), - Test("ORG 2", "ORG 0x08000001 ; BYTE 1 ; ORG 0x0800000A ; BYTE 10", b"\x00\x01" + b"\x00" * 8 + b"\x0A"), - Test("ORG 3", "ORG 0x10000000 ; BYTE 1", None), - Test("ORG 4", "ORG -1 ; BYTE 1", None), - - # ALIGN - Test("ALIGN 1", "ORG 1 ; ALIGN 4 ; WORD CURRENTOFFSET", b"\x00\x00\x00\x00\x04\x00\x00\x00"), - Test("ALIGN 2", "ORG 4 ; ALIGN 4 ; WORD CURRENTOFFSET", b"\x00\x00\x00\x00\x04\x00\x00\x00"), - Test("ALIGN 3", "ORG 1 ; ALIGN 0 ; WORD CURRENTOFFSET", None), - Test("ALIGN 4", "ORG 1 ; ALIGN -1 ; WORD CURRENTOFFSET", None), - - # FILL - Test("FILL 1", "ORG 0 ; FILL 0x10", b"\x00" * 0x10), - Test("FILL 2", "ORG 4 ; FILL 0x10 0xFF", b"\x00\x00\x00\x00" + b"\xFF" * 0x10), - - # ASSERT - Test("ASSERT 1", "ASSERT 0", b""), - Test("ASSERT 2", "ASSERT -1", None), - Test("ASSERT 3", "ASSERT 1 < 0", None), - Test("ASSERT 4", "ASSERT 1 - 2", None) + T("POIN 1", "ORG 0 ; POIN 4", b"\x04\x00\x00\x08"), + T("POIN 2", "ORG 0 ; POIN 0", b"\x00\x00\x00\x00"), + T("POIN 3", "ORG 0 ; POIN 0x08000000", b"\x00\x00\x00\x08"), + T("POIN 4", "ORG 0 ; POIN 0x02000000", b"\x00\x00\x00\x02"), ] EXPRESSION_TESTS = [ - Test("UNDCOERCE 1", 'A := 0 ; ORG 0 ; BYTE (A || 1) ?? 0', b"\x01"), - Test("UNDCOERCE 2", 'ORG 0 ; BYTE (A || 1) ?? 0', b"\x00"), + T("UNDCOERCE 1", 'A := 0 ; ORG 0 ; BYTE (A || 1) ?? 0', b"\x01"), + T("UNDCOERCE 2", 'ORG 0 ; BYTE (A || 1) ?? 0', b"\x00"), ] PREPROC_TESTS = [ # '#define' traditional nominal behavior - Test("Define 1", '#define Value 0xFA \n ORG 0 ; BYTE Value', b"\xFA"), - Test("Define 2", '#define Macro(a) "0xFA + (a)" \n ORG 0 ; BYTE Macro(2)', b"\xFC"), - Test("Define 3", '#define Value \n #ifdef Value \n ORG 0 ; BYTE 1 \n #endif', b"\x01"), + T("Define 1", '#define Value 0xFA \n ORG 0 ; BYTE Value', b"\xFA"), + T("Define 2", '#define Macro(a) "0xFA + (a)" \n ORG 0 ; BYTE Macro(2)', b"\xFC"), + T("Define 3", '#define Value \n #ifdef Value \n ORG 0 ; BYTE 1 \n #endif', b"\x01"), # '#define' a second time overrides the first definition - Test("Define override", '#define Value 1 \n #define Value 2 \n ORG 0 ; BYTE Value', b"\x02"), + T("Define override", '#define Value 1 \n #define Value 2 \n ORG 0 ; BYTE Value', b"\x02"), # '#define' using a vector as argument (extra commas) - Test("Define vector argument", '#define Macro(a) "BYTE 1" \n ORG 0 ; Macro([1, 2, 3])', b"\x01"), + T("Define vector argument", '#define Macro(a) "BYTE 1" \n ORG 0 ; Macro([1, 2, 3])', b"\x01"), # '#define ... "..."' with escaped newlines inside string - Test("Multi-line string define", '#define SomeLongMacro(A, B, C) "\\\n ALIGN 4 ; \\\n WORD C ; \\\n SHORT B ; \\\n BYTE A" \n ORG 0 ; SomeLongMacro(0xAA, 0xBB, 0xCC)', b"\xCC\x00\x00\x00\xBB\x00\xAA"), + T("Multi-line string define", '#define SomeLongMacro(A, B, C) "\\\n ALIGN 4 ; \\\n WORD C ; \\\n SHORT B ; \\\n BYTE A" \n ORG 0 ; SomeLongMacro(0xAA, 0xBB, 0xCC)', b"\xCC\x00\x00\x00\xBB\x00\xAA"), # '#define ...' multi-token without quotes - Test("Multi-token define 1", '#define Value (1 + 2) \n ORG 0 ; BYTE Value', b"\x03"), - Test("Multi-token define 2", '#define Macro(a, b) (a + b) \n ORG 0 ; BYTE Macro(1, 2)', b"\x03"), - Test("Multi-token define 2", '#define Macro(a, b) BYTE a a + b b \n ORG 0 ; Macro(1, 2)', b"\x01\x03\x02"), + T("Multi-token define 1", '#define Value (1 + 2) \n ORG 0 ; BYTE Value', b"\x03"), + T("Multi-token define 2", '#define Macro(a, b) (a + b) \n ORG 0 ; BYTE Macro(1, 2)', b"\x03"), + T("Multi-token define 2", '#define Macro(a, b) BYTE a a + b b \n ORG 0 ; Macro(1, 2)', b"\x01\x03\x02"), # '#ifdef' - Test("Ifdef", 'ORG 0 \n #define Value \n #ifdef Value \n BYTE 1 \n #else \n BYTE 0 \n #endif', b"\x01"), + T("Ifdef", 'ORG 0 \n #define Value \n #ifdef Value \n BYTE 1 \n #else \n BYTE 0 \n #endif', b"\x01"), # '#ifndef' - Test("Ifndef", 'ORG 0 \n #define Value \n #ifndef Value \n BYTE 1 \n #else \n BYTE 0 \n #endif', b"\x00"), + T("Ifndef", 'ORG 0 \n #define Value \n #ifndef Value \n BYTE 1 \n #else \n BYTE 0 \n #endif', b"\x00"), # '#define MyMacro MyMacro' (MyMacro shouldn't expand) - Test("Non-productive macros 1", '#define MyMacro MyMacro \n ORG 0 ; MyMacro: ; BYTE 1', b'\x01'), - Test("Non-productive macros 2", '#define MyMacro MyMacro \n ORG 0 ; BYTE IsDefined(MyMacro)', b'\x01'), - Test("Non-productive macros 3", '#define MyMacro MyMacro \n ORG 0 ; #ifdef MyMacro \n BYTE 1 \n #else \n BYTE 0 \n #endif', b'\x01'), + T("Non-productive macros 1", '#define MyMacro MyMacro \n ORG 0 ; MyMacro: ; BYTE 1', b'\x01'), + T("Non-productive macros 2", '#define MyMacro MyMacro \n ORG 0 ; BYTE IsDefined(MyMacro)', b'\x01'), + T("Non-productive macros 3", '#define MyMacro MyMacro \n ORG 0 ; #ifdef MyMacro \n BYTE 1 \n #else \n BYTE 0 \n #endif', b'\x01'), - # Test("IFDEF 2", 'ORG 0 \n #define A \n #define B \n #ifdef A B \n BYTE 1 \n #else \n BYTE 0 \n #endif', b"\x01"), + # T("IFDEF 2", 'ORG 0 \n #define A \n #define B \n #ifdef A B \n BYTE 1 \n #else \n BYTE 0 \n #endif', b"\x01"), # '#undef' - Test("Undef 1", '#define Value 1 \n #undef Value \n ORG 0 ; BYTE Value', None), - Test("Undef 2", '#define Value 1 \n #undef Value \n #ifndef Value \n ORG 0 ; BYTE 1 \n #endif', b"\x01"), + T("Undef 1", '#define Value 1 \n #undef Value \n ORG 0 ; BYTE Value', None), + T("Undef 2", '#define Value 1 \n #undef Value \n #ifndef Value \n ORG 0 ; BYTE 1 \n #endif', b"\x01"), ] -ALL_TEST_CASES = BASIC_TESTS + EXPRESSION_TESTS + PREPROC_TESTS +import statements + +ALL_TEST_CASES = BASIC_TESTS + statements.TESTS + EXPRESSION_TESTS + PREPROC_TESTS def main(args): import argparse diff --git a/Tests/statements.py b/Tests/statements.py new file mode 100644 index 0000000..b5f7d82 --- /dev/null +++ b/Tests/statements.py @@ -0,0 +1,169 @@ +from ea_test import EATest as T + + +TESTS = [ + # ================= + # = ORG Statement = + # ================= + + # Nominal behavior + T("ORG Basic", + "ORG 1 ; BYTE 1 ; ORG 10 ; BYTE 10", + b"\x00\x01" + b"\x00" * 8 + b"\x0A"), + + # Also works backwards + T("ORG Backwards", + "ORG 10 ; BYTE 10 ; ORG 1 ; BYTE 1", + b"\x00\x01" + b"\x00" * 8 + b"\x0A"), + + # Addresses are Offsets + T("ORG Addresses", + "ORG 0x08000001 ; BYTE 1 ; ORG 0x0800000A ; BYTE 10", + b"\x00\x01" + b"\x00" * 8 + b"\x0A"), + + # Error on offset too big + T("ORG Overflow", + "ORG 0x10000000 ; BYTE 1", + None), + + # Error on offset too small/negative + T("ORG Underflow", + "ORG -1 ; BYTE 1", + None), + + # ======================= + # = PUSH/POP Statements = + # ======================= + + # Nominal behavior + T("PUSH POP Basic", + "ORG 4 ; PUSH ; ORG 1 ; POP ; BYTE CURRENTOFFSET", + b"\x00\x00\x00\x00\x04"), + + T("PUSH POP Override", + "ORG 0 ; PUSH ; BYTE 0xAA ; POP ; BYTE 0xBB", + b"\xBB"), + + T("POP Naked", + "ORG 0 ; BYTE 0 ; POP", + None), + + # =================== + # = ALIGN Statement = + # =================== + + T("ALIGN Basic", + "ORG 1 ; ALIGN 4 ; BYTE CURRENTOFFSET", + b"\x00\x00\x00\x00\x04"), + + T("ALIGN Aligned", + "ORG 4 ; ALIGN 4 ; BYTE CURRENTOFFSET", + b"\x00\x00\x00\x00\x04"), + + T("ALIGN Zero", + "ORG 1 ; ALIGN 0 ; BYTE CURRENTOFFSET", + None), + + T("ALIGN Negative", + "ORG 1 ; ALIGN -1 ; BYTE CURRENTOFFSET", + None), + + T("ALIGN Offset", + "ORG 2 ; ALIGN 4 1 ; BYTE CURRENTOFFSET", + b"\x00\x00\x00\x00\x00\x05"), + + T("ALIGN Offset Aligned", + "ORG 1 ; ALIGN 4 1 ; BYTE CURRENTOFFSET", + b"\x00\x01"), + + # ================== + # = FILL Statement = + # ================== + + T("FILL Basic", + "ORG 0 ; FILL 0x10", + b"\x00" * 0x10), + + T("FILL Value", + "ORG 4 ; FILL 0x10 0xFF", + b"\x00\x00\x00\x00" + b"\xFF" * 0x10), + + T("FILL Zero", + "ORG 0 ; FILL 0", + None), + + T("FILL Negative", + "ORG -1 ; FILL 0", + None), + + # ==================== + # = ASSERT Statement = + # ==================== + + T("ASSERT Traditional", + "ASSERT 0", + b""), + + T("ASSERT Traditional Failure", + "ASSERT -1", + None), + + T("ASSERT Conditional", + "ASSERT 1 > 0", + b""), + + T("ASSERT Conditional Failure", + "ASSERT 1 < 0", + None), + + T("ASSERT Traditional Expression Failure", + "ASSERT 1 - 2", + None), + + # ================== + # = UTF8 Statement = + # ================== + + T("UTF8", + "ORG 0 ;UTF8 \"Hello World\"", + b"Hello World"), + + # ==================== + # = BASE64 Statement = + # ==================== + + T("BASE64", + " BASE64 \"RXZlbnQgQXNzZW1ibGVy\"", + b"Event Assembler"), + + # ===================== + # = PROTECT Statement = + # ===================== + + T("PROTECT Basic", + "PROTECT 0 4 ; ORG 0 ; BYTE 1", + None), + + T("PROTECT Edge 1", + "PROTECT 0 4 ; ORG 3 ; BYTE 1", + None), + + T("PROTECT Edge 2", + "PROTECT 0 4 ; ORG 4 ; BYTE 1", + b"\x00\x00\x00\x00\x01"), + + T("PROTECT Late", + "ORG 0 ; BYTE 1 ; PROTECT 0 4", + b"\x01"), + + # default PROTECT end is start + 4 + T("PROTECT Default range 1", + "PROTECT 0 ; ORG 4 ; BYTE 1", + b"\x00\x00\x00\x00\x01"), + + # default PROTECT end is start + 4 + T("PROTECT Default range 2", + "PROTECT 0 ; ORG 3 ; BYTE 1", + None), + +] From fe6d4aa3d124b3793fd68612f92e757066935bbe Mon Sep 17 00:00:00 2001 From: Nat_776 Date: Tue, 30 Apr 2024 21:41:26 +0200 Subject: [PATCH 46/59] add tests gitignore --- Tests/.gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 Tests/.gitignore diff --git a/Tests/.gitignore b/Tests/.gitignore new file mode 100644 index 0000000..c18dd8d --- /dev/null +++ b/Tests/.gitignore @@ -0,0 +1 @@ +__pycache__/ From cf37d5dc10adbbb87dcfb7dbf7dd61d3721d743e Mon Sep 17 00:00:00 2001 From: Nat_776 Date: Wed, 1 May 2024 15:22:17 +0200 Subject: [PATCH 47/59] fixes --- ColorzCore/Parser/EAParseConsumer.cs | 1 - ColorzCore/Parser/EAParser.cs | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/ColorzCore/Parser/EAParseConsumer.cs b/ColorzCore/Parser/EAParseConsumer.cs index f4bf598..7bd1976 100644 --- a/ColorzCore/Parser/EAParseConsumer.cs +++ b/ColorzCore/Parser/EAParseConsumer.cs @@ -247,7 +247,6 @@ public void OnProtectStatement(Location location, IAtomNode beginAtom, IAtomNode } } - Logger.Message(location, $"{beginValue} {length}"); protectedRegions.Add((beginValue, length, location)); } else diff --git a/ColorzCore/Parser/EAParser.cs b/ColorzCore/Parser/EAParser.cs index f3b111b..e9a2b5b 100644 --- a/ColorzCore/Parser/EAParser.cs +++ b/ColorzCore/Parser/EAParser.cs @@ -81,7 +81,7 @@ public IList ParseAll(IEnumerable tokenStream) { if (tokens.Current.Type != TokenType.NEWLINE || tokens.MoveNext()) { - ParseLine(tokens, ParseConsumer.GlobalScope); + ParseLine(tokens, ParseConsumer.CurrentScope); } } From ccc0350e9a48c577f323cbbaf7ce0929586a9e23 Mon Sep 17 00:00:00 2001 From: Nat_776 Date: Wed, 1 May 2024 15:55:42 +0200 Subject: [PATCH 48/59] add extra tests --- Tests/run_tests.py | 4 ++-- Tests/symbols.py | 48 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 Tests/symbols.py diff --git a/Tests/run_tests.py b/Tests/run_tests.py index 3b4f91a..7aa8f2c 100644 --- a/Tests/run_tests.py +++ b/Tests/run_tests.py @@ -59,9 +59,9 @@ ] -import statements +import statements, symbols -ALL_TEST_CASES = BASIC_TESTS + statements.TESTS + EXPRESSION_TESTS + PREPROC_TESTS +ALL_TEST_CASES = BASIC_TESTS + statements.TESTS + symbols.TESTS + EXPRESSION_TESTS + PREPROC_TESTS def main(args): import argparse diff --git a/Tests/symbols.py b/Tests/symbols.py new file mode 100644 index 0000000..13c99b1 --- /dev/null +++ b/Tests/symbols.py @@ -0,0 +1,48 @@ +from ea_test import EATest as T + + +TESTS = [ + T("Label Basic", + "ORG 4 ; MyLabel: ; ORG 0 ; BYTE MyLabel", + b'\x04'), + + T("Label None", + "ORG 0 ; BYTE MyLabel", + None), + + T("Label Address", + "ORG 4 ; MyLabel: ; ORG 0 ; WORD MyLabel", + b'\x04\x00\x00\x08'), + + T("Label POIN", + "ORG 4 ; MyLabel: ; ORG 0 ; POIN MyLabel", + b'\x04\x00\x00\x08'), + + T("Label Forward", + "ORG 0 ; WORD MyLabel ; MyLabel:", + b'\x04\x00\x00\x08'), + + T("Symbol Basic", + "MySymbol := 0xBEEF ; ORG 0 ; SHORT MySymbol", + b'\xEF\xBE'), + + T("Symbol Reference Forward", + "ORG 0 ; SHORT MySymbol ; MySymbol := 0xBEEF", + b'\xEF\xBE'), + + T("Symbol Evaluate Forward", + "MySymbol := MyLabel + 0xA0 ; ORG 0 ; BYTE MySymbol ; MyLabel:", + b'\xA1'), + + T("Scope Basic", + "ORG 0 ; { MyLabel: BYTE MyLabel + 1 ; }", + b'\x01'), + + T("Scope Failure", + "ORG 0 ; { MyLabel: BYTE 0 ; } BYTE MyLabel", + None), + + T("Scope Up", + "ORG 0 ; MyLabel: { BYTE MyLabel + 1 ; }", + b'\x01'), +] From 9487fc39f6d6528259952c798277e225fcb0fda5 Mon Sep 17 00:00:00 2001 From: Nat_776 Date: Sat, 4 May 2024 10:22:51 +0200 Subject: [PATCH 49/59] SetSymbol macro detection, parse fixes, many small internal simplifications --- ColorzCore/EAInterpreter.cs | 66 +++++++--- ColorzCore/EAOptions.cs | 3 + ColorzCore/Parser/AST/MacroInvocationNode.cs | 10 +- ColorzCore/Parser/AST/OperatorNode.cs | 13 +- ColorzCore/Parser/AtomParser.cs | 14 +- .../{ => Diagnostics}/DiagnosticsHelpers.cs | 24 +++- .../Diagnostics/SetSymbolMacroDetector.cs | 110 ++++++++++++++++ ColorzCore/Parser/EAParseConsumer.cs | 13 +- ColorzCore/Parser/EAParser.cs | 123 ++++++++++-------- ColorzCore/Parser/ParseConsumerChain.cs | 76 +++++++++++ ColorzCore/Parser/Pool.cs | 4 +- ColorzCore/Preprocessor/Definition.cs | 5 + ColorzCore/Preprocessor/DirectiveHandler.cs | 14 +- .../Directives/DefineDirective.cs | 4 +- .../Preprocessor/Directives/IDirective.cs | 2 +- .../Directives/IfDefinedDirective.cs | 43 +++--- .../Preprocessor/Directives/PoolDirective.cs | 2 +- .../Directives/SimpleDirective.cs | 4 +- .../Directives/UndefineDirective.cs | 2 +- ColorzCore/Preprocessor/Macros/AddToPool.cs | 4 +- .../Preprocessor/Macros/BuiltInMacro.cs | 2 +- ColorzCore/Preprocessor/Macros/ErrorMacro.cs | 2 +- ColorzCore/Preprocessor/Macros/IMacro.cs | 2 +- ColorzCore/Preprocessor/Macros/IsDefined.cs | 2 +- .../Preprocessor/Macros/IsSymbolDefined.cs | 46 +++---- ColorzCore/Preprocessor/Macros/ReadDataAt.cs | 4 +- ColorzCore/Preprocessor/Macros/StringMacro.cs | 7 +- ColorzCore/Preprocessor/Macros/UserMacro.cs | 2 +- 28 files changed, 424 insertions(+), 179 deletions(-) rename ColorzCore/Parser/{ => Diagnostics}/DiagnosticsHelpers.cs (87%) create mode 100644 ColorzCore/Parser/Diagnostics/SetSymbolMacroDetector.cs create mode 100644 ColorzCore/Parser/ParseConsumerChain.cs diff --git a/ColorzCore/EAInterpreter.cs b/ColorzCore/EAInterpreter.cs index 017c902..e15ab29 100644 --- a/ColorzCore/EAInterpreter.cs +++ b/ColorzCore/EAInterpreter.cs @@ -3,6 +3,7 @@ using ColorzCore.Lexer; using ColorzCore.Parser; using ColorzCore.Parser.AST; +using ColorzCore.Parser.Diagnostics; using ColorzCore.Preprocessor; using ColorzCore.Preprocessor.Directives; using ColorzCore.Preprocessor.Macros; @@ -14,18 +15,20 @@ namespace ColorzCore { - //Class to excapsulate all steps in EA script interpretation. + // Class to excapsulate all steps in EA script interpretation. + // TODO: this is just another Program + // the name is problematic as EAParseConsumer would really like to be called EAInterpreter class EAInterpreter { private Dictionary> allRaws; private EAParser myParser; - private EAParseConsumer myParseConsumer; + private EAParseConsumer myInterpreter; private string iFile; private Stream sin; private Logger Logger { get; } private IOutput output; - public EAInterpreter(IOutput output, string game, string? rawsFolder, string rawsExtension, Stream sin, string inFileName, Logger logger) + public EAInterpreter(IOutput output, string? game, string? rawsFolder, string rawsExtension, Stream sin, string inFileName, Logger logger) { this.output = output; @@ -47,17 +50,40 @@ public EAInterpreter(IOutput output, string game, string? rawsFolder, string raw Logger = logger; iFile = inFileName; + myInterpreter = new EAParseConsumer(logger); + + ParseConsumerChain parseConsumers = new ParseConsumerChain(); + + if (EAOptions.IsWarningEnabled(EAOptions.Warnings.SetSymbolMacros)) + { + parseConsumers.Add(new SetSymbolMacroDetector(logger)); + } + + // add the interpreter last + parseConsumers.Add(myInterpreter); + + myParser = new EAParser(logger, allRaws, parseConsumers, myInterpreter.BindIdentifier); + + myParser.Definitions["__COLORZ_CORE__"] = new Definition(); + + myParser.Definitions["__COLORZ_CORE_VERSION__"] = new Definition( + new Token(TokenType.NUMBER, new Location("builtin", 0, 0), "20240502")); + + if (game != null) + { + myParser.Definitions[$"_{game}_"] = new Definition(); + } + IncludeFileSearcher includeSearcher = new IncludeFileSearcher(); includeSearcher.IncludeDirectories.Add(AppDomain.CurrentDomain.BaseDirectory); foreach (string path in EAOptions.IncludePaths) + { includeSearcher.IncludeDirectories.Add(path); + } - myParseConsumer = new EAParseConsumer(logger); - myParser = new EAParser(logger, allRaws, new DirectiveHandler(includeSearcher), myParseConsumer); - - myParser.Definitions[$"_{game}_"] = new Definition(); - myParser.Definitions["__COLORZ_CORE__"] = new Definition(); + myParser.DirectiveHandler.Directives["include"] = new IncludeDirective() { FileSearcher = includeSearcher }; + myParser.DirectiveHandler.Directives["incbin"] = new IncludeBinaryDirective() { FileSearcher = includeSearcher }; if (EAOptions.IsExtensionEnabled(EAOptions.Extensions.ReadDataMacros) && output is ROM rom) { @@ -78,6 +104,8 @@ public EAInterpreter(IOutput output, string game, string? rawsFolder, string raw if (EAOptions.IsExtensionEnabled(EAOptions.Extensions.IncludeTools)) { + myParser.Definitions["__has_incext"] = new Definition(); + IncludeFileSearcher toolSearcher = new IncludeFileSearcher { AllowRelativeInclude = false }; toolSearcher.IncludeDirectories.Add(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "Tools")); @@ -96,6 +124,8 @@ public EAInterpreter(IOutput output, string game, string? rawsFolder, string raw if (EAOptions.IsExtensionEnabled(EAOptions.Extensions.AddToPool)) { + myParser.Definitions["__has_pool"] = new Definition(); + Pool pool = new Pool(); myParser.Macros.BuiltInMacros.Add("AddToPool", new AddToPool(pool)); @@ -111,10 +141,12 @@ public bool Interpret() foreach ((string name, string body) in EAOptions.PreDefintions) { - myParser.ParseAll(tokenizer.TokenizeLine($"#define {name} {body}", "cmd", 0)); + myParser.ParseAll(tokenizer.TokenizeLine($"#define {name} \"{body}\"", "cmd", 0)); } - IList lines = new List(myParser.ParseAll(tokenizer.Tokenize(sin, iFile))); + myParser.ParseAll(tokenizer.Tokenize(sin, iFile)); + + IList lines = myInterpreter.HandleEndOfInput(); /* First pass on AST: Identifier resolution. * @@ -182,7 +214,7 @@ public bool Interpret() public bool WriteNocashSymbols(TextWriter output) { - for (int i = 0; i < myParseConsumer.AllScopes.Count; i++) + for (int i = 0; i < myInterpreter.AllScopes.Count; i++) { string manglePrefix = string.Empty; @@ -197,7 +229,7 @@ public bool WriteNocashSymbols(TextWriter output) break; } - Closure scope = myParseConsumer.AllScopes[i]; + Closure scope = myInterpreter.AllScopes[i]; foreach (KeyValuePair pair in scope.LocalSymbols()) { @@ -261,14 +293,16 @@ static Raw CreateRaw(string name, int byteSize, int alignment, bool isPointer) }; } - private static Dictionary> SelectRaws(string game, IList allRaws) + private static Dictionary> SelectRaws(string? game, IList allRaws) { Dictionary> result = new Dictionary>(); - foreach (Raw r in allRaws) + foreach (Raw raw in allRaws) { - if (r.Game.Count == 0 || r.Game.Contains(game)) - result.AddTo(r.Name, r); + if (raw.Game.Count == 0 || (game != null && raw.Game.Contains(game))) + { + result.AddTo(raw.Name, raw); + } } return result; diff --git a/ColorzCore/EAOptions.cs b/ColorzCore/EAOptions.cs index 7b4dac0..e3e45ca 100644 --- a/ColorzCore/EAOptions.cs +++ b/ColorzCore/EAOptions.cs @@ -26,6 +26,9 @@ public enum Warnings : long // warn on expansion of unguarded expression within macro () UnguardedExpressionMacros = 16, + // warn on macro expanded into "PUSH ; ORG value ; name : ; POP" + SetSymbolMacros = 32, + Extra = UnguardedExpressionMacros, All = long.MaxValue & ~Extra, } diff --git a/ColorzCore/Parser/AST/MacroInvocationNode.cs b/ColorzCore/Parser/AST/MacroInvocationNode.cs index cec08a4..d743686 100644 --- a/ColorzCore/Parser/AST/MacroInvocationNode.cs +++ b/ColorzCore/Parser/AST/MacroInvocationNode.cs @@ -21,15 +21,13 @@ public MacroException(MacroInvocationNode min) : base(min.invokeToken.Content) private readonly EAParser p; private readonly Token invokeToken; - private readonly ImmutableStack scope; public IList> Parameters { get; } - public MacroInvocationNode(EAParser p, Token invokeTok, IList> parameters, ImmutableStack scopes) + public MacroInvocationNode(EAParser p, Token invokeTok, IList> parameters) { this.p = p; - this.invokeToken = invokeTok; - this.Parameters = parameters; - this.scope = scopes; + invokeToken = invokeTok; + Parameters = parameters; } public ParamType Type => ParamType.MACRO; @@ -54,7 +52,7 @@ public string PrettyPrint() public IEnumerable ExpandMacro() { - return p.Macros.GetMacro(invokeToken.Content, Parameters.Count).ApplyMacro(invokeToken, Parameters, scope); + return p.Macros.GetMacro(invokeToken.Content, Parameters.Count).ApplyMacro(invokeToken, Parameters); } public Either TryEvaluate() diff --git a/ColorzCore/Parser/AST/OperatorNode.cs b/ColorzCore/Parser/AST/OperatorNode.cs index 393946b..d848cf9 100644 --- a/ColorzCore/Parser/AST/OperatorNode.cs +++ b/ColorzCore/Parser/AST/OperatorNode.cs @@ -101,7 +101,7 @@ public override IEnumerable ToTokens() public override int? TryEvaluate(Action handler, EvaluationPhase evaluationPhase) { /* undefined-coalescing operator is special because - * 1. it should only be evaluated at final evaluation. + * 1. it should not be evaluated early. * 2. it is legal for its left operand to fail evaluation. */ if (OperatorToken.Type == TokenType.UNDEFINED_COALESCE_OP) { @@ -109,17 +109,12 @@ public override IEnumerable ToTokens() switch (evaluationPhase) { - case EvaluationPhase.Immediate: - handler(new Exception("Invalid use of '??'.")); - return null; case EvaluationPhase.Early: - /* NOTE: you'd think one could optimize this by reducing this if left can be evaluated early - * but that would allow simplifying expressions even in contexts where '??' makes no sense - * (for example: 'ORG SomePossiblyUndefinedLabel ?? SomeOtherLabel') - * I don't think that's desirable */ + /* NOTE: maybe one could optimize this by reducing this if left can be evaluated early? */ handler(new Exception("The value of a '??' expression cannot be resolved early.")); return null; - case EvaluationPhase.Final: + + default: return TryCoalesceUndefined(handler); } } diff --git a/ColorzCore/Parser/AtomParser.cs b/ColorzCore/Parser/AtomParser.cs index 2cde2dd..ead4f6f 100644 --- a/ColorzCore/Parser/AtomParser.cs +++ b/ColorzCore/Parser/AtomParser.cs @@ -3,6 +3,7 @@ using ColorzCore.IO; using ColorzCore.Lexer; using ColorzCore.Parser.AST; +using ColorzCore.Parser.Diagnostics; namespace ColorzCore.Parser { @@ -33,7 +34,7 @@ public static class AtomParser public static bool IsInfixOperator(Token token) => precedences.ContainsKey(token.Type); - public static IAtomNode? ParseAtom(this EAParser self, MergeableGenerator tokens, ImmutableStack scopes, bool expandDefs = true) + public static IAtomNode? ParseAtom(this EAParser self, MergeableGenerator tokens) { //Use Shift Reduce Parsing Token localHead = tokens.Current; @@ -96,7 +97,7 @@ public static class AtomParser case TokenType.OPEN_PAREN: { tokens.MoveNext(); - IAtomNode? interior = self.ParseAtom(tokens, scopes); + IAtomNode? interior = self.ParseAtom(tokens); if (tokens.Current.Type != TokenType.CLOSE_PAREN) { self.Logger.Error(tokens.Current.Location, "Unmatched open parenthesis (currently at " + tokens.Current.Type + ")."); @@ -120,7 +121,7 @@ public static class AtomParser { //Assume unary negation. tokens.MoveNext(); - IAtomNode? interior = self.ParseAtom(tokens, scopes); + IAtomNode? interior = self.ParseAtom(tokens); if (interior == null) { self.Logger.Error(lookAhead.Location, "Expected expression after unary operator."); @@ -164,22 +165,21 @@ public static class AtomParser switch (lookAhead.Type) { case TokenType.IDENTIFIER: - if (expandDefs && self.ExpandIdentifier(tokens, scopes, true)) + if (self.ExpandIdentifier(tokens, true)) { continue; } grammarSymbols.Push(new Left(lookAhead.Content.ToUpperInvariant() switch { - "CURRENTOFFSET" => new NumberNode(lookAhead, self.ParseConsumer.CurrentOffset), "__LINE__" => new NumberNode(lookAhead, lookAhead.GetSourceLocation().line), - _ => new IdentifierNode(lookAhead, scopes), + _ => self.BindIdentifier(lookAhead), })); break; case TokenType.MAYBE_MACRO: - self.ExpandIdentifier(tokens, scopes, true); + self.ExpandIdentifier(tokens, true); continue; case TokenType.NUMBER: grammarSymbols.Push(new Left(new NumberNode(lookAhead))); diff --git a/ColorzCore/Parser/DiagnosticsHelpers.cs b/ColorzCore/Parser/Diagnostics/DiagnosticsHelpers.cs similarity index 87% rename from ColorzCore/Parser/DiagnosticsHelpers.cs rename to ColorzCore/Parser/Diagnostics/DiagnosticsHelpers.cs index 3dbcb28..3ae6196 100644 --- a/ColorzCore/Parser/DiagnosticsHelpers.cs +++ b/ColorzCore/Parser/Diagnostics/DiagnosticsHelpers.cs @@ -5,7 +5,7 @@ using ColorzCore.Lexer; using ColorzCore.Parser.AST; -namespace ColorzCore.Parser +namespace ColorzCore.Parser.Diagnostics { public static class DiagnosticsHelpers { @@ -209,5 +209,27 @@ public static bool DoesOperationSpanMultipleMacrosUnintuitively(OperatorNode ope ParamType.MACRO => "Macro", _ => "", }; + + // absolute as in "not relative to the value of a symbol" + public static bool IsAbsoluteAtom(IAtomNode node) => node switch + { + IdentifierNode => false, + + OperatorNode operatorNode => operatorNode.OperatorToken.Type switch + { + // A + B is not absolute if either one is relative, but not both + TokenType.ADD_OP => IsAbsoluteAtom(operatorNode.Left) == IsAbsoluteAtom(operatorNode.Right), + + // A - B is not absolute if A is relative and not B + TokenType.SUB_OP => IsAbsoluteAtom(operatorNode.Left) || !IsAbsoluteAtom(operatorNode.Right), + + // A ?? B is not absolute if A and B aren't absolute + TokenType.UNDEFINED_COALESCE_OP => IsAbsoluteAtom(operatorNode.Left) || IsAbsoluteAtom(operatorNode.Right), + + _ => true, + }, + + _ => true, + }; } } diff --git a/ColorzCore/Parser/Diagnostics/SetSymbolMacroDetector.cs b/ColorzCore/Parser/Diagnostics/SetSymbolMacroDetector.cs new file mode 100644 index 0000000..0752a6f --- /dev/null +++ b/ColorzCore/Parser/Diagnostics/SetSymbolMacroDetector.cs @@ -0,0 +1,110 @@ + +using System.Collections.Generic; +using ColorzCore.DataTypes; +using ColorzCore.IO; +using ColorzCore.Parser.AST; +using ColorzCore.Raws; + +namespace ColorzCore.Parser.Diagnostics +{ + public class SetSymbolMacroDetector : IParseConsumer + { + public Logger Logger { get; } + + private enum State + { + AwaitingPush, + AwaitingOrgAbsolute, + AwaitingLabel, + AwaitingPop, + AwaitingEndOfMacro, + } + + State suspectState = State.AwaitingPush; + Location? suspectLocation = null; + + public SetSymbolMacroDetector(Logger logger) + { + Logger = logger; + suspectState = State.AwaitingPush; + suspectLocation = null; + } + + private void OnSequenceBroken(Location location) + { + if (suspectState == State.AwaitingEndOfMacro && location.macroLocation != suspectLocation?.macroLocation) + { + Logger.Warning(suspectLocation, + "This looks like the expansion of a \"SetSymbol\" macro.\n" + + "SetSymbol macros are macros defined as `PUSH ; ORG value ; name : ; POP`.\n" + + "Because of changes to the behavior of labels, their use may introduce bugs.\n" + + "Consider using symbol assignment instead by writing `name := value`."); + } + + suspectState = State.AwaitingPush; + suspectLocation = null; + } + + public void OnPushStatement(Location location) + { + OnSequenceBroken(location); + + suspectState = State.AwaitingOrgAbsolute; + suspectLocation = location; + } + + public void OnOrgStatement(Location location, IAtomNode offsetNode) + { + if (suspectState == State.AwaitingOrgAbsolute && location.macroLocation == suspectLocation?.macroLocation) + { + suspectState = State.AwaitingLabel; + } + else + { + OnSequenceBroken(location); + } + } + + public void OnLabel(Location location, string name) + { + if (suspectState == State.AwaitingLabel && location.macroLocation == suspectLocation?.macroLocation) + { + suspectState = State.AwaitingPop; + } + else + { + OnSequenceBroken(location); + } + } + + public void OnPopStatement(Location location) + { + if (suspectState == State.AwaitingPop && location.macroLocation == suspectLocation?.macroLocation) + { + suspectState = State.AwaitingEndOfMacro; + } + else + { + OnSequenceBroken(location); + } + } + + public void OnOpenScope(Location location) => OnSequenceBroken(location); + + public void OnCloseScope(Location location) => OnSequenceBroken(location); + + public void OnRawStatement(Location location, Raw raw, IList parameters) => OnSequenceBroken(location); + + public void OnAssertStatement(Location location, IAtomNode node) => OnSequenceBroken(location); + + public void OnProtectStatement(Location location, IAtomNode beginAtom, IAtomNode? endAtom) => OnSequenceBroken(location); + + public void OnAlignStatement(Location location, IAtomNode alignNode, IAtomNode? offsetNode) => OnSequenceBroken(location); + + public void OnFillStatement(Location location, IAtomNode amountNode, IAtomNode? valueNode) => OnSequenceBroken(location); + + public void OnSymbolAssignment(Location location, string name, IAtomNode atom) => OnSequenceBroken(location); + + public void OnData(Location location, byte[] data) => OnSequenceBroken(location); + } +} diff --git a/ColorzCore/Parser/EAParseConsumer.cs b/ColorzCore/Parser/EAParseConsumer.cs index 7bd1976..57d615f 100644 --- a/ColorzCore/Parser/EAParseConsumer.cs +++ b/ColorzCore/Parser/EAParseConsumer.cs @@ -20,7 +20,7 @@ public class EAParseConsumer : IParseConsumer private readonly Stack<(int, bool)> pastOffsets; // currentOffset, offsetInitialized private readonly IList<(int, int, Location)> protectedRegions; - private bool diagnosedOverflow; // true until first overflow diagnostic. + private bool diagnosedOverflow; // false until first overflow diagnostic. private bool offsetInitialized; // false until first ORG, used for diagnostics private int currentOffset; @@ -33,7 +33,7 @@ public EAParseConsumer(Logger logger) pastOffsets = new Stack<(int, bool)>(); protectedRegions = new List<(int, int, Location)>(); currentOffset = 0; - diagnosedOverflow = true; + diagnosedOverflow = false; offsetInitialized = false; Logger = logger; } @@ -66,6 +66,15 @@ public static int ConvertToOffset(int value) return value; } + public IAtomNode BindIdentifier(Token identifierToken) + { + return identifierToken.Content.ToUpperInvariant() switch + { + "CURRENTOFFSET" => new NumberNode(identifierToken, EAOptions.BaseAddress + CurrentOffset), + _ => new IdentifierNode(identifierToken, CurrentScope), + }; + } + // Helper method for statement handlers private int? EvaluteAtom(IAtomNode node) { diff --git a/ColorzCore/Parser/EAParser.cs b/ColorzCore/Parser/EAParser.cs index e9a2b5b..44978ac 100644 --- a/ColorzCore/Parser/EAParser.cs +++ b/ColorzCore/Parser/EAParser.cs @@ -2,6 +2,7 @@ using ColorzCore.IO; using ColorzCore.Lexer; using ColorzCore.Parser.AST; +using ColorzCore.Parser.Diagnostics; using ColorzCore.Preprocessor; using ColorzCore.Preprocessor.Macros; using ColorzCore.Raws; @@ -44,17 +45,24 @@ public bool IsIncluding } } - public EAParseConsumer ParseConsumer { get; } + public delegate IAtomNode BindIdentifierFunc(Token identifierToken); - public EAParser(Logger log, Dictionary> raws, DirectiveHandler directiveHandler, EAParseConsumer parseConsumer) + public IParseConsumer ParseConsumer { get; } + + // TODO: IParseContextProvider or something like that? + // could also provide expanded format strings and stuff + public BindIdentifierFunc BindIdentifier { get; } + + public EAParser(Logger log, Dictionary> raws, IParseConsumer parseConsumer, BindIdentifierFunc bindIdentifier) { Logger = log; Raws = raws; Macros = new MacroCollection(this); Definitions = new Dictionary(); Inclusion = ImmutableStack.Nil; - DirectiveHandler = directiveHandler; + DirectiveHandler = new DirectiveHandler(); ParseConsumer = parseConsumer; + BindIdentifier = bindIdentifier; } public bool IsReservedName(string name) @@ -72,7 +80,7 @@ public bool IsValidMacroName(string name, int paramNum) return !Macros.HasMacro(name, paramNum) && !IsReservedName(name); } - public IList ParseAll(IEnumerable tokenStream) + public void ParseAll(IEnumerable tokenStream) { MergeableGenerator tokens = new MergeableGenerator(tokenStream); tokens.MoveNext(); @@ -81,11 +89,9 @@ public IList ParseAll(IEnumerable tokenStream) { if (tokens.Current.Type != TokenType.NEWLINE || tokens.MoveNext()) { - ParseLine(tokens, ParseConsumer.CurrentScope); + ParseLine(tokens); } } - - return ParseConsumer.HandleEndOfInput(); } public static readonly HashSet SpecialCodes = new HashSet() @@ -100,13 +106,13 @@ public IList ParseAll(IEnumerable tokenStream) "PROTECT", "ALIGN", "FILL", - "UTF8", + "UTF8", // TODO: remove en favor of generic STRING with .tbl support "BASE64", // "SECTION", // TODO // "DSECTION", // TODO }; - private void ParseStatement(MergeableGenerator tokens, ImmutableStack scopes) + private void ParseStatement(MergeableGenerator tokens) { // NOTE: here previously lied en ExpandIdentifier loop // though because this is only called from ParseLine after the corresponding check, this is not needed @@ -135,7 +141,7 @@ private void ParseStatement(MergeableGenerator tokens, ImmutableStack parameters = tokens.Current.Type switch { TokenType.NEWLINE or TokenType.SEMICOLON => new List(), - _ => ParseParamList(tokens, scopes), + _ => ParseParamList(tokens), }; string upperCodeIdentifier = head.Content.ToUpperInvariant(); @@ -201,7 +207,7 @@ private void ParseStatement(MergeableGenerator tokens, ImmutableStack tokens) { - IAtomNode? atom = this.ParseAtom(tokens, ParseConsumer.CurrentScope, true); + IAtomNode? atom = this.ParseAtom(tokens); if (atom != null) { @@ -383,20 +389,17 @@ private void ParseFillStatement(Token head, IList parameters) private void ParseMessageStatement(Token head, IList parameters) { - ImmutableStack scopes = ParseConsumer.CurrentScope; - Logger.Message(head.Location, PrettyPrintParamsForMessage(parameters, scopes)); + Logger.Message(head.Location, PrettyPrintParamsForMessage(parameters)); } private void ParseWarningStatement(Token head, IList parameters) { - ImmutableStack scopes = ParseConsumer.CurrentScope; - Logger.Warning(head.Location, PrettyPrintParamsForMessage(parameters, scopes)); + Logger.Warning(head.Location, PrettyPrintParamsForMessage(parameters)); } private void ParseErrorStatement(Token head, IList parameters) { - ImmutableStack scopes = ParseConsumer.CurrentScope; - Logger.Error(head.Location, PrettyPrintParamsForMessage(parameters, scopes)); + Logger.Error(head.Location, PrettyPrintParamsForMessage(parameters)); } private void ParseUtf8Statement(Token head, IList parameters) @@ -502,18 +505,16 @@ public IList> ParseMacroParamList(MergeableGenerator tokens) return parameters; } - private IList ParseParamList(MergeableGenerator tokens, ImmutableStack scopes, bool expandFirstDef = true) + private IList ParseParamList(MergeableGenerator tokens) { IList paramList = new List(); - bool first = true; - while (tokens.Current.Type != TokenType.NEWLINE && tokens.Current.Type != TokenType.SEMICOLON && !tokens.EOS) + while (!IsTokenAlwaysPastEndOfStatement(tokens.Current) && !tokens.EOS) { Token localHead = tokens.Current; - ParseParam(tokens, scopes, expandFirstDef || !first).IfJust( + ParseParam(tokens).IfJust( n => paramList.Add(n), () => Logger.Error(localHead.Location, "Expected parameter.")); - first = false; } if (tokens.Current.Type == TokenType.SEMICOLON) @@ -524,48 +525,49 @@ private IList ParseParamList(MergeableGenerator tokens, Immut return paramList; } - public IList ParsePreprocParamList(MergeableGenerator tokens, ImmutableStack scopes, bool allowsFirstExpanded) + public IList ParsePreprocParamList(MergeableGenerator tokens) { - IList temp = ParseParamList(tokens, scopes, allowsFirstExpanded); + IList temp = ParseParamList(tokens); for (int i = 0; i < temp.Count; i++) { - if (temp[i].Type == ParamType.STRING && ((StringNode)temp[i]).IsValidIdentifier()) + if (temp[i] is StringNode stringNode && stringNode.IsValidIdentifier()) { - temp[i] = ((StringNode)temp[i]).ToIdentifier(scopes); + // TODO: what is this for? can we omit it? + temp[i] = BindIdentifier(stringNode.SourceToken); } } return temp; } - private IParamNode? ParseParam(MergeableGenerator tokens, ImmutableStack scopes, bool expandDefs = true) + private IParamNode? ParseParam(MergeableGenerator tokens) { Token localHead = tokens.Current; switch (localHead.Type) { case TokenType.OPEN_BRACKET: - return new ListNode(localHead.Location, ParseList(tokens, scopes)); + return new ListNode(localHead.Location, ParseList(tokens)); case TokenType.STRING: tokens.MoveNext(); return new StringNode(localHead); case TokenType.MAYBE_MACRO: //TODO: Move this and the one in ExpandId to a separate ParseMacroNode that may return an Invocation. - if (expandDefs && ExpandIdentifier(tokens, scopes, true)) + if (ExpandIdentifier(tokens, true)) { - return ParseParam(tokens, scopes); + return ParseParam(tokens); } else { tokens.MoveNext(); IList> param = ParseMacroParamList(tokens); //TODO: Smart errors if trying to redefine a macro with the same num of params. - return new MacroInvocationNode(this, localHead, param, scopes); + return new MacroInvocationNode(this, localHead, param); } case TokenType.IDENTIFIER: - if (expandDefs && ExpandIdentifier(tokens, scopes, true)) + if (ExpandIdentifier(tokens, true)) { - return ParseParam(tokens, scopes, expandDefs); + return ParseParam(tokens); } else { @@ -576,16 +578,16 @@ public IList ParsePreprocParamList(MergeableGenerator tokens, return new StringNode(new Token(TokenType.STRING, localHead.Location, localHead.GetSourceLocation().file)); default: - return this.ParseAtom(tokens, scopes, expandDefs); + return this.ParseAtom(tokens); } } default: - return this.ParseAtom(tokens, scopes, expandDefs); + return this.ParseAtom(tokens); } } - private IList ParseList(MergeableGenerator tokens, ImmutableStack scopes) + private IList ParseList(MergeableGenerator tokens) { Token localHead = tokens.Current; tokens.MoveNext(); @@ -593,7 +595,7 @@ private IList ParseList(MergeableGenerator tokens, ImmutableSt IList atoms = new List(); while (tokens.Current.Type != TokenType.NEWLINE && tokens.Current.Type != TokenType.CLOSE_BRACKET) { - IAtomNode? res = this.ParseAtom(tokens, scopes); + IAtomNode? res = this.ParseAtom(tokens); res.IfJust( n => atoms.Add(n), () => Logger.Error(tokens.Current.Location, "Expected atomic value, got " + tokens.Current.Type + ".")); @@ -614,7 +616,7 @@ private IList ParseList(MergeableGenerator tokens, ImmutableSt return atoms; } - public void ParseLine(MergeableGenerator tokens, ImmutableStack scopes) + public void ParseLine(MergeableGenerator tokens) { if (IsIncluding) { @@ -629,7 +631,7 @@ public void ParseLine(MergeableGenerator tokens, ImmutableStack { case TokenType.IDENTIFIER: case TokenType.MAYBE_MACRO: - if (ExpandIdentifier(tokens, scopes)) + if (ExpandIdentifier(tokens)) { // NOTE: we check here if we didn't end up with something that can't be a statement @@ -640,7 +642,7 @@ public void ParseLine(MergeableGenerator tokens, ImmutableStack case TokenType.OPEN_BRACE: case TokenType.PREPROCESSOR_DIRECTIVE: // recursion! - ParseLine(tokens, scopes); + ParseLine(tokens); return; default: @@ -654,7 +656,7 @@ public void ParseLine(MergeableGenerator tokens, ImmutableStack } else { - ParseStatement(tokens, scopes); + ParseStatement(tokens); } break; @@ -670,7 +672,7 @@ public void ParseLine(MergeableGenerator tokens, ImmutableStack break; case TokenType.PREPROCESSOR_DIRECTIVE: - ParsePreprocessor(tokens, scopes); + ParsePreprocessor(tokens); break; case TokenType.OPEN_BRACKET: @@ -710,7 +712,7 @@ public void ParseLine(MergeableGenerator tokens, ImmutableStack if (hasNext) { - ParsePreprocessor(tokens, scopes); + ParsePreprocessor(tokens); } else { @@ -719,11 +721,11 @@ public void ParseLine(MergeableGenerator tokens, ImmutableStack } } - private void ParsePreprocessor(MergeableGenerator tokens, ImmutableStack scopes) + private void ParsePreprocessor(MergeableGenerator tokens) { Token head = tokens.Current; tokens.MoveNext(); - DirectiveHandler.HandleDirective(this, head, tokens, scopes); + DirectiveHandler.HandleDirective(this, head, tokens); } /*** @@ -731,7 +733,7 @@ private void ParsePreprocessor(MergeableGenerator tokens, ImmutableStack< * Postcondition: tokens.Current is fully reduced (i.e. not a macro, and not a definition) * Returns: true iff tokens was actually expanded. */ - public bool ExpandIdentifier(MergeableGenerator tokens, ImmutableStack scopes, bool insideExpression = false) + public bool ExpandIdentifier(MergeableGenerator tokens, bool insideExpression = false) { // function-like macros if (tokens.Current.Type == TokenType.MAYBE_MACRO) @@ -749,7 +751,7 @@ public bool ExpandIdentifier(MergeableGenerator tokens, ImmutableStack tokens, IEnumerable tokens) { - // TODO: also check for OPEN_BRACE? - - while (tokens.Current.Type != TokenType.NEWLINE && tokens.Current.Type != TokenType.SEMICOLON && tokens.MoveNext()) { } + while (!IsTokenAlwaysPastEndOfStatement(tokens.Current) && tokens.MoveNext()) + { + } if (tokens.Current.Type == TokenType.SEMICOLON) { @@ -825,13 +827,13 @@ public void IgnoreRestOfLine(MergeableGenerator tokens) /// token stream /// If non-null, will expand any macros as they are encountered using this scope /// The resulting list of tokens - public IList GetRestOfLine(MergeableGenerator tokens, ImmutableStack? scopesForMacros) + public IList GetRestOfLine(MergeableGenerator tokens) { IList result = new List(); while (tokens.Current.Type != TokenType.NEWLINE) { - if (scopesForMacros == null || !ExpandIdentifier(tokens, scopesForMacros)) + if (!ExpandIdentifier(tokens)) { result.Add(tokens.Current); tokens.MoveNext(); @@ -841,18 +843,27 @@ public IList GetRestOfLine(MergeableGenerator tokens, ImmutableSta return result; } - private string PrettyPrintParamsForMessage(IList parameters, ImmutableStack scopes) + private static bool IsTokenAlwaysPastEndOfStatement(Token token) => token.Type switch + { + TokenType.NEWLINE => true, + TokenType.SEMICOLON => true, + TokenType.OPEN_BRACE => true, + TokenType.CLOSE_BRACE => true, + _ => false, + }; + + private string PrettyPrintParamsForMessage(IList parameters) { return string.Join(" ", parameters.Select(parameter => parameter switch { - StringNode node => ExpandUserFormatString(scopes, parameter.MyLocation, node.Value), + StringNode node => ExpandUserFormatString(parameter.MyLocation, node.Value), _ => parameter.PrettyPrint(), })); } private static readonly Regex formatItemRegex = new Regex(@"\{(?[^:}]+)(?:\:(?[^:}]*))?\}"); - private string ExpandUserFormatString(ImmutableStack scopes, Location baseLocation, string stringValue) + private string ExpandUserFormatString(Location baseLocation, string stringValue) { string UserFormatStringError(Location loc, string message, string details) { @@ -872,7 +883,7 @@ string UserFormatStringError(Location loc, string message, string details) tokens.MoveNext(); - IAtomNode? node = this.ParseAtom(tokens, scopes); + IAtomNode? node = this.ParseAtom(tokens); if (node == null || tokens.Current.Type != TokenType.NEWLINE) { diff --git a/ColorzCore/Parser/ParseConsumerChain.cs b/ColorzCore/Parser/ParseConsumerChain.cs new file mode 100644 index 0000000..cd8879d --- /dev/null +++ b/ColorzCore/Parser/ParseConsumerChain.cs @@ -0,0 +1,76 @@ + +using System.Collections.Generic; +using ColorzCore.DataTypes; +using ColorzCore.Parser.AST; +using ColorzCore.Raws; + +namespace ColorzCore.Parser +{ + public class ParseConsumerChain : List, IParseConsumer + { + public void OnAlignStatement(Location location, IAtomNode alignNode, IAtomNode? offsetNode) + { + ForEach(pc => pc.OnAlignStatement(location, alignNode, offsetNode)); + } + + public void OnAssertStatement(Location location, IAtomNode node) + { + ForEach(pc => pc.OnAssertStatement(location, node)); + } + + public void OnCloseScope(Location location) + { + ForEach(pc => pc.OnCloseScope(location)); + } + + public void OnData(Location location, byte[] data) + { + ForEach(pc => pc.OnData(location, data)); + } + + public void OnFillStatement(Location location, IAtomNode amountNode, IAtomNode? valueNode) + { + ForEach(pc => pc.OnFillStatement(location, amountNode, valueNode)); + } + + public void OnLabel(Location location, string name) + { + ForEach(pc => pc.OnLabel(location, name)); + } + + public void OnOpenScope(Location location) + { + ForEach(pc => pc.OnOpenScope(location)); + } + + public void OnOrgStatement(Location location, IAtomNode offsetNode) + { + ForEach(pc => pc.OnOrgStatement(location, offsetNode)); + } + + public void OnPopStatement(Location location) + { + ForEach(pc => pc.OnPopStatement(location)); + } + + public void OnProtectStatement(Location location, IAtomNode beginAtom, IAtomNode? endAtom) + { + ForEach(pc => pc.OnProtectStatement(location, beginAtom, endAtom)); + } + + public void OnPushStatement(Location location) + { + ForEach(pc => pc.OnPushStatement(location)); + } + + public void OnRawStatement(Location location, Raw raw, IList parameters) + { + ForEach(pc => pc.OnRawStatement(location, raw, parameters)); + } + + public void OnSymbolAssignment(Location location, string name, IAtomNode atom) + { + ForEach(pc => pc.OnSymbolAssignment(location, name, atom)); + } + } +} \ No newline at end of file diff --git a/ColorzCore/Parser/Pool.cs b/ColorzCore/Parser/Pool.cs index 0e7d95e..f02ece1 100644 --- a/ColorzCore/Parser/Pool.cs +++ b/ColorzCore/Parser/Pool.cs @@ -9,12 +9,10 @@ public class Pool { public struct PooledLine { - public ImmutableStack Scope { get; private set; } public List Tokens { get; private set; } - public PooledLine(ImmutableStack scope, List tokens) + public PooledLine(List tokens) { - Scope = scope; Tokens = tokens; } } diff --git a/ColorzCore/Preprocessor/Definition.cs b/ColorzCore/Preprocessor/Definition.cs index 905648b..b65cdc2 100644 --- a/ColorzCore/Preprocessor/Definition.cs +++ b/ColorzCore/Preprocessor/Definition.cs @@ -21,6 +21,11 @@ public Definition() replacement = null; } + public Definition(Token token) + { + replacement = new List { token }; + } + public Definition(IList defn) { replacement = defn; diff --git a/ColorzCore/Preprocessor/DirectiveHandler.cs b/ColorzCore/Preprocessor/DirectiveHandler.cs index 6d4a740..cbbcb3c 100644 --- a/ColorzCore/Preprocessor/DirectiveHandler.cs +++ b/ColorzCore/Preprocessor/DirectiveHandler.cs @@ -2,7 +2,6 @@ using ColorzCore.IO; using ColorzCore.Lexer; using ColorzCore.Parser; -using ColorzCore.Parser.AST; using ColorzCore.Preprocessor.Directives; using System; using System.Collections.Generic; @@ -12,17 +11,14 @@ namespace ColorzCore.Preprocessor { public class DirectiveHandler { + // TODO: do we need this class? Could we not just have this part of EAParser? + public Dictionary Directives { get; } - public DirectiveHandler(IncludeFileSearcher includeSearcher) + public DirectiveHandler() { - // TODO: move out from this directives that need external context - // (already done for pool, but could be done for includes as well) - Directives = new Dictionary { - { "include", new IncludeDirective { FileSearcher = includeSearcher } }, - { "incbin", new IncludeBinaryDirective { FileSearcher = includeSearcher } }, { "ifdef", new IfDefinedDirective(false) }, { "ifndef", new IfDefinedDirective(true) }, { "if", new IfDirective() }, @@ -33,7 +29,7 @@ public DirectiveHandler(IncludeFileSearcher includeSearcher) }; } - public void HandleDirective(EAParser p, Token directive, MergeableGenerator tokens, ImmutableStack scopes) + public void HandleDirective(EAParser p, Token directive, MergeableGenerator tokens) { string directiveName = directive.Content.Substring(1); @@ -41,7 +37,7 @@ public void HandleDirective(EAParser p, Token directive, MergeableGenerator true; - public void Execute(EAParser p, Token self, MergeableGenerator tokens, ImmutableStack scopes) + public void Execute(EAParser p, Token self, MergeableGenerator tokens) { Token nextToken = tokens.Current; IList? parameters; @@ -41,7 +41,7 @@ public void Execute(EAParser p, Token self, MergeableGenerator tokens, Im return; } - IList? macroBody = ExpandMacroBody(p, p.GetRestOfLine(tokens, scopes)); + IList? macroBody = ExpandMacroBody(p, p.GetRestOfLine(tokens)); if (parameters != null) { diff --git a/ColorzCore/Preprocessor/Directives/IDirective.cs b/ColorzCore/Preprocessor/Directives/IDirective.cs index de0eb36..730249b 100644 --- a/ColorzCore/Preprocessor/Directives/IDirective.cs +++ b/ColorzCore/Preprocessor/Directives/IDirective.cs @@ -15,7 +15,7 @@ public interface IDirective * * Return: If a string is returned, it is interpreted as an error. */ - void Execute(EAParser p, Token self, MergeableGenerator tokens, ImmutableStack scopes); + void Execute(EAParser p, Token self, MergeableGenerator tokens); /*** * Whether requires the parser to be taking in tokens. diff --git a/ColorzCore/Preprocessor/Directives/IfDefinedDirective.cs b/ColorzCore/Preprocessor/Directives/IfDefinedDirective.cs index a917668..cd871be 100644 --- a/ColorzCore/Preprocessor/Directives/IfDefinedDirective.cs +++ b/ColorzCore/Preprocessor/Directives/IfDefinedDirective.cs @@ -5,17 +5,14 @@ using ColorzCore.IO; using ColorzCore.Lexer; using ColorzCore.Parser; -using ColorzCore.Parser.AST; namespace ColorzCore.Preprocessor.Directives { - class IfDefinedDirective : SimpleDirective + class IfDefinedDirective : IDirective { - public override int MinParams => 1; + // This directive does not inherit SimpleDirective so as to avoid having its parameter expanded - public override int? MaxParams => 1; - - public override bool RequireInclusion => false; + public bool RequireInclusion => false; public bool Inverted { get; } @@ -24,27 +21,31 @@ public IfDefinedDirective(bool invert) Inverted = invert; } - public override void Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) + public void Execute(EAParser p, Token self, MergeableGenerator tokens) { - bool flag = true; - - foreach (IParamNode parameter in parameters) + if (tokens.Current.Type != TokenType.IDENTIFIER) { - string? identifier; + p.Logger.Error(self.Location, $"Invalid use of directive '{self.Content}': expected macro name, got {tokens.Current}."); + p.IgnoreRestOfLine(tokens); + } + else + { + string identifier = tokens.Current.Content; + tokens.MoveNext(); - if (parameter.Type == ParamType.ATOM && (identifier = ((IAtomNode)parameter).GetIdentifier()) != null) - { - // TODO: Built in definitions? - bool isDefined = p.Macros.ContainsName(identifier) || p.Definitions.ContainsKey(identifier); - flag &= Inverted ? !isDefined : isDefined; - } - else + // here we could parse a potential parameter list/specifier + + bool isDefined = p.Macros.ContainsName(identifier) || p.Definitions.ContainsKey(identifier); + bool flag = Inverted ? !isDefined : isDefined; + + p.Inclusion = new ImmutableStack(flag, p.Inclusion); + + if (tokens.Current.Type != TokenType.NEWLINE) { - p.Logger.Error(parameter.MyLocation, "Definition name must be an identifier."); + p.Logger.Error(self.Location, $"Garbage at the end of directive '{self.Content}' (got {tokens.Current})."); + p.IgnoreRestOfLine(tokens); } } - - p.Inclusion = new ImmutableStack(flag, p.Inclusion); } } } diff --git a/ColorzCore/Preprocessor/Directives/PoolDirective.cs b/ColorzCore/Preprocessor/Directives/PoolDirective.cs index 194d9b4..4e98ecb 100644 --- a/ColorzCore/Preprocessor/Directives/PoolDirective.cs +++ b/ColorzCore/Preprocessor/Directives/PoolDirective.cs @@ -34,7 +34,7 @@ public override void Execute(EAParser p, Token self, IList parameter while (!tempGenerator.EOS) { - p.ParseLine(tempGenerator, line.Scope); + p.ParseLine(tempGenerator); } } diff --git a/ColorzCore/Preprocessor/Directives/SimpleDirective.cs b/ColorzCore/Preprocessor/Directives/SimpleDirective.cs index 1930349..b895a7c 100644 --- a/ColorzCore/Preprocessor/Directives/SimpleDirective.cs +++ b/ColorzCore/Preprocessor/Directives/SimpleDirective.cs @@ -26,11 +26,11 @@ public abstract class SimpleDirective : IDirective */ public abstract int? MaxParams { get; } - public void Execute(EAParser p, Token self, MergeableGenerator tokens, ImmutableStack scopes) + public void Execute(EAParser p, Token self, MergeableGenerator tokens) { // Note: Not a ParseParamList because no commas. // HACK: #if wants its parameters to be expanded, but other directives (define, ifdef, undef, etc) do not - IList parameters = p.ParsePreprocParamList(tokens, scopes, self.Content == "#if"); + IList parameters = p.ParsePreprocParamList(tokens); if (MinParams <= parameters.Count && (!MaxParams.HasValue || parameters.Count <= MaxParams)) { diff --git a/ColorzCore/Preprocessor/Directives/UndefineDirective.cs b/ColorzCore/Preprocessor/Directives/UndefineDirective.cs index 544127b..d301bfa 100644 --- a/ColorzCore/Preprocessor/Directives/UndefineDirective.cs +++ b/ColorzCore/Preprocessor/Directives/UndefineDirective.cs @@ -13,7 +13,7 @@ class UndefineDirective : IDirective { public bool RequireInclusion => true; - public void Execute(EAParser p, Token self, MergeableGenerator tokens, ImmutableStack scopes) + public void Execute(EAParser p, Token self, MergeableGenerator tokens) { if (tokens.Current.Type == TokenType.NEWLINE) { diff --git a/ColorzCore/Preprocessor/Macros/AddToPool.cs b/ColorzCore/Preprocessor/Macros/AddToPool.cs index 1bb4248..5b98b80 100644 --- a/ColorzCore/Preprocessor/Macros/AddToPool.cs +++ b/ColorzCore/Preprocessor/Macros/AddToPool.cs @@ -21,7 +21,7 @@ public AddToPool(Pool pool) Pool = pool; } - public override IEnumerable ApplyMacro(Token head, IList> parameters, ImmutableStack scopes) + public override IEnumerable ApplyMacro(Token head, IList> parameters) { List line = new List(6 + parameters[0].Count); @@ -45,7 +45,7 @@ public override IEnumerable ApplyMacro(Token head, IList> pa line.AddRange(parameters[0]); line.Add(new Token(TokenType.NEWLINE, head.Location, "\n")); - Pool.Lines.Add(new Pool.PooledLine(scopes, line)); + Pool.Lines.Add(new Pool.PooledLine(line)); yield return new Token(TokenType.IDENTIFIER, head.Location, labelName); } diff --git a/ColorzCore/Preprocessor/Macros/BuiltInMacro.cs b/ColorzCore/Preprocessor/Macros/BuiltInMacro.cs index c4f0c16..31b97b3 100644 --- a/ColorzCore/Preprocessor/Macros/BuiltInMacro.cs +++ b/ColorzCore/Preprocessor/Macros/BuiltInMacro.cs @@ -10,6 +10,6 @@ namespace ColorzCore.Preprocessor.Macros public abstract class BuiltInMacro : IMacro { public abstract bool ValidNumParams(int num); - public abstract IEnumerable ApplyMacro(Token head, IList> parameters, ImmutableStack scopes); + public abstract IEnumerable ApplyMacro(Token head, IList> parameters); } } diff --git a/ColorzCore/Preprocessor/Macros/ErrorMacro.cs b/ColorzCore/Preprocessor/Macros/ErrorMacro.cs index 498f870..460f710 100644 --- a/ColorzCore/Preprocessor/Macros/ErrorMacro.cs +++ b/ColorzCore/Preprocessor/Macros/ErrorMacro.cs @@ -23,7 +23,7 @@ public ErrorMacro(EAParser parser, string message, ValidateNumParamsIndirect val this.validateNumParamsIndirect = validateNumParamsIndirect; } - public override IEnumerable ApplyMacro(Token head, IList> parameters, ImmutableStack scopes) + public override IEnumerable ApplyMacro(Token head, IList> parameters) { parser.Logger.Error(head.Location, message); yield return new Token(TokenType.NUMBER, head.Location, "0"); diff --git a/ColorzCore/Preprocessor/Macros/IMacro.cs b/ColorzCore/Preprocessor/Macros/IMacro.cs index a631559..1aa11cf 100644 --- a/ColorzCore/Preprocessor/Macros/IMacro.cs +++ b/ColorzCore/Preprocessor/Macros/IMacro.cs @@ -11,6 +11,6 @@ namespace ColorzCore.Preprocessor.Macros { public interface IMacro { - IEnumerable ApplyMacro(Token head, IList> parameters, ImmutableStack scopes); + IEnumerable ApplyMacro(Token head, IList> parameters); } } diff --git a/ColorzCore/Preprocessor/Macros/IsDefined.cs b/ColorzCore/Preprocessor/Macros/IsDefined.cs index f75c1ac..2cd6218 100644 --- a/ColorzCore/Preprocessor/Macros/IsDefined.cs +++ b/ColorzCore/Preprocessor/Macros/IsDefined.cs @@ -15,7 +15,7 @@ public IsDefined(EAParser parent) ParentParser = parent; } - public override IEnumerable ApplyMacro(Token head, IList> parameters, ImmutableStack scopes) + public override IEnumerable ApplyMacro(Token head, IList> parameters) { if (parameters[0].Count != 1) { diff --git a/ColorzCore/Preprocessor/Macros/IsSymbolDefined.cs b/ColorzCore/Preprocessor/Macros/IsSymbolDefined.cs index bbb2949..b5531a1 100644 --- a/ColorzCore/Preprocessor/Macros/IsSymbolDefined.cs +++ b/ColorzCore/Preprocessor/Macros/IsSymbolDefined.cs @@ -2,13 +2,12 @@ using System.Collections.Generic; using ColorzCore.DataTypes; using ColorzCore.Lexer; -using ColorzCore.Parser; namespace ColorzCore.Preprocessor.Macros { public class IsSymbolDefined : BuiltInMacro { - public override IEnumerable ApplyMacro(Token head, IList> parameters, ImmutableStack scopes) + public override IEnumerable ApplyMacro(Token head, IList> parameters) { if (parameters[0].Count != 1) { @@ -17,40 +16,29 @@ public override IEnumerable ApplyMacro(Token head, IList> pa } else { - Token token = parameters[0][0]; + MacroLocation macroLocation = new MacroLocation(head.Content, head.Location); + Location location = head.Location.MacroClone(macroLocation); - if ((token.Type == TokenType.IDENTIFIER) && IsReallyDefined(scopes, token.Content)) - { - yield return MakeTrueToken(head.Location); - } - else - { - yield return MakeFalseToken(head.Location); - } - } - } + Token identifierToken = parameters[0][0]; - public override bool ValidNumParams(int num) - { - return num == 1; - } + // This used to be more involved, but now it is just a dummy + // ((id || 1) ?? 0) - protected static bool IsReallyDefined(ImmutableStack scopes, string name) - { - for (ImmutableStack it = scopes; it != ImmutableStack.Nil; it = it.Tail) - { - if (it.Head.HasLocalSymbol(name)) - { - return true; - } + yield return new Token(TokenType.OPEN_PAREN, location, "("); + yield return new Token(TokenType.OPEN_PAREN, location, "("); + yield return identifierToken.MacroClone(macroLocation); + yield return new Token(TokenType.LOGOR_OP, location, "||"); + yield return new Token(TokenType.NUMBER, location, "1"); + yield return new Token(TokenType.CLOSE_PAREN, location, ")"); + yield return new Token(TokenType.UNDEFINED_COALESCE_OP, location, "??"); + yield return new Token(TokenType.NUMBER, location, "0"); + yield return new Token(TokenType.CLOSE_PAREN, location, ")"); } - - return false; } - protected static Token MakeTrueToken(Location location) + public override bool ValidNumParams(int num) { - return new Token(TokenType.NUMBER, location, "1"); + return num == 1; } protected static Token MakeFalseToken(Location location) diff --git a/ColorzCore/Preprocessor/Macros/ReadDataAt.cs b/ColorzCore/Preprocessor/Macros/ReadDataAt.cs index 402d3f7..b91235a 100644 --- a/ColorzCore/Preprocessor/Macros/ReadDataAt.cs +++ b/ColorzCore/Preprocessor/Macros/ReadDataAt.cs @@ -22,14 +22,14 @@ public ReadDataAt(EAParser parser, ROM rom, int readLength) this.readLength = readLength; } - public override IEnumerable ApplyMacro(Token head, IList> parameters, ImmutableStack scopes) + public override IEnumerable ApplyMacro(Token head, IList> parameters) { // HACK: hack MergeableGenerator tokens = new MergeableGenerator( Enumerable.Repeat(new Token(TokenType.NEWLINE, head.Location, "\n"), 1)); tokens.PrependEnumerator(parameters[0].GetEnumerator()); - IAtomNode? atom = parser.ParseAtom(tokens, scopes, true); + IAtomNode? atom = parser.ParseAtom(tokens); if (tokens.Current.Type != TokenType.NEWLINE) { diff --git a/ColorzCore/Preprocessor/Macros/StringMacro.cs b/ColorzCore/Preprocessor/Macros/StringMacro.cs index 4e88686..ae14506 100644 --- a/ColorzCore/Preprocessor/Macros/StringMacro.cs +++ b/ColorzCore/Preprocessor/Macros/StringMacro.cs @@ -10,11 +10,10 @@ namespace ColorzCore.Preprocessor.Macros { class StringMacro : BuiltInMacro { - public override IEnumerable ApplyMacro(Token head, IList> parameters, ImmutableStack scopes) + public override IEnumerable ApplyMacro(Token head, IList> parameters) { - yield return new Token(TokenType.IDENTIFIER, head.Location, "BYTE"); - foreach (byte num in Encoding.ASCII.GetBytes(parameters[0][0].Content.ToCharArray())) //TODO: Errors if not adherent? - yield return new Token(TokenType.NUMBER, head.Location, num.ToString()); + yield return new Token(TokenType.IDENTIFIER, head.Location, "UTF8"); + yield return new Token(TokenType.STRING, parameters[0][0].Location, parameters[0][0].Content); } public override bool ValidNumParams(int num) diff --git a/ColorzCore/Preprocessor/Macros/UserMacro.cs b/ColorzCore/Preprocessor/Macros/UserMacro.cs index cb5c750..e5e8d74 100644 --- a/ColorzCore/Preprocessor/Macros/UserMacro.cs +++ b/ColorzCore/Preprocessor/Macros/UserMacro.cs @@ -27,7 +27,7 @@ public UserMacro(IList parameters, IList macroBody) /*** * Precondition: parameters.Count = max(keys(idToParamNum)) */ - public IEnumerable ApplyMacro(Token head, IList> parameters, ImmutableStack scopes) + public IEnumerable ApplyMacro(Token head, IList> parameters) { MacroLocation macroLocation = new MacroLocation(head.Content, head.Location); From 0b10055181cd8a4f6d1cb5690ec280b05067b106 Mon Sep 17 00:00:00 2001 From: Nat_776 Date: Sat, 4 May 2024 12:03:04 +0200 Subject: [PATCH 50/59] fix compile, UTF8 -> STRING, replace error with warning for some specific ASSERTs that got broken with CURRENTOFFSET changes --- ColorzCore/Parser/AST/NumberNode.cs | 13 ++-- .../Parser/Diagnostics/DiagnosticsHelpers.cs | 16 +++++ ColorzCore/Parser/EAParseConsumer.cs | 20 +++++- ColorzCore/Parser/EAParser.cs | 65 +++++++++++++++---- ColorzCore/Preprocessor/Macros/StringMacro.cs | 11 +++- 5 files changed, 103 insertions(+), 22 deletions(-) diff --git a/ColorzCore/Parser/AST/NumberNode.cs b/ColorzCore/Parser/AST/NumberNode.cs index a6ca3d0..52c74b3 100644 --- a/ColorzCore/Parser/AST/NumberNode.cs +++ b/ColorzCore/Parser/AST/NumberNode.cs @@ -10,24 +10,27 @@ namespace ColorzCore.Parser.AST { public class NumberNode : AtomNodeKernel { + public Token SourceToken { get; } public int Value { get; } - public override Location MyLocation { get; } - public override int Precedence { get { return 11; } } + public override Location MyLocation => SourceToken.Location; + public override int Precedence => int.MaxValue; public NumberNode(Token num) { - MyLocation = num.Location; + SourceToken = num; Value = num.Content.ToInt(); } + public NumberNode(Token text, int value) { - MyLocation = text.Location; + SourceToken = text; Value = value; } + public NumberNode(Location loc, int value) { - MyLocation = loc; + SourceToken = new Token(TokenType.NUMBER, loc, value.ToString()); Value = value; } diff --git a/ColorzCore/Parser/Diagnostics/DiagnosticsHelpers.cs b/ColorzCore/Parser/Diagnostics/DiagnosticsHelpers.cs index 3ae6196..b212ea7 100644 --- a/ColorzCore/Parser/Diagnostics/DiagnosticsHelpers.cs +++ b/ColorzCore/Parser/Diagnostics/DiagnosticsHelpers.cs @@ -211,6 +211,7 @@ public static bool DoesOperationSpanMultipleMacrosUnintuitively(OperatorNode ope }; // absolute as in "not relative to the value of a symbol" + // NOTE: currently unused (I considered using this for SetSymbol detection) public static bool IsAbsoluteAtom(IAtomNode node) => node switch { IdentifierNode => false, @@ -231,5 +232,20 @@ public static bool DoesOperationSpanMultipleMacrosUnintuitively(OperatorNode ope _ => true, }; + + public static bool IsSubtractionOfCurrentOffset(IAtomNode node) + { + if (node is OperatorNode operatorNode && operatorNode.OperatorToken.Type == TokenType.SUB_OP) + { + // the "CURRENTOFFSET" node is a number node whose source token is an identifier + if (operatorNode.Right is NumberNode numberNode) + { + Token token = numberNode.SourceToken; + return token.Type == TokenType.IDENTIFIER && token.Content.ToUpperInvariant() == "CURRENTOFFSET"; + } + } + + return false; + } } } diff --git a/ColorzCore/Parser/EAParseConsumer.cs b/ColorzCore/Parser/EAParseConsumer.cs index 57d615f..aa577ad 100644 --- a/ColorzCore/Parser/EAParseConsumer.cs +++ b/ColorzCore/Parser/EAParseConsumer.cs @@ -4,6 +4,7 @@ using ColorzCore.IO; using ColorzCore.Lexer; using ColorzCore.Parser.AST; +using ColorzCore.Parser.Diagnostics; using ColorzCore.Raws; namespace ColorzCore.Parser @@ -203,6 +204,7 @@ static bool IsBooleanResultHelper(IAtomNode node) } bool isBoolean = IsBooleanResultHelper(node); + bool isSubtractionOfCurrentOffset = DiagnosticsHelpers.IsSubtractionOfCurrentOffset(node); if (EvaluteAtom(node) is int result) { @@ -212,7 +214,23 @@ static bool IsBooleanResultHelper(IAtomNode node) } else if (!isBoolean && result < 0) { - Logger.Error(location, $"Assertion failed with value {result}."); + /* users may do something like ASSERT UpperBound - CURRENTOFFSET + * If UpperBound is an offset rather than an address, this will now certainly fail. + * as CURRENTOFFSET was changed to be expanded into an address rather than a ROM offset. + * we do not want this to break too hard so we try to emit a warning rather than an error. */ + + if (isSubtractionOfCurrentOffset && result + EAOptions.BaseAddress >= 0) + { + Logger.Warning(location, + $"Assertion would fail with value {result} if CURRENTOFFSET is treated as an address.\n" + + "ColorzCore was recently changed to work in terms of addresses rather that ROM offsets.\n" + + "Consider changing the left operand to an address, or if you need to keep compatibility,\n" + + "You can also bitwise AND CURRENTOFFSET to keep it within the desired bounds."); + } + else + { + Logger.Error(location, $"Assertion failed with value {result}."); + } } } else diff --git a/ColorzCore/Parser/EAParser.cs b/ColorzCore/Parser/EAParser.cs index 44978ac..e979218 100644 --- a/ColorzCore/Parser/EAParser.cs +++ b/ColorzCore/Parser/EAParser.cs @@ -106,7 +106,7 @@ public void ParseAll(IEnumerable tokenStream) "PROTECT", "ALIGN", "FILL", - "UTF8", // TODO: remove en favor of generic STRING with .tbl support + "STRING", "BASE64", // "SECTION", // TODO // "DSECTION", // TODO @@ -190,8 +190,8 @@ private void ParseStatement(MergeableGenerator tokens) ParseErrorStatement(head, parameters); break; - case "UTF8": - ParseUtf8Statement(head, parameters); + case "STRING": + ParseStringStatement(head, parameters); break; case "BASE64": @@ -402,25 +402,62 @@ private void ParseErrorStatement(Token head, IList parameters) Logger.Error(head.Location, PrettyPrintParamsForMessage(parameters)); } - private void ParseUtf8Statement(Token head, IList parameters) - { - using MemoryStream memoryStream = new MemoryStream(); - foreach (IParamNode parameter in parameters) + private void ParseStringStatement(Token head, IList parameters) + { + void HandleStringStatement(Token head, string inputString, string? encodingName) { - if (parameter is StringNode node) + if (encodingName != null && encodingName != "UTF-8") { - byte[] utf8Bytes = Encoding.UTF8.GetBytes(node.Value); - memoryStream.Write(utf8Bytes, 0, utf8Bytes.Length); + Logger.Error(head.Location, + $"Custom STRING encoding is not yet supported (requested encoding: `{encodingName}`."); } else { - Logger.Error(head.Location, $"expects a String (got {DiagnosticsHelpers.PrettyParamType(parameter.Type)})."); + byte[] utf8Bytes = Encoding.UTF8.GetBytes(inputString); + ParseConsumer.OnData(head.Location, utf8Bytes); } } - byte[] bytes = memoryStream.ToArray(); - ParseConsumer.OnData(head.Location, bytes); + // NOTE: this is copy-pasted from ParseStatementTwoParam but adjusted for string param + + if (parameters.Count == 1) + { + if (parameters[0] is StringNode firstNode) + { + HandleStringStatement(head, firstNode.Value, null); + } + else + { + Logger.Error(parameters[0].MyLocation, + $"STRING expects a String (got {DiagnosticsHelpers.PrettyParamType(parameters[0].Type)})."); + } + } + else if (parameters.Count == 2) + { + if (parameters[0] is StringNode firstNode) + { + if (parameters[1] is StringNode secondNode) + { + HandleStringStatement(head, firstNode.Value, secondNode.Value); + } + else + { + Logger.Error(parameters[1].MyLocation, + $"STRING expects a String (got {DiagnosticsHelpers.PrettyParamType(parameters[1].Type)})."); + } + } + else + { + Logger.Error(parameters[0].MyLocation, + $"STRING expects a String (got {DiagnosticsHelpers.PrettyParamType(parameters[0].Type)})."); + } + } + else + { + Logger.Error(head.Location, + $"A STRING statement expects 1 to 2 parameters, got {parameters.Count}."); + } } private void ParseBase64Statement(Token head, IList parameters) @@ -534,7 +571,7 @@ public IList ParsePreprocParamList(MergeableGenerator tokens) if (temp[i] is StringNode stringNode && stringNode.IsValidIdentifier()) { // TODO: what is this for? can we omit it? - temp[i] = BindIdentifier(stringNode.SourceToken); + temp[i] = BindIdentifier(stringNode.MyToken); } } diff --git a/ColorzCore/Preprocessor/Macros/StringMacro.cs b/ColorzCore/Preprocessor/Macros/StringMacro.cs index ae14506..94558d2 100644 --- a/ColorzCore/Preprocessor/Macros/StringMacro.cs +++ b/ColorzCore/Preprocessor/Macros/StringMacro.cs @@ -12,8 +12,15 @@ class StringMacro : BuiltInMacro { public override IEnumerable ApplyMacro(Token head, IList> parameters) { - yield return new Token(TokenType.IDENTIFIER, head.Location, "UTF8"); - yield return new Token(TokenType.STRING, parameters[0][0].Location, parameters[0][0].Content); + MacroLocation macroLocation = new MacroLocation(head.Content, head.Location); + + Token token = parameters[0][0]; + Location location = token.Location.MacroClone(macroLocation); + + yield return new Token(TokenType.IDENTIFIER, location, "STRING"); + yield return new Token(TokenType.STRING, location, token.Content); + yield return new Token(TokenType.STRING, location, "UTF-8"); + yield return new Token(TokenType.SEMICOLON, location, ";"); } public override bool ValidNumParams(int num) From e0072f076f4b11da2be2aabb8a2ca487acd17a4b Mon Sep 17 00:00:00 2001 From: Nat_776 Date: Sat, 4 May 2024 12:34:58 +0200 Subject: [PATCH 51/59] put parts Parser goes into new Interpreter namespace. rename EAInterpreter to EADriver and EAParseConsumer to EAInterpreter --- ColorzCore/{EAInterpreter.cs => EADriver.cs} | 17 +++--- ColorzCore/IO/ASM.cs | 10 ++-- .../{Parser => Interpreter}/AtomVisitor.cs | 2 +- .../{Parser => Interpreter}/BaseClosure.cs | 2 +- ColorzCore/{Parser => Interpreter}/Closure.cs | 2 +- .../Diagnostics/DiagnosticsHelpers.cs | 3 +- .../Diagnostics/SetSymbolMacroDetector.cs | 5 +- .../EAInterpreter.cs} | 17 +++--- .../EvaluationPhase.cs | 2 +- ColorzCore/Parser/AST/AtomNodeKernel.cs | 3 +- ColorzCore/Parser/AST/DataNode.cs | 10 ++-- ColorzCore/Parser/AST/IAtomNode.cs | 3 +- ColorzCore/Parser/AST/ILineNode.cs | 2 +- ColorzCore/Parser/AST/IParamNode.cs | 1 + ColorzCore/Parser/AST/IdentifierNode.cs | 58 +++++++++++-------- ColorzCore/Parser/AST/ListNode.cs | 1 + ColorzCore/Parser/AST/MacroInvocationNode.cs | 1 + ColorzCore/Parser/AST/NumberNode.cs | 3 +- ColorzCore/Parser/AST/OperatorNode.cs | 2 +- ColorzCore/Parser/AST/RawNode.cs | 1 + ColorzCore/Parser/AST/StringNode.cs | 22 +++---- ColorzCore/Parser/AST/UnaryOperatorNode.cs | 1 + ColorzCore/Parser/AtomParser.cs | 2 +- ColorzCore/Parser/EAParser.cs | 7 ++- .../Preprocessor/Directives/IfDirective.cs | 1 + .../Directives/IncludeExternalDirective.cs | 1 + .../Directives/IncludeToolEventDirective.cs | 1 + .../Directives/UndefineDirective.cs | 1 - ColorzCore/Preprocessor/Macros/ReadDataAt.cs | 9 +-- ColorzCore/Program.cs | 6 +- ColorzCore/Raws/AtomicParam.cs | 4 +- 31 files changed, 104 insertions(+), 96 deletions(-) rename ColorzCore/{EAInterpreter.cs => EADriver.cs} (92%) rename ColorzCore/{Parser => Interpreter}/AtomVisitor.cs (96%) rename ColorzCore/{Parser => Interpreter}/BaseClosure.cs (86%) rename ColorzCore/{Parser => Interpreter}/Closure.cs (95%) rename ColorzCore/{Parser => Interpreter}/Diagnostics/DiagnosticsHelpers.cs (99%) rename ColorzCore/{Parser => Interpreter}/Diagnostics/SetSymbolMacroDetector.cs (97%) rename ColorzCore/{Parser/EAParseConsumer.cs => Interpreter/EAInterpreter.cs} (97%) rename ColorzCore/{Parser => Interpreter}/EvaluationPhase.cs (92%) diff --git a/ColorzCore/EAInterpreter.cs b/ColorzCore/EADriver.cs similarity index 92% rename from ColorzCore/EAInterpreter.cs rename to ColorzCore/EADriver.cs index e15ab29..4a85845 100644 --- a/ColorzCore/EAInterpreter.cs +++ b/ColorzCore/EADriver.cs @@ -1,9 +1,10 @@ using ColorzCore.DataTypes; +using ColorzCore.Interpreter; +using ColorzCore.Interpreter.Diagnostics; using ColorzCore.IO; using ColorzCore.Lexer; using ColorzCore.Parser; using ColorzCore.Parser.AST; -using ColorzCore.Parser.Diagnostics; using ColorzCore.Preprocessor; using ColorzCore.Preprocessor.Directives; using ColorzCore.Preprocessor.Macros; @@ -15,20 +16,18 @@ namespace ColorzCore { - // Class to excapsulate all steps in EA script interpretation. - // TODO: this is just another Program - // the name is problematic as EAParseConsumer would really like to be called EAInterpreter - class EAInterpreter + // Class to excapsulate all steps in EA script interpretation (lexing -> parsing -> interpretation -> commit). + class EADriver { private Dictionary> allRaws; private EAParser myParser; - private EAParseConsumer myInterpreter; + private EAInterpreter myInterpreter; private string iFile; private Stream sin; private Logger Logger { get; } private IOutput output; - public EAInterpreter(IOutput output, string? game, string? rawsFolder, string rawsExtension, Stream sin, string inFileName, Logger logger) + public EADriver(IOutput output, string? game, string? rawsFolder, string rawsExtension, Stream sin, string inFileName, Logger logger) { this.output = output; @@ -50,7 +49,7 @@ public EAInterpreter(IOutput output, string? game, string? rawsFolder, string ra Logger = logger; iFile = inFileName; - myInterpreter = new EAParseConsumer(logger); + myInterpreter = new EAInterpreter(logger); ParseConsumerChain parseConsumers = new ParseConsumerChain(); @@ -234,7 +233,7 @@ public bool WriteNocashSymbols(TextWriter output) foreach (KeyValuePair pair in scope.LocalSymbols()) { string name = pair.Key; - int address = EAParseConsumer.ConvertToAddress(pair.Value); + int address = EAInterpreter.ConvertToAddress(pair.Value); output.WriteLine($"{address:X8} {manglePrefix}{name}"); } diff --git a/ColorzCore/IO/ASM.cs b/ColorzCore/IO/ASM.cs index a0c3df0..66f5cc8 100644 --- a/ColorzCore/IO/ASM.cs +++ b/ColorzCore/IO/ASM.cs @@ -2,9 +2,7 @@ using System.IO; using System.Collections.Generic; using System.Linq; -using System.Text; -using System.Threading.Tasks; -using ColorzCore.Parser; +using ColorzCore.Interpreter; namespace ColorzCore.IO { @@ -20,7 +18,7 @@ public ASM(StreamWriter asmStream, StreamWriter ldsStream) private void WriteToASM(int position, byte[] data) { - string sectionName = $".ea_{EAParseConsumer.ConvertToAddress(position):x}"; + string sectionName = $".ea_{EAInterpreter.ConvertToAddress(position):x}"; asmStream.WriteLine($".section {sectionName},\"ax\",%progbits"); asmStream.WriteLine($".global {sectionName}"); asmStream.WriteLine($"{sectionName}:"); @@ -32,8 +30,8 @@ private void WriteToASM(int position, byte[] data) private void WriteToLDS(int position) { - string sectionName = $".ea_{EAParseConsumer.ConvertToAddress(position):x}"; - ldsStream.WriteLine($". = 0x{EAParseConsumer.ConvertToAddress(position):x};"); + string sectionName = $".ea_{EAInterpreter.ConvertToAddress(position):x}"; + ldsStream.WriteLine($". = 0x{EAInterpreter.ConvertToAddress(position):x};"); ldsStream.WriteLine($"{sectionName} : {{*.o({sectionName})}}"); } diff --git a/ColorzCore/Parser/AtomVisitor.cs b/ColorzCore/Interpreter/AtomVisitor.cs similarity index 96% rename from ColorzCore/Parser/AtomVisitor.cs rename to ColorzCore/Interpreter/AtomVisitor.cs index fc1ac9f..023c73c 100644 --- a/ColorzCore/Parser/AtomVisitor.cs +++ b/ColorzCore/Interpreter/AtomVisitor.cs @@ -1,7 +1,7 @@ using ColorzCore.Parser.AST; -namespace ColorzCore.Parser +namespace ColorzCore.Interpreter { public abstract class AtomVisitor { diff --git a/ColorzCore/Parser/BaseClosure.cs b/ColorzCore/Interpreter/BaseClosure.cs similarity index 86% rename from ColorzCore/Parser/BaseClosure.cs rename to ColorzCore/Interpreter/BaseClosure.cs index 0f69b04..5fbe44c 100644 --- a/ColorzCore/Parser/BaseClosure.cs +++ b/ColorzCore/Interpreter/BaseClosure.cs @@ -1,4 +1,4 @@ -namespace ColorzCore.Parser +namespace ColorzCore.Interpreter { class BaseClosure : Closure { diff --git a/ColorzCore/Parser/Closure.cs b/ColorzCore/Interpreter/Closure.cs similarity index 95% rename from ColorzCore/Parser/Closure.cs rename to ColorzCore/Interpreter/Closure.cs index 268a12c..c2aa981 100644 --- a/ColorzCore/Parser/Closure.cs +++ b/ColorzCore/Interpreter/Closure.cs @@ -2,7 +2,7 @@ using System.Collections.Generic; using ColorzCore.Parser.AST; -namespace ColorzCore.Parser +namespace ColorzCore.Interpreter { public class Closure { diff --git a/ColorzCore/Parser/Diagnostics/DiagnosticsHelpers.cs b/ColorzCore/Interpreter/Diagnostics/DiagnosticsHelpers.cs similarity index 99% rename from ColorzCore/Parser/Diagnostics/DiagnosticsHelpers.cs rename to ColorzCore/Interpreter/Diagnostics/DiagnosticsHelpers.cs index b212ea7..c05fe21 100644 --- a/ColorzCore/Parser/Diagnostics/DiagnosticsHelpers.cs +++ b/ColorzCore/Interpreter/Diagnostics/DiagnosticsHelpers.cs @@ -3,9 +3,10 @@ using System.Text; using ColorzCore.DataTypes; using ColorzCore.Lexer; +using ColorzCore.Parser; using ColorzCore.Parser.AST; -namespace ColorzCore.Parser.Diagnostics +namespace ColorzCore.Interpreter.Diagnostics { public static class DiagnosticsHelpers { diff --git a/ColorzCore/Parser/Diagnostics/SetSymbolMacroDetector.cs b/ColorzCore/Interpreter/Diagnostics/SetSymbolMacroDetector.cs similarity index 97% rename from ColorzCore/Parser/Diagnostics/SetSymbolMacroDetector.cs rename to ColorzCore/Interpreter/Diagnostics/SetSymbolMacroDetector.cs index 0752a6f..30bf2e7 100644 --- a/ColorzCore/Parser/Diagnostics/SetSymbolMacroDetector.cs +++ b/ColorzCore/Interpreter/Diagnostics/SetSymbolMacroDetector.cs @@ -1,11 +1,12 @@ - +using System; using System.Collections.Generic; using ColorzCore.DataTypes; using ColorzCore.IO; +using ColorzCore.Parser; using ColorzCore.Parser.AST; using ColorzCore.Raws; -namespace ColorzCore.Parser.Diagnostics +namespace ColorzCore.Interpreter.Diagnostics { public class SetSymbolMacroDetector : IParseConsumer { diff --git a/ColorzCore/Parser/EAParseConsumer.cs b/ColorzCore/Interpreter/EAInterpreter.cs similarity index 97% rename from ColorzCore/Parser/EAParseConsumer.cs rename to ColorzCore/Interpreter/EAInterpreter.cs index aa577ad..6080680 100644 --- a/ColorzCore/Parser/EAParseConsumer.cs +++ b/ColorzCore/Interpreter/EAInterpreter.cs @@ -3,13 +3,14 @@ using ColorzCore.DataTypes; using ColorzCore.IO; using ColorzCore.Lexer; +using ColorzCore.Parser; using ColorzCore.Parser.AST; -using ColorzCore.Parser.Diagnostics; +using ColorzCore.Interpreter.Diagnostics; using ColorzCore.Raws; -namespace ColorzCore.Parser +namespace ColorzCore.Interpreter { - public class EAParseConsumer : IParseConsumer + public class EAInterpreter : IParseConsumer { public int CurrentOffset => currentOffset; @@ -25,12 +26,14 @@ public class EAParseConsumer : IParseConsumer private bool offsetInitialized; // false until first ORG, used for diagnostics private int currentOffset; - Logger Logger { get; } + public Logger Logger { get; } - public EAParseConsumer(Logger logger) + public EAInterpreter(Logger logger) { - AllScopes = new List() { new BaseClosure() }; - CurrentScope = GlobalScope = new ImmutableStack(AllScopes.First()!, ImmutableStack.Nil); + Closure headScope = new BaseClosure(); + AllScopes = new List() { headScope }; + GlobalScope = new ImmutableStack(headScope, ImmutableStack.Nil); + CurrentScope = GlobalScope; pastOffsets = new Stack<(int, bool)>(); protectedRegions = new List<(int, int, Location)>(); currentOffset = 0; diff --git a/ColorzCore/Parser/EvaluationPhase.cs b/ColorzCore/Interpreter/EvaluationPhase.cs similarity index 92% rename from ColorzCore/Parser/EvaluationPhase.cs rename to ColorzCore/Interpreter/EvaluationPhase.cs index e96fb8e..3bda589 100644 --- a/ColorzCore/Parser/EvaluationPhase.cs +++ b/ColorzCore/Interpreter/EvaluationPhase.cs @@ -1,6 +1,6 @@ using System; -namespace ColorzCore.Parser +namespace ColorzCore.Interpreter { public enum EvaluationPhase { diff --git a/ColorzCore/Parser/AST/AtomNodeKernel.cs b/ColorzCore/Parser/AST/AtomNodeKernel.cs index fb5ff9e..5c6dcb9 100644 --- a/ColorzCore/Parser/AST/AtomNodeKernel.cs +++ b/ColorzCore/Parser/AST/AtomNodeKernel.cs @@ -1,9 +1,8 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Text; -using System.Threading.Tasks; using ColorzCore.DataTypes; +using ColorzCore.Interpreter; using ColorzCore.Lexer; namespace ColorzCore.Parser.AST diff --git a/ColorzCore/Parser/AST/DataNode.cs b/ColorzCore/Parser/AST/DataNode.cs index f0dad5c..e0d6af2 100644 --- a/ColorzCore/Parser/AST/DataNode.cs +++ b/ColorzCore/Parser/AST/DataNode.cs @@ -1,18 +1,16 @@ using ColorzCore.DataTypes; +using ColorzCore.Interpreter; using ColorzCore.IO; -using ColorzCore.Lexer; using System; using System.Collections.Generic; using System.Linq; -using System.Text; -using System.Threading.Tasks; namespace ColorzCore.Parser.AST { class DataNode : ILineNode { - private int offset; - private byte[] data; + private readonly int offset; + private readonly byte[] data; public DataNode(int offset, byte[] data) { @@ -24,7 +22,7 @@ public DataNode(int offset, byte[] data) public string PrettyPrint(int indentation) { - return String.Format("Raw Data Block of Length {0}", Size); + return $"Raw Data Block of Length {Size}"; } public void WriteData(IOutput output) diff --git a/ColorzCore/Parser/AST/IAtomNode.cs b/ColorzCore/Parser/AST/IAtomNode.cs index 31a6294..5fd98df 100644 --- a/ColorzCore/Parser/AST/IAtomNode.cs +++ b/ColorzCore/Parser/AST/IAtomNode.cs @@ -1,10 +1,9 @@ using ColorzCore.DataTypes; +using ColorzCore.Interpreter; using ColorzCore.Lexer; using System; using System.Collections.Generic; using System.Linq; -using System.Text; -using System.Threading.Tasks; namespace ColorzCore.Parser.AST { diff --git a/ColorzCore/Parser/AST/ILineNode.cs b/ColorzCore/Parser/AST/ILineNode.cs index 8a59918..7544a00 100644 --- a/ColorzCore/Parser/AST/ILineNode.cs +++ b/ColorzCore/Parser/AST/ILineNode.cs @@ -1,6 +1,6 @@ using ColorzCore.DataTypes; +using ColorzCore.Interpreter; using ColorzCore.IO; -using ColorzCore.Lexer; using System; using System.Collections.Generic; using System.Linq; diff --git a/ColorzCore/Parser/AST/IParamNode.cs b/ColorzCore/Parser/AST/IParamNode.cs index 9f0e7aa..e04e7db 100644 --- a/ColorzCore/Parser/AST/IParamNode.cs +++ b/ColorzCore/Parser/AST/IParamNode.cs @@ -5,6 +5,7 @@ using System.Text; using System.Threading.Tasks; using ColorzCore.Lexer; +using ColorzCore.Interpreter; namespace ColorzCore.Parser.AST { diff --git a/ColorzCore/Parser/AST/IdentifierNode.cs b/ColorzCore/Parser/AST/IdentifierNode.cs index bd571f9..f717bde 100644 --- a/ColorzCore/Parser/AST/IdentifierNode.cs +++ b/ColorzCore/Parser/AST/IdentifierNode.cs @@ -1,4 +1,5 @@ using ColorzCore.DataTypes; +using ColorzCore.Interpreter; using ColorzCore.Lexer; using System; using System.Collections.Generic; @@ -8,29 +9,33 @@ namespace ColorzCore.Parser.AST { public class IdentifierNode : AtomNodeKernel { - private Token identifier; - readonly ImmutableStack scope; + private readonly ImmutableStack boundScope; - public override int Precedence { get { return 11; } } - public override Location MyLocation { get { return identifier.Location; } } + public Token IdentifierToken { get; } - public IdentifierNode(Token id, ImmutableStack scopes) + public override int Precedence => int.MaxValue; + public override Location MyLocation => IdentifierToken.Location; + + public IdentifierNode(Token identifierToken, ImmutableStack scope) { - identifier = id; - scope = scopes; + IdentifierToken = identifierToken; + boundScope = scope; } private int ToInt(EvaluationPhase evaluationPhase) { - ImmutableStack temp = scope; - while (!temp.IsEmpty) + if (boundScope != null) { - if (temp.Head.HasLocalSymbol(identifier.Content)) - return temp.Head.GetSymbol(identifier.Content, evaluationPhase); - else - temp = temp.Tail; + for (ImmutableStack it = boundScope; !it.IsEmpty; it = it.Tail) + { + if (it.Head.HasLocalSymbol(IdentifierToken.Content)) + { + return it.Head.GetSymbol(IdentifierToken.Content, evaluationPhase); + } + } } - throw new UndefinedIdentifierException(identifier); + + throw new UndefinedIdentifierException(IdentifierToken); } public override int? TryEvaluate(Action handler, EvaluationPhase evaluationPhase) @@ -53,7 +58,7 @@ private int ToInt(EvaluationPhase evaluationPhase) public override string? GetIdentifier() { - return identifier.Content; + return IdentifierToken.Content; } public override string PrettyPrint() @@ -64,28 +69,33 @@ public override string PrettyPrint() } catch (UndefinedIdentifierException) { - return identifier.Content; + return IdentifierToken.Content; } catch (Closure.SymbolComputeException) { - return identifier.Content; + return IdentifierToken.Content; } } - public override IEnumerable ToTokens() { yield return identifier; } + public override string ToString() + { + return IdentifierToken.Content; + } + public override IEnumerable ToTokens() + { + yield return IdentifierToken; + } + + // TODO: move this outside of this class public class UndefinedIdentifierException : Exception { public Token CausedError { get; set; } + public UndefinedIdentifierException(Token causedError) : base($"Undefined identifier `{causedError.Content}`") { - this.CausedError = causedError; + CausedError = causedError; } } - - public override string ToString() - { - return identifier.Content; - } } } diff --git a/ColorzCore/Parser/AST/ListNode.cs b/ColorzCore/Parser/AST/ListNode.cs index 41d3cf1..ee873f8 100644 --- a/ColorzCore/Parser/AST/ListNode.cs +++ b/ColorzCore/Parser/AST/ListNode.cs @@ -1,4 +1,5 @@ using ColorzCore.DataTypes; +using ColorzCore.Interpreter; using ColorzCore.Lexer; using System; using System.Collections.Generic; diff --git a/ColorzCore/Parser/AST/MacroInvocationNode.cs b/ColorzCore/Parser/AST/MacroInvocationNode.cs index d743686..0e73875 100644 --- a/ColorzCore/Parser/AST/MacroInvocationNode.cs +++ b/ColorzCore/Parser/AST/MacroInvocationNode.cs @@ -1,4 +1,5 @@ using ColorzCore.DataTypes; +using ColorzCore.Interpreter; using ColorzCore.Lexer; using System; using System.Collections.Generic; diff --git a/ColorzCore/Parser/AST/NumberNode.cs b/ColorzCore/Parser/AST/NumberNode.cs index 52c74b3..b694c08 100644 --- a/ColorzCore/Parser/AST/NumberNode.cs +++ b/ColorzCore/Parser/AST/NumberNode.cs @@ -2,9 +2,8 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Text; -using System.Threading.Tasks; using ColorzCore.DataTypes; +using ColorzCore.Interpreter; namespace ColorzCore.Parser.AST { diff --git a/ColorzCore/Parser/AST/OperatorNode.cs b/ColorzCore/Parser/AST/OperatorNode.cs index d848cf9..7354920 100644 --- a/ColorzCore/Parser/AST/OperatorNode.cs +++ b/ColorzCore/Parser/AST/OperatorNode.cs @@ -1,6 +1,6 @@ using ColorzCore.DataTypes; +using ColorzCore.Interpreter; using ColorzCore.Lexer; -using ColorzCore.Parser; using System; using System.Collections.Generic; using System.Linq; diff --git a/ColorzCore/Parser/AST/RawNode.cs b/ColorzCore/Parser/AST/RawNode.cs index 5b9f3f9..5005f06 100644 --- a/ColorzCore/Parser/AST/RawNode.cs +++ b/ColorzCore/Parser/AST/RawNode.cs @@ -1,4 +1,5 @@ using ColorzCore.DataTypes; +using ColorzCore.Interpreter; using ColorzCore.IO; using ColorzCore.Lexer; using ColorzCore.Raws; diff --git a/ColorzCore/Parser/AST/StringNode.cs b/ColorzCore/Parser/AST/StringNode.cs index 4b30e33..65a7102 100644 --- a/ColorzCore/Parser/AST/StringNode.cs +++ b/ColorzCore/Parser/AST/StringNode.cs @@ -1,4 +1,5 @@ using ColorzCore.DataTypes; +using ColorzCore.Interpreter; using ColorzCore.Lexer; using System; using System.Collections.Generic; @@ -12,21 +13,17 @@ namespace ColorzCore.Parser.AST public class StringNode : IParamNode { public static readonly Regex idRegex = new Regex("^([a-zA-Z_][a-zA-Z0-9_]*)$"); - public Token MyToken { get; } - public Location MyLocation => MyToken.Location; + public Token SourceToken { get; } + + public Location MyLocation => SourceToken.Location; public ParamType Type => ParamType.STRING; - public string Value => MyToken.Content; + public string Value => SourceToken.Content; public StringNode(Token value) { - MyToken = value; - } - - public IEnumerable ToBytes() - { - return Encoding.ASCII.GetBytes(ToString()); + SourceToken = value; } public override string ToString() @@ -39,18 +36,13 @@ public string PrettyPrint() return $"\"{Value}\""; } - public IEnumerable ToTokens() { yield return MyToken; } + public IEnumerable ToTokens() { yield return SourceToken; } public bool IsValidIdentifier() { return idRegex.IsMatch(Value); } - public IdentifierNode ToIdentifier(ImmutableStack scope) - { - return new IdentifierNode(MyToken, scope); - } - public IAtomNode? AsAtom() => null; public IParamNode SimplifyExpressions(Action handler, EvaluationPhase evaluationPhase) => this; } diff --git a/ColorzCore/Parser/AST/UnaryOperatorNode.cs b/ColorzCore/Parser/AST/UnaryOperatorNode.cs index 06a140c..c7d2ea8 100644 --- a/ColorzCore/Parser/AST/UnaryOperatorNode.cs +++ b/ColorzCore/Parser/AST/UnaryOperatorNode.cs @@ -4,6 +4,7 @@ using System.Text; using System.Threading.Tasks; using ColorzCore.DataTypes; +using ColorzCore.Interpreter; using ColorzCore.Lexer; namespace ColorzCore.Parser.AST diff --git a/ColorzCore/Parser/AtomParser.cs b/ColorzCore/Parser/AtomParser.cs index ead4f6f..312f54b 100644 --- a/ColorzCore/Parser/AtomParser.cs +++ b/ColorzCore/Parser/AtomParser.cs @@ -1,9 +1,9 @@ using System.Collections.Generic; using ColorzCore.DataTypes; +using ColorzCore.Interpreter.Diagnostics; using ColorzCore.IO; using ColorzCore.Lexer; using ColorzCore.Parser.AST; -using ColorzCore.Parser.Diagnostics; namespace ColorzCore.Parser { diff --git a/ColorzCore/Parser/EAParser.cs b/ColorzCore/Parser/EAParser.cs index e979218..5be66b6 100644 --- a/ColorzCore/Parser/EAParser.cs +++ b/ColorzCore/Parser/EAParser.cs @@ -1,8 +1,9 @@ using ColorzCore.DataTypes; +using ColorzCore.Interpreter; +using ColorzCore.Interpreter.Diagnostics; using ColorzCore.IO; using ColorzCore.Lexer; using ColorzCore.Parser.AST; -using ColorzCore.Parser.Diagnostics; using ColorzCore.Preprocessor; using ColorzCore.Preprocessor.Macros; using ColorzCore.Raws; @@ -571,7 +572,7 @@ public IList ParsePreprocParamList(MergeableGenerator tokens) if (temp[i] is StringNode stringNode && stringNode.IsValidIdentifier()) { // TODO: what is this for? can we omit it? - temp[i] = BindIdentifier(stringNode.MyToken); + temp[i] = BindIdentifier(stringNode.SourceToken); } } @@ -928,7 +929,7 @@ string UserFormatStringError(Location loc, string message, string details) } if (node.TryEvaluate(e => Logger.Error(node.MyLocation, e.Message), - EvaluationPhase.Early) is int value) + EvaluationPhase.Immediate) is int value) { try { diff --git a/ColorzCore/Preprocessor/Directives/IfDirective.cs b/ColorzCore/Preprocessor/Directives/IfDirective.cs index c864f6a..459b24c 100644 --- a/ColorzCore/Preprocessor/Directives/IfDirective.cs +++ b/ColorzCore/Preprocessor/Directives/IfDirective.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Linq; using ColorzCore.DataTypes; +using ColorzCore.Interpreter; using ColorzCore.IO; using ColorzCore.Lexer; using ColorzCore.Parser; diff --git a/ColorzCore/Preprocessor/Directives/IncludeExternalDirective.cs b/ColorzCore/Preprocessor/Directives/IncludeExternalDirective.cs index f5469ef..522bdcc 100644 --- a/ColorzCore/Preprocessor/Directives/IncludeExternalDirective.cs +++ b/ColorzCore/Preprocessor/Directives/IncludeExternalDirective.cs @@ -8,6 +8,7 @@ using ColorzCore.Parser.AST; using ColorzCore.IO; using System.IO; +using ColorzCore.Interpreter; namespace ColorzCore.Preprocessor.Directives { diff --git a/ColorzCore/Preprocessor/Directives/IncludeToolEventDirective.cs b/ColorzCore/Preprocessor/Directives/IncludeToolEventDirective.cs index 7ea7e0f..49b8c74 100644 --- a/ColorzCore/Preprocessor/Directives/IncludeToolEventDirective.cs +++ b/ColorzCore/Preprocessor/Directives/IncludeToolEventDirective.cs @@ -8,6 +8,7 @@ using ColorzCore.Parser.AST; using ColorzCore.IO; using System.IO; +using ColorzCore.Interpreter; namespace ColorzCore.Preprocessor.Directives { diff --git a/ColorzCore/Preprocessor/Directives/UndefineDirective.cs b/ColorzCore/Preprocessor/Directives/UndefineDirective.cs index d301bfa..21a1c4e 100644 --- a/ColorzCore/Preprocessor/Directives/UndefineDirective.cs +++ b/ColorzCore/Preprocessor/Directives/UndefineDirective.cs @@ -5,7 +5,6 @@ using ColorzCore.IO; using ColorzCore.Lexer; using ColorzCore.Parser; -using ColorzCore.Parser.AST; namespace ColorzCore.Preprocessor.Directives { diff --git a/ColorzCore/Preprocessor/Macros/ReadDataAt.cs b/ColorzCore/Preprocessor/Macros/ReadDataAt.cs index b91235a..5cbeaa7 100644 --- a/ColorzCore/Preprocessor/Macros/ReadDataAt.cs +++ b/ColorzCore/Preprocessor/Macros/ReadDataAt.cs @@ -1,11 +1,12 @@ +using System; +using System.Collections.Generic; +using System.Linq; using ColorzCore.DataTypes; +using ColorzCore.Interpreter; using ColorzCore.IO; using ColorzCore.Lexer; using ColorzCore.Parser; using ColorzCore.Parser.AST; -using System; -using System.Collections.Generic; -using System.Linq; namespace ColorzCore.Preprocessor.Macros { @@ -38,7 +39,7 @@ public override IEnumerable ApplyMacro(Token head, IList> pa } else if (atom?.TryEvaluate(e => parser.Logger.Error(atom.MyLocation, e.Message), EvaluationPhase.Immediate) is int offset) { - offset = EAParseConsumer.ConvertToOffset(offset); + offset = EAInterpreter.ConvertToOffset(offset); if (offset >= 0 && offset <= EAOptions.MaximumBinarySize - readLength) { diff --git a/ColorzCore/Program.cs b/ColorzCore/Program.cs index ca1f7c9..5a6033a 100644 --- a/ColorzCore/Program.cs +++ b/ColorzCore/Program.cs @@ -406,17 +406,17 @@ static int Main(string[] args) if (EAOptions.QuietMessages) log.IgnoredKinds.Add(Logger.MessageKind.MESSAGE); - EAInterpreter myInterpreter = new EAInterpreter(output, game, rawsFolder, rawsExtension, inStream, inFileName, log); + EADriver myDriver = new EADriver(output, game, rawsFolder, rawsExtension, inStream, inFileName, log); ExecTimer.Timer.AddTimingPoint(ExecTimer.KEY_RAWPROC); - bool success = myInterpreter.Interpret(); + bool success = myDriver.Interpret(); if (success && EAOptions.ProduceNocashSym) { using StreamWriter symOut = File.CreateText(Path.ChangeExtension(outFileName, "sym")); - if (!(success = myInterpreter.WriteNocashSymbols(symOut))) + if (!(success = myDriver.WriteNocashSymbols(symOut))) { log.Message(Logger.MessageKind.ERROR, "Error trying to write no$gba symbol file."); } diff --git a/ColorzCore/Raws/AtomicParam.cs b/ColorzCore/Raws/AtomicParam.cs index 1f1e093..b79f28f 100644 --- a/ColorzCore/Raws/AtomicParam.cs +++ b/ColorzCore/Raws/AtomicParam.cs @@ -1,8 +1,8 @@ using System; using System.Collections; -using ColorzCore.Parser; using ColorzCore.Parser.AST; using ColorzCore.DataTypes; +using ColorzCore.Interpreter; namespace ColorzCore.Raws { @@ -33,7 +33,7 @@ public void Set(byte[] data, int value) { if (pointer) { - value = EAParseConsumer.ConvertToAddress(value); + value = EAInterpreter.ConvertToAddress(value); } data.SetBits(Position, Length, value); From f443830f0579e2168a3d07f10ebd66cafe57c0c0 Mon Sep 17 00:00:00 2001 From: Nat_776 Date: Sat, 4 May 2024 15:08:01 +0200 Subject: [PATCH 52/59] tbl and custom encoding support --- ColorzCore/EADriver.cs | 9 +- ColorzCore/IO/TblEncoding.cs | 152 ++++++++++++++++++ ColorzCore/Interpreter/StringProcessor.cs | 134 +++++++++++++++ ColorzCore/Parser/EAParser.cs | 82 ++-------- .../IncludeEncodingTableDirective.cs | 67 ++++++++ 5 files changed, 377 insertions(+), 67 deletions(-) create mode 100644 ColorzCore/IO/TblEncoding.cs create mode 100644 ColorzCore/Interpreter/StringProcessor.cs create mode 100644 ColorzCore/Preprocessor/Directives/IncludeEncodingTableDirective.cs diff --git a/ColorzCore/EADriver.cs b/ColorzCore/EADriver.cs index 4a85845..5acddcc 100644 --- a/ColorzCore/EADriver.cs +++ b/ColorzCore/EADriver.cs @@ -61,7 +61,9 @@ public EADriver(IOutput output, string? game, string? rawsFolder, string rawsExt // add the interpreter last parseConsumers.Add(myInterpreter); - myParser = new EAParser(logger, allRaws, parseConsumers, myInterpreter.BindIdentifier); + StringProcessor stringProcessor = new StringProcessor(); + + myParser = new EAParser(logger, allRaws, parseConsumers, myInterpreter.BindIdentifier, stringProcessor); myParser.Definitions["__COLORZ_CORE__"] = new Definition(); @@ -84,6 +86,11 @@ public EADriver(IOutput output, string? game, string? rawsFolder, string rawsExt myParser.DirectiveHandler.Directives["include"] = new IncludeDirective() { FileSearcher = includeSearcher }; myParser.DirectiveHandler.Directives["incbin"] = new IncludeBinaryDirective() { FileSearcher = includeSearcher }; + myParser.DirectiveHandler.Directives["inctbl"] = new IncludeEncodingTableDirective(stringProcessor) + { + FileSearcher = includeSearcher + }; + if (EAOptions.IsExtensionEnabled(EAOptions.Extensions.ReadDataMacros) && output is ROM rom) { myParser.Definitions["__has_read_data_macros"] = new Definition(); diff --git a/ColorzCore/IO/TblEncoding.cs b/ColorzCore/IO/TblEncoding.cs new file mode 100644 index 0000000..68541b8 --- /dev/null +++ b/ColorzCore/IO/TblEncoding.cs @@ -0,0 +1,152 @@ +using System; +using System.Collections.Generic; +using System.IO; + +namespace ColorzCore.IO +{ + public class TblEncoding + { + // https://datacrystal.romhacking.net/wiki/Text_Table + + private class EncodingTableNode + { + public byte[]? encoding; + public IDictionary nextTable = new Dictionary(); + } + + private readonly IDictionary rootTable; + + private TblEncoding() + { + rootTable = new Dictionary(); + } + + public byte[] ConvertToBytes(string inputString) + { + return ConvertToBytes(inputString, 0, inputString.Length); + } + + public byte[] ConvertToBytes(string inputString, int start, int length) + { + using MemoryStream memoryStream = new MemoryStream(); + using BinaryWriter binaryWriter = new BinaryWriter(memoryStream); + + int offset = start; + int end = start + length; + + while (offset < end) + { + // find longest match by following tree branches and keeping track of last encodable match + + byte[]? currentMatch = null; + int currentMatchLength = 0; + + IDictionary table = rootTable; + + for (int j = 0; offset + j < end; j++) + { + if (!table.TryGetValue(inputString[offset + j], out EncodingTableNode? node)) + { + break; + } + + table = node.nextTable; + + if (node.encoding != null) + { + currentMatch = node.encoding; + currentMatchLength = j + 1; + } + } + + if (currentMatch == null) + { + // TODO: better exception? + throw new Exception($"Could not encode character: '{inputString[offset]}'"); + } + else + { + binaryWriter.Write(currentMatch); + offset += currentMatchLength; + } + } + + return memoryStream.ToArray(); + } + + public void MapSequence(string sequence, byte[] bytes) + { + if (!rootTable.TryGetValue(sequence[0], out EncodingTableNode? mapping)) + { + mapping = rootTable[sequence[0]] = new EncodingTableNode(); + } + + // Find node corresponding to character sequence + for (int i = 1; i < sequence.Length; i++) + { + IDictionary thisTable = mapping.nextTable; + + if (!thisTable.TryGetValue(sequence[i], out mapping)) + { + mapping = thisTable[sequence[i]] = new EncodingTableNode(); + } + } + + mapping.encoding = bytes; + } + + // TODO: this could be elsewhere + private static byte[] HexToBytes(string hex, int offset, int length) + { + byte[] result = new byte[length / 2]; + + for (int i = 0; i < result.Length; i++) + { + // this could be optimized... + result[i] = Convert.ToByte(hex.Substring(offset + i * 2, 2), 16); + } + + return result; + } + + public static TblEncoding FromTextReader(TextReader reader) + { + TblEncoding result = new TblEncoding(); + string? line; + + char[] equalForSplit = new char[] { '=' }; + + while ((line = reader.ReadLine()) != null) + { + string[] parts; + + // this is never null but might be white space + if (string.IsNullOrWhiteSpace(line)) + { + continue; + } + + line = line.TrimStart(null); + + switch (line[0]) + { + case '*': + result.MapSequence("\n", HexToBytes(line, 1, line.Length - 1)); + break; + + case '/': + // What is this? + // not very well documented + break; + + default: + parts = line.Split(equalForSplit, 2); + result.MapSequence(parts[1], HexToBytes(parts[0], 0, parts[0].Length)); + break; + } + } + + return result; + } + } +} diff --git a/ColorzCore/Interpreter/StringProcessor.cs b/ColorzCore/Interpreter/StringProcessor.cs new file mode 100644 index 0000000..5b1a2f4 --- /dev/null +++ b/ColorzCore/Interpreter/StringProcessor.cs @@ -0,0 +1,134 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Text; +using System.Text.RegularExpressions; +using ColorzCore.DataTypes; +using ColorzCore.Interpreter.Diagnostics; +using ColorzCore.IO; +using ColorzCore.Lexer; +using ColorzCore.Parser; +using ColorzCore.Parser.AST; + +namespace ColorzCore.Interpreter +{ + // String handling the various processing of strings (encoding, formatting) + public class StringProcessor + { + public IDictionary TableEncodings { get; } = new Dictionary(); + public Logger Logger { get; } + + public byte[] EncodeString(Location locationForLogger, string inputString, string? encodingName) + { + switch (encodingName.ToUpperInvariant()) + { + case "UTF-8": + case "UTF8": + try + { + return Encoding.UTF8.GetBytes(inputString); + } + catch (Exception e) + { + Logger.Error(locationForLogger, + $"Failed to encode string '{inputString}': {e.Message}."); + return Array.Empty(); + } + } + + if (TableEncodings.TryGetValue(encodingName, out TblEncoding? encoding)) + { + try + { + return encoding.ConvertToBytes(inputString); + } + catch (Exception e) + { + Logger.Error(locationForLogger, + $"Failed to encode string '{inputString}': {e.Message}."); + } + } + else + { + Logger.Error(locationForLogger, + $"Unknown encoding: '{encodingName}'."); + } + + return Array.Empty(); + } + + private static readonly Regex formatItemRegex = new Regex(@"\{(?[^:}]+)(?:\:(?[^:}]*))?\}"); + + string UserFormatStringError(Location loc, string message, string details) + { + Logger.Error(loc, $"An error occurred while expanding format string ({message})."); + return $"<{message}: {details}>"; + } + + public string ExpandUserFormatString(Location location, EAParser parser, string stringValue) + { + return formatItemRegex.Replace(stringValue, match => + { + string expr = match.Groups["expr"].Value!; + string? format = match.Groups["format"].Value; + + return ExpandFormatItem(parser, location.OffsetBy(match.Index), expr, format); + }); + } + + public string ExpandFormatItem(EAParser parser, Location location, string expr, string? format = null) + { + MergeableGenerator tokens = new MergeableGenerator( + Tokenizer.TokenizeLine($"{expr} \n", location)); + + tokens.MoveNext(); + + IParamNode? node = parser.ParseParam(tokens); + + if (node == null || tokens.Current.Type != TokenType.NEWLINE) + { + return UserFormatStringError(location, "bad expression", $"'{expr}'"); + } + + switch (node) + { + case IAtomNode atom: + if (atom.TryEvaluate(e => Logger.Error(atom.MyLocation, e.Message), + EvaluationPhase.Immediate) is int value) + { + try + { + // TODO: do we need to raise an error when result == format? + // this happens (I think) when a custom format specifier with no substitution + return value.ToString(format, CultureInfo.InvariantCulture); + } + catch (FormatException e) + { + return UserFormatStringError(node.MyLocation, + "bad format specifier", $"'{format}' ({e.Message})"); + } + } + else + { + return UserFormatStringError(node.MyLocation, + "failed to evaluate expression", $"'{expr}'"); + } + + case StringNode stringNode: + if (!string.IsNullOrEmpty(format)) + { + return UserFormatStringError(node.MyLocation, + "string items cannot specify format", $"'{format}'"); + } + else + { + return stringNode.Value; + } + + default: + return UserFormatStringError(node.MyLocation, + "invalid format item type", $"'{DiagnosticsHelpers.PrettyParamType(node.Type)}'"); + } + } + } +} diff --git a/ColorzCore/Parser/EAParser.cs b/ColorzCore/Parser/EAParser.cs index 5be66b6..2ce5962 100644 --- a/ColorzCore/Parser/EAParser.cs +++ b/ColorzCore/Parser/EAParser.cs @@ -51,10 +51,10 @@ public bool IsIncluding public IParseConsumer ParseConsumer { get; } // TODO: IParseContextProvider or something like that? - // could also provide expanded format strings and stuff public BindIdentifierFunc BindIdentifier { get; } + public StringProcessor StringProcessor { get; } - public EAParser(Logger log, Dictionary> raws, IParseConsumer parseConsumer, BindIdentifierFunc bindIdentifier) + public EAParser(Logger log, Dictionary> raws, IParseConsumer parseConsumer, BindIdentifierFunc bindIdentifier, StringProcessor stringProcessor) { Logger = log; Raws = raws; @@ -64,6 +64,7 @@ public EAParser(Logger log, Dictionary> raws, IParseConsumer DirectiveHandler = new DirectiveHandler(); ParseConsumer = parseConsumer; BindIdentifier = bindIdentifier; + StringProcessor = stringProcessor; } public bool IsReservedName(string name) @@ -403,20 +404,19 @@ private void ParseErrorStatement(Token head, IList parameters) Logger.Error(head.Location, PrettyPrintParamsForMessage(parameters)); } - private void ParseStringStatement(Token head, IList parameters) { - void HandleStringStatement(Token head, string inputString, string? encodingName) + void HandleStringStatement(Token head, StringNode node, string? encodingName) { - if (encodingName != null && encodingName != "UTF-8") - { - Logger.Error(head.Location, - $"Custom STRING encoding is not yet supported (requested encoding: `{encodingName}`."); - } - else + string formattedString = StringProcessor.ExpandUserFormatString( + node.MyLocation, this, node.Value); + + byte[] encodedString = StringProcessor.EncodeString( + head.Location, formattedString, encodingName); + + if (encodedString.Length != 0) { - byte[] utf8Bytes = Encoding.UTF8.GetBytes(inputString); - ParseConsumer.OnData(head.Location, utf8Bytes); + ParseConsumer.OnData(head.Location, encodedString); } } @@ -426,7 +426,7 @@ void HandleStringStatement(Token head, string inputString, string? encodingName) { if (parameters[0] is StringNode firstNode) { - HandleStringStatement(head, firstNode.Value, null); + HandleStringStatement(head, firstNode, null); } else { @@ -440,7 +440,7 @@ void HandleStringStatement(Token head, string inputString, string? encodingName) { if (parameters[1] is StringNode secondNode) { - HandleStringStatement(head, firstNode.Value, secondNode.Value); + HandleStringStatement(head, firstNode, secondNode.Value); } else { @@ -579,7 +579,7 @@ public IList ParsePreprocParamList(MergeableGenerator tokens) return temp; } - private IParamNode? ParseParam(MergeableGenerator tokens) + public IParamNode? ParseParam(MergeableGenerator tokens) { Token localHead = tokens.Current; switch (localHead.Type) @@ -894,59 +894,9 @@ private string PrettyPrintParamsForMessage(IList parameters) { return string.Join(" ", parameters.Select(parameter => parameter switch { - StringNode node => ExpandUserFormatString(parameter.MyLocation, node.Value), + StringNode node => StringProcessor.ExpandUserFormatString(parameter.MyLocation, this, node.Value), _ => parameter.PrettyPrint(), })); } - - private static readonly Regex formatItemRegex = new Regex(@"\{(?[^:}]+)(?:\:(?[^:}]*))?\}"); - - private string ExpandUserFormatString(Location baseLocation, string stringValue) - { - string UserFormatStringError(Location loc, string message, string details) - { - Logger.Error(loc, $"An error occurred while expanding format string ({message})."); - return $"<{message}: {details}>"; - } - - return formatItemRegex.Replace(stringValue, match => - { - string expr = match.Groups["expr"].Value!; - string? format = match.Groups["format"].Value; - - Location itemLocation = baseLocation.OffsetBy(match.Index); - - MergeableGenerator tokens = new MergeableGenerator( - Tokenizer.TokenizeLine($"{expr} \n", itemLocation)); - - tokens.MoveNext(); - - IAtomNode? node = this.ParseAtom(tokens); - - if (node == null || tokens.Current.Type != TokenType.NEWLINE) - { - return UserFormatStringError(itemLocation, "bad expression", $"'{expr}'"); - } - - if (node.TryEvaluate(e => Logger.Error(node.MyLocation, e.Message), - EvaluationPhase.Immediate) is int value) - { - try - { - // TODO: do we need to raise an error when result == format? - // this happens (I think) when a custom format specifier with no substitution - return value.ToString(format, CultureInfo.InvariantCulture); - } - catch (FormatException e) - { - return UserFormatStringError(node.MyLocation, "bad format specifier", $"'{format}' ({e.Message})"); - } - } - else - { - return UserFormatStringError(node.MyLocation, "failed to evaluate expression", $"'{expr}'"); - } - }); - } } } diff --git a/ColorzCore/Preprocessor/Directives/IncludeEncodingTableDirective.cs b/ColorzCore/Preprocessor/Directives/IncludeEncodingTableDirective.cs new file mode 100644 index 0000000..e9263e0 --- /dev/null +++ b/ColorzCore/Preprocessor/Directives/IncludeEncodingTableDirective.cs @@ -0,0 +1,67 @@ +using System; +using System.Collections.Generic; +using System.IO; +using ColorzCore.DataTypes; +using ColorzCore.Interpreter; +using ColorzCore.IO; +using ColorzCore.Lexer; +using ColorzCore.Parser; +using ColorzCore.Parser.AST; + +namespace ColorzCore.Preprocessor.Directives +{ + public class IncludeEncodingTableDirective : BaseIncludeDirective + { + private readonly StringProcessor stringProcessor; + + public IncludeEncodingTableDirective(StringProcessor stringProcessor) + { + this.stringProcessor = stringProcessor; + } + + public override int MinParams => 2; + + public override int? MaxParams => 2; + + /* HACK: this is set by HandleInclude and read by Execute + * because we depend BaseIncludeDirective Execute, there's no elegant way of passing this around. */ + private TblEncoding? lastTblEncoding = null; + + public override void Execute(EAParser p, Token self, IList parameters, MergeableGenerator tokens) + { + string? encodingName = parameters[0].ToString(); + + if (encodingName != null) + { + if (stringProcessor.TableEncodings.ContainsKey(encodingName)) + { + p.Logger.Error(self.Location, $"String encoding '{encodingName}' already exists."); + } + else + { + base.Execute(p, self, new List() { parameters[1] }, tokens); + + if (lastTblEncoding != null) + { + stringProcessor.TableEncodings[encodingName] = lastTblEncoding; + lastTblEncoding = null; + } + else + { + p.Logger.Error(self.Location, $"Could not load encoding from table file '{parameters[1]}'."); + } + } + } + else + { + p.Logger.Error(self.Location, $"{self.Content} expected encoding name as first parameter."); + } + } + + public override void HandleInclude(EAParser p, Token self, string path, MergeableGenerator tokens) + { + using TextReader textReader = new StreamReader(path); + lastTblEncoding = TblEncoding.FromTextReader(textReader); + } + } +} From 4eee1760921b0e35fa864ae0ed3ea52b2902e335 Mon Sep 17 00:00:00 2001 From: Nat_776 Date: Sat, 4 May 2024 18:48:26 +0200 Subject: [PATCH 53/59] token location fixes --- ColorzCore/EADriver.cs | 5 +- ColorzCore/Lexer/Token.cs | 9 +-- ColorzCore/Lexer/Tokenizer.cs | 116 ++++++++++++++++------------------ 3 files changed, 58 insertions(+), 72 deletions(-) diff --git a/ColorzCore/EADriver.cs b/ColorzCore/EADriver.cs index 5acddcc..00e62e5 100644 --- a/ColorzCore/EADriver.cs +++ b/ColorzCore/EADriver.cs @@ -68,7 +68,7 @@ public EADriver(IOutput output, string? game, string? rawsFolder, string rawsExt myParser.Definitions["__COLORZ_CORE__"] = new Definition(); myParser.Definitions["__COLORZ_CORE_VERSION__"] = new Definition( - new Token(TokenType.NUMBER, new Location("builtin", 0, 0), "20240502")); + new Token(TokenType.NUMBER, new Location("builtin", 0, 0), "20240504")); if (game != null) { @@ -147,7 +147,8 @@ public bool Interpret() foreach ((string name, string body) in EAOptions.PreDefintions) { - myParser.ParseAll(tokenizer.TokenizeLine($"#define {name} \"{body}\"", "cmd", 0)); + Location location = new Location("CMD", 0, 1); + myParser.ParseAll(tokenizer.TokenizePhrase($"#define {name} \"{body}\"", location)); } myParser.ParseAll(tokenizer.Tokenize(sin, iFile)); diff --git a/ColorzCore/Lexer/Token.cs b/ColorzCore/Lexer/Token.cs index 2d96884..61eaa90 100644 --- a/ColorzCore/Lexer/Token.cs +++ b/ColorzCore/Lexer/Token.cs @@ -14,14 +14,7 @@ public class Token public int LineNumber => Location.line; public int ColumnNumber => Location.column; - public Token(TokenType type, string fileName, int lineNum, int colNum, string original = "") - { - Type = type; - Location = new Location(fileName, lineNum, colNum + 1); - Content = original; - } - - public Token(TokenType type, Location location, string content) + public Token(TokenType type, Location location, string content = "") { Type = type; Location = location; diff --git a/ColorzCore/Lexer/Tokenizer.cs b/ColorzCore/Lexer/Tokenizer.cs index 3de3d35..672c9d7 100644 --- a/ColorzCore/Lexer/Tokenizer.cs +++ b/ColorzCore/Lexer/Tokenizer.cs @@ -59,7 +59,7 @@ public Tokenizer() multilineCommentNesting = 0; } - public IEnumerable TokenizePhrase(string line, string fileName, int lineNum, int startOffs, int endOffs, int offset = 0) + public IEnumerable TokenizePhrase(string line, int startOffs, int endOffs, Location location) { bool afterInclude = false, afterDirective = false, afterWhitespace = false; @@ -97,44 +97,44 @@ public IEnumerable TokenizePhrase(string line, string fileName, int lineN switch (nextChar) { case ';': - yield return new Token(TokenType.SEMICOLON, fileName, lineNum, curCol + offset); + yield return new Token(TokenType.SEMICOLON, location.OffsetBy(curCol)); break; case ':': if (curCol + 1 < endOffs && line[curCol + 1] == '=') // ':=' { - yield return new Token(TokenType.ASSIGN, fileName, lineNum, curCol + offset); + yield return new Token(TokenType.ASSIGN, location.OffsetBy(curCol)); curCol++; break; } - yield return new Token(TokenType.COLON, fileName, lineNum, curCol + offset); + yield return new Token(TokenType.COLON, location.OffsetBy(curCol)); break; case '{': - yield return new Token(TokenType.OPEN_BRACE, fileName, lineNum, curCol + offset); + yield return new Token(TokenType.OPEN_BRACE, location.OffsetBy(curCol)); break; case '}': - yield return new Token(TokenType.CLOSE_BRACE, fileName, lineNum, curCol + offset); + yield return new Token(TokenType.CLOSE_BRACE, location.OffsetBy(curCol)); break; case '[': - yield return new Token(TokenType.OPEN_BRACKET, fileName, lineNum, curCol + offset); + yield return new Token(TokenType.OPEN_BRACKET, location.OffsetBy(curCol)); break; case ']': - yield return new Token(TokenType.CLOSE_BRACKET, fileName, lineNum, curCol + offset); + yield return new Token(TokenType.CLOSE_BRACKET, location.OffsetBy(curCol)); break; case '(': - yield return new Token(TokenType.OPEN_PAREN, fileName, lineNum, curCol + offset); + yield return new Token(TokenType.OPEN_PAREN, location.OffsetBy(curCol)); break; case ')': - yield return new Token(TokenType.CLOSE_PAREN, fileName, lineNum, curCol + offset); + yield return new Token(TokenType.CLOSE_PAREN, location.OffsetBy(curCol)); break; case '*': - yield return new Token(TokenType.MUL_OP, fileName, lineNum, curCol + offset); + yield return new Token(TokenType.MUL_OP, location.OffsetBy(curCol)); break; case '%': - yield return new Token(TokenType.MOD_OP, fileName, lineNum, curCol + offset); + yield return new Token(TokenType.MOD_OP, location.OffsetBy(curCol)); break; case ',': - yield return new Token(TokenType.COMMA, fileName, lineNum, curCol + offset); + yield return new Token(TokenType.COMMA, location.OffsetBy(curCol)); break; case '/': if (curCol + 1 < endOffs && line[curCol + 1] == '/') @@ -150,11 +150,11 @@ public IEnumerable TokenizePhrase(string line, string fileName, int lineN } else { - yield return new Token(TokenType.DIV_OP, fileName, lineNum, curCol + offset); + yield return new Token(TokenType.DIV_OP, location.OffsetBy(curCol)); } break; case '+': - yield return new Token(TokenType.ADD_OP, fileName, lineNum, curCol + offset); + yield return new Token(TokenType.ADD_OP, location.OffsetBy(curCol)); break; case '-': if (afterWhitespace && afterDirective) @@ -163,65 +163,65 @@ public IEnumerable TokenizePhrase(string line, string fileName, int lineN if (wsDelimited.Success) { string match = wsDelimited.Value; - yield return new Token(TokenType.STRING, fileName, lineNum, curCol, IOUtility.UnescapePath(match)); + yield return new Token(TokenType.STRING, location.OffsetBy(curCol), IOUtility.UnescapePath(match)); curCol += match.Length; continue; } } - yield return new Token(TokenType.SUB_OP, fileName, lineNum, curCol + offset); + yield return new Token(TokenType.SUB_OP, location.OffsetBy(curCol)); break; case '&': if (curCol + 1 < endOffs && line[curCol + 1] == '&') { - yield return new Token(TokenType.LOGAND_OP, fileName, lineNum, curCol + offset); + yield return new Token(TokenType.LOGAND_OP, location.OffsetBy(curCol)); curCol++; break; } - yield return new Token(TokenType.AND_OP, fileName, lineNum, curCol + offset); + yield return new Token(TokenType.AND_OP, location.OffsetBy(curCol)); break; case '^': - yield return new Token(TokenType.XOR_OP, fileName, lineNum, curCol + offset); + yield return new Token(TokenType.XOR_OP, location.OffsetBy(curCol)); break; case '|': if (curCol + 1 < endOffs && line[curCol + 1] == '|') { - yield return new Token(TokenType.LOGOR_OP, fileName, lineNum, curCol + offset); + yield return new Token(TokenType.LOGOR_OP, location.OffsetBy(curCol)); curCol++; break; } - yield return new Token(TokenType.OR_OP, fileName, lineNum, curCol + offset); + yield return new Token(TokenType.OR_OP, location.OffsetBy(curCol)); break; case '\"': { curCol++; Match quoteInterior = stringRegex.Match(line, curCol, endOffs - curCol); string match = quoteInterior.Value; - yield return new Token(TokenType.STRING, fileName, lineNum, curCol + offset, /*IOUtility.UnescapeString(*/match/*)*/); + yield return new Token(TokenType.STRING, location.OffsetBy(curCol), /*IOUtility.UnescapeString(*/match/*)*/); curCol += match.Length; if (curCol == endOffs || line[curCol] != '\"') { - yield return new Token(TokenType.ERROR, fileName, lineNum, curCol, "Unclosed string."); + yield return new Token(TokenType.ERROR, location.OffsetBy(curCol), "Unclosed string."); } break; } case '<': if (curCol + 1 < endOffs && line[curCol + 1] == '<') { - yield return new Token(TokenType.LSHIFT_OP, fileName, lineNum, curCol + offset); + yield return new Token(TokenType.LSHIFT_OP, location.OffsetBy(curCol)); curCol++; break; } else if (curCol + 1 < endOffs && line[curCol + 1] == '=') { - yield return new Token(TokenType.COMPARE_LE, fileName, lineNum, curCol + offset); + yield return new Token(TokenType.COMPARE_LE, location.OffsetBy(curCol)); curCol++; break; } else { - yield return new Token(TokenType.COMPARE_LT, fileName, lineNum, curCol + offset); + yield return new Token(TokenType.COMPARE_LT, location.OffsetBy(curCol)); break; } case '>': @@ -229,78 +229,69 @@ public IEnumerable TokenizePhrase(string line, string fileName, int lineN { if (curCol + 2 < endOffs && line[curCol + 2] == '>') { - yield return new Token(TokenType.SIGNED_RSHIFT_OP, fileName, lineNum, curCol + offset); + yield return new Token(TokenType.SIGNED_RSHIFT_OP, location.OffsetBy(curCol)); curCol += 2; } else { - yield return new Token(TokenType.RSHIFT_OP, fileName, lineNum, curCol + offset); + yield return new Token(TokenType.RSHIFT_OP, location.OffsetBy(curCol)); curCol++; } break; } else if (curCol + 1 < endOffs && line[curCol + 1] == '=') { - yield return new Token(TokenType.COMPARE_GE, fileName, lineNum, curCol + offset); + yield return new Token(TokenType.COMPARE_GE, location.OffsetBy(curCol)); curCol++; break; } else { - yield return new Token(TokenType.COMPARE_GT, fileName, lineNum, curCol + offset); + yield return new Token(TokenType.COMPARE_GT, location.OffsetBy(curCol)); break; } case '=': if (curCol + 1 < endOffs && line[curCol + 1] == '=') { - yield return new Token(TokenType.COMPARE_EQ, fileName, lineNum, curCol + offset); + yield return new Token(TokenType.COMPARE_EQ, location.OffsetBy(curCol)); curCol++; break; } else { - yield return new Token(TokenType.ERROR, fileName, lineNum, curCol, "="); + yield return new Token(TokenType.ERROR, location.OffsetBy(curCol), "="); break; } case '!': if (curCol + 1 < endOffs && line[curCol + 1] == '=') { - yield return new Token(TokenType.COMPARE_NE, fileName, lineNum, curCol + offset); + yield return new Token(TokenType.COMPARE_NE, location.OffsetBy(curCol)); curCol++; break; } else { - yield return new Token(TokenType.LOGNOT_OP, fileName, lineNum, curCol + offset); + yield return new Token(TokenType.LOGNOT_OP, location.OffsetBy(curCol)); break; } case '~': - yield return new Token(TokenType.NOT_OP, fileName, lineNum, curCol + offset); + yield return new Token(TokenType.NOT_OP, location.OffsetBy(curCol)); break; case '?': if (curCol + 1 < endOffs && line[curCol + 1] == '?') { - yield return new Token(TokenType.UNDEFINED_COALESCE_OP, fileName, lineNum, curCol + offset); + yield return new Token(TokenType.UNDEFINED_COALESCE_OP, location.OffsetBy(curCol)); curCol++; break; } else { - yield return new Token(TokenType.ERROR, fileName, lineNum, curCol, "?"); + yield return new Token(TokenType.ERROR, location.OffsetBy(curCol), "?"); break; } case '\n': - yield return new Token(TokenType.NEWLINE, fileName, lineNum, curCol + offset); + yield return new Token(TokenType.NEWLINE, location.OffsetBy(curCol)); break; - case '\\': - if (curCol + 1 < endOffs && line[curCol + 1] == '\n') - { - curCol++; - continue; - } - yield return new Token(TokenType.ERROR, fileName, lineNum, curCol, "\\"); - break; - default: if (afterInclude) { @@ -308,7 +299,7 @@ public IEnumerable TokenizePhrase(string line, string fileName, int lineN if (winPath.Success) { string match = winPath.Value; - yield return new Token(TokenType.STRING, fileName, lineNum, curCol, IOUtility.UnescapePath(match)); + yield return new Token(TokenType.STRING, location.OffsetBy(curCol), IOUtility.UnescapePath(match)); curCol += match.Length; afterInclude = false; continue; @@ -322,17 +313,17 @@ public IEnumerable TokenizePhrase(string line, string fileName, int lineN if (idMatch.Success) { string match = idMatch.Value; - int idCol = curCol + offset; + int idCol = curCol; curCol += match.Length; if (curCol < endOffs && line[curCol] == '(') - yield return new Token(TokenType.MAYBE_MACRO, fileName, lineNum, idCol, match); + yield return new Token(TokenType.MAYBE_MACRO, location.OffsetBy(idCol), match); else - yield return new Token(TokenType.IDENTIFIER, fileName, lineNum, idCol, match); + yield return new Token(TokenType.IDENTIFIER, location.OffsetBy(idCol), match); if (curCol < endOffs && (char.IsLetterOrDigit(line[curCol]) | line[curCol] == '_')) { Match idMatch2 = new Regex("[a-zA-Z0-9_]+").Match(line, curCol, endOffs - curCol); match = idMatch2.Value; - yield return new Token(TokenType.ERROR, fileName, lineNum, curCol, $"Identifier longer than {MAX_ID_LENGTH} characters."); + yield return new Token(TokenType.ERROR, location.OffsetBy(curCol), $"Identifier longer than {MAX_ID_LENGTH} characters."); curCol += match.Length; } continue; @@ -344,7 +335,7 @@ public IEnumerable TokenizePhrase(string line, string fileName, int lineN //Verify that next token isn't start of an identifier if (curCol + match.Length >= endOffs || (!char.IsLetter(line[curCol + match.Length]) && line[curCol + match.Length] != '_')) { - yield return new Token(TokenType.NUMBER, fileName, lineNum, curCol + offset, match.TrimEnd()); + yield return new Token(TokenType.NUMBER, location.OffsetBy(curCol), match.TrimEnd()); curCol += match.Length; continue; } @@ -353,7 +344,7 @@ public IEnumerable TokenizePhrase(string line, string fileName, int lineN if (directiveMatch.Success) { string match = directiveMatch.Value; - yield return new Token(TokenType.PREPROCESSOR_DIRECTIVE, fileName, lineNum, curCol + offset, match); + yield return new Token(TokenType.PREPROCESSOR_DIRECTIVE, location.OffsetBy(curCol), match); curCol += match.Length; if (match.Substring(1).Equals("include") || match.Substring(1).Equals("incbin")) { @@ -364,7 +355,7 @@ public IEnumerable TokenizePhrase(string line, string fileName, int lineN } } string restOfWord = new Regex("\\G\\S+").Match(line, curCol, endOffs - curCol).Value; - yield return new Token(TokenType.ERROR, fileName, lineNum, curCol, restOfWord); + yield return new Token(TokenType.ERROR, location.OffsetBy(curCol), restOfWord); curCol += restOfWord.Length; continue; } @@ -374,14 +365,14 @@ public IEnumerable TokenizePhrase(string line, string fileName, int lineN } } - public IEnumerable TokenizeLine(string line, string fileName, int lineNum, int offset = 0) + public IEnumerable TokenizePhrase(string line, Location location) { - return TokenizePhrase(line, fileName, lineNum, 0, line.Length, offset); + return TokenizePhrase(line, 0, line.Length, location); } public static IEnumerable TokenizeLine(string line, Location location) { - return new Tokenizer().TokenizeLine(line, location.file, location.line, location.column); + return new Tokenizer().TokenizePhrase(line, 0, line.Length, location); } /*** @@ -397,17 +388,18 @@ public IEnumerable Tokenize(Stream input, string fileName) string line = sin.ReadLine()!; //allow escaping newlines - while (line.Length > 0 && line.Substring(line.Length - 1) == "\\") + while (line.Length > 0 && line[line.Length - 1] == '\\') { curLine++; line = line.Substring(0, line.Length - 1) + " " + sin.ReadLine(); } - foreach (Token t in TokenizeLine(line, fileName, curLine)) + Location location = new Location(fileName, curLine, 1); + foreach (Token t in TokenizeLine(line, location)) { yield return t; } - yield return new Token(TokenType.NEWLINE, fileName, curLine, line.Length); + yield return new Token(TokenType.NEWLINE, location.OffsetBy(line.Length)); curLine++; } } From 79d803f997cf6bceaf82791897231ea58a1d82e7 Mon Sep 17 00:00:00 2001 From: Nat_776 Date: Sat, 4 May 2024 18:49:57 +0200 Subject: [PATCH 54/59] do not translate offset 0 to address 0 except for POIN --- ColorzCore/Interpreter/EAInterpreter.cs | 8 +------- ColorzCore/Raws/AtomicParam.cs | 2 +- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/ColorzCore/Interpreter/EAInterpreter.cs b/ColorzCore/Interpreter/EAInterpreter.cs index 6080680..f132421 100644 --- a/ColorzCore/Interpreter/EAInterpreter.cs +++ b/ColorzCore/Interpreter/EAInterpreter.cs @@ -46,13 +46,7 @@ public EAInterpreter(Logger logger) public static int ConvertToAddress(int value) { - /* - NOTE: Offset 0 is always converted to a null address - If one wants to instead refer to ROM offset 0 they would want to use the address directly instead. - If ROM offset 0 is already address 0 then this is a moot point. - */ - - if (value > 0 && value < EAOptions.MaximumBinarySize) + if (value >= 0 && value < EAOptions.MaximumBinarySize) { value += EAOptions.BaseAddress; } diff --git a/ColorzCore/Raws/AtomicParam.cs b/ColorzCore/Raws/AtomicParam.cs index b79f28f..4cb1ac4 100644 --- a/ColorzCore/Raws/AtomicParam.cs +++ b/ColorzCore/Raws/AtomicParam.cs @@ -31,7 +31,7 @@ public void Set(byte[] data, IParamNode input) public void Set(byte[] data, int value) { - if (pointer) + if (pointer && value != 0) { value = EAInterpreter.ConvertToAddress(value); } From 0a474d05cbcf9623a42f310c25ff7006e36ae92c Mon Sep 17 00:00:00 2001 From: Nat_776 Date: Sat, 4 May 2024 18:53:42 +0200 Subject: [PATCH 55/59] fix STRING with no encoding + fix tests --- ColorzCore/EADriver.cs | 2 +- ColorzCore/Interpreter/StringProcessor.cs | 7 +++++++ Tests/statements.py | 11 ++++++----- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/ColorzCore/EADriver.cs b/ColorzCore/EADriver.cs index 00e62e5..b6a7d86 100644 --- a/ColorzCore/EADriver.cs +++ b/ColorzCore/EADriver.cs @@ -61,7 +61,7 @@ public EADriver(IOutput output, string? game, string? rawsFolder, string rawsExt // add the interpreter last parseConsumers.Add(myInterpreter); - StringProcessor stringProcessor = new StringProcessor(); + StringProcessor stringProcessor = new StringProcessor(logger); myParser = new EAParser(logger, allRaws, parseConsumers, myInterpreter.BindIdentifier, stringProcessor); diff --git a/ColorzCore/Interpreter/StringProcessor.cs b/ColorzCore/Interpreter/StringProcessor.cs index 5b1a2f4..cdaf706 100644 --- a/ColorzCore/Interpreter/StringProcessor.cs +++ b/ColorzCore/Interpreter/StringProcessor.cs @@ -18,8 +18,15 @@ public class StringProcessor public IDictionary TableEncodings { get; } = new Dictionary(); public Logger Logger { get; } + public StringProcessor(Logger logger) + { + Logger = logger; + } + public byte[] EncodeString(Location locationForLogger, string inputString, string? encodingName) { + encodingName ??= "UTF-8"; + switch (encodingName.ToUpperInvariant()) { case "UTF-8": diff --git a/Tests/statements.py b/Tests/statements.py index b5f7d82..083a05d 100644 --- a/Tests/statements.py +++ b/Tests/statements.py @@ -120,12 +120,13 @@ "ASSERT 1 - 2", None), - # ================== - # = UTF8 Statement = - # ================== + # ==================== + # = STRING Statement = + # ==================== + # incomplete - T("UTF8", - "ORG 0 ;UTF8 \"Hello World\"", + T("STRING Basic", + "ORG 0 ; STRING \"Hello World\"", b"Hello World"), # ==================== From 3c2f36953af7fd6e8256326f935d1b6b70f2b914 Mon Sep 17 00:00:00 2001 From: Nat_776 Date: Sat, 4 May 2024 20:22:34 +0200 Subject: [PATCH 56/59] fix major a oopsie (multi-line comments broke) --- ColorzCore/Lexer/Tokenizer.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ColorzCore/Lexer/Tokenizer.cs b/ColorzCore/Lexer/Tokenizer.cs index 672c9d7..f780d28 100644 --- a/ColorzCore/Lexer/Tokenizer.cs +++ b/ColorzCore/Lexer/Tokenizer.cs @@ -395,7 +395,7 @@ public IEnumerable Tokenize(Stream input, string fileName) } Location location = new Location(fileName, curLine, 1); - foreach (Token t in TokenizeLine(line, location)) + foreach (Token t in TokenizePhrase(line, location)) { yield return t; } From 08dff627e4e64e85040fefd9c271a764421aa8bc Mon Sep 17 00:00:00 2001 From: Nat_776 Date: Sat, 4 May 2024 21:32:06 +0200 Subject: [PATCH 57/59] fix "garbage" characters printed on Windows cmd (ANSI terminal escape codes are not handled) --- .../Diagnostics/DiagnosticsHelpers.cs | 74 ----------------- .../Diagnostics/EmphasisExpressionPrinter.cs | 83 +++++++++++++++++++ 2 files changed, 83 insertions(+), 74 deletions(-) create mode 100644 ColorzCore/Interpreter/Diagnostics/EmphasisExpressionPrinter.cs diff --git a/ColorzCore/Interpreter/Diagnostics/DiagnosticsHelpers.cs b/ColorzCore/Interpreter/Diagnostics/DiagnosticsHelpers.cs index c05fe21..5d50daa 100644 --- a/ColorzCore/Interpreter/Diagnostics/DiagnosticsHelpers.cs +++ b/ColorzCore/Interpreter/Diagnostics/DiagnosticsHelpers.cs @@ -10,80 +10,6 @@ namespace ColorzCore.Interpreter.Diagnostics { public static class DiagnosticsHelpers { - // Helper class for printing expressions (IAtomNode) with some bits emphasized - private class EmphasisExpressionPrinter : AtomVisitor - { - readonly StringBuilder stringBuilder = new StringBuilder(); - readonly StringBuilder underlineBuilder = new StringBuilder(); - - readonly Func emphasisPredicate; - bool wasEmphasized = false; - - public EmphasisExpressionPrinter(Func predicate) - { - emphasisPredicate = predicate; - // targetMacroLocation = macroLocation; - } - - public string PrintExpression(IAtomNode expression) - { - stringBuilder.Clear(); - underlineBuilder.Clear(); - - Visit(expression); - - return $"{stringBuilder}\n{underlineBuilder}"; - } - - private void AppendString(bool strong, string value) - { - if (strong) - { - stringBuilder.Append("\x1B[1;37m"); - stringBuilder.Append(value); - stringBuilder.Append("\x1B[0m"); - } - else - { - stringBuilder.Append(value); - } - - underlineBuilder.Append(strong ? '~' : ' ', value.Length); - - wasEmphasized = strong; - } - - protected override void VisitNode(OperatorNode node) - { - AppendString(emphasisPredicate(node.Left.MyLocation), "("); - - Visit(node.Left); - - AppendString(emphasisPredicate(node.OperatorToken.Location), $" {node.OperatorString} "); - - Visit(node.Right); - - AppendString(wasEmphasized, ")"); - } - - protected override void VisitNode(UnaryOperatorNode node) - { - AppendString(emphasisPredicate(node.OperatorToken.Location), node.OperatorString); - - Visit(node.Inner); - } - - protected override void VisitNode(IdentifierNode node) - { - AppendString(emphasisPredicate(node.MyLocation), node.GetIdentifier()!); - } - - protected override void VisitNode(NumberNode node) - { - AppendString(emphasisPredicate(node.MyLocation), node.PrettyPrint()); - } - } - // Gets string representation of expression with emphasis on locations for which the predicate is true public static string GetEmphasizedExpression(IAtomNode node, Func emphasisPredicate) { diff --git a/ColorzCore/Interpreter/Diagnostics/EmphasisExpressionPrinter.cs b/ColorzCore/Interpreter/Diagnostics/EmphasisExpressionPrinter.cs new file mode 100644 index 0000000..05ae83c --- /dev/null +++ b/ColorzCore/Interpreter/Diagnostics/EmphasisExpressionPrinter.cs @@ -0,0 +1,83 @@ +using System; +using System.Runtime.InteropServices; +using System.Text; +using ColorzCore.DataTypes; +using ColorzCore.Parser.AST; + +namespace ColorzCore.Interpreter.Diagnostics +{ + // Helper class for printing expressions (IAtomNode) with some bits emphasized + public class EmphasisExpressionPrinter : AtomVisitor + { + readonly StringBuilder stringBuilder = new StringBuilder(); + readonly StringBuilder underlineBuilder = new StringBuilder(); + + readonly Func emphasisPredicate; + bool wasEmphasized = false; + + public EmphasisExpressionPrinter(Func predicate) + { + emphasisPredicate = predicate; + // targetMacroLocation = macroLocation; + } + + public string PrintExpression(IAtomNode expression) + { + stringBuilder.Clear(); + underlineBuilder.Clear(); + + Visit(expression); + + return $"{stringBuilder}\n{underlineBuilder}"; + } + + private void AppendString(bool strong, string value) + { + // HACK: on Windows, ANSI terminal escape codes don't (always?) work + if (strong && !RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + stringBuilder.Append("\x1B[1;37m"); + stringBuilder.Append(value); + stringBuilder.Append("\x1B[0m"); + } + else + { + stringBuilder.Append(value); + } + + underlineBuilder.Append(strong ? '~' : ' ', value.Length); + + wasEmphasized = strong; + } + + protected override void VisitNode(OperatorNode node) + { + AppendString(emphasisPredicate(node.Left.MyLocation), "("); + + Visit(node.Left); + + AppendString(emphasisPredicate(node.OperatorToken.Location), $" {node.OperatorString} "); + + Visit(node.Right); + + AppendString(wasEmphasized, ")"); + } + + protected override void VisitNode(UnaryOperatorNode node) + { + AppendString(emphasisPredicate(node.OperatorToken.Location), node.OperatorString); + + Visit(node.Inner); + } + + protected override void VisitNode(IdentifierNode node) + { + AppendString(emphasisPredicate(node.MyLocation), node.GetIdentifier()!); + } + + protected override void VisitNode(NumberNode node) + { + AppendString(emphasisPredicate(node.MyLocation), node.PrettyPrint()); + } + } +} From 5e5126ad9354e9939c02c53c5e7bd09d6c8a3e40 Mon Sep 17 00:00:00 2001 From: Nat_776 Date: Sun, 5 May 2024 11:57:57 +0200 Subject: [PATCH 58/59] remove many unused classes and methods --- ColorzCore/DataTypes/CaseInsensitiveString.cs | 36 -------------- ColorzCore/EAOptions.cs | 17 +++++-- .../Diagnostics/EmphasisExpressionPrinter.cs | 4 +- ColorzCore/Parser/AST/AtomNodeKernel.cs | 11 ----- ColorzCore/Parser/AST/IAtomNode.cs | 2 - ColorzCore/Parser/AST/IParamNode.cs | 12 ++--- ColorzCore/Parser/AST/IdentifierNode.cs | 29 ++++++----- ColorzCore/Parser/AST/ListNode.cs | 33 ------------- ColorzCore/Parser/AST/MacroInvocationNode.cs | 15 +----- ColorzCore/Parser/AST/NumberNode.cs | 5 -- ColorzCore/Parser/AST/OperatorNode.cs | 13 ----- ColorzCore/Parser/AST/StringNode.cs | 10 ---- ColorzCore/Parser/AST/UnaryOperatorNode.cs | 9 +--- ColorzCore/Parser/EAParser.cs | 9 +++- .../Directives/IncludeToolEventDirective.cs | 6 ++- ColorzCore/Preprocessor/MacroCollection.cs | 2 - .../Preprocessor/Macros/IsSymbolDefined.cs | 49 ------------------- ColorzCore/Preprocessor/Macros/StringMacro.cs | 7 +++ ColorzCore/Program.cs | 47 +----------------- ColorzCore/Raws/AtomicParam.cs | 3 +- 20 files changed, 61 insertions(+), 258 deletions(-) delete mode 100644 ColorzCore/DataTypes/CaseInsensitiveString.cs delete mode 100644 ColorzCore/Preprocessor/Macros/IsSymbolDefined.cs diff --git a/ColorzCore/DataTypes/CaseInsensitiveString.cs b/ColorzCore/DataTypes/CaseInsensitiveString.cs deleted file mode 100644 index cbc65e7..0000000 --- a/ColorzCore/DataTypes/CaseInsensitiveString.cs +++ /dev/null @@ -1,36 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace ColorzCore.DataTypes -{ - class CaseInsensitiveString - { - private string data; - public string String { - get { return data; } - set - { - data = value.ToUpper(); - } - } - public CaseInsensitiveString(string input) - { - data = input.ToUpper(); - } - public override int GetHashCode() - { - return String.GetHashCode(); - } - public override bool Equals(object? obj) - { - return obj != null && String == obj.ToString(); - } - public override string ToString() - { - return String; - } - } -} diff --git a/ColorzCore/EAOptions.cs b/ColorzCore/EAOptions.cs index e3e45ca..d6d05dd 100644 --- a/ColorzCore/EAOptions.cs +++ b/ColorzCore/EAOptions.cs @@ -18,18 +18,23 @@ public enum Warnings : long ReDefine = 2, // warn on write before ORG + // NOTE: currently no way to disable UninitializedOffset = 4, // warn on unintuitive macro expansions (#define A 1 + 2 ... BYTE A * 2 ) UnintuitiveExpressionMacros = 8, - // warn on expansion of unguarded expression within macro () + // warn on expansion of unguarded expression within macro UnguardedExpressionMacros = 16, // warn on macro expanded into "PUSH ; ORG value ; name : ; POP" + // NOTE: currently no way to disable SetSymbolMacros = 32, - Extra = UnguardedExpressionMacros, + // warn on use of legacy features that were superceded (such as the String macro) + LegacyFeatures = 64, + + Extra = UnguardedExpressionMacros | LegacyFeatures, All = long.MaxValue & ~Extra, } @@ -47,8 +52,8 @@ public enum Extensions : long // enable AddToPool and #pool AddToPool = 4, + Extra = All, All = long.MaxValue, - Default = IncludeTools | AddToPool, } public static bool WarningsAreErrors { get; set; } @@ -57,6 +62,8 @@ public enum Extensions : long public static bool MonochromeLog { get; set; } public static bool BenchmarkBuildTimes { get; set; } public static bool ProduceNocashSym { get; set; } + + // NOTE: currently this is not exposed to users public static bool TranslateBackslashesInPaths { get; set; } = true; public static int BaseAddress { get; set; } = 0x8000000; @@ -67,7 +74,9 @@ public enum Extensions : long public static List<(string, string)> PreDefintions { get; } = new List<(string, string)>(); public static Warnings EnabledWarnings { get; set; } = Warnings.All; - public static Extensions EnabledExtensions { get; set; } = Extensions.Default; + + // NOTE: currently this is not exposed to users + public static Extensions EnabledExtensions { get; set; } = Extensions.All; public static bool IsWarningEnabled(Warnings warning) => EnabledWarnings.HasFlag(warning); public static bool IsExtensionEnabled(Extensions extension) => EnabledExtensions.HasFlag(extension); diff --git a/ColorzCore/Interpreter/Diagnostics/EmphasisExpressionPrinter.cs b/ColorzCore/Interpreter/Diagnostics/EmphasisExpressionPrinter.cs index 05ae83c..bb4a23d 100644 --- a/ColorzCore/Interpreter/Diagnostics/EmphasisExpressionPrinter.cs +++ b/ColorzCore/Interpreter/Diagnostics/EmphasisExpressionPrinter.cs @@ -34,7 +34,7 @@ public string PrintExpression(IAtomNode expression) private void AppendString(bool strong, string value) { // HACK: on Windows, ANSI terminal escape codes don't (always?) work - if (strong && !RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + if (strong && !RuntimeInformation.IsOSPlatform(OSPlatform.Windows) && !EAOptions.MonochromeLog) { stringBuilder.Append("\x1B[1;37m"); stringBuilder.Append(value); @@ -72,7 +72,7 @@ protected override void VisitNode(UnaryOperatorNode node) protected override void VisitNode(IdentifierNode node) { - AppendString(emphasisPredicate(node.MyLocation), node.GetIdentifier()!); + AppendString(emphasisPredicate(node.MyLocation), node.IdentifierToken.Content); } protected override void VisitNode(NumberNode node) diff --git a/ColorzCore/Parser/AST/AtomNodeKernel.cs b/ColorzCore/Parser/AST/AtomNodeKernel.cs index 5c6dcb9..deff126 100644 --- a/ColorzCore/Parser/AST/AtomNodeKernel.cs +++ b/ColorzCore/Parser/AST/AtomNodeKernel.cs @@ -13,13 +13,7 @@ public abstract class AtomNodeKernel : IAtomNode public ParamType Type => ParamType.ATOM; - public virtual string? GetIdentifier() - { - return null; - } - public abstract string PrettyPrint(); - public abstract IEnumerable ToTokens(); public abstract Location MyLocation { get; } public abstract int? TryEvaluate(Action handler, EvaluationPhase evaluationPhase); @@ -28,10 +22,5 @@ public IParamNode SimplifyExpressions(Action handler, EvaluationPhase { return this.Simplify(handler, evaluationPhase); } - - public IAtomNode? AsAtom() - { - return this; - } } } diff --git a/ColorzCore/Parser/AST/IAtomNode.cs b/ColorzCore/Parser/AST/IAtomNode.cs index 5fd98df..262ceb9 100644 --- a/ColorzCore/Parser/AST/IAtomNode.cs +++ b/ColorzCore/Parser/AST/IAtomNode.cs @@ -11,8 +11,6 @@ public interface IAtomNode : IParamNode { //TODO: Simplify() partial evaluation as much as is defined, to save on memory space. int Precedence { get; } - string? GetIdentifier(); - IEnumerable ToTokens(); int? TryEvaluate(Action handler, EvaluationPhase evaluationPhase); //Simplifies the AST as much as possible. } diff --git a/ColorzCore/Parser/AST/IParamNode.cs b/ColorzCore/Parser/AST/IParamNode.cs index e04e7db..b0f6336 100644 --- a/ColorzCore/Parser/AST/IParamNode.cs +++ b/ColorzCore/Parser/AST/IParamNode.cs @@ -1,10 +1,7 @@ -using ColorzCore.DataTypes; -using System; +using System; using System.Collections.Generic; using System.Linq; -using System.Text; -using System.Threading.Tasks; -using ColorzCore.Lexer; +using ColorzCore.DataTypes; using ColorzCore.Interpreter; namespace ColorzCore.Parser.AST @@ -15,8 +12,9 @@ public interface IParamNode ParamType Type { get; } string PrettyPrint(); Location MyLocation { get; } - IParamNode SimplifyExpressions(Action handler, EvaluationPhase evaluationPhase); //TODO: Abstract this into a general traverse method. - IAtomNode? AsAtom(); + + // TODO: Abstract this into a general traverse method. + IParamNode SimplifyExpressions(Action handler, EvaluationPhase evaluationPhase); } public static class ParamExtensions diff --git a/ColorzCore/Parser/AST/IdentifierNode.cs b/ColorzCore/Parser/AST/IdentifierNode.cs index f717bde..56ca345 100644 --- a/ColorzCore/Parser/AST/IdentifierNode.cs +++ b/ColorzCore/Parser/AST/IdentifierNode.cs @@ -26,11 +26,24 @@ private int ToInt(EvaluationPhase evaluationPhase) { if (boundScope != null) { - for (ImmutableStack it = boundScope; !it.IsEmpty; it = it.Tail) + if (evaluationPhase == EvaluationPhase.Early) { - if (it.Head.HasLocalSymbol(IdentifierToken.Content)) + /* We only check the immediate scope. + * As an outer scope symbol may get shadowed by an upcoming inner scope symbol. */ + + if (boundScope.Head.HasLocalSymbol(IdentifierToken.Content)) + { + return boundScope.Head.GetSymbol(IdentifierToken.Content, evaluationPhase); + } + } + else + { + for (ImmutableStack it = boundScope; !it.IsEmpty; it = it.Tail) { - return it.Head.GetSymbol(IdentifierToken.Content, evaluationPhase); + if (it.Head.HasLocalSymbol(IdentifierToken.Content)) + { + return it.Head.GetSymbol(IdentifierToken.Content, evaluationPhase); + } } } } @@ -56,11 +69,6 @@ private int ToInt(EvaluationPhase evaluationPhase) } } - public override string? GetIdentifier() - { - return IdentifierToken.Content; - } - public override string PrettyPrint() { try @@ -82,11 +90,6 @@ public override string ToString() return IdentifierToken.Content; } - public override IEnumerable ToTokens() - { - yield return IdentifierToken; - } - // TODO: move this outside of this class public class UndefinedIdentifierException : Exception { diff --git a/ColorzCore/Parser/AST/ListNode.cs b/ColorzCore/Parser/AST/ListNode.cs index ee873f8..82a3fa6 100644 --- a/ColorzCore/Parser/AST/ListNode.cs +++ b/ColorzCore/Parser/AST/ListNode.cs @@ -5,7 +5,6 @@ using System.Collections.Generic; using System.Linq; using System.Text; -using System.Threading.Tasks; namespace ColorzCore.Parser.AST { @@ -49,36 +48,6 @@ public string PrettyPrint() sb.Append(']'); return sb.ToString(); } - public IEnumerable ToTokens() - { - //Similar code to ParenthesizedAtom - IList> temp = new List>(); - foreach (IAtomNode n in Interior) - { - temp.Add(new List(n.ToTokens())); - } - Location myStart = MyLocation; - Location myEnd = temp.Count > 0 ? temp.Last().Last().Location : MyLocation; - yield return new Token(TokenType.OPEN_BRACKET, new Location(myStart.file, myStart.line, myStart.column - 1), "["); - for (int i = 0; i < temp.Count; i++) - { - foreach (Token t in temp[i]) - { - yield return t; - } - if (i < temp.Count - 1) - { - Location tempEnd = temp[i].Last().Location; - yield return new Token(TokenType.COMMA, new Location(tempEnd.file, tempEnd.line, tempEnd.column + temp[i].Last().Content.Length), ","); - } - } - yield return new Token(TokenType.CLOSE_BRACKET, new Location(myEnd.file, myEnd.line, myEnd.column + (temp.Count > 0 ? temp.Last().Last().Content.Length : 1)), "]"); - } - - public Either TryEvaluate() - { - return new Right("Expected atomic parameter."); - } public IParamNode SimplifyExpressions(Action handler, EvaluationPhase evaluationPhase) { @@ -91,7 +60,5 @@ public IParamNode SimplifyExpressions(Action handler, EvaluationPhase } public int NumCoords { get { return Interior.Count; } } - - public IAtomNode? AsAtom() { return null; } } } diff --git a/ColorzCore/Parser/AST/MacroInvocationNode.cs b/ColorzCore/Parser/AST/MacroInvocationNode.cs index 0e73875..5d7668a 100644 --- a/ColorzCore/Parser/AST/MacroInvocationNode.cs +++ b/ColorzCore/Parser/AST/MacroInvocationNode.cs @@ -9,6 +9,7 @@ namespace ColorzCore.Parser.AST { + // TODO: what do we need this for? class MacroInvocationNode : IParamNode { public class MacroException : Exception @@ -51,22 +52,8 @@ public string PrettyPrint() return sb.ToString(); } - public IEnumerable ExpandMacro() - { - return p.Macros.GetMacro(invokeToken.Content, Parameters.Count).ApplyMacro(invokeToken, Parameters); - } - - public Either TryEvaluate() - { - return new Right("Expected atomic parameter."); - } - - public string Name { get { return invokeToken.Content; } } - public Location MyLocation { get { return invokeToken.Location; } } - public IAtomNode? AsAtom() { return null; } - public IParamNode SimplifyExpressions(Action handler, EvaluationPhase evaluationPhase) { handler(new MacroException(this)); diff --git a/ColorzCore/Parser/AST/NumberNode.cs b/ColorzCore/Parser/AST/NumberNode.cs index b694c08..ed64eb9 100644 --- a/ColorzCore/Parser/AST/NumberNode.cs +++ b/ColorzCore/Parser/AST/NumberNode.cs @@ -33,11 +33,6 @@ public NumberNode(Location loc, int value) Value = value; } - public override IEnumerable ToTokens() - { - yield return new Token(TokenType.NUMBER, MyLocation, Value.ToString()); - } - public override int? TryEvaluate(Action handler, EvaluationPhase evaluationPhase) { return Value; diff --git a/ColorzCore/Parser/AST/OperatorNode.cs b/ColorzCore/Parser/AST/OperatorNode.cs index 7354920..3d02f17 100644 --- a/ColorzCore/Parser/AST/OperatorNode.cs +++ b/ColorzCore/Parser/AST/OperatorNode.cs @@ -54,19 +54,6 @@ public override string PrettyPrint() return $"({Left.PrettyPrint()} {OperatorString} {Right.PrettyPrint()})"; } - public override IEnumerable ToTokens() - { - foreach (Token t in Left.ToTokens()) - { - yield return t; - } - yield return OperatorToken; - foreach (Token t in Right.ToTokens()) - { - yield return t; - } - } - private int? TryCoalesceUndefined(Action handler) { List? leftExceptions = null; diff --git a/ColorzCore/Parser/AST/StringNode.cs b/ColorzCore/Parser/AST/StringNode.cs index 65a7102..eb918be 100644 --- a/ColorzCore/Parser/AST/StringNode.cs +++ b/ColorzCore/Parser/AST/StringNode.cs @@ -12,8 +12,6 @@ namespace ColorzCore.Parser.AST { public class StringNode : IParamNode { - public static readonly Regex idRegex = new Regex("^([a-zA-Z_][a-zA-Z0-9_]*)$"); - public Token SourceToken { get; } public Location MyLocation => SourceToken.Location; @@ -36,14 +34,6 @@ public string PrettyPrint() return $"\"{Value}\""; } - public IEnumerable ToTokens() { yield return SourceToken; } - - public bool IsValidIdentifier() - { - return idRegex.IsMatch(Value); - } - - public IAtomNode? AsAtom() => null; public IParamNode SimplifyExpressions(Action handler, EvaluationPhase evaluationPhase) => this; } } diff --git a/ColorzCore/Parser/AST/UnaryOperatorNode.cs b/ColorzCore/Parser/AST/UnaryOperatorNode.cs index c7d2ea8..2f15f30 100644 --- a/ColorzCore/Parser/AST/UnaryOperatorNode.cs +++ b/ColorzCore/Parser/AST/UnaryOperatorNode.cs @@ -21,7 +21,7 @@ public UnaryOperatorNode(Token token, IAtomNode inside) Inner = inside; } - public override int Precedence => 11; + public override int Precedence => int.MaxValue; public override Location MyLocation => OperatorToken.Location; public string OperatorString => OperatorToken.Type switch @@ -37,13 +37,6 @@ public override string PrettyPrint() return OperatorString + Inner.PrettyPrint(); } - public override IEnumerable ToTokens() - { - yield return OperatorToken; - foreach (Token t in Inner.ToTokens()) - yield return t; - } - public override int? TryEvaluate(Action handler, EvaluationPhase evaluationPhase) { int? inner = Inner.TryEvaluate(handler, evaluationPhase); diff --git a/ColorzCore/Parser/EAParser.cs b/ColorzCore/Parser/EAParser.cs index 2ce5962..d17308b 100644 --- a/ColorzCore/Parser/EAParser.cs +++ b/ColorzCore/Parser/EAParser.cs @@ -563,13 +563,20 @@ private IList ParseParamList(MergeableGenerator tokens) return paramList; } + private static readonly Regex idRegex = new Regex("^([a-zA-Z_][a-zA-Z0-9_]*)$"); + public IList ParsePreprocParamList(MergeableGenerator tokens) { + static bool IsValidIdentifier(string value) + { + return idRegex.IsMatch(value); + } + IList temp = ParseParamList(tokens); for (int i = 0; i < temp.Count; i++) { - if (temp[i] is StringNode stringNode && stringNode.IsValidIdentifier()) + if (temp[i] is StringNode stringNode && IsValidIdentifier(stringNode.Value)) { // TODO: what is this for? can we omit it? temp[i] = BindIdentifier(stringNode.SourceToken); diff --git a/ColorzCore/Preprocessor/Directives/IncludeToolEventDirective.cs b/ColorzCore/Preprocessor/Directives/IncludeToolEventDirective.cs index 49b8c74..531ffcb 100644 --- a/ColorzCore/Preprocessor/Directives/IncludeToolEventDirective.cs +++ b/ColorzCore/Preprocessor/Directives/IncludeToolEventDirective.cs @@ -45,7 +45,11 @@ public override void Execute(EAParser parse, Token self, IList param StringBuilder argumentBuilder = new StringBuilder(); for (int i = 1; i < parameters.Count; i++) { - parameters[i].AsAtom().IfJust((IAtomNode n) => { parameters[i] = n.Simplify(EvaluationPhase.Early); }); + if (parameters[i] is IAtomNode atom) + { + parameters[i] = atom.Simplify(EvaluationPhase.Early); + } + argumentBuilder.Append(parameters[i].PrettyPrint()); argumentBuilder.Append(' '); } diff --git a/ColorzCore/Preprocessor/MacroCollection.cs b/ColorzCore/Preprocessor/MacroCollection.cs index 0bafbd3..6d0c58b 100644 --- a/ColorzCore/Preprocessor/MacroCollection.cs +++ b/ColorzCore/Preprocessor/MacroCollection.cs @@ -22,8 +22,6 @@ public MacroCollection(EAParser parent) { { "String", new StringMacro() }, { "IsDefined", new IsDefined(parent) }, - { "IsSymbolDefined", new IsSymbolDefined() }, - { "IsLabelDefined", new IsSymbolDefined() }, // alias }; } diff --git a/ColorzCore/Preprocessor/Macros/IsSymbolDefined.cs b/ColorzCore/Preprocessor/Macros/IsSymbolDefined.cs deleted file mode 100644 index b5531a1..0000000 --- a/ColorzCore/Preprocessor/Macros/IsSymbolDefined.cs +++ /dev/null @@ -1,49 +0,0 @@ -using System; -using System.Collections.Generic; -using ColorzCore.DataTypes; -using ColorzCore.Lexer; - -namespace ColorzCore.Preprocessor.Macros -{ - public class IsSymbolDefined : BuiltInMacro - { - public override IEnumerable ApplyMacro(Token head, IList> parameters) - { - if (parameters[0].Count != 1) - { - // TODO: err somehow - yield return MakeFalseToken(head.Location); - } - else - { - MacroLocation macroLocation = new MacroLocation(head.Content, head.Location); - Location location = head.Location.MacroClone(macroLocation); - - Token identifierToken = parameters[0][0]; - - // This used to be more involved, but now it is just a dummy - // ((id || 1) ?? 0) - - yield return new Token(TokenType.OPEN_PAREN, location, "("); - yield return new Token(TokenType.OPEN_PAREN, location, "("); - yield return identifierToken.MacroClone(macroLocation); - yield return new Token(TokenType.LOGOR_OP, location, "||"); - yield return new Token(TokenType.NUMBER, location, "1"); - yield return new Token(TokenType.CLOSE_PAREN, location, ")"); - yield return new Token(TokenType.UNDEFINED_COALESCE_OP, location, "??"); - yield return new Token(TokenType.NUMBER, location, "0"); - yield return new Token(TokenType.CLOSE_PAREN, location, ")"); - } - } - - public override bool ValidNumParams(int num) - { - return num == 1; - } - - protected static Token MakeFalseToken(Location location) - { - return new Token(TokenType.NUMBER, location, "0"); - } - } -} diff --git a/ColorzCore/Preprocessor/Macros/StringMacro.cs b/ColorzCore/Preprocessor/Macros/StringMacro.cs index 94558d2..88b0cd4 100644 --- a/ColorzCore/Preprocessor/Macros/StringMacro.cs +++ b/ColorzCore/Preprocessor/Macros/StringMacro.cs @@ -17,6 +17,13 @@ public override IEnumerable ApplyMacro(Token head, IList> pa Token token = parameters[0][0]; Location location = token.Location.MacroClone(macroLocation); + if (EAOptions.IsWarningEnabled(EAOptions.Warnings.LegacyFeatures)) + { + yield return new Token(TokenType.IDENTIFIER, location, "WARNING"); + yield return new Token(TokenType.STRING, location, "Consider using the STRING statement rather than the legacy String macro."); + yield return new Token(TokenType.SEMICOLON, location, ";"); + } + yield return new Token(TokenType.IDENTIFIER, location, "STRING"); yield return new Token(TokenType.STRING, location, token.Content); yield return new Token(TokenType.STRING, location, "UTF-8"); diff --git a/ColorzCore/Program.cs b/ColorzCore/Program.cs index 5a6033a..6e7ffbc 100644 --- a/ColorzCore/Program.cs +++ b/ColorzCore/Program.cs @@ -16,15 +16,11 @@ class Program { "unintuitive-expression-macros" , EAOptions.Warnings.UnintuitiveExpressionMacros }, { "unguarded-expression-macros", EAOptions.Warnings.UnguardedExpressionMacros }, { "redefine", EAOptions.Warnings.ReDefine }, + { "legacy", EAOptions.Warnings.LegacyFeatures }, { "all", EAOptions.Warnings.All }, { "extra", EAOptions.Warnings.Extra }, }; - private static readonly IDictionary extensionNames = new Dictionary() - { - { "read-data-macros", EAOptions.Extensions.ReadDataMacros } - }; - private static readonly string[] helpstringarr = { "EA Colorz Core. Usage:", "./ColorzCore [-opts]", @@ -83,9 +79,6 @@ class Program " Sets the maximum size of the binary. Defaults to 0x02000000.", "-romoffset:", " Compatibility alias for --base-address:", - "--extensions:[no-]:...", - " Enable or disable extensions. By default, no extension is enabled.", - " Possible values: " + string.Join(", ", extensionNames.Keys), "-h|--help", " Display this message and exit.", "" @@ -303,44 +296,6 @@ static int Main(string[] args) break; - case "--extensions": - if (flag.Length == 1) - { - EAOptions.EnabledExtensions |= EAOptions.Extensions.All; - } - else - { - foreach (string extension in flag[1].Split(':')) - { - string name = extension; - bool invert = false; - - if (name.StartsWith("no-")) - { - name = name.Substring(3); - invert = true; - } - - if (extensionNames.TryGetValue(name, out EAOptions.Extensions extFlag)) - { - if (invert) - { - EAOptions.EnabledExtensions &= ~extFlag; - } - else - { - EAOptions.EnabledExtensions |= extFlag; - } - } - else - { - Console.Error.WriteLine($"Unrecognized extension: {name}"); - } - } - } - - break; - default: Console.Error.WriteLine($"Unrecognized flag: {flag[0]}"); return EXIT_FAILURE; diff --git a/ColorzCore/Raws/AtomicParam.cs b/ColorzCore/Raws/AtomicParam.cs index 4cb1ac4..39699a8 100644 --- a/ColorzCore/Raws/AtomicParam.cs +++ b/ColorzCore/Raws/AtomicParam.cs @@ -24,9 +24,10 @@ public AtomicParam(string name, int position, int length, bool isPointer) pointer = isPointer; } + // Precondition: input is an IAtomNode public void Set(byte[] data, IParamNode input) { - Set(data, input.AsAtom()!.CoerceInt()); + Set(data, (input as IAtomNode)!.CoerceInt()); } public void Set(byte[] data, int value) From ead413d0987c14a8d7152f613e76c361f72325bf Mon Sep 17 00:00:00 2001 From: Nat_776 Date: Sun, 5 May 2024 12:38:45 +0200 Subject: [PATCH 59/59] add more tests --- Tests/README.md | 2 + Tests/directives.py | 102 +++++++++++++++++++++++++++++++++++++++++++ Tests/expressions.py | 90 ++++++++++++++++++++++++++++++++++++++ Tests/run_tests.py | 57 +++--------------------- 4 files changed, 200 insertions(+), 51 deletions(-) create mode 100644 Tests/directives.py create mode 100644 Tests/expressions.py diff --git a/Tests/README.md b/Tests/README.md index e3bca9c..1a8705c 100644 --- a/Tests/README.md +++ b/Tests/README.md @@ -1 +1,3 @@ Run using `python run_tests.py path/to/ColorzCore.exe`. + +Tests are incomplete. Do not assume that because all tests are passing the program works fine. diff --git a/Tests/directives.py b/Tests/directives.py new file mode 100644 index 0000000..c2d3d98 --- /dev/null +++ b/Tests/directives.py @@ -0,0 +1,102 @@ +from ea_test import EATest as T + + +TESTS = [ + # ===================== + # = #define Directive = + # ===================== + + # '#define' traditional nominal behavior + T("Define Basic object-like", + '#define Value 0xFA \n ORG 0 ; BYTE Value', + b"\xFA"), + + T("Define Basic function-like", + '#define Macro(a) "0xFA + (a)" \n ORG 0 ; BYTE Macro(2)', + b"\xFC"), + + # '#define' a second time overrides the first definition + # NOTE: this is probably not something that we want to freeze + # T("Define override", + # '#define Value 1 \n #define Value 2 \n ORG 0 ; BYTE Value', + # b"\x02"), + + # '#define' using a vector as argument (extra commas) + T("Define vector argument", + '#define Macro(a) "BYTE 1" \n ORG 0 ; Macro([1, 2, 3])', + b"\x01"), + + # '#define ... "..."' with escaped newlines inside string + T("Multi-line string define", + '#define SomeLongMacro(A, B, C) "\\\n ALIGN 4 ; \\\n WORD C ; \\\n SHORT B ; \\\n BYTE A" \n' + + 'ORG 0 ; SomeLongMacro(0xAA, 0xBB, 0xCC)', + b"\xCC\x00\x00\x00\xBB\x00\xAA"), + + T("Define eager expansion", + "#define Value 1 \n #define OtherValue Value \n #undef Value \n #define Value 2 \n ORG 0 ; BYTE OtherValue", + b'\x01'), + + # '#define ...' multi-token without quotes + T("Multi-token define 1", + '#define Value (1 + 2) \n ORG 0 ; BYTE Value', + b"\x03"), + + T("Multi-token define 2", + '#define Macro(a, b) (a + b) \n ORG 0 ; BYTE Macro(1, 2)', + b"\x03"), + + T("Multi-token define 2", + '#define Macro(a, b) BYTE a a + b b \n ORG 0 ; Macro(1, 2)', + b"\x01\x03\x02"), + + # Those would fail on + T("Define uinintuitive atomic 1", + '#define Value 1 + 2 \n ORG 0 ; BYTE Value * 2', + b"\x05"), # 1 * 2 * 2 = 5 + + T("Define uinintuitive atomic 1", + '#define Value "1 + 2" \n ORG 0 ; BYTE Value * 2', + b"\x05"), # 1 * 2 * 2 = 5 + + # '#define MyMacro MyMacro' (MyMacro shouldn't expand) + T("Non-productive macros 1", + '#define MyMacro MyMacro \n ORG 0 ; MyMacro: ; BYTE 1', + b'\x01'), + + T("Non-productive macros 2", + '#define MyMacro MyMacro \n ORG 0 ; BYTE IsDefined(MyMacro)', + b'\x01'), + + T("Non-productive macros 3", + '#define MyMacro MyMacro \n ORG 0 ; #ifdef MyMacro \n BYTE 1 \n #else \n BYTE 0 \n #endif', + b'\x01'), + + # ==================== + # = #undef Directive = + # ==================== + + T("Undef 1", + '#define Value 1 \n #undef Value \n ORG 0 ; BYTE Value', + None), + + T("Undef 2", + '#define Value 1 \n #undef Value \n #ifndef Value \n ORG 0 ; BYTE 1 \n #endif', + b"\x01"), + + T("Undef multiple", + '#define ValueA \n #define ValueB \n #undef ValueA ValueB \n #ifndef ValueA \n #ifndef ValueB \n' + + 'BYTE 1 \n #endif \n #endif', + b'\x01'), + + # ======================== + # = #if[n]def Directives = + # ======================== + + # '#ifdef' + T("Ifdef", 'ORG 0 \n #define Value \n #ifdef Value \n BYTE 1 \n #else \n BYTE 0 \n #endif', b"\x01"), + + # '#ifndef' + T("Ifndef", 'ORG 0 \n #define Value \n #ifndef Value \n BYTE 1 \n #else \n BYTE 0 \n #endif', b"\x00"), + + # TODO: #if, #include, #incbin, #incext, #inctext, #pool +] diff --git a/Tests/expressions.py b/Tests/expressions.py new file mode 100644 index 0000000..5b740ca --- /dev/null +++ b/Tests/expressions.py @@ -0,0 +1,90 @@ +from ea_test import EATest as T + +def byte(expr): + return f"ORG 0 ; BYTE {expr} ;" + +TESTS = [ + T("Operator '+'", byte('1 + 2'), b'\x03'), + + T("Operator '-' 1", byte('2 - 1'), b'\x01'), + T("Operator '-' 2", byte('1 - 2'), b'\xFF'), + + T("Operator '*'", byte('3 * 2'), b'\x06'), + + T("Operator '/' 1", byte('6 / 2'), b'\x03'), + T("Operator '/' 2", byte('5 / 2'), b'\x02'), # +2 (round towards zero) + T("Operator '/' 3", byte('(-5) / 2'), b'\xFE'), # -2 (round towards zero) + + T("Operator '%' 1", byte('5 % 2'), b'\x01'), # +1 + T("Operator '%' 2", byte('(-5) % 2'), b'\xFF'), # -1 + + T("Operator '<<'", byte('3 << 2'), b'\x0C'), # 12 + T("Operator '>>'", byte('3 >> 1'), b'\x01'), # 12 + T("Operator '>>>'", byte('0x80000000 >>> 25'), b'\xC0'), # 0b11000000 + + T("Operator '<' 1", byte('1 < 2'), b'\x01'), + T("Operator '<' 2", byte('2 < 1'), b'\x00'), + T("Operator '<' 3", byte('2 < 2'), b'\x00'), + + T("Operator '<=' 1", byte('1 <= 2'), b'\x01'), + T("Operator '<=' 2", byte('2 <= 1'), b'\x00'), + T("Operator '<=' 3", byte('2 <= 2'), b'\x01'), + + T("Operator '==' 1", byte('1 == 2'), b'\x00'), + T("Operator '==' 2", byte('2 == 1'), b'\x00'), + T("Operator '==' 3", byte('2 == 2'), b'\x01'), + + T("Operator '!=' 1", byte('1 != 2'), b'\x01'), + T("Operator '!=' 2", byte('2 != 1'), b'\x01'), + T("Operator '!=' 3", byte('2 != 2'), b'\x00'), + + T("Operator '>=' 1", byte('1 >= 2'), b'\x00'), + T("Operator '>=' 2", byte('2 >= 1'), b'\x01'), + T("Operator '>=' 3", byte('2 >= 2'), b'\x01'), + + T("Operator '>' 1", byte('1 > 2'), b'\x00'), + T("Operator '>' 2", byte('2 > 1'), b'\x01'), + T("Operator '>' 3", byte('2 > 2'), b'\x00'), + + T("Operator '&' 1", byte('3 & 6'), b'\x02'), + T("Operator '&' 2", byte('1 & 6'), b'\x00'), + + T("Operator '|' 1", byte('1 | 12'), b'\x0D'), # 0b1101 + T("Operator '|' 2", byte('1 | 1'), b'\x01'), + + T("Operator '^' 1", byte('3 ^ 6'), b'\x05'), + T("Operator '^' 2", byte('1 ^ 6'), b'\x07'), + + T("Operator '&&' 1", byte('0 && 1'), b'\x00'), + T("Operator '&&' 2", byte('1 && 1'), b'\x01'), + T("Operator '&&' 3", byte('1 && 10'), b'\x0A'), + + T("Operator '||' 1", byte('0 || 1'), b'\x01'), + T("Operator '||' 2", byte('1 || 1'), b'\x01'), + T("Operator '||' 3", byte('1 || 10'), b'\x01'), + T("Operator '||' 4", byte('0 || 10'), b'\x0A'), + T("Operator '||' 5", byte('8 || 1'), b'\x08'), + + T("Operator '??' 1", f'A := 0 ;' + byte('(A || 1) ?? 0'), b"\x01"), + T("Operator '??' 2", byte('(A || 1) ?? 0'), b"\x00"), + + T("Operator unary '-' 1", byte('-1'), b'\xFF'), + T("Operator unary '-' 2", byte('-(1 + 2)'), b'\xFD'), + + T("Operator unary '~' 1", byte('~0'), b'\xFF'), + T("Operator unary '~' 2", byte('~3'), b'\xFC'), + T("Operator unary '~' 3", byte('~(-1)'), b'\x00'), + + T("Operator unary '!' 1", byte('!76'), b'\x00'), + T("Operator unary '!' 2", byte('!0'), b'\x01'), + T("Operator unary '!' 3", byte('!!7'), b'\x01'), + + T("Precedence 1 ('+', '*')", byte('1 + 2 * 3'), b'\x07'), # +7 + T("Precedence 2 ('-', '*')", byte('1 - 2 * 3'), b'\xFB'), # -5 + T("Precedence 3 ('+', '/')", byte('4 + 6 / 2'), b'\x07'), # +7 (not +5) + T("Precedence 4 ('+', '%')", byte('5 + 5 % 2'), b'\x06'), # +6 (not +0) + T("Precedence 5 ('<<', '+')", byte('2 << 1 + 5'), bytes((128,))), # not 9 + T("Precedence 6 ('>>', '+')", byte('0xFF >> 1 + 5'), b'\x03'), + T("Precedence 7 ('>>>', '+')", byte('0x80000000 >>> 20 + 5'), b'\xC0'), + # TODO: more +] diff --git a/Tests/run_tests.py b/Tests/run_tests.py index 7aa8f2c..19b375a 100644 --- a/Tests/run_tests.py +++ b/Tests/run_tests.py @@ -7,61 +7,16 @@ T("Precedence 1", "ORG 0 ; BYTE 1 + 2 * 10", b"\x15"), # POIN - T("POIN 1", "ORG 0 ; POIN 4", b"\x04\x00\x00\x08"), - T("POIN 2", "ORG 0 ; POIN 0", b"\x00\x00\x00\x00"), - T("POIN 3", "ORG 0 ; POIN 0x08000000", b"\x00\x00\x00\x08"), - T("POIN 4", "ORG 0 ; POIN 0x02000000", b"\x00\x00\x00\x02"), + T("POIN Offset", "ORG 0 ; POIN 4", b"\x04\x00\x00\x08"), + T("POIN NULL", "ORG 0 ; POIN 0", b"\x00\x00\x00\x00"), + T("POIN Address", "ORG 0 ; POIN 0x08000000", b"\x00\x00\x00\x08"), + T("POIN RAM", "ORG 0 ; POIN 0x02000000", b"\x00\x00\x00\x02"), ] -EXPRESSION_TESTS = [ - T("UNDCOERCE 1", 'A := 0 ; ORG 0 ; BYTE (A || 1) ?? 0', b"\x01"), - T("UNDCOERCE 2", 'ORG 0 ; BYTE (A || 1) ?? 0', b"\x00"), -] - - -PREPROC_TESTS = [ - # '#define' traditional nominal behavior - T("Define 1", '#define Value 0xFA \n ORG 0 ; BYTE Value', b"\xFA"), - T("Define 2", '#define Macro(a) "0xFA + (a)" \n ORG 0 ; BYTE Macro(2)', b"\xFC"), - T("Define 3", '#define Value \n #ifdef Value \n ORG 0 ; BYTE 1 \n #endif', b"\x01"), - - # '#define' a second time overrides the first definition - T("Define override", '#define Value 1 \n #define Value 2 \n ORG 0 ; BYTE Value', b"\x02"), - - # '#define' using a vector as argument (extra commas) - T("Define vector argument", '#define Macro(a) "BYTE 1" \n ORG 0 ; Macro([1, 2, 3])', b"\x01"), - - # '#define ... "..."' with escaped newlines inside string - T("Multi-line string define", '#define SomeLongMacro(A, B, C) "\\\n ALIGN 4 ; \\\n WORD C ; \\\n SHORT B ; \\\n BYTE A" \n ORG 0 ; SomeLongMacro(0xAA, 0xBB, 0xCC)', b"\xCC\x00\x00\x00\xBB\x00\xAA"), - - # '#define ...' multi-token without quotes - T("Multi-token define 1", '#define Value (1 + 2) \n ORG 0 ; BYTE Value', b"\x03"), - T("Multi-token define 2", '#define Macro(a, b) (a + b) \n ORG 0 ; BYTE Macro(1, 2)', b"\x03"), - T("Multi-token define 2", '#define Macro(a, b) BYTE a a + b b \n ORG 0 ; Macro(1, 2)', b"\x01\x03\x02"), - - # '#ifdef' - T("Ifdef", 'ORG 0 \n #define Value \n #ifdef Value \n BYTE 1 \n #else \n BYTE 0 \n #endif', b"\x01"), - - # '#ifndef' - T("Ifndef", 'ORG 0 \n #define Value \n #ifndef Value \n BYTE 1 \n #else \n BYTE 0 \n #endif', b"\x00"), - - # '#define MyMacro MyMacro' (MyMacro shouldn't expand) - T("Non-productive macros 1", '#define MyMacro MyMacro \n ORG 0 ; MyMacro: ; BYTE 1', b'\x01'), - T("Non-productive macros 2", '#define MyMacro MyMacro \n ORG 0 ; BYTE IsDefined(MyMacro)', b'\x01'), - T("Non-productive macros 3", '#define MyMacro MyMacro \n ORG 0 ; #ifdef MyMacro \n BYTE 1 \n #else \n BYTE 0 \n #endif', b'\x01'), - - # T("IFDEF 2", 'ORG 0 \n #define A \n #define B \n #ifdef A B \n BYTE 1 \n #else \n BYTE 0 \n #endif', b"\x01"), - - # '#undef' - T("Undef 1", '#define Value 1 \n #undef Value \n ORG 0 ; BYTE Value', None), - T("Undef 2", '#define Value 1 \n #undef Value \n #ifndef Value \n ORG 0 ; BYTE 1 \n #endif', b"\x01"), -] - - -import statements, symbols +import statements, symbols, directives, expressions -ALL_TEST_CASES = BASIC_TESTS + statements.TESTS + symbols.TESTS + EXPRESSION_TESTS + PREPROC_TESTS +ALL_TEST_CASES = BASIC_TESTS + statements.TESTS + symbols.TESTS + directives.TESTS + expressions.TESTS def main(args): import argparse