diff --git a/.github/workflows/dotnetcore.yml b/.github/workflows/dotnetcore.yml index 57bbadf2..576e5b4e 100644 --- a/.github/workflows/dotnetcore.yml +++ b/.github/workflows/dotnetcore.yml @@ -13,6 +13,7 @@ on: - dev permissions: pull-requests: write + checks: write jobs: build: env: @@ -23,7 +24,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, windows-latest] + os: [ubuntu-latest] steps: - uses: actions/checkout@v2 - name: Setup .NET Core @@ -35,7 +36,7 @@ jobs: - name: Build with dotnet run: dotnet build --configuration Release - name: Test with dotnet - uses: b3b00/coverlet-action@1.3.5-alpha1 + uses: b3b00/coverlet-action@d4de331462c1706fd9f07b4b2a565f256fd95a23 id: 'coverlet' if: env.RUN_TESTS with: @@ -44,17 +45,22 @@ jobs: threshold: 80 outputFormat: 'lcov' excludes: '[program]*,[expressionParser]*,[jsonparser]*,[while]*,[indentedWhile]*,[SimpleExpressionParser]*,[GenericLexerWithCallbacks]*,[indented]*,[postProcessedLexerParser]*,[XML]*,[SimpleTemplate]*,[SlowEOS]*' - + - name: Publish Test Results + uses: EnricoMi/publish-unit-test-result-action@3a74b2957438d0b6e2e61d67b05318aa25c9e6c6 + if: always() + with: + files: | + **/*.trx - name: coveralls uses: coverallsapp/github-action@v1.1.1 - if: matrix.os == 'windows-latest' && env.RUN_TESTS + if: matrix.os == 'ubuntu-latest' && env.RUN_TESTS with: fail-on-error: false github-token: ${{secrets.GITHUB_TOKEN }} path-to-lcov: ${{steps.coverlet.outputs.coverageFile}} #base-path: ${{github.workspace}} - name: publish nuget - if: ${{success() && matrix.os == 'windows-latest' && env.PUBLISH_NUGET}} + if: ${{success() && matrix.os == 'ubuntu-latest' && env.PUBLISH_NUGET}} id: publish_nuget uses: alirezanet/publish-nuget@v3.0.0 with: @@ -64,7 +70,7 @@ jobs: VERSION_FILE_PATH: ${{env.MAIN_CSPROJ}} INCLUDE_SYMBOLS: true - name: Create Release - if: ${{ success() && matrix.os == 'windows-latest' && steps.publish_nuget.outputs.VERSION != '' && steps.publish_nuget.outputs.VERSION != null }} + if: ${{ success() && matrix.os == 'ubuntu-latest' && steps.publish_nuget.outputs.VERSION != '' && steps.publish_nuget.outputs.VERSION != null }} id: create_release uses: b3b00/create-release@1.0.7 env: @@ -76,7 +82,7 @@ jobs: prerelease: false failsOnCreationError: false - name: Upload Release nuget Asset - if: ${{ success() && matrix.os == 'windows-latest' && steps.create_release.outputs.upload_url != '' && steps.create_release.outputs.upload_url != null }} + if: ${{ success() && matrix.os == 'ubuntu-latest' && steps.create_release.outputs.upload_url != '' && steps.create_release.outputs.upload_url != null }} id: upload-release-nuget-asset uses: actions/upload-release-asset@v1 env: @@ -87,7 +93,7 @@ jobs: asset_name: ${{ steps.publish_nuget.outputs.PACKAGE_NAME }} asset_content_type: application/zip - name: Upload Release symbols nuget Asset - if: ${{ success() && matrix.os == 'windows-latest' && steps.create_release.outputs.upload_url != '' && steps.create_release.outputs.upload_url != null }} + if: ${{ success() && matrix.os == 'ubuntu-latest' && steps.create_release.outputs.upload_url != '' && steps.create_release.outputs.upload_url != null }} id: upload-release-nuget-symbols-asset uses: actions/upload-release-asset@v1 env: diff --git a/.gitignore b/.gitignore index a2176161..b419611d 100644 --- a/.gitignore +++ b/.gitignore @@ -49,3 +49,4 @@ BenchmarkDotNet.Artifacts AotTester/* *.dtp *.dtp.state +src/samples/HandMadeExpres* diff --git a/src/benchCurrent/Program.cs b/src/benchCurrent/Program.cs index 9b432b6d..1b8c7684 100644 --- a/src/benchCurrent/Program.cs +++ b/src/benchCurrent/Program.cs @@ -1,5 +1,6 @@ using System; using BenchmarkDotNet.Running; +using sly.parser.generator; namespace benchCurrent { @@ -15,7 +16,15 @@ private static void Bench() { // var summary3 = BenchmarkRunner.Run(); //var summary4 = BenchmarkRunner.Run(); - var summary5 = BenchmarkRunner.Run(); + + // StackExpressionBench bench = new StackExpressionBench(); + // bench.parserType = ParserType.LL_RECURSIVE_DESCENT; + // bench.Setup(); + // bench.BenchLargeExpression(); + + //var summary5 = BenchmarkRunner.Run(); + + var summary6 = BenchmarkRunner.Run(); } static void Main(string[] args) diff --git a/src/benchCurrent/StackEbnfExpressionBench.cs b/src/benchCurrent/StackEbnfExpressionBench.cs new file mode 100644 index 00000000..a10ab70d --- /dev/null +++ b/src/benchCurrent/StackEbnfExpressionBench.cs @@ -0,0 +1,91 @@ +using System; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Columns; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Toolchains.CsProj; +using simpleExpressionParser; +using sly.parser.generator; + + +namespace benchCurrent +{ + + [MemoryDiagnoser] + + [Config(typeof(Config))] + //[Config(typeof(ConfigWithPercentage))] + public class StackEbnfExpressionBench + { + + + private class Config : ManualConfig + { + public Config() + { + SummaryStyle = BenchmarkDotNet.Reports.SummaryStyle.Default.WithRatioStyle(RatioStyle.Trend); + var baseJob = Job.MediumRun.With(CsProjCoreToolchain.NetCoreApp80); + } + } + + private string expression = ""; + + [GlobalSetup] + public void Setup() + { + expression = GetExpression(1000); + } + + //[Params(ParserType.LL_RECURSIVE_DESCENT,ParserType.LL_STACK )] + //public ParserType parserType { get; set; } + + public string GetExpression(int max) + { + var rnd = new Random(); + //int width = rnd.Next(100, max); + char[] ops = new[] { '+', '-', '*' }; + var getOp = () => ops[rnd.Next(0, ops.Length)]; + var expr = rnd.Next(0, 100).ToString(); + for (int i = 0; i < max; i++) + { + var op = getOp(); + var right = rnd.Next(0, 100); + expr += $"{op} {right}"; + + } +Console.WriteLine(expr); + return expr; + } + + + [Benchmark(Baseline = true)] + public void recursive() => BenchLargeExpression(ParserType.EBNF_LL_RECURSIVE_DESCENT); + + [Benchmark] + public void stacked() => BenchLargeExpression(ParserType.EBNF_LL_STACK); + + public void BenchLargeExpression(ParserType type) + { + var instance = new SimpleExpressionParser(); + ParserBuilder builder = + new ParserBuilder(); + var parser = builder.BuildParser(instance, type, "root"); + if (!parser.IsOk) + { + foreach (var error in parser.Errors) + { + Console.WriteLine(error.Message); + } + Environment.Exit(1); + } + var r = parser.Result.Parse(expression); + ; + } + + + + + + } + +} diff --git a/src/benchCurrent/StackExpressionBench.cs b/src/benchCurrent/StackExpressionBench.cs new file mode 100644 index 00000000..5aef4112 --- /dev/null +++ b/src/benchCurrent/StackExpressionBench.cs @@ -0,0 +1,93 @@ +using System; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Columns; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Toolchains.CsProj; +using expressionparser; +using simpleExpressionParser; +using sly.parser; +using sly.parser.generator; +using ExpressionToken = simpleExpressionParser.ExpressionToken; + +namespace benchCurrent +{ + + [MemoryDiagnoser] + + [Config(typeof(Config))] + //[Config(typeof(ConfigWithPercentage))] + public class StackExpressionBench + { + + + private class Config : ManualConfig + { + public Config() + { + SummaryStyle = BenchmarkDotNet.Reports.SummaryStyle.Default.WithRatioStyle(RatioStyle.Trend); + var baseJob = Job.MediumRun.With(CsProjCoreToolchain.NetCoreApp80); + } + } + + private string expression = ""; + + [GlobalSetup] + public void Setup() + { + expression = GetExpression(1000); + } + + //[Params(ParserType.LL_RECURSIVE_DESCENT,ParserType.LL_STACK )] + //public ParserType parserType { get; set; } + + public string GetExpression(int max) + { + var rnd = new Random(); + //int width = rnd.Next(100, max); + char[] ops = new[] { '+', '-', '*' }; + var getOp = () => ops[rnd.Next(0, ops.Length)]; + var expr = rnd.Next(0, 100).ToString(); + for (int i = 0; i < max; i++) + { + var op = getOp(); + var right = rnd.Next(0, 100); + expr += $"{op} {right}"; + + } +Console.WriteLine(expr); + return expr; + } + + + [Benchmark(Baseline = true)] + public void recursive() => BenchLargeExpression(ParserType.LL_RECURSIVE_DESCENT); + + [Benchmark] + public void stacked() => BenchLargeExpression(ParserType.LL_STACK); + + public void BenchLargeExpression(ParserType type) + { + var instance = new ExpressionParser(); + ParserBuilder builder = + new ParserBuilder(); + var parser = builder.BuildParser(instance, type, "expression"); + if (!parser.IsOk) + { + foreach (var error in parser.Errors) + { + Console.WriteLine(error.Message); + } + Environment.Exit(1); + } + var r = parser.Result.Parse(expression); + ; + } + + + + + + } + +} diff --git a/src/profiler/Program.cs b/src/profiler/Program.cs index ad631574..e4ecd5a9 100644 --- a/src/profiler/Program.cs +++ b/src/profiler/Program.cs @@ -1,18 +1,66 @@ using System; +using System.Collections.Generic; using System.Diagnostics; +using System.Diagnostics.Metrics; using System.IO; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; using csly.indentedWhileLang.parser; using csly.whileLang.model; +using expressionparser; using jsonparser; using jsonparser.JsonModel; +using simpleExpressionParser; +using sly.buildresult; +using sly.parser; using sly.parser.generator; +using ExpressionToken = simpleExpressionParser.ExpressionToken; namespace profiler { class Program { - static void ProfileWhile() + static string GetExpression(int max) + { + var rnd = new Random(); + //int width = rnd.Next(100, max); + char[] ops = new[] { '+', '-', '*' }; + var getOp = () => ops[rnd.Next(0, ops.Length)]; + var expr = rnd.Next(0, 100).ToString(); + for (int i = 0; i < max; i++) + { + var op = getOp(); + var right = rnd.Next(0, 100); + expr += $"{op} {right}"; + + } + return expr; + } + + static void ProfileStackEbnfExpresion() + { + var expresion = GetExpression(1000); + var instance = new SimpleExpressionParser(); + ParserBuilder builder = + new ParserBuilder(); + var parser = builder.BuildParser(instance, ParserType.EBNF_LL_STACK, "root"); + if (!parser.IsOk) + { + foreach (var error in parser.Errors) + { + Console.WriteLine(error.Message); + } + + Environment.Exit(1); + } + + var r = parser.Result.Parse(expresion); + ; + } + + static void ProfileWhile() { string source = @" r:=1 @@ -113,11 +161,212 @@ print i7 static void Main(string[] args) { - //ProfileJson(); - for (int i = 0; i < 15; i++) - { - ProfileWhile(); - } + Dictionary> timings = new Dictionary>(); + //ProfileExpressions(10000, 100, true, timings); + ProfileEbnfExpressions(10000, 100, true, timings); + } + + static void ProfileExpressions(int max, int step, bool progression, + Dictionary> timings) + { + var instance = new ExpressionParser(); + ParserBuilder builder = + new ParserBuilder(); + + var types = new List() { ParserType.LL_STACK, ParserType.LL_RECURSIVE_DESCENT }; + foreach (var type in types) + { + Console.WriteLine(); + Console.WriteLine(type); + var b = builder.BuildParser(instance, type, "expression"); + if (b.IsError) + { + foreach (var error in b.Errors) + { + Console.WriteLine(error.Message); + } + return; + } + + + + if (progression) + { + for (int i = 2; i < max; i += step) + { + Console.Write(i); + + Stopwatch watch = new Stopwatch(); + watch.Start(); + SingleExpressionProfile(i, b); + watch.Stop(); + + Dictionary timing; + if (!timings.TryGetValue(i, out timing)) + { + timing = new Dictionary(); + } + + timing[type] = watch.ElapsedMilliseconds; + timings[i] = timing; + var pos = Console.GetCursorPosition(); + var l = i.ToString().Length; + Console.SetCursorPosition(pos.Left - l, pos.Top); + WriteTimings(timings, types); + } + } + else + { + SingleExpressionProfile(max, b); + } + } + + + + + } + + + static void ProfileEbnfExpressions(int max, int step, bool progression, + Dictionary> timings) + { + var instance = new SimpleExpressionParser(); + ParserBuilder builder = + new ParserBuilder(); + + var types = new List() { ParserType.EBNF_LL_STACK, ParserType.EBNF_LL_RECURSIVE_DESCENT }; + foreach (var type in types) + { + Console.WriteLine(); + Console.WriteLine(type); + var b = builder.BuildParser(instance, type, $"{nameof(SimpleExpressionParser)}_expressions"); + if (b.IsError) + { + foreach (var error in b.Errors) + { + Console.WriteLine(error.Message); + } + return; + } + + + + if (progression) + { + for (int i = 2; i < max; i += step) + { + Console.Write(i); + + Stopwatch watch = new Stopwatch(); + watch.Start(); + SingleEbnfExpressionProfile(i, b); + watch.Stop(); + + Dictionary timing; + if (!timings.TryGetValue(i, out timing)) + { + timing = new Dictionary(); + } + + timing[type] = watch.ElapsedMilliseconds; + timings[i] = timing; + var pos = Console.GetCursorPosition(); + var l = i.ToString().Length; + Console.SetCursorPosition(pos.Left - l, pos.Top); + WriteTimings(timings, types); + } + } + else + { + SingleEbnfExpressionProfile(max, b); + } + } + + + + + } + + private static void WriteTimings(Dictionary> timings, List types) + { + var csvBuilder = new StringBuilder(); + csvBuilder.Append("time"); + foreach (var type in types) + { + csvBuilder.Append($";{type}"); + } + foreach (var line in timings) + { + csvBuilder.Append($"\n{line.Key}"); + foreach (var type in types) + { + csvBuilder.Append(";"); + if (line.Value.TryGetValue(type, out var value)) + { + csvBuilder.Append(value); + } + } + } + if (File.Exists("c:/tmp/progression.csv")) + { + File.Delete("c:/tmp/progresssion.csv"); + } + + File.WriteAllText("c:/tmp/progression.csv",csvBuilder.ToString()); + } + + private static void SingleExpressionProfile(int max, BuildResult> b) + { + var rnd = new Random(); + //int width = rnd.Next(100, max); + char[] ops = new[] { '+', '-', '*' }; + var getOp = () => ops[rnd.Next(0, ops.Length)]; + var expression = rnd.Next(0, 100).ToString(); + for(int i = 0; i < max; i++) + { + var op = getOp(); + var right = rnd.Next(0, 100); + expression += $"{op} {right}"; + + } + //Console.WriteLine($"parsing {expression}"); + var result = b.Result.Parse(expression); + if (result.IsError) + { + File.WriteAllLines("c:/tmp/progress_errors.txt", new[] { expression }); + File.WriteAllLines("c:/tmp/progress_errors.txt", result.Errors.Select(x => x.ErrorMessage).ToArray() ); + Console.WriteLine($"error parsing {expression}"); + Environment.Exit(max); + + return; + } + } + + private static void SingleEbnfExpressionProfile(int max, BuildResult> b) + { + var rnd = new Random(); + //int width = rnd.Next(100, max); + char[] ops = new[] { '+', '-', '*' }; + var getOp = () => ops[rnd.Next(0, ops.Length)]; + var expression = rnd.Next(0, 100).ToString(); + for(int i = 0; i < max; i++) + { + var op = getOp(); + var right = rnd.Next(0, 100); + expression += $"{op} {right}"; + + } + //Console.WriteLine($"parsing {expression}"); + var result = b.Result.Parse(expression); + if (result.IsError) + { + File.WriteAllLines("c:/tmp/progress_errors.txt", new[] { expression }); + File.WriteAllLines("c:/tmp/progress_errors.txt", result.Errors.Select(x => x.ErrorMessage).ToArray() ); + Console.WriteLine($"error parsing {expression}"); + Environment.Exit(max); + + return; + } } private static void ProfileJson() diff --git a/src/profiler/profiler.csproj b/src/profiler/profiler.csproj index 0418b786..659e582f 100644 --- a/src/profiler/profiler.csproj +++ b/src/profiler/profiler.csproj @@ -8,6 +8,7 @@ + diff --git a/src/samples/ParserExample/Program.cs b/src/samples/ParserExample/Program.cs index 0fe7b7d0..19bf5272 100644 --- a/src/samples/ParserExample/Program.cs +++ b/src/samples/ParserExample/Program.cs @@ -871,6 +871,22 @@ private static BuildResult> BuildParserExpressio return builder.BuildParser(parserInstance, ParserType.EBNF_LL_RECURSIVE_DESCENT, StartingRule); } + private static void Schtak() + { + // Console.WriteLine("basic"); + // Stacker.Stack(); + // Console.WriteLine("a bit more complicated"); + // Stacker.MoreStack(); + // Console.WriteLine("even more complicated"); + // Stacker.EvenMoreStack(); + // Console.WriteLine("and finally an expression"); + // Stacker.Expression(); + //Console.WriteLine("one more with rules parser"); + //Stacker.Rules(); + Console.WriteLine("EBNF many (*)"); + Stacker.TestEbnfSimpleZeroOrMore(); + } + public static void TestAssociativityFactorExpressionParser() { @@ -1357,7 +1373,9 @@ print a } private static void Main(string[] args) { - testIssue516(); + TestPrefixPostfixStack(); + //Schtak(); + //testIssue516(); //TestIssue507(); //TestFStrings(); //TestIssue495(); @@ -1848,6 +1866,21 @@ private static void TestFStrings() IndentedWhileTests tests = new IndentedWhileTests(); tests.TestFString(); } + + public static void TestPrefixPostfixStack() + { + var startingRule = $"{nameof(SimpleExpressionParserWithContext)}_expressions"; + var parserInstance = new SimpleExpressionParserWithContext(); + var builder = new ParserBuilder("en"); + var buildResult = builder.BuildParser(parserInstance, ParserType.EBNF_LL_STACK, startingRule); + + Check.That(buildResult.IsError).IsFalse(); + var parser = buildResult.Result; + Console.Clear(); + var res = parser.ParseWithContext("- a !", new Dictionary {{"a", 3}}); + Check.That(res.IsOk).IsTrue(); + Check.That(res.Result).IsEqualTo(3*2*1); + } } public enum TestGrammarToken diff --git a/src/samples/ParserExample/stack/EvenSimplerStackParser.cs b/src/samples/ParserExample/stack/EvenSimplerStackParser.cs new file mode 100644 index 00000000..17a0f75f --- /dev/null +++ b/src/samples/ParserExample/stack/EvenSimplerStackParser.cs @@ -0,0 +1,15 @@ +using sly.lexer; +using sly.parser.generator; + +namespace ParserExample; + +[ParserRoot("root")] +public class EvenSimplerStackParser +{ + [Production("root : expr")] + public string root(string e) => e; + + [Production("expr : INT INT")] + public string expr(Token i, Token j) => i.Value + "," + j.Value; + +} \ No newline at end of file diff --git a/src/samples/ParserExample/stack/SimpleStackLexer.cs b/src/samples/ParserExample/stack/SimpleStackLexer.cs new file mode 100644 index 00000000..5fca08b9 --- /dev/null +++ b/src/samples/ParserExample/stack/SimpleStackLexer.cs @@ -0,0 +1,10 @@ +using sly.lexer; + +namespace ParserExample; + +public enum SimpleStackLexer +{ + EOS, + [Int] INT, + [Sugar("+")] PLUS +} \ No newline at end of file diff --git a/src/samples/ParserExample/stack/SimpleStackParser.cs b/src/samples/ParserExample/stack/SimpleStackParser.cs new file mode 100644 index 00000000..23be5650 --- /dev/null +++ b/src/samples/ParserExample/stack/SimpleStackParser.cs @@ -0,0 +1,22 @@ +using sly.lexer; +using sly.parser.generator; + +namespace ParserExample; + +[ParserRoot("root")] +public class SimpleStackParser +{ + [Production("root : expr")] + public int root(int e) => e; + + [Production("expr : INT PLUS expr")] + public int expr(Token e1, Token plus, int e2) => e1.IntValue + e2; + + [Production("expr : term")] + // [Production("expr : INT")] + //public int expr2(Token e) => e.IntValue; + public int expr2(int e) => e; + + [Production("term : INT")] + public int term(Token i) => i.IntValue; +} \ No newline at end of file diff --git a/src/samples/ParserExample/stack/SimplerStackLexer.cs b/src/samples/ParserExample/stack/SimplerStackLexer.cs new file mode 100644 index 00000000..34ec4b1a --- /dev/null +++ b/src/samples/ParserExample/stack/SimplerStackLexer.cs @@ -0,0 +1,9 @@ +using sly.lexer; + +namespace ParserExample; + +public enum SimplerStackLexer +{ + EOS, + [Int] INT, +} \ No newline at end of file diff --git a/src/samples/ParserExample/stack/SimplerStackParser.cs b/src/samples/ParserExample/stack/SimplerStackParser.cs new file mode 100644 index 00000000..fe04f0c1 --- /dev/null +++ b/src/samples/ParserExample/stack/SimplerStackParser.cs @@ -0,0 +1,18 @@ +using sly.lexer; +using sly.parser.generator; + +namespace ParserExample; + +[ParserRoot("root")] +public class SimplerStackParser +{ + [Production("root : expr")] + public string root(string e) => e; + + [Production("expr : INT expr")] + public string expr(Token i, string e) => i.Value + "," + e; + + [Production("expr : INT")] + public string expr2(Token i) => i.Value; + +} \ No newline at end of file diff --git a/src/samples/ParserExample/stack/Stacker.cs b/src/samples/ParserExample/stack/Stacker.cs new file mode 100644 index 00000000..ccaf650d --- /dev/null +++ b/src/samples/ParserExample/stack/Stacker.cs @@ -0,0 +1,550 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using expressionparser; +using indented; +using jsonparser; +using jsonparser.JsonModel; +using NFluent; +using ParserTests; +using ParserTests.Issue239; +using ParserTests.stack; +using sly.lexer; +using sly.lexer.fluent; +using sly.parser; +using sly.parser.fluent; +using sly.parser.generator; +using sly.parser.syntax.grammar; +using sly.parser.syntax.tree; +using postProcessedLexerParser.expressionModel; + +namespace ParserExample; + +public enum P +{ + [Sugar("+")] e, + [Keyword("true")] True, + [Keyword("false")] False, +} + +public enum L +{ + [Keyword("A")] A = 1, + [Keyword("B")] B = 2, + [Sugar("+")] PLUS = 3, + [Sugar("-")] MINUS = 4 +} + + +public class SimpleEBNFMany +{ + [Production("root : astar")] + public string Root(string astar) + { + return astar; + } + + [Production("rootplus : aplus")] + public string RootPlus(string aplus) + { + return aplus; + } + + [Production("astar : A*")] + public string Astar(List> all) + { + return string.Join(",",all.Select(x => x.Value)); + } + + [Production("aplus : A+")] + public string Aplus(List> all) + { + return string.Join(",",all.Select(x => x.Value)); + } +} + +public class Visitor { + + [Production("root : a [ PLUS | MINUS ] b")] + public string Root(string a, Token op, string b) + { + return "a <" + op.Value + "> b"; + } + + [Production("a : A")] + public string A(Token a) + { + return a.Value; + } + + [Production("b : B")] + public string B(Token b) + { + return b.Value; + } +} + +public class Stacker +{ + public static void Stack() + { + var instance = new EvenSimplerStackParser(); + ParserBuilder builder = new ParserBuilder(); + var parser = builder.BuildParser(instance, ParserType.LL_STACK, "root"); + if (parser.IsOk) + { + var r = parser.Result.Parse("1 2"); + if (r.IsOk) + { + Console.WriteLine($"PARSE OK !!! >{r.Result}<"); + } + else + { + foreach (var error in r.Errors) + { + Console.WriteLine(error.ErrorMessage); + } + } + } + else + { + foreach (var error in parser.Errors) + { + Console.WriteLine(error.Message); + } + } + } + + public static void MoreStack() + { + var instance = new SimplerStackParser(); + ParserBuilder builder = new ParserBuilder(); + var parser = builder.BuildParser(instance, ParserType.LL_STACK, "root"); + if (parser.IsOk) + { + var r = parser.Result.Parse("1"); + Check.That(r).IsOkParsing(); + Check.That(r.Result).IsEqualTo("1"); + r = parser.Result.Parse("1 2"); + Check.That(r).IsOkParsing(); + Check.That(r.Result).IsEqualTo("1,2"); + r = parser.Result.Parse("1 2 3 4 5"); + Check.That(r).IsOkParsing(); + Check.That(r.Result).IsEqualTo("1,2,3,4,5"); + } + else + { + foreach (var error in parser.Errors) + { + Console.WriteLine(error.Message); + } + } + } + + public static void FluentStack() + { + var lexer = FluentLexerBuilder.NewBuilder() + .Int(ExpressionToken.INT); + var parser = FluentParserBuilder + .NewBuilder(new SimplerStackParser(), "root", "en") + .WithLexerbuilder(lexer) + .Production("root : expr", (object[] args) => { return (string)args[0]; }) + .Production("expr : INT", (args) => { return ((Token)args[0]).Value; }) + .Production("expr : INT expr", + (args) => { return ((Token)args[0]).Value + "," + (string)args[1]; }) + .BuildParser(ParserType.LL_STACK); + Check.That(parser).IsOk(); + var result = parser.Result.Parse("1"); + Check.That(result).IsOkParsing(); + Check.That(result.Result).IsEqualTo("1"); + result = parser.Result.Parse("1 2 3 4 5"); + Check.That(result).IsOkParsing(); + Check.That(result.Result).IsEqualTo("1,2,3,4,5"); + } + + public static void EvenMoreStack() + { + var instance = new SimpleStackParser(); + ParserBuilder builder = new ParserBuilder(); + var parser = builder.BuildParser(instance, ParserType.LL_STACK, "root"); + if (parser.IsOk) + { + var r = parser.Result.Parse("1+2+3+4"); + Check.That(r).IsOkParsing(); + Check.That(r.Result).IsEqualTo(10); + r = parser.Result.Parse("1+2+3+4+5+6+7+8+9+10"); + Check.That(r).IsOkParsing(); + Check.That(r.Result).IsEqualTo(55); + } + else + { + foreach (var error in parser.Errors) + { + Console.WriteLine(error.Message); + } + } + } + + public static void Expression() + { + var instance = new ExpressionParser(); + ParserBuilder builder2 = + new ParserBuilder(); + var parser = builder2.BuildParser(instance, ParserType.LL_STACK, "expression"); + if (parser.IsOk) + { + string source = "2+2"; + ; + Console.WriteLine($"start parsing {source}"); + var r = parser.Result.Parse(source); + Console.WriteLine($"parsing done : {(r.IsOk ? "OK" : "KO")}"); + Check.That(r).IsOkParsing(); + Check.That(r.Result).IsEqualTo(4); + source = "1+2+3+4+5+6+7+8+9*10"; + Console.WriteLine($"start parsing {source}"); + r = parser.Result.Parse(source); + Check.That(r).IsOkParsing(); + Check.That(r.Result).IsEqualTo(126); + Console.WriteLine("parsing done !!! OOH YEAH !! :: "+r.Result); + ; + } + else + { + foreach (var error in parser.Errors) + { + Console.WriteLine(error.Message); + } + } + } + + + public static void ParseVisitorAPlusB() + { + var builder = new ParserBuilder("en"); + var instance = new Visitor(); + + string source = "A + B"; + string start = "root"; + RuleParserType.ParserType = ParserType.LL_RECURSIVE_DESCENT; + + var grammarParser = builder.BuildParser(instance, ParserType.EBNF_LL_RECURSIVE_DESCENT, start); + + Check.That(grammarParser).IsOk(); + var parser = grammarParser.Result; + + var r = parser.Parse(source,start); + Check.That(r).IsOkParsing(); + string expected = r.Result.ToString(); + + RuleParserType.ParserType = ParserType.LL_STACK; + + grammarParser = builder.BuildParser(instance, ParserType.EBNF_LL_RECURSIVE_DESCENT, start); + + Check.That(grammarParser).IsOk(); + parser = grammarParser.Result; + r = parser.Parse(source, start); + Check.That(r).IsOkParsing(); + var actual = r.Result.ToString(); + Check.That(actual).IsEqualTo(expected); + } + + public static void Rules() + { + var ruleparser = new RuleParser(); + string source = "[ PLUS | MINUS ]"; + string start = "choiceclause"; + + // + // LL_RECURSIVE => OK => get expected output + // + + var parser = GetParser>(ruleparser, ParserType.LL_RECURSIVE_DESCENT, "rule"); + + var r = parser.Parse(source, start); + Check.That(r).IsOkParsing(); + + var tree = r.SyntaxTree; + Check.That(r.SyntaxTree).IsInstanceOf>>(); + var root = r.SyntaxTree as SyntaxNode>; + Check.That(root).IsNotNull(); + var expected = (r.Result as IClause).Dump(); + + + // + // LL_STACK => get output and compare to expected (if parse succeeded at all) + // + + + parser = GetParser>(ruleparser, ParserType.LL_STACK, "rule"); + + r = parser.Parse(source, start); + Check.That(r).IsOkParsing(); + tree = r.SyntaxTree; + var actual = (r.Result as IClause).Dump(); + Check.That(actual).IsEqualTo(expected); + } + + public static void List() + { + var lexer = FluentLexerBuilder.NewBuilder() + .Keyword(L.A, "A") + .Keyword(L.B, "B"); + ; + var p = FluentParserBuilder.NewBuilder(new Dumb(), "r", "en") + .Production("r : cs", (args) => { return args[0].ToString(); }) + .Production("cs : c cs", (args) => + { + var head = (string)args[0]; + var tail = (string)args[1]; + + return head + ".." + tail; + }) + .Production("cs : c", (args) => { return (string)args[0]; }) + .Production("c : A", (args) => { return ((Token)args[0]).Value; }) + .Production("c : B", (args) => { return ((Token)args[0]).Value; }) + .WithLexerbuilder(lexer) + .BuildParser(ParserType.LL_STACK); + Check.That(p.IsOk); + var t = p.Result.Parse("A B", "cs"); + Check.That(t).IsOkParsing(); + Check.That(t.Result).IsEqualTo("A..B"); + Console.WriteLine("OK : "+t.Result); + Console.WriteLine(t.SyntaxTree.Dump(" ")); + } + + public static Parser GetParser(object instance, ParserType type, string root) where IN : struct , Enum + { + ParserBuilder builder = new ParserBuilder(); + var built = builder.BuildParser(instance, type, root); + Check.That(built).IsOk(); + Check.That(built.Result).IsNotNull(); + return built.Result; + } + + public static void Test239() + { + string source = "INT[d] ID SEMI[d]"; + string start = "clauses"; + var ruleparser = new RuleParser(); + var parser = GetParser>(ruleparser, ParserType.LL_RECURSIVE_DESCENT, "clauses"); + + var r = parser.Parse(source, start); + Check.That(r).IsOkParsing(); + Check.That(r.Result).IsInstanceOf < ClauseSequence>(); + var rule = r.Result as ClauseSequence; + var expected = rule.Dump(); + + parser = GetParser>(ruleparser, ParserType.LL_STACK, "rule"); + + r = parser.Parse(source, start); + Check.That(r).IsOkParsing(); + Check.That(r.Result).IsInstanceOf < ClauseSequence>(); + rule = r.Result as ClauseSequence; + var actual = rule.Dump(); + Check.That(actual).IsEqualTo(expected); + + + + + } + + public static void TestPostProcessedLexer() + { + string start = "rule"; + string source = "rule : IDENTIFIER LPAREN[d] FormulaParser_expressions (COMMA FormulaParser_expressions)* "; + //string source = "IDENTIFIER LPAREN[d] FormulaParser_expressions (COMMA FormulaParser_expressions)* "; +// string source = "FormulaParser_expressions (COMMA FormulaParser_expressions)* "; + //string source = "(COMMA FormulaParser_expressions)* "; + //string source = "( COMMA FormulaParser_expressions )"; + var ruleparser = new RuleParser(); + + + // + // LL_RECURSIVE => OK => get expected output + // + + var parser = GetParser>(ruleparser, ParserType.LL_RECURSIVE_DESCENT, "rule"); + + var r = parser.Parse(source, start); + Check.That(r).IsOkParsing(); + + var tree = r.SyntaxTree; + Check.That(r.SyntaxTree).IsInstanceOf>>(); + var root = r.SyntaxTree as SyntaxNode>; + Check.That(root).IsNotNull(); + Check.That(r.Result).IsInstanceOf> (); + var rule = r.Result as Rule; + var expected = rule.Dump(); + + + // + // LL_STACK => get output and compare to expected (if parse succeeded at all) + // + + + parser = GetParser>(ruleparser, ParserType.LL_STACK, "rule"); + + r = parser.Parse(source, start); + Check.That(r).IsOkParsing(); + tree = r.SyntaxTree; + Check.That(r.Result).IsInstanceOf> (); + rule = r.Result as Rule; + var actual = rule.Dump(); + Check.That(actual).IsEqualTo(expected); +Console.WriteLine("***************************************"); +Console.WriteLine("*** YAHOO ! WE'VE DONE A GREAT JOB"); +Console.WriteLine($"*** {actual} == {expected}"); +Console.WriteLine("***************************************"); + } + + public static void TestEbnfSimpleZeroOrMore() + { + var ebnf = new SimpleEBNFMany(); + RuleParserType.ParserType = ParserType.LL_RECURSIVE_DESCENT; + var parser = GetParser(ebnf, ParserType.EBNF_LL_STACK, "root"); + + Check.That(parser).IsNotNull(); + + var parseResult = parser.Parse(" A A A A"); + Check.That(parseResult).IsOkParsing(); + var result = parseResult.Result; + Check.That(result).IsEqualTo("A,A,A,A"); + Console.WriteLine("parse A A A A : OK"); + + parseResult = parser.Parse(" A "); + Check.That(parseResult).IsOkParsing(); + result = parseResult.Result; + Check.That(result).IsEqualTo("A"); + Console.WriteLine("parse A : OK"); + + parseResult = parser.Parse(" "); + Check.That(parseResult).IsOkParsing(); + result = parseResult.Result; + Check.That(result).IsEqualTo(""); + Console.WriteLine("parse (empty) : OK"); + } + + public static void TestIndentedNotClosing() + { + var source =@" +if truc == 1 + un = 1 + deux = 2"; + ParserBuilder builder = new ParserBuilder(); + var instance = new IndentedParser(); + var parserRes = builder.BuildParser(instance, ParserType.EBNF_LL_STACK, "root"); + Check.That(parserRes.IsOk).IsTrue(); + + var parser = parserRes.Result; + Check.That(parser).IsNotNull(); + var parseResult = parser.Parse(source); + Check.That(parseResult).Not.IsOkParsing(); + Check.That(parseResult.Errors).CountIs(1); + var error = parseResult.Errors[0]; + Check.That(error.ErrorType).IsEqualTo(ErrorType.UnexpectedEOS); + } + + public static void TestEbnfSimpleOneOrMore() + { + var ebnf = new SimpleEBNFMany(); + RuleParserType.ParserType = ParserType.LL_RECURSIVE_DESCENT; + var parser = GetParser(ebnf, ParserType.EBNF_LL_STACK, "rootplus"); + + Check.That(parser).IsNotNull(); + + var parseResult = parser.Parse(" A A A A"); + Check.That(parseResult).IsOkParsing(); + var result = parseResult.Result; + Check.That(result).IsEqualTo("A,A,A,A"); + + parseResult = parser.Parse(" A "); + Check.That(parseResult).IsOkParsing(); + result = parseResult.Result; + Check.That(result).IsEqualTo("A"); + + parseResult = parser.Parse(" "); + Check.That(parseResult).Not.IsOkParsing(); + Check.That(parseResult.Errors).CountIs(1); + var error = parseResult.Errors[0]; + Check.That(error.ErrorType).IsEqualTo(ErrorType.UnexpectedEOS); + + } + + public static void TestFluentOption() + { + var lexer = FluentLexerBuilder.NewBuilder() + .Keyword(L.A, "A") + .Keyword(L.B, "B"); + + var buildResult = FluentEBNFParserBuilder.NewBuilder(new FluentTests(), "root", "en") + .WithLexerbuilder(lexer) + .Production("root : o", (object[] args) => + { + return (string)args[0]; + }) + .Production("o : A B? A", (args) => + { + var a1 = (Token)args[0]; + var b = (Token)args[1]; + var a2 = (Token)args[0]; + if (b.IsEmpty) + { + return "ah ah !"; + } + else + { + return "ABBA"; + } + }) + .BuildParser(ParserType.EBNF_LL_RECURSIVE_DESCENT); + + Check.That(buildResult).IsOk(); + var parser = buildResult.Result; + var result = parser.Parse("A A"); + Check.That(result).IsOkParsing(); + Check.That(result.Result).IsEqualTo("ah ah !"); + result = parser.Parse("A B A"); + Check.That(result).IsOkParsing(); + Check.That(result.Result).IsEqualTo("ABBA"); + } + + public static void TestIndented() + { + var source =@" +if truc == 1 + un = 1 + deux = 2"; + ParserBuilder builder = new ParserBuilder(); + var instance = new IndentedParser(); + var parserRes = builder.BuildParser(instance, ParserType.EBNF_LL_STACK, "root"); + Check.That(parserRes.IsOk).IsTrue(); + + var parser = parserRes.Result; + Check.That(parser).IsNotNull(); + var parseResult = parser.Parse(source); + Check.That(parseResult).Not.IsOkParsing(); + Check.That(parseResult.Errors).IsSingle(); + var error = parseResult.Errors[0]; + Check.That(error.ErrorType).IsEqualTo(ErrorType.UnexpectedEOS); + + } + + public static void TestIncompleteJsonObject() + { + var jsonParser = new EbnfJsonGenericParser(); + var builder = new ParserBuilder(); + + var build = builder.BuildParser(jsonParser, ParserType.EBNF_LL_STACK, "root"); + Check.That(build).IsOk(); + var parser = build.Result; + var r = parser.Parse("{\"prop\":\"value\",\"prop2\":[1,2,3"); + Check.That(r).Not.IsOkParsing(); + Check.That(r.Errors).CountIs(1); + var error = r.Errors[0]; + Check.That(error.ErrorType).IsEqualTo(ErrorType.UnexpectedEOS); + foreach (var rError in r.Errors) + { + Console.WriteLine(rError.ContextualErrorMessage); + } + } +} \ No newline at end of file diff --git a/src/samples/postProcessedLexerParser/PostProcessedLexerParserBuilder.cs b/src/samples/postProcessedLexerParser/PostProcessedLexerParserBuilder.cs index a5151672..17286d39 100644 --- a/src/samples/postProcessedLexerParser/PostProcessedLexerParserBuilder.cs +++ b/src/samples/postProcessedLexerParser/PostProcessedLexerParserBuilder.cs @@ -10,7 +10,7 @@ namespace postProcessedLexerParser public class PostProcessedLexerParserBuilder { - private static List> postProcessFormula(List> tokens) + public static List> postProcessFormula(List> tokens) { var mayLeft = new List() { @@ -50,6 +50,7 @@ public static Parser buildPostProcessedLexerParser() var builder = new ParserBuilder(); var build = builder.BuildParser(parserInstance, ParserType.EBNF_LL_RECURSIVE_DESCENT, $"{nameof(FormulaParser)}_expressions", lexerPostProcess: postProcessFormula); + if (build.IsError) { foreach (var error in build.Errors) diff --git a/src/sly/buildresult/ErrorCodes.cs b/src/sly/buildresult/ErrorCodes.cs index 26596785..0a8bd720 100644 --- a/src/sly/buildresult/ErrorCodes.cs +++ b/src/sly/buildresult/ErrorCodes.cs @@ -72,6 +72,6 @@ public enum ErrorCodes #endregion - + PARSER_RULE_SYNTAX_ERROR } } \ No newline at end of file diff --git a/src/sly/parser/fluent/FluentEBNFParserBuilder.cs b/src/sly/parser/fluent/FluentEBNFParserBuilder.cs index 9ebda8c7..b8f5d7cb 100644 --- a/src/sly/parser/fluent/FluentEBNFParserBuilder.cs +++ b/src/sly/parser/fluent/FluentEBNFParserBuilder.cs @@ -65,7 +65,7 @@ private FluentEBNFParserBuilder(string i18N, object parserInstance, string rootR _grammarParser = grammar.Result; } - public BuildResult> BuildSyntaxParser(BuildResult> result) + public BuildResult> BuildSyntaxParser(BuildResult> result, ParserType parserType = ParserType.EBNF_LL_RECURSIVE_DESCENT) { // build configuration _configuration = new ParserConfiguration(); @@ -102,7 +102,7 @@ public BuildResult> BuildSyntaxParser(BuildResult>(result.Errors); } - var syntaxParser = b.BuildSyntaxParser(_configuration, ParserType.EBNF_LL_RECURSIVE_DESCENT, _rootRule); + var syntaxParser = b.BuildSyntaxParser(_configuration, parserType, _rootRule); var checkResult =b.CheckParser(_configuration); if (!checkResult.IsOk) { @@ -117,10 +117,10 @@ public BuildResult> BuildSyntaxParser(BuildResult>(syntaxParser); } - public BuildResult> BuildParser() + public BuildResult> BuildParser(ParserType parserType = ParserType.EBNF_LL_RECURSIVE_DESCENT) { var buildResult = new BuildResult>(); - var syntaxParserResult = BuildSyntaxParser(buildResult); + var syntaxParserResult = BuildSyntaxParser(buildResult, parserType); if (buildResult.IsError) { var result = new BuildResult>(); diff --git a/src/sly/parser/fluent/FluentParserBuilder.cs b/src/sly/parser/fluent/FluentParserBuilder.cs index afd00e7d..3871bbea 100644 --- a/src/sly/parser/fluent/FluentParserBuilder.cs +++ b/src/sly/parser/fluent/FluentParserBuilder.cs @@ -45,7 +45,8 @@ private FluentParserBuilder(string i18N, object parserInstance, string rootRule) _rootRule = rootRule; } - public ISyntaxParser BuildSyntaxParser(BuildResult> result) + public ISyntaxParser BuildSyntaxParser(BuildResult> result, + ParserType parserType = ParserType.LL_RECURSIVE_DESCENT) { // build configuration _configuration = new ParserConfiguration(); @@ -59,15 +60,15 @@ public ISyntaxParser BuildSyntaxParser(BuildResult("en"); - var syntaxParser = b.BuildSyntaxParser(_configuration, ParserType.LL_RECURSIVE_DESCENT, _rootRule); + var syntaxParser = b.BuildSyntaxParser(_configuration, parserType, _rootRule); // initialize starting tokens syntaxParser.Init(_configuration,_rootRule); return syntaxParser; } - public BuildResult> BuildParser() + public BuildResult> BuildParser(ParserType parserType = ParserType.LL_RECURSIVE_DESCENT) { - var syntaxParser = BuildSyntaxParser(new BuildResult>()); + var syntaxParser = BuildSyntaxParser(new BuildResult>(), parserType); var tokens = _configuration.GetAllExplicitTokenClauses(); if (tokens != null && tokens.Any()) diff --git a/src/sly/parser/fluent/IFluentEbnfParserBuilder.cs b/src/sly/parser/fluent/IFluentEbnfParserBuilder.cs index f6c9d8aa..e5816383 100644 --- a/src/sly/parser/fluent/IFluentEbnfParserBuilder.cs +++ b/src/sly/parser/fluent/IFluentEbnfParserBuilder.cs @@ -46,9 +46,9 @@ public interface IFluentEbnfParserBuilder where IN : struct, Enum IFluentEbnfRuleBuilder Postfix(string operation, int precedence, Func visitor); - public BuildResult> BuildSyntaxParser(BuildResult> result); + public BuildResult> BuildSyntaxParser(BuildResult> result, ParserType parserType = ParserType.EBNF_LL_RECURSIVE_DESCENT); - public BuildResult> BuildParser(); + public BuildResult> BuildParser(ParserType parserType = ParserType.EBNF_LL_RECURSIVE_DESCENT); public IFluentEbnfParserBuilder WithLexerbuilder(IFluentLexerBuilder lexerBuilder); diff --git a/src/sly/parser/fluent/IFluentParserBuilder.cs b/src/sly/parser/fluent/IFluentParserBuilder.cs index 5d3e50f0..8c79a752 100644 --- a/src/sly/parser/fluent/IFluentParserBuilder.cs +++ b/src/sly/parser/fluent/IFluentParserBuilder.cs @@ -19,9 +19,9 @@ public interface IFluentParserBuilder where IN : struct, Enum IFluentParserBuilder Production(string ruleString, Func visitor); - public ISyntaxParser BuildSyntaxParser(BuildResult> result); + public ISyntaxParser BuildSyntaxParser(BuildResult> result, ParserType parserType = ParserType.LL_RECURSIVE_DESCENT); - public BuildResult> BuildParser(); + public BuildResult> BuildParser(ParserType parserType = ParserType.LL_RECURSIVE_DESCENT); public IFluentParserBuilder WithLexerbuilder(IFluentLexerBuilder lexerBuilder); diff --git a/src/sly/parser/generator/EBNFParserBuilder.cs b/src/sly/parser/generator/EBNFParserBuilder.cs index bb36b75d..a90d452f 100644 --- a/src/sly/parser/generator/EBNFParserBuilder.cs +++ b/src/sly/parser/generator/EBNFParserBuilder.cs @@ -9,6 +9,7 @@ using sly.parser.generator.visitor; using sly.parser.llparser.bnf; using sly.parser.llparser.ebnf; +using sly.parser.parser.llparser.ebnf.stackist; using sly.parser.syntax.grammar; namespace sly.parser.generator @@ -44,7 +45,7 @@ public override BuildResult> BuildParser(object parserInstance, var ruleparser = new RuleParser(); var builder = new ParserBuilder>(I18N); - var grammarParser = builder.BuildParser(ruleparser, ParserType.LL_RECURSIVE_DESCENT, "rule").Result; + var grammarParser = builder.BuildParser(ruleparser, RuleParserType.ParserType, "rule").Result; var result = new BuildResult>(); @@ -54,6 +55,15 @@ public override BuildResult> BuildParser(object parserInstance, try { configuration = ExtractEbnfParserConfiguration(parserInstance.GetType(), grammarParser); + if (configuration.IsError) + { + foreach (var error in configuration.Errors) + { + result.AddError(new ParserInitializationError(ErrorLevel.FATAL, error, + ErrorCodes.PARSER_RULE_SYNTAX_ERROR)); + } + return result; + } configuration.UseMemoization = useMemoization; configuration.BroadenTokenWindow = broadenTokenWindow; configuration.AutoCloseIndentations = autoCloseIndentations; @@ -116,6 +126,11 @@ public override ISyntaxParser BuildSyntaxParser(ParserConfiguration(conf, rootRule, I18N); } + + if (parserType == ParserType.EBNF_LL_STACK) + { + parser = new EBNFStackDescentSyntaxParser(I18N, conf); + } return parser; } @@ -197,7 +212,7 @@ protected virtual ParserConfiguration ExtractEbnfParserConfiguration(Ty .Select(e => e.ErrorMessage) .Aggregate((e1, e2) => e1 + "\n" + e2); message = $"rule error [{ruleString}] : {message}"; - throw new ParserConfigurationException(message); + conf.AddError(message); } } }); diff --git a/src/sly/parser/generator/ExpressionRulesGenerator.cs b/src/sly/parser/generator/ExpressionRulesGenerator.cs index ca4d3d66..0120680e 100644 --- a/src/sly/parser/generator/ExpressionRulesGenerator.cs +++ b/src/sly/parser/generator/ExpressionRulesGenerator.cs @@ -283,7 +283,11 @@ private void GenerateExpressionParser(ParserConfiguration configuration rule.IsByPassRule = true; rule.IsExpressionRule = true; rule.ExpressionAffix = Affix.NotOperator; - rule.SetLambdaVisitor((args) => (OUT)args[0]); + // BUG ? : what if a parse context exists ! + rule.SetLambdaVisitor((args) => + { + return (OUT)args[0]; + }); configuration.NonTerminals[entrypoint.Name] = entrypoint; entrypoint.Rules.Add(rule); } diff --git a/src/sly/parser/generator/ParserBuilder.cs b/src/sly/parser/generator/ParserBuilder.cs index 0523cbe6..98fafec6 100644 --- a/src/sly/parser/generator/ParserBuilder.cs +++ b/src/sly/parser/generator/ParserBuilder.cs @@ -9,7 +9,10 @@ using sly.lexer.fsm; using sly.parser.generator.visitor; using sly.parser.llparser.bnf; +using sly.parser.llparser.bnf.stackist; +using sly.parser.llparser.ebnf; using sly.parser.parser; +using sly.parser.parser.llparser.ebnf.stackist; using sly.parser.syntax.grammar; namespace sly.parser.generator @@ -90,6 +93,55 @@ public virtual BuildResult> BuildParser(object parserInstance, P extensionBuilder, lexerPostProcess); break; } + case ParserType.LL_STACK: + { + var configuration = ExtractParserConfiguration(parserInstance.GetType()); + var (foundRecursion, recursions) = LeftRecursionChecker.CheckLeftRecursion(configuration); + if (foundRecursion) + { + var recs = string.Join("\n", + recursions.Select, string>(x => string.Join(" > ", x))); + result.AddError(new ParserInitializationError(ErrorLevel.FATAL, + i18n.I18N.Instance.GetText(I18N, I18NMessage.LeftRecursion, recs), + ErrorCodes.PARSER_LEFT_RECURSIVE)); + return result; + } + + configuration.StartingRule = rootRule; + var syntaxParser = new StackDescentSyntaxParser("en", configuration); + var visitor = new SyntaxTreeVisitor(configuration, parserInstance); + parser = new Parser(I18N, syntaxParser, visitor); + + parser.Instance = parserInstance; + parser.Configuration = configuration; + result.Result = parser; + break; + } + + case ParserType.EBNF_LL_STACK: + { + // use EBNFParserBuilder to extract configuration + var builder = new EBNFParserBuilder(I18N); + result = builder.BuildParser(parserInstance, ParserType.EBNF_LL_RECURSIVE_DESCENT, rootRule, + extensionBuilder, lexerPostProcess); + if (result.IsError) + { + return result; + } + + var configuration = result.Result.Configuration; + + + configuration.StartingRule = rootRule; + var syntaxParser = new EBNFStackDescentSyntaxParser("en", configuration); + var visitor = new EBNFSyntaxTreeVisitor(configuration, parserInstance); + parser = new Parser(I18N, syntaxParser, visitor); + + parser.Instance = parserInstance; + parser.Configuration = configuration; + result.Result = parser; + break; + } } parser = result.Result; @@ -144,6 +196,23 @@ public virtual ISyntaxParser BuildSyntaxParser(ParserConfiguration(conf, rootRule, I18N); } + else if (parserType == ParserType.LL_STACK) + { + parser = new StackDescentSyntaxParser(I18N, conf); + } + else if (parserType == ParserType.EBNF_LL_STACK) + { + // use EBNFParserBuilder to extract configuration + var builder = new EBNFParserBuilder(I18N); + var result = builder.BuildParser(new {}, ParserType.EBNF_LL_RECURSIVE_DESCENT, rootRule); + + + var configuration = result.Result.Configuration; + + + configuration.StartingRule = rootRule; + parser = new EBNFStackDescentSyntaxParser(I18N, configuration); + } return parser; } diff --git a/src/sly/parser/generator/ParserConfiguration.cs b/src/sly/parser/generator/ParserConfiguration.cs index b3d5b614..77d6c0b9 100644 --- a/src/sly/parser/generator/ParserConfiguration.cs +++ b/src/sly/parser/generator/ParserConfiguration.cs @@ -9,6 +9,10 @@ namespace sly.parser.generator { public class ParserConfiguration where IN : struct, Enum { + public bool IsError { get; set; } = false; + + public List Errors = new List(); + public string StartingRule { get; set; } public Dictionary> NonTerminals { get; set; } @@ -72,6 +76,12 @@ public List> GetRulesForNonTerminal(string nonTerminal) return new List>(); } + public void AddError(string error) + { + IsError = true; + Errors.Add(error); + } + [ExcludeFromCodeCoverage] public string Dump() { diff --git a/src/sly/parser/generator/ParserType.cs b/src/sly/parser/generator/ParserType.cs index 57b28d7a..de10740a 100644 --- a/src/sly/parser/generator/ParserType.cs +++ b/src/sly/parser/generator/ParserType.cs @@ -3,6 +3,8 @@ public enum ParserType { LL_RECURSIVE_DESCENT = 1, - EBNF_LL_RECURSIVE_DESCENT = 2 + EBNF_LL_RECURSIVE_DESCENT = 2, + LL_STACK = 3, + EBNF_LL_STACK = 4, } } \ No newline at end of file diff --git a/src/sly/parser/generator/RuleParser.cs b/src/sly/parser/generator/RuleParser.cs index ab485f4c..2f8af332 100644 --- a/src/sly/parser/generator/RuleParser.cs +++ b/src/sly/parser/generator/RuleParser.cs @@ -4,8 +4,16 @@ namespace sly.parser.generator { - public class RuleParser where IN : struct, Enum + + public static class RuleParserType { + public static ParserType ParserType = ParserType.LL_RECURSIVE_DESCENT; + } + public class RuleParser where IN : struct, Enum + { + + + #region rules grammar [Production("rule : IDENTIFIER COLON clauses")] diff --git a/src/sly/parser/parser/IEnumerableExtensions.cs b/src/sly/parser/parser/IEnumerableExtensions.cs new file mode 100644 index 00000000..7c37280a --- /dev/null +++ b/src/sly/parser/parser/IEnumerableExtensions.cs @@ -0,0 +1,29 @@ +using System; +using System.Collections.Generic; + +namespace sly.lexer.fsm; + +public static class IEnumerableExtensions +{ + public static IEnumerable DistinctWithPredicate(this IEnumerable source, Func predicate) + { + var items = new List(); + foreach (var element in source) + { + bool isDuplicate = false; + foreach (var existing in items) + { + if (predicate(element, existing)) + { + isDuplicate = true; + break; + } + } + if (!isDuplicate) + { + items.Add(element); + yield return element; + } + } + } +} \ No newline at end of file diff --git a/src/sly/parser/parser/Parser.cs b/src/sly/parser/parser/Parser.cs index 608bd89e..de310158 100644 --- a/src/sly/parser/parser/Parser.cs +++ b/src/sly/parser/parser/Parser.cs @@ -112,12 +112,44 @@ public ParseResult ParseWithContext(IList> tokens, object par var result = new ParseResult(); var cleaner = new SyntaxTreeCleaner(); + if (!typeof(IN).Name.Contains("Ebnf")) + { + ; + } var syntaxResult = SyntaxParser.Parse(tokens.ToArray(), startingNonTerminal); + syntaxResult.UsesOperations = Configuration.UsesOperations; syntaxResult = cleaner.CleanSyntaxTree(syntaxResult); if (!syntaxResult.IsError && syntaxResult.Root != null) { - + if (!syntaxResult.IsEnded) + { + result.Errors = new List(); + // TODO something wrong can happen here not ended without error + var unexpectedTokens = syntaxResult.GetErrors(); + var byEnding = unexpectedTokens.GroupBy(x => x.UnexpectedToken.Position).OrderBy(x => x.Key); + var errors = new List(); + foreach (var expecting in byEnding) + { + var expectingTokens = expecting.SelectMany(x => x.ExpectedTokens ?? new List>()).Distinct(); + var expectedTokens = expectingTokens?.ToArray(); + if (expectedTokens != null) + { + var expected = new UnexpectedTokenSyntaxError(expecting.First().UnexpectedToken, LexemeLabels, I18n, + expectedTokens); + errors.Add(expected); + } + else + { + var expected = new UnexpectedTokenSyntaxError(expecting.First().UnexpectedToken, LexemeLabels, I18n, + new LeadingToken[]{}); + errors.Add(expected); + } + } + result.Errors.AddRange(errors); + result.IsError = true; + return result; + } var r = Visitor.VisitSyntaxTree(syntaxResult.Root,parsingContext ?? new NoContext()); result.Result = r; result.SyntaxTree = syntaxResult.Root; diff --git a/src/sly/parser/parser/SyntaxParseResult.cs b/src/sly/parser/parser/SyntaxParseResult.cs index a2a7826e..dd222e25 100644 --- a/src/sly/parser/parser/SyntaxParseResult.cs +++ b/src/sly/parser/parser/SyntaxParseResult.cs @@ -8,6 +8,7 @@ namespace sly.parser { public class SyntaxParseResult where IN : struct, Enum { + private bool _isEnded; public ISyntaxNode Root { get; set; } public bool IsError { get; set; } @@ -20,7 +21,11 @@ public class SyntaxParseResult where IN : struct, Enum public int EndingPosition { get; set; } - public bool IsEnded { get; set; } + public bool IsEnded + { + get => _isEnded; + set => _isEnded = value; + } private void InitErrors() { @@ -32,24 +37,46 @@ private void InitErrors() public void AddErrors(IList> errors) { - InitErrors(); - foreach (var error in errors) + if (errors != null) { - AddError(error); + InitErrors(); + foreach (var error in errors) + { + AddError(error); + } } } public void AddError(UnexpectedTokenSyntaxError error) { InitErrors(); + if (Errors.Any()) + { + int compare = error.CompareTo(Errors.First()); + bool eq = error.Equals(Errors.First()); + } Errors.Add(error); } - public IList> GetErrors() => Errors?.ToList(); + public IList> GetErrors() => + Errors == null ? new List>() : Errors?.ToList(); public List> Expecting {get; set;} public bool HasByPassNodes { get; set; } = false; public bool UsesOperations { get; set; } + + public string Dump() + { + if (IsOk) + { + return "OK : \n" + Root.Dump(" "); + } + else + { + return "KO "+Errors.First().ErrorMessage; + } + + } } } \ No newline at end of file diff --git a/src/sly/parser/parser/UnexpectedTokenSyntaxError.cs b/src/sly/parser/parser/UnexpectedTokenSyntaxError.cs index be3e1110..3f72c672 100644 --- a/src/sly/parser/parser/UnexpectedTokenSyntaxError.cs +++ b/src/sly/parser/parser/UnexpectedTokenSyntaxError.cs @@ -45,6 +45,8 @@ public class UnexpectedTokenSyntaxError : ParseError, IComparable where T : s { private readonly string _i18N; + public string I18n => _i18N; + private readonly Dictionary> _labels = new Dictionary>(); public UnexpectedTokenSyntaxError(Token unexpectedToken, Dictionary> labels, string i18n=null, params LeadingToken[] expectedTokens ) { @@ -68,7 +70,6 @@ public UnexpectedTokenSyntaxError(Token unexpectedToken, string i18n = null, { _i18N = i18n; ErrorType = unexpectedToken.IsEOS ? ErrorType.UnexpectedEOS : ErrorType.UnexpectedToken; - UnexpectedToken = unexpectedToken; if (expectedTokens != null) { @@ -154,11 +155,21 @@ private string GetMessageForExpectedToken(LeadingToken expected) else { var lbl = expected.ToString(); - if (_labels.TryGetValue(expected.TokenId, out var labels) && labels.TryGetValue(_i18N, out var label)) + if (_labels != null && _labels.TryGetValue(expected.TokenId, out var labels) && labels.TryGetValue(_i18N, out var label)) { lbl = label; } + if (expected.IsIndent) + { + lbl = "INDENT"; + } + + if (expected.IsUnindent) + { + lbl = "UINDENT"; + } + message.Append(lbl); } @@ -180,5 +191,13 @@ protected override string GetContextualMessage(string fullSource) var position = UnexpectedToken.Position; return GetContextualMessage(fullSource,position.Line,position.Column, message); } + + // public string Discriminant() + // { + // StringBuilder builder = new StringBuilder(); + // builder.Append(UnexpectedToken.TokenID.ToString()); + // builder.Append(UnexpectedToken.Position.Index); + // return builder.ToString(); + // } } } \ No newline at end of file diff --git a/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.Expressions.cs b/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.Expressions.cs index 9dc2e72d..8bd0d32e 100644 --- a/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.Expressions.cs +++ b/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.Expressions.cs @@ -6,10 +6,10 @@ namespace sly.parser.llparser.bnf; -public partial class RecursiveDescentSyntaxParser where IN : struct, Enum +public class ExpressionRuleManager where IN : struct, Enum { - protected SyntaxNode ManageExpressionRules(Rule rule, SyntaxNode node) + public static SyntaxNode ManageExpressionRules(Rule rule, SyntaxNode node) { var operatorIndex = -1; switch (rule.IsExpressionRule) diff --git a/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.cs b/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.cs index 5c669593..6003a5d8 100644 --- a/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.cs +++ b/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.cs @@ -203,7 +203,7 @@ public virtual SyntaxParseResult Parse(Token[] tokens, Rule(nonTerminalName, children); else node = new SyntaxNode(rule.NodeName ?? nonTerminalName, children); - node = ManageExpressionRules(rule, node); + node = ExpressionRuleManager.ManageExpressionRules(rule, node); if (node.IsByPassNode) // inutile de créer un niveau supplémentaire result.Root = children[0]; result.Root = node; diff --git a/src/sly/parser/parser/llparser/bnf/stackist/ErrorAggregator.cs b/src/sly/parser/parser/llparser/bnf/stackist/ErrorAggregator.cs new file mode 100644 index 00000000..22f48063 --- /dev/null +++ b/src/sly/parser/parser/llparser/bnf/stackist/ErrorAggregator.cs @@ -0,0 +1,23 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace sly.parser.llparser.bnf.stackist; + +public class ErrorAggregator +{ + public static List> Aggregate(List> errors) where IN : struct, Enum + { + List> errorList = new(); + var groups = errors.GroupBy(x => x.UnexpectedToken.ToString()); + foreach (IGrouping> g in groups) + { + var first = g.First(); + var expected = g.SelectMany(x => x.ExpectedTokens).Distinct().ToList(); + var error = new UnexpectedTokenSyntaxError(first.UnexpectedToken, first.I18n, expected); + errorList.Add(error); + } + + return errorList; + } +} \ No newline at end of file diff --git a/src/sly/parser/parser/llparser/bnf/stackist/StackDescentSyntaxParser.cs b/src/sly/parser/parser/llparser/bnf/stackist/StackDescentSyntaxParser.cs new file mode 100644 index 00000000..2261e125 --- /dev/null +++ b/src/sly/parser/parser/llparser/bnf/stackist/StackDescentSyntaxParser.cs @@ -0,0 +1,478 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using sly.lexer; +using sly.lexer.fsm; +using sly.parser.generator; +using sly.parser.syntax.grammar; +using sly.parser.syntax.tree; + + +namespace sly.parser.llparser.bnf.stackist; + +public enum StackStateType +{ + Terminal, + NonTerminal, + Rule, + Root, + Extension, + Result +} + +public partial class StackDescentSyntaxParser : ISyntaxParser where IN : struct, Enum +{ + + private const bool DEBUG = false; + public Dictionary> LexemeLabels { get; set; } + + public ParserConfiguration Configuration { get; set; } + + public string StartingNonTerminal { get; set; } + + public string I18n { get; set; } + + + public StackDescentSyntaxParser() + { + + } + public StackDescentSyntaxParser(string i18n, + ParserConfiguration configuration) + { + I18n = i18n; + Init(configuration, configuration.StartingRule); + } + + public virtual void Init(ParserConfiguration configuration, string root) + { + Configuration = configuration; + RecursiveDescentSyntaxParser recursive = + new RecursiveDescentSyntaxParser(configuration, configuration.StartingRule, I18n); + recursive.Init(configuration, configuration.StartingRule); + } + + public string Dump() => Configuration.Dump(); + + + public virtual void ParseExtension(StackState state, Stack> stack) + { + + } + + public virtual bool IsExtension(GrammarNode clause) + { + return false; + } + + public SyntaxParseResult Parse(Token[] tokens, string startingNonTerminal = null) + { + + + var stack = new Stack>(); + var root = new RootStackState(); + var start = startingNonTerminal ?? Configuration.StartingRule; + if (string.IsNullOrEmpty(start)) + { + throw new Exception("No starting rule defined"); + } + + NonTerminalClause startNode = new NonTerminalClause(start); + StackState state = new NonTerminalStackState(root, startNode) + { + Position = 0, + Tokens = tokens + }; + var toks = string.Join(" ", tokens.Select(x => x.Value)); + //Log($"start :: {toks}",stack); + + stack.Push(root); + stack.Push(state); + + var current = stack.Pop(); + while (current != null) + { + + + switch (current.Type) + { + case StackStateType.Rule: + { + //Log(ruleState.Progress(), stack); + ParseRule(current as RuleStackState, stack); + break; + } + case StackStateType.NonTerminal: + { + ////Log(nonTerminalState.Progress(Configuration), stack); + ParseNonTerminal(current as NonTerminalStackState, stack); + break; + } + case StackStateType.Terminal: + // //Log(current.DebugString, stack); + ParseTerminal(current as TerminalStackState, stack); + break; + case StackStateType.Root: + { + return current.Result; + } + default: + { + ParseExtension(current, stack); + break; + } + } + current = stack.Pop(); + } + + return null; + } + + private void ParseNonTerminal(NonTerminalStackState state, Stack> stack) + { + NonTerminalClause nonTerminal = state.NonTerminal; + if (Configuration.NonTerminals.TryGetValue(nonTerminal.NonTerminalName, out var nonTerminalClause)) + { + if (state.Index >= nonTerminalClause.Rules.Count && state.Result != null) + { + var realResult = state.Result; + // success may have happened previously ! + if (realResult.IsError && state.Successes.Any(x => x.IsOk)) + { + realResult = state.Successes.OrderBy(x => x.EndingPosition).Last(); + } + else if (!state.Successes.Any(x => x.IsOk) && state.Errors.Any()) + { + // only errors : return error result merging errors + var max = state.Errors.Max(x => x.EndingPosition); + realResult = new SyntaxParseResult() + { + IsError = true, + EndingPosition = max, + }; + + var errors = state.Errors.Where(x => x.EndingPosition == max) + .SelectMany(x => x.GetErrors()).ToList(); + errors = ErrorAggregator.Aggregate(errors); + realResult.AddErrors(errors); + } + else if (state.Successes.Count(x => x.IsOk) > 1) + { + realResult = state.Successes.OrderBy(x => x.EndingPosition).Last(); + } + + state.Parent.SetResult(realResult); + + return; + } + + if (state.Index >= nonTerminalClause.Rules.Count && state.Result == null) + { + var result = new SyntaxParseResult(); + result.IsError = true; + result.EndingPosition = state.Position; + var expected = nonTerminalClause.Rules.SelectMany(x => x.PossibleLeadingTokens).Distinct().ToArray(); + + if (state.Successes.Any()) + { + // //Log($"{state.NonTerminal.NonTerminalName}<<{state.Index}>> : no more alternative but at least one match has been found ",stack,1); + var last = state.Successes.OrderBy(x => x.EndingPosition).Last(); + state.Parent.SetResult(result); + + + return; + } + + if (state.Position >= state.Tokens.Length) + { + var error = new UnexpectedTokenSyntaxError(state.Tokens.Last(), LexemeLabels, I18n, + expected); + result.AddError(error); + } + else + { + var error = new UnexpectedTokenSyntaxError(state.Tokens[state.Position], LexemeLabels, I18n, + expected); + result.AddError(error); + } + + state.Parent.SetResult(result); + return; + } + else if (!state.Successes.Any(x => x.IsOk) && state.Errors.Any()) + { + // only errors : return error result merging errors + var max = state.Errors.Max(x => x.EndingPosition); + var realResult = new SyntaxParseResult() + { + IsError = true, + EndingPosition = max, + }; + + var errors = state.Errors.Where(x => x.EndingPosition == max) + .SelectMany(x => x.GetErrors()) + .DistinctWithPredicate(( x, y) => x.Line == y.Line && x.Column == y.Column).ToList(); + realResult.AddErrors(errors); + } + if (state.Index > 0 && state.Result != null && state.Result.IsOk) + { + // here : last rule returned OK => returning right now + state.Parent.SetResult(state.Result); + bool hasParseEnded = state.Result.EndingPosition >= state.Tokens.Length-1; + // other rules may beter match if parse has not ended + if (hasParseEnded) + { + return; + } + } + + var rules = nonTerminalClause.Rules; + if (state.Index >= rules.Count) + { + state.Parent.SetResult(state.Result); + + return; + } + + // first stack self shifting + var rule = rules[state.Index]; + state.Index++; + stack.Push(state); + + + if (state.Position >= state.Tokens.Length) + { + return; + } + + if (rule.Match(state.Tokens, state.Position, Configuration)) + { + PushClause(rule,stack,state); + } + else + { + var result = new SyntaxParseResult(); + var token = state.Tokens[state.Position]; + + result.IsError = true; + var expected = rule.PossibleLeadingTokens; + result.EndingPosition = state.Position; + + result.AddError(new UnexpectedTokenSyntaxError(token, LexemeLabels, I18n, expected.ToArray())); + state.Parent.SetResult(result); + } + } + else + { + throw new Exception($"ERRRRRROR >{state.NonTerminal.NonTerminalName}< not found"); + } + } + + + public void ParseTerminal(TerminalStackState state, Stack> stack) + { + var terminalState = state as TerminalStackState; + TerminalClause terminal = terminalState.Terminal; + if (terminalState.Position >= state.Tokens.Length) + { + //Log($"end of stream found expected {terminal.ExpectedToken}", stack, 2); + var resultEos = new SyntaxParseResult(); + var eosToken = terminalState.Tokens[terminalState.Position-1]; // get EOS token + resultEos.AddError(new UnexpectedTokenSyntaxError(eosToken, LexemeLabels, I18n, terminal.ExpectedToken)); + resultEos.IsError = true; + resultEos.EndingPosition = terminalState.Position; + resultEos.Expecting = new List>() { terminal.ExpectedToken }; + state.Parent.SetResult(resultEos); + return; + } + + var result = new SyntaxParseResult(); + var token = terminalState.Tokens[terminalState.Position]; + var isError = !terminal.Check(token); + result.IsError = isError; + result.EndingPosition = !result.IsError ? terminalState.Position + 1 : terminalState.Position; + if (isError) + { + //Log($"error found {token} expected {terminal.ExpectedToken}",stack,1); + result.AddError(new UnexpectedTokenSyntaxError(token, LexemeLabels, I18n, terminal.ExpectedToken)); + } + + token.Discarded = terminal.Discarded; + token.IsExplicit = terminal.IsExplicitToken; + result.Root = new SyntaxLeaf(token, terminal.Discarded); + result.HasByPassNodes = false; + if (result.IsError) + { + result.AddError(new UnexpectedTokenSyntaxError(token, LexemeLabels, I18n, terminal.ExpectedToken)); + } + + + state.Parent.SetResult(result); + } + + private void ParseRule(RuleStackState state, Stack> stack) + { + var rule = state.Rule; + + if (state.Index > 0 && state.IsEnded) + { + + + if (state.Parent is NonTerminalStackState parentState) + { + var result = new SyntaxParseResult(); + + if (state.LastResult.IsError) + { + var allErrors = state.Children + .Where(x => x != null) + .SelectMany(x => x.GetErrors()) + .Distinct().ToList(); + + var max = allErrors.Max(x => x.UnexpectedToken.Position.Index); + var realResult = new SyntaxParseResult() + { + IsError = true, + EndingPosition = max, + }; + + // if (allErrors.Count == 3) + // { + // var grouped = allErrors.GroupBy(x => x.Discriminant() + // ).ToList(); + // var expecting = grouped + // .Select(x => x + // .SelectMany(y => y.ExpectedTokens) + // .DistinctWithPredicate((w,z) => z.TokenId.Equals(w.TokenId))) + // ; + // } + // TODO aggregate errors + var errors = allErrors.Where(x => x.UnexpectedToken.Position.Index == max) + .ToList(); + realResult.AddErrors(errors); + + parentState.SetResult(realResult); + } + else + { + string name = ""; + if (!string.IsNullOrEmpty(state.Rule.NodeName)) + { + name = state.Rule.NodeName; + } + else + { + name = state.Rule.NonTerminalName; + } + + SyntaxNode node = null; + if (rule.IsSubRule) + { + node = new GroupSyntaxNode(rule.NodeName,state.Children.Select(x => x.Root).ToList()); + } + else + { + node = new SyntaxNode(name, + state.Children.Select(x => x.Root).ToList()); + } + + node.IsByPassNode = rule.IsByPassRule; + node.Visitor = state.Rule.GetVisitorMethod(); + node.LambdaVisitor = state.Rule.getLambdaVisitor(null); + node.Visitor = state.Rule.GetVisitorMethod(); + result.IsEnded = state.Tokens[state.LastResult.EndingPosition].IsEOS; + result.AddErrors(state.Result.GetErrors()); + + node = ExpressionRuleManager.ManageExpressionRules(state.Rule, node); + result.Root = node; + // send new position upward + result.EndingPosition = state.Children.Last().EndingPosition; + parentState.SetResult(result); + } + } + else + { + throw new Exception( + $"HOOPS something very bad happened here ! terminal's parent should not be a {state.Parent.GetType().Name} : {state.Parent}"); + } + + return; + } + + if (rule != null) + { + // new position is ending position of the last result + int newPosition = state.Index > 0 ? state.LastResult.EndingPosition : state.Position; + + + state.Position = newPosition; + + + var clause = rule.Clauses[state.Index]; + // first self stack with index shift + state.Index++; + state.Position = newPosition; + + stack.Push(state); + + // then push the clause + + + PushClause(clause, stack, state); + } + } + + + public void PushClause(GrammarNode clause, Stack> stack, StackState parent) + { + if (IsExtension(clause)) + { + PushClauseExtension(clause, stack, parent); + return; + } + switch (clause) + { + case Rule rule: + { + var ruleState = new RuleStackState(parent, rule) + { + Tokens = parent.Tokens, + Position = parent.Position + }; + stack.Push(ruleState); + break; + } + case TerminalClause terminalClause: + { + var terminalState = new TerminalStackState(parent, terminalClause) + { + Tokens = parent.Tokens, + Position = parent.Position + }; + stack.Push(terminalState); + break; + } + case NonTerminalClause nonTerminalClause: + { + var nonTerminalState = new NonTerminalStackState(parent, nonTerminalClause) + { + Tokens = parent.Tokens, + Position = parent.Position + }; + stack.Push(nonTerminalState); + break; + } + default: + { + PushClauseExtension(clause, stack, parent); + break; + } + } + } + + public virtual void PushClauseExtension(GrammarNode clause, Stack> stack, StackState parent) + { + + } + +} \ No newline at end of file diff --git a/src/sly/parser/parser/llparser/bnf/stackist/state/NonTerminalStackState.cs b/src/sly/parser/parser/llparser/bnf/stackist/state/NonTerminalStackState.cs new file mode 100644 index 00000000..97b9c1f4 --- /dev/null +++ b/src/sly/parser/parser/llparser/bnf/stackist/state/NonTerminalStackState.cs @@ -0,0 +1,63 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Text; +using sly.parser.generator; +using sly.parser.syntax.grammar; + +namespace sly.parser.llparser.bnf.stackist; + + +[DebuggerDisplay("{DebugString}")] +public class NonTerminalStackState : StackState where IN : struct, Enum +{ + public NonTerminalClause NonTerminal { get; set; } + + public int Index { get; set; } + + public List> Successes { get; set; } + + public List> Errors { get; set; } + + public override StackStateType Type => StackStateType.NonTerminal; + + public bool IsError => Result != null && Result.IsError; + + public NonTerminalStackState(StackState parent, NonTerminalClause nonTerminal, + StackState sibling = null) : base(parent) + { + NonTerminal = nonTerminal; + Index = 0; + Successes = new List>(); + Errors = new List>(); + } + + public void AddSuccess(SyntaxParseResult success) + { + Successes.Add(success); + } + + public void AddError(SyntaxParseResult success) + { + Errors.Add(success); + } + + public void SetResult(SyntaxParseResult result) + { + Result = result; + if (result.IsOk) + { + AddSuccess(result); + } + else + { + AddError(result); + } + } + + public override string ToString() + { + return "non Terminal: " + NonTerminal.NonTerminalName; + } +} \ No newline at end of file diff --git a/src/sly/parser/parser/llparser/bnf/stackist/state/ResultState.cs b/src/sly/parser/parser/llparser/bnf/stackist/state/ResultState.cs new file mode 100644 index 00000000..ffd45694 --- /dev/null +++ b/src/sly/parser/parser/llparser/bnf/stackist/state/ResultState.cs @@ -0,0 +1,14 @@ +using System; + +namespace sly.parser.llparser.bnf.stackist; + +public class ResultState : StackState where IN : struct, Enum +{ + public override StackStateType Type => StackStateType.Result; + public ResultState(StackState parent, SyntaxParseResult result) : base(parent) + { + Result = result; + + } + +} \ No newline at end of file diff --git a/src/sly/parser/parser/llparser/bnf/stackist/state/RootStackState.cs b/src/sly/parser/parser/llparser/bnf/stackist/state/RootStackState.cs new file mode 100644 index 00000000..5f2e7218 --- /dev/null +++ b/src/sly/parser/parser/llparser/bnf/stackist/state/RootStackState.cs @@ -0,0 +1,23 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; + +namespace sly.parser.llparser.bnf.stackist; + +[DebuggerDisplay("root")] +public class RootStackState : StackState where IN : struct, Enum +{ + public override StackStateType Type => StackStateType.Root; + + public RootStackState() : base() + { + + } + + public void SetResult(SyntaxParseResult result) + { + Result = result; + } + + +} \ No newline at end of file diff --git a/src/sly/parser/parser/llparser/bnf/stackist/state/RuleStackState.cs b/src/sly/parser/parser/llparser/bnf/stackist/state/RuleStackState.cs new file mode 100644 index 00000000..14a455c5 --- /dev/null +++ b/src/sly/parser/parser/llparser/bnf/stackist/state/RuleStackState.cs @@ -0,0 +1,109 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Text; +using sly.lexer; +using sly.parser.syntax.grammar; + +namespace sly.parser.llparser.bnf.stackist; + +public static class RuleExt +{ + public static string Progress(this RuleStackState state) where IN : struct, Enum + { + StringBuilder b = new StringBuilder(); + b.Append("<<").Append(state.Id).Append($">>[{state.Index}] @").Append(state.Position).Append(" :: "); + b.Append(state.Rule.NonTerminalName).Append(" : "); + for (int i = 0; i < state.Rule.Clauses.Count; i++) + { + if (i == state.Index) + { + b.Append(">"); + } + + b.Append(state.Rule.Clauses[i].Dump()); + b.Append(" "); + } + + if (state.Index >= state.Rule.Clauses.Count) + { + var last = state.Children.FirstOrDefault(x => x != null); + + var isError = last != null && last.IsError; + b.Append(" DONE :").Append(isError ? " KO" : " OK"); + + } + return b.ToString(); + } +} + + +[DebuggerDisplay("{DebugString}")] +public class RuleStackState : StackState where IN : struct, Enum +{ + + private static int Counter = 0; + + public int Id { get; set; } + public override string DebugString => $"Rule <<{Id}>> {Rule.RuleString} [{Index}] @{Position}"; + + public int Index { get; set; } + + public override StackStateType Type => StackStateType.Rule; + public Rule Rule { get; set; } + + public bool IsEnded => Index >= Rule.Clauses.Count || LastResult == null || LastResult.IsError; + + public SyntaxParseResult LastResult => Children.Last(x => x != null); + + public RuleStackState(StackState parent, Rule rule) : base(parent) + { + Rule = rule; + Index = 0; + Id = Counter++; + Children = new List>(rule.Clauses.Count); + for (int i = 0; i < rule.Clauses.Count; i++) + { + Children.Add(null); + } + } + + public List> Children { get; set; } = new (); + + public override void SetResult(SyntaxParseResult result) + { + base.SetResult(result); + AddChild(result); + } + + public void AddChild(SyntaxParseResult result) + { + if (Children.Any(x => x == null)) + { + ; + } + if (result == null) + { + return; + } + + try + { + Children[Index-1] = result; + } + catch (Exception e) + { + ; + } + + //Children.Add(result); + } + + + + public override string ToString() + { + return "Rule: " + Rule.RuleString; + } +} \ No newline at end of file diff --git a/src/sly/parser/parser/llparser/bnf/stackist/state/StackState.cs b/src/sly/parser/parser/llparser/bnf/stackist/state/StackState.cs new file mode 100644 index 00000000..9361f60f --- /dev/null +++ b/src/sly/parser/parser/llparser/bnf/stackist/state/StackState.cs @@ -0,0 +1,57 @@ +using System; +using sly.lexer; +using sly.parser.syntax.grammar; + +namespace sly.parser.llparser.bnf.stackist; + +public class StackState where IN : struct, Enum +{ + + public virtual string DebugString + { + get + { + return "StackState : " + Type.ToString() + " : " + Position + " : " + (Parent != null ? Parent.Position.ToString() : "null"); + } + } + public int Position { get; set; } + + public Token CurrentToken => Tokens[Position]; + + public virtual StackStateType Type { get; } + + public Token[] Tokens { get; set; } + + public StackState Parent { get; set; } + + public SyntaxParseResult Result { get; protected set; } = null; + + public StackState(StackState parent) + { + Parent = parent; + } + + public StackState(StackState parent, TerminalClause terminal) + { + Parent = parent; + Type = StackStateType.Terminal; + } + + + public StackState() + { + Parent = null; + Type = StackStateType.Root; + } + + public virtual void SetResult(SyntaxParseResult result) + { + if (result == null) + { + ; + } + Result = result; + } + + +} \ No newline at end of file diff --git a/src/sly/parser/parser/llparser/bnf/stackist/state/TerminalStackState.cs b/src/sly/parser/parser/llparser/bnf/stackist/state/TerminalStackState.cs new file mode 100644 index 00000000..568f1155 --- /dev/null +++ b/src/sly/parser/parser/llparser/bnf/stackist/state/TerminalStackState.cs @@ -0,0 +1,27 @@ +using System; +using System.Diagnostics; +using sly.parser.syntax.grammar; + +namespace sly.parser.llparser.bnf.stackist; + +[DebuggerDisplay("{DebugString}")] +public class TerminalStackState : StackState where IN : struct, Enum +{ + public TerminalClause Terminal { get; set; } + + public override string DebugString => $"Terminal {Terminal.ExpectedToken} @{Position}"; + + public StackState Sibling { get; set; } + + public override StackStateType Type => StackStateType.Terminal; + public TerminalStackState(StackState parent, TerminalClause terminal) : base(parent) + { + Terminal = terminal; + + } + + public override string ToString() + { + return "Terminal: " + Terminal.ExpectedToken.ToString(); + } +} \ No newline at end of file diff --git a/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.Expressions.cs b/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.Expressions.cs index 7eed7962..975a143f 100644 --- a/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.Expressions.cs +++ b/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.Expressions.cs @@ -2,6 +2,7 @@ using sly.lexer; using sly.parser; using sly.parser.generator; +using sly.parser.llparser.bnf; using sly.parser.syntax.grammar; using sly.parser.syntax.tree; @@ -94,7 +95,7 @@ public virtual SyntaxParseResult ParseInfixExpressionRule(Token[] t currentPosition = thirdResult.EndingPosition; var finalNode = new SyntaxNode(rule.NodeName ?? nonTerminalName, children); finalNode.ExpressionAffix = rule.ExpressionAffix; - finalNode = ManageExpressionRules(rule, finalNode); + finalNode = ExpressionRuleManager.ManageExpressionRules(rule, finalNode); var finalResult = new SyntaxParseResult(); finalResult.Root = finalNode; finalResult.IsEnded = currentPosition >= tokens.Length - 1 @@ -116,7 +117,7 @@ public virtual SyntaxParseResult ParseInfixExpressionRule(Token[] t node = new GroupSyntaxNode(nonTerminalName, children); else node = new SyntaxNode(nonTerminalName, children); - node = ManageExpressionRules(rule, node); + node = ExpressionRuleManager.ManageExpressionRules(rule, node); if (node.IsByPassNode) // inutile de créer un niveau supplémentaire result.Root = children[0]; result.Root = node; diff --git a/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.cs b/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.cs index 17789a62..2d4407a3 100644 --- a/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.cs +++ b/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.cs @@ -145,7 +145,7 @@ public override SyntaxParseResult Parse(Token[] tokens, Rule(nonTerminalName, children); - node = ManageExpressionRules(rule, node); + node = ExpressionRuleManager.ManageExpressionRules(rule, node); result.Root = node; result.IsEnded = currentPosition >= tokens.Length - 1 || currentPosition == tokens.Length - 2 && @@ -169,7 +169,7 @@ public override SyntaxParseResult Parse(Token[] tokens, Rule.ManageExpressionRules(rule, node); result.Root = node; result.IsEnded = tokens[result.EndingPosition].IsEOS || node.IsEpsilon && tokens[result.EndingPosition+1].IsEOS; diff --git a/src/sly/parser/parser/llparser/ebnf/stackist/EBNFStackDescentSyntaxParser.cs b/src/sly/parser/parser/llparser/ebnf/stackist/EBNFStackDescentSyntaxParser.cs new file mode 100644 index 00000000..beee42de --- /dev/null +++ b/src/sly/parser/parser/llparser/ebnf/stackist/EBNFStackDescentSyntaxParser.cs @@ -0,0 +1,483 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using sly.lexer; +using sly.lexer.fsm; +using sly.parser.generator; +using sly.parser.llparser.bnf; +using sly.parser.llparser.bnf.stackist; +using sly.parser.llparser.ebnf; +using sly.parser.parser.llparser.ebnf.stackist.state; +using sly.parser.syntax.grammar; +using sly.parser.syntax.tree; + +namespace sly.parser.parser.llparser.ebnf.stackist; + +public enum EbnfStackStateType +{ + Infix, + Prefix, + Postfix, + OneOrMore, + ZeroOrMore, + Choice, + Option, + None, +} + +public class EBNFStackDescentSyntaxParser : StackDescentSyntaxParser where IN : struct, Enum +{ + public EBNFStackDescentSyntaxParser(string i18n, ParserConfiguration configuration) + { + I18n = i18n; + Configuration = configuration; + } + + public override void Init(ParserConfiguration configuration, string root) + { + Configuration = configuration; + EBNFRecursiveDescentSyntaxParser recursive = + new EBNFRecursiveDescentSyntaxParser(configuration, configuration.StartingRule, I18n); + recursive.Init(configuration, configuration.StartingRule); + } + + public override void ParseExtension(StackState state, Stack> stack) + { + var ebnfState = state as EbnfStackState; + + switch (ebnfState.EbnfStackType) + { + case EbnfStackStateType.Infix: + { + ParseInfixExpressionRule(ebnfState as InfixExpressionStackState, stack); + break; + } + case EbnfStackStateType.ZeroOrMore: + { + ParseZeroOrMore(ebnfState as ZeroOrMoreStackState, stack); + break; + } + case EbnfStackStateType.OneOrMore: + { + ParseOneOrMore(ebnfState as OneOrMoreStackState, stack); + break; + } + case EbnfStackStateType.Option: + { + ParseOption(ebnfState as OptionStackState, stack); + break; + } + case EbnfStackStateType.Choice: + { + ParseChoice(ebnfState as ChoiceStackState, stack); + break; + } + default : + { + ; + break; + } + } + } + + private void ParseOption(OptionStackState state, Stack> stack) + { + if (state.Result == null) + { + // push itself + stack.Push(state); + + PushClause(state.OptionalClause, stack, state); + } + else + { + var innerResult = state.Result; + var result = new SyntaxParseResult(); + result.IsError = false; + result.EndingPosition = innerResult.EndingPosition; + if (state.Result.IsOk) + { + var node = new OptionSyntaxNode($"{state.OptionalClause.ToString()}?", + new List>() { innerResult.Root }, null); + result.Root = node; + node.IsGroupOption = state.OptionalClause is NonTerminalClause nt && nt.IsGroup; + + + var children = new List> { innerResult.Root }; + result.Root = + new OptionSyntaxNode("nevermind", children, null); + result.EndingPosition = innerResult.EndingPosition; + result.HasByPassNodes = innerResult.HasByPassNodes; + state.Parent.SetResult(result); + } + else if (state.Result.IsError) + { + if (state.OptionalClause is TerminalClause) + { + result.Root = new SyntaxLeaf(Token.Empty(), false); + state.Parent.SetResult(result); + } + else if (state.OptionalClause is NonTerminalClause nonTerminalClause) + { + result = new SyntaxParseResult(); + result.AddErrors(innerResult.GetErrors()); + //result.IsError = true; + var children = new List> { innerResult.Root }; + if (innerResult.IsError) children.Clear(); + var node = new OptionSyntaxNode(nonTerminalClause.NonTerminalName, children, + null); + node.IsGroupOption = nonTerminalClause.IsGroup; + result.Root = node; + result.EndingPosition = state.Position; + state.Parent.SetResult(result); + ; + } + else if (state.OptionalClause is ChoiceClause choiceClause) + { + if (choiceClause.IsTerminalChoice) + { + result.Root = new SyntaxLeaf(Token.Empty(), false); + state.Parent.SetResult(result); + } + else if (choiceClause.IsNonTerminalChoice) + { + result = new SyntaxParseResult(); + result.AddErrors(innerResult.GetErrors()); + //result.IsError = true; + var children = new List> { innerResult.Root }; + if (innerResult.IsError) children.Clear(); + var node = new OptionSyntaxNode("", children, + null); + node.IsGroupOption = false; + result.Root = node; + result.EndingPosition = state.Position; + state.Parent.SetResult(result); + } + } + } + } + } + + private void ParseChoice(ChoiceStackState state, Stack> stack) + { + // match has been found return immediatly + if (state.Result != null && state.Result.IsOk) + { + state.Parent.SetResult(state.Result); + state.Index = 0; + return; + } + + // no more choice => failing + if (state.Index >= state.Choice.Choices.Count) + { + SyntaxParseResult result = new SyntaxParseResult(); + result.IsError = true; + result.EndingPosition = state.Result.EndingPosition; + + var max = state.Children.Max(x => x.EndingPosition); + + var errors = state.Children.Where(x => x.EndingPosition == max) + .SelectMany(x => x.GetErrors()) + .DistinctWithPredicate(( x, y) => x.Line == y.Line && x.Column == y.Column).ToList(); + + result.AddErrors(errors); + state.Parent.SetResult(result); + return; + } + + + state.Index++; + stack.Push(state); + if (state.Index - 1 < 0 || state.Index - 1 >= state.Choice.Choices.Count) + { + ; + } + + var next = state.Choice.Choices[state.Index - 1]; + if (next is TerminalClause terminalClause) + { + terminalClause.Discarded = state.Choice.IsDiscarded; + } + + PushClause(next, stack, state); + } + + + private void ParseZeroOrMore(ZeroOrMoreStackState state, Stack> stack) + { + // either first time we evaluate sub clause or previous evaluation is ok + if (state.Result == null || state.IsOk) + { + state.Position = state.Children.Count > 0 ? state.Children.Last().EndingPosition : state.Position; + // push self + state.Index++; + stack.Push(state); + + // keep on trying + PushClause(state.RepeatedClause, stack, state); + } + else + { + var result = new SyntaxParseResult(); + result.AddErrors(state.Result.GetErrors()); + result.IsError = false; // zero or more is always ok , either empty or with many values + var manyNode = new ManySyntaxNode($"{state.RepeatedClause.ToString()}*"); + + manyNode.IsManyGroups = state.RepeatedClause is NonTerminalClause nt && nt.IsGroup; + if (!manyNode.IsManyGroups) + { + manyNode.IsManyTokens = state.RepeatedClause is TerminalClause || + state.RepeatedClause is ChoiceClause tc && tc.IsTerminalChoice; + manyNode.IsManyValues = state.RepeatedClause is NonTerminalClause || + state.RepeatedClause is ChoiceClause ntc && ntc.IsNonTerminalChoice; + } + + + if (state.Children.Any()) + { + foreach (var child in state.Children) + { + if (child.Root != null) + { + result.EndingPosition = Math.Max(result.EndingPosition, child.EndingPosition); + manyNode.Add(child.Root); + } + } + } + else if (state.Children.Count == 0) + { + // If no children then there is no move : ending position does is starting position + result.EndingPosition = state.Position; + } + else if (result.GetErrors().Any(x => x.ErrorType == ErrorType.UnexpectedEOS)) + { + // eos reached , rewind to start ? + result.EndingPosition = state.Position; + } + + + result.Root = manyNode; + state.Parent.SetResult(result); + } + } + + private void ParseOneOrMore(OneOrMoreStackState state, Stack> stack) + { + // either first time we evaluate sub clause or previous evaluation is ok + if (state.Result == null || state.IsOk) + { + state.Position = state.Children.Count > 0 ? state.Children.Last().EndingPosition : state.Position; + state.Index++; + // push self + stack.Push(state); + + // keep on trying + PushClause(state.RepeatedClause, stack, state); + return; + } + else + { + if (state.Children.Count == 0) + { + // error expecting at least one ... + state.Parent.SetResult(state.Result); + return; + } + + var result = new SyntaxParseResult(); + result.AddErrors(state.Result.GetErrors()); + result.IsError = true; + var manyNode = new ManySyntaxNode($"{state.RepeatedClause.ToString()}*"); + manyNode.IsManyTokens = state.RepeatedClause is TerminalClause; + manyNode.IsManyValues = state.RepeatedClause is NonTerminalClause; + manyNode.IsManyGroups = state.RepeatedClause is NonTerminalClause nt && nt.IsGroup; + + if (state.RepeatedClause is ChoiceClause choice) + { + manyNode.IsManyGroups = false; + manyNode.IsManyTokens = choice.IsTerminalChoice; + manyNode.IsManyValues = choice.IsNonTerminalChoice; + } + + if (state.Children.Any()) + { + result.IsError = false; + foreach (var child in state.Children) + { + if (child.Root != null) + { + result.EndingPosition = Math.Max(result.EndingPosition, child.EndingPosition); + manyNode.Add(child.Root); + } + } + } + + result.Root = manyNode; + state.Parent.SetResult(result); + } + } + + private void ParseInfixExpressionRule(InfixExpressionStackState state, Stack> stack) + { + if (state.ExpressionState == ExpressionRuleState.NotStarted) + { + state.ExpressionState = ExpressionRuleState.Left; + stack.Push(state); + var nextClause = state.Rule.Clauses[0]; + PushClause(nextClause, stack, state); + return; + } + if (state.Result.IsOk) + { + if (state.ExpressionState == ExpressionRuleState.Done) + { + var children = new List>(); + children.Add(state.Left.Root); + children.Add(state.Operator.Root); + children.Add(state.Right.Root); + int currentPosition = state.Right.EndingPosition; + var node = new SyntaxNode(state.Rule.NodeName ?? state.Rule.NonTerminalName, children); + node.ExpressionAffix = state.Rule.ExpressionAffix; + node = ExpressionRuleManager.ManageExpressionRules(state.Rule, node); + node.IsByPassNode = state.Rule.IsByPassRule; + var op = (state.Operator.Root as SyntaxLeaf).Token; + var key = op.IsExplicit ? op.Value : op.TokenID.ToString(); + node.Operation = state.Rule.GetOperation(key); + var finalResult = new SyntaxParseResult(); + finalResult.Root = node; + finalResult.IsEnded = currentPosition >= state.Tokens.Length - 1 + || currentPosition == state.Tokens.Length - 2 && + state.Tokens[state.Tokens.Length - 1].IsEOS; + finalResult.EndingPosition = currentPosition; + state.Parent.SetResult(finalResult); + return; + } + + if (state.ExpressionState == ExpressionRuleState.Left) + { + state.Left = state.Result; + state.Position = state.Left.EndingPosition; + state.ExpressionState = ExpressionRuleState.Operator; + stack.Push(state); + var nextClause = state.Rule.Clauses[1]; + PushClause(nextClause, stack, state); + return; + } + if (state.ExpressionState == ExpressionRuleState.Operator) + { + state.Operator = state.Result; + state.ExpressionState = ExpressionRuleState.Right; + state.Position = state.Operator.EndingPosition; + stack.Push(state); + var nextClause = state.Rule.Clauses[2]; + PushClause(nextClause, stack, state); + return; + } + if (state.ExpressionState == ExpressionRuleState.Right) + { + state.Right = state.Result; + state.ExpressionState = ExpressionRuleState.Done; + state.Position = state.Right.EndingPosition; + stack.Push(state); + return; + } + } + else + { + if (state.Result.IsError) + { + if (state.ExpressionState == ExpressionRuleState.Operator) + { + // return Left (ok) + state.Parent.SetResult(state.Left); + return; + } + + if (state.ExpressionState == ExpressionRuleState.Right) + { + // fail on operator parsing => return expression with left operand + state.Parent.SetResult(state.Left); + return; + } + + // fail otherwise => return current result (left if left or right if done) + state.Parent.SetResult(state.Result); + return; + } + } + } + + public override bool IsExtension(GrammarNode clause) + { + if (clause is Rule rule && rule.IsInfixExpressionRule) + { + // only infix . prefix is a regular rule and postfix are managed through 2 generic rules + return true; + } + if (clause is Rule rule2 && rule2.IsExpressionRule) + { + ; + } + + return (clause is OptionClause || clause is ManyClause || clause is ChoiceClause); + } + + public override void PushClauseExtension(GrammarNode clause, Stack> stack, + StackState parent) + { + switch (clause) + { + case Rule rule when rule.IsInfixExpressionRule: + { + var state = new InfixExpressionStackState(rule, parent) + { + Tokens = parent.Tokens, + Position = parent.Position + }; + stack.Push(state); + break; + } + case ZeroOrMoreClause zeroOrMore: + { + var state = new ZeroOrMoreStackState(zeroOrMore, parent) + { + Tokens = parent.Tokens, + Position = parent.Position + }; + stack.Push(state); + break; + } + case OneOrMoreClause oneOrMore: + { + var state = new OneOrMoreStackState(oneOrMore, parent) + { + Tokens = parent.Tokens, + Position = parent.Position + }; + stack.Push(state); + break; + } + case OptionClause option: + { + var state = new OptionStackState(option, parent) + { + Tokens = parent.Tokens, + Position = parent.Position + }; + stack.Push(state); + break; + } + case ChoiceClause choice: + { + var state = new ChoiceStackState(choice, parent) + { + Tokens = parent.Tokens, + Position = parent.Position + }; + stack.Push(state); + break; + } + } + } +} \ No newline at end of file diff --git a/src/sly/parser/parser/llparser/ebnf/stackist/state/ChoiceStackState.cs b/src/sly/parser/parser/llparser/ebnf/stackist/state/ChoiceStackState.cs new file mode 100644 index 00000000..a5306e49 --- /dev/null +++ b/src/sly/parser/parser/llparser/ebnf/stackist/state/ChoiceStackState.cs @@ -0,0 +1,101 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using sly.parser.llparser.bnf.stackist; +using sly.parser.syntax.grammar; + +namespace sly.parser.parser.llparser.ebnf.stackist.state; + + +public abstract class EbnfStackState : StackState where IN : struct, Enum +{ + public virtual EbnfStackStateType EbnfStackType => EbnfStackStateType.None; + + public EbnfStackState() + { + + } + + public EbnfStackState(StackState parent) : base(parent) + { + + } + + public override StackStateType Type => StackStateType.Extension; +} + +[DebuggerDisplay("{DebugString}")] +public class ChoiceStackState : EbnfStackState where IN : struct, Enum +{ + + public override EbnfStackStateType EbnfStackType => EbnfStackStateType.Choice; + + private List> _children; + + public List> Children => _children; + + public override string DebugString => $"Choice {Choice.Dump()} [{Index}] @{Position}"; + + public int Index { get; set; } + + public ChoiceClause Choice { get; set; } + + public ChoiceStackState(ChoiceClause choice, StackState parent) : base(parent) + { + Choice = choice; + Index = 0; + _children = new List>(); + } + + public override void SetResult(SyntaxParseResult result) + { + base.SetResult(result); + // if (result.IsOk) + // { + AddChild(result); + // } + } + + + public void AddChild(SyntaxParseResult result) + { + if (result.IsOk) + if (Children.Any(x => x == null)) + { + _children.Add(result); + ; + } + if (result == null) + { + return; + } + + try + { + if (Index - 1 <= Children.Count - 1) + { + var previous = Children[Index - 1]; + if (previous != null && previous.EndingPosition < result.EndingPosition) + { + Children[Index - 1] = result; + } + } + else + { + Children.Add(result); + } + } + catch (Exception e) + { + ; + } + + //Children.Add(result); + } + + public override string ToString() + { + return "Choice: " + Choice.Dump(); + } +} \ No newline at end of file diff --git a/src/sly/parser/parser/llparser/ebnf/stackist/state/ExpressionRuleStackState.cs b/src/sly/parser/parser/llparser/ebnf/stackist/state/ExpressionRuleStackState.cs new file mode 100644 index 00000000..10825a83 --- /dev/null +++ b/src/sly/parser/parser/llparser/ebnf/stackist/state/ExpressionRuleStackState.cs @@ -0,0 +1,13 @@ +using System.Collections.Generic; +using System.Linq; + +namespace sly.parser.parser.llparser.ebnf.stackist.state; + +public enum ExpressionRuleState +{ + NotStarted, + Left, + Operator, + Right, + Done +} \ No newline at end of file diff --git a/src/sly/parser/parser/llparser/ebnf/stackist/state/InfixExpressionStackState.cs b/src/sly/parser/parser/llparser/ebnf/stackist/state/InfixExpressionStackState.cs new file mode 100644 index 00000000..7a06a0ae --- /dev/null +++ b/src/sly/parser/parser/llparser/ebnf/stackist/state/InfixExpressionStackState.cs @@ -0,0 +1,35 @@ +using System; +using System.Diagnostics; +using sly.parser.generator; +using sly.parser.llparser.bnf.stackist; +using sly.parser.syntax.grammar; + +namespace sly.parser.parser.llparser.ebnf.stackist.state; + +[DebuggerDisplay("{DebugString}")] +public class InfixExpressionStackState : EbnfStackState where IN : struct, Enum +{ + public override string DebugString => $"expression {Rule.Dump()} [{ExpressionState}] @{Position}"; + + public SyntaxParseResult Left; + public SyntaxParseResult Operator; + public SyntaxParseResult Right; + + public override EbnfStackStateType EbnfStackType => EbnfStackStateType.Infix; + public ExpressionRuleState ExpressionState { get; set; } + + public Rule Rule { get; set; } + + private Affix Affix => Rule.ExpressionAffix; + + public InfixExpressionStackState(Rule rule, StackState parent) : base(parent) + { + Rule = rule; + ExpressionState = ExpressionRuleState.NotStarted; + } + + public override string ToString() + { + return "Expresion: " + Rule.Dump(); + } +} \ No newline at end of file diff --git a/src/sly/parser/parser/llparser/ebnf/stackist/state/OneOrMoreStackState.cs b/src/sly/parser/parser/llparser/ebnf/stackist/state/OneOrMoreStackState.cs new file mode 100644 index 00000000..6758eca9 --- /dev/null +++ b/src/sly/parser/parser/llparser/ebnf/stackist/state/OneOrMoreStackState.cs @@ -0,0 +1,83 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using sly.parser.llparser.bnf.stackist; +using sly.parser.syntax.grammar; + +namespace sly.parser.parser.llparser.ebnf.stackist.state; + +public class OneOrMoreStackState:EbnfStackState where IN : struct, Enum +{ + public override EbnfStackStateType EbnfStackType => EbnfStackStateType.OneOrMore; + + private readonly OneOrMoreClause _clause; + private readonly StackState _parent; + + + + public int Index { get; set; } + + private List> _children; + + public List> Children => _children; + + public IClause RepeatedClause => _clause.Clause; + + public OneOrMoreStackState(OneOrMoreClause oneOrMore, StackState parent) + { + _clause = oneOrMore; + Parent = parent; + _children = new List>(); + } + + public bool IsOk => Result != null && Result.IsOk; + + + public override void SetResult(SyntaxParseResult result) + { + base.SetResult(result); + if (result.IsOk) + { + AddChild(result); + } + } + + + public void AddChild(SyntaxParseResult result) + { + if (result.IsOk) + if (Children.Any(x => x == null)) + { + _children.Add(result); + ; + } + if (result == null) + { + return; + } + + try + { + if (Index - 1 <= Children.Count - 1) + { + var previous = Children[Index - 1]; + if (previous != null && previous.EndingPosition < result.EndingPosition) + { + Children[Index - 1] = result; + } + } + else + { + Children.Add(result); + } + } + catch (Exception e) + { + ; + } + + //Children.Add(result); + } + + +} \ No newline at end of file diff --git a/src/sly/parser/parser/llparser/ebnf/stackist/state/OptionStackState.cs b/src/sly/parser/parser/llparser/ebnf/stackist/state/OptionStackState.cs new file mode 100644 index 00000000..bd6790f9 --- /dev/null +++ b/src/sly/parser/parser/llparser/ebnf/stackist/state/OptionStackState.cs @@ -0,0 +1,46 @@ +using System; +using System.Collections.Generic; +using sly.parser.llparser.bnf.stackist; +using sly.parser.syntax.grammar; + +namespace sly.parser.parser.llparser.ebnf.stackist.state; + +public class OptionStackState : EbnfStackState where IN : struct, Enum +{ + private readonly OptionClause _clause; + private readonly StackState _parent; + + public override EbnfStackStateType EbnfStackType => EbnfStackStateType.Option; + + private List> _children; + + public List> Children => _children; + + public IClause OptionalClause => _clause.Clause; + + + public OptionStackState(OptionClause option, StackState parent) + { + _clause = option; + Parent = parent; + _children = new List>(); + } + + public override void SetResult(SyntaxParseResult result) + { + base.SetResult(result); + AddChild(result); + } + + public bool IsOk => Result != null && Result.IsOk; + + + public void AddChild(SyntaxParseResult result) + { + if (result.IsOk) + { + _children.Add(result); + } + } + +} \ No newline at end of file diff --git a/src/sly/parser/parser/llparser/ebnf/stackist/state/PostfixExpressionStackState.cs b/src/sly/parser/parser/llparser/ebnf/stackist/state/PostfixExpressionStackState.cs new file mode 100644 index 00000000..ce50677a --- /dev/null +++ b/src/sly/parser/parser/llparser/ebnf/stackist/state/PostfixExpressionStackState.cs @@ -0,0 +1,36 @@ +using System; +using System.Diagnostics; +using sly.parser.generator; +using sly.parser.llparser.bnf.stackist; +using sly.parser.syntax.grammar; + +namespace sly.parser.parser.llparser.ebnf.stackist.state; + +[DebuggerDisplay("{DebugString}")] +public class PostfixExpressionStackState : EbnfStackState where IN : struct, Enum +{ + public override string DebugString => $"expression {Rule.Dump()} [{ExpressionState}] @{Position}"; + + public SyntaxParseResult Left; + public SyntaxParseResult Operator; + public SyntaxParseResult Right; + + public override EbnfStackStateType EbnfStackType => EbnfStackStateType.Postfix; + + public ExpressionRuleState ExpressionState { get; set; } + + public Rule Rule { get; set; } + + private Affix Affix => Rule.ExpressionAffix; + + public PostfixExpressionStackState(Rule rule, StackState parent) : base(parent) + { + Rule = rule; + ExpressionState = ExpressionRuleState.NotStarted; + } + + public override string ToString() + { + return "Expresion: " + Rule.Dump(); + } +} \ No newline at end of file diff --git a/src/sly/parser/parser/llparser/ebnf/stackist/state/PrefixExpressionStackState.cs b/src/sly/parser/parser/llparser/ebnf/stackist/state/PrefixExpressionStackState.cs new file mode 100644 index 00000000..034e2b2a --- /dev/null +++ b/src/sly/parser/parser/llparser/ebnf/stackist/state/PrefixExpressionStackState.cs @@ -0,0 +1,35 @@ +using System; +using System.Diagnostics; +using sly.parser.generator; +using sly.parser.llparser.bnf.stackist; +using sly.parser.syntax.grammar; + +namespace sly.parser.parser.llparser.ebnf.stackist.state; + +[DebuggerDisplay("{DebugString}")] +public class PrefixExpressionStackState : EbnfStackState where IN : struct, Enum +{ + public override string DebugString => $"expression {Rule.Dump()} [{ExpressionState}] @{Position}"; + + public SyntaxParseResult Left; + public SyntaxParseResult Operator; + public SyntaxParseResult Right; + + public override EbnfStackStateType EbnfStackType => EbnfStackStateType.Prefix; + public ExpressionRuleState ExpressionState { get; set; } + + public Rule Rule { get; set; } + + private Affix Affix => Rule.ExpressionAffix; + + public PrefixExpressionStackState(Rule rule, StackState parent) : base(parent) + { + Rule = rule; + ExpressionState = ExpressionRuleState.NotStarted; + } + + public override string ToString() + { + return "Expresion: " + Rule.Dump(); + } +} \ No newline at end of file diff --git a/src/sly/parser/parser/llparser/ebnf/stackist/state/ZeroOrMoreStackState.cs b/src/sly/parser/parser/llparser/ebnf/stackist/state/ZeroOrMoreStackState.cs new file mode 100644 index 00000000..6ced17f6 --- /dev/null +++ b/src/sly/parser/parser/llparser/ebnf/stackist/state/ZeroOrMoreStackState.cs @@ -0,0 +1,80 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using sly.parser.llparser.bnf.stackist; +using sly.parser.syntax.grammar; + +namespace sly.parser.parser.llparser.ebnf.stackist.state; + +public class ZeroOrMoreStackState : EbnfStackState where IN : struct, Enum +{ + private readonly ZeroOrMoreClause _clause; + private readonly StackState _parent; + + public int Index { get; set; } + + public override EbnfStackStateType EbnfStackType => EbnfStackStateType.ZeroOrMore; + + private List> _children; + + public List> Children => _children; + + public IClause RepeatedClause => _clause.Clause; + + + public ZeroOrMoreStackState(ZeroOrMoreClause zeroOrMore, StackState parent) + { + _clause = zeroOrMore; + Parent = parent; + _children = new List>(); + } + + + + public bool IsOk => Result != null && Result.IsOk; + + + public override void SetResult(SyntaxParseResult result) + { + base.SetResult(result); + if (result.IsOk) + { + AddChild(result); + } + } + + public void AddChild(SyntaxParseResult result) + { + if (Children.Any(x => x == null)) + { + ; + } + if (result == null) + { + return; + } + + try + { + if (Index - 1 <= Children.Count - 1) + { + var previous = Children[Index - 1]; + if (previous != null && previous.EndingPosition < result.EndingPosition) + { + Children[Index - 1] = result; + } + } + else + { + Children.Add(result); + } + } + catch (Exception e) + { + ; + } + + //Children.Add(result); + } + +} \ No newline at end of file diff --git a/src/sly/parser/syntax/grammar/LeadingToken.cs b/src/sly/parser/syntax/grammar/LeadingToken.cs index 343a4c1a..7e1d8958 100644 --- a/src/sly/parser/syntax/grammar/LeadingToken.cs +++ b/src/sly/parser/syntax/grammar/LeadingToken.cs @@ -49,7 +49,6 @@ public bool Match(Token token) { return token.IsUnIndent; } - return TokenId.Equals(token.TokenID); } diff --git a/src/sly/parser/syntax/tree/SyntaxNode.cs b/src/sly/parser/syntax/tree/SyntaxNode.cs index 40c716c2..6857d8cf 100644 --- a/src/sly/parser/syntax/tree/SyntaxNode.cs +++ b/src/sly/parser/syntax/tree/SyntaxNode.cs @@ -130,7 +130,7 @@ public string Dump(string tab) expressionSuffix = $">{expressionSuffix}<"; } - builder.AppendLine($"{tab}+ {Name} {(IsByPassNode ? "===":"")}"); + builder.AppendLine($"{tab}+ {Name} {(IsByPassNode ? "===":"")} {(Visitor != null ? Visitor.Name : "no visitor?")}() {expressionSuffix}"); foreach (var child in Children) { diff --git a/tests/ParserTests/EBNFStackTests.cs b/tests/ParserTests/EBNFStackTests.cs new file mode 100644 index 00000000..18c5d0f3 --- /dev/null +++ b/tests/ParserTests/EBNFStackTests.cs @@ -0,0 +1,1191 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using indented; +using jsonparser; +using jsonparser.JsonModel; +using NFluent; +using simpleExpressionParser; +using sly.buildresult; +using sly.lexer; +using sly.parser; +using sly.parser.generator; +using sly.parser.llparser.ebnf; +using sly.parser.parser; +using sly.parser.parser.llparser.ebnf.stackist; +using sly.parser.syntax.grammar; +using Xunit; +using ExpressionToken = simpleExpressionParser.ExpressionToken; +using String = System.String; + +namespace ParserTests +{ + + + [Collection("stack")] + public class EBNFStackTests + { + public enum TokenType + { + [Lexeme("a")] a = 1, + [Lexeme("b")] b = 2, + [Lexeme("c")] c = 3, + [Lexeme("e")] e = 4, + [Lexeme("f")] f = 5, + [Lexeme("[ \\t]+", true)] WS = 100, + [Lexeme("\\n\\r]+", true, true)] EOL = 101 + } + + + [Production("R : A B c ")] + public string R(string A, string B, Token c) + { + var result = "R("; + result += A + ","; + result += B + ","; + result += c.Value; + result += ")"; + return result; + } + + [Production("R : G+ ")] + public string RManyNT(List gs) + { + if (gs.Any()) + { + var result = "R("; + result += gs + .Select(g => g.ToString()) + .Aggregate((s1, s2) => s1 + "," + s2); + result += ")"; + return result; + } + + return ""; + } + + [Production("G : e f ")] + public string RManyNT(Token e, Token f) + { + var result = $"G({e.Value},{f.Value})"; + return result; + } + + [Production("A : a + ")] + public string A(List> astr) + { + var result = "A("; + result += astr + .Select(a => a.Value) + .Aggregate((a1, a2) => a1 + ", " + a2); + result += ")"; + return result; + } + + [Production("B : b * ")] + public string B(List> bstr) + { + if (bstr.Any()) + { + var result = "B("; + result += bstr + .Select(b => b.Value) + .Aggregate((b1, b2) => b1 + ", " + b2); + result += ")"; + return result; + } + + return "B()"; + } + + [Production("Ba : b* a")] + public string Ba(List> bstr, Token a) { + var result = "Ba("; + if (bstr.Any()) + { + result += bstr + .Select(b => b.Value) + .Aggregate((b1, b2) => b1 + ", " + b2); + result += ", "; + } + + result += a.Value; + result += ")"; + return result; + } + [Production("BA : b* A")] + public string BA(List> bstr, string a) { + var result = "BA("; + if (bstr.Any()) + { + result += bstr + .Select(b => b.Value) + .Aggregate((b1, b2) => b1 + ", " + b2); + result += ", "; + } + result += a; + result += ")"; + return result; + } + + + + + private Parser Parser; + + private BuildResult> BuildParser() + { + var parserInstance = new EBNFStackTests(); + var builder = new ParserBuilder(); + var result = builder.BuildParser(parserInstance, ParserType.EBNF_LL_STACK, "R"); + return result; + } + + + private BuildResult> BuildEbnfJsonParser() + { + var parserInstance = new EbnfJsonParser(); + var builder = new ParserBuilder(); + + var result = + builder.BuildParser(parserInstance, ParserType.EBNF_LL_STACK, "root"); + return result; + } + + private BuildResult> BuildOptionParser() + { + var parserInstance = new OptionTestParser(); + var builder = new ParserBuilder(); + + var result = + builder.BuildParser(parserInstance, ParserType.EBNF_LL_STACK, "root"); + return result; + } + + private BuildResult> BuildGroupParser() + { + var parserInstance = new GroupTestParser(); + var builder = new ParserBuilder(); + + var result = + builder.BuildParser(parserInstance, ParserType.EBNF_LL_STACK, "rootGroup"); + return result; + } + + + + [Fact] + public void TestBuildGroupParser() + { + var buildResult = BuildGroupParser(); + Check.That(buildResult).IsOk(); + } + + [Fact] + public void TestEmptyOptionalNonTerminal() + { + var buildResult = BuildOptionParser(); + Check.That(buildResult).IsOk(); + var optionParser = buildResult.Result; + + var result = optionParser.Parse("a c", "root2"); + Check.That(result).IsOkParsing(); + Check.That(result.Result).IsEqualTo("R(a,,c)"); + } + + [Fact] + public void TestEmptyOptionTerminalInMiddle() + { + var buildResult = BuildOptionParser(); + Check.That(buildResult).IsOk(); + var optionParser = buildResult.Result; + + var result = optionParser.Parse("a c", "root2"); + Check.That(result).IsOkParsing(); + Check.That(result.Result).IsEqualTo("R(a,,c)"); + } + + + [Fact] + public void TestEmptyTerminalOption() + { + var buildResult = BuildOptionParser(); + Check.That(buildResult).IsOk(); + var optionParser = buildResult.Result; + + var result = optionParser.Parse("a b", "root3"); + Check.That(result).IsOkParsing(); + Check.That(result.Result).IsEqualTo("R(a,B(b),)"); + } + + [Fact] + public void TestErrorMissingClosingBracket() + { + var jsonParser = new EbnfJsonGenericParser(); + var builder = new ParserBuilder(); + var build = builder.BuildParser(jsonParser, ParserType.EBNF_LL_STACK, "root"); + var parserTest = build.Result; + ParseResult r = null; + try + { + r = parserTest.Parse("{"); + } + catch (Exception e) + { + var stack = e.StackTrace; + var message = e.Message; + } + + Check.That(r).Not.IsOkParsing(); + } + + [Fact] + public void TestGroupSyntaxManyParser() + { + var buildResult = BuildGroupParser(); + Check.That(buildResult).IsOk(); + var groupParser = buildResult.Result; + var res = groupParser.Parse("a ,a , a ,a,a", "rootMany"); + Check.That(res).IsOkParsing(); + Check.That(res.Result).IsEqualTo("R(a,a,a,a,a)"); + } + + [Fact] + public void TestGroupSyntaxChoicesParser() + { + var buildResult = BuildGroupParser(); + Check.That(buildResult).IsOk(); + var groupParser = buildResult.Result; + var res = groupParser.Parse("a ;a ", "rootGroupChoice"); + + Check.That(res).IsOkParsing(); + Check.That(res.Result).IsEqualTo("R(a,a)"); + + res = groupParser.Parse("a ,a ", "rootGroupChoice"); + + Check.That(res).IsOkParsing(); + Check.That(res.Result).IsEqualTo("R(a,a)"); + } + + [Fact] + public void TestGroupSyntaxChoicesManyParser() + { + var buildResult = BuildGroupParser(); + Check.That(buildResult).IsOk(); + var groupParser = buildResult.Result; + var res = groupParser.Parse("a ;a,a ; a,a ", "rootGroupChoiceMany"); + Check.That(res).IsOkParsing(); + Check.That(res.Result).IsEqualTo("R(a,a,a,a,a)"); // rootMany + } + + [Fact] + public void TestGroupSyntaxOptionIsSome() + { + var buildResult = BuildGroupParser(); + Check.That(buildResult).IsOk(); + var groupParser = buildResult.Result; + var res = groupParser.Parse("a ; a ", "rootOption"); + Check.That(res).IsOkParsing(); + Check.That(res.Result).IsEqualTo("R(a;a)"); + } + + [Fact] + public void TestGroupSyntaxOptionIsNone() + { + var buildResult = BuildGroupParser(); + Check.That(buildResult).IsOk(); + var groupParser = buildResult.Result; + var res = groupParser.Parse("a ", "rootOption"); + Check.That(res).IsOkParsing(); + Check.That(res.Result).IsEqualTo("R(a;)"); + } + + [Fact] + public void TestGroupSyntaxParser() + { + var buildResult = BuildGroupParser(); + Check.That(buildResult).IsOk(); + var groupParser = buildResult.Result; + var res = groupParser.Parse("a ,a"); + + Check.That(res).IsOkParsing(); + Check.That(res.Result).IsEqualTo("R(a; {,,,a})"); + } + + + [Fact] + public void TestJsonList() + { + var buildResult = BuildEbnfJsonParser(); + Check.That(buildResult).IsOk(); + var jsonParser = buildResult.Result; + + var result = jsonParser.Parse("[1,2,3,4]"); + Check.That(result).IsOkParsing(); + Check.That(result.Result.IsList).IsTrue(); + + var list = (JList) result.Result; + Check.That(list.Count).IsEqualTo(4); + + Check.That(list).HasItem(0,1); + Check.That(list).HasItem(1,2); + Check.That(list).HasItem( 2,3); + Check.That(list).HasItem( 3,4); + } + + [Fact] + public void TestJsonObject() + { + var buildResult = BuildEbnfJsonParser(); + Check.That(buildResult).IsOk(); + var jsonParser = buildResult.Result; + var result = jsonParser.Parse("{\"one\":1,\"two\":2,\"three\":\"trois\" }"); + Check.That(result).IsOkParsing(); + Check.That(result.Result.IsObject).IsTrue(); + + var o = (JObject) result.Result; + Check.That(o.Count).IsEqualTo(3); + Check.That(o.Count).IsEqualTo(3); + Check.That(o).HasProperty("one", 1); + Check.That(o).HasProperty("two", 2); + Check.That(o).HasProperty("three", "trois"); + } + + [Fact] + public void TestNonEmptyOptionalNonTerminal() + { + var buildResult = BuildOptionParser(); + Check.That(buildResult).IsOk(); + var optionParser = buildResult.Result; + + var result = optionParser.Parse("a b c", "root2"); + Check.That(result).IsOkParsing(); + Check.That(result.Result).IsEqualTo("R(a,B(b),c)"); + } + + + [Fact] + public void TestNonEmptyTerminalOption() + { + var buildResult = BuildOptionParser(); + Check.That(buildResult).IsOk(); + var optionParser = buildResult.Result; + + var result = optionParser.Parse("a b c", "root"); + Check.That(result).IsOkParsing(); + Check.That(result.Result).IsEqualTo("R(a,b,c)"); + } + + + [Fact] + public void TestOneOrMoreNonTerminal() + { + var buildResult = BuildParser(); + Check.That(buildResult).IsOk(); + Parser = buildResult.Result; + var result = Parser.Parse("e f e f"); + Check.That(result).IsOkParsing(); + Check.That(result.Result).IsEqualTo("R(G(e,f),G(e,f))"); + } + + + [Fact] + public void TestOneOrMoreWithMany() + { + var buildResult = BuildParser(); + Check.That(buildResult).IsOk(); + Parser = buildResult.Result; + var result = Parser.Parse("aaa b c"); + Check.That(result).IsOkParsing(); + Check.That(result.Result).IsEqualTo("R(A(a, a, a),B(b),c)"); + } + + [Fact] + public void TestOneOrMoreWithOne() + { + var buildResult = BuildParser(); + Check.That(buildResult).IsOk(); + Parser = buildResult.Result; + var result = Parser.Parse(" b c"); + Check.That(result).Not.IsOkParsing(); + + } + + [Fact] + public void TestParseBuild() + { + var buildResult = BuildParser(); + Check.That(buildResult).IsOk(); + Parser = buildResult.Result; + Check.That(Parser.SyntaxParser).IsInstanceOf>(); + Check.That(Parser.Configuration.NonTerminals).CountIs(6); + + var nt = Parser.Configuration.NonTerminals["R"]; + Check.That(nt.Rules).CountIs(2); + nt = Parser.Configuration.NonTerminals["A"]; + Check.That(nt.Rules).CountIs(1); + var rule = nt.Rules[0]; + Check.That(rule.Clauses).CountIs(1); + Check.That(rule.Clauses[0]).IsInstanceOf>(); + nt = Parser.Configuration.NonTerminals["B"]; + Check.That((nt.Rules)).CountIs(1); + rule = nt.Rules[0]; + Check.That((rule.Clauses)).CountIs(1); + Check.That(rule.Clauses[0]).IsInstanceOf>(); + + } + + [Fact] + public void TestZeroOrMoreWithMany() + { + var buildResult = BuildParser(); + Check.That(buildResult).IsOk(); + Parser = buildResult.Result; + var result = Parser.Parse("a bb c"); + Check.That(result).IsOkParsing(); + Check.That(result.Result).IsEqualTo("R(A(a),B(b, b),c)"); + } + + [Fact] + public void TestZeroOrMoreStarterFollowedByTerminal() + { + var buildResult = BuildParser(); + Check.That(buildResult).IsOk(); + Parser = buildResult.Result; + var result = Parser.Parse("bbb a","Ba"); + Check.That(result).IsOkParsing(); + Check.That(result.Result).IsEqualTo("Ba(b, b, b, a)"); + result = Parser.Parse("a","Ba"); + Check.That(result).IsOkParsing(); + Check.That(result.Result).IsEqualTo("Ba(a)"); + } + + [Fact] + public void TestZeroOrMoreStarterFollowedByNonTerminal() + { + var buildResult = BuildParser(); + Check.That(buildResult).IsOk(); + Parser = buildResult.Result; + var result = Parser.Parse("bbb a","BA"); + Check.That(result).IsOkParsing(); + Check.That(result.Result).IsEqualTo("BA(b, b, b, A(a))"); + result = Parser.Parse("a","BA"); + Check.That(result).IsOkParsing(); + Check.That(result.Result).IsEqualTo("BA(A(a))"); + } + + [Fact] + public void TestZeroOrMoreWithNone() + { + var buildResult = BuildParser(); + Check.That(buildResult).IsOk(); + Parser = buildResult.Result; + var result = Parser.Parse("a c"); + Check.That(result).IsOkParsing(); + Check.That(result.Result).IsEqualTo("R(A(a),B(),c)"); + } + + [Fact] + public void TestZeroOrMoreWithOne() + { + var buildResult = BuildParser(); + Check.That(buildResult).IsOk(); + Parser = buildResult.Result; + var result = Parser.Parse("a b c"); + Check.That(result).IsOkParsing(); + Check.That(result.Result).IsEqualTo("R(A(a),B(b),c)"); + } + + + #region CONTEXTS + + private BuildResult> buildSimpleExpressionParserWithContext(ParserType parserType = ParserType.EBNF_LL_STACK) + { + var startingRule = $"{nameof(SimpleExpressionParserWithContext)}_expressions"; + var parserInstance = new SimpleExpressionParserWithContext(); + var builder = new ParserBuilder(); + var parser = builder.BuildParser(parserInstance, parserType, startingRule); + return parser; + } + + [Fact] + public void TestContextualParsing() + { + var buildResult = buildSimpleExpressionParserWithContext(); + Check.That(buildResult).IsOk(); + //Check.That(buildResult).IsOkParsing(); + var parser = buildResult.Result; + var res = parser.ParseWithContext("2 + a", new Dictionary {{"a", 2}}); + Check.That(res).IsOkParsing(); + Check.That(res.Result).IsEqualTo(4); + } + + [Fact] + public void TestContextualParsing2() + { + var buildResult = buildSimpleExpressionParserWithContext(); + + Check.That(buildResult).IsOk(); + var parser = buildResult.Result; + var res = parser.ParseWithContext("2 + a * b", new Dictionary {{"a", 2}, {"b", 3}}); + Check.That(res.IsOk).IsTrue(); + Check.That(res.Result).IsEqualTo(8); + } + + [Fact] + public void TestContextualParsingPrefixAndPostfix() + { + var buildResult = buildSimpleExpressionParserWithContext(); + + Check.That(buildResult).IsOk(); + var parser = buildResult.Result; + var res = parser.ParseWithContext("- a", new Dictionary {{"a", 3}}); + Check.That(res.IsOk).IsTrue(); + Check.That(res.Result).IsEqualTo(-3); + } + + [Fact] + public void TestContextualParsingWithEbnf() + { + var buildResult = buildSimpleExpressionParserWithContext(ParserType.EBNF_LL_STACK); + + Check.That(buildResult).IsOk(); + var parser = buildResult.Result; + var res = parser.ParseWithContext("2 + a * b", new Dictionary {{"a", 2}, {"b", 3}}); + Check.That(res.IsOk).IsTrue(); + Check.That(res.Result).IsEqualTo(8); + } + + [Fact] + public void TestBug100() + { + var startingRule = $"testNonTerm"; + var parserInstance = new Bugfix100Test(); + var builder = new ParserBuilder(); + var builtParser = builder.BuildParser(parserInstance, ParserType.EBNF_LL_STACK, startingRule); + Check.That(builtParser).IsOk(); + Check.That(builtParser.Result).IsNotNull(); + var parser = builtParser.Result; + var conf = parser.Configuration; + var expected = new List() {GroupTestToken.A, GroupTestToken.COMMA}; + + var nonTerm = conf.NonTerminals["testNonTerm"]; + Check.That(nonTerm).IsNotNull(); + Check.That(nonTerm.GetPossibleLeadingTokens()).CountIs(2); + Check.That(nonTerm.GetPossibleLeadingTokens().Select(x => x.TokenId)).Contains(expected); + + var term = conf.NonTerminals["testTerm"]; + Check.That(term).IsNotNull(); + Check.That(term.GetPossibleLeadingTokens()).CountIs(2); + Check.That(term.GetPossibleLeadingTokens().Select(x => x.TokenId)).Contains(expected); + } + + #endregion + + [Fact] + public void TestBug104() + { + var startingRule = $"testNonTerm"; + var parserInstance = new Bugfix104Test(); + var builder = new ParserBuilder(); + var builtParser = builder.BuildParser(parserInstance, ParserType.EBNF_LL_STACK, startingRule); + Check.That(builtParser).IsOk(); + Check.That(builtParser.Errors).IsEmpty(); + } + + [Fact] + public void TestAlternateChoiceTerminal() + { + var startingRule = $"choice"; + var parserInstance = new AlternateChoiceTestTerminal(); + var builder = new ParserBuilder(); + var builtParser = builder.BuildParser(parserInstance, ParserType.EBNF_LL_STACK, startingRule); + Check.That(builtParser).IsOk(); + Check.That(builtParser.Errors).IsEmpty(); + var parseResult = builtParser.Result.Parse("a", "choice"); + Check.That(parseResult.IsOk).IsTrue(); + Check.That(parseResult.Result).IsEqualTo("a"); + parseResult = builtParser.Result.Parse("b", "choice"); + Check.That(parseResult.IsOk).IsTrue(); + Check.That(parseResult.Result).IsEqualTo("b"); + parseResult = builtParser.Result.Parse("c", "choice"); + Check.That(parseResult.IsOk).IsTrue(); + Check.That(parseResult.Result).IsEqualTo("c"); + parseResult = builtParser.Result.Parse("d", "choice"); + Check.That(parseResult.IsOk).IsFalse(); + } + + [Fact] + public void TestAlternateChoiceNonTerminal() + { + var startingRule = $"choice"; + var parserInstance = new AlternateChoiceTestNonTerminal(); + var builder = new ParserBuilder(); + var builtParser = builder.BuildParser(parserInstance, ParserType.EBNF_LL_STACK, startingRule); + Check.That(builtParser).IsOk(); + Check.That(builtParser.Errors).IsEmpty(); + var parseResult = builtParser.Result.Parse("a", "choice"); + Check.That(parseResult.IsOk).IsTrue(); + Check.That(parseResult.Result).IsEqualTo("A(a)"); + parseResult = builtParser.Result.Parse("b", "choice"); + Check.That(parseResult.IsOk).IsTrue(); + Check.That(parseResult.Result).IsEqualTo("B(b)"); + parseResult = builtParser.Result.Parse("c", "choice"); + Check.That(parseResult.IsOk).IsTrue(); + Check.That(parseResult.Result).IsEqualTo("C(c)");; + parseResult = builtParser.Result.Parse("d", "choice"); + Check.That(parseResult.IsOk).IsFalse(); + } + + [Fact] + public void TestAlternateChoiceOneOrMoreNonTerminal() + { + var startingRule = $"choice"; + var parserInstance = new AlternateChoiceTestOneOrMoreNonTerminal(); + var builder = new ParserBuilder(); + var builtParser = builder.BuildParser(parserInstance, ParserType.EBNF_LL_STACK, startingRule); + Check.That(builtParser).IsOk(); + Check.That(builtParser.Errors).IsEmpty(); + var parseResult = builtParser.Result.Parse("a b", "choice"); + Check.That(parseResult).IsOkParsing(); + Check.That(parseResult.Result).IsEqualTo("A(a) B(b)"); + + parseResult = builtParser.Result.Parse("b", "choice"); + Check.That(parseResult.IsOk).IsTrue(); + Check.That(parseResult.Result).IsEqualTo("B(b)"); + parseResult = builtParser.Result.Parse("c", "choice"); + Check.That(parseResult.IsOk).IsTrue(); + Check.That(parseResult.Result).IsEqualTo("C(c)"); + parseResult = builtParser.Result.Parse("d", "choice"); + Check.That(parseResult.IsOk).IsFalse(); + } + + [Fact] + public void TestAlternateChoiceZeroOrMoreTerminal() + { + var startingRule = $"choice"; + var parserInstance = new AlternateChoiceTestZeroOrMoreTerminal(); + var builder = new ParserBuilder(); + var builtParser = builder.BuildParser(parserInstance, ParserType.EBNF_LL_STACK, startingRule); + Check.That(builtParser).IsOk(); + Check.That(builtParser.Errors).IsEmpty(); + var parseResult = builtParser.Result.Parse("a b c", "choice"); + Check.That(parseResult.IsOk).IsTrue(); + Check.That(parseResult.Result).IsEqualTo("a,b,c"); + parseResult = builtParser.Result.Parse("b", "choice"); + Check.That(parseResult.IsOk).IsTrue(); + } + + [Fact] + public void TestAlternateChoiceOneOrMoreTerminal() + { + var startingRule = $"choice"; + var parserInstance = new AlternateChoiceTestOneOrMoreTerminal(); + var builder = new ParserBuilder(); + var builtParser = builder.BuildParser(parserInstance, ParserType.EBNF_LL_STACK, startingRule); + Check.That(builtParser).IsOk(); + Check.That(builtParser.Errors).IsEmpty(); + var parseResult = builtParser.Result.Parse("a b c", "choice"); + Check.That(parseResult).IsOkParsing(); + Check.That(parseResult.Result).IsEqualTo("a,b,c"); + parseResult = builtParser.Result.Parse("b", "choice"); + Check.That(parseResult.IsOk).IsTrue(); + } + + [Fact] + public void TestAlternateChoiceOptionTerminal() + { + var startingRule = $"choice"; + var parserInstance = new AlternateChoiceTestOptionTerminal(); + var builder = new ParserBuilder(); + var builtParser = builder.BuildParser(parserInstance, ParserType.EBNF_LL_STACK, startingRule); + Check.That(builtParser).IsOk(); + Check.That(builtParser.Errors).IsEmpty(); + var parseResult = builtParser.Result.Parse("a b", "choice"); + Check.That(parseResult.IsOk).IsTrue(); + Check.That(parseResult.Result).IsEqualTo("a,b"); + parseResult = builtParser.Result.Parse("a", "choice"); + Check.That(parseResult.IsOk).IsTrue(); + Check.That(parseResult.Result).IsEqualTo("a,"); + } + + [Fact] + public void TestAlternateChoiceOptionNonTerminal() + { + var startingRule = $"choice"; + var parserInstance = new AlternateChoiceTestOptionNonTerminal(); + var builder = new ParserBuilder(); + var builtParser = builder.BuildParser(parserInstance, ParserType.EBNF_LL_STACK, startingRule); + Check.That(builtParser).IsOk(); + Check.That(builtParser.Errors).IsEmpty(); + var parseResult = builtParser.Result.Parse("a b f", "choice"); + Check.That(parseResult.IsOk).IsTrue(); + Check.That(parseResult.Result).IsEqualTo("a,b,f"); + parseResult = builtParser.Result.Parse("a", "choice"); + Check.That(parseResult.IsOk).IsTrue(); + Check.That(parseResult.Result).IsEqualTo("a,,"); + parseResult = builtParser.Result.Parse("a b ", "choice"); + Check.That(parseResult.IsOk).IsTrue(); + Check.That(parseResult.Result).IsEqualTo("a,b,"); + parseResult = builtParser.Result.Parse("a f", "choice"); + Check.That(parseResult.IsOk).IsTrue(); + Check.That(parseResult.Result).IsEqualTo("a,,f"); + + } + + [Fact] + public void TestAlternateChoiceOptionDiscardedTerminal() + { + var startingRule = $"choice"; + var parserInstance = new AlternateChoiceTestOptionDiscardedTerminal(); + var builder = new ParserBuilder(); + var builtParser = builder.BuildParser(parserInstance, ParserType.EBNF_LL_STACK, startingRule); + Check.That(builtParser).IsOk(); + Check.That(builtParser.Errors).IsEmpty(); + var parseResult = builtParser.Result.Parse("a b", "choice"); + Check.That(parseResult.IsOk).IsTrue(); + Check.That(parseResult.Result).IsEqualTo("a"); + parseResult = builtParser.Result.Parse("a", "choice"); + Check.That(parseResult).Not.IsOkParsing(); + Check.That(parseResult.Errors).CountIs(1); + Check.That(parseResult.Errors[0].ErrorType).IsEqualTo(ErrorType.UnexpectedEOS); + } + + [Fact] + public void TestAlternateChoiceErrorMixedTerminalAndNonTerminal() + { + var startingRule = $"choice"; + var parserInstance = new AlternateChoiceTestError(); + var builder = new ParserBuilder("en"); + var builtParser = builder.BuildParser(parserInstance, ParserType.EBNF_LL_STACK, startingRule); + Check.That(builtParser).Not.IsOk(); + Check.That(builtParser.Errors).CountIs(2); + Check.That(builtParser.Errors.Select(x => x.Code)).Contains(ErrorCodes.PARSER_MIXED_CHOICES, + ErrorCodes.PARSER_NON_TERMINAL_CHOICE_CANNOT_BE_DISCARDED); + Check.That(builtParser.Errors.Select(x => x.Message)).Contains("choice : [ a | b | C | D] contains [ a(T) | b(T) | C(NT) | D(NT) ] with mixed terminal and nonterminal."); + + } + + + + [Fact] + public void TestAlternateChoiceInGroupLeftRecursion() + { + var startingRule = $"choiceInGroup"; + var parserInstance = new LeftRecWithChoiceInGroup(); + var builder = new ParserBuilder(); + var builtParser = builder.BuildParser(parserInstance, ParserType.EBNF_LL_STACK, startingRule); + Check.That(builtParser).Not.IsOk(); + Check.That(builtParser.Errors).CountIs(1); + Check.That(builtParser.Errors.First().Code).IsEqualTo(ErrorCodes.PARSER_LEFT_RECURSIVE); + } + + + [Fact] + public void TestIssue507TransitiveEmptyStarter() + { + var startingRule = $"x"; + var parserInstance = new Issue507TransitiveEmptyStarterParser(); + var builder = new ParserBuilder(); + var builtParser = builder.BuildParser(parserInstance, ParserType.EBNF_LL_STACK, startingRule); + Check.That(builtParser).IsOk(); + var parser = builtParser.Result; + var parserResultNotEmpty = parser.Parse("a a a"); + Check.That(parserResultNotEmpty).IsOkParsing(); + Check.That(parserResultNotEmpty.Result).IsEqualTo("a,a,a"); + + var parserResultEmpty = parser.Parse(""); + Check.That(parserResultEmpty).IsOkParsing(); + Check.That(parserResultEmpty.Result).IsEqualTo("empty"); + } + + [Fact] + public void TestIssue507MoreTransitiveEmptyStarter() + { + var startingRule = $"x"; + var parserInstance = new Issue507MoreTransitiveEmptyStarterParser(); + var builder = new ParserBuilder(); + var builtParser = builder.BuildParser(parserInstance, ParserType.EBNF_LL_STACK, startingRule); + Check.That(builtParser).IsOk(); + var parser = builtParser.Result; + var parserResultNotEmpty = parser.Parse("a a a"); + Check.That(parserResultNotEmpty).IsOkParsing(); + Check.That(parserResultNotEmpty.Result).IsEqualTo("a,a,a"); + + var parserResultEmpty = parser.Parse(""); + Check.That(parserResultEmpty).IsOkParsing(); + Check.That(parserResultEmpty.Result).IsEqualTo("empty"); + } + + [Fact] + public void TestIssue190() + { + var startingRule = $"root"; + var parserInstance = new Issue190parser(); + var builder = new ParserBuilder(); + var builtParser = builder.BuildParser(parserInstance, ParserType.EBNF_LL_STACK, startingRule); + Check.That(builtParser).IsOk(); + var parser = builtParser.Result; + var parserResultNotTrue = parser.Parse("not true"); + Check.That(parserResultNotTrue.IsOk).IsTrue(); + Check.That(parserResultNotTrue.Result).IsFalse(); + var parserResultTrue = parser.Parse("yes"); + Check.That(parserResultTrue.IsOk).IsTrue(); + Check.That(parserResultTrue.Result).IsTrue(); + } + + [Fact] + public void TestIssue193() + { + RuleParserType.ParserType = ParserType.LL_RECURSIVE_DESCENT; + var builtParser = BuildParser(); + Check.That(builtParser).IsOk(); + Check.That(builtParser.Result).IsNotNull(); + var parser = builtParser.Result; + var test = parser.Parse("a b"); + + Check.That(test).Not.IsOkParsing(); + Check.That(test.Errors).CountIs(1); + var error = test.Errors[0] as UnexpectedTokenSyntaxError; + // TODO : parser does not return furthest error. + Check.That(error).IsNotNull(); + Check.That(error.ErrorType).IsEqualTo(ErrorType.UnexpectedEOS); + } + + [Fact] + public void TestIssue213() + { + var parserInstance = new DoNotIgnoreCommentsParser(); + var builder = new ParserBuilder(); + var builtParser = builder.BuildParser(parserInstance, ParserType.EBNF_LL_STACK, "main"); + + Check.That(builtParser.IsOk).IsTrue(); + Check.That(builtParser.Result).IsNotNull(); + + var parser = builtParser.Result; + + var test = parser.Parse("a /*commented b*/b"); + Check.That(test).IsOkParsing(); + Check.That(test.Result).IsInstanceOf(); + + var list = test.Result as IdentifierList; + Check.That(list.Ids).CountIs(2); + Check.That(list.Ids[0].IsCommented).IsFalse(); + Check.That(list.Ids[0].Name).IsEqualTo("a"); + Check.That(list.Ids[1].IsCommented).IsTrue(); + Check.That(list.Ids[1].Name).IsEqualTo("b"); + Check.That(list.Ids[1].Comment).IsEqualTo("commented b"); + } + + + [Fact] + public void TestIndentedParser() + { + var source =@"if truc == 1 + un = 1 + deux = 2 +else + trois = 3 + quatre = 4 + +"; + ParserBuilder builder = new ParserBuilder(); + var instance = new IndentedParser(); + var parserRes = builder.BuildParser(instance, ParserType.EBNF_LL_STACK, "root"); + Check.That(parserRes).IsOk(); + + var parser = parserRes.Result; + Check.That(parser).IsNotNull(); + var parseResult = parser.Parse(source); + Check.That(parseResult).IsOkParsing(); + + var ast = parseResult.Result; + Check.That(ast).IsNotNull(); + Check.That(ast).IsInstanceOf(); + + Block root = ast as Block; + Check.That(root.Statements).CountIs(1); + Check.That(root.Statements.First()).IsInstanceOf(); + + IfThenElse ifthenelse = root.Statements.First() as IfThenElse; + Check.That(ifthenelse.Cond).IsNotNull(); + Check.That(ifthenelse.Then).IsNotNull(); + Check.That(ifthenelse.Else).IsNotNull(); + Check.That(ifthenelse.Then.Statements).CountIs(2); + Check.That(ifthenelse.Else.Statements).CountIs(2); + } + + [Fact] + public void TestIndentedParserNestedBlocks() + { + + var source =@" +// this is a informative comment +if truc == 1 + un = 1 + deux = 2 +else + trois = 3 + quatre = 4 + if bidule ==89 + toto = 28 +final = 9999 +"; + ParserBuilder builder = new ParserBuilder(); + var instance = new IndentedParser(); + var parserRes = builder.BuildParser(instance, ParserType.EBNF_LL_STACK, "root"); + Check.That(parserRes).IsOk(); + var parser = parserRes.Result; + Check.That(parser).IsNotNull(); + var parseResult = parser.Parse(source); + Check.That(parseResult).IsOkParsing(); + + var ast = parseResult.Result; + Check.That(ast).IsNotNull(); + Check.That(ast).IsInstanceOf(); + + Block root = ast as Block; + Check.That(root.Statements).CountIs(2); + Check.That(root.Statements.First()).IsInstanceOf(); + + IfThenElse ifthenelse = root.Statements.First() as IfThenElse; + Check.That(ifthenelse.Comment).IsNotNull(); + Check.That(ifthenelse.Comment.Trim()).IsEqualTo("this is a informative comment"); + + Check.That(ifthenelse.Cond).IsNotNull(); + Check.That(ifthenelse.Then).IsNotNull(); + Check.That(ifthenelse.Else).IsNotNull(); + + Check.That(ifthenelse.Then.Statements).CountIs(2); + Check.That(ifthenelse.Else.Statements).CountIs(3); + + var lastelseStatement = ifthenelse.Else.Statements.Last(); + Check.That(lastelseStatement).IsInstanceOf(); + var nestedIf = lastelseStatement as IfThenElse; + Check.That(nestedIf.Then).IsNotNull(); + Check.That(nestedIf.Cond).IsNotNull(); + + var lastStatement = root.Statements.Last(); + Check.That(lastStatement).IsInstanceOf(); + + var finalSet = lastStatement as Set; + Check.That(finalSet.Id.Name).IsEqualTo("final"); + Check.That(finalSet.Value.Value).IsEqualTo(9999); + + } + + [Fact] + public void TestIndentedParserWithEolAwareness() + { + var source =@"// information +if truc == 1 + un = 1 + deux = 2 +else + trois = 3 + quatre = 4 + +"; + ParserBuilder builder = new ParserBuilder(); + var instance = new IndentedParser2(); + var parserRes = builder.BuildParser(instance, ParserType.EBNF_LL_STACK, "root"); + Check.That(parserRes.IsOk).IsTrue(); + var parser = parserRes.Result; + Check.That(parser).IsNotNull(); + var parseResult = parser.Parse(source); + Check.That(parseResult).IsOkParsing(); + var ast = parseResult.Result; + Check.That(ast).IsNotNull(); + Check.That(ast).IsInstanceOf(); + + Block root = ast as Block; + Check.That(root.Statements).CountIs(1); + + Check.That(root.Statements.First()).IsInstanceOf(); + IfThenElse ifthenelse = root.Statements.First() as IfThenElse; + Check.That(ifthenelse.IsCommented).IsTrue(); + + Check.That(ifthenelse.Comment.Trim()).IsEqualTo("information"); + Check.That(ifthenelse.Cond).IsNotNull(); + Check.That(ifthenelse.Then).IsNotNull(); + Check.That(ifthenelse.Then.Statements).CountIs(2); + Check.That(ifthenelse.Else).IsNotNull(); + Check.That(ifthenelse.Else.Statements).CountIs(2); + } + + [Fact] + public void TestIndentedParserWithEolAwareness2() + { + var source =@"// information +if truc == 1 + un = 1 + deux = 2 +else + trois = 3 + quatre = 4 + +"; + ParserBuilder builder = new ParserBuilder(); + var instance = new IndentedParser2(); + var parserRes = builder.BuildParser(instance, ParserType.EBNF_LL_STACK, "root"); + Check.That(parserRes.IsOk).IsTrue(); + var parser = parserRes.Result; + Check.That(parser).IsNotNull(); + + var parseResult = parser.Parse(source); + Check.That(parseResult).IsOkParsing(); + var ast = parseResult.Result; + Check.That(ast).IsNotNull(); + Check.That(ast).IsInstanceOf(); + Block root = ast as Block; + Check.That(root.Statements).CountIs(1); + Check.That(root.Statements.First()).IsInstanceOf(); + IfThenElse ifthenelse = root.Statements.First() as IfThenElse; + Check.That(ifthenelse.IsCommented).IsTrue(); + Check.That(ifthenelse.Comment).Contains("information"); + Check.That(ifthenelse.Cond).IsNotNull(); + Check.That(ifthenelse.Then).IsNotNull(); + Check.That(ifthenelse.Then.Statements).CountIs(2); + Check.That(ifthenelse.Else).IsNotNull(); + Check.That(ifthenelse.Else.Statements).CountIs(2); + } + + [Fact] + public void TestIssue213WithChannels() + { + var parserInstance = new DoNotIgnoreCommentsWithChannelsParser(); + var builder = new ParserBuilder(); + var builtParser = builder.BuildParser(parserInstance, ParserType.EBNF_LL_STACK, "main"); + + Check.That(builtParser.IsOk).IsTrue(); + Check.That(builtParser.Result).IsNotNull(); + var parser = builtParser.Result; + + var test = parser.Parse(@" +a + +b1 +// commented b [1] +/*commented b [2]*/ +b2 + +c +// comment c @1 +// commented c @2 +// commented c @3 + +test + +// commented d before +d +// commented d after + +"); + + Check.That(test).IsOkParsing(); + Check.That(test.Result).IsNotNull(); + Check.That(test.Result).IsInstanceOf(); + var list = test.Result as IdentifierList; + Check.That(list.Ids).CountIs(6); + + var id = list.Ids[0]; + Check.That(id.Name).IsEqualTo("a"); + Check.That(id.IsCommented).IsFalse(); + + id = list.Ids[1]; + Check.That(id.IsCommented).IsTrue(); + Check.That(id.Name).IsEqualTo("b1"); + Check.That(id.Comment.Trim()).IsEqualTo("commented b [1]\ncommented b [2]"); + + id = list.Ids[2]; + Check.That(id.IsCommented).IsTrue(); + Check.That(id.Name).IsEqualTo("b2"); + Check.That(id.Comment.Trim()).IsEqualTo("commented b [1]\ncommented b [2]"); + + id = list.Ids[3]; + Check.That(id.IsCommented).IsTrue(); + Check.That(id.Name).IsEqualTo("c"); + var comments = id.Comment; + Check.That(id.Comment.Trim()).IsEqualTo("comment c @1\ncommented c @2\ncommented c @3"); + + id = list.Ids[4]; + Check.That(id.Name).IsEqualTo("test"); + Check.That(id.IsCommented).IsTrue(); // catches comment from c and d + Check.That(id.Comment.Trim()).IsEqualTo("comment c @1\ncommented c @2\ncommented c @3\ncommented d before"); + + id = list.Ids[5]; + Check.That(id.IsCommented).IsTrue(); + Check.That(id.Name).IsEqualTo("d"); + comments = id.Comment; + Check.That(id.Comment.Trim()).IsEqualTo("commented d before\ncommented d after"); + + test = parser.Parse(@"a +// commented b +b"); + + Check.That(test.IsOk).IsTrue(); + Check.That(test.Result).IsNotNull(); + Check.That(test.Result).IsInstanceOf(); + list = test.Result as IdentifierList; + Check.That(list.Ids).CountIs(2); + Check.That(list.Ids[0].IsCommented).IsFalse(); + Check.That(list.Ids[0].Name).IsEqualTo("a"); + Check.That(list.Ids[1].IsCommented).IsTrue(); + Check.That(list.Ids[1].Name).IsEqualTo("b"); + Check.That(list.Ids[1].Comment.Trim()).IsEqualTo("commented b"); + ; + + } + + [Fact] + public void TestNotClosingIndents() + { + var source =@" +if truc == 1 + un = 1 + deux = 2"; + ParserBuilder builder = new ParserBuilder(); + var instance = new IndentedParser(); + var parserRes = builder.BuildParser(instance, ParserType.EBNF_LL_STACK, "root"); + Check.That(parserRes.IsOk).IsTrue(); + + var parser = parserRes.Result; + Check.That(parser).IsNotNull(); + var parseResult = parser.Parse(source); + Check.That(parseResult).Not.IsOkParsing(); + Check.That(parseResult.Errors).CountIs(1); + var error = parseResult.Errors[0]; + Check.That(error.ErrorType).IsEqualTo(ErrorType.UnexpectedEOS); + } + + + //[Fact] : repeat are not (yet) managed by stack parser + public void TestRepeat() + { + ParserBuilder builder = new ParserBuilder(); + var instance = new RepeatParser(); + var parserRes = builder.BuildParser(instance, ParserType.EBNF_LL_STACK, "root"); + Check.That(parserRes).IsOk(); + + var parser = parserRes.Result; + Check.That(parser).IsNotNull(); + var parseResult = parser.Parse("a1b2c3d4e5."); + Check.That(parseResult).IsOkParsing(); + Check.That(parseResult.Result).IsEqualTo("a(1),b(2),c(3),d(4),e(5)"); + parseResult = parser.Parse("."); + Check.That(parseResult).IsOkParsing(); + Check.That(parseResult.Result).IsEqualTo(""); + parseResult = parser.Parse("a1b2c3d4e5f6g7"); + Check.That(parseResult).Not.IsOkParsing(); + Check.That(parseResult.Errors).CountIs(1); + var error = parseResult.Errors[0] as UnexpectedTokenSyntaxError; + Check.That(error).IsNotNull(); + Check.That(error.ErrorType).IsEqualTo(ErrorType.UnexpectedToken); + + Check.That(error.UnexpectedToken.TokenID).IsEqualTo(BasicToken.ID); + Check.That(error.UnexpectedToken.Value).IsEqualTo("g"); + + } + } +} diff --git a/tests/ParserTests/EBNFTests.cs b/tests/ParserTests/EBNFTests.cs index 8475db16..f142cca6 100644 --- a/tests/ParserTests/EBNFTests.cs +++ b/tests/ParserTests/EBNFTests.cs @@ -1219,6 +1219,20 @@ public void TestContextualParsing() Check.That(res.IsOk).IsTrue(); Check.That(res.Result).IsEqualTo(4); } + + + [Fact] + public void TestContextualParsingprefixPostfix() + { + var buildResult = buildSimpleExpressionParserWithContext(); + + Check.That(buildResult.IsError).IsFalse(); + var parser = buildResult.Result; + var res = parser.ParseWithContext(" - a ! ", new Dictionary {{"a", 3}}); + Check.That(res.IsOk).IsTrue(); + Check.That(res.Result).IsEqualTo(-(3*2*1)); + } + [Fact] public void TestContextualParsingPrefixPostfix() @@ -1232,6 +1246,7 @@ public void TestContextualParsingPrefixPostfix() Check.That(res.Result).IsEqualTo(-6); } + [Fact] public void TestContextualParsing2() { diff --git a/tests/ParserTests/ExpressionTests.cs b/tests/ParserTests/ExpressionTests.cs index c5505f63..415ed53a 100644 --- a/tests/ParserTests/ExpressionTests.cs +++ b/tests/ParserTests/ExpressionTests.cs @@ -12,15 +12,28 @@ public ExpressionTests() { var parserInstance = new ExpressionParser(); var builder = new ParserBuilder(); - Parser = builder.BuildParser(parserInstance, ParserType.LL_RECURSIVE_DESCENT, "expression").Result; + RecursiveParser = builder.BuildParser(parserInstance, ParserType.LL_RECURSIVE_DESCENT, "expression").Result; + StackParser = builder.BuildParser(parserInstance, ParserType.LL_STACK, "expression").Result; + + } - private readonly Parser Parser; + private readonly Parser RecursiveParser; + + private readonly Parser StackParser; [Fact] public void TestFactorDivide() { - var r = Parser.Parse("42/2"); + var r = RecursiveParser.Parse("42/2"); + Check.That(r.IsError).IsFalse(); + Check.That(r.Result).IsEqualTo(21); + } + + [Fact] + public void TestFactorDivideStack() + { + var r = StackParser.Parse("42/2"); Check.That(r.IsError).IsFalse(); Check.That(r.Result).IsEqualTo(21); } @@ -28,7 +41,15 @@ public void TestFactorDivide() [Fact] public void TestFactorTimes() { - var r = Parser.Parse("2*2"); + var r = RecursiveParser.Parse("2*2"); + Check.That(r.IsError).IsFalse(); + Check.That(r.Result).IsEqualTo(4); + } + + [Fact] + public void TestFactorTimesStack() + { + var r = StackParser.Parse("2*2"); Check.That(r.IsError).IsFalse(); Check.That(r.Result).IsEqualTo(4); } @@ -36,7 +57,15 @@ public void TestFactorTimes() [Fact] public void TestGroup() { - var r = Parser.Parse("(2 + 2)"); + var r = RecursiveParser.Parse("(2 + 2)"); + Check.That(r.IsError).IsFalse(); + Check.That(r.Result).IsEqualTo(4); + } + + [Fact] + public void TestGroupStack() + { + var r = StackParser.Parse("(2 + 2)"); Check.That(r.IsError).IsFalse(); Check.That(r.Result).IsEqualTo(4); } @@ -44,7 +73,15 @@ public void TestGroup() [Fact] public void TestGroup2() { - var r = Parser.Parse("6 * (2 + 2)"); + var r = RecursiveParser.Parse("6 * (2 + 2)"); + Check.That(r.IsError).IsFalse(); + Check.That(r.Result).IsEqualTo(24); + } + + [Fact] + public void TestGroup2Stack() + { + var r = StackParser.Parse("6 * (2 + 2)"); Check.That(r.IsError).IsFalse(); Check.That(r.Result).IsEqualTo(24); } @@ -52,7 +89,15 @@ public void TestGroup2() [Fact] public void TestPrecedence() { - var r = Parser.Parse("6 * 2 + 2"); + var r = RecursiveParser.Parse("6 * 2 + 2"); + Check.That(r.IsError).IsFalse(); + Check.That(r.Result).IsEqualTo(14); + } + + [Fact] + public void TestPrecedenceStack() + { + var r = StackParser.Parse("6 * 2 + 2"); Check.That(r.IsError).IsFalse(); Check.That(r.Result).IsEqualTo(14); } @@ -60,7 +105,15 @@ public void TestPrecedence() [Fact] public void TestSingleNegativeValue() { - var r = Parser.Parse("-1"); + var r = RecursiveParser.Parse("-1"); + Check.That(r.IsError).IsFalse(); + Check.That(r.Result).IsEqualTo(-1); + } + + [Fact] + public void TestSingleNegativeValueStack() + { + var r = StackParser.Parse("-1"); Check.That(r.IsError).IsFalse(); Check.That(r.Result).IsEqualTo(-1); } @@ -69,7 +122,15 @@ public void TestSingleNegativeValue() [Fact] public void TestSingleValue() { - var r = Parser.Parse("1"); + var r = RecursiveParser.Parse("1"); + Check.That(r.IsError).IsFalse(); + Check.That(r.Result).IsEqualTo(1); + } + + [Fact] + public void TestSingleValueStack() + { + var r = StackParser.Parse("1"); Check.That(r.IsError).IsFalse(); Check.That(r.Result).IsEqualTo(1); } @@ -77,7 +138,15 @@ public void TestSingleValue() [Fact] public void TestTermMinus() { - var r = Parser.Parse("1 - 1"); + var r = RecursiveParser.Parse("1 - 1"); + Check.That(r.IsError).IsFalse(); + Check.That(r.Result).IsEqualTo(0); + } + + [Fact] + public void TestTermMinusStack() + { + var r = StackParser.Parse("1 - 1"); Check.That(r.IsError).IsFalse(); Check.That(r.Result).IsEqualTo(0); } @@ -85,23 +154,43 @@ public void TestTermMinus() [Fact] public void TestTermPlus() { - var r = Parser.Parse("1 + 1"); + var r = RecursiveParser.Parse("1 + 1"); Check.That(r.IsError).IsFalse(); Check.That(r.Result).IsEqualTo(2); } [Fact] - public void TestIssue351NotReachingEOS() + public void TestTermPlusStack() { - var r = Parser.Parse("1 + 1 + 1"); + var r = StackParser.Parse("1 + 1"); Check.That(r.IsError).IsFalse(); + Check.That(r.Result).IsEqualTo(2); + } + + [Fact] + public void TestIssue351NotReachingEOS() + { + var r = RecursiveParser.Parse("1 + 1 + 1"); + Check.That(r).IsOkParsing(); - r = Parser.Parse("1 + 1 + "); - Check.That(r.IsError).IsTrue(); + r = RecursiveParser.Parse("1 + 1 + "); + Check.That(r).Not.IsOkParsing(); Check.That(r.Errors).CountIs(1); + var error = r.Errors[0]; + Check.That(error.ErrorType).IsEqualTo(ErrorType.UnexpectedEOS); + } + + [Fact] + public void TestIssue351NotReachingEOSStack() + { + var r = StackParser.Parse("1 + 1 + 1"); + Check.That(r).IsOkParsing(); - - + r = RecursiveParser.Parse("1 + 1 + "); + Check.That(r).Not.IsOkParsing(); + Check.That(r.Errors).CountIs(1); + var error = r.Errors[0]; + Check.That(error.ErrorType).IsEqualTo(ErrorType.UnexpectedEOS); } } } \ No newline at end of file diff --git a/tests/ParserTests/Issue225_223/IndexOutOfRangeTests.cs b/tests/ParserTests/Issue225_223/IndexOutOfRangeTests.cs index e774b34f..fd9d9b60 100644 --- a/tests/ParserTests/Issue225_223/IndexOutOfRangeTests.cs +++ b/tests/ParserTests/Issue225_223/IndexOutOfRangeTests.cs @@ -24,11 +24,8 @@ private static Expression Parse(string query) var buildResult = builder.BuildParser(parserInstance, ParserType.EBNF_LL_RECURSIVE_DESCENT, "expression"); - // if (buildResult.IsError) - // throw new AggregateException( - // buildResult.Errors - // .Select(e => new Exception($"{e.Level} {e.Code} {e.Message}")) - // ); + + Check.That(buildResult).IsOk(); var parser = buildResult.Result; var queryExpression = parser.Parse(query.Trim()); diff --git a/tests/ParserTests/Issue239/Issue239Lexer.cs b/tests/ParserTests/Issue239/Issue239Lexer.cs index 146c4b78..ef18a620 100644 --- a/tests/ParserTests/Issue239/Issue239Lexer.cs +++ b/tests/ParserTests/Issue239/Issue239Lexer.cs @@ -1,3 +1,4 @@ +using sly.i18n; using sly.lexer; namespace ParserTests.Issue239 @@ -5,14 +6,22 @@ namespace ParserTests.Issue239 public enum Issue239Lexer { [AlphaNumDashId] + [LexemeLabel("en","identifier")] ID, + + [LexemeLabel("en","int keyword")] [Keyword("int")] INT, + + [LexemeLabel("en","int literal")] [Int] INT_LITERAL, [Sugar("=")] + [LexemeLabel("en","equal")] + ASSIGN, [Sugar(";")] + [LexemeLabel("en","semicolon")] SEMI diff --git a/tests/ParserTests/Issue239/Issue239Tests.cs b/tests/ParserTests/Issue239/Issue239Tests.cs index a6bb0ff6..12502b7e 100644 --- a/tests/ParserTests/Issue239/Issue239Tests.cs +++ b/tests/ParserTests/Issue239/Issue239Tests.cs @@ -26,7 +26,7 @@ public static void TestOk() { var parser = BuildParser(); var parseResult = parser.Parse("int x; int y; a = 12;"); - Check.That(parseResult.IsOk).IsTrue(); + Check.That(parseResult).IsOkParsing(); Check.That(parseResult.Result).IsInstanceOf>(); var lst = parseResult.Result as List; Check.That(lst).CountIs(3); diff --git a/tests/ParserTests/PostProcessedLexerTests.cs b/tests/ParserTests/PostProcessedLexerTests.cs index e39bc4c2..d64a51d6 100644 --- a/tests/ParserTests/PostProcessedLexerTests.cs +++ b/tests/ParserTests/PostProcessedLexerTests.cs @@ -1,7 +1,9 @@ using System.Collections.Generic; using NFluent; +using postProcessedLexerParser; using Xunit; using postProcessedLexerParser.expressionModel; +using sly.parser.generator; namespace ParserTests { @@ -10,10 +12,16 @@ public class PostProcessedLexerTests [Fact] public void TestPostLexerProcessing() { + RuleParserType.ParserType = ParserType.LL_STACK; var Parser = postProcessedLexerParser.PostProcessedLexerParserBuilder.buildPostProcessedLexerParser(); + var parserInstance = new FormulaParser(); + var builder = new ParserBuilder(); + var build = builder.BuildParser(parserInstance, ParserType.EBNF_LL_RECURSIVE_DESCENT, $"{nameof(FormulaParser)}_expressions", + lexerPostProcess: PostProcessedLexerParserBuilder.postProcessFormula); + Check.That(build).IsOk(); var r = Parser.Parse("2 * x"); - Check.That(r.IsError).IsFalse(); + Check.That(r).IsOkParsing(); var res = r.Result.Evaluate(new ExpressionContext(new Dictionary() { { "x", 2 } })); diff --git a/tests/ParserTests/samples/FluentIndentedWhileTests.cs b/tests/ParserTests/samples/FluentIndentedWhileTests.cs index 2896d9f6..c95ce7e1 100644 --- a/tests/ParserTests/samples/FluentIndentedWhileTests.cs +++ b/tests/ParserTests/samples/FluentIndentedWhileTests.cs @@ -238,6 +238,7 @@ public void TestNestedIfThenElse() [Fact] + public void TestFString() { var buildResult = buildParser(); diff --git a/tests/ParserTests/samples/IndentedWhileTests.cs b/tests/ParserTests/samples/IndentedWhileTests.cs index b4ff28ad..e34d3431 100644 --- a/tests/ParserTests/samples/IndentedWhileTests.cs +++ b/tests/ParserTests/samples/IndentedWhileTests.cs @@ -13,6 +13,7 @@ namespace ParserTests.samples { + [CollectionDefinition("while", DisableParallelization = true)] public class IndentedWhileTests { diff --git a/tests/ParserTests/samples/JsonGenericStackTests.cs b/tests/ParserTests/samples/JsonGenericStackTests.cs new file mode 100644 index 00000000..ed40edc4 --- /dev/null +++ b/tests/ParserTests/samples/JsonGenericStackTests.cs @@ -0,0 +1,213 @@ +using jsonparser; +using jsonparser.JsonModel; +using NFluent; +using sly.parser; +using sly.parser.generator; +using Xunit; + +namespace ParserTests.samples +{ + [Collection("stack")] + public class JsonGenericStackTests + { + public JsonGenericStackTests() + { + var jsonParser = new EbnfJsonGenericParser(); + var builder = new ParserBuilder(); + + var build = builder.BuildParser(jsonParser, ParserType.EBNF_LL_STACK, "root"); + Check.That(build).IsOk(); + Parser = build.Result; + } + + private static Parser Parser; + + + + [Fact] + public void TestDoubleValue() + { + var r = Parser.Parse("0.1"); + Check.That(r).IsOkParsing(); + Check.That(r.Result.IsValue).IsTrue(); + Check.That(((JValue) r.Result).IsDouble).IsTrue(); + Check.That(((JValue) r.Result).GetValue()).IsEqualTo(0.1d); + } + + + [Fact] + public void TestEmptyListValue() + { + var r = Parser.Parse("[]"); + Check.That(r).IsOkParsing(); + + Check.That(r.Result.IsList).IsTrue(); + Check.That(((JList) r.Result)).CountIs(0); + } + + [Fact] + public void TestEmptyObjectValue() + { + var r = Parser.Parse("{}"); + Check.That(r).IsOkParsing(); + Check.That(r.Result.IsObject).IsTrue(); + Check.That(((JObject) r.Result)).CountIs(0); + } + + [Fact] + public void TestFalseBooleanValue() + { + var r = Parser.Parse("false"); + Check.That(r).IsOkParsing(); + Check.That(r.Result.IsValue).IsTrue(); + var val = (JValue) r.Result; + Check.That(val.IsBool).IsTrue(); + Check.That(val.GetValue()).IsFalse(); + } + + [Fact] + public void TestIntValue() + { + var r = Parser.Parse("1"); + Check.That(r).IsOkParsing(); + + Check.That(r.Result.IsValue).IsTrue(); + Check.That(((JValue) r.Result).IsInt).IsTrue(); + + Check.That(((JValue) r.Result).GetValue()).IsEqualTo(1); + } + + [Fact] + public void TestManyListValue() + { + var r = Parser.Parse("[1,2]"); + Check.That(r).IsOkParsing(); + Check.That(r.Result.IsList).IsTrue(); + var list = (JList) r.Result; + Check.That(list).CountIs(2); + Check.That(list).HasItem(0, 1); + Check.That(list).HasItem(1, 2); + } + + [Fact] + public void TestManyMixedListValue() + { + var r = Parser.Parse("[1,null,{},true,42.58]"); + Check.That(r).IsOkParsing(); + object val = r.Result; + Check.That(r.Result.IsList).IsTrue(); + var list = (JList) r.Result; + Check.That((JList) r.Result).CountIs(5); + Check.That(list).HasItem(0, 1); + Check.That(((JList) r.Result)[1].IsNull).IsTrue(); + Check.That(list).HasObjectItem(2,0); + Check.That(list).HasItem(3, true); + Check.That(list).HasItem(4, 42.58d); + } + + [Fact] + public void TestManyNestedPropertyObjectValue() + { + var json = "{\"p1\":\"v1\",\"p2\":\"v2\",\"p3\":{\"inner1\":1}}"; + + var r = Parser.Parse(json); + Check.That(r).IsOkParsing(); + Check.That(r.Result.IsObject).IsTrue(); + var values = (JObject) r.Result; + Check.That(values).CountIs(3); + Check.That(values).HasProperty("p1", "v1"); + Check.That(values).HasProperty("p1", "v1"); + Check.That(values).HasProperty("p2", "v2"); + + Check.That(values).HasObjectProperty("p3"); + var inner = values["p3"]; + Check.That(inner.IsObject).IsTrue(); + var innerObj = (JObject) inner; + + Check.That(innerObj).CountIs(1); + Check.That(innerObj).HasProperty("inner1", 1); + } + + [Fact] + public void TestManyPropertyObjectValue() + { + var json = "{\"p1\":\"v1\",\"p2\":\"v2\"}"; + json = "{\"p1\":\"v1\" , \"p2\":\"v2\" }"; + var r = Parser.Parse(json); + Check.That(r).IsOkParsing(); + + Check.That(r.Result.IsObject).IsTrue(); + var values = (JObject) r.Result; + Check.That(values).CountIs(2); + Check.That(values).HasProperty("p1", "v1"); + Check.That(values).HasProperty("p2", "v2"); + } + + [Fact] + public void TestNullValue() + { + var r = Parser.Parse("null"); + Check.That(r).IsOkParsing(); + Check.That(r.Result.IsNull).IsTrue(); + } + + [Fact] + public void TestListSingleValue() + { + var r = Parser.Parse("[1]"); + Check.That(r).IsOkParsing(); + + Check.That(r.Result.IsList).IsTrue(); + var list = (JList) r.Result; + Check.That(list).CountIs(1); + Check.That(list).HasItem(0, 1); + } + + + [Fact] + public void TestSinglePropertyObjectValue() + { + var r = Parser.Parse("{\"prop\":\"value\"}"); + Check.That(r.IsError).IsFalse(); + Check.That(r.Result).IsNotNull(); + Check.That(r.Result.IsObject).IsTrue(); + var values = (JObject) r.Result; + Check.That(values).CountIs(1); + Check.That(values).HasProperty("prop", "value"); + } + + [Fact] + public void TestStringValue() + { + var val = "hello"; + var r = Parser.Parse("\"" + val + "\""); + Check.That(r).IsOkParsing(); + Check.That(r.Result.IsValue).IsTrue(); + Check.That(((JValue) r.Result).IsString).IsTrue(); + Check.That(((JValue) r.Result).GetValue()).IsEqualTo(val); + } + + [Fact] + public void TestTrueBooleanValue() + { + var r = Parser.Parse("true"); + Check.That(r).IsOkParsing(); + + Check.That(r.Result.IsValue).IsTrue(); + var val = (JValue) r.Result; + Check.That(val.IsBool).IsTrue(); + Check.That(val.IsBool).IsTrue(); + Check.That(val.GetValue()).IsTrue(); + } + + [Fact] + public void TestIncompleteObject() + { + var r = Parser.Parse("{\"prop\":\"value\",\"prop2\":[1,2,3"); + Check.That(r).Not.IsOkParsing(); + Check.That(r.Errors).CountIs(1); + var error = r.Errors[0]; + Check.That(error.ErrorType).IsEqualTo(ErrorType.UnexpectedEOS); + } + } +} \ No newline at end of file diff --git a/tests/ParserTests/samples/XmlStackTests.cs b/tests/ParserTests/samples/XmlStackTests.cs new file mode 100644 index 00000000..b68dd035 --- /dev/null +++ b/tests/ParserTests/samples/XmlStackTests.cs @@ -0,0 +1,115 @@ +using NFluent; +using sly.parser; +using sly.parser.generator; +using XML; +using Xunit; + +namespace ParserTests.samples; + +public class XmlStackTests +{ + + [Fact] + public void TestXmlParserWithLexerModesOk() + { + ParserBuilder builder = new ParserBuilder(); + var xmlparser = new MinimalXmlParser(); + var r = builder.BuildParser(xmlparser, ParserType.EBNF_LL_RECURSIVE_DESCENT, "document"); + Check.That(r.IsError).IsFalse(); + var parser = r.Result; + var parsed = parser.Parse(@" + + + + + + + + inner inner content + + + +"); + Check.That(parsed).IsOkParsing(); + // var tree = parsed.SyntaxTree; + // var graphviz = new GraphVizEBNFSyntaxTreeVisitor(); + // var root = graphviz.VisitTree(tree); + // string graph = graphviz.Graph.Compile(); + // File.Delete("c:\\tmp\\tree.dot"); + // File.AppendAllText("c:\\tmp\\tree.dot", graph); + } + + + [Fact] + public void TestXmlParserWithLexerModesKo() + { + ParserBuilder builder = new ParserBuilder(); + var xmlparser = new MinimalXmlParser(); + var r = builder.BuildParser(xmlparser, ParserType.EBNF_LL_RECURSIVE_DESCENT, "document"); + Check.That(r.IsError).IsFalse(); + var parser = r.Result; + var parsed = parser.Parse(@" + + + + + + + + inner inner content +"); + Check.That(parsed).Not.IsOkParsing(); + var error = parsed.Errors[0]; + Check.That(error.ErrorType).IsEqualTo(ErrorType.UnexpectedEOS); + } + + [Fact] + public void TestXmlLeadingMiscs() + { + ParserBuilder builder = new ParserBuilder(); + var xmlparser = new MinimalXmlParser(); + var r = builder.BuildParser(xmlparser, ParserType.EBNF_LL_RECURSIVE_DESCENT, "document"); + Check.That(r).IsOk(); + var parser = r.Result; + var parsed = parser.Parse(@" + + + +data + + + +"); + Check.That(parsed).IsOkParsing(); + var rr = parsed.Result; + Check.That(rr).Contains("pi(xml :: version = 1.0)") + .And.Contains("comment( starting doc )") + .And.Contains("comment( ending doc )"); + ; + } + + [Fact] + public void TestXmlLeadingMiscs2() + { + ParserBuilder builder = new ParserBuilder(); + var xmlparser = new MinimalXmlParser2(); + var r = builder.BuildParser(xmlparser, ParserType.EBNF_LL_RECURSIVE_DESCENT, "document"); + Check.That(r).IsOk(); + var parser = r.Result; + var parsed = parser.Parse(@" + + + +data + + + +"); + Check.That(parsed).IsOkParsing(); + var rr = parsed.Result; + Check.That(rr).Contains("pi(xml :: version = 1.0)") + .And.Contains("comment( starting doc )") + .And.Contains("comment( ending doc )"); + ; + } +} \ No newline at end of file diff --git a/tests/ParserTests/stack/BasicGroup.cs b/tests/ParserTests/stack/BasicGroup.cs new file mode 100644 index 00000000..74f1e035 --- /dev/null +++ b/tests/ParserTests/stack/BasicGroup.cs @@ -0,0 +1,20 @@ +using sly.lexer; +using sly.parser.generator; +using sly.parser.parser; + +namespace ParserTests.stack; + +public class BasicGroup +{ + [Production("root : g")] + public string Root(string g) => g; + + [Production("g : ( a b) ")] + public string G(Group group) => group.Value(0)+" "+group.Value(1); + + [Production("a : A")] + public string A(Token a) => a.Value; + + [Production("b: B")] + public string B(Token b) => b.Value; +} \ No newline at end of file diff --git a/tests/ParserTests/stack/L.cs b/tests/ParserTests/stack/L.cs new file mode 100644 index 00000000..1422681f --- /dev/null +++ b/tests/ParserTests/stack/L.cs @@ -0,0 +1,11 @@ +using sly.lexer; + +namespace ParserTests.stack; + +public enum L +{ + [Keyword("A")] A = 1, + [Keyword("B")] B = 2, + [Sugar("+")] PLUS = 3, + [Sugar("-")] MINUS = 4 +} \ No newline at end of file diff --git a/tests/ParserTests/stack/ManyGroup.cs b/tests/ParserTests/stack/ManyGroup.cs new file mode 100644 index 00000000..ba3e380f --- /dev/null +++ b/tests/ParserTests/stack/ManyGroup.cs @@ -0,0 +1,25 @@ +using System.Collections.Generic; +using System.Linq; +using sly.lexer; +using sly.parser.generator; +using sly.parser.parser; + +namespace ParserTests.stack; + +public class ManyGroup +{ + [Production("root : g")] + public string Root(string g) => g; + + [Production("g : ( a b)* ")] + public string G(List> groups) + { + return string.Join(",",groups.Select(x => x.Value(0)+" "+x.Value(1))); + } + + [Production("a : A")] + public string A(Token a) => a.Value; + + [Production("b: B")] + public string B(Token b) => b.Value; +} \ No newline at end of file diff --git a/tests/ParserTests/stack/ManyTerminalChoice.cs b/tests/ParserTests/stack/ManyTerminalChoice.cs new file mode 100644 index 00000000..c2f30372 --- /dev/null +++ b/tests/ParserTests/stack/ManyTerminalChoice.cs @@ -0,0 +1,18 @@ +using System.Collections.Generic; +using System.Linq; +using sly.lexer; +using sly.parser.generator; + +namespace ParserTests.stack; + +public class ManyTerminalChoice +{ + [Production("root : c")] + public string Root(string c) => c; + + [Production("c : [ A | B ]*")] + public string C(List> abs) + { + return string.Join(",", abs.Select(x => x.Value)); + } +} \ No newline at end of file diff --git a/tests/ParserTests/stack/NonTerminalChoice.cs b/tests/ParserTests/stack/NonTerminalChoice.cs new file mode 100644 index 00000000..9124a572 --- /dev/null +++ b/tests/ParserTests/stack/NonTerminalChoice.cs @@ -0,0 +1,19 @@ +using sly.lexer; +using sly.parser.generator; + +namespace ParserTests.stack; + +public class NonTerminalChoice +{ + [Production("root : c")] + public string Root(string c) => c; + + [Production("c : [ a | b ]")] + public string C(string ab) => ab; + + [Production("a : A")] + public string A(Token a) => a.Value; + + [Production("b: B")] + public string B(Token b) => b.Value; +} \ No newline at end of file diff --git a/tests/ParserTests/stack/OptionGroup.cs b/tests/ParserTests/stack/OptionGroup.cs new file mode 100644 index 00000000..6e23177f --- /dev/null +++ b/tests/ParserTests/stack/OptionGroup.cs @@ -0,0 +1,27 @@ +using sly.lexer; +using sly.parser.generator; +using sly.parser.parser; + +namespace ParserTests.stack; + +public class OptionGroup +{ + [Production("root : g")] + public string Root(string g) => g; + + [Production("g : ( a b)? ")] + public string G(ValueOption> optionalGroup) + { + return optionalGroup.Match(grp => + { + return grp.Value(0)+" "+grp.Value(1); + }, + () => "nothing"); + } + + [Production("a : A")] + public string A(Token a) => a.Value; + + [Production("b: B")] + public string B(Token b) => b.Value; +} \ No newline at end of file diff --git a/tests/ParserTests/stack/OptionTerminalChoice.cs b/tests/ParserTests/stack/OptionTerminalChoice.cs new file mode 100644 index 00000000..a9600f95 --- /dev/null +++ b/tests/ParserTests/stack/OptionTerminalChoice.cs @@ -0,0 +1,21 @@ +using sly.lexer; +using sly.parser.generator; + +namespace ParserTests.stack; + +public class OptionTerminalChoice +{ + [Production("root : c")] + public string Root(string c) => c; + + [Production("c : [ A | B ]?")] + public string C(Token ab) + { + if (ab.IsEmpty) + { + return "nothing"; + } + + return ab.Value; + } +} \ No newline at end of file diff --git a/tests/ParserTests/stack/P.cs b/tests/ParserTests/stack/P.cs new file mode 100644 index 00000000..1b8e48a7 --- /dev/null +++ b/tests/ParserTests/stack/P.cs @@ -0,0 +1,10 @@ +using sly.lexer; + +namespace ParserTests.stack; + +public enum P +{ + [Sugar("+")] e, + [Keyword("true")] True, + [Keyword("false")] False, +} \ No newline at end of file diff --git a/tests/ParserTests/stack/SimpleEBNFMany.cs b/tests/ParserTests/stack/SimpleEBNFMany.cs new file mode 100644 index 00000000..eb50f110 --- /dev/null +++ b/tests/ParserTests/stack/SimpleEBNFMany.cs @@ -0,0 +1,33 @@ +using System.Collections.Generic; +using System.Linq; +using sly.lexer; +using sly.parser.generator; + +namespace ParserTests.stack; + +public class SimpleEBNFMany +{ + [Production("root : astar")] + public string Root(string astar) + { + return astar; + } + + [Production("rootplus : aplus")] + public string RootPlus(string aplus) + { + return aplus; + } + + [Production("astar : A*")] + public string Astar(List> all) + { + return string.Join(",",all.Select(x => x.Value)); + } + + [Production("aplus : A+")] + public string Aplus(List> all) + { + return string.Join(",",all.Select(x => x.Value)); + } +} \ No newline at end of file diff --git a/tests/ParserTests/stack/StackParserTests.cs b/tests/ParserTests/stack/StackParserTests.cs new file mode 100644 index 00000000..a901690d --- /dev/null +++ b/tests/ParserTests/stack/StackParserTests.cs @@ -0,0 +1,514 @@ +using System; +using expressionparser; +using NFluent; +using ParserTests.Issue239; +using sly.lexer; +using sly.lexer.fluent; +using sly.parser; +using sly.parser.fluent; +using sly.parser.generator; +using sly.parser.parser; +using sly.parser.syntax.grammar; +using sly.parser.syntax.tree; +using Xunit; + + +namespace ParserTests.stack; + +public class Dumb +{ + +} + +[ParserRoot("root")] +public class SimplerStackParser +{ + [Production("root : expr")] + public string root(string e) => e; + + [Production("expr : INT expr")] + public string expr(Token i, string e) => i.Value + "," + e; + + [Production("expr : INT")] + public string expr2(Token i) => i.Value; + +} + +public class StackParserTests +{ + + [Fact] + public void basic() + { + var instance = new SimplerStackParser(); + ParserBuilder builder = new ParserBuilder(); + var parser = builder.BuildParser(instance, ParserType.LL_STACK, "root"); + Check.That(parser).IsOk(); + var r = parser.Result.Parse("1"); + Check.That(r).IsOkParsing(); + Check.That(r.Result).IsEqualTo("1"); + r = parser.Result.Parse("1 2"); + Check.That(r).IsOkParsing(); + Check.That(r.Result).IsEqualTo("1,2"); + r = parser.Result.Parse("1 2 3 4 5"); + Check.That(r).IsOkParsing(); + Check.That(r.Result).IsEqualTo("1,2,3,4,5"); + } + + [Fact] + public void expression() + { + var instance = new ExpressionParser(); + ParserBuilder builder = new ParserBuilder(); + var parser = builder.BuildParser(instance, ParserType.LL_STACK, "expression"); + Check.That(parser).IsOk(); + var r = parser.Result.Parse("1"); + Check.That(r).IsOkParsing(); + Check.That(r.Result).IsEqualTo(1); + r = parser.Result.Parse("2 + 2"); + Check.That(r).IsOkParsing(); + //Check.That(r.Result).IsEqualTo(4); + r = parser.Result.Parse("1 + 2 + 3 + 4 * 5"); + Check.That(r).IsOkParsing(); + Check.That(r.Result).IsEqualTo(1+2+3+4*5); + } + + [Fact] + public void basicFluent() + { + var lexer = FluentLexerBuilder.NewBuilder() + .Int(ExpressionToken.INT); + var parser = FluentParserBuilder.NewBuilder(new SimplerStackParser(), "root", "en") + .WithLexerbuilder(lexer) + .Production("root : expr", (object[] args) => + { + return (string)args[0]; + }) + .Production("expr : INT", (args) => + { + return ((Token)args[0]).Value; + }) + .Production("expr : INT expr", (args) => + { + return ((Token)args[0]).Value + "," + (string)args[1]; + }) + .BuildParser(ParserType.LL_RECURSIVE_DESCENT); + Check.That(parser).IsOk(); + var result = parser.Result.Parse("1"); + Check.That(result).IsOkParsing(); + Check.That(result.Result).IsEqualTo("1"); + result = parser.Result.Parse("1 2 3 4 5"); + Check.That(result).IsOkParsing(); + Check.That(result.Result).IsEqualTo("1,2,3,4,5"); + + + + } + + [Fact] + public void RuleChoices() + { + var ruleparser = new RuleParser(); + var builder = new ParserBuilder>("en"); + + var grammarParser = builder.BuildParser(ruleparser, ParserType.LL_STACK, "rule"); + string source = "True|False"; + string start = "choices"; + Check.That(grammarParser).IsOk(); + var parser = grammarParser.Result; + var r = parser.Parse(source, start); + Check.That(r).IsOkParsing(); + var actual = r.Result.ToString(); + Check.That(actual).IsEqualTo("[ True(T) | False(T) ]"); + + } + + [Fact] + public void RuleChoices2() + { + var ruleparser = new RuleParser(); + + string source = "[ PLUS | MINUS ]"; + string start = "choiceclause"; + + // + // LL_RECURSIVE => OK => get expected output + // + + var parser = GetParser>(ruleparser, ParserType.LL_RECURSIVE_DESCENT, "rule"); + // string source = "root : A [PLUS|MINUS] B"; + // string start = "rule"; + + var r = parser.Parse(source, start); + Check.That(r).IsOkParsing(); + + var tree = r.SyntaxTree; + Check.That(r.SyntaxTree).IsInstanceOf>>(); + var root = r.SyntaxTree as SyntaxNode>; + Check.That(root).IsNotNull(); + var expected = (r.Result as IClause).Dump(); + + + // + // LL_STACK => get output and compare to expected (if parse succeeded at all) + // + + + parser = GetParser>(ruleparser, ParserType.LL_STACK, "rule"); + + r = parser.Parse(source, start); + Check.That(r).IsOkParsing(); + + var actualTreeDump = root.Dump(" "); + var actual = (r.Result as IClause).Dump(); + Check.That(actual).IsEqualTo(expected); + + } + + [Fact] + public void RuleChoicesRule() + { + var ruleparser = new RuleParser(); + + string source = "rule : A [ PLUS | MINUS ] B"; + string start = "rule"; + + var parser = GetParser>(ruleparser, ParserType.LL_RECURSIVE_DESCENT, "rule"); + + var r = parser.Parse(source, start); + Check.That(r).IsOkParsing(); + + var tree = r.SyntaxTree; + Check.That(r.SyntaxTree).IsInstanceOf>>(); + var root = r.SyntaxTree as SyntaxNode>; + Check.That(root).IsNotNull(); + var expected = (r.Result as Rule).Dump(); + + // LL_STACK => get output and compare to expected (if parse succeeded at all) + + parser = GetParser>(ruleparser, ParserType.LL_STACK, "rule"); + + r = parser.Parse(source, start); + Check.That(r).IsOkParsing(); + + var actualTreeDump = root.Dump(" "); + var actual = (r.Result as Rule).Dump(); + Check.That(actual).IsEqualTo(expected); + + } + + [Fact] + public void ChoicesVisitorTest() + { + var builder = new ParserBuilder("en"); + var instance = new Visitor(); + + string source = "A + B"; + string start = "root"; + RuleParserType.ParserType = ParserType.LL_RECURSIVE_DESCENT; + + var grammarParser = builder.BuildParser(instance, ParserType.EBNF_LL_RECURSIVE_DESCENT, start); + Check.That(grammarParser).IsOk(); + var parser = grammarParser.Result; + + var r = parser.Parse(source,start); + Check.That(r).IsOkParsing(); + string expected = r.Result.ToString(); + + RuleParserType.ParserType = ParserType.LL_STACK; + + grammarParser = builder.BuildParser(instance, ParserType.EBNF_LL_RECURSIVE_DESCENT, start); + Check.That(grammarParser).IsOk(); + parser = grammarParser.Result; + + r = parser.Parse(source, start); + RuleParserType.ParserType = ParserType.LL_RECURSIVE_DESCENT; + Check.That(r).IsOkParsing(); + var actual = r.Result.ToString(); + Check.That(actual).IsEqualTo(expected); + + + } + + [Fact] + public void TestIssue239() + { + string source = "INT[d] ID SEMI[d]"; + string start = "clauses"; + var ruleparser = new RuleParser(); + var parser = GetParser>(ruleparser, ParserType.LL_RECURSIVE_DESCENT, "clauses"); + + var r = parser.Parse(source, start); + Check.That(r).IsOkParsing(); + Check.That(r.Result).IsInstanceOf < ClauseSequence>(); + var rule = r.Result as ClauseSequence; + var expected = rule.Dump(); + + parser = GetParser>(ruleparser, ParserType.LL_STACK, "rule"); + + r = parser.Parse(source, start); + Check.That(r).IsOkParsing(); + Check.That(r.Result).IsInstanceOf < ClauseSequence>(); + rule = r.Result as ClauseSequence; + var actual = rule.Dump(); + Check.That(actual).IsEqualTo(expected); + } + + [Fact] + public void TestEbnfZeroOrMore() + { + var ebnf = new SimpleEBNFMany(); + RuleParserType.ParserType = ParserType.LL_RECURSIVE_DESCENT; + var parser = GetParser(ebnf, ParserType.EBNF_LL_STACK, "root"); + + Check.That(parser).IsNotNull(); + + var parseResult = parser.Parse(" A A A A"); + Check.That(parseResult).IsOkParsing(); + var result = parseResult.Result; + Check.That(result).IsEqualTo("A,A,A,A"); + + parseResult = parser.Parse(" A "); + Check.That(parseResult).IsOkParsing(); + result = parseResult.Result; + Check.That(result).IsEqualTo("A"); + + parseResult = parser.Parse(" "); + Check.That(parseResult).IsOkParsing(); + result = parseResult.Result; + Check.That(result).IsEqualTo(""); + } + + [Fact] + public void TestEbnfOneOrMore() + { + var ebnf = new SimpleEBNFMany(); + RuleParserType.ParserType = ParserType.LL_RECURSIVE_DESCENT; + var parser = GetParser(ebnf, ParserType.EBNF_LL_STACK, "rootplus"); + + Check.That(parser).IsNotNull(); + + var parseResult = parser.Parse(" A A A A"); + Check.That(parseResult).IsOkParsing(); + var result = parseResult.Result; + Check.That(result).IsEqualTo("A,A,A,A"); + + parseResult = parser.Parse(" A "); + Check.That(parseResult).IsOkParsing(); + result = parseResult.Result; + Check.That(result).IsEqualTo("A"); + + parseResult = parser.Parse(" "); + Check.That(parseResult).Not.IsOkParsing(); + Check.That(parseResult.Errors).CountIs(1); + var error = parseResult.Errors[0]; + Check.That(error.ErrorType).IsEqualTo(ErrorType.UnexpectedEOS); + } + + [Fact] + void TestOptionTerminal() + { + var lexer = FluentLexerBuilder.NewBuilder() + .Keyword(L.A, "A") + .Keyword(L.B, "B"); + + var buildResult = FluentEBNFParserBuilder.NewBuilder(new FluentTests(), "root", "en") + .WithLexerbuilder(lexer) + .Production("root : o", (object[] args) => + { + return (string)args[0]; + }) + .Production("o : A B? A", (args) => + { + var a1 = (Token)args[0]; + var b = (Token)args[1]; + var a2 = (Token)args[0]; + if (b.IsEmpty) + { + return "ah ah !"; + } + else + { + return "ABBA"; + } + }) + .BuildParser(ParserType.EBNF_LL_STACK); + + Check.That(buildResult).IsOk(); + var parser = buildResult.Result; + var result = parser.Parse("A A"); + Check.That(result).IsOkParsing(); + Check.That(result.Result).IsEqualTo("ah ah !"); + result = parser.Parse("A B A"); + Check.That(result).IsOkParsing(); + Check.That(result.Result).IsEqualTo("ABBA"); + } + + [Fact] + void TestOptionNonTerminal() + { + var lexer = FluentLexerBuilder.NewBuilder() + .Keyword(L.A, "A") + .Keyword(L.B, "B"); + + var buildResult = FluentEBNFParserBuilder.NewBuilder(new FluentTests(), "root", "en") + .WithLexerbuilder(lexer) + .Production("root : o", (object[] args) => + { + return (string)args[0]; + }) + .Production("o : a b? a", (args) => + { + var a1 = (string)args[0]; + var b = (ValueOption)args[1]; + var a2 = (string)args[0]; + if (b.IsNone) + { + return "ah ah !"; + } + else + { + return "ABBA"; + } + }) + .Production("a : A", (args => + { + return "a"; + })) + .Production("b : B", (args => + { + return "b"; + })) + .BuildParser(ParserType.EBNF_LL_RECURSIVE_DESCENT); + + Check.That(buildResult).IsOk(); + var parser = buildResult.Result; + var result = parser.Parse("A A"); + Check.That(result).IsOkParsing(); + Check.That(result.Result).IsEqualTo("ah ah !"); + result = parser.Parse("A B A"); + Check.That(result).IsOkParsing(); + Check.That(result.Result).IsEqualTo("ABBA"); + } + + public static Parser GetParser(object instance, ParserType type, string root) where IN : struct , Enum + { + ParserBuilder builder = new ParserBuilder(); + var built = builder.BuildParser(instance, type, root); + Check.That(built).IsOk(); + Check.That(built.Result).IsNotNull(); + return built.Result; + } + + + [Fact] + void TestChoiceTerminal() + { + RuleParserType.ParserType = ParserType.LL_RECURSIVE_DESCENT; + var parser = GetParser(new TerminalChoice(), ParserType.EBNF_LL_STACK, "root"); + + Check.That(parser).IsNotNull(); + var result = parser.Parse("A"); + Check.That(result).IsOkParsing(); + Check.That(result.Result).IsEqualTo("A"); + result = parser.Parse("B"); + Check.That(result).IsOkParsing(); + Check.That(result.Result).IsEqualTo("B"); + } + + [Fact] + void TestManyChoiceTerminal() + { + RuleParserType.ParserType = ParserType.LL_RECURSIVE_DESCENT; + var parser = GetParser(new ManyTerminalChoice(), ParserType.EBNF_LL_STACK, "root"); + + Check.That(parser).IsNotNull(); + var result = parser.Parse("A"); + Check.That(result).IsOkParsing(); + Check.That(result.Result).IsEqualTo("A"); + result = parser.Parse("A B B A"); + Check.That(result).IsOkParsing(); + Check.That(result.Result).IsEqualTo("A,B,B,A"); + result = parser.Parse(""); + Check.That(result).IsOkParsing(); + Check.That(result.Result).IsEqualTo(""); + } + + [Fact] + void TestOptionChoiceTerminal() + { + RuleParserType.ParserType = ParserType.LL_RECURSIVE_DESCENT; + var parser = GetParser(new OptionTerminalChoice(), ParserType.EBNF_LL_STACK, "root"); + + Check.That(parser).IsNotNull(); + // var result = parser.Parse("A"); + // Check.That(result).IsOkParsing(); + // Check.That(result.Result).IsEqualTo("A"); + // result = parser.Parse("B"); + // Check.That(result).IsOkParsing(); + // Check.That(result.Result).IsEqualTo("B"); + var result = parser.Parse(""); + Check.That(result).IsOkParsing(); + Check.That(result.Result).IsEqualTo("nothing"); + } + + [Fact] + void TestChoiceNonTerminal() + { + RuleParserType.ParserType = ParserType.LL_RECURSIVE_DESCENT; + var parser = GetParser(new NonTerminalChoice(), ParserType.EBNF_LL_STACK, "root"); + + Check.That(parser).IsNotNull(); + var result = parser.Parse("A"); + Check.That(result).IsOkParsing(); + Check.That(result.Result).IsEqualTo("A"); + result = parser.Parse("B"); + Check.That(result).IsOkParsing(); + Check.That(result.Result).IsEqualTo("B"); + } + + [Fact] + void TestBasicGroup() + { + RuleParserType.ParserType = ParserType.LL_RECURSIVE_DESCENT; + var parser = GetParser(new BasicGroup(), ParserType.EBNF_LL_STACK, "root"); + + Check.That(parser).IsNotNull(); + var result = parser.Parse("A B"); + Check.That(result).IsOkParsing(); + Check.That(result.Result).IsEqualTo("A B"); + result = parser.Parse("A A"); + Check.That(result).Not.IsOkParsing(); + } + + [Fact] + void TestManyGroup() + { + RuleParserType.ParserType = ParserType.LL_RECURSIVE_DESCENT; + var parser = GetParser(new ManyGroup(), ParserType.EBNF_LL_STACK, "root"); + + Check.That(parser).IsNotNull(); + var result = parser.Parse("A B"); + Check.That(result).IsOkParsing(); + Check.That(result.Result).IsEqualTo("A B"); + result = parser.Parse("A B A B"); + Check.That(result).IsOkParsing(); + Check.That(result.Result).IsEqualTo("A B,A B"); + } + + [Fact] + void TestOptionGroup() + { + RuleParserType.ParserType = ParserType.LL_RECURSIVE_DESCENT; + var parser = GetParser(new OptionGroup(), ParserType.EBNF_LL_STACK, "root"); + + Check.That(parser).IsNotNull(); + var result = parser.Parse("A B"); + Check.That(result).IsOkParsing(); + Check.That(result.Result).IsEqualTo("A B"); + result = parser.Parse(""); + Check.That(result).IsOkParsing(); + Check.That(result.Result).IsEqualTo("nothing"); + } + +} \ No newline at end of file diff --git a/tests/ParserTests/stack/TerminalChoice.cs b/tests/ParserTests/stack/TerminalChoice.cs new file mode 100644 index 00000000..1af21194 --- /dev/null +++ b/tests/ParserTests/stack/TerminalChoice.cs @@ -0,0 +1,15 @@ +using sly.lexer; +using sly.parser.generator; +using sly.parser.parser; + +namespace ParserTests.stack; + +public class TerminalChoice +{ + [Production("root : c")] + public string Root(string c) => c; + + [Production("c : [ A | B ]")] + public string C(Token ab) => ab.Value; +} + diff --git a/tests/ParserTests/stack/Visitor.cs b/tests/ParserTests/stack/Visitor.cs new file mode 100644 index 00000000..fe8c6183 --- /dev/null +++ b/tests/ParserTests/stack/Visitor.cs @@ -0,0 +1,32 @@ +using System; +using sly.lexer; +using sly.parser.generator; + +namespace ParserTests.stack; + +public class Visitor : IDisposable { + + public void Dispose() + { + + RuleParserType.ParserType = ParserType.LL_RECURSIVE_DESCENT; + } + + [Production("root : a [PLUS|MINUS] b")] + public string Root(string a, Token op, string b) + { + return "a <" + op.Value + "> b"; + } + + [Production("a : A")] + public string A(Token a) + { + return a.Value; + } + + [Production("b : B")] + public string B(Token b) + { + return b.Value; + } +} \ No newline at end of file