diff --git a/src/Repr/Common/Level.fram b/src/Repr/Common/Level.fram index 1a0b248..e616b6f 100644 --- a/src/Repr/Common/Level.fram +++ b/src/Repr/Common/Level.fram @@ -14,3 +14,37 @@ pub data Level = {## The highest precedence level. ##} | LTop + +{## Equality of levels. ##} +pub method equal (l1 : Level) (l2 : Level) = + match l1, l2 with + | LBot, LBot => True + | LBot, _ => False + | LTop, LTop => True + | LTop, _ => False + | LNum n1, LNum n2 => n1 == n2 + | LNum _, _ => False + end + +{## Strict order on levels. ##} +pub method lt (l1 : Level) (l2 : Level) = + match l1, l2 with + | LBot, LBot => False + | LBot, _ => True + | _, LBot => False + | LTop, _ => False + | _, LTop => True + | LNum n1, LNum n2 => n1 < n2 + end + +{## Non-strict order on levels. ##} +pub method le (l1 : Level) (l2 : Level) = + l1 == l2 || l1 < l2 + +{## Convert level to string. ##} +pub method toString (level : Level) = + match level with + | LBot => "bot" + | LTop => "top" + | LNum n => n.toString + end diff --git a/src/Transform/SplitNTermLevels.fram b/src/Transform/SplitNTermLevels.fram index b84c9e5..4e133e4 100644 --- a/src/Transform/SplitNTermLevels.fram +++ b/src/Transform/SplitNTermLevels.fram @@ -9,8 +9,189 @@ of the original non-terminal. Such non-terminals contain only productions of the corresponding level and a single production that calls the next level. ##} import open Repr/RichGrammar +import List +import Map +import Utils/UID + +let (Map { module LevelMap }) = Map.make { Key = Level } +let (Map { module TagMap }) = Map.make { Key = String } + +type LevelSet = LevelMap.T Unit +type TagSet = TagMap.T Unit + +let collectLevels (prods : List NTermProd) = + let levelSet = + List.foldLeft + (fn (levelSet : LevelSet) (prod : NTermProd) => + levelSet.add prod.level ()) + LevelMap.empty + prods + in + levelSet.fold (fn {key} _ levels => key :: levels) [] >.rev + +let levelProds (level : Level) (prods : List NTermProd) = + List.filter + (fn (prod : NTermProd) => prod.level == level) + prods + +type ProdsByLevel = LevelMap.T (List NTermProd) + +let addProdToLevel (prodsByLevel : ProdsByLevel) (prod : NTermProd) = + let prods = + prodsByLevel.find prod.level >.unwrapOr [] + in + prodsByLevel.add prod.level (prod :: prods) + +let prodsByLevel prods = + List.foldLeft addProdToLevel LevelMap.empty prods + >.map (fn {key} prods => prods.rev) + +let lookupProds (level : Level) (prodsByLevel : ProdsByLevel) = + prodsByLevel.findErr + {~onError = fn _ => impossible ()} + level + +type LevelIds = List (Pair Level NTermId) +type NTermLevelIds = Pair String LevelIds +type IdTable = NTermMap.T NTermLevelIds + +parameter ~onError + +let lookupNTermLevelIds (idTable : IdTable) (id : NTermId) = + idTable.findErr + {~onError = fn _ => impossible ()} + id + +let lookupLevelIds idTable id = + snd (lookupNTermLevelIds idTable id) + +let invalidLevelError (ntName : String) (level : Level) = + "Requested level " + level.toString + + " for non-terminal " + ntName + + " is stronger than every production level" + +let rec pickLevelId ntName (level : Level) (levelIds : LevelIds) = + match levelIds with + | [] => impossible () + | (level', id) :: [] => + if level <= level' then id else ~onError (invalidLevelError ntName level) + | (level', id) :: rest => + if level <= level' then id else pickLevelId ntName level rest + end + +let lookupNTermId (idTable : IdTable) id level = + let (ntName, levelIds) = lookupNTermLevelIds idTable id in + pickLevelId ntName level levelIds + +let rewriteSymbol (idTable : IdTable) sym = + match sym with + | PS_Token _ => sym + | PS_NTerm {var, level} id => + PS_NTerm {var, level} (lookupNTermId idTable id level) + end + +let rewriteProd (idTable : IdTable) (NTermProd {module Prod}) = + NTermProd + { module Prod + , symbols = List.map (rewriteSymbol idTable) Prod.symbols + } + +let collectTags (tags : List Tag) = + let tagSet = + List.foldLeft + (fn (tagSet : TagSet) tag => tagSet.add tag ()) + TagMap.empty + tags + in + tagSet.fold (fn {key} _ tags => key :: tags) [] >.rev + +let tagsForLevels var levels prodsByLevel = + levels + |> List.concatMap + (fn level => + lookupProds level prodsByLevel + |> List.concatMap (fn (prod : NTermProd) => prod.tags)) + |> List.map (fn ((tag, _) : Pair Tag (TagCond Var)) => tag) + |> collectTags + |> List.map (fn tag => (tag, TC_Tag tag var)) + +let forwardVar (nt : NTerm) = + Var {id = UID.fresh (), name = None, typ = nt.valueType} + +let forwardProd (nt : NTerm) level nextLevel nextId tagLevels prodsByLevel = + let var = forwardVar nt in + NTermProd + { symbols = + [PS_NTerm {var, level = nextLevel} nextId] + , level + , tags = tagsForLevels var tagLevels prodsByLevel + , unless = TC_False + , action = AForward + } + +let splitNTermName (nt : NTerm) (id : NTermId) (level : Level) = + if id == nt.id then nt.name else nt.name + "@" + level.toString + +let rec splitLevels + (idTable : IdTable) + (nt : NTerm) + (prodsByLevel : ProdsByLevel) + (levelIds : LevelIds) = + match levelIds with + | [] => [] + | (level, id) :: rest => + let prods = + lookupProds level prodsByLevel + |> List.map (rewriteProd idTable) + let prods = + match rest with + | [] => prods + | (nextLevel, nextId) :: _ => + prods + + [forwardProd + nt level nextLevel nextId (List.map fst rest) prodsByLevel] + end + in + NTerm + { id + , name = splitNTermName nt id level + , valueType = nt.valueType + , prods + } :: splitLevels idTable nt prodsByLevel rest + end + +let splitNTerm (idTable : IdTable) (nt : NTerm) = + match lookupLevelIds idTable nt.id with + | [] => [nt] + | levelIds => + let prodsByLevel = prodsByLevel nt.prods in + splitLevels idTable nt prodsByLevel levelIds + end + +let freshLevelIds (nt : NTerm) levels = + match levels with + | [] => [] + | level :: levels => + (level, nt.id) :: + List.map (fn level => (level, NTermId.fresh ())) levels + end + +let addNTermLevelIds (idTable : IdTable) (nt : NTerm) = + idTable.add nt.id (nt.name, freshLevelIds nt (collectLevels nt.prods)) + +pub let transformErr (RichGrammar {module G}) : RichGrammar = + let idTable = + List.foldLeft addNTermLevelIds NTermMap.empty G.nterms + in + RichGrammar + { module G + , nterms = + G.nterms + |> List.concatMap (splitNTerm idTable) + } {## Split each non-terminal into multiple levels. ##} -pub let transform (g : RichGrammar) : RichGrammar = - # TODO: Implement this function. - g +pub let transform g : RichGrammar = + transformErr + {~onError = fn msg => runtimeError ("SplitNTermLevels: " + msg)} + g diff --git a/test.sh b/test.sh new file mode 100755 index 0000000..5809704 --- /dev/null +++ b/test.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +set -u + +if [ $# -ne 1 ]; then + echo "USAGE: ./test.sh TEST_SUITE" + exit 1 +fi + +if [ ! -f "$1" ]; then + echo "ERROR: test suite file not found: $1" + exit 1 +fi + +if [ ! -r "$1" ]; then + echo "ERROR: test suite file is not readable: $1" + exit 1 +fi + +binary="${DBL:-dbl}" +if ! command -v "$binary" > /dev/null; then + echo "ERROR: dbl executable not found in PATH" + exit 1 +fi + +if [ -z "${DBL_LIB:-}" ]; then + dbl_path=$(command -v "$binary") + dbl_prefix=$(dirname "$(dirname "$dbl_path")") + if [ -d "$dbl_prefix/lib/dbl/stdlib" ]; then + export DBL_LIB="$dbl_prefix/lib/dbl/stdlib" + else + echo "ERROR: DBL_LIB is not set and DBL stdlib was not found next to '$dbl_path'" + exit 1 + fi +fi + +flags="" +total_tests=0 +passed_tests=0 + +function simple_test { + total_tests=$((total_tests + 1)) + + local file="$1" + local cmd=("$binary") + if [ -n "$flags" ]; then + # shellcheck disable=SC2206 + cmd+=($flags) + fi + cmd+=("$file") + + echo "${cmd[*]}" + if "${cmd[@]}"; then + passed_tests=$((passed_tests + 1)) + else + echo "Test file failed: $file" + fi +} + +function run_with_flags { + local flags="$2" + "$1" +} + +source "$1" + +echo "Passed: ${passed_tests}/${total_tests}" + +if [ "$passed_tests" -eq "$total_tests" ]; then + exit 0 +else + exit 1 +fi diff --git a/test/TestAll.fram b/test/TestAll.fram new file mode 100644 index 0000000..fa057c0 --- /dev/null +++ b/test/TestAll.fram @@ -0,0 +1 @@ +import TransformTests/SplitNTermLevels diff --git a/test/TransformTests/SplitNTermLevels.fram b/test/TransformTests/SplitNTermLevels.fram new file mode 100644 index 0000000..656aeeb --- /dev/null +++ b/test/TransformTests/SplitNTermLevels.fram @@ -0,0 +1,327 @@ +{# This file is part of FramYard, released under MIT license. + See LICENSE for details. + #} + +import open Testing +import /List +import open Repr/RichGrammar +import Transform/SplitNTermLevels +import Utils/UID + +let ~onError _ = + runtimeError "Unhandled ~onError in SplitNTermLevels test module!" + +let pos = + Position + { fname = "" + , line = 1 + , cnum = 0 + , bol = 0 + , length = 0 + } + +let action name = ACode {pos, code = name} + +let tokenProdWithTags level token tags = + NTermProd + { symbols = [PS_Token {var = None} token] + , level + , tags + , unless = TC_False + , action = action token + } + +let tokenProd level token = + tokenProdWithTags level token [] + +let var name = + Var {id = UID.fresh (), name = Some name, typ = TToken "Expr"} + +let exprSym id level name = + PS_NTerm {var = var name, level} id + +let infixProdWithTags level leftLevel token rightLevel tags exprId = + NTermProd + { symbols = + [ exprSym exprId leftLevel "lhs" + , PS_Token {var = None} token + , exprSym exprId rightLevel "rhs" + ] + , level + , tags + , unless = TC_False + , action = action token + } + +let parenProd exprId = + NTermProd + { symbols = + [ PS_Token {var = None} "LPAREN" + , exprSym exprId (LNum 0) "inner" + , PS_Token {var = None} "RPAREN" + ] + , level = LTop + , tags = [] + , unless = TC_False + , action = action "paren" + } + +let mkNTerm id name prods = + NTerm {id, name, valueType = TToken name, prods} + +let mkGrammar nterms = + RichGrammar + { tokens = [] + , datatypes = [] + , typeDefs = [] + , nterms + , treeSymbols = [] + , preambleCode = [] + } + +let isForwardProd (NTermProd {symbols, tags, unless, action}) = + match (symbols, unless, action) with + | ([PS_NTerm _], TC_False, AForward) => True + | _ => False + end + +let sameVar (v1 : Var) (v2 : Var) = + v1.id == v2.id + +let hasForwardedTag tag (prod : NTermProd) = + let tagUsesVar var = + List.exists + (fn ((prodTag, cond) : Pair Tag (TagCond Var)) => + prodTag == tag && + match cond with + | TC_Tag tag' var' => tag' == tag && sameVar var var' + | _ => False + end) + prod.tags + in + match prod.symbols with + | [PS_NTerm {var} _] => tagUsesVar var + | _ => False + end + +let countProds (f : NTermProd ->[] Bool) (nterms : List NTerm) = + List.foldLeft + (fn acc (NTerm {prods}) => + acc + List.length (List.filter f prods)) + 0 + nterms + +let allProds (nterms : List NTerm) = + List.concatMap (fn (NTerm {prods}) => prods) nterms + +let hasProdAtLevel (level : Level) (nterm : NTerm) = + List.exists + (fn (prod : NTermProd) => + prod.level == level && not (isForwardProd prod)) + nterm.prods + +let countNTermsWithLevel (level : Level) (nterms : List NTerm) = + List.length (List.filter (hasProdAtLevel level) nterms) + +let hasTaggedForward tag (nterm : NTerm) = + List.exists + (fn (prod : NTermProd) => isForwardProd prod && hasForwardedTag tag prod) + nterm.prods + +let countForwardedTag tag (prod : NTermProd) = + match prod.symbols with + | [PS_NTerm {var} _] => + List.length + (List.filter + (fn ((prodTag, cond) : Pair Tag (TagCond Var)) => + prodTag == tag && + match cond with + | TC_Tag tag' var' => tag' == tag && sameVar var var' + | _ => False + end) + prod.tags) + | _ => 0 + end + +let countForwardedTags tag nterms = + nterms + |> allProds + |> List.filter isForwardProd + |> List.foldLeft + (fn count (prod : NTermProd) => count + countForwardedTag tag prod) + 0 + +let tagSourceVar = var "tag-source" + +let sampleGrammar () = + let exprId = NTermId.fresh () in + let expr = + mkNTerm exprId "Expr" + [ infixProdWithTags + (LNum 0) + (LNum 0) + "PLUS" + (LNum 10) + [] + exprId + , infixProdWithTags + (LNum 10) + (LNum 10) + "STAR" + LTop + [("multiplicative", TC_Tag "multiplicative" tagSourceVar)] + exprId + , tokenProdWithTags LTop "NUMBER" [("atomic", TC_True)] + , parenProd exprId + ] + in + mkGrammar [expr] + +let sample = sampleGrammar () + +let transformedSample = + SplitNTermLevels.transform sample + +let sampleExprId = + match sample.nterms with + | [nt] => nt.id + | _ => impossible () + end + +let emptyGrammar () = + mkGrammar [mkNTerm (NTermId.fresh ()) "Empty" []] + +let emptySample = + emptyGrammar () + +let transformedEmptySample = + SplitNTermLevels.transform emptySample + +let duplicateTagGrammar () = + let exprId = NTermId.fresh () in + mkGrammar + [ mkNTerm exprId "Expr" + [ tokenProd (LNum 0) "LOW" + , tokenProdWithTags (LNum 10) "HIGH_A" [("dup", TC_True)] + , tokenProdWithTags (LNum 10) "HIGH_B" [("dup", TC_True)] + ] + ] + +let duplicateTagSample = + duplicateTagGrammar () + +let transformedDuplicateTagSample = + SplitNTermLevels.transform duplicateTagSample + +let invalidLevelGrammar () = + let exprId = NTermId.fresh () in + mkGrammar + [ mkNTerm exprId "Expr" + [ NTermProd + { symbols = [exprSym exprId (LNum 21) "bad"] + , level = LNum 20 + , tags = [] + , unless = TC_False + , action = action "bad-level" + } + ] + ] + +let invalidLevelSample = + invalidLevelGrammar () + +let invalidLevelReportsError = + handle + ~onError = effect _ / _ => True + return _ => False + in + SplitNTermLevels.transformErr invalidLevelSample + +let _ = + +testSuite "SplitNTermLevels" (fn _ => + + testCase "creates one non-terminal per production level" (fn _ => + let result = transformedSample in + assertEq 3 (List.length result.nterms); + expectEq 1 (countNTermsWithLevel (LNum 0) result.nterms); + expectEq 1 (countNTermsWithLevel (LNum 10) result.nterms); + expectEq 1 (countNTermsWithLevel LTop result.nterms)); + + testCase "keeps original id and name on the lowest level" (fn _ => + let result = transformedSample in + let originalNTerms = + List.filter (fn (nt : NTerm) => nt.id == sampleExprId) result.nterms + in + assertEq 1 (List.length originalNTerms); + match originalNTerms with + | [nt] => + expectEq "Expr" nt.name; + assertTrue (hasProdAtLevel (LNum 0) nt) + | _ => assertTrue False + end); + + testCase "keeps original productions only on their own levels" (fn _ => + let result = transformedSample in + let nonForwardProds = + allProds result.nterms |> List.filter (fn prod => not (isForwardProd prod)) + in + assertEq 4 (List.length nonForwardProds); + expectEq 1 + (List.length + (List.filter + (fn (prod : NTermProd) => prod.level == LNum 0) + nonForwardProds)); + expectEq 1 + (List.length + (List.filter + (fn (prod : NTermProd) => prod.level == LNum 10) + nonForwardProds)); + expectEq 2 + (List.length + (List.filter + (fn (prod : NTermProd) => prod.level == LTop) + nonForwardProds))); + + testCase "adds forwarding productions between adjacent levels" (fn _ => + let result = transformedSample in + assertEq 2 (countProds isForwardProd result.nterms)); + + testCase "forwards semantic action and tags from the level above" (fn _ => + let result = transformedSample in + let bottomLevelWithTaggedForward = + result.nterms + |> List.filter + (fn (nterm : NTerm) => + hasProdAtLevel (LNum 0) nterm && + hasTaggedForward "multiplicative" nterm) + in + assertEq 1 (List.length bottomLevelWithTaggedForward)); + + testCase "forwards tags through multiple levels" (fn _ => + let result = transformedSample in + let bottomLevelWithTaggedForward = + result.nterms + |> List.filter + (fn (nterm : NTerm) => + hasProdAtLevel (LNum 0) nterm && + hasTaggedForward "atomic" nterm) + in + assertEq 1 (List.length bottomLevelWithTaggedForward)); + + testCase "preserves non-terminals without productions" (fn _ => + let result = transformedEmptySample in + assertEq 1 (List.length result.nterms); + match result.nterms with + | [nt] => + expectEq "Empty" nt.name; + expectEq 0 (List.length nt.prods) + | _ => assertTrue False + end); + + testCase "deduplicates forwarded tags" (fn _ => + let result = transformedDuplicateTagSample in + assertEq 1 (countForwardedTags "dup" result.nterms)); + + testCase "reports references above the strongest production level" (fn _ => + assertTrue invalidLevelReportsError)) diff --git a/test/test_suite b/test/test_suite new file mode 100644 index 0000000..38dc532 --- /dev/null +++ b/test/test_suite @@ -0,0 +1,5 @@ +function framyard_tests { + simple_test test/TestAll.fram +} + +run_with_flags framyard_tests "-L src -L test"