diff --git a/org.eclipse.xtext.tests/src/org/eclipse/xtext/xtext/generator/hoisting/HoistingProcessorTest.java b/org.eclipse.xtext.tests/src/org/eclipse/xtext/xtext/generator/hoisting/HoistingProcessorTest.java index 9980a8b15..1894885c2 100644 --- a/org.eclipse.xtext.tests/src/org/eclipse/xtext/xtext/generator/hoisting/HoistingProcessorTest.java +++ b/org.eclipse.xtext.tests/src/org/eclipse/xtext/xtext/generator/hoisting/HoistingProcessorTest.java @@ -523,6 +523,27 @@ public class HoistingProcessorTest extends AbstractXtextTests { assertTrue(guard.hasTerminal()); assertEquals("((" + getSyntaxForKeywordToken("a", 1) + " || (p0)) && (" + getSyntaxForKeywordToken("b", 1) + " || (p1)))", guard.render()); } + + @Test + public void testAlternativesWithPrefixPathsAndNonEofContext_expectGuardBasedOnContext() throws Exception { + // @formatter:off + String model = + MODEL_PREAMBLE + + "S: a=A 's' ;\n" + + "A: {A} $$ p0 $$?=> 'a' \n" + + " | {A} $$ p1 $$?=> 'a' 'b' ;"; + // @formatter:off + XtextResource resource = getResourceFromString(model); + Grammar grammar = ((Grammar) resource.getContents().get(0)); + hoistingProcessor.init(grammar); + AbstractRule rule = getRule(grammar, "A"); + + HoistingGuard guard = hoistingProcessor.findHoistingGuard(rule.getAlternatives()); + assertFalse(guard.isTrivial()); + assertTrue(guard.hasTerminal()); + System.out.println(guard.toString()); + assertEquals("((" + getSyntaxForKeywordToken("s", 2) + " || (p0)) && (" + getSyntaxForKeywordToken("b", 2) + " || (p1)))", guard.render()); + } @Test public void testAlternativeWithDifferentEnumRule() throws Exception { diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/guards/AlternativeTokenSequenceGuard.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/guards/AlternativeTokenSequenceGuard.java index 3cfeae55e..8f9829efd 100644 --- a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/guards/AlternativeTokenSequenceGuard.java +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/guards/AlternativeTokenSequenceGuard.java @@ -8,6 +8,7 @@ *******************************************************************************/ package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.guards; +import java.util.Arrays; import java.util.Collection; import java.util.stream.Collectors; @@ -40,5 +41,18 @@ public class AlternativeTokenSequenceGuard implements TokenGuard { return result; } + + @Override + public String toString() { + return "AlternativeTokenSequenceGuard (\n" + + alternatives.stream() + .map(Object::toString) + .map(s -> Arrays.stream(s.split("\n")) + .map(l -> "\t" + l) + .collect(Collectors.joining("\n")) + ).map(s -> s + "\n") + .collect(Collectors.joining("\n")) + + ")\n"; + } } diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/guards/AlternativesGuard.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/guards/AlternativesGuard.java index e687ba91d..5fca19f52 100644 --- a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/guards/AlternativesGuard.java +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/guards/AlternativesGuard.java @@ -8,6 +8,7 @@ *******************************************************************************/ package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.guards; +import java.util.Arrays; import java.util.LinkedList; import java.util.List; import java.util.function.Predicate; @@ -69,4 +70,17 @@ public class AlternativesGuard implements HoistingGuard { Collector.Characteristics.CONCURRENT ); } + + @Override + public String toString() { + return "AlternativesGuard (\n" + + paths.stream() + .map(Object::toString) + .map(s -> Arrays.stream(s.split("\n")) + .map(l -> "\t" + l) + .collect(Collectors.joining("\n")) + ).map(s -> s + "\n") + .collect(Collectors.joining("\n")) + + ")\n"; + } } diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/guards/GroupGuard.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/guards/GroupGuard.java index 001b34395..ed6c9da61 100644 --- a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/guards/GroupGuard.java +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/guards/GroupGuard.java @@ -8,6 +8,7 @@ *******************************************************************************/ package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.guards; +import java.util.Arrays; import java.util.Collection; import java.util.LinkedList; import java.util.List; @@ -85,5 +86,18 @@ public class GroupGuard implements HoistingGuard { public boolean hasTerminal() { return hasTerminal; } + + @Override + public String toString() { + return "GroupGuard (\n" + + elementGuards.stream() + .map(Object::toString) + .map(s -> Arrays.stream(s.split("\n")) + .map(l -> "\t" + l) + .collect(Collectors.joining("\n")) + ).map(s -> s + "\n") + .collect(Collectors.joining("\n")) + + ")\n"; + } } diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/guards/MergedPathGuard.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/guards/MergedPathGuard.java index 903c6f192..29524e93f 100644 --- a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/guards/MergedPathGuard.java +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/guards/MergedPathGuard.java @@ -8,6 +8,7 @@ *******************************************************************************/ package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.guards; +import java.util.Arrays; import java.util.LinkedList; import java.util.List; import java.util.stream.Collectors; @@ -61,5 +62,17 @@ public class MergedPathGuard implements HoistingGuard { // only need to check first element since all paths should be identical return pathGuards.get(0).hasTerminal(); } - + + @Override + public String toString() { + return "MergedPathGuard (\n" + + pathGuards.stream() + .map(Object::toString) + .map(s -> Arrays.stream(s.split("\n")) + .map(l -> "\t" + l) + .collect(Collectors.joining("\n")) + ).map(s -> s + "\n") + .collect(Collectors.joining("\n")) + + ")\n"; + } } diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/guards/PathGuard.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/guards/PathGuard.java index 04327268f..1eccccfc9 100644 --- a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/guards/PathGuard.java +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/guards/PathGuard.java @@ -8,8 +8,10 @@ *******************************************************************************/ package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.guards; +import java.util.Arrays; import java.util.LinkedList; import java.util.List; +import java.util.stream.Collectors; import org.eclipse.xtext.util.Pair; @@ -83,4 +85,16 @@ public class PathGuard implements HoistingGuard { return result; } + + @Override + public String toString() { + return "PathGuard (\n" + + "\ttokenGuard:\n\t" + Arrays.stream(tokenGuard.toString().split("\n")) + .map(l -> "\t" + l) + .collect(Collectors.joining("\n")) + "\n" + + "\thoistngGuard:\n\t" + Arrays.stream(hoistngGuard.toString().split("\n")) + .map(l -> "\t" + l) + .collect(Collectors.joining("\n")) + "\n" + + ")\n"; + } } diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/guards/PredicateGuard.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/guards/PredicateGuard.java index 58b5ccbbc..7eb61c119 100644 --- a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/guards/PredicateGuard.java +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/guards/PredicateGuard.java @@ -37,4 +37,11 @@ public class PredicateGuard implements HoistingGuard { return false; } + @Override + public String toString() { + return "PredicateGuard (\n" + + "\t" + JavaCodeUtils.getSource(element.getCode()).trim() + "\n" + + ")\n"; + } + } diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/guards/SingleTokenGuard.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/guards/SingleTokenGuard.java index 8bee003c1..30f285790 100644 --- a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/guards/SingleTokenGuard.java +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/guards/SingleTokenGuard.java @@ -24,4 +24,11 @@ public class SingleTokenGuard implements TokenGuard { public String render() { return token.negatedCondition(); } + + @Override + public String toString() { + return "SingleTokenGuard (\n" + + "\t" + token + "\n" + + ")\n"; + } } diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/guards/TokenSequenceGuard.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/guards/TokenSequenceGuard.java index b17cb2fc9..ecd860369 100644 --- a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/guards/TokenSequenceGuard.java +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/guards/TokenSequenceGuard.java @@ -8,6 +8,7 @@ *******************************************************************************/ package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.guards; +import java.util.Arrays; import java.util.Collection; import java.util.stream.Collectors; @@ -40,4 +41,17 @@ public class TokenSequenceGuard implements TokenGuard { return result; } + + @Override + public String toString() { + return "TokenSequenceGuard (\n" + + sequence.stream() + .map(Object::toString) + .map(s -> Arrays.stream(s.split("\n")) + .map(l -> "\t" + l) + .collect(Collectors.joining("\n")) + ).map(s -> s + "\n") + .collect(Collectors.joining("\n")) + + ")\n"; + } } diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/pathAnalysis/TokenAnalysis.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/pathAnalysis/TokenAnalysis.java index ae9f27cb7..6c9a30c3e 100644 --- a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/pathAnalysis/TokenAnalysis.java +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/pathAnalysis/TokenAnalysis.java @@ -56,28 +56,39 @@ public class TokenAnalysis { this.grammar = grammar; } + private AbstractElement getContainer(AbstractElement element) { + EObject tmp = element.eContainer(); + while (!(tmp instanceof AbstractElement)) { + if (tmp == null) { + return null; + } + tmp = tmp.eContainer(); + } + return (AbstractElement) tmp; + } + private CompoundElement getCompoundContainer(AbstractElement element) { if (element instanceof CompoundElement) { // get container of compoundElement since getContainerOfType // would return the same element - EObject tmp = element.eContainer(); - while (!(tmp instanceof AbstractElement)) { - if (tmp == null) { - return null; - } - tmp = tmp.eContainer(); + element = getContainer(element); + if (element == null) { + return null; } - element = (AbstractElement) tmp; } return getContainerOfType(element, CompoundElement.class); } - private List getNextElementsInContext(AbstractElement last) { - + private List getNextElementsInContext(AbstractElement last) { CompoundElement container = getCompoundContainer(last); - while (container instanceof Alternatives) { + while (container instanceof Alternatives || + last.eContainer() instanceof Assignment + ) { // skip alternatives since they have to be covered separately - last = container; + last = getContainer(last); + if (last == null) { + return Arrays.asList((AbstractElement) null); + } container = getCompoundContainer(last); } @@ -89,7 +100,11 @@ public class TokenAnalysis { } else if (container instanceof Group) { List elements = container.getElements(); int index = elements.indexOf(last); - log.info(index); + if (index < 0) { + log.error("context analysis: element not part of compound"); + log.info(last.eClass().getName()); + log.info(abstractElementToString(container)); + } if (index < elements.size() - 1) { return Arrays.asList(elements.get(index + 1)); } else { @@ -98,7 +113,7 @@ public class TokenAnalysis { } } else if (container == null) { // end of rule - AbstractRule rule = containingRule(last); + AbstractRule rule = containingRule(last); List calls = findAllRuleCalls(grammar, rule); if (calls.isEmpty()) { @@ -121,7 +136,7 @@ public class TokenAnalysis { private TokenAnalysisPaths getTokenPathsContext(AbstractElement last, TokenAnalysisPaths prefix) { List context = getNextElementsInContext(last); - + TokenAnalysisPaths result = TokenAnalysisPaths.empty(prefix); if (context.isEmpty()) { @@ -129,10 +144,10 @@ public class TokenAnalysis { throw new TokenAnalysisAbortedException("context analysis failed: no context"); } - for (AbstractElement element : context) { + for (AbstractElement element : context) { TokenAnalysisPaths path = new TokenAnalysisPaths(prefix); path = getTokenPaths(element, path, false, false); - if (!path.isDone()) { + if (!path.isDone() && element != null) { path = getTokenPathsContext(element, path); } if (path.isDone()) { @@ -265,8 +280,6 @@ public class TokenAnalysis { do { TokenAnalysisPaths tokenPaths = getTokenPathsTrivial(path, result); - result = result.merge(tokenPaths); - if (tokenPaths.isDone()) { result = result.merge(tokenPaths); break; diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/EofToken.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/EofToken.java index 4fac9ae77..3a3e6b2a7 100644 --- a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/EofToken.java +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/EofToken.java @@ -45,5 +45,10 @@ public class EofToken implements Token { return false; return true; } + + @Override + public String toString() { + return "EofToken(" + position + ")\n"; + } } diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/KeywordToken.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/KeywordToken.java index f56a2c8c2..8a67c4c77 100644 --- a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/KeywordToken.java +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/KeywordToken.java @@ -29,7 +29,7 @@ public class KeywordToken implements Token { @Override public String toString() { - return "keyword " + keyword.getValue(); + return "keywordToken(" + keyword.getValue() + ", " + position + ")\n"; } @Override diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/TerminalRuleToken.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/TerminalRuleToken.java index 99a82b1cf..5611cebfb 100644 --- a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/TerminalRuleToken.java +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/TerminalRuleToken.java @@ -29,7 +29,7 @@ public class TerminalRuleToken implements Token { @Override public String toString() { - return "terminal " + rule.getName(); + return "TerminalToken(" + rule.getName() + ", " + position + ")\n"; } @Override