From 0a6b1b69c9101dff4cc0cdbb4370bd46440cc6a6 Mon Sep 17 00:00:00 2001 From: overflowerror Date: Thu, 9 Dec 2021 19:30:19 +0100 Subject: [PATCH] added support for context analysis with optional cardinalities unordered groups not yet working properly --- .../hoisting/HoistingProcessorTest.java | 41 ++++++++++--- .../antlr/hoisting/HoistingProcessor.java | 60 ++++++++++++------- .../hoisting/guards/AlternativesGuard.java | 6 +- .../hoisting/pathAnalysis/TokenAnalysis.java | 33 ++++++++++ 4 files changed, 112 insertions(+), 28 deletions(-) diff --git a/org.eclipse.xtext.tests/src/org/eclipse/xtext/xtext/generator/hoisting/HoistingProcessorTest.java b/org.eclipse.xtext.tests/src/org/eclipse/xtext/xtext/generator/hoisting/HoistingProcessorTest.java index 1894885c2..ed6a3ac2f 100644 --- a/org.eclipse.xtext.tests/src/org/eclipse/xtext/xtext/generator/hoisting/HoistingProcessorTest.java +++ b/org.eclipse.xtext.tests/src/org/eclipse/xtext/xtext/generator/hoisting/HoistingProcessorTest.java @@ -291,8 +291,8 @@ public class HoistingProcessorTest extends AbstractXtextTests { assertEquals("((" + getSyntaxForKeywordToken("s", 1) + " || (p1)) && (" + getSyntaxForKeywordToken("a", 1) + " || (p0)))", guard.render()); } - @Test(expected = UnsupportedConstructException.class) - public void testCardinalityQuestionmarkWithoutContext_expectUnsupportedConstruct() throws Exception { + @Test + public void testCardinalityQuestionmarkWithoutContext_expectNoContextCheck() throws Exception { // @formatter:off String model = MODEL_PREAMBLE + @@ -303,9 +303,31 @@ public class HoistingProcessorTest extends AbstractXtextTests { hoistingProcessor.init(grammar); AbstractRule rule = getRule(grammar, "S"); - hoistingProcessor.findHoistingGuard(rule.getAlternatives()); + HoistingGuard guard = hoistingProcessor.findHoistingGuard(rule.getAlternatives()); + assertFalse(guard.isTrivial()); + assertTrue(guard.hasTerminal()); + assertEquals("(" + getSyntaxForKeywordToken("a", 1) + " || (p0))", guard.render()); } + @Test + public void testCardinalityQuestionmarkWithExternalContext_expectContextCheck() throws Exception { + // @formatter:off + String model = + MODEL_PREAMBLE + + "S: A 'a' 'b';\n" + + "A: ($$ p0 $$?=> 'a')?;"; + // @formatter:off + XtextResource resource = getResourceFromString(model); + Grammar grammar = ((Grammar) resource.getContents().get(0)); + hoistingProcessor.init(grammar); + AbstractRule rule = getRule(grammar, "A"); + + HoistingGuard guard = hoistingProcessor.findHoistingGuard(rule.getAlternatives()); + assertFalse(guard.isTrivial()); + assertTrue(guard.hasTerminal()); + assertEquals("(" + getSyntaxForKeywordToken("a", 1) + " || (p0))", guard.render()); + } + @Test public void testCardinalityStarWithContext() throws Exception { // @formatter:off @@ -341,8 +363,8 @@ public class HoistingProcessorTest extends AbstractXtextTests { assertFalse(guard.hasTerminal()); } - @Test(expected = UnsupportedConstructException.class) - public void testCardinalityStarWithoutContext_expectUnsupporedConstruct() throws Exception { + @Test + public void testCardinalityStarWithoutContext_expectContextCheck() throws Exception { // @formatter:off String model = MODEL_PREAMBLE + @@ -353,7 +375,10 @@ public class HoistingProcessorTest extends AbstractXtextTests { hoistingProcessor.init(grammar); AbstractRule rule = getRule(grammar, "S"); - hoistingProcessor.findHoistingGuard(rule.getAlternatives()); + HoistingGuard guard = hoistingProcessor.findHoistingGuard(rule.getAlternatives()); + assertFalse(guard.isTrivial()); + assertTrue(guard.hasTerminal()); + assertEquals("(" + getSyntaxForKeywordToken("a", 1) + " || (p0))", guard.render()); } @@ -410,7 +435,8 @@ public class HoistingProcessorTest extends AbstractXtextTests { assertEquals("((" + getSyntaxForKeywordToken("a", 1) + " || (p0)) && (" + getSyntaxForKeywordToken("b", 1) + " || (p1)))", guard.render()); } - @Test(expected = UnsupportedConstructException.class) + // TODO: this is no longer unsupported + // @Test(expected = UnsupportedConstructException.class) public void testUnorderedGroupWithEmptyPathsWithoutContext_expectUnsupportedConstruct() throws Exception { // @formatter:off String model = @@ -443,6 +469,7 @@ public class HoistingProcessorTest extends AbstractXtextTests { assertEquals("((" + getSyntaxForKeywordToken("s", 1) + " || (p2)) && (" + getSyntaxForKeywordToken("a", 1) + " || (p0)) && (" + getSyntaxForKeywordToken("b", 1) + " || (p1)))", guard.render()); } + // TODO: this is no longer unsupported @Test(expected = UnsupportedConstructException.class) public void testUnorderedGroupWithoutMandatoryContentWithoutContext_expectUnsupportedConstruct() throws Exception { // @formatter:off diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/HoistingProcessor.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/HoistingProcessor.java index 366c8b59d..3d8c05739 100644 --- a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/HoistingProcessor.java +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/HoistingProcessor.java @@ -70,7 +70,45 @@ public class HoistingProcessor { analysis = new TokenAnalysis(config, grammar); } - // TODO: handling for TokenAnalysisAbortedException + private HoistingGuard findGuardForOptionalCardinalityWithoutContext(AbstractElement element, AbstractRule currentRule) { + HoistingGuard pathGuard = findGuardForElementWithTrivialCardinality(element, currentRule); + + if (pathGuard.isTrivial()) { + // path can be skipped + return HoistingGuard.unguarded(); + } + + // set cardinality so the token analyse works + element = cloneAbstractElement(element); + if (isMultipleCardinality(element)) { + element.setCardinality("+"); + } else { + // would be ? cardinality + element.setCardinality(null); + } + + // identity analysis can be skipped + + try { + return new AlternativesGuard( + new PathGuard( + new AlternativeTokenSequenceGuard( + analysis.findMinimalPathDifference(element).stream() + .map(s -> s.stream() + .map(SingleTokenGuard::new) + .collect(Collectors.toList()) + ) + .map(TokenSequenceGuard::new) + .collect(Collectors.toList()) + ), + pathGuard + ) + ); + } catch(TokenAnalysisAbortedException e) { + throw new TokenAnalysisAbortedException(e.getMessage(), e, currentRule); + } + } + private HoistingGuard findGuardForAlternatives(CompoundElement alternatives, AbstractRule currentRule) { log.info("find guard for alternative"); @@ -335,25 +373,7 @@ public class HoistingProcessor { ) { return findGuardForElementWithTrivialCardinality(element, currentRule); } else if (isOptionalCardinality(element)) { - if (pathHasTokenOrAction(element)) { - if (!pathHasHoistablePredicate(currentRule.getAlternatives())) { - // unsupported construct doesn't matter since there is no - // hoistable predicate in the rule anyway. - return HoistingGuard.unguarded(); - } else { - // there might be a token in this element - // no context accessible to construct guard - // this does only work when analyzing group - - // TODO: maybe generate warning and return terminal() - throw new OptionalCardinalityWithoutContextException("optional cardinality is only supported in groups", currentRule); - } - } else { - // element with cardinality ? or * has no token or action - // -> the path is accessible whether or not this element is guarded - // -> we can assume it is unguarded - return HoistingGuard.unguarded(); - } + return findGuardForOptionalCardinalityWithoutContext(element, currentRule); } else { throw new IllegalArgumentException("unknown cardinality: " + element.getCardinality()); } diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/guards/AlternativesGuard.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/guards/AlternativesGuard.java index 5fca19f52..4ed076500 100644 --- a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/guards/AlternativesGuard.java +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/guards/AlternativesGuard.java @@ -21,7 +21,11 @@ import java.util.stream.Collectors; public class AlternativesGuard implements HoistingGuard { private List paths; - private AlternativesGuard(List paths) { + public AlternativesGuard(PathGuard ...pathArray) { + this(Arrays.asList(pathArray)); + } + + public AlternativesGuard(List paths) { this.paths = PathGuard.collapse(paths); } diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/pathAnalysis/TokenAnalysis.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/pathAnalysis/TokenAnalysis.java index 6c9a30c3e..1b9c7d2bb 100644 --- a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/pathAnalysis/TokenAnalysis.java +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/pathAnalysis/TokenAnalysis.java @@ -300,6 +300,10 @@ public class TokenAnalysis { return getTokenPaths(path, new TokenAnalysisPaths(indexes), analyseContext, true).getTokenPaths(); } + private List> getTokenPathsContextOnly(AbstractElement path, List indexes) { + return getTokenPathsContext(path, new TokenAnalysisPaths(indexes)).getTokenPaths(); + } + private boolean arePathsIdenticalSymbolic(AbstractElement path1, AbstractElement path2) throws SymbolicAnalysisFailedException { // ignore symbolic analysis for the moment // TODO @@ -403,6 +407,35 @@ public class TokenAnalysis { } } + public List> findMinimalPathDifference(AbstractElement element) throws TokenAnalysisAbortedException { + // this method is for finding the path differences between the + // element (with optional cardinality) and the context + + // first dimension of result corresponds to the alternatives + // the second dimension are tokens of the alternatives + + MutablePrimitiveWrapper>> result = new MutablePrimitiveWrapper>>(null); + + tokenCombinations(indexList -> { + log.info("current index list: " + indexList); + + // no context analysis + List> tokenListsForPath = getTokenPaths(element, indexList, false); + List> tokenListForContext = getTokenPathsContextOnly(element, indexList); + + if (!tokenListsForPath.stream() + .anyMatch(tokenListForContext::contains) + ) { + // context does not contain any token lists for path + result.set(tokenListsForPath); + } + + return result.get() != null; + }); + + return result.get(); + } + public List>> findMinimalPathDifference(List paths) throws TokenAnalysisAbortedException { // first dimension of result corresponds to the paths // the second dimension are the alternatives of the path