From be80d5ffbce24a99ccf394479ed4f332fd22a49b Mon Sep 17 00:00:00 2001 From: overflowerror <mail@overflowerror.com> Date: Fri, 17 Dec 2021 21:18:59 +0100 Subject: [PATCH] added basic support for nested prefix alternatives after minimal path difference analysis failed, flatten paths (limited by token limit; justification: identity check would error out if paths are not distinguishable within the limit) and recompute alternative guard. now nested prefixes will be collapsed with all corresponding guard conditions --- .../antlr/hoisting/HoistingProcessor.java | 92 ++++++++++++++++++- .../NestedPrefixAlternativesException.java | 53 +++++++++++ .../hoisting/pathAnalysis/TokenAnalysis.java | 19 +++- .../parser/antlr/hoisting/token/EofToken.java | 7 ++ .../antlr/hoisting/token/KeywordToken.java | 7 ++ .../hoisting/token/TerminalRuleToken.java | 11 ++- .../parser/antlr/hoisting/token/Token.java | 4 +- .../src/org/eclipse/xtext/GrammarUtil.java | 12 +-- 8 files changed, 185 insertions(+), 20 deletions(-) create mode 100644 org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/exceptions/NestedPrefixAlternativesException.java diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/HoistingProcessor.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/HoistingProcessor.java index f95266dd4..2d325419e 100644 --- a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/HoistingProcessor.java +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/HoistingProcessor.java @@ -18,6 +18,9 @@ import java.util.List; import java.util.stream.Collectors; import org.apache.log4j.Logger; + +import static org.eclipse.emf.ecore.util.EcoreUtil.*; + import org.eclipse.xtext.AbstractElement; import org.eclipse.xtext.AbstractRule; import org.eclipse.xtext.AbstractSemanticPredicate; @@ -28,12 +31,15 @@ import org.eclipse.xtext.CompoundElement; import org.eclipse.xtext.Grammar; import org.eclipse.xtext.Group; import org.eclipse.xtext.JavaAction; +import org.eclipse.xtext.JavaCode; import org.eclipse.xtext.RuleCall; import org.eclipse.xtext.UnorderedGroup; import org.eclipse.xtext.XtextFactory; import org.eclipse.xtext.util.Tuples; + import static org.eclipse.xtext.GrammarUtil.*; +import org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.exceptions.NestedPrefixAlternativesException; import org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.exceptions.OptionalCardinalityWithoutContextException; import org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.exceptions.TokenAnalysisAbortedException; import org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.exceptions.UnsupportedConstructException; @@ -78,7 +84,7 @@ public class HoistingProcessor { } // set cardinality so the token analyse works - element = cloneAbstractElement(element); + element = copy(element); if (isMultipleCardinality(element)) { element.setCardinality("+"); } else { @@ -107,6 +113,70 @@ public class HoistingProcessor { throw new TokenAnalysisAbortedException(e.getMessage(), e, currentRule); } } + + private AbstractElement getNopElement() { + JavaCode virtualJavaCodeForAction = XtextFactory.eINSTANCE.createJavaCode(); + virtualJavaCodeForAction.setSource("$$ /* nop */ $$"); + + JavaAction virtualNopJavaAction = XtextFactory.eINSTANCE.createJavaAction(); + virtualNopJavaAction.setCode(virtualJavaCodeForAction); + + return virtualNopJavaAction; + } + + private CompoundElement flattenPaths(CompoundElement original, List<AbstractElement> paths, List<? extends HoistingGuard> guards) { + // work on copy of original to preserve eParent + CompoundElement flattened = copy(original); + setElementsOfCompoundElement(flattened, + StreamUtils.zip( + paths.stream() + .map(analysis::getAllPossiblePaths), + guards.stream() + .map(HoistingGuard.class::cast), + (List<List<AbstractElement>> e, HoistingGuard g) -> Tuples.pair(e, g) + ).filter(t -> t.getFirst() != null) + .flatMap(t -> { + HoistingGuard guard = t.getSecond(); + + // we can reuse objects since setElementsOfCompundElement + // will create copy anyway + AbstractElement nop = getNopElement(); + AbstractSemanticPredicate renderedVirtualPredicate; + + if (!guard.isTrivial()) { + JavaCode virtualJavaCodeForPredicate = XtextFactory.eINSTANCE.createJavaCode(); + virtualJavaCodeForPredicate.setSource("$$ " + guard.render() + " $$"); + + renderedVirtualPredicate = XtextFactory.eINSTANCE.createGatedSemanticPredicate(); + renderedVirtualPredicate.setCode(virtualJavaCodeForPredicate); + } else { + renderedVirtualPredicate = null; + } + + return t.getFirst() + .stream() + .map(a -> { + List<AbstractElement> groupElements = new ArrayList<>(a.size() + 2); + + if (renderedVirtualPredicate != null) { + groupElements.add(renderedVirtualPredicate); + } + + // virtual action to prevent hoisting + groupElements.add(nop); + + groupElements.addAll(a); + + + Group virtualGroup = XtextFactory.eINSTANCE.createGroup(); + setElementsOfCompoundElement(virtualGroup, groupElements); + + return virtualGroup; + }); + }) + ); + return flattened; + } private HoistingGuard findGuardForAlternatives(CompoundElement alternatives, AbstractRule currentRule) { log.info("find guard for alternative"); @@ -171,6 +241,18 @@ public class HoistingProcessor { (TokenGuard tokenGuard, MergedPathGuard pathGuard) -> Tuples.pair(tokenGuard, pathGuard) ).map(p -> new PathGuard(p.getFirst(), p.getSecond())) .collect(AlternativesGuard.collector()); + } catch(NestedPrefixAlternativesException e) { + // nested prefix alternatives + // -> flatten paths to alternative and try again + // this is very inefficient + log.warn("nested prefix alternatives detected"); + log.warn("avoid these since they can't be handled efficiency"); + log.info(abstractElementToString(alternatives)); + + CompoundElement flattened = flattenPaths(alternatives, paths, guards); + log.info(abstractElementToString(flattened)); + + return findGuardForAlternatives(flattened, currentRule); } catch(TokenAnalysisAbortedException e) { throw new TokenAnalysisAbortedException(e.getMessage(), e, currentRule); } @@ -225,7 +307,7 @@ public class HoistingProcessor { // we need a clone of the element because we need to set the cardinality without changing the // original syntax tree - AbstractElement clonedElement = cloneAbstractElement(element); + AbstractElement clonedElement = copy(element); clonedElement.setCardinality(null); // make copy of every element because we can't use any element twice @@ -243,13 +325,13 @@ public class HoistingProcessor { remainingElementsInGroupIncludingCurrent.add(0, clonedElement); Group virtualPathRemaining = XtextFactory.eINSTANCE.createGroup(); - addElementsToCompoundElement(virtualPathRemaining, remainingElementsInGroup); + setElementsOfCompoundElement(virtualPathRemaining, remainingElementsInGroup); Group virtualPathRemainingPlusCurrent = XtextFactory.eINSTANCE.createGroup(); - addElementsToCompoundElement(virtualPathRemainingPlusCurrent, remainingElementsInGroupIncludingCurrent); + setElementsOfCompoundElement(virtualPathRemainingPlusCurrent, remainingElementsInGroupIncludingCurrent); Alternatives virtualAlternatives = XtextFactory.eINSTANCE.createAlternatives(); - addElementsToCompoundElement(virtualAlternatives, Arrays.asList(virtualPathRemaining, virtualPathRemainingPlusCurrent)); + setElementsOfCompoundElement(virtualAlternatives, Arrays.asList(virtualPathRemaining, virtualPathRemainingPlusCurrent)); // get Guard for virtual alternatives HoistingGuard guard = findGuardForElementWithTrivialCardinality(virtualAlternatives, currentRule); diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/exceptions/NestedPrefixAlternativesException.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/exceptions/NestedPrefixAlternativesException.java new file mode 100644 index 000000000..aa5f8db97 --- /dev/null +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/exceptions/NestedPrefixAlternativesException.java @@ -0,0 +1,53 @@ +/******************************************************************************* + * Copyright (c) 2021 itemis AG (http://www.itemis.eu) and others. + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *******************************************************************************/ +package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.exceptions; + +import org.eclipse.xtext.AbstractRule; + +/** + * @author overflow - Initial contribution and API + */ +public class NestedPrefixAlternativesException extends TokenAnalysisAbortedException { + private static final long serialVersionUID = -1309434841547765441L; + + public NestedPrefixAlternativesException() { + super(); + // TODO Auto-generated constructor stub + } + + public NestedPrefixAlternativesException(String message, AbstractRule rule) { + super(message, rule); + // TODO Auto-generated constructor stub + } + + public NestedPrefixAlternativesException(String message, Throwable cause, AbstractRule rule) { + super(message, cause, rule); + // TODO Auto-generated constructor stub + } + + public NestedPrefixAlternativesException(String message, Throwable cause) { + super(message, cause); + // TODO Auto-generated constructor stub + } + + public NestedPrefixAlternativesException(String message) { + super(message); + // TODO Auto-generated constructor stub + } + + public NestedPrefixAlternativesException(Throwable cause, AbstractRule rule) { + super(cause, rule); + // TODO Auto-generated constructor stub + } + + public NestedPrefixAlternativesException(Throwable cause) { + super(cause); + // TODO Auto-generated constructor stub + } +} diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/pathAnalysis/TokenAnalysis.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/pathAnalysis/TokenAnalysis.java index 81231a4f2..c8727c318 100644 --- a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/pathAnalysis/TokenAnalysis.java +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/pathAnalysis/TokenAnalysis.java @@ -30,12 +30,11 @@ import org.eclipse.xtext.CompoundElement; import org.eclipse.xtext.Grammar; import org.eclipse.xtext.GrammarUtil; import org.eclipse.xtext.Group; -import org.eclipse.xtext.Keyword; -import org.eclipse.xtext.ParserRule; import org.eclipse.xtext.RuleCall; import org.eclipse.xtext.UnorderedGroup; import org.eclipse.xtext.util.XtextSwitch; import org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.HoistingConfiguration; +import org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.exceptions.NestedPrefixAlternativesException; import org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.exceptions.SymbolicAnalysisFailedException; import org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.exceptions.TokenAnalysisAbortedException; import org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.token.Token; @@ -402,7 +401,6 @@ public class TokenAnalysis { } } - private void tokenCombinations(Function<List<Integer>, Boolean> callback) { MutablePrimitiveWrapper<Integer> limit = new MutablePrimitiveWrapper<>(config.getTokenLimit()); @@ -416,7 +414,9 @@ public class TokenAnalysis { if (limit.get().equals(config.getTokenLimit())) { throw new TokenAnalysisAbortedException("token limit exhausted while searching for minimal differences"); } else { - throw new TokenAnalysisAbortedException("path length exhausted while searching for minimal differences"); + // path length exhausted while searching for minimal differences + // this indicates nested prefix alternatives + throw new NestedPrefixAlternativesException(); } } private boolean tokenCombinations(long prefix, int prefixLength, int ones, Function<List<Integer>, Boolean> callback, MutablePrimitiveWrapper<Integer> limit) { @@ -528,4 +528,15 @@ public class TokenAnalysis { return result; } + + public List<List<AbstractElement>> getAllPossiblePaths(AbstractElement path) { + return getTokenPaths(path, new TokenAnalysisPaths(range(0, config.getTokenLimit() + 1)), false, false) + .getTokenPaths() + .stream() + .map(l -> l.stream() + .map(Token::getElement) + .collect(Collectors.toList())) + .collect(Collectors.toList()); + + } } diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/EofToken.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/EofToken.java index 3a3e6b2a7..ab3bca0b6 100644 --- a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/EofToken.java +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/EofToken.java @@ -8,6 +8,8 @@ *******************************************************************************/ package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.token; +import org.eclipse.xtext.AbstractElement; + /** * @author overflow - Initial contribution and API */ @@ -51,4 +53,9 @@ public class EofToken implements Token { return "EofToken(" + position + ")\n"; } + @Override + public AbstractElement getElement() { + return null; + } + } diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/KeywordToken.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/KeywordToken.java index 8a67c4c77..757257f1b 100644 --- a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/KeywordToken.java +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/KeywordToken.java @@ -8,6 +8,7 @@ *******************************************************************************/ package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.token; +import org.eclipse.xtext.AbstractElement; import org.eclipse.xtext.Keyword; /** @@ -59,4 +60,10 @@ public class KeywordToken implements Token { return false; return true; } + + @Override + public AbstractElement getElement() { + return keyword; + } + } diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/TerminalRuleToken.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/TerminalRuleToken.java index 5611cebfb..224bb0161 100644 --- a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/TerminalRuleToken.java +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/TerminalRuleToken.java @@ -8,16 +8,20 @@ *******************************************************************************/ package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.token; +import org.eclipse.xtext.AbstractElement; +import org.eclipse.xtext.RuleCall; import org.eclipse.xtext.TerminalRule; /** * @author overflow - Initial contribution and API */ public class TerminalRuleToken implements Token { + private RuleCall call; private TerminalRule rule; private int position; - public TerminalRuleToken(TerminalRule rule, int position) { + TerminalRuleToken(RuleCall call, TerminalRule rule, int position) { + this.call = call; this.rule = rule; this.position = position; } @@ -60,5 +64,8 @@ public class TerminalRuleToken implements Token { return true; } - + @Override + public AbstractElement getElement() { + return call; + } } diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/Token.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/Token.java index bc04b26d7..7e1c6ba30 100644 --- a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/Token.java +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/Token.java @@ -21,6 +21,8 @@ import org.eclipse.xtext.TerminalRule; public interface Token { String negatedCondition(); + AbstractElement getElement(); + static boolean isToken(AbstractElement element) { if (element == null) { return true; @@ -43,7 +45,7 @@ public interface Token { } else if (element instanceof RuleCall) { AbstractRule rule = ((RuleCall) element).getRule(); if (rule instanceof TerminalRule) { - return new TerminalRuleToken((TerminalRule) rule, position); + return new TerminalRuleToken((RuleCall) element, (TerminalRule) rule, position); } } else if (element instanceof EnumLiteralDeclaration) { return new KeywordToken(((EnumLiteralDeclaration) element).getLiteral(), position); diff --git a/org.eclipse.xtext/src/org/eclipse/xtext/GrammarUtil.java b/org.eclipse.xtext/src/org/eclipse/xtext/GrammarUtil.java index aa985911e..b7c4429c5 100644 --- a/org.eclipse.xtext/src/org/eclipse/xtext/GrammarUtil.java +++ b/org.eclipse.xtext/src/org/eclipse/xtext/GrammarUtil.java @@ -709,20 +709,16 @@ public class GrammarUtil { return null; } - public static AbstractElement cloneAbstractElement(AbstractElement element) { - return (AbstractElement) EcoreUtil.copy(element); - } - - public static void addElementsToCompoundElement(CompoundElement element, Stream<? extends AbstractElement> elements) { + public static void setElementsOfCompoundElement(CompoundElement element, Stream<? extends AbstractElement> elements) { EStructuralFeature compoundElementElementsFeature = XtextPackage.Literals.COMPOUND_ELEMENT.getEStructuralFeature(XtextPackage.COMPOUND_ELEMENT__ELEMENTS); element.eSet(compoundElementElementsFeature, elements - .map(GrammarUtil::cloneAbstractElement) + .map(EcoreUtil::copy) .collect(Collectors.toList())); } - public static void addElementsToCompoundElement(CompoundElement element, Collection<? extends AbstractElement> elements) { - addElementsToCompoundElement(element, elements.stream()); + public static void setElementsOfCompoundElement(CompoundElement element, Collection<? extends AbstractElement> elements) { + setElementsOfCompoundElement(element, elements.stream()); } private static void findAllRuleCalls(List<RuleCall> calls, AbstractElement element, AbstractRule rule) {