From be80d5ffbce24a99ccf394479ed4f332fd22a49b Mon Sep 17 00:00:00 2001
From: overflowerror <mail@overflowerror.com>
Date: Fri, 17 Dec 2021 21:18:59 +0100
Subject: [PATCH] added basic support for nested prefix alternatives
after minimal path difference analysis failed, flatten paths (limited by
token limit; justification: identity check would error out if paths are
not distinguishable within the limit) and recompute alternative guard.
now nested prefixes will be collapsed with all corresponding guard
conditions
---
.../antlr/hoisting/HoistingProcessor.java | 92 ++++++++++++++++++-
.../NestedPrefixAlternativesException.java | 53 +++++++++++
.../hoisting/pathAnalysis/TokenAnalysis.java | 19 +++-
.../parser/antlr/hoisting/token/EofToken.java | 7 ++
.../antlr/hoisting/token/KeywordToken.java | 7 ++
.../hoisting/token/TerminalRuleToken.java | 11 ++-
.../parser/antlr/hoisting/token/Token.java | 4 +-
.../src/org/eclipse/xtext/GrammarUtil.java | 12 +--
8 files changed, 185 insertions(+), 20 deletions(-)
create mode 100644 org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/exceptions/NestedPrefixAlternativesException.java
diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/HoistingProcessor.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/HoistingProcessor.java
index f95266dd4..2d325419e 100644
--- a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/HoistingProcessor.java
+++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/HoistingProcessor.java
@@ -18,6 +18,9 @@ import java.util.List;
import java.util.stream.Collectors;
import org.apache.log4j.Logger;
+
+import static org.eclipse.emf.ecore.util.EcoreUtil.*;
+
import org.eclipse.xtext.AbstractElement;
import org.eclipse.xtext.AbstractRule;
import org.eclipse.xtext.AbstractSemanticPredicate;
@@ -28,12 +31,15 @@ import org.eclipse.xtext.CompoundElement;
import org.eclipse.xtext.Grammar;
import org.eclipse.xtext.Group;
import org.eclipse.xtext.JavaAction;
+import org.eclipse.xtext.JavaCode;
import org.eclipse.xtext.RuleCall;
import org.eclipse.xtext.UnorderedGroup;
import org.eclipse.xtext.XtextFactory;
import org.eclipse.xtext.util.Tuples;
+
import static org.eclipse.xtext.GrammarUtil.*;
+import org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.exceptions.NestedPrefixAlternativesException;
import org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.exceptions.OptionalCardinalityWithoutContextException;
import org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.exceptions.TokenAnalysisAbortedException;
import org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.exceptions.UnsupportedConstructException;
@@ -78,7 +84,7 @@ public class HoistingProcessor {
}
// set cardinality so the token analyse works
- element = cloneAbstractElement(element);
+ element = copy(element);
if (isMultipleCardinality(element)) {
element.setCardinality("+");
} else {
@@ -107,6 +113,70 @@ public class HoistingProcessor {
throw new TokenAnalysisAbortedException(e.getMessage(), e, currentRule);
}
}
+
+ private AbstractElement getNopElement() {
+ JavaCode virtualJavaCodeForAction = XtextFactory.eINSTANCE.createJavaCode();
+ virtualJavaCodeForAction.setSource("$$ /* nop */ $$");
+
+ JavaAction virtualNopJavaAction = XtextFactory.eINSTANCE.createJavaAction();
+ virtualNopJavaAction.setCode(virtualJavaCodeForAction);
+
+ return virtualNopJavaAction;
+ }
+
+ private CompoundElement flattenPaths(CompoundElement original, List<AbstractElement> paths, List<? extends HoistingGuard> guards) {
+ // work on copy of original to preserve eParent
+ CompoundElement flattened = copy(original);
+ setElementsOfCompoundElement(flattened,
+ StreamUtils.zip(
+ paths.stream()
+ .map(analysis::getAllPossiblePaths),
+ guards.stream()
+ .map(HoistingGuard.class::cast),
+ (List<List<AbstractElement>> e, HoistingGuard g) -> Tuples.pair(e, g)
+ ).filter(t -> t.getFirst() != null)
+ .flatMap(t -> {
+ HoistingGuard guard = t.getSecond();
+
+ // we can reuse objects since setElementsOfCompundElement
+ // will create copy anyway
+ AbstractElement nop = getNopElement();
+ AbstractSemanticPredicate renderedVirtualPredicate;
+
+ if (!guard.isTrivial()) {
+ JavaCode virtualJavaCodeForPredicate = XtextFactory.eINSTANCE.createJavaCode();
+ virtualJavaCodeForPredicate.setSource("$$ " + guard.render() + " $$");
+
+ renderedVirtualPredicate = XtextFactory.eINSTANCE.createGatedSemanticPredicate();
+ renderedVirtualPredicate.setCode(virtualJavaCodeForPredicate);
+ } else {
+ renderedVirtualPredicate = null;
+ }
+
+ return t.getFirst()
+ .stream()
+ .map(a -> {
+ List<AbstractElement> groupElements = new ArrayList<>(a.size() + 2);
+
+ if (renderedVirtualPredicate != null) {
+ groupElements.add(renderedVirtualPredicate);
+ }
+
+ // virtual action to prevent hoisting
+ groupElements.add(nop);
+
+ groupElements.addAll(a);
+
+
+ Group virtualGroup = XtextFactory.eINSTANCE.createGroup();
+ setElementsOfCompoundElement(virtualGroup, groupElements);
+
+ return virtualGroup;
+ });
+ })
+ );
+ return flattened;
+ }
private HoistingGuard findGuardForAlternatives(CompoundElement alternatives, AbstractRule currentRule) {
log.info("find guard for alternative");
@@ -171,6 +241,18 @@ public class HoistingProcessor {
(TokenGuard tokenGuard, MergedPathGuard pathGuard) -> Tuples.pair(tokenGuard, pathGuard)
).map(p -> new PathGuard(p.getFirst(), p.getSecond()))
.collect(AlternativesGuard.collector());
+ } catch(NestedPrefixAlternativesException e) {
+ // nested prefix alternatives
+ // -> flatten paths to alternative and try again
+ // this is very inefficient
+ log.warn("nested prefix alternatives detected");
+ log.warn("avoid these since they can't be handled efficiency");
+ log.info(abstractElementToString(alternatives));
+
+ CompoundElement flattened = flattenPaths(alternatives, paths, guards);
+ log.info(abstractElementToString(flattened));
+
+ return findGuardForAlternatives(flattened, currentRule);
} catch(TokenAnalysisAbortedException e) {
throw new TokenAnalysisAbortedException(e.getMessage(), e, currentRule);
}
@@ -225,7 +307,7 @@ public class HoistingProcessor {
// we need a clone of the element because we need to set the cardinality without changing the
// original syntax tree
- AbstractElement clonedElement = cloneAbstractElement(element);
+ AbstractElement clonedElement = copy(element);
clonedElement.setCardinality(null);
// make copy of every element because we can't use any element twice
@@ -243,13 +325,13 @@ public class HoistingProcessor {
remainingElementsInGroupIncludingCurrent.add(0, clonedElement);
Group virtualPathRemaining = XtextFactory.eINSTANCE.createGroup();
- addElementsToCompoundElement(virtualPathRemaining, remainingElementsInGroup);
+ setElementsOfCompoundElement(virtualPathRemaining, remainingElementsInGroup);
Group virtualPathRemainingPlusCurrent = XtextFactory.eINSTANCE.createGroup();
- addElementsToCompoundElement(virtualPathRemainingPlusCurrent, remainingElementsInGroupIncludingCurrent);
+ setElementsOfCompoundElement(virtualPathRemainingPlusCurrent, remainingElementsInGroupIncludingCurrent);
Alternatives virtualAlternatives = XtextFactory.eINSTANCE.createAlternatives();
- addElementsToCompoundElement(virtualAlternatives, Arrays.asList(virtualPathRemaining, virtualPathRemainingPlusCurrent));
+ setElementsOfCompoundElement(virtualAlternatives, Arrays.asList(virtualPathRemaining, virtualPathRemainingPlusCurrent));
// get Guard for virtual alternatives
HoistingGuard guard = findGuardForElementWithTrivialCardinality(virtualAlternatives, currentRule);
diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/exceptions/NestedPrefixAlternativesException.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/exceptions/NestedPrefixAlternativesException.java
new file mode 100644
index 000000000..aa5f8db97
--- /dev/null
+++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/exceptions/NestedPrefixAlternativesException.java
@@ -0,0 +1,53 @@
+/*******************************************************************************
+ * Copyright (c) 2021 itemis AG (http://www.itemis.eu) and others.
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *******************************************************************************/
+package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.exceptions;
+
+import org.eclipse.xtext.AbstractRule;
+
+/**
+ * @author overflow - Initial contribution and API
+ */
+public class NestedPrefixAlternativesException extends TokenAnalysisAbortedException {
+ private static final long serialVersionUID = -1309434841547765441L;
+
+ public NestedPrefixAlternativesException() {
+ super();
+ // TODO Auto-generated constructor stub
+ }
+
+ public NestedPrefixAlternativesException(String message, AbstractRule rule) {
+ super(message, rule);
+ // TODO Auto-generated constructor stub
+ }
+
+ public NestedPrefixAlternativesException(String message, Throwable cause, AbstractRule rule) {
+ super(message, cause, rule);
+ // TODO Auto-generated constructor stub
+ }
+
+ public NestedPrefixAlternativesException(String message, Throwable cause) {
+ super(message, cause);
+ // TODO Auto-generated constructor stub
+ }
+
+ public NestedPrefixAlternativesException(String message) {
+ super(message);
+ // TODO Auto-generated constructor stub
+ }
+
+ public NestedPrefixAlternativesException(Throwable cause, AbstractRule rule) {
+ super(cause, rule);
+ // TODO Auto-generated constructor stub
+ }
+
+ public NestedPrefixAlternativesException(Throwable cause) {
+ super(cause);
+ // TODO Auto-generated constructor stub
+ }
+}
diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/pathAnalysis/TokenAnalysis.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/pathAnalysis/TokenAnalysis.java
index 81231a4f2..c8727c318 100644
--- a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/pathAnalysis/TokenAnalysis.java
+++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/pathAnalysis/TokenAnalysis.java
@@ -30,12 +30,11 @@ import org.eclipse.xtext.CompoundElement;
import org.eclipse.xtext.Grammar;
import org.eclipse.xtext.GrammarUtil;
import org.eclipse.xtext.Group;
-import org.eclipse.xtext.Keyword;
-import org.eclipse.xtext.ParserRule;
import org.eclipse.xtext.RuleCall;
import org.eclipse.xtext.UnorderedGroup;
import org.eclipse.xtext.util.XtextSwitch;
import org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.HoistingConfiguration;
+import org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.exceptions.NestedPrefixAlternativesException;
import org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.exceptions.SymbolicAnalysisFailedException;
import org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.exceptions.TokenAnalysisAbortedException;
import org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.token.Token;
@@ -402,7 +401,6 @@ public class TokenAnalysis {
}
}
-
private void tokenCombinations(Function<List<Integer>, Boolean> callback) {
MutablePrimitiveWrapper<Integer> limit = new MutablePrimitiveWrapper<>(config.getTokenLimit());
@@ -416,7 +414,9 @@ public class TokenAnalysis {
if (limit.get().equals(config.getTokenLimit())) {
throw new TokenAnalysisAbortedException("token limit exhausted while searching for minimal differences");
} else {
- throw new TokenAnalysisAbortedException("path length exhausted while searching for minimal differences");
+ // path length exhausted while searching for minimal differences
+ // this indicates nested prefix alternatives
+ throw new NestedPrefixAlternativesException();
}
}
private boolean tokenCombinations(long prefix, int prefixLength, int ones, Function<List<Integer>, Boolean> callback, MutablePrimitiveWrapper<Integer> limit) {
@@ -528,4 +528,15 @@ public class TokenAnalysis {
return result;
}
+
+ public List<List<AbstractElement>> getAllPossiblePaths(AbstractElement path) {
+ return getTokenPaths(path, new TokenAnalysisPaths(range(0, config.getTokenLimit() + 1)), false, false)
+ .getTokenPaths()
+ .stream()
+ .map(l -> l.stream()
+ .map(Token::getElement)
+ .collect(Collectors.toList()))
+ .collect(Collectors.toList());
+
+ }
}
diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/EofToken.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/EofToken.java
index 3a3e6b2a7..ab3bca0b6 100644
--- a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/EofToken.java
+++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/EofToken.java
@@ -8,6 +8,8 @@
*******************************************************************************/
package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.token;
+import org.eclipse.xtext.AbstractElement;
+
/**
* @author overflow - Initial contribution and API
*/
@@ -51,4 +53,9 @@ public class EofToken implements Token {
return "EofToken(" + position + ")\n";
}
+ @Override
+ public AbstractElement getElement() {
+ return null;
+ }
+
}
diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/KeywordToken.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/KeywordToken.java
index 8a67c4c77..757257f1b 100644
--- a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/KeywordToken.java
+++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/KeywordToken.java
@@ -8,6 +8,7 @@
*******************************************************************************/
package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.token;
+import org.eclipse.xtext.AbstractElement;
import org.eclipse.xtext.Keyword;
/**
@@ -59,4 +60,10 @@ public class KeywordToken implements Token {
return false;
return true;
}
+
+ @Override
+ public AbstractElement getElement() {
+ return keyword;
+ }
+
}
diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/TerminalRuleToken.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/TerminalRuleToken.java
index 5611cebfb..224bb0161 100644
--- a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/TerminalRuleToken.java
+++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/TerminalRuleToken.java
@@ -8,16 +8,20 @@
*******************************************************************************/
package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.token;
+import org.eclipse.xtext.AbstractElement;
+import org.eclipse.xtext.RuleCall;
import org.eclipse.xtext.TerminalRule;
/**
* @author overflow - Initial contribution and API
*/
public class TerminalRuleToken implements Token {
+ private RuleCall call;
private TerminalRule rule;
private int position;
- public TerminalRuleToken(TerminalRule rule, int position) {
+ TerminalRuleToken(RuleCall call, TerminalRule rule, int position) {
+ this.call = call;
this.rule = rule;
this.position = position;
}
@@ -60,5 +64,8 @@ public class TerminalRuleToken implements Token {
return true;
}
-
+ @Override
+ public AbstractElement getElement() {
+ return call;
+ }
}
diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/Token.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/Token.java
index bc04b26d7..7e1c6ba30 100644
--- a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/Token.java
+++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/Token.java
@@ -21,6 +21,8 @@ import org.eclipse.xtext.TerminalRule;
public interface Token {
String negatedCondition();
+ AbstractElement getElement();
+
static boolean isToken(AbstractElement element) {
if (element == null) {
return true;
@@ -43,7 +45,7 @@ public interface Token {
} else if (element instanceof RuleCall) {
AbstractRule rule = ((RuleCall) element).getRule();
if (rule instanceof TerminalRule) {
- return new TerminalRuleToken((TerminalRule) rule, position);
+ return new TerminalRuleToken((RuleCall) element, (TerminalRule) rule, position);
}
} else if (element instanceof EnumLiteralDeclaration) {
return new KeywordToken(((EnumLiteralDeclaration) element).getLiteral(), position);
diff --git a/org.eclipse.xtext/src/org/eclipse/xtext/GrammarUtil.java b/org.eclipse.xtext/src/org/eclipse/xtext/GrammarUtil.java
index aa985911e..b7c4429c5 100644
--- a/org.eclipse.xtext/src/org/eclipse/xtext/GrammarUtil.java
+++ b/org.eclipse.xtext/src/org/eclipse/xtext/GrammarUtil.java
@@ -709,20 +709,16 @@ public class GrammarUtil {
return null;
}
- public static AbstractElement cloneAbstractElement(AbstractElement element) {
- return (AbstractElement) EcoreUtil.copy(element);
- }
-
- public static void addElementsToCompoundElement(CompoundElement element, Stream<? extends AbstractElement> elements) {
+ public static void setElementsOfCompoundElement(CompoundElement element, Stream<? extends AbstractElement> elements) {
EStructuralFeature compoundElementElementsFeature = XtextPackage.Literals.COMPOUND_ELEMENT.getEStructuralFeature(XtextPackage.COMPOUND_ELEMENT__ELEMENTS);
element.eSet(compoundElementElementsFeature, elements
- .map(GrammarUtil::cloneAbstractElement)
+ .map(EcoreUtil::copy)
.collect(Collectors.toList()));
}
- public static void addElementsToCompoundElement(CompoundElement element, Collection<? extends AbstractElement> elements) {
- addElementsToCompoundElement(element, elements.stream());
+ public static void setElementsOfCompoundElement(CompoundElement element, Collection<? extends AbstractElement> elements) {
+ setElementsOfCompoundElement(element, elements.stream());
}
private static void findAllRuleCalls(List<RuleCall> calls, AbstractElement element, AbstractRule rule) {