From be80d5ffbce24a99ccf394479ed4f332fd22a49b Mon Sep 17 00:00:00 2001
From: overflowerror <mail@overflowerror.com>
Date: Fri, 17 Dec 2021 21:18:59 +0100
Subject: [PATCH] added basic support for nested prefix alternatives

after minimal path difference analysis failed, flatten paths (limited by
token limit; justification: identity check would error out if paths are
not distinguishable within the limit) and recompute alternative guard.
now nested prefixes will be collapsed with all corresponding guard
conditions
---
 .../antlr/hoisting/HoistingProcessor.java     | 92 ++++++++++++++++++-
 .../NestedPrefixAlternativesException.java    | 53 +++++++++++
 .../hoisting/pathAnalysis/TokenAnalysis.java  | 19 +++-
 .../parser/antlr/hoisting/token/EofToken.java |  7 ++
 .../antlr/hoisting/token/KeywordToken.java    |  7 ++
 .../hoisting/token/TerminalRuleToken.java     | 11 ++-
 .../parser/antlr/hoisting/token/Token.java    |  4 +-
 .../src/org/eclipse/xtext/GrammarUtil.java    | 12 +--
 8 files changed, 185 insertions(+), 20 deletions(-)
 create mode 100644 org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/exceptions/NestedPrefixAlternativesException.java

diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/HoistingProcessor.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/HoistingProcessor.java
index f95266dd4..2d325419e 100644
--- a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/HoistingProcessor.java
+++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/HoistingProcessor.java
@@ -18,6 +18,9 @@ import java.util.List;
 import java.util.stream.Collectors;
 
 import org.apache.log4j.Logger;
+
+import static org.eclipse.emf.ecore.util.EcoreUtil.*;
+
 import org.eclipse.xtext.AbstractElement;
 import org.eclipse.xtext.AbstractRule;
 import org.eclipse.xtext.AbstractSemanticPredicate;
@@ -28,12 +31,15 @@ import org.eclipse.xtext.CompoundElement;
 import org.eclipse.xtext.Grammar;
 import org.eclipse.xtext.Group;
 import org.eclipse.xtext.JavaAction;
+import org.eclipse.xtext.JavaCode;
 import org.eclipse.xtext.RuleCall;
 import org.eclipse.xtext.UnorderedGroup;
 import org.eclipse.xtext.XtextFactory;
 import org.eclipse.xtext.util.Tuples;
+
 import static org.eclipse.xtext.GrammarUtil.*;
 
+import org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.exceptions.NestedPrefixAlternativesException;
 import org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.exceptions.OptionalCardinalityWithoutContextException;
 import org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.exceptions.TokenAnalysisAbortedException;
 import org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.exceptions.UnsupportedConstructException;
@@ -78,7 +84,7 @@ public class HoistingProcessor {
 		}
 		
 		// set cardinality so the token analyse works
-		element = cloneAbstractElement(element);
+		element = copy(element);
 		if (isMultipleCardinality(element)) {
 			element.setCardinality("+");
 		} else {
@@ -107,6 +113,70 @@ public class HoistingProcessor {
 			throw new TokenAnalysisAbortedException(e.getMessage(), e, currentRule);
 		}
 	}
+
+	private AbstractElement getNopElement() {
+		JavaCode virtualJavaCodeForAction = XtextFactory.eINSTANCE.createJavaCode();
+		virtualJavaCodeForAction.setSource("$$ /* nop */ $$");
+		
+		JavaAction virtualNopJavaAction = XtextFactory.eINSTANCE.createJavaAction();
+		virtualNopJavaAction.setCode(virtualJavaCodeForAction);
+		
+		return virtualNopJavaAction;
+	}
+	
+	private CompoundElement flattenPaths(CompoundElement original, List<AbstractElement> paths, List<? extends HoistingGuard> guards) {
+		// work on copy of original to preserve eParent
+		CompoundElement flattened = copy(original);
+		setElementsOfCompoundElement(flattened,
+			StreamUtils.zip(
+				paths.stream()
+					.map(analysis::getAllPossiblePaths),
+				guards.stream()
+					.map(HoistingGuard.class::cast),
+				(List<List<AbstractElement>> e, HoistingGuard g) -> Tuples.pair(e, g)
+			).filter(t -> t.getFirst() != null)
+			.flatMap(t -> {
+				HoistingGuard guard = t.getSecond();
+				
+				// we can reuse objects since setElementsOfCompundElement
+				// will create copy anyway
+				AbstractElement nop = getNopElement();
+				AbstractSemanticPredicate renderedVirtualPredicate;
+				
+				if (!guard.isTrivial()) {
+					JavaCode virtualJavaCodeForPredicate = XtextFactory.eINSTANCE.createJavaCode();
+					virtualJavaCodeForPredicate.setSource("$$ " + guard.render() + " $$");
+				
+					renderedVirtualPredicate = XtextFactory.eINSTANCE.createGatedSemanticPredicate();
+					renderedVirtualPredicate.setCode(virtualJavaCodeForPredicate);
+				} else {
+					renderedVirtualPredicate = null;
+				}
+				
+				return t.getFirst()
+					.stream()
+					.map(a -> {
+						List<AbstractElement> groupElements = new ArrayList<>(a.size() + 2);
+						
+						if (renderedVirtualPredicate != null) {
+							groupElements.add(renderedVirtualPredicate);
+						}
+
+						// virtual action to prevent hoisting
+						groupElements.add(nop);
+						
+						groupElements.addAll(a);
+
+
+						Group virtualGroup = XtextFactory.eINSTANCE.createGroup();
+						setElementsOfCompoundElement(virtualGroup, groupElements);
+								
+						return virtualGroup;
+					});
+			})
+		);
+		return flattened;
+	}
 	
 	private HoistingGuard findGuardForAlternatives(CompoundElement alternatives, AbstractRule currentRule) {
 		log.info("find guard for alternative");
@@ -171,6 +241,18 @@ public class HoistingProcessor {
 					(TokenGuard tokenGuard, MergedPathGuard pathGuard) -> Tuples.pair(tokenGuard, pathGuard)
 				).map(p -> new PathGuard(p.getFirst(), p.getSecond()))
 				.collect(AlternativesGuard.collector());
+			} catch(NestedPrefixAlternativesException e) {
+				// nested prefix alternatives
+				// -> flatten paths to alternative and try again
+				// this is very inefficient
+				log.warn("nested prefix alternatives detected");
+				log.warn("avoid these since they can't be handled efficiency");
+				log.info(abstractElementToString(alternatives));
+				
+				CompoundElement flattened = flattenPaths(alternatives, paths, guards);
+				log.info(abstractElementToString(flattened));
+				
+				return findGuardForAlternatives(flattened, currentRule);
 			} catch(TokenAnalysisAbortedException e) {
 				throw new TokenAnalysisAbortedException(e.getMessage(), e, currentRule);
 			}
@@ -225,7 +307,7 @@ public class HoistingProcessor {
 				
 				// we need a clone of the element because we need to set the cardinality without changing the
 				// original syntax tree
-				AbstractElement clonedElement = cloneAbstractElement(element);
+				AbstractElement clonedElement = copy(element);
 				clonedElement.setCardinality(null);
 				
 				// make copy of every element because we can't use any element twice
@@ -243,13 +325,13 @@ public class HoistingProcessor {
 				remainingElementsInGroupIncludingCurrent.add(0, clonedElement);
 						
 				Group virtualPathRemaining = XtextFactory.eINSTANCE.createGroup();
-				addElementsToCompoundElement(virtualPathRemaining, remainingElementsInGroup);
+				setElementsOfCompoundElement(virtualPathRemaining, remainingElementsInGroup);
 				
 				Group virtualPathRemainingPlusCurrent = XtextFactory.eINSTANCE.createGroup();
-				addElementsToCompoundElement(virtualPathRemainingPlusCurrent, remainingElementsInGroupIncludingCurrent);
+				setElementsOfCompoundElement(virtualPathRemainingPlusCurrent, remainingElementsInGroupIncludingCurrent);
 				
 				Alternatives virtualAlternatives = XtextFactory.eINSTANCE.createAlternatives();
-				addElementsToCompoundElement(virtualAlternatives, Arrays.asList(virtualPathRemaining, virtualPathRemainingPlusCurrent));
+				setElementsOfCompoundElement(virtualAlternatives, Arrays.asList(virtualPathRemaining, virtualPathRemainingPlusCurrent));
 				
 				// get Guard for virtual alternatives
 				HoistingGuard guard = findGuardForElementWithTrivialCardinality(virtualAlternatives, currentRule);
diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/exceptions/NestedPrefixAlternativesException.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/exceptions/NestedPrefixAlternativesException.java
new file mode 100644
index 000000000..aa5f8db97
--- /dev/null
+++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/exceptions/NestedPrefixAlternativesException.java
@@ -0,0 +1,53 @@
+/*******************************************************************************
+ * Copyright (c) 2021 itemis AG (http://www.itemis.eu) and others.
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ * 
+ * SPDX-License-Identifier: EPL-2.0
+ *******************************************************************************/
+package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.exceptions;
+
+import org.eclipse.xtext.AbstractRule;
+
+/**
+ * @author overflow - Initial contribution and API
+ */
+public class NestedPrefixAlternativesException extends TokenAnalysisAbortedException {
+	private static final long serialVersionUID = -1309434841547765441L;
+
+	public NestedPrefixAlternativesException() {
+		super();
+		// TODO Auto-generated constructor stub
+	}
+
+	public NestedPrefixAlternativesException(String message, AbstractRule rule) {
+		super(message, rule);
+		// TODO Auto-generated constructor stub
+	}
+
+	public NestedPrefixAlternativesException(String message, Throwable cause, AbstractRule rule) {
+		super(message, cause, rule);
+		// TODO Auto-generated constructor stub
+	}
+
+	public NestedPrefixAlternativesException(String message, Throwable cause) {
+		super(message, cause);
+		// TODO Auto-generated constructor stub
+	}
+
+	public NestedPrefixAlternativesException(String message) {
+		super(message);
+		// TODO Auto-generated constructor stub
+	}
+
+	public NestedPrefixAlternativesException(Throwable cause, AbstractRule rule) {
+		super(cause, rule);
+		// TODO Auto-generated constructor stub
+	}
+
+	public NestedPrefixAlternativesException(Throwable cause) {
+		super(cause);
+		// TODO Auto-generated constructor stub
+	}
+}
diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/pathAnalysis/TokenAnalysis.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/pathAnalysis/TokenAnalysis.java
index 81231a4f2..c8727c318 100644
--- a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/pathAnalysis/TokenAnalysis.java
+++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/pathAnalysis/TokenAnalysis.java
@@ -30,12 +30,11 @@ import org.eclipse.xtext.CompoundElement;
 import org.eclipse.xtext.Grammar;
 import org.eclipse.xtext.GrammarUtil;
 import org.eclipse.xtext.Group;
-import org.eclipse.xtext.Keyword;
-import org.eclipse.xtext.ParserRule;
 import org.eclipse.xtext.RuleCall;
 import org.eclipse.xtext.UnorderedGroup;
 import org.eclipse.xtext.util.XtextSwitch;
 import org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.HoistingConfiguration;
+import org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.exceptions.NestedPrefixAlternativesException;
 import org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.exceptions.SymbolicAnalysisFailedException;
 import org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.exceptions.TokenAnalysisAbortedException;
 import org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.token.Token;
@@ -402,7 +401,6 @@ public class TokenAnalysis {
 		}
 	}
 	
-	
 	private void tokenCombinations(Function<List<Integer>, Boolean> callback) {
 		MutablePrimitiveWrapper<Integer> limit = new MutablePrimitiveWrapper<>(config.getTokenLimit());
 		
@@ -416,7 +414,9 @@ public class TokenAnalysis {
 		if (limit.get().equals(config.getTokenLimit())) {
 			throw new TokenAnalysisAbortedException("token limit exhausted while searching for minimal differences");
 		} else {
-			throw new TokenAnalysisAbortedException("path length exhausted while searching for minimal differences");
+			// path length exhausted while searching for minimal differences
+			// this indicates nested prefix alternatives
+			throw new NestedPrefixAlternativesException();
 		}
 	}
 	private boolean tokenCombinations(long prefix, int prefixLength, int ones, Function<List<Integer>, Boolean> callback, MutablePrimitiveWrapper<Integer> limit) {
@@ -528,4 +528,15 @@ public class TokenAnalysis {
 		
 		return result;
 	}
+	
+	public List<List<AbstractElement>> getAllPossiblePaths(AbstractElement path) {
+		return getTokenPaths(path, new TokenAnalysisPaths(range(0, config.getTokenLimit() + 1)), false, false)
+				.getTokenPaths()
+				.stream()
+				.map(l -> l.stream()
+					.map(Token::getElement)
+					.collect(Collectors.toList()))
+				.collect(Collectors.toList());
+				
+	}
 }
diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/EofToken.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/EofToken.java
index 3a3e6b2a7..ab3bca0b6 100644
--- a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/EofToken.java
+++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/EofToken.java
@@ -8,6 +8,8 @@
  *******************************************************************************/
 package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.token;
 
+import org.eclipse.xtext.AbstractElement;
+
 /**
  * @author overflow - Initial contribution and API
  */
@@ -51,4 +53,9 @@ public class EofToken implements Token {
 		return "EofToken(" + position + ")\n";
 	}
 
+	@Override
+	public AbstractElement getElement() {
+		return null;
+	}
+
 }
diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/KeywordToken.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/KeywordToken.java
index 8a67c4c77..757257f1b 100644
--- a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/KeywordToken.java
+++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/KeywordToken.java
@@ -8,6 +8,7 @@
  *******************************************************************************/
 package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.token;
 
+import org.eclipse.xtext.AbstractElement;
 import org.eclipse.xtext.Keyword;
 
 /**
@@ -59,4 +60,10 @@ public class KeywordToken implements Token {
 			return false;
 		return true;
 	}
+	
+	@Override
+	public AbstractElement getElement() {
+		return keyword;
+	}
+	
 }
diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/TerminalRuleToken.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/TerminalRuleToken.java
index 5611cebfb..224bb0161 100644
--- a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/TerminalRuleToken.java
+++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/TerminalRuleToken.java
@@ -8,16 +8,20 @@
  *******************************************************************************/
 package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.token;
 
+import org.eclipse.xtext.AbstractElement;
+import org.eclipse.xtext.RuleCall;
 import org.eclipse.xtext.TerminalRule;
 
 /**
  * @author overflow - Initial contribution and API
  */
 public class TerminalRuleToken implements Token {
+	private RuleCall call;
 	private TerminalRule rule;
 	private int position;
 	
-	public TerminalRuleToken(TerminalRule rule, int position) {
+	TerminalRuleToken(RuleCall call, TerminalRule rule, int position) {
+		this.call = call;
 		this.rule = rule;
 		this.position = position;
 	}
@@ -60,5 +64,8 @@ public class TerminalRuleToken implements Token {
 		return true;
 	}
 	
-	
+	@Override
+	public AbstractElement getElement() {
+		return call;
+	}
 }
diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/Token.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/Token.java
index bc04b26d7..7e1c6ba30 100644
--- a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/Token.java
+++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/Token.java
@@ -21,6 +21,8 @@ import org.eclipse.xtext.TerminalRule;
 public interface Token {
 	String negatedCondition();
 	
+	AbstractElement getElement();
+	
 	static boolean isToken(AbstractElement element) {
 		if (element == null) {
 			return true;
@@ -43,7 +45,7 @@ public interface Token {
 		} else if (element instanceof RuleCall) {
 			AbstractRule rule = ((RuleCall) element).getRule();
 			if (rule instanceof TerminalRule) {
-				return new TerminalRuleToken((TerminalRule) rule, position);
+				return new TerminalRuleToken((RuleCall) element, (TerminalRule) rule, position);
 			}
 		} else if (element instanceof EnumLiteralDeclaration) {
 			return new KeywordToken(((EnumLiteralDeclaration) element).getLiteral(), position);
diff --git a/org.eclipse.xtext/src/org/eclipse/xtext/GrammarUtil.java b/org.eclipse.xtext/src/org/eclipse/xtext/GrammarUtil.java
index aa985911e..b7c4429c5 100644
--- a/org.eclipse.xtext/src/org/eclipse/xtext/GrammarUtil.java
+++ b/org.eclipse.xtext/src/org/eclipse/xtext/GrammarUtil.java
@@ -709,20 +709,16 @@ public class GrammarUtil {
 		return null;
 	}
 	
-	public static AbstractElement cloneAbstractElement(AbstractElement element) {
-		return (AbstractElement) EcoreUtil.copy(element);
-	}
-	
-	public static void addElementsToCompoundElement(CompoundElement element, Stream<? extends AbstractElement> elements) {
+	public static void setElementsOfCompoundElement(CompoundElement element, Stream<? extends AbstractElement> elements) {
 		EStructuralFeature compoundElementElementsFeature = XtextPackage.Literals.COMPOUND_ELEMENT.getEStructuralFeature(XtextPackage.COMPOUND_ELEMENT__ELEMENTS);
 		
 		element.eSet(compoundElementElementsFeature, elements
-				.map(GrammarUtil::cloneAbstractElement)
+				.map(EcoreUtil::copy)
 				.collect(Collectors.toList()));
 	}
 	
-	public static void addElementsToCompoundElement(CompoundElement element, Collection<? extends AbstractElement> elements) {
-		addElementsToCompoundElement(element, elements.stream());
+	public static void setElementsOfCompoundElement(CompoundElement element, Collection<? extends AbstractElement> elements) {
+		setElementsOfCompoundElement(element, elements.stream());
 	}
 	
 	private static void findAllRuleCalls(List<RuleCall> calls, AbstractElement element, AbstractRule rule) {