hoisting algorithms are done (except symbolic analysis & hoisting

support for predicates in non-trivial cardinalities)

code generation part is still missing

org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/AlternativeTokenSequenceGuard.java

@@ -0,0 +1,44 @@
+/*******************************************************************************
+ * Copyright (c) 2021 itemis AG (http://www.itemis.eu) and others.
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ * 
+ * SPDX-License-Identifier: EPL-2.0
+ *******************************************************************************/
+package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting;
+
+import java.util.Collection;
+import java.util.stream.Collectors;
+
+/**
+ * @author overflow - Initial contribution and API
+ */
+public class AlternativeTokenSequenceGuard implements TokenGuard {
+	private Collection<? extends TokenSequenceGuard> alternatives;
+	
+	public AlternativeTokenSequenceGuard(Collection<? extends TokenSequenceGuard> alternatives) {
+		this.alternatives = alternatives;
+	}
+	
+	@Override
+	public String render() {
+		boolean addParentheses = alternatives.size() != 1;
+		String result = "";
+		
+		if (addParentheses) {
+			result += "(";
+		}
+		
+		result += alternatives.stream()
+				.map(TokenGuard::render)
+				.collect(Collectors.joining(" && "));
+
+		if (addParentheses) {
+			result += ")";
+		}
+		
+		return result;
+	}
+
+}

org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/Hoisting.java

@@ -8,11 +8,15 @@
  *******************************************************************************/
 package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting;
 
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Objects;
 import java.util.List;
 import java.util.Set;
+import java.util.function.Function;
 import java.util.stream.Collectors;
+import java.util.stream.IntStream;
 
 import org.eclipse.xtext.AbstractElement;
 import org.eclipse.xtext.AbstractRule;
@@ -36,7 +40,9 @@ import org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.utils.StreamUtils
 public class Hoisting {
 	private Map<String, HoistingGuard> ruleCache = new HashMap<>();
 	private Map<Group, HoistingGuard> groupCache = new HashMap<>();
-
+	
+	private static final int TOKEN_ANALYSIS_LIMIT = 10;
+	
 	private boolean isParserRule(AbstractElement element) {
 		return (element instanceof RuleCall) && (((RuleCall) element).getRule() instanceof ParserRule);
 	}
@@ -185,11 +191,51 @@ public class Hoisting {
 	}
 	
 	private boolean arePathsIdenticalSymbolic(AbstractElement path1, AbstractElement path2) throws SymbolicAnalysisFailedException {
-		return false;
+		// ignore symbolic analysis for the moment
+		// TODO
+		throw new SymbolicAnalysisFailedException(); 
+	}
+	
+	private List<Integer> range(int i, int j) {
+		return IntStream.rangeClosed(i, j).boxed().collect(Collectors.toList());
 	}
 	
 	private boolean arePathsIdenticalFallback(AbstractElement path1, AbstractElement path2) {
-		return false;
+		for (int i = 1; i < TOKEN_ANALYSIS_LIMIT; i++) {
+			Set<List<Token>> tokenListSet1;
+			Set<List<Token>> tokenListSet2;
+			
+			List<Integer> range = range(1,  i);
+			
+			try {
+				tokenListSet1 = getTokenForIndexes(path1, range);
+			} catch (TokenAnalysisAbortedException e) {
+				tokenListSet1 = null;
+			}
+			
+			try {
+				tokenListSet2 = getTokenForIndexes(path2, range);
+			} catch (TokenAnalysisAbortedException e) {
+				tokenListSet2 = null;
+			}
+			
+			if (tokenListSet1 == null && tokenListSet2 == null) {
+				return true;
+			}
+			if (tokenListSet1 == null || tokenListSet2 == null) {
+				// the paths have different length
+				return false;
+			}
+			if (!tokenListSet1.equals(tokenListSet2)) {
+				// TODO: hashCode method of Token classes
+				return false;
+			}
+		}
+		
+		// we can't analyze the paths any further
+		// we can assume that the paths are identical because the path diff analysis would cause an error anyway.
+		// TODO maybe warning?
+		return true;
 	}
 	
 	private boolean arePathsIdentical(AbstractElement path1, AbstractElement path2) {
@@ -200,10 +246,75 @@ public class Hoisting {
 		}
 	}
 	
-	private List<Set<Token>> findMinimalPathDifference(List<AbstractElement> paths) {
-		return null;
+	
+	private void tokenCombinations(Function<List<Integer>, Boolean> callback) {
+		for (int i = 1; i <= TOKEN_ANALYSIS_LIMIT; i++) {
+			if (tokenCombinations(0, 0, i, callback)) {
+				return;
+			}
+		}
+	}
+	private boolean tokenCombinations(long prefix, int prefixLength, int ones, Function<List<Integer>, Boolean> callback) {
+		if (ones == 0) {
+			List<Integer> indexes = new ArrayList<>(TOKEN_ANALYSIS_LIMIT);
+			for (int i = 0; i < TOKEN_ANALYSIS_LIMIT; i++) {
+				if ((prefix & (1 << i)) > 0) {
+					indexes.add(i + 1);
+				}
+			}
+			return callback.apply(indexes);
+		} else if (prefixLength + ones > TOKEN_ANALYSIS_LIMIT) {
+			// prefix is too long
+			return false;
+		} else {
+			for (int i = prefixLength; i < TOKEN_ANALYSIS_LIMIT - ones + 1; i++) {
+				long current = prefix | (1 << i);
+				if (tokenCombinations(current, i + 1, ones - 1, callback)) {
+					return true;
+				}
+			}
+			return false;
+		}
 	}
 	
+	private List<Set<List<Token>>> findMinimalPathDifference(List<AbstractElement> paths) throws TokenAnalysisAbortedException {
+		List<Set<List<Token>>> result = paths.stream()
+				.map(p -> (Set<List<Token>>) null)
+				.collect(Collectors.toList());
+		
+		tokenCombinations(indexList -> {
+			// will throw TokenAnalysisAborted if any path is too short
+			List<Set<List<Token>>> tokenListSets = paths.stream()
+					.map(p -> getTokenForIndexes(p, indexList))
+					.collect(Collectors.toList());
+			
+			int size = result.size();
+			for (int i = 0; i < size; i++) {
+				if (result.get(i) == null) {
+					continue;
+				}
+				
+				Set<List<Token>> tokenSet = tokenListSets.get(i);
+				if (!tokenSet.stream()
+					.anyMatch(tokenList -> tokenListSets.stream()
+							.filter(s -> s != tokenSet)
+							.anyMatch(s -> s.contains(tokenList))
+					)
+				) {
+					// token list set is unique for path i
+					result.set(i, tokenSet);
+				}
+				
+			}
+			
+			return result.stream()
+					.allMatch(Objects::nonNull);
+		});
+		
+		return result;
+	}
+	
+	// TODO: handling for TokenAnalysisAbortedException
 	private HoistingGuard findGuardForAlternatives(Alternatives alternatives) {
 		List<AbstractElement> paths = alternatives.getElements();
 		List<MergedPathGuard> guards = paths.stream()
@@ -225,15 +336,22 @@ public class Hoisting {
 			}
 		}
 		
+		// if all paths are empty the above step will eliminate all paths
+		// -> size = 1
 		if (size > 1) {
 			return StreamUtils.zip(
 				findMinimalPathDifference(paths).stream()
-					.map(s -> s.stream()
-						.map(TokenGuard::new)
+					.map(a -> a.stream()
+						.map(s -> s.stream()
+								.map(SingleTokenGuard::new)
+								.collect(Collectors.toList())
+						)
+						.map(TokenSequenceGuard::new)
 						.collect(Collectors.toSet())
-					),
+					)
+					.map(AlternativeTokenSequenceGuard::new),
 				guards.stream(),
-				(Set<TokenGuard> tokenSet, MergedPathGuard guard) -> Tuples.pair(tokenSet, guard)
+				(TokenGuard tokenGuard, MergedPathGuard pathGuard) -> Tuples.pair(tokenGuard, pathGuard)
 			).map(p -> new PathGuard(p.getFirst(), p.getSecond()))
 			.collect(AlternativesGuard.collector());
 		} else {

org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/KeywordToken.java

@@ -13,7 +13,7 @@ import org.eclipse.xtext.Keyword;
 /**
  * @author overflow - Initial contribution and API
  */
-public class KeywordToken extends Token {
+public class KeywordToken implements Token {
 	private Keyword keyword;
 	private int position;
 	
@@ -21,4 +21,9 @@ public class KeywordToken extends Token {
 		this.keyword = keyword;
 		this.position = position;
 	}
+
+	@Override
+	public String negatedCondition() {
+		return "!\"" + keyword.getValue().replace("\"", "\\\"") + "\".equals(input.LT(" + position + ").getText()";
+	}
 }

org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/PathGuard.java

@@ -8,19 +8,15 @@
  *******************************************************************************/
 package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting;
 
-import java.util.Collection;
-import java.util.LinkedList;
-import java.util.List;
-
 /**
  * @author overflow - Initial contribution and API
  */
 public class PathGuard implements HoistingGuard {
-	private Collection<TokenGuard> tokenGuards;
+	private TokenGuard tokenGuard;
 	private HoistingGuard hoistngGuard;
 	
-	public PathGuard(Collection<TokenGuard> tokenGuards, HoistingGuard hoistingGuard) {
-		this.tokenGuards = tokenGuards;
+	public PathGuard(TokenGuard tokenGuard, HoistingGuard hoistingGuard) {
+		this.tokenGuard = tokenGuard;
 		this.hoistngGuard = hoistingGuard;
 	}
 	
@@ -32,13 +28,13 @@ public class PathGuard implements HoistingGuard {
 	@Override
 	public boolean hasTerminal() {
 		// empty paths are only allowed when all paths are empty
-		// in that case a MergedPathGuard is returned.
+		// in that case a MergedPathGuard is returned by findGuardForAlternatives.
 		return true;
 	}
 	
 	@Override
 	public String render() {
-		// TODO
-		return null;
+		// parentheses needed since tokenGuard is never empty
+		return "(" + tokenGuard.render() + " || " + hoistngGuard.render() + ")";
 	}
 }

org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/SingleTokenGuard.java

@@ -0,0 +1,25 @@
+/*******************************************************************************
+ * Copyright (c) 2021 itemis AG (http://www.itemis.eu) and others.
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ * 
+ * SPDX-License-Identifier: EPL-2.0
+ *******************************************************************************/
+package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting;
+
+/**
+ * @author overflow - Initial contribution and API
+ */
+public class SingleTokenGuard implements TokenGuard {
+	private Token token;
+	
+	public SingleTokenGuard(Token token) {
+		this.token = token;
+	}
+
+	@Override
+	public String render() {
+		return token.negatedCondition();
+	}
+}

org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/TerminalRuleToken.java

@@ -13,7 +13,7 @@ import org.eclipse.xtext.TerminalRule;
 /**
  * @author overflow - Initial contribution and API
  */
-public class TerminalRuleToken extends Token {
+public class TerminalRuleToken implements Token {
 	private TerminalRule rule;
 	private int position;
 	
@@ -21,4 +21,9 @@ public class TerminalRuleToken extends Token {
 		this.rule = rule;
 		this.position = position;
 	}
+
+	@Override
+	public String negatedCondition() {
+		return "input.LT(" + position + ").getType() != " + rule.getName();
+	}
 }

org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/Token.java

@@ -17,17 +17,14 @@ import org.eclipse.xtext.TerminalRule;
 /**
  * @author overflow - Initial contribution and API
  */
-public class Token {
+public interface Token {
+	String negatedCondition();
 	
 	static boolean isToken(AbstractElement element) {
 		if (element instanceof Keyword) {
 			return true;
 		} else if (element instanceof RuleCall) {
-			if (((RuleCall) element).getRule() instanceof TerminalRule) {
-				return true;
-			} else {
-				return false;
-			}
+			return (((RuleCall) element).getRule() instanceof TerminalRule);
 		} else {
 			return false;
 		}

org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/TokenAnalysisAbortedException.java

@@ -11,6 +11,6 @@ package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting;
 /**
  * @author overflow - Initial contribution and API
  */
-public class TokenAnalysisAbortedException extends Exception {
+public class TokenAnalysisAbortedException extends RuntimeException {
 	private static final long serialVersionUID = 4303267001950479292L;
 }

org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/TokenAnalysisPaths.java

@@ -9,7 +9,6 @@
 package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting;
 
 import java.util.HashSet;
-import java.util.LinkedList;
 import java.util.List;
 import java.util.Set;
 import java.util.stream.Collectors;

org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/TokenGuard.java

@@ -11,21 +11,9 @@ package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting;
 /**
  * @author overflow - Initial contribution and API
  */
-public class TokenGuard implements Guard {
-	private Token token;
-	
-	public TokenGuard(Token token) {
-		this.token = token;
-	}
-	
+public interface TokenGuard extends Guard {
 	@Override
-	public boolean isTrivial() {
+	default boolean isTrivial() {
 		return false;
 	}
-
-	@Override
-	public String render() {
-		return "not implemented";
-	}
-
 }

org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/TokenSequenceGuard.java

@@ -0,0 +1,43 @@
+/*******************************************************************************
+ * Copyright (c) 2021 itemis AG (http://www.itemis.eu) and others.
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ * 
+ * SPDX-License-Identifier: EPL-2.0
+ *******************************************************************************/
+package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting;
+
+import java.util.Collection;
+import java.util.stream.Collectors;
+
+/**
+ * @author overflow - Initial contribution and API
+ */
+public class TokenSequenceGuard implements TokenGuard {
+	private Collection<? extends TokenGuard> sequence;
+	
+	public TokenSequenceGuard(Collection<? extends TokenGuard> sequence) {
+		this.sequence = sequence;
+	}
+
+	@Override
+	public String render() {
+		boolean addParentheses = sequence.size() != 1;
+		String result = "";
+		
+		if (addParentheses) {
+			result += "(";
+		}
+		
+		result += sequence.stream()
+				.map(TokenGuard::render)
+				.collect(Collectors.joining(" || "));
+
+		if (addParentheses) {
+			result += ")";
+		}
+		
+		return result;
+	}
+}