cleanup in HoistingProcessor and TokenAnalysis

org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/HoistingProcessor.java

@@ -223,8 +223,6 @@ public class HoistingProcessor {
 		return flattened;
 	}
 	
-	boolean hasSeen = false;
-	
 	private HoistingGuard findGuardForAlternatives(CompoundElement alternatives, AbstractRule currentRule, boolean skipCache) {
 		if (config.isDebug())
 			log.info("find guard for alternative");
@@ -315,11 +313,6 @@ public class HoistingProcessor {
 					throw new NestedPrefixAlternativesException("nested prefix alternatives can't be analysed because of too many paths");
 				}
 				
-				/*if (hasSeen) {
-					throw new RuntimeException();
-				}
-				hasSeen=true;*/
-				
 				return findGuardForAlternatives(flattened, currentRule, true);
 			} catch(TokenAnalysisAbortedException e) {
 				throw new TokenAnalysisAbortedException(e.getMessage(), e, currentRule);
@@ -335,8 +328,6 @@ public class HoistingProcessor {
 		// if A and B are optional the guard for the alternatives need to check the context
 		// if not the alternatives are actual alternatives
 		
-		// TODO: check if hasTerminal is valid
-		
 		return findGuardForAlternatives(element, currentRule, skipCache);
 	}
 	
@@ -347,10 +338,8 @@ public class HoistingProcessor {
 			HoistingGuard guard = findGuardForElement(element, currentRule, skipCache);
 			groupGuard.add(guard);
 			
-			// if cardinality is + and there is a terminal we don't need to consider
+			// no need to consider the following elements if the current one has a terminal
-			// the following elements
+			// following elements can't be hoisted anyway
-			// if cardinality is + and there is no token the guard won't change even if the
-			// element could be repeated
 			if (guard.hasTerminal()) {
 				groupGuard.setHasTerminal();
 				break;
@@ -360,72 +349,17 @@ public class HoistingProcessor {
 		return groupGuard;
 	}
 	
-	// TODO: make private
 	public HoistingGuard findGuardForRule(AbstractRule rule) {
+		// public so the AntlrGrammarGenerator can access it if debug mode is enabled
+		
 		if (config.isDebug())
 			log.info("finding guard for rule: " + rule.getName());
 		return findGuardForElement(rule.getAlternatives(), rule, false);
 	}
 	
-	private boolean pathHasTokenOrAction(AbstractElement path) {
-		// we are only interested in whether or not there could be any tokens on this path
-		// -> ignore cardinality
-		
-		if (Token.isToken(path)) {
-			return true;
-		} else if (isEnumRuleCall(path)) {
-			return true;
-		} else if (path instanceof JavaAction) {
-			return true;
-		} else if (isParserRuleCall(path)) {
-			// can not be self recursive since ANTLR uses LL(*) and therefore does not support left recursion
-			return pathHasTokenOrAction(((RuleCall) path).getRule().getAlternatives());
-		} else if (path instanceof Assignment) {
-			return pathHasTokenOrAction(((Assignment) path).getTerminal());
-		} else if (path instanceof CompoundElement) {
-			return ((CompoundElement) path).getElements().stream()
-					.anyMatch(this::pathHasTokenOrAction);
-		} else {
-			// Actions, Predicates, ...
-			return false;
-		}
-	}
-	
-	private boolean pathHasHoistablePredicate(AbstractElement path) {
-		// we are only interested in whether or not there could be any hoistable predicate on this path
-		// -> ignore cardinality
-		
-		if (path instanceof AbstractSemanticPredicate) {
-			return true;
-		} else if (isParserRuleCall(path)) {
-			// can not be self recursive since ANTLR uses LL(*) and therefore does not support left recursion
-			return pathHasHoistablePredicate(((RuleCall) path).getRule().getAlternatives());
-		} else if (path instanceof Assignment) {
-			return pathHasHoistablePredicate(((Assignment) path).getTerminal());
-		} else if (path instanceof Alternatives ||
-		           path instanceof UnorderedGroup) {
-			return ((CompoundElement) path).getElements().stream()
-					.anyMatch(this::pathHasHoistablePredicate);
-		} else if (path instanceof Group) {
-			for (AbstractElement element : ((CompoundElement) path).getElements()) {
-				if (pathHasHoistablePredicate(element)) {
-					return true;
-				}
-				if (pathHasTokenOrAction(element)) {
-					return false;
-				}
-			}
-			return false;
-		} else {
-			// Tokens, EnumRule, Actions, JavaActions, ...
-			return false;
-		}
-	}
-	
 	public HoistingGuard findHoistingGuardIgnoreCardinality(AbstractElement element) {
 		if (config.isDebug())
 			log.info("hoisting (trivial) guard of: \n" + abstractElementToString(element));
-		// should only be called for valid AST elements, so element can never be floating
 		
 		AbstractRule rule = containingParserRule(element);
 		if (element instanceof UnorderedGroup) {
@@ -439,12 +373,10 @@ public class HoistingProcessor {
 		if (config.isDebug())
 			log.info("hoisting guard of: \n" + abstractElementToString(element));
 		
-		// should only be called for valid AST elements, so element can never be floating
 		return findGuardForElement(element, containingParserRule(element), false);
 	}
 	
 	private HoistingGuard findGuardForElement(AbstractElement element, AbstractRule currentRule, boolean skipCache) {
-
 		String path = null; 
 		HoistingGuard guard;
 
@@ -458,7 +390,8 @@ public class HoistingProcessor {
 			}
 		}
 		
-		if (isTrivialCardinality(element) ||
+		if (
+			isTrivialCardinality(element) || 
 			isOneOrMoreCardinality(element)
 		) {
 			guard = findGuardForElementWithTrivialCardinality(element, currentRule, skipCache);
@@ -483,14 +416,18 @@ public class HoistingProcessor {
 		} else if (element instanceof Group) {
 			return findGuardForGroup((Group) element, currentRule, skipCache);
 		} else if (element instanceof AbstractSemanticPredicate) {
-			return new PredicateGuard((AbstractSemanticPredicate) element);
+			// TODO: change to GatedSemanticPredicate
+			return new PredicateGuard((AbstractSemanticPredicate) element);	
 		} else if (Token.isToken(element)) {
 			return HoistingGuard.terminal();
-		} else if (isParserRuleCall(element) ||
+		} else if (
-		           isEnumRuleCall(element)) {
+			isParserRuleCall(element) ||
+			isEnumRuleCall(element)
+		) {
 			RuleCall call = (RuleCall) element;
 			return findGuardForRule(call.getRule());
 		} else if (element instanceof Action) {
+			// this is an Xtext action, not a JavaAction
 			return HoistingGuard.unguarded();
 		} else if (element instanceof JavaAction) {
 			return HoistingGuard.action();
@@ -499,12 +436,7 @@ public class HoistingProcessor {
 		} else if (element instanceof Assignment) {
 			return findGuardForElement(((Assignment) element).getTerminal(), currentRule, skipCache);
 		} else {
-			if (!pathHasHoistablePredicate(currentRule.getAlternatives())) {
+			throw new UnsupportedConstructException("element not supported: " + element.toString(), currentRule);
-				// unsupported construct but rule doesn't contain hoistable predicates
-				return HoistingGuard.unguarded();
-			} else {
-				throw new UnsupportedConstructException("element not supported: " + element.toString(), currentRule);
-			}
 		}
 	}
 }

org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/exceptions/EmptyRecursionPathInContextAnalysisException.java

@@ -1,47 +0,0 @@
-/*******************************************************************************
- * Copyright (c) 2022 itemis AG (http://www.itemis.eu) and others.
- * This program and the accompanying materials are made available under the
- * terms of the Eclipse Public License 2.0 which is available at
- * http://www.eclipse.org/legal/epl-2.0.
- * 
- * SPDX-License-Identifier: EPL-2.0
- *******************************************************************************/
-package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.exceptions;
-
-import org.eclipse.xtext.AbstractRule;
-
-/**
- * @author overflow - Initial contribution and API
- */
-public class EmptyRecursionPathInContextAnalysisException extends HoistingException {
-	private static final long serialVersionUID = 3820353828411603508L;
-
-	public EmptyRecursionPathInContextAnalysisException() {
-		super();
-	}
-
-	public EmptyRecursionPathInContextAnalysisException(String message, AbstractRule rule) {
-		super(message, rule);
-	}
-
-	public EmptyRecursionPathInContextAnalysisException(String message, Throwable cause, AbstractRule rule) {
-		super(message, cause, rule);
-	}
-
-	public EmptyRecursionPathInContextAnalysisException(String message, Throwable cause) {
-		super(message, cause);
-	}
-
-	public EmptyRecursionPathInContextAnalysisException(String message) {
-		super(message);
-	}
-
-	public EmptyRecursionPathInContextAnalysisException(Throwable cause, AbstractRule rule) {
-		super(cause, rule);
-	}
-
-	public EmptyRecursionPathInContextAnalysisException(Throwable cause) {
-		super(cause);
-	}
-
-}

org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/exceptions/OptionalCardinalityWithoutContextException.java

@@ -1,47 +0,0 @@
-/*******************************************************************************
- * Copyright (c) 2021 itemis AG (http://www.itemis.eu) and others.
- * This program and the accompanying materials are made available under the
- * terms of the Eclipse Public License 2.0 which is available at
- * http://www.eclipse.org/legal/epl-2.0.
- * 
- * SPDX-License-Identifier: EPL-2.0
- *******************************************************************************/
-package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.exceptions;
-
-import org.eclipse.xtext.AbstractRule;
-
-/**
- * @author overflow - Initial contribution and API
- */
-public class OptionalCardinalityWithoutContextException extends UnsupportedConstructException {
-	private static final long serialVersionUID = -3958747238452206971L;
-
-	public OptionalCardinalityWithoutContextException() {
-		super();
-	}
-
-	public OptionalCardinalityWithoutContextException(String message, AbstractRule rule) {
-		super(message, rule);
-	}
-
-	public OptionalCardinalityWithoutContextException(String message, Throwable cause, AbstractRule rule) {
-		super(message, cause, rule);
-	}
-
-	public OptionalCardinalityWithoutContextException(String message, Throwable cause) {
-		super(message, cause);
-	}
-
-	public OptionalCardinalityWithoutContextException(String message) {
-		super(message);
-	}
-
-	public OptionalCardinalityWithoutContextException(Throwable cause, AbstractRule rule) {
-		super(cause, rule);
-	}
-
-	public OptionalCardinalityWithoutContextException(Throwable cause) {
-		super(cause);
-	}
-	
-}

org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/exceptions/SymbolicAnalysisFailedException.java

@@ -1,17 +0,0 @@
-/*******************************************************************************
- * Copyright (c) 2021 itemis AG (http://www.itemis.eu) and others.
- * This program and the accompanying materials are made available under the
- * terms of the Eclipse Public License 2.0 which is available at
- * http://www.eclipse.org/legal/epl-2.0.
- * 
- * SPDX-License-Identifier: EPL-2.0
- *******************************************************************************/
-package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.exceptions;
-
-/**
- * @author overflow - Initial contribution and API
- */
-public class SymbolicAnalysisFailedException extends Exception {
-	private static final long serialVersionUID = 4185473673062321988L;
-
-}

org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/pathAnalysis/TokenAnalysis.java

@@ -8,6 +8,7 @@
  *******************************************************************************/
 package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.pathAnalysis;
 
+import static org.eclipse.xtext.GrammarUtil.*;
 import static org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.utils.DebugUtils.*;
 
 import java.util.ArrayList;
@@ -38,14 +39,11 @@ import org.eclipse.xtext.UnorderedGroup;
 import org.eclipse.xtext.util.XtextSwitch;
 import org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.HoistingConfiguration;
 import org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.exceptions.NestedPrefixAlternativesException;
-import org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.exceptions.SymbolicAnalysisFailedException;
 import org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.exceptions.TokenAnalysisAbortedException;
 import org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.exceptions.UnsupportedConstructException;
 import org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.token.Token;
 import org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.utils.DebugUtils;
-import org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.utils.MutablePrimitiveWrapper;
+import org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.utils.MutableWrapper;
-
-import static org.eclipse.xtext.GrammarUtil.*;
 
 /**
  * @author overflow - Initial contribution and API
@@ -71,15 +69,17 @@ public class TokenAnalysis {
 		AbstractElement _last = last;
 		AbstractElement container = last;
 		
-		while(container != null && (
+		while(
+			container != null && (
 				container == last ||
 				!(container instanceof CompoundElement) ||
 				container instanceof Alternatives
 			)
 		) {
 			EObject _container = _last;
-			while (_container == _last || 
+			while (
-			       !(_container instanceof AbstractElement)
+				_container == _last || 
+				!(_container instanceof AbstractElement)
 			) {
 				_container = _container.eContainer();
 				if (_container == null)
@@ -89,7 +89,7 @@ public class TokenAnalysis {
 			container = (AbstractElement) _container;
 			
 			if (considerCardinalities && last != _last && isMultipleCardinality(_last)) {
-				// last is + or * quantified
+				// last is + or * quantified, so it may be repeated -> add to result
 				result.add(_last);
 			}
 		}
@@ -100,23 +100,22 @@ public class TokenAnalysis {
 		if (compoundContainer == null) {
 			if (config.isDebug())
 				log.info("no container");
+			
 			// no container element; this is last element in a rule definition
 			AbstractRule rule = containingRule(last);
-			List<RuleCall> calls = findAllRuleCalls(grammar, rule).stream()
-					.filter(Predicate.not(visited::contains))
-					.collect(Collectors.toList());
 			
 			if (isStartRule(grammar, rule)) {
 				// context is EOF
 				result.add(null);
 			}
 			
-			for (RuleCall call : calls) {
+			findAllRuleCalls(grammar, rule).stream()
-				Set<AbstractElement> _visited = new HashSet<>(visited);
+				.filter(Predicate.not(visited::contains))
-				_visited.add(call);
+				.forEach(call -> {
-				result.addAll(getNextElementsInContext(call, considerCardinalities, _visited));
+					Set<AbstractElement> _visited = new HashSet<>(visited);
-			}
+					_visited.add(call);
-			
+					result.addAll(getNextElementsInContext(call, considerCardinalities, _visited));
+				});
 		} else if (compoundContainer instanceof Group) {
 			if (config.isDebug())
 				log.info("group container");
@@ -134,11 +133,13 @@ public class TokenAnalysis {
 			// skip simple non-token-elements
 			while (index < size - 1) {
 				next = elements.get(index + 1);
-				if (!(
+				if (
+					!(
 						(next instanceof Action) ||
 						(next instanceof JavaAction) ||
 						(next instanceof AbstractSemanticPredicate)
-				)) {
+					)
+				) {
 					break;
 				}
 					
@@ -175,9 +176,10 @@ public class TokenAnalysis {
 		return getTokenPathsContext(last, prefix, true, new HashSet<>());
 	}
 	
-	private TokenAnalysisPaths getTokenPathsContext(AbstractElement last, TokenAnalysisPaths prefix, boolean considerCardinalities, Set<AbstractElement> callStack) {
+	private TokenAnalysisPaths getTokenPathsContext(AbstractElement last, TokenAnalysisPaths prefix, boolean considerCardinalities, Set<AbstractElement> visitedElements) {
 		if (config.isDebug())
-			log.info("get context for: " + abstractElementToShortString(last) + (considerCardinalities ? " with" : " without") + " cardinalities");
+			log.info("get context for: " + abstractElementToShortString(last) + 
+				(considerCardinalities ? " with" : " without") + " cardinalities");
 		
 		List<AbstractElement> context = getNextElementsInContext(last, considerCardinalities);
 		
@@ -192,11 +194,13 @@ public class TokenAnalysis {
 				log.info("context element: " + abstractElementToShortString(element));
 			TokenAnalysisPaths path = new TokenAnalysisPaths(prefix);
 			path.resetProgress();
+			
 			// shortcut endless loops instead of throwing exception
 			path = getTokenPaths(element, path, false, false, true);
+			
 			if (!path.isDone() && element != null) {
 				boolean _considerCardinalities = considerCardinalities;
-				if (callStack.contains(element) && !path.hasProgress()) {
+				if (visitedElements.contains(element) && !path.hasProgress()) {
 					if (_considerCardinalities) {
 						_considerCardinalities = false;
 					} else {
@@ -206,9 +210,9 @@ public class TokenAnalysis {
 						continue;
 					}
 				}
-				Set<AbstractElement> localCallStack = new HashSet<>(callStack);
+				Set<AbstractElement> localVisitedElements = new HashSet<>(visitedElements);
-				localCallStack.add(element);
+				localVisitedElements.add(element);
-				path = getTokenPathsContext(element, path, _considerCardinalities, localCallStack);
+				path = getTokenPathsContext(element, path, _considerCardinalities, localVisitedElements);
 			}
 			if (path.isDone()) {
 				result = result.merge(path);
@@ -221,12 +225,16 @@ public class TokenAnalysis {
 		}
 		
 		if (actualNumberOfElements == 0) {
-			// TODO: is this special case n)ecessary?
+			// necessary in case all possible path are ignored
+			// (considerCardinalities is false, all elements were visited and there was no progress)
+
+			if (config.isDebug())
+				log.info("context analysis failed: no context");
 			throw new TokenAnalysisAbortedException("context analysis failed: no context");
 		}
 		
 		if (config.isDebug())
-			log.info("done");
+			log.info("context analysis done");
 		return result;
 	}
 	
@@ -314,12 +322,13 @@ public class TokenAnalysis {
 			};
 			@Override
 			public TokenAnalysisPaths caseRuleCall(RuleCall call) {
-				if (isParserRuleCall(call) || 
+				if (
-				    isEnumRuleCall(call)
+					isParserRuleCall(call) || 
+					isEnumRuleCall(call)
 				) {
 					return getTokenPaths(call.getRule().getAlternatives(), prefix, false, false, shortcutEndlessLoops);
 				} else {
-					// go to default case
+					// probably terminal rule call -> go to default case
 					return null;
 				}
 			};
@@ -333,7 +342,7 @@ public class TokenAnalysis {
 					return result;
 				} else {
 					// Actions, Predicates, JavaActions, ...
-					// create new state with out empty flag
+					// create new state without empty flag
 					return new TokenAnalysisPaths(prefix);
 				}
 			};
@@ -363,6 +372,7 @@ public class TokenAnalysis {
 			return result;
 		}
 		
+		// use actual cardinality from path
 		return getTokenPaths(path, path.getCardinality(), prefix, analyseContext, needsLength, shortcutEndlessLoops);
 	}
 	
@@ -441,16 +451,24 @@ public class TokenAnalysis {
 		return result;
 	}
 	
-	private List<List<Token>> getTokenPaths(AbstractElement path, List<Integer> indexes, boolean analyseContext) throws TokenAnalysisAbortedException {
+	private TokenAnalysisPaths getTokenPaths(AbstractElement path, List<Integer> indexes, boolean analyseContext, boolean needsLength, boolean shortcutEndlessLoops) {
-		return getTokenPaths(path, new TokenAnalysisPaths(indexes), analyseContext, true, false).getTokenPaths();
+		return getTokenPaths(path, new TokenAnalysisPaths(indexes), analyseContext, needsLength, shortcutEndlessLoops);
-	}
-
-	private List<List<Token>> getTokenPaths(AbstractElement path, String virtualCardinality, List<Integer> indexes, boolean analyseContext) throws TokenAnalysisAbortedException {
-		return getTokenPaths(path, virtualCardinality, new TokenAnalysisPaths(indexes), analyseContext, true, false).getTokenPaths();
 	}
 	
-	private List<List<Token>> getTokenPathsContextOnly(AbstractElement path, List<Integer> indexes) {
+	private TokenAnalysisPaths getTokenPathsNoLength(AbstractElement path, List<Integer> indexes) {
-		return getTokenPathsContext(path, new TokenAnalysisPaths(indexes)).getTokenPaths();
+		return getTokenPaths(path, new TokenAnalysisPaths(indexes), false, false, false);
+	}
+	
+	private TokenAnalysisPaths getTokenPathsContext(AbstractElement path, List<Integer> indexes) {
+		return getTokenPaths(path, new TokenAnalysisPaths(indexes), true, true, false);
+	}
+	
+	private TokenAnalysisPaths getTokenPaths(AbstractElement path, String virtualCardinality, List<Integer> indexes, boolean analyseContext, boolean needsLength, boolean shortcutEndlessLoops) {
+		return getTokenPaths(path, virtualCardinality, new TokenAnalysisPaths(indexes), analyseContext, needsLength, shortcutEndlessLoops);
+	}
+	
+	private TokenAnalysisPaths getTokenPathsContextOnly(AbstractElement path, List<Integer> indexes) throws TokenAnalysisAbortedException {
+		return getTokenPathsContext(path, new TokenAnalysisPaths(indexes));
 	}
 	
 	private List<Integer> range(int i, int j) {
@@ -463,50 +481,47 @@ public class TokenAnalysis {
 			log.info("path2: " + abstractElementToString(path2));
 		}
 		
-		int i;
 		// + 1, because otherwise identical paths of length token limit can't be checked
 		int limit = config.getTokenLimit() + 1;
-		for (i = 0; i < limit; i++) {
+		for (int i = 0; i < limit; i++) {
 			TokenAnalysisPaths tokenPaths1;
 			TokenAnalysisPaths tokenPaths2;
 			
 			List<Integer> range = range(0,  i);
 			
 			// there shouldn't be a TokenAnalysisAbortedException if needsLength is false
-			tokenPaths1 = getTokenPaths(path1, new TokenAnalysisPaths(range), false, false, false);
+			tokenPaths1 = getTokenPathsNoLength(path1, range);
-			tokenPaths2 = getTokenPaths(path2, new TokenAnalysisPaths(range), false, false, false);
+			tokenPaths2 = getTokenPathsNoLength(path2, range);
 			
 			Set<Set<Token>> tokenListSet1 = tokenPaths1.getTokenPaths().stream().map(HashSet::new).collect(Collectors.toSet());
 			Set<Set<Token>> tokenListSet2 = tokenPaths2.getTokenPaths().stream().map(HashSet::new).collect(Collectors.toSet());
 			
-			int maxPosition1 = tokenPaths1.getMaxPosition();
-			int maxPosition2 = tokenPaths2.getMaxPosition();
-			
 			if (!tokenListSet1.equals(tokenListSet2)) {
 				return false;
 			}
 			
-			if (maxPosition1 < i + 1) {
+			if (tokenPaths1.getMaxPosition() < i + 1) {
 				// different max positions would have failed the equals-Operation
 				// if the max position is smaller than i + 1 the end of the path has been reached
+				// -> the paths are identical
 				return true;
 			}
 		}
 		
 		// token analysis limit reached
 		// we can't analyze the paths any further
-		// TODO maybe assume paths are equal and show warning instead of exception
 		throw new TokenAnalysisAbortedException("token limit exhausted while looking for identical paths");
 	}
 	
 	private void tokenCombinations(Function<List<Integer>, Boolean> callback) {
-		MutablePrimitiveWrapper<Integer> limit = new MutablePrimitiveWrapper<>(config.getTokenLimit());
+		MutableWrapper<Integer> limit = new MutableWrapper<>(config.getTokenLimit());
 		
 		for (int i = 1; i <= limit.get(); i++) {
 			if (tokenCombinations(0, 0, i, callback, limit)) {
 				return;
 			}
 		}
+		
 		// we tried all possible combinations
 		// -> abort
 		if (limit.get().equals(config.getTokenLimit())) {
@@ -517,10 +532,11 @@ public class TokenAnalysis {
 			throw new NestedPrefixAlternativesException();
 		}
 	}
-	private boolean tokenCombinations(long prefix, int prefixLength, int ones, Function<List<Integer>, Boolean> callback, MutablePrimitiveWrapper<Integer> limit) {
+	private boolean tokenCombinations(long prefix, int prefixLength, int ones, Function<List<Integer>, Boolean> callback, MutableWrapper<Integer> limit) {
 		if (ones == 0) {
 			List<Integer> indexes = new ArrayList<>(limit.get());
-			for (int i = 0; i < limit.get(); i++) {
+			int l = limit.get();
+			for (int i = 0; i < l; i++) {
 				if ((prefix & (1 << i)) > 0) {
 					indexes.add(i);
 				}
@@ -530,6 +546,7 @@ public class TokenAnalysis {
 			// prefix is too long
 			return false;
 		} else {
+			// we can not cache the value of limit here since it might be modified during the recursion
 			for (int i = prefixLength; i < limit.get() - ones + 1; i++) {
 				long current = prefix | (1 << i);
 				try {
@@ -554,16 +571,22 @@ public class TokenAnalysis {
 		// this method is for finding the path differences between the 
 		// element (with optional cardinality) and the context
 		
+		// virtualCardinality is what should be used instead of the actual cardinality of the root element
+		// this way ? and * can both be handled by this method
+		
 		// first dimension of result corresponds to the alternatives
 		// the second dimension are tokens of the alternatives
+		// no need return the context tokens
 		
-		MutablePrimitiveWrapper<List<List<Token>>> result = new MutablePrimitiveWrapper<List<List<Token>>>(null);
+		MutableWrapper<List<List<Token>>> result = new MutableWrapper<List<List<Token>>>(null);
 		
 		tokenCombinations(indexList -> {
 			
 			// no context analysis // TODO why?
-			List<List<Token>> tokenListsForPath = getTokenPaths(element, virtualCardinality, indexList, false);
+			List<List<Token>> tokenListsForPath = getTokenPaths(element, virtualCardinality, indexList, false, true, false)
-			List<List<Token>> tokenListForContext = getTokenPathsContextOnly(element, indexList);
+				.getTokenPaths();
+			List<List<Token>> tokenListForContext = getTokenPathsContextOnly(element, indexList)
+				.getTokenPaths();
 				
 			if (!tokenListsForPath.stream()
 					.anyMatch(tokenListForContext::contains)
@@ -602,7 +625,7 @@ public class TokenAnalysis {
 						if (config.isDebug())
 							log.info("next path: " + p);
 					})
-					.map(p -> getTokenPaths(p, indexList, true))
+					.map(p -> getTokenPathsContext(p, indexList).getTokenPaths())
 					.collect(Collectors.toList());
 			
 			int size = result.size();
@@ -615,7 +638,7 @@ public class TokenAnalysis {
 				if (!tokenListOfCurrentPath.stream()
 					.anyMatch(tokenList -> tokenListsForPaths.stream()
 							.filter(s -> s != tokenListOfCurrentPath)
-							// does any other path contain a similar token list (= alternative)
+							// does any other path contain a similar token list (= alternative) ?
 							.anyMatch(s -> s.contains(tokenList))
 					)
 				) {
@@ -625,6 +648,7 @@ public class TokenAnalysis {
 				
 			}
 			
+			// abort when all paths have an identifying token sequence set
 			return result.stream()
 					.allMatch(Objects::nonNull);
 		});
@@ -634,7 +658,7 @@ public class TokenAnalysis {
 	
 	public List<List<AbstractElement>> getAllPossiblePaths(AbstractElement path) {
 		// token limit + 2 so identity analysis will recognize paths that are identical up to the token limit on the flattened tree
-		return getTokenPaths(path, new TokenAnalysisPaths(range(0, config.getTokenLimit() + 2)), false, false, true)
+		return getTokenPaths(path, range(0, config.getTokenLimit() + 2), false, false, true)
 				.getTokenPaths()
 				.stream()
 				.map(l -> l.stream()

org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/utils/MutableWrapper.java

@@ -13,13 +13,13 @@ import java.util.function.Function;
 /**
  * @author overflow - Initial contribution and API
  */
-public class MutablePrimitiveWrapper<T> {
+public class MutableWrapper<T> {
 	private T value;
 	
-	public MutablePrimitiveWrapper() {
+	public MutableWrapper() {
 	}	
 	
-	public MutablePrimitiveWrapper(T value) {
+	public MutableWrapper(T value) {
 		this.value = value;
 	}