basic hoisting algorithm

token path analysis still missing; some guard render methods still
missing

org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/AlternativesGuard.java

@@ -0,0 +1,58 @@
+/*******************************************************************************
+ * Copyright (c) 2021 itemis AG (http://www.itemis.eu) and others.
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ * 
+ * SPDX-License-Identifier: EPL-2.0
+ *******************************************************************************/
+package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.stream.Collector;
+import java.util.stream.Collectors;
+
+/**
+ * @author overflow - Initial contribution and API
+ */
+public class AlternativesGuard implements HoistingGuard {
+	private List<PathGuard> paths;
+	
+	private AlternativesGuard(List<PathGuard> paths) {
+		this.paths = paths;
+	}
+	
+	@Override
+	public boolean isTrivial() {
+		return paths.stream().anyMatch(Guard::isTrivial);
+	}
+
+	@Override
+	public String render() {
+		return "(" + paths.stream()
+					.map(Guard::render)
+					.collect(Collectors.joining(" || ")) +
+				")";
+	}
+
+	@Override
+	public boolean hasTerminal() {
+		// empty paths are only allowed when all paths are empty
+		// in that case a MergedPathGuard ist returned.
+		return true;
+	}
+	
+	public static Collector<PathGuard, List<PathGuard>, AlternativesGuard> collector() {
+		return Collector.of(
+			LinkedList::new, 
+			List::add,
+			(l, r) -> {
+				l.addAll(r);
+				return l;
+			},
+			AlternativesGuard::new, 
+			Collector.Characteristics.CONCURRENT
+		);
+	}
+}

org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/GroupGuard.java

@@ -0,0 +1,54 @@
+/*******************************************************************************
+ * Copyright (c) 2021 itemis AG (http://www.itemis.eu) and others.
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ * 
+ * SPDX-License-Identifier: EPL-2.0
+ *******************************************************************************/
+package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * @author overflow - Initial contribution and API
+ */
+public class GroupGuard implements HoistingGuard {
+	private List<Guard> elementGuards = new LinkedList<>();
+	private boolean hasTerminal = false;
+	
+	public void add(Guard guard) {
+		if (!guard.isTrivial())
+			elementGuards.add(guard);
+	}
+	
+	public void setHasTerminal() {
+		hasTerminal = true;
+	}
+	
+	@Override
+	public boolean isTrivial() {
+		return elementGuards.isEmpty();
+	}
+
+	@Override
+	public String render() {
+		if (elementGuards.size() == 1) {
+			return elementGuards.get(0).render();
+		} else {
+			return "(" + 
+					elementGuards.stream()
+						.map(Guard::render)
+						.collect(Collectors.joining(" && ")) +
+					")";
+		}
+	}
+
+	@Override
+	public boolean hasTerminal() {
+		return hasTerminal;
+	}
+
+}

org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/Guard.java

@@ -0,0 +1,17 @@
+/*******************************************************************************
+ * Copyright (c) 2021 itemis AG (http://www.itemis.eu) and others.
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ * 
+ * SPDX-License-Identifier: EPL-2.0
+ *******************************************************************************/
+package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting;
+
+/**
+ * @author overflow - Initial contribution and API
+ */
+public interface Guard {
+	boolean isTrivial();
+	String render();
+}

org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/Hoisting.java

@@ -0,0 +1,292 @@
+/*******************************************************************************
+ * Copyright (c) 2021 itemis AG (http://www.itemis.eu) and others.
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ * 
+ * SPDX-License-Identifier: EPL-2.0
+ *******************************************************************************/
+package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.eclipse.xtext.AbstractElement;
+import org.eclipse.xtext.AbstractRule;
+import org.eclipse.xtext.AbstractSemanticPredicate;
+import org.eclipse.xtext.Action;
+import org.eclipse.xtext.Alternatives;
+import org.eclipse.xtext.Assignment;
+import org.eclipse.xtext.EnumRule;
+import org.eclipse.xtext.Group;
+import org.eclipse.xtext.Keyword;
+import org.eclipse.xtext.ParserRule;
+import org.eclipse.xtext.RuleCall;
+import org.eclipse.xtext.TerminalRule;
+import org.eclipse.xtext.UnorderedGroup;
+import org.eclipse.xtext.util.Tuples;
+import org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.utils.StreamUtils;
+
+/**
+ * @author overflow - Initial contribution and API
+ */
+public class Hoisting {
+	private Map<String, HoistingGuard> ruleCache = new HashMap<>();
+	private Map<Group, HoistingGuard> groupCache = new HashMap<>();
+
+	private boolean isParserRule(AbstractElement element) {
+		return (element instanceof RuleCall) && (((RuleCall) element).getRule() instanceof ParserRule);
+	}
+	
+	private boolean cardinalityAllowsEmpty(AbstractElement element) {
+		String cardinality = element.getCardinality();
+		return cardinality.equals("?") || cardinality.equals("*");
+	}
+	
+	private boolean cardinalityAllowsRepetition(AbstractElement element) {
+		String cardinality = element.getCardinality();
+		return cardinality.equals("+") || cardinality.equals("*");
+	}
+	
+	private TokenAnalysisPaths getTokenForIndexes(Alternatives path, TokenAnalysisPaths prefix, boolean needsLength) throws TokenAnalysisAbortedException {
+		if (prefix.isDone()) {
+			return prefix;
+		}
+		
+		TokenAnalysisPaths result;
+		if (cardinalityAllowsEmpty(path)) {
+			result = prefix;
+			if (needsLength) {
+				// analysis is not done but there are no more mandatory tokens
+				throw new TokenAnalysisAbortedException();
+			}
+		} else {
+			result = TokenAnalysisPaths.empty();
+		}
+		
+		boolean loop = cardinalityAllowsRepetition(path);
+		
+		do {
+			boolean allDone = true;
+			
+			for (AbstractElement element : path.getElements()) {
+				TokenAnalysisPaths current = new TokenAnalysisPaths(prefix);
+				current = getTokenForIndexes(element, current, needsLength); // will check for needsLength
+				
+				if (!current.isDone()) {
+					allDone = false;
+				}
+				
+				result = result.merge(current);
+			}
+			
+			if (allDone) {
+				break;
+			}
+			
+			prefix = result;
+			
+			// repeat until all further extensions of prefix are done 
+		} while(loop);
+	
+		return result;
+	}
+	
+	private TokenAnalysisPaths getTokenForIndexes(Group path, TokenAnalysisPaths prefix, boolean needsLength) throws TokenAnalysisAbortedException {
+		if (prefix.isDone()) {
+			return prefix;
+		}
+		
+		TokenAnalysisPaths result; 
+		TokenAnalysisPaths current = new TokenAnalysisPaths(prefix);
+		
+		if (cardinalityAllowsEmpty(path)) {
+			result = prefix;
+			if (needsLength) {
+				// analysis is not done but there are no more mandatory tokens
+				throw new TokenAnalysisAbortedException();
+			}
+		} else {
+			result = TokenAnalysisPaths.empty();
+		}
+		
+		boolean loop = cardinalityAllowsRepetition(path);
+		
+		do {
+			for (AbstractElement element : path.getElements()) {
+				current = getTokenForIndexes(element, current, false);
+				
+				if (current.isDone()) {
+					// no need to look further
+					return result.merge(current);
+				}
+			}
+			
+			if (needsLength && !current.isDone()) {
+				// analysis is not done but there are no more mandatory tokens
+				throw new TokenAnalysisAbortedException();
+			}
+			
+			result = result.merge(current);
+			current = new TokenAnalysisPaths(result);
+		} while(loop);
+		
+		// if cardinality is trivial or ? return result
+		return result;
+	}
+	
+	private TokenAnalysisPaths getTokenForIndexes(AbstractElement path, TokenAnalysisPaths prefix, boolean needsLength) throws TokenAnalysisAbortedException {
+		if (prefix.isDone()) {
+			return prefix;
+		}
+		
+		TokenAnalysisPaths result;
+		
+		if (cardinalityAllowsEmpty(path)) {
+			result = prefix;
+			if (needsLength) {
+				throw new TokenAnalysisAbortedException();
+			}
+		} else {
+			result = TokenAnalysisPaths.empty();
+		}
+		
+		TokenAnalysisPaths current = new TokenAnalysisPaths(prefix);
+		
+		boolean loop = cardinalityAllowsRepetition(path);
+		do {
+			if (Token.isToken(path)) {
+				current.add(path);
+			} else if (isParserRule(path)) {
+				current = getTokenForIndexes(((RuleCall) path).getRule().getAlternatives(), current, false);
+			} else {
+				throw new IllegalArgumentException("unknown element: " + path.eClass().getName());
+			}
+			
+			result = result.merge(current);
+			
+			if (current.isDone()) {
+				return result;
+			}
+			
+			if (needsLength) {
+				throw new TokenAnalysisAbortedException();
+			}
+		} while(loop);
+		
+		return result;
+	}
+	
+	private Set<List<Token>> getTokenForIndexes(AbstractElement path, List<Integer> indexes) throws TokenAnalysisAbortedException {
+		return getTokenForIndexes(path, new TokenAnalysisPaths(indexes), true).getTokenPaths();
+	}
+	
+	private boolean arePathsIdenticalSymbolic(AbstractElement path1, AbstractElement path2) throws SymbolicAnalysisFailedException {
+		return false;
+	}
+	
+	private boolean arePathsIdenticalFallback(AbstractElement path1, AbstractElement path2) {
+		return false;
+	}
+	
+	private boolean arePathsIdentical(AbstractElement path1, AbstractElement path2) {
+		try {
+			return arePathsIdenticalSymbolic(path1, path2);
+		} catch (SymbolicAnalysisFailedException e) {
+			return arePathsIdenticalFallback(path1, path2);
+		}
+	}
+	
+	private List<Set<Token>> findMinimalPathDifference(List<AbstractElement> paths) {
+		return null;
+	}
+	
+	private HoistingGuard findGuardForAlternatives(Alternatives alternatives) {
+		List<AbstractElement> paths = alternatives.getElements();
+		List<MergedPathGuard> guards = paths.stream()
+				.map(this::findGuardForElement)
+				.map(MergedPathGuard::new)
+				.collect(Collectors.toList());
+		
+		int size = paths.size();
+		for (int i = 0; i < size; i++) {
+			for (int j = i + 1; j < size; j++) {
+				if (arePathsIdentical(paths.get(i), paths.get(j))) {
+					guards.get(i).add(guards.get(j));
+					
+					paths.remove(j);
+					guards.remove(j);
+					j--;
+					size--;
+				}
+			}
+		}
+		
+		if (size > 1) {
+			return StreamUtils.zip(
+				findMinimalPathDifference(paths).stream()
+					.map(s -> s.stream()
+						.map(TokenGuard::new)
+						.collect(Collectors.toSet())
+					),
+				guards.stream(),
+				(Set<TokenGuard> tokenSet, MergedPathGuard guard) -> Tuples.pair(tokenSet, guard)
+			).map(p -> new PathGuard(p.getFirst(), p.getSecond()))
+			.collect(AlternativesGuard.collector());
+		} else {
+			return guards.get(0);
+		}
+	}
+	
+	private HoistingGuard findGuardForGroup(Group group) {
+		GroupGuard groupGuard = new GroupGuard();
+		
+		for (AbstractElement element : group.getElements()) {
+			// TODO: findGuardForElement doesn's support non-trivial cardinalities
+			HoistingGuard guard = findGuardForElement(element);
+			groupGuard.add(guard);
+			
+			if (guard.hasTerminal()) {
+				groupGuard.setHasTerminal();
+				break;
+			}
+		}
+		
+		return groupGuard;
+	}
+	
+	private HoistingGuard findGuardForElement(AbstractElement element) {
+		if (element instanceof Alternatives) {
+			return findGuardForAlternatives((Alternatives) element);
+		} else if (element instanceof Group) {
+			return groupCache.computeIfAbsent((Group) element, this::findGuardForGroup);
+		} else if (element instanceof AbstractSemanticPredicate) {
+			return new PredicateGuard((AbstractSemanticPredicate) element);
+		} else if (element instanceof Keyword) {
+			return HoistingGuard.terminal();
+		} else if (element instanceof RuleCall) {
+			RuleCall call = (RuleCall) element;
+			AbstractRule rule = call.getRule();
+			if (rule instanceof TerminalRule || rule instanceof EnumRule) {
+				return HoistingGuard.terminal();
+			} else {
+				// TODO: findGuardForElement can't deal with cardinalities
+				return ruleCache.computeIfAbsent(rule.getName(), s -> findGuardForElement(rule.getAlternatives()));
+			}
+		} else if (element instanceof Action) {
+			// TODO: Maybe find better indicator for "we don't care about this element"
+			return HoistingGuard.unguarded();
+		} else if (element instanceof UnorderedGroup) {
+			// TODO: No support for Unordered Groups yet.
+			throw new UnsupportedOperationException("unordered groups are not yet supported");
+		} else if (element instanceof Assignment) {
+			// TODO: findGuardForElement can't deal with cardinalities
+			return findGuardForElement(((Assignment) element).getTerminal());
+		} else {
+			throw new UnsupportedOperationException("element not supported: " + element.toString());
+		}
+	}
+}

org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/HoistingGuard.java

@@ -0,0 +1,54 @@
+/*******************************************************************************
+ * Copyright (c) 2021 itemis AG (http://www.itemis.eu) and others.
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ * 
+ * SPDX-License-Identifier: EPL-2.0
+ *******************************************************************************/
+package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting;
+
+/**
+ * @author overflow - Initial contribution and API
+ */
+public interface HoistingGuard extends Guard {
+	boolean hasTerminal();
+	
+	static public HoistingGuard unguarded() {
+		return new HoistingGuard() {
+			@Override
+			public String render() {
+				return "true";
+			}
+			
+			@Override
+			public boolean isTrivial() {
+				return true;
+			}
+			
+			@Override
+			public boolean hasTerminal() {
+				return false;
+			}
+		};
+	}
+	
+	static public HoistingGuard terminal() {
+		return new HoistingGuard() {
+			@Override
+			public String render() {
+				return "true";
+			}
+			
+			@Override
+			public boolean isTrivial() {
+				return true;
+			}
+			
+			@Override
+			public boolean hasTerminal() {
+				return true;
+			}
+		};
+	}
+}

org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/KeywordToken.java

@@ -0,0 +1,24 @@
+/*******************************************************************************
+ * Copyright (c) 2021 itemis AG (http://www.itemis.eu) and others.
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ * 
+ * SPDX-License-Identifier: EPL-2.0
+ *******************************************************************************/
+package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting;
+
+import org.eclipse.xtext.Keyword;
+
+/**
+ * @author overflow - Initial contribution and API
+ */
+public class KeywordToken extends Token {
+	private Keyword keyword;
+	private int position;
+	
+	public KeywordToken(Keyword keyword, int position) {
+		this.keyword = keyword;
+		this.position = position;
+	}
+}

org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/MergedPathGuard.java

@@ -0,0 +1,57 @@
+/*******************************************************************************
+ * Copyright (c) 2021 itemis AG (http://www.itemis.eu) and others.
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ * 
+ * SPDX-License-Identifier: EPL-2.0
+ *******************************************************************************/
+package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * @author overflow - Initial contribution and API
+ */
+public class MergedPathGuard implements HoistingGuard {
+	private List<HoistingGuard> pathGuards = new LinkedList<>();
+	
+	public MergedPathGuard(HoistingGuard guard) {
+		add(guard);
+	}
+	
+	public void add(HoistingGuard guard) {
+		pathGuards.add(guard);
+	}
+	
+	public void add(MergedPathGuard mergedPathGuard) {
+		pathGuards.addAll(mergedPathGuard.pathGuards);
+	}
+	
+	@Override
+	public boolean isTrivial() {
+		return pathGuards.stream().anyMatch(Guard::isTrivial);
+	}
+
+	@Override
+	public String render() {
+		if (pathGuards.size() == 1) {
+			return pathGuards.get(0).render();
+		} else {
+			return "(" + 
+					pathGuards.stream()
+						.map(Guard::render)
+						.collect(Collectors.joining(" || ")) +
+					")";
+		}
+	}
+
+	@Override
+	public boolean hasTerminal() {
+		// only need to check first element since all paths should be identical
+		return pathGuards.get(0).hasTerminal();
+	}
+
+}

org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/NotATokenException.java

@@ -0,0 +1,17 @@
+/*******************************************************************************
+ * Copyright (c) 2021 itemis AG (http://www.itemis.eu) and others.
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ * 
+ * SPDX-License-Identifier: EPL-2.0
+ *******************************************************************************/
+package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting;
+
+/**
+ * @author overflow - Initial contribution and API
+ */
+public class NotATokenException extends RuntimeException {
+	private static final long serialVersionUID = 643265533068524552L;
+
+}

org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/PathGuard.java

@@ -0,0 +1,44 @@
+/*******************************************************************************
+ * Copyright (c) 2021 itemis AG (http://www.itemis.eu) and others.
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ * 
+ * SPDX-License-Identifier: EPL-2.0
+ *******************************************************************************/
+package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting;
+
+import java.util.Collection;
+import java.util.LinkedList;
+import java.util.List;
+
+/**
+ * @author overflow - Initial contribution and API
+ */
+public class PathGuard implements HoistingGuard {
+	private Collection<TokenGuard> tokenGuards;
+	private HoistingGuard hoistngGuard;
+	
+	public PathGuard(Collection<TokenGuard> tokenGuards, HoistingGuard hoistingGuard) {
+		this.tokenGuards = tokenGuards;
+		this.hoistngGuard = hoistingGuard;
+	}
+	
+	@Override
+	public boolean isTrivial() {
+		return hoistngGuard.isTrivial();
+	}
+
+	@Override
+	public boolean hasTerminal() {
+		// empty paths are only allowed when all paths are empty
+		// in that case a MergedPathGuard is returned.
+		return true;
+	}
+	
+	@Override
+	public String render() {
+		// TODO
+		return null;
+	}
+}

org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/PredicateGuard.java

@@ -0,0 +1,39 @@
+/*******************************************************************************
+ * Copyright (c) 2021 itemis AG (http://www.itemis.eu) and others.
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ * 
+ * SPDX-License-Identifier: EPL-2.0
+ *******************************************************************************/
+package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting;
+
+import org.eclipse.xtext.AbstractSemanticPredicate;
+
+/**
+ * @author overflow - Initial contribution and API
+ */
+public class PredicateGuard implements HoistingGuard {
+
+	private AbstractSemanticPredicate element;
+	
+	public PredicateGuard(AbstractSemanticPredicate element) {
+		this.element = element;
+	}
+
+	@Override
+	public boolean isTrivial() {
+		return false;
+	}
+
+	@Override
+	public String render() {
+		return element.getCode().getSource();
+	}
+
+	@Override
+	public boolean hasTerminal() {
+		return false;
+	}
+
+}

org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/SymbolicAnalysisFailedException.java

@@ -0,0 +1,17 @@
+/*******************************************************************************
+ * Copyright (c) 2021 itemis AG (http://www.itemis.eu) and others.
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ * 
+ * SPDX-License-Identifier: EPL-2.0
+ *******************************************************************************/
+package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting;
+
+/**
+ * @author overflow - Initial contribution and API
+ */
+public class SymbolicAnalysisFailedException extends Exception {
+	private static final long serialVersionUID = 4185473673062321988L;
+
+}

org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/TerminalRuleToken.java

@@ -0,0 +1,24 @@
+/*******************************************************************************
+ * Copyright (c) 2021 itemis AG (http://www.itemis.eu) and others.
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ * 
+ * SPDX-License-Identifier: EPL-2.0
+ *******************************************************************************/
+package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting;
+
+import org.eclipse.xtext.TerminalRule;
+
+/**
+ * @author overflow - Initial contribution and API
+ */
+public class TerminalRuleToken extends Token {
+	private TerminalRule rule;
+	private int position;
+	
+	public TerminalRuleToken(TerminalRule rule, int position) {
+		this.rule = rule;
+		this.position = position;
+	}
+}

org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/Token.java

@@ -0,0 +1,48 @@
+/*******************************************************************************
+ * Copyright (c) 2021 itemis AG (http://www.itemis.eu) and others.
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ * 
+ * SPDX-License-Identifier: EPL-2.0
+ *******************************************************************************/
+package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting;
+
+import org.eclipse.xtext.AbstractElement;
+import org.eclipse.xtext.AbstractRule;
+import org.eclipse.xtext.Keyword;
+import org.eclipse.xtext.RuleCall;
+import org.eclipse.xtext.TerminalRule;
+
+/**
+ * @author overflow - Initial contribution and API
+ */
+public class Token {
+	
+	static boolean isToken(AbstractElement element) {
+		if (element instanceof Keyword) {
+			return true;
+		} else if (element instanceof RuleCall) {
+			if (((RuleCall) element).getRule() instanceof TerminalRule) {
+				return true;
+			} else {
+				return false;
+			}
+		} else {
+			return false;
+		}
+	}
+	
+	static Token fromElement(AbstractElement element, int position) {
+		if (element instanceof Keyword) {
+			return new KeywordToken((Keyword) element, position);
+		} else if (element instanceof RuleCall) {
+			AbstractRule rule = ((RuleCall) element).getRule();
+			if (rule instanceof TerminalRule) {
+				return new TerminalRuleToken((TerminalRule) rule, position);
+			}
+		}
+		
+		throw new NotATokenException();
+	}
+}

org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/TokenAnalysisAbortedException.java

@@ -0,0 +1,16 @@
+/*******************************************************************************
+ * Copyright (c) 2021 itemis AG (http://www.itemis.eu) and others.
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ * 
+ * SPDX-License-Identifier: EPL-2.0
+ *******************************************************************************/
+package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting;
+
+/**
+ * @author overflow - Initial contribution and API
+ */
+public class TokenAnalysisAbortedException extends Exception {
+	private static final long serialVersionUID = 4303267001950479292L;
+}

org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/TokenAnalysisPath.java

@@ -0,0 +1,60 @@
+/*******************************************************************************
+ * Copyright (c) 2021 itemis AG (http://www.itemis.eu) and others.
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ * 
+ * SPDX-License-Identifier: EPL-2.0
+ *******************************************************************************/
+package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting;
+
+import java.util.LinkedList;
+import java.util.List;
+
+import org.eclipse.xtext.AbstractElement;
+
+/**
+ * @author overflow - Initial contribution and API
+ */
+public class TokenAnalysisPath {
+	private List<Token> path = new LinkedList<>();
+	private List<Integer> remainingIndexes;
+	private int position = 1;
+	
+	public TokenAnalysisPath(List<Integer> indexes) {
+		this.remainingIndexes = indexes;
+	}
+	
+	public TokenAnalysisPath(TokenAnalysisPath prefix) {
+		this(prefix.remainingIndexes);
+		path = new LinkedList<>(prefix.path);
+		position = prefix.position;
+	}
+	
+	public boolean isDone() {
+		return remainingIndexes.isEmpty();
+	}
+	
+	private void shift() {
+		position++;
+		int size = remainingIndexes.size();
+		for (int i = 0; i < size; i++) {
+			remainingIndexes.set(i, remainingIndexes.get(i) - 1);
+		}
+	}
+	
+	public void add(AbstractElement element) {
+		if (isDone())
+			return;
+		
+		if (remainingIndexes.get(0) <= 0) {
+			path.add(Token.fromElement(element, position));
+			remainingIndexes.remove(0);
+		}
+		shift();
+	}
+	
+	public List<Token> getTokenPath() {
+		return path;
+	}
+}

org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/TokenAnalysisPaths.java

@@ -0,0 +1,67 @@
+/*******************************************************************************
+ * Copyright (c) 2021 itemis AG (http://www.itemis.eu) and others.
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ * 
+ * SPDX-License-Identifier: EPL-2.0
+ *******************************************************************************/
+package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting;
+
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.eclipse.xtext.AbstractElement;
+
+/**
+ * @author overflow - Initial contribution and API
+ */
+public class TokenAnalysisPaths {
+	private Set<TokenAnalysisPath> tokenPaths = new HashSet<>();
+	private boolean isEmpty = false;
+	
+	public Set<List<Token>> getTokenPaths() {
+		return tokenPaths.stream()
+				.map(TokenAnalysisPath::getTokenPath)
+				.collect(Collectors.toSet());
+	}
+	
+	private TokenAnalysisPaths() {
+	}
+	
+	public TokenAnalysisPaths(List<Integer> indexes) {
+		tokenPaths.add(new TokenAnalysisPath(indexes));
+	}
+	
+	public TokenAnalysisPaths(TokenAnalysisPaths prefix) {
+		this.tokenPaths = prefix.tokenPaths.stream()
+				.map(TokenAnalysisPath::new)
+				.collect(Collectors.toSet());
+	}
+	
+	public boolean isDone() {
+		return tokenPaths.stream().allMatch(TokenAnalysisPath::isDone);
+	}
+	
+	public void add(AbstractElement element) {
+		tokenPaths.forEach(p -> p.add(element));
+	}
+	
+	public TokenAnalysisPaths merge(TokenAnalysisPaths other) {
+		if (isEmpty) {
+			return other;
+		} else {
+			this.tokenPaths.addAll(other.tokenPaths);
+			return this;
+		}
+	}
+	
+	static public TokenAnalysisPaths empty() {
+		TokenAnalysisPaths empty = new TokenAnalysisPaths();
+		empty.isEmpty = true;
+		return empty;
+	}
+}

org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/TokenGuard.java

@@ -0,0 +1,31 @@
+/*******************************************************************************
+ * Copyright (c) 2021 itemis AG (http://www.itemis.eu) and others.
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ * 
+ * SPDX-License-Identifier: EPL-2.0
+ *******************************************************************************/
+package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting;
+
+/**
+ * @author overflow - Initial contribution and API
+ */
+public class TokenGuard implements Guard {
+	private Token token;
+	
+	public TokenGuard(Token token) {
+		this.token = token;
+	}
+	
+	@Override
+	public boolean isTrivial() {
+		return false;
+	}
+
+	@Override
+	public String render() {
+		return "not implemented";
+	}
+
+}

org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/utils/StreamUtils.java

@@ -0,0 +1,40 @@
+/*******************************************************************************
+ * Copyright (c) 2021 itemis AG (http://www.itemis.eu) and others.
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ * 
+ * SPDX-License-Identifier: EPL-2.0
+ *******************************************************************************/
+package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.utils;
+
+import java.util.Iterator;
+import java.util.Spliterator;
+import java.util.Spliterators;
+import java.util.function.BiFunction;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
+
+/**
+ * @author overflow - Initial contribution and API
+ */
+public class StreamUtils {
+	public static <A, B, C> Stream<C> zip(Stream<A> stream1, Stream<B> stream2, BiFunction<A, B, C> combine) {
+		Iterator<A> iter1 = stream1.iterator();
+		Iterator<B> iter2 = stream2.iterator();
+		
+		Spliterator<C> spliterator = Spliterators.spliteratorUnknownSize(new Iterator<C>() {
+			@Override
+			public boolean hasNext() {
+				return iter1.hasNext() && iter2.hasNext();
+			}
+			
+			@Override
+			public C next() {
+				return combine.apply(iter1.next(), iter2.next());
+			}
+		}, Spliterator.NONNULL);
+		
+		return StreamSupport.stream(spliterator, false);
+	}
+}