fixed: minimal token difference returning wrong token

org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/AlternativesGuard.java

@@ -10,6 +10,7 @@ package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting;
 
 import java.util.LinkedList;
 import java.util.List;
+import java.util.function.Predicate;
 import java.util.stream.Collector;
 import java.util.stream.Collectors;
 
@@ -25,21 +26,22 @@ public class AlternativesGuard implements HoistingGuard {
 	
 	@Override
 	public boolean isTrivial() {
-		return paths.stream().anyMatch(Guard::isTrivial);
+		return paths.stream().allMatch(Guard::isTrivial);
 	}
 
 	@Override
 	public String render() {
 		return "(" + paths.stream()
+					.filter(Predicate.not(Guard::isTrivial))
 					.map(Guard::render)
-					.collect(Collectors.joining(" || ")) +
+					.collect(Collectors.joining(" && ")) +
 				")";
 	}
 
 	@Override
 	public boolean hasTerminal() {
 		// empty paths are only allowed when all paths are empty
-		// in that case a MergedPathGuard ist returned.
+		// in that case a MergedPathGuard is returned.
 		return true;
 	}
 	

org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/HoistingProcessor.java

@@ -74,7 +74,7 @@ public class HoistingProcessor {
 				throw new TokenAnalysisAbortedException();
 			}
 		} else {
-			result = TokenAnalysisPaths.empty();
+			result = TokenAnalysisPaths.empty(prefix);
 		}
 		
 		boolean loop = cardinalityAllowsRepetition(path);
@@ -120,12 +120,12 @@ public class HoistingProcessor {
 				throw new TokenAnalysisAbortedException();
 			}
 		} else {
-			result = TokenAnalysisPaths.empty();
+			result = TokenAnalysisPaths.empty(prefix);
 		}
 		
 		boolean loop = cardinalityAllowsRepetition(path);
 		
-		log.info(prefix);
+		log.info("group: " + prefix);
 		
 		do {
 			for (AbstractElement element : path.getElements()) {
@@ -133,6 +133,9 @@ public class HoistingProcessor {
 				
 				if (current.isDone()) {
 					// no need to look further
+					
+					log.info("done: " + current);
+					
 					return result.merge(current);
 				}
 			}
@@ -163,7 +166,7 @@ public class HoistingProcessor {
 				throw new TokenAnalysisAbortedException();
 			}
 		} else {
-			result = TokenAnalysisPaths.empty();
+			result = TokenAnalysisPaths.empty(prefix);
 		}
 		
 		TokenAnalysisPaths current = new TokenAnalysisPaths(prefix);
@@ -204,7 +207,7 @@ public class HoistingProcessor {
 		           path instanceof AbstractSemanticPredicate
 				) {
 			// TODO: make sure empty token analysis paths don't cause problems down the line
-			return TokenAnalysisPaths.empty();
+			return TokenAnalysisPaths.empty(prefix);
 		} else {
 			return getTokenForIndexesDefault(path, prefix, needsLength);
 		}
@@ -310,9 +313,13 @@ public class HoistingProcessor {
 			log.info("current index list: " + indexList);
 			
 			// will throw TokenAnalysisAborted if any path is too short
+			// TODO: when TokenAnalysisAborted, add new index
 			List<Set<List<Token>>> tokenListSets = paths.stream()
+					.peek(p -> log.info("next path: " + p))
 					.map(p -> getTokenForIndexes(p, indexList))
 					.collect(Collectors.toList());
+
+			log.info("sets: " + tokenListSets);
 			
 			int size = result.size();
 			for (int i = 0; i < size; i++) {
@@ -320,15 +327,16 @@ public class HoistingProcessor {
 					continue;
 				}
 				
-				Set<List<Token>> tokenSet = tokenListSets.get(i);
-				if (!tokenSet.stream()
+				Set<List<Token>> tokenListSet = tokenListSets.get(i);
+				if (!tokenListSet.stream()
 					.anyMatch(tokenList -> tokenListSets.stream()
-							.filter(s -> s != tokenSet)
+							.filter(s -> s != tokenListSet)
 							.anyMatch(s -> s.contains(tokenList))
 					)
 				) {
+					log.info("set:  " + tokenListSet);
 					// token list set is unique for path i
-					result.set(i, tokenSet);
+					result.set(i, tokenListSet);
 				}
 				
 			}
@@ -368,6 +376,8 @@ public class HoistingProcessor {
 			}
 		}
 		
+		log.info("paths:" + paths);
+		
 		log.info("minimal path difference");
 		
 		// if all paths are empty the above step will eliminate all paths

org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/KeywordToken.java

@@ -24,11 +24,39 @@ public class KeywordToken implements Token {
 
 	@Override
 	public String negatedCondition() {
-		return "!\"" + keyword.getValue().replace("\"", "\\\"") + "\".equals(input.LT(" + position + ").getText()";
+		return "!\"" + keyword.getValue().replace("\"", "\\\"") + "\".equals(input.LT(" + position + ").getText())";
 	}
 	
 	@Override
 	public String toString() {
 		return "keyword " + keyword.getValue();
 	}
+
+	@Override
+	public int hashCode() {
+		final int prime = 31;
+		int result = 1;
+		result = prime * result + ((keyword == null) ? 0 : keyword.getValue().hashCode());
+		result = prime * result + position;
+		return result;
+	}
+
+	@Override
+	public boolean equals(Object obj) {
+		if (this == obj)
+			return true;
+		if (obj == null)
+			return false;
+		if (getClass() != obj.getClass())
+			return false;
+		KeywordToken other = (KeywordToken) obj;
+		if (keyword == null) {
+			if (other.keyword != null)
+				return false;
+		} else if (!keyword.getValue().equals(other.keyword.getValue()))
+			return false;
+		if (position != other.position)
+			return false;
+		return true;
+	}
 }

org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/TerminalRuleToken.java

@@ -31,4 +31,34 @@ public class TerminalRuleToken implements Token {
 	public String toString() {
 		return "terminal " + rule.getName();
 	}
+
+	@Override
+	public int hashCode() {
+		final int prime = 31;
+		int result = 1;
+		result = prime * result + position;
+		result = prime * result + ((rule == null) ? 0 : rule.getName().hashCode());
+		return result;
+	}
+
+	@Override
+	public boolean equals(Object obj) {
+		if (this == obj)
+			return true;
+		if (obj == null)
+			return false;
+		if (getClass() != obj.getClass())
+			return false;
+		TerminalRuleToken other = (TerminalRuleToken) obj;
+		if (position != other.position)
+			return false;
+		if (rule == null) {
+			if (other.rule != null)
+				return false;
+		} else if (!rule.getName().equals(other.rule.getName()))
+			return false;
+		return true;
+	}
+	
+	
 }

org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/TokenAnalysisPath.java

@@ -23,7 +23,7 @@ public class TokenAnalysisPath {
 	private int position = 1;
 	
 	public TokenAnalysisPath(List<Integer> indexes) {
-		this.remainingIndexes = indexes;
+		this.remainingIndexes = new LinkedList<>(indexes);
 	}
 	
 	public TokenAnalysisPath(TokenAnalysisPath prefix) {

org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/TokenAnalysisPaths.java

@@ -28,9 +28,6 @@ public class TokenAnalysisPaths {
 				.collect(Collectors.toSet());
 	}
 	
-	private TokenAnalysisPaths() {
-	}
-	
 	public TokenAnalysisPaths(List<Integer> indexes) {
 		tokenPaths.add(new TokenAnalysisPath(indexes));
 	}
@@ -42,7 +39,7 @@ public class TokenAnalysisPaths {
 	}
 	
 	public boolean isDone() {
-		return tokenPaths.stream().allMatch(TokenAnalysisPath::isDone);
+		return !isEmpty && tokenPaths.stream().allMatch(TokenAnalysisPath::isDone);
 	}
 	
 	public void add(AbstractElement element) {
@@ -58,8 +55,8 @@ public class TokenAnalysisPaths {
 		}
 	}
 	
-	static public TokenAnalysisPaths empty() {
-		TokenAnalysisPaths empty = new TokenAnalysisPaths();
+	static public TokenAnalysisPaths empty(TokenAnalysisPaths prefix) {
+		TokenAnalysisPaths empty = new TokenAnalysisPaths(prefix);
 		empty.isEmpty = true;
 		return empty;
 	}