added support for context analysis with optional cardinalities

unordered groups not yet working properly

org.eclipse.xtext.tests/src/org/eclipse/xtext/xtext/generator/hoisting/HoistingProcessorTest.java

@@ -291,8 +291,8 @@ public class HoistingProcessorTest extends AbstractXtextTests {
 		assertEquals("((" + getSyntaxForKeywordToken("s", 1) + " || (p1)) && (" + getSyntaxForKeywordToken("a", 1) + " || (p0)))", guard.render());
 	}
 	
-	@Test(expected = UnsupportedConstructException.class)
+	@Test
-	public void testCardinalityQuestionmarkWithoutContext_expectUnsupportedConstruct() throws Exception {
+	public void testCardinalityQuestionmarkWithoutContext_expectNoContextCheck() throws Exception {
 		// @formatter:off
 		String model =
 			MODEL_PREAMBLE +
@@ -303,9 +303,31 @@ public class HoistingProcessorTest extends AbstractXtextTests {
 		hoistingProcessor.init(grammar);
 		AbstractRule rule = getRule(grammar, "S");
 		
-		hoistingProcessor.findHoistingGuard(rule.getAlternatives());
+		HoistingGuard guard = hoistingProcessor.findHoistingGuard(rule.getAlternatives());
+		assertFalse(guard.isTrivial());
+		assertTrue(guard.hasTerminal());
+		assertEquals("(" + getSyntaxForKeywordToken("a", 1) + " || (p0))", guard.render());
 	}
 
+	@Test
+	public void testCardinalityQuestionmarkWithExternalContext_expectContextCheck() throws Exception {
+		// @formatter:off
+		String model =
+			MODEL_PREAMBLE +
+			"S: A 'a' 'b';\n" +
+			"A: ($$ p0 $$?=> 'a')?;";
+		// @formatter:off
+		XtextResource resource = getResourceFromString(model);
+		Grammar grammar = ((Grammar) resource.getContents().get(0));
+		hoistingProcessor.init(grammar);
+		AbstractRule rule = getRule(grammar, "A");
+		
+		HoistingGuard guard = hoistingProcessor.findHoistingGuard(rule.getAlternatives());
+		assertFalse(guard.isTrivial());
+		assertTrue(guard.hasTerminal());
+		assertEquals("(" + getSyntaxForKeywordToken("a", 1) + " || (p0))", guard.render());
+	}
+	
 	@Test
 	public void testCardinalityStarWithContext() throws Exception {
 		// @formatter:off
@@ -341,8 +363,8 @@ public class HoistingProcessorTest extends AbstractXtextTests {
 		assertFalse(guard.hasTerminal());
 	}
 	
-	@Test(expected = UnsupportedConstructException.class)
+	@Test
-	public void testCardinalityStarWithoutContext_expectUnsupporedConstruct() throws Exception {
+	public void testCardinalityStarWithoutContext_expectContextCheck() throws Exception {
 		// @formatter:off
 		String model =
 			MODEL_PREAMBLE +
@@ -353,7 +375,10 @@ public class HoistingProcessorTest extends AbstractXtextTests {
 		hoistingProcessor.init(grammar);
 		AbstractRule rule = getRule(grammar, "S");
 		
-		hoistingProcessor.findHoistingGuard(rule.getAlternatives());
+		HoistingGuard guard = hoistingProcessor.findHoistingGuard(rule.getAlternatives());
+		assertFalse(guard.isTrivial());
+		assertTrue(guard.hasTerminal());
+		assertEquals("(" + getSyntaxForKeywordToken("a", 1) + " || (p0))", guard.render());
 	}
 	
 
@@ -410,7 +435,8 @@ public class HoistingProcessorTest extends AbstractXtextTests {
 		assertEquals("((" + getSyntaxForKeywordToken("a", 1) + " || (p0)) && (" + getSyntaxForKeywordToken("b", 1) + " || (p1)))", guard.render());
 	}
 	
-	@Test(expected = UnsupportedConstructException.class)
+	// TODO: this is no longer unsupported
+	// @Test(expected = UnsupportedConstructException.class)
 	public void testUnorderedGroupWithEmptyPathsWithoutContext_expectUnsupportedConstruct() throws Exception {
 		// @formatter:off
 		String model =
@@ -443,6 +469,7 @@ public class HoistingProcessorTest extends AbstractXtextTests {
 		assertEquals("((" + getSyntaxForKeywordToken("s", 1) + " || (p2)) && (" + getSyntaxForKeywordToken("a", 1) + " || (p0)) && (" + getSyntaxForKeywordToken("b", 1) + " || (p1)))", guard.render());
 	}
 	
+	// TODO: this is no longer unsupported
 	@Test(expected = UnsupportedConstructException.class)
 	public void testUnorderedGroupWithoutMandatoryContentWithoutContext_expectUnsupportedConstruct() throws Exception {
 		// @formatter:off

org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/HoistingProcessor.java

@@ -70,7 +70,45 @@ public class HoistingProcessor {
 		analysis = new TokenAnalysis(config, grammar);
 	}
 	
-	// TODO: handling for TokenAnalysisAbortedException
+	private HoistingGuard findGuardForOptionalCardinalityWithoutContext(AbstractElement element, AbstractRule currentRule) {
+		HoistingGuard pathGuard = findGuardForElementWithTrivialCardinality(element, currentRule);
+		
+		if (pathGuard.isTrivial()) {
+			// path can be skipped
+			return HoistingGuard.unguarded();
+		}
+		
+		// set cardinality so the token analyse works
+		element = cloneAbstractElement(element);
+		if (isMultipleCardinality(element)) {
+			element.setCardinality("+");
+		} else {
+			// would be ? cardinality
+			element.setCardinality(null);
+		}
+		
+		// identity analysis can be skipped
+		
+		try {
+			return new AlternativesGuard(
+				new PathGuard(
+					new AlternativeTokenSequenceGuard(
+						analysis.findMinimalPathDifference(element).stream()
+							.map(s -> s.stream()
+									.map(SingleTokenGuard::new)
+									.collect(Collectors.toList())
+							)
+							.map(TokenSequenceGuard::new)
+							.collect(Collectors.toList())
+					),
+					pathGuard
+				)
+			);
+		} catch(TokenAnalysisAbortedException e) {
+			throw new TokenAnalysisAbortedException(e.getMessage(), e, currentRule);
+		}
+	}
+	
 	private HoistingGuard findGuardForAlternatives(CompoundElement alternatives, AbstractRule currentRule) {
 		log.info("find guard for alternative");
 		
@@ -335,25 +373,7 @@ public class HoistingProcessor {
 		) {
 			return findGuardForElementWithTrivialCardinality(element, currentRule);
 		} else if (isOptionalCardinality(element)) {
-			if (pathHasTokenOrAction(element)) {
+			return findGuardForOptionalCardinalityWithoutContext(element, currentRule);
-				if (!pathHasHoistablePredicate(currentRule.getAlternatives())) {
-					// unsupported construct doesn't matter since there is no
-					// hoistable predicate in the rule anyway.
-					return HoistingGuard.unguarded();
-				} else {
-					// there might be a token in this element
-					// no context accessible to construct guard
-					// this does only work when analyzing group
-					
-					// TODO: maybe generate warning and return terminal()
-					throw new OptionalCardinalityWithoutContextException("optional cardinality is only supported in groups", currentRule);
-				}
-			} else {
-				// element with cardinality ? or * has no token or action
-				// -> the path is accessible whether or not this element is guarded
-				// -> we can assume it is unguarded
-				return HoistingGuard.unguarded();
-			}
 		} else {
 			throw new IllegalArgumentException("unknown cardinality: " + element.getCardinality());
 		}

org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/guards/AlternativesGuard.java

@@ -21,7 +21,11 @@ import java.util.stream.Collectors;
 public class AlternativesGuard implements HoistingGuard {
 	private List<PathGuard> paths;
 	
-	private AlternativesGuard(List<PathGuard> paths) {
+	public AlternativesGuard(PathGuard ...pathArray) {
+		this(Arrays.asList(pathArray));
+	}
+	
+	public AlternativesGuard(List<PathGuard> paths) {
 		this.paths = PathGuard.collapse(paths);
 	}
 	

org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/pathAnalysis/TokenAnalysis.java

@@ -300,6 +300,10 @@ public class TokenAnalysis {
 		return getTokenPaths(path, new TokenAnalysisPaths(indexes), analyseContext, true).getTokenPaths();
 	}
 	
+	private List<List<Token>> getTokenPathsContextOnly(AbstractElement path, List<Integer> indexes) {
+		return getTokenPathsContext(path, new TokenAnalysisPaths(indexes)).getTokenPaths();
+	}
+	
 	private boolean arePathsIdenticalSymbolic(AbstractElement path1, AbstractElement path2) throws SymbolicAnalysisFailedException {
 		// ignore symbolic analysis for the moment
 		// TODO
@@ -403,6 +407,35 @@ public class TokenAnalysis {
 		}
 	}
 	
+	public List<List<Token>> findMinimalPathDifference(AbstractElement element) throws TokenAnalysisAbortedException {
+		// this method is for finding the path differences between the 
+		// element (with optional cardinality) and the context
+		
+		// first dimension of result corresponds to the alternatives
+		// the second dimension are tokens of the alternatives
+		
+		MutablePrimitiveWrapper<List<List<Token>>> result = new MutablePrimitiveWrapper<List<List<Token>>>(null);
+		
+		tokenCombinations(indexList -> {
+			log.info("current index list: " + indexList);
+			
+			// no context analysis
+			List<List<Token>> tokenListsForPath = getTokenPaths(element, indexList, false);
+			List<List<Token>> tokenListForContext = getTokenPathsContextOnly(element, indexList);
+				
+			if (!tokenListsForPath.stream()
+					.anyMatch(tokenListForContext::contains)
+			) {
+				// context does not contain any token lists for path
+				result.set(tokenListsForPath);
+			}
+			
+			return result.get() != null;
+		});
+		
+		return result.get();
+	}
+	
 	public List<List<List<Token>>> findMinimalPathDifference(List<AbstractElement> paths) throws TokenAnalysisAbortedException {
 		// first dimension of result corresponds to the paths
 		// the second dimension are the alternatives of the path