- AbstractAntlrParser handles lookahead in match() mor sophisticated

- fixed tests
- fixed test.xml

plugins/org.eclipse.xtext/src/org/eclipse/xtext/parser/antlr/AbstractAntlrParser.java

@@ -103,7 +103,7 @@ public abstract class AbstractAntlrParser extends Parser {
 	}
 
 	private EObject getGrammarElement(String grammarElementID) {
-		return grammar.eResource().getResourceSet().getEObject(URI.createURI(grammarElementID),true);
+		return grammar.eResource().getResourceSet().getEObject(URI.createURI(grammarElementID), true);
 	}
 
 	private Map<Integer, String> antlrTypeToLexerName = null;
@@ -169,9 +169,9 @@ public abstract class AbstractAntlrParser extends Parser {
 		leafNode.setHidden(isHidden);
 		if (isSemanticChannel(token))
 			appendError(leafNode);
-		if(token.getType() == Token.INVALID_TOKEN_TYPE) {
+		if (token.getType() == Token.INVALID_TOKEN_TYPE) {
 			SyntaxError error = ParsetreeFactory.eINSTANCE.createSyntaxError();
-			String lexerErrorMessage = ((XtextTokenStream)input).getLexerErrorMessage(token);
+			String lexerErrorMessage = ((XtextTokenStream) input).getLexerErrorMessage(token);
 			error.setMessage(lexerErrorMessage);
 			leafNode.setSyntaxError(error);
 		}
@@ -354,7 +354,7 @@ public abstract class AbstractAntlrParser extends Parser {
 
 		for (Token deferredLookaheadToken : deferredLookaheadMap.keySet()) {
 			List<CompositeNode> nodesDecidingOnToken = deferredLookaheadMap.get(deferredLookaheadToken);
-			while(nodesDecidingOnToken.indexOf(source) != -1) {
+			while (nodesDecidingOnToken.indexOf(source) != -1) {
 				nodesDecidingOnToken.set(nodesDecidingOnToken.indexOf(source), target);
 			}
 		}
@@ -370,8 +370,12 @@ public abstract class AbstractAntlrParser extends Parser {
 	 */
 	@Override
 	public void match(IntStream input, int ttype, BitSet follow) throws RecognitionException {
+		XtextTokenStream xtextTokenStream = (XtextTokenStream) input;
+		int numLookaheadBeforeMatch = xtextTokenStream.getLookaheadTokens().size();
 		super.match(input, ttype, follow);
-		((XtextTokenStream) input).removeLastLookaheadToken();
+		if (xtextTokenStream.getLookaheadTokens().size() > numLookaheadBeforeMatch) {
+			xtextTokenStream.removeLastLookaheadToken();
+		}
 	}
 
 	protected InputStream getTokenFile() {

plugins/org.eclipse.xtext/src/org/eclipse/xtext/parser/impl/PartialParsingUtil.java

@@ -170,8 +170,6 @@ public class PartialParsingUtil {
 
 		boolean lookaheadChanged = false;
 		for (CompositeNode node : nodesEnclosingRegion) {
-			// EList<LeafNode> parentsLookaheadNodes = ((CompositeNode)
-			// node.getNode()).getLookaheadLeafNodes();
 			EList<LeafNode> parentsLookaheadNodes = node.getLookaheadLeafNodes();
 			if (!parentsLookaheadNodes.isEmpty()) {
 				int index = lookaheadNodes.indexOf(parentsLookaheadNodes.get(0));
@@ -189,41 +187,12 @@ public class PartialParsingUtil {
 				if (nodeIsBeforeRegion(lastLookaheadLeafNode, offset)) {
 					// last lookahead token is before changed region
 					// and parent has consumed all current lookahead tokens
-					// NodeUtil.dumpCompositeNodeInfo("Possible entry node: "
-					// , node);
+					 NodeUtil.dumpCompositeNodeInfo("Possible entry node: "
+					 , node);
 					validReplaceRootNodes.add(node);
 					lookaheadChanged = false;
 				}
 			}
-			// List<NodeWithCachedOffset> lookaheadNodes = new
-			// ArrayList<NodeWithCachedOffset>();
-			// int numConsumedLookaheadTokens = 0;
-			// List<CompositeNode> validReplaceRootNodes = new
-			// ArrayList<CompositeNode>();
-			//
-			// for (NodeWithCachedOffset node : nodesEnclosingRegion) {
-			// List<NodeWithCachedOffset> parentsLookaheadNodes =
-			// getParentsLookaheadNodes(node);
-			// if (!parentsLookaheadNodes.isEmpty()) {
-			// mergeLookaheadNodes(lookaheadNodes, parentsLookaheadNodes);
-			// }
-			// if (((CompositeNode) node.getNode()).getLookaheadConsumed() > 0)
-			// {
-			// // parent has consumed lookahead tokens.
-			// numConsumedLookaheadTokens += ((CompositeNode)
-			// node.getNode()).getLookaheadConsumed();
-			// if (numConsumedLookaheadTokens == lookaheadNodes.size()) {
-			// // parent has consumed all current lookahead tokens
-			// NodeWithCachedOffset leafNode =
-			// lookaheadNodes.get(numConsumedLookaheadTokens - 1);
-			// if (nodeIsBeforeRegion(leafNode, offset)) {
-			// // last lookahead token is before changed region
-			// //NodeUtil.dumpCompositeNodeInfo("Possible entry node: "
-			// // , node);
-			// validReplaceRootNodes.add((CompositeNode) node.getNode());
-			// }
-			// }
-			// }
 		}
 		return validReplaceRootNodes;
 	}

plugins/org.eclipse.xtext/src/org/eclipse/xtext/parsetree/NodeUtil.java

@@ -118,8 +118,7 @@ public class NodeUtil {
         for (LeafNode lookaheadNode : node.getLookaheadLeafNodes()) {
         	System.out.print("\""+ lookaheadNode.getText() +"\" ");
 		}
-        System.out.print(" (" + node.getOffset() + ", " + node.getLength() + ")");
-        System.out.println("}");
+        System.out.println("}   (" + node.getOffset() + ", " + node.getLength() + ")");
     }
     
 }

tests/org.eclipse.xtext.generator.tests/src/org/eclipse/xtext/parser/PartialParserReplaceTest.java

@@ -12,6 +12,7 @@ import org.eclipse.emf.common.util.EList;
 import org.eclipse.xtext.parser.impl.PartialParsingPointers;
 import org.eclipse.xtext.parser.impl.PartialParsingUtil;
 import org.eclipse.xtext.parsetree.CompositeNode;
+import org.eclipse.xtext.parsetree.NodeUtil;
 import org.eclipse.xtext.parsetree.SyntaxError;
 import org.eclipse.xtext.testlanguages.LookaheadLanguageStandaloneSetup;
 import org.eclipse.xtext.testlanguages.SimpleExpressionsStandaloneSetup;
@@ -26,16 +27,13 @@ public class PartialParserReplaceTest extends AbstractPartialParserTest {
 		with(SimpleExpressionsStandaloneSetup.class);
 		String model = "(a+b+c)*(c/d)";
 		replaceAndReparse(model, 2, 2, "+hugo+egon", "(a+hugo+egon+c)");
-//TODO repair
-//		replaceAndReparse(model, 8, 5, "egon", "egon");
+		replaceAndReparse(model, 8, 5, "egon", "egon");
 		replaceAndReparse(model, 1, 2, "", "(b+c)");
 		replaceAndReparse(model, 6, 3, "*", "(a+b+c*c/d)");
-//		replaceAndReparse(model, 3, 1, "(x+y+z)", "(x+y+z)");
-		
-		replaceAndReparse("a b", 1,1,"+","a+b");
-		// TODO: breaking case
-//		replaceAndReparse(model, 3, 1, "x)+(b", "x)+(b");
-		
+		replaceAndReparse(model, 3, 1, "(x+y+z)", "(x+y+z)");
+		replaceAndReparse("a b", 1, 1, "+", "a+b");
+		replaceAndReparse(model, 3, 1, "x)+(b", "x)+(b");
+
 	}
 
 	public void testLookahead() throws Exception {
@@ -49,16 +47,18 @@ public class PartialParserReplaceTest extends AbstractPartialParserTest {
 	private void replaceAndReparse(String model, int offset, int length, String change, String expectedReparseRegion)
 			throws Exception {
 		CompositeNode rootNode = getRootNode(model);
+		if (DEBUG) {
+			NodeUtil.dumpCompositeNodes("", rootNode);
+		}
 		PartialParsingPointers parsingPointers = PartialParsingUtil.calculatePartialParsingPointers(rootNode, offset,
 				length);
 		String reparseRegion = PartialParsingUtil.insertChangeIntoReplaceRegion(parsingPointers
 				.getDefaultReplaceRootNode(), offset, length, change);
 		assertEquals(expectedReparseRegion, reparseRegion);
-		IParseResult partiallyReparse = PartialParsingUtil.reparse(getParser(), rootNode, offset, length,
-				change);
+		IParseResult partiallyReparse = PartialParsingUtil.reparse(getParser(), rootNode, offset, length, change);
 		EList<SyntaxError> errors = partiallyReparse.getRootNode().allSyntaxErrors();
 		for (SyntaxError syntaxError : errors) {
-			System.out.println(model+offset+length+change+":"+syntaxError.getMessage());
+			System.out.println(model + offset + length + change + ":" + syntaxError.getMessage());
 		}
 		assertTrue(partiallyReparse.getRootNode().allSyntaxErrors().isEmpty());
 		String expectedReparseModel = model.substring(0, offset) + change + model.substring(offset + length);

tests/org.eclipse.xtext.tests/test.xml

@@ -16,7 +16,7 @@
 	<property name="eclipse-home" value="${basedir}\..\.." />
 
 	<!-- sets the properties plugin-name, and library-file -->
-	<property name="plugin-name" value="org.eclipse.xtext.servce.tests" />
+	<property name="plugin-name" value="org.eclipse.xtext.tests" />
 	<property name="library-file" value="${eclipse-home}/plugins/org.eclipse.test/library.xml" />
 
 	<!-- This target holds all initialization code that needs to be done for -->