From 69cfb157be5cd463b6cf92d7941070cfd0b47f27 Mon Sep 17 00:00:00 2001 From: jkohnlein Date: Tue, 8 Jul 2008 16:51:20 +0000 Subject: [PATCH] - AbstractAntlrParser handles lookahead in match() mor sophisticated - fixed tests - fixed test.xml --- .../parser/antlr/AbstractAntlrParser.java | 14 +++++--- .../xtext/parser/impl/PartialParsingUtil.java | 35 ++----------------- .../org/eclipse/xtext/parsetree/NodeUtil.java | 3 +- .../parser/PartialParserReplaceTest.java | 22 ++++++------ tests/org.eclipse.xtext.tests/test.xml | 2 +- 5 files changed, 24 insertions(+), 52 deletions(-) diff --git a/plugins/org.eclipse.xtext/src/org/eclipse/xtext/parser/antlr/AbstractAntlrParser.java b/plugins/org.eclipse.xtext/src/org/eclipse/xtext/parser/antlr/AbstractAntlrParser.java index 2b8260c49..dcfd3a5bb 100644 --- a/plugins/org.eclipse.xtext/src/org/eclipse/xtext/parser/antlr/AbstractAntlrParser.java +++ b/plugins/org.eclipse.xtext/src/org/eclipse/xtext/parser/antlr/AbstractAntlrParser.java @@ -103,7 +103,7 @@ public abstract class AbstractAntlrParser extends Parser { } private EObject getGrammarElement(String grammarElementID) { - return grammar.eResource().getResourceSet().getEObject(URI.createURI(grammarElementID),true); + return grammar.eResource().getResourceSet().getEObject(URI.createURI(grammarElementID), true); } private Map antlrTypeToLexerName = null; @@ -169,9 +169,9 @@ public abstract class AbstractAntlrParser extends Parser { leafNode.setHidden(isHidden); if (isSemanticChannel(token)) appendError(leafNode); - if(token.getType() == Token.INVALID_TOKEN_TYPE) { + if (token.getType() == Token.INVALID_TOKEN_TYPE) { SyntaxError error = ParsetreeFactory.eINSTANCE.createSyntaxError(); - String lexerErrorMessage = ((XtextTokenStream)input).getLexerErrorMessage(token); + String lexerErrorMessage = ((XtextTokenStream) input).getLexerErrorMessage(token); error.setMessage(lexerErrorMessage); leafNode.setSyntaxError(error); } @@ -354,7 +354,7 @@ public abstract class AbstractAntlrParser extends Parser { for (Token deferredLookaheadToken : deferredLookaheadMap.keySet()) { List nodesDecidingOnToken = deferredLookaheadMap.get(deferredLookaheadToken); - while(nodesDecidingOnToken.indexOf(source) != -1) { + while (nodesDecidingOnToken.indexOf(source) != -1) { nodesDecidingOnToken.set(nodesDecidingOnToken.indexOf(source), target); } } @@ -370,8 +370,12 @@ public abstract class AbstractAntlrParser extends Parser { */ @Override public void match(IntStream input, int ttype, BitSet follow) throws RecognitionException { + XtextTokenStream xtextTokenStream = (XtextTokenStream) input; + int numLookaheadBeforeMatch = xtextTokenStream.getLookaheadTokens().size(); super.match(input, ttype, follow); - ((XtextTokenStream) input).removeLastLookaheadToken(); + if (xtextTokenStream.getLookaheadTokens().size() > numLookaheadBeforeMatch) { + xtextTokenStream.removeLastLookaheadToken(); + } } protected InputStream getTokenFile() { diff --git a/plugins/org.eclipse.xtext/src/org/eclipse/xtext/parser/impl/PartialParsingUtil.java b/plugins/org.eclipse.xtext/src/org/eclipse/xtext/parser/impl/PartialParsingUtil.java index a9c81d828..dc3a64edb 100644 --- a/plugins/org.eclipse.xtext/src/org/eclipse/xtext/parser/impl/PartialParsingUtil.java +++ b/plugins/org.eclipse.xtext/src/org/eclipse/xtext/parser/impl/PartialParsingUtil.java @@ -170,8 +170,6 @@ public class PartialParsingUtil { boolean lookaheadChanged = false; for (CompositeNode node : nodesEnclosingRegion) { - // EList parentsLookaheadNodes = ((CompositeNode) - // node.getNode()).getLookaheadLeafNodes(); EList parentsLookaheadNodes = node.getLookaheadLeafNodes(); if (!parentsLookaheadNodes.isEmpty()) { int index = lookaheadNodes.indexOf(parentsLookaheadNodes.get(0)); @@ -189,41 +187,12 @@ public class PartialParsingUtil { if (nodeIsBeforeRegion(lastLookaheadLeafNode, offset)) { // last lookahead token is before changed region // and parent has consumed all current lookahead tokens - // NodeUtil.dumpCompositeNodeInfo("Possible entry node: " - // , node); + NodeUtil.dumpCompositeNodeInfo("Possible entry node: " + , node); validReplaceRootNodes.add(node); lookaheadChanged = false; } } - // List lookaheadNodes = new - // ArrayList(); - // int numConsumedLookaheadTokens = 0; - // List validReplaceRootNodes = new - // ArrayList(); - // - // for (NodeWithCachedOffset node : nodesEnclosingRegion) { - // List parentsLookaheadNodes = - // getParentsLookaheadNodes(node); - // if (!parentsLookaheadNodes.isEmpty()) { - // mergeLookaheadNodes(lookaheadNodes, parentsLookaheadNodes); - // } - // if (((CompositeNode) node.getNode()).getLookaheadConsumed() > 0) - // { - // // parent has consumed lookahead tokens. - // numConsumedLookaheadTokens += ((CompositeNode) - // node.getNode()).getLookaheadConsumed(); - // if (numConsumedLookaheadTokens == lookaheadNodes.size()) { - // // parent has consumed all current lookahead tokens - // NodeWithCachedOffset leafNode = - // lookaheadNodes.get(numConsumedLookaheadTokens - 1); - // if (nodeIsBeforeRegion(leafNode, offset)) { - // // last lookahead token is before changed region - // //NodeUtil.dumpCompositeNodeInfo("Possible entry node: " - // // , node); - // validReplaceRootNodes.add((CompositeNode) node.getNode()); - // } - // } - // } } return validReplaceRootNodes; } diff --git a/plugins/org.eclipse.xtext/src/org/eclipse/xtext/parsetree/NodeUtil.java b/plugins/org.eclipse.xtext/src/org/eclipse/xtext/parsetree/NodeUtil.java index ce8e6f5c8..b4ee4a0b4 100644 --- a/plugins/org.eclipse.xtext/src/org/eclipse/xtext/parsetree/NodeUtil.java +++ b/plugins/org.eclipse.xtext/src/org/eclipse/xtext/parsetree/NodeUtil.java @@ -118,8 +118,7 @@ public class NodeUtil { for (LeafNode lookaheadNode : node.getLookaheadLeafNodes()) { System.out.print("\""+ lookaheadNode.getText() +"\" "); } - System.out.print(" (" + node.getOffset() + ", " + node.getLength() + ")"); - System.out.println("}"); + System.out.println("} (" + node.getOffset() + ", " + node.getLength() + ")"); } } diff --git a/tests/org.eclipse.xtext.generator.tests/src/org/eclipse/xtext/parser/PartialParserReplaceTest.java b/tests/org.eclipse.xtext.generator.tests/src/org/eclipse/xtext/parser/PartialParserReplaceTest.java index 53895a1cd..1fc78b9f6 100644 --- a/tests/org.eclipse.xtext.generator.tests/src/org/eclipse/xtext/parser/PartialParserReplaceTest.java +++ b/tests/org.eclipse.xtext.generator.tests/src/org/eclipse/xtext/parser/PartialParserReplaceTest.java @@ -12,6 +12,7 @@ import org.eclipse.emf.common.util.EList; import org.eclipse.xtext.parser.impl.PartialParsingPointers; import org.eclipse.xtext.parser.impl.PartialParsingUtil; import org.eclipse.xtext.parsetree.CompositeNode; +import org.eclipse.xtext.parsetree.NodeUtil; import org.eclipse.xtext.parsetree.SyntaxError; import org.eclipse.xtext.testlanguages.LookaheadLanguageStandaloneSetup; import org.eclipse.xtext.testlanguages.SimpleExpressionsStandaloneSetup; @@ -26,16 +27,13 @@ public class PartialParserReplaceTest extends AbstractPartialParserTest { with(SimpleExpressionsStandaloneSetup.class); String model = "(a+b+c)*(c/d)"; replaceAndReparse(model, 2, 2, "+hugo+egon", "(a+hugo+egon+c)"); -//TODO repair -// replaceAndReparse(model, 8, 5, "egon", "egon"); + replaceAndReparse(model, 8, 5, "egon", "egon"); replaceAndReparse(model, 1, 2, "", "(b+c)"); replaceAndReparse(model, 6, 3, "*", "(a+b+c*c/d)"); -// replaceAndReparse(model, 3, 1, "(x+y+z)", "(x+y+z)"); - - replaceAndReparse("a b", 1,1,"+","a+b"); - // TODO: breaking case -// replaceAndReparse(model, 3, 1, "x)+(b", "x)+(b"); - + replaceAndReparse(model, 3, 1, "(x+y+z)", "(x+y+z)"); + replaceAndReparse("a b", 1, 1, "+", "a+b"); + replaceAndReparse(model, 3, 1, "x)+(b", "x)+(b"); + } public void testLookahead() throws Exception { @@ -49,16 +47,18 @@ public class PartialParserReplaceTest extends AbstractPartialParserTest { private void replaceAndReparse(String model, int offset, int length, String change, String expectedReparseRegion) throws Exception { CompositeNode rootNode = getRootNode(model); + if (DEBUG) { + NodeUtil.dumpCompositeNodes("", rootNode); + } PartialParsingPointers parsingPointers = PartialParsingUtil.calculatePartialParsingPointers(rootNode, offset, length); String reparseRegion = PartialParsingUtil.insertChangeIntoReplaceRegion(parsingPointers .getDefaultReplaceRootNode(), offset, length, change); assertEquals(expectedReparseRegion, reparseRegion); - IParseResult partiallyReparse = PartialParsingUtil.reparse(getParser(), rootNode, offset, length, - change); + IParseResult partiallyReparse = PartialParsingUtil.reparse(getParser(), rootNode, offset, length, change); EList errors = partiallyReparse.getRootNode().allSyntaxErrors(); for (SyntaxError syntaxError : errors) { - System.out.println(model+offset+length+change+":"+syntaxError.getMessage()); + System.out.println(model + offset + length + change + ":" + syntaxError.getMessage()); } assertTrue(partiallyReparse.getRootNode().allSyntaxErrors().isEmpty()); String expectedReparseModel = model.substring(0, offset) + change + model.substring(offset + length); diff --git a/tests/org.eclipse.xtext.tests/test.xml b/tests/org.eclipse.xtext.tests/test.xml index c86f70b3d..0784feb75 100644 --- a/tests/org.eclipse.xtext.tests/test.xml +++ b/tests/org.eclipse.xtext.tests/test.xml @@ -16,7 +16,7 @@ - +