- AbstractAntlrParser handles lookahead in match() mor sophisticated

- fixed tests
- fixed test.xml
This commit is contained in:
jkohnlein 2008-07-08 16:51:20 +00:00
parent 3452a7aacc
commit 69cfb157be
5 changed files with 24 additions and 52 deletions

View file

@ -103,7 +103,7 @@ public abstract class AbstractAntlrParser extends Parser {
}
private EObject getGrammarElement(String grammarElementID) {
return grammar.eResource().getResourceSet().getEObject(URI.createURI(grammarElementID),true);
return grammar.eResource().getResourceSet().getEObject(URI.createURI(grammarElementID), true);
}
private Map<Integer, String> antlrTypeToLexerName = null;
@ -169,9 +169,9 @@ public abstract class AbstractAntlrParser extends Parser {
leafNode.setHidden(isHidden);
if (isSemanticChannel(token))
appendError(leafNode);
if(token.getType() == Token.INVALID_TOKEN_TYPE) {
if (token.getType() == Token.INVALID_TOKEN_TYPE) {
SyntaxError error = ParsetreeFactory.eINSTANCE.createSyntaxError();
String lexerErrorMessage = ((XtextTokenStream)input).getLexerErrorMessage(token);
String lexerErrorMessage = ((XtextTokenStream) input).getLexerErrorMessage(token);
error.setMessage(lexerErrorMessage);
leafNode.setSyntaxError(error);
}
@ -354,7 +354,7 @@ public abstract class AbstractAntlrParser extends Parser {
for (Token deferredLookaheadToken : deferredLookaheadMap.keySet()) {
List<CompositeNode> nodesDecidingOnToken = deferredLookaheadMap.get(deferredLookaheadToken);
while(nodesDecidingOnToken.indexOf(source) != -1) {
while (nodesDecidingOnToken.indexOf(source) != -1) {
nodesDecidingOnToken.set(nodesDecidingOnToken.indexOf(source), target);
}
}
@ -370,8 +370,12 @@ public abstract class AbstractAntlrParser extends Parser {
*/
@Override
public void match(IntStream input, int ttype, BitSet follow) throws RecognitionException {
XtextTokenStream xtextTokenStream = (XtextTokenStream) input;
int numLookaheadBeforeMatch = xtextTokenStream.getLookaheadTokens().size();
super.match(input, ttype, follow);
((XtextTokenStream) input).removeLastLookaheadToken();
if (xtextTokenStream.getLookaheadTokens().size() > numLookaheadBeforeMatch) {
xtextTokenStream.removeLastLookaheadToken();
}
}
protected InputStream getTokenFile() {

View file

@ -170,8 +170,6 @@ public class PartialParsingUtil {
boolean lookaheadChanged = false;
for (CompositeNode node : nodesEnclosingRegion) {
// EList<LeafNode> parentsLookaheadNodes = ((CompositeNode)
// node.getNode()).getLookaheadLeafNodes();
EList<LeafNode> parentsLookaheadNodes = node.getLookaheadLeafNodes();
if (!parentsLookaheadNodes.isEmpty()) {
int index = lookaheadNodes.indexOf(parentsLookaheadNodes.get(0));
@ -189,41 +187,12 @@ public class PartialParsingUtil {
if (nodeIsBeforeRegion(lastLookaheadLeafNode, offset)) {
// last lookahead token is before changed region
// and parent has consumed all current lookahead tokens
// NodeUtil.dumpCompositeNodeInfo("Possible entry node: "
// , node);
NodeUtil.dumpCompositeNodeInfo("Possible entry node: "
, node);
validReplaceRootNodes.add(node);
lookaheadChanged = false;
}
}
// List<NodeWithCachedOffset> lookaheadNodes = new
// ArrayList<NodeWithCachedOffset>();
// int numConsumedLookaheadTokens = 0;
// List<CompositeNode> validReplaceRootNodes = new
// ArrayList<CompositeNode>();
//
// for (NodeWithCachedOffset node : nodesEnclosingRegion) {
// List<NodeWithCachedOffset> parentsLookaheadNodes =
// getParentsLookaheadNodes(node);
// if (!parentsLookaheadNodes.isEmpty()) {
// mergeLookaheadNodes(lookaheadNodes, parentsLookaheadNodes);
// }
// if (((CompositeNode) node.getNode()).getLookaheadConsumed() > 0)
// {
// // parent has consumed lookahead tokens.
// numConsumedLookaheadTokens += ((CompositeNode)
// node.getNode()).getLookaheadConsumed();
// if (numConsumedLookaheadTokens == lookaheadNodes.size()) {
// // parent has consumed all current lookahead tokens
// NodeWithCachedOffset leafNode =
// lookaheadNodes.get(numConsumedLookaheadTokens - 1);
// if (nodeIsBeforeRegion(leafNode, offset)) {
// // last lookahead token is before changed region
// //NodeUtil.dumpCompositeNodeInfo("Possible entry node: "
// // , node);
// validReplaceRootNodes.add((CompositeNode) node.getNode());
// }
// }
// }
}
return validReplaceRootNodes;
}

View file

@ -118,8 +118,7 @@ public class NodeUtil {
for (LeafNode lookaheadNode : node.getLookaheadLeafNodes()) {
System.out.print("\""+ lookaheadNode.getText() +"\" ");
}
System.out.print(" (" + node.getOffset() + ", " + node.getLength() + ")");
System.out.println("}");
System.out.println("} (" + node.getOffset() + ", " + node.getLength() + ")");
}
}

View file

@ -12,6 +12,7 @@ import org.eclipse.emf.common.util.EList;
import org.eclipse.xtext.parser.impl.PartialParsingPointers;
import org.eclipse.xtext.parser.impl.PartialParsingUtil;
import org.eclipse.xtext.parsetree.CompositeNode;
import org.eclipse.xtext.parsetree.NodeUtil;
import org.eclipse.xtext.parsetree.SyntaxError;
import org.eclipse.xtext.testlanguages.LookaheadLanguageStandaloneSetup;
import org.eclipse.xtext.testlanguages.SimpleExpressionsStandaloneSetup;
@ -26,16 +27,13 @@ public class PartialParserReplaceTest extends AbstractPartialParserTest {
with(SimpleExpressionsStandaloneSetup.class);
String model = "(a+b+c)*(c/d)";
replaceAndReparse(model, 2, 2, "+hugo+egon", "(a+hugo+egon+c)");
//TODO repair
// replaceAndReparse(model, 8, 5, "egon", "egon");
replaceAndReparse(model, 8, 5, "egon", "egon");
replaceAndReparse(model, 1, 2, "", "(b+c)");
replaceAndReparse(model, 6, 3, "*", "(a+b+c*c/d)");
// replaceAndReparse(model, 3, 1, "(x+y+z)", "(x+y+z)");
replaceAndReparse("a b", 1,1,"+","a+b");
// TODO: breaking case
// replaceAndReparse(model, 3, 1, "x)+(b", "x)+(b");
replaceAndReparse(model, 3, 1, "(x+y+z)", "(x+y+z)");
replaceAndReparse("a b", 1, 1, "+", "a+b");
replaceAndReparse(model, 3, 1, "x)+(b", "x)+(b");
}
public void testLookahead() throws Exception {
@ -49,16 +47,18 @@ public class PartialParserReplaceTest extends AbstractPartialParserTest {
private void replaceAndReparse(String model, int offset, int length, String change, String expectedReparseRegion)
throws Exception {
CompositeNode rootNode = getRootNode(model);
if (DEBUG) {
NodeUtil.dumpCompositeNodes("", rootNode);
}
PartialParsingPointers parsingPointers = PartialParsingUtil.calculatePartialParsingPointers(rootNode, offset,
length);
String reparseRegion = PartialParsingUtil.insertChangeIntoReplaceRegion(parsingPointers
.getDefaultReplaceRootNode(), offset, length, change);
assertEquals(expectedReparseRegion, reparseRegion);
IParseResult partiallyReparse = PartialParsingUtil.reparse(getParser(), rootNode, offset, length,
change);
IParseResult partiallyReparse = PartialParsingUtil.reparse(getParser(), rootNode, offset, length, change);
EList<SyntaxError> errors = partiallyReparse.getRootNode().allSyntaxErrors();
for (SyntaxError syntaxError : errors) {
System.out.println(model+offset+length+change+":"+syntaxError.getMessage());
System.out.println(model + offset + length + change + ":" + syntaxError.getMessage());
}
assertTrue(partiallyReparse.getRootNode().allSyntaxErrors().isEmpty());
String expectedReparseModel = model.substring(0, offset) + change + model.substring(offset + length);

View file

@ -16,7 +16,7 @@
<property name="eclipse-home" value="${basedir}\..\.." />
<!-- sets the properties plugin-name, and library-file -->
<property name="plugin-name" value="org.eclipse.xtext.servce.tests" />
<property name="plugin-name" value="org.eclipse.xtext.tests" />
<property name="library-file" value="${eclipse-home}/plugins/org.eclipse.test/library.xml" />
<!-- This target holds all initialization code that needs to be done for -->