mirror of
https://github.com/sigmasternchen/xtext-core
synced 2025-03-16 00:38:56 +00:00
- AbstractAntlrParser handles lookahead in match() mor sophisticated
- fixed tests - fixed test.xml
This commit is contained in:
parent
3452a7aacc
commit
69cfb157be
5 changed files with 24 additions and 52 deletions
|
@ -103,7 +103,7 @@ public abstract class AbstractAntlrParser extends Parser {
|
|||
}
|
||||
|
||||
private EObject getGrammarElement(String grammarElementID) {
|
||||
return grammar.eResource().getResourceSet().getEObject(URI.createURI(grammarElementID),true);
|
||||
return grammar.eResource().getResourceSet().getEObject(URI.createURI(grammarElementID), true);
|
||||
}
|
||||
|
||||
private Map<Integer, String> antlrTypeToLexerName = null;
|
||||
|
@ -169,9 +169,9 @@ public abstract class AbstractAntlrParser extends Parser {
|
|||
leafNode.setHidden(isHidden);
|
||||
if (isSemanticChannel(token))
|
||||
appendError(leafNode);
|
||||
if(token.getType() == Token.INVALID_TOKEN_TYPE) {
|
||||
if (token.getType() == Token.INVALID_TOKEN_TYPE) {
|
||||
SyntaxError error = ParsetreeFactory.eINSTANCE.createSyntaxError();
|
||||
String lexerErrorMessage = ((XtextTokenStream)input).getLexerErrorMessage(token);
|
||||
String lexerErrorMessage = ((XtextTokenStream) input).getLexerErrorMessage(token);
|
||||
error.setMessage(lexerErrorMessage);
|
||||
leafNode.setSyntaxError(error);
|
||||
}
|
||||
|
@ -354,7 +354,7 @@ public abstract class AbstractAntlrParser extends Parser {
|
|||
|
||||
for (Token deferredLookaheadToken : deferredLookaheadMap.keySet()) {
|
||||
List<CompositeNode> nodesDecidingOnToken = deferredLookaheadMap.get(deferredLookaheadToken);
|
||||
while(nodesDecidingOnToken.indexOf(source) != -1) {
|
||||
while (nodesDecidingOnToken.indexOf(source) != -1) {
|
||||
nodesDecidingOnToken.set(nodesDecidingOnToken.indexOf(source), target);
|
||||
}
|
||||
}
|
||||
|
@ -370,8 +370,12 @@ public abstract class AbstractAntlrParser extends Parser {
|
|||
*/
|
||||
@Override
|
||||
public void match(IntStream input, int ttype, BitSet follow) throws RecognitionException {
|
||||
XtextTokenStream xtextTokenStream = (XtextTokenStream) input;
|
||||
int numLookaheadBeforeMatch = xtextTokenStream.getLookaheadTokens().size();
|
||||
super.match(input, ttype, follow);
|
||||
((XtextTokenStream) input).removeLastLookaheadToken();
|
||||
if (xtextTokenStream.getLookaheadTokens().size() > numLookaheadBeforeMatch) {
|
||||
xtextTokenStream.removeLastLookaheadToken();
|
||||
}
|
||||
}
|
||||
|
||||
protected InputStream getTokenFile() {
|
||||
|
|
|
@ -170,8 +170,6 @@ public class PartialParsingUtil {
|
|||
|
||||
boolean lookaheadChanged = false;
|
||||
for (CompositeNode node : nodesEnclosingRegion) {
|
||||
// EList<LeafNode> parentsLookaheadNodes = ((CompositeNode)
|
||||
// node.getNode()).getLookaheadLeafNodes();
|
||||
EList<LeafNode> parentsLookaheadNodes = node.getLookaheadLeafNodes();
|
||||
if (!parentsLookaheadNodes.isEmpty()) {
|
||||
int index = lookaheadNodes.indexOf(parentsLookaheadNodes.get(0));
|
||||
|
@ -189,41 +187,12 @@ public class PartialParsingUtil {
|
|||
if (nodeIsBeforeRegion(lastLookaheadLeafNode, offset)) {
|
||||
// last lookahead token is before changed region
|
||||
// and parent has consumed all current lookahead tokens
|
||||
// NodeUtil.dumpCompositeNodeInfo("Possible entry node: "
|
||||
// , node);
|
||||
NodeUtil.dumpCompositeNodeInfo("Possible entry node: "
|
||||
, node);
|
||||
validReplaceRootNodes.add(node);
|
||||
lookaheadChanged = false;
|
||||
}
|
||||
}
|
||||
// List<NodeWithCachedOffset> lookaheadNodes = new
|
||||
// ArrayList<NodeWithCachedOffset>();
|
||||
// int numConsumedLookaheadTokens = 0;
|
||||
// List<CompositeNode> validReplaceRootNodes = new
|
||||
// ArrayList<CompositeNode>();
|
||||
//
|
||||
// for (NodeWithCachedOffset node : nodesEnclosingRegion) {
|
||||
// List<NodeWithCachedOffset> parentsLookaheadNodes =
|
||||
// getParentsLookaheadNodes(node);
|
||||
// if (!parentsLookaheadNodes.isEmpty()) {
|
||||
// mergeLookaheadNodes(lookaheadNodes, parentsLookaheadNodes);
|
||||
// }
|
||||
// if (((CompositeNode) node.getNode()).getLookaheadConsumed() > 0)
|
||||
// {
|
||||
// // parent has consumed lookahead tokens.
|
||||
// numConsumedLookaheadTokens += ((CompositeNode)
|
||||
// node.getNode()).getLookaheadConsumed();
|
||||
// if (numConsumedLookaheadTokens == lookaheadNodes.size()) {
|
||||
// // parent has consumed all current lookahead tokens
|
||||
// NodeWithCachedOffset leafNode =
|
||||
// lookaheadNodes.get(numConsumedLookaheadTokens - 1);
|
||||
// if (nodeIsBeforeRegion(leafNode, offset)) {
|
||||
// // last lookahead token is before changed region
|
||||
// //NodeUtil.dumpCompositeNodeInfo("Possible entry node: "
|
||||
// // , node);
|
||||
// validReplaceRootNodes.add((CompositeNode) node.getNode());
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
}
|
||||
return validReplaceRootNodes;
|
||||
}
|
||||
|
|
|
@ -118,8 +118,7 @@ public class NodeUtil {
|
|||
for (LeafNode lookaheadNode : node.getLookaheadLeafNodes()) {
|
||||
System.out.print("\""+ lookaheadNode.getText() +"\" ");
|
||||
}
|
||||
System.out.print(" (" + node.getOffset() + ", " + node.getLength() + ")");
|
||||
System.out.println("}");
|
||||
System.out.println("} (" + node.getOffset() + ", " + node.getLength() + ")");
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -12,6 +12,7 @@ import org.eclipse.emf.common.util.EList;
|
|||
import org.eclipse.xtext.parser.impl.PartialParsingPointers;
|
||||
import org.eclipse.xtext.parser.impl.PartialParsingUtil;
|
||||
import org.eclipse.xtext.parsetree.CompositeNode;
|
||||
import org.eclipse.xtext.parsetree.NodeUtil;
|
||||
import org.eclipse.xtext.parsetree.SyntaxError;
|
||||
import org.eclipse.xtext.testlanguages.LookaheadLanguageStandaloneSetup;
|
||||
import org.eclipse.xtext.testlanguages.SimpleExpressionsStandaloneSetup;
|
||||
|
@ -26,16 +27,13 @@ public class PartialParserReplaceTest extends AbstractPartialParserTest {
|
|||
with(SimpleExpressionsStandaloneSetup.class);
|
||||
String model = "(a+b+c)*(c/d)";
|
||||
replaceAndReparse(model, 2, 2, "+hugo+egon", "(a+hugo+egon+c)");
|
||||
//TODO repair
|
||||
// replaceAndReparse(model, 8, 5, "egon", "egon");
|
||||
replaceAndReparse(model, 8, 5, "egon", "egon");
|
||||
replaceAndReparse(model, 1, 2, "", "(b+c)");
|
||||
replaceAndReparse(model, 6, 3, "*", "(a+b+c*c/d)");
|
||||
// replaceAndReparse(model, 3, 1, "(x+y+z)", "(x+y+z)");
|
||||
|
||||
replaceAndReparse("a b", 1,1,"+","a+b");
|
||||
// TODO: breaking case
|
||||
// replaceAndReparse(model, 3, 1, "x)+(b", "x)+(b");
|
||||
|
||||
replaceAndReparse(model, 3, 1, "(x+y+z)", "(x+y+z)");
|
||||
replaceAndReparse("a b", 1, 1, "+", "a+b");
|
||||
replaceAndReparse(model, 3, 1, "x)+(b", "x)+(b");
|
||||
|
||||
}
|
||||
|
||||
public void testLookahead() throws Exception {
|
||||
|
@ -49,16 +47,18 @@ public class PartialParserReplaceTest extends AbstractPartialParserTest {
|
|||
private void replaceAndReparse(String model, int offset, int length, String change, String expectedReparseRegion)
|
||||
throws Exception {
|
||||
CompositeNode rootNode = getRootNode(model);
|
||||
if (DEBUG) {
|
||||
NodeUtil.dumpCompositeNodes("", rootNode);
|
||||
}
|
||||
PartialParsingPointers parsingPointers = PartialParsingUtil.calculatePartialParsingPointers(rootNode, offset,
|
||||
length);
|
||||
String reparseRegion = PartialParsingUtil.insertChangeIntoReplaceRegion(parsingPointers
|
||||
.getDefaultReplaceRootNode(), offset, length, change);
|
||||
assertEquals(expectedReparseRegion, reparseRegion);
|
||||
IParseResult partiallyReparse = PartialParsingUtil.reparse(getParser(), rootNode, offset, length,
|
||||
change);
|
||||
IParseResult partiallyReparse = PartialParsingUtil.reparse(getParser(), rootNode, offset, length, change);
|
||||
EList<SyntaxError> errors = partiallyReparse.getRootNode().allSyntaxErrors();
|
||||
for (SyntaxError syntaxError : errors) {
|
||||
System.out.println(model+offset+length+change+":"+syntaxError.getMessage());
|
||||
System.out.println(model + offset + length + change + ":" + syntaxError.getMessage());
|
||||
}
|
||||
assertTrue(partiallyReparse.getRootNode().allSyntaxErrors().isEmpty());
|
||||
String expectedReparseModel = model.substring(0, offset) + change + model.substring(offset + length);
|
||||
|
|
|
@ -16,7 +16,7 @@
|
|||
<property name="eclipse-home" value="${basedir}\..\.." />
|
||||
|
||||
<!-- sets the properties plugin-name, and library-file -->
|
||||
<property name="plugin-name" value="org.eclipse.xtext.servce.tests" />
|
||||
<property name="plugin-name" value="org.eclipse.xtext.tests" />
|
||||
<property name="library-file" value="${eclipse-home}/plugins/org.eclipse.test/library.xml" />
|
||||
|
||||
<!-- This target holds all initialization code that needs to be done for -->
|
||||
|
|
Loading…
Reference in a new issue