diff --git a/org.eclipse.xtext.tests/src/org/eclipse/xtext/xtext/generator/hoisting/HoistingGeneratorBenchmark.xtend b/org.eclipse.xtext.tests/src/org/eclipse/xtext/xtext/generator/hoisting/HoistingGeneratorBenchmark.xtend new file mode 100644 index 000000000..1639e7b8c --- /dev/null +++ b/org.eclipse.xtext.tests/src/org/eclipse/xtext/xtext/generator/hoisting/HoistingGeneratorBenchmark.xtend @@ -0,0 +1,96 @@ +/******************************************************************************* + * Copyright (c) 2022 itemis AG (http://www.itemis.eu) and others. + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *******************************************************************************/ +package org.eclipse.xtext.xtext.generator.hoisting + +import org.eclipse.xtext.tests.AbstractXtextTests +import org.eclipse.xtext.XtextStandaloneSetup +import org.junit.Test +import org.eclipse.xtext.Grammar +import org.eclipse.xtext.xtext.generator.DefaultGeneratorModule +import com.google.inject.Guice +import org.eclipse.xtext.xtext.generator.parser.antlr.AntlrOptions +import org.eclipse.xtext.generator.InMemoryFileSystemAccess +import org.eclipse.xtext.xtext.generator.model.IXtextGeneratorFileSystemAccess +import com.google.inject.Injector +import org.eclipse.xtext.xtext.generator.parser.antlr.AntlrDebugProductionGrammarGenerator +import org.eclipse.xtext.xtext.generator.parser.antlr.AntlrDebugContentAssistGrammarGenerator + +/** + * @author overflow - Initial contribution and API + */ +class HoistingGeneratorBenchmark extends AbstractXtextTests { + + val REPETITIONS = 1000 + + override void setUp() throws Exception { + super.setUp() + with(XtextStandaloneSetup) + } + + + + @Test + def void testFullTranslationBenchmark() { + for (var i = 0; i < REPETITIONS; i++) { + ''' + grammar com.foo.bar with org.eclipse.xtext.common.Terminals + import "http://www.eclipse.org/emf/2002/Ecore" as ecore + generate myPack 'http://mypack' + + setup $$ + java.util.Stack symbolTable = new java.util.Stack(); + $$ + + Statements: + defs+=VarDef+ + ; + + VarDef: + $$ !symbolTable.contains( input.LT( 1 ).getText() ) $$?=> + name=ID '=' value=Expr + $$ symbolTable.push(((org.xtext.example.mydsl.myDsl.VarDef) current).getName()); $$ + ; + + Expr: + $$ symbolTable.contains( input.LT( 0 ).getText() ) $$?=> + var=ID + | builtin=ID '(' args += Expr ( args += Expr ',' )* ')' + | num=INT + ; + '''.doFullTranslateRun + } + } + + protected def void doFullTranslateRun(CharSequence xtextGrammar) { + val grammar = super.getModel(xtextGrammar.toString) as Grammar + val injector = Guice.createInjector(new DefaultGeneratorModule) + val inMem = new InMemFSA + val options = new AntlrOptions + System.out.println("production grammar") + injector.getInstance(AntlrDebugProductionGrammarGenerator).generate(grammar, options, inMem) + System.out.println("content assist grammar"); + injector.getInstance(AntlrDebugContentAssistGrammarGenerator).generate(grammar, options, inMem) + } + + static class InMemFSA extends InMemoryFileSystemAccess implements IXtextGeneratorFileSystemAccess { + + override getPath() { + "./" + } + + override isOverwrite() { + true + } + + override initialize(Injector injector) { + injector.injectMembers(this) + } + + } +} \ No newline at end of file diff --git a/org.eclipse.xtext.tests/xtend-gen/org/eclipse/xtext/xtext/generator/hoisting/HoistingGeneratorBenchmark.java b/org.eclipse.xtext.tests/xtend-gen/org/eclipse/xtext/xtext/generator/hoisting/HoistingGeneratorBenchmark.java new file mode 100644 index 000000000..5e6420f3c --- /dev/null +++ b/org.eclipse.xtext.tests/xtend-gen/org/eclipse/xtext/xtext/generator/hoisting/HoistingGeneratorBenchmark.java @@ -0,0 +1,134 @@ +/** + * Copyright (c) 2022 itemis AG (http://www.itemis.eu) and others. + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.xtext.xtext.generator.hoisting; + +import com.google.inject.Guice; +import com.google.inject.Injector; +import org.eclipse.emf.ecore.EObject; +import org.eclipse.xtend2.lib.StringConcatenation; +import org.eclipse.xtext.Grammar; +import org.eclipse.xtext.XtextStandaloneSetup; +import org.eclipse.xtext.generator.InMemoryFileSystemAccess; +import org.eclipse.xtext.tests.AbstractXtextTests; +import org.eclipse.xtext.xbase.lib.Exceptions; +import org.eclipse.xtext.xtext.generator.DefaultGeneratorModule; +import org.eclipse.xtext.xtext.generator.model.IXtextGeneratorFileSystemAccess; +import org.eclipse.xtext.xtext.generator.parser.antlr.AntlrDebugContentAssistGrammarGenerator; +import org.eclipse.xtext.xtext.generator.parser.antlr.AntlrDebugProductionGrammarGenerator; +import org.eclipse.xtext.xtext.generator.parser.antlr.AntlrOptions; +import org.junit.Test; + +/** + * @author overflow - Initial contribution and API + */ +@SuppressWarnings("all") +public class HoistingGeneratorBenchmark extends AbstractXtextTests { + public static class InMemFSA extends InMemoryFileSystemAccess implements IXtextGeneratorFileSystemAccess { + @Override + public String getPath() { + return "./"; + } + + @Override + public boolean isOverwrite() { + return true; + } + + @Override + public void initialize(final Injector injector) { + injector.injectMembers(this); + } + } + + private final int REPETITIONS = 1000; + + @Override + public void setUp() throws Exception { + super.setUp(); + this.with(XtextStandaloneSetup.class); + } + + @Test + public void testFullTranslationBenchmark() { + for (int i = 0; (i < this.REPETITIONS); i++) { + StringConcatenation _builder = new StringConcatenation(); + _builder.append("grammar com.foo.bar with org.eclipse.xtext.common.Terminals"); + _builder.newLine(); + _builder.append("import \"http://www.eclipse.org/emf/2002/Ecore\" as ecore"); + _builder.newLine(); + _builder.append("generate myPack \'http://mypack\'"); + _builder.newLine(); + _builder.newLine(); + _builder.append("setup $$"); + _builder.newLine(); + _builder.append("\t"); + _builder.append("java.util.Stack symbolTable = new java.util.Stack();"); + _builder.newLine(); + _builder.append("$$"); + _builder.newLine(); + _builder.newLine(); + _builder.append("Statements:"); + _builder.newLine(); + _builder.append(" "); + _builder.append("defs+=VarDef+"); + _builder.newLine(); + _builder.append(";"); + _builder.newLine(); + _builder.newLine(); + _builder.append("VarDef: "); + _builder.newLine(); + _builder.append(" "); + _builder.append("$$ !symbolTable.contains( input.LT( 1 ).getText() ) $$?=>"); + _builder.newLine(); + _builder.append(" "); + _builder.append("name=ID \'=\' value=Expr"); + _builder.newLine(); + _builder.append(" "); + _builder.append("$$ symbolTable.push(((org.xtext.example.mydsl.myDsl.VarDef) current).getName()); $$"); + _builder.newLine(); + _builder.append(";"); + _builder.newLine(); + _builder.newLine(); + _builder.append("Expr: "); + _builder.newLine(); + _builder.append(" "); + _builder.append("$$ symbolTable.contains( input.LT( 0 ).getText() ) $$?=>"); + _builder.newLine(); + _builder.append(" "); + _builder.append("var=ID"); + _builder.newLine(); + _builder.append(" "); + _builder.append("| builtin=ID \'(\' args += Expr ( args += Expr \',\' )* \')\'"); + _builder.newLine(); + _builder.append(" "); + _builder.append("| num=INT"); + _builder.newLine(); + _builder.append(";"); + _builder.newLine(); + this.doFullTranslateRun(_builder); + } + } + + protected void doFullTranslateRun(final CharSequence xtextGrammar) { + try { + EObject _model = super.getModel(xtextGrammar.toString()); + final Grammar grammar = ((Grammar) _model); + DefaultGeneratorModule _defaultGeneratorModule = new DefaultGeneratorModule(); + final Injector injector = Guice.createInjector(_defaultGeneratorModule); + final HoistingGeneratorBenchmark.InMemFSA inMem = new HoistingGeneratorBenchmark.InMemFSA(); + final AntlrOptions options = new AntlrOptions(); + System.out.println("production grammar"); + injector.getInstance(AntlrDebugProductionGrammarGenerator.class).generate(grammar, options, inMem); + System.out.println("content assist grammar"); + injector.getInstance(AntlrDebugContentAssistGrammarGenerator.class).generate(grammar, options, inMem); + } catch (Throwable _e) { + throw Exceptions.sneakyThrow(_e); + } + } +} diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/HoistingProcessor.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/HoistingProcessor.java index 928b4e7df..88c929ae6 100644 --- a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/HoistingProcessor.java +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/HoistingProcessor.java @@ -11,8 +11,6 @@ package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; -import java.util.Iterator; -import java.util.LinkedList; import java.util.Map; import java.util.List; import java.util.stream.Collectors; @@ -82,8 +80,8 @@ public class HoistingProcessor { analysis = new TokenAnalysis(config, grammar); } - private HoistingGuard findGuardForOptionalCardinalityWithoutContext(AbstractElement element, AbstractRule currentRule) { - HoistingGuard pathGuard = findGuardForElementWithTrivialCardinality(element, currentRule); + private HoistingGuard findGuardForOptionalCardinalityWithoutContext(AbstractElement element, AbstractRule currentRule, boolean skipCache) { + HoistingGuard pathGuard = findGuardForElementWithTrivialCardinality(element, currentRule, skipCache); if (pathGuard.isTrivial()) { // path can be skipped @@ -227,12 +225,12 @@ public class HoistingProcessor { boolean hasSeen = false; - private HoistingGuard findGuardForAlternatives(CompoundElement alternatives, AbstractRule currentRule) { + private HoistingGuard findGuardForAlternatives(CompoundElement alternatives, AbstractRule currentRule, boolean skipCache) { log.info("find guard for alternative"); List paths = new ArrayList<>(alternatives.getElements()); List guards = paths.stream() - .map(p -> findGuardForElement(p, currentRule)) + .map(p -> findGuardForElement(p, currentRule, skipCache)) .map(MergedPathGuard::new) .collect(Collectors.toList()); @@ -324,7 +322,7 @@ public class HoistingProcessor { } hasSeen=true;*/ - return findGuardForAlternatives(flattened, currentRule); + return findGuardForAlternatives(flattened, currentRule, true); } catch(TokenAnalysisAbortedException e) { throw new TokenAnalysisAbortedException(e.getMessage(), e, currentRule); } @@ -333,7 +331,7 @@ public class HoistingProcessor { } } - private HoistingGuard findGuardForUnorderedGroup(UnorderedGroup element, AbstractRule currentRule) { + private HoistingGuard findGuardForUnorderedGroup(UnorderedGroup element, AbstractRule currentRule, boolean skipCache) { // Unordered group (A & B) is the same as (A | B)+ or (A | B)* (if A and B are optional) // but the cardinality doesn't matter for hoisting // if A and B are optional the guard for the alternatives need to check the context @@ -341,14 +339,14 @@ public class HoistingProcessor { // TODO: check if hasTerminal is valid - return findGuardForAlternatives(element, currentRule); + return findGuardForAlternatives(element, currentRule, skipCache); } - private HoistingGuard findGuardForGroup(Group group, AbstractRule currentRule) { + private HoistingGuard findGuardForGroup(Group group, AbstractRule currentRule, boolean skipCache) { GroupGuard groupGuard = new GroupGuard(); for(AbstractElement element : group.getElements()) { - HoistingGuard guard = findGuardForElement(element, currentRule); + HoistingGuard guard = findGuardForElement(element, currentRule, skipCache); groupGuard.add(guard); // if cardinality is + and there is a terminal we don't need to consider @@ -367,7 +365,7 @@ public class HoistingProcessor { // TODO: make private public HoistingGuard findGuardForRule(AbstractRule rule) { log.info("finding guard for rule: " + rule.getName()); - return findGuardForElement(rule.getAlternatives(), rule); + return findGuardForElement(rule.getAlternatives(), rule, false); } private boolean pathHasTokenOrAction(AbstractElement path) { @@ -394,7 +392,6 @@ public class HoistingProcessor { } } - @SuppressWarnings("unused") private boolean pathHasHoistablePredicate(AbstractElement path) { // we are only interested in whether or not there could be any hoistable predicate on this path // -> ignore cardinality @@ -433,9 +430,9 @@ public class HoistingProcessor { AbstractRule rule = containingParserRule(element); if (element instanceof UnorderedGroup) { - return findGuardForAlternatives(((CompoundElement) element), rule); + return findGuardForAlternatives(((CompoundElement) element), rule, false); } else { - return findGuardForElementWithTrivialCardinality(element, rule); + return findGuardForElementWithTrivialCardinality(element, rule, false); } } @@ -444,40 +441,47 @@ public class HoistingProcessor { log.info("hoisting guard of: \n" + abstractElementToString(element)); // should only be called for valid AST elements, so element can never be floating - return findGuardForElement(element, containingParserRule(element)); + return findGuardForElement(element, containingParserRule(element), false); } - private HoistingGuard findGuardForElement(AbstractElement element, AbstractRule currentRule) { - String path = getPathOfElement(element); - - HoistingGuard guard = null;//elementCache.get(path); - if (guard != null) { - log.info("from cache: " + path); - return guard; + private HoistingGuard findGuardForElement(AbstractElement element, AbstractRule currentRule, boolean skipCache) { + + String path = null; + HoistingGuard guard; + + if (!skipCache) { + path = getPathOfElement(element); + guard = elementCache.get(path); + if (guard != null) { + log.info("from cache: " + path); + return guard; + } } if (isTrivialCardinality(element) || isOneOrMoreCardinality(element) ) { - guard = findGuardForElementWithTrivialCardinality(element, currentRule); + guard = findGuardForElementWithTrivialCardinality(element, currentRule, skipCache); } else if (isOptionalCardinality(element)) { - guard = findGuardForOptionalCardinalityWithoutContext(element, currentRule); + guard = findGuardForOptionalCardinalityWithoutContext(element, currentRule, skipCache); } else { throw new IllegalArgumentException("unknown cardinality: " + element.getCardinality()); } - elementCache.put(path, guard); + if (!skipCache) { + elementCache.put(path, guard); + } return guard; } - private HoistingGuard findGuardForElementWithTrivialCardinality(AbstractElement element, AbstractRule currentRule) { + private HoistingGuard findGuardForElementWithTrivialCardinality(AbstractElement element, AbstractRule currentRule, boolean skipCache) { if (config.isDebug()) log.info(currentRule.getName() + ": " + abstractElementToShortString(element)); if (element instanceof Alternatives) { - return findGuardForAlternatives((Alternatives) element, currentRule); + return findGuardForAlternatives((Alternatives) element, currentRule, skipCache); } else if (element instanceof Group) { - return findGuardForGroup((Group) element, currentRule); + return findGuardForGroup((Group) element, currentRule, skipCache); } else if (element instanceof AbstractSemanticPredicate) { return new PredicateGuard((AbstractSemanticPredicate) element); } else if (Token.isToken(element)) { @@ -491,9 +495,9 @@ public class HoistingProcessor { } else if (element instanceof JavaAction) { return HoistingGuard.action(); } else if (element instanceof UnorderedGroup) { - return findGuardForUnorderedGroup((UnorderedGroup) element, currentRule); + return findGuardForUnorderedGroup((UnorderedGroup) element, currentRule, skipCache); } else if (element instanceof Assignment) { - return findGuardForElement(((Assignment) element).getTerminal(), currentRule); + return findGuardForElement(((Assignment) element).getTerminal(), currentRule, skipCache); } else { if (!pathHasHoistablePredicate(currentRule.getAlternatives())) { // unsupported construct but rule doesn't contain hoistable predicates diff --git a/org.eclipse.xtext/src/org/eclipse/xtext/GrammarUtil.java b/org.eclipse.xtext/src/org/eclipse/xtext/GrammarUtil.java index a1431d804..7e8582afc 100644 --- a/org.eclipse.xtext/src/org/eclipse/xtext/GrammarUtil.java +++ b/org.eclipse.xtext/src/org/eclipse/xtext/GrammarUtil.java @@ -769,4 +769,38 @@ public class GrammarUtil { // and changes to the copied path will be considered for .equals() return grammar.getRules().get(0).getName().equals(rule.getName()); } + + public static String getPathOfElement(AbstractRule element) { + return element.getName(); + } + + public static String getPathOfElement(AbstractElement element) { + String path = element.eClass().getName(); + EObject container = element; + EObject _element = element; + while ( + container != null && ( + container == element || ( + !(container instanceof AbstractElement) && + !(container instanceof AbstractRule) + ) + ) + ) { + _element = container; + container = container.eContainer(); + } + + if (container == null) { + System.out.println("no container: " + element.eClass().getName()); + return path; + } else if (container instanceof CompoundElement) { + CompoundElement compoundElement = (CompoundElement) container; + int i = compoundElement.getElements().indexOf(_element); + return getPathOfElement(compoundElement) + "." + i + "." + path; + } else if (container instanceof AbstractRule) { + return getPathOfElement((AbstractRule) container) + "." + path; + } else { + return getPathOfElement((AbstractElement) container) + "." + path; + } + } }