From e2984ad177e15bd70a58019b51fcf131a4ccd973 Mon Sep 17 00:00:00 2001 From: overflowerror Date: Fri, 28 Jan 2022 20:18:35 +0100 Subject: [PATCH] added benchmark for hoisting & fixed caching use path in grammar as key for caching elements; using the elements itself doesn't work because the reference will be different for the content assist grammar run. this does not work for generated elements; the only place that generated elements are necessary is when dealing with nested identical paths, in this case caching is disabled. the benchmark test case does not validate the result since we are only interested in the runtime. --- .../hoisting/HoistingGeneratorBenchmark.xtend | 96 +++++++++++++ .../hoisting/HoistingGeneratorBenchmark.java | 134 ++++++++++++++++++ .../antlr/hoisting/HoistingProcessor.java | 66 +++++---- .../src/org/eclipse/xtext/GrammarUtil.java | 34 +++++ 4 files changed, 299 insertions(+), 31 deletions(-) create mode 100644 org.eclipse.xtext.tests/src/org/eclipse/xtext/xtext/generator/hoisting/HoistingGeneratorBenchmark.xtend create mode 100644 org.eclipse.xtext.tests/xtend-gen/org/eclipse/xtext/xtext/generator/hoisting/HoistingGeneratorBenchmark.java diff --git a/org.eclipse.xtext.tests/src/org/eclipse/xtext/xtext/generator/hoisting/HoistingGeneratorBenchmark.xtend b/org.eclipse.xtext.tests/src/org/eclipse/xtext/xtext/generator/hoisting/HoistingGeneratorBenchmark.xtend new file mode 100644 index 000000000..1639e7b8c --- /dev/null +++ b/org.eclipse.xtext.tests/src/org/eclipse/xtext/xtext/generator/hoisting/HoistingGeneratorBenchmark.xtend @@ -0,0 +1,96 @@ +/******************************************************************************* + * Copyright (c) 2022 itemis AG (http://www.itemis.eu) and others. + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *******************************************************************************/ +package org.eclipse.xtext.xtext.generator.hoisting + +import org.eclipse.xtext.tests.AbstractXtextTests +import org.eclipse.xtext.XtextStandaloneSetup +import org.junit.Test +import org.eclipse.xtext.Grammar +import org.eclipse.xtext.xtext.generator.DefaultGeneratorModule +import com.google.inject.Guice +import org.eclipse.xtext.xtext.generator.parser.antlr.AntlrOptions +import org.eclipse.xtext.generator.InMemoryFileSystemAccess +import org.eclipse.xtext.xtext.generator.model.IXtextGeneratorFileSystemAccess +import com.google.inject.Injector +import org.eclipse.xtext.xtext.generator.parser.antlr.AntlrDebugProductionGrammarGenerator +import org.eclipse.xtext.xtext.generator.parser.antlr.AntlrDebugContentAssistGrammarGenerator + +/** + * @author overflow - Initial contribution and API + */ +class HoistingGeneratorBenchmark extends AbstractXtextTests { + + val REPETITIONS = 1000 + + override void setUp() throws Exception { + super.setUp() + with(XtextStandaloneSetup) + } + + + + @Test + def void testFullTranslationBenchmark() { + for (var i = 0; i < REPETITIONS; i++) { + ''' + grammar com.foo.bar with org.eclipse.xtext.common.Terminals + import "http://www.eclipse.org/emf/2002/Ecore" as ecore + generate myPack 'http://mypack' + + setup $$ + java.util.Stack symbolTable = new java.util.Stack(); + $$ + + Statements: + defs+=VarDef+ + ; + + VarDef: + $$ !symbolTable.contains( input.LT( 1 ).getText() ) $$?=> + name=ID '=' value=Expr + $$ symbolTable.push(((org.xtext.example.mydsl.myDsl.VarDef) current).getName()); $$ + ; + + Expr: + $$ symbolTable.contains( input.LT( 0 ).getText() ) $$?=> + var=ID + | builtin=ID '(' args += Expr ( args += Expr ',' )* ')' + | num=INT + ; + '''.doFullTranslateRun + } + } + + protected def void doFullTranslateRun(CharSequence xtextGrammar) { + val grammar = super.getModel(xtextGrammar.toString) as Grammar + val injector = Guice.createInjector(new DefaultGeneratorModule) + val inMem = new InMemFSA + val options = new AntlrOptions + System.out.println("production grammar") + injector.getInstance(AntlrDebugProductionGrammarGenerator).generate(grammar, options, inMem) + System.out.println("content assist grammar"); + injector.getInstance(AntlrDebugContentAssistGrammarGenerator).generate(grammar, options, inMem) + } + + static class InMemFSA extends InMemoryFileSystemAccess implements IXtextGeneratorFileSystemAccess { + + override getPath() { + "./" + } + + override isOverwrite() { + true + } + + override initialize(Injector injector) { + injector.injectMembers(this) + } + + } +} \ No newline at end of file diff --git a/org.eclipse.xtext.tests/xtend-gen/org/eclipse/xtext/xtext/generator/hoisting/HoistingGeneratorBenchmark.java b/org.eclipse.xtext.tests/xtend-gen/org/eclipse/xtext/xtext/generator/hoisting/HoistingGeneratorBenchmark.java new file mode 100644 index 000000000..5e6420f3c --- /dev/null +++ b/org.eclipse.xtext.tests/xtend-gen/org/eclipse/xtext/xtext/generator/hoisting/HoistingGeneratorBenchmark.java @@ -0,0 +1,134 @@ +/** + * Copyright (c) 2022 itemis AG (http://www.itemis.eu) and others. + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.xtext.xtext.generator.hoisting; + +import com.google.inject.Guice; +import com.google.inject.Injector; +import org.eclipse.emf.ecore.EObject; +import org.eclipse.xtend2.lib.StringConcatenation; +import org.eclipse.xtext.Grammar; +import org.eclipse.xtext.XtextStandaloneSetup; +import org.eclipse.xtext.generator.InMemoryFileSystemAccess; +import org.eclipse.xtext.tests.AbstractXtextTests; +import org.eclipse.xtext.xbase.lib.Exceptions; +import org.eclipse.xtext.xtext.generator.DefaultGeneratorModule; +import org.eclipse.xtext.xtext.generator.model.IXtextGeneratorFileSystemAccess; +import org.eclipse.xtext.xtext.generator.parser.antlr.AntlrDebugContentAssistGrammarGenerator; +import org.eclipse.xtext.xtext.generator.parser.antlr.AntlrDebugProductionGrammarGenerator; +import org.eclipse.xtext.xtext.generator.parser.antlr.AntlrOptions; +import org.junit.Test; + +/** + * @author overflow - Initial contribution and API + */ +@SuppressWarnings("all") +public class HoistingGeneratorBenchmark extends AbstractXtextTests { + public static class InMemFSA extends InMemoryFileSystemAccess implements IXtextGeneratorFileSystemAccess { + @Override + public String getPath() { + return "./"; + } + + @Override + public boolean isOverwrite() { + return true; + } + + @Override + public void initialize(final Injector injector) { + injector.injectMembers(this); + } + } + + private final int REPETITIONS = 1000; + + @Override + public void setUp() throws Exception { + super.setUp(); + this.with(XtextStandaloneSetup.class); + } + + @Test + public void testFullTranslationBenchmark() { + for (int i = 0; (i < this.REPETITIONS); i++) { + StringConcatenation _builder = new StringConcatenation(); + _builder.append("grammar com.foo.bar with org.eclipse.xtext.common.Terminals"); + _builder.newLine(); + _builder.append("import \"http://www.eclipse.org/emf/2002/Ecore\" as ecore"); + _builder.newLine(); + _builder.append("generate myPack \'http://mypack\'"); + _builder.newLine(); + _builder.newLine(); + _builder.append("setup $$"); + _builder.newLine(); + _builder.append("\t"); + _builder.append("java.util.Stack symbolTable = new java.util.Stack();"); + _builder.newLine(); + _builder.append("$$"); + _builder.newLine(); + _builder.newLine(); + _builder.append("Statements:"); + _builder.newLine(); + _builder.append(" "); + _builder.append("defs+=VarDef+"); + _builder.newLine(); + _builder.append(";"); + _builder.newLine(); + _builder.newLine(); + _builder.append("VarDef: "); + _builder.newLine(); + _builder.append(" "); + _builder.append("$$ !symbolTable.contains( input.LT( 1 ).getText() ) $$?=>"); + _builder.newLine(); + _builder.append(" "); + _builder.append("name=ID \'=\' value=Expr"); + _builder.newLine(); + _builder.append(" "); + _builder.append("$$ symbolTable.push(((org.xtext.example.mydsl.myDsl.VarDef) current).getName()); $$"); + _builder.newLine(); + _builder.append(";"); + _builder.newLine(); + _builder.newLine(); + _builder.append("Expr: "); + _builder.newLine(); + _builder.append(" "); + _builder.append("$$ symbolTable.contains( input.LT( 0 ).getText() ) $$?=>"); + _builder.newLine(); + _builder.append(" "); + _builder.append("var=ID"); + _builder.newLine(); + _builder.append(" "); + _builder.append("| builtin=ID \'(\' args += Expr ( args += Expr \',\' )* \')\'"); + _builder.newLine(); + _builder.append(" "); + _builder.append("| num=INT"); + _builder.newLine(); + _builder.append(";"); + _builder.newLine(); + this.doFullTranslateRun(_builder); + } + } + + protected void doFullTranslateRun(final CharSequence xtextGrammar) { + try { + EObject _model = super.getModel(xtextGrammar.toString()); + final Grammar grammar = ((Grammar) _model); + DefaultGeneratorModule _defaultGeneratorModule = new DefaultGeneratorModule(); + final Injector injector = Guice.createInjector(_defaultGeneratorModule); + final HoistingGeneratorBenchmark.InMemFSA inMem = new HoistingGeneratorBenchmark.InMemFSA(); + final AntlrOptions options = new AntlrOptions(); + System.out.println("production grammar"); + injector.getInstance(AntlrDebugProductionGrammarGenerator.class).generate(grammar, options, inMem); + System.out.println("content assist grammar"); + injector.getInstance(AntlrDebugContentAssistGrammarGenerator.class).generate(grammar, options, inMem); + } catch (Throwable _e) { + throw Exceptions.sneakyThrow(_e); + } + } +} diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/HoistingProcessor.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/HoistingProcessor.java index 928b4e7df..88c929ae6 100644 --- a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/HoistingProcessor.java +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/HoistingProcessor.java @@ -11,8 +11,6 @@ package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; -import java.util.Iterator; -import java.util.LinkedList; import java.util.Map; import java.util.List; import java.util.stream.Collectors; @@ -82,8 +80,8 @@ public class HoistingProcessor { analysis = new TokenAnalysis(config, grammar); } - private HoistingGuard findGuardForOptionalCardinalityWithoutContext(AbstractElement element, AbstractRule currentRule) { - HoistingGuard pathGuard = findGuardForElementWithTrivialCardinality(element, currentRule); + private HoistingGuard findGuardForOptionalCardinalityWithoutContext(AbstractElement element, AbstractRule currentRule, boolean skipCache) { + HoistingGuard pathGuard = findGuardForElementWithTrivialCardinality(element, currentRule, skipCache); if (pathGuard.isTrivial()) { // path can be skipped @@ -227,12 +225,12 @@ public class HoistingProcessor { boolean hasSeen = false; - private HoistingGuard findGuardForAlternatives(CompoundElement alternatives, AbstractRule currentRule) { + private HoistingGuard findGuardForAlternatives(CompoundElement alternatives, AbstractRule currentRule, boolean skipCache) { log.info("find guard for alternative"); List paths = new ArrayList<>(alternatives.getElements()); List guards = paths.stream() - .map(p -> findGuardForElement(p, currentRule)) + .map(p -> findGuardForElement(p, currentRule, skipCache)) .map(MergedPathGuard::new) .collect(Collectors.toList()); @@ -324,7 +322,7 @@ public class HoistingProcessor { } hasSeen=true;*/ - return findGuardForAlternatives(flattened, currentRule); + return findGuardForAlternatives(flattened, currentRule, true); } catch(TokenAnalysisAbortedException e) { throw new TokenAnalysisAbortedException(e.getMessage(), e, currentRule); } @@ -333,7 +331,7 @@ public class HoistingProcessor { } } - private HoistingGuard findGuardForUnorderedGroup(UnorderedGroup element, AbstractRule currentRule) { + private HoistingGuard findGuardForUnorderedGroup(UnorderedGroup element, AbstractRule currentRule, boolean skipCache) { // Unordered group (A & B) is the same as (A | B)+ or (A | B)* (if A and B are optional) // but the cardinality doesn't matter for hoisting // if A and B are optional the guard for the alternatives need to check the context @@ -341,14 +339,14 @@ public class HoistingProcessor { // TODO: check if hasTerminal is valid - return findGuardForAlternatives(element, currentRule); + return findGuardForAlternatives(element, currentRule, skipCache); } - private HoistingGuard findGuardForGroup(Group group, AbstractRule currentRule) { + private HoistingGuard findGuardForGroup(Group group, AbstractRule currentRule, boolean skipCache) { GroupGuard groupGuard = new GroupGuard(); for(AbstractElement element : group.getElements()) { - HoistingGuard guard = findGuardForElement(element, currentRule); + HoistingGuard guard = findGuardForElement(element, currentRule, skipCache); groupGuard.add(guard); // if cardinality is + and there is a terminal we don't need to consider @@ -367,7 +365,7 @@ public class HoistingProcessor { // TODO: make private public HoistingGuard findGuardForRule(AbstractRule rule) { log.info("finding guard for rule: " + rule.getName()); - return findGuardForElement(rule.getAlternatives(), rule); + return findGuardForElement(rule.getAlternatives(), rule, false); } private boolean pathHasTokenOrAction(AbstractElement path) { @@ -394,7 +392,6 @@ public class HoistingProcessor { } } - @SuppressWarnings("unused") private boolean pathHasHoistablePredicate(AbstractElement path) { // we are only interested in whether or not there could be any hoistable predicate on this path // -> ignore cardinality @@ -433,9 +430,9 @@ public class HoistingProcessor { AbstractRule rule = containingParserRule(element); if (element instanceof UnorderedGroup) { - return findGuardForAlternatives(((CompoundElement) element), rule); + return findGuardForAlternatives(((CompoundElement) element), rule, false); } else { - return findGuardForElementWithTrivialCardinality(element, rule); + return findGuardForElementWithTrivialCardinality(element, rule, false); } } @@ -444,40 +441,47 @@ public class HoistingProcessor { log.info("hoisting guard of: \n" + abstractElementToString(element)); // should only be called for valid AST elements, so element can never be floating - return findGuardForElement(element, containingParserRule(element)); + return findGuardForElement(element, containingParserRule(element), false); } - private HoistingGuard findGuardForElement(AbstractElement element, AbstractRule currentRule) { - String path = getPathOfElement(element); - - HoistingGuard guard = null;//elementCache.get(path); - if (guard != null) { - log.info("from cache: " + path); - return guard; + private HoistingGuard findGuardForElement(AbstractElement element, AbstractRule currentRule, boolean skipCache) { + + String path = null; + HoistingGuard guard; + + if (!skipCache) { + path = getPathOfElement(element); + guard = elementCache.get(path); + if (guard != null) { + log.info("from cache: " + path); + return guard; + } } if (isTrivialCardinality(element) || isOneOrMoreCardinality(element) ) { - guard = findGuardForElementWithTrivialCardinality(element, currentRule); + guard = findGuardForElementWithTrivialCardinality(element, currentRule, skipCache); } else if (isOptionalCardinality(element)) { - guard = findGuardForOptionalCardinalityWithoutContext(element, currentRule); + guard = findGuardForOptionalCardinalityWithoutContext(element, currentRule, skipCache); } else { throw new IllegalArgumentException("unknown cardinality: " + element.getCardinality()); } - elementCache.put(path, guard); + if (!skipCache) { + elementCache.put(path, guard); + } return guard; } - private HoistingGuard findGuardForElementWithTrivialCardinality(AbstractElement element, AbstractRule currentRule) { + private HoistingGuard findGuardForElementWithTrivialCardinality(AbstractElement element, AbstractRule currentRule, boolean skipCache) { if (config.isDebug()) log.info(currentRule.getName() + ": " + abstractElementToShortString(element)); if (element instanceof Alternatives) { - return findGuardForAlternatives((Alternatives) element, currentRule); + return findGuardForAlternatives((Alternatives) element, currentRule, skipCache); } else if (element instanceof Group) { - return findGuardForGroup((Group) element, currentRule); + return findGuardForGroup((Group) element, currentRule, skipCache); } else if (element instanceof AbstractSemanticPredicate) { return new PredicateGuard((AbstractSemanticPredicate) element); } else if (Token.isToken(element)) { @@ -491,9 +495,9 @@ public class HoistingProcessor { } else if (element instanceof JavaAction) { return HoistingGuard.action(); } else if (element instanceof UnorderedGroup) { - return findGuardForUnorderedGroup((UnorderedGroup) element, currentRule); + return findGuardForUnorderedGroup((UnorderedGroup) element, currentRule, skipCache); } else if (element instanceof Assignment) { - return findGuardForElement(((Assignment) element).getTerminal(), currentRule); + return findGuardForElement(((Assignment) element).getTerminal(), currentRule, skipCache); } else { if (!pathHasHoistablePredicate(currentRule.getAlternatives())) { // unsupported construct but rule doesn't contain hoistable predicates diff --git a/org.eclipse.xtext/src/org/eclipse/xtext/GrammarUtil.java b/org.eclipse.xtext/src/org/eclipse/xtext/GrammarUtil.java index a1431d804..7e8582afc 100644 --- a/org.eclipse.xtext/src/org/eclipse/xtext/GrammarUtil.java +++ b/org.eclipse.xtext/src/org/eclipse/xtext/GrammarUtil.java @@ -769,4 +769,38 @@ public class GrammarUtil { // and changes to the copied path will be considered for .equals() return grammar.getRules().get(0).getName().equals(rule.getName()); } + + public static String getPathOfElement(AbstractRule element) { + return element.getName(); + } + + public static String getPathOfElement(AbstractElement element) { + String path = element.eClass().getName(); + EObject container = element; + EObject _element = element; + while ( + container != null && ( + container == element || ( + !(container instanceof AbstractElement) && + !(container instanceof AbstractRule) + ) + ) + ) { + _element = container; + container = container.eContainer(); + } + + if (container == null) { + System.out.println("no container: " + element.eClass().getName()); + return path; + } else if (container instanceof CompoundElement) { + CompoundElement compoundElement = (CompoundElement) container; + int i = compoundElement.getElements().indexOf(_element); + return getPathOfElement(compoundElement) + "." + i + "." + path; + } else if (container instanceof AbstractRule) { + return getPathOfElement((AbstractRule) container) + "." + path; + } else { + return getPathOfElement((AbstractElement) container) + "." + path; + } + } }