added benchmark for hoisting & fixed caching

use path in grammar as key for caching elements; using the elements
itself doesn't work because the reference will be different for the
content assist grammar run.
this does not work for generated elements; the only place that generated
elements are necessary is when dealing with nested identical paths, in
this case caching is disabled.

the benchmark test case does not validate the result since we are only
interested in the runtime.

org.eclipse.xtext.tests/src/org/eclipse/xtext/xtext/generator/hoisting/HoistingGeneratorBenchmark.xtend

@@ -0,0 +1,96 @@
+/*******************************************************************************
+ * Copyright (c) 2022 itemis AG (http://www.itemis.eu) and others.
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ * 
+ * SPDX-License-Identifier: EPL-2.0
+ *******************************************************************************/
+package org.eclipse.xtext.xtext.generator.hoisting
+
+import org.eclipse.xtext.tests.AbstractXtextTests
+import org.eclipse.xtext.XtextStandaloneSetup
+import org.junit.Test
+import org.eclipse.xtext.Grammar
+import org.eclipse.xtext.xtext.generator.DefaultGeneratorModule
+import com.google.inject.Guice
+import org.eclipse.xtext.xtext.generator.parser.antlr.AntlrOptions
+import org.eclipse.xtext.generator.InMemoryFileSystemAccess
+import org.eclipse.xtext.xtext.generator.model.IXtextGeneratorFileSystemAccess
+import com.google.inject.Injector
+import org.eclipse.xtext.xtext.generator.parser.antlr.AntlrDebugProductionGrammarGenerator
+import org.eclipse.xtext.xtext.generator.parser.antlr.AntlrDebugContentAssistGrammarGenerator
+
+/**
+ * @author overflow - Initial contribution and API
+ */
+class HoistingGeneratorBenchmark extends AbstractXtextTests {
+	
+	val REPETITIONS = 1000
+	
+	override void setUp() throws Exception {
+		super.setUp()
+		with(XtextStandaloneSetup)
+	}
+	
+	
+	
+	@Test
+	def void testFullTranslationBenchmark() {
+		for (var i = 0; i < REPETITIONS; i++) {
+			'''
+				grammar com.foo.bar with org.eclipse.xtext.common.Terminals
+				import "http://www.eclipse.org/emf/2002/Ecore" as ecore
+				generate myPack 'http://mypack'
+				
+				setup $$
+					java.util.Stack<String> symbolTable = new java.util.Stack<String>();
+				$$
+				
+				Statements:
+				            defs+=VarDef+
+				;
+				
+				VarDef: 
+				        $$ !symbolTable.contains( input.LT( 1 ).getText() ) $$?=>
+				        name=ID '=' value=Expr
+				        $$ symbolTable.push(((org.xtext.example.mydsl.myDsl.VarDef) current).getName()); $$
+				;
+				
+				Expr: 
+				      $$ symbolTable.contains( input.LT( 0 ).getText() ) $$?=>
+				      var=ID
+				    | builtin=ID '(' args += Expr ( args += Expr ',' )* ')'
+				    | num=INT
+				;
+			'''.doFullTranslateRun
+		}
+	}
+	
+	protected def void doFullTranslateRun(CharSequence xtextGrammar) {
+		val grammar = super.getModel(xtextGrammar.toString) as Grammar
+		val injector = Guice.createInjector(new DefaultGeneratorModule)
+		val inMem = new InMemFSA
+		val options = new AntlrOptions
+		System.out.println("production grammar")
+		injector.getInstance(AntlrDebugProductionGrammarGenerator).generate(grammar, options, inMem)
+		System.out.println("content assist grammar");
+		injector.getInstance(AntlrDebugContentAssistGrammarGenerator).generate(grammar, options, inMem)
+	}
+	
+	static class InMemFSA extends InMemoryFileSystemAccess implements IXtextGeneratorFileSystemAccess {
+		
+		override getPath() {
+			"./"
+		}
+		
+		override isOverwrite() {
+			true
+		}
+		
+		override initialize(Injector injector) {
+			injector.injectMembers(this)
+		}
+		
+	}
+}

org.eclipse.xtext.tests/xtend-gen/org/eclipse/xtext/xtext/generator/hoisting/HoistingGeneratorBenchmark.java

@@ -0,0 +1,134 @@
+/**
+ * Copyright (c) 2022 itemis AG (http://www.itemis.eu) and others.
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ * 
+ * SPDX-License-Identifier: EPL-2.0
+ */
+package org.eclipse.xtext.xtext.generator.hoisting;
+
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import org.eclipse.emf.ecore.EObject;
+import org.eclipse.xtend2.lib.StringConcatenation;
+import org.eclipse.xtext.Grammar;
+import org.eclipse.xtext.XtextStandaloneSetup;
+import org.eclipse.xtext.generator.InMemoryFileSystemAccess;
+import org.eclipse.xtext.tests.AbstractXtextTests;
+import org.eclipse.xtext.xbase.lib.Exceptions;
+import org.eclipse.xtext.xtext.generator.DefaultGeneratorModule;
+import org.eclipse.xtext.xtext.generator.model.IXtextGeneratorFileSystemAccess;
+import org.eclipse.xtext.xtext.generator.parser.antlr.AntlrDebugContentAssistGrammarGenerator;
+import org.eclipse.xtext.xtext.generator.parser.antlr.AntlrDebugProductionGrammarGenerator;
+import org.eclipse.xtext.xtext.generator.parser.antlr.AntlrOptions;
+import org.junit.Test;
+
+/**
+ * @author overflow - Initial contribution and API
+ */
+@SuppressWarnings("all")
+public class HoistingGeneratorBenchmark extends AbstractXtextTests {
+  public static class InMemFSA extends InMemoryFileSystemAccess implements IXtextGeneratorFileSystemAccess {
+    @Override
+    public String getPath() {
+      return "./";
+    }
+    
+    @Override
+    public boolean isOverwrite() {
+      return true;
+    }
+    
+    @Override
+    public void initialize(final Injector injector) {
+      injector.injectMembers(this);
+    }
+  }
+  
+  private final int REPETITIONS = 1000;
+  
+  @Override
+  public void setUp() throws Exception {
+    super.setUp();
+    this.with(XtextStandaloneSetup.class);
+  }
+  
+  @Test
+  public void testFullTranslationBenchmark() {
+    for (int i = 0; (i < this.REPETITIONS); i++) {
+      StringConcatenation _builder = new StringConcatenation();
+      _builder.append("grammar com.foo.bar with org.eclipse.xtext.common.Terminals");
+      _builder.newLine();
+      _builder.append("import \"http://www.eclipse.org/emf/2002/Ecore\" as ecore");
+      _builder.newLine();
+      _builder.append("generate myPack \'http://mypack\'");
+      _builder.newLine();
+      _builder.newLine();
+      _builder.append("setup $$");
+      _builder.newLine();
+      _builder.append("\t");
+      _builder.append("java.util.Stack<String> symbolTable = new java.util.Stack<String>();");
+      _builder.newLine();
+      _builder.append("$$");
+      _builder.newLine();
+      _builder.newLine();
+      _builder.append("Statements:");
+      _builder.newLine();
+      _builder.append("            ");
+      _builder.append("defs+=VarDef+");
+      _builder.newLine();
+      _builder.append(";");
+      _builder.newLine();
+      _builder.newLine();
+      _builder.append("VarDef: ");
+      _builder.newLine();
+      _builder.append("        ");
+      _builder.append("$$ !symbolTable.contains( input.LT( 1 ).getText() ) $$?=>");
+      _builder.newLine();
+      _builder.append("        ");
+      _builder.append("name=ID \'=\' value=Expr");
+      _builder.newLine();
+      _builder.append("        ");
+      _builder.append("$$ symbolTable.push(((org.xtext.example.mydsl.myDsl.VarDef) current).getName()); $$");
+      _builder.newLine();
+      _builder.append(";");
+      _builder.newLine();
+      _builder.newLine();
+      _builder.append("Expr: ");
+      _builder.newLine();
+      _builder.append("      ");
+      _builder.append("$$ symbolTable.contains( input.LT( 0 ).getText() ) $$?=>");
+      _builder.newLine();
+      _builder.append("      ");
+      _builder.append("var=ID");
+      _builder.newLine();
+      _builder.append("    ");
+      _builder.append("| builtin=ID \'(\' args += Expr ( args += Expr \',\' )* \')\'");
+      _builder.newLine();
+      _builder.append("    ");
+      _builder.append("| num=INT");
+      _builder.newLine();
+      _builder.append(";");
+      _builder.newLine();
+      this.doFullTranslateRun(_builder);
+    }
+  }
+  
+  protected void doFullTranslateRun(final CharSequence xtextGrammar) {
+    try {
+      EObject _model = super.getModel(xtextGrammar.toString());
+      final Grammar grammar = ((Grammar) _model);
+      DefaultGeneratorModule _defaultGeneratorModule = new DefaultGeneratorModule();
+      final Injector injector = Guice.createInjector(_defaultGeneratorModule);
+      final HoistingGeneratorBenchmark.InMemFSA inMem = new HoistingGeneratorBenchmark.InMemFSA();
+      final AntlrOptions options = new AntlrOptions();
+      System.out.println("production grammar");
+      injector.<AntlrDebugProductionGrammarGenerator>getInstance(AntlrDebugProductionGrammarGenerator.class).generate(grammar, options, inMem);
+      System.out.println("content assist grammar");
+      injector.<AntlrDebugContentAssistGrammarGenerator>getInstance(AntlrDebugContentAssistGrammarGenerator.class).generate(grammar, options, inMem);
+    } catch (Throwable _e) {
+      throw Exceptions.sneakyThrow(_e);
+    }
+  }
+}

org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/HoistingProcessor.java

@@ -11,8 +11,6 @@ package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
-import java.util.Iterator;
-import java.util.LinkedList;
 import java.util.Map;
 import java.util.List;
 import java.util.stream.Collectors;
@@ -82,8 +80,8 @@ public class HoistingProcessor {
 		analysis = new TokenAnalysis(config, grammar);
 	}
 	
-	private HoistingGuard findGuardForOptionalCardinalityWithoutContext(AbstractElement element, AbstractRule currentRule) {
-		HoistingGuard pathGuard = findGuardForElementWithTrivialCardinality(element, currentRule);
+	private HoistingGuard findGuardForOptionalCardinalityWithoutContext(AbstractElement element, AbstractRule currentRule, boolean skipCache) {
+		HoistingGuard pathGuard = findGuardForElementWithTrivialCardinality(element, currentRule, skipCache);
 		
 		if (pathGuard.isTrivial()) {
 			// path can be skipped
@@ -227,12 +225,12 @@ public class HoistingProcessor {
 	
 	boolean hasSeen = false;
 	
-	private HoistingGuard findGuardForAlternatives(CompoundElement alternatives, AbstractRule currentRule) {
+	private HoistingGuard findGuardForAlternatives(CompoundElement alternatives, AbstractRule currentRule, boolean skipCache) {
 		log.info("find guard for alternative");
 		
 		List<AbstractElement> paths = new ArrayList<>(alternatives.getElements());
 		List<MergedPathGuard> guards = paths.stream()
-				.map(p -> findGuardForElement(p, currentRule))
+				.map(p -> findGuardForElement(p, currentRule, skipCache))
 				.map(MergedPathGuard::new)
 				.collect(Collectors.toList());
 		
@@ -324,7 +322,7 @@ public class HoistingProcessor {
 				}
 				hasSeen=true;*/
 				
-				return findGuardForAlternatives(flattened, currentRule);
+				return findGuardForAlternatives(flattened, currentRule, true);
 			} catch(TokenAnalysisAbortedException e) {
 				throw new TokenAnalysisAbortedException(e.getMessage(), e, currentRule);
 			}
@@ -333,7 +331,7 @@ public class HoistingProcessor {
 		}
 	}
 	
-	private HoistingGuard findGuardForUnorderedGroup(UnorderedGroup element, AbstractRule currentRule) {
+	private HoistingGuard findGuardForUnorderedGroup(UnorderedGroup element, AbstractRule currentRule, boolean skipCache) {
 		// Unordered group (A & B) is the same as (A | B)+ or (A | B)* (if A and B are optional)
 		// but the cardinality doesn't matter for hoisting
 		// if A and B are optional the guard for the alternatives need to check the context
@@ -341,14 +339,14 @@ public class HoistingProcessor {
 		
 		// TODO: check if hasTerminal is valid
 		
-		return findGuardForAlternatives(element, currentRule);
+		return findGuardForAlternatives(element, currentRule, skipCache);
 	}
 	
-	private HoistingGuard findGuardForGroup(Group group, AbstractRule currentRule) {
+	private HoistingGuard findGuardForGroup(Group group, AbstractRule currentRule, boolean skipCache) {
 		GroupGuard groupGuard = new GroupGuard();
 		
 		for(AbstractElement element : group.getElements()) {
-			HoistingGuard guard = findGuardForElement(element, currentRule);
+			HoistingGuard guard = findGuardForElement(element, currentRule, skipCache);
 			groupGuard.add(guard);
 			
 			// if cardinality is + and there is a terminal we don't need to consider
@@ -367,7 +365,7 @@ public class HoistingProcessor {
 	// TODO: make private
 	public HoistingGuard findGuardForRule(AbstractRule rule) {
 		log.info("finding guard for rule: " + rule.getName());
-		return findGuardForElement(rule.getAlternatives(), rule);
+		return findGuardForElement(rule.getAlternatives(), rule, false);
 	}
 	
 	private boolean pathHasTokenOrAction(AbstractElement path) {
@@ -394,7 +392,6 @@ public class HoistingProcessor {
 		}
 	}
 	
-	@SuppressWarnings("unused")
 	private boolean pathHasHoistablePredicate(AbstractElement path) {
 		// we are only interested in whether or not there could be any hoistable predicate on this path
 		// -> ignore cardinality
@@ -433,9 +430,9 @@ public class HoistingProcessor {
 		
 		AbstractRule rule = containingParserRule(element);
 		if (element instanceof UnorderedGroup) {
-			return findGuardForAlternatives(((CompoundElement) element), rule);
+			return findGuardForAlternatives(((CompoundElement) element), rule, false);
 		} else {
-			return findGuardForElementWithTrivialCardinality(element, rule);
+			return findGuardForElementWithTrivialCardinality(element, rule, false);
 		}
 	}
 	
@@ -444,40 +441,47 @@ public class HoistingProcessor {
 			log.info("hoisting guard of: \n" + abstractElementToString(element));
 		
 		// should only be called for valid AST elements, so element can never be floating
-		return findGuardForElement(element, containingParserRule(element));
+		return findGuardForElement(element, containingParserRule(element), false);
 	}
 	
-	private HoistingGuard findGuardForElement(AbstractElement element, AbstractRule currentRule) {
-		String path = getPathOfElement(element);
+	private HoistingGuard findGuardForElement(AbstractElement element, AbstractRule currentRule, boolean skipCache) {
 
-		HoistingGuard guard = null;//elementCache.get(path);
+		String path = null; 
+		HoistingGuard guard;
+
+		if (!skipCache) {
+			path = getPathOfElement(element);
+			guard = elementCache.get(path);
 			if (guard != null) {
 				log.info("from cache: " + path);
 				return guard;
 			}
+		}
 		
 		if (isTrivialCardinality(element) ||
 			isOneOrMoreCardinality(element)
 		) {
-			guard = findGuardForElementWithTrivialCardinality(element, currentRule);
+			guard = findGuardForElementWithTrivialCardinality(element, currentRule, skipCache);
 		} else if (isOptionalCardinality(element)) {
-			guard = findGuardForOptionalCardinalityWithoutContext(element, currentRule);
+			guard = findGuardForOptionalCardinalityWithoutContext(element, currentRule, skipCache);
 		} else {
 			throw new IllegalArgumentException("unknown cardinality: " + element.getCardinality());
 		}
 		
+		if (!skipCache) {
 			elementCache.put(path, guard);
+		}
 		return guard;
 	}
 	
-	private HoistingGuard findGuardForElementWithTrivialCardinality(AbstractElement element, AbstractRule currentRule) {
+	private HoistingGuard findGuardForElementWithTrivialCardinality(AbstractElement element, AbstractRule currentRule, boolean skipCache) {
 		if (config.isDebug())
 			log.info(currentRule.getName() + ": " + abstractElementToShortString(element));
 		
 		if (element instanceof Alternatives) {
-			return findGuardForAlternatives((Alternatives) element, currentRule);
+			return findGuardForAlternatives((Alternatives) element, currentRule, skipCache);
 		} else if (element instanceof Group) {
-			return findGuardForGroup((Group) element, currentRule);
+			return findGuardForGroup((Group) element, currentRule, skipCache);
 		} else if (element instanceof AbstractSemanticPredicate) {
 			return new PredicateGuard((AbstractSemanticPredicate) element);
 		} else if (Token.isToken(element)) {
@@ -491,9 +495,9 @@ public class HoistingProcessor {
 		} else if (element instanceof JavaAction) {
 			return HoistingGuard.action();
 		} else if (element instanceof UnorderedGroup) {
-			return findGuardForUnorderedGroup((UnorderedGroup) element, currentRule);
+			return findGuardForUnorderedGroup((UnorderedGroup) element, currentRule, skipCache);
 		} else if (element instanceof Assignment) {
-			return findGuardForElement(((Assignment) element).getTerminal(), currentRule);
+			return findGuardForElement(((Assignment) element).getTerminal(), currentRule, skipCache);
 		} else {
 			if (!pathHasHoistablePredicate(currentRule.getAlternatives())) {
 				// unsupported construct but rule doesn't contain hoistable predicates

org.eclipse.xtext/src/org/eclipse/xtext/GrammarUtil.java

@@ -769,4 +769,38 @@ public class GrammarUtil {
 		// and changes to the copied path will be considered for .equals()
 		return grammar.getRules().get(0).getName().equals(rule.getName());
 	}
+	
+	public static String getPathOfElement(AbstractRule element) {
+		return element.getName();
+	}
+	
+	public static String getPathOfElement(AbstractElement element) {
+		String path = element.eClass().getName();
+		EObject container = element;
+		EObject _element = element;
+		while (
+			container != null && (
+				container == element || (
+					!(container instanceof AbstractElement) &&
+					!(container instanceof AbstractRule)
+				)
+			)
+		) {
+			_element = container;
+			container = container.eContainer();
+		}
+		
+		if (container == null) {
+			System.out.println("no container: " + element.eClass().getName());
+			return path;
+		} else if (container instanceof CompoundElement) {
+			CompoundElement compoundElement = (CompoundElement) container;
+			int i = compoundElement.getElements().indexOf(_element);
+			return getPathOfElement(compoundElement) + "." + i + "." + path;
+		} else if (container instanceof AbstractRule) {
+			return getPathOfElement((AbstractRule) container) + "." + path;
+		} else {
+			return getPathOfElement((AbstractElement) container) + "." + path;
+		}
+	}
 }