From b7f69d8a65a84610b31cc829a9870e95cbb55953 Mon Sep 17 00:00:00 2001 From: overflowerror Date: Fri, 28 Jan 2022 21:51:13 +0100 Subject: [PATCH] cleanup in HoistingProcessor and TokenAnalysis --- .../antlr/hoisting/HoistingProcessor.java | 96 ++---------- ...cursionPathInContextAnalysisException.java | 47 ------ ...nalCardinalityWithoutContextException.java | 47 ------ .../SymbolicAnalysisFailedException.java | 17 --- .../hoisting/pathAnalysis/TokenAnalysis.java | 138 ++++++++++-------- ...mitiveWrapper.java => MutableWrapper.java} | 6 +- 6 files changed, 98 insertions(+), 253 deletions(-) delete mode 100644 org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/exceptions/EmptyRecursionPathInContextAnalysisException.java delete mode 100644 org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/exceptions/OptionalCardinalityWithoutContextException.java delete mode 100644 org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/exceptions/SymbolicAnalysisFailedException.java rename org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/utils/{MutablePrimitiveWrapper.java => MutableWrapper.java} (87%) diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/HoistingProcessor.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/HoistingProcessor.java index 336a6a222..8ac460ff8 100644 --- a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/HoistingProcessor.java +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/HoistingProcessor.java @@ -223,8 +223,6 @@ public class HoistingProcessor { return flattened; } - boolean hasSeen = false; - private HoistingGuard findGuardForAlternatives(CompoundElement alternatives, AbstractRule currentRule, boolean skipCache) { if (config.isDebug()) log.info("find guard for alternative"); @@ -315,11 +313,6 @@ public class HoistingProcessor { throw new NestedPrefixAlternativesException("nested prefix alternatives can't be analysed because of too many paths"); } - /*if (hasSeen) { - throw new RuntimeException(); - } - hasSeen=true;*/ - return findGuardForAlternatives(flattened, currentRule, true); } catch(TokenAnalysisAbortedException e) { throw new TokenAnalysisAbortedException(e.getMessage(), e, currentRule); @@ -335,8 +328,6 @@ public class HoistingProcessor { // if A and B are optional the guard for the alternatives need to check the context // if not the alternatives are actual alternatives - // TODO: check if hasTerminal is valid - return findGuardForAlternatives(element, currentRule, skipCache); } @@ -347,10 +338,8 @@ public class HoistingProcessor { HoistingGuard guard = findGuardForElement(element, currentRule, skipCache); groupGuard.add(guard); - // if cardinality is + and there is a terminal we don't need to consider - // the following elements - // if cardinality is + and there is no token the guard won't change even if the - // element could be repeated + // no need to consider the following elements if the current one has a terminal + // following elements can't be hoisted anyway if (guard.hasTerminal()) { groupGuard.setHasTerminal(); break; @@ -360,72 +349,17 @@ public class HoistingProcessor { return groupGuard; } - // TODO: make private public HoistingGuard findGuardForRule(AbstractRule rule) { + // public so the AntlrGrammarGenerator can access it if debug mode is enabled + if (config.isDebug()) log.info("finding guard for rule: " + rule.getName()); return findGuardForElement(rule.getAlternatives(), rule, false); } - private boolean pathHasTokenOrAction(AbstractElement path) { - // we are only interested in whether or not there could be any tokens on this path - // -> ignore cardinality - - if (Token.isToken(path)) { - return true; - } else if (isEnumRuleCall(path)) { - return true; - } else if (path instanceof JavaAction) { - return true; - } else if (isParserRuleCall(path)) { - // can not be self recursive since ANTLR uses LL(*) and therefore does not support left recursion - return pathHasTokenOrAction(((RuleCall) path).getRule().getAlternatives()); - } else if (path instanceof Assignment) { - return pathHasTokenOrAction(((Assignment) path).getTerminal()); - } else if (path instanceof CompoundElement) { - return ((CompoundElement) path).getElements().stream() - .anyMatch(this::pathHasTokenOrAction); - } else { - // Actions, Predicates, ... - return false; - } - } - - private boolean pathHasHoistablePredicate(AbstractElement path) { - // we are only interested in whether or not there could be any hoistable predicate on this path - // -> ignore cardinality - - if (path instanceof AbstractSemanticPredicate) { - return true; - } else if (isParserRuleCall(path)) { - // can not be self recursive since ANTLR uses LL(*) and therefore does not support left recursion - return pathHasHoistablePredicate(((RuleCall) path).getRule().getAlternatives()); - } else if (path instanceof Assignment) { - return pathHasHoistablePredicate(((Assignment) path).getTerminal()); - } else if (path instanceof Alternatives || - path instanceof UnorderedGroup) { - return ((CompoundElement) path).getElements().stream() - .anyMatch(this::pathHasHoistablePredicate); - } else if (path instanceof Group) { - for (AbstractElement element : ((CompoundElement) path).getElements()) { - if (pathHasHoistablePredicate(element)) { - return true; - } - if (pathHasTokenOrAction(element)) { - return false; - } - } - return false; - } else { - // Tokens, EnumRule, Actions, JavaActions, ... - return false; - } - } - public HoistingGuard findHoistingGuardIgnoreCardinality(AbstractElement element) { if (config.isDebug()) log.info("hoisting (trivial) guard of: \n" + abstractElementToString(element)); - // should only be called for valid AST elements, so element can never be floating AbstractRule rule = containingParserRule(element); if (element instanceof UnorderedGroup) { @@ -439,12 +373,10 @@ public class HoistingProcessor { if (config.isDebug()) log.info("hoisting guard of: \n" + abstractElementToString(element)); - // should only be called for valid AST elements, so element can never be floating return findGuardForElement(element, containingParserRule(element), false); } private HoistingGuard findGuardForElement(AbstractElement element, AbstractRule currentRule, boolean skipCache) { - String path = null; HoistingGuard guard; @@ -458,7 +390,8 @@ public class HoistingProcessor { } } - if (isTrivialCardinality(element) || + if ( + isTrivialCardinality(element) || isOneOrMoreCardinality(element) ) { guard = findGuardForElementWithTrivialCardinality(element, currentRule, skipCache); @@ -483,14 +416,18 @@ public class HoistingProcessor { } else if (element instanceof Group) { return findGuardForGroup((Group) element, currentRule, skipCache); } else if (element instanceof AbstractSemanticPredicate) { - return new PredicateGuard((AbstractSemanticPredicate) element); + // TODO: change to GatedSemanticPredicate + return new PredicateGuard((AbstractSemanticPredicate) element); } else if (Token.isToken(element)) { return HoistingGuard.terminal(); - } else if (isParserRuleCall(element) || - isEnumRuleCall(element)) { + } else if ( + isParserRuleCall(element) || + isEnumRuleCall(element) + ) { RuleCall call = (RuleCall) element; return findGuardForRule(call.getRule()); } else if (element instanceof Action) { + // this is an Xtext action, not a JavaAction return HoistingGuard.unguarded(); } else if (element instanceof JavaAction) { return HoistingGuard.action(); @@ -499,12 +436,7 @@ public class HoistingProcessor { } else if (element instanceof Assignment) { return findGuardForElement(((Assignment) element).getTerminal(), currentRule, skipCache); } else { - if (!pathHasHoistablePredicate(currentRule.getAlternatives())) { - // unsupported construct but rule doesn't contain hoistable predicates - return HoistingGuard.unguarded(); - } else { - throw new UnsupportedConstructException("element not supported: " + element.toString(), currentRule); - } + throw new UnsupportedConstructException("element not supported: " + element.toString(), currentRule); } } } diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/exceptions/EmptyRecursionPathInContextAnalysisException.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/exceptions/EmptyRecursionPathInContextAnalysisException.java deleted file mode 100644 index 8dc175c03..000000000 --- a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/exceptions/EmptyRecursionPathInContextAnalysisException.java +++ /dev/null @@ -1,47 +0,0 @@ -/******************************************************************************* - * Copyright (c) 2022 itemis AG (http://www.itemis.eu) and others. - * This program and the accompanying materials are made available under the - * terms of the Eclipse Public License 2.0 which is available at - * http://www.eclipse.org/legal/epl-2.0. - * - * SPDX-License-Identifier: EPL-2.0 - *******************************************************************************/ -package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.exceptions; - -import org.eclipse.xtext.AbstractRule; - -/** - * @author overflow - Initial contribution and API - */ -public class EmptyRecursionPathInContextAnalysisException extends HoistingException { - private static final long serialVersionUID = 3820353828411603508L; - - public EmptyRecursionPathInContextAnalysisException() { - super(); - } - - public EmptyRecursionPathInContextAnalysisException(String message, AbstractRule rule) { - super(message, rule); - } - - public EmptyRecursionPathInContextAnalysisException(String message, Throwable cause, AbstractRule rule) { - super(message, cause, rule); - } - - public EmptyRecursionPathInContextAnalysisException(String message, Throwable cause) { - super(message, cause); - } - - public EmptyRecursionPathInContextAnalysisException(String message) { - super(message); - } - - public EmptyRecursionPathInContextAnalysisException(Throwable cause, AbstractRule rule) { - super(cause, rule); - } - - public EmptyRecursionPathInContextAnalysisException(Throwable cause) { - super(cause); - } - -} diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/exceptions/OptionalCardinalityWithoutContextException.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/exceptions/OptionalCardinalityWithoutContextException.java deleted file mode 100644 index b115b6eff..000000000 --- a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/exceptions/OptionalCardinalityWithoutContextException.java +++ /dev/null @@ -1,47 +0,0 @@ -/******************************************************************************* - * Copyright (c) 2021 itemis AG (http://www.itemis.eu) and others. - * This program and the accompanying materials are made available under the - * terms of the Eclipse Public License 2.0 which is available at - * http://www.eclipse.org/legal/epl-2.0. - * - * SPDX-License-Identifier: EPL-2.0 - *******************************************************************************/ -package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.exceptions; - -import org.eclipse.xtext.AbstractRule; - -/** - * @author overflow - Initial contribution and API - */ -public class OptionalCardinalityWithoutContextException extends UnsupportedConstructException { - private static final long serialVersionUID = -3958747238452206971L; - - public OptionalCardinalityWithoutContextException() { - super(); - } - - public OptionalCardinalityWithoutContextException(String message, AbstractRule rule) { - super(message, rule); - } - - public OptionalCardinalityWithoutContextException(String message, Throwable cause, AbstractRule rule) { - super(message, cause, rule); - } - - public OptionalCardinalityWithoutContextException(String message, Throwable cause) { - super(message, cause); - } - - public OptionalCardinalityWithoutContextException(String message) { - super(message); - } - - public OptionalCardinalityWithoutContextException(Throwable cause, AbstractRule rule) { - super(cause, rule); - } - - public OptionalCardinalityWithoutContextException(Throwable cause) { - super(cause); - } - -} diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/exceptions/SymbolicAnalysisFailedException.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/exceptions/SymbolicAnalysisFailedException.java deleted file mode 100644 index 6e7ea4a26..000000000 --- a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/exceptions/SymbolicAnalysisFailedException.java +++ /dev/null @@ -1,17 +0,0 @@ -/******************************************************************************* - * Copyright (c) 2021 itemis AG (http://www.itemis.eu) and others. - * This program and the accompanying materials are made available under the - * terms of the Eclipse Public License 2.0 which is available at - * http://www.eclipse.org/legal/epl-2.0. - * - * SPDX-License-Identifier: EPL-2.0 - *******************************************************************************/ -package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.exceptions; - -/** - * @author overflow - Initial contribution and API - */ -public class SymbolicAnalysisFailedException extends Exception { - private static final long serialVersionUID = 4185473673062321988L; - -} diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/pathAnalysis/TokenAnalysis.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/pathAnalysis/TokenAnalysis.java index b899e29cd..5143422fa 100644 --- a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/pathAnalysis/TokenAnalysis.java +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/pathAnalysis/TokenAnalysis.java @@ -8,6 +8,7 @@ *******************************************************************************/ package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.pathAnalysis; +import static org.eclipse.xtext.GrammarUtil.*; import static org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.utils.DebugUtils.*; import java.util.ArrayList; @@ -38,14 +39,11 @@ import org.eclipse.xtext.UnorderedGroup; import org.eclipse.xtext.util.XtextSwitch; import org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.HoistingConfiguration; import org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.exceptions.NestedPrefixAlternativesException; -import org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.exceptions.SymbolicAnalysisFailedException; import org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.exceptions.TokenAnalysisAbortedException; import org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.exceptions.UnsupportedConstructException; import org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.token.Token; import org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.utils.DebugUtils; -import org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.utils.MutablePrimitiveWrapper; - -import static org.eclipse.xtext.GrammarUtil.*; +import org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.utils.MutableWrapper; /** * @author overflow - Initial contribution and API @@ -71,15 +69,17 @@ public class TokenAnalysis { AbstractElement _last = last; AbstractElement container = last; - while(container != null && ( + while( + container != null && ( container == last || !(container instanceof CompoundElement) || container instanceof Alternatives ) ) { EObject _container = _last; - while (_container == _last || - !(_container instanceof AbstractElement) + while ( + _container == _last || + !(_container instanceof AbstractElement) ) { _container = _container.eContainer(); if (_container == null) @@ -89,7 +89,7 @@ public class TokenAnalysis { container = (AbstractElement) _container; if (considerCardinalities && last != _last && isMultipleCardinality(_last)) { - // last is + or * quantified + // last is + or * quantified, so it may be repeated -> add to result result.add(_last); } } @@ -100,23 +100,22 @@ public class TokenAnalysis { if (compoundContainer == null) { if (config.isDebug()) log.info("no container"); + // no container element; this is last element in a rule definition AbstractRule rule = containingRule(last); - List calls = findAllRuleCalls(grammar, rule).stream() - .filter(Predicate.not(visited::contains)) - .collect(Collectors.toList()); if (isStartRule(grammar, rule)) { // context is EOF result.add(null); } - for (RuleCall call : calls) { - Set _visited = new HashSet<>(visited); - _visited.add(call); - result.addAll(getNextElementsInContext(call, considerCardinalities, _visited)); - } - + findAllRuleCalls(grammar, rule).stream() + .filter(Predicate.not(visited::contains)) + .forEach(call -> { + Set _visited = new HashSet<>(visited); + _visited.add(call); + result.addAll(getNextElementsInContext(call, considerCardinalities, _visited)); + }); } else if (compoundContainer instanceof Group) { if (config.isDebug()) log.info("group container"); @@ -134,11 +133,13 @@ public class TokenAnalysis { // skip simple non-token-elements while (index < size - 1) { next = elements.get(index + 1); - if (!( + if ( + !( (next instanceof Action) || (next instanceof JavaAction) || (next instanceof AbstractSemanticPredicate) - )) { + ) + ) { break; } @@ -175,9 +176,10 @@ public class TokenAnalysis { return getTokenPathsContext(last, prefix, true, new HashSet<>()); } - private TokenAnalysisPaths getTokenPathsContext(AbstractElement last, TokenAnalysisPaths prefix, boolean considerCardinalities, Set callStack) { + private TokenAnalysisPaths getTokenPathsContext(AbstractElement last, TokenAnalysisPaths prefix, boolean considerCardinalities, Set visitedElements) { if (config.isDebug()) - log.info("get context for: " + abstractElementToShortString(last) + (considerCardinalities ? " with" : " without") + " cardinalities"); + log.info("get context for: " + abstractElementToShortString(last) + + (considerCardinalities ? " with" : " without") + " cardinalities"); List context = getNextElementsInContext(last, considerCardinalities); @@ -192,11 +194,13 @@ public class TokenAnalysis { log.info("context element: " + abstractElementToShortString(element)); TokenAnalysisPaths path = new TokenAnalysisPaths(prefix); path.resetProgress(); + // shortcut endless loops instead of throwing exception path = getTokenPaths(element, path, false, false, true); + if (!path.isDone() && element != null) { boolean _considerCardinalities = considerCardinalities; - if (callStack.contains(element) && !path.hasProgress()) { + if (visitedElements.contains(element) && !path.hasProgress()) { if (_considerCardinalities) { _considerCardinalities = false; } else { @@ -206,9 +210,9 @@ public class TokenAnalysis { continue; } } - Set localCallStack = new HashSet<>(callStack); - localCallStack.add(element); - path = getTokenPathsContext(element, path, _considerCardinalities, localCallStack); + Set localVisitedElements = new HashSet<>(visitedElements); + localVisitedElements.add(element); + path = getTokenPathsContext(element, path, _considerCardinalities, localVisitedElements); } if (path.isDone()) { result = result.merge(path); @@ -221,12 +225,16 @@ public class TokenAnalysis { } if (actualNumberOfElements == 0) { - // TODO: is this special case n)ecessary? + // necessary in case all possible path are ignored + // (considerCardinalities is false, all elements were visited and there was no progress) + + if (config.isDebug()) + log.info("context analysis failed: no context"); throw new TokenAnalysisAbortedException("context analysis failed: no context"); } if (config.isDebug()) - log.info("done"); + log.info("context analysis done"); return result; } @@ -314,12 +322,13 @@ public class TokenAnalysis { }; @Override public TokenAnalysisPaths caseRuleCall(RuleCall call) { - if (isParserRuleCall(call) || - isEnumRuleCall(call) + if ( + isParserRuleCall(call) || + isEnumRuleCall(call) ) { return getTokenPaths(call.getRule().getAlternatives(), prefix, false, false, shortcutEndlessLoops); } else { - // go to default case + // probably terminal rule call -> go to default case return null; } }; @@ -333,7 +342,7 @@ public class TokenAnalysis { return result; } else { // Actions, Predicates, JavaActions, ... - // create new state with out empty flag + // create new state without empty flag return new TokenAnalysisPaths(prefix); } }; @@ -363,6 +372,7 @@ public class TokenAnalysis { return result; } + // use actual cardinality from path return getTokenPaths(path, path.getCardinality(), prefix, analyseContext, needsLength, shortcutEndlessLoops); } @@ -441,16 +451,24 @@ public class TokenAnalysis { return result; } - private List> getTokenPaths(AbstractElement path, List indexes, boolean analyseContext) throws TokenAnalysisAbortedException { - return getTokenPaths(path, new TokenAnalysisPaths(indexes), analyseContext, true, false).getTokenPaths(); - } - - private List> getTokenPaths(AbstractElement path, String virtualCardinality, List indexes, boolean analyseContext) throws TokenAnalysisAbortedException { - return getTokenPaths(path, virtualCardinality, new TokenAnalysisPaths(indexes), analyseContext, true, false).getTokenPaths(); + private TokenAnalysisPaths getTokenPaths(AbstractElement path, List indexes, boolean analyseContext, boolean needsLength, boolean shortcutEndlessLoops) { + return getTokenPaths(path, new TokenAnalysisPaths(indexes), analyseContext, needsLength, shortcutEndlessLoops); } - private List> getTokenPathsContextOnly(AbstractElement path, List indexes) { - return getTokenPathsContext(path, new TokenAnalysisPaths(indexes)).getTokenPaths(); + private TokenAnalysisPaths getTokenPathsNoLength(AbstractElement path, List indexes) { + return getTokenPaths(path, new TokenAnalysisPaths(indexes), false, false, false); + } + + private TokenAnalysisPaths getTokenPathsContext(AbstractElement path, List indexes) { + return getTokenPaths(path, new TokenAnalysisPaths(indexes), true, true, false); + } + + private TokenAnalysisPaths getTokenPaths(AbstractElement path, String virtualCardinality, List indexes, boolean analyseContext, boolean needsLength, boolean shortcutEndlessLoops) { + return getTokenPaths(path, virtualCardinality, new TokenAnalysisPaths(indexes), analyseContext, needsLength, shortcutEndlessLoops); + } + + private TokenAnalysisPaths getTokenPathsContextOnly(AbstractElement path, List indexes) throws TokenAnalysisAbortedException { + return getTokenPathsContext(path, new TokenAnalysisPaths(indexes)); } private List range(int i, int j) { @@ -463,50 +481,47 @@ public class TokenAnalysis { log.info("path2: " + abstractElementToString(path2)); } - int i; // + 1, because otherwise identical paths of length token limit can't be checked int limit = config.getTokenLimit() + 1; - for (i = 0; i < limit; i++) { + for (int i = 0; i < limit; i++) { TokenAnalysisPaths tokenPaths1; TokenAnalysisPaths tokenPaths2; List range = range(0, i); // there shouldn't be a TokenAnalysisAbortedException if needsLength is false - tokenPaths1 = getTokenPaths(path1, new TokenAnalysisPaths(range), false, false, false); - tokenPaths2 = getTokenPaths(path2, new TokenAnalysisPaths(range), false, false, false); + tokenPaths1 = getTokenPathsNoLength(path1, range); + tokenPaths2 = getTokenPathsNoLength(path2, range); Set> tokenListSet1 = tokenPaths1.getTokenPaths().stream().map(HashSet::new).collect(Collectors.toSet()); Set> tokenListSet2 = tokenPaths2.getTokenPaths().stream().map(HashSet::new).collect(Collectors.toSet()); - int maxPosition1 = tokenPaths1.getMaxPosition(); - int maxPosition2 = tokenPaths2.getMaxPosition(); - if (!tokenListSet1.equals(tokenListSet2)) { return false; } - if (maxPosition1 < i + 1) { + if (tokenPaths1.getMaxPosition() < i + 1) { // different max positions would have failed the equals-Operation // if the max position is smaller than i + 1 the end of the path has been reached + // -> the paths are identical return true; } } // token analysis limit reached // we can't analyze the paths any further - // TODO maybe assume paths are equal and show warning instead of exception throw new TokenAnalysisAbortedException("token limit exhausted while looking for identical paths"); } private void tokenCombinations(Function, Boolean> callback) { - MutablePrimitiveWrapper limit = new MutablePrimitiveWrapper<>(config.getTokenLimit()); + MutableWrapper limit = new MutableWrapper<>(config.getTokenLimit()); for (int i = 1; i <= limit.get(); i++) { if (tokenCombinations(0, 0, i, callback, limit)) { return; } } + // we tried all possible combinations // -> abort if (limit.get().equals(config.getTokenLimit())) { @@ -517,10 +532,11 @@ public class TokenAnalysis { throw new NestedPrefixAlternativesException(); } } - private boolean tokenCombinations(long prefix, int prefixLength, int ones, Function, Boolean> callback, MutablePrimitiveWrapper limit) { + private boolean tokenCombinations(long prefix, int prefixLength, int ones, Function, Boolean> callback, MutableWrapper limit) { if (ones == 0) { List indexes = new ArrayList<>(limit.get()); - for (int i = 0; i < limit.get(); i++) { + int l = limit.get(); + for (int i = 0; i < l; i++) { if ((prefix & (1 << i)) > 0) { indexes.add(i); } @@ -530,6 +546,7 @@ public class TokenAnalysis { // prefix is too long return false; } else { + // we can not cache the value of limit here since it might be modified during the recursion for (int i = prefixLength; i < limit.get() - ones + 1; i++) { long current = prefix | (1 << i); try { @@ -554,16 +571,22 @@ public class TokenAnalysis { // this method is for finding the path differences between the // element (with optional cardinality) and the context + // virtualCardinality is what should be used instead of the actual cardinality of the root element + // this way ? and * can both be handled by this method + // first dimension of result corresponds to the alternatives // the second dimension are tokens of the alternatives + // no need return the context tokens - MutablePrimitiveWrapper>> result = new MutablePrimitiveWrapper>>(null); + MutableWrapper>> result = new MutableWrapper>>(null); tokenCombinations(indexList -> { // no context analysis // TODO why? - List> tokenListsForPath = getTokenPaths(element, virtualCardinality, indexList, false); - List> tokenListForContext = getTokenPathsContextOnly(element, indexList); + List> tokenListsForPath = getTokenPaths(element, virtualCardinality, indexList, false, true, false) + .getTokenPaths(); + List> tokenListForContext = getTokenPathsContextOnly(element, indexList) + .getTokenPaths(); if (!tokenListsForPath.stream() .anyMatch(tokenListForContext::contains) @@ -602,7 +625,7 @@ public class TokenAnalysis { if (config.isDebug()) log.info("next path: " + p); }) - .map(p -> getTokenPaths(p, indexList, true)) + .map(p -> getTokenPathsContext(p, indexList).getTokenPaths()) .collect(Collectors.toList()); int size = result.size(); @@ -615,7 +638,7 @@ public class TokenAnalysis { if (!tokenListOfCurrentPath.stream() .anyMatch(tokenList -> tokenListsForPaths.stream() .filter(s -> s != tokenListOfCurrentPath) - // does any other path contain a similar token list (= alternative) + // does any other path contain a similar token list (= alternative) ? .anyMatch(s -> s.contains(tokenList)) ) ) { @@ -625,6 +648,7 @@ public class TokenAnalysis { } + // abort when all paths have an identifying token sequence set return result.stream() .allMatch(Objects::nonNull); }); @@ -634,7 +658,7 @@ public class TokenAnalysis { public List> getAllPossiblePaths(AbstractElement path) { // token limit + 2 so identity analysis will recognize paths that are identical up to the token limit on the flattened tree - return getTokenPaths(path, new TokenAnalysisPaths(range(0, config.getTokenLimit() + 2)), false, false, true) + return getTokenPaths(path, range(0, config.getTokenLimit() + 2), false, false, true) .getTokenPaths() .stream() .map(l -> l.stream() diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/utils/MutablePrimitiveWrapper.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/utils/MutableWrapper.java similarity index 87% rename from org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/utils/MutablePrimitiveWrapper.java rename to org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/utils/MutableWrapper.java index 10731e838..16a015be2 100644 --- a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/utils/MutablePrimitiveWrapper.java +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/utils/MutableWrapper.java @@ -13,13 +13,13 @@ import java.util.function.Function; /** * @author overflow - Initial contribution and API */ -public class MutablePrimitiveWrapper { +public class MutableWrapper { private T value; - public MutablePrimitiveWrapper() { + public MutableWrapper() { } - public MutablePrimitiveWrapper(T value) { + public MutableWrapper(T value) { this.value = value; }