diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/AlternativeTokenSequenceGuard.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/AlternativeTokenSequenceGuard.java new file mode 100644 index 000000000..581e34aac --- /dev/null +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/AlternativeTokenSequenceGuard.java @@ -0,0 +1,44 @@ +/******************************************************************************* + * Copyright (c) 2021 itemis AG (http://www.itemis.eu) and others. + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *******************************************************************************/ +package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting; + +import java.util.Collection; +import java.util.stream.Collectors; + +/** + * @author overflow - Initial contribution and API + */ +public class AlternativeTokenSequenceGuard implements TokenGuard { + private Collection alternatives; + + public AlternativeTokenSequenceGuard(Collection alternatives) { + this.alternatives = alternatives; + } + + @Override + public String render() { + boolean addParentheses = alternatives.size() != 1; + String result = ""; + + if (addParentheses) { + result += "("; + } + + result += alternatives.stream() + .map(TokenGuard::render) + .collect(Collectors.joining(" && ")); + + if (addParentheses) { + result += ")"; + } + + return result; + } + +} diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/Hoisting.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/Hoisting.java index 864162b86..b5918c0b9 100644 --- a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/Hoisting.java +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/Hoisting.java @@ -8,11 +8,15 @@ *******************************************************************************/ package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting; +import java.util.ArrayList; import java.util.HashMap; import java.util.Map; +import java.util.Objects; import java.util.List; import java.util.Set; +import java.util.function.Function; import java.util.stream.Collectors; +import java.util.stream.IntStream; import org.eclipse.xtext.AbstractElement; import org.eclipse.xtext.AbstractRule; @@ -36,7 +40,9 @@ import org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.utils.StreamUtils public class Hoisting { private Map ruleCache = new HashMap<>(); private Map groupCache = new HashMap<>(); - + + private static final int TOKEN_ANALYSIS_LIMIT = 10; + private boolean isParserRule(AbstractElement element) { return (element instanceof RuleCall) && (((RuleCall) element).getRule() instanceof ParserRule); } @@ -185,11 +191,51 @@ public class Hoisting { } private boolean arePathsIdenticalSymbolic(AbstractElement path1, AbstractElement path2) throws SymbolicAnalysisFailedException { - return false; + // ignore symbolic analysis for the moment + // TODO + throw new SymbolicAnalysisFailedException(); + } + + private List range(int i, int j) { + return IntStream.rangeClosed(i, j).boxed().collect(Collectors.toList()); } private boolean arePathsIdenticalFallback(AbstractElement path1, AbstractElement path2) { - return false; + for (int i = 1; i < TOKEN_ANALYSIS_LIMIT; i++) { + Set> tokenListSet1; + Set> tokenListSet2; + + List range = range(1, i); + + try { + tokenListSet1 = getTokenForIndexes(path1, range); + } catch (TokenAnalysisAbortedException e) { + tokenListSet1 = null; + } + + try { + tokenListSet2 = getTokenForIndexes(path2, range); + } catch (TokenAnalysisAbortedException e) { + tokenListSet2 = null; + } + + if (tokenListSet1 == null && tokenListSet2 == null) { + return true; + } + if (tokenListSet1 == null || tokenListSet2 == null) { + // the paths have different length + return false; + } + if (!tokenListSet1.equals(tokenListSet2)) { + // TODO: hashCode method of Token classes + return false; + } + } + + // we can't analyze the paths any further + // we can assume that the paths are identical because the path diff analysis would cause an error anyway. + // TODO maybe warning? + return true; } private boolean arePathsIdentical(AbstractElement path1, AbstractElement path2) { @@ -200,10 +246,75 @@ public class Hoisting { } } - private List> findMinimalPathDifference(List paths) { - return null; + + private void tokenCombinations(Function, Boolean> callback) { + for (int i = 1; i <= TOKEN_ANALYSIS_LIMIT; i++) { + if (tokenCombinations(0, 0, i, callback)) { + return; + } + } + } + private boolean tokenCombinations(long prefix, int prefixLength, int ones, Function, Boolean> callback) { + if (ones == 0) { + List indexes = new ArrayList<>(TOKEN_ANALYSIS_LIMIT); + for (int i = 0; i < TOKEN_ANALYSIS_LIMIT; i++) { + if ((prefix & (1 << i)) > 0) { + indexes.add(i + 1); + } + } + return callback.apply(indexes); + } else if (prefixLength + ones > TOKEN_ANALYSIS_LIMIT) { + // prefix is too long + return false; + } else { + for (int i = prefixLength; i < TOKEN_ANALYSIS_LIMIT - ones + 1; i++) { + long current = prefix | (1 << i); + if (tokenCombinations(current, i + 1, ones - 1, callback)) { + return true; + } + } + return false; + } } + private List>> findMinimalPathDifference(List paths) throws TokenAnalysisAbortedException { + List>> result = paths.stream() + .map(p -> (Set>) null) + .collect(Collectors.toList()); + + tokenCombinations(indexList -> { + // will throw TokenAnalysisAborted if any path is too short + List>> tokenListSets = paths.stream() + .map(p -> getTokenForIndexes(p, indexList)) + .collect(Collectors.toList()); + + int size = result.size(); + for (int i = 0; i < size; i++) { + if (result.get(i) == null) { + continue; + } + + Set> tokenSet = tokenListSets.get(i); + if (!tokenSet.stream() + .anyMatch(tokenList -> tokenListSets.stream() + .filter(s -> s != tokenSet) + .anyMatch(s -> s.contains(tokenList)) + ) + ) { + // token list set is unique for path i + result.set(i, tokenSet); + } + + } + + return result.stream() + .allMatch(Objects::nonNull); + }); + + return result; + } + + // TODO: handling for TokenAnalysisAbortedException private HoistingGuard findGuardForAlternatives(Alternatives alternatives) { List paths = alternatives.getElements(); List guards = paths.stream() @@ -225,15 +336,22 @@ public class Hoisting { } } + // if all paths are empty the above step will eliminate all paths + // -> size = 1 if (size > 1) { return StreamUtils.zip( findMinimalPathDifference(paths).stream() - .map(s -> s.stream() - .map(TokenGuard::new) + .map(a -> a.stream() + .map(s -> s.stream() + .map(SingleTokenGuard::new) + .collect(Collectors.toList()) + ) + .map(TokenSequenceGuard::new) .collect(Collectors.toSet()) - ), + ) + .map(AlternativeTokenSequenceGuard::new), guards.stream(), - (Set tokenSet, MergedPathGuard guard) -> Tuples.pair(tokenSet, guard) + (TokenGuard tokenGuard, MergedPathGuard pathGuard) -> Tuples.pair(tokenGuard, pathGuard) ).map(p -> new PathGuard(p.getFirst(), p.getSecond())) .collect(AlternativesGuard.collector()); } else { diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/KeywordToken.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/KeywordToken.java index ca772a17c..2b5d81262 100644 --- a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/KeywordToken.java +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/KeywordToken.java @@ -13,7 +13,7 @@ import org.eclipse.xtext.Keyword; /** * @author overflow - Initial contribution and API */ -public class KeywordToken extends Token { +public class KeywordToken implements Token { private Keyword keyword; private int position; @@ -21,4 +21,9 @@ public class KeywordToken extends Token { this.keyword = keyword; this.position = position; } + + @Override + public String negatedCondition() { + return "!\"" + keyword.getValue().replace("\"", "\\\"") + "\".equals(input.LT(" + position + ").getText()"; + } } diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/PathGuard.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/PathGuard.java index 3de698ac8..014892c06 100644 --- a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/PathGuard.java +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/PathGuard.java @@ -8,19 +8,15 @@ *******************************************************************************/ package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting; -import java.util.Collection; -import java.util.LinkedList; -import java.util.List; - /** * @author overflow - Initial contribution and API */ public class PathGuard implements HoistingGuard { - private Collection tokenGuards; + private TokenGuard tokenGuard; private HoistingGuard hoistngGuard; - public PathGuard(Collection tokenGuards, HoistingGuard hoistingGuard) { - this.tokenGuards = tokenGuards; + public PathGuard(TokenGuard tokenGuard, HoistingGuard hoistingGuard) { + this.tokenGuard = tokenGuard; this.hoistngGuard = hoistingGuard; } @@ -32,13 +28,13 @@ public class PathGuard implements HoistingGuard { @Override public boolean hasTerminal() { // empty paths are only allowed when all paths are empty - // in that case a MergedPathGuard is returned. + // in that case a MergedPathGuard is returned by findGuardForAlternatives. return true; } @Override public String render() { - // TODO - return null; + // parentheses needed since tokenGuard is never empty + return "(" + tokenGuard.render() + " || " + hoistngGuard.render() + ")"; } } diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/SingleTokenGuard.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/SingleTokenGuard.java new file mode 100644 index 000000000..39f571787 --- /dev/null +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/SingleTokenGuard.java @@ -0,0 +1,25 @@ +/******************************************************************************* + * Copyright (c) 2021 itemis AG (http://www.itemis.eu) and others. + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *******************************************************************************/ +package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting; + +/** + * @author overflow - Initial contribution and API + */ +public class SingleTokenGuard implements TokenGuard { + private Token token; + + public SingleTokenGuard(Token token) { + this.token = token; + } + + @Override + public String render() { + return token.negatedCondition(); + } +} diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/TerminalRuleToken.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/TerminalRuleToken.java index d5f01d058..98e1cea08 100644 --- a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/TerminalRuleToken.java +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/TerminalRuleToken.java @@ -13,7 +13,7 @@ import org.eclipse.xtext.TerminalRule; /** * @author overflow - Initial contribution and API */ -public class TerminalRuleToken extends Token { +public class TerminalRuleToken implements Token { private TerminalRule rule; private int position; @@ -21,4 +21,9 @@ public class TerminalRuleToken extends Token { this.rule = rule; this.position = position; } + + @Override + public String negatedCondition() { + return "input.LT(" + position + ").getType() != " + rule.getName(); + } } diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/Token.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/Token.java index 7df646925..11c24a653 100644 --- a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/Token.java +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/Token.java @@ -17,17 +17,14 @@ import org.eclipse.xtext.TerminalRule; /** * @author overflow - Initial contribution and API */ -public class Token { +public interface Token { + String negatedCondition(); static boolean isToken(AbstractElement element) { if (element instanceof Keyword) { return true; } else if (element instanceof RuleCall) { - if (((RuleCall) element).getRule() instanceof TerminalRule) { - return true; - } else { - return false; - } + return (((RuleCall) element).getRule() instanceof TerminalRule); } else { return false; } diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/TokenAnalysisAbortedException.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/TokenAnalysisAbortedException.java index 68b5163b4..12bbb0654 100644 --- a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/TokenAnalysisAbortedException.java +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/TokenAnalysisAbortedException.java @@ -11,6 +11,6 @@ package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting; /** * @author overflow - Initial contribution and API */ -public class TokenAnalysisAbortedException extends Exception { +public class TokenAnalysisAbortedException extends RuntimeException { private static final long serialVersionUID = 4303267001950479292L; } diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/TokenAnalysisPaths.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/TokenAnalysisPaths.java index 0ef3c9acb..45f1b9c0c 100644 --- a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/TokenAnalysisPaths.java +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/TokenAnalysisPaths.java @@ -9,7 +9,6 @@ package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting; import java.util.HashSet; -import java.util.LinkedList; import java.util.List; import java.util.Set; import java.util.stream.Collectors; diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/TokenGuard.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/TokenGuard.java index 0df2a1262..048e2aedf 100644 --- a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/TokenGuard.java +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/TokenGuard.java @@ -11,21 +11,9 @@ package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting; /** * @author overflow - Initial contribution and API */ -public class TokenGuard implements Guard { - private Token token; - - public TokenGuard(Token token) { - this.token = token; - } - +public interface TokenGuard extends Guard { @Override - public boolean isTrivial() { + default boolean isTrivial() { return false; } - - @Override - public String render() { - return "not implemented"; - } - } diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/TokenSequenceGuard.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/TokenSequenceGuard.java new file mode 100644 index 000000000..5712341e0 --- /dev/null +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/TokenSequenceGuard.java @@ -0,0 +1,43 @@ +/******************************************************************************* + * Copyright (c) 2021 itemis AG (http://www.itemis.eu) and others. + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *******************************************************************************/ +package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting; + +import java.util.Collection; +import java.util.stream.Collectors; + +/** + * @author overflow - Initial contribution and API + */ +public class TokenSequenceGuard implements TokenGuard { + private Collection sequence; + + public TokenSequenceGuard(Collection sequence) { + this.sequence = sequence; + } + + @Override + public String render() { + boolean addParentheses = sequence.size() != 1; + String result = ""; + + if (addParentheses) { + result += "("; + } + + result += sequence.stream() + .map(TokenGuard::render) + .collect(Collectors.joining(" || ")); + + if (addParentheses) { + result += ")"; + } + + return result; + } +}