diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/AlternativesGuard.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/AlternativesGuard.java new file mode 100644 index 000000000..0f54860d7 --- /dev/null +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/AlternativesGuard.java @@ -0,0 +1,58 @@ +/******************************************************************************* + * Copyright (c) 2021 itemis AG (http://www.itemis.eu) and others. + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *******************************************************************************/ +package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting; + +import java.util.LinkedList; +import java.util.List; +import java.util.stream.Collector; +import java.util.stream.Collectors; + +/** + * @author overflow - Initial contribution and API + */ +public class AlternativesGuard implements HoistingGuard { + private List paths; + + private AlternativesGuard(List paths) { + this.paths = paths; + } + + @Override + public boolean isTrivial() { + return paths.stream().anyMatch(Guard::isTrivial); + } + + @Override + public String render() { + return "(" + paths.stream() + .map(Guard::render) + .collect(Collectors.joining(" || ")) + + ")"; + } + + @Override + public boolean hasTerminal() { + // empty paths are only allowed when all paths are empty + // in that case a MergedPathGuard ist returned. + return true; + } + + public static Collector, AlternativesGuard> collector() { + return Collector.of( + LinkedList::new, + List::add, + (l, r) -> { + l.addAll(r); + return l; + }, + AlternativesGuard::new, + Collector.Characteristics.CONCURRENT + ); + } +} diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/GroupGuard.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/GroupGuard.java new file mode 100644 index 000000000..6b3603312 --- /dev/null +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/GroupGuard.java @@ -0,0 +1,54 @@ +/******************************************************************************* + * Copyright (c) 2021 itemis AG (http://www.itemis.eu) and others. + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *******************************************************************************/ +package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting; + +import java.util.LinkedList; +import java.util.List; +import java.util.stream.Collectors; + +/** + * @author overflow - Initial contribution and API + */ +public class GroupGuard implements HoistingGuard { + private List elementGuards = new LinkedList<>(); + private boolean hasTerminal = false; + + public void add(Guard guard) { + if (!guard.isTrivial()) + elementGuards.add(guard); + } + + public void setHasTerminal() { + hasTerminal = true; + } + + @Override + public boolean isTrivial() { + return elementGuards.isEmpty(); + } + + @Override + public String render() { + if (elementGuards.size() == 1) { + return elementGuards.get(0).render(); + } else { + return "(" + + elementGuards.stream() + .map(Guard::render) + .collect(Collectors.joining(" && ")) + + ")"; + } + } + + @Override + public boolean hasTerminal() { + return hasTerminal; + } + +} diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/Guard.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/Guard.java new file mode 100644 index 000000000..b5c6ef491 --- /dev/null +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/Guard.java @@ -0,0 +1,17 @@ +/******************************************************************************* + * Copyright (c) 2021 itemis AG (http://www.itemis.eu) and others. + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *******************************************************************************/ +package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting; + +/** + * @author overflow - Initial contribution and API + */ +public interface Guard { + boolean isTrivial(); + String render(); +} diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/Hoisting.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/Hoisting.java new file mode 100644 index 000000000..864162b86 --- /dev/null +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/Hoisting.java @@ -0,0 +1,292 @@ +/******************************************************************************* + * Copyright (c) 2021 itemis AG (http://www.itemis.eu) and others. + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *******************************************************************************/ +package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting; + +import java.util.HashMap; +import java.util.Map; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import org.eclipse.xtext.AbstractElement; +import org.eclipse.xtext.AbstractRule; +import org.eclipse.xtext.AbstractSemanticPredicate; +import org.eclipse.xtext.Action; +import org.eclipse.xtext.Alternatives; +import org.eclipse.xtext.Assignment; +import org.eclipse.xtext.EnumRule; +import org.eclipse.xtext.Group; +import org.eclipse.xtext.Keyword; +import org.eclipse.xtext.ParserRule; +import org.eclipse.xtext.RuleCall; +import org.eclipse.xtext.TerminalRule; +import org.eclipse.xtext.UnorderedGroup; +import org.eclipse.xtext.util.Tuples; +import org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.utils.StreamUtils; + +/** + * @author overflow - Initial contribution and API + */ +public class Hoisting { + private Map ruleCache = new HashMap<>(); + private Map groupCache = new HashMap<>(); + + private boolean isParserRule(AbstractElement element) { + return (element instanceof RuleCall) && (((RuleCall) element).getRule() instanceof ParserRule); + } + + private boolean cardinalityAllowsEmpty(AbstractElement element) { + String cardinality = element.getCardinality(); + return cardinality.equals("?") || cardinality.equals("*"); + } + + private boolean cardinalityAllowsRepetition(AbstractElement element) { + String cardinality = element.getCardinality(); + return cardinality.equals("+") || cardinality.equals("*"); + } + + private TokenAnalysisPaths getTokenForIndexes(Alternatives path, TokenAnalysisPaths prefix, boolean needsLength) throws TokenAnalysisAbortedException { + if (prefix.isDone()) { + return prefix; + } + + TokenAnalysisPaths result; + if (cardinalityAllowsEmpty(path)) { + result = prefix; + if (needsLength) { + // analysis is not done but there are no more mandatory tokens + throw new TokenAnalysisAbortedException(); + } + } else { + result = TokenAnalysisPaths.empty(); + } + + boolean loop = cardinalityAllowsRepetition(path); + + do { + boolean allDone = true; + + for (AbstractElement element : path.getElements()) { + TokenAnalysisPaths current = new TokenAnalysisPaths(prefix); + current = getTokenForIndexes(element, current, needsLength); // will check for needsLength + + if (!current.isDone()) { + allDone = false; + } + + result = result.merge(current); + } + + if (allDone) { + break; + } + + prefix = result; + + // repeat until all further extensions of prefix are done + } while(loop); + + return result; + } + + private TokenAnalysisPaths getTokenForIndexes(Group path, TokenAnalysisPaths prefix, boolean needsLength) throws TokenAnalysisAbortedException { + if (prefix.isDone()) { + return prefix; + } + + TokenAnalysisPaths result; + TokenAnalysisPaths current = new TokenAnalysisPaths(prefix); + + if (cardinalityAllowsEmpty(path)) { + result = prefix; + if (needsLength) { + // analysis is not done but there are no more mandatory tokens + throw new TokenAnalysisAbortedException(); + } + } else { + result = TokenAnalysisPaths.empty(); + } + + boolean loop = cardinalityAllowsRepetition(path); + + do { + for (AbstractElement element : path.getElements()) { + current = getTokenForIndexes(element, current, false); + + if (current.isDone()) { + // no need to look further + return result.merge(current); + } + } + + if (needsLength && !current.isDone()) { + // analysis is not done but there are no more mandatory tokens + throw new TokenAnalysisAbortedException(); + } + + result = result.merge(current); + current = new TokenAnalysisPaths(result); + } while(loop); + + // if cardinality is trivial or ? return result + return result; + } + + private TokenAnalysisPaths getTokenForIndexes(AbstractElement path, TokenAnalysisPaths prefix, boolean needsLength) throws TokenAnalysisAbortedException { + if (prefix.isDone()) { + return prefix; + } + + TokenAnalysisPaths result; + + if (cardinalityAllowsEmpty(path)) { + result = prefix; + if (needsLength) { + throw new TokenAnalysisAbortedException(); + } + } else { + result = TokenAnalysisPaths.empty(); + } + + TokenAnalysisPaths current = new TokenAnalysisPaths(prefix); + + boolean loop = cardinalityAllowsRepetition(path); + do { + if (Token.isToken(path)) { + current.add(path); + } else if (isParserRule(path)) { + current = getTokenForIndexes(((RuleCall) path).getRule().getAlternatives(), current, false); + } else { + throw new IllegalArgumentException("unknown element: " + path.eClass().getName()); + } + + result = result.merge(current); + + if (current.isDone()) { + return result; + } + + if (needsLength) { + throw new TokenAnalysisAbortedException(); + } + } while(loop); + + return result; + } + + private Set> getTokenForIndexes(AbstractElement path, List indexes) throws TokenAnalysisAbortedException { + return getTokenForIndexes(path, new TokenAnalysisPaths(indexes), true).getTokenPaths(); + } + + private boolean arePathsIdenticalSymbolic(AbstractElement path1, AbstractElement path2) throws SymbolicAnalysisFailedException { + return false; + } + + private boolean arePathsIdenticalFallback(AbstractElement path1, AbstractElement path2) { + return false; + } + + private boolean arePathsIdentical(AbstractElement path1, AbstractElement path2) { + try { + return arePathsIdenticalSymbolic(path1, path2); + } catch (SymbolicAnalysisFailedException e) { + return arePathsIdenticalFallback(path1, path2); + } + } + + private List> findMinimalPathDifference(List paths) { + return null; + } + + private HoistingGuard findGuardForAlternatives(Alternatives alternatives) { + List paths = alternatives.getElements(); + List guards = paths.stream() + .map(this::findGuardForElement) + .map(MergedPathGuard::new) + .collect(Collectors.toList()); + + int size = paths.size(); + for (int i = 0; i < size; i++) { + for (int j = i + 1; j < size; j++) { + if (arePathsIdentical(paths.get(i), paths.get(j))) { + guards.get(i).add(guards.get(j)); + + paths.remove(j); + guards.remove(j); + j--; + size--; + } + } + } + + if (size > 1) { + return StreamUtils.zip( + findMinimalPathDifference(paths).stream() + .map(s -> s.stream() + .map(TokenGuard::new) + .collect(Collectors.toSet()) + ), + guards.stream(), + (Set tokenSet, MergedPathGuard guard) -> Tuples.pair(tokenSet, guard) + ).map(p -> new PathGuard(p.getFirst(), p.getSecond())) + .collect(AlternativesGuard.collector()); + } else { + return guards.get(0); + } + } + + private HoistingGuard findGuardForGroup(Group group) { + GroupGuard groupGuard = new GroupGuard(); + + for (AbstractElement element : group.getElements()) { + // TODO: findGuardForElement doesn's support non-trivial cardinalities + HoistingGuard guard = findGuardForElement(element); + groupGuard.add(guard); + + if (guard.hasTerminal()) { + groupGuard.setHasTerminal(); + break; + } + } + + return groupGuard; + } + + private HoistingGuard findGuardForElement(AbstractElement element) { + if (element instanceof Alternatives) { + return findGuardForAlternatives((Alternatives) element); + } else if (element instanceof Group) { + return groupCache.computeIfAbsent((Group) element, this::findGuardForGroup); + } else if (element instanceof AbstractSemanticPredicate) { + return new PredicateGuard((AbstractSemanticPredicate) element); + } else if (element instanceof Keyword) { + return HoistingGuard.terminal(); + } else if (element instanceof RuleCall) { + RuleCall call = (RuleCall) element; + AbstractRule rule = call.getRule(); + if (rule instanceof TerminalRule || rule instanceof EnumRule) { + return HoistingGuard.terminal(); + } else { + // TODO: findGuardForElement can't deal with cardinalities + return ruleCache.computeIfAbsent(rule.getName(), s -> findGuardForElement(rule.getAlternatives())); + } + } else if (element instanceof Action) { + // TODO: Maybe find better indicator for "we don't care about this element" + return HoistingGuard.unguarded(); + } else if (element instanceof UnorderedGroup) { + // TODO: No support for Unordered Groups yet. + throw new UnsupportedOperationException("unordered groups are not yet supported"); + } else if (element instanceof Assignment) { + // TODO: findGuardForElement can't deal with cardinalities + return findGuardForElement(((Assignment) element).getTerminal()); + } else { + throw new UnsupportedOperationException("element not supported: " + element.toString()); + } + } +} diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/HoistingGuard.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/HoistingGuard.java new file mode 100644 index 000000000..96d61b009 --- /dev/null +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/HoistingGuard.java @@ -0,0 +1,54 @@ +/******************************************************************************* + * Copyright (c) 2021 itemis AG (http://www.itemis.eu) and others. + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *******************************************************************************/ +package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting; + +/** + * @author overflow - Initial contribution and API + */ +public interface HoistingGuard extends Guard { + boolean hasTerminal(); + + static public HoistingGuard unguarded() { + return new HoistingGuard() { + @Override + public String render() { + return "true"; + } + + @Override + public boolean isTrivial() { + return true; + } + + @Override + public boolean hasTerminal() { + return false; + } + }; + } + + static public HoistingGuard terminal() { + return new HoistingGuard() { + @Override + public String render() { + return "true"; + } + + @Override + public boolean isTrivial() { + return true; + } + + @Override + public boolean hasTerminal() { + return true; + } + }; + } +} diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/KeywordToken.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/KeywordToken.java new file mode 100644 index 000000000..ca772a17c --- /dev/null +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/KeywordToken.java @@ -0,0 +1,24 @@ +/******************************************************************************* + * Copyright (c) 2021 itemis AG (http://www.itemis.eu) and others. + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *******************************************************************************/ +package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting; + +import org.eclipse.xtext.Keyword; + +/** + * @author overflow - Initial contribution and API + */ +public class KeywordToken extends Token { + private Keyword keyword; + private int position; + + public KeywordToken(Keyword keyword, int position) { + this.keyword = keyword; + this.position = position; + } +} diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/MergedPathGuard.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/MergedPathGuard.java new file mode 100644 index 000000000..18bc33a98 --- /dev/null +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/MergedPathGuard.java @@ -0,0 +1,57 @@ +/******************************************************************************* + * Copyright (c) 2021 itemis AG (http://www.itemis.eu) and others. + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *******************************************************************************/ +package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting; + +import java.util.LinkedList; +import java.util.List; +import java.util.stream.Collectors; + +/** + * @author overflow - Initial contribution and API + */ +public class MergedPathGuard implements HoistingGuard { + private List pathGuards = new LinkedList<>(); + + public MergedPathGuard(HoistingGuard guard) { + add(guard); + } + + public void add(HoistingGuard guard) { + pathGuards.add(guard); + } + + public void add(MergedPathGuard mergedPathGuard) { + pathGuards.addAll(mergedPathGuard.pathGuards); + } + + @Override + public boolean isTrivial() { + return pathGuards.stream().anyMatch(Guard::isTrivial); + } + + @Override + public String render() { + if (pathGuards.size() == 1) { + return pathGuards.get(0).render(); + } else { + return "(" + + pathGuards.stream() + .map(Guard::render) + .collect(Collectors.joining(" || ")) + + ")"; + } + } + + @Override + public boolean hasTerminal() { + // only need to check first element since all paths should be identical + return pathGuards.get(0).hasTerminal(); + } + +} diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/NotATokenException.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/NotATokenException.java new file mode 100644 index 000000000..9d88c3901 --- /dev/null +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/NotATokenException.java @@ -0,0 +1,17 @@ +/******************************************************************************* + * Copyright (c) 2021 itemis AG (http://www.itemis.eu) and others. + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *******************************************************************************/ +package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting; + +/** + * @author overflow - Initial contribution and API + */ +public class NotATokenException extends RuntimeException { + private static final long serialVersionUID = 643265533068524552L; + +} diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/PathGuard.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/PathGuard.java new file mode 100644 index 000000000..3de698ac8 --- /dev/null +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/PathGuard.java @@ -0,0 +1,44 @@ +/******************************************************************************* + * Copyright (c) 2021 itemis AG (http://www.itemis.eu) and others. + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *******************************************************************************/ +package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting; + +import java.util.Collection; +import java.util.LinkedList; +import java.util.List; + +/** + * @author overflow - Initial contribution and API + */ +public class PathGuard implements HoistingGuard { + private Collection tokenGuards; + private HoistingGuard hoistngGuard; + + public PathGuard(Collection tokenGuards, HoistingGuard hoistingGuard) { + this.tokenGuards = tokenGuards; + this.hoistngGuard = hoistingGuard; + } + + @Override + public boolean isTrivial() { + return hoistngGuard.isTrivial(); + } + + @Override + public boolean hasTerminal() { + // empty paths are only allowed when all paths are empty + // in that case a MergedPathGuard is returned. + return true; + } + + @Override + public String render() { + // TODO + return null; + } +} diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/PredicateGuard.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/PredicateGuard.java new file mode 100644 index 000000000..0343ed4a3 --- /dev/null +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/PredicateGuard.java @@ -0,0 +1,39 @@ +/******************************************************************************* + * Copyright (c) 2021 itemis AG (http://www.itemis.eu) and others. + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *******************************************************************************/ +package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting; + +import org.eclipse.xtext.AbstractSemanticPredicate; + +/** + * @author overflow - Initial contribution and API + */ +public class PredicateGuard implements HoistingGuard { + + private AbstractSemanticPredicate element; + + public PredicateGuard(AbstractSemanticPredicate element) { + this.element = element; + } + + @Override + public boolean isTrivial() { + return false; + } + + @Override + public String render() { + return element.getCode().getSource(); + } + + @Override + public boolean hasTerminal() { + return false; + } + +} diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/SymbolicAnalysisFailedException.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/SymbolicAnalysisFailedException.java new file mode 100644 index 000000000..6e0985723 --- /dev/null +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/SymbolicAnalysisFailedException.java @@ -0,0 +1,17 @@ +/******************************************************************************* + * Copyright (c) 2021 itemis AG (http://www.itemis.eu) and others. + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *******************************************************************************/ +package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting; + +/** + * @author overflow - Initial contribution and API + */ +public class SymbolicAnalysisFailedException extends Exception { + private static final long serialVersionUID = 4185473673062321988L; + +} diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/TerminalRuleToken.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/TerminalRuleToken.java new file mode 100644 index 000000000..d5f01d058 --- /dev/null +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/TerminalRuleToken.java @@ -0,0 +1,24 @@ +/******************************************************************************* + * Copyright (c) 2021 itemis AG (http://www.itemis.eu) and others. + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *******************************************************************************/ +package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting; + +import org.eclipse.xtext.TerminalRule; + +/** + * @author overflow - Initial contribution and API + */ +public class TerminalRuleToken extends Token { + private TerminalRule rule; + private int position; + + public TerminalRuleToken(TerminalRule rule, int position) { + this.rule = rule; + this.position = position; + } +} diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/Token.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/Token.java new file mode 100644 index 000000000..7df646925 --- /dev/null +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/Token.java @@ -0,0 +1,48 @@ +/******************************************************************************* + * Copyright (c) 2021 itemis AG (http://www.itemis.eu) and others. + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *******************************************************************************/ +package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting; + +import org.eclipse.xtext.AbstractElement; +import org.eclipse.xtext.AbstractRule; +import org.eclipse.xtext.Keyword; +import org.eclipse.xtext.RuleCall; +import org.eclipse.xtext.TerminalRule; + +/** + * @author overflow - Initial contribution and API + */ +public class Token { + + static boolean isToken(AbstractElement element) { + if (element instanceof Keyword) { + return true; + } else if (element instanceof RuleCall) { + if (((RuleCall) element).getRule() instanceof TerminalRule) { + return true; + } else { + return false; + } + } else { + return false; + } + } + + static Token fromElement(AbstractElement element, int position) { + if (element instanceof Keyword) { + return new KeywordToken((Keyword) element, position); + } else if (element instanceof RuleCall) { + AbstractRule rule = ((RuleCall) element).getRule(); + if (rule instanceof TerminalRule) { + return new TerminalRuleToken((TerminalRule) rule, position); + } + } + + throw new NotATokenException(); + } +} diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/TokenAnalysisAbortedException.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/TokenAnalysisAbortedException.java new file mode 100644 index 000000000..68b5163b4 --- /dev/null +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/TokenAnalysisAbortedException.java @@ -0,0 +1,16 @@ +/******************************************************************************* + * Copyright (c) 2021 itemis AG (http://www.itemis.eu) and others. + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *******************************************************************************/ +package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting; + +/** + * @author overflow - Initial contribution and API + */ +public class TokenAnalysisAbortedException extends Exception { + private static final long serialVersionUID = 4303267001950479292L; +} diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/TokenAnalysisPath.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/TokenAnalysisPath.java new file mode 100644 index 000000000..616e13a15 --- /dev/null +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/TokenAnalysisPath.java @@ -0,0 +1,60 @@ +/******************************************************************************* + * Copyright (c) 2021 itemis AG (http://www.itemis.eu) and others. + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *******************************************************************************/ +package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting; + +import java.util.LinkedList; +import java.util.List; + +import org.eclipse.xtext.AbstractElement; + +/** + * @author overflow - Initial contribution and API + */ +public class TokenAnalysisPath { + private List path = new LinkedList<>(); + private List remainingIndexes; + private int position = 1; + + public TokenAnalysisPath(List indexes) { + this.remainingIndexes = indexes; + } + + public TokenAnalysisPath(TokenAnalysisPath prefix) { + this(prefix.remainingIndexes); + path = new LinkedList<>(prefix.path); + position = prefix.position; + } + + public boolean isDone() { + return remainingIndexes.isEmpty(); + } + + private void shift() { + position++; + int size = remainingIndexes.size(); + for (int i = 0; i < size; i++) { + remainingIndexes.set(i, remainingIndexes.get(i) - 1); + } + } + + public void add(AbstractElement element) { + if (isDone()) + return; + + if (remainingIndexes.get(0) <= 0) { + path.add(Token.fromElement(element, position)); + remainingIndexes.remove(0); + } + shift(); + } + + public List getTokenPath() { + return path; + } +} diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/TokenAnalysisPaths.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/TokenAnalysisPaths.java new file mode 100644 index 000000000..0ef3c9acb --- /dev/null +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/TokenAnalysisPaths.java @@ -0,0 +1,67 @@ +/******************************************************************************* + * Copyright (c) 2021 itemis AG (http://www.itemis.eu) and others. + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *******************************************************************************/ +package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting; + +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import org.eclipse.xtext.AbstractElement; + +/** + * @author overflow - Initial contribution and API + */ +public class TokenAnalysisPaths { + private Set tokenPaths = new HashSet<>(); + private boolean isEmpty = false; + + public Set> getTokenPaths() { + return tokenPaths.stream() + .map(TokenAnalysisPath::getTokenPath) + .collect(Collectors.toSet()); + } + + private TokenAnalysisPaths() { + } + + public TokenAnalysisPaths(List indexes) { + tokenPaths.add(new TokenAnalysisPath(indexes)); + } + + public TokenAnalysisPaths(TokenAnalysisPaths prefix) { + this.tokenPaths = prefix.tokenPaths.stream() + .map(TokenAnalysisPath::new) + .collect(Collectors.toSet()); + } + + public boolean isDone() { + return tokenPaths.stream().allMatch(TokenAnalysisPath::isDone); + } + + public void add(AbstractElement element) { + tokenPaths.forEach(p -> p.add(element)); + } + + public TokenAnalysisPaths merge(TokenAnalysisPaths other) { + if (isEmpty) { + return other; + } else { + this.tokenPaths.addAll(other.tokenPaths); + return this; + } + } + + static public TokenAnalysisPaths empty() { + TokenAnalysisPaths empty = new TokenAnalysisPaths(); + empty.isEmpty = true; + return empty; + } +} diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/TokenGuard.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/TokenGuard.java new file mode 100644 index 000000000..0df2a1262 --- /dev/null +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/TokenGuard.java @@ -0,0 +1,31 @@ +/******************************************************************************* + * Copyright (c) 2021 itemis AG (http://www.itemis.eu) and others. + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *******************************************************************************/ +package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting; + +/** + * @author overflow - Initial contribution and API + */ +public class TokenGuard implements Guard { + private Token token; + + public TokenGuard(Token token) { + this.token = token; + } + + @Override + public boolean isTrivial() { + return false; + } + + @Override + public String render() { + return "not implemented"; + } + +} diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/utils/StreamUtils.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/utils/StreamUtils.java new file mode 100644 index 000000000..2ceb90296 --- /dev/null +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/utils/StreamUtils.java @@ -0,0 +1,40 @@ +/******************************************************************************* + * Copyright (c) 2021 itemis AG (http://www.itemis.eu) and others. + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + *******************************************************************************/ +package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.utils; + +import java.util.Iterator; +import java.util.Spliterator; +import java.util.Spliterators; +import java.util.function.BiFunction; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; + +/** + * @author overflow - Initial contribution and API + */ +public class StreamUtils { + public static Stream zip(Stream stream1, Stream stream2, BiFunction combine) { + Iterator iter1 = stream1.iterator(); + Iterator iter2 = stream2.iterator(); + + Spliterator spliterator = Spliterators.spliteratorUnknownSize(new Iterator() { + @Override + public boolean hasNext() { + return iter1.hasNext() && iter2.hasNext(); + } + + @Override + public C next() { + return combine.apply(iter1.next(), iter2.next()); + } + }, Spliterator.NONNULL); + + return StreamSupport.stream(spliterator, false); + } +}