sigmasternchen/xtext-core
1
0
Fork 0
mirror of https://github.com/sigmasternchen/xtext-core synced 2025-03-15 08:18:55 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

added basic support for nested prefix alternatives

Browse source 
after minimal path difference analysis failed, flatten paths (limited by
token limit; justification: identity check would error out if paths are
not distinguishable within the limit) and recompute alternative guard.
now nested prefixes will be collapsed with all corresponding guard
conditions
This commit is contained in:
overflowerror 2021-12-17 21:18:59 +01:00
parent 45d2df5c33
commit be80d5ffbc
8 changed files with 185 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

92
org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/HoistingProcessor.java
View file

@ -18,6 +18,9 @@ import java.util.List;
import java.util.stream.Collectors;
import org.apache.log4j.Logger;
import static org.eclipse.emf.ecore.util.EcoreUtil.*;
import org.eclipse.xtext.AbstractElement;
import org.eclipse.xtext.AbstractRule;
import org.eclipse.xtext.AbstractSemanticPredicate;
@ -28,12 +31,15 @@ import org.eclipse.xtext.CompoundElement;
import org.eclipse.xtext.Grammar;
import org.eclipse.xtext.Group;
import org.eclipse.xtext.JavaAction;
import org.eclipse.xtext.JavaCode;
import org.eclipse.xtext.RuleCall;
import org.eclipse.xtext.UnorderedGroup;
import org.eclipse.xtext.XtextFactory;
import org.eclipse.xtext.util.Tuples;
import static org.eclipse.xtext.GrammarUtil.*;
import org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.exceptions.NestedPrefixAlternativesException;
import org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.exceptions.OptionalCardinalityWithoutContextException;
import org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.exceptions.TokenAnalysisAbortedException;
import org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.exceptions.UnsupportedConstructException;
@ -78,7 +84,7 @@ public class HoistingProcessor {
}
// set cardinality so the token analyse works
element = cloneAbstractElement(element);
element = copy(element);
if (isMultipleCardinality(element)) {
element.setCardinality("+");
} else {
@ -108,6 +114,70 @@ public class HoistingProcessor {
}
}
private AbstractElement getNopElement() {
JavaCode virtualJavaCodeForAction = XtextFactory.eINSTANCE.createJavaCode();
virtualJavaCodeForAction.setSource("$$ /* nop */ $$");
JavaAction virtualNopJavaAction = XtextFactory.eINSTANCE.createJavaAction();
virtualNopJavaAction.setCode(virtualJavaCodeForAction);
return virtualNopJavaAction;
}
private CompoundElement flattenPaths(CompoundElement original, List<AbstractElement> paths, List<? extends HoistingGuard> guards) {
// work on copy of original to preserve eParent
CompoundElement flattened = copy(original);
setElementsOfCompoundElement(flattened,
StreamUtils.zip(
paths.stream()
.map(analysis::getAllPossiblePaths),
guards.stream()
.map(HoistingGuard.class::cast),
(List<List<AbstractElement>> e, HoistingGuard g) -> Tuples.pair(e, g)
).filter(t -> t.getFirst() != null)
.flatMap(t -> {
HoistingGuard guard = t.getSecond();
// we can reuse objects since setElementsOfCompundElement
// will create copy anyway
AbstractElement nop = getNopElement();
AbstractSemanticPredicate renderedVirtualPredicate;
if (!guard.isTrivial()) {
JavaCode virtualJavaCodeForPredicate = XtextFactory.eINSTANCE.createJavaCode();
virtualJavaCodeForPredicate.setSource("$$ " + guard.render() + " $$");
renderedVirtualPredicate = XtextFactory.eINSTANCE.createGatedSemanticPredicate();
renderedVirtualPredicate.setCode(virtualJavaCodeForPredicate);
} else {
renderedVirtualPredicate = null;
}
return t.getFirst()
.stream()
.map(a -> {
List<AbstractElement> groupElements = new ArrayList<>(a.size() + 2);
if (renderedVirtualPredicate != null) {
groupElements.add(renderedVirtualPredicate);
}
// virtual action to prevent hoisting
groupElements.add(nop);
groupElements.addAll(a);
Group virtualGroup = XtextFactory.eINSTANCE.createGroup();
setElementsOfCompoundElement(virtualGroup, groupElements);
return virtualGroup;
});
})
);
return flattened;
}
private HoistingGuard findGuardForAlternatives(CompoundElement alternatives, AbstractRule currentRule) {
log.info("find guard for alternative");
@ -171,6 +241,18 @@ public class HoistingProcessor {
(TokenGuard tokenGuard, MergedPathGuard pathGuard) -> Tuples.pair(tokenGuard, pathGuard)
).map(p -> new PathGuard(p.getFirst(), p.getSecond()))
.collect(AlternativesGuard.collector());
} catch(NestedPrefixAlternativesException e) {
// nested prefix alternatives
// -> flatten paths to alternative and try again
// this is very inefficient
log.warn("nested prefix alternatives detected");
log.warn("avoid these since they can't be handled efficiency");
log.info(abstractElementToString(alternatives));
CompoundElement flattened = flattenPaths(alternatives, paths, guards);
log.info(abstractElementToString(flattened));
return findGuardForAlternatives(flattened, currentRule);
} catch(TokenAnalysisAbortedException e) {
throw new TokenAnalysisAbortedException(e.getMessage(), e, currentRule);
}
@ -225,7 +307,7 @@ public class HoistingProcessor {
// we need a clone of the element because we need to set the cardinality without changing the
// original syntax tree
AbstractElement clonedElement = cloneAbstractElement(element);
AbstractElement clonedElement = copy(element);
clonedElement.setCardinality(null);
// make copy of every element because we can't use any element twice
@ -243,13 +325,13 @@ public class HoistingProcessor {
remainingElementsInGroupIncludingCurrent.add(0, clonedElement);
Group virtualPathRemaining = XtextFactory.eINSTANCE.createGroup();
addElementsToCompoundElement(virtualPathRemaining, remainingElementsInGroup);
setElementsOfCompoundElement(virtualPathRemaining, remainingElementsInGroup);
Group virtualPathRemainingPlusCurrent = XtextFactory.eINSTANCE.createGroup();
addElementsToCompoundElement(virtualPathRemainingPlusCurrent, remainingElementsInGroupIncludingCurrent);
setElementsOfCompoundElement(virtualPathRemainingPlusCurrent, remainingElementsInGroupIncludingCurrent);
Alternatives virtualAlternatives = XtextFactory.eINSTANCE.createAlternatives();
addElementsToCompoundElement(virtualAlternatives, Arrays.asList(virtualPathRemaining, virtualPathRemainingPlusCurrent));
setElementsOfCompoundElement(virtualAlternatives, Arrays.asList(virtualPathRemaining, virtualPathRemainingPlusCurrent));
// get Guard for virtual alternatives
HoistingGuard guard = findGuardForElementWithTrivialCardinality(virtualAlternatives, currentRule);

53
org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/exceptions/NestedPrefixAlternativesException.java Normal file
View file

@ -0,0 +1,53 @@
/*******************************************************************************
* Copyright (c) 2021 itemis AG (http://www.itemis.eu) and others.
* This program and the accompanying materials are made available under the
* terms of the Eclipse Public License 2.0 which is available at
* http://www.eclipse.org/legal/epl-2.0.
*
* SPDX-License-Identifier: EPL-2.0
*******************************************************************************/
package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.exceptions;
import org.eclipse.xtext.AbstractRule;
/**
* @author overflow - Initial contribution and API
*/
public class NestedPrefixAlternativesException extends TokenAnalysisAbortedException {
private static final long serialVersionUID = -1309434841547765441L;
public NestedPrefixAlternativesException() {
super();
// TODO Auto-generated constructor stub
}
public NestedPrefixAlternativesException(String message, AbstractRule rule) {
super(message, rule);
// TODO Auto-generated constructor stub
}
public NestedPrefixAlternativesException(String message, Throwable cause, AbstractRule rule) {
super(message, cause, rule);
// TODO Auto-generated constructor stub
}
public NestedPrefixAlternativesException(String message, Throwable cause) {
super(message, cause);
// TODO Auto-generated constructor stub
}
public NestedPrefixAlternativesException(String message) {
super(message);
// TODO Auto-generated constructor stub
}
public NestedPrefixAlternativesException(Throwable cause, AbstractRule rule) {
super(cause, rule);
// TODO Auto-generated constructor stub
}
public NestedPrefixAlternativesException(Throwable cause) {
super(cause);
// TODO Auto-generated constructor stub
}
}

19
org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/pathAnalysis/TokenAnalysis.java
View file

@ -30,12 +30,11 @@ import org.eclipse.xtext.CompoundElement;
import org.eclipse.xtext.Grammar;
import org.eclipse.xtext.GrammarUtil;
import org.eclipse.xtext.Group;
import org.eclipse.xtext.Keyword;
import org.eclipse.xtext.ParserRule;
import org.eclipse.xtext.RuleCall;
import org.eclipse.xtext.UnorderedGroup;
import org.eclipse.xtext.util.XtextSwitch;
import org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.HoistingConfiguration;
import org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.exceptions.NestedPrefixAlternativesException;
import org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.exceptions.SymbolicAnalysisFailedException;
import org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.exceptions.TokenAnalysisAbortedException;
import org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.token.Token;
@ -402,7 +401,6 @@ public class TokenAnalysis {
}
}
private void tokenCombinations(Function<List<Integer>, Boolean> callback) {
MutablePrimitiveWrapper<Integer> limit = new MutablePrimitiveWrapper<>(config.getTokenLimit());
@ -416,7 +414,9 @@ public class TokenAnalysis {
if (limit.get().equals(config.getTokenLimit())) {
throw new TokenAnalysisAbortedException("token limit exhausted while searching for minimal differences");
} else {
throw new TokenAnalysisAbortedException("path length exhausted while searching for minimal differences");
// path length exhausted while searching for minimal differences
// this indicates nested prefix alternatives
throw new NestedPrefixAlternativesException();
}
}
private boolean tokenCombinations(long prefix, int prefixLength, int ones, Function<List<Integer>, Boolean> callback, MutablePrimitiveWrapper<Integer> limit) {
@ -528,4 +528,15 @@ public class TokenAnalysis {
return result;
}
public List<List<AbstractElement>> getAllPossiblePaths(AbstractElement path) {
return getTokenPaths(path, new TokenAnalysisPaths(range(0, config.getTokenLimit() + 1)), false, false)
.getTokenPaths()
.stream()
.map(l -> l.stream()
.map(Token::getElement)
.collect(Collectors.toList()))
.collect(Collectors.toList());
}
}

7
org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/EofToken.java
View file

@ -8,6 +8,8 @@
*******************************************************************************/
package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.token;
import org.eclipse.xtext.AbstractElement;
/**
* @author overflow - Initial contribution and API
*/
@ -51,4 +53,9 @@ public class EofToken implements Token {
return "EofToken(" + position + ")\n";
}
@Override
public AbstractElement getElement() {
return null;
}
}

7
org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/KeywordToken.java
View file

@ -8,6 +8,7 @@
*******************************************************************************/
package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.token;
import org.eclipse.xtext.AbstractElement;
import org.eclipse.xtext.Keyword;
/**
@ -59,4 +60,10 @@ public class KeywordToken implements Token {
return false;
return true;
}
@Override
public AbstractElement getElement() {
return keyword;
}
}

11
org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/TerminalRuleToken.java
View file

@ -8,16 +8,20 @@
*******************************************************************************/
package org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.token;
import org.eclipse.xtext.AbstractElement;
import org.eclipse.xtext.RuleCall;
import org.eclipse.xtext.TerminalRule;
/**
* @author overflow - Initial contribution and API
*/
public class TerminalRuleToken implements Token {
private RuleCall call;
private TerminalRule rule;
private int position;
public TerminalRuleToken(TerminalRule rule, int position) {
TerminalRuleToken(RuleCall call, TerminalRule rule, int position) {
this.call = call;
this.rule = rule;
this.position = position;
}
@ -60,5 +64,8 @@ public class TerminalRuleToken implements Token {
return true;
}
@Override
public AbstractElement getElement() {
return call;
}
}

4
org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/hoisting/token/Token.java
View file

@ -21,6 +21,8 @@ import org.eclipse.xtext.TerminalRule;
public interface Token {
String negatedCondition();
AbstractElement getElement();
static boolean isToken(AbstractElement element) {
if (element == null) {
return true;
@ -43,7 +45,7 @@ public interface Token {
} else if (element instanceof RuleCall) {
AbstractRule rule = ((RuleCall) element).getRule();
if (rule instanceof TerminalRule) {
return new TerminalRuleToken((TerminalRule) rule, position);
return new TerminalRuleToken((RuleCall) element, (TerminalRule) rule, position);
}
} else if (element instanceof EnumLiteralDeclaration) {
return new KeywordToken(((EnumLiteralDeclaration) element).getLiteral(), position);

12
org.eclipse.xtext/src/org/eclipse/xtext/GrammarUtil.java
View file

@ -709,20 +709,16 @@ public class GrammarUtil {
return null;
}
public static AbstractElement cloneAbstractElement(AbstractElement element) {
return (AbstractElement) EcoreUtil.copy(element);
}
public static void addElementsToCompoundElement(CompoundElement element, Stream<? extends AbstractElement> elements) {
public static void setElementsOfCompoundElement(CompoundElement element, Stream<? extends AbstractElement> elements) {
EStructuralFeature compoundElementElementsFeature = XtextPackage.Literals.COMPOUND_ELEMENT.getEStructuralFeature(XtextPackage.COMPOUND_ELEMENT__ELEMENTS);
element.eSet(compoundElementElementsFeature, elements
.map(GrammarUtil::cloneAbstractElement)
.map(EcoreUtil::copy)
.collect(Collectors.toList()));
}
public static void addElementsToCompoundElement(CompoundElement element, Collection<? extends AbstractElement> elements) {
addElementsToCompoundElement(element, elements.stream());
public static void setElementsOfCompoundElement(CompoundElement element, Collection<? extends AbstractElement> elements) {
setElementsOfCompoundElement(element, elements.stream());
}
private static void findAllRuleCalls(List<RuleCall> calls, AbstractElement element, AbstractRule rule) {