mirror of
https://github.com/sigmasternchen/xtext-core
synced 2025-03-15 16:28:56 +00:00
Reuse parts of the AbstractAntlrGeneratorFragment2
This commit is contained in:
Sebastian Zarnekow
parent
6245f3e6e6
commit
231b4ef862
3 changed files with 45 additions and 216 deletions
|
|
@ -22,6 +22,7 @@ import org.eclipse.xtext.ParserRule
|
|||
import org.eclipse.xtext.UnorderedGroup
|
||||
import org.eclipse.xtext.generator.LineSeparatorHarmonizer
|
||||
import org.eclipse.xtext.util.Strings
|
||||
import org.eclipse.xtext.xtext.generator.AbstractXtextGeneratorFragment
|
||||
import org.eclipse.xtext.xtext.generator.CodeConfig
|
||||
import org.eclipse.xtext.xtext.generator.Issues
|
||||
import org.eclipse.xtext.xtext.generator.model.IXtextGeneratorFileSystemAccess
|
||||
|
|
@ -31,11 +32,10 @@ import org.eclipse.xtext.xtext.generator.parser.antlr.splitting.AntlrCodeQuality
|
|||
import org.eclipse.xtext.xtext.generator.parser.antlr.splitting.AntlrLexerSplitter
|
||||
import org.eclipse.xtext.xtext.generator.parser.antlr.splitting.AntlrParserSplitter
|
||||
import org.eclipse.xtext.xtext.generator.parser.antlr.splitting.BacktrackingGuardForUnorderedGroupsRemover
|
||||
import org.eclipse.xtext.xtext.generator.parser.antlr.splitting.BacktrackingGuardRemover
|
||||
import org.eclipse.xtext.xtext.generator.parser.antlr.splitting.PartialClassExtractor
|
||||
import org.eclipse.xtext.xtext.generator.parser.antlr.splitting.SyntacticPredicateFixup
|
||||
import org.eclipse.xtext.xtext.generator.parser.antlr.splitting.UnorderedGroupsSplitter
|
||||
import org.eclipse.xtext.xtext.generator.parser.antlr.splitting.BacktrackingGuardRemover
|
||||
import org.eclipse.xtext.xtext.generator.AbstractXtextGeneratorFragment
|
||||
|
||||
abstract class AbstractAntlrGeneratorFragment2 extends AbstractXtextGeneratorFragment {
|
||||
@Inject @Accessors(PROTECTED_GETTER) AntlrToolFacade antlrTool
|
||||
|
|
@ -169,11 +169,12 @@ abstract class AbstractAntlrGeneratorFragment2 extends AbstractXtextGeneratorFra
|
|||
}
|
||||
}
|
||||
|
||||
def protected improveCodeQuality(IXtextGeneratorFileSystemAccess fsa, TypeReference lexer, TypeReference parser) {
|
||||
var lexerContent = fsa.readTextFile(lexer.javaPath).toString
|
||||
lexerContent = codeQualityHelper.stripUnnecessaryComments(lexerContent, options)
|
||||
fsa.generateFile(lexer.javaPath, lexerContent)
|
||||
|
||||
def protected void improveCodeQuality(IXtextGeneratorFileSystemAccess fsa, TypeReference lexer, TypeReference parser) {
|
||||
improveLexerCodeQuality(fsa, lexer)
|
||||
improveParserCodeQuality(fsa, parser)
|
||||
}
|
||||
|
||||
def protected void improveParserCodeQuality(IXtextGeneratorFileSystemAccess fsa, TypeReference parser) {
|
||||
var parserContent = fsa.readTextFile(parser.javaPath).toString
|
||||
parserContent = codeQualityHelper.stripUnnecessaryComments(parserContent, options)
|
||||
parserContent = codeQualityHelper.removeDuplicateBitsets(parserContent, options)
|
||||
|
|
@ -181,6 +182,12 @@ abstract class AbstractAntlrGeneratorFragment2 extends AbstractXtextGeneratorFra
|
|||
fsa.generateFile(parser.javaPath, parserContent)
|
||||
}
|
||||
|
||||
def protected void improveLexerCodeQuality(IXtextGeneratorFileSystemAccess fsa, TypeReference lexer) {
|
||||
var lexerContent = fsa.readTextFile(lexer.javaPath).toString
|
||||
lexerContent = codeQualityHelper.stripUnnecessaryComments(lexerContent, options)
|
||||
fsa.generateFile(lexer.javaPath, lexerContent)
|
||||
}
|
||||
|
||||
def protected void cleanupLexerTokensFile(AntlrGrammar lexerGrammar, KeywordHelper helper, IXtextGeneratorFileSystemAccess fsa) {
|
||||
if (options.backtrackLexer) {
|
||||
val provider = createLexerTokensProvider(lexerGrammar, helper, fsa)
|
||||
|
|
|
|||
|
|
@ -7,42 +7,23 @@
|
|||
*******************************************************************************/
|
||||
package org.eclipse.xtext.xtext.generator.parser.antlr.ex;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.nio.charset.Charset;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
|
||||
import org.eclipse.emf.common.util.WrappedException;
|
||||
import org.eclipse.xtend2.lib.StringConcatenationClient;
|
||||
import org.eclipse.xtext.generator.LineSeparatorHarmonizer;
|
||||
import org.eclipse.xtext.parser.antlr.Lexer;
|
||||
import org.eclipse.xtext.util.Strings;
|
||||
import org.eclipse.xtext.xtext.generator.AbstractXtextGeneratorFragment;
|
||||
import org.eclipse.xtext.xtext.generator.CodeConfig;
|
||||
import org.eclipse.xtext.xtext.generator.Issues;
|
||||
import org.eclipse.xtext.xtext.generator.model.GuiceModuleAccess;
|
||||
import org.eclipse.xtext.xtext.generator.model.IXtextGeneratorFileSystemAccess;
|
||||
import org.eclipse.xtext.xtext.generator.parser.antlr.AntlrToolFacade;
|
||||
import org.eclipse.xtext.xtext.generator.parser.antlr.postProcessing.SuppressWarningsProcessor;
|
||||
import org.eclipse.xtext.xtext.generator.parser.antlr.splitting.AntlrLexerSplitter;
|
||||
import org.eclipse.xtext.xtext.generator.parser.antlr.splitting.internal.LexerSpecialStateTransitionSplitter;
|
||||
import org.eclipse.xtext.xtext.generator.model.TypeReference;
|
||||
import org.eclipse.xtext.xtext.generator.parser.antlr.AbstractAntlrGeneratorFragment2;
|
||||
|
||||
import com.google.common.annotations.Beta;
|
||||
import com.google.common.collect.Lists;
|
||||
import com.google.common.io.Files;
|
||||
import com.google.inject.Inject;
|
||||
|
||||
/**
|
||||
* @since 2.20
|
||||
* @author dietrich - Initial contribution and API
|
||||
*/
|
||||
@Beta
|
||||
public class ExternalAntlrLexerFragment extends AbstractXtextGeneratorFragment {
|
||||
public class ExternalAntlrLexerFragment extends AbstractAntlrGeneratorFragment2 {
|
||||
|
||||
@Inject CodeConfig codeConfig;
|
||||
|
||||
private String lexerGrammar;
|
||||
|
||||
private boolean highlighting;
|
||||
|
|
@ -51,48 +32,8 @@ public class ExternalAntlrLexerFragment extends AbstractXtextGeneratorFragment {
|
|||
|
||||
private boolean contentAssist;
|
||||
|
||||
private boolean classSplitting = false;
|
||||
|
||||
private boolean specialStateSwitchSplitting = false;
|
||||
|
||||
private int casesPerSpecialStateSwitch = LexerSpecialStateTransitionSplitter.CASES_PER_SPECIAL_STATE_SWITCH;
|
||||
|
||||
private List<String> antlrParams = Lists.newArrayList();
|
||||
|
||||
private String getLineDelimiter() {
|
||||
return codeConfig.getLineDelimiter();
|
||||
}
|
||||
|
||||
public void addAntlrParam(String param) {
|
||||
antlrParams.add(param);
|
||||
}
|
||||
|
||||
public String[] getAntlrParams() {
|
||||
List<String> params = new ArrayList<String>(antlrParams);
|
||||
// setting the default conversion timeout to 100secs.
|
||||
// There seem to be no practical situations where the NFA conversion would hang,
|
||||
// so Terence suggested here [1] to remove the option all together
|
||||
// [1] - http://antlr.1301665.n2.nabble.com/Xconversiontimeout-td5294411.html
|
||||
if (!params.contains("-Xconversiontimeout")) {
|
||||
params.add("-Xconversiontimeout");
|
||||
params.add("100000");
|
||||
}
|
||||
String[] result = params.toArray(new String[params.size()]);
|
||||
return result;
|
||||
}
|
||||
|
||||
private AntlrToolFacade antlrTool = new AntlrToolFacade();
|
||||
|
||||
public void setAntlrTool(AntlrToolFacade facade) {
|
||||
this.antlrTool = facade;
|
||||
}
|
||||
|
||||
public AntlrToolFacade getAntlrTool() {
|
||||
return antlrTool;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void generate() {
|
||||
protected void doGenerate() {
|
||||
// i am not sure when to generate what
|
||||
if (runtime) {
|
||||
StringConcatenationClient binding = new StringConcatenationClient() {
|
||||
|
|
@ -166,85 +107,35 @@ public class ExternalAntlrLexerFragment extends AbstractXtextGeneratorFragment {
|
|||
String srcPath = srcFsa.getPath();
|
||||
String grammarFile = srcPath + "/" + getLexerGrammar().replace('.', '/') + ".g";
|
||||
String generateTo = "";
|
||||
if (lexerGrammar.lastIndexOf('.') != -1) {
|
||||
generateTo = lexerGrammar.substring(0, lexerGrammar.lastIndexOf('.'));
|
||||
if (getLexerGrammar().lastIndexOf('.') != -1) {
|
||||
generateTo = getLexerGrammar().substring(0, getLexerGrammar().lastIndexOf('.'));
|
||||
}
|
||||
generateTo = srcGenPath + "/" + generateTo.replace('.', '/');
|
||||
addAntlrParam("-fo");
|
||||
addAntlrParam(generateTo);
|
||||
final String encoding = codeConfig.getEncoding();
|
||||
final String encoding = getCodeConfig().getEncoding();
|
||||
getAntlrTool().runWithEncodingAndParams(grammarFile, encoding, getAntlrParams());
|
||||
Charset charset = Charset.forName(encoding);
|
||||
String javaFile = srcGenPath+"/"+getLexerGrammar().replace('.', '/')+".java";
|
||||
splitLexerIfEnabled(javaFile, charset);
|
||||
suppressWarningsImpl(javaFile, charset);
|
||||
normalizeLineDelimiters(javaFile, charset);
|
||||
normalizeTokens(javaFile, charset);
|
||||
|
||||
TypeReference lexerType = new TypeReference(getLexerGrammar());
|
||||
splitParserAndLexerIfEnabled(srcGenFsa, null /* parser */, lexerType);
|
||||
normalizeTokens(srcGenFsa, getLexerGrammar().replace('.', '/') + ".tokens");
|
||||
suppressWarnings(srcGenFsa, lexerType);
|
||||
normalizeLineDelimiters(srcGenFsa, lexerType);
|
||||
}
|
||||
|
||||
/**
|
||||
* @since 2.9
|
||||
*/
|
||||
protected void splitLexerIfEnabled(String lexerJavaFile, Charset encoding) {
|
||||
if (isClassSplitting()) {
|
||||
String content = readFileIntoString(lexerJavaFile, encoding);
|
||||
AntlrLexerSplitter splitter = new AntlrLexerSplitter(content);
|
||||
splitter.setAllowDFAStaticClasses(false);
|
||||
splitter.setCasesPerSpecialStateSwitch(casesPerSpecialStateSwitch);
|
||||
writeStringIntoFile(lexerJavaFile, splitter.transform(), encoding);
|
||||
}
|
||||
@Override
|
||||
protected void splitParserClassFile(IXtextGeneratorFileSystemAccess fsa, TypeReference parser) {
|
||||
// no-op
|
||||
}
|
||||
|
||||
private void normalizeTokens(String grammarFileName, Charset encoding) {
|
||||
String tokenFile = toTokenFileName(grammarFileName);
|
||||
String content = readFileIntoString(tokenFile, encoding);
|
||||
content = new NewlineNormalizer(getLineDelimiter()).normalizeLineDelimiters(content);
|
||||
List<String> splitted = Strings.split(content, getLineDelimiter());
|
||||
Collections.sort(splitted);
|
||||
content = Strings.concat(getLineDelimiter(), splitted) + getLineDelimiter();
|
||||
writeStringIntoFile(tokenFile, content, encoding);
|
||||
@Override
|
||||
protected void improveParserCodeQuality(IXtextGeneratorFileSystemAccess fsa, TypeReference parser) {
|
||||
// no-op
|
||||
}
|
||||
|
||||
private void normalizeLineDelimiters(String textFile, Charset encoding) {
|
||||
String content = readFileIntoString(textFile, encoding);
|
||||
content = new NewlineNormalizer(getLineDelimiter()) {
|
||||
// Antlr tries to outsmart us by using a line length that depends on the system
|
||||
// line delimiter when it splits a very long String (encoded DFA) into a
|
||||
// string concatenation
|
||||
// Here we join these lines again.
|
||||
@Override
|
||||
public String normalizeLineDelimiters(CharSequence content) {
|
||||
String result = super.normalizeLineDelimiters(content);
|
||||
result = result.replaceAll("\"\\+(\\r)?\\n\\s+\"", "");
|
||||
return result;
|
||||
}
|
||||
}.normalizeLineDelimiters(content);
|
||||
writeStringIntoFile(textFile, content, encoding);
|
||||
}
|
||||
|
||||
private String toTokenFileName(String grammarFileName) {
|
||||
return grammarFileName.replaceAll("\\.java$", ".tokens");
|
||||
}
|
||||
|
||||
/**
|
||||
* @deprecated use {@link #suppressWarningsImpl(String, Charset)} instead
|
||||
*/
|
||||
@Deprecated
|
||||
protected void suppressWarningsImpl(String javaFile) {
|
||||
suppressWarningsImpl(javaFile, Charset.defaultCharset());
|
||||
}
|
||||
|
||||
/**
|
||||
* @since 2.7
|
||||
*/
|
||||
protected void suppressWarningsImpl(String javaFile, Charset encoding) {
|
||||
String content = readFileIntoString(javaFile, encoding);
|
||||
content = new SuppressWarningsProcessor().process(content);
|
||||
writeStringIntoFile(javaFile, content, encoding);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void checkConfiguration(Issues issues) {
|
||||
super.checkConfiguration(issues);
|
||||
if (contentAssist && highlighting || runtime && highlighting || contentAssist && runtime) {
|
||||
issues.addError("Only one of those flags is allowed: contentAssist, runtime, highlighting flag");
|
||||
}
|
||||
|
|
@ -283,81 +174,4 @@ public class ExternalAntlrLexerFragment extends AbstractXtextGeneratorFragment {
|
|||
return contentAssist;
|
||||
}
|
||||
|
||||
/**
|
||||
* @since 2.9
|
||||
*/
|
||||
public boolean isClassSplitting() {
|
||||
return classSplitting;
|
||||
}
|
||||
|
||||
/**
|
||||
* @since 2.9
|
||||
*/
|
||||
public void setClassSplitting(boolean value) {
|
||||
this.classSplitting = value;
|
||||
}
|
||||
|
||||
/**
|
||||
* @since 2.9
|
||||
*/
|
||||
public boolean isSpecialStateSwitchSplitting() {
|
||||
return specialStateSwitchSplitting;
|
||||
}
|
||||
|
||||
/**
|
||||
* @since 2.9
|
||||
*/
|
||||
public void setSpecialStateSwitchSplitting(boolean value) {
|
||||
this.specialStateSwitchSplitting = value;
|
||||
}
|
||||
|
||||
/**
|
||||
* @since 2.9
|
||||
*/
|
||||
public int getCasesPerSpecialStateSwitch(){
|
||||
return casesPerSpecialStateSwitch;
|
||||
}
|
||||
|
||||
/**
|
||||
* @since 2.9
|
||||
*/
|
||||
public void setCasesPerSpecialStateSwitch(final String casesPerSpecialStateSwitch){
|
||||
int _parseInt = Integer.parseInt(casesPerSpecialStateSwitch);
|
||||
this.casesPerSpecialStateSwitch = _parseInt;
|
||||
}
|
||||
|
||||
private String readFileIntoString(String filename, Charset encoding) {
|
||||
try {
|
||||
String result = Files.asCharSource(new File(filename), encoding).read();
|
||||
return result;
|
||||
} catch (IOException e) {
|
||||
throw new WrappedException(e);
|
||||
}
|
||||
}
|
||||
|
||||
private void writeStringIntoFile(String filename, String content, Charset encoding) {
|
||||
try {
|
||||
Files.asCharSink(new File(filename), encoding).write(content);
|
||||
} catch (IOException e) {
|
||||
throw new WrappedException(e);
|
||||
}
|
||||
}
|
||||
|
||||
public static class NewlineNormalizer extends LineSeparatorHarmonizer {
|
||||
|
||||
private final String lineDelimiterToUse;
|
||||
|
||||
public NewlineNormalizer(String lineDelimiterToUse) {
|
||||
this.lineDelimiterToUse = lineDelimiterToUse;
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts the given content into a {@link CharSequence} that uses only the configured {@link #lineDelimiterToUse}
|
||||
*/
|
||||
public String normalizeLineDelimiters(CharSequence content) {
|
||||
return replaceLineSeparators(content, lineDelimiterToUse);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -219,9 +219,11 @@ public abstract class AbstractAntlrGeneratorFragment2 extends AbstractXtextGener
|
|||
}
|
||||
|
||||
protected void improveCodeQuality(final IXtextGeneratorFileSystemAccess fsa, final TypeReference lexer, final TypeReference parser) {
|
||||
String lexerContent = fsa.readTextFile(lexer.getJavaPath()).toString();
|
||||
lexerContent = this.codeQualityHelper.stripUnnecessaryComments(lexerContent, this.options);
|
||||
fsa.generateFile(lexer.getJavaPath(), lexerContent);
|
||||
this.improveLexerCodeQuality(fsa, lexer);
|
||||
this.improveParserCodeQuality(fsa, parser);
|
||||
}
|
||||
|
||||
protected void improveParserCodeQuality(final IXtextGeneratorFileSystemAccess fsa, final TypeReference parser) {
|
||||
String parserContent = fsa.readTextFile(parser.getJavaPath()).toString();
|
||||
parserContent = this.codeQualityHelper.stripUnnecessaryComments(parserContent, this.options);
|
||||
parserContent = this.codeQualityHelper.removeDuplicateBitsets(parserContent, this.options);
|
||||
|
|
@ -229,6 +231,12 @@ public abstract class AbstractAntlrGeneratorFragment2 extends AbstractXtextGener
|
|||
fsa.generateFile(parser.getJavaPath(), parserContent);
|
||||
}
|
||||
|
||||
protected void improveLexerCodeQuality(final IXtextGeneratorFileSystemAccess fsa, final TypeReference lexer) {
|
||||
String lexerContent = fsa.readTextFile(lexer.getJavaPath()).toString();
|
||||
lexerContent = this.codeQualityHelper.stripUnnecessaryComments(lexerContent, this.options);
|
||||
fsa.generateFile(lexer.getJavaPath(), lexerContent);
|
||||
}
|
||||
|
||||
protected void cleanupLexerTokensFile(final AntlrGrammar lexerGrammar, final KeywordHelper helper, final IXtextGeneratorFileSystemAccess fsa) {
|
||||
try {
|
||||
boolean _isBacktrackLexer = this.options.isBacktrackLexer();
|
||||
|
|
|
|||
Loading…
Reference in a new issue