diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/AbstractAntlrGeneratorFragment2.xtend b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/AbstractAntlrGeneratorFragment2.xtend index 5899ccfcf..ea006effc 100644 --- a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/AbstractAntlrGeneratorFragment2.xtend +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/AbstractAntlrGeneratorFragment2.xtend @@ -22,6 +22,7 @@ import org.eclipse.xtext.ParserRule import org.eclipse.xtext.UnorderedGroup import org.eclipse.xtext.generator.LineSeparatorHarmonizer import org.eclipse.xtext.util.Strings +import org.eclipse.xtext.xtext.generator.AbstractXtextGeneratorFragment import org.eclipse.xtext.xtext.generator.CodeConfig import org.eclipse.xtext.xtext.generator.Issues import org.eclipse.xtext.xtext.generator.model.IXtextGeneratorFileSystemAccess @@ -31,11 +32,10 @@ import org.eclipse.xtext.xtext.generator.parser.antlr.splitting.AntlrCodeQuality import org.eclipse.xtext.xtext.generator.parser.antlr.splitting.AntlrLexerSplitter import org.eclipse.xtext.xtext.generator.parser.antlr.splitting.AntlrParserSplitter import org.eclipse.xtext.xtext.generator.parser.antlr.splitting.BacktrackingGuardForUnorderedGroupsRemover +import org.eclipse.xtext.xtext.generator.parser.antlr.splitting.BacktrackingGuardRemover import org.eclipse.xtext.xtext.generator.parser.antlr.splitting.PartialClassExtractor import org.eclipse.xtext.xtext.generator.parser.antlr.splitting.SyntacticPredicateFixup import org.eclipse.xtext.xtext.generator.parser.antlr.splitting.UnorderedGroupsSplitter -import org.eclipse.xtext.xtext.generator.parser.antlr.splitting.BacktrackingGuardRemover -import org.eclipse.xtext.xtext.generator.AbstractXtextGeneratorFragment abstract class AbstractAntlrGeneratorFragment2 extends AbstractXtextGeneratorFragment { @Inject @Accessors(PROTECTED_GETTER) AntlrToolFacade antlrTool @@ -169,11 +169,12 @@ abstract class AbstractAntlrGeneratorFragment2 extends AbstractXtextGeneratorFra } } - def protected improveCodeQuality(IXtextGeneratorFileSystemAccess fsa, TypeReference lexer, TypeReference parser) { - var lexerContent = fsa.readTextFile(lexer.javaPath).toString - lexerContent = codeQualityHelper.stripUnnecessaryComments(lexerContent, options) - fsa.generateFile(lexer.javaPath, lexerContent) - + def protected void improveCodeQuality(IXtextGeneratorFileSystemAccess fsa, TypeReference lexer, TypeReference parser) { + improveLexerCodeQuality(fsa, lexer) + improveParserCodeQuality(fsa, parser) + } + + def protected void improveParserCodeQuality(IXtextGeneratorFileSystemAccess fsa, TypeReference parser) { var parserContent = fsa.readTextFile(parser.javaPath).toString parserContent = codeQualityHelper.stripUnnecessaryComments(parserContent, options) parserContent = codeQualityHelper.removeDuplicateBitsets(parserContent, options) @@ -181,6 +182,12 @@ abstract class AbstractAntlrGeneratorFragment2 extends AbstractXtextGeneratorFra fsa.generateFile(parser.javaPath, parserContent) } + def protected void improveLexerCodeQuality(IXtextGeneratorFileSystemAccess fsa, TypeReference lexer) { + var lexerContent = fsa.readTextFile(lexer.javaPath).toString + lexerContent = codeQualityHelper.stripUnnecessaryComments(lexerContent, options) + fsa.generateFile(lexer.javaPath, lexerContent) + } + def protected void cleanupLexerTokensFile(AntlrGrammar lexerGrammar, KeywordHelper helper, IXtextGeneratorFileSystemAccess fsa) { if (options.backtrackLexer) { val provider = createLexerTokensProvider(lexerGrammar, helper, fsa) diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/ex/ExternalAntlrLexerFragment.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/ex/ExternalAntlrLexerFragment.java index 34c07b6e2..5a328de45 100644 --- a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/ex/ExternalAntlrLexerFragment.java +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/ex/ExternalAntlrLexerFragment.java @@ -7,42 +7,23 @@ *******************************************************************************/ package org.eclipse.xtext.xtext.generator.parser.antlr.ex; -import java.io.File; -import java.io.IOException; -import java.nio.charset.Charset; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -import org.eclipse.emf.common.util.WrappedException; import org.eclipse.xtend2.lib.StringConcatenationClient; -import org.eclipse.xtext.generator.LineSeparatorHarmonizer; import org.eclipse.xtext.parser.antlr.Lexer; -import org.eclipse.xtext.util.Strings; -import org.eclipse.xtext.xtext.generator.AbstractXtextGeneratorFragment; -import org.eclipse.xtext.xtext.generator.CodeConfig; import org.eclipse.xtext.xtext.generator.Issues; import org.eclipse.xtext.xtext.generator.model.GuiceModuleAccess; import org.eclipse.xtext.xtext.generator.model.IXtextGeneratorFileSystemAccess; -import org.eclipse.xtext.xtext.generator.parser.antlr.AntlrToolFacade; -import org.eclipse.xtext.xtext.generator.parser.antlr.postProcessing.SuppressWarningsProcessor; -import org.eclipse.xtext.xtext.generator.parser.antlr.splitting.AntlrLexerSplitter; -import org.eclipse.xtext.xtext.generator.parser.antlr.splitting.internal.LexerSpecialStateTransitionSplitter; +import org.eclipse.xtext.xtext.generator.model.TypeReference; +import org.eclipse.xtext.xtext.generator.parser.antlr.AbstractAntlrGeneratorFragment2; import com.google.common.annotations.Beta; -import com.google.common.collect.Lists; -import com.google.common.io.Files; -import com.google.inject.Inject; /** * @since 2.20 * @author dietrich - Initial contribution and API */ @Beta -public class ExternalAntlrLexerFragment extends AbstractXtextGeneratorFragment { +public class ExternalAntlrLexerFragment extends AbstractAntlrGeneratorFragment2 { - @Inject CodeConfig codeConfig; - private String lexerGrammar; private boolean highlighting; @@ -51,48 +32,8 @@ public class ExternalAntlrLexerFragment extends AbstractXtextGeneratorFragment { private boolean contentAssist; - private boolean classSplitting = false; - - private boolean specialStateSwitchSplitting = false; - - private int casesPerSpecialStateSwitch = LexerSpecialStateTransitionSplitter.CASES_PER_SPECIAL_STATE_SWITCH; - - private List antlrParams = Lists.newArrayList(); - - private String getLineDelimiter() { - return codeConfig.getLineDelimiter(); - } - - public void addAntlrParam(String param) { - antlrParams.add(param); - } - - public String[] getAntlrParams() { - List params = new ArrayList(antlrParams); - // setting the default conversion timeout to 100secs. - // There seem to be no practical situations where the NFA conversion would hang, - // so Terence suggested here [1] to remove the option all together - // [1] - http://antlr.1301665.n2.nabble.com/Xconversiontimeout-td5294411.html - if (!params.contains("-Xconversiontimeout")) { - params.add("-Xconversiontimeout"); - params.add("100000"); - } - String[] result = params.toArray(new String[params.size()]); - return result; - } - - private AntlrToolFacade antlrTool = new AntlrToolFacade(); - - public void setAntlrTool(AntlrToolFacade facade) { - this.antlrTool = facade; - } - - public AntlrToolFacade getAntlrTool() { - return antlrTool; - } - @Override - public void generate() { + protected void doGenerate() { // i am not sure when to generate what if (runtime) { StringConcatenationClient binding = new StringConcatenationClient() { @@ -166,85 +107,35 @@ public class ExternalAntlrLexerFragment extends AbstractXtextGeneratorFragment { String srcPath = srcFsa.getPath(); String grammarFile = srcPath + "/" + getLexerGrammar().replace('.', '/') + ".g"; String generateTo = ""; - if (lexerGrammar.lastIndexOf('.') != -1) { - generateTo = lexerGrammar.substring(0, lexerGrammar.lastIndexOf('.')); + if (getLexerGrammar().lastIndexOf('.') != -1) { + generateTo = getLexerGrammar().substring(0, getLexerGrammar().lastIndexOf('.')); } generateTo = srcGenPath + "/" + generateTo.replace('.', '/'); addAntlrParam("-fo"); addAntlrParam(generateTo); - final String encoding = codeConfig.getEncoding(); + final String encoding = getCodeConfig().getEncoding(); getAntlrTool().runWithEncodingAndParams(grammarFile, encoding, getAntlrParams()); - Charset charset = Charset.forName(encoding); - String javaFile = srcGenPath+"/"+getLexerGrammar().replace('.', '/')+".java"; - splitLexerIfEnabled(javaFile, charset); - suppressWarningsImpl(javaFile, charset); - normalizeLineDelimiters(javaFile, charset); - normalizeTokens(javaFile, charset); + + TypeReference lexerType = new TypeReference(getLexerGrammar()); + splitParserAndLexerIfEnabled(srcGenFsa, null /* parser */, lexerType); + normalizeTokens(srcGenFsa, getLexerGrammar().replace('.', '/') + ".tokens"); + suppressWarnings(srcGenFsa, lexerType); + normalizeLineDelimiters(srcGenFsa, lexerType); } - /** - * @since 2.9 - */ - protected void splitLexerIfEnabled(String lexerJavaFile, Charset encoding) { - if (isClassSplitting()) { - String content = readFileIntoString(lexerJavaFile, encoding); - AntlrLexerSplitter splitter = new AntlrLexerSplitter(content); - splitter.setAllowDFAStaticClasses(false); - splitter.setCasesPerSpecialStateSwitch(casesPerSpecialStateSwitch); - writeStringIntoFile(lexerJavaFile, splitter.transform(), encoding); - } + @Override + protected void splitParserClassFile(IXtextGeneratorFileSystemAccess fsa, TypeReference parser) { + // no-op } - private void normalizeTokens(String grammarFileName, Charset encoding) { - String tokenFile = toTokenFileName(grammarFileName); - String content = readFileIntoString(tokenFile, encoding); - content = new NewlineNormalizer(getLineDelimiter()).normalizeLineDelimiters(content); - List splitted = Strings.split(content, getLineDelimiter()); - Collections.sort(splitted); - content = Strings.concat(getLineDelimiter(), splitted) + getLineDelimiter(); - writeStringIntoFile(tokenFile, content, encoding); + @Override + protected void improveParserCodeQuality(IXtextGeneratorFileSystemAccess fsa, TypeReference parser) { + // no-op } - private void normalizeLineDelimiters(String textFile, Charset encoding) { - String content = readFileIntoString(textFile, encoding); - content = new NewlineNormalizer(getLineDelimiter()) { - // Antlr tries to outsmart us by using a line length that depends on the system - // line delimiter when it splits a very long String (encoded DFA) into a - // string concatenation - // Here we join these lines again. - @Override - public String normalizeLineDelimiters(CharSequence content) { - String result = super.normalizeLineDelimiters(content); - result = result.replaceAll("\"\\+(\\r)?\\n\\s+\"", ""); - return result; - } - }.normalizeLineDelimiters(content); - writeStringIntoFile(textFile, content, encoding); - } - - private String toTokenFileName(String grammarFileName) { - return grammarFileName.replaceAll("\\.java$", ".tokens"); - } - - /** - * @deprecated use {@link #suppressWarningsImpl(String, Charset)} instead - */ - @Deprecated - protected void suppressWarningsImpl(String javaFile) { - suppressWarningsImpl(javaFile, Charset.defaultCharset()); - } - - /** - * @since 2.7 - */ - protected void suppressWarningsImpl(String javaFile, Charset encoding) { - String content = readFileIntoString(javaFile, encoding); - content = new SuppressWarningsProcessor().process(content); - writeStringIntoFile(javaFile, content, encoding); - } - @Override public void checkConfiguration(Issues issues) { + super.checkConfiguration(issues); if (contentAssist && highlighting || runtime && highlighting || contentAssist && runtime) { issues.addError("Only one of those flags is allowed: contentAssist, runtime, highlighting flag"); } @@ -283,81 +174,4 @@ public class ExternalAntlrLexerFragment extends AbstractXtextGeneratorFragment { return contentAssist; } - /** - * @since 2.9 - */ - public boolean isClassSplitting() { - return classSplitting; - } - - /** - * @since 2.9 - */ - public void setClassSplitting(boolean value) { - this.classSplitting = value; - } - - /** - * @since 2.9 - */ - public boolean isSpecialStateSwitchSplitting() { - return specialStateSwitchSplitting; - } - - /** - * @since 2.9 - */ - public void setSpecialStateSwitchSplitting(boolean value) { - this.specialStateSwitchSplitting = value; - } - - /** - * @since 2.9 - */ - public int getCasesPerSpecialStateSwitch(){ - return casesPerSpecialStateSwitch; - } - - /** - * @since 2.9 - */ - public void setCasesPerSpecialStateSwitch(final String casesPerSpecialStateSwitch){ - int _parseInt = Integer.parseInt(casesPerSpecialStateSwitch); - this.casesPerSpecialStateSwitch = _parseInt; - } - - private String readFileIntoString(String filename, Charset encoding) { - try { - String result = Files.asCharSource(new File(filename), encoding).read(); - return result; - } catch (IOException e) { - throw new WrappedException(e); - } - } - - private void writeStringIntoFile(String filename, String content, Charset encoding) { - try { - Files.asCharSink(new File(filename), encoding).write(content); - } catch (IOException e) { - throw new WrappedException(e); - } - } - - public static class NewlineNormalizer extends LineSeparatorHarmonizer { - - private final String lineDelimiterToUse; - - public NewlineNormalizer(String lineDelimiterToUse) { - this.lineDelimiterToUse = lineDelimiterToUse; - } - - /** - * Converts the given content into a {@link CharSequence} that uses only the configured {@link #lineDelimiterToUse} - */ - public String normalizeLineDelimiters(CharSequence content) { - return replaceLineSeparators(content, lineDelimiterToUse); - } - - } - } diff --git a/org.eclipse.xtext.xtext.generator/xtend-gen/org/eclipse/xtext/xtext/generator/parser/antlr/AbstractAntlrGeneratorFragment2.java b/org.eclipse.xtext.xtext.generator/xtend-gen/org/eclipse/xtext/xtext/generator/parser/antlr/AbstractAntlrGeneratorFragment2.java index d88f4d79b..d04b99048 100644 --- a/org.eclipse.xtext.xtext.generator/xtend-gen/org/eclipse/xtext/xtext/generator/parser/antlr/AbstractAntlrGeneratorFragment2.java +++ b/org.eclipse.xtext.xtext.generator/xtend-gen/org/eclipse/xtext/xtext/generator/parser/antlr/AbstractAntlrGeneratorFragment2.java @@ -219,9 +219,11 @@ public abstract class AbstractAntlrGeneratorFragment2 extends AbstractXtextGener } protected void improveCodeQuality(final IXtextGeneratorFileSystemAccess fsa, final TypeReference lexer, final TypeReference parser) { - String lexerContent = fsa.readTextFile(lexer.getJavaPath()).toString(); - lexerContent = this.codeQualityHelper.stripUnnecessaryComments(lexerContent, this.options); - fsa.generateFile(lexer.getJavaPath(), lexerContent); + this.improveLexerCodeQuality(fsa, lexer); + this.improveParserCodeQuality(fsa, parser); + } + + protected void improveParserCodeQuality(final IXtextGeneratorFileSystemAccess fsa, final TypeReference parser) { String parserContent = fsa.readTextFile(parser.getJavaPath()).toString(); parserContent = this.codeQualityHelper.stripUnnecessaryComments(parserContent, this.options); parserContent = this.codeQualityHelper.removeDuplicateBitsets(parserContent, this.options); @@ -229,6 +231,12 @@ public abstract class AbstractAntlrGeneratorFragment2 extends AbstractXtextGener fsa.generateFile(parser.getJavaPath(), parserContent); } + protected void improveLexerCodeQuality(final IXtextGeneratorFileSystemAccess fsa, final TypeReference lexer) { + String lexerContent = fsa.readTextFile(lexer.getJavaPath()).toString(); + lexerContent = this.codeQualityHelper.stripUnnecessaryComments(lexerContent, this.options); + fsa.generateFile(lexer.getJavaPath(), lexerContent); + } + protected void cleanupLexerTokensFile(final AntlrGrammar lexerGrammar, final KeywordHelper helper, final IXtextGeneratorFileSystemAccess fsa) { try { boolean _isBacktrackLexer = this.options.isBacktrackLexer();