From 6245f3e6e689117bcff80e12862767774651b474 Mon Sep 17 00:00:00 2001 From: Christian Dietrich Date: Fri, 8 Nov 2019 12:37:20 +0100 Subject: [PATCH 1/3] provided replacement for ExternalAntlrLexerFragment Signed-off-by: Christian Dietrich --- .../META-INF/MANIFEST.MF | 3 + .../antlr/ex/ExternalAntlrLexerFragment.java | 363 ++++++++++++++++++ 2 files changed, 366 insertions(+) create mode 100644 org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/ex/ExternalAntlrLexerFragment.java diff --git a/org.eclipse.xtext.xtext.generator/META-INF/MANIFEST.MF b/org.eclipse.xtext.xtext.generator/META-INF/MANIFEST.MF index ce370dd96..e8a22ef13 100644 --- a/org.eclipse.xtext.xtext.generator/META-INF/MANIFEST.MF +++ b/org.eclipse.xtext.xtext.generator/META-INF/MANIFEST.MF @@ -45,6 +45,9 @@ Export-Package: org.eclipse.xtext.xtext.generator, org.eclipse.xtext.xbase, org.eclipse.xtext.tests, org.eclipse.xtend.core", + org.eclipse.xtext.xtext.generator.parser.antlr.ex;x-friends:="org.eclipse.emf.mwe2.language, + org.eclipse.xtext.ui.codetemplates, + org.eclipse.xtext.ui.tests", org.eclipse.xtext.xtext.generator.parser.antlr.postProcessing;x-internal:=true, org.eclipse.xtext.xtext.generator.parser.antlr.splitting;x-friends:="org.eclipse.xtext.generator, org.eclipse.xtext.eclipse.tests, diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/ex/ExternalAntlrLexerFragment.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/ex/ExternalAntlrLexerFragment.java new file mode 100644 index 000000000..34c07b6e2 --- /dev/null +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/ex/ExternalAntlrLexerFragment.java @@ -0,0 +1,363 @@ +/******************************************************************************* + * Copyright (c) 2019 itemis AG (http://www.itemis.eu) and others. + * All rights reserved. This program and the accompanying materials + * are made available under the terms of the Eclipse Public License v1.0 + * which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v10.html + *******************************************************************************/ +package org.eclipse.xtext.xtext.generator.parser.antlr.ex; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import org.eclipse.emf.common.util.WrappedException; +import org.eclipse.xtend2.lib.StringConcatenationClient; +import org.eclipse.xtext.generator.LineSeparatorHarmonizer; +import org.eclipse.xtext.parser.antlr.Lexer; +import org.eclipse.xtext.util.Strings; +import org.eclipse.xtext.xtext.generator.AbstractXtextGeneratorFragment; +import org.eclipse.xtext.xtext.generator.CodeConfig; +import org.eclipse.xtext.xtext.generator.Issues; +import org.eclipse.xtext.xtext.generator.model.GuiceModuleAccess; +import org.eclipse.xtext.xtext.generator.model.IXtextGeneratorFileSystemAccess; +import org.eclipse.xtext.xtext.generator.parser.antlr.AntlrToolFacade; +import org.eclipse.xtext.xtext.generator.parser.antlr.postProcessing.SuppressWarningsProcessor; +import org.eclipse.xtext.xtext.generator.parser.antlr.splitting.AntlrLexerSplitter; +import org.eclipse.xtext.xtext.generator.parser.antlr.splitting.internal.LexerSpecialStateTransitionSplitter; + +import com.google.common.annotations.Beta; +import com.google.common.collect.Lists; +import com.google.common.io.Files; +import com.google.inject.Inject; + +/** + * @since 2.20 + * @author dietrich - Initial contribution and API + */ +@Beta +public class ExternalAntlrLexerFragment extends AbstractXtextGeneratorFragment { + + @Inject CodeConfig codeConfig; + + private String lexerGrammar; + + private boolean highlighting; + + private boolean runtime; + + private boolean contentAssist; + + private boolean classSplitting = false; + + private boolean specialStateSwitchSplitting = false; + + private int casesPerSpecialStateSwitch = LexerSpecialStateTransitionSplitter.CASES_PER_SPECIAL_STATE_SWITCH; + + private List antlrParams = Lists.newArrayList(); + + private String getLineDelimiter() { + return codeConfig.getLineDelimiter(); + } + + public void addAntlrParam(String param) { + antlrParams.add(param); + } + + public String[] getAntlrParams() { + List params = new ArrayList(antlrParams); + // setting the default conversion timeout to 100secs. + // There seem to be no practical situations where the NFA conversion would hang, + // so Terence suggested here [1] to remove the option all together + // [1] - http://antlr.1301665.n2.nabble.com/Xconversiontimeout-td5294411.html + if (!params.contains("-Xconversiontimeout")) { + params.add("-Xconversiontimeout"); + params.add("100000"); + } + String[] result = params.toArray(new String[params.size()]); + return result; + } + + private AntlrToolFacade antlrTool = new AntlrToolFacade(); + + public void setAntlrTool(AntlrToolFacade facade) { + this.antlrTool = facade; + } + + public AntlrToolFacade getAntlrTool() { + return antlrTool; + } + + @Override + public void generate() { + // i am not sure when to generate what + if (runtime) { + StringConcatenationClient binding = new StringConcatenationClient() { + @Override + protected void appendTo(TargetStringConcatenation target) { + target.append("binder.bind(" + Lexer.class.getName() + ".class)"+ + ".annotatedWith(com.google.inject.name.Names.named(" + + "org.eclipse.xtext.parser.antlr.LexerBindings.RUNTIME" + + ")).to(" + lexerGrammar +".class);"); + } + }; + new GuiceModuleAccess.BindingFactory().addConfiguredBinding("RuntimeLexer", binding).contributeTo(getLanguage().getRuntimeGenModule()); + } + if (highlighting) { + if (this.getProjectConfig().getEclipsePlugin().getRoot() != null) { + StringConcatenationClient binding = new StringConcatenationClient() { + @Override + protected void appendTo(TargetStringConcatenation target) { + target.append("binder.bind(" + Lexer.class.getName() + ".class)"+ + ".annotatedWith(com.google.inject.name.Names.named(" + + "org.eclipse.xtext.ide.LexerIdeBindings.HIGHLIGHTING" + + ")).to(" + lexerGrammar +".class);"); + } + }; + new GuiceModuleAccess.BindingFactory().addConfiguredBinding("HighlightingLexer", binding).contributeTo(getLanguage().getEclipsePluginGenModule()); + } + } + if (contentAssist) { + if (this.getProjectConfig().getGenericIde().getRoot() != null) { + StringConcatenationClient binding = new StringConcatenationClient() { + @Override + protected void appendTo(TargetStringConcatenation target) { + target.append("binder.bind(org.eclipse.xtext.ide.editor.contentassist.antlr.internal.Lexer.class)"+ + ".annotatedWith(com.google.inject.name.Names.named(" + + "org.eclipse.xtext.ide.LexerIdeBindings.CONTENT_ASSIST" + + ")).to(" + lexerGrammar +".class);"); + } + }; + new GuiceModuleAccess.BindingFactory().addConfiguredBinding("ContentAssistLexer", binding).contributeTo(getLanguage().getIdeGenModule()); + + } + if (this.getProjectConfig().getEclipsePlugin().getRoot() != null) { + StringConcatenationClient binding = new StringConcatenationClient() { + @Override + protected void appendTo(TargetStringConcatenation target) { + target.append("binder.bind(org.eclipse.xtext.ide.editor.contentassist.antlr.internal.Lexer.class)"+ + ".annotatedWith(com.google.inject.name.Names.named(" + + "org.eclipse.xtext.ide.LexerIdeBindings.CONTENT_ASSIST" + + ")).to(" + lexerGrammar +".class);"); + } + }; + new GuiceModuleAccess.BindingFactory().addConfiguredBinding("ContentAssistLexer", binding).contributeTo(getLanguage().getEclipsePluginGenModule()); + } + } + + + + + IXtextGeneratorFileSystemAccess srcGenFsa = this.getProjectConfig().getRuntime().getSrcGen(); + IXtextGeneratorFileSystemAccess srcFsa = this.getProjectConfig().getRuntime().getSrc(); + if (contentAssist || highlighting) { + if (this.getProjectConfig().getGenericIde().getRoot() != null) { + srcGenFsa = this.getProjectConfig().getGenericIde().getSrcGen(); + srcFsa = this.getProjectConfig().getGenericIde().getSrc(); + } else { + srcGenFsa = this.getProjectConfig().getEclipsePlugin().getSrcGen(); + srcFsa = this.getProjectConfig().getEclipsePlugin().getSrc(); + } + } + String srcGenPath = srcGenFsa.getPath(); + String srcPath = srcFsa.getPath(); + String grammarFile = srcPath + "/" + getLexerGrammar().replace('.', '/') + ".g"; + String generateTo = ""; + if (lexerGrammar.lastIndexOf('.') != -1) { + generateTo = lexerGrammar.substring(0, lexerGrammar.lastIndexOf('.')); + } + generateTo = srcGenPath + "/" + generateTo.replace('.', '/'); + addAntlrParam("-fo"); + addAntlrParam(generateTo); + final String encoding = codeConfig.getEncoding(); + getAntlrTool().runWithEncodingAndParams(grammarFile, encoding, getAntlrParams()); + Charset charset = Charset.forName(encoding); + String javaFile = srcGenPath+"/"+getLexerGrammar().replace('.', '/')+".java"; + splitLexerIfEnabled(javaFile, charset); + suppressWarningsImpl(javaFile, charset); + normalizeLineDelimiters(javaFile, charset); + normalizeTokens(javaFile, charset); + } + + /** + * @since 2.9 + */ + protected void splitLexerIfEnabled(String lexerJavaFile, Charset encoding) { + if (isClassSplitting()) { + String content = readFileIntoString(lexerJavaFile, encoding); + AntlrLexerSplitter splitter = new AntlrLexerSplitter(content); + splitter.setAllowDFAStaticClasses(false); + splitter.setCasesPerSpecialStateSwitch(casesPerSpecialStateSwitch); + writeStringIntoFile(lexerJavaFile, splitter.transform(), encoding); + } + } + + private void normalizeTokens(String grammarFileName, Charset encoding) { + String tokenFile = toTokenFileName(grammarFileName); + String content = readFileIntoString(tokenFile, encoding); + content = new NewlineNormalizer(getLineDelimiter()).normalizeLineDelimiters(content); + List splitted = Strings.split(content, getLineDelimiter()); + Collections.sort(splitted); + content = Strings.concat(getLineDelimiter(), splitted) + getLineDelimiter(); + writeStringIntoFile(tokenFile, content, encoding); + } + + private void normalizeLineDelimiters(String textFile, Charset encoding) { + String content = readFileIntoString(textFile, encoding); + content = new NewlineNormalizer(getLineDelimiter()) { + // Antlr tries to outsmart us by using a line length that depends on the system + // line delimiter when it splits a very long String (encoded DFA) into a + // string concatenation + // Here we join these lines again. + @Override + public String normalizeLineDelimiters(CharSequence content) { + String result = super.normalizeLineDelimiters(content); + result = result.replaceAll("\"\\+(\\r)?\\n\\s+\"", ""); + return result; + } + }.normalizeLineDelimiters(content); + writeStringIntoFile(textFile, content, encoding); + } + + private String toTokenFileName(String grammarFileName) { + return grammarFileName.replaceAll("\\.java$", ".tokens"); + } + + /** + * @deprecated use {@link #suppressWarningsImpl(String, Charset)} instead + */ + @Deprecated + protected void suppressWarningsImpl(String javaFile) { + suppressWarningsImpl(javaFile, Charset.defaultCharset()); + } + + /** + * @since 2.7 + */ + protected void suppressWarningsImpl(String javaFile, Charset encoding) { + String content = readFileIntoString(javaFile, encoding); + content = new SuppressWarningsProcessor().process(content); + writeStringIntoFile(javaFile, content, encoding); + } + + @Override + public void checkConfiguration(Issues issues) { + if (contentAssist && highlighting || runtime && highlighting || contentAssist && runtime) { + issues.addError("Only one of those flags is allowed: contentAssist, runtime, highlighting flag"); + } + } + + + public void setLexerGrammar(String lexerGrammar) { + this.lexerGrammar = lexerGrammar; + } + + public String getLexerGrammar() { + return lexerGrammar; + } + + public void setHighlighting(boolean highlighting) { + this.highlighting = highlighting; + } + + public boolean isHighlighting() { + return highlighting; + } + + public void setRuntime(boolean runtime) { + this.runtime = runtime; + } + + public boolean isRuntime() { + return runtime; + } + + public void setContentAssist(boolean contentAssist) { + this.contentAssist = contentAssist; + } + + public boolean isContentAssist() { + return contentAssist; + } + + /** + * @since 2.9 + */ + public boolean isClassSplitting() { + return classSplitting; + } + + /** + * @since 2.9 + */ + public void setClassSplitting(boolean value) { + this.classSplitting = value; + } + + /** + * @since 2.9 + */ + public boolean isSpecialStateSwitchSplitting() { + return specialStateSwitchSplitting; + } + + /** + * @since 2.9 + */ + public void setSpecialStateSwitchSplitting(boolean value) { + this.specialStateSwitchSplitting = value; + } + + /** + * @since 2.9 + */ + public int getCasesPerSpecialStateSwitch(){ + return casesPerSpecialStateSwitch; + } + + /** + * @since 2.9 + */ + public void setCasesPerSpecialStateSwitch(final String casesPerSpecialStateSwitch){ + int _parseInt = Integer.parseInt(casesPerSpecialStateSwitch); + this.casesPerSpecialStateSwitch = _parseInt; + } + + private String readFileIntoString(String filename, Charset encoding) { + try { + String result = Files.asCharSource(new File(filename), encoding).read(); + return result; + } catch (IOException e) { + throw new WrappedException(e); + } + } + + private void writeStringIntoFile(String filename, String content, Charset encoding) { + try { + Files.asCharSink(new File(filename), encoding).write(content); + } catch (IOException e) { + throw new WrappedException(e); + } + } + + public static class NewlineNormalizer extends LineSeparatorHarmonizer { + + private final String lineDelimiterToUse; + + public NewlineNormalizer(String lineDelimiterToUse) { + this.lineDelimiterToUse = lineDelimiterToUse; + } + + /** + * Converts the given content into a {@link CharSequence} that uses only the configured {@link #lineDelimiterToUse} + */ + public String normalizeLineDelimiters(CharSequence content) { + return replaceLineSeparators(content, lineDelimiterToUse); + } + + } + +} From 231b4ef862c7e6632087159dac329e1afcb02f2a Mon Sep 17 00:00:00 2001 From: Sebastian Zarnekow Date: Sun, 24 Nov 2019 15:20:39 +0100 Subject: [PATCH 2/3] Reuse parts of the AbstractAntlrGeneratorFragment2 --- .../AbstractAntlrGeneratorFragment2.xtend | 21 +- .../antlr/ex/ExternalAntlrLexerFragment.java | 226 ++---------------- .../AbstractAntlrGeneratorFragment2.java | 14 +- 3 files changed, 45 insertions(+), 216 deletions(-) diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/AbstractAntlrGeneratorFragment2.xtend b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/AbstractAntlrGeneratorFragment2.xtend index 5899ccfcf..ea006effc 100644 --- a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/AbstractAntlrGeneratorFragment2.xtend +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/AbstractAntlrGeneratorFragment2.xtend @@ -22,6 +22,7 @@ import org.eclipse.xtext.ParserRule import org.eclipse.xtext.UnorderedGroup import org.eclipse.xtext.generator.LineSeparatorHarmonizer import org.eclipse.xtext.util.Strings +import org.eclipse.xtext.xtext.generator.AbstractXtextGeneratorFragment import org.eclipse.xtext.xtext.generator.CodeConfig import org.eclipse.xtext.xtext.generator.Issues import org.eclipse.xtext.xtext.generator.model.IXtextGeneratorFileSystemAccess @@ -31,11 +32,10 @@ import org.eclipse.xtext.xtext.generator.parser.antlr.splitting.AntlrCodeQuality import org.eclipse.xtext.xtext.generator.parser.antlr.splitting.AntlrLexerSplitter import org.eclipse.xtext.xtext.generator.parser.antlr.splitting.AntlrParserSplitter import org.eclipse.xtext.xtext.generator.parser.antlr.splitting.BacktrackingGuardForUnorderedGroupsRemover +import org.eclipse.xtext.xtext.generator.parser.antlr.splitting.BacktrackingGuardRemover import org.eclipse.xtext.xtext.generator.parser.antlr.splitting.PartialClassExtractor import org.eclipse.xtext.xtext.generator.parser.antlr.splitting.SyntacticPredicateFixup import org.eclipse.xtext.xtext.generator.parser.antlr.splitting.UnorderedGroupsSplitter -import org.eclipse.xtext.xtext.generator.parser.antlr.splitting.BacktrackingGuardRemover -import org.eclipse.xtext.xtext.generator.AbstractXtextGeneratorFragment abstract class AbstractAntlrGeneratorFragment2 extends AbstractXtextGeneratorFragment { @Inject @Accessors(PROTECTED_GETTER) AntlrToolFacade antlrTool @@ -169,11 +169,12 @@ abstract class AbstractAntlrGeneratorFragment2 extends AbstractXtextGeneratorFra } } - def protected improveCodeQuality(IXtextGeneratorFileSystemAccess fsa, TypeReference lexer, TypeReference parser) { - var lexerContent = fsa.readTextFile(lexer.javaPath).toString - lexerContent = codeQualityHelper.stripUnnecessaryComments(lexerContent, options) - fsa.generateFile(lexer.javaPath, lexerContent) - + def protected void improveCodeQuality(IXtextGeneratorFileSystemAccess fsa, TypeReference lexer, TypeReference parser) { + improveLexerCodeQuality(fsa, lexer) + improveParserCodeQuality(fsa, parser) + } + + def protected void improveParserCodeQuality(IXtextGeneratorFileSystemAccess fsa, TypeReference parser) { var parserContent = fsa.readTextFile(parser.javaPath).toString parserContent = codeQualityHelper.stripUnnecessaryComments(parserContent, options) parserContent = codeQualityHelper.removeDuplicateBitsets(parserContent, options) @@ -181,6 +182,12 @@ abstract class AbstractAntlrGeneratorFragment2 extends AbstractXtextGeneratorFra fsa.generateFile(parser.javaPath, parserContent) } + def protected void improveLexerCodeQuality(IXtextGeneratorFileSystemAccess fsa, TypeReference lexer) { + var lexerContent = fsa.readTextFile(lexer.javaPath).toString + lexerContent = codeQualityHelper.stripUnnecessaryComments(lexerContent, options) + fsa.generateFile(lexer.javaPath, lexerContent) + } + def protected void cleanupLexerTokensFile(AntlrGrammar lexerGrammar, KeywordHelper helper, IXtextGeneratorFileSystemAccess fsa) { if (options.backtrackLexer) { val provider = createLexerTokensProvider(lexerGrammar, helper, fsa) diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/ex/ExternalAntlrLexerFragment.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/ex/ExternalAntlrLexerFragment.java index 34c07b6e2..5a328de45 100644 --- a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/ex/ExternalAntlrLexerFragment.java +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/ex/ExternalAntlrLexerFragment.java @@ -7,42 +7,23 @@ *******************************************************************************/ package org.eclipse.xtext.xtext.generator.parser.antlr.ex; -import java.io.File; -import java.io.IOException; -import java.nio.charset.Charset; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -import org.eclipse.emf.common.util.WrappedException; import org.eclipse.xtend2.lib.StringConcatenationClient; -import org.eclipse.xtext.generator.LineSeparatorHarmonizer; import org.eclipse.xtext.parser.antlr.Lexer; -import org.eclipse.xtext.util.Strings; -import org.eclipse.xtext.xtext.generator.AbstractXtextGeneratorFragment; -import org.eclipse.xtext.xtext.generator.CodeConfig; import org.eclipse.xtext.xtext.generator.Issues; import org.eclipse.xtext.xtext.generator.model.GuiceModuleAccess; import org.eclipse.xtext.xtext.generator.model.IXtextGeneratorFileSystemAccess; -import org.eclipse.xtext.xtext.generator.parser.antlr.AntlrToolFacade; -import org.eclipse.xtext.xtext.generator.parser.antlr.postProcessing.SuppressWarningsProcessor; -import org.eclipse.xtext.xtext.generator.parser.antlr.splitting.AntlrLexerSplitter; -import org.eclipse.xtext.xtext.generator.parser.antlr.splitting.internal.LexerSpecialStateTransitionSplitter; +import org.eclipse.xtext.xtext.generator.model.TypeReference; +import org.eclipse.xtext.xtext.generator.parser.antlr.AbstractAntlrGeneratorFragment2; import com.google.common.annotations.Beta; -import com.google.common.collect.Lists; -import com.google.common.io.Files; -import com.google.inject.Inject; /** * @since 2.20 * @author dietrich - Initial contribution and API */ @Beta -public class ExternalAntlrLexerFragment extends AbstractXtextGeneratorFragment { +public class ExternalAntlrLexerFragment extends AbstractAntlrGeneratorFragment2 { - @Inject CodeConfig codeConfig; - private String lexerGrammar; private boolean highlighting; @@ -51,48 +32,8 @@ public class ExternalAntlrLexerFragment extends AbstractXtextGeneratorFragment { private boolean contentAssist; - private boolean classSplitting = false; - - private boolean specialStateSwitchSplitting = false; - - private int casesPerSpecialStateSwitch = LexerSpecialStateTransitionSplitter.CASES_PER_SPECIAL_STATE_SWITCH; - - private List antlrParams = Lists.newArrayList(); - - private String getLineDelimiter() { - return codeConfig.getLineDelimiter(); - } - - public void addAntlrParam(String param) { - antlrParams.add(param); - } - - public String[] getAntlrParams() { - List params = new ArrayList(antlrParams); - // setting the default conversion timeout to 100secs. - // There seem to be no practical situations where the NFA conversion would hang, - // so Terence suggested here [1] to remove the option all together - // [1] - http://antlr.1301665.n2.nabble.com/Xconversiontimeout-td5294411.html - if (!params.contains("-Xconversiontimeout")) { - params.add("-Xconversiontimeout"); - params.add("100000"); - } - String[] result = params.toArray(new String[params.size()]); - return result; - } - - private AntlrToolFacade antlrTool = new AntlrToolFacade(); - - public void setAntlrTool(AntlrToolFacade facade) { - this.antlrTool = facade; - } - - public AntlrToolFacade getAntlrTool() { - return antlrTool; - } - @Override - public void generate() { + protected void doGenerate() { // i am not sure when to generate what if (runtime) { StringConcatenationClient binding = new StringConcatenationClient() { @@ -166,85 +107,35 @@ public class ExternalAntlrLexerFragment extends AbstractXtextGeneratorFragment { String srcPath = srcFsa.getPath(); String grammarFile = srcPath + "/" + getLexerGrammar().replace('.', '/') + ".g"; String generateTo = ""; - if (lexerGrammar.lastIndexOf('.') != -1) { - generateTo = lexerGrammar.substring(0, lexerGrammar.lastIndexOf('.')); + if (getLexerGrammar().lastIndexOf('.') != -1) { + generateTo = getLexerGrammar().substring(0, getLexerGrammar().lastIndexOf('.')); } generateTo = srcGenPath + "/" + generateTo.replace('.', '/'); addAntlrParam("-fo"); addAntlrParam(generateTo); - final String encoding = codeConfig.getEncoding(); + final String encoding = getCodeConfig().getEncoding(); getAntlrTool().runWithEncodingAndParams(grammarFile, encoding, getAntlrParams()); - Charset charset = Charset.forName(encoding); - String javaFile = srcGenPath+"/"+getLexerGrammar().replace('.', '/')+".java"; - splitLexerIfEnabled(javaFile, charset); - suppressWarningsImpl(javaFile, charset); - normalizeLineDelimiters(javaFile, charset); - normalizeTokens(javaFile, charset); + + TypeReference lexerType = new TypeReference(getLexerGrammar()); + splitParserAndLexerIfEnabled(srcGenFsa, null /* parser */, lexerType); + normalizeTokens(srcGenFsa, getLexerGrammar().replace('.', '/') + ".tokens"); + suppressWarnings(srcGenFsa, lexerType); + normalizeLineDelimiters(srcGenFsa, lexerType); } - /** - * @since 2.9 - */ - protected void splitLexerIfEnabled(String lexerJavaFile, Charset encoding) { - if (isClassSplitting()) { - String content = readFileIntoString(lexerJavaFile, encoding); - AntlrLexerSplitter splitter = new AntlrLexerSplitter(content); - splitter.setAllowDFAStaticClasses(false); - splitter.setCasesPerSpecialStateSwitch(casesPerSpecialStateSwitch); - writeStringIntoFile(lexerJavaFile, splitter.transform(), encoding); - } + @Override + protected void splitParserClassFile(IXtextGeneratorFileSystemAccess fsa, TypeReference parser) { + // no-op } - private void normalizeTokens(String grammarFileName, Charset encoding) { - String tokenFile = toTokenFileName(grammarFileName); - String content = readFileIntoString(tokenFile, encoding); - content = new NewlineNormalizer(getLineDelimiter()).normalizeLineDelimiters(content); - List splitted = Strings.split(content, getLineDelimiter()); - Collections.sort(splitted); - content = Strings.concat(getLineDelimiter(), splitted) + getLineDelimiter(); - writeStringIntoFile(tokenFile, content, encoding); + @Override + protected void improveParserCodeQuality(IXtextGeneratorFileSystemAccess fsa, TypeReference parser) { + // no-op } - private void normalizeLineDelimiters(String textFile, Charset encoding) { - String content = readFileIntoString(textFile, encoding); - content = new NewlineNormalizer(getLineDelimiter()) { - // Antlr tries to outsmart us by using a line length that depends on the system - // line delimiter when it splits a very long String (encoded DFA) into a - // string concatenation - // Here we join these lines again. - @Override - public String normalizeLineDelimiters(CharSequence content) { - String result = super.normalizeLineDelimiters(content); - result = result.replaceAll("\"\\+(\\r)?\\n\\s+\"", ""); - return result; - } - }.normalizeLineDelimiters(content); - writeStringIntoFile(textFile, content, encoding); - } - - private String toTokenFileName(String grammarFileName) { - return grammarFileName.replaceAll("\\.java$", ".tokens"); - } - - /** - * @deprecated use {@link #suppressWarningsImpl(String, Charset)} instead - */ - @Deprecated - protected void suppressWarningsImpl(String javaFile) { - suppressWarningsImpl(javaFile, Charset.defaultCharset()); - } - - /** - * @since 2.7 - */ - protected void suppressWarningsImpl(String javaFile, Charset encoding) { - String content = readFileIntoString(javaFile, encoding); - content = new SuppressWarningsProcessor().process(content); - writeStringIntoFile(javaFile, content, encoding); - } - @Override public void checkConfiguration(Issues issues) { + super.checkConfiguration(issues); if (contentAssist && highlighting || runtime && highlighting || contentAssist && runtime) { issues.addError("Only one of those flags is allowed: contentAssist, runtime, highlighting flag"); } @@ -283,81 +174,4 @@ public class ExternalAntlrLexerFragment extends AbstractXtextGeneratorFragment { return contentAssist; } - /** - * @since 2.9 - */ - public boolean isClassSplitting() { - return classSplitting; - } - - /** - * @since 2.9 - */ - public void setClassSplitting(boolean value) { - this.classSplitting = value; - } - - /** - * @since 2.9 - */ - public boolean isSpecialStateSwitchSplitting() { - return specialStateSwitchSplitting; - } - - /** - * @since 2.9 - */ - public void setSpecialStateSwitchSplitting(boolean value) { - this.specialStateSwitchSplitting = value; - } - - /** - * @since 2.9 - */ - public int getCasesPerSpecialStateSwitch(){ - return casesPerSpecialStateSwitch; - } - - /** - * @since 2.9 - */ - public void setCasesPerSpecialStateSwitch(final String casesPerSpecialStateSwitch){ - int _parseInt = Integer.parseInt(casesPerSpecialStateSwitch); - this.casesPerSpecialStateSwitch = _parseInt; - } - - private String readFileIntoString(String filename, Charset encoding) { - try { - String result = Files.asCharSource(new File(filename), encoding).read(); - return result; - } catch (IOException e) { - throw new WrappedException(e); - } - } - - private void writeStringIntoFile(String filename, String content, Charset encoding) { - try { - Files.asCharSink(new File(filename), encoding).write(content); - } catch (IOException e) { - throw new WrappedException(e); - } - } - - public static class NewlineNormalizer extends LineSeparatorHarmonizer { - - private final String lineDelimiterToUse; - - public NewlineNormalizer(String lineDelimiterToUse) { - this.lineDelimiterToUse = lineDelimiterToUse; - } - - /** - * Converts the given content into a {@link CharSequence} that uses only the configured {@link #lineDelimiterToUse} - */ - public String normalizeLineDelimiters(CharSequence content) { - return replaceLineSeparators(content, lineDelimiterToUse); - } - - } - } diff --git a/org.eclipse.xtext.xtext.generator/xtend-gen/org/eclipse/xtext/xtext/generator/parser/antlr/AbstractAntlrGeneratorFragment2.java b/org.eclipse.xtext.xtext.generator/xtend-gen/org/eclipse/xtext/xtext/generator/parser/antlr/AbstractAntlrGeneratorFragment2.java index d88f4d79b..d04b99048 100644 --- a/org.eclipse.xtext.xtext.generator/xtend-gen/org/eclipse/xtext/xtext/generator/parser/antlr/AbstractAntlrGeneratorFragment2.java +++ b/org.eclipse.xtext.xtext.generator/xtend-gen/org/eclipse/xtext/xtext/generator/parser/antlr/AbstractAntlrGeneratorFragment2.java @@ -219,9 +219,11 @@ public abstract class AbstractAntlrGeneratorFragment2 extends AbstractXtextGener } protected void improveCodeQuality(final IXtextGeneratorFileSystemAccess fsa, final TypeReference lexer, final TypeReference parser) { - String lexerContent = fsa.readTextFile(lexer.getJavaPath()).toString(); - lexerContent = this.codeQualityHelper.stripUnnecessaryComments(lexerContent, this.options); - fsa.generateFile(lexer.getJavaPath(), lexerContent); + this.improveLexerCodeQuality(fsa, lexer); + this.improveParserCodeQuality(fsa, parser); + } + + protected void improveParserCodeQuality(final IXtextGeneratorFileSystemAccess fsa, final TypeReference parser) { String parserContent = fsa.readTextFile(parser.getJavaPath()).toString(); parserContent = this.codeQualityHelper.stripUnnecessaryComments(parserContent, this.options); parserContent = this.codeQualityHelper.removeDuplicateBitsets(parserContent, this.options); @@ -229,6 +231,12 @@ public abstract class AbstractAntlrGeneratorFragment2 extends AbstractXtextGener fsa.generateFile(parser.getJavaPath(), parserContent); } + protected void improveLexerCodeQuality(final IXtextGeneratorFileSystemAccess fsa, final TypeReference lexer) { + String lexerContent = fsa.readTextFile(lexer.getJavaPath()).toString(); + lexerContent = this.codeQualityHelper.stripUnnecessaryComments(lexerContent, this.options); + fsa.generateFile(lexer.getJavaPath(), lexerContent); + } + protected void cleanupLexerTokensFile(final AntlrGrammar lexerGrammar, final KeywordHelper helper, final IXtextGeneratorFileSystemAccess fsa) { try { boolean _isBacktrackLexer = this.options.isBacktrackLexer(); From b78d337a9c4caa4fee333f5addaf75b613d18ad6 Mon Sep 17 00:00:00 2001 From: Christian Dietrich Date: Tue, 26 Nov 2019 10:25:26 +0100 Subject: [PATCH 3/3] review feedback Signed-off-by: Christian Dietrich --- .../generator/parser/antlr/ex/ExternalAntlrLexerFragment.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/ex/ExternalAntlrLexerFragment.java b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/ex/ExternalAntlrLexerFragment.java index 5a328de45..3567501c7 100644 --- a/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/ex/ExternalAntlrLexerFragment.java +++ b/org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/ex/ExternalAntlrLexerFragment.java @@ -18,6 +18,8 @@ import org.eclipse.xtext.xtext.generator.parser.antlr.AbstractAntlrGeneratorFrag import com.google.common.annotations.Beta; /** + * Use this fragment if you want to use an external lexer , e.g: in case the generated lexer has to be customized. + * * @since 2.20 * @author dietrich - Initial contribution and API */ @@ -34,7 +36,6 @@ public class ExternalAntlrLexerFragment extends AbstractAntlrGeneratorFragment2 @Override protected void doGenerate() { - // i am not sure when to generate what if (runtime) { StringConcatenationClient binding = new StringConcatenationClient() { @Override