sigmasternchen/xtext-core
1
0
Fork 0
mirror of https://github.com/sigmasternchen/xtext-core synced 2025-03-15 16:28:56 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #1287 from eclipse/cd_externalLexer

Browse source 
provide a new generator infrastructure external lexer fragment
This commit is contained in:
Christian Dietrich 2019-11-27 11:13:02 +01:00 committed by GitHub
commit cd774217e1
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 206 additions and 10 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
org.eclipse.xtext.xtext.generator/META-INF/MANIFEST.MF
View file

@ -45,6 +45,9 @@ Export-Package: org.eclipse.xtext.xtext.generator,
org.eclipse.xtext.xbase,
org.eclipse.xtext.tests,
org.eclipse.xtend.core",
org.eclipse.xtext.xtext.generator.parser.antlr.ex;x-friends:="org.eclipse.emf.mwe2.language,
org.eclipse.xtext.ui.codetemplates,
org.eclipse.xtext.ui.tests",
org.eclipse.xtext.xtext.generator.parser.antlr.postProcessing;x-internal:=true,
org.eclipse.xtext.xtext.generator.parser.antlr.splitting;x-friends:="org.eclipse.xtext.generator,
org.eclipse.xtext.eclipse.tests,

21
org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/AbstractAntlrGeneratorFragment2.xtend
View file

@ -22,6 +22,7 @@ import org.eclipse.xtext.ParserRule
import org.eclipse.xtext.UnorderedGroup
import org.eclipse.xtext.generator.LineSeparatorHarmonizer
import org.eclipse.xtext.util.Strings
import org.eclipse.xtext.xtext.generator.AbstractXtextGeneratorFragment
import org.eclipse.xtext.xtext.generator.CodeConfig
import org.eclipse.xtext.xtext.generator.Issues
import org.eclipse.xtext.xtext.generator.model.IXtextGeneratorFileSystemAccess
@ -31,11 +32,10 @@ import org.eclipse.xtext.xtext.generator.parser.antlr.splitting.AntlrCodeQuality
import org.eclipse.xtext.xtext.generator.parser.antlr.splitting.AntlrLexerSplitter
import org.eclipse.xtext.xtext.generator.parser.antlr.splitting.AntlrParserSplitter
import org.eclipse.xtext.xtext.generator.parser.antlr.splitting.BacktrackingGuardForUnorderedGroupsRemover
import org.eclipse.xtext.xtext.generator.parser.antlr.splitting.BacktrackingGuardRemover
import org.eclipse.xtext.xtext.generator.parser.antlr.splitting.PartialClassExtractor
import org.eclipse.xtext.xtext.generator.parser.antlr.splitting.SyntacticPredicateFixup
import org.eclipse.xtext.xtext.generator.parser.antlr.splitting.UnorderedGroupsSplitter
import org.eclipse.xtext.xtext.generator.parser.antlr.splitting.BacktrackingGuardRemover
import org.eclipse.xtext.xtext.generator.AbstractXtextGeneratorFragment
abstract class AbstractAntlrGeneratorFragment2 extends AbstractXtextGeneratorFragment {
@Inject @Accessors(PROTECTED_GETTER) AntlrToolFacade antlrTool
@ -169,11 +169,12 @@ abstract class AbstractAntlrGeneratorFragment2 extends AbstractXtextGeneratorFra
}
}
def protected improveCodeQuality(IXtextGeneratorFileSystemAccess fsa, TypeReference lexer, TypeReference parser) {
var lexerContent = fsa.readTextFile(lexer.javaPath).toString
lexerContent = codeQualityHelper.stripUnnecessaryComments(lexerContent, options)
fsa.generateFile(lexer.javaPath, lexerContent)
def protected void improveCodeQuality(IXtextGeneratorFileSystemAccess fsa, TypeReference lexer, TypeReference parser) {
improveLexerCodeQuality(fsa, lexer)
improveParserCodeQuality(fsa, parser)
}
def protected void improveParserCodeQuality(IXtextGeneratorFileSystemAccess fsa, TypeReference parser) {
var parserContent = fsa.readTextFile(parser.javaPath).toString
parserContent = codeQualityHelper.stripUnnecessaryComments(parserContent, options)
parserContent = codeQualityHelper.removeDuplicateBitsets(parserContent, options)
@ -181,6 +182,12 @@ abstract class AbstractAntlrGeneratorFragment2 extends AbstractXtextGeneratorFra
fsa.generateFile(parser.javaPath, parserContent)
}
def protected void improveLexerCodeQuality(IXtextGeneratorFileSystemAccess fsa, TypeReference lexer) {
var lexerContent = fsa.readTextFile(lexer.javaPath).toString
lexerContent = codeQualityHelper.stripUnnecessaryComments(lexerContent, options)
fsa.generateFile(lexer.javaPath, lexerContent)
}
def protected void cleanupLexerTokensFile(AntlrGrammar lexerGrammar, KeywordHelper helper, IXtextGeneratorFileSystemAccess fsa) {
if (options.backtrackLexer) {
val provider = createLexerTokensProvider(lexerGrammar, helper, fsa)

178
org.eclipse.xtext.xtext.generator/src/org/eclipse/xtext/xtext/generator/parser/antlr/ex/ExternalAntlrLexerFragment.java Normal file
View file

@ -0,0 +1,178 @@
/*******************************************************************************
* Copyright (c) 2019 itemis AG (http://www.itemis.eu) and others.
* All rights reserved. This program and the accompanying materials
* are made available under the terms of the Eclipse Public License v1.0
* which accompanies this distribution, and is available at
* http://www.eclipse.org/legal/epl-v10.html
*******************************************************************************/
package org.eclipse.xtext.xtext.generator.parser.antlr.ex;
import org.eclipse.xtend2.lib.StringConcatenationClient;
import org.eclipse.xtext.parser.antlr.Lexer;
import org.eclipse.xtext.xtext.generator.Issues;
import org.eclipse.xtext.xtext.generator.model.GuiceModuleAccess;
import org.eclipse.xtext.xtext.generator.model.IXtextGeneratorFileSystemAccess;
import org.eclipse.xtext.xtext.generator.model.TypeReference;
import org.eclipse.xtext.xtext.generator.parser.antlr.AbstractAntlrGeneratorFragment2;
import com.google.common.annotations.Beta;
/**
* Use this fragment if you want to use an external lexer , e.g: in case the generated lexer has to be customized.
*
* @since 2.20
* @author dietrich - Initial contribution and API
*/
@Beta
public class ExternalAntlrLexerFragment extends AbstractAntlrGeneratorFragment2 {
private String lexerGrammar;
private boolean highlighting;
private boolean runtime;
private boolean contentAssist;
@Override
protected void doGenerate() {
if (runtime) {
StringConcatenationClient binding = new StringConcatenationClient() {
@Override
protected void appendTo(TargetStringConcatenation target) {
target.append("binder.bind(" + Lexer.class.getName() + ".class)"+
".annotatedWith(com.google.inject.name.Names.named(" +
"org.eclipse.xtext.parser.antlr.LexerBindings.RUNTIME" +
")).to(" + lexerGrammar +".class);");
}
};
new GuiceModuleAccess.BindingFactory().addConfiguredBinding("RuntimeLexer", binding).contributeTo(getLanguage().getRuntimeGenModule());
}
if (highlighting) {
if (this.getProjectConfig().getEclipsePlugin().getRoot() != null) {
StringConcatenationClient binding = new StringConcatenationClient() {
@Override
protected void appendTo(TargetStringConcatenation target) {
target.append("binder.bind(" + Lexer.class.getName() + ".class)"+
".annotatedWith(com.google.inject.name.Names.named(" +
"org.eclipse.xtext.ide.LexerIdeBindings.HIGHLIGHTING" +
")).to(" + lexerGrammar +".class);");
}
};
new GuiceModuleAccess.BindingFactory().addConfiguredBinding("HighlightingLexer", binding).contributeTo(getLanguage().getEclipsePluginGenModule());
}
}
if (contentAssist) {
if (this.getProjectConfig().getGenericIde().getRoot() != null) {
StringConcatenationClient binding = new StringConcatenationClient() {
@Override
protected void appendTo(TargetStringConcatenation target) {
target.append("binder.bind(org.eclipse.xtext.ide.editor.contentassist.antlr.internal.Lexer.class)"+
".annotatedWith(com.google.inject.name.Names.named(" +
"org.eclipse.xtext.ide.LexerIdeBindings.CONTENT_ASSIST" +
")).to(" + lexerGrammar +".class);");
}
};
new GuiceModuleAccess.BindingFactory().addConfiguredBinding("ContentAssistLexer", binding).contributeTo(getLanguage().getIdeGenModule());
}
if (this.getProjectConfig().getEclipsePlugin().getRoot() != null) {
StringConcatenationClient binding = new StringConcatenationClient() {
@Override
protected void appendTo(TargetStringConcatenation target) {
target.append("binder.bind(org.eclipse.xtext.ide.editor.contentassist.antlr.internal.Lexer.class)"+
".annotatedWith(com.google.inject.name.Names.named(" +
"org.eclipse.xtext.ide.LexerIdeBindings.CONTENT_ASSIST" +
")).to(" + lexerGrammar +".class);");
}
};
new GuiceModuleAccess.BindingFactory().addConfiguredBinding("ContentAssistLexer", binding).contributeTo(getLanguage().getEclipsePluginGenModule());
}
}
IXtextGeneratorFileSystemAccess srcGenFsa = this.getProjectConfig().getRuntime().getSrcGen();
IXtextGeneratorFileSystemAccess srcFsa = this.getProjectConfig().getRuntime().getSrc();
if (contentAssist || highlighting) {
if (this.getProjectConfig().getGenericIde().getRoot() != null) {
srcGenFsa = this.getProjectConfig().getGenericIde().getSrcGen();
srcFsa = this.getProjectConfig().getGenericIde().getSrc();
} else {
srcGenFsa = this.getProjectConfig().getEclipsePlugin().getSrcGen();
srcFsa = this.getProjectConfig().getEclipsePlugin().getSrc();
}
}
String srcGenPath = srcGenFsa.getPath();
String srcPath = srcFsa.getPath();
String grammarFile = srcPath + "/" + getLexerGrammar().replace('.', '/') + ".g";
String generateTo = "";
if (getLexerGrammar().lastIndexOf('.') != -1) {
generateTo = getLexerGrammar().substring(0, getLexerGrammar().lastIndexOf('.'));
}
generateTo = srcGenPath + "/" + generateTo.replace('.', '/');
addAntlrParam("-fo");
addAntlrParam(generateTo);
final String encoding = getCodeConfig().getEncoding();
getAntlrTool().runWithEncodingAndParams(grammarFile, encoding, getAntlrParams());
TypeReference lexerType = new TypeReference(getLexerGrammar());
splitParserAndLexerIfEnabled(srcGenFsa, null /* parser */, lexerType);
normalizeTokens(srcGenFsa, getLexerGrammar().replace('.', '/') + ".tokens");
suppressWarnings(srcGenFsa, lexerType);
normalizeLineDelimiters(srcGenFsa, lexerType);
}
@Override
protected void splitParserClassFile(IXtextGeneratorFileSystemAccess fsa, TypeReference parser) {
// no-op
}
@Override
protected void improveParserCodeQuality(IXtextGeneratorFileSystemAccess fsa, TypeReference parser) {
// no-op
}
@Override
public void checkConfiguration(Issues issues) {
super.checkConfiguration(issues);
if (contentAssist && highlighting || runtime && highlighting || contentAssist && runtime) {
issues.addError("Only one of those flags is allowed: contentAssist, runtime, highlighting flag");
}
}
public void setLexerGrammar(String lexerGrammar) {
this.lexerGrammar = lexerGrammar;
}
public String getLexerGrammar() {
return lexerGrammar;
}
public void setHighlighting(boolean highlighting) {
this.highlighting = highlighting;
}
public boolean isHighlighting() {
return highlighting;
}
public void setRuntime(boolean runtime) {
this.runtime = runtime;
}
public boolean isRuntime() {
return runtime;
}
public void setContentAssist(boolean contentAssist) {
this.contentAssist = contentAssist;
}
public boolean isContentAssist() {
return contentAssist;
}
}

14
org.eclipse.xtext.xtext.generator/xtend-gen/org/eclipse/xtext/xtext/generator/parser/antlr/AbstractAntlrGeneratorFragment2.java
View file

@ -219,9 +219,11 @@ public abstract class AbstractAntlrGeneratorFragment2 extends AbstractXtextGener
}
protected void improveCodeQuality(final IXtextGeneratorFileSystemAccess fsa, final TypeReference lexer, final TypeReference parser) {
String lexerContent = fsa.readTextFile(lexer.getJavaPath()).toString();
lexerContent = this.codeQualityHelper.stripUnnecessaryComments(lexerContent, this.options);
fsa.generateFile(lexer.getJavaPath(), lexerContent);
this.improveLexerCodeQuality(fsa, lexer);
this.improveParserCodeQuality(fsa, parser);
}
protected void improveParserCodeQuality(final IXtextGeneratorFileSystemAccess fsa, final TypeReference parser) {
String parserContent = fsa.readTextFile(parser.getJavaPath()).toString();
parserContent = this.codeQualityHelper.stripUnnecessaryComments(parserContent, this.options);
parserContent = this.codeQualityHelper.removeDuplicateBitsets(parserContent, this.options);
@ -229,6 +231,12 @@ public abstract class AbstractAntlrGeneratorFragment2 extends AbstractXtextGener
fsa.generateFile(parser.getJavaPath(), parserContent);
}
protected void improveLexerCodeQuality(final IXtextGeneratorFileSystemAccess fsa, final TypeReference lexer) {
String lexerContent = fsa.readTextFile(lexer.getJavaPath()).toString();
lexerContent = this.codeQualityHelper.stripUnnecessaryComments(lexerContent, this.options);
fsa.generateFile(lexer.getJavaPath(), lexerContent);
}
protected void cleanupLexerTokensFile(final AntlrGrammar lexerGrammar, final KeywordHelper helper, final IXtextGeneratorFileSystemAccess fsa) {
try {
boolean _isBacktrackLexer = this.options.isBacktrackLexer();