added tests for hoisting specific renders

tests predicate rendering in alternatives, in non-trivial cardinalities,
and setup block rendering. in both the production grammar and content
assist grammar.

added debug grammar generators for production grammar and content assist
grammar which use DebugGrammarNaming to avoid null-ptr because of
missing adapters in ecore objects. (existing AntlrDebugGrammarGenerator
can't be used because setup blocks are handled by AntlrGrammarGenerator
and AntlrContentAssistGrammarGenerator respectively, not
AbstractAntlrGrammarGenerator.)

fixed small details in rendered output (removed guard of rule comment
and trimmed setup block).
This commit is contained in:
overflowerror 2022-01-27 20:37:07 +01:00
parent 07845a3590
commit 00ff0407b0
8 changed files with 2423 additions and 5 deletions
org.eclipse.xtext.tests
src/org/eclipse/xtext/xtext/generator/hoisting
xtend-gen/org/eclipse/xtext/xtext/generator/hoisting
org.eclipse.xtext.xtext.generator

View file

@ -0,0 +1,750 @@
/*******************************************************************************
* Copyright (c) 2022 itemis AG (http://www.itemis.eu) and others.
* This program and the accompanying materials are made available under the
* terms of the Eclipse Public License 2.0 which is available at
* http://www.eclipse.org/legal/epl-2.0.
*
* SPDX-License-Identifier: EPL-2.0
*******************************************************************************/
package org.eclipse.xtext.xtext.generator.hoisting
import org.eclipse.xtext.tests.AbstractXtextTests
import org.eclipse.xtext.XtextStandaloneSetup
import org.junit.Test
import org.eclipse.xtext.Grammar
import org.eclipse.xtext.xtext.generator.DefaultGeneratorModule
import com.google.inject.Guice
import org.eclipse.xtext.xtext.generator.parser.antlr.AntlrOptions
import org.eclipse.xtext.generator.InMemoryFileSystemAccess
import org.eclipse.xtext.xtext.generator.model.IXtextGeneratorFileSystemAccess
import com.google.inject.Injector
import org.eclipse.xtext.xtext.generator.parser.antlr.AntlrDebugProductionGrammarGenerator
import org.eclipse.xtext.xtext.generator.parser.antlr.AntlrDebugContentAssistGrammarGenerator
/**
* @author overflow - Initial contribution and API
*/
class HoistingGeneratorTest extends AbstractXtextTests {
override void setUp() throws Exception {
super.setUp()
with(XtextStandaloneSetup)
}
@Test
def void testRenderingOfGuardConditionInAlternativesInProductionGrammar() {
'''
grammar com.foo.bar
import "http://www.eclipse.org/emf/2002/Ecore" as ecore
generate myPack 'http://mypack'
S: $$ p0 $$?=> 'a'
| $$ p1 $$?=> 'b';
'''.assertTranslatesToProductionGrammar('''
grammar DebugInternalbar;
options {
}
@lexer::header {
package com.foo.parser.antlr.internal;
// Hack: Use our own Lexer superclass by means of import.
// Currently there is no other way to specify the superclass for the lexer.
import org.eclipse.xtext.parser.antlr.Lexer;
}
@parser::header {
package com.foo.parser.antlr.internal;
}
@parser::members {
private barGrammarAccess grammarAccess;
public InternalbarParser(TokenStream input, barGrammarAccess grammarAccess) {
this(input);
this.grammarAccess = grammarAccess;
registerRules(grammarAccess.getGrammar());
}
@Override
protected String getFirstRuleName() {
return "S";
}
@Override
protected barGrammarAccess getGrammarAccess() {
return grammarAccess;
}
// no init block
}
@rulecatch {
catch (RecognitionException re) {
recover(input,re);
appendSkippedTokens();
}
}
// Entry rule entryRuleS
entryRuleS returns [EObject current=null]:
{ newCompositeNode(grammarAccess.getSRule()); }
iv_ruleS=ruleS
{ $current=$iv_ruleS.current; }
EOF;
// Rule S
ruleS returns [EObject current=null]
@init {
enterRule();
}
@after {
leaveRule();
}:
(
{(p0)}?=>
(
(
)
otherlv_1='a'
{
newLeafNode(otherlv_1, grammarAccess.getSAccess().getAKeyword_0_1());
}
)
|
{(p1)}?=>
(
(
)
otherlv_3='b'
{
newLeafNode(otherlv_3, grammarAccess.getSAccess().getBKeyword_1_1());
}
)
)
;
''')
}
@Test
def void testRenderingOfGuardConditionInNonTrivialCardinalitityInProductionGrammar() {
'''
grammar com.foo.bar
import "http://www.eclipse.org/emf/2002/Ecore" as ecore
generate myPack 'http://mypack'
S: ($$ p0 $$?=> 'a')*;
'''.assertTranslatesToProductionGrammar('''
grammar DebugInternalbar;
options {
}
@lexer::header {
package com.foo.parser.antlr.internal;
// Hack: Use our own Lexer superclass by means of import.
// Currently there is no other way to specify the superclass for the lexer.
import org.eclipse.xtext.parser.antlr.Lexer;
}
@parser::header {
package com.foo.parser.antlr.internal;
}
@parser::members {
private barGrammarAccess grammarAccess;
public InternalbarParser(TokenStream input, barGrammarAccess grammarAccess) {
this(input);
this.grammarAccess = grammarAccess;
registerRules(grammarAccess.getGrammar());
}
@Override
protected String getFirstRuleName() {
return "S";
}
@Override
protected barGrammarAccess getGrammarAccess() {
return grammarAccess;
}
// no init block
}
@rulecatch {
catch (RecognitionException re) {
recover(input,re);
appendSkippedTokens();
}
}
// Entry rule entryRuleS
entryRuleS returns [EObject current=null]:
{ newCompositeNode(grammarAccess.getSRule()); }
iv_ruleS=ruleS
{ $current=$iv_ruleS.current; }
EOF;
// Rule S
ruleS returns [EObject current=null]
@init {
enterRule();
}
@after {
leaveRule();
}:
(
{(p0)}?=>
(
)
otherlv_1='a'
{
newLeafNode(otherlv_1, grammarAccess.getSAccess().getAKeyword_1());
}
)*
;
''')
}
@Test
def void testRenderingOfSetupBlockInProductionGrammar() {
'''
grammar com.foo.bar
import "http://www.eclipse.org/emf/2002/Ecore" as ecore
generate myPack 'http://mypack'
setup $$ s0 $$
S: 'a';
'''.assertTranslatesToProductionGrammar('''
grammar DebugInternalbar;
options {
}
@lexer::header {
package com.foo.parser.antlr.internal;
// Hack: Use our own Lexer superclass by means of import.
// Currently there is no other way to specify the superclass for the lexer.
import org.eclipse.xtext.parser.antlr.Lexer;
}
@parser::header {
package com.foo.parser.antlr.internal;
}
@parser::members {
private barGrammarAccess grammarAccess;
public InternalbarParser(TokenStream input, barGrammarAccess grammarAccess) {
this(input);
this.grammarAccess = grammarAccess;
registerRules(grammarAccess.getGrammar());
}
@Override
protected String getFirstRuleName() {
return "S";
}
@Override
protected barGrammarAccess getGrammarAccess() {
return grammarAccess;
}
// init block
s0
}
@rulecatch {
catch (RecognitionException re) {
recover(input,re);
appendSkippedTokens();
}
}
// Entry rule entryRuleS
entryRuleS returns [String current=null]:
{ newCompositeNode(grammarAccess.getSRule()); }
iv_ruleS=ruleS
{ $current=$iv_ruleS.current.getText(); }
EOF;
// Rule S
ruleS returns [AntlrDatatypeRuleToken current=new AntlrDatatypeRuleToken()]
@init {
enterRule();
}
@after {
leaveRule();
}:
kw='a'
{
$current.merge(kw);
newLeafNode(kw, grammarAccess.getSAccess().getAKeyword());
}
;
''')
}
@Test
def void testRenderingOfGuardConditionInAlternativesInContentAssistGrammar() {
'''
grammar com.foo.bar
import "http://www.eclipse.org/emf/2002/Ecore" as ecore
generate myPack 'http://mypack'
S: $$ p0 $$?=> 'a'
| $$ p1 $$?=> 'b';
'''.assertTranslatesToContentAssistGrammar('''
grammar DebugInternalbar;
options {
}
@lexer::header {
package com.foo.parser.antlr.internal;
// Hack: Use our own Lexer superclass by means of import.
// Currently there is no other way to specify the superclass for the lexer.
import org.eclipse.xtext.parser.antlr.Lexer;
}
@parser::header {
package com.foo.parser.antlr.internal;
}
@parser::members {
private barGrammarAccess grammarAccess;
public void setGrammarAccess(barGrammarAccess grammarAccess) {
this.grammarAccess = grammarAccess;
}
@Override
protected Grammar getGrammar() {
return grammarAccess.getGrammar();
}
@Override
protected String getValueForTokenName(String tokenName) {
return tokenName;
}
// no init block
}
// Entry rule entryRuleS
entryRuleS
:
{ before(grammarAccess.getSRule()); }
ruleS
{ after(grammarAccess.getSRule()); }
EOF
;
// Rule S
ruleS
@init {
int stackSize = keepStackSize();
}
:
(
{ before(grammarAccess.getSAccess().getAlternatives()); }
(
rule__S__Alternatives
)
{ after(grammarAccess.getSAccess().getAlternatives()); }
)
;
finally {
restoreStackSize(stackSize);
}
rule__S__Alternatives
@init {
int stackSize = keepStackSize();
}
:
{(p0)}?=>(
{ before(grammarAccess.getSAccess().getGroup_0()); }
(
rule__S__Group_0__0
)
{ after(grammarAccess.getSAccess().getGroup_0()); }
)
|
{(p1)}?=>(
{ before(grammarAccess.getSAccess().getGroup_1()); }
(
rule__S__Group_1__0
)
{ after(grammarAccess.getSAccess().getGroup_1()); }
)
;
finally {
restoreStackSize(stackSize);
}
rule__S__Group_0__0
@init {
int stackSize = keepStackSize();
}
:
rule__S__Group_0__0__Impl
rule__S__Group_0__1
;
finally {
restoreStackSize(stackSize);
}
rule__S__Group_0__0__Impl
@init {
int stackSize = keepStackSize();
}
:
(
{ before(grammarAccess.getSAccess().getGatedSemanticPredicate_0_0()); }
(
)
{ after(grammarAccess.getSAccess().getGatedSemanticPredicate_0_0()); }
)
;
finally {
restoreStackSize(stackSize);
}
rule__S__Group_0__1
@init {
int stackSize = keepStackSize();
}
:
rule__S__Group_0__1__Impl
;
finally {
restoreStackSize(stackSize);
}
rule__S__Group_0__1__Impl
@init {
int stackSize = keepStackSize();
}
:
(
{ before(grammarAccess.getSAccess().getAKeyword_0_1()); }
'a'
{ after(grammarAccess.getSAccess().getAKeyword_0_1()); }
)
;
finally {
restoreStackSize(stackSize);
}
rule__S__Group_1__0
@init {
int stackSize = keepStackSize();
}
:
rule__S__Group_1__0__Impl
rule__S__Group_1__1
;
finally {
restoreStackSize(stackSize);
}
rule__S__Group_1__0__Impl
@init {
int stackSize = keepStackSize();
}
:
(
{ before(grammarAccess.getSAccess().getGatedSemanticPredicate_1_0()); }
(
)
{ after(grammarAccess.getSAccess().getGatedSemanticPredicate_1_0()); }
)
;
finally {
restoreStackSize(stackSize);
}
rule__S__Group_1__1
@init {
int stackSize = keepStackSize();
}
:
rule__S__Group_1__1__Impl
;
finally {
restoreStackSize(stackSize);
}
rule__S__Group_1__1__Impl
@init {
int stackSize = keepStackSize();
}
:
(
{ before(grammarAccess.getSAccess().getBKeyword_1_1()); }
'b'
{ after(grammarAccess.getSAccess().getBKeyword_1_1()); }
)
;
finally {
restoreStackSize(stackSize);
}
''')
}
@Test
def void testRenderingOfGuardConditionInNonTrivialCardinalitityInContentAssistGrammar() {
'''
grammar com.foo.bar
import "http://www.eclipse.org/emf/2002/Ecore" as ecore
generate myPack 'http://mypack'
S: ($$ p0 $$?=> 'a')*;
'''.assertTranslatesToContentAssistGrammar('''
grammar DebugInternalbar;
options {
}
@lexer::header {
package com.foo.parser.antlr.internal;
// Hack: Use our own Lexer superclass by means of import.
// Currently there is no other way to specify the superclass for the lexer.
import org.eclipse.xtext.parser.antlr.Lexer;
}
@parser::header {
package com.foo.parser.antlr.internal;
}
@parser::members {
private barGrammarAccess grammarAccess;
public void setGrammarAccess(barGrammarAccess grammarAccess) {
this.grammarAccess = grammarAccess;
}
@Override
protected Grammar getGrammar() {
return grammarAccess.getGrammar();
}
@Override
protected String getValueForTokenName(String tokenName) {
return tokenName;
}
// no init block
}
// Entry rule entryRuleS
entryRuleS
:
{ before(grammarAccess.getSRule()); }
ruleS
{ after(grammarAccess.getSRule()); }
EOF
;
// Rule S
ruleS
@init {
int stackSize = keepStackSize();
}
:
(
{ before(grammarAccess.getSAccess().getGroup()); }
(
{(p0)}?=>
rule__S__Group__0
)
*
{ after(grammarAccess.getSAccess().getGroup()); }
)
;
finally {
restoreStackSize(stackSize);
}
rule__S__Group__0
@init {
int stackSize = keepStackSize();
}
:
rule__S__Group__0__Impl
rule__S__Group__1
;
finally {
restoreStackSize(stackSize);
}
rule__S__Group__0__Impl
@init {
int stackSize = keepStackSize();
}
:
(
{ before(grammarAccess.getSAccess().getGatedSemanticPredicate_0()); }
(
)
{ after(grammarAccess.getSAccess().getGatedSemanticPredicate_0()); }
)
;
finally {
restoreStackSize(stackSize);
}
rule__S__Group__1
@init {
int stackSize = keepStackSize();
}
:
rule__S__Group__1__Impl
;
finally {
restoreStackSize(stackSize);
}
rule__S__Group__1__Impl
@init {
int stackSize = keepStackSize();
}
:
(
{ before(grammarAccess.getSAccess().getAKeyword_1()); }
'a'
{ after(grammarAccess.getSAccess().getAKeyword_1()); }
)
;
finally {
restoreStackSize(stackSize);
}
''')
}
@Test
def void testRenderingOfSetupBlockInContentAssistGrammar() {
'''
grammar com.foo.bar
import "http://www.eclipse.org/emf/2002/Ecore" as ecore
generate myPack 'http://mypack'
setup $$ s0 $$
S: 'a';
'''.assertTranslatesToContentAssistGrammar('''
grammar DebugInternalbar;
options {
}
@lexer::header {
package com.foo.parser.antlr.internal;
// Hack: Use our own Lexer superclass by means of import.
// Currently there is no other way to specify the superclass for the lexer.
import org.eclipse.xtext.parser.antlr.Lexer;
}
@parser::header {
package com.foo.parser.antlr.internal;
}
@parser::members {
private barGrammarAccess grammarAccess;
public void setGrammarAccess(barGrammarAccess grammarAccess) {
this.grammarAccess = grammarAccess;
}
@Override
protected Grammar getGrammar() {
return grammarAccess.getGrammar();
}
@Override
protected String getValueForTokenName(String tokenName) {
return tokenName;
}
// init block
s0
}
// Entry rule entryRuleS
entryRuleS
:
{ before(grammarAccess.getSRule()); }
ruleS
{ after(grammarAccess.getSRule()); }
EOF
;
// Rule S
ruleS
@init {
int stackSize = keepStackSize();
}
:
(
{ before(grammarAccess.getSAccess().getAKeyword()); }
'a'
{ after(grammarAccess.getSAccess().getAKeyword()); }
)
;
finally {
restoreStackSize(stackSize);
}
''')
}
protected def void assertTranslatesToProductionGrammar(CharSequence xtextGrammar, String expectedDebugGrammar) {
val grammar = super.getModel(xtextGrammar.toString) as Grammar
val injector = Guice.createInjector(new DefaultGeneratorModule)
val inMem = new InMemFSA
val options = new AntlrOptions
injector.getInstance(AntlrDebugProductionGrammarGenerator).generate(grammar, options, inMem)
assertEquals(expectedDebugGrammar, inMem.allFiles.values.head.toString)
}
protected def void assertTranslatesToContentAssistGrammar(CharSequence xtextGrammar, String expectedDebugGrammar) {
val grammar = super.getModel(xtextGrammar.toString) as Grammar
val injector = Guice.createInjector(new DefaultGeneratorModule)
val inMem = new InMemFSA
val options = new AntlrOptions
injector.getInstance(AntlrDebugContentAssistGrammarGenerator).generate(grammar, options, inMem)
assertEquals(expectedDebugGrammar, inMem.allFiles.values.head.toString)
}
static class InMemFSA extends InMemoryFileSystemAccess implements IXtextGeneratorFileSystemAccess {
override getPath() {
"./"
}
override isOverwrite() {
true
}
override initialize(Injector injector) {
injector.injectMembers(this)
}
}
}

View file

@ -293,7 +293,7 @@ abstract class AbstractAntlrGrammarGenerator {
protected def String compileEBNF(AbstractRule it, AntlrOptions options) '''
// Rule «originalElement.name»
«IF it instanceof ParserRule»
«IF it instanceof ParserRule && originalGrammar.isDebug»
// Guard: «findGuardForRule.renderDescription»
«ENDIF»
«ruleName»«compileInit(options)»:

View file

@ -0,0 +1,28 @@
/**
* Copyright (c) 2015, 2020 itemis AG (http://www.itemis.eu) and others.
* This program and the accompanying materials are made available under the
* terms of the Eclipse Public License 2.0 which is available at
* http://www.eclipse.org/legal/epl-2.0.
*
* SPDX-License-Identifier: EPL-2.0
*/
package org.eclipse.xtext.xtext.generator.parser.antlr;
import org.eclipse.xtext.Grammar;
import com.google.inject.Inject;
public class AntlrDebugContentAssistGrammarGenerator extends AntlrContentAssistGrammarGenerator {
@Inject
private DebugGrammarNaming naming;
@Override
protected GrammarNaming getGrammarNaming() {
return naming;
}
@Override
protected String compileParserImports(Grammar it, AntlrOptions options) {
return "";
}
}

View file

@ -0,0 +1,28 @@
/**
* Copyright (c) 2015, 2020 itemis AG (http://www.itemis.eu) and others.
* This program and the accompanying materials are made available under the
* terms of the Eclipse Public License 2.0 which is available at
* http://www.eclipse.org/legal/epl-2.0.
*
* SPDX-License-Identifier: EPL-2.0
*/
package org.eclipse.xtext.xtext.generator.parser.antlr;
import org.eclipse.xtext.Grammar;
import com.google.inject.Inject;
public class AntlrDebugProductionGrammarGenerator extends AntlrGrammarGenerator {
@Inject
private DebugGrammarNaming naming;
@Override
protected GrammarNaming getGrammarNaming() {
return naming;
}
@Override
protected String compileParserImports(Grammar it, AntlrOptions options) {
return "";
}
}

View file

@ -17,7 +17,7 @@ import org.eclipse.xtext.xtext.generator.parser.antlr.hoisting.guards.Guard;
public class JavaCodeUtils {
static public String getSource(JavaCode code) {
String source = code.getSource();
return source.substring(2, source.length() - 2);
return source.substring(2, source.length() - 2).trim();
}
static public String formatCodeForGrammar(String code) {

View file

@ -29,7 +29,7 @@ public class PredicateGuard implements HoistingGuard {
@Override
public String render() {
return "(" + JavaCodeUtils.getSource(element.getCode()).trim() + ")";
return "(" + JavaCodeUtils.getSource(element.getCode()) + ")";
}
@Override
@ -40,7 +40,7 @@ public class PredicateGuard implements HoistingGuard {
@Override
public String toString() {
return "PredicateGuard (\n" +
"\t" + JavaCodeUtils.getSource(element.getCode()).trim() + "\n" +
"\t" + JavaCodeUtils.getSource(element.getCode()) + "\n" +
")\n";
}

View file

@ -641,7 +641,7 @@ public abstract class AbstractAntlrGrammarGenerator {
_builder.append(_name);
_builder.newLineIfNotEmpty();
{
if ((it instanceof ParserRule)) {
if (((it instanceof ParserRule) && this.originalGrammar.isDebug())) {
_builder.append("// Guard: ");
String _renderDescription = this._hoistingProcessor.findGuardForRule(it).renderDescription();
_builder.append(_renderDescription);