| Trouble migrating grammar from oAW to TMF Xtext [message #55843] |
Tue, 07 July 2009 01:53  |
Eclipse User |
|
|
|
A grammar built for analyzing an existing general purpose language
(bearing a resemblance to COBOL) that worked in oAW Xtext is presenting an
intractable problem in migrating to TMF Xtext. I was successful in
cleaning up all of the "Decision can match input such as "'XXX'" using
multiple alternatives" messages except for a couple emitted when ANTL
processes one of the grammar rules.
ANTLR3 produces the same message in ANTLRworks, except that when I try to
remove code specific to Xtext so that I can work with the grammar in
ANTLRworks, the problem goes away. It's pretty time consuming to take out
one bit of code at a time, so I'm hoping for some guidance in locating the
problem.
The code below is extracted from the Xtext-generated file. I added in EOF
and RULE_WS so I could check that the parser generated by ANTLR works (in
a trivial case). With the Xtext-generated ANTLR source, two warnings are
emitted portions of ruleVariable - at the decision for 'POS' and at the
decision for 'LENGTH'
[22:34:39] warning(200): InternalPdl.g:2115:2: Decision can match input
such as "'POS'" using multiple alternatives: 1, 2
As a result, alternative(s) 2 were disabled for that input
[22:34:39] warning(200): InternalPdl.g:2140:4: Decision can match input
such as "'LENGTH'" using multiple alternatives: 1, 2
As a result, alternative(s) 2 were disabled for that input
The full grammar Xtext generated is around 4500 lines and has 250
terminals (based on T250 being the last one in the __.g file). (Note that
more RULE_WS would be needed to parse the actual language successfully in
ANTLRworks.)
grammar Foo;
ruleQS :
((
ruleLiteral
|
ruleVariable
) EOF
);
ruleLiteral :
(
RULE_STRING
|
RULE_INT
);
ruleVariable :
(
ruleValue
( 'POS'
(
ruleBracketExpression
))?
('LENGTH'
(
ruleBracketExpression
))?);
ruleValue:
(
ruleSystemValue
|
ruleDataValue
|
ruleQSBuild
);
ruleDataValue :
(
ruleSymbol
(
ruleMemoryStructureAbbrev
)?(
ruleBracketExpression
)?( RULE_WS 'IN'
(RULE_WS 'FORM'
|'SFA'
)?(RULE_WS
ruleSymbol
))?);
ruleQSBuild :
('QSBUILD'
(
ruleSymbol
)'='
'('
(
ruleQS
)(','
(
ruleQS
))*')'
);
ruleBracketExpression :
('['
(
ruleExpression
)']'
);
ruleExpression :
(
ruleTerm
(
ruleOperator
ruleExpression
)?);
ruleTerm :
(
ruleQS
|
ruleParenthesizedExpression
);
ruleParenthesizedExpression :
('('
(
ruleExpression
)')'
);
ruleSystemValue :
((
ruleSimpleSysSym
)
|
ruleSysSymQS
|
ruleSysSymName
|
ruleSysSymInName
|
ruleSysSymIndexed
|
ruleListIndex
|
ruleListLookup
);
ruleListIndex :
((
ruleListKeyword
)(
ruleSymbol
)(
ruleBracketExpression
));
ruleListLookup :
((
ruleListlookupKeyword
)(
ruleSymbol
)'USING'
(
ruleQS
));
ruleSysSymQS :
(('DAYOFWEEK'
|'DAYOFYEAR'
|'DAYSINMONTH'
|'DAYSINYEAR'
|'LENGTHOF'
|'NEXTBUSDAY'
|'SIZEOF'
|'SQLLENGTH'
|'SYSITEMSINSET'
|'SYSPARAM'
)
ruleQS
);
ruleSysSymName :
(('LASTINDEXOF'
|'LISTINDEX'
|'RECORDSIN'
|'SQLCURSOR'
)(
ruleSymbol
));
ruleSysSymInName :
(('SYSITEMNO'
|'SYSRECNO'
)'IN'
(
ruleSymbol
));
ruleSysSymIndexed :
(('SYSERRORLINE'
|'SYSMESSAGE'
)(
ruleBracketExpression
));
ruleSymbol:
(RULE_ID
|
ruleDynamicSymbol
);
ruleDynamicSymbol :
(('%'
RULE_ID
)
|(('%'
)?'^' ruleQS
'.'
));
ruleMemoryStructureAbbrev :
(( '#X'
)
|( '#C'
)
|( '#S'
));
ruleListKeyword :
(( 'LIST'
)
|( 'TABLE'
));
ruleListlookupKeyword :
(( 'LISTLOOKUP'
)
|( 'TABLELOOKUP'
));
ruleSimpleSysSym :
(( 'SYSTIME'
)
|( 'SYSEXITKEY'
)
|( 'SYSFKFNKEY'
)
|( 'FIELDEXITBYTE'
)
|( 'SYSRAWDATETIME'
)
|( 'PREVFKPROCEDURE'
));
ruleOperator :
(( '+'
)
|( '-'
)
|( '/'
)
|( '*'
));
RULE_ML_COMMENT : '{*' ( options {greedy=false;} : . )*'*}';
RULE_ID : ('a'..'z'|'A'..'Z'|'_') ('a'..'z'|'A'..'Z'|'_'|'0'..'9')*;
RULE_INT : ('0'..'9')+;
RULE_STRING : ('"' ('\\'
('b'|'t'|'n'|'f'|'r'|'"'|'\''|'\\')|~(('\\'|'"')))* '"'|'\'' ('\\'
('b'|'t'|'n'|'f'|'r'|'"'|'\''|'\\')|~(('\\'|'\'')))* '\'');
RULE_SL_COMMENT : '//' ~(('\n'|'\r'))* ('\r'? '\n')?;
RULE_WS : (' '|'\t'|'\r'|'\n')+;
RULE_ANY_OTHER : .;
------------------------file.xtext-------------------------- ---------------
grammar xx.Foo with org.eclipse.xtext.common.Terminals
generate Foo "http://www.example.org/foo"
/*
enum MemoryStructureAbbrev:
Stack="#X" | Common="#C" | Scratchpad="#S" ;
QS:
( Literal | Variable ) ;
Literal:
toss=STRING | toss1=INT ( "." toss2=INT )? ;
Variable:
val=Value ( "POS" pos=BracketExpression )? ("LENGTH"
len=BracketExpression )? ;
Value:
SystemValue | DataValue | QSBuild ;
DataValue:
sym=Symbol toss=MemoryStructureAbbrev? (index=BracketExpression)? ( "IN"
( "FORM" | "SFA" )? struct=Symbol )? ;
QSBuild:
"QSBUILD" name=Symbol "=" "(" first=QS ( "," list+=QS )* ")" ;
BracketExpression:
"[" exp=Expression "]" ;
Expression:
left=Term ( op=Operator right=Expression )? ;
Term:
QS | ParenthesizedExpression ;
ParenthesizedExpression:
"(" exp=Expression ")" ;
SystemValue:
( sim=SimpleSysSym | SysSymQS | SysSymName | SysSymInName | SysSymIndexed
| ListIndex | ListLookup ) ;
ListIndex:
toss=ListKeyword toss1=Symbol toss2=BracketExpression ;
enum ListKeyword:
List = "LIST" | Table = "TABLE" ;
ListLookup:
toss=ListlookupKeyword toss1=Symbol "USING" toss2=QS ;
enum ListlookupKeyword:
ListLookup = "LISTLOOKUP" | TableLookup="TABLELOOKUP" ;
enum SimpleSysSym:
// SYSDIR = "SYSDIR" | SYSEOL = "SYSEOL" | SYSVOL = "SYSVOL" |
//SQLAREA = "SQLAREA" | SQLCODE = "SQLCODE" | SYSDATE = "SYSDATE" |
// SYSDBMS = "SYSDBMS" | SYSNODE = "SYSNODE" | SYSPATH = "SYSPATH" |
SYSTIME = "SYSTIME" |
//SYSINDEX = "SYSINDEX" | SYSLEVEL = "SYSLEVEL" | SYSLINENO = "SYSLINENO"
| SYSPAGENO = "SYSPAGENO" |
SYSEXITKEY = "SYSEXITKEY" | SYSFKFNKEY = "SYSFKFNKEY" | //SYSEXITBYTE =
"SYSEXITBYTE" | SYSUSERNAME = "SYSUSERNAME" |
//FIELDEXITKEY = "FIELDEXITKEY" | SYSCURRFIELD = "SYSCURRFIELD" |
SYSFKEXITKEY = "SYSFKEXITKEY" |
//SYSLONGLIMIT = "SYSLONGLIMIT" |
FIELDEXITBYTE = "FIELDEXITBYTE" | // SYSBREAKLEVEL = "SYSBREAKLEVEL" |
SYSSIGNONNAME = "SYSSIGNONNAME" |
//CURRFUNCKEYSET = "CURRFUNCKEYSET" |
SYSRAWDATETIME = "SYSRAWDATETIME" | PREVFKPROCEDURE = "PREVFKPROCEDURE"
//|
//SYSUSERINITIALS = "SYSUSERINITIALS" | SYSCURRFIELDNAME =
"SYSCURRFIELDNAME" | SQLMESSAGE = "SQLMESSAGE"
; // The generated parser exceeds the Java class size limit if the system
symbols are enumerated.
SysSymQS:
( "DAYOFWEEK" | "DAYOFYEAR" | "DAYSINMONTH" | "DAYSINYEAR" | "LENGTHOF" |
"NEXTBUSDAY" | "SIZEOF" | "SQLLENGTH" | "SYSITEMSINSET" | "SYSPARAM" ) QS ;
SysSymName:
( "LASTINDEXOF" | "LISTINDEX" | "RECORDSIN" | "SQLCURSOR" ) sym=Symbol ;
SysSymInName:
( "SYSITEMNO" | "SYSRECNO" ) "IN" sym=Symbol ;
SysSymIndexed:
( "SYSERRORLINE" | "SYSMESSAGE" ) index=BracketExpression ;
Symbol:
ID | DynamicSymbol ;
DynamicSymbol:
(("%" ID) | ("%"? "^" QS "." ));
enum Operator:
Plus="+" | Dash="-" | Whack="/" | Splat="*" ;
terminal ML_COMMENT:
'{*' -> '*}' ;
terminal ID:
('a'..'z'|'A'..'Z'|'_') ('a'..'z'|'A'..'Z'|'_'|'0'..'9')* ;
|
|
|
| Re: Trouble migrating grammar from oAW to TMF Xtext [message #55871 is a reply to message #55843] |
Tue, 07 July 2009 03:04  |
| Eclipse User |
|
|
|
Hi John,
the problem with your grammar is the QS rule combined with the rules
Variable, Value, SystemValue and ListLookup.
Kind of simplified a QS may be a Variable which in turn starts with a
Value.
A Value may be a SystemValue which may be a ListLookup. This leads to
the fact, that a ListLookup is sort of a Value. The ListLookup ends with
a QS, meaning it may end with a Variable (which starts with a Value -
and so on).
To summerize this in one line, we have a situation that can be expressed as:
Variable == Variable 'POS'? 'LENGTH'?
which is kind of equal to
Variable == Value 'POS'? 'LENGTH'? 'POS'? 'LENGTH'?
At this point it is not possible to decide whether a POS keyword in your
input matches the first or the second POS keyword in your grammar.
To overcome this situation please activate backtracking for your
grammar. To do so replace the following fragment in your mwe-file
<fragment class="org.eclipse.xtext.generator.AntlrDelegatingFragment" />
and use this one instead:
<fragment class="de.itemis.xtext.antlr.XtextAntlrGeneratorFragment">
<options backtrack="true"/>
</fragment>
Btw: Backtracking was enabled by default in oAW Xtext.
Hope that helps,
Sebastian
Am 07.07.2009 7:53 Uhr, schrieb John Bito:
> A grammar built for analyzing an existing general purpose language
> (bearing a resemblance to COBOL) that worked in oAW Xtext is presenting
> an intractable problem in migrating to TMF Xtext. I was successful in
> cleaning up all of the "Decision can match input such as "'XXX'" using
> multiple alternatives" messages except for a couple emitted when ANTL
> processes one of the grammar rules.
>
> ANTLR3 produces the same message in ANTLRworks, except that when I try
> to remove code specific to Xtext so that I can work with the grammar in
> ANTLRworks, the problem goes away. It's pretty time consuming to take
> out one bit of code at a time, so I'm hoping for some guidance in
> locating the problem.
> The code below is extracted from the Xtext-generated file. I added in
> EOF and RULE_WS so I could check that the parser generated by ANTLR
> works (in a trivial case). With the Xtext-generated ANTLR source, two
> warnings are emitted portions of ruleVariable - at the decision for
> 'POS' and at the decision for 'LENGTH'
> [22:34:39] warning(200): InternalPdl.g:2115:2: Decision can match input
> such as "'POS'" using multiple alternatives: 1, 2
> As a result, alternative(s) 2 were disabled for that input
> [22:34:39] warning(200): InternalPdl.g:2140:4: Decision can match input
> such as "'LENGTH'" using multiple alternatives: 1, 2
> As a result, alternative(s) 2 were disabled for that input
>
> The full grammar Xtext generated is around 4500 lines and has 250
> terminals (based on T250 being the last one in the __.g file). (Note
> that more RULE_WS would be needed to parse the actual language
> successfully in ANTLRworks.)
>
>
> grammar Foo;
>
>
> ruleQS :
> ((
> ruleLiteral
> |
> ruleVariable
> ) EOF );
>
> ruleLiteral :
> (
> RULE_STRING
> |
> RULE_INT
> );
>
>
> ruleVariable :
> (
> ruleValue ( 'POS' (
> ruleBracketExpression ))?
> ('LENGTH' (
> ruleBracketExpression ))?);
>
> ruleValue:
> (
> ruleSystemValue
> |
> ruleDataValue
> |
> ruleQSBuild
> );
>
> ruleDataValue :
> (
> ruleSymbol (
> ruleMemoryStructureAbbrev )?(
> ruleBracketExpression )?( RULE_WS 'IN' (RULE_WS 'FORM'
> |'SFA' )?(RULE_WS
> ruleSymbol ))?);
>
> ruleQSBuild :
> ('QSBUILD' (
> ruleSymbol )'=' '(' (
> ruleQS )(',' (
> ruleQS ))*')' );
>
> ruleBracketExpression :
> ('[' (
> ruleExpression )']' );
>
> ruleExpression :
> (
> ruleTerm (
> ruleOperator ruleExpression )?);
>
> ruleTerm :
> (
> ruleQS
> |
> ruleParenthesizedExpression
> );
>
> ruleParenthesizedExpression :
> ('(' (
> ruleExpression )')' );
>
> ruleSystemValue :
> ((
> ruleSimpleSysSym )
> |
> ruleSysSymQS
> |
> ruleSysSymName
> |
> ruleSysSymInName
> |
> ruleSysSymIndexed
> |
> ruleListIndex
> |
> ruleListLookup
> );
>
> ruleListIndex :
> ((
> ruleListKeyword )(
> ruleSymbol )(
> ruleBracketExpression ));
>
> ruleListLookup :
> ((
> ruleListlookupKeyword )(
> ruleSymbol )'USING' (
> ruleQS ));
>
> ruleSysSymQS :
> (('DAYOFWEEK' |'DAYOFYEAR' |'DAYSINMONTH' |'DAYSINYEAR' |'LENGTHOF'
> |'NEXTBUSDAY' |'SIZEOF' |'SQLLENGTH' |'SYSITEMSINSET' |'SYSPARAM' )
> ruleQS
> );
>
> ruleSysSymName :
> (('LASTINDEXOF' |'LISTINDEX' |'RECORDSIN' |'SQLCURSOR' )(
> ruleSymbol ));
>
> ruleSysSymInName :
> (('SYSITEMNO' |'SYSRECNO' )'IN' (
> ruleSymbol ));
>
> ruleSysSymIndexed :
> (('SYSERRORLINE' |'SYSMESSAGE' )(
> ruleBracketExpression ));
>
>
> ruleSymbol:
> (RULE_ID
> |
> ruleDynamicSymbol
> );
>
> ruleDynamicSymbol :
> (('%' RULE_ID
> )
> |(('%' )?'^' ruleQS
> '.' ));
>
>
> ruleMemoryStructureAbbrev :
> (( '#X' )
> |( '#C' )
> |( '#S' ));
>
> ruleListKeyword :
> (( 'LIST' )
> |( 'TABLE'
> ));
>
> ruleListlookupKeyword :
> (( 'LISTLOOKUP' )
> |( 'TABLELOOKUP'
> ));
>
> ruleSimpleSysSym :
> (( 'SYSTIME'
> )
> |( 'SYSEXITKEY' )
> |( 'SYSFKFNKEY' )
> |( 'FIELDEXITBYTE' )
> |( 'SYSRAWDATETIME' )
> |( 'PREVFKPROCEDURE'
> ));
>
> ruleOperator :
> (( '+' )
> |( '-' )
> |( '/' )
> |( '*' ));
>
>
>
> RULE_ML_COMMENT : '{*' ( options {greedy=false;} : . )*'*}';
>
> RULE_ID : ('a'..'z'|'A'..'Z'|'_') ('a'..'z'|'A'..'Z'|'_'|'0'..'9')*;
>
> RULE_INT : ('0'..'9')+;
>
> RULE_STRING : ('"' ('\\'
> ('b'|'t'|'n'|'f'|'r'|'"'|'\''|'\\')|~(('\\'|'"')))* '"'|'\'' ('\\'
> ('b'|'t'|'n'|'f'|'r'|'"'|'\''|'\\')|~(('\\'|'\'')))* '\'');
>
> RULE_SL_COMMENT : '//' ~(('\n'|'\r'))* ('\r'? '\n')?;
>
> RULE_WS : (' '|'\t'|'\r'|'\n')+;
>
> RULE_ANY_OTHER : .;
>
> ------------------------file.xtext-------------------------- ---------------
> grammar xx.Foo with org.eclipse.xtext.common.Terminals
>
> generate Foo "http://www.example.org/foo"
> /*
> enum MemoryStructureAbbrev:
> Stack="#X" | Common="#C" | Scratchpad="#S" ;
>
> QS:
> ( Literal | Variable ) ;
>
> Literal:
> toss=STRING | toss1=INT ( "." toss2=INT )? ;
>
> Variable:
> val=Value ( "POS" pos=BracketExpression )? ("LENGTH"
> len=BracketExpression )? ;
>
> Value:
> SystemValue | DataValue | QSBuild ;
>
> DataValue:
> sym=Symbol toss=MemoryStructureAbbrev? (index=BracketExpression)? ( "IN"
> ( "FORM" | "SFA" )? struct=Symbol )? ;
>
> QSBuild:
> "QSBUILD" name=Symbol "=" "(" first=QS ( "," list+=QS )* ")" ;
>
> BracketExpression:
> "[" exp=Expression "]" ;
>
> Expression:
> left=Term ( op=Operator right=Expression )? ;
>
> Term:
> QS | ParenthesizedExpression ;
>
> ParenthesizedExpression:
> "(" exp=Expression ")" ;
>
> SystemValue:
> ( sim=SimpleSysSym | SysSymQS | SysSymName | SysSymInName |
> SysSymIndexed | ListIndex | ListLookup ) ;
>
> ListIndex:
> toss=ListKeyword toss1=Symbol toss2=BracketExpression ;
>
> enum ListKeyword:
> List = "LIST" | Table = "TABLE" ;
>
> ListLookup:
> toss=ListlookupKeyword toss1=Symbol "USING" toss2=QS ;
>
> enum ListlookupKeyword:
> ListLookup = "LISTLOOKUP" | TableLookup="TABLELOOKUP" ;
>
> enum SimpleSysSym:
> // SYSDIR = "SYSDIR" | SYSEOL = "SYSEOL" | SYSVOL = "SYSVOL" | //SQLAREA
> = "SQLAREA" | SQLCODE = "SQLCODE" | SYSDATE = "SYSDATE" | // SYSDBMS =
> "SYSDBMS" | SYSNODE = "SYSNODE" | SYSPATH = "SYSPATH" | SYSTIME =
> "SYSTIME" |
> //SYSINDEX = "SYSINDEX" | SYSLEVEL = "SYSLEVEL" | SYSLINENO =
> "SYSLINENO" | SYSPAGENO = "SYSPAGENO" | SYSEXITKEY = "SYSEXITKEY" |
> SYSFKFNKEY = "SYSFKFNKEY" | //SYSEXITBYTE = "SYSEXITBYTE" | SYSUSERNAME
> = "SYSUSERNAME" |
> //FIELDEXITKEY = "FIELDEXITKEY" | SYSCURRFIELD = "SYSCURRFIELD" |
> SYSFKEXITKEY = "SYSFKEXITKEY" | //SYSLONGLIMIT = "SYSLONGLIMIT" |
> FIELDEXITBYTE = "FIELDEXITBYTE" | // SYSBREAKLEVEL = "SYSBREAKLEVEL" |
> SYSSIGNONNAME = "SYSSIGNONNAME" | //CURRFUNCKEYSET = "CURRFUNCKEYSET" |
> SYSRAWDATETIME = "SYSRAWDATETIME" | PREVFKPROCEDURE = "PREVFKPROCEDURE"
> //| //SYSUSERINITIALS = "SYSUSERINITIALS" | SYSCURRFIELDNAME =
> "SYSCURRFIELDNAME" | SQLMESSAGE = "SQLMESSAGE" ; // The generated parser
> exceeds the Java class size limit if the system symbols are enumerated.
>
> SysSymQS:
> ( "DAYOFWEEK" | "DAYOFYEAR" | "DAYSINMONTH" | "DAYSINYEAR" | "LENGTHOF"
> | "NEXTBUSDAY" | "SIZEOF" | "SQLLENGTH" | "SYSITEMSINSET" | "SYSPARAM" )
> QS ;
>
> SysSymName:
> ( "LASTINDEXOF" | "LISTINDEX" | "RECORDSIN" | "SQLCURSOR" ) sym=Symbol ;
>
> SysSymInName:
> ( "SYSITEMNO" | "SYSRECNO" ) "IN" sym=Symbol ;
>
> SysSymIndexed:
> ( "SYSERRORLINE" | "SYSMESSAGE" ) index=BracketExpression ;
>
> Symbol:
> ID | DynamicSymbol ;
>
> DynamicSymbol:
> (("%" ID) | ("%"? "^" QS "." ));
>
> enum Operator:
> Plus="+" | Dash="-" | Whack="/" | Splat="*" ;
> terminal ML_COMMENT:
> '{*' -> '*}' ;
>
> terminal ID:
> ('a'..'z'|'A'..'Z'|'_') ('a'..'z'|'A'..'Z'|'_'|'0'..'9')* ;
>
>
>
>
|
|
|
| Re: Trouble migrating grammar from oAW to TMF Xtext [message #55900 is a reply to message #55871] |
Tue, 07 July 2009 03:21 |
| Eclipse User |
|
|
|
Hi John,
please don't forget to enable backtracking for the ui-part as well. We
use a second ANTLR parser to compute the content proposals.
Replace
<!-- java-based API for content assistance -->
<fragment
class=" org.eclipse.xtext.ui.generator.contentAssist.JavaBasedConten tAssistFragment "/>
<!-- the following fragment tries to use the Antlr based content assist
fragment which can be downloaded from http://www.itemis.com
and will be ignored if it's not available. -->
<fragment class="de.itemis.xtext.antlr.XtextAntlrUiGeneratorFragment" >
</fragment>
<!-- <fragment
class="de.itemis.xtext.antlr.XtextAntlrUiGeneratorFragment"/ > -->
and use
<fragment class="de.itemis.xtext.antlr.XtextAntlrUiGeneratorFragment" >
<options backtrack="true"/>
</fragment>
instead.
Regards,
Sebastian
Am 07.07.2009 9:04 Uhr, schrieb Sebastian Zarnekow:
> Hi John,
>
> the problem with your grammar is the QS rule combined with the rules
> Variable, Value, SystemValue and ListLookup.
>
> Kind of simplified a QS may be a Variable which in turn starts with a
> Value.
> A Value may be a SystemValue which may be a ListLookup. This leads to
> the fact, that a ListLookup is sort of a Value. The ListLookup ends with
> a QS, meaning it may end with a Variable (which starts with a Value -
> and so on).
> To summerize this in one line, we have a situation that can be expressed
> as:
>
> Variable == Variable 'POS'? 'LENGTH'?
>
> which is kind of equal to
>
> Variable == Value 'POS'? 'LENGTH'? 'POS'? 'LENGTH'?
>
> At this point it is not possible to decide whether a POS keyword in your
> input matches the first or the second POS keyword in your grammar.
>
> To overcome this situation please activate backtracking for your
> grammar. To do so replace the following fragment in your mwe-file
>
> <fragment class="org.eclipse.xtext.generator.AntlrDelegatingFragment" />
>
> and use this one instead:
>
> <fragment class="de.itemis.xtext.antlr.XtextAntlrGeneratorFragment">
> <options backtrack="true"/>
> </fragment>
>
> Btw: Backtracking was enabled by default in oAW Xtext.
>
> Hope that helps,
> Sebastian
>
> Am 07.07.2009 7:53 Uhr, schrieb John Bito:
>> A grammar built for analyzing an existing general purpose language
>> (bearing a resemblance to COBOL) that worked in oAW Xtext is presenting
>> an intractable problem in migrating to TMF Xtext. I was successful in
>> cleaning up all of the "Decision can match input such as "'XXX'" using
>> multiple alternatives" messages except for a couple emitted when ANTL
>> processes one of the grammar rules.
>>
>> ANTLR3 produces the same message in ANTLRworks, except that when I try
>> to remove code specific to Xtext so that I can work with the grammar in
>> ANTLRworks, the problem goes away. It's pretty time consuming to take
>> out one bit of code at a time, so I'm hoping for some guidance in
>> locating the problem.
>> The code below is extracted from the Xtext-generated file. I added in
>> EOF and RULE_WS so I could check that the parser generated by ANTLR
>> works (in a trivial case). With the Xtext-generated ANTLR source, two
>> warnings are emitted portions of ruleVariable - at the decision for
>> 'POS' and at the decision for 'LENGTH'
>> [22:34:39] warning(200): InternalPdl.g:2115:2: Decision can match input
>> such as "'POS'" using multiple alternatives: 1, 2
>> As a result, alternative(s) 2 were disabled for that input
>> [22:34:39] warning(200): InternalPdl.g:2140:4: Decision can match input
>> such as "'LENGTH'" using multiple alternatives: 1, 2
>> As a result, alternative(s) 2 were disabled for that input
>>
>> The full grammar Xtext generated is around 4500 lines and has 250
>> terminals (based on T250 being the last one in the __.g file). (Note
>> that more RULE_WS would be needed to parse the actual language
>> successfully in ANTLRworks.)
>>
>>
>> grammar Foo;
>>
>>
>> ruleQS :
>> ((
>> ruleLiteral
>> |
>> ruleVariable
>> ) EOF );
>>
>> ruleLiteral :
>> (
>> RULE_STRING
>> |
>> RULE_INT
>> );
>>
>>
>> ruleVariable :
>> (
>> ruleValue ( 'POS' (
>> ruleBracketExpression ))?
>> ('LENGTH' (
>> ruleBracketExpression ))?);
>>
>> ruleValue:
>> (
>> ruleSystemValue
>> |
>> ruleDataValue
>> |
>> ruleQSBuild
>> );
>>
>> ruleDataValue :
>> (
>> ruleSymbol (
>> ruleMemoryStructureAbbrev )?(
>> ruleBracketExpression )?( RULE_WS 'IN' (RULE_WS 'FORM'
>> |'SFA' )?(RULE_WS
>> ruleSymbol ))?);
>>
>> ruleQSBuild :
>> ('QSBUILD' (
>> ruleSymbol )'=' '(' (
>> ruleQS )(',' (
>> ruleQS ))*')' );
>>
>> ruleBracketExpression :
>> ('[' (
>> ruleExpression )']' );
>>
>> ruleExpression :
>> (
>> ruleTerm (
>> ruleOperator ruleExpression )?);
>>
>> ruleTerm :
>> (
>> ruleQS
>> |
>> ruleParenthesizedExpression
>> );
>>
>> ruleParenthesizedExpression :
>> ('(' (
>> ruleExpression )')' );
>>
>> ruleSystemValue :
>> ((
>> ruleSimpleSysSym )
>> |
>> ruleSysSymQS
>> |
>> ruleSysSymName
>> |
>> ruleSysSymInName
>> |
>> ruleSysSymIndexed
>> |
>> ruleListIndex
>> |
>> ruleListLookup
>> );
>>
>> ruleListIndex :
>> ((
>> ruleListKeyword )(
>> ruleSymbol )(
>> ruleBracketExpression ));
>>
>> ruleListLookup :
>> ((
>> ruleListlookupKeyword )(
>> ruleSymbol )'USING' (
>> ruleQS ));
>>
>> ruleSysSymQS :
>> (('DAYOFWEEK' |'DAYOFYEAR' |'DAYSINMONTH' |'DAYSINYEAR' |'LENGTHOF'
>> |'NEXTBUSDAY' |'SIZEOF' |'SQLLENGTH' |'SYSITEMSINSET' |'SYSPARAM' )
>> ruleQS
>> );
>>
>> ruleSysSymName :
>> (('LASTINDEXOF' |'LISTINDEX' |'RECORDSIN' |'SQLCURSOR' )(
>> ruleSymbol ));
>>
>> ruleSysSymInName :
>> (('SYSITEMNO' |'SYSRECNO' )'IN' (
>> ruleSymbol ));
>>
>> ruleSysSymIndexed :
>> (('SYSERRORLINE' |'SYSMESSAGE' )(
>> ruleBracketExpression ));
>>
>>
>> ruleSymbol:
>> (RULE_ID
>> |
>> ruleDynamicSymbol
>> );
>>
>> ruleDynamicSymbol :
>> (('%' RULE_ID
>> )
>> |(('%' )?'^' ruleQS
>> '.' ));
>>
>>
>> ruleMemoryStructureAbbrev :
>> (( '#X' )
>> |( '#C' )
>> |( '#S' ));
>>
>> ruleListKeyword :
>> (( 'LIST' )
>> |( 'TABLE'
>> ));
>>
>> ruleListlookupKeyword :
>> (( 'LISTLOOKUP' )
>> |( 'TABLELOOKUP'
>> ));
>>
>> ruleSimpleSysSym :
>> (( 'SYSTIME'
>> )
>> |( 'SYSEXITKEY' )
>> |( 'SYSFKFNKEY' )
>> |( 'FIELDEXITBYTE' )
>> |( 'SYSRAWDATETIME' )
>> |( 'PREVFKPROCEDURE'
>> ));
>>
>> ruleOperator :
>> (( '+' )
>> |( '-' )
>> |( '/' )
>> |( '*' ));
>>
>>
>>
>> RULE_ML_COMMENT : '{*' ( options {greedy=false;} : . )*'*}';
>>
>> RULE_ID : ('a'..'z'|'A'..'Z'|'_') ('a'..'z'|'A'..'Z'|'_'|'0'..'9')*;
>>
>> RULE_INT : ('0'..'9')+;
>>
>> RULE_STRING : ('"' ('\\'
>> ('b'|'t'|'n'|'f'|'r'|'"'|'\''|'\\')|~(('\\'|'"')))* '"'|'\'' ('\\'
>> ('b'|'t'|'n'|'f'|'r'|'"'|'\''|'\\')|~(('\\'|'\'')))* '\'');
>>
>> RULE_SL_COMMENT : '//' ~(('\n'|'\r'))* ('\r'? '\n')?;
>>
>> RULE_WS : (' '|'\t'|'\r'|'\n')+;
>>
>> RULE_ANY_OTHER : .;
>>
>> ------------------------file.xtext-------------------------- ---------------
>>
>> grammar xx.Foo with org.eclipse.xtext.common.Terminals
>>
>> generate Foo "http://www.example.org/foo"
>> /*
>> enum MemoryStructureAbbrev:
>> Stack="#X" | Common="#C" | Scratchpad="#S" ;
>>
>> QS:
>> ( Literal | Variable ) ;
>>
>> Literal:
>> toss=STRING | toss1=INT ( "." toss2=INT )? ;
>>
>> Variable:
>> val=Value ( "POS" pos=BracketExpression )? ("LENGTH"
>> len=BracketExpression )? ;
>>
>> Value:
>> SystemValue | DataValue | QSBuild ;
>>
>> DataValue:
>> sym=Symbol toss=MemoryStructureAbbrev? (index=BracketExpression)? ( "IN"
>> ( "FORM" | "SFA" )? struct=Symbol )? ;
>>
>> QSBuild:
>> "QSBUILD" name=Symbol "=" "(" first=QS ( "," list+=QS )* ")" ;
>>
>> BracketExpression:
>> "[" exp=Expression "]" ;
>>
>> Expression:
>> left=Term ( op=Operator right=Expression )? ;
>>
>> Term:
>> QS | ParenthesizedExpression ;
>>
>> ParenthesizedExpression:
>> "(" exp=Expression ")" ;
>>
>> SystemValue:
>> ( sim=SimpleSysSym | SysSymQS | SysSymName | SysSymInName |
>> SysSymIndexed | ListIndex | ListLookup ) ;
>>
>> ListIndex:
>> toss=ListKeyword toss1=Symbol toss2=BracketExpression ;
>>
>> enum ListKeyword:
>> List = "LIST" | Table = "TABLE" ;
>>
>> ListLookup:
>> toss=ListlookupKeyword toss1=Symbol "USING" toss2=QS ;
>>
>> enum ListlookupKeyword:
>> ListLookup = "LISTLOOKUP" | TableLookup="TABLELOOKUP" ;
>>
>> enum SimpleSysSym:
>> // SYSDIR = "SYSDIR" | SYSEOL = "SYSEOL" | SYSVOL = "SYSVOL" | //SQLAREA
>> = "SQLAREA" | SQLCODE = "SQLCODE" | SYSDATE = "SYSDATE" | // SYSDBMS =
>> "SYSDBMS" | SYSNODE = "SYSNODE" | SYSPATH = "SYSPATH" | SYSTIME =
>> "SYSTIME" |
>> //SYSINDEX = "SYSINDEX" | SYSLEVEL = "SYSLEVEL" | SYSLINENO =
>> "SYSLINENO" | SYSPAGENO = "SYSPAGENO" | SYSEXITKEY = "SYSEXITKEY" |
>> SYSFKFNKEY = "SYSFKFNKEY" | //SYSEXITBYTE = "SYSEXITBYTE" | SYSUSERNAME
>> = "SYSUSERNAME" |
>> //FIELDEXITKEY = "FIELDEXITKEY" | SYSCURRFIELD = "SYSCURRFIELD" |
>> SYSFKEXITKEY = "SYSFKEXITKEY" | //SYSLONGLIMIT = "SYSLONGLIMIT" |
>> FIELDEXITBYTE = "FIELDEXITBYTE" | // SYSBREAKLEVEL = "SYSBREAKLEVEL" |
>> SYSSIGNONNAME = "SYSSIGNONNAME" | //CURRFUNCKEYSET = "CURRFUNCKEYSET" |
>> SYSRAWDATETIME = "SYSRAWDATETIME" | PREVFKPROCEDURE = "PREVFKPROCEDURE"
>> //| //SYSUSERINITIALS = "SYSUSERINITIALS" | SYSCURRFIELDNAME =
>> "SYSCURRFIELDNAME" | SQLMESSAGE = "SQLMESSAGE" ; // The generated parser
>> exceeds the Java class size limit if the system symbols are enumerated.
>>
>> SysSymQS:
>> ( "DAYOFWEEK" | "DAYOFYEAR" | "DAYSINMONTH" | "DAYSINYEAR" | "LENGTHOF"
>> | "NEXTBUSDAY" | "SIZEOF" | "SQLLENGTH" | "SYSITEMSINSET" | "SYSPARAM" )
>> QS ;
>>
>> SysSymName:
>> ( "LASTINDEXOF" | "LISTINDEX" | "RECORDSIN" | "SQLCURSOR" ) sym=Symbol ;
>>
>> SysSymInName:
>> ( "SYSITEMNO" | "SYSRECNO" ) "IN" sym=Symbol ;
>>
>> SysSymIndexed:
>> ( "SYSERRORLINE" | "SYSMESSAGE" ) index=BracketExpression ;
>>
>> Symbol:
>> ID | DynamicSymbol ;
>>
>> DynamicSymbol:
>> (("%" ID) | ("%"? "^" QS "." ));
>>
>> enum Operator:
>> Plus="+" | Dash="-" | Whack="/" | Splat="*" ;
>> terminal ML_COMMENT:
>> '{*' -> '*}' ;
>>
>> terminal ID:
>> ('a'..'z'|'A'..'Z'|'_') ('a'..'z'|'A'..'Z'|'_'|'0'..'9')* ;
>>
>>
>>
>>
>
|
|
|
|
| Re: Trouble migrating grammar from oAW to TMF Xtext [message #56141 is a reply to message #55900] |
Tue, 07 July 2009 13:35 |
| Eclipse User |
|
|
|
One small point: Sebastian's posting appeared to indicate that
!-- java-based API for content assistance -->
<fragment
class=" org.eclipse.xtext.ui.generator.contentAssist.JavaBasedConten tAssistFragment "/>
should be removed, but it is actually only the following element (with
DelegatingGeneratorFragment) that should be removed from the MWE file.
|
|
|
Powered by
FUDForum. Page generated in 0.02966 seconds
Home