| [cdt-patch] Fix Scanner Bugs |
|
Fixed Bug36475 - Scanner does
not concatenate strings
Fixed Bug36509 - Scanner turns strings into
identifiers when expanding macros
Fixed Bug36521 - Scanner gets confused over
commas in function like macros
Fixed Bug36695 - Scanner looses escaping on
chars (ie '\4' to '4')
Moved ScannerFailedTest::testBug36475 to
ScannerTestCase::testBug36475()
Moved ScannerFailedTest::testBug36509 to
ScannerTestCase::testBug36509()
Moved ScannerFailedTest::testBug36521 to
ScannerTestCase::testBug36521() Added
ScannerTestCase::testBug36695()
Updated
ScannerTestCase::testBug36047
Updated
ScannerTestCase::testBug36045
|
Index: ChangeLog
===================================================================
RCS file: /home/tools/org.eclipse.cdt.ui.tests/ChangeLog,v
retrieving revision 1.33
diff -u -r1.33 ChangeLog
--- ChangeLog 17 Apr 2003 18:59:18 -0000 1.33
+++ ChangeLog 21 Apr 2003 14:52:24 -0000
@@ -1,3 +1,11 @@
+2003-04-17 Andrew Niefer
+ Added ScannerTestCase::testBug36695()
+ Moved ScannerFailedTest::testBug36521 to ScannerTestCase::testBug36521()
+ Moved ScannerFailedTest::testBug36509 to ScannerTestCase::testBug36509()
+ Moved ScannerFailedTest::testBug36475 to ScannerTestCase::testBug36475()
+ Updated ScannerTestCase::testBug36047
+ Updated ScannerTestCase::testBug36045
+
2003-04-17 John Camelon
Updated DOMTests::testBug36600().
Updated LineNumberTest::testDOMLineNos().
Index: failures/org/eclipse/cdt/core/parser/failedTests/ScannerFailedTest.java
===================================================================
RCS file: /home/tools/org.eclipse.cdt.ui.tests/failures/org/eclipse/cdt/core/parser/failedTests/ScannerFailedTest.java,v
retrieving revision 1.5
diff -u -r1.5 ScannerFailedTest.java
--- failures/org/eclipse/cdt/core/parser/failedTests/ScannerFailedTest.java 16 Apr 2003 14:35:12 -0000 1.5
+++ failures/org/eclipse/cdt/core/parser/failedTests/ScannerFailedTest.java 21 Apr 2003 14:52:24 -0000
@@ -36,105 +36,29 @@
public static Test suite()
{
TestSuite suite = new TestSuite();
-
- suite.addTest( new ScannerFailedTest( "testBug36475" ) );
- suite.addTest( new ScannerFailedTest( "testBug36509" ) );
- suite.addTest( new ScannerFailedTest( "testBug36521" ) );
-
+
+ suite.addTest( new ScannerFailedTest( "testBug36701" ) );
+
return suite;
}
- public void testBug36475() throws Exception
+ public void testBug36701() throws Exception
{
boolean testPassed = false;
- try
- {
- StringWriter writer = new StringWriter();
- writer.write( " \"A\" \"B\" \"C\" " );
+ try{
+ StringWriter writer = new StringWriter();
+ writer.write( "#define str(s) # s\n" );
+ writer.write( "str( @ \n )\n");
initializeScanner( writer.toString() );
-
- validateString( "ABC" );
- validateEOF();
-
+ validateString( "@ \\\\n" );
+ validateEOF();
+
testPassed = true;
-
+ } catch( AssertionFailedError e ){
+ //expected failure
}
- catch( Throwable e )
- {
- if( !(e instanceof AssertionFailedError) ){
- fail( "Unexpected Error: " + e.getMessage() );
- }
- }
-
- if( testPassed )
- fail( "The expected error did not occur." );
- }
- public void testBug36509() throws Exception
- {
- boolean testPassed = false;
-
- try{
- StringWriter writer = new StringWriter();
- writer.write("#define debug(s, t) printf(\"x\" # s \"= %d, x\" # t \"= %s\", \\\n");
- writer.write(" x ## s, x ## t) \n");
-
- initializeScanner( writer.toString() );
- //printf("x" "1" "=%d, x" "2" "= %s", x1, x2);
- validateIdentifier( "printf" );
- validateToken( Token.tLPAREN );
- validateString("x");
- validateString("1");
- validateString("= %d, x");
- validateString("2");
- validateString("= %s");
- validateToken(Token.tCOMMA);
- validateIdentifier("x1");
- validateToken(Token.tCOMMA);
- validateIdentifier("x2");
- validateToken(Token.tRPAREN);
- validateToken(Token.tSEMI);
-
- testPassed = true;
- }
- catch( Throwable e )
- {
- if( !(e instanceof AssertionFailedError) ){
- fail( "Unexpected Error: " + e.getMessage() );
- }
- }
-
- if( testPassed )
- fail( "The expected error did not occur." );
- }
- public void testBug36521() throws Exception
- {
- boolean testPassed = false;
-
- try{
- StringWriter writer = new StringWriter();
- writer.write("#define str(s) # s\n");
- writer.write("fputs(str(strncmp(\"abc\\0d\", \"abc\", \'\\4\')\n");
- writer.write(" == 0) str(: @\\n), s);\n");
-
- initializeScanner( writer.toString() );
- validateIdentifier("fputs");
- validateToken(Token.tLPAREN);
- validateString("strncmp(\\\"abc\\\\0d\\\", \\\"abc\\\", '\\\\4') == 0");
- validateString(": @\\n");
- validateToken(Token.tCOMMA);
- validateIdentifier("s");
- validateToken(Token.tRPAREN);
- validateToken(Token.tSEMI);
-
- testPassed = true;
- }
- catch( ScannerException e )
- {
- if( !e.getMessage().equals( "Improper use of macro str" ) )
- fail( "Unexpected Error: " + e.getMessage() );
- }
-
+
if( testPassed )
fail( "The expected error did not occur." );
}
Index: parser/org/eclipse/cdt/core/parser/tests/ScannerTestCase.java
===================================================================
RCS file: /home/tools/org.eclipse.cdt.ui.tests/parser/org/eclipse/cdt/core/parser/tests/ScannerTestCase.java,v
retrieving revision 1.14
diff -u -r1.14 ScannerTestCase.java
--- parser/org/eclipse/cdt/core/parser/tests/ScannerTestCase.java 16 Apr 2003 12:30:40 -0000 1.14
+++ parser/org/eclipse/cdt/core/parser/tests/ScannerTestCase.java 21 Apr 2003 14:52:25 -0000
@@ -1072,6 +1072,16 @@
assertTrue(false);
}
}
+ public void validateChar( String expected ) throws ScannerException
+ {
+ try {
+ Token t= scanner.nextToken();
+ assertTrue(t.getType() == Token.tCHAR );
+ assertEquals( t.getImage(), expected );
+ } catch (Parser.EndOfFile e) {
+ assertTrue(false);
+ }
+ }
public void validateString( String expectedImage ) throws ScannerException
{
@@ -1219,8 +1229,7 @@
buffer.append( '"');
buffer.append( "\n\n");
initializeScanner( buffer.toString());
- validateString( "\\\"");
- validateString( "\\\\");
+ validateString( "\\\"\\\\");
}
public void testConditionalWithBraces()
@@ -1343,15 +1352,68 @@
writer.write( "MAD_VERSION\n" );
initializeScanner( writer.toString() );
- validateString( "2" );
- validateString( "." );
- validateString( "1" );
- validateString( "." );
- validateString( "3" );
- validateString( "." );
- validateString( "boo" );
+ validateString( "2.1.3.boo" );
validateEOF();
}
-
+
+ public void testBug36475() throws Exception
+ {
+ StringWriter writer = new StringWriter();
+ writer.write( " \"A\" \"B\" \"C\" " );
+
+ initializeScanner( writer.toString() );
+
+ validateString( "ABC" );
+ validateEOF();
+ }
+
+ public void testBug36509() throws Exception
+ {
+ StringWriter writer = new StringWriter();
+ writer.write("#define debug(s, t) printf(\"x\" # s \"= %d, x\" # t \"= %s\", \\\n");
+ writer.write(" x ## s, x ## t) \n");
+ writer.write("debug(1, 2);");
+
+ initializeScanner( writer.toString() );
+ //printf("x1=%d, x2= %s", x1, x2);
+ validateIdentifier( "printf" );
+ validateToken( Token.tLPAREN );
+ validateString("x1= %d, x2= %s");
+ validateToken(Token.tCOMMA);
+ validateIdentifier("x1");
+ validateToken(Token.tCOMMA);
+ validateIdentifier("x2");
+ validateToken(Token.tRPAREN);
+ validateToken(Token.tSEMI);
+ validateEOF();
+ }
+
+ public void testBug36695() throws Exception
+ {
+ StringWriter writer = new StringWriter();
+ writer.write("\'\\4\' \'\\n\'");
+ initializeScanner( writer.toString() );
+
+ validateChar( "\\4" );
+ validateChar( "\\n" );
+ validateEOF();
+ }
+
+ public void testBug36521() throws Exception
+ {
+ StringWriter writer = new StringWriter();
+ writer.write("#define str(s) # s\n");
+ writer.write("fputs(str(strncmp(\"abc\\0d\", \"abc\", \'\\4\')\n");
+ writer.write(" == 0), s);\n");
+
+ initializeScanner( writer.toString() );
+ validateIdentifier("fputs");
+ validateToken(Token.tLPAREN);
+ validateString("strncmp ( \\\"abc\\\\0d\\\" , \\\"abc\\\" , '\\\\4' ) == 0");
+ validateToken(Token.tCOMMA);
+ validateIdentifier("s");
+ validateToken(Token.tRPAREN);
+ validateToken(Token.tSEMI);
+ }
}
Index: parser/ChangeLog
===================================================================
RCS file: /home/tools/org.eclipse.cdt.core/parser/ChangeLog,v
retrieving revision 1.33
diff -u -r1.33 ChangeLog
--- parser/ChangeLog 17 Apr 2003 18:59:16 -0000 1.33
+++ parser/ChangeLog 21 Apr 2003 14:52:11 -0000
@@ -1,3 +1,9 @@
+2003-04-21 Andrew Niefer
+ Fixed Bug36475 - Scanner does not concatenate strings
+ Fixed Bug36509 - Scanner turns strings into identifiers when expanding macros
+ Fixed Bug36521 - Scanner gets confused over commas in function like macros
+ Fixed Bug36695 - Scanner looses escaping on chars (ie '\4' to '4')
+
2003-04-17 John Camelon
Fixed error in Elaborated Enumeration Types.
Fixed Bug36559 - Parsing Templates...
Index: parser/org/eclipse/cdt/internal/core/parser/Scanner.java
===================================================================
RCS file: /home/tools/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Scanner.java,v
retrieving revision 1.21
diff -u -r1.21 Scanner.java
--- parser/org/eclipse/cdt/internal/core/parser/Scanner.java 16 Apr 2003 12:30:46 -0000 1.21
+++ parser/org/eclipse/cdt/internal/core/parser/Scanner.java 21 Apr 2003 14:52:11 -0000
@@ -300,6 +300,7 @@
private static HashMap ppDirectives = new HashMap();
private Token currentToken = null;
+ private Token cachedToken = null;
private boolean passOnToClient = true;
private BranchTracker branches = new BranchTracker();
@@ -436,7 +437,12 @@
protected Token nextToken( boolean pasting ) throws ScannerException, Parser.EndOfFile
{
-
+ if( cachedToken != null ){
+ setCurrentToken( cachedToken );
+ cachedToken = null;
+ return currentToken;
+ }
+
count++;
boolean madeMistake = false;
int c = getChar();
@@ -510,10 +516,33 @@
if (c != NOCHAR )
{
int type = wideString ? Token.tLSTRING : Token.tSTRING;
- return newToken(
- type,
- buff.toString(),
- contextStack.getCurrentContext());
+
+ //If the next token is going to be a string as well, we need to concatenate
+ //it with this token.
+ Token returnToken = newToken( type, buff.toString(), contextStack.getCurrentContext());
+ Token next = null;
+ try{
+ next = nextToken( true );
+ } catch( Parser.EndOfFile e ){
+ next = null;
+ }
+
+ while( next != null && next.type == returnToken.type ){
+ returnToken.image += next.image;
+ returnToken.setNext( null );
+ currentToken = returnToken;
+ try{
+ next = nextToken( true );
+ } catch( Parser.EndOfFile e ){
+ next = null;
+ }
+ }
+
+ cachedToken = next;
+ currentToken = returnToken;
+ returnToken.setNext( null );
+
+ return returnToken;
} else {
if (throwExceptionOnUnboundedString)
@@ -916,9 +945,16 @@
} else {
switch (c) {
case '\'' :
- c = getChar();
- int next = getChar();
- if( next == '\'' )
+ c = getChar( true );
+ int next = getChar( true );
+ if( c == '\\' ){
+ c = next;
+ next = getChar( true );
+ if( next == '\'' )
+ return newToken( Token.tCHAR, '\\' + new Character( (char)c ).toString(), contextStack.getCurrentContext() );
+ else if( throwExceptionOnBadCharacterRead )
+ throw new ScannerException( "Invalid character '" + (char)c + "' read @ offset " + contextStack.getCurrentContext().getOffset() + " of file " + contextStack.getCurrentContext().getFilename() );
+ } else if( next == '\'' )
return newToken( Token.tCHAR, new Character( (char)c ).toString(), contextStack.getCurrentContext() );
else
if( throwExceptionOnBadCharacterRead )
@@ -1714,7 +1750,7 @@
if (bracketCount == 0)
break;
buffer.append((char) c);
- c = getChar();
+ c = getChar( true );
}
String betweenTheBrackets = buffer.toString().trim();
@@ -1724,11 +1760,15 @@
Token t = null;
String str = new String();
boolean space = false;
+ int nParen = 0;
try{
while (true) {
t = tokenizer.nextToken(false);
- if( t.type == Token.tCOMMA )
- {
+ if( t.type == Token.tLPAREN ){
+ nParen++;
+ } else if ( t.type == Token.tRPAREN ){
+ nParen--;
+ } else if( t.type == Token.tCOMMA && nParen == 0 ){
parameterValues.add( str );
str = "";
space = false;
@@ -1738,11 +1778,13 @@
if( space )
str += ' ';
- if( t.type == Token.tSTRING )
- str += '\"' + t.image + '\"';
- else
- str += t.image;
-
+ switch( t.type )
+ {
+ case Token.tSTRING: str += '\"' + t.image + '\"'; break;
+ case Token.tLSTRING: str += "L\"" + t.image + '\"'; break;
+ case Token.tCHAR: str += '\'' + t.image + '\''; break;
+ default: str += t.image; break;
+ }
space = true;
}
} catch (Parser.EndOfFile e) {
@@ -1812,7 +1854,13 @@
buffer.append('\"');
}
} else {
- buffer.append(t.image);
+ switch( t.type )
+ {
+ case Token.tSTRING: buffer.append('\"' + t.image + '\"'); break;
+ case Token.tLSTRING: buffer.append("L\"" + t.image + '\"'); break;
+ case Token.tCHAR: buffer.append('\'' + t.image + '\''); break;
+ default: buffer.append(t.image); break;
+ }
}
boolean pastingNext = false;
Index: parser/org/eclipse/cdt/internal/core/parser/Token.java
===================================================================
RCS file: /home/tools/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Token.java,v
retrieving revision 1.10
diff -u -r1.10 Token.java
--- parser/org/eclipse/cdt/internal/core/parser/Token.java 16 Apr 2003 12:30:46 -0000 1.10
+++ parser/org/eclipse/cdt/internal/core/parser/Token.java 21 Apr 2003 14:52:11 -0000
@@ -17,6 +17,10 @@
image = i;
filename = context.getFilename();
offset = context.getOffset() - image.length() - context.undoStackSize();
+
+ if( type == tLSTRING || type == tSTRING || type == tCHAR ){
+ offset--;
+ }
}
public Token(int t, String i) {