| [cdt-patch] For Head: Scanner bug fixes and test updates |
|
This
patch is meant to be applied to the head branch. (NOT
1_1)
core:
Fixed Bug 36287 - Parser failure with new CDT 1.1
parser
Fixed Bug 37011 - Scanner: #define A "//"
not properly handled
UI.tests
Updated FractionalAutomatedTest to use threads
Modified
ScannerTestCase::testBug36287
Added
ScannerTestCase::testBug37011
Added
ScannerTestCase::testOtherPreprocessorDefines
Some notes about the updates to
FractionalAutomatedTest:
- launches parses on a seperate thread in order to detect if they are
looping infinitely.
- currently using deprecated Thread.stop() to stop
looping parses
- new properties settting "timeOut", how long to wait (in milliseconds)
until deciding the parse is taking too long and must be looping. Default
is 5000.
- setting "outDir" replaces tempFile, copies of code causing loops or
exceptions are placed in the outDir
-Andrew
|
Index: parser/ChangeLog
===================================================================
RCS file: /home/tools/org.eclipse.cdt.core/parser/ChangeLog,v
retrieving revision 1.45
diff -u -r1.45 ChangeLog
--- parser/ChangeLog 30 Apr 2003 22:14:55 -0000 1.45
+++ parser/ChangeLog 1 May 2003 19:06:33 -0000
@@ -1,3 +1,7 @@
+2003-05-01 Andrew Niefer
+ Fixed Bug 36287 - Parser failure with new CDT 1.1 parser
+ Fixed Bug 37011 - Scanner: #define A "//" not properly handled
+
2003-04-30 John Camelon
Added some Javadoc to IParser.java and Parser.java.
Index: parser/org/eclipse/cdt/internal/core/parser/Scanner.java
===================================================================
RCS file: /home/tools/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Scanner.java,v
retrieving revision 1.25
diff -u -r1.25 Scanner.java
--- parser/org/eclipse/cdt/internal/core/parser/Scanner.java 27 Apr 2003 20:28:14 -0000 1.25
+++ parser/org/eclipse/cdt/internal/core/parser/Scanner.java 1 May 2003 19:06:34 -0000
@@ -120,17 +120,28 @@
StringBuffer buffer = new StringBuffer();
skipOverWhitespace();
int c = getChar();
-
+ boolean inString = false;
+ boolean inChar = false;
while (true) {
while ((c != '\n')
&& (c != '\r')
&& (c != '\\')
&& (c != '/')
+ && (c != '"' || ( c == '"' && inChar ) )
+ && (c != '\'' || ( c == '\'' && inString ) )
&& (c != NOCHAR)) {
buffer.append((char) c);
- c = getChar();
+ c = getChar( true );
}
+
if (c == '/') {
+ //only care about comments outside of a quote
+ if( inString || inChar ){
+ buffer.append( (char) c );
+ c = getChar( true );
+ continue;
+ }
+
// we need to peek ahead at the next character to see if
// this is a comment or not
int next = getChar();
@@ -143,7 +154,7 @@
if (skipOverMultilineComment())
break;
else
- c = getChar();
+ c = getChar( true );
continue;
} else {
// we are not in a comment
@@ -151,12 +162,35 @@
c = next;
continue;
}
- } else {
- if (c != '\\') {
- ungetChar(c);
+ } else if( c == '"' ){
+ inString = !inString;
+ buffer.append((char) c);
+ c = getChar( true );
+ continue;
+ } else if( c == '\'' ){
+ inChar = !inChar;
+ buffer.append((char) c);
+ c = getChar( true );
+ continue;
+ } else if( c == '\\' ){
+ c = getChar(true);
+ if( c == '\r' ){
+ c = getChar(true);
+ if( c == '\n' ){
+ c = getChar(true);
+ }
+ } else if( c == '\n' ){
+ c = getChar(true);
} else {
- c = getChar();
+ buffer.append('\\');
+ if( c == '"' || c == '\'' ){
+ buffer.append((char)c);
+ c = getChar( true );
+ }
}
+ continue;
+ } else {
+ ungetChar(c);
break;
}
}
@@ -645,13 +679,23 @@
c = getChar();
- if (c == 'x') {
- if( ! firstCharZero && floatingPoint )
- {
- ungetChar( c );
- return newToken( Token.tDOT, ".", contextStack.getCurrentContext() );
+ if( ! firstCharZero && floatingPoint && !(c >= '0' && c <= '9') ){
+ //if pasting, there could actually be a float here instead of just a .
+ if( buff.toString().equals( "." ) ){
+ if( c == '*' ){
+ return newToken( Token.tDOTSTAR, ".*", contextStack.getCurrentContext() );
+ } else if( c == '.' ){
+ if( getChar() == '.' )
+ return newToken( Token.tELIPSE, "..." );
+ else
+ throw new ScannerException( "Invalid floating point @ offset " + contextStack.getCurrentContext().getOffset() );
+ } else {
+ ungetChar( c );
+ return newToken( Token.tDOT, ".", contextStack.getCurrentContext() );
+ }
}
- else if( ! firstCharZero )
+ } else if (c == 'x') {
+ if( ! firstCharZero )
throw new ScannerException( "Invalid Hexidecimal @ offset " + contextStack.getCurrentContext().getOffset() );
hex = true;
@@ -668,11 +712,6 @@
if( c == '.' )
{
buff.append( (char)c);
- if( floatingPoint || hex ) {
- if( buff.toString().equals( "..") && getChar() == '.' )
- return newToken( Token.tELIPSE, "..." );
- throw new ScannerException( "Invalid floating point @ offset " + contextStack.getCurrentContext().getOffset() );
- }
floatingPoint = true;
c= getChar();
Index: ChangeLog
===================================================================
RCS file: /home/tools/org.eclipse.cdt.ui.tests/ChangeLog,v
retrieving revision 1.48
diff -u -r1.48 ChangeLog
--- ChangeLog 28 Apr 2003 20:00:35 -0000 1.48
+++ ChangeLog 1 May 2003 19:05:53 -0000
@@ -1,3 +1,9 @@
+2003-05-01 Andrew Niefer
+ Updated FractionalAutomatedTest to use threads
+ Modified ScannerTestCase::testBug36287
+ Added ScannerTestCase::testBug37011
+ Added ScannerTestCase::testOtherPreprocessorDefines
+
2003-04-28 John Camelon
Moved testBug36730() & testBug37019() from DOMFailedTests to DOMTests.
Index: parser/org/eclipse/cdt/core/parser/tests/FractionalAutomatedTest.java
===================================================================
RCS file: /home/tools/org.eclipse.cdt.ui.tests/parser/org/eclipse/cdt/core/parser/tests/FractionalAutomatedTest.java,v
retrieving revision 1.2
diff -u -r1.2 FractionalAutomatedTest.java
--- parser/org/eclipse/cdt/core/parser/tests/FractionalAutomatedTest.java 24 Apr 2003 21:01:24 -0000 1.2
+++ parser/org/eclipse/cdt/core/parser/tests/FractionalAutomatedTest.java 1 May 2003 19:05:53 -0000
@@ -14,14 +14,13 @@
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
+import java.io.IOException;
import java.io.StringWriter;
import java.util.NoSuchElementException;
import java.util.StringTokenizer;
-import junit.framework.AssertionFailedError;
import junit.framework.Test;
-import org.eclipse.cdt.internal.core.parser.IParser;
import org.eclipse.cdt.internal.core.parser.Parser;
import org.eclipse.core.runtime.Path;
@@ -57,8 +56,8 @@
stepSize = Integer.parseInt( properties.getProperty( "stepSize", "50" ) );
windowSize = Integer.parseInt( properties.getProperty( "windowSize", "200" ) );
-
- tempFile = properties.getProperty( "tempFile", "" );
+ timeOut = Integer.parseInt( properties.getProperty( "timeOut", "5000" ));
+ outputDir = properties.getProperty( "outDir", "" );
if( sourceInfo.equals("") )
throw new FileNotFoundException();
@@ -91,95 +90,163 @@
return frame.createSuite();
}
- static private void outputTempFile( String code ) {
- if( tempFile == null || tempFile.equals("") )
- return;
+ static private String outputFile( String code ) {
+ if( outputDir == null || outputDir.equals("") )
+ return "";
- File output = new File( tempFile );
+ File output = new File( outputDir );
try{
if( output.exists() ){
- output.delete();
+ if( output.isFile() ){
+ output.delete();
+ output.createNewFile();
+ FileOutputStream stream = new FileOutputStream( output );
+ stream.write( code.getBytes() );
+ stream.flush();
+ stream.close();
+ return outputDir;
+ }
+ } else {
+ output.mkdir();
}
-
- output.createNewFile();
- FileOutputStream stream = new FileOutputStream( output );
+ File file = new File( outputDir + "/" + failures++ + ".tmp" );
+ if( file.exists() )
+ file.delete();
+ file.createNewFile();
+ FileOutputStream stream = new FileOutputStream( file );
stream.write( code.getBytes() );
stream.flush();
stream.close();
+
+ return file.getCanonicalPath();
+
} catch ( Exception e )
{}
+ return "";
+ }
+
+ static private void reportHang( String code, String file ){
+ String output = outputFile( code.toString() );
+ if( output.equals("") )
+ output = "Parser hang while parsing " + file + "\n";
+ else
+ output = "Parser hang while parsing " + output + "\n";
+
+ if( report != null ){
+ try{
+ report.write( output.getBytes() );
+ } catch ( IOException e ) {}
+ }
+
+ fail( output );
+ }
+
+ static private void reportException( String code, String file, String exception ){
+ String output = outputFile( code.toString() );
+
+ if( output.equals("") )
+ output = exception.getClass().toString() + " encountered in " + file + "\n";
+ else
+ output = exception.getClass().toString() + " encountered in " + output + "\n";
+
+ if( report != null ){
+ try{
+ report.write( output.getBytes() );
+ } catch ( IOException e ) {}
+ }
+
+ fail( output );
}
public void doFile() throws Throwable {
assertNotNull( fileList );
- File file = null;
- IParser parser = null;
-
- try{
- file = (File)fileList.removeFirst();
- FileInputStream stream = new FileInputStream( file );
+ File file = (File)fileList.removeFirst();
+ FileInputStream stream = new FileInputStream( file );
- String filePath = file.getCanonicalPath();
- String nature = (String)natures.get( filePath );
+ String filePath = file.getCanonicalPath();
+ String nature = (String)natures.get( filePath );
- boolean cppNature = nature.equalsIgnoreCase("cpp");
-
- StringWriter code = new StringWriter();
-
- byte b[] = new byte[stepSize];
- int n = stream.read( b );
- while( n != -1 ){
- code.write( new String( b ) );
- parser = new Parser( code.toString(), nullCallback, true);
- parser.setCppNature( cppNature );
- parser.mapLineNumbers(true);
-
- outputTempFile( code.toString() );
- parser.parse();
-
- n = stream.read( b );
+ boolean cppNature = nature.equalsIgnoreCase("cpp");
+
+ StringWriter code = new StringWriter();
+
+ ParseThread thread = new ParseThread();
+
+ byte b[] = new byte[stepSize];
+ int n = stream.read( b );
+ while( n != -1 ){
+ code.write( new String( b ) );
+
+ thread.code = code.toString();
+ thread.cppNature = cppNature;
+ thread.start();
+ thread.join( timeOut );
+
+ if( thread.isAlive() ){
+ //Use deprecated Thread.stop() for now
+ //alternative is to create a callback which could stop the parse on a flag
+ //by throwing something, but that has the disadvantage of being unable to
+ //stop any loops that don't involve callbacks.
+ thread.stop();
+ reportHang( code.toString(), filePath );
+ } else if( thread.result != null ) {
+ reportException( code.toString(), filePath, thread.result );
}
- String fullCode = code.toString();
- String windowedCode = null;
- int length = fullCode.length();
- int curPos = 0;
-
- while( curPos + windowSize < length){
- windowedCode = fullCode.substring( 0, curPos );
- windowedCode += "\n" + fullCode.substring( curPos + windowSize, length );
-
- parser = new Parser( windowedCode, nullCallback, true );
+ n = stream.read( b );
+ }
+
+ String fullCode = code.toString();
+ String windowedCode = null;
+ int length = fullCode.length();
+ int curPos = 0;
+
+ while( curPos + windowSize < length){
+ windowedCode = fullCode.substring( 0, curPos );
+ windowedCode += "\n" + fullCode.substring( curPos + windowSize, length );
+
+ thread.code = windowedCode;
+ thread.cppNature = cppNature;
+ thread.file = filePath;
+ thread.start();
+ thread.join( timeOut );
+
+ if( thread.isAlive() )
+ {
+ thread.stop();
+ reportHang( windowedCode, filePath );
+ } else if( thread.result != null ) {
+ reportException( windowedCode, filePath, thread.result );
+ }
+
+ curPos += stepSize;
+ }
+ }
+
+ static class ParseThread extends Thread{
+ public String code;
+ public boolean cppNature;
+ public String file;
+ public String result;
+
+ public void run(){
+ try{
+ result = null;
+ Parser parser = new Parser( code, nullCallback, true);
parser.setCppNature( cppNature );
parser.mapLineNumbers(true);
-
- outputTempFile( windowedCode );
parser.parse();
-
- curPos += stepSize;
- }
- }
- catch( Throwable e )
- {
- String output = null;
- if( e instanceof AssertionFailedError ){
- output = file.getCanonicalPath() + ": Parse failed on line ";
- output += parser.getLineNumberForOffset(parser.getLastErrorOffset()) + "\n";
- } else {
- output = file.getCanonicalPath() + ": " + e.getClass().toString();
- output += " on line " + parser.getLineNumberForOffset(parser.getLastErrorOffset()) + "\n";
- }
- if( report != null ){
- report.write( output.getBytes() );
+ } catch ( Exception e ){
+ result = e.getClass().toString();
}
-
- fail( output );
}
}
-
+
static private int stepSize = 50;
static private int windowSize = 200;
- static private String tempFile = null;
+ static private int timeOut = 5000;
+ static private String outputDir = null;
+ static private int failures = 0;
}
Index: parser/org/eclipse/cdt/core/parser/tests/ScannerTestCase.java
===================================================================
RCS file: /home/tools/org.eclipse.cdt.ui.tests/parser/org/eclipse/cdt/core/parser/tests/ScannerTestCase.java,v
retrieving revision 1.19
diff -u -r1.19 ScannerTestCase.java
--- parser/org/eclipse/cdt/core/parser/tests/ScannerTestCase.java 27 Apr 2003 20:28:16 -0000 1.19
+++ parser/org/eclipse/cdt/core/parser/tests/ScannerTestCase.java 1 May 2003 19:05:55 -0000
@@ -995,7 +995,7 @@
public void testBug36287() throws Exception
{
- initializeScanner( "X::X( const X & rtg_arg ) : U( rtg_arg ) , Z( rtg_arg.Z ) , br( rtg_arg.br ){}" );
+ initializeScanner( "X::X( const X & rtg_arg ) : U( rtg_arg ) , Z( rtg_arg.Z ) , er( rtg_arg.er ){}" );
validateIdentifier("X");
validateToken( Token.tCOLONCOLON);
validateIdentifier("X");
@@ -1018,15 +1018,27 @@
validateIdentifier( "Z");
validateToken( Token.tRPAREN );
validateToken( Token.tCOMMA );
- validateIdentifier( "br");
+ validateIdentifier( "er");
validateToken( Token.tLPAREN );
validateIdentifier( "rtg_arg");
validateToken( Token.tDOT );
- validateIdentifier( "br");
+ validateIdentifier( "er");
validateToken( Token.tRPAREN );
validateToken( Token.tLBRACE);
validateToken( Token.tRBRACE);
validateEOF();
+
+ initializeScanner( "foo.*bar");
+ validateIdentifier("foo");
+ validateToken( Token.tDOTSTAR );
+ validateIdentifier("bar");
+ validateEOF();
+
+ initializeScanner( "foo...bar");
+ validateIdentifier("foo");
+ validateToken( Token.tELIPSE );
+ validateIdentifier("bar");
+ validateEOF();
}
public void testBug35892()
@@ -1308,5 +1320,40 @@
initializeScanner( writer.toString() );
validateEOF();
+ }
+
+ public void testBug37011() throws Exception{
+ StringWriter writer = new StringWriter();
+ writer.write( "#define A \"//\"");
+
+ initializeScanner( writer.toString() );
+
+ validateEOF();
+ validateDefinition("A", "\"//\"");
+ }
+
+ public void testOtherPreprocessorDefines() throws Exception{
+ StringWriter writer = new StringWriter();
+ writer.write( "#define A a//boo\n" );
+ writer.write( "#define B a /*boo*/ a\n" );
+ writer.write( "#define C a \" //boo \"\n" );
+ writer.write( "#define D a \\\"//boo\n" );
+ writer.write( "#define E a \\n \"\\\"\"\n" );
+ writer.write( "#define F a\\\n b\n" );
+ writer.write( "#define G a '\"'//boo\n" );
+ writer.write( "#define H a '\\'//b'\"/*bo\\o*/\" b\n" );
+
+ initializeScanner( writer.toString() );
+
+ validateEOF();
+
+ validateDefinition("A", "a");
+ validateDefinition("B", "a a");
+ validateDefinition("C", "a \" //boo \"");
+ validateDefinition("D", "a \\\"");
+ validateDefinition("E", "a \\n \"\\\"\"");
+ validateDefinition("F", "a b");
+ validateDefinition("G", "a '\"'");
+ validateDefinition("H", "a '\\'//b'\"/*bo\\o*/\" b");
}
}