| [cdt-patch] Applied Patch - Parser robustness improvements |
|
Fixed infinite loop problems in the parser by making the end of file condition a parser backtrack exception instead of a token.
Doug Schaefer Rational Software - IBM Software Group
|
Index: parser/ChangeLog
===================================================================
RCS file: /home/tools/org.eclipse.cdt.core/parser/ChangeLog,v
retrieving revision 1.3
diff -u -r1.3 ChangeLog
--- parser/ChangeLog 17 Mar 2003 15:22:05 -0000 1.3
+++ parser/ChangeLog 17 Mar 2003 19:22:06 -0000
@@ -1,3 +1,7 @@
+2003-03-17 Doug Schaefer
+ Changed EOF to be a Backtrack exception instead of a token so simplify
+ error handling.
+
2003-03-13 John Camelon
Moved ## token to IScanner from Scanner.
Updated IParserCallback and implementations to deal with Elaborated Type Specifiers.
Index: parser/org/eclipse/cdt/internal/core/parser/IScanner.java
===================================================================
RCS file: /home/tools/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/IScanner.java,v
retrieving revision 1.3
diff -u -r1.3 IScanner.java
--- parser/org/eclipse/cdt/internal/core/parser/IScanner.java 17 Mar 2003 15:22:05 -0000 1.3
+++ parser/org/eclipse/cdt/internal/core/parser/IScanner.java 17 Mar 2003 19:22:06 -0000
@@ -26,7 +26,7 @@
public void addIncludePath(String includePath);
public void overwriteIncludePath( List newIncludePaths );
- public Token nextToken() throws ScannerException;
+ public Token nextToken() throws ScannerException, Parser.EndOfFile;
public void setQuickScan(boolean qs);
public void setCallback(IParserCallback c);
Index: parser/org/eclipse/cdt/internal/core/parser/Parser.java
===================================================================
RCS file: /home/tools/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Parser.java,v
retrieving revision 1.9
diff -u -r1.9 Parser.java
--- parser/org/eclipse/cdt/internal/core/parser/Parser.java 17 Mar 2003 15:22:05 -0000 1.9
+++ parser/org/eclipse/cdt/internal/core/parser/Parser.java 17 Mar 2003 19:22:07 -0000
@@ -32,7 +32,6 @@
quickParse = quick;
scanner.setQuickScan(quick);
scanner.setCallback(c);
- //fetchToken();
}
public Parser(IScanner s, IParserCallback c) throws Exception {
@@ -78,13 +77,16 @@
public void translationUnit() throws Exception {
Object translationUnit = callback.translationUnitBegin();
Token lastBacktrack = null;
- Token lastToken = null;
- while (LT(1) != Token.tEOF) {
+ Token lastToken;
+ while (true) {
try {
- lastToken = currToken;
+ lastToken = LA(1);
declaration( translationUnit );
- if( currToken == lastToken )
+ if( LA(1) == lastToken )
consumeToNextSemicolon();
+ } catch (EndOfFile e) {
+ // Good
+ break;
} catch (Backtrack b) {
// Mark as failure and try to reach a recovery point
parsePassed = false;
@@ -102,12 +104,11 @@
callback.translationUnitEnd(translationUnit);
}
- protected void consumeToNextSemicolon() {
- for (int t = LT(1); t != Token.tEOF; t = LT(1)) {
+ protected void consumeToNextSemicolon() throws EndOfFile {
+ consume();
+ // TODO - we should really check for matching braces too
+ while (LT(1) != Token.tSEMI) {
consume();
- // TO DO: we should really check for matching braces too
- if (t == Token.tSEMI)
- break;
}
}
@@ -205,9 +206,7 @@
// and look for the left brace;
consume();
while (LT(1) != Token.tLBRACE) {
- if (consume().getType() == Token.tEOF)
- // Oops, couldn't find it
- throw backtrack;
+ consume();
}
// Falling through on purpose
case Token.tLBRACE:
@@ -225,9 +224,6 @@
case Token.tLBRACE:
++depth;
break;
- case Token.tEOF:
- // Oops, no match
- throw backtrack;
}
}
} else {
@@ -461,9 +457,6 @@
case Token.tLBRACE:
++depth;
break;
- case Token.tEOF:
- // Oops, no match
- throw backtrack;
}
}
}
@@ -529,6 +522,7 @@
// parameterDeclarationClause
Object clause = callback.argumentsBegin(declarator);
consume();
+ boolean seenParameter = false;
parameterDeclarationLoop:
for (;;) {
switch (LT(1)) {
@@ -540,9 +534,13 @@
break;
case Token.tCOMMA:
consume();
+ seenParameter = false;
break;
default:
- parameterDeclaration( clause );
+ if (seenParameter)
+ throw backtrack;
+ parameterDeclaration( clause );
+ seenParameter = true;
}
}
callback.argumentsEnd(clause);
@@ -633,9 +631,6 @@
case Token.tLBRACE:
++depth;
break;
- case Token.tEOF:
- // Oops, no match
- throw backtrack;
}
}
}
@@ -1362,24 +1357,32 @@
}
// Backtracking
- private static class Backtrack extends Exception {
+ public static class Backtrack extends Exception {
}
private static Backtrack backtrack = new Backtrack();
+ // End of file generally causes backtracking
+ public static class EndOfFile extends Backtrack {
+ }
+
+ public static EndOfFile endOfFile = new EndOfFile();
+
// Token management
private IScanner scanner;
private Token currToken;
- private Token fetchToken() {
+ private Token fetchToken() throws EndOfFile {
try {
return scanner.nextToken();
+ } catch (EndOfFile e) {
+ throw e;
} catch (Exception e) {
return null;
}
}
- protected Token LA(int i) {
+ protected Token LA(int i) throws EndOfFile {
if (i < 1)
// can't go backwards
return null;
@@ -1390,34 +1393,37 @@
Token retToken = currToken;
for (; i > 1; --i) {
- if (retToken.getNext() == null)
- fetchToken();
retToken = retToken.getNext();
+ if (retToken == null)
+ retToken = fetchToken();
}
return retToken;
}
- protected int LT(int i) {
+ protected int LT(int i) throws EndOfFile {
return LA(i).type;
}
- protected Token consume() {
- if (currToken.getNext() == null)
- fetchToken();
+ protected Token consume() throws EndOfFile {
+ if (currToken == null)
+ currToken = fetchToken();
+
Token retToken = currToken;
currToken = currToken.getNext();
return retToken;
}
- protected Token consume(int type) throws Exception {
+ protected Token consume(int type) throws Backtrack {
if (LT(1) == type)
return consume();
else
throw backtrack;
}
- protected Token mark() {
+ protected Token mark() throws EndOfFile {
+ if (currToken == null)
+ currToken = fetchToken();
return currToken;
}
Index: parser/org/eclipse/cdt/internal/core/parser/Scanner.java
===================================================================
RCS file: /home/tools/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Scanner.java,v
retrieving revision 1.5
diff -u -r1.5 Scanner.java
--- parser/org/eclipse/cdt/internal/core/parser/Scanner.java 17 Mar 2003 15:22:05 -0000 1.5
+++ parser/org/eclipse/cdt/internal/core/parser/Scanner.java 17 Mar 2003 19:22:08 -0000
@@ -366,6 +366,9 @@
private int getChar() {
int c = NOCHAR;
+ if (currentContext == null)
+ // past the end of file
+ return c;
boolean done;
do {
@@ -436,12 +439,12 @@
- public Token nextToken() throws ScannerException {
+ public Token nextToken() throws ScannerException, Parser.EndOfFile {
return nextToken( true );
}
- protected Token nextToken( boolean pasting ) throws ScannerException
+ protected Token nextToken( boolean pasting ) throws ScannerException, Parser.EndOfFile
{
count++;
@@ -1102,10 +1105,9 @@
break;
}
- return newToken(
- Token.tEOF,
- "Bad Char: " + (char) c,
- currentContext);
+ // Bad character
+ // TODO - does this need it's own exception
+ throw Parser.endOfFile;
}
}
@@ -1113,9 +1115,7 @@
throw new ScannerException("End of file encountered without terminating #endif");
// we're done
- if (currentToken != null)
- currentToken.setNext(Token.EOF);
- return Token.EOF;
+ throw Parser.endOfFile;
}
static {
@@ -1254,7 +1254,8 @@
ExpressionEvaluator evaluator = new ExpressionEvaluator();
Scanner trial =
new Scanner(
- new StringReader(expression),
+ // Semicolon makes this valid C (hopefully)
+ new StringReader(expression + ";"),
EXPRESSION,
definitions);
Parser parser = new Parser(trial, evaluator);
@@ -1373,7 +1374,7 @@
handleInclusion(f.trim(), useIncludePath );
}
- protected void poundDefine() throws ScannerException {
+ protected void poundDefine() throws ScannerException, Parser.EndOfFile {
skipOverWhitespace();
// definition
String key = getNextIdentifier();
@@ -1424,9 +1425,13 @@
null);
Token t = helperScanner.nextToken(false);
- while (t.type != Token.tEOF) {
- macroReplacementTokens.add(t);
- t = helperScanner.nextToken(false);
+ try {
+ while (true) {
+ macroReplacementTokens.add(t);
+ t = helperScanner.nextToken(false);
+ }
+ } catch (Parser.EndOfFile e) {
+ // Good
}
IMacroDescriptor descriptor = new MacroDescriptor();
Index: parser/org/eclipse/cdt/internal/core/parser/Token.java
===================================================================
RCS file: /home/tools/org.eclipse.cdt.core/parser/org/eclipse/cdt/internal/core/parser/Token.java,v
retrieving revision 1.3
diff -u -r1.3 Token.java
--- parser/org/eclipse/cdt/internal/core/parser/Token.java 11 Mar 2003 02:42:47 -0000 1.3
+++ parser/org/eclipse/cdt/internal/core/parser/Token.java 17 Mar 2003 19:22:09 -0000
@@ -24,8 +24,6 @@
image = i;
}
- public static Token EOF = new Token(Token.tEOF, "<EOF>");
-
public String toString()
{
return "Token type=" + type + " image =" + image + " offset=" + offset;
@@ -47,7 +45,6 @@
public void setNext(Token t) { next = t; }
// Token types
- static public final int tEOF = 0;
static public final int tIDENTIFIER = 1;
static public final int tINTEGER = 2;
static public final int tCOLONCOLON = 3;
Index: parser/org/eclipse/cdt/core/parser/tests/ExprEvalTest.java
===================================================================
RCS file: /home/tools/org.eclipse.cdt.ui.tests/parser/org/eclipse/cdt/core/parser/tests/ExprEvalTest.java,v
retrieving revision 1.2
diff -u -r1.2 ExprEvalTest.java
--- parser/org/eclipse/cdt/core/parser/tests/ExprEvalTest.java 4 Mar 2003 18:25:45 -0000 1.2
+++ parser/org/eclipse/cdt/core/parser/tests/ExprEvalTest.java 17 Mar 2003 19:22:38 -0000
@@ -25,17 +25,17 @@
}
public void testInteger() throws Exception {
- runTest("5", 5);
+ runTest("5;", 5);
}
public void testRelational() throws Exception {
- runTest("1 < 2", 1);
- runTest("2 < 1", 0);
- runTest("2 == 1 + 1", 1);
- runTest("2 != 1 + 1", 0);
+ runTest("1 < 2;", 1);
+ runTest("2 < 1;", 0);
+ runTest("2 == 1 + 1;", 1);
+ runTest("2 != 1 + 1;", 0);
}
public void testBracketed() throws Exception {
- runTest("2 * (3 + 4)", 14);
+ runTest("2 * (3 + 4);", 14);
}
}
Index: parser/org/eclipse/cdt/core/parser/tests/ScannerTestCase.java
===================================================================
RCS file: /home/tools/org.eclipse.cdt.ui.tests/parser/org/eclipse/cdt/core/parser/tests/ScannerTestCase.java,v
retrieving revision 1.5
diff -u -r1.5 ScannerTestCase.java
--- parser/org/eclipse/cdt/core/parser/tests/ScannerTestCase.java 11 Mar 2003 02:42:43 -0000 1.5
+++ parser/org/eclipse/cdt/core/parser/tests/ScannerTestCase.java 17 Mar 2003 19:22:39 -0000
@@ -8,6 +8,7 @@
import junit.framework.TestSuite;
import org.eclipse.cdt.internal.core.parser.IMacroDescriptor;
+import org.eclipse.cdt.internal.core.parser.Parser;
import org.eclipse.cdt.internal.core.parser.Scanner;
import org.eclipse.cdt.internal.core.parser.ScannerException;
import org.eclipse.cdt.internal.core.parser.Token;
@@ -176,16 +177,19 @@
try
{
Token t= scanner.nextToken();
- while ((t != null) && (t.type != Token.tEOF))
+ while (t != null)
{
if (verbose)
System.out.println("Token t = " + t);
- if ((t.type < Token.tEOF) || (t.type > Token.tLAST))
+ if ((t.type < 1) || (t.type > Token.tLAST))
System.out.println("Unknown type for token " + t);
t= scanner.nextToken();
}
}
+ catch (Parser.EndOfFile e)
+ {
+ }
catch (ScannerException se)
{
throw se;
@@ -245,7 +249,7 @@
validateToken(Token.tASSIGN);
validateInteger("5");
validateToken(Token.tSEMI);
- validateToken(Token.tEOF);
+ validateEOF();
// the case we were failing against in ctype.h
// this is a definition, not a macro!
@@ -445,7 +449,7 @@
validateEOF();
initializeScanner("#define SYMBOL 5\n#ifndef SYMBOL\nint counter(SYMBOL);\n#endif");
- validateToken(Token.tEOF);
+ validateEOF();
initializeScanner("#ifndef DEFINED\n#define DEFINED 100\n#endif\nint count = DEFINED;");
validateToken(Token.t_int);
@@ -522,7 +526,7 @@
validateBalance();
initializeScanner("#ifndef FOO\n#define FOO 4\n#else\n#undef FOO\n#define FOO 6\n#endif");
- validateToken(Token.tEOF);
+ validateEOF();
validateBalance();
validateDefinition("FOO", "4");
@@ -833,7 +837,7 @@
}
}
- public void testQuickScan()
+ public void testQuickScan() throws Parser.EndOfFile
{
try
{
@@ -970,29 +974,45 @@
public void validateIdentifier(String expectedImage) throws ScannerException
{
- Token t= scanner.nextToken();
- assertTrue(t.type == Token.tIDENTIFIER);
- assertTrue(t.image.equals(expectedImage));
+ try {
+ Token t= scanner.nextToken();
+ assertTrue(t.type == Token.tIDENTIFIER);
+ assertTrue(t.image.equals(expectedImage));
+ } catch (Parser.EndOfFile e) {
+ assertTrue(false);
+ }
}
public void validateInteger(String expectedImage) throws ScannerException
{
- Token t= scanner.nextToken();
- assertTrue(t.type == Token.tINTEGER);
- assertTrue(t.image.equals(expectedImage));
+ try {
+ Token t= scanner.nextToken();
+ assertTrue(t.type == Token.tINTEGER);
+ assertTrue(t.image.equals(expectedImage));
+ } catch (Parser.EndOfFile e) {
+ assertTrue(false);
+ }
}
public void validateString(String expectedImage) throws ScannerException
{
- Token t= scanner.nextToken();
- assertTrue(t.type == Token.tSTRING);
- assertTrue(t.image.equals(expectedImage));
+ try {
+ Token t= scanner.nextToken();
+ assertTrue(t.type == Token.tSTRING);
+ assertTrue(t.image.equals(expectedImage));
+ } catch (Parser.EndOfFile e) {
+ assertTrue(false);
+ }
}
public void validateToken(int tokenType) throws ScannerException
{
- Token t= scanner.nextToken();
- assertTrue(t.type == tokenType);
+ try {
+ Token t= scanner.nextToken();
+ assertTrue(t.type == tokenType);
+ } catch (Parser.EndOfFile e) {
+ assertTrue(false);
+ }
}
public void validateBalance(int expected)
@@ -1007,7 +1027,10 @@
public void validateEOF() throws ScannerException
{
- validateToken(Token.tEOF);
+ try {
+ assertNull(scanner.nextToken());
+ } catch (Parser.EndOfFile e) {
+ }
}
public void validateDefinition(String name, String value)
@@ -1022,7 +1045,7 @@
{
String definition= null;
definition= (String) scanner.getDefinition(name);
- this.assertNotNull(definition);
+ assertNotNull(definition);
int intValue= (Integer.valueOf((String) definition)).intValue();
assertEquals(value, intValue);
}
@@ -1105,7 +1128,7 @@
validateInteger( "0");
validateToken( Token.tSEMI );
validateToken( Token.tRBRACE );
- validateToken( Token.tEOF );
+ validateEOF();
}
} catch( ScannerException se )
{