package java_cup;
|
|
import java.util.Hashtable;
|
|
import java_cup.runtime.str_token;
|
import java_cup.runtime.token;
|
|
/** This class implements a small scanner (aka lexical analyzer or lexer) for
|
* the JavaCup specification. This scanner reads characters from standard
|
* input (System.in) and returns integers corresponding to the terminal
|
* number of the next token. Once end of input is reached the EOF token is
|
* returned on every subsequent call.<p>
|
* Tokens currently returned include: <pre>
|
* Symbol Constant Returned Symbol Constant Returned
|
* ------ ----------------- ------ -----------------
|
* "package" PACKAGE "import" IMPORT
|
* "code" CODE "action" ACTION
|
* "parser" PARSER "terminal" TERMINAL
|
* "non" NON "init" INIT
|
* "scan" SCAN "with" WITH
|
* "start" START ; SEMI
|
* , COMMA * STAR
|
* . DOT : COLON
|
* ::= COLON_COLON_EQUALS | BAR
|
* identifier ID {:...:} CODE_STRING
|
* "debug" DEBUG
|
* </pre>
|
* All symbol constants are defined in sym.java which is generated by
|
* JavaCup from parser.cup.<p>
|
*
|
* In addition to the scanner proper (called first via init() then with
|
* next_token() to get each token) this class provides simple error and
|
* warning routines and keeps a count of errors and warnings that is
|
* publicly accessible.<p>
|
*
|
* This class is "static" (i.e., it has only static members and methods).
|
*
|
* @version last updated: 11/25/95
|
* @author Scott Hudson
|
*/
|
public class lexer {
|
|
/*-----------------------------------------------------------*/
|
/*--- Constructor(s) ----------------------------------------*/
|
/*-----------------------------------------------------------*/
|
|
/** The only constructor is private, so no instances can be created. */
|
private lexer() { }
|
|
/*-----------------------------------------------------------*/
|
/*--- Static (Class) Variables ------------------------------*/
|
/*-----------------------------------------------------------*/
|
|
/** First character of lookahead. */
|
protected static int next_char;
|
|
/** Second character of lookahead. */
|
protected static int next_char2;
|
|
/*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/
|
|
/** EOF constant. */
|
protected static final int EOF_CHAR = -1;
|
|
/*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/
|
|
/** Table of keywords. Keywords are initially treated as identifiers.
|
* Just before they are returned we look them up in this table to see if
|
* they match one of the keywords. The string of the name is the key here,
|
* which indexes Integer objects holding the symbol number.
|
*/
|
protected static Hashtable keywords = new Hashtable(23);
|
|
/*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/
|
|
/** Table of single character symbols. For ease of implementation, we
|
* store all unambiguous single character tokens in this table of Integer
|
* objects keyed by Integer objects with the numerical value of the
|
* appropriate char (currently Character objects have a bug which precludes
|
* their use in tables).
|
*/
|
protected static Hashtable char_symbols = new Hashtable(11);
|
|
/*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/
|
|
/** Current line number for use in error messages. */
|
protected static int current_line = 1;
|
|
/*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/
|
|
/** Character position in current line. */
|
protected static int current_position = 1;
|
|
/*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/
|
|
/** Count of total errors detected so far. */
|
public static int error_count = 0;
|
|
/*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/
|
|
/** Count of warnings issued so far */
|
public static int warning_count = 0;
|
|
/*-----------------------------------------------------------*/
|
/*--- Static Methods ----------------------------------------*/
|
/*-----------------------------------------------------------*/
|
|
/** Initialize the scanner. This sets up the keywords and char_symbols
|
* tables and reads the first two characters of lookahead.
|
*/
|
public static void init() throws java.io.IOException
|
{
|
/* set up the keyword table */
|
keywords.put("package", new Integer(sym.PACKAGE));
|
keywords.put("import", new Integer(sym.IMPORT));
|
keywords.put("code", new Integer(sym.CODE));
|
keywords.put("action", new Integer(sym.ACTION));
|
keywords.put("parser", new Integer(sym.PARSER));
|
keywords.put("terminal", new Integer(sym.TERMINAL));
|
keywords.put("non", new Integer(sym.NON));
|
keywords.put("init", new Integer(sym.INIT));
|
keywords.put("scan", new Integer(sym.SCAN));
|
keywords.put("with", new Integer(sym.WITH));
|
keywords.put("start", new Integer(sym.START));
|
keywords.put("debug", new Integer(sym.DEBUG));
|
|
/* set up the table of single character symbols */
|
char_symbols.put(new Integer(';'), new Integer(sym.SEMI));
|
char_symbols.put(new Integer(','), new Integer(sym.COMMA));
|
char_symbols.put(new Integer('*'), new Integer(sym.STAR));
|
char_symbols.put(new Integer('.'), new Integer(sym.DOT));
|
char_symbols.put(new Integer('|'), new Integer(sym.BAR));
|
|
/* read two characters of lookahead */
|
next_char = System.in.read();
|
if (next_char == EOF_CHAR)
|
next_char2 = EOF_CHAR;
|
else
|
next_char2 = System.in.read();
|
}
|
|
/*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/
|
|
/** Advance the scanner one character in the input stream. This moves
|
* next_char2 to next_char and then reads a new next_char2.
|
*/
|
protected static void advance() throws java.io.IOException
|
{
|
int old_char;
|
|
old_char = next_char;
|
next_char = next_char2;
|
if (next_char == EOF_CHAR)
|
next_char2 = EOF_CHAR;
|
else
|
next_char2 = System.in.read();
|
|
/* count this */
|
current_position++;
|
if (old_char == '\n')
|
{
|
current_line++;
|
current_position = 1;
|
}
|
}
|
|
/*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/
|
|
/** Emit an error message. The message will be marked with both the
|
* current line number and the position in the line. Error messages
|
* are printed on standard error (System.err).
|
* @param message the message to print.
|
*/
|
public static void emit_error(String message)
|
{
|
System.err.println("Error at " + current_line + "(" + current_position +
|
"): " + message);
|
error_count++;
|
}
|
|
/*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/
|
|
/** Emit a warning message. The message will be marked with both the
|
* current line number and the position in the line. Messages are
|
* printed on standard error (System.err).
|
* @param message the message to print.
|
*/
|
public static void emit_warn(String message)
|
{
|
System.err.println("Warning at " + current_line + "(" + current_position +
|
"): " + message);
|
warning_count++;
|
}
|
|
/*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/
|
|
/** Determine if a character is ok to start an id.
|
* @param ch the character in question.
|
*/
|
protected static boolean id_start_char(int ch)
|
{
|
return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') ||
|
(ch == '_');
|
|
// later need to deal with non-8-bit chars here
|
}
|
|
/*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/
|
|
/** Determine if a character is ok for the middle of an id.
|
* @param ch the character in question.
|
*/
|
protected static boolean id_char(int ch)
|
{
|
return id_start_char(ch) || (ch >= '0' && ch <= '9');
|
}
|
|
/*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/
|
|
/** Try to look up a single character symbol, returns -1 for not found.
|
* @param ch the character in question.
|
*/
|
protected static int find_single_char(int ch)
|
{
|
Integer result;
|
|
result = (Integer)char_symbols.get(new Integer((char)ch));
|
if (result == null)
|
return -1;
|
else
|
return result.intValue();
|
}
|
|
/*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/
|
|
/** Handle swallowing up a comment. Both old style C and new style C++
|
* comments are handled.
|
*/
|
protected static void swallow_comment() throws java.io.IOException
|
{
|
/* next_char == '/' at this point */
|
|
/* is it a traditional comment */
|
if (next_char2 == '*')
|
{
|
/* swallow the opener */
|
advance(); advance();
|
|
/* swallow the comment until end of comment or EOF */
|
for (;;)
|
{
|
/* if its EOF we have an error */
|
if (next_char == EOF_CHAR)
|
{
|
emit_error("Specification file ends inside a comment");
|
return;
|
}
|
|
/* if we can see the closer we are done */
|
if (next_char == '*' && next_char2 == '/')
|
{
|
advance();
|
advance();
|
return;
|
}
|
|
/* otherwise swallow char and move on */
|
advance();
|
}
|
}
|
|
/* is its a new style comment */
|
if (next_char2 == '/')
|
{
|
/* swallow the opener */
|
advance(); advance();
|
|
/* swallow to '\n', '\f', or EOF */
|
while (next_char != '\n' && next_char != '\f' && next_char!=EOF_CHAR)
|
advance();
|
|
return;
|
|
}
|
|
/* shouldn't get here, but... if we get here we have an error */
|
emit_error("Malformed comment in specification -- ignored");
|
advance();
|
}
|
|
/*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/
|
|
/** Swallow up a code string. Code strings begin with "{:" and include
|
all characters up to the first occurrence of ":}" (there is no way to
|
include ":}" inside a code string). The routine returns an str_token
|
object suitable for return by the scanner.
|
*/
|
protected static token do_code_string() throws java.io.IOException
|
{
|
StringBuffer result = new StringBuffer();
|
|
/* at this point we have lookahead of "{:" -- swallow that */
|
advance(); advance();
|
|
/* save chars until we see ":}" */
|
while (!(next_char == ':' && next_char2 == '}'))
|
{
|
/* if we have run off the end issue a message and break out of loop */
|
if (next_char == EOF_CHAR)
|
{
|
emit_error("Specification file ends inside a code string");
|
break;
|
}
|
|
/* otherwise record the char and move on */
|
result.append(new Character((char)next_char));
|
advance();
|
}
|
|
/* advance past the closer and build a return token */
|
advance(); advance();
|
return new str_token(sym.CODE_STRING, result.toString());
|
}
|
|
/*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/
|
|
/** Process an identifier. Identifiers begin with a letter, underscore,
|
* or dollar sign, which is followed by zero or more letters, numbers,
|
* underscores or dollar signs. This routine returns an str_token suitable
|
* for return by the scanner.
|
*/
|
protected static token do_id() throws java.io.IOException
|
{
|
StringBuffer result = new StringBuffer();
|
String result_str;
|
Integer keyword_num;
|
char buffer[] = new char[1];
|
|
/* next_char holds first character of id */
|
buffer[0] = (char)next_char;
|
result.append(buffer,0,1);
|
advance();
|
|
/* collect up characters while they fit in id */
|
while(id_char(next_char))
|
{
|
buffer[0] = (char)next_char;
|
result.append(buffer,0,1);
|
advance();
|
}
|
|
/* extract a string and try to look it up as a keyword */
|
result_str = result.toString();
|
keyword_num = (Integer)keywords.get(result_str);
|
|
/* if we found something, return that keyword */
|
if (keyword_num != null)
|
return new token(keyword_num.intValue());
|
|
/* otherwise build and return an id token with an attached string */
|
return new str_token(sym.ID, result_str);
|
}
|
|
/*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/
|
|
/** Return one token. This is the main external interface to the scanner.
|
* It consumes sufficient characters to determine the next input token
|
* and returns it. To help with debugging, this routine actually calls
|
* real_next_token() which does the work. If you need to debug the
|
* parser, this can be changed to call debug_next_token() which prints
|
* a debugging message before returning the token.
|
*/
|
public static token next_token() throws java.io.IOException
|
{
|
return real_next_token();
|
}
|
|
/*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/
|
|
/** Debugging version of next_token(). This routine calls the real scanning
|
* routine, prints a message on System.out indicating what the token is,
|
* then returns it.
|
*/
|
public static token debug_next_token() throws java.io.IOException
|
{
|
token result = real_next_token();
|
System.out.println("# next_token() => " + result.sym);
|
return result;
|
}
|
|
/*. . . . . . . . . . . . . . . . . . . . . . . . . . . . . .*/
|
|
/** The actual routine to return one token. This is normally called from
|
* next_token(), but for debugging purposes can be called indirectly from
|
* debug_next_token().
|
*/
|
protected static token real_next_token() throws java.io.IOException
|
{
|
int sym_num;
|
|
for (;;)
|
{
|
/* look for white space */
|
if (next_char == ' ' || next_char == '\t' || next_char == '\n' ||
|
next_char == '\f' || next_char == '\r')
|
{
|
/* advance past it and try the next character */
|
advance();
|
continue;
|
}
|
|
/* look for a single character symbol */
|
sym_num = find_single_char(next_char);
|
if (sym_num != -1)
|
{
|
/* found one -- advance past it and return a token for it */
|
advance();
|
return new token(sym_num);
|
}
|
|
/* look for : or ::= */
|
if (next_char == ':')
|
{
|
/* if we don't have a second ':' return COLON */
|
if (next_char2 != ':')
|
{
|
advance();
|
return new token(sym.COLON);
|
}
|
|
/* move forward and look for the '=' */
|
advance();
|
if (next_char2 == '=')
|
{
|
advance(); advance();
|
return new token(sym.COLON_COLON_EQUALS);
|
}
|
else
|
{
|
/* return just the colon (already consumed) */
|
return new token(sym.COLON);
|
}
|
}
|
|
/* look for a comment */
|
if (next_char == '/' && (next_char2 == '*' || next_char2 == '/'))
|
{
|
/* swallow then continue the scan */
|
swallow_comment();
|
continue;
|
}
|
|
/* look for start of code string */
|
if (next_char == '{' && next_char2 == ':')
|
return do_code_string();
|
|
/* look for an id or keyword */
|
if (id_start_char(next_char)) return do_id();
|
|
/* look for EOF */
|
if (next_char == EOF_CHAR) return new token(sym.EOF);
|
|
/* if we get here, we have an unrecognized character */
|
emit_warn("Unrecognized character '" +
|
new Character((char)next_char) + "'(" + next_char +
|
") -- ignored");
|
|
/* advance past it */
|
advance();
|
}
|
}
|
|
/*-----------------------------------------------------------*/
|
};
|
