package ij.macro;
import ij.*;
import java.io.*;

/** This class converts an imageJ macro file file into a token stream. */
public class Tokenizer implements MacroConstants {

    private StreamTokenizer st;
    private int token;
    private String tokenString;
    private double tokenValue;
    private Program pgm;
    private int lineNumber;

    /** Uses a StreamTokenizer to convert an ImageJ macro file into a token stream. */
    public Program tokenize(String program) {
        if (program.contains("/*") && program.contains("*/"))
            program = addSpacesToEmptyLines(program);
        st = new StreamTokenizer(new StringReader(program));
        //st.eolIsSignificant(true);
        st.ordinaryChar('-');
        st.ordinaryChar('/');
        st.ordinaryChar('.');
        st.wordChars('_', '_');
        st.whitespaceChars(128, 255);
        st.slashStarComments(true);
        st.slashSlashComments(true);
        pgm = new Program();
        do {
            getToken();
            addToken();
        } while (token!=EOF);
        if (pgm.hasFunctions)
        	addUserFunctions();
        return pgm;
    }

    final void getToken() {
        try {
            token = st.nextToken();
            lineNumber = st.lineno();
            String ret = null;
            int nextToken;
            switch (st.ttype) {
				case StreamTokenizer.TT_EOF:
					ret = "EOF";
					token = EOF;
					break;
				case StreamTokenizer.TT_WORD:
					ret = st.sval;
					token = WORD;
					break;
				case StreamTokenizer.TT_NUMBER:
					ret = ""+st.nval;
					tokenValue = st.nval;
					if (tokenValue==0.0)
						tokenValue = getHexConstant();
					else if (tryScientificNotation())
						ret += st.sval;
					token = NUMBER;
					break;
				case '"': case '\'':
					ret = ""+st.sval;
					token = STRING_CONSTANT;
					break;
				case '+':
					nextToken = st.nextToken();
					if (nextToken=='+')
						token = PLUS_PLUS;
					else if (nextToken=='=')
						token = PLUS_EQUAL;
					else
						st.pushBack();
					break;
				case '-':
					nextToken = st.nextToken();
					if (nextToken=='-')
						token = MINUS_MINUS;
					else if (nextToken=='=')
						token = MINUS_EQUAL;
					else
						st.pushBack();
					break;
				case '*':
					nextToken = st.nextToken();
					if (nextToken=='=')
						token = MUL_EQUAL;
					else
						st.pushBack();
					break;
				case '/':
					nextToken = st.nextToken();
					if (nextToken=='=')
						token = DIV_EQUAL;
					else
						st.pushBack();
					break;
				case '=':
					nextToken = st.nextToken();
					if (nextToken=='=')
						token = EQ;
					else
						st.pushBack();
					break;
				case '!':
					nextToken = st.nextToken();
					if (nextToken=='=')
						token = NEQ;
					else
						st.pushBack();
					break;
				case '>':
					nextToken = st.nextToken();
					if (nextToken=='=')
						token = GTE;
					else if (nextToken=='>')
						token = SHIFT_RIGHT;
					else {
						st.pushBack();
						token = GT;
					}
					break;
				case '<':
					nextToken = st.nextToken();
					if (nextToken=='=')
						token = LTE;
					else if (nextToken=='<')
						token = SHIFT_LEFT;
					else {
						st.pushBack();
						token = LT;
					}
					break;
				case '&':
					nextToken = st.nextToken();
					if (nextToken=='&')
						token = LOGICAL_AND;
					else
						st.pushBack();
					break;
				case '|':
					nextToken = st.nextToken();
					if (nextToken=='|')
						token = LOGICAL_OR;
					else
						st.pushBack();
					break;
				default:
            }
            tokenString = ret;
        } catch (Exception e) {
            return;
        }
    }

    final void addToken() {
        	int tok = token;
        	switch (token) {
        		case WORD:
				Symbol symbol = pgm.lookupWord(tokenString);
				if (symbol!=null) {
					int type = symbol.getFunctionType();
					if (type==0) {
						tok = symbol.type;
						switch (tok) {
							case FUNCTION: pgm.hasFunctions=true; break;
							case VAR: pgm.hasVars=true; break;
							case MACRO: pgm.macroCount++; break;
						}
					} else
						tok = type;
					tok += pgm.symTabLoc<=PREDEFINED_FUNCTION&&nextToken<=ARRAY_FUNCTION)) {
					address = address2 = code[i+1]>>TOK_SHIFT;
					if (nextToken!=WORD) { // override built in function
                		pgm.addSymbol(new Symbol(WORD, pgm.getSymbolTable()[address].str));
                		address2 = pgm.stLoc;
                		code[i+1] = WORD + (address2<=PREDEFINED_FUNCTION&&token<=ARRAY_FUNCTION))
						&& (code[j]>>TOK_SHIFT)==address && (j==0||(code[j-1]&0xfff)!=FUNCTION)) {
							code[j] = USER_FUNCTION;
							code[j] += address2<>TOK_SHIFT)+" "+USER_FUNCTION+" "+address);
						} else if (token==EOF)
							break;
					}
					//IJ.log(i+"  "+pgm.decodeToken(nextToken, address));
				}					
			} else if (token==EOF)
				break;
		}
	}
	
	private String addSpacesToEmptyLines(String pgm) {
		StringBuilder sb = new StringBuilder();
		int len = pgm.length();
		for (int jj=0; jj