/**File Name :     Tokenizer.java
  *  a random access file and returns tokens separated by '<','>',
  *  space,tab,and newline characters. It also return the separators. 
  *  This is built mainly to parse HTML files. 
  */
import java.io.*;
import java.util.StringTokenizer;
public class  Tokenizer
{
     protected  RandomAccessFile inFile;
     protected  StringTokenizer token;
     protected  long length;
     protected  long filePointer;
     Tokenizer(RandomAccessFile inf) throws IOException
     {
        inFile = inf;
        length = inFile.length();
        token = new StringTokenizer(inFile.readLine(),"\"<>=    ",true);
        filePointer = inFile.getFilePointer();
     }
     protected String getNextToken() throws IOException
     {
        while(true)
        {
            if(token.hasMoreTokens())
            {
                String str = token.nextToken();               
                if (!(str.equals(" ") || str.equals("   ")))
                    return(str);
            }
            else
        if(filePointer < length)
        {
        token=new StringTokenizer(inFile.readLine(),"\"<>=    " ,true);
                filePointer = inFile.getFilePointer();
                if(token.hasMoreTokens())
                {
                    String str = token.nextToken();               
                    if (!(str.equals(" ") || str.equals("   "))) 
                        return(str);
                 }
        }
        else{
                inFile.close();
                return("<>");
            }
       }
   }
}