The Java Program: TokenizerText.java

  1 // TokenizerText.java
  2 
  3 import java.io.FileReader;
  4 import java.io.StreamTokenizer;
  5 import java.io.IOException;
  6 
  7 public class TokenizerText { 
  8    public static void main (final String[] args) {
  9       if (args.length < 1) {     
 10          System.err.println("Usage: java TokenizerTest <src>");
 11          return;
 12       }
 13 
 14       try {
 15          final StreamTokenizer stok = new StreamTokenizer(new FileReader(args[0]));
 16 
 17          stok.wordChars ('_', '_');       // "letters" of words include '_'
 18          stok.whitespaceChars(';', ';');  // let ';' also be a separator
 19 
 20          int token;
 21 
 22          // token is filled with a code indicating what type of item was just read.
 23          while ((token = stok.nextToken()) != StreamTokenizer.TT_EOF) {         
 24             switch (token) {             
 25             case StreamTokenizer.TT_NUMBER:
 26                // If a number is read, value is in the double nval.
 27                System.out.println("Number: " + stok.nval);
 28                break;
 29 
 30             case StreamTokenizer.TT_WORD:
 31                // If a word is read, value is in the String variable sval.
 32                System.out.println("Word: " + stok.sval);
 33                break;
 34 
 35             case '"':
 36                System.out.println("String: " + stok.sval);
 37                break;
 38 
 39             case '\'':
 40                System.out.println("Char: " + stok.sval);
 41                break;
 42 
 43             default:
 44                System.out.println("Other: " + stok.toString());
 45                break;   
 46             }    
 47          }
 48 
 49       } catch (IOException e) {          
 50          System.err.println(e);          
 51          return;        
 52       } 
 53    } 
 54 }