#!/bin/bash

#%#
# Copyright 2007 Logan Lee
# Released under GPL
#%#

#%%#
# 0 Tokenizer that outputs tokens one per line

# 1 Plan outline
# 1.1 Reader that reads from file then returns all or some of the string
# 1.2 Tokenizer that cuts up the string(s) then outputs them one per line.

# 2 Support code

# 2.1 Reader
TEXTFILE=$1
MIDLINE_SEPARATOR=":"
function getStringFromFile {
    FILE=$TEXTFILE
    while read LINE; do
	echo "$LINE" && echo "^"
    done < $FILE
}

# 2.2 Tokenizer
function tokenizer {
    # delimiter used here is " ".
    STRING=$(getStringFromFile)
    TOKEN_REMAINDER=$(echo $STRING | cut -d" " -f1)
    while [ -n "$STRING" ]; do
	echo -e $(echo $STRING | cut -d" " -f1)
	if [ "$(echo $STRING | cut -d" " -f1)" = "$STRING" ]; then
	    break
	fi
	# Quick fix for late midline insertion of newline
	if [ "$(echo $STRING | cut -d" " -f 2)" = ":" ]; then
	    echo "^"
	fi
	STRING=$(echo $STRING | cut --complement -d" " -f1 )
    done
}

# 3 Main
# Test code is sufficient.
#%%#

ARG[1]=$1; ARG[2]=$2	
function main {
    HELP="$0 </path/to/text_file>"
    ARG1=${ARG[1]}; ARG2=${ARG[2]}
    if [ -z "$ARG1" ] || [ -n "$ARG2" ]; then
	echo $HELP
    else
	tokenizer
    fi
}

# main 
main

