alrm/config/tokenizer.go

package config

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"strings"
	"unicode"
)

const (
	TK_NONE = iota
	TK_VAL
	TK_QUOTE
	TK_COMMENT
)

type Tokenizer struct {
	curline int
	repline int
	file    *os.File
	reader  *bufio.Reader
	text    string
	err     error
}

func NewTokenizer(fn string) (*Tokenizer, error) {
	var err error
	tk := &Tokenizer{curline: 1}
	tk.file, err = os.Open(fn)
	if err != nil {
		return nil, err
	}

	tk.reader = bufio.NewReader(tk.file)
	return tk, nil
}

func (t *Tokenizer) Close() error {
	return t.file.Close()
}

func (t *Tokenizer) Scan() bool {
	t.repline = t.curline
	state := TK_NONE
	t.text = ""

	var b strings.Builder
	var quo rune
	for {
		var r rune
		r, _, t.err = t.reader.ReadRune()
		if t.err != nil {
			break
		}
		if r == unicode.ReplacementChar {
			t.err = fmt.Errorf("invalid utf-8 encoding on line %s", t.repline)
			break
		}

		switch state {
		case TK_NONE:
			// When between values, increment both the reported line
			// and the current line, since there's not yet anything
			// to report
			if r == '\n' {
				t.repline++
				t.curline++
			}

			// If we're between values and we encounter a space
			// or a control character, ignore it
			if unicode.IsSpace(r) || unicode.IsControl(r) {
				continue
			}

			// If we're between values and we encounter a #, it's
			// the beginning of a comment
			if r == '#' {
				state = TK_COMMENT
				continue
			}

			// If we're between values and we get a quote character
			// treat it as the beginning of a string literal
			if r == '"' || r == '\'' || r == '`' {
				state = TK_QUOTE
				quo = r
				continue
			}

			b.WriteRune(r)
			state = TK_VAL

		case TK_VAL:
			// In values, only increment the current line, so
			// if an error is reported, it reports the line
			// the value starts on
			if r == '\n' {
				t.curline++
			}

			// If we're in a normal value and we encounter a space
			// or a control character, end value
			if unicode.IsSpace(r) || unicode.IsControl(r) {
				goto end
			}
			b.WriteRune(r)

		case TK_QUOTE:
			// In quotes, only increment the current line, so
			// if an error is reported, it reports the line
			// the quoted value starts on
			if r == '\n' {
				t.curline++
			}

			// End this quote if it's another quote of the same rune
			if r == quo {
				goto end
			}
			b.WriteRune(r)

		case TK_COMMENT:
			// Comments are ignored, until a new line is encounter
			// at which point, increment the current and reported line
			if r == '\n' {
				t.curline++
				t.repline++
				state = TK_NONE
			}
			continue
		}
	}

end:
	if t.err == nil || t.err == io.EOF {
		t.text = b.String()
	}
	return t.err == nil
}

func (t *Tokenizer) Text() string {
	return t.text
}

func (t *Tokenizer) Line() int {
	return t.repline
}

func (t *Tokenizer) Err() error {
	if t.err == io.EOF {
		return nil
	}
	return t.err
}