You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
122 lines
1.9 KiB
122 lines
1.9 KiB
package config
|
|
|
|
import (
|
|
"bufio"
|
|
"fmt"
|
|
"os"
|
|
)
|
|
|
|
type Tokenizer struct {
|
|
line int
|
|
file *os.File
|
|
scanner *bufio.Scanner
|
|
}
|
|
|
|
func NewTokenizer(fn string) (*Tokenizer, error) {
|
|
var err error
|
|
tk := &Tokenizer{line: 1}
|
|
tk.file, err = os.Open(fn)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
tk.scanner = bufio.NewScanner(tk.file)
|
|
tk.scanner.Split(tk.Split)
|
|
return tk, nil
|
|
}
|
|
|
|
func (t *Tokenizer) Close() error {
|
|
return t.file.Close()
|
|
}
|
|
|
|
func (t *Tokenizer) Scan() bool {
|
|
return t.scanner.Scan()
|
|
}
|
|
|
|
func (t *Tokenizer) Text() string {
|
|
return t.scanner.Text()
|
|
}
|
|
|
|
func (t *Tokenizer) Line() int {
|
|
return t.line
|
|
}
|
|
|
|
func (t *Tokenizer) Err() error {
|
|
return t.scanner.Err()
|
|
}
|
|
|
|
func (t *Tokenizer) Split(data []byte, atEOF bool) (int, []byte, error) {
|
|
if atEOF && len(data) == 0 {
|
|
return 0, nil, nil
|
|
}
|
|
|
|
var ignoreline bool
|
|
var started bool
|
|
var startidx int
|
|
var quote byte
|
|
|
|
for i := 0; i < len(data); i++ {
|
|
c := data[i]
|
|
//fmt.Printf("%c (%t) (%t)\n", c, started, ignoreline)
|
|
switch c {
|
|
case '\f', '\n', '\r':
|
|
if started {
|
|
return i, data[startidx:i], nil
|
|
}
|
|
|
|
t.line++
|
|
if ignoreline {
|
|
ignoreline = false
|
|
continue
|
|
}
|
|
fallthrough
|
|
|
|
case ' ', '\t', '\v':
|
|
if started && quote == 0 {
|
|
return i + 1, data[startidx:i], nil
|
|
}
|
|
|
|
case '\'', '"', '`':
|
|
// When the quote ends
|
|
if quote == c {
|
|
// if we've gotten data, return it
|
|
if started {
|
|
return i + 1, data[startidx:i], nil
|
|
}
|
|
// if we haven't return nothing
|
|
return i + 1, []byte{}, nil
|
|
}
|
|
|
|
// start a quoted string
|
|
if !ignoreline && quote == 0 {
|
|
quote = c
|
|
}
|
|
|
|
case '#':
|
|
if !started {
|
|
ignoreline = true
|
|
}
|
|
|
|
default:
|
|
if !ignoreline && !started {
|
|
started = true
|
|
startidx = i
|
|
}
|
|
}
|
|
}
|
|
|
|
if atEOF {
|
|
if quote != 0 {
|
|
return 0, nil, fmt.Errorf("unterminated quote")
|
|
}
|
|
|
|
if ignoreline {
|
|
return len(data), nil, nil
|
|
}
|
|
if started {
|
|
return len(data), data[startidx:], nil
|
|
}
|
|
}
|
|
|
|
return 0, nil, nil
|
|
}
|