|
|
|
@ -2,26 +2,37 @@ package config |
|
|
|
|
|
|
|
import ( |
|
|
|
"bufio" |
|
|
|
"fmt" |
|
|
|
"io" |
|
|
|
"os" |
|
|
|
"strings" |
|
|
|
"unicode" |
|
|
|
) |
|
|
|
|
|
|
|
const ( |
|
|
|
TK_NONE = iota |
|
|
|
TK_VAL |
|
|
|
TK_QUOTE |
|
|
|
TK_COMMENT |
|
|
|
) |
|
|
|
|
|
|
|
type Tokenizer struct { |
|
|
|
line int |
|
|
|
curline int |
|
|
|
repline int |
|
|
|
file *os.File |
|
|
|
scanner *bufio.Scanner |
|
|
|
reader *bufio.Reader |
|
|
|
text string |
|
|
|
err error |
|
|
|
} |
|
|
|
|
|
|
|
func NewTokenizer(fn string) (*Tokenizer, error) { |
|
|
|
var err error |
|
|
|
tk := &Tokenizer{line: 1} |
|
|
|
tk := &Tokenizer{curline: 1} |
|
|
|
tk.file, err = os.Open(fn) |
|
|
|
if err != nil { |
|
|
|
return nil, err |
|
|
|
} |
|
|
|
|
|
|
|
tk.scanner = bufio.NewScanner(tk.file) |
|
|
|
tk.scanner.Split(tk.Split) |
|
|
|
tk.reader = bufio.NewReader(tk.file) |
|
|
|
return tk, nil |
|
|
|
} |
|
|
|
|
|
|
|
@ -30,93 +41,114 @@ func (t *Tokenizer) Close() error { |
|
|
|
} |
|
|
|
|
|
|
|
func (t *Tokenizer) Scan() bool { |
|
|
|
return t.scanner.Scan() |
|
|
|
} |
|
|
|
|
|
|
|
func (t *Tokenizer) Text() string { |
|
|
|
return t.scanner.Text() |
|
|
|
} |
|
|
|
|
|
|
|
func (t *Tokenizer) Line() int { |
|
|
|
return t.line |
|
|
|
} |
|
|
|
|
|
|
|
func (t *Tokenizer) Err() error { |
|
|
|
return t.scanner.Err() |
|
|
|
} |
|
|
|
|
|
|
|
func (t *Tokenizer) Split(data []byte, atEOF bool) (int, []byte, error) { |
|
|
|
if atEOF && len(data) == 0 { |
|
|
|
return 0, nil, nil |
|
|
|
} |
|
|
|
t.repline = t.curline |
|
|
|
state := TK_NONE |
|
|
|
t.text = "" |
|
|
|
|
|
|
|
var b strings.Builder |
|
|
|
var quo rune |
|
|
|
for { |
|
|
|
var r rune |
|
|
|
r, _, t.err = t.reader.ReadRune() |
|
|
|
if t.err != nil { |
|
|
|
break |
|
|
|
} |
|
|
|
|
|
|
|
var ignoreline bool |
|
|
|
var started bool |
|
|
|
var startidx int |
|
|
|
var quote byte |
|
|
|
|
|
|
|
for i := 0; i < len(data); i++ { |
|
|
|
c := data[i] |
|
|
|
//fmt.Printf("%c (%t) (%t)\n", c, started, ignoreline)
|
|
|
|
switch c { |
|
|
|
case '\f', '\n', '\r': |
|
|
|
if started { |
|
|
|
return i, data[startidx:i], nil |
|
|
|
switch state { |
|
|
|
case TK_NONE: |
|
|
|
// When between values, increment both the reported line
|
|
|
|
// and the current line, since there's not yet anything
|
|
|
|
// to report
|
|
|
|
if r == '\n' { |
|
|
|
t.repline++ |
|
|
|
t.curline++ |
|
|
|
} |
|
|
|
|
|
|
|
t.line++ |
|
|
|
if ignoreline { |
|
|
|
ignoreline = false |
|
|
|
// If we're between values and we encounter a space
|
|
|
|
// or a control character, ignore it
|
|
|
|
if unicode.IsSpace(r) || unicode.IsControl(r) { |
|
|
|
continue |
|
|
|
} |
|
|
|
fallthrough |
|
|
|
|
|
|
|
case ' ', '\t', '\v': |
|
|
|
if started && quote == 0 { |
|
|
|
return i + 1, data[startidx:i], nil |
|
|
|
// If we're between values and we encounter a #, it's
|
|
|
|
// the beginning of a comment
|
|
|
|
if r == '#' { |
|
|
|
state = TK_COMMENT |
|
|
|
continue |
|
|
|
} |
|
|
|
|
|
|
|
case '\'', '"', '`': |
|
|
|
// When the quote ends
|
|
|
|
if quote == c { |
|
|
|
// if we've gotten data, return it
|
|
|
|
if started { |
|
|
|
return i + 1, data[startidx:i], nil |
|
|
|
} |
|
|
|
// if we haven't return nothing
|
|
|
|
return i + 1, []byte{}, nil |
|
|
|
// If we're between values and we get a quote character
|
|
|
|
// treat it as the beginning of a string literal
|
|
|
|
if r == '"' || r == '\'' || r == '`' { |
|
|
|
state = TK_QUOTE |
|
|
|
quo = r |
|
|
|
continue |
|
|
|
} |
|
|
|
|
|
|
|
// start a quoted string
|
|
|
|
if !ignoreline && quote == 0 { |
|
|
|
quote = c |
|
|
|
b.WriteRune(r) |
|
|
|
state = TK_VAL |
|
|
|
|
|
|
|
case TK_VAL: |
|
|
|
// In values, only increment the current line, so
|
|
|
|
// if an error is reported, it reports the line
|
|
|
|
// the value starts on
|
|
|
|
if r == '\n' { |
|
|
|
t.curline++ |
|
|
|
} |
|
|
|
|
|
|
|
case '#': |
|
|
|
if !started { |
|
|
|
ignoreline = true |
|
|
|
// If we're in a normal value and we encounter a space
|
|
|
|
// or a control character, end value
|
|
|
|
if unicode.IsSpace(r) || unicode.IsControl(r) { |
|
|
|
goto end |
|
|
|
} |
|
|
|
b.WriteRune(r) |
|
|
|
|
|
|
|
case TK_QUOTE: |
|
|
|
// In quotes, only increment the current line, so
|
|
|
|
// if an error is reported, it reports the line
|
|
|
|
// the quoted value starts on
|
|
|
|
if r == '\n' { |
|
|
|
t.curline++ |
|
|
|
} |
|
|
|
|
|
|
|
default: |
|
|
|
if !ignoreline && !started { |
|
|
|
started = true |
|
|
|
startidx = i |
|
|
|
// End this quote if it's another quote of the same rune
|
|
|
|
if r == quo { |
|
|
|
goto end |
|
|
|
} |
|
|
|
b.WriteRune(r) |
|
|
|
|
|
|
|
case TK_COMMENT: |
|
|
|
// Comments are ignored, until a new line is encounter
|
|
|
|
// at which point, increment the current and reported line
|
|
|
|
if r == '\n' { |
|
|
|
t.curline++ |
|
|
|
t.repline++ |
|
|
|
state = TK_NONE |
|
|
|
} |
|
|
|
continue |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
if atEOF { |
|
|
|
if quote != 0 { |
|
|
|
return 0, nil, fmt.Errorf("unterminated quote") |
|
|
|
} |
|
|
|
|
|
|
|
if ignoreline { |
|
|
|
return len(data), nil, nil |
|
|
|
} |
|
|
|
if started { |
|
|
|
return len(data), data[startidx:], nil |
|
|
|
end: |
|
|
|
if t.err == nil || t.err == io.EOF { |
|
|
|
if b.Len() > 0 { |
|
|
|
t.text = b.String() |
|
|
|
} |
|
|
|
} |
|
|
|
return t.err == nil |
|
|
|
} |
|
|
|
|
|
|
|
return 0, nil, nil |
|
|
|
func (t *Tokenizer) Text() string { |
|
|
|
return t.text |
|
|
|
} |
|
|
|
|
|
|
|
func (t *Tokenizer) Line() int { |
|
|
|
return t.repline |
|
|
|
} |
|
|
|
|
|
|
|
func (t *Tokenizer) Err() error { |
|
|
|
if t.err == io.EOF { |
|
|
|
return nil |
|
|
|
} |
|
|
|
return t.err |
|
|
|
} |
