improved tokenizer, added tests

2021-01-18 07:29:38 -09:00
parent b675d8714e
commit cd73181a62
11 changed files with 210 additions and 149 deletions
--- a/config/testdata/comments-inline.tok
+++ b/config/testdata/comments-inline.tok
@ -0,0 +1,5 @@
+one #one
+"two#three"
+# "three"
+four
+# EOF
--- a/config/testdata/comments.tok
+++ b/config/testdata/comments.tok
@ -1,5 +1,6 @@
-# one two three
-one two three
-#four five six
-four five six #seven eight nine
-# EOF
+# one
+one
+#two
+two
+   # three
+   three
--- a/config/testdata/quotes-multiline.tok
+++ b/config/testdata/quotes-multiline.tok
@ -0,0 +1,6 @@
+"one
+two"   'three
+four'
+
+`five
+  six`
--- a/config/testdata/quotes.tok
+++ b/config/testdata/quotes.tok
@ -1,8 +1,3 @@
-"one" "two three"
+"one" 'two' `three`

-"four five"
-
-" #six" "" "seven" "ei" "ght"
-
-"multi
-line"
+`four` 'five' "six"
--- a/config/testdata/simple-multiline.tok
+++ b/config/testdata/simple-multiline.tok
--- a/config/testdata/simple-spaces.tok
+++ b/config/testdata/simple-spaces.tok
@ -0,0 +1 @@
+one two three    four five six
--- a/config/testdata/simple.tok
+++ b/config/testdata/simple.tok
@ -1 +0,0 @@
-one two three four five six