support token stream

base62/base62_test.go

@@ -1,29 +1,31 @@
 package base62
 
 import (
-  "fmt"
-  "testing"
-  assert "github.com/pilu/miniassert"
+	"fmt"
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
 )
 
 func TestEncode(t *testing.T) {
-  assert.Equal(t, "0", Encode(0))
-  assert.Equal(t, "1B", Encode(99))
+	So("0", ShouldEqual, Encode(0))
+	So("0", ShouldEqual, Encode(0))
+	So("1B", ShouldEqual, Encode(99))
 }
 
 func TestDecode(t *testing.T) {
-  assert.Equal(t, 0,  Decode("0"))
-  assert.Equal(t, 99, Decode("1B"))
+	So(0, ShouldEqual, Decode("0"))
+	So(99, ShouldEqual, Decode("1B"))
 }
 
 func ExampleEncode() {
-  fmt.Println(Encode(99))
-  // Output:
-  // 1B
+	fmt.Println(Encode(99))
+	// Output:
+	// 1B
 }
 
 func ExampleDecode() {
-  fmt.Println(Decode("1B"))
-  // Output:
-  // 99
+	fmt.Println(Decode("1B"))
+	// Output:
+	// 99
 }

go.mod

@@ -1,10 +1,17 @@
 module github.com/gogather/com
 
-go 1.16
+go 1.18
 
 require (
 	github.com/pilu/miniassert v0.0.0-20140522125902-bee63581261a
-	github.com/smartystreets/goconvey v1.6.4
+	github.com/smartystreets/goconvey v1.8.1
 	github.com/toolkits/file v0.0.0-20160325033739-a5b3c5147e07
-	golang.org/x/net v0.0.0-20210614182718-04defd469f4e
+	golang.org/x/net v0.24.0
+)
+
+require (
+	github.com/gopherjs/gopherjs v1.17.2 // indirect
+	github.com/jtolds/gls v4.20.0+incompatible // indirect
+	github.com/smarty/assertions v1.15.0 // indirect
+	github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d // indirect
 )

stream/token.go

@@ -0,0 +1,44 @@
+package stream
+
+type TokenStream[T any] interface {
+	Next() *T
+	Prev() *T
+	Peek() *T
+	EOF() bool
+}
+
+type commonTokenStream[T any] struct {
+	pos    int
+	tokens []T
+}
+
+func NewTokenStream[T any](input []T) TokenStream[T] {
+	return &commonTokenStream[T]{tokens: input, pos: -1}
+}
+
+func (s *commonTokenStream[T]) Next() *T {
+	if s.pos < len(s.tokens)-1 {
+		s.pos++
+		return &s.tokens[s.pos]
+	}
+	return nil
+}
+
+func (s *commonTokenStream[T]) Prev() *T {
+	if s.pos > 0 {
+		s.pos--
+		return &s.tokens[s.pos]
+	}
+	return nil
+}
+
+func (s *commonTokenStream[T]) Peek() *T {
+	if s.pos < len(s.tokens)-1 {
+		return &s.tokens[s.pos+1]
+	}
+	return nil
+}
+
+func (s *commonTokenStream[T]) EOF() bool {
+	return s.pos == len(s.tokens)-1
+}

stream/token_test.go

@@ -0,0 +1,18 @@
+package stream
+
+import (
+	"fmt"
+	"testing"
+)
+
+func TestTokenStream(t *testing.T) {
+	input := "x = 1 + 2 * 3"
+	stream := NewTokenStream([]byte(input))
+
+	for !stream.EOF() {
+		token := stream.Next()
+		fmt.Printf("Token -> %c\n", *token)
+	}
+
+	fmt.Println("stream:", stream)
+}