merge from main

Signed-off-by: hfuss <[email protected]>

.github/workflows/go.yml

@@ -18,7 +18,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v4
         with:
-          go-version: "1.21"
+          go-version: "1.22"
 
       - name: Build and Test
         run: make

go.mod

@@ -1,6 +1,6 @@
 module github.com/hyperledger/firefly-common
 
-go 1.21
+go 1.22
 
 require (
 	github.com/DATA-DOG/go-sqlmock v1.5.2

pkg/ffapi/handler.go

@@ -181,7 +181,9 @@ func (hs *HandlerFactory) RouteHandler(route *Route) http.HandlerFunc {
 				fallthrough
 			case strings.HasPrefix(strings.ToLower(contentType), "application/json"):
 				if jsonInput != nil {
-					err = json.NewDecoder(req.Body).Decode(&jsonInput)
+					d := json.NewDecoder(req.Body)
+					d.UseNumber()
+					err = d.Decode(&jsonInput)
 				}
 			case strings.HasPrefix(strings.ToLower(contentType), "text/plain"):
 			default:

pkg/ffapi/handler_test.go

@@ -21,8 +21,6 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"github.com/getkin/kin-openapi/openapi3"
-	"github.com/stretchr/testify/require"
 	"io"
 	"mime/multipart"
 	"net/http"
@@ -31,13 +29,24 @@ import (
 	"testing"
 	"time"
 
+	"github.com/getkin/kin-openapi/openapi3"
+	"github.com/stretchr/testify/require"
+
 	"github.com/gorilla/mux"
 	"github.com/hyperledger/firefly-common/pkg/config"
 	"github.com/hyperledger/firefly-common/pkg/httpserver"
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 )
 
+const largeParamLiteral = `{
+	"largeNumberParam": 10000000000000000000000000001
+}`
+
+const scientificParamLiteral = `{
+	"scientificNumberParam": 1e+24
+}`
+
 const configDir = "../../test/data/config"
 
 func newTestHandlerFactory(basePath string, basePathParams []*PathParam) *HandlerFactory {
@@ -112,6 +121,62 @@ func TestRouteServePOST201WithParams(t *testing.T) {
 	assert.Equal(t, "value2", resJSON["output1"])
 }
 
+func TestRouteServePOST201WithParamsLargeNumber(t *testing.T) {
+	s, _, done := newTestServer(t, []*Route{{
+		Name:            "testRoute",
+		Path:            "/test/{something}",
+		Method:          "POST",
+		PathParams:      []*PathParam{},
+		QueryParams:     []*QueryParam{},
+		JSONInputValue:  func() interface{} { return make(map[string]interface{}) },
+		JSONOutputValue: func() interface{} { return make(map[string]interface{}) },
+		JSONOutputCodes: []int{201},
+		JSONHandler: func(r *APIRequest) (output interface{}, err error) {
+			assert.Equal(t, r.Input, map[string]interface{}{"largeNumberParam": json.Number("10000000000000000000000000001")})
+			// Echo the input back as the response
+			return r.Input, nil
+		},
+	}}, "", nil)
+	defer done()
+
+	res, err := http.Post(fmt.Sprintf("http://%s/test/stuff", s.Addr()), "application/json", bytes.NewReader([]byte(largeParamLiteral)))
+	assert.NoError(t, err)
+	assert.Equal(t, 201, res.StatusCode)
+	var resJSON map[string]interface{}
+	d := json.NewDecoder(res.Body)
+	d.UseNumber()
+	d.Decode(&resJSON)
+	assert.Equal(t, json.Number("10000000000000000000000000001"), resJSON["largeNumberParam"])
+}
+
+func TestRouteServePOST201WithParamsScientificNumber(t *testing.T) {
+	s, _, done := newTestServer(t, []*Route{{
+		Name:            "testRoute",
+		Path:            "/test/{something}",
+		Method:          "POST",
+		PathParams:      []*PathParam{},
+		QueryParams:     []*QueryParam{},
+		JSONInputValue:  func() interface{} { return make(map[string]interface{}) },
+		JSONOutputValue: func() interface{} { return make(map[string]interface{}) },
+		JSONOutputCodes: []int{201},
+		JSONHandler: func(r *APIRequest) (output interface{}, err error) {
+			assert.Equal(t, r.Input, map[string]interface{}{"scientificNumberParam": json.Number("1e+24")})
+			// Echo the input back as the response
+			return r.Input, nil
+		},
+	}}, "", nil)
+	defer done()
+
+	res, err := http.Post(fmt.Sprintf("http://%s/test/stuff", s.Addr()), "application/json", bytes.NewReader([]byte(scientificParamLiteral)))
+	assert.NoError(t, err)
+	assert.Equal(t, 201, res.StatusCode)
+	var resJSON map[string]interface{}
+	d := json.NewDecoder(res.Body)
+	d.UseNumber()
+	d.Decode(&resJSON)
+	assert.Equal(t, json.Number("1e+24"), resJSON["scientificNumberParam"])
+}
+
 func TestJSONHTTPResponseEncodeFail(t *testing.T) {
 	s, _, done := newTestServer(t, []*Route{{
 		Name:            "testRoute",

pkg/fftls/fftls.go

@@ -89,20 +89,34 @@ func NewTLSConfig(ctx context.Context, config *Config, tlsType TLSType) (*tls.Co
 
 	tlsConfig.RootCAs = rootCAs
 
+	var configuredCert *tls.Certificate
 	// For mTLS we need both the cert and key
 	if config.CertFile != "" && config.KeyFile != "" {
 		// Read the key pair to create certificate
 		cert, err := tls.LoadX509KeyPair(config.CertFile, config.KeyFile)
 		if err != nil {
 			return nil, i18n.WrapError(ctx, err, i18n.MsgInvalidKeyPairFiles)
 		}
-		tlsConfig.Certificates = []tls.Certificate{cert}
+		configuredCert = &cert
 	} else if config.Cert != "" && config.Key != "" {
 		cert, err := tls.X509KeyPair([]byte(config.Cert), []byte(config.Key))
 		if err != nil {
 			return nil, i18n.WrapError(ctx, err, i18n.MsgInvalidKeyPairFiles)
 		}
-		tlsConfig.Certificates = []tls.Certificate{cert}
+		configuredCert = &cert
+	}
+
+	if configuredCert != nil {
+		// Rather than letting Golang pick a certificate it thinks matches from the list of one,
+		// we directly supply it the one we have in all cases.
+		tlsConfig.GetClientCertificate = func(_ *tls.CertificateRequestInfo) (*tls.Certificate, error) {
+			log.L(ctx).Debugf("Supplying client certificate")
+			return configuredCert, nil
+		}
+		tlsConfig.GetCertificate = func(_ *tls.ClientHelloInfo) (*tls.Certificate, error) {
+			log.L(ctx).Debugf("Supplying server certificate")
+			return configuredCert, nil
+		}
 	}
 
 	if tlsType == ServerType {

pkg/fftypes/jsonany.go

@@ -1,4 +1,4 @@
-// Copyright © 2023 Kaleido, Inc.
+// Copyright © 2024 Kaleido, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -21,6 +21,7 @@ import (
 	"crypto/sha256"
 	"database/sql/driver"
 	"encoding/json"
+	"strings"
 
 	"github.com/hyperledger/firefly-common/pkg/i18n"
 	"github.com/hyperledger/firefly-common/pkg/log"
@@ -76,7 +77,18 @@ func (h *JSONAny) Unmarshal(ctx context.Context, v interface{}) error {
 	if h == nil {
 		return i18n.NewError(ctx, i18n.MsgNilOrNullObject)
 	}
-	return json.Unmarshal([]byte(*h), v)
+
+	if _, ok := v.(*float64); ok {
+		return i18n.NewError(ctx, i18n.MsgUnmarshalToFloat64NotSupported)
+	}
+
+	d := json.NewDecoder(strings.NewReader(h.String()))
+	d.UseNumber()
+	if err := d.Decode(v); err != nil {
+		return err
+	}
+
+	return nil
 }
 
 func (h *JSONAny) Hash() *Bytes32 {

pkg/fftypes/jsonany_test.go

@@ -181,6 +181,51 @@ func TestUnmarshal(t *testing.T) {
 	assert.Equal(t, "value1", myObj.Key1)
 }
 
+func TestUnmarshalHugeNumber(t *testing.T) {
+
+	var myInt64Variable int64
+	var myFloat64Variable float64
+	ctx := context.Background()
+	var h *JSONAny
+	var myObj struct {
+		Key1 interface{} `json:"key1"`
+		Key2 JSONAny     `json:"key2"`
+		Key3 JSONAny     `json:"key3"`
+	}
+
+	h = JSONAnyPtr(`{"key1":123456789123456789123456789, "key2":123456789123456789123456789, "key3":1234}`)
+	err := h.Unmarshal(ctx, &myObj)
+	assert.NoError(t, err)
+	assert.Equal(t, json.Number("123456789123456789123456789"), myObj.Key1)
+
+	assert.NoError(t, err)
+	assert.Equal(t, "123456789123456789123456789", myObj.Key2.String())
+
+	err = myObj.Key2.Unmarshal(ctx, &myInt64Variable)
+	assert.Error(t, err)
+	assert.Regexp(t, "cannot unmarshal number 123456789123456789123456789 into Go value of type int64", err)
+
+	err = myObj.Key3.Unmarshal(ctx, &myInt64Variable)
+	assert.NoError(t, err)
+	assert.Equal(t, int64(1234), myInt64Variable)
+
+	err = myObj.Key2.Unmarshal(ctx, &myFloat64Variable)
+	assert.Error(t, err)
+	assert.Regexp(t, "FF00249", err)
+}
+
+func TestUnmarshalHugeNumberError(t *testing.T) {
+
+	var h *JSONAny
+	var myObj struct {
+		Key1 interface{} `json:"key1"`
+	}
+
+	h = JSONAnyPtr(`{"key1":1234567891invalidchars234569}`)
+	err := h.Unmarshal(context.Background(), &myObj)
+	assert.Error(t, err)
+}
+
 func TestNilHash(t *testing.T) {
 	assert.Nil(t, (*JSONAny)(nil).Hash())
 }

pkg/fswatcher/fswatcher.go

@@ -1,4 +1,4 @@
-// Copyright © 2023 Kaleido, Inc.
+// Copyright © 2024 Kaleido, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -20,6 +20,7 @@ import (
 	"context"
 	"os"
 	"path"
+	"time"
 
 	"github.com/fsnotify/fsnotify"
 	"github.com/hyperledger/firefly-common/pkg/fftypes"
@@ -35,9 +36,19 @@ import (
 // - Only fires if the data in the file is different to the last notification
 // - Does not reload the config - that's the caller's responsibility
 func Watch(ctx context.Context, fullFilePath string, onChange, onClose func()) error {
+	return sync(ctx, fullFilePath, onChange, onClose, nil, nil)
+}
+
+// Reconcile behaves the same as Watch, except it allows for running the onSync func on a provided
+// interval. The default re-sync internal is 1m.
+func Reconcile(ctx context.Context, fullFilePath string, onChange, onClose, onSync func(), resyncInterval *time.Duration) error {
+	return sync(ctx, fullFilePath, onChange, onClose, onSync, resyncInterval)
+}
+
+func sync(ctx context.Context, fullFilePath string, onChange, onClose, onSync func(), resyncInterval *time.Duration) error {
 	filePath := path.Dir(fullFilePath)
 	fileName := path.Base(fullFilePath)
-	log.L(ctx).Debugf("Starting file listener for '%s' in directory '%s'", fileName, filePath)
+	log.L(ctx).Debugf("Starting file reconciler for '%s' in directory '%s'", fileName, filePath)
 
 	watcher, err := fsnotify.NewWatcher()
 	if err == nil {
@@ -46,7 +57,7 @@ func Watch(ctx context.Context, fullFilePath string, onChange, onClose func()) e
 			if onClose != nil {
 				onClose()
 			}
-		}, watcher.Events, watcher.Errors)
+		}, onSync, resyncInterval, watcher.Events, watcher.Errors)
 		err = watcher.Add(filePath)
 	}
 	if err != nil {
@@ -56,9 +67,18 @@ func Watch(ctx context.Context, fullFilePath string, onChange, onClose func()) e
 	return nil
 }
 
-func fsListenerLoop(ctx context.Context, fullFilePath string, onChange, onClose func(), events chan fsnotify.Event, errors chan error) {
+func fsListenerLoop(ctx context.Context, fullFilePath string, onChange, onClose, onSync func(), resyncInterval *time.Duration, events chan fsnotify.Event, errors chan error) {
 	defer onClose()
 
+	timeout := resyncInterval
+	if timeout == nil {
+		timeout = func() *time.Duration {
+			defaultTimeout := time.Minute
+			return &defaultTimeout
+		}()
+	}
+	log.L(ctx).Debugf("re-sync interval set to '%s'", *timeout)
+
 	var lastHash *fftypes.Bytes32
 	for {
 		select {
@@ -83,6 +103,15 @@ func fsListenerLoop(ctx context.Context, fullFilePath string, onChange, onClose
 					lastHash = dataHash
 				}
 			}
+		case <-time.After(*timeout):
+			if onSync != nil {
+				data, err := os.ReadFile(fullFilePath)
+				if err == nil {
+					dataHash := fftypes.HashString(string(data))
+					log.L(ctx).Infof("Config file re-sync. Event=Resync Name=%s Size=%d Hash=%s", fullFilePath, len(data), dataHash)
+					onSync()
+				}
+			}
 		case err, ok := <-errors:
 			if ok {
 				log.L(ctx).Errorf("FSEvent error: %s", err)

pkg/fswatcher/fswatcher_test.go

@@ -1,4 +1,4 @@
-// Copyright © 2022 Kaleido, Inc.
+// Copyright © 2024 Kaleido, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -21,14 +21,15 @@ import (
 	"fmt"
 	"os"
 	"testing"
+	"time"
 
 	"github.com/fsnotify/fsnotify"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/viper"
 	"github.com/stretchr/testify/assert"
 )
 
-func TestFileListenerE2E(t *testing.T) {
+func TestFileReconcilerE2E(t *testing.T) {
 
 	logrus.SetLevel(logrus.DebugLevel)
 	tmpDir := t.TempDir()
@@ -47,14 +48,20 @@ func TestFileListenerE2E(t *testing.T) {
 	// Start listener on config file
 	fsListenerDone := make(chan struct{})
 	fsListenerFired := make(chan bool)
+	reSyncFired := make(chan bool)
+	reSyncInterval := 1 * time.Second
 	ctx, cancelCtx := context.WithCancel(context.Background())
-	err := Watch(ctx, filePath, func() {
+	err := Reconcile(ctx, filePath, func() {
 		err := viper.ReadInConfig()
 		assert.NoError(t, err)
 		fsListenerFired <- true
 	}, func() {
 		close(fsListenerDone)
-	})
+	}, func() {
+		err := viper.ReadInConfig()
+		assert.NoError(t, err)
+		reSyncFired <- true
+	}, &reSyncInterval)
 	assert.NoError(t, err)
 
 	// Delete and rename in another file
@@ -63,6 +70,7 @@ func TestFileListenerE2E(t *testing.T) {
 	os.Rename(fmt.Sprintf("%s/another.yaml", tmpDir), fmt.Sprintf("%s/test.yaml", tmpDir))
 	<-fsListenerFired
 	assert.Equal(t, "two", viper.Get("ut_conf"))
+	<-reSyncFired
 
 	defer func() {
 		cancelCtx()
@@ -74,7 +82,7 @@ func TestFileListenerE2E(t *testing.T) {
 
 }
 
-func TestFileListenerFail(t *testing.T) {
+func TestFileWatcherFail(t *testing.T) {
 
 	logrus.SetLevel(logrus.DebugLevel)
 	tmpDir := t.TempDir()
@@ -95,7 +103,7 @@ func TestFileListenerLogError(t *testing.T) {
 	defer cancelCtx()
 	errors := make(chan error)
 	fsListenerDone := make(chan struct{})
-	go fsListenerLoop(ctx, "somefile", func() {}, func() { close(fsListenerDone) }, make(chan fsnotify.Event), errors)
+	go fsListenerLoop(ctx, "somefile", func() {}, func() { close(fsListenerDone) }, nil, nil, make(chan fsnotify.Event), errors)
 
 	errors <- fmt.Errorf("pop")
 	cancelCtx()