From 5c1aa1e14af75ffd7f9f1f6e88ffd88b294aeaee Mon Sep 17 00:00:00 2001 From: nick Date: Tue, 20 Feb 2024 18:21:18 +0900 Subject: [PATCH] fix: test hanging error fix --- node/pkg/admin/tests/main_test.go | 1 + node/pkg/db/pgsql.go | 16 ++ node/pkg/fetcher/fetcher.go | 2 +- node/pkg/fetcher/fetcher_test.go | 248 +---------------------------- node/pkg/fetcher/main_test.go | 249 +++++++++++++++++++++++++++++- 5 files changed, 266 insertions(+), 250 deletions(-) diff --git a/node/pkg/admin/tests/main_test.go b/node/pkg/admin/tests/main_test.go index 2f578ffbb..7f28ce186 100644 --- a/node/pkg/admin/tests/main_test.go +++ b/node/pkg/admin/tests/main_test.go @@ -55,6 +55,7 @@ func cleanup() { func TestMain(m *testing.M) { // setup code := m.Run() + db.ClosePool() db.CloseRedis() // teardown diff --git a/node/pkg/db/pgsql.go b/node/pkg/db/pgsql.go index 3261c7e1f..f03f7be47 100644 --- a/node/pkg/db/pgsql.go +++ b/node/pkg/db/pgsql.go @@ -43,6 +43,20 @@ func loadPgsqlConnectionString() string { return os.Getenv("DATABASE_URL") } +func QueryWithoutResult(ctx context.Context, queryString string, args map[string]any) error { + pool, err := GetPool(ctx) + if err != nil { + return err + } + rows, err := query(pool, queryString, args) + if err != nil { + return err + } + rows.Close() + return nil +} + +// Using this raw function is highly unrecommended since rows should be manually closed func Query(ctx context.Context, queryString string, args map[string]any) (pgx.Rows, error) { pool, err := GetPool(ctx) if err != nil { @@ -83,6 +97,7 @@ func queryRow[T any](pool *pgxpool.Pool, _query string, args map[string]any) (T, if errors.Is(err, pgx.ErrNoRows) { return result, nil } + defer rows.Close() return result, err } @@ -98,6 +113,7 @@ func queryRows[T any](pool *pgxpool.Pool, _query string, args map[string]any) ([ if errors.Is(err, pgx.ErrNoRows) { return results, nil } + defer rows.Close() return results, err } diff --git a/node/pkg/fetcher/fetcher.go b/node/pkg/fetcher/fetcher.go index f48b379ca..fe0929878 100644 --- a/node/pkg/fetcher/fetcher.go +++ b/node/pkg/fetcher/fetcher.go @@ -53,7 +53,7 @@ func (f *Fetcher) runAdapter(ctx context.Context) error { } func (f *Fetcher) insertPgsql(ctx context.Context, name string, value float64) error { - _, err := db.Query(ctx, InsertLocalAggregateQuery, map[string]any{"name": name, "value": int64(value)}) + err := db.QueryWithoutResult(ctx, InsertLocalAggregateQuery, map[string]any{"name": name, "value": int64(value)}) return err } diff --git a/node/pkg/fetcher/fetcher_test.go b/node/pkg/fetcher/fetcher_test.go index b541a53d3..002d06d2b 100644 --- a/node/pkg/fetcher/fetcher_test.go +++ b/node/pkg/fetcher/fetcher_test.go @@ -3,258 +3,13 @@ package fetcher import ( "context" "encoding/json" - "strconv" "testing" - "bisonai.com/orakl/node/pkg/admin/adapter" - "bisonai.com/orakl/node/pkg/admin/tests" - "bisonai.com/orakl/node/pkg/admin/utils" "bisonai.com/orakl/node/pkg/bus" "bisonai.com/orakl/node/pkg/db" - "github.com/gofiber/fiber/v2" "github.com/stretchr/testify/assert" ) -var sampleData = `{ - "name": "BNB-USDT", - "feeds": [{ - "name": "Bybit-BNB-USDT", - "definition": { - "url": "https://api.bybit.com/derivatives/v3/public/tickers?symbol=BNBUSDT", - "headers": { - "Content-Type": "application/json" - }, - "method": "GET", - "reducers": [ - { - "function": "PARSE", - "args": [ - "result", - "list" - ] - }, - { - "function": "INDEX", - "args": 0 - }, - { - "function": "PARSE", - "args": [ - "lastPrice" - ] - }, - { - "function": "POW10", - "args": 8 - }, - { - "function": "ROUND" - } - ] - } - }, - { - "name": "Binance-BNB-USDT", - "definition": { - "url": "https://api.binance.com/api/v3/avgPrice?symbol=BNBUSDT", - "headers": { - "Content-Type": "application/json" - }, - "method": "GET", - "reducers": [ - { - "function": "PARSE", - "args": [ - "price" - ] - }, - { - "function": "POW10", - "args": 8 - }, - { - "function": "ROUND" - } - ] - } - }, - { - "name": "Coinbase-BNB-USDT", - "definition": { - "url": "https://api.coinbase.com/v2/exchange-rates?currency=BNB", - "headers": { - "Content-Type": "application/json" - }, - "method": "GET", - "reducers": [ - { - "function": "PARSE", - "args": [ - "data", - "rates", - "USDT" - ] - }, - { - "function": "POW10", - "args": 8 - }, - { - "function": "ROUND" - } - ] - } - }, - { - "name": "Kucoin-BNB-USDT", - "definition": { - "url": "https://api.kucoin.com/api/v1/market/orderbook/level1?symbol=BNB-USDT", - "headers": { - "Content-Type": "application/json" - }, - "method": "GET", - "reducers": [ - { - "function": "PARSE", - "args": [ - "data", - "price" - ] - }, - { - "function": "POW10", - "args": 8 - }, - { - "function": "ROUND" - } - ] - } - }, - { - "name": "Btse-BNB-USDT", - "definition": { - "url": "https://api.btse.com/spot/api/v3.2/price?symbol=BNB-USDT", - "headers": { - "Content-Type": "application/json" - }, - "method": "GET", - "reducers": [ - { - "function": "INDEX", - "args": 0 - }, - { - "function": "PARSE", - "args": [ - "indexPrice" - ] - }, - { - "function": "POW10", - "args": 8 - }, - { - "function": "ROUND" - } - ] - } - }, - { - "name": "Gateio-BNB-USDT", - "definition": { - "url": "https://api.gateio.ws/api/v4/spot/tickers?currency_pair=BNB_USDT", - "headers": { - "Content-Type": "application/json" - }, - "method": "GET", - "reducers": [ - { - "function": "INDEX", - "args": 0 - }, - { - "function": "PARSE", - "args": [ - "last" - ] - }, - { - "function": "POW10", - "args": 8 - }, - { - "function": "ROUND" - } - ] - } - }, - { - "name": "Coinex-BNB-USDT", - "definition": { - "url": "https://api.coinex.com/v1/market/ticker?market=BNBUSDT", - "headers": { - "Content-Type": "application/json" - }, - "method": "GET", - "reducers": [ - { - "function": "PARSE", - "args": [ - "data", - "ticker", - "last" - ] - }, - { - "function": "POW10", - "args": 8 - }, - { - "function": "ROUND" - } - ] - } - }] -}` - -var insertResult adapter.AdapterModel - -func setup() (*fiber.App, error) { - app, err := utils.Setup("") - if err != nil { - return nil, err - } - v1 := app.Group("/api/v1") - adapter.Routes(v1) - - return app, nil -} - -func insertSampleData(app *fiber.App, ctx context.Context) error { - var insertData adapter.AdapterInsertModel - err := json.Unmarshal([]byte(sampleData), &insertData) - if err != nil { - return err - } - - tmp, err := tests.PostRequest[adapter.AdapterModel](app, "/api/v1/adapter", insertData) - if err != nil { - return err - } - - insertResult = tmp - return nil -} - -func cleanupSampleData(app *fiber.App, ctx context.Context) error { - _, err := tests.DeleteRequest[adapter.AdapterModel](app, "/api/v1/adapter/"+strconv.FormatInt(*insertResult.Id, 10), nil) - if err != nil { - return err - } - return nil -} - func TestFetcherInitialize(t *testing.T) { admin, err := setup() if err != nil { @@ -266,6 +21,7 @@ func TestFetcherInitialize(t *testing.T) { } defer admin.Shutdown() defer cleanupSampleData(admin, context.Background()) + b := bus.NewMessageBus() fetcher := NewFetcher(b) fetcher.initialize(context.Background()) @@ -323,7 +79,7 @@ func TestFetcherRunAdapter(t *testing.T) { assert.NotNil(t, pgResult) // cleanup aggregate from db - _, err = db.Query(context.Background(), "DELETE FROM local_aggregates WHERE name = @name", map[string]any{"name": fetcher.Adapters[0].Name}) + err = db.QueryWithoutResult(context.Background(), "DELETE FROM local_aggregates WHERE name = @name", map[string]any{"name": fetcher.Adapters[0].Name}) if err != nil { t.Fatalf("error cleaning up from db: %v", err) } diff --git a/node/pkg/fetcher/main_test.go b/node/pkg/fetcher/main_test.go index 4dff008ae..25ce3ec8c 100644 --- a/node/pkg/fetcher/main_test.go +++ b/node/pkg/fetcher/main_test.go @@ -2,17 +2,260 @@ package fetcher import ( "context" + "encoding/json" "os" + "strconv" "testing" + "bisonai.com/orakl/node/pkg/admin/adapter" + "bisonai.com/orakl/node/pkg/admin/tests" + "bisonai.com/orakl/node/pkg/admin/utils" "bisonai.com/orakl/node/pkg/db" + "github.com/gofiber/fiber/v2" ) +var sampleData = `{ + "name": "BNB-USDT", + "feeds": [{ + "name": "Bybit-BNB-USDT", + "definition": { + "url": "https://api.bybit.com/derivatives/v3/public/tickers?symbol=BNBUSDT", + "headers": { + "Content-Type": "application/json" + }, + "method": "GET", + "reducers": [ + { + "function": "PARSE", + "args": [ + "result", + "list" + ] + }, + { + "function": "INDEX", + "args": 0 + }, + { + "function": "PARSE", + "args": [ + "lastPrice" + ] + }, + { + "function": "POW10", + "args": 8 + }, + { + "function": "ROUND" + } + ] + } + }, + { + "name": "Binance-BNB-USDT", + "definition": { + "url": "https://api.binance.com/api/v3/avgPrice?symbol=BNBUSDT", + "headers": { + "Content-Type": "application/json" + }, + "method": "GET", + "reducers": [ + { + "function": "PARSE", + "args": [ + "price" + ] + }, + { + "function": "POW10", + "args": 8 + }, + { + "function": "ROUND" + } + ] + } + }, + { + "name": "Coinbase-BNB-USDT", + "definition": { + "url": "https://api.coinbase.com/v2/exchange-rates?currency=BNB", + "headers": { + "Content-Type": "application/json" + }, + "method": "GET", + "reducers": [ + { + "function": "PARSE", + "args": [ + "data", + "rates", + "USDT" + ] + }, + { + "function": "POW10", + "args": 8 + }, + { + "function": "ROUND" + } + ] + } + }, + { + "name": "Kucoin-BNB-USDT", + "definition": { + "url": "https://api.kucoin.com/api/v1/market/orderbook/level1?symbol=BNB-USDT", + "headers": { + "Content-Type": "application/json" + }, + "method": "GET", + "reducers": [ + { + "function": "PARSE", + "args": [ + "data", + "price" + ] + }, + { + "function": "POW10", + "args": 8 + }, + { + "function": "ROUND" + } + ] + } + }, + { + "name": "Btse-BNB-USDT", + "definition": { + "url": "https://api.btse.com/spot/api/v3.2/price?symbol=BNB-USDT", + "headers": { + "Content-Type": "application/json" + }, + "method": "GET", + "reducers": [ + { + "function": "INDEX", + "args": 0 + }, + { + "function": "PARSE", + "args": [ + "indexPrice" + ] + }, + { + "function": "POW10", + "args": 8 + }, + { + "function": "ROUND" + } + ] + } + }, + { + "name": "Gateio-BNB-USDT", + "definition": { + "url": "https://api.gateio.ws/api/v4/spot/tickers?currency_pair=BNB_USDT", + "headers": { + "Content-Type": "application/json" + }, + "method": "GET", + "reducers": [ + { + "function": "INDEX", + "args": 0 + }, + { + "function": "PARSE", + "args": [ + "last" + ] + }, + { + "function": "POW10", + "args": 8 + }, + { + "function": "ROUND" + } + ] + } + }, + { + "name": "Coinex-BNB-USDT", + "definition": { + "url": "https://api.coinex.com/v1/market/ticker?market=BNBUSDT", + "headers": { + "Content-Type": "application/json" + }, + "method": "GET", + "reducers": [ + { + "function": "PARSE", + "args": [ + "data", + "ticker", + "last" + ] + }, + { + "function": "POW10", + "args": 8 + }, + { + "function": "ROUND" + } + ] + } + }] +}` + +var insertResult adapter.AdapterModel + +func setup() (*fiber.App, error) { + app, err := utils.Setup("") + if err != nil { + return nil, err + } + v1 := app.Group("/api/v1") + adapter.Routes(v1) + + return app, nil +} + +func insertSampleData(app *fiber.App, ctx context.Context) error { + var insertData adapter.AdapterInsertModel + err := json.Unmarshal([]byte(sampleData), &insertData) + if err != nil { + return err + } + + tmp, err := tests.PostRequest[adapter.AdapterModel](app, "/api/v1/adapter", insertData) + if err != nil { + return err + } + + insertResult = tmp + return nil +} + +func cleanupSampleData(app *fiber.App, ctx context.Context) error { + _, err := tests.DeleteRequest[adapter.AdapterModel](app, "/api/v1/adapter/"+strconv.FormatInt(*insertResult.Id, 10), nil) + if err != nil { + return err + } + return nil +} + func TestMain(m *testing.M) { // setup - - db.GetPool(context.Background()) - db.GetRedisConn(context.Background()) code := m.Run() db.ClosePool()