diff --git a/utils/api.go b/utils/api.go index 4d99158f9..0b5dc2c6a 100644 --- a/utils/api.go +++ b/utils/api.go @@ -24,7 +24,13 @@ func GetDataFromAPI(dataSourceURLStruct types.DataSourceURL, localCache *cache.L Timeout: time.Duration(HTTPTimeout) * time.Second, } - cachedData, found := localCache.Read(dataSourceURLStruct.URL) + cacheKey, err := generateCacheKey(dataSourceURLStruct.URL, dataSourceURLStruct.Body) + if err != nil { + log.Errorf("Error in generating cache key for API %s: %v", dataSourceURLStruct.URL, err) + return nil, err + } + + cachedData, found := localCache.Read(cacheKey) if found { log.Debugf("Getting Data for URL %s from local cache...", dataSourceURLStruct.URL) return cachedData, nil @@ -36,7 +42,7 @@ func GetDataFromAPI(dataSourceURLStruct types.DataSourceURL, localCache *cache.L } // Storing the data into cache - localCache.Update(response, dataSourceURLStruct.URL, time.Now().Add(time.Second*time.Duration(core.StateLength)).Unix()) + localCache.Update(response, cacheKey, time.Now().Add(time.Second*time.Duration(core.StateLength)).Unix()) return response, nil } diff --git a/utils/api_test.go b/utils/api_test.go index b6b219f4b..f01627e6b 100644 --- a/utils/api_test.go +++ b/utils/api_test.go @@ -150,6 +150,19 @@ func TestGetDataFromAPI(t *testing.T) { want: nil, wantErr: true, }, + { + name: "Gneerating cache key throws error", + args: args{ + urlStruct: types.DataSourceURL{ + URL: "http://example.com", + Body: map[string]interface{}{ + "key": func() {}, // functions cannot be marshaled and will cause an error + }, + }, + }, + want: nil, + wantErr: true, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/utils/asset_test.go b/utils/asset_test.go index 98dde03de..88216dea9 100644 --- a/utils/asset_test.go +++ b/utils/asset_test.go @@ -554,95 +554,95 @@ func TestGetAllCollections(t *testing.T) { func TestGetDataToCommitFromJobs(t *testing.T) { jobsArray := []bindings.StructsJob{ - {Id: 1, SelectorType: 1, Weight: 100, - Power: 2, Name: "ethusd_gemini", Selector: "last", - Url: `{"type": "GET","url": "https://api.gemini.com/v1/pubticker/ethusd","body": {},"header": {}}`, - }, {Id: 2, SelectorType: 1, Weight: 100, + {Id: 1, SelectorType: 0, Weight: 10, Power: 2, Name: "ethusd_gemini", Selector: "last", - Url: `{"type": "GET","url": "https://api.gemini.com/v1/pubticker/ethusd","body": {},"header": {}}`, + Url: "https://api.gemini.com/v1/pubticker/ethusd", + }, + {Id: 2, SelectorType: 0, Weight: 20, + Power: 2, Name: "ethusd_kraken", Selector: "result.XETHZUSD.c[0]", + Url: `{"type": "GET","url": "https://api.kraken.com/0/public/Ticker?pair=ETHUSD","body": {},"header": {}}`, + }, + {Id: 3, SelectorType: 0, Weight: 30, + Power: 2, Name: "ethusd_kucoin", Selector: "data.ETH", + Url: `{"type": "GET","url": "https://api.kucoin.com/api/v1/prices?base=USD¤cies=ETH","body": {},"header": {}}`, + }, + {Id: 4, SelectorType: 0, Weight: 40, + Power: 2, Name: "ethusd_coinbase", Selector: "data.amount", + Url: `{"type": "GET","url": "https://api.coinbase.com/v2/prices/ETH-USD/spot","body": {},"header": {}}`, + }, + // This job returns an error which will not add any value to data or weight array + {Id: 5, SelectorType: 0, Weight: 10, + Power: 2, Name: "ethusd_gemini_incorrect", Selector: "last1", + Url: `{"type": "GET","url": "https://api.gemini.com/v1/pubticker/ethusd1","body": {},"header": {}}`, + }, + {Id: 6, SelectorType: 0, Weight: 100, + Power: 6, Name: "ethusd_uniswapv2", Selector: "result", + Url: `{"type": "POST","url": "https://rpc.ankr.com/eth","body": {"jsonrpc":"2.0","id":7269270904970082,"method":"eth_call","params":[{"from":"0x0000000000000000000000000000000000000000","data":"0xd06ca61f0000000000000000000000000000000000000000000000000de0b6b3a76400000000000000000000000000000000000000000000000000000000000000000040000000000000000000000000000000000000000000000000000000000000000200000000000000000000000050de6856358cc35f3a9a57eaaa34bd4cb707d2cd0000000000000000000000008e870d67f660d95d5be530380d0ec0bd388289e1","to":"0x7a250d5630b4cf539739df2c5dacb4c659f2488d"},"latest"]},"header": {"content-type": "application/json"}, "returnType": "hexArray[1]"}`, + }, + {Id: 7, SelectorType: 0, Weight: 100, + Power: 2, Name: "ethusd_uniswapv3", Selector: "result", + Url: `{"type": "POST","url": "https://rpc.ankr.com/eth","body": {"jsonrpc":"2.0","method":"eth_call","params":[{"to":"0xb27308f9f90d607463bb33ea1bebb41c27ce5ab6","data":"0xf7729d43000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2000000000000000000000000a0b86991c6218b36c1d19d4a2e9eb0ce3606eb480000000000000000000000000000000000000000000000000000000000000bb80000000000000000000000000000000000000000000000000de0b6b3a76400000000000000000000000000000000000000000000000000000000000000000000"},"latest"],"id":5},"header": {"content-type": "application/json"}, "returnType": "hex"}`, + }, + // This is a duplicate job to check if cache is working correctly for POST Jobs + {Id: 8, SelectorType: 0, Weight: 100, + Power: 2, Name: "ethusd_uniswapv3_duplicate", Selector: "result", + Url: `{"type": "POST","url": "https://rpc.ankr.com/eth","body": {"jsonrpc":"2.0","method":"eth_call","params":[{"to":"0xb27308f9f90d607463bb33ea1bebb41c27ce5ab6","data":"0xf7729d43000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2000000000000000000000000a0b86991c6218b36c1d19d4a2e9eb0ce3606eb480000000000000000000000000000000000000000000000000000000000000bb80000000000000000000000000000000000000000000000000de0b6b3a76400000000000000000000000000000000000000000000000000000000000000000000"},"latest"],"id":5},"header": {"content-type": "application/json"}, "returnType": "hex"}`, + }, + // This is a duplicate job to check if cache is working correctly for GET Jobs + {Id: 9, SelectorType: 0, Weight: 10, + Power: 2, Name: "ethusd_gemini_duplicate", Selector: "last", + Url: "https://api.gemini.com/v1/pubticker/ethusd", }, } type args struct { - jobPath string - jobPathErr error - overrideJobData map[string]*types.StructsJob - overrideJobDataErr error - dataToAppend *big.Int - dataToAppendErr error + jobs []bindings.StructsJob } + tests := []struct { - name string - args args - want []*big.Int - wantErr bool + name string + args args + wantArrayLength int + wantErr bool }{ { - name: "Test 1: When GetDataToCommitFromJobs() executes successfully", + name: "Test 1: Getting values from set of jobs of length 4", args: args{ - jobPath: "", - overrideJobData: map[string]*types.StructsJob{"1": { - Id: 2, SelectorType: 1, Weight: 100, - Power: 2, Name: "ethusd_gemini", Selector: "last", - Url: `{"type": "GET","url": "https://api.gemini.com/v1/pubticker/ethusd","body": {},"header": {}}`, - }}, - dataToAppend: big.NewInt(1), + jobs: jobsArray[:4], }, - want: []*big.Int{big.NewInt(1), big.NewInt(1)}, - wantErr: false, + wantArrayLength: 4, }, { - name: "Test 2: When there is an error in getting overrideJobData", + name: "Test 2: Getting values from set of jobs of length 2", args: args{ - jobPath: "", - overrideJobDataErr: errors.New("overrideJobData error"), - dataToAppend: big.NewInt(1), + jobs: jobsArray[:2], }, - want: []*big.Int{big.NewInt(1), big.NewInt(1)}, - wantErr: false, + wantArrayLength: 2, }, { - name: "Test 3: When there is an error in getting jobPath", + name: "Test 3: Getting values from whole set of jobs of length 9 but job at index 5 reports an error", args: args{ - jobPathErr: errors.New("jobPath error"), - overrideJobData: map[string]*types.StructsJob{}, - dataToAppend: big.NewInt(1), + jobs: jobsArray, }, - want: []*big.Int{big.NewInt(1), big.NewInt(1)}, - wantErr: false, - }, - { - name: "Test 4: When there is an error in getting dataToAppend", - args: args{ - jobPath: "", - overrideJobData: map[string]*types.StructsJob{}, - dataToAppendErr: errors.New("dataToAppend error"), - }, - want: nil, - wantErr: false, + wantArrayLength: 8, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - utilsMock := new(mocks.Utils) - pathMock := new(mocks.PathUtils) + UtilsInterface = &UtilsStruct{} + lc := cache.NewLocalCache(time.Second * 20) - optionsPackageStruct := OptionsPackageStruct{ - UtilsInterface: utilsMock, - PathInterface: pathMock, - } - utils := StartRazor(optionsPackageStruct) - - utilsMock.On("GetDataToCommitFromJob", mock.Anything, mock.Anything).Return(tt.args.dataToAppend, tt.args.dataToAppendErr) - - got, _, err := utils.GetDataToCommitFromJobs(jobsArray, &cache.LocalCache{}) + gotDataArray, gotWeightArray, err := UtilsInterface.GetDataToCommitFromJobs(tt.args.jobs, lc) if (err != nil) != tt.wantErr { - t.Errorf("GetDataToCommitFromJobs() error = %v, wantErr %v", err, tt.wantErr) + t.Errorf("GetDataToCommitFromJob() error = %v, wantErr %v", err, tt.wantErr) return } - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("GetDataToCommitFromJobs() got = %v, want %v", got, tt.want) + + if len(gotDataArray) != tt.wantArrayLength || len(gotWeightArray) != tt.wantArrayLength { + t.Errorf("GetDataToCommitFromJobs() got = %v, want %v", gotDataArray, tt.wantArrayLength) } + fmt.Println("Got Data Array: ", gotDataArray) + fmt.Println("Got WeightArray: ", gotWeightArray) }) } } diff --git a/utils/hash.go b/utils/hash.go index 104a9364b..5a54b2182 100644 --- a/utils/hash.go +++ b/utils/hash.go @@ -1,10 +1,12 @@ package utils import ( + "encoding/json" "fmt" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/crypto" + solsha3 "github.com/miguelmota/go-solidity-sha3" ) func EcRecover(data, sig hexutil.Bytes) (common.Address, error) { @@ -23,3 +25,19 @@ func SignHash(data []byte) []byte { msg := fmt.Sprintf("\x19Ethereum Signed Message:\n%d%s", len(data), data) return crypto.Keccak256([]byte(msg)) } + +func generateCacheKey(url string, body map[string]interface{}) (string, error) { + var bodyString string + if body != nil { + // Convert the body to a JSON string + bodyBytes, err := json.Marshal(body) + if err != nil { + log.Error("Error in marshalling body: ", err) + return "", err + } + bodyString = string(bodyBytes) + } + + hash := solsha3.SoliditySHA3([]string{"string", "string"}, []interface{}{url, bodyString}) + return common.BytesToHash(hash).Hex(), nil +} diff --git a/utils/hash_test.go b/utils/hash_test.go index 0033da8f6..ea05c00d2 100644 --- a/utils/hash_test.go +++ b/utils/hash_test.go @@ -65,3 +65,67 @@ func TestEcRecover(t *testing.T) { }) } } + +func Test_generateCacheKey(t *testing.T) { + type args struct { + url string + body map[string]interface{} + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + { + name: "Test when the request is of type GET with body as nil", + args: args{ + url: "https:api.gemini.com/v1/pubticker", + body: nil, + }, + want: "0x9e9d7684772b773287cfd315aea97e574b7059bf63599ad1e008b9f695ab46e2", + wantErr: false, + }, + { + name: "Test when the request is of type POST with body", + args: args{ + url: "https://staging-v3.skalenodes.com/v1/staging-aware-chief-gianfar", + body: map[string]interface{}{"jsonrpc": "2.0", "method": "eth_chainId", "params": nil, "id": 0}, + }, + want: "0xecd57ac05ee0584932ac9b969f6e8a851b7a5f1bbd46ae756f48ad9e2747a0ff", + wantErr: false, + }, + { + name: "Test when url and body is nil", + args: args{ + url: "", + body: nil, + }, + want: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", + wantErr: false, + }, + { + name: "Test when marshalling fails", + args: args{ + url: "http://example.com", + body: map[string]interface{}{ + "key": func() {}, // functions cannot be marshaled and will cause an error + }, + }, + want: "", + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := generateCacheKey(tt.args.url, tt.args.body) + if (err != nil) != tt.wantErr { + t.Errorf("generateCacheKey() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("generateCacheKey() got = %v, want %v", got, tt.want) + } + }) + } +}