refactor: used hash of url and body as a cache key

utils/api.go

@@ -24,7 +24,13 @@ func GetDataFromAPI(dataSourceURLStruct types.DataSourceURL, localCache *cache.L
 		Timeout: time.Duration(HTTPTimeout) * time.Second,
 	}
 
-	cachedData, found := localCache.Read(dataSourceURLStruct.URL)
+	cacheKey, err := generateCacheKey(dataSourceURLStruct.URL, dataSourceURLStruct.Body)
+	if err != nil {
+		log.Errorf("Error in generating cache key for API %s: %v", dataSourceURLStruct.URL, err)
+		return nil, err
+	}
+
+	cachedData, found := localCache.Read(cacheKey)
 	if found {
 		log.Debugf("Getting Data for URL %s from local cache...", dataSourceURLStruct.URL)
 		return cachedData, nil
@@ -36,7 +42,7 @@ func GetDataFromAPI(dataSourceURLStruct types.DataSourceURL, localCache *cache.L
 	}
 
 	// Storing the data into cache
-	localCache.Update(response, dataSourceURLStruct.URL, time.Now().Add(time.Second*time.Duration(core.StateLength)).Unix())
+	localCache.Update(response, cacheKey, time.Now().Add(time.Second*time.Duration(core.StateLength)).Unix())
 	return response, nil
 }
 

utils/api_test.go

@@ -150,6 +150,19 @@ func TestGetDataFromAPI(t *testing.T) {
 			want:    nil,
 			wantErr: true,
 		},
+		{
+			name: "Gneerating cache key throws error",
+			args: args{
+				urlStruct: types.DataSourceURL{
+					URL: "http://example.com",
+					Body: map[string]interface{}{
+						"key": func() {}, // functions cannot be marshaled and will cause an error
+					},
+				},
+			},
+			want:    nil,
+			wantErr: true,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

utils/asset_test.go

@@ -554,95 +554,95 @@ func TestGetAllCollections(t *testing.T) {
 
 func TestGetDataToCommitFromJobs(t *testing.T) {
 	jobsArray := []bindings.StructsJob{
-		{Id: 1, SelectorType: 1, Weight: 100,
-			Power: 2, Name: "ethusd_gemini", Selector: "last",
-			Url: `{"type": "GET","url": "https://api.gemini.com/v1/pubticker/ethusd","body": {},"header": {}}`,
-		}, {Id: 2, SelectorType: 1, Weight: 100,
+		{Id: 1, SelectorType: 0, Weight: 10,
 			Power: 2, Name: "ethusd_gemini", Selector: "last",
-			Url: `{"type": "GET","url": "https://api.gemini.com/v1/pubticker/ethusd","body": {},"header": {}}`,
+			Url: "https://api.gemini.com/v1/pubticker/ethusd",
+		},
+		{Id: 2, SelectorType: 0, Weight: 20,
+			Power: 2, Name: "ethusd_kraken", Selector: "result.XETHZUSD.c[0]",
+			Url: `{"type": "GET","url": "https://api.kraken.com/0/public/Ticker?pair=ETHUSD","body": {},"header": {}}`,
+		},
+		{Id: 3, SelectorType: 0, Weight: 30,
+			Power: 2, Name: "ethusd_kucoin", Selector: "data.ETH",
+			Url: `{"type": "GET","url": "https://api.kucoin.com/api/v1/prices?base=USD&currencies=ETH","body": {},"header": {}}`,
+		},
+		{Id: 4, SelectorType: 0, Weight: 40,
+			Power: 2, Name: "ethusd_coinbase", Selector: "data.amount",
+			Url: `{"type": "GET","url": "https://api.coinbase.com/v2/prices/ETH-USD/spot","body": {},"header": {}}`,
+		},
+		// This job returns an error which will not add any value to data or weight array
+		{Id: 5, SelectorType: 0, Weight: 10,
+			Power: 2, Name: "ethusd_gemini_incorrect", Selector: "last1",
+			Url: `{"type": "GET","url": "https://api.gemini.com/v1/pubticker/ethusd1","body": {},"header": {}}`,
+		},
+		{Id: 6, SelectorType: 0, Weight: 100,
+			Power: 6, Name: "ethusd_uniswapv2", Selector: "result",
+			Url: `{"type": "POST","url": "https://rpc.ankr.com/eth","body": {"jsonrpc":"2.0","id":7269270904970082,"method":"eth_call","params":[{"from":"0x0000000000000000000000000000000000000000","data":"0xd06ca61f0000000000000000000000000000000000000000000000000de0b6b3a76400000000000000000000000000000000000000000000000000000000000000000040000000000000000000000000000000000000000000000000000000000000000200000000000000000000000050de6856358cc35f3a9a57eaaa34bd4cb707d2cd0000000000000000000000008e870d67f660d95d5be530380d0ec0bd388289e1","to":"0x7a250d5630b4cf539739df2c5dacb4c659f2488d"},"latest"]},"header": {"content-type": "application/json"}, "returnType": "hexArray[1]"}`,
+		},
+		{Id: 7, SelectorType: 0, Weight: 100,
+			Power: 2, Name: "ethusd_uniswapv3", Selector: "result",
+			Url: `{"type": "POST","url": "https://rpc.ankr.com/eth","body": {"jsonrpc":"2.0","method":"eth_call","params":[{"to":"0xb27308f9f90d607463bb33ea1bebb41c27ce5ab6","data":"0xf7729d43000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2000000000000000000000000a0b86991c6218b36c1d19d4a2e9eb0ce3606eb480000000000000000000000000000000000000000000000000000000000000bb80000000000000000000000000000000000000000000000000de0b6b3a76400000000000000000000000000000000000000000000000000000000000000000000"},"latest"],"id":5},"header": {"content-type": "application/json"}, "returnType": "hex"}`,
+		},
+		// This is a duplicate job to check if cache is working correctly for POST Jobs
+		{Id: 8, SelectorType: 0, Weight: 100,
+			Power: 2, Name: "ethusd_uniswapv3_duplicate", Selector: "result",
+			Url: `{"type": "POST","url": "https://rpc.ankr.com/eth","body": {"jsonrpc":"2.0","method":"eth_call","params":[{"to":"0xb27308f9f90d607463bb33ea1bebb41c27ce5ab6","data":"0xf7729d43000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2000000000000000000000000a0b86991c6218b36c1d19d4a2e9eb0ce3606eb480000000000000000000000000000000000000000000000000000000000000bb80000000000000000000000000000000000000000000000000de0b6b3a76400000000000000000000000000000000000000000000000000000000000000000000"},"latest"],"id":5},"header": {"content-type": "application/json"}, "returnType": "hex"}`,
+		},
+		// This is a duplicate job to check if cache is working correctly for GET Jobs
+		{Id: 9, SelectorType: 0, Weight: 10,
+			Power: 2, Name: "ethusd_gemini_duplicate", Selector: "last",
+			Url: "https://api.gemini.com/v1/pubticker/ethusd",
 		},
 	}
 
 	type args struct {
-		jobPath            string
-		jobPathErr         error
-		overrideJobData    map[string]*types.StructsJob
-		overrideJobDataErr error
-		dataToAppend       *big.Int
-		dataToAppendErr    error
+		jobs []bindings.StructsJob
 	}
+
 	tests := []struct {
-		name    string
-		args    args
-		want    []*big.Int
-		wantErr bool
+		name            string
+		args            args
+		wantArrayLength int
+		wantErr         bool
 	}{
 		{
-			name: "Test 1: When GetDataToCommitFromJobs() executes successfully",
+			name: "Test 1: Getting values from set of jobs of length 4",
 			args: args{
-				jobPath: "",
-				overrideJobData: map[string]*types.StructsJob{"1": {
-					Id: 2, SelectorType: 1, Weight: 100,
-					Power: 2, Name: "ethusd_gemini", Selector: "last",
-					Url: `{"type": "GET","url": "https://api.gemini.com/v1/pubticker/ethusd","body": {},"header": {}}`,
-				}},
-				dataToAppend: big.NewInt(1),
+				jobs: jobsArray[:4],
 			},
-			want:    []*big.Int{big.NewInt(1), big.NewInt(1)},
-			wantErr: false,
+			wantArrayLength: 4,
 		},
 		{
-			name: "Test 2: When there is an error in getting overrideJobData",
+			name: "Test 2: Getting values from set of jobs of length 2",
 			args: args{
-				jobPath:            "",
-				overrideJobDataErr: errors.New("overrideJobData error"),
-				dataToAppend:       big.NewInt(1),
+				jobs: jobsArray[:2],
 			},
-			want:    []*big.Int{big.NewInt(1), big.NewInt(1)},
-			wantErr: false,
+			wantArrayLength: 2,
 		},
 		{
-			name: "Test 3: When there is an error in getting jobPath",
+			name: "Test 3: Getting values from whole set of jobs of length 9 but job at last index reports an error",
 			args: args{
-				jobPathErr:      errors.New("jobPath error"),
-				overrideJobData: map[string]*types.StructsJob{},
-				dataToAppend:    big.NewInt(1),
+				jobs: jobsArray,
 			},
-			want:    []*big.Int{big.NewInt(1), big.NewInt(1)},
-			wantErr: false,
-		},
-		{
-			name: "Test 4: When there is an error in getting dataToAppend",
-			args: args{
-				jobPath:         "",
-				overrideJobData: map[string]*types.StructsJob{},
-				dataToAppendErr: errors.New("dataToAppend error"),
-			},
-			want:    nil,
-			wantErr: false,
+			wantArrayLength: 8,
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			utilsMock := new(mocks.Utils)
-			pathMock := new(mocks.PathUtils)
+			UtilsInterface = &UtilsStruct{}
+			lc := cache.NewLocalCache(time.Second * 20)
 
-			optionsPackageStruct := OptionsPackageStruct{
-				UtilsInterface: utilsMock,
-				PathInterface:  pathMock,
-			}
-			utils := StartRazor(optionsPackageStruct)
-
-			utilsMock.On("GetDataToCommitFromJob", mock.Anything, mock.Anything).Return(tt.args.dataToAppend, tt.args.dataToAppendErr)
-
-			got, _, err := utils.GetDataToCommitFromJobs(jobsArray, &cache.LocalCache{})
+			gotDataArray, gotWeightArray, err := UtilsInterface.GetDataToCommitFromJobs(tt.args.jobs, lc)
 			if (err != nil) != tt.wantErr {
-				t.Errorf("GetDataToCommitFromJobs() error = %v, wantErr %v", err, tt.wantErr)
+				t.Errorf("GetDataToCommitFromJob() error = %v, wantErr %v", err, tt.wantErr)
 				return
 			}
-			if !reflect.DeepEqual(got, tt.want) {
-				t.Errorf("GetDataToCommitFromJobs() got = %v, want %v", got, tt.want)
+
+			if len(gotDataArray) != tt.wantArrayLength || len(gotWeightArray) != tt.wantArrayLength {
+				t.Errorf("GetDataToCommitFromJobs() got = %v, want %v", gotDataArray, tt.wantArrayLength)
 			}
+			fmt.Println("Got Data Array: ", gotDataArray)
+			fmt.Println("Got WeightArray: ", gotWeightArray)
 		})
 	}
 }

utils/hash.go

@@ -1,10 +1,12 @@
 package utils
 
 import (
+	"encoding/json"
 	"fmt"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/crypto"
+	solsha3 "github.com/miguelmota/go-solidity-sha3"
 )
 
 func EcRecover(data, sig hexutil.Bytes) (common.Address, error) {
@@ -23,3 +25,19 @@ func SignHash(data []byte) []byte {
 	msg := fmt.Sprintf("\x19Ethereum Signed Message:\n%d%s", len(data), data)
 	return crypto.Keccak256([]byte(msg))
 }
+
+func generateCacheKey(url string, body map[string]interface{}) (string, error) {
+	var bodyString string
+	if body != nil {
+		// Convert the body to a JSON string
+		bodyBytes, err := json.Marshal(body)
+		if err != nil {
+			log.Error("Error in marshalling body: ", err)
+			return "", err
+		}
+		bodyString = string(bodyBytes)
+	}
+
+	hash := solsha3.SoliditySHA3([]string{"string", "string"}, []interface{}{url, bodyString})
+	return common.BytesToHash(hash).Hex(), nil
+}

utils/hash_test.go

@@ -65,3 +65,67 @@ func TestEcRecover(t *testing.T) {
 		})
 	}
 }
+
+func Test_generateCacheKey(t *testing.T) {
+	type args struct {
+		url  string
+		body map[string]interface{}
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    string
+		wantErr bool
+	}{
+		{
+			name: "Test when the request is of type GET with body as nil",
+			args: args{
+				url:  "https:api.gemini.com/v1/pubticker",
+				body: nil,
+			},
+			want:    "0x9e9d7684772b773287cfd315aea97e574b7059bf63599ad1e008b9f695ab46e2",
+			wantErr: false,
+		},
+		{
+			name: "Test when the request is of type POST with body",
+			args: args{
+				url:  "https://staging-v3.skalenodes.com/v1/staging-aware-chief-gianfar",
+				body: map[string]interface{}{"jsonrpc": "2.0", "method": "eth_chainId", "params": nil, "id": 0},
+			},
+			want:    "0xecd57ac05ee0584932ac9b969f6e8a851b7a5f1bbd46ae756f48ad9e2747a0ff",
+			wantErr: false,
+		},
+		{
+			name: "Test when url and body is nil",
+			args: args{
+				url:  "",
+				body: nil,
+			},
+			want:    "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470",
+			wantErr: false,
+		},
+		{
+			name: "Test when marshalling fails",
+			args: args{
+				url: "http://example.com",
+				body: map[string]interface{}{
+					"key": func() {}, // functions cannot be marshaled and will cause an error
+				},
+			},
+			want:    "",
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := generateCacheKey(tt.args.url, tt.args.body)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("generateCacheKey() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if got != tt.want {
+				t.Errorf("generateCacheKey() got = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}