Skip to content

Commit

Permalink
Merge pull request #15827 from serathius/robustness-k8s-2
Browse files Browse the repository at this point in the history 
tests/robustness: Implement delete request for kubernetes scenario
  • Loading branch information
@serathius
serathius authored May 5, 2023
2 parents 79eabc1 + 40f71ef commit 249c0d7
Show file tree
Hide file tree
Showing 7 changed files with 251 additions and 158 deletions.
26 changes: 19 additions & 7 deletions tests/robustness/client.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,23 +96,35 @@ func (c *recordingClient) Delete(ctx context.Context, key string) error {
return nil
}

func (c *recordingClient) CompareAndSet(ctx context.Context, key, value string, expectedRevision int64) error {
func (c *recordingClient) CompareRevisionAndDelete(ctx context.Context, key string, expectedRevision int64) error {
callTime := time.Since(c.baseTime)
resp, err := c.compareRevisionTxn(ctx, key, expectedRevision, clientv3.OpDelete(key)).Commit()
returnTime := time.Since(c.baseTime)
c.history.AppendCompareRevisionAndDelete(key, expectedRevision, callTime, returnTime, resp, err)
return err
}

func (c *recordingClient) CompareRevisionAndPut(ctx context.Context, key, value string, expectedRevision int64) error {
callTime := time.Since(c.baseTime)
resp, err := c.compareRevisionTxn(ctx, key, expectedRevision, clientv3.OpPut(key, value)).Commit()
returnTime := time.Since(c.baseTime)
c.history.AppendCompareRevisionAndPut(key, expectedRevision, value, callTime, returnTime, resp, err)
return err
}

func (c *recordingClient) compareRevisionTxn(ctx context.Context, key string, expectedRevision int64, op clientv3.Op) clientv3.Txn {
txn := c.client.Txn(ctx)
var cmp clientv3.Cmp
if expectedRevision == 0 {
cmp = clientv3.Compare(clientv3.CreateRevision(key), "=", 0)
} else {
cmp = clientv3.Compare(clientv3.ModRevision(key), "=", expectedRevision)
}
resp, err := txn.If(
return txn.If(
cmp,
).Then(
clientv3.OpPut(key, value),
).Commit()
returnTime := time.Since(c.baseTime)
c.history.AppendCompareAndSet(key, expectedRevision, value, callTime, returnTime, resp, err)
return err
op,
)
}

func (c *recordingClient) Txn(ctx context.Context, cmp []clientv3.Cmp, ops []clientv3.Op) error {
Expand Down
45 changes: 26 additions & 19 deletions tests/robustness/linearizability_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,18 +41,18 @@ var (
maximalQPS: 200,
clientCount: 8,
requestProgress: false,
traffic: traffic{
traffic: etcdTraffic{
keyCount: 10,
leaseTTL: DefaultLeaseTTL,
largePutSize: 32769,
writes: []requestChance{
{operation: Put, chance: 45},
{operation: LargePut, chance: 5},
{operation: Delete, chance: 10},
{operation: MultiOpTxn, chance: 10},
{operation: PutWithLease, chance: 10},
{operation: LeaseRevoke, chance: 10},
{operation: CompareAndSet, chance: 10},
writeChoices: []choiceWeight{
{choice: string(Put), weight: 45},
{choice: string(LargePut), weight: 5},
{choice: string(Delete), weight: 10},
{choice: string(MultiOpTxn), weight: 10},
{choice: string(PutWithLease), weight: 10},
{choice: string(LeaseRevoke), weight: 10},
{choice: string(CompareAndSet), weight: 10},
},
},
}
Expand All @@ -62,14 +62,14 @@ var (
maximalQPS: 1000,
clientCount: 12,
requestProgress: false,
traffic: traffic{
traffic: etcdTraffic{
keyCount: 10,
largePutSize: 32769,
leaseTTL: DefaultLeaseTTL,
writes: []requestChance{
{operation: Put, chance: 85},
{operation: MultiOpTxn, chance: 10},
{operation: LargePut, chance: 5},
writeChoices: []choiceWeight{
{choice: string(Put), weight: 85},
{choice: string(MultiOpTxn), weight: 10},
{choice: string(LargePut), weight: 5},
},
},
}
Expand All @@ -79,7 +79,14 @@ var (
maximalQPS: 1000,
clientCount: 12,
traffic: kubernetesTraffic{
keyCount: 5,
averageKeyCount: 5,
resource: "pods",
namespace: "default",
writeChoices: []choiceWeight{
{choice: string(KubernetesUpdate), weight: 75},
{choice: string(KubernetesDelete), weight: 15},
{choice: string(KubernetesCreate), weight: 10},
},
},
}
ReqProgTraffic = trafficConfig{
Expand All @@ -88,13 +95,13 @@ var (
maximalQPS: 1000,
clientCount: 12,
requestProgress: true,
traffic: traffic{
traffic: etcdTraffic{
keyCount: 10,
largePutSize: 8196,
leaseTTL: DefaultLeaseTTL,
writes: []requestChance{
{operation: Put, chance: 95},
{operation: LargePut, chance: 5},
writeChoices: []choiceWeight{
{choice: string(Put), weight: 95},
{choice: string(LargePut), weight: 5},
},
},
}
Expand Down
10 changes: 5 additions & 5 deletions tests/robustness/model/describe_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,17 +80,17 @@ func TestModelDescribe(t *testing.T) {
expectDescribe: `delete("key6") -> err: "failed"`,
},
{
req: compareAndSetRequest("key7", 7, "77"),
resp: compareAndSetResponse(false, 7),
req: compareRevisionAndPutRequest("key7", 7, "77"),
resp: compareRevisionAndPutResponse(false, 7),
expectDescribe: `if(mod_rev(key7)==7).then(put("key7", "77")) -> txn failed, rev: 7`,
},
{
req: compareAndSetRequest("key8", 8, "88"),
resp: compareAndSetResponse(true, 8),
req: compareRevisionAndPutRequest("key8", 8, "88"),
resp: compareRevisionAndPutResponse(true, 8),
expectDescribe: `if(mod_rev(key8)==8).then(put("key8", "88")) -> ok, rev: 8`,
},
{
req: compareAndSetRequest("key9", 9, "99"),
req: compareRevisionAndPutRequest("key9", 9, "99"),
resp: failedResponse(errors.New("failed")),
expectDescribe: `if(mod_rev(key9)==9).then(put("key9", "99")) -> err: "failed"`,
},
Expand Down
28 changes: 14 additions & 14 deletions tests/robustness/model/deterministic_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ func TestModelBase(t *testing.T) {
{
name: "First Txn can start from non-zero revision",
operations: []testOperation{
{req: compareAndSetRequest("key", 0, "42"), resp: compareAndSetResponse(false, 42).EtcdResponse},
{req: compareRevisionAndPutRequest("key", 0, "42"), resp: compareRevisionAndPutResponse(false, 42).EtcdResponse},
},
},
{
Expand Down Expand Up @@ -157,10 +157,10 @@ func TestModelBase(t *testing.T) {
name: "Txn sets new value if value matches expected",
operations: []testOperation{
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1).EtcdResponse},
{req: compareAndSetRequest("key", 1, "2"), resp: compareAndSetResponse(true, 1).EtcdResponse, failure: true},
{req: compareAndSetRequest("key", 1, "2"), resp: compareAndSetResponse(false, 2).EtcdResponse, failure: true},
{req: compareAndSetRequest("key", 1, "2"), resp: compareAndSetResponse(false, 1).EtcdResponse, failure: true},
{req: compareAndSetRequest("key", 1, "2"), resp: compareAndSetResponse(true, 2).EtcdResponse},
{req: compareRevisionAndPutRequest("key", 1, "2"), resp: compareRevisionAndPutResponse(true, 1).EtcdResponse, failure: true},
{req: compareRevisionAndPutRequest("key", 1, "2"), resp: compareRevisionAndPutResponse(false, 2).EtcdResponse, failure: true},
{req: compareRevisionAndPutRequest("key", 1, "2"), resp: compareRevisionAndPutResponse(false, 1).EtcdResponse, failure: true},
{req: compareRevisionAndPutRequest("key", 1, "2"), resp: compareRevisionAndPutResponse(true, 2).EtcdResponse},
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1).EtcdResponse, failure: true},
{req: getRequest("key"), resp: getResponse("key", "1", 1, 2).EtcdResponse, failure: true},
{req: getRequest("key"), resp: getResponse("key", "1", 2, 2).EtcdResponse, failure: true},
Expand All @@ -172,19 +172,19 @@ func TestModelBase(t *testing.T) {
name: "Txn can expect on empty key",
operations: []testOperation{
{req: getRequest("key1"), resp: emptyGetResponse(1).EtcdResponse},
{req: compareAndSetRequest("key1", 0, "2"), resp: compareAndSetResponse(true, 2).EtcdResponse},
{req: compareAndSetRequest("key2", 0, "3"), resp: compareAndSetResponse(true, 3).EtcdResponse},
{req: compareAndSetRequest("key3", 4, "4"), resp: compareAndSetResponse(false, 4).EtcdResponse, failure: true},
{req: compareRevisionAndPutRequest("key1", 0, "2"), resp: compareRevisionAndPutResponse(true, 2).EtcdResponse},
{req: compareRevisionAndPutRequest("key2", 0, "3"), resp: compareRevisionAndPutResponse(true, 3).EtcdResponse},
{req: compareRevisionAndPutRequest("key3", 4, "4"), resp: compareRevisionAndPutResponse(false, 4).EtcdResponse, failure: true},
},
},
{
name: "Txn doesn't do anything if value doesn't match expected",
operations: []testOperation{
{req: getRequest("key"), resp: getResponse("key", "1", 1, 1).EtcdResponse},
{req: compareAndSetRequest("key", 2, "3"), resp: compareAndSetResponse(true, 2).EtcdResponse, failure: true},
{req: compareAndSetRequest("key", 2, "3"), resp: compareAndSetResponse(true, 1).EtcdResponse, failure: true},
{req: compareAndSetRequest("key", 2, "3"), resp: compareAndSetResponse(false, 2).EtcdResponse, failure: true},
{req: compareAndSetRequest("key", 2, "3"), resp: compareAndSetResponse(false, 1).EtcdResponse},
{req: compareRevisionAndPutRequest("key", 2, "3"), resp: compareRevisionAndPutResponse(true, 2).EtcdResponse, failure: true},
{req: compareRevisionAndPutRequest("key", 2, "3"), resp: compareRevisionAndPutResponse(true, 1).EtcdResponse, failure: true},
{req: compareRevisionAndPutRequest("key", 2, "3"), resp: compareRevisionAndPutResponse(false, 2).EtcdResponse, failure: true},
{req: compareRevisionAndPutRequest("key", 2, "3"), resp: compareRevisionAndPutResponse(false, 1).EtcdResponse},
{req: getRequest("key"), resp: getResponse("key", "2", 1, 1).EtcdResponse, failure: true},
{req: getRequest("key"), resp: getResponse("key", "2", 2, 2).EtcdResponse, failure: true},
{req: getRequest("key"), resp: getResponse("key", "3", 1, 1).EtcdResponse, failure: true},
Expand Down Expand Up @@ -317,7 +317,7 @@ func TestModelBase(t *testing.T) {
{req: leaseRevokeRequest(1), resp: leaseRevokeResponse(3).EtcdResponse},
{req: putRequest("key", "4"), resp: putResponse(4).EtcdResponse},
{req: getRequest("key"), resp: getResponse("key", "4", 4, 4).EtcdResponse},
{req: compareAndSetRequest("key", 4, "5"), resp: compareAndSetResponse(true, 5).EtcdResponse},
{req: compareRevisionAndPutRequest("key", 4, "5"), resp: compareRevisionAndPutResponse(true, 5).EtcdResponse},
{req: deleteRequest("key"), resp: deleteResponse(1, 6).EtcdResponse},
{req: defragmentRequest(), resp: defragmentResponse(6).EtcdResponse},
},
Expand All @@ -336,7 +336,7 @@ func TestModelBase(t *testing.T) {
{req: defragmentRequest(), resp: defragmentResponse(4).EtcdResponse},
{req: getRequest("key"), resp: getResponse("key", "4", 4, 4).EtcdResponse},
{req: defragmentRequest(), resp: defragmentResponse(4).EtcdResponse},
{req: compareAndSetRequest("key", 4, "5"), resp: compareAndSetResponse(true, 5).EtcdResponse},
{req: compareRevisionAndPutRequest("key", 4, "5"), resp: compareRevisionAndPutResponse(true, 5).EtcdResponse},
{req: defragmentRequest(), resp: defragmentResponse(5).EtcdResponse},
{req: deleteRequest("key"), resp: deleteResponse(1, 6).EtcdResponse},
{req: defragmentRequest(), resp: defragmentResponse(6).EtcdResponse},
Expand Down
45 changes: 40 additions & 5 deletions tests/robustness/model/history.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -180,8 +180,31 @@ func (h *AppendableHistory) AppendDelete(key string, start, end time.Duration, r
})
}

func (h *AppendableHistory) AppendCompareAndSet(key string, expectedRevision int64, value string, start, end time.Duration, resp *clientv3.TxnResponse, err error) {
request := compareAndSetRequest(key, expectedRevision, value)
func (h *AppendableHistory) AppendCompareRevisionAndDelete(key string, expectedRevision int64, start, end time.Duration, resp *clientv3.TxnResponse, err error) {
request := compareRevisionAndDeleteRequest(key, expectedRevision)
if err != nil {
h.appendFailed(request, start, err)
return
}
var revision int64
if resp != nil && resp.Header != nil {
revision = resp.Header.Revision
}
var deleted int64
if resp != nil && len(resp.Responses) > 0 {
deleted = resp.Responses[0].GetResponseDeleteRange().Deleted
}
h.successful = append(h.successful, porcupine.Operation{
ClientId: h.id,
Input: request,
Call: start.Nanoseconds(),
Output: compareRevisionAndDeleteResponse(resp.Succeeded, deleted, revision),
Return: end.Nanoseconds(),
})

}
func (h *AppendableHistory) AppendCompareRevisionAndPut(key string, expectedRevision int64, value string, start, end time.Duration, resp *clientv3.TxnResponse, err error) {
request := compareRevisionAndPutRequest(key, expectedRevision, value)
if err != nil {
h.appendFailed(request, start, err)
return
Expand All @@ -194,7 +217,7 @@ func (h *AppendableHistory) AppendCompareAndSet(key string, expectedRevision int
ClientId: h.id,
Input: request,
Call: start.Nanoseconds(),
Output: compareAndSetResponse(resp.Succeeded, revision),
Output: compareRevisionAndPutResponse(resp.Succeeded, revision),
Return: end.Nanoseconds(),
})
}
Expand Down Expand Up @@ -376,18 +399,30 @@ func deleteResponse(deleted int64, revision int64) EtcdNonDeterministicResponse
return EtcdNonDeterministicResponse{EtcdResponse: EtcdResponse{Txn: &TxnResponse{OpsResult: []EtcdOperationResult{{Deleted: deleted}}}, Revision: revision}}
}

func compareAndSetRequest(key string, expectedRevision int64, value string) EtcdRequest {
func compareRevisionAndDeleteRequest(key string, expectedRevision int64) EtcdRequest {
return txnRequest([]EtcdCondition{{Key: key, ExpectedRevision: expectedRevision}}, []EtcdOperation{{Type: Delete, Key: key}})
}

func compareRevisionAndPutRequest(key string, expectedRevision int64, value string) EtcdRequest {
return txnRequest([]EtcdCondition{{Key: key, ExpectedRevision: expectedRevision}}, []EtcdOperation{{Type: Put, Key: key, Value: ToValueOrHash(value)}})
}

func compareAndSetResponse(succeeded bool, revision int64) EtcdNonDeterministicResponse {
func compareRevisionAndPutResponse(succeeded bool, revision int64) EtcdNonDeterministicResponse {
var result []EtcdOperationResult
if succeeded {
result = []EtcdOperationResult{{}}
}
return txnResponse(result, succeeded, revision)
}

func compareRevisionAndDeleteResponse(succeeded bool, deleted, revision int64) EtcdNonDeterministicResponse {
var result []EtcdOperationResult
if succeeded {
result = []EtcdOperationResult{{Deleted: deleted}}
}
return txnResponse(result, succeeded, revision)
}

func txnRequest(conds []EtcdCondition, onSuccess []EtcdOperation) EtcdRequest {
return EtcdRequest{Type: Txn, Txn: &TxnRequest{Conds: conds, Ops: onSuccess}}
}
Expand Down
Loading

0 comments on commit 249c0d7

Please sign in to comment.