Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Live CD logs on DO #924

Draft
wants to merge 3 commits into
base: main
Choose a base branch
from
Draft

Live CD logs on DO #924

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
686d9af
Handle multiple deployments
lionello Dec 13, 2024
51de160
Update godo: use UpdateAllSourceVersions
lionello Dec 13, 2024
041869e
wip
lionello Dec 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions src/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ require (
github.com/awslabs/goformation/v7 v7.13.1
github.com/bufbuild/connect-go v1.10.0
github.com/compose-spec/compose-go/v2 v2.4.3
github.com/digitalocean/godo v1.118.0
github.com/digitalocean/godo v1.131.1
github.com/docker/docker v25.0.6+incompatible
github.com/google/uuid v1.6.0
github.com/googleapis/gax-go/v2 v2.13.0
Expand All @@ -42,7 +42,7 @@ require (
github.com/spf13/cobra v1.8.0
github.com/spf13/pflag v1.0.5
golang.org/x/mod v0.17.0
golang.org/x/oauth2 v0.23.0
golang.org/x/oauth2 v0.24.0
golang.org/x/sys v0.26.0
golang.org/x/term v0.25.0
google.golang.org/api v0.203.0
Expand Down Expand Up @@ -143,6 +143,6 @@ require (
golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3 // indirect; compose-go is using the older slices.sortFunc API
golang.org/x/sync v0.8.0 // indirect
golang.org/x/text v0.19.0 // indirect
golang.org/x/time v0.7.0 // indirect
golang.org/x/time v0.8.0 // indirect
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
)
12 changes: 6 additions & 6 deletions src/go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,8 +124,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/digitalocean/godo v1.118.0 h1:lkzGFQmACrVCp7UqH1sAi4JK/PWwlc5aaxubgorKmC4=
github.com/digitalocean/godo v1.118.0/go.mod h1:Vk0vpCot2HOAJwc5WE8wljZGtJ3ZtWIc8MQ8rF38sdo=
github.com/digitalocean/godo v1.131.1 h1:2QsRwjNukKgOQbflMxOsTDoC05o5UKBpqQMFKXegYKE=
github.com/digitalocean/godo v1.131.1/go.mod h1:PU8JB6I1XYkQIdHFop8lLAY9ojp6M0XcU0TWaQSxbrc=
github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0=
github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
github.com/docker/docker v25.0.6+incompatible h1:5cPwbwriIcsua2REJe8HqQV+6WlWc1byg2QSXzBxBGg=
Expand Down Expand Up @@ -360,8 +360,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug
golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs=
golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE=
golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
Expand Down Expand Up @@ -395,8 +395,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM=
golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ=
golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/time v0.8.0 h1:9i3RxcPv3PZnitoVGMPDKZSq1xW1gK1Xy3ArNOGZfEg=
golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
Expand Down
12 changes: 8 additions & 4 deletions src/pkg/cli/client/byoc/aws/byoc.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -930,13 +930,13 @@ func (b *ByocAws) DeleteConfig(ctx context.Context, secrets *defangv1.Secrets) e
return nil
}

type awsObj struct{ obj s3types.Object }
type s3Obj struct{ obj s3types.Object }

func (a awsObj) Name() string {
func (a s3Obj) Name() string {
return *a.obj.Key
}

func (a awsObj) Size() int64 {
func (a s3Obj) Size() int64 {
return *a.obj.Size
}

Expand All @@ -955,6 +955,10 @@ func (b *ByocAws) BootstrapList(ctx context.Context) ([]string, error) {
}

s3client := s3.NewFromConfig(cfg)
return ListPulumiStacks(ctx, s3client, bucketName)
}

func ListPulumiStacks(ctx context.Context, s3client *s3.Client, bucketName string) ([]string, error) {
prefix := `.pulumi/stacks/` // TODO: should we filter on `projectName`?

term.Debug("Listing stacks in bucket:", bucketName)
Expand All @@ -970,7 +974,7 @@ func (b *ByocAws) BootstrapList(ctx context.Context) ([]string, error) {
if obj.Key == nil || obj.Size == nil {
continue
}
stack, err := b.ParsePulumiStackObject(ctx, awsObj{obj}, bucketName, prefix, func(ctx context.Context, bucket, path string) ([]byte, error) {
stack, err := byoc.ParsePulumiStackObject(ctx, s3Obj{obj}, bucketName, prefix, func(ctx context.Context, bucket, path string) ([]byte, error) {
getObjectOutput, err := s3client.GetObject(ctx, &s3.GetObjectInput{
Bucket: &bucket,
Key: &path,
Expand Down
10 changes: 5 additions & 5 deletions src/pkg/cli/client/byoc/aws/byoc_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,15 +88,15 @@ func (f FakeLoader) LoadProjectName(ctx context.Context) (string, error) {
return f.ProjectName, nil
}

//go:embed test_ecs_events/*.json
//go:embed testdata/*.json
var testDir embed.FS

//go:embed test_ecs_events/*.events
//go:embed testdata/*.events
var expectedDir embed.FS

func TestSubscribe(t *testing.T) {
t.Skip("Pending test")
tests, err := testDir.ReadDir("test_ecs_events")
tests, err := testDir.ReadDir("testdata")
if err != nil {
t.Fatalf("failed to load ecs events test files: %v", err)
}
Expand Down Expand Up @@ -125,7 +125,7 @@ func TestSubscribe(t *testing.T) {
go func() {
defer wg.Done()

filename := path.Join("test_ecs_events", name+".events")
filename := path.Join("testdata", name+".events")
ef, _ := expectedDir.ReadFile(filename)
dec := json.NewDecoder(bytes.NewReader(ef))

Expand All @@ -148,7 +148,7 @@ func TestSubscribe(t *testing.T) {
}
}()

data, err := testDir.ReadFile(path.Join("test_ecs_events", tt.Name()))
data, err := testDir.ReadFile(path.Join("testdata", tt.Name()))
if err != nil {
t.Fatalf("failed to read test file: %v", err)
}
Expand Down
0 ...cs_events/build-failure-o4epidtq3j3b.json → .../testdata/build-failure-o4epidtq3j3b.json
View file
Open in desktop
File renamed without changes.
0 .../aws/test_ecs_events/build-failure.events → ...nt/byoc/aws/testdata/build-failure.events
View file
Open in desktop
File renamed without changes.
0 ...ts/failure-then-success-c1v6g2m5qlvm.json → ...ta/failure-then-success-c1v6g2m5qlvm.json
View file
Open in desktop
File renamed without changes.
0 ...st_ecs_events/failure-then-success.events → .../aws/testdata/failure-then-success.events
View file
Open in desktop
File renamed without changes.
0 ...nts/healthcheck-failure-8ui3h0yf5xqg.json → ...ata/healthcheck-failure-8ui3h0yf5xqg.json
View file
Open in desktop
File renamed without changes.
0 ...est_ecs_events/healthcheck-failure.events → ...c/aws/testdata/healthcheck-failure.events
View file
Open in desktop
File renamed without changes.
0 ...nts/processexit-failure-se3n0qmzhzpm.json → ...ata/processexit-failure-se3n0qmzhzpm.json
View file
Open in desktop
File renamed without changes.
0 ...est_ecs_events/processexit-failure.events → ...c/aws/testdata/processexit-failure.events
View file
Open in desktop
File renamed without changes.
0 ...test_ecs_events/success-f249u7ap07ef.json → ...oc/aws/testdata/success-f249u7ap07ef.json
View file
Open in desktop
File renamed without changes.
0 ...t/byoc/aws/test_ecs_events/success.events → ...i/client/byoc/aws/testdata/success.events
View file
Open in desktop
File renamed without changes.
48 changes: 0 additions & 48 deletions src/pkg/cli/client/byoc/baseclient.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ package byoc

import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
Expand Down Expand Up @@ -166,50 +165,3 @@ func (b *ByocBaseClient) GetProjectDomain(projectName, zone string) string {
func GetPrivateDomain(projectName string) string {
return DnsSafeLabel(projectName) + ".internal"
}

type Obj interface {
Name() string
Size() int64
}

func (b *ByocBaseClient) ParsePulumiStackObject(ctx context.Context, obj Obj, bucket, prefix string, objLoader func(ctx context.Context, bucket, object string) ([]byte, error)) (string, error) {
// The JSON file for an empty stack is ~600 bytes; we add a margin of 100 bytes to account for the length of the stack/project names
if !strings.HasSuffix(obj.Name(), ".json") || obj.Size() < 700 {
return "", nil
}
// Cut off the prefix and the .json suffix
stack := (obj.Name())[len(prefix) : len(obj.Name())-5]
// Check the contents of the JSON file, because the size is not a reliable indicator of a valid stack
data, err := objLoader(ctx, bucket, obj.Name())
if err != nil {
return "", fmt.Errorf("failed to get Pulumi state object %q: %w", obj.Name(), err)
}
var state struct {
Version int `json:"version"`
Checkpoint struct {
// Stack string `json:"stack"` TODO: could use this instead of deriving the stack name from the key
Latest struct {
Resources []struct{} `json:"resources,omitempty"`
PendingOperations []struct {
Resource struct {
Urn string `json:"urn"`
}
} `json:"pending_operations,omitempty"`
}
}
}
if err := json.Unmarshal(data, &state); err != nil {
return "", fmt.Errorf("Failed to decode Pulumi state %q: %w", obj.Name(), err)
} else if state.Version != 3 {
term.Debug("Skipping Pulumi state with version", state.Version)
} else if len(state.Checkpoint.Latest.PendingOperations) > 0 {
for _, op := range state.Checkpoint.Latest.PendingOperations {
parts := strings.Split(op.Resource.Urn, "::") // prefix::project::type::resource => urn:provider:stack::project::plugin:file:class::name
stack += fmt.Sprintf(" (pending %q)", parts[3])
}
} else if len(state.Checkpoint.Latest.Resources) == 0 {
return "", nil // skip: no resources and no pending operations
}

return stack, nil
}
Loading
Loading