Skip to content

Commit

Permalink
Implement a plugin caching mechanism (#284)
Browse files Browse the repository at this point in the history
* WIP on the github cache

* More WIP on the cache

* Finish impl and add cache tests

* Go crap

* More go crap

* Fix x86 arch
  • Loading branch information
UnstoppableMango authored Jul 24, 2024
1 parent 97b44c4 commit 437b752
Show file tree
Hide file tree
Showing 10 changed files with 167 additions and 53 deletions.
6 changes: 3 additions & 3 deletions cli/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ require (
github.com/cyphar/filepath-securejoin v0.2.4 // indirect
github.com/distribution/reference v0.6.0 // indirect
github.com/djherbis/times v1.5.0 // indirect
github.com/docker/docker v27.1.0+incompatible // indirect
github.com/docker/docker v27.1.1+incompatible // indirect
github.com/docker/go-connections v0.5.0 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/edsrzf/mmap-go v1.1.0 // indirect
Expand Down Expand Up @@ -83,8 +83,8 @@ require (
github.com/pkg/term v1.1.0 // indirect
github.com/pulumi/appdash v0.0.0-20231130102222-75f619a67231 // indirect
github.com/pulumi/esc v0.9.1 // indirect
github.com/pulumi/pulumi/pkg/v3 v3.125.0 // indirect
github.com/pulumi/pulumi/sdk/v3 v3.125.0 // indirect
github.com/pulumi/pulumi/pkg/v3 v3.126.0 // indirect
github.com/pulumi/pulumi/sdk/v3 v3.126.0 // indirect
github.com/rivo/uniseg v0.4.4 // indirect
github.com/rogpeppe/go-internal v1.12.0 // indirect
github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 // indirect
Expand Down
12 changes: 6 additions & 6 deletions cli/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -56,8 +56,8 @@ github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5Qvfr
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
github.com/djherbis/times v1.5.0 h1:79myA211VwPhFTqUk8xehWrsEO+zcIZj0zT8mXPVARU=
github.com/djherbis/times v1.5.0/go.mod h1:5q7FDLvbNg1L/KaBmPcWlVR9NmoKo3+ucqUA3ijQhA0=
github.com/docker/docker v27.1.0+incompatible h1:rEHVQc4GZ0MIQKifQPHSFGV/dVgaZafgRf8fCPtDYBs=
github.com/docker/docker v27.1.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/docker v27.1.1+incompatible h1:hO/M4MtV36kzKldqnA37IWhebRA+LnqqcqDja6kVaKY=
github.com/docker/docker v27.1.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c=
github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc=
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
Expand Down Expand Up @@ -202,10 +202,10 @@ github.com/pulumi/appdash v0.0.0-20231130102222-75f619a67231 h1:vkHw5I/plNdTr435
github.com/pulumi/appdash v0.0.0-20231130102222-75f619a67231/go.mod h1:murToZ2N9hNJzewjHBgfFdXhZKjY3z5cYC1VXk+lbFE=
github.com/pulumi/esc v0.9.1 h1:HH5eEv8sgyxSpY5a8yePyqFXzA8cvBvapfH8457+mIs=
github.com/pulumi/esc v0.9.1/go.mod h1:oEJ6bOsjYlQUpjf70GiX+CXn3VBmpwFDxUTlmtUN84c=
github.com/pulumi/pulumi/pkg/v3 v3.125.0 h1:3iG071wxtmfthzcDbAvfSDUDSQngzL/eKylmz69+Njo=
github.com/pulumi/pulumi/pkg/v3 v3.125.0/go.mod h1:L+M81tuBoWAk7lBRUxFoj1kGaLPVYA98/zmWkviUpEQ=
github.com/pulumi/pulumi/sdk/v3 v3.125.0 h1:hou7x/qf9G3878g4+DmBU+IEMJz66w+ZhwJONymjANE=
github.com/pulumi/pulumi/sdk/v3 v3.125.0/go.mod h1:p1U24en3zt51agx+WlNboSOV8eLlPWYAkxMzVEXKbnY=
github.com/pulumi/pulumi/pkg/v3 v3.126.0 h1:XaZU1ehjHN2I5ihkfwxK/UFMDiCDM9FSt2TBnbldAx4=
github.com/pulumi/pulumi/pkg/v3 v3.126.0/go.mod h1:1P4/oK9zceOJUm48QQl/TqjDN68lfsdnTR1FITTFddw=
github.com/pulumi/pulumi/sdk/v3 v3.126.0 h1:6GQVhwG2jgnG7wjRiWgrq0/sU39onctAiBcvTlqb20s=
github.com/pulumi/pulumi/sdk/v3 v3.126.0/go.mod h1:p1U24en3zt51agx+WlNboSOV8eLlPWYAkxMzVEXKbnY=
github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis=
Expand Down
2 changes: 1 addition & 1 deletion cli/um/cmd/from.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ func init() {
var fromCmd = cli.NewFromCmd(
func(opts uml.ConverterOptions) (uml.Converter, error) {
opts.Log.Debug("getting plugin name")
source, err := plugin.ForTarget(*opts.Target)
source, err := uml.PluginForTarget(*opts.Target)
if err != nil {
return nil, err
}
Expand Down
2 changes: 1 addition & 1 deletion cli/um/cmd/gen.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ func init() {
var genCmd = cli.NewGenCmd(
func(opts uml.GeneratorOptions) (uml.Generator, error) {
opts.Log.Debug("getting plugin name")
source, err := plugin.ForTarget(opts.Target)
source, err := uml.PluginForTarget(opts.Target)
if err != nil {
return nil, err
}
Expand Down
2 changes: 1 addition & 1 deletion pkg/cache/cache.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import (
type Cache interface {
Add(string, io.Reader) error
Get(string) (io.ReadCloser, error)
Path(...string) string
Path(string) (string, error)
Remove(string) error
}

Expand Down
8 changes: 5 additions & 3 deletions pkg/cache/fs.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,11 @@ func NewFsCache(path string, logger *slog.Logger) Cache {
}

// Path implements Cache.
func (c *fsCache) Path(rel ...string) string {
segments := append([]string{c.path}, rel...)
return path.Join(segments...)
func (c *fsCache) Path(name string) (string, error) {
p := path.Join(c.path, name)
_, err := os.Stat(p)

return p, err
}

// Add implements Cache.
Expand Down
2 changes: 0 additions & 2 deletions pkg/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -58,8 +58,6 @@ github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5Qvfr
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
github.com/djherbis/times v1.5.0 h1:79myA211VwPhFTqUk8xehWrsEO+zcIZj0zT8mXPVARU=
github.com/djherbis/times v1.5.0/go.mod h1:5q7FDLvbNg1L/KaBmPcWlVR9NmoKo3+ucqUA3ijQhA0=
github.com/docker/docker v27.1.0+incompatible h1:rEHVQc4GZ0MIQKifQPHSFGV/dVgaZafgRf8fCPtDYBs=
github.com/docker/docker v27.1.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/docker v27.1.1+incompatible h1:hO/M4MtV36kzKldqnA37IWhebRA+LnqqcqDja6kVaKY=
github.com/docker/docker v27.1.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c=
Expand Down
59 changes: 59 additions & 0 deletions pkg/plugin/cache.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
package plugin

import (
"context"
"log/slog"

"github.com/google/go-github/v63/github"
"github.com/unstoppablemango/tdl/pkg/cache"
)

type PluginCache interface {
PathFor(string) (string, error)
}

// There's a better way to structure these types...
// and I'll figure it out later
type pluginCache struct {
githubClient
cache cache.Cache
log *slog.Logger
}

func NewCache(client *github.Client, path string, logger *slog.Logger) PluginCache {
fsCache := cache.NewFsCache(path, logger)
gh := githubClient{
client: client,
cache: fsCache,
log: logger,
}

return &pluginCache{
githubClient: gh,
cache: fsCache,
log: logger,
}
}

// PathFor implements PluginCache.
func (c *pluginCache) PathFor(name string) (string, error) {
p, err := c.cache.Path(name)
if err == nil {
return p, nil
}

if err = c.populate(context.Background()); err != nil {
return "", err
}

return c.cache.Path(name)
}

func (c *pluginCache) populate(ctx context.Context) error {
asset, err := c.getLatestAsset(ctx)
if err != nil {
return err
}

return c.cacheAsset(ctx, asset)
}
41 changes: 41 additions & 0 deletions pkg/plugin/cache_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
package plugin_test

import (
"log/slog"
"os"

"github.com/google/go-github/v63/github"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/unstoppablemango/tdl/pkg/plugin"
)

var _ = Describe("PluginCache", func() {
var cacheDir string

BeforeEach(func() {
if _, found := os.LookupEnv("CI"); !found {
Skip("Only running cache test in CI")
}

By("creating a temporary directory")
tmp, err := os.MkdirTemp("", "plugin-cache-test")
Expect(err).NotTo(HaveOccurred())
cacheDir = tmp
})

AfterEach(func() {
_ = os.RemoveAll(cacheDir)
})

It("should retrieve path", func() {
gh := github.NewClient(nil)
cache := plugin.NewCache(gh, cacheDir, slog.Default())

// TODO: Why is uml2ts not in this
result, err := cache.PathFor("uml2go")

Expect(err).NotTo(HaveOccurred())
Expect(result).NotTo(BeEmpty())
})
})
86 changes: 50 additions & 36 deletions pkg/plugin/github.go
Original file line number Diff line number Diff line change
@@ -1,13 +1,18 @@
package plugin

import (
"archive/tar"
"compress/gzip"
"context"
"errors"
"fmt"
"io"
"log/slog"
"net/http"
"runtime"

"github.com/google/go-github/v63/github"
"github.com/unstoppablemango/tdl/pkg/cache"
"github.com/unstoppablemango/tdl/pkg/logging"
)

var (
Expand All @@ -31,67 +36,76 @@ func init() {
ext = "zip"
}

assetName = fmt.Sprintf("tdl_%s_%s.%s",
os,
runtime.GOARCH,
ext,
)
arch := runtime.GOARCH
if arch == "amd64" {
arch = "x86_64"
}

assetName = fmt.Sprintf("tdl_%s_%s.%s", os, arch, ext)
}

type GitHubClient struct {
type githubClient struct {
client *github.Client
cache cache.Cache
log *slog.Logger
}

func NewGitHubClient(client *github.Client, cache cache.Cache) GitHubClient {
return GitHubClient{client: client, cache: cache}
}

func (c GitHubClient) GetPlugin(ctx context.Context) (string, error) {
asset, err := c.getReleaseAsset(ctx)
if err != nil {
return "", err
}

if err = c.cacheAsset(ctx, asset); err != nil {
return "", err
}

return c.cache.Path(assetName), nil
}

func (c GitHubClient) getReleaseAsset(ctx context.Context) (*github.ReleaseAsset, error) {
log := logging.FromContext(ctx)

log.Debug("fetching latest release")
func (c githubClient) getLatestAsset(ctx context.Context) (*github.ReleaseAsset, error) {
c.log.Debug("fetching latest release")
release, _, err := c.client.Repositories.GetLatestRelease(ctx, owner, repo)
if err != nil {
return nil, err
}

log.Debug("searching for asset", "asset", assetName)
c.log.Debug("searching for asset", "asset", assetName)
for _, asset := range release.Assets {
if *asset.Name == assetName {
return asset, nil
}

log.Debug("skipping asset", "asset", asset.Name)
c.log.Debug("skipping asset", "asset", asset.Name)
}

return nil, fmt.Errorf("unable to find asset %s", assetName)
}

func (c GitHubClient) cacheAsset(ctx context.Context, asset *github.ReleaseAsset) error {
log := logging.FromContext(ctx)
func (c githubClient) cacheAsset(ctx context.Context, asset *github.ReleaseAsset) error {
log := c.log.With("asset", asset.Name)

log.Debug("downloading release", "asset", asset.Name)
reader, _, err := c.client.Repositories.DownloadReleaseAsset(ctx, owner, repo, *asset.ID, nil)
log.Debug("downloading release")
reader, _, err := c.client.Repositories.DownloadReleaseAsset(ctx, owner, repo, *asset.ID, http.DefaultClient)
if err != nil {
return err
}
if reader == nil {
return errors.New("github redirects not supported")
}

defer reader.Close()

log.Debug("writing asset to cache")
return c.cache.Add(assetName, reader)
gzipStream, err := gzip.NewReader(reader)
if err != nil {
return err
}

tarFile := tar.NewReader(gzipStream)
for {
h, err := tarFile.Next()
if err == io.EOF {
log.Debug("reached end of tar stream")
break
}
if err != nil {
return err
}

log.Debug("caching tar entry", "name", h.Name)
err = c.cache.Add(h.Name, tarFile)
if err != nil {
return err
}
}

log.Debug("finished caching asset")
return nil
}

0 comments on commit 437b752

Please sign in to comment.