Skip to content

Commit

Permalink
feat: support pkgfile frontend.
Browse files Browse the repository at this point in the history
This supports `Pkgfile` projects.

Targets can be set as below in `.kres.yaml`:

```yaml
---
kind: pkgfile.Build
spec:
  targets:
    - toolchain
  reproducibleTargetName: toolchain
  additionalTargets:
    nonfree:
      - nonfree-kmod-nvidia-pkg
```

Signed-off-by: Noel Georgi <[email protected]>
  • Loading branch information
frezbo committed Nov 2, 2023
1 parent 3a2980e commit 1eacef1
Show file tree
Hide file tree
Showing 17 changed files with 498 additions and 78 deletions.
5 changes: 4 additions & 1 deletion cmd/kres/cmd/gen.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import (
"github.com/siderolabs/kres/internal/output/codecov"
"github.com/siderolabs/kres/internal/output/conform"
"github.com/siderolabs/kres/internal/output/dockerfile"
"github.com/siderolabs/kres/internal/output/dockerignore"
"github.com/siderolabs/kres/internal/output/drone"
"github.com/siderolabs/kres/internal/output/ghworkflow"
"github.com/siderolabs/kres/internal/output/github"
Expand Down Expand Up @@ -50,6 +51,7 @@ func runGen() error {

outputs := []output.Writer{
output.Wrap[dockerfile.Compiler](dockerfile.NewOutput()),
output.Wrap[dockerignore.Compiler](dockerignore.NewOutput()),
output.Wrap[makefile.Compiler](makefile.NewOutput()),
output.Wrap[golangci.Compiler](golangci.NewOutput()),
output.Wrap[license.Compiler](license.NewOutput()),
Expand All @@ -65,7 +67,8 @@ func runGen() error {
var err error

options := meta.Options{
GoContainerVersion: config.GolangContainerImageVersion,
GoContainerVersion: config.GolangContainerImageVersion,
ContainerImageFrontend: config.ContainerImageFrontendDockerfile,
}

options.Config, err = config.NewProvider(".kres.yaml")
Expand Down
8 changes: 8 additions & 0 deletions internal/config/constants.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,14 @@ const (
CIProviderDrone = "drone"
// CIProviderGitHubActions is the github actions ci provider.
CIProviderGitHubActions = "ghaction"
// ContainerImageFrontendDockerfile is the dockerfile frontend.
ContainerImageFrontendDockerfile = "Dockerfile"
// ContainerImageFrontendPkgfile is the pkgfile frontend.
ContainerImageFrontendPkgfile = "Pkgfile"

// BldrImageVersion is the version of bldr image.
// renovate: datasource=github-releases depName=siderolabs/bldr
BldrImageVersion = "v0.2.3"

// BuildKitContainerVersion is the version of buildkit container image.
// renovate: datasource=docker versioning=docker depName=moby/buildkit
Expand Down
52 changes: 16 additions & 36 deletions internal/output/dockerfile/dockerfile.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,19 +17,17 @@ import (
)

const (
dockerfile = "Dockerfile"
dockerignore = ".dockerignore"
syntax = "docker/dockerfile-upstream:" + config.DockerfileFrontendImageVersion
filename = "Dockerfile"
syntax = "docker/dockerfile-upstream:" + config.DockerfileFrontendImageVersion
)

// Output implements Dockerfile and .dockerignore generation.
type Output struct {
output.FileAdapter

args []*step.ArgStep
stages map[string]*Stage

allowedLocalPaths []string
stages map[string]*Stage
args []*step.ArgStep
enabled bool
}

// NewOutput creates new dockerfile output.
Expand All @@ -46,18 +44,25 @@ func (o *Output) Compile(compiler Compiler) error {
return compiler.CompileDockerfile(o)
}

// Enable should be called to enable config generation.
func (o *Output) Enable() {
o.enabled = true
}

// Filenames implements output.FileWriter interface.
func (o *Output) Filenames() []string {
return []string{dockerfile, dockerignore}
if !o.enabled {
return nil
}

return []string{filename}
}

// GenerateFile implements output.FileWriter interface.
func (o *Output) GenerateFile(filename string, w io.Writer) error {
switch filename {
case dockerfile:
case filename:
return o.dockerfile(w)
case dockerignore:
return o.dockerignore(w)
default:
panic("unexpected filename: " + filename)
}
Expand All @@ -83,13 +88,6 @@ func (o *Output) Arg(arg *step.ArgStep) *Output {
return o
}

// AllowLocalPath adds path to the list of paths to be copied into the context.
func (o *Output) AllowLocalPath(paths ...string) *Output {
o.allowedLocalPaths = append(o.allowedLocalPaths, paths...)

return o
}

func (o *Output) dockerfile(w io.Writer) error {
if _, err := fmt.Fprintf(w, "# syntax = %s\n\n", syntax); err != nil {
return err
Expand Down Expand Up @@ -129,24 +127,6 @@ func (o *Output) dockerfile(w io.Writer) error {
return nil
}

func (o *Output) dockerignore(w io.Writer) error {
if _, err := w.Write([]byte(output.Preamble("# "))); err != nil {
return err
}

if _, err := fmt.Fprintln(w, "*"); err != nil {
return err
}

for _, path := range o.allowedLocalPaths {
if _, err := fmt.Fprintf(w, "!%s\n", path); err != nil {
return err
}
}

return nil
}

// Compiler is implemented by project blocks which support Dockerfile generate.
type Compiler interface {
CompileDockerfile(*Output) error
Expand Down
83 changes: 83 additions & 0 deletions internal/output/dockerignore/dockerignore.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

// Package dockerignore implements output to .dockerignore.
package dockerignore

import (
"fmt"
"io"

"github.com/siderolabs/kres/internal/output"
)

const (
filename = ".dockerignore"
)

// Output implements .dockerignore generation.
type Output struct {
output.FileAdapter

allowedLocalPaths []string
}

// NewOutput creates new dockerignore output.
func NewOutput() *Output {
output := &Output{}

output.FileWriter = output

return output
}

// Compile implements [output.TypedWriter] interface.
func (o *Output) Compile(compiler Compiler) error {
return compiler.CompileDockerignore(o)
}

// Filenames implements output.FileWriter interface.
func (o *Output) Filenames() []string {
return []string{filename}
}

// AllowLocalPath adds path to the list of paths to be copied into the context.
func (o *Output) AllowLocalPath(paths ...string) *Output {
o.allowedLocalPaths = append(o.allowedLocalPaths, paths...)

return o
}

// GenerateFile implements output.FileWriter interface.
func (o *Output) GenerateFile(filename string, w io.Writer) error {
switch filename {
case filename:
return o.dockerignore(w)
default:
panic("unexpected filename: " + filename)
}
}

func (o *Output) dockerignore(w io.Writer) error {
if _, err := w.Write([]byte(output.Preamble("# "))); err != nil {
return err
}

if _, err := fmt.Fprintln(w, "*"); err != nil {
return err
}

for _, path := range o.allowedLocalPaths {
if _, err := fmt.Fprintf(w, "!%s\n", path); err != nil {
return err
}
}

return nil
}

// Compiler is implemented by project blocks which support Dockerfile generate.
type Compiler interface {
CompileDockerignore(*Output) error
}
63 changes: 48 additions & 15 deletions internal/output/ghworkflow/gh_workflow.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ const (
HostedRunner = "self-hosted"
// GenericRunner is the name of the generic runner.
GenericRunner = "generic"
// PkgsRunner is the name of the default runner for packages.
PkgsRunner = "pkgs"
// DefaultSkipCondition is the default condition to skip the workflow.
DefaultSkipCondition = "(!startsWith(github.head_ref, 'renovate/') && !startsWith(github.head_ref, 'dependabot/'))"

Expand Down Expand Up @@ -165,23 +167,22 @@ func (o *Output) AddSlackNotify(workflow string) {
o.workflows[slackWorkflow].Workflows = append(o.workflows[slackWorkflow].Workflows, workflow)
}

// OverrideDefaultJobCondition overrides default job condition.
func (o *Output) OverrideDefaultJobCondition(condition string) {
o.workflows[ciWorkflow].Jobs["default"].If = condition
// SetDefaultJobRunnerAsPkgs sets default job runner as pkgs.
func (o *Output) SetDefaultJobRunnerAsPkgs() {
o.workflows[ciWorkflow].Jobs["default"].RunsOn = []string{
HostedRunner,
PkgsRunner,
}
}

// AddPullRequestLabelCondition adds condition to default job to also run on PRs with labels.
func (o *Output) AddPullRequestLabelCondition() {
o.workflows[ciWorkflow].On.PullRequest.Types = []string{
"opened",
"synchronize",
"reopened",
"labeled",
}
// OverwriteDefaultJobStepsAsPkgs overwrites default job steps as pkgs.
// Note that calling this method will overwrite any existing steps.
func (o *Output) OverwriteDefaultJobStepsAsPkgs() {
o.workflows[ciWorkflow].Jobs["default"].Steps = DefaultPkgsSteps()
}

// DefaultSteps returns default steps for the workflow.
func DefaultSteps() []*Step {
// CommonSteps returns common steps for the workflow.
func CommonSteps() []*Step {
return []*Step{
{
Name: "checkout",
Expand All @@ -191,15 +192,43 @@ func DefaultSteps() []*Step {
Name: "Unshallow",
Run: "git fetch --prune --unshallow\n",
},
{
}
}

// DefaultSteps returns default steps for the workflow.
func DefaultSteps() []*Step {
return append(
CommonSteps(),
&Step{
Name: "Set up Docker Buildx",
Uses: fmt.Sprintf("docker/setup-buildx-action@%s", config.SetupBuildxActionVersion),
With: map[string]string{
"driver": "remote",
"endpoint": "tcp://localhost:1234",
},
},
}
)
}

// DefaultPkgsSteps returns default pkgs steps for the workflow.
func DefaultPkgsSteps() []*Step {
armbuildkitdEnpointConfig := `
- endpoint: tcp://buildkit-arm64.ci.svc.cluster.local:1234
platforms: linux/arm64
`

return append(
CommonSteps(),
&Step{
Name: "Set up Docker Buildx",
Uses: fmt.Sprintf("docker/setup-buildx-action@%s", config.SetupBuildxActionVersion),
With: map[string]string{
"driver": "remote",
"endpoint": "tcp://localhost:1234",
"append": strings.TrimPrefix(armbuildkitdEnpointConfig, "\n"),
},
},
)
}

// DefaultServices returns default services for the workflow.
Expand All @@ -221,6 +250,10 @@ func DefaultServices() map[string]Service {
func MakeStep(name string, args ...string) *Step {
command := fmt.Sprintf("make %s\n", name)

if name == "" {
command = "make\n"
}

if len(args) > 0 {
command = fmt.Sprintf("make %s %s\n", name, strings.Join(args, " "))
}
Expand Down
11 changes: 7 additions & 4 deletions internal/output/makefile/group.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,13 @@ import (

// Descriptions (used as keys) of some predefined variable groups.
const (
VariableGroupCommon = "common variables"
VariableGroupDocker = "docker build settings"
VariableGroupHelp = "help menu"
VariableGroupExtra = "extra variables"
VariableGroupCommon = "common variables"
VariableGroupDocker = "docker build settings"
VariableGroupExtra = "extra variables"
VariableGroupHelp = "help menu"
VariableGroupSourceDateEpoch = "source date epoch of first commit"
VariableGroupTargets = "targets defines all the available targets"
VariableGroupAdditionalTargets = "additional targets defines all the additional targets"
)

// VariableGroup is a way to group nicely variables in Makefile.
Expand Down
7 changes: 6 additions & 1 deletion internal/project/auto/builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,11 @@ func (builder *builder) build() error {
detect: builder.DetectCI,
build: builder.BuildCI,
},
{
detect: builder.DetectPkgFile,
build: builder.BuildPkgFile,
mandatory: true,
},
{
detect: builder.DetectJS,
build: builder.BuildJS,
Expand Down Expand Up @@ -100,7 +105,7 @@ func (builder *builder) build() error {
}

if !mandatoryReached {
return errors.New("no Go or JS files were found")
return errors.New("no Go or JS or Pkgfile files were found")
}

if len(builder.lintInputs) > 0 {
Expand Down
5 changes: 5 additions & 0 deletions internal/project/auto/golang.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (

"golang.org/x/mod/modfile"

"github.com/siderolabs/kres/internal/config"
"github.com/siderolabs/kres/internal/dag"
"github.com/siderolabs/kres/internal/project/common"
"github.com/siderolabs/kres/internal/project/golang"
Expand All @@ -22,6 +23,10 @@ import (

// DetectGolang checks if project at rootPath is Go-based project.
func (builder *builder) DetectGolang() (bool, error) {
if builder.meta.ContainerImageFrontend != config.ContainerImageFrontendDockerfile {
return false, nil
}

var lookupDirs []string

err := filepath.Walk(builder.rootPath, func(path string, info os.FileInfo, err error) error {
Expand Down
Loading

0 comments on commit 1eacef1

Please sign in to comment.