diff --git a/pkg/agent/containerd/config_linux.go b/pkg/agent/containerd/config_linux.go index b5219aee4ec5..ae546bbb127c 100644 --- a/pkg/agent/containerd/config_linux.go +++ b/pkg/agent/containerd/config_linux.go @@ -27,17 +27,6 @@ const ( runtimesPath = "/usr/local/nvidia/toolkit:/opt/kwasm/bin" ) -func getContainerdArgs(cfg *config.Node) []string { - args := []string{ - "containerd", - "-c", cfg.Containerd.Config, - "-a", cfg.Containerd.Address, - "--state", cfg.Containerd.State, - "--root", cfg.Containerd.Root, - } - return args -} - // SetupContainerdConfig generates the containerd.toml, using a template combined with various // runtime configurations and registry mirror settings provided by the administrator. func SetupContainerdConfig(cfg *config.Node) error { diff --git a/pkg/agent/containerd/config_test.go b/pkg/agent/containerd/config_test.go index 98a948a22435..807458059302 100644 --- a/pkg/agent/containerd/config_test.go +++ b/pkg/agent/containerd/config_test.go @@ -1473,12 +1473,18 @@ func Test_UnitGetHostConfigs(t *testing.T) { nodeConfig := &config.Node{ Containerd: config.Containerd{ + Address: "/run/k3s/containerd/containerd.sock", + Opt: "/var/lib/rancher/k3s/agent/containerd", Registry: tempDir + "/hosts.d", + Root: "/var/lib/rancher/k3s/agent/containerd", + State: "/run/k3s/containerd", }, AgentConfig: config.Agent{ ImageServiceSocket: "containerd-stargz-grpc.sock", Registry: registry.Registry, Snapshotter: "stargz", + CNIBinDir: "/bin", + CNIConfDir: "/var/lib/rancher/k3s/agent/etc/cni/net.d", }, } diff --git a/pkg/agent/containerd/config_windows.go b/pkg/agent/containerd/config_windows.go index c12b88c84dfc..f4cdc525765d 100644 --- a/pkg/agent/containerd/config_windows.go +++ b/pkg/agent/containerd/config_windows.go @@ -13,14 +13,6 @@ import ( "k8s.io/cri-client/pkg/util" ) -func getContainerdArgs(cfg *config.Node) []string { - args := []string{ - "containerd", - "-c", cfg.Containerd.Config, - } - return args -} - // SetupContainerdConfig generates the containerd.toml, using a template combined with various // runtime configurations and registry mirror settings provided by the administrator. func SetupContainerdConfig(cfg *config.Node) error { diff --git a/pkg/agent/containerd/containerd.go b/pkg/agent/containerd/containerd.go index ae15ac9bd629..4d320ddbfa1b 100644 --- a/pkg/agent/containerd/containerd.go +++ b/pkg/agent/containerd/containerd.go @@ -50,7 +50,7 @@ const ( // Run configures and starts containerd as a child process. Once it is up, images are preloaded // or pulled from files found in the agent images directory. func Run(ctx context.Context, cfg *config.Node) error { - args := getContainerdArgs(cfg) + args := []string{"containerd", "-c", cfg.Containerd.Config} stdOut := io.Writer(os.Stdout) stdErr := io.Writer(os.Stderr) diff --git a/pkg/agent/templates/templates.go b/pkg/agent/templates/templates.go index 9a66b9007472..b84ff314fa28 100644 --- a/pkg/agent/templates/templates.go +++ b/pkg/agent/templates/templates.go @@ -1,8 +1,11 @@ package templates import ( + "bufio" "bytes" + "io" "net/url" + "strings" "text/template" "github.com/rancher/wharfie/pkg/registries" @@ -98,7 +101,7 @@ func ParseTemplateFromConfig(templateBuffer string, config interface{}) (string, if err := t.Execute(out, config); err != nil { return "", err } - return out.String(), nil + return trimEmpty(out) } func ParseHostsTemplateFromConfig(templateBuffer string, config interface{}) (string, error) { @@ -107,5 +110,21 @@ func ParseHostsTemplateFromConfig(templateBuffer string, config interface{}) (st if err := t.Execute(out, config); err != nil { return "", err } - return out.String(), nil + return trimEmpty(out) +} + +// trimEmpty removes excess empty lines from the rendered template +func trimEmpty(r io.Reader) (string, error) { + builder := strings.Builder{} + scanner := bufio.NewScanner(r) + for scanner.Scan() { + line := scanner.Text() + if strings.TrimSpace(line) != "" { + if strings.HasPrefix(line, "[") { + builder.WriteString("\n") + } + builder.WriteString(line + "\n") + } + } + return builder.String(), scanner.Err() } diff --git a/pkg/agent/templates/templates_linux.go b/pkg/agent/templates/templates_linux.go index dffce1737ccb..66457c612ec0 100644 --- a/pkg/agent/templates/templates_linux.go +++ b/pkg/agent/templates/templates_linux.go @@ -1,4 +1,5 @@ -//go:build linux +//go:build linux && !windows_test +// +build linux,!windows_test package templates @@ -9,100 +10,116 @@ import ( const ContainerdConfigTemplate = ` {{- /* */ -}} # File generated by {{ .Program }}. DO NOT EDIT. Use config.toml.tmpl instead. -version = 2 +version = 3 +root = {{ printf "%q" .NodeConfig.Containerd.Root }} +state = {{ printf "%q" .NodeConfig.Containerd.State }} -[plugins."io.containerd.internal.v1.opt"] +[grpc] + address = {{ deschemify .NodeConfig.Containerd.Address | printf "%q" }} + +[plugins.'io.containerd.internal.v1.opt'] path = "{{ .NodeConfig.Containerd.Opt }}" -[plugins."io.containerd.grpc.v1.cri"] + +[plugins.'io.containerd.grpc.v1.cri'] stream_server_address = "127.0.0.1" stream_server_port = "10010" + +[plugins.'io.containerd.cri.v1.runtime'] enable_selinux = {{ .NodeConfig.SELinux }} enable_unprivileged_ports = {{ .EnableUnprivileged }} enable_unprivileged_icmp = {{ .EnableUnprivileged }} device_ownership_from_security_context = {{ .NonrootDevices }} -{{- if .DisableCgroup}} +{{ if .DisableCgroup}} disable_cgroup = true -{{end}} -{{- if .IsRunningInUserNS }} +{{ end }} + +{{ if .IsRunningInUserNS }} disable_apparmor = true restrict_oom_score_adj = true -{{end}} - -{{- if .NodeConfig.AgentConfig.PauseImage }} - sandbox_image = "{{ .NodeConfig.AgentConfig.PauseImage }}" -{{end}} +{{ end }} -{{- if .NodeConfig.AgentConfig.Snapshotter }} -[plugins."io.containerd.grpc.v1.cri".containerd] +{{ if .NodeConfig.AgentConfig.Snapshotter }} +[plugins.'io.containerd.cri.v1.images'] snapshotter = "{{ .NodeConfig.AgentConfig.Snapshotter }}" disable_snapshot_annotations = {{ if eq .NodeConfig.AgentConfig.Snapshotter "stargz" }}false{{else}}true{{end}} - {{ if .NodeConfig.DefaultRuntime }}default_runtime_name = "{{ .NodeConfig.DefaultRuntime }}"{{end}} -{{ if eq .NodeConfig.AgentConfig.Snapshotter "stargz" }} -{{ if .NodeConfig.AgentConfig.ImageServiceSocket }} -[plugins."io.containerd.snapshotter.v1.stargz"] -cri_keychain_image_service_path = "{{ .NodeConfig.AgentConfig.ImageServiceSocket }}" -[plugins."io.containerd.snapshotter.v1.stargz".cri_keychain] -enable_keychain = true -{{end}} +{{ end }} + +{{ if .NodeConfig.AgentConfig.PauseImage }} +[plugins.'io.containerd.cri.v1.images'.pinned_images] + sandbox = "{{ .NodeConfig.AgentConfig.PauseImage }}" +{{ end }} -[plugins."io.containerd.snapshotter.v1.stargz".registry] +[plugins.'io.containerd.cri.v1.images'.registry] config_path = "{{ .NodeConfig.Containerd.Registry }}" -{{ if .PrivateRegistryConfig }} -{{range $k, $v := .PrivateRegistryConfig.Configs }} -{{ if $v.Auth }} -[plugins."io.containerd.snapshotter.v1.stargz".registry.configs."{{$k}}".auth] - {{ if $v.Auth.Username }}username = {{ printf "%q" $v.Auth.Username }}{{end}} - {{ if $v.Auth.Password }}password = {{ printf "%q" $v.Auth.Password }}{{end}} - {{ if $v.Auth.Auth }}auth = {{ printf "%q" $v.Auth.Auth }}{{end}} - {{ if $v.Auth.IdentityToken }}identitytoken = {{ printf "%q" $v.Auth.IdentityToken }}{{end}} -{{end}} -{{end}} -{{end}} -{{end}} -{{end}} - -{{- if not .NodeConfig.NoFlannel }} -[plugins."io.containerd.grpc.v1.cri".cni] +{{ if not .NodeConfig.NoFlannel }} +[plugins.'io.containerd.cri.v1.runtime'.cni] bin_dir = "{{ .NodeConfig.AgentConfig.CNIBinDir }}" conf_dir = "{{ .NodeConfig.AgentConfig.CNIConfDir }}" -{{end}} +{{ end }} + +{{ if or .NodeConfig.Containerd.BlockIOConfig .NodeConfig.Containerd.RDTConfig }} +[plugins.'io.containerd.service.v1.tasks-service'] + {{ if .NodeConfig.Containerd.BlockIOConfig }}blockio_config_file = "{{ .NodeConfig.Containerd.BlockIOConfig }}"{{ end }} + {{ if .NodeConfig.Containerd.RDTConfig }}rdt_config_file = "{{ .NodeConfig.Containerd.RDTConfig }}"{{ end }} +{{ end }} -{{- if or .NodeConfig.Containerd.BlockIOConfig .NodeConfig.Containerd.RDTConfig }} -[plugins."io.containerd.service.v1.tasks-service"] - {{ if .NodeConfig.Containerd.BlockIOConfig }}blockio_config_file = "{{ .NodeConfig.Containerd.BlockIOConfig }}"{{end}} - {{ if .NodeConfig.Containerd.RDTConfig }}rdt_config_file = "{{ .NodeConfig.Containerd.RDTConfig }}"{{end}} -{{end}} +{{ if .NodeConfig.DefaultRuntime }} +[plugins.'io.containerd.cri.v1.runtime'.containerd] + default_runtime_name = "{{ .NodeConfig.DefaultRuntime }}" +{{ end }} -[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] +[plugins.'io.containerd.cri.v1.runtime'.containerd.runtimes.runc] runtime_type = "io.containerd.runc.v2" -[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] +[plugins.'io.containerd.cri.v1.runtime'.containerd.runtimes.runc.options] SystemdCgroup = {{ .SystemdCgroup }} -[plugins."io.containerd.grpc.v1.cri".registry] +{{ range $k, $v := .ExtraRuntimes }} +[plugins.'io.containerd.cri.v1.runtime'.containerd.runtimes.'{{ $k }}'] + runtime_type = "{{$v.RuntimeType}}" +[plugins.'io.containerd.cri.v1.runtime'.containerd.runtimes.'{{ $k }}'.options] + BinaryName = "{{ $v.BinaryName }}" + SystemdCgroup = {{ $.SystemdCgroup }} +{{ end }} + +{{- if .PrivateRegistryConfig }} +{{- range $k, $v := .PrivateRegistryConfig.Configs }} +{{ if $v.Auth }} +[plugins.'io.containerd.grpc.v1.cri'.registry.configs.'{{ $k }}'.auth] + {{ if $v.Auth.Username }}username = {{ printf "%q" $v.Auth.Username }}{{ end }} + {{ if $v.Auth.Password }}password = {{ printf "%q" $v.Auth.Password }}{{ end }} + {{ if $v.Auth.Auth }}auth = {{ printf "%q" $v.Auth.Auth }}{{ end }} + {{ if $v.Auth.IdentityToken }}identitytoken = {{ printf "%q" $v.Auth.IdentityToken }}{{ end }} +{{ end }} +{{ end }} +{{ end }} + +{{ if eq .NodeConfig.AgentConfig.Snapshotter "stargz" }} +{{ if .NodeConfig.AgentConfig.ImageServiceSocket }} +[plugins.'io.containerd.snapshotter.v1.stargz'] + cri_keychain_image_service_path = "{{ .NodeConfig.AgentConfig.ImageServiceSocket }}" + +[plugins.'io.containerd.snapshotter.v1.stargz'.cri_keychain] + enable_keychain = true +{{ end }} + +[plugins.'io.containerd.snapshotter.v1.stargz'.registry] config_path = "{{ .NodeConfig.Containerd.Registry }}" {{ if .PrivateRegistryConfig }} -{{range $k, $v := .PrivateRegistryConfig.Configs }} +{{- range $k, $v := .PrivateRegistryConfig.Configs }} {{ if $v.Auth }} -[plugins."io.containerd.grpc.v1.cri".registry.configs."{{$k}}".auth] - {{ if $v.Auth.Username }}username = {{ printf "%q" $v.Auth.Username }}{{end}} - {{ if $v.Auth.Password }}password = {{ printf "%q" $v.Auth.Password }}{{end}} - {{ if $v.Auth.Auth }}auth = {{ printf "%q" $v.Auth.Auth }}{{end}} - {{ if $v.Auth.IdentityToken }}identitytoken = {{ printf "%q" $v.Auth.IdentityToken }}{{end}} -{{end}} -{{end}} -{{end}} - -{{range $k, $v := .ExtraRuntimes}} -[plugins."io.containerd.grpc.v1.cri".containerd.runtimes."{{$k}}"] - runtime_type = "{{$v.RuntimeType}}" -[plugins."io.containerd.grpc.v1.cri".containerd.runtimes."{{$k}}".options] - BinaryName = "{{$v.BinaryName}}" - SystemdCgroup = {{ $.SystemdCgroup }} -{{end}} +[plugins.'io.containerd.snapshotter.v1.stargz'.registry.configs.'{{ $k }}'.auth] + {{ if $v.Auth.Username }}username = {{ printf "%q" $v.Auth.Username }}{{ end }} + {{ if $v.Auth.Password }}password = {{ printf "%q" $v.Auth.Password }}{{ end }} + {{ if $v.Auth.Auth }}auth = {{ printf "%q" $v.Auth.Auth }}{{ end }} + {{ if $v.Auth.IdentityToken }}identitytoken = {{ printf "%q" $v.Auth.IdentityToken }}{{ end }} +{{ end }} +{{ end }} +{{ end }} +{{ end }} ` // Linux config templates do not need fixups diff --git a/pkg/agent/templates/templates_win.go b/pkg/agent/templates/templates_win.go new file mode 100644 index 000000000000..95c5ab279aa8 --- /dev/null +++ b/pkg/agent/templates/templates_win.go @@ -0,0 +1,93 @@ +//go:build windows || windows_test +// +build windows windows_test + +// test with: go test -tags windows_test +// note that this requires the filename not end with '_windows.go' + +package templates + +import ( + "net/url" + "strings" + "text/template" +) + +const ContainerdConfigTemplate = ` +{{- /* */ -}} +# File generated by {{ .Program }}. DO NOT EDIT. Use config.toml.tmpl instead. +version = 3 +root = {{ printf "%q" .NodeConfig.Containerd.Root }} +state = {{ printf "%q" .NodeConfig.Containerd.State }} + +[grpc] + address = {{ deschemify .NodeConfig.Containerd.Address | printf "%q" }} + +[plugins."io.containerd.internal.v1.opt"] + path = {{ printf "%q" .NodeConfig.Containerd.Opt }} + +[plugins.'io.containerd.grpc.v1.cri'] + stream_server_address = "127.0.0.1" + stream_server_port = "10010" + +[plugins.'io.containerd.cri.v1.runtime'] + enable_selinux = false + +[plugins.'io.containerd.grpc.v1.cri'] + stream_server_address = "127.0.0.1" + stream_server_port = "10010" + +[plugins.'io.containerd.cri.v1.images'] + snapshotter = "windows" + +{{ if .NodeConfig.AgentConfig.PauseImage }} +[plugins.'io.containerd.cri.v1.images'.pinned_images] + sandbox = "{{ .NodeConfig.AgentConfig.PauseImage }}" +{{ end }} + +[plugins.'io.containerd.cri.v1.images'.registry] + config_path = "{{ .NodeConfig.Containerd.Registry }}" + +{{ if not .NodeConfig.NoFlannel }} +[plugins.'io.containerd.cri.v1.runtime'.cni] + bin_dir = "{{ .NodeConfig.AgentConfig.CNIBinDir }}" + conf_dir = "{{ .NodeConfig.AgentConfig.CNIConfDir }}" +{{ end }} + +[plugins.'io.containerd.cri.v1.runtime'.containerd] + default_runtime_name = "runhcs-wcow-process" + +[plugins.'io.containerd.cri.v1.runtime'.containerd.runtimes.runhcs-wcow-process] + runtime_type = "io.containerd.runhcs.v1" + +[plugins.'io.containerd.runtime.v2.task'] + platforms = ["windows/amd64", "linux/amd64"] + +[plugins.'io.containerd.service.v1.diff-service'] + default = ["windows", "windows-lcow"] + +{{- if .PrivateRegistryConfig }} +{{- range $k, $v := .PrivateRegistryConfig.Configs }} +{{ if $v.Auth }} +[plugins.'io.containerd.grpc.v1.cri'.registry.configs.'{{ $k }}'.auth] + {{ if $v.Auth.Username }}username = {{ printf "%q" $v.Auth.Username }}{{ end }} + {{ if $v.Auth.Password }}password = {{ printf "%q" $v.Auth.Password }}{{ end }} + {{ if $v.Auth.Auth }}auth = {{ printf "%q" $v.Auth.Auth }}{{ end }} + {{ if $v.Auth.IdentityToken }}identitytoken = {{ printf "%q" $v.Auth.IdentityToken }}{{ end }} +{{ end }} +{{ end }} +{{ end }} +` + +// Windows config templates need named pipe addresses fixed up +var templateFuncs = template.FuncMap{ + "deschemify": func(s string) string { + if strings.HasPrefix(s, "npipe:") { + u, err := url.Parse(s) + if err != nil { + return "" + } + return u.Path + } + return s + }, +} diff --git a/pkg/agent/templates/templates_windows.go b/pkg/agent/templates/templates_windows.go deleted file mode 100644 index 5cccd7e43d08..000000000000 --- a/pkg/agent/templates/templates_windows.go +++ /dev/null @@ -1,158 +0,0 @@ -//go:build windows -// +build windows - -package templates - -import ( - "net/url" - "strings" - "text/template" -) - -const ContainerdConfigTemplate = ` -{{- /* */ -}} -# File generated by {{ .Program }}. DO NOT EDIT. Use config.toml.tmpl instead. -version = 2 -root = {{ printf "%q" .NodeConfig.Containerd.Root }} -state = {{ printf "%q" .NodeConfig.Containerd.State }} -plugin_dir = "" -disabled_plugins = [] -required_plugins = [] -oom_score = 0 - -[grpc] - address = "{{ deschemify .NodeConfig.Containerd.Address }}" - tcp_address = "" - tcp_tls_cert = "" - tcp_tls_key = "" - uid = 0 - gid = 0 - max_recv_message_size = 16777216 - max_send_message_size = 16777216 - -[ttrpc] - address = "" - uid = 0 - gid = 0 - -[debug] - address = "" - uid = 0 - gid = 0 - level = "" - -[metrics] - address = "" - grpc_histogram = false - -[cgroup] - path = "" - -[timeouts] - "io.containerd.timeout.shim.cleanup" = "5s" - "io.containerd.timeout.shim.load" = "5s" - "io.containerd.timeout.shim.shutdown" = "3s" - "io.containerd.timeout.task.state" = "2s" - -[plugins] - [plugins."io.containerd.gc.v1.scheduler"] - pause_threshold = 0.02 - deletion_threshold = 0 - mutation_threshold = 100 - schedule_delay = "0s" - startup_delay = "100ms" - [plugins."io.containerd.grpc.v1.cri"] - disable_tcp_service = true - stream_server_address = "127.0.0.1" - stream_server_port = "0" - stream_idle_timeout = "4h0m0s" - enable_selinux = false - selinux_category_range = 0 - sandbox_image = "{{ .NodeConfig.AgentConfig.PauseImage }}" - stats_collect_period = 10 - systemd_cgroup = false - enable_tls_streaming = false - max_container_log_line_size = 16384 - disable_cgroup = false - disable_apparmor = false - restrict_oom_score_adj = false - max_concurrent_downloads = 3 - disable_proc_mount = false - unset_seccomp_profile = "" - tolerate_missing_hugetlb_controller = false - disable_hugetlb_controller = false - ignore_image_defined_volumes = false - [plugins."io.containerd.grpc.v1.cri".containerd] - snapshotter = "windows" - default_runtime_name = "runhcs-wcow-process" - no_pivot = false - disable_snapshot_annotations = false - discard_unpacked_layers = false - [plugins."io.containerd.grpc.v1.cri".containerd.default_runtime] - runtime_type = "" - runtime_engine = "" - runtime_root = "" - privileged_without_host_devices = false - base_runtime_spec = "" - [plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime] - runtime_type = "" - runtime_engine = "" - runtime_root = "" - privileged_without_host_devices = false - base_runtime_spec = "" - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] - [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runhcs-wcow-process] - runtime_type = "io.containerd.runhcs.v1" - runtime_engine = "" - runtime_root = "" - privileged_without_host_devices = false - base_runtime_spec = "" - [plugins."io.containerd.grpc.v1.cri".cni] - bin_dir = {{ printf "%q" .NodeConfig.AgentConfig.CNIBinDir }} - conf_dir = {{ printf "%q" .NodeConfig.AgentConfig.CNIConfDir }} - max_conf_num = 1 - conf_template = "" - [plugins."io.containerd.grpc.v1.cri".registry] - config_path = {{ printf "%q" .NodeConfig.Containerd.Registry }} - - {{ if .PrivateRegistryConfig }} - {{range $k, $v := .PrivateRegistryConfig.Configs }} - {{ if $v.Auth }} - [plugins."io.containerd.grpc.v1.cri".registry.configs.auth."{{$k}}"] - {{ if $v.Auth.Username }}username = {{ printf "%q" $v.Auth.Username }}{{end}} - {{ if $v.Auth.Password }}password = {{ printf "%q" $v.Auth.Password }}{{end}} - {{ if $v.Auth.Auth }}auth = {{ printf "%q" $v.Auth.Auth }}{{end}} - {{ if $v.Auth.IdentityToken }}identitytoken = {{ printf "%q" $v.Auth.IdentityToken }}{{end}} - {{end}} - {{end}} - {{end}} - [plugins."io.containerd.grpc.v1.cri".image_decryption] - key_model = "" - [plugins."io.containerd.grpc.v1.cri".x509_key_pair_streaming] - tls_cert_file = "" - tls_key_file = "" - [plugins."io.containerd.internal.v1.opt"] - path = {{ printf "%q" .NodeConfig.Containerd.Opt }} - [plugins."io.containerd.internal.v1.restart"] - interval = "10s" - [plugins."io.containerd.metadata.v1.bolt"] - content_sharing_policy = "shared" - [plugins."io.containerd.runtime.v2.task"] - platforms = ["windows/amd64", "linux/amd64"] - [plugins."io.containerd.service.v1.diff-service"] - default = ["windows", "windows-lcow"] -` - -// Windows config templates need named pipe addresses fixed up -var templateFuncs = template.FuncMap{ - "deschemify": func(s string) string { - if strings.HasPrefix(s, "npipe:") { - u, err := url.Parse(s) - if err != nil { - return "" - } - return u.Path - } - return s - }, -}