Merge pull request containerd#9681 from dmcgowan/cri-runtime-plugin

Add CRI Service plugin type

cmd/containerd/builtins/cri.go

@@ -19,6 +19,7 @@
 package builtins
 
 import (
-	_ "github.com/containerd/containerd/v2/pkg/cri"
+	_ "github.com/containerd/containerd/v2/plugins/cri"
 	_ "github.com/containerd/containerd/v2/plugins/cri/images"
+	_ "github.com/containerd/containerd/v2/plugins/cri/runtime"
 )

contrib/fuzz/builtins.go

@@ -19,10 +19,11 @@ package fuzz
 import (
 	// base containerd imports
 	_ "github.com/containerd/containerd/v2/core/runtime/v2"
-	_ "github.com/containerd/containerd/v2/pkg/cri"
 	_ "github.com/containerd/containerd/v2/pkg/events/plugin"
 	_ "github.com/containerd/containerd/v2/pkg/nri/plugin"
+	_ "github.com/containerd/containerd/v2/plugins/cri"
 	_ "github.com/containerd/containerd/v2/plugins/cri/images"
+	_ "github.com/containerd/containerd/v2/plugins/cri/runtime"
 	_ "github.com/containerd/containerd/v2/plugins/diff/walking/plugin"
 	_ "github.com/containerd/containerd/v2/plugins/gc"
 	_ "github.com/containerd/containerd/v2/plugins/imageverifier"

contrib/fuzz/cri_server_fuzzer.go

@@ -29,6 +29,7 @@ import (
 	"github.com/containerd/containerd/v2/pkg/cri/server"
 	"github.com/containerd/containerd/v2/pkg/cri/server/images"
 	"github.com/containerd/containerd/v2/pkg/oci"
+	"github.com/containerd/errdefs"
 )
 
 func FuzzCRIServer(data []byte) int {
@@ -42,7 +43,6 @@ func FuzzCRIServer(data []byte) int {
 	}
 	defer client.Close()
 
-	config := criconfig.Config{}
 	imageConfig := criconfig.ImageConfig{}
 
 	imageService, err := images.NewService(imageConfig, &images.CRIImageServiceOptions{
@@ -52,10 +52,10 @@ func FuzzCRIServer(data []byte) int {
 		panic(err)
 	}
 
-	c, rs, err := server.NewCRIService(config, &server.CRIServiceOptions{
-		ImageService: imageService,
-		Client:       client,
-		BaseOCISpecs: map[string]*oci.Spec{},
+	c, rs, err := server.NewCRIService(&server.CRIServiceOptions{
+		RuntimeService: &fakeRuntimeService{},
+		ImageService:   imageService,
+		Client:         client,
 	})
 	if err != nil {
 		panic(err)
@@ -68,6 +68,16 @@ func FuzzCRIServer(data []byte) int {
 	})
 }
 
+type fakeRuntimeService struct{}
+
+func (fakeRuntimeService) Config() criconfig.Config {
+	return criconfig.Config{}
+}
+
+func (fakeRuntimeService) LoadOCISpec(string) (*oci.Spec, error) {
+	return nil, errdefs.ErrNotFound
+}
+
 type service struct {
 	server.CRIService
 	runtime.RuntimeServiceServer

integration/build_local_containerd_helper_test.go

@@ -38,6 +38,7 @@ import (
 	_ "github.com/containerd/containerd/v2/core/runtime/v2/runc/options"
 	_ "github.com/containerd/containerd/v2/pkg/events/plugin"
 	_ "github.com/containerd/containerd/v2/plugins/cri/images"
+	_ "github.com/containerd/containerd/v2/plugins/cri/runtime"
 	_ "github.com/containerd/containerd/v2/plugins/diff/walking/plugin"
 	_ "github.com/containerd/containerd/v2/plugins/gc"
 	_ "github.com/containerd/containerd/v2/plugins/leases"

integration/main_test.go

@@ -51,7 +51,7 @@ import (
 	dialer "github.com/containerd/containerd/v2/integration/remote/util"
 	criconfig "github.com/containerd/containerd/v2/pkg/cri/config"
 	"github.com/containerd/containerd/v2/pkg/cri/constants"
-	"github.com/containerd/containerd/v2/pkg/cri/server/base"
+	"github.com/containerd/containerd/v2/pkg/cri/types"
 	"github.com/containerd/containerd/v2/pkg/cri/util"
 )
 
@@ -686,7 +686,7 @@ func CRIConfig() (*criconfig.Config, error) {
 }
 
 // SandboxInfo gets sandbox info.
-func SandboxInfo(id string) (*runtime.PodSandboxStatus, *base.SandboxInfo, error) {
+func SandboxInfo(id string) (*runtime.PodSandboxStatus, *types.SandboxInfo, error) {
 	client, err := RawRuntimeClient()
 	if err != nil {
 		return nil, nil, fmt.Errorf("failed to get raw runtime client: %w", err)
@@ -699,7 +699,7 @@ func SandboxInfo(id string) (*runtime.PodSandboxStatus, *base.SandboxInfo, error
 		return nil, nil, fmt.Errorf("failed to get sandbox status: %w", err)
 	}
 	status := resp.GetStatus()
-	var info base.SandboxInfo
+	var info types.SandboxInfo
 	if err := json.Unmarshal([]byte(resp.GetInfo()["info"]), &info); err != nil {
 		return nil, nil, fmt.Errorf("failed to unmarshal sandbox info: %w", err)
 	}

integration/sandbox_run_rollback_test.go

@@ -36,7 +36,7 @@ import (
 	criapiv1 "k8s.io/cri-api/pkg/apis/runtime/v1"
 
 	"github.com/containerd/containerd/v2/internal/failpoint"
-	"github.com/containerd/containerd/v2/pkg/cri/server/base"
+	"github.com/containerd/containerd/v2/pkg/cri/types"
 )
 
 const (
@@ -299,7 +299,7 @@ func TestRunPodSandboxAndTeardownCNISlow(t *testing.T) {
 }
 
 // sbserverSandboxInfo gets sandbox info.
-func sbserverSandboxInfo(id string) (*criapiv1.PodSandboxStatus, *base.SandboxInfo, error) {
+func sbserverSandboxInfo(id string) (*criapiv1.PodSandboxStatus, *types.SandboxInfo, error) {
 	client, err := RawRuntimeClient()
 	if err != nil {
 		return nil, nil, fmt.Errorf("failed to get raw runtime client: %w", err)
@@ -312,7 +312,7 @@ func sbserverSandboxInfo(id string) (*criapiv1.PodSandboxStatus, *base.SandboxIn
 		return nil, nil, fmt.Errorf("failed to get sandbox status: %w", err)
 	}
 	status := resp.GetStatus()
-	var info base.SandboxInfo
+	var info types.SandboxInfo
 	if err := json.Unmarshal([]byte(resp.GetInfo()["info"]), &info); err != nil {
 		return nil, nil, fmt.Errorf("failed to unmarshal sandbox info: %w", err)
 	}

pkg/cri/config/config.go

@@ -26,6 +26,7 @@ import (
 	"github.com/containerd/log"
 	"github.com/pelletier/go-toml/v2"
 	runtime "k8s.io/cri-api/pkg/apis/runtime/v1"
+	"k8s.io/kubelet/pkg/cri/streaming"
 
 	runhcsoptions "github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/options"
 	runcoptions "github.com/containerd/containerd/v2/core/runtime/v2/runc/options"
@@ -312,33 +313,18 @@ type ImageConfig struct {
 	StatsCollectPeriod int `toml:"stats_collect_period" json:"statsCollectPeriod"`
 }
 
-// PluginConfig contains toml config related to CRI plugin,
+// RuntimeConfig contains toml config related to CRI plugin,
 // it is a subset of Config.
-type PluginConfig struct {
+type RuntimeConfig struct {
 	// ContainerdConfig contains config related to containerd
 	ContainerdConfig `toml:"containerd" json:"containerd"`
 	// CniConfig contains config related to cni
 	CniConfig `toml:"cni" json:"cni"`
-	// DisableTCPService disables serving CRI on the TCP server.
-	DisableTCPService bool `toml:"disable_tcp_service" json:"disableTCPService"`
-	// StreamServerAddress is the ip address streaming server is listening on.
-	StreamServerAddress string `toml:"stream_server_address" json:"streamServerAddress"`
-	// StreamServerPort is the port streaming server is listening on.
-	StreamServerPort string `toml:"stream_server_port" json:"streamServerPort"`
-	// StreamIdleTimeout is the maximum time a streaming connection
-	// can be idle before the connection is automatically closed.
-	// The string is in the golang duration format, see:
-	//   https://golang.org/pkg/time/#ParseDuration
-	StreamIdleTimeout string `toml:"stream_idle_timeout" json:"streamIdleTimeout"`
 	// EnableSelinux indicates to enable the selinux support.
 	EnableSelinux bool `toml:"enable_selinux" json:"enableSelinux"`
 	// SelinuxCategoryRange allows the upper bound on the category range to be set.
 	// If not specified or set to 0, defaults to 1024 from the selinux package.
 	SelinuxCategoryRange int `toml:"selinux_category_range" json:"selinuxCategoryRange"`
-	// EnableTLSStreaming indicates to enable the TLS streaming support.
-	EnableTLSStreaming bool `toml:"enable_tls_streaming" json:"enableTLSStreaming"`
-	// X509KeyPairStreaming is a x509 key pair used for TLS streaming
-	X509KeyPairStreaming `toml:"x509_key_pair_streaming" json:"x509KeyPairStreaming"`
 	// MaxContainerLogLineSize is the maximum log line size in bytes for a container.
 	// Log line longer than the limit will be split into multiple lines. Non-positive
 	// value means no limit.
@@ -418,10 +404,10 @@ type X509KeyPairStreaming struct {
 	TLSKeyFile string `toml:"tls_key_file" json:"tlsKeyFile"`
 }
 
-// Config contains all configurations for cri server.
+// Config contains all configurations for CRI runtime plugin.
 type Config struct {
-	// PluginConfig is the config for CRI plugin.
-	PluginConfig
+	// RuntimeConfig is the config for CRI runtime.
+	RuntimeConfig
 	// ContainerdRootDir is the root directory path for containerd.
 	ContainerdRootDir string `json:"containerdRootDir"`
 	// ContainerdEndpoint is the containerd endpoint path.
@@ -433,6 +419,25 @@ type Config struct {
 	StateDir string `json:"stateDir"`
 }
 
+// ServerConfig contains all the configuration for the CRI API server.
+type ServerConfig struct {
+	// DisableTCPService disables serving CRI on the TCP server.
+	DisableTCPService bool `toml:"disable_tcp_service" json:"disableTCPService"`
+	// StreamServerAddress is the ip address streaming server is listening on.
+	StreamServerAddress string `toml:"stream_server_address" json:"streamServerAddress"`
+	// StreamServerPort is the port streaming server is listening on.
+	StreamServerPort string `toml:"stream_server_port" json:"streamServerPort"`
+	// StreamIdleTimeout is the maximum time a streaming connection
+	// can be idle before the connection is automatically closed.
+	// The string is in the golang duration format, see:
+	//   https://golang.org/pkg/time/#ParseDuration
+	StreamIdleTimeout string `toml:"stream_idle_timeout" json:"streamIdleTimeout"`
+	// EnableTLSStreaming indicates to enable the TLS streaming support.
+	EnableTLSStreaming bool `toml:"enable_tls_streaming" json:"enableTLSStreaming"`
+	// X509KeyPairStreaming is a x509 key pair used for TLS streaming
+	X509KeyPairStreaming `toml:"x509_key_pair_streaming" json:"x509KeyPairStreaming"`
+}
+
 const (
 	// RuntimeUntrusted is the implicit runtime defined for ContainerdConfig.UntrustedWorkloadRuntime
 	RuntimeUntrusted = "untrusted"
@@ -494,8 +499,8 @@ func ValidateImageConfig(ctx context.Context, c *ImageConfig) ([]deprecation.War
 	return warnings, nil
 }
 
-// ValidatePluginConfig validates the given plugin configuration.
-func ValidatePluginConfig(ctx context.Context, c *PluginConfig) ([]deprecation.Warning, error) {
+// ValidateRuntimeConfig validates the given runtime configuration.
+func ValidateRuntimeConfig(ctx context.Context, c *RuntimeConfig) ([]deprecation.Warning, error) {
 	var warnings []deprecation.Warning
 	if c.ContainerdConfig.Runtimes == nil {
 		c.ContainerdConfig.Runtimes = make(map[string]Runtime)
@@ -520,13 +525,6 @@ func ValidatePluginConfig(ctx context.Context, c *PluginConfig) ([]deprecation.W
 		}
 	}
 
-	// Validation for stream_idle_timeout
-	if c.StreamIdleTimeout != "" {
-		if _, err := time.ParseDuration(c.StreamIdleTimeout); err != nil {
-			return warnings, fmt.Errorf("invalid stream idle timeout: %w", err)
-		}
-	}
-
 	// Validation for drain_exec_sync_io_timeout
 	if c.DrainExecSyncIOTimeout != "" {
 		if _, err := time.ParseDuration(c.DrainExecSyncIOTimeout); err != nil {
@@ -539,6 +537,18 @@ func ValidatePluginConfig(ctx context.Context, c *PluginConfig) ([]deprecation.W
 	return warnings, nil
 }
 
+// ValidateServerConfig validates the given server configuration.
+func ValidateServerConfig(ctx context.Context, c *ServerConfig) ([]deprecation.Warning, error) {
+	var warnings []deprecation.Warning
+	// Validation for stream_idle_timeout
+	if c.StreamIdleTimeout != "" {
+		if _, err := time.ParseDuration(c.StreamIdleTimeout); err != nil {
+			return warnings, fmt.Errorf("invalid stream idle timeout: %w", err)
+		}
+	}
+	return warnings, nil
+}
+
 func (config *Config) GetSandboxRuntime(podSandboxConfig *runtime.PodSandboxConfig, runtimeHandler string) (Runtime, error) {
 	if untrustedWorkload(podSandboxConfig) {
 		// If the untrusted annotation is provided, runtimeHandler MUST be empty.
@@ -627,3 +637,17 @@ func getRuntimeOptionsType(t string) interface{} {
 		return &runtimeoptions.Options{}
 	}
 }
+
+func DefaultServerConfig() ServerConfig {
+	return ServerConfig{
+		DisableTCPService:   true,
+		StreamServerAddress: "127.0.0.1",
+		StreamServerPort:    "0",
+		StreamIdleTimeout:   streaming.DefaultConfig.StreamIdleTimeout.String(), // 4 hour
+		EnableTLSStreaming:  false,
+		X509KeyPairStreaming: X509KeyPairStreaming{
+			TLSKeyFile:  "",
+			TLSCertFile: "",
+		},
+	}
+}

pkg/cri/config/config_kernel_linux.go

@@ -28,7 +28,7 @@ import (
 
 var kernelGreaterEqualThan = kernel.GreaterEqualThan
 
-func ValidateEnableUnprivileged(ctx context.Context, c *PluginConfig) error {
+func ValidateEnableUnprivileged(ctx context.Context, c *RuntimeConfig) error {
 	if c.EnableUnprivilegedICMP || c.EnableUnprivilegedPorts {
 		fourDotEleven := kernel.KernelVersion{Kernel: 4, Major: 11}
 		ok, err := kernelGreaterEqualThan(fourDotEleven)

pkg/cri/config/config_kernel_linux_test.go

@@ -32,13 +32,13 @@ func TestValidateEnableUnprivileged(t *testing.T) {
 
 	tests := []struct {
 		name          string
-		config        *PluginConfig
+		config        *RuntimeConfig
 		kernelGreater bool
 		expectedErr   string
 	}{
 		{
 			name: "disable unprivileged_icmp and unprivileged_port",
-			config: &PluginConfig{
+			config: &RuntimeConfig{
 				ContainerdConfig: ContainerdConfig{
 					DefaultRuntimeName: RuntimeDefault,
 					Runtimes: map[string]Runtime{
@@ -54,7 +54,7 @@ func TestValidateEnableUnprivileged(t *testing.T) {
 		},
 		{
 			name: "enable unprivileged_icmp or unprivileged_port, but kernel version is smaller than 4.11",
-			config: &PluginConfig{
+			config: &RuntimeConfig{
 				ContainerdConfig: ContainerdConfig{
 					DefaultRuntimeName: RuntimeDefault,
 					Runtimes: map[string]Runtime{
@@ -71,7 +71,7 @@ func TestValidateEnableUnprivileged(t *testing.T) {
 		},
 		{
 			name: "enable unprivileged_icmp or unprivileged_port, but kernel version is greater than or equal 4.11",
-			config: &PluginConfig{
+			config: &RuntimeConfig{
 				ContainerdConfig: ContainerdConfig{
 					DefaultRuntimeName: RuntimeDefault,
 					Runtimes: map[string]Runtime{

pkg/cri/config/config_kernel_other.go

@@ -22,6 +22,6 @@ import (
 	"context"
 )
 
-func ValidateEnableUnprivileged(ctx context.Context, c *PluginConfig) error {
+func ValidateEnableUnprivileged(ctx context.Context, c *RuntimeConfig) error {
 	return nil
 }