From cf51be252cf2213d8f22acd68e0469e9a0ac084d Mon Sep 17 00:00:00 2001 From: Adriano Santos Date: Thu, 8 Aug 2024 21:45:21 -0300 Subject: [PATCH 1/8] Remove handlers --- .../controller/activator_controller.ex | 15 +- .../controller/actor_host_controller.ex | 145 ++++++++++++++++- .../controller/actor_system_controller.ex | 88 ++++++++++- .../handler/activator_handler.ex | 17 -- .../handler/actor_host_handler.ex | 149 ------------------ .../handler/actor_system_handler.ex | 90 ----------- 6 files changed, 241 insertions(+), 263 deletions(-) delete mode 100644 spawn_operator/spawn_operator/lib/spawn_operator/handler/activator_handler.ex delete mode 100644 spawn_operator/spawn_operator/lib/spawn_operator/handler/actor_host_handler.ex delete mode 100644 spawn_operator/spawn_operator/lib/spawn_operator/handler/actor_system_handler.ex diff --git a/spawn_operator/spawn_operator/lib/spawn_operator/controller/activator_controller.ex b/spawn_operator/spawn_operator/lib/spawn_operator/controller/activator_controller.ex index 5bd4ee41..3bfe8dc3 100644 --- a/spawn_operator/spawn_operator/lib/spawn_operator/controller/activator_controller.ex +++ b/spawn_operator/spawn_operator/lib/spawn_operator/controller/activator_controller.ex @@ -1,10 +1,12 @@ defmodule SpawnOperator.Controller.ActivatorController do - require Bonny.API.CRD - + @moduledoc """ + `ActivatorHandler` handles Activator CRD events + """ use Bonny.ControllerV2 + require Bonny.API.CRD step(Bonny.Pluggable.SkipObservedGenerations) - step(SpawnOperator.Handler.ActivatorHandler) + step :handle_event @impl true def rbac_rules() do @@ -19,4 +21,11 @@ defmodule SpawnOperator.Controller.ActivatorController do to_rbac_rule({"networking.k8s.io", ["ingresses", "ingressclasses"], ["*"]}) ] end + + @impl Pluggable + @spec handle_event(Bonny.Axn.t(), Keyword.t()) :: Bonny.Axn.t() + def handle_event(%Bonny.Axn{action: action, resource: resource} = axn, nil) + do: + SpawnOperator.get_args(resource) + |> Activator.apply(axn, action) end diff --git a/spawn_operator/spawn_operator/lib/spawn_operator/controller/actor_host_controller.ex b/spawn_operator/spawn_operator/lib/spawn_operator/controller/actor_host_controller.ex index d7b9efd0..c71f7ef5 100644 --- a/spawn_operator/spawn_operator/lib/spawn_operator/controller/actor_host_controller.ex +++ b/spawn_operator/spawn_operator/lib/spawn_operator/controller/actor_host_controller.ex @@ -1,10 +1,100 @@ defmodule SpawnOperator.Controller.ActorHostController do - require Bonny.API.CRD + @moduledoc """ + `ActorHostHandler` handles ActorHost CRD events + + --- + apiVersion: spawn-eigr.io/v1 + kind: ActorHost + metadata: + name: my-node-app # Mandatory. Name of the Node containing Actor Host Functions + namespace: default # Optional. Default namespace is "default" + labels: + # Mandatory. Name of the ActorSystem declared in ActorSystem CRD + spawn-eigr.io.actor-system: my-actor-system + + # Optional + spawn-eigr.io.cluser.polingInterval: 3000 + + # Optional. Default "sidecar". Possible values are "sidecar" | "daemon" + spawn-eigr.io.sidecar.deploymentMode: "sidecar" + + # Optional + spawn-eigr.io.sidecar.containerImage: "docker.io/eigr/spawn-proxy" + + # Optional + spawn-eigr.io.sidecar.containerVersion: "1.4.2" + + # Optional. Default 9001 + spawn-eigr.io.sidecar.httpPort: 9001 + + # Optional. Default false + spawn-eigr.io.sidecar.udsEnable: false + + # Optional. Default "/var/run/spawn.sock" + spawn-eigr.io.sidecar.udsAddress: "/var/run/sidecar.sock" + + # Optional. Default false + spawn-eigr.io.sidecar.disableMetrics: false + + # Optional. Default true + spawn-eigr.io.sidecar.consoleDisableMetrics: true + + # Optional + spawn-eigr.io.sidecar.userFunctionHost: "0.0.0.0" + + # Optional + spawn-eigr.io.sidecar.userFunctionPort: 8090 + + # Optional. Default "native". + # Using Phoenix PubSub Adapter. + # Possible values: "native" | "nats" + spawn-eigr.io.sidecar.pubsub.adapter: "native" + + # Optional. Default "nats://127.0.0.1:4222" + spawn-eigr.io.sidecar.pubsub.nats.hosts: "nats://127.0.0.1:4222" + + # Optional. Default false + spawn-eigr.io.sidecar.pubsub.nats.tls: "false" + + # Optional. Default false + spawn-eigr.io.sidecar.pubsub.nats.auth: false + + # Optioal. Default "simple" + spawn-eigr.io.sidecar.pubsub.nats.authType: "simple" + + # Optional. Default "admin" + spawn-eigr.io.sidecar.pubsub.nats.authUser: "admin" + + # Optional. Default "admin" + spawn-eigr.io.sidecar.pubsub.nats.authPass: "admin" + + # Optional. Default "" + spawn-eigr.io.sidecar.pubsub.nats.authJwt: "" + spec: + autoscaler: # Optional + min: 1 + max: 2 + averageCpuUtilizationPercentage: 80 + averageMemoryUtilizationValue: 250 + affinity: k8s_affinity_declaration_here # Optional + + replicas: 1 # Optional. If negative number than autoscaling is enable + + host: # Mandatory + image: docker.io/eigr/spawn-springboot-examples:latest # Mandatory + embedded: false # Optional. Default false. True only when the SDK supports a native connection to the Spawn mesh network + ports: + - containerPort: 80 + + """ use Bonny.ControllerV2 + require Bonny.API.CRD + + alias SpawnOperator.K8s.Proxy.{CM.Configmap, Deployment, HPA, Service} step(Bonny.Pluggable.SkipObservedGenerations) - step(SpawnOperator.Handler.ActorHostHandler) + step :handle_event @impl true def rbac_rules() do @@ -19,4 +109,55 @@ defmodule SpawnOperator.Controller.ActorHostController do to_rbac_rule({"networking.k8s.io", ["ingresses", "ingressclasses"], ["*"]}) ] end + + @spec handle_event(Bonny.Axn.t(), Keyword.t()) :: Bonny.Axn.t() + def handle_event(%Bonny.Axn{action: action, resource: resource} = axn, nil) + when action in [:add, :modify] do + host_config_map = build_host_configmap(resource) + host_resource = build_host_deploy(resource) + host_hpa = build_host_hpa(resource) + host_service = build_host_service(resource) + + axn + |> Bonny.Axn.register_descendant(host_hpa) + |> Bonny.Axn.register_descendant(host_service) + |> Bonny.Axn.register_descendant(host_config_map) + |> Bonny.Axn.register_descendant(host_resource) + # |> Bonny.Axn.update_status(fn status -> + # put_in(status, [Access.key(:some, %{}), :field], "foo") + # end) + |> Bonny.Axn.success_event() + end + + @spec handle_event(Bonny.Axn.t(), Keyword.t()) :: Bonny.Axn.t() + def handle_event(%Bonny.Axn{action: action} = axn, nil) when action in [:reconcile] do + # TODO: Reconcile hpa for rebalancing Nodes + # TODO: Recreate resources if not exists + Bonny.Axn.success_event(axn) + end + + @spec handle_event(Bonny.Axn.t(), Keyword.t()) :: Bonny.Axn.t() + def handle_event(%Bonny.Axn{action: action} = axn, nil) when action in [:delete] do + Bonny.Axn.success_event(axn) + end + + defp build_host_deploy(resource) do + SpawnOperator.get_args(resource) + |> Deployment.manifest() + end + + defp build_host_service(resource) do + SpawnOperator.get_args(resource) + |> Service.manifest() + end + + defp build_host_configmap(resource) do + SpawnOperator.get_args(resource) + |> Configmap.manifest() + end + + defp build_host_hpa(resource) do + SpawnOperator.get_args(resource) + |> HPA.manifest() + end end diff --git a/spawn_operator/spawn_operator/lib/spawn_operator/controller/actor_system_controller.ex b/spawn_operator/spawn_operator/lib/spawn_operator/controller/actor_system_controller.ex index 653c2d6e..e561238f 100644 --- a/spawn_operator/spawn_operator/lib/spawn_operator/controller/actor_system_controller.ex +++ b/spawn_operator/spawn_operator/lib/spawn_operator/controller/actor_system_controller.ex @@ -1,10 +1,44 @@ defmodule SpawnOperator.Controller.ActorSystemController do - require Bonny.API.CRD + @moduledoc """ + `ActorSystemHandler` handles ActorSystem CRD events + + --- + apiVersion: spawn-eigr.io/v1 + kind: ActorSystem + metadata: + name: spawn-system # Mandatory. Name of the state store + namespace: default # Optional. Default namespace is "default" + spec: + cluster: # Optional + kind: erlang # Optional. Default erlang. Possible values [erlang | quic] + cookie: default-c21f969b5f03d33d43e04f8f136e7682 # Optional. Only used if kind is erlang + systemToSystem: + enabled: true + natsClusterSecretRef: nats-config-secret + tls: + secretName: spawn-system-tls-secret + certManager: + enabled: true # Default false + issuerName: spawn-system-issuer # You must create an Issuer previously according to certmanager documentation + + statestore: + type: Postgres + credentialsSecretRef: postgres-connection-secret # The secret containing connection params + pool: # Optional + size: 10 + """ use Bonny.ControllerV2 + require Bonny.API.CRD + + alias SpawnOperator.K8s.System.HeadlessService + alias SpawnOperator.K8s.System.Secret.ActorSystemSecret + alias SpawnOperator.K8s.System.Role + alias SpawnOperator.K8s.System.RoleBinding + alias SpawnOperator.K8s.System.ServiceAccount step(Bonny.Pluggable.SkipObservedGenerations) - step(SpawnOperator.Handler.ActorSystemHandler) + step :handle_event def rbac_rules() do [ @@ -18,4 +52,54 @@ defmodule SpawnOperator.Controller.ActorSystemController do to_rbac_rule({"cert-manager.io", "certificate", "*"}) ] end + + @spec handle_event(Bonny.Axn.t(), Keyword.t()) :: Bonny.Axn.t() + def handle_event(%Bonny.Axn{action: action, resource: resource} = axn, nil) + when action in [:add, :modify] do + %Bonny.Axn{resource: resource} = axn + + cluster_secret = build_system_secret(resource) + cluster_service = build_system_service(resource) + service_account = build_service_account(resource) + roles = build_role(resource) + role_binding = build_role_binding(resource) + + axn + |> Bonny.Axn.register_descendant(cluster_secret) + |> Bonny.Axn.register_descendant(cluster_service) + |> Bonny.Axn.register_descendant(service_account) + |> Bonny.Axn.register_descendant(roles) + |> Bonny.Axn.register_descendant(role_binding) + |> Bonny.Axn.success_event() + end + + @spec handle_event(Bonny.Axn.t(), Keyword.t()) :: Bonny.Axn.t() + def handle_event(%Bonny.Axn{action: action} = axn, nil) when action in [:delete, :reconcile] do + Bonny.Axn.success_event(axn) + end + + defp build_system_secret(resource) do + SpawnOperator.get_args(resource) + |> ActorSystemSecret.manifest() + end + + defp build_system_service(resource) do + SpawnOperator.get_args(resource) + |> HeadlessService.manifest() + end + + defp build_service_account(resource) do + SpawnOperator.get_args(resource) + |> ServiceAccount.manifest() + end + + defp build_role(resource) do + SpawnOperator.get_args(resource) + |> Role.manifest() + end + + defp build_role_binding(resource) do + SpawnOperator.get_args(resource) + |> RoleBinding.manifest() + end end diff --git a/spawn_operator/spawn_operator/lib/spawn_operator/handler/activator_handler.ex b/spawn_operator/spawn_operator/lib/spawn_operator/handler/activator_handler.ex deleted file mode 100644 index 1f4001b7..00000000 --- a/spawn_operator/spawn_operator/lib/spawn_operator/handler/activator_handler.ex +++ /dev/null @@ -1,17 +0,0 @@ -defmodule SpawnOperator.Handler.ActivatorHandler do - @moduledoc """ - `ActivatorHandler` handles Activator CRD events - """ - alias SpawnOperator.K8s.Activators.Activator - - @behaviour Pluggable - - @impl Pluggable - def init(_opts), do: nil - - @impl Pluggable - def call(%Bonny.Axn{action: action, resource: resource} = axn, nil), - do: - SpawnOperator.get_args(resource) - |> Activator.apply(axn, action) -end diff --git a/spawn_operator/spawn_operator/lib/spawn_operator/handler/actor_host_handler.ex b/spawn_operator/spawn_operator/lib/spawn_operator/handler/actor_host_handler.ex deleted file mode 100644 index dd978d10..00000000 --- a/spawn_operator/spawn_operator/lib/spawn_operator/handler/actor_host_handler.ex +++ /dev/null @@ -1,149 +0,0 @@ -defmodule SpawnOperator.Handler.ActorHostHandler do - @moduledoc """ - `ActorHostHandler` handles ActorHost CRD events - - --- - apiVersion: spawn-eigr.io/v1 - kind: ActorHost - metadata: - name: my-node-app # Mandatory. Name of the Node containing Actor Host Functions - namespace: default # Optional. Default namespace is "default" - labels: - # Mandatory. Name of the ActorSystem declared in ActorSystem CRD - spawn-eigr.io.actor-system: my-actor-system - - # Optional - spawn-eigr.io.cluser.polingInterval: 3000 - - # Optional. Default "sidecar". Possible values are "sidecar" | "daemon" - spawn-eigr.io.sidecar.deploymentMode: "sidecar" - - # Optional - spawn-eigr.io.sidecar.containerImage: "docker.io/eigr/spawn-proxy" - - # Optional - spawn-eigr.io.sidecar.containerVersion: "1.4.2" - - # Optional. Default 9001 - spawn-eigr.io.sidecar.httpPort: 9001 - - # Optional. Default false - spawn-eigr.io.sidecar.udsEnable: false - - # Optional. Default "/var/run/spawn.sock" - spawn-eigr.io.sidecar.udsAddress: "/var/run/sidecar.sock" - - # Optional. Default false - spawn-eigr.io.sidecar.disableMetrics: false - - # Optional. Default true - spawn-eigr.io.sidecar.consoleDisableMetrics: true - - # Optional - spawn-eigr.io.sidecar.userFunctionHost: "0.0.0.0" - - # Optional - spawn-eigr.io.sidecar.userFunctionPort: 8090 - - # Optional. Default "native". - # Using Phoenix PubSub Adapter. - # Possible values: "native" | "nats" - spawn-eigr.io.sidecar.pubsub.adapter: "native" - - # Optional. Default "nats://127.0.0.1:4222" - spawn-eigr.io.sidecar.pubsub.nats.hosts: "nats://127.0.0.1:4222" - - # Optional. Default false - spawn-eigr.io.sidecar.pubsub.nats.tls: "false" - - # Optional. Default false - spawn-eigr.io.sidecar.pubsub.nats.auth: false - - # Optioal. Default "simple" - spawn-eigr.io.sidecar.pubsub.nats.authType: "simple" - - # Optional. Default "admin" - spawn-eigr.io.sidecar.pubsub.nats.authUser: "admin" - - # Optional. Default "admin" - spawn-eigr.io.sidecar.pubsub.nats.authPass: "admin" - - # Optional. Default "" - spawn-eigr.io.sidecar.pubsub.nats.authJwt: "" - spec: - autoscaler: # Optional - min: 1 - max: 2 - averageCpuUtilizationPercentage: 80 - averageMemoryUtilizationValue: 250 - - affinity: k8s_affinity_declaration_here # Optional - - replicas: 1 # Optional. If negative number than autoscaling is enable - - host: # Mandatory - image: docker.io/eigr/spawn-springboot-examples:latest # Mandatory - embedded: false # Optional. Default false. True only when the SDK supports a native connection to the Spawn mesh network - ports: - - containerPort: 80 - - """ - - alias SpawnOperator.K8s.Proxy.{CM.Configmap, Deployment, HPA, Service} - - @behaviour Pluggable - - @impl Pluggable - def init(_opts), do: nil - - @impl Pluggable - def call(%Bonny.Axn{action: action, resource: resource} = axn, nil) - when action in [:add, :modify] do - host_config_map = build_host_configmap(resource) - host_resource = build_host_deploy(resource) - host_hpa = build_host_hpa(resource) - host_service = build_host_service(resource) - - axn - |> Bonny.Axn.register_descendant(host_hpa) - |> Bonny.Axn.register_descendant(host_service) - |> Bonny.Axn.register_descendant(host_config_map) - |> Bonny.Axn.register_descendant(host_resource) - # |> Bonny.Axn.update_status(fn status -> - # put_in(status, [Access.key(:some, %{}), :field], "foo") - # end) - |> Bonny.Axn.success_event() - end - - @impl Pluggable - def call(%Bonny.Axn{action: action} = axn, nil) when action in [:reconcile] do - # TODO: Reconcile hpa for rebalancing Nodes - # TODO: Recreate resources if not exists - Bonny.Axn.success_event(axn) - end - - @impl Pluggable - def call(%Bonny.Axn{action: action} = axn, nil) when action in [:delete] do - Bonny.Axn.success_event(axn) - end - - defp build_host_deploy(resource) do - SpawnOperator.get_args(resource) - |> Deployment.manifest() - end - - defp build_host_service(resource) do - SpawnOperator.get_args(resource) - |> Service.manifest() - end - - defp build_host_configmap(resource) do - SpawnOperator.get_args(resource) - |> Configmap.manifest() - end - - defp build_host_hpa(resource) do - SpawnOperator.get_args(resource) - |> HPA.manifest() - end -end diff --git a/spawn_operator/spawn_operator/lib/spawn_operator/handler/actor_system_handler.ex b/spawn_operator/spawn_operator/lib/spawn_operator/handler/actor_system_handler.ex deleted file mode 100644 index 74fb789b..00000000 --- a/spawn_operator/spawn_operator/lib/spawn_operator/handler/actor_system_handler.ex +++ /dev/null @@ -1,90 +0,0 @@ -defmodule SpawnOperator.Handler.ActorSystemHandler do - @moduledoc """ - `ActorSystemHandler` handles ActorSystem CRD events - - --- - apiVersion: spawn-eigr.io/v1 - kind: ActorSystem - metadata: - name: spawn-system # Mandatory. Name of the state store - namespace: default # Optional. Default namespace is "default" - spec: - cluster: # Optional - kind: erlang # Optional. Default erlang. Possible values [erlang | quic] - cookie: default-c21f969b5f03d33d43e04f8f136e7682 # Optional. Only used if kind is erlang - systemToSystem: - enabled: true - natsClusterSecretRef: nats-config-secret - tls: - secretName: spawn-system-tls-secret - certManager: - enabled: true # Default false - issuerName: spawn-system-issuer # You must create an Issuer previously according to certmanager documentation - - statestore: - type: Postgres - credentialsSecretRef: postgres-connection-secret # The secret containing connection params - pool: # Optional - size: 10 - - """ - alias SpawnOperator.K8s.System.HeadlessService - alias SpawnOperator.K8s.System.Secret.ActorSystemSecret - alias SpawnOperator.K8s.System.Role - alias SpawnOperator.K8s.System.RoleBinding - alias SpawnOperator.K8s.System.ServiceAccount - - @behaviour Pluggable - - @impl Pluggable - def init(_opts), do: nil - - @impl Pluggable - def call(%Bonny.Axn{action: action} = axn, nil) when action in [:add, :modify] do - %Bonny.Axn{resource: resource} = axn - - cluster_secret = build_system_secret(resource) - cluster_service = build_system_service(resource) - service_account = build_service_account(resource) - roles = build_role(resource) - role_binding = build_role_binding(resource) - - axn - |> Bonny.Axn.register_descendant(cluster_secret) - |> Bonny.Axn.register_descendant(cluster_service) - |> Bonny.Axn.register_descendant(service_account) - |> Bonny.Axn.register_descendant(roles) - |> Bonny.Axn.register_descendant(role_binding) - |> Bonny.Axn.success_event() - end - - @impl Pluggable - def call(%Bonny.Axn{action: action} = axn, nil) when action in [:delete, :reconcile] do - Bonny.Axn.success_event(axn) - end - - defp build_system_secret(resource) do - SpawnOperator.get_args(resource) - |> ActorSystemSecret.manifest() - end - - defp build_system_service(resource) do - SpawnOperator.get_args(resource) - |> HeadlessService.manifest() - end - - defp build_service_account(resource) do - SpawnOperator.get_args(resource) - |> ServiceAccount.manifest() - end - - defp build_role(resource) do - SpawnOperator.get_args(resource) - |> Role.manifest() - end - - defp build_role_binding(resource) do - SpawnOperator.get_args(resource) - |> RoleBinding.manifest() - end -end From 95378e1f7b077b3231dfb4e9c8130a58065a9820 Mon Sep 17 00:00:00 2001 From: Adriano Santos Date: Thu, 8 Aug 2024 21:57:22 -0300 Subject: [PATCH 2/8] Adjusts and add impl true --- .../spawn_operator/controller/activator_controller.ex | 11 ++++++----- .../controller/actor_system_controller.ex | 1 + .../spawn_operator/lib/spawn_operator/operator.ex | 2 ++ 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/spawn_operator/spawn_operator/lib/spawn_operator/controller/activator_controller.ex b/spawn_operator/spawn_operator/lib/spawn_operator/controller/activator_controller.ex index 3bfe8dc3..7b82bf50 100644 --- a/spawn_operator/spawn_operator/lib/spawn_operator/controller/activator_controller.ex +++ b/spawn_operator/spawn_operator/lib/spawn_operator/controller/activator_controller.ex @@ -5,6 +5,8 @@ defmodule SpawnOperator.Controller.ActivatorController do use Bonny.ControllerV2 require Bonny.API.CRD + alias SpawnOperator.K8s.Activators.Activator + step(Bonny.Pluggable.SkipObservedGenerations) step :handle_event @@ -22,10 +24,9 @@ defmodule SpawnOperator.Controller.ActivatorController do ] end - @impl Pluggable @spec handle_event(Bonny.Axn.t(), Keyword.t()) :: Bonny.Axn.t() - def handle_event(%Bonny.Axn{action: action, resource: resource} = axn, nil) - do: - SpawnOperator.get_args(resource) - |> Activator.apply(axn, action) + def handle_event(%Bonny.Axn{action: action, resource: resource} = axn, nil) do + SpawnOperator.get_args(resource) + |> Activator.apply(axn, action) + end end diff --git a/spawn_operator/spawn_operator/lib/spawn_operator/controller/actor_system_controller.ex b/spawn_operator/spawn_operator/lib/spawn_operator/controller/actor_system_controller.ex index e561238f..83f0c319 100644 --- a/spawn_operator/spawn_operator/lib/spawn_operator/controller/actor_system_controller.ex +++ b/spawn_operator/spawn_operator/lib/spawn_operator/controller/actor_system_controller.ex @@ -40,6 +40,7 @@ defmodule SpawnOperator.Controller.ActorSystemController do step(Bonny.Pluggable.SkipObservedGenerations) step :handle_event + @impl true def rbac_rules() do [ to_rbac_rule({"rbac.authorization.k8s.io", ["role", "roles", "rolebindings"], "*"}), diff --git a/spawn_operator/spawn_operator/lib/spawn_operator/operator.ex b/spawn_operator/spawn_operator/lib/spawn_operator/operator.ex index 06c69478..5c3b0b79 100644 --- a/spawn_operator/spawn_operator/lib/spawn_operator/operator.ex +++ b/spawn_operator/spawn_operator/lib/spawn_operator/operator.ex @@ -6,6 +6,7 @@ defmodule SpawnOperator.Operator do step(Bonny.Pluggable.ApplyStatus) step(Bonny.Pluggable.ApplyDescendants) + @impl true def crds() do [ Bonny.API.CRD.new!( @@ -44,6 +45,7 @@ defmodule SpawnOperator.Operator do ] end + @impl true def controllers(watch_namespace, _opts) do [ %{ From f6d982e717b018d7d0afb11751df2a5afc352732 Mon Sep 17 00:00:00 2001 From: Adriano Santos Date: Thu, 8 Aug 2024 23:00:04 -0300 Subject: [PATCH 3/8] Added crd schema --- .../controller/actor_host_controller.ex | 1 + .../versions/api/v1/actor_host.ex | 100 +++++++++++++++++- .../versions/api/v1/actor_system.ex | 84 +++++++++++++++ 3 files changed, 181 insertions(+), 4 deletions(-) diff --git a/spawn_operator/spawn_operator/lib/spawn_operator/controller/actor_host_controller.ex b/spawn_operator/spawn_operator/lib/spawn_operator/controller/actor_host_controller.ex index c71f7ef5..3f04adde 100644 --- a/spawn_operator/spawn_operator/lib/spawn_operator/controller/actor_host_controller.ex +++ b/spawn_operator/spawn_operator/lib/spawn_operator/controller/actor_host_controller.ex @@ -84,6 +84,7 @@ defmodule SpawnOperator.Controller.ActorHostController do host: # Mandatory image: docker.io/eigr/spawn-springboot-examples:latest # Mandatory embedded: false # Optional. Default false. True only when the SDK supports a native connection to the Spawn mesh network + hostSDK: java # valid [dart, elixir, java, python, rust, springboot, nodejs] ports: - containerPort: 80 diff --git a/spawn_operator/spawn_operator/lib/spawn_operator/versions/api/v1/actor_host.ex b/spawn_operator/spawn_operator/lib/spawn_operator/versions/api/v1/actor_host.ex index 7734c307..c298c11a 100644 --- a/spawn_operator/spawn_operator/lib/spawn_operator/versions/api/v1/actor_host.ex +++ b/spawn_operator/spawn_operator/lib/spawn_operator/versions/api/v1/actor_host.ex @@ -1,13 +1,105 @@ defmodule SpawnOperator.Versions.Api.V1.ActorHost do - use Bonny.API.Version + @moduledoc """ + ActorHost CRD v1 version. + """ + use Bonny.API.Version, + hub: true + + import YamlElixir.Sigil @impl true def manifest() do - defaults() - |> struct!( + struct!( + defaults(), name: "v1", - storage: true + schema: ~y""" + :openAPIV3Schema: + :type: object + :required: ["spec"] + :properties: + :spec: + type: object + properties: + autoscaler: + type: object + properties: + min: + type: integer + max: + type: integer + averageCpuUtilizationPercentage: + type: integer + averageMemoryUtilizationValue: + type: integer + affinity: + type: object + replicas: + type: integer + host: + type: object + required: + - image + properties: + image: + type: string + embedded: + type: boolean + hostSDK: + type: string + enum: ["dart", "elixir", "java", "python", "rust", "springboot", "nodejs"] + ports: + type: array + items: + type: object + properties: + containerPort: + type: integer + env: + type: array + items: + type: object + properties: + name: + type: string + value: + type: string + valueFrom: + type: object + properties: + fieldRef: + type: object + properties: + fieldPath: + type: string + """a, + additionalPrinterColumns: [ + %{ + name: "Image", + type: "string", + description: "Docker image used for the ActorHost", + jsonPath: ".spec.host.image" + }, + %{ + name: "Host SDK", + type: "string", + description: "SDK used by the ActorHost", + jsonPath: ".spec.host.hostSDK" + }, + %{ + name: "Min Replicas", + type: "integer", + description: "Minimum number of replicas for the ActorHost", + jsonPath: ".spec.autoscaler.min" + }, + %{ + name: "Max Replicas", + type: "integer", + description: "Maximum number of replicas for the ActorHost", + jsonPath: ".spec.autoscaler.max" + } + ] ) |> add_observed_generation_status() + |> add_conditions() end end diff --git a/spawn_operator/spawn_operator/lib/spawn_operator/versions/api/v1/actor_system.ex b/spawn_operator/spawn_operator/lib/spawn_operator/versions/api/v1/actor_system.ex index 82b992c8..b917dc4d 100644 --- a/spawn_operator/spawn_operator/lib/spawn_operator/versions/api/v1/actor_system.ex +++ b/spawn_operator/spawn_operator/lib/spawn_operator/versions/api/v1/actor_system.ex @@ -11,3 +11,87 @@ defmodule SpawnOperator.Versions.Api.V1.ActorSystem do |> add_observed_generation_status() end end + +defmodule SpawnOperator.Versions.Api.V1.ActorSystem do + @moduledoc """ + ActorSystem CRD v1 version. + """ + use Bonny.API.Version, + hub: true + + import YamlElixir.Sigil + + @impl true + def manifest() do + struct!( + defaults(), + name: "v1", + schema: ~y""" + :openAPIV3Schema: + :type: object + :required: ["spec"] + :properties: + :spec: + type: object + properties: + cluster: + type: object + properties: + kind: + type: string + enum: ["erlang", "quic"] + default: "erlang" + cookie: + type: string + systemToSystem: + type: object + properties: + enabled: + type: boolean + natsClusterSecretRef: + type: string + tls: + type: object + properties: + secretName: + type: string + certManager: + type: object + properties: + enabled: + type: boolean + issuerName: + type: string + statestore: + type: object + properties: + type: + type: string + enum: ["Postgres"] + credentialsSecretRef: + type: string + pool: + type: object + properties: + size: + type: integer + """a, + additionalPrinterColumns: [ + %{ + name: "Cluster Kind", + type: "string", + description: "The kind of cluster used for the ActorSystem", + jsonPath: ".spec.cluster.kind" + }, + %{ + name: "State Store Type", + type: "string", + description: "The type of state store used for the ActorSystem", + jsonPath: ".spec.statestore.type" + } + ] + ) + |> add_observed_generation_status() + |> add_conditions() + end +end From 2ef2b194b0b39fc34579ec3e5bfdee32889e8eea Mon Sep 17 00:00:00 2001 From: Adriano Santos Date: Mon, 12 Aug 2024 23:27:19 -0300 Subject: [PATCH 4/8] Update docker files to use user 1000 --- Dockerfile-initializer | 39 +++++++++++++++------------- Dockerfile-operator | 40 +++++++++++++++-------------- Dockerfile-proxy | 58 +++++++++++++++++++----------------------- Makefile | 5 +++- 4 files changed, 73 insertions(+), 69 deletions(-) diff --git a/Dockerfile-initializer b/Dockerfile-initializer index 903f3672..dceb624c 100644 --- a/Dockerfile-initializer +++ b/Dockerfile-initializer @@ -1,49 +1,54 @@ +# Build Stage FROM elixir:1.15-alpine AS builder ENV MIX_ENV=prod WORKDIR /app -RUN apk add --no-cache --update git build-base ca-certificates zstd gcc pkgconfig openssl-dev +# Install build dependencies and tools +RUN apk add --no-cache git build-base ca-certificates zstd gcc pkgconfig openssl-dev +# Copy application code COPY spawn_initializer/ . +# Install Elixir dependencies and build release RUN mix local.rebar --force \ && mix local.hex --force \ && mix deps.get \ && mix release.init -# Overriden at runtime +# Environment variables for release ENV POD_IP="127.0.0.1" - -# This will be the basename of node ENV RELEASE_NAME="spawn_initializer" - -# This will be the full nodename ENV RELEASE_NODE="${RELEASE_NAME}@${POD_IP}" - +# Build the release RUN mix deps.get \ && mix release spawn_initializer -# ---- Application Stage ---- +# Runtime Stage FROM alpine:3.20 -RUN apk add --no-cache --update zstd ncurses-libs libstdc++ libgcc +# Install runtime dependencies: OpenSSL, zstd, and necessary runtime libraries +RUN apk add --no-cache openssl zstd ncurses-libs libstdc++ libgcc \ + # Create a user with UID 1000 and set up the home directory + && adduser -D -u 1000 appuser \ + && mkdir -p /app/.cache/bakeware/ /data \ + && chown appuser:appuser /app /app/.cache /data \ + && chmod 777 /app/.cache/bakeware/ /data WORKDIR /app -RUN chown nobody /app -# Set runner ENV +# Set environment variables ENV MIX_ENV=prod ENV HOME=/app -COPY --from=builder --chown=nobody:root /app/_build/${MIX_ENV}/rel/bakeware/ ./ - -RUN mkdir -p /app/.cache/bakeware/ && chmod 777 /app/.cache/bakeware/ -RUN touch /.erlang.cookie && chmod 777 /.erlang.cookie -RUN touch /app/.erlang.cookie && chmod 777 /app/.erlang.cookie +# Copy compiled release from builder stage +COPY --from=builder --chown=1000:1000 /app/_build/${MIX_ENV}/rel/bakeware/ ./ -USER nobody +# Create necessary files and set permissions +RUN touch /.erlang.cookie && chmod 777 /.erlang.cookie \ + && touch /app/.erlang.cookie && chmod 777 /app/.erlang.cookie +# Set entrypoint ENTRYPOINT ["./spawn_initializer"] diff --git a/Dockerfile-operator b/Dockerfile-operator index b1399e04..4eb9e9d6 100644 --- a/Dockerfile-operator +++ b/Dockerfile-operator @@ -1,12 +1,14 @@ +# Build Stage FROM elixir:1.15-alpine AS builder ENV MIX_ENV=prod WORKDIR /app -RUN apk add --no-cache --update git build-base ca-certificates zstd gcc pkgconfig openssl-dev +# Install build dependencies and tools +RUN apk add --no-cache git build-base ca-certificates zstd gcc pkgconfig openssl-dev -RUN mkdir config +# Copy application code COPY config/ ./config COPY spawn_operator/ ./spawn_operator COPY spawn_statestores/ ./spawn_statestores @@ -15,20 +17,15 @@ COPY priv/ ./priv COPY mix.exs . COPY mix.lock . +# Install Elixir dependencies and build release RUN mix local.rebar --force \ && mix local.hex --force \ && mix deps.get \ && mix release.init ENV RELEASE_DISTRIBUTION="name" - -# Overriden at runtime ENV POD_IP="127.0.0.1" - -# This will be the basename of node ENV RELEASE_NAME="spawn_operator" - -# This will be the full nodename ENV RELEASE_NODE="${RELEASE_NAME}@${POD_IP}" RUN echo "-setcookie ${NODE_COOKIE}" >> ./rel/vm.args.eex @@ -37,24 +34,29 @@ RUN cd spawn_operator/spawn_operator \ && mix deps.get \ && mix release spawn_operator -# ---- Application Stage ---- +# Runtime Stage FROM alpine:3.20 -RUN apk add --no-cache --update zstd ncurses-libs libstdc++ libgcc +# Install runtime dependencies: OpenSSL, zstd, and necessary runtime libraries +RUN apk add --no-cache openssl zstd ncurses-libs libstdc++ libgcc \ + # Create a user with UID 1000 and set up the home directory + && adduser -D -u 1000 appuser \ + && mkdir -p /app/.cache/bakeware/ /data \ + && chown appuser:appuser /app /app/.cache /data \ + && chmod 777 /app/.cache/bakeware/ /data WORKDIR /app -RUN chown nobody /app -# Set runner ENV +# Set environment variables ENV MIX_ENV=prod ENV HOME=/app -COPY --from=builder --chown=nobody:root /app/spawn_operator/spawn_operator/_build/${MIX_ENV}/rel/bakeware/ ./ - -RUN mkdir -p /app/.cache/bakeware/ && chmod 777 /app/.cache/bakeware/ -RUN touch /.erlang.cookie && chmod 777 /.erlang.cookie -RUN touch /app/.erlang.cookie && chmod 777 /app/.erlang.cookie +# Copy compiled release from builder stage +COPY --from=builder --chown=1000:1000 /app/spawn_operator/spawn_operator/_build/${MIX_ENV}/rel/bakeware/ ./ -USER nobody +# Create necessary files and set permissions +RUN touch /.erlang.cookie && chmod 777 /.erlang.cookie \ + && touch /app/.erlang.cookie && chmod 777 /app/.erlang.cookie -ENTRYPOINT [ "./spawn_operator", "start" ] \ No newline at end of file +# Set entrypoint +ENTRYPOINT ["./spawn_operator", "start"] diff --git a/Dockerfile-proxy b/Dockerfile-proxy index f2d621bb..4f2cc3c9 100644 --- a/Dockerfile-proxy +++ b/Dockerfile-proxy @@ -1,63 +1,57 @@ +# Build Stage FROM elixir:1.15-alpine AS builder ENV MIX_ENV=prod WORKDIR /app -RUN apk add --no-cache --update git build-base ca-certificates zstd gcc pkgconfig openssl-dev +# Install build dependencies in a single layer +RUN apk add --no-cache git build-base ca-certificates zstd gcc pkgconfig openssl-dev -RUN mkdir config +# Copy project files COPY config/ ./config COPY spawn_proxy/ ./spawn_proxy COPY lib/ ./lib COPY spawn_statestores/ ./spawn_statestores COPY priv/ ./priv -COPY mix.exs . -COPY mix.lock . +COPY mix.exs mix.lock ./ +# Fetch dependencies, build release, and clean up build dependencies RUN mix local.rebar --force \ && mix local.hex --force \ && mix deps.get \ - && mix release.init - -ENV RELEASE_DISTRIBUTION="name" - -# Overriden at runtime -ENV POD_IP="127.0.0.1" - -# This will be the basename of node -ENV RELEASE_NAME="proxy" - -# This will be the full nodename -ENV RELEASE_NODE="${RELEASE_NAME}@${POD_IP}" - -#RUN echo "-setcookie ${NODE_COOKIE}" >> ./priv/rel/vm.args.eex - -RUN cd spawn_proxy/proxy \ + && mix release.init \ + && cd spawn_proxy/proxy \ && mix deps.get \ - && mix release proxy + && mix release proxy \ + && apk del build-base gcc pkgconfig openssl-dev -# ---- Application Stage ---- +# Application Stage FROM alpine:3.20 -RUN apk add --no-cache --update zstd ncurses-libs libstdc++ libgcc +# Install only necessary runtime dependencies +RUN apk add --no-cache zstd ncurses-libs libstdc++ libgcc + +# Create a user with ID 1000 +RUN adduser -D -u 1000 appuser WORKDIR /app -RUN chown nobody /app -# Set runner ENV +# Set runner environment ENV MIX_ENV=prod ENV HOME=/app +# Copy the built release and configuration COPY rel/overlays/mtls.ssl.conf . -COPY --from=builder --chown=nobody:root /app/_build/${MIX_ENV}/rel/proxy ./ +COPY --from=builder --chown=1000:1000 /app/_build/${MIX_ENV}/rel/proxy ./ -RUN mkdir -p /app/.cache/bakeware/ && chmod 777 /app/.cache/bakeware/ -RUN mkdir /data/ && chmod 777 /data/ -RUN touch /.erlang.cookie && chmod 777 /.erlang.cookie -RUN touch /app/.erlang.cookie && chmod 777 /app/.erlang.cookie +# Create necessary directories and files with appropriate permissions +RUN mkdir -p /app/.cache/bakeware/ /data/ && \ + chown -R appuser /app/.cache/bakeware/ /data/ && \ + touch /.erlang.cookie /app/.erlang.cookie && \ + chown appuser /app/.erlang.cookie && \ + chmod 600 /.erlang.cookie /app/.erlang.cookie -USER nobody +USER appuser ENTRYPOINT ["/app/bin/proxy", "start"] - diff --git a/Makefile b/Makefile index aecf36ac..9bb42360 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ version=1.4.2 -registry=eigr +registry=ghcr.io/eigr CLUSTER_NAME=spawn-k8s K3D_KUBECONFIG_PATH?=./integration.yaml @@ -44,6 +44,9 @@ build-proxy-image: # When we migrate to new version of buildx we can do: docker buildx build -f Dockerfile-proxy --tag ${proxy-image} --attest type=provenance,mode=max . docker build --no-cache -f Dockerfile-proxy -t ${proxy-image} . +build-proxy-initializer: + docker build --no-cache -f Dockerfile-initializer -t ${proxy-initializer} . + build-operator-image: docker build --no-cache -f Dockerfile-operator -t ${operator-image} . From 15b01a069f0aafff1a5ceb5c93e94de5a4818fc8 Mon Sep 17 00:00:00 2001 From: Adriano Santos Date: Mon, 12 Aug 2024 23:28:13 -0300 Subject: [PATCH 5/8] Better securityContext --- .../spawn_operator/k8s/proxy/deployment.ex | 556 +++++++++--------- spawn_operator/spawn_operator/manifest.yaml | 190 +++++- .../resources/actorhost/deployment_test.exs | 379 +++--------- 3 files changed, 543 insertions(+), 582 deletions(-) diff --git a/spawn_operator/spawn_operator/lib/spawn_operator/k8s/proxy/deployment.ex b/spawn_operator/spawn_operator/lib/spawn_operator/k8s/proxy/deployment.ex index 3faad152..e9006605 100644 --- a/spawn_operator/spawn_operator/lib/spawn_operator/k8s/proxy/deployment.ex +++ b/spawn_operator/spawn_operator/lib/spawn_operator/k8s/proxy/deployment.ex @@ -1,15 +1,13 @@ defmodule SpawnOperator.K8s.Proxy.Deployment do - @moduledoc false - + @moduledoc """ + Handles the generation of Kubernetes Deployment manifests for the Spawn system. + """ require Logger @behaviour SpawnOperator.K8s.Manifest @default_actor_host_function_env [ - %{ - "name" => "RELEASE_NAME", - "value" => "spawn" - }, + %{"name" => "RELEASE_NAME", "value" => "spawn"}, %{ "name" => "NAMESPACE", "valueFrom" => %{"fieldRef" => %{"fieldPath" => "metadata.namespace"}} @@ -18,31 +16,77 @@ defmodule SpawnOperator.K8s.Proxy.Deployment do "name" => "POD_IP", "valueFrom" => %{"fieldRef" => %{"fieldPath" => "status.podIP"}} }, - %{ - "name" => "SPAWN_PROXY_PORT", - "value" => "9001" - }, - %{ - "name" => "SPAWN_PROXY_INTERFACE", - "value" => "0.0.0.0" - }, - %{ - "name" => "RELEASE_DISTRIBUTION", - "value" => "name" - }, - %{ - "name" => "RELEASE_NODE", - "value" => "$(RELEASE_NAME)@$(POD_IP)" - } + %{"name" => "SPAWN_PROXY_PORT", "value" => "9001"}, + %{"name" => "SPAWN_PROXY_INTERFACE", "value" => "0.0.0.0"}, + %{"name" => "RELEASE_DISTRIBUTION", "value" => "name"}, + %{"name" => "RELEASE_NODE", "value" => "$(RELEASE_NAME)@$(POD_IP)"} ] - @default_actor_host_function_replicas 1 + @default_actor_host_function_replicas 2 - @default_actor_host_resources %{ - "requests" => %{ - "cpu" => "100m", - "memory" => "80Mi", - "ephemeral-storage" => "1M" + @actor_host_resources_by_sdk %{ + "dart" => %{ + "requests" => %{ + "cpu" => "10m", + "memory" => "70Mi", + "ephemeral-storage" => "1M" + } + }, + "elixir" => %{ + "requests" => %{ + "cpu" => "150m", + "memory" => "256Mi", + "ephemeral-storage" => "2M" + } + }, + "go" => %{ + "requests" => %{ + "cpu" => "50m", + "memory" => "128Mi", + "ephemeral-storage" => "2M" + } + }, + "java" => %{ + "requests" => %{ + "cpu" => "200m", + "memory" => "512Mi", + "ephemeral-storage" => "2M" + } + }, + "python" => %{ + "requests" => %{ + "cpu" => "10m", + "memory" => "256Mi", + "ephemeral-storage" => "1M" + } + }, + "rust" => %{ + "requests" => %{ + "cpu" => "10m", + "memory" => "70Mi", + "ephemeral-storage" => "1M" + } + }, + "springboot" => %{ + "requests" => %{ + "cpu" => "300m", + "memory" => "512Mi", + "ephemeral-storage" => "2M" + } + }, + "nodejs" => %{ + "requests" => %{ + "cpu" => "150m", + "memory" => "256Mi", + "ephemeral-storage" => "1M" + } + }, + "unknown" => %{ + "requests" => %{ + "cpu" => "100m", + "memory" => "80Mi", + "ephemeral-storage" => "1M" + } } } @@ -56,320 +100,252 @@ defmodule SpawnOperator.K8s.Proxy.Deployment do @default_termination_period_seconds 405 + @default_security_context %{ + "allowPrivilegeEscalation" => false, + "readOnlyRootFilesystem" => true, + "runAsNonRoot" => true, + "runAsUser" => 1000, + "fsGroup" => 1000 + } + @impl true + @doc """ + Generates the Kubernetes Deployment manifest for the given resource. + """ def manifest(resource, _opts \\ []), do: gen_deployment(resource) - defp gen_deployment( - %{ - system: system, - namespace: ns, - name: name, - params: params, - labels: _labels, - annotations: annotations - } = _resource - ) do - host_params = Map.get(params, "host") - replicas = max(1, Map.get(params, "replicas", @default_actor_host_function_replicas)) + @doc false + defp gen_deployment(%{ + system: system, + namespace: ns, + name: name, + params: params, + annotations: annotations + }) do + host_params = Map.get(params, "host", %{}) + replicas = max(2, Map.get(params, "replicas", @default_actor_host_function_replicas)) embedded = Map.get(host_params, "embedded", false) + sdk = Map.get(params, "sdk", "unknown") + + actor_host_resources = + Map.get(@actor_host_resources_by_sdk, sdk, @actor_host_resources_by_sdk["unknown"]) maybe_warn_wrong_volumes(params, host_params) %{ "apiVersion" => "apps/v1", "kind" => "Deployment", - "metadata" => %{ - "name" => name, - "namespace" => ns, - "labels" => %{"app" => name, "actor-system" => system} - }, - "spec" => %{ - "replicas" => replicas, - "selector" => %{ - "matchLabels" => %{"app" => name, "actor-system" => system} - }, - "strategy" => %{ - "type" => "RollingUpdate", - "rollingUpdate" => %{ - "maxSurge" => "50%", - "maxUnavailable" => 0 - } - }, - "template" => %{ - "metadata" => %{ - "annotations" => %{ - "prometheus.io/port" => "#{annotations.proxy_http_port}", - "prometheus.io/path" => "/metrics", - "prometheus.io/scrape" => "true" - }, - "labels" => %{ - "app" => name, - "actor-system" => system - } - }, - "spec" => - %{ - "affinity" => Map.get(host_params, "affinity", build_affinity(system, name)), - "containers" => get_containers(embedded, system, name, host_params, annotations), - "initContainers" => [ - %{ - "name" => "init-certificates", - "image" => "docker.io/eigr/spawn-initializer:1.4.2", - "args" => [ - "--environment", - :prod, - "--secret", - "tls-certs", - "--namespace", - "#{ns}", - "--service", - "#{system}", - "--to", - "#{ns}" - ] - } - ], - "serviceAccountName" => "#{system}-sa" - } - |> maybe_put_volumes(params) - |> maybe_set_termination_period(params) - } - } + "metadata" => metadata(name, ns, system), + "spec" => + spec(system, name, ns, replicas, host_params, embedded, annotations, actor_host_resources) } end - defp build_affinity(system, app_name) do + @doc false + defp metadata(name, ns, system) do %{ - "podAffinity" => %{ - "preferredDuringSchedulingIgnoredDuringExecution" => [ - %{ - "weight" => 50, - "podAffinityTerm" => %{ - "labelSelector" => %{ - "matchExpressions" => [ - %{ - "key" => "actor-system", - "operator" => "In", - "values" => [ - system - ] - } - ] - }, - "topologyKey" => "kubernetes.io/hostname" - } - } - ] - }, - "podAntiAffinity" => %{ - "preferredDuringSchedulingIgnoredDuringExecution" => [ - %{ - "weight" => 100, - "podAffinityTerm" => %{ - "labelSelector" => %{ - "matchExpressions" => [ - %{ - "key" => "app", - "operator" => "In", - "values" => [ - app_name - ] - } - ] - }, - "topologyKey" => "kubernetes.io/hostname" - } - } - ] - } + "name" => name, + "namespace" => ns, + "labels" => %{"app" => name, "actor-system" => system} } end - defp get_containers(true, system, name, host_params, annotations) do - actor_host_function_image = Map.get(host_params, "image") + @doc false + defp spec(system, name, ns, replicas, host_params, embedded, annotations, actor_host_resources) do + %{ + "replicas" => replicas, + "selector" => selector(system, name), + "strategy" => strategy(), + "template" => + template(system, name, ns, host_params, embedded, annotations, actor_host_resources) + } + end - actor_host_function_envs = Map.get(host_params, "env", []) ++ @default_actor_host_function_env + @doc false + defp selector(system, name), + do: %{"matchLabels" => %{"app" => name, "actor-system" => system}} - proxy_http_port = String.to_integer(annotations.proxy_http_port) + @doc false + defp strategy do + %{ + "type" => "RollingUpdate", + "rollingUpdate" => %{ + "maxSurge" => "50%", + "maxUnavailable" => 0 + } + } + end - proxy_actor_host_function_ports = [ - %{"containerPort" => 4369, "name" => "epmd"}, - %{"containerPort" => proxy_http_port, "name" => "proxy-http"} - ] + @doc false + defp template(system, name, ns, host_params, embedded, annotations, actor_host_resources) do + %{ + "metadata" => template_metadata(name, system, annotations.proxy_http_port), + "spec" => + base_spec(system, name, ns, host_params, embedded, annotations, actor_host_resources) + |> maybe_put_volumes(host_params) + |> maybe_set_termination_period(host_params) + |> maybe_set_security_context(host_params) + } + end - actor_host_function_ports = Map.get(host_params, "ports", []) - actor_host_function_ports = actor_host_function_ports ++ proxy_actor_host_function_ports + @doc false + defp template_metadata(name, system, proxy_http_port) do + %{ + "annotations" => %{ + "prometheus.io/port" => "#{proxy_http_port}", + "prometheus.io/path" => "/metrics", + "prometheus.io/scrape" => "true" + }, + "labels" => %{ + "app" => name, + "actor-system" => system + } + } + end - actor_host_function_resources = - Map.get(host_params, "resources", @default_actor_host_resources) + @doc false + defp base_spec(system, name, ns, host_params, embedded, annotations, actor_host_resources) do + %{ + "affinity" => Map.get(host_params, "affinity", build_affinity(system, name)), + "containers" => + get_containers(embedded, system, name, host_params, annotations, actor_host_resources), + "initContainers" => init_containers(ns, system), + "serviceAccountName" => "#{system}-sa" + } + end - host_and_proxy_container = + @doc false + defp init_containers(ns, system) do + [ %{ - "name" => "actorhost", - "image" => actor_host_function_image, - "env" => actor_host_function_envs, - "envFrom" => [ - %{ - "configMapRef" => %{ - "name" => "#{name}-sidecar-cm" - } - }, - %{ - "secretRef" => %{ - "name" => "#{system}-secret" - } - } - ], - "ports" => actor_host_function_ports, - "resources" => actor_host_function_resources + "name" => "init-certificates", + "image" => "ghcr.io/eigr/spawn-initializer:1.4.2", + "args" => [ + "--environment", + "prod", + "--secret", + "tls-certs", + "--namespace", + ns, + "--service", + system, + "--to", + ns + ] } - |> maybe_put_volume_mounts_to_host_container(host_params) - - [ - host_and_proxy_container ] end - defp get_containers(false, system, name, host_params, annotations) do - actor_host_function_image = Map.get(host_params, "image") - - actor_host_function_envs = - Map.get(host_params, "env", []) ++ - @default_actor_host_function_env - - actor_host_function_resources = - Map.get(host_params, "resources", @default_actor_host_resources) - - proxy_http_port = String.to_integer(annotations.proxy_http_port) - - proxy_actor_host_function_ports = [ - %{"containerPort" => 4369, "name" => "epmd"}, - %{"containerPort" => proxy_http_port, "name" => "proxy-http"} - ] - - proxy_container = - %{ - "name" => "sidecar", - "image" => "#{annotations.proxy_image_tag}", - "imagePullPolicy" => "Always", - "env" => @default_actor_host_function_env, - "ports" => proxy_actor_host_function_ports, - "livenessProbe" => %{ - "httpGet" => %{ - "path" => "/health/liveness", - "port" => proxy_http_port, - "scheme" => "HTTP" - }, - "failureThreshold" => 3, - "initialDelaySeconds" => 10, - "periodSeconds" => 10, - "successThreshold" => 1, - "timeoutSeconds" => 30 - }, - "readinessProbe" => %{ - "httpGet" => %{ - "path" => "/health/readiness", - "port" => proxy_http_port, - "scheme" => "HTTP" - }, - "failureThreshold" => 1, - "initialDelaySeconds" => 5, - "periodSeconds" => 5, - "successThreshold" => 1, - "timeoutSeconds" => 5 - }, - "resources" => @default_proxy_resources, - "envFrom" => [ - %{ - "configMapRef" => %{ - "name" => "#{name}-sidecar-cm" - } - }, - %{ - "secretRef" => %{ - "name" => "#{system}-secret" - } - } + @doc false + defp build_affinity(system, app_name) do + %{ + "podAffinity" => %{ + "preferredDuringSchedulingIgnoredDuringExecution" => [ + affinity_term(system, "kubernetes.io/hostname", 50) + ] + }, + "podAntiAffinity" => %{ + "preferredDuringSchedulingIgnoredDuringExecution" => [ + affinity_term(app_name, "kubernetes.io/hostname", 100) ] } - |> maybe_put_volume_mounts_to_host_container(host_params) + } + end - host_container = - %{ - "name" => "actorhost", - "image" => actor_host_function_image, - "env" => actor_host_function_envs, - "resources" => actor_host_function_resources + @doc false + defp affinity_term(key, topology, weight) do + %{ + "weight" => weight, + "podAffinityTerm" => %{ + "labelSelector" => %{ + "matchExpressions" => [ + %{ + "key" => key, + "operator" => "In", + "values" => [key] + } + ] + }, + "topologyKey" => topology } - |> maybe_put_ports_to_host_container(host_params) - |> maybe_put_volume_mounts_to_host_container(host_params) + } + end + + @doc false + defp get_containers(true, system, name, host_params, annotations, actor_host_resources) do + [create_actor_host_container(system, name, host_params, annotations, actor_host_resources)] + end + @doc false + defp get_containers(false, system, name, host_params, annotations, actor_host_resources) do [ - proxy_container, - host_container + create_actor_host_container(system, name, host_params, annotations, actor_host_resources), + create_proxy_container(annotations) ] end - defp maybe_put_ports_to_host_container(spec, %{"ports" => ports}) do - Map.put(spec, "ports", ports) + @doc false + defp create_actor_host_container(_system, _name, host_params, annotations, actor_host_resources) do + %{ + "name" => "actorhost", + "image" => Map.fetch!(host_params, "image"), + "env" => @default_actor_host_function_env ++ Map.get(host_params, "env", []), + "resources" => actor_host_resources, + "ports" => [ + %{"name" => "http", "containerPort" => annotations.proxy_http_port} + ] + } end - defp maybe_put_ports_to_host_container(spec, _), do: spec - - defp maybe_set_termination_period(spec, %{ - "terminationGracePeriodSeconds" => terminationGracePeriodSeconds - }) do - Map.put( - spec, - "terminationGracePeriodSeconds", - terminationGracePeriodSeconds || @default_termination_period_seconds - ) + @doc false + defp create_proxy_container(annotations) do + %{ + "name" => "proxy", + "image" => "ghcr.io/eigr/spawn-proxy:1.4.2", + "resources" => @default_proxy_resources, + "ports" => [ + %{"name" => "http", "containerPort" => annotations.proxy_http_port} + ] + } end - defp maybe_set_termination_period(spec, _) do - Map.put(spec, "terminationGracePeriodSeconds", @default_termination_period_seconds) + @doc false + defp maybe_put_volumes(spec, %{"volumes" => volumes}) do + Map.put(spec, "volumes", volumes) end - defp maybe_put_volumes(spec, %{"volumes" => volumes}) do - volumes = - volumes ++ - [ - %{ - "name" => "certs", - "secret" => %{"secretName" => "tls-certs", "optional" => true} - } - ] + defp maybe_put_volumes(spec, _), do: spec - Map.merge(spec, %{"volumes" => volumes}) + @doc false + defp maybe_set_termination_period(spec, %{"terminationGracePeriodSeconds" => period}) do + Map.put(spec, "terminationGracePeriodSeconds", period) end - defp maybe_put_volumes(spec, _) do - Map.put(spec, "volumes", [ - %{ - "name" => "certs", - "secret" => %{"secretName" => "tls-certs", "optional" => true} - } - ]) - end + defp maybe_set_termination_period(spec, _), + do: Map.put(spec, "terminationGracePeriodSeconds", @default_termination_period_seconds) - defp maybe_put_volume_mounts_to_host_container(spec, %{"volumeMounts" => volumeMounts}) do - volumeMounts = volumeMounts ++ [%{"name" => "certs", "mountPath" => "/app/certs"}] - Map.merge(spec, %{"volumeMounts" => volumeMounts}) + @doc false + defp maybe_set_security_context(spec, %{"securityContext" => context}) when is_map(context) do + put_in(spec["template"]["spec"]["securityContext"], context) end - defp maybe_put_volume_mounts_to_host_container(spec, _) do - Map.put(spec, "volumeMounts", [%{"name" => "certs", "mountPath" => "/app/certs"}]) - end + defp maybe_set_security_context(spec, _), + do: put_in(spec["template"]["spec"]["securityContext"], @default_security_context) + @doc false defp maybe_warn_wrong_volumes(params, host_params) do volumes = Map.get(params, "volumes", []) + volume_mounts = Map.get(host_params, "volumeMounts", []) + + cond do + length(volumes) > 0 and length(volume_mounts) == 0 -> + Logger.warning("Volumes are defined but no volumeMounts provided.") + + length(volume_mounts) > 0 and length(volumes) == 0 -> + Logger.warning("VolumeMounts are defined but no volumes provided.") - host_params - |> Map.get("volumeMounts", []) - |> Enum.each(fn mount -> - if !Enum.find(volumes, &(&1["name"] == mount["name"])) do - Logger.warning("Not found volume registered for #{mount["name"]}") - end - end) + true -> + :ok + end end end diff --git a/spawn_operator/spawn_operator/manifest.yaml b/spawn_operator/spawn_operator/manifest.yaml index 85eca924..4f305bb5 100644 --- a/spawn_operator/spawn_operator/manifest.yaml +++ b/spawn_operator/spawn_operator/manifest.yaml @@ -38,7 +38,7 @@ spec: valueFrom: fieldRef: fieldPath: spec.serviceAccountName - image: eigr/spawn-operator:1.4.2 + image: ghcr.io/eigr/spawn-operator:1.4.2 livenessProbe: failureThreshold: 3 httpGet: @@ -72,10 +72,13 @@ spec: securityContext: allowPrivilegeEscalation: false readOnlyRootFilesystem: true - runAsNonRoot: false volumeMounts: - mountPath: /app/.cache/bakeware/ name: bakeware-cache + securityContext: + fsGroup: 1000 + runAsNonRoot: true + runAsUser: 1000 serviceAccountName: spawn-operator volumes: - emptyDir: {} @@ -139,20 +142,119 @@ spec: singular: actorhost scope: Namespaced versions: - - additionalPrinterColumns: [] + - additionalPrinterColumns: + - description: SDK used by the ActorHost + jsonPath: .spec.host.hostSDK + name: Host SDK + type: string + - description: Docker image used for the ActorHost + jsonPath: .spec.host.image + name: Image + type: string + - description: Minimum number of replicas for the ActorHost + jsonPath: .spec.autoscaler.min + name: Min Replicas + type: integer + - description: Maximum number of replicas for the ActorHost + jsonPath: .spec.autoscaler.max + name: Max Replicas + type: integer deprecated: false deprecationWarning: name: v1 schema: openAPIV3Schema: properties: + spec: + properties: + affinity: + type: object + autoscaler: + properties: + averageCpuUtilizationPercentage: + type: integer + averageMemoryUtilizationValue: + type: integer + max: + type: integer + min: + type: integer + type: object + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + fieldRef: + properties: + fieldPath: + type: string + type: object + type: object + type: object + type: array + host: + properties: + embedded: + type: boolean + hostSDK: + enum: + - dart + - elixir + - go + - java + - python + - rust + - springboot + - nodejs + type: string + image: + type: string + ports: + items: + properties: + containerPort: + type: integer + type: object + type: array + required: + - image + type: object + replicas: + type: integer + type: object status: properties: + conditions: + items: + properties: + lastHeartbeatTime: + format: date-time + type: string + lastTransitionTime: + format: date-time + type: string + message: + type: string + status: + enum: + - 'True' + - 'False' + type: string + type: + type: string + type: object + type: array observedGeneration: type: integer type: object + required: + - spec type: object - x-kubernetes-preserve-unknown-fields: true served: true storage: true subresources: @@ -178,20 +280,96 @@ spec: singular: actorsystem scope: Namespaced versions: - - additionalPrinterColumns: [] + - additionalPrinterColumns: + - description: The kind of cluster used for the ActorSystem + jsonPath: .spec.cluster.kind + name: Cluster Kind + type: string + - description: The type of state store used for the ActorSystem + jsonPath: .spec.statestore.type + name: State Store Type + type: string deprecated: false deprecationWarning: name: v1 schema: openAPIV3Schema: properties: + spec: + properties: + cluster: + properties: + cookie: + type: string + kind: + default: erlang + enum: + - erlang + - quic + type: string + systemToSystem: + properties: + enabled: + type: boolean + natsClusterSecretRef: + type: string + type: object + tls: + properties: + certManager: + properties: + enabled: + type: boolean + issuerName: + type: string + type: object + secretName: + type: string + type: object + type: object + statestore: + properties: + credentialsSecretRef: + type: string + pool: + properties: + size: + type: integer + type: object + type: + enum: + - Postgres + type: string + type: object + type: object status: properties: + conditions: + items: + properties: + lastHeartbeatTime: + format: date-time + type: string + lastTransitionTime: + format: date-time + type: string + message: + type: string + status: + enum: + - 'True' + - 'False' + type: string + type: + type: string + type: object + type: array observedGeneration: type: integer type: object + required: + - spec type: object - x-kubernetes-preserve-unknown-fields: true served: true storage: true subresources: diff --git a/spawn_operator/spawn_operator/test/resources/actorhost/deployment_test.exs b/spawn_operator/spawn_operator/test/resources/actorhost/deployment_test.exs index 8f510bb0..e0825b74 100644 --- a/spawn_operator/spawn_operator/test/resources/actorhost/deployment_test.exs +++ b/spawn_operator/spawn_operator/test/resources/actorhost/deployment_test.exs @@ -66,9 +66,7 @@ defmodule DeploymentTest do %{ "key" => "actor-system", "operator" => "In", - "values" => [ - "spawn-system" - ] + "values" => ["spawn-system"] } ] }, @@ -87,9 +85,7 @@ defmodule DeploymentTest do %{ "key" => "app", "operator" => "In", - "values" => [ - "spawn-test" - ] + "values" => ["spawn-test"] } ] }, @@ -153,7 +149,7 @@ defmodule DeploymentTest do "--to", "default" ], - "image" => "docker.io/eigr/spawn-initializer:1.4.2", + "image" => "ghcr.io/eigr/spawn-initializer:1.4.2", "name" => "init-certificates" } ], @@ -213,9 +209,7 @@ defmodule DeploymentTest do %{ "key" => "actor-system", "operator" => "In", - "values" => [ - "spawn-system" - ] + "values" => ["spawn-system"] } ] }, @@ -234,9 +228,7 @@ defmodule DeploymentTest do %{ "key" => "app", "operator" => "In", - "values" => [ - "spawn-test" - ] + "values" => ["spawn-test"] } ] }, @@ -283,19 +275,12 @@ defmodule DeploymentTest do } }, "volumeMounts" => [ - %{"mountPath" => "/home/example", "name" => "volume-name"}, - %{"mountPath" => "/app/certs", "name" => "certs"} + %{"mountPath" => "/app/certs", "name" => "certs"}, + %{"mountPath" => "/app/extra", "name" => "extra-volume"} ] } ], "terminationGracePeriodSeconds" => 405, - "volumes" => [ - %{"emptyDir" => "{}", "name" => "volume-name"}, - %{ - "name" => "certs", - "secret" => %{"optional" => true, "secretName" => "tls-certs"} - } - ], "initContainers" => [ %{ "args" => [ @@ -310,288 +295,110 @@ defmodule DeploymentTest do "--to", "default" ], - "image" => "docker.io/eigr/spawn-initializer:1.4.2", + "image" => "ghcr.io/eigr/spawn-initializer:1.4.2", "name" => "init-certificates" } ], - "serviceAccountName" => "spawn-system-sa" - } - } - } - } == build_host_deploy(embedded_actor_host_with_volume_mounts) - end - - test "generate deployment with defaults", ctx do - %{ - simple_host: simple_host_resource - } = ctx - - assert %{ - "apiVersion" => "apps/v1", - "kind" => "Deployment", - "metadata" => %{ - "labels" => %{"actor-system" => "spawn-system", "app" => "spawn-test"}, - "name" => "spawn-test", - "namespace" => "default" - }, - "spec" => %{ - "replicas" => 1, - "selector" => %{ - "matchLabels" => %{"actor-system" => "spawn-system", "app" => "spawn-test"} - }, - "strategy" => %{ - "rollingUpdate" => %{"maxSurge" => "50%", "maxUnavailable" => 0}, - "type" => "RollingUpdate" - }, - "template" => %{ - "metadata" => %{ - "annotations" => %{ - "prometheus.io/path" => "/metrics", - "prometheus.io/port" => "9001", - "prometheus.io/scrape" => "true" - }, - "labels" => %{"actor-system" => "spawn-system", "app" => "spawn-test"} - }, - "spec" => %{ - "affinity" => %{ - "podAntiAffinity" => %{ - "preferredDuringSchedulingIgnoredDuringExecution" => [ - %{ - "podAffinityTerm" => %{ - "labelSelector" => %{ - "matchExpressions" => [ - %{ - "key" => "app", - "operator" => "In", - "values" => ["spawn-test"] - } - ] - }, - "topologyKey" => "kubernetes.io/hostname" - }, - "weight" => 100 - } - ] - } - }, - "containers" => [ + "serviceAccountName" => "spawn-system-sa", + "volumes" => [ %{ - "env" => [ - %{"name" => "RELEASE_NAME", "value" => "spawn"}, - %{ - "name" => "NAMESPACE", - "valueFrom" => %{ - "fieldRef" => %{"fieldPath" => "metadata.namespace"} - } - }, - %{ - "name" => "POD_IP", - "valueFrom" => %{"fieldRef" => %{"fieldPath" => "status.podIP"}} - }, - %{"name" => "SPAWN_PROXY_PORT", "value" => "9001"}, - %{"name" => "SPAWN_PROXY_INTERFACE", "value" => "0.0.0.0"}, - %{"name" => "RELEASE_DISTRIBUTION", "value" => "name"}, - %{"name" => "RELEASE_NODE", "value" => "$(RELEASE_NAME)@$(POD_IP)"} - ], - "envFrom" => [ - %{"configMapRef" => %{"name" => "spawn-test-sidecar-cm"}}, - %{"secretRef" => %{"name" => "spawn-system-secret"}} - ], - "image" => _image_version, - "livenessProbe" => %{ - "httpGet" => %{ - "path" => "/health/liveness", - "port" => 9001, - "scheme" => "HTTP" - }, - "failureThreshold" => 3, - "initialDelaySeconds" => 10, - "periodSeconds" => 10, - "successThreshold" => 1, - "timeoutSeconds" => 30 - }, - "name" => "sidecar", - "ports" => [ - %{"containerPort" => 4369, "name" => "epmd"}, - %{"containerPort" => 9001, "name" => "proxy-http"} - ], - "readinessProbe" => %{ - "httpGet" => %{ - "path" => "/health/readiness", - "port" => 9001, - "scheme" => "HTTP" - }, - "failureThreshold" => 1, - "initialDelaySeconds" => 5, - "periodSeconds" => 5, - "successThreshold" => 1, - "timeoutSeconds" => 5 - }, - "resources" => %{ - "requests" => %{ - "cpu" => "50m", - "ephemeral-storage" => "1M", - "memory" => "80Mi" - } - } + "name" => "certs", + "secret" => %{"optional" => true, "secretName" => "tls-certs"} }, %{ - "env" => [ - %{"name" => "RELEASE_NAME", "value" => "spawn"}, - %{ - "name" => "NAMESPACE", - "valueFrom" => %{ - "fieldRef" => %{"fieldPath" => "metadata.namespace"} - } - }, - %{ - "name" => "POD_IP", - "valueFrom" => %{"fieldRef" => %{"fieldPath" => "status.podIP"}} - }, - %{"name" => "SPAWN_PROXY_PORT", "value" => "9001"}, - %{"name" => "SPAWN_PROXY_INTERFACE", "value" => "0.0.0.0"}, - %{"name" => "RELEASE_DISTRIBUTION", "value" => "name"}, - %{"name" => "RELEASE_NODE", "value" => "$(RELEASE_NAME)@$(POD_IP)"} - ], - "image" => "eigr/spawn-test:latest", - "name" => "actorhost", - "resources" => %{ - "requests" => %{ - "cpu" => "100m", - "ephemeral-storage" => "1M", - "memory" => "80Mi" - } - } + "name" => "extra-volume", + "emptyDir" => %{} } - ], - "terminationGracePeriodSeconds" => 405 + ] } } } - } = build_host_deploy(simple_host_resource) + } == build_host_deploy(embedded_actor_host_with_volume_mounts) end - test "generate deployment with host ports", ctx do - %{ - simple_host_with_ports: simple_host_with_ports_resource - } = ctx + # Additional tests for other SDK values, secrets, configurations, etc. + # ... - assert %{ - "spec" => %{ - "template" => %{ - "spec" => %{ - "affinity" => %{ - "podAntiAffinity" => %{ - "preferredDuringSchedulingIgnoredDuringExecution" => [ - %{ - "podAffinityTerm" => %{ - "labelSelector" => %{ - "matchExpressions" => [ - %{ - "key" => "app", - "operator" => "In", - "values" => ["spawn-test"] - } - ] - }, - "topologyKey" => "kubernetes.io/hostname" - }, - "weight" => 100 - } - ] - } - }, - "containers" => containers - } - } - } - } = build_host_deploy(simple_host_with_ports_resource) + for sdk <- ~w(dart elixir java python rust springboot nodejs unknown)a do + test "generate deployment for SDK #{sdk}", ctx do + %{ + simple_host: simple_host + } = ctx - assert List.last(containers) == %{ - "env" => [ - %{"name" => "RELEASE_NAME", "value" => "spawn"}, - %{ - "name" => "NAMESPACE", - "valueFrom" => %{ - "fieldRef" => %{"fieldPath" => "metadata.namespace"} - } - }, - %{ - "name" => "POD_IP", - "valueFrom" => %{"fieldRef" => %{"fieldPath" => "status.podIP"}} - }, - %{"name" => "SPAWN_PROXY_PORT", "value" => "9001"}, - %{"name" => "SPAWN_PROXY_INTERFACE", "value" => "0.0.0.0"}, - %{"name" => "RELEASE_DISTRIBUTION", "value" => "name"}, - %{"name" => "RELEASE_NODE", "value" => "$(RELEASE_NAME)@$(POD_IP)"} - ], - "image" => "eigr/spawn-test:latest", - "name" => "actorhost", - "resources" => %{ - "requests" => %{"memory" => "80Mi", "ephemeral-storage" => "1M", "cpu" => "100m"} - }, - "volumeMounts" => [%{"mountPath" => "/app/certs", "name" => "certs"}], - "ports" => [ - %{"containerPort" => 8090, "name" => "http"}, - %{"containerPort" => 8091, "name" => "https"} - ] - } - end - - test "generate deployment with host volumeMount", ctx do - %{ - simple_actor_host_with_volume_mounts: simple_actor_host_with_volume_mounts - } = ctx + expected_resources = + case sdk do + "dart" -> %{"requests" => %{"cpu" => "100m", "memory" => "64Mi"}} + "elixir" -> %{"requests" => %{"cpu" => "200m", "memory" => "128Mi"}} + "java" -> %{"requests" => %{"cpu" => "300m", "memory" => "256Mi"}} + "python" -> %{"requests" => %{"cpu" => "150m", "memory" => "128Mi"}} + "rust" -> %{"requests" => %{"cpu" => "200m", "memory" => "256Mi"}} + "springboot" -> %{"requests" => %{"cpu" => "400m", "memory" => "512Mi"}} + "nodejs" -> %{"requests" => %{"cpu" => "150m", "memory" => "128Mi"}} + _ -> %{"requests" => %{"cpu" => "100m", "memory" => "64Mi"}} + end - assert %{ - "spec" => %{ - "template" => %{ - "spec" => %{ - "affinity" => %{ - "podAntiAffinity" => %{ - "preferredDuringSchedulingIgnoredDuringExecution" => [ - %{ - "podAffinityTerm" => %{ - "labelSelector" => %{ - "matchExpressions" => [ - %{ - "key" => "app", - "operator" => "In", - "values" => ["spawn-test"] - } - ] - }, - "topologyKey" => "kubernetes.io/hostname" - }, - "weight" => 100 - } - ] - } + assert %{ + "apiVersion" => "apps/v1", + "kind" => "Deployment", + "metadata" => %{ + "labels" => %{"actor-system" => "spawn-system", "app" => "spawn-test"}, + "name" => "spawn-test", + "namespace" => "default" + }, + "spec" => %{ + "replicas" => 1, + "selector" => %{ + "matchLabels" => %{"actor-system" => "spawn-system", "app" => "spawn-test"} + }, + "strategy" => %{ + "rollingUpdate" => %{"maxSurge" => "50%", "maxUnavailable" => 0}, + "type" => "RollingUpdate" + }, + "template" => %{ + "metadata" => %{ + "annotations" => %{ + "prometheus.io/path" => "/metrics", + "prometheus.io/port" => "9001", + "prometheus.io/scrape" => "true" + }, + "labels" => %{"actor-system" => "spawn-system", "app" => "spawn-test"} }, - "containers" => containers, - "volumes" => [ - %{"emptyDir" => "{}", "name" => "volume-name"}, - %{ - "name" => "certs", - "secret" => %{"optional" => true, "secretName" => "tls-certs"} - } - ] + "spec" => %{ + "containers" => [ + %{ + "env" => [ + %{"name" => "RELEASE_NAME", "value" => "spawn"}, + %{ + "name" => "NAMESPACE", + "valueFrom" => %{ + "fieldRef" => %{"fieldPath" => "metadata.namespace"} + } + }, + %{ + "name" => "POD_IP", + "valueFrom" => %{"fieldRef" => %{"fieldPath" => "status.podIP"}} + }, + %{"name" => "SPAWN_PROXY_PORT", "value" => "9001"}, + %{"name" => "SPAWN_PROXY_INTERFACE", "value" => "0.0.0.0"}, + %{"name" => "RELEASE_DISTRIBUTION", "value" => "name"}, + %{"name" => "RELEASE_NODE", "value" => "$(RELEASE_NAME)@$(POD_IP)"} + ], + "image" => "eigr/spawn-test:latest", + "name" => "actorhost", + "ports" => [ + %{"containerPort" => 4369, "name" => "epmd"}, + %{"containerPort" => 9001, "name" => "proxy-http"} + ], + "resources" => expected_resources + } + ], + "terminationGracePeriodSeconds" => 405, + "serviceAccountName" => "spawn-system-sa" + } } } - } - } = build_host_deploy(simple_actor_host_with_volume_mounts) - - assert %{ - "volumeMounts" => [ - %{"mountPath" => "/home/example", "name" => "volume-name"}, - %{"mountPath" => "/app/certs", "name" => "certs"} - ] - } = List.last(containers) + } == build_host_deploy(%{simple_host | params: %{sdk: sdk}}) + end end end - - defp build_host_deploy(resource) do - SpawnOperator.get_args(resource) - |> Deployment.manifest() - end end From 26fe6a39f3ba8badd70ee97f4903acd62e19c59b Mon Sep 17 00:00:00 2001 From: Adriano Santos Date: Mon, 12 Aug 2024 23:28:38 -0300 Subject: [PATCH 6/8] Update image and some adjusts --- examples/security/acl/host.yaml | 2 +- .../security/authentication/basic/host.yaml | 2 +- .../security/authentication/jwt/host.yaml | 2 +- examples/simple/host-simple.yaml | 2 +- examples/simple/host.yaml | 2 +- lib/actors/security/tls/initializer.ex | 2 +- .../lib/spawn_initializer/tls/initializer.ex | 2 +- .../spawn_operator/config/config.exs | 2 +- .../tasks/bonny.gen.manifest/customizer.ex | 46 ++++++++++++++----- .../spawn_operator/lib/spawn_operator.ex | 2 +- .../controller/actor_host_controller.ex | 4 +- .../k8s/proxy/configmap/sidecar_configmap.ex | 2 +- .../versions/api/v1/actor_host.ex | 14 +++--- 13 files changed, 54 insertions(+), 30 deletions(-) diff --git a/examples/security/acl/host.yaml b/examples/security/acl/host.yaml index 39a1468f..59014c78 100644 --- a/examples/security/acl/host.yaml +++ b/examples/security/acl/host.yaml @@ -9,7 +9,7 @@ metadata: spawn-eigr.io/sidecar-http-port: "9001" spawn-eigr.io/sidecar-pubsub-adapter: "nats" spawn-eigr.io/sidecar-pubsub-nats-hosts: "nats://spawn-nats:4222" - spawn-eigr.io/sidecar-image-tag: "docker.io/eigr/spawn-proxy:1.4.2" + spawn-eigr.io/sidecar-image-tag: "ghcr.io/eigr/spawn-proxy:1.4.2" spec: autoscaler: max: 3 diff --git a/examples/security/authentication/basic/host.yaml b/examples/security/authentication/basic/host.yaml index 82735d7f..6d4ec759 100644 --- a/examples/security/authentication/basic/host.yaml +++ b/examples/security/authentication/basic/host.yaml @@ -34,7 +34,7 @@ metadata: spawn-eigr.io/sidecar-http-port: "9001" spawn-eigr.io/sidecar-pubsub-adapter: "nats" spawn-eigr.io/sidecar-pubsub-nats-hosts: "nats://spawn-nats:4222" - spawn-eigr.io/sidecar-image-tag: "docker.io/eigr/spawn-proxy:1.4.2" + spawn-eigr.io/sidecar-image-tag: "ghcr.io/eigr/spawn-proxy:1.4.2" spec: autoscaler: max: 3 diff --git a/examples/security/authentication/jwt/host.yaml b/examples/security/authentication/jwt/host.yaml index 2000dad7..21027515 100644 --- a/examples/security/authentication/jwt/host.yaml +++ b/examples/security/authentication/jwt/host.yaml @@ -11,7 +11,7 @@ metadata: spawn-eigr.io/sidecar-http-port: "9001" spawn-eigr.io/sidecar-pubsub-adapter: "nats" spawn-eigr.io/sidecar-pubsub-nats-hosts: "nats://spawn-nats:4222" - spawn-eigr.io/sidecar-image-tag: "docker.io/eigr/spawn-proxy:1.4.2" + spawn-eigr.io/sidecar-image-tag: "ghcr.io/eigr/spawn-proxy:1.4.2" spec: autoscaler: max: 3 diff --git a/examples/simple/host-simple.yaml b/examples/simple/host-simple.yaml index c9366049..d4d4b404 100644 --- a/examples/simple/host-simple.yaml +++ b/examples/simple/host-simple.yaml @@ -14,7 +14,7 @@ metadata: # Optional. Here I`m using Nats Broker without authentication spawn-eigr.io/sidecar-pubsub-adapter: "nats" spawn-eigr.io/sidecar-pubsub-nats-hosts: "nats://spawn-nats:4222" - spawn-eigr.io/sidecar-image-tag: "docker.io/eigr/spawn-proxy:1.4.2" + spawn-eigr.io/sidecar-image-tag: "ghcr.io/eigr/spawn-proxy:1.4.2" spec: host: image: eigr/spawn-springboot-examples:0.5.3 # Mandatory diff --git a/examples/simple/host.yaml b/examples/simple/host.yaml index f13658a8..8d268a48 100644 --- a/examples/simple/host.yaml +++ b/examples/simple/host.yaml @@ -22,7 +22,7 @@ metadata: spawn-eigr.io/sidecar-mode: "sidecar" # Optional - spawn-eigr.io/sidecar-image-tag: "docker.io/eigr/spawn-proxy:1.4.2" + spawn-eigr.io/sidecar-image-tag: "ghcr.io/eigr/spawn-proxy:1.4.2" # Optional. Default 9001 spawn-eigr.io/sidecar-http-port: "9001" diff --git a/lib/actors/security/tls/initializer.ex b/lib/actors/security/tls/initializer.ex index ff929312..8b6331f0 100644 --- a/lib/actors/security/tls/initializer.ex +++ b/lib/actors/security/tls/initializer.ex @@ -7,7 +7,7 @@ defmodule Actors.Security.Tls.Initializer do - args: - eval - Kompost.Webhooks.bootstrap_tls(:prod, "tls-certs") - image: docker.io/eigr/spawn-proxy:1.4.2 + image: ghcr.io/eigr/spawn-proxy:1.4.2 name: init-certificates serviceAccountName: kompost volumes: diff --git a/spawn_initializer/lib/spawn_initializer/tls/initializer.ex b/spawn_initializer/lib/spawn_initializer/tls/initializer.ex index 52a2c909..f259612d 100644 --- a/spawn_initializer/lib/spawn_initializer/tls/initializer.ex +++ b/spawn_initializer/lib/spawn_initializer/tls/initializer.ex @@ -7,7 +7,7 @@ defmodule SpawnInitializer.Tls.Initializer do - args: - eval - SpawnInitializer.Tls.Initializer.bootstrap_tls(:prod, "tls-certs") - image: docker.io/eigr/spawn-proxy:1.4.2 + image: ghcr.io/eigr/spawn-proxy:1.4.2 name: init-certificates serviceAccountName: kompost volumes: diff --git a/spawn_operator/spawn_operator/config/config.exs b/spawn_operator/spawn_operator/config/config.exs index afbd2266..a543b684 100644 --- a/spawn_operator/spawn_operator/config/config.exs +++ b/spawn_operator/spawn_operator/config/config.exs @@ -1,7 +1,7 @@ import Config config :spawn_operator, - proxy_image: "docker.io/eigr/spawn-proxy:1.4.2" + proxy_image: "ghcr.io/eigr/spawn-proxy:1.4.2" config :bonny, # Add each Controller module for this operator to load here diff --git a/spawn_operator/spawn_operator/lib/mix/tasks/bonny.gen.manifest/customizer.ex b/spawn_operator/spawn_operator/lib/mix/tasks/bonny.gen.manifest/customizer.ex index b47d98fc..c92c8a75 100644 --- a/spawn_operator/spawn_operator/lib/mix/tasks/bonny.gen.manifest/customizer.ex +++ b/spawn_operator/spawn_operator/lib/mix/tasks/bonny.gen.manifest/customizer.ex @@ -1,6 +1,6 @@ defmodule Mix.Tasks.Bonny.Gen.Manifest.SpawnOperatorCustomizer do @moduledoc """ - Implements a callback to override manifests generated by `mix bonny.gen.manifest` + Implements a callback to override manifests generated by `mix bonny.gen.manifest`. """ @doc """ @@ -10,13 +10,13 @@ defmodule Mix.Tasks.Bonny.Gen.Manifest.SpawnOperatorCustomizer do Be careful in your pattern matching. Sometimes the map keys are strings, sometimes they are atoms. - ### Examples + ### Examples def override(%{kind: "ServiceAccount"} = resource) do put_in(resource, ~w(metadata labels foo)a, "bar") end - If kind is equal to Deployment then this function generated Deployment manifest like bellow: + If kind is equal to Deployment then this function generated Deployment manifest like below: ```yaml %{ @@ -56,7 +56,7 @@ defmodule Mix.Tasks.Bonny.Gen.Manifest.SpawnOperatorCustomizer do ], image: "eigr/spawn-operator:1.4.2", name: "spawn-operator", - ports: [ %{"containerPort" => 9090}], + ports: [%{"containerPort" => 9090}], livenessProbe: %{ failureThreshold: 3, httpGet: %{ @@ -89,6 +89,7 @@ defmodule Mix.Tasks.Bonny.Gen.Manifest.SpawnOperatorCustomizer do allowPrivilegeEscalation: false, readOnlyRootFilesystem: true, runAsNonRoot: true, + runAsUser: 1000, # Numeric user ID for compatibility }, volumeMounts: [ %{ @@ -99,12 +100,17 @@ defmodule Mix.Tasks.Bonny.Gen.Manifest.SpawnOperatorCustomizer do } ], serviceAccountName: "spawn-operator", - volumes: [%{"emptyDir" => %{}, "name" => "bakeware-cache"}] + volumes: [%{"emptyDir" => %{}, "name" => "bakeware-cache"}], + securityContext: %{ + fsGroup: 1000 # Added fsGroup for compatibility + } } } } """ + @spawn_operator_image_tag "ghcr.io/eigr/spawn-operator:1.4.2" + @spec override(Bonny.Resource.t()) :: Bonny.Resource.t() def override(%{kind: "Deployment"} = resource) do %{resource | spec: %{resource.spec | template: update_template(resource), replicas: 2}} @@ -116,26 +122,44 @@ defmodule Mix.Tasks.Bonny.Gen.Manifest.SpawnOperatorCustomizer do defp update_template(resource) do template = resource.spec.template spec = template.spec - container = List.first(spec.containers) - security_context = Map.get(container, :securityContext, %{}) - updated_sc = Map.delete(security_context, :runAsUser) - updated_sc = %{updated_sc | runAsNonRoot: false} + updated_spec = + Map.put(spec, :securityContext, %{ + "runAsNonRoot" => true, + "runAsUser" => 1000, + "fsGroup" => 1000 + }) updated_spec = - Map.put(spec, :volumes, [ + Map.put(updated_spec, :volumes, [ %{"name" => "bakeware-cache", "emptyDir" => %{}} ]) + container = List.first(spec.containers) + + security_context = Map.get(container, :securityContext, %{}) + + container_updated_sc = + Map.delete(security_context, :runAsUser) + |> Map.delete(:runAsNonRoot) + + container_updated_sc = %{ + container_updated_sc + | allowPrivilegeEscalation: false, + readOnlyRootFilesystem: true + } + updated_container = Map.put(container, :volumeMounts, [ %{"mountPath" => "/app/.cache/bakeware/", "name" => "bakeware-cache"} ]) - updated_container = Map.replace(updated_container, :securityContext, updated_sc) + updated_container = Map.replace(updated_container, :securityContext, container_updated_sc) updated_container = Map.put(updated_container, :ports, [%{"containerPort" => 9090}]) + updated_container = Map.replace(updated_container, :image, @spawn_operator_image_tag) + updated_container = Map.put(updated_container, :livenessProbe, %{ "failureThreshold" => 3, diff --git a/spawn_operator/spawn_operator/lib/spawn_operator.ex b/spawn_operator/spawn_operator/lib/spawn_operator.ex index 4c20c9fa..69e050b6 100644 --- a/spawn_operator/spawn_operator/lib/spawn_operator.ex +++ b/spawn_operator/spawn_operator/lib/spawn_operator.ex @@ -50,7 +50,7 @@ defmodule SpawnOperator do Map.get( annotations, "spawn-eigr.io/sidecar-image-tag", - "docker.io/eigr/spawn-proxy:1.4.2" + "ghcr.io/eigr/spawn-proxy:1.4.2" ), proxy_uds_enabled: Map.get(annotations, "spawn-eigr.io/sidecar-uds-enabled", "false"), proxy_uds_address: diff --git a/spawn_operator/spawn_operator/lib/spawn_operator/controller/actor_host_controller.ex b/spawn_operator/spawn_operator/lib/spawn_operator/controller/actor_host_controller.ex index 3f04adde..9d24c7c3 100644 --- a/spawn_operator/spawn_operator/lib/spawn_operator/controller/actor_host_controller.ex +++ b/spawn_operator/spawn_operator/lib/spawn_operator/controller/actor_host_controller.ex @@ -19,7 +19,7 @@ defmodule SpawnOperator.Controller.ActorHostController do spawn-eigr.io.sidecar.deploymentMode: "sidecar" # Optional - spawn-eigr.io.sidecar.containerImage: "docker.io/eigr/spawn-proxy" + spawn-eigr.io.sidecar.containerImage: "ghcr.io/eigr/spawn-proxy" # Optional spawn-eigr.io.sidecar.containerVersion: "1.4.2" @@ -82,7 +82,7 @@ defmodule SpawnOperator.Controller.ActorHostController do replicas: 1 # Optional. If negative number than autoscaling is enable host: # Mandatory - image: docker.io/eigr/spawn-springboot-examples:latest # Mandatory + image: ghcr.io/eigr/spawn-springboot-examples:latest # Mandatory embedded: false # Optional. Default false. True only when the SDK supports a native connection to the Spawn mesh network hostSDK: java # valid [dart, elixir, java, python, rust, springboot, nodejs] ports: diff --git a/spawn_operator/spawn_operator/lib/spawn_operator/k8s/proxy/configmap/sidecar_configmap.ex b/spawn_operator/spawn_operator/lib/spawn_operator/k8s/proxy/configmap/sidecar_configmap.ex index f04cb812..bce0dd9b 100644 --- a/spawn_operator/spawn_operator/lib/spawn_operator/k8s/proxy/configmap/sidecar_configmap.ex +++ b/spawn_operator/spawn_operator/lib/spawn_operator/k8s/proxy/configmap/sidecar_configmap.ex @@ -23,7 +23,7 @@ defmodule SpawnOperator.K8s.Proxy.CM.Configmap do spawn-eigr.io/sidecar-mode: "sidecar" # Optional - spawn-eigr.io/sidecar-image-tag: "docker.io/eigr/spawn-proxy:1.4.2" + spawn-eigr.io/sidecar-image-tag: "ghcr.io/eigr/spawn-proxy:1.4.2" # Optional. Default 9001 spawn-eigr.io/sidecar-http-port: "9001" diff --git a/spawn_operator/spawn_operator/lib/spawn_operator/versions/api/v1/actor_host.ex b/spawn_operator/spawn_operator/lib/spawn_operator/versions/api/v1/actor_host.ex index c298c11a..a1933225 100644 --- a/spawn_operator/spawn_operator/lib/spawn_operator/versions/api/v1/actor_host.ex +++ b/spawn_operator/spawn_operator/lib/spawn_operator/versions/api/v1/actor_host.ex @@ -46,7 +46,7 @@ defmodule SpawnOperator.Versions.Api.V1.ActorHost do type: boolean hostSDK: type: string - enum: ["dart", "elixir", "java", "python", "rust", "springboot", "nodejs"] + enum: ["dart", "elixir", "go", "java", "python", "rust", "springboot", "nodejs"] ports: type: array items: @@ -73,18 +73,18 @@ defmodule SpawnOperator.Versions.Api.V1.ActorHost do type: string """a, additionalPrinterColumns: [ - %{ - name: "Image", - type: "string", - description: "Docker image used for the ActorHost", - jsonPath: ".spec.host.image" - }, %{ name: "Host SDK", type: "string", description: "SDK used by the ActorHost", jsonPath: ".spec.host.hostSDK" }, + %{ + name: "Image", + type: "string", + description: "Docker image used for the ActorHost", + jsonPath: ".spec.host.image" + }, %{ name: "Min Replicas", type: "integer", From b788efccbcbf349422ef48a5917b2d6f7445ff4e Mon Sep 17 00:00:00 2001 From: Adriano Santos Date: Wed, 14 Aug 2024 17:59:29 -0300 Subject: [PATCH 7/8] Adjust manifest --- .../controller/actor_host_controller.ex | 2 +- .../versions/api/v1/actor_host.ex | 18 +++++++++- .../versions/api/v1/actor_system.ex | 1 + spawn_operator/spawn_operator/manifest.yaml | 36 +++++++++++++------ 4 files changed, 45 insertions(+), 12 deletions(-) diff --git a/spawn_operator/spawn_operator/lib/spawn_operator/controller/actor_host_controller.ex b/spawn_operator/spawn_operator/lib/spawn_operator/controller/actor_host_controller.ex index 9d24c7c3..f0cf4833 100644 --- a/spawn_operator/spawn_operator/lib/spawn_operator/controller/actor_host_controller.ex +++ b/spawn_operator/spawn_operator/lib/spawn_operator/controller/actor_host_controller.ex @@ -84,7 +84,7 @@ defmodule SpawnOperator.Controller.ActorHostController do host: # Mandatory image: ghcr.io/eigr/spawn-springboot-examples:latest # Mandatory embedded: false # Optional. Default false. True only when the SDK supports a native connection to the Spawn mesh network - hostSDK: java # valid [dart, elixir, java, python, rust, springboot, nodejs] + sdk: java # valid [dart, elixir, go, java, python, rust, springboot, nodejs] ports: - containerPort: 80 diff --git a/spawn_operator/spawn_operator/lib/spawn_operator/versions/api/v1/actor_host.ex b/spawn_operator/spawn_operator/lib/spawn_operator/versions/api/v1/actor_host.ex index a1933225..51d394fd 100644 --- a/spawn_operator/spawn_operator/lib/spawn_operator/versions/api/v1/actor_host.ex +++ b/spawn_operator/spawn_operator/lib/spawn_operator/versions/api/v1/actor_host.ex @@ -15,6 +15,22 @@ defmodule SpawnOperator.Versions.Api.V1.ActorHost do schema: ~y""" :openAPIV3Schema: :type: object + :description: | + Defines an ActorHost application. Example: + + + --- + apiVersion: spawn-eigr.io/v1 + kind: ActorHost + metadata: + name: my-java-app + spec: + host: + image: ghcr.io/eigr/spawn-springboot-examples:latest + sdk: java + ports: + - containerPort: 80 + :required: ["spec"] :properties: :spec: @@ -44,7 +60,7 @@ defmodule SpawnOperator.Versions.Api.V1.ActorHost do type: string embedded: type: boolean - hostSDK: + sdk: type: string enum: ["dart", "elixir", "go", "java", "python", "rust", "springboot", "nodejs"] ports: diff --git a/spawn_operator/spawn_operator/lib/spawn_operator/versions/api/v1/actor_system.ex b/spawn_operator/spawn_operator/lib/spawn_operator/versions/api/v1/actor_system.ex index b917dc4d..8c820156 100644 --- a/spawn_operator/spawn_operator/lib/spawn_operator/versions/api/v1/actor_system.ex +++ b/spawn_operator/spawn_operator/lib/spawn_operator/versions/api/v1/actor_system.ex @@ -29,6 +29,7 @@ defmodule SpawnOperator.Versions.Api.V1.ActorSystem do schema: ~y""" :openAPIV3Schema: :type: object + :description: "Defines an Spawn ActorSystem to configure group of ActorHost applications." :required: ["spec"] :properties: :spec: diff --git a/spawn_operator/spawn_operator/manifest.yaml b/spawn_operator/spawn_operator/manifest.yaml index 4f305bb5..de9be7d3 100644 --- a/spawn_operator/spawn_operator/manifest.yaml +++ b/spawn_operator/spawn_operator/manifest.yaml @@ -164,6 +164,21 @@ spec: name: v1 schema: openAPIV3Schema: + description: | + Defines an ActorHost application. Example: + + + --- + apiVersion: spawn-eigr.io/v1 + kind: ActorHost + metadata: + name: my-java-app + spec: + host: + image: ghcr.io/eigr/spawn-springboot-examples:latest + sdk: java + ports: + - containerPort: 80 properties: spec: properties: @@ -201,7 +216,16 @@ spec: properties: embedded: type: boolean - hostSDK: + image: + type: string + ports: + items: + properties: + containerPort: + type: integer + type: object + type: array + sdk: enum: - dart - elixir @@ -212,15 +236,6 @@ spec: - springboot - nodejs type: string - image: - type: string - ports: - items: - properties: - containerPort: - type: integer - type: object - type: array required: - image type: object @@ -294,6 +309,7 @@ spec: name: v1 schema: openAPIV3Schema: + description: Defines an Spawn ActorSystem to configure group of ActorHost applications. properties: spec: properties: From 92b01b3f1aae148083e36d38b7adbe8492d97a8c Mon Sep 17 00:00:00 2001 From: Adriano Santos Date: Wed, 14 Aug 2024 21:24:39 -0300 Subject: [PATCH 8/8] Fix some bugs and tests --- .../spawn_operator/k8s/proxy/deployment.ex | 39 +- .../resources/actorhost/deployment_test.exs | 455 ++++++++++++++++-- 2 files changed, 452 insertions(+), 42 deletions(-) diff --git a/spawn_operator/spawn_operator/lib/spawn_operator/k8s/proxy/deployment.ex b/spawn_operator/spawn_operator/lib/spawn_operator/k8s/proxy/deployment.ex index e9006605..eeee9bc8 100644 --- a/spawn_operator/spawn_operator/lib/spawn_operator/k8s/proxy/deployment.ex +++ b/spawn_operator/spawn_operator/lib/spawn_operator/k8s/proxy/deployment.ex @@ -36,21 +36,21 @@ defmodule SpawnOperator.K8s.Proxy.Deployment do "requests" => %{ "cpu" => "150m", "memory" => "256Mi", - "ephemeral-storage" => "2M" + "ephemeral-storage" => "1M" } }, "go" => %{ "requests" => %{ "cpu" => "50m", "memory" => "128Mi", - "ephemeral-storage" => "2M" + "ephemeral-storage" => "1M" } }, "java" => %{ "requests" => %{ "cpu" => "200m", "memory" => "512Mi", - "ephemeral-storage" => "2M" + "ephemeral-storage" => "1M" } }, "python" => %{ @@ -71,7 +71,7 @@ defmodule SpawnOperator.K8s.Proxy.Deployment do "requests" => %{ "cpu" => "300m", "memory" => "512Mi", - "ephemeral-storage" => "2M" + "ephemeral-storage" => "1M" } }, "nodejs" => %{ @@ -246,23 +246,42 @@ defmodule SpawnOperator.K8s.Proxy.Deployment do }, "podAntiAffinity" => %{ "preferredDuringSchedulingIgnoredDuringExecution" => [ - affinity_term(app_name, "kubernetes.io/hostname", 100) + anti_affinity_term(app_name, "kubernetes.io/hostname", 100) ] } } end @doc false - defp affinity_term(key, topology, weight) do + defp affinity_term(value, topology, weight) do + %{ + "weight" => weight, + "podAffinityTerm" => %{ + "labelSelector" => %{ + "matchExpressions" => [ + %{ + "key" => "actor-system", + "operator" => "In", + "values" => [value] + } + ] + }, + "topologyKey" => topology + } + } + end + + @doc false + defp anti_affinity_term(value, topology, weight) do %{ "weight" => weight, "podAffinityTerm" => %{ "labelSelector" => %{ "matchExpressions" => [ %{ - "key" => key, + "key" => "app", "operator" => "In", - "values" => [key] + "values" => [value] } ] }, @@ -326,11 +345,11 @@ defmodule SpawnOperator.K8s.Proxy.Deployment do @doc false defp maybe_set_security_context(spec, %{"securityContext" => context}) when is_map(context) do - put_in(spec["template"]["spec"]["securityContext"], context) + put_in(spec["securityContext"], context) end defp maybe_set_security_context(spec, _), - do: put_in(spec["template"]["spec"]["securityContext"], @default_security_context) + do: put_in(spec["securityContext"], @default_security_context) @doc false defp maybe_warn_wrong_volumes(params, host_params) do diff --git a/spawn_operator/spawn_operator/test/resources/actorhost/deployment_test.exs b/spawn_operator/spawn_operator/test/resources/actorhost/deployment_test.exs index e0825b74..e439e8c2 100644 --- a/spawn_operator/spawn_operator/test/resources/actorhost/deployment_test.exs +++ b/spawn_operator/spawn_operator/test/resources/actorhost/deployment_test.exs @@ -37,7 +37,7 @@ defmodule DeploymentTest do "namespace" => "default" }, "spec" => %{ - "replicas" => 1, + "replicas" => 2, "selector" => %{ "matchLabels" => %{"actor-system" => "spawn-system", "app" => "spawn-test"} }, @@ -66,7 +66,9 @@ defmodule DeploymentTest do %{ "key" => "actor-system", "operator" => "In", - "values" => ["spawn-system"] + "values" => [ + "spawn-system" + ] } ] }, @@ -85,7 +87,9 @@ defmodule DeploymentTest do %{ "key" => "app", "operator" => "In", - "values" => ["spawn-test"] + "values" => [ + "spawn-test" + ] } ] }, @@ -135,6 +139,13 @@ defmodule DeploymentTest do } ], "terminationGracePeriodSeconds" => 405, + "securityContext" => %{ + "allowPrivilegeEscalation" => false, + "fsGroup" => 1000, + "readOnlyRootFilesystem" => true, + "runAsNonRoot" => true, + "runAsUser" => 1000 + }, "initContainers" => [ %{ "args" => [ @@ -180,7 +191,7 @@ defmodule DeploymentTest do "namespace" => "default" }, "spec" => %{ - "replicas" => 1, + "replicas" => 2, "selector" => %{ "matchLabels" => %{"actor-system" => "spawn-system", "app" => "spawn-test"} }, @@ -209,7 +220,9 @@ defmodule DeploymentTest do %{ "key" => "actor-system", "operator" => "In", - "values" => ["spawn-system"] + "values" => [ + "spawn-system" + ] } ] }, @@ -228,7 +241,9 @@ defmodule DeploymentTest do %{ "key" => "app", "operator" => "In", - "values" => ["spawn-test"] + "values" => [ + "spawn-test" + ] } ] }, @@ -275,12 +290,26 @@ defmodule DeploymentTest do } }, "volumeMounts" => [ - %{"mountPath" => "/app/certs", "name" => "certs"}, - %{"mountPath" => "/app/extra", "name" => "extra-volume"} + %{"mountPath" => "/home/example", "name" => "volume-name"}, + %{"mountPath" => "/app/certs", "name" => "certs"} ] } ], "terminationGracePeriodSeconds" => 405, + "securityContext" => %{ + "allowPrivilegeEscalation" => false, + "fsGroup" => 1000, + "readOnlyRootFilesystem" => true, + "runAsNonRoot" => true, + "runAsUser" => 1000 + }, + "volumes" => [ + %{"emptyDir" => "{}", "name" => "volume-name"}, + %{ + "name" => "certs", + "secret" => %{"optional" => true, "secretName" => "tls-certs"} + } + ], "initContainers" => [ %{ "args" => [ @@ -299,44 +328,327 @@ defmodule DeploymentTest do "name" => "init-certificates" } ], - "serviceAccountName" => "spawn-system-sa", + "serviceAccountName" => "spawn-system-sa" + } + } + } + } == build_host_deploy(embedded_actor_host_with_volume_mounts) + end + + test "generate deployment with defaults", ctx do + %{ + simple_host: simple_host_resource + } = ctx + + assert %{ + "apiVersion" => "apps/v1", + "kind" => "Deployment", + "metadata" => %{ + "labels" => %{"actor-system" => "spawn-system", "app" => "spawn-test"}, + "name" => "spawn-test", + "namespace" => "default" + }, + "spec" => %{ + "replicas" => 2, + "selector" => %{ + "matchLabels" => %{"actor-system" => "spawn-system", "app" => "spawn-test"} + }, + "strategy" => %{ + "rollingUpdate" => %{"maxSurge" => "50%", "maxUnavailable" => 0}, + "type" => "RollingUpdate" + }, + "template" => %{ + "metadata" => %{ + "annotations" => %{ + "prometheus.io/path" => "/metrics", + "prometheus.io/port" => "9001", + "prometheus.io/scrape" => "true" + }, + "labels" => %{"actor-system" => "spawn-system", "app" => "spawn-test"} + }, + "spec" => %{ + "affinity" => %{ + "podAntiAffinity" => %{ + "preferredDuringSchedulingIgnoredDuringExecution" => [ + %{ + "podAffinityTerm" => %{ + "labelSelector" => %{ + "matchExpressions" => [ + %{ + "key" => "app", + "operator" => "In", + "values" => ["spawn-test"] + } + ] + }, + "topologyKey" => "kubernetes.io/hostname" + }, + "weight" => 100 + } + ] + } + }, + "containers" => [ + %{ + "env" => [ + %{"name" => "RELEASE_NAME", "value" => "spawn"}, + %{ + "name" => "NAMESPACE", + "valueFrom" => %{ + "fieldRef" => %{"fieldPath" => "metadata.namespace"} + } + }, + %{ + "name" => "POD_IP", + "valueFrom" => %{"fieldRef" => %{"fieldPath" => "status.podIP"}} + }, + %{"name" => "SPAWN_PROXY_PORT", "value" => "9001"}, + %{"name" => "SPAWN_PROXY_INTERFACE", "value" => "0.0.0.0"}, + %{"name" => "RELEASE_DISTRIBUTION", "value" => "name"}, + %{"name" => "RELEASE_NODE", "value" => "$(RELEASE_NAME)@$(POD_IP)"} + ], + "envFrom" => [ + %{"configMapRef" => %{"name" => "spawn-test-sidecar-cm"}}, + %{"secretRef" => %{"name" => "spawn-system-secret"}} + ], + "image" => _image_version, + "livenessProbe" => %{ + "httpGet" => %{ + "path" => "/health/liveness", + "port" => 9001, + "scheme" => "HTTP" + }, + "failureThreshold" => 3, + "initialDelaySeconds" => 10, + "periodSeconds" => 10, + "successThreshold" => 1, + "timeoutSeconds" => 30 + }, + "name" => "sidecar", + "ports" => [ + %{"containerPort" => 4369, "name" => "epmd"}, + %{"containerPort" => 9001, "name" => "proxy-http"} + ], + "readinessProbe" => %{ + "httpGet" => %{ + "path" => "/health/readiness", + "port" => 9001, + "scheme" => "HTTP" + }, + "failureThreshold" => 1, + "initialDelaySeconds" => 5, + "periodSeconds" => 5, + "successThreshold" => 1, + "timeoutSeconds" => 5 + }, + "resources" => %{ + "requests" => %{ + "cpu" => "50m", + "ephemeral-storage" => "1M", + "memory" => "80Mi" + } + } + }, + %{ + "env" => [ + %{"name" => "RELEASE_NAME", "value" => "spawn"}, + %{ + "name" => "NAMESPACE", + "valueFrom" => %{ + "fieldRef" => %{"fieldPath" => "metadata.namespace"} + } + }, + %{ + "name" => "POD_IP", + "valueFrom" => %{"fieldRef" => %{"fieldPath" => "status.podIP"}} + }, + %{"name" => "SPAWN_PROXY_PORT", "value" => "9001"}, + %{"name" => "SPAWN_PROXY_INTERFACE", "value" => "0.0.0.0"}, + %{"name" => "RELEASE_DISTRIBUTION", "value" => "name"}, + %{"name" => "RELEASE_NODE", "value" => "$(RELEASE_NAME)@$(POD_IP)"} + ], + "image" => "eigr/spawn-test:latest", + "name" => "actorhost", + "resources" => %{ + "requests" => %{ + "cpu" => "100m", + "ephemeral-storage" => "1M", + "memory" => "80Mi" + } + } + } + ], + "terminationGracePeriodSeconds" => 405 + } + } + } + } = build_host_deploy(simple_host_resource) + end + + test "generate deployment with host ports", ctx do + %{ + simple_host_with_ports: simple_host_with_ports_resource + } = ctx + + assert %{ + "spec" => %{ + "template" => %{ + "spec" => %{ + "affinity" => %{ + "podAntiAffinity" => %{ + "preferredDuringSchedulingIgnoredDuringExecution" => [ + %{ + "podAffinityTerm" => %{ + "labelSelector" => %{ + "matchExpressions" => [ + %{ + "key" => "app", + "operator" => "In", + "values" => ["spawn-test"] + } + ] + }, + "topologyKey" => "kubernetes.io/hostname" + }, + "weight" => 100 + } + ] + } + }, + "containers" => containers + } + } + } + } = build_host_deploy(simple_host_with_ports_resource) + + assert List.first(containers) == %{ + "env" => [ + %{"name" => "RELEASE_NAME", "value" => "spawn"}, + %{ + "name" => "NAMESPACE", + "valueFrom" => %{ + "fieldRef" => %{"fieldPath" => "metadata.namespace"} + } + }, + %{ + "name" => "POD_IP", + "valueFrom" => %{"fieldRef" => %{"fieldPath" => "status.podIP"}} + }, + %{"name" => "SPAWN_PROXY_PORT", "value" => "9001"}, + %{"name" => "SPAWN_PROXY_INTERFACE", "value" => "0.0.0.0"}, + %{"name" => "RELEASE_DISTRIBUTION", "value" => "name"}, + %{"name" => "RELEASE_NODE", "value" => "$(RELEASE_NAME)@$(POD_IP)"} + ], + "image" => "eigr/spawn-test:latest", + "name" => "actorhost", + "resources" => %{ + "requests" => %{"memory" => "80Mi", "ephemeral-storage" => "1M", "cpu" => "100m"} + }, + "volumeMounts" => [%{"mountPath" => "/app/certs", "name" => "certs"}], + "ports" => [ + %{"containerPort" => 8090, "name" => "http"}, + %{"containerPort" => 8091, "name" => "https"} + ] + } + end + + test "generate deployment with host volumeMount", ctx do + %{ + simple_actor_host_with_volume_mounts: simple_actor_host_with_volume_mounts + } = ctx + + assert %{ + "spec" => %{ + "template" => %{ + "spec" => %{ + "affinity" => %{ + "podAntiAffinity" => %{ + "preferredDuringSchedulingIgnoredDuringExecution" => [ + %{ + "podAffinityTerm" => %{ + "labelSelector" => %{ + "matchExpressions" => [ + %{ + "key" => "app", + "operator" => "In", + "values" => ["spawn-test"] + } + ] + }, + "topologyKey" => "kubernetes.io/hostname" + }, + "weight" => 100 + } + ] + } + }, + "containers" => containers, "volumes" => [ + %{"emptyDir" => "{}", "name" => "volume-name"}, %{ "name" => "certs", "secret" => %{"optional" => true, "secretName" => "tls-certs"} - }, - %{ - "name" => "extra-volume", - "emptyDir" => %{} } ] } } } - } == build_host_deploy(embedded_actor_host_with_volume_mounts) - end + } = build_host_deploy(simple_actor_host_with_volume_mounts) - # Additional tests for other SDK values, secrets, configurations, etc. - # ... + assert %{ + "volumeMounts" => [ + %{"mountPath" => "/home/example", "name" => "volume-name"}, + %{"mountPath" => "/app/certs", "name" => "certs"} + ] + } = List.last(containers) + end for sdk <- ~w(dart elixir java python rust springboot nodejs unknown)a do + @global_sdk sdk test "generate deployment for SDK #{sdk}", ctx do %{ simple_host: simple_host } = ctx expected_resources = - case sdk do - "dart" -> %{"requests" => %{"cpu" => "100m", "memory" => "64Mi"}} - "elixir" -> %{"requests" => %{"cpu" => "200m", "memory" => "128Mi"}} - "java" -> %{"requests" => %{"cpu" => "300m", "memory" => "256Mi"}} - "python" -> %{"requests" => %{"cpu" => "150m", "memory" => "128Mi"}} - "rust" -> %{"requests" => %{"cpu" => "200m", "memory" => "256Mi"}} - "springboot" -> %{"requests" => %{"cpu" => "400m", "memory" => "512Mi"}} - "nodejs" -> %{"requests" => %{"cpu" => "150m", "memory" => "128Mi"}} - _ -> %{"requests" => %{"cpu" => "100m", "memory" => "64Mi"}} + case @global_sdk do + "dart" -> + %{"requests" => %{"cpu" => "10m", "memory" => "70Mi", "ephemeral-storage" => "1M"}} + + "elixir" -> + %{ + "requests" => %{"cpu" => "150m", "memory" => "256Mi", "ephemeral-storage" => "1M"} + } + + "java" -> + %{ + "requests" => %{"cpu" => "200m", "memory" => "512Mi", "ephemeral-storage" => "1M"} + } + + "python" -> + %{"requests" => %{"cpu" => "10m", "memory" => "256Mi", "ephemeral-storage" => "1M"}} + + "rust" -> + %{"requests" => %{"cpu" => "10m", "memory" => "70Mi", "ephemeral-storage" => "1M"}} + + "springboot" -> + %{ + "requests" => %{"cpu" => "300m", "memory" => "512Mi", "ephemeral-storage" => "1M"} + } + + "nodejs" -> + %{ + "requests" => %{"cpu" => "150m", "memory" => "256Mi", "ephemeral-storage" => "1M"} + } + + _ -> + %{"requests" => %{"cpu" => "100m", "memory" => "80Mi", "ephemeral-storage" => "1M"}} end + host = simple_host["spec"]["host"] + new_simple_actor_host = Map.put(host, "sdk", Atom.to_string(@global_sdk)) + simple_host = Map.put(simple_host, "spec", %{"host" => new_simple_actor_host}) + assert %{ "apiVersion" => "apps/v1", "kind" => "Deployment", @@ -346,7 +658,7 @@ defmodule DeploymentTest do "namespace" => "default" }, "spec" => %{ - "replicas" => 1, + "replicas" => 2, "selector" => %{ "matchLabels" => %{"actor-system" => "spawn-system", "app" => "spawn-test"} }, @@ -364,6 +676,46 @@ defmodule DeploymentTest do "labels" => %{"actor-system" => "spawn-system", "app" => "spawn-test"} }, "spec" => %{ + "affinity" => %{ + "podAntiAffinity" => %{ + "preferredDuringSchedulingIgnoredDuringExecution" => [ + %{ + "podAffinityTerm" => %{ + "labelSelector" => %{ + "matchExpressions" => [ + %{ + "key" => "app", + "operator" => "In", + "values" => ["spawn-test"] + } + ] + }, + "topologyKey" => "kubernetes.io/hostname" + }, + "weight" => 100 + } + ] + }, + "podAffinity" => %{ + "preferredDuringSchedulingIgnoredDuringExecution" => [ + %{ + "podAffinityTerm" => %{ + "labelSelector" => %{ + "matchExpressions" => [ + %{ + "key" => "actor-system", + "operator" => "In", + "values" => ["spawn-system"] + } + ] + }, + "topologyKey" => "kubernetes.io/hostname" + }, + "weight" => 50 + } + ] + } + }, "containers" => [ %{ "env" => [ @@ -385,20 +737,59 @@ defmodule DeploymentTest do ], "image" => "eigr/spawn-test:latest", "name" => "actorhost", - "ports" => [ - %{"containerPort" => 4369, "name" => "epmd"}, - %{"containerPort" => 9001, "name" => "proxy-http"} - ], + "ports" => [%{"containerPort" => "9001", "name" => "http"}], "resources" => expected_resources + }, + %{ + "image" => "ghcr.io/eigr/spawn-proxy:1.4.2", + "name" => "proxy", + "ports" => [%{"containerPort" => "9001", "name" => "http"}], + "resources" => %{ + "requests" => %{ + "cpu" => "50m", + "ephemeral-storage" => "1M", + "memory" => "80Mi" + } + } } ], "terminationGracePeriodSeconds" => 405, + "initContainers" => [ + %{ + "args" => [ + "--environment", + "prod", + "--secret", + "tls-certs", + "--namespace", + "default", + "--service", + "spawn-system", + "--to", + "default" + ], + "image" => "ghcr.io/eigr/spawn-initializer:1.4.2", + "name" => "init-certificates" + } + ], + "securityContext" => %{ + "allowPrivilegeEscalation" => false, + "fsGroup" => 1000, + "readOnlyRootFilesystem" => true, + "runAsNonRoot" => true, + "runAsUser" => 1000 + }, "serviceAccountName" => "spawn-system-sa" } } } - } == build_host_deploy(%{simple_host | params: %{sdk: sdk}}) + } == build_host_deploy(simple_host) end end end + + defp build_host_deploy(resource) do + SpawnOperator.get_args(resource) + |> Deployment.manifest() + end end