Add policy demo to handle topology spread

anydatacenter/30-policy-demo/.terraform.lock.hcl

@@ -0,0 +1,22 @@
+# This file is maintained automatically by "terraform init".
+# Manual edits may be lost in future updates.
+
+provider "registry.terraform.io/hashicorp/kubernetes" {
+  version     = "2.34.0"
+  constraints = "~> 2.34"
+  hashes = [
+    "h1:QOiO85qZnkUm7kAtuPkfblchuKPWUqRdNVWE5agpr8k=",
+    "zh:076b451dc8629c49f4260de6d43595e98ac5f1bdbebb01d112659ef94d99451f",
+    "zh:0c29855dbd3c6ba82fce680fa5ac969d4e09e20fecb4ed40166b778bd19895a4",
+    "zh:583b4dfcea4d8392dd7904c00b2ff41bbae78d238e8b72e5ad580370a24a4ecb",
+    "zh:5e20844d8d1af052381d00de4febd4055ad0f3c3c02795c361265b9ef72a1075",
+    "zh:766b7ab7c4727c62b5887c3922e0467c4cc355ba0dc3aabe465ebb86bc1caabb",
+    "zh:776a5000b441d7c8262d17d4a4aa4aa9760ae64de4cb7172961d9e007e0be1e5",
+    "zh:7838f509235116e55adeeecbe6def3da1b66dd3c4ce0de02fc7dc66a60e1d630",
+    "zh:931e5581ec66c145c1d29198bd23fddc8d0c5cbf4cda22e02dba65644c7842f2",
+    "zh:95e728efa2a31a63b879fd093507466e509e3bfc9325eb35ea3dc28fed15c6f7",
+    "zh:972b9e3ca2b6a1057dcf5003fc78cabb0dd8847580bddeb52d885ebd64df38ea",
+    "zh:ef6114217965d55f5bddbd7a316b8f85f15b8a77c075fcbed95813039d522e0a",
+    "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c",
+  ]
+}

anydatacenter/30-policy-demo/kyvernoPolicies.tf

@@ -0,0 +1,11 @@
+resource "kubernetes_manifest" "kyverno_policy_topology_spread" {
+  # To ensure the correct namespace is used, we could do a patch on the manifest
+  # For the sake of the demo and to ensure the kyverno tests are running correctly
+  # the namespace is hardcoded in the policy
+  manifest = yamldecode(file("${path.module}/kyvernoPolicies/rossumTopologySpread.yaml"))
+  depends_on = [
+    kubernetes_namespace.rossum,
+    kubernetes_deployment.pre_policy_sleeper,
+    kubernetes_deployment.pre_policy_sleeper_without_topology_spread,
+  ]
+}

anydatacenter/30-policy-demo/kyvernoPolicies/rossumTopologySpread.yaml

@@ -0,0 +1,116 @@
+apiVersion: kyverno.io/v1
+kind: Policy
+metadata:
+  name: enforce-topology-spread
+  namespace: rossum
+  annotations:
+    policies.kyverno.io/title: Spread pods based on zone topology
+    policies.kyverno.io/subject: Pod
+spec:
+  # Depending on the desired outcome for the applications, there could be multiple solutions
+  # to the problem, naturally, the most complex and difficult was chosen in order to:
+  # 1. Preserve existing topology spread if defined
+  # 2. Inject topology spread if not defined
+  # 3. Override any settings of topology spread based on zones
+  #
+  # see https://github.com/kyverno/kyverno/issues/10655 for details why it cannot
+  # be achieved using the merge patch
+  rules:
+    # Check if existing zone topology spread is defined and overwrite it
+    - name: enforce-zone-topology-spread-configuration
+      match:
+        any:
+          - resources:
+              kinds:
+                - Deployment
+                - StatefulSet
+              selector:
+                matchLabels:
+                  # Additional validation policies would be needed to ensure the label is present on every resource
+                  app.kubernetes.io/name: "*"
+              operations:
+                - CREATE
+                - UPDATE
+      mutate:
+        foreach:
+          - list: "request.object.spec.template.spec.topologySpreadConstraints || []"
+            # Use precondition to mutate
+            preconditions:
+              any:
+                - key: "{{ element.topologyKey }}"
+                  operator: Equals
+                  value: topology.kubernetes.io/zone
+            patchesJson6902: |-
+              - path: /spec/template/spec/topologySpreadConstraints/{{elementIndex}}
+                op: replace
+                value:
+                  maxSkew: 1
+                  topologyKey: topology.kubernetes.io/zone
+                  whenUnsatisfiable: ScheduleAnyway
+                  labelSelector:
+                    matchLabels:
+                      app.kubernetes.io/name: '{{request.object.spec.selector.matchLabels."app.kubernetes.io/name"}}'
+    # Check if the zone topology spread is not defined and inject it
+    - name: inject-zone-topology-spread
+      match:
+        any:
+          - resources:
+              kinds:
+                - Deployment
+                - StatefulSet
+              selector:
+                matchLabels:
+                  # Additional validation policies would be needed to ensure the label is present on every resource
+                  app.kubernetes.io/name: "*"
+              operations:
+                - CREATE
+                - UPDATE
+      mutate:
+        foreach:
+          - list: "request.object.spec.template.spec.topologySpreadConstraints || []"
+            # Use precondition to mutate
+            preconditions:
+              any:
+                - key: "{{ request.object.spec.template.spec.topologySpreadConstraints[].topologyKey }}"
+                  operator: AllNotIn
+                  value:
+                    - topology.kubernetes.io/zone
+            patchesJson6902: |-
+              - path: /spec/template/spec/topologySpreadConstraints/0
+                op: add
+                value:
+                  maxSkew: 1
+                  topologyKey: topology.kubernetes.io/zone
+                  whenUnsatisfiable: ScheduleAnyway
+                  labelSelector:
+                    matchLabels:
+                      app.kubernetes.io/name: '{{request.object.spec.selector.matchLabels."app.kubernetes.io/name"}}'
+    # Create topology spread if it does not exist
+    - name: create-topology-spread
+      match:
+        any:
+          - resources:
+              kinds:
+                - Deployment
+                - StatefulSet
+              selector:
+                matchLabels:
+                  # Additional validation policies would be needed to ensure the label is present on every resource
+                  app.kubernetes.io/name: "*"
+              operations:
+                - CREATE
+                - UPDATE
+      mutate:
+        patchStrategicMerge:
+          spec:
+            template:
+              spec:
+                # Ensure the zone topology spread is present if undefined
+                +(topologySpreadConstraints):
+                  - maxSkew: 1
+                    topologyKey: topology.kubernetes.io/zone
+                    # Depending on the workload and requirements, ScheduleAnyway or DoNotSchedule might be chosen
+                    whenUnsatisfiable: ScheduleAnyway
+                    labelSelector:
+                      matchLabels:
+                        app.kubernetes.io/name: '{{request.object.spec.selector.matchLabels."app.kubernetes.io/name"}}'

anydatacenter/30-policy-demo/main.tf

@@ -0,0 +1,13 @@
+terraform {
+  backend "local" {
+    path = "30-policy-demo.tfstate"
+  }
+}
+
+variable "kubectx" {
+  type = string
+}
+
+provider "kubernetes" {
+  config_context = var.kubectx
+}

anydatacenter/30-policy-demo/postPolicy.tf

@@ -0,0 +1,55 @@
+resource "kubernetes_deployment" "post_policy_sleeper" {
+  metadata {
+    name      = "post-policy-sleeper"
+    namespace = kubernetes_namespace.rossum.metadata[0].name
+    labels = {
+      "app.kubernetes.io/name"    = "post-policy-sleeper"
+      "app.kubernetes.io/version" = "v5"
+    }
+  }
+
+  spec {
+    replicas = 3
+
+    selector {
+      match_labels = {
+        "app.kubernetes.io/name" = "post-policy-sleeper"
+      }
+    }
+
+    template {
+      metadata {
+        labels = {
+          "app.kubernetes.io/name" = "post-policy-sleeper"
+        }
+      }
+
+      spec {
+        container {
+          name  = "sleepy"
+          image = "busybox"
+          command = [
+            "sh",
+            "-c",
+            "while true; do sleep 60; done"
+          ]
+        }
+
+        security_context {
+          run_as_user  = 1000
+          run_as_group = 1000
+        }
+      }
+    }
+  }
+
+  lifecycle {
+    ignore_changes = [
+      # Injected by kyverno policy on create
+      spec[0].template[0].spec[0].topology_spread_constraint
+    ]
+  }
+
+  # Execute after the kyverno policy is in place
+  depends_on = [kubernetes_manifest.kyverno_policy_topology_spread]
+}

anydatacenter/30-policy-demo/prePolicy.tf

@@ -0,0 +1,110 @@
+# Deployment will be added before the kyverno policy is created
+resource "kubernetes_deployment" "pre_policy_sleeper" {
+  metadata {
+    name      = "pre-policy-sleeper"
+    namespace = kubernetes_namespace.rossum.metadata[0].name
+    labels = {
+      "app.kubernetes.io/name"    = "pre-policy-sleeper"
+      "app.kubernetes.io/version" = "v3"
+    }
+  }
+
+  spec {
+    replicas = 3
+
+    selector {
+      match_labels = {
+        "app.kubernetes.io/name" = "pre-policy-sleeper"
+      }
+    }
+
+    template {
+      metadata {
+        labels = {
+          "app.kubernetes.io/name" = "pre-policy-sleeper"
+        }
+      }
+
+      spec {
+        topology_spread_constraint {
+          max_skew           = 1
+          topology_key       = "topology.kubernetes.io/hostname"
+          when_unsatisfiable = "ScheduleAnyway"
+          label_selector {
+            match_labels = {
+              "app.kubernetes.io/name" = "pre-policy-sleeper"
+            }
+          }
+        }
+
+        container {
+          name  = "sleepy"
+          image = "busybox"
+          command = [
+            "sh",
+            "-c",
+            "while true; do sleep 60; done"
+          ]
+        }
+
+        security_context {
+          run_as_user  = 1000
+          run_as_group = 1000
+        }
+      }
+    }
+  }
+}
+
+resource "kubernetes_deployment" "pre_policy_sleeper_without_topology_spread" {
+  metadata {
+    name      = "pre-policy-sleeper-without-topology-spread"
+    namespace = kubernetes_namespace.rossum.metadata[0].name
+    labels = {
+      "app.kubernetes.io/name"    = "pre-policy-sleeper-without-topology-spread"
+      "app.kubernetes.io/version" = "v2"
+    }
+  }
+
+  spec {
+    replicas = 3
+
+    selector {
+      match_labels = {
+        "app.kubernetes.io/name" = "pre-policy-sleeper-without-topology-spread"
+      }
+    }
+
+    template {
+      metadata {
+        labels = {
+          "app.kubernetes.io/name" = "pre-policy-sleeper-without-topology-spread"
+        }
+      }
+
+      spec {
+        container {
+          name  = "sleepy"
+          image = "busybox"
+          command = [
+            "sh",
+            "-c",
+            "while true; do sleep 60; done"
+          ]
+        }
+
+        security_context {
+          run_as_user  = 1000
+          run_as_group = 1000
+        }
+      }
+    }
+  }
+
+  lifecycle {
+    ignore_changes = [
+      # Injected by kyverno policy on update
+      spec[0].template[0].spec[0].topology_spread_constraint
+    ]
+  }
+}

anydatacenter/30-policy-demo/rossum.tf

@@ -0,0 +1,5 @@
+resource "kubernetes_namespace" "rossum" {
+  metadata {
+    name = "rossum"
+  }
+}

anydatacenter/30-policy-demo/versions.tf

@@ -0,0 +1,3 @@
+module "versions" {
+  source = "../../modules/versions"
+}