chore(helm): add unit tests for helm chart (#6557)

This PR adds a minimum set of Helm tests for the Helm chart.
It's heavily based on the approach in [1], but uses a golden-files-based approach instead.
It also runs helm template directly instead of importing the entire Kubernetes API.
Golden files can be updated by running go test ./helm/tests -update or by running make update-golden-files.

[1] https://github.com/coder/enterprise-helm

Fixes #6552

.gitignore

@@ -30,6 +30,7 @@ site/playwright-report/*
 
 # Make target for updating golden files.
 cli/testdata/.gen-golden
+helm/tests/testdata/.gen-golden
 
 # Build
 /build/

.prettierignore

@@ -33,6 +33,7 @@ site/playwright-report/*
 
 # Make target for updating golden files.
 cli/testdata/.gen-golden
+helm/tests/testdata/.gen-golden
 
 # Build
 /build/

Makefile

@@ -516,13 +516,17 @@ coderd/apidoc/swagger.json: $(shell find ./scripts/apidocgen $(FIND_EXCLUSIONS)
 	./scripts/apidocgen/generate.sh
 	yarn run --cwd=site format:write:only ../docs/api ../docs/manifest.json ../coderd/apidoc/swagger.json
 
-update-golden-files: cli/testdata/.gen-golden
+update-golden-files: cli/testdata/.gen-golden helm/tests/testdata/.gen-golden
 .PHONY: update-golden-files
 
 cli/testdata/.gen-golden: $(wildcard cli/testdata/*.golden) $(GO_SRC_FILES)
 	go test ./cli -run=TestCommandHelp -update
 	touch "$@"
 
+helm/tests/testdata/.gen-golden: $(wildcard helm/tests/testdata/*.golden) $(GO_SRC_FILES)
+	go test ./helm/tests -run=TestUpdateGoldenFiles -update
+	touch "$@"
+
 # Generate a prettierrc for the site package that uses relative paths for
 # overrides. This allows us to share the same prettier config between the
 # site and the root of the repo.

flake.lock

@@ -37,11 +37,11 @@
     },
     "flake-utils_2": {
       "locked": {
-        "lastModified": 1667395993,
-        "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=",
+        "lastModified": 1676283394,
+        "narHash": "sha256-XX2f9c3iySLCw54rJ/CZs+ZK6IQy7GXNY4nSOyu2QG4=",
         "owner": "numtide",
         "repo": "flake-utils",
-        "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f",
+        "rev": "3db36a8b464d0c4532ba1c7dda728f4576d6d073",
         "type": "github"
       },
       "original": {
@@ -67,11 +67,11 @@
     },
     "nixpkgs_2": {
       "locked": {
-        "lastModified": 1676110339,
-        "narHash": "sha256-kOS/L8OOL2odpCOM11IevfHxcUeE0vnZUQ74EOiwXcs=",
+        "lastModified": 1678654296,
+        "narHash": "sha256-aVfw3ThpY7vkUeF1rFy10NAkpKDS2imj3IakrzT0Occ=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "e5530aba13caff5a4f41713f1265b754dc2abfd8",
+        "rev": "5a1dc8acd977ff3dccd1328b7c4a6995429a656b",
         "type": "github"
       },
       "original": {

flake.nix

@@ -29,6 +29,7 @@
             gopls
             gotestsum
             jq
+            kubernetes-helm
             nfpm
             nodePackages.typescript
             nodePackages.typescript-language-server

helm/tests/chart_test.go

@@ -0,0 +1,154 @@
+package tests // nolint: testpackage
+
+import (
+	"bytes"
+	"flag"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"golang.org/x/xerrors"
+)
+
+// These tests run `helm template` with the values file specified in each test
+// and compare the output to the contents of the corresponding golden file.
+// All values and golden files are located in the `testdata` directory.
+// To update golden files, run `go test . -update`.
+
+// UpdateGoldenFiles is a flag that can be set to update golden files.
+var UpdateGoldenFiles = flag.Bool("update", false, "Update golden files")
+
+var TestCases = []TestCase{
+	{
+		name:          "default_values",
+		expectedError: "",
+	},
+	{
+		name:          "missing_values",
+		expectedError: `You must specify the coder.image.tag value if you're installing the Helm chart directly from Git.`,
+	},
+	{
+		name:          "tls",
+		expectedError: "",
+	},
+}
+
+type TestCase struct {
+	name          string // Name of the test case. This is used to control which values and golden file are used.
+	expectedError string // Expected error from running `helm template`.
+}
+
+func (tc TestCase) valuesFilePath() string {
+	return filepath.Join("./testdata", tc.name+".yaml")
+}
+
+func (tc TestCase) goldenFilePath() string {
+	return filepath.Join("./testdata", tc.name+".golden")
+}
+
+func TestRenderChart(t *testing.T) {
+	t.Parallel()
+	if *UpdateGoldenFiles {
+		t.Skip("Golden files are being updated. Skipping test.")
+	}
+	if _, runningInCI := os.LookupEnv("CI"); runningInCI {
+		switch runtime.GOOS {
+		case "windows", "darwin":
+			t.Skip("Skipping tests on Windows and macOS in CI")
+		}
+	}
+
+	// Ensure that Helm is available in $PATH
+	helmPath := lookupHelm(t)
+	for _, tc := range TestCases {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+
+			// Ensure that the values file exists.
+			valuesFilePath := tc.valuesFilePath()
+			if _, err := os.Stat(valuesFilePath); os.IsNotExist(err) {
+				t.Fatalf("values file %q does not exist", valuesFilePath)
+			}
+
+			// Run helm template with the values file.
+			templateOutput, err := runHelmTemplate(t, helmPath, "..", valuesFilePath)
+			if tc.expectedError != "" {
+				require.Error(t, err, "helm template should have failed")
+				require.Contains(t, templateOutput, tc.expectedError, "helm template output should contain expected error")
+			} else {
+				require.NoError(t, err, "helm template should not have failed")
+				require.NotEmpty(t, templateOutput, "helm template output should not be empty")
+				goldenFilePath := tc.goldenFilePath()
+				goldenBytes, err := os.ReadFile(goldenFilePath)
+				require.NoError(t, err, "failed to read golden file %q", goldenFilePath)
+
+				// Remove carriage returns to make tests pass on Windows.
+				goldenBytes = bytes.Replace(goldenBytes, []byte("\r"), []byte(""), -1)
+				expected := string(goldenBytes)
+
+				require.NoError(t, err, "failed to load golden file %q")
+				require.Equal(t, expected, templateOutput)
+			}
+		})
+	}
+}
+
+func TestUpdateGoldenFiles(t *testing.T) {
+	t.Parallel()
+	if !*UpdateGoldenFiles {
+		t.Skip("Run with -update to update golden files")
+	}
+
+	helmPath := lookupHelm(t)
+	for _, tc := range TestCases {
+		if tc.expectedError != "" {
+			t.Logf("skipping test case %q with render error", tc.name)
+			continue
+		}
+
+		valuesPath := tc.valuesFilePath()
+		templateOutput, err := runHelmTemplate(t, helmPath, "..", valuesPath)
+
+		require.NoError(t, err, "failed to run `helm template -f %q`", valuesPath)
+
+		goldenFilePath := tc.goldenFilePath()
+		err = os.WriteFile(goldenFilePath, []byte(templateOutput), 0o644) // nolint:gosec
+		require.NoError(t, err, "failed to write golden file %q", goldenFilePath)
+	}
+	t.Log("Golden files updated. Please review the changes and commit them.")
+}
+
+// runHelmTemplate runs helm template on the given chart with the given values and
+// returns the raw output.
+func runHelmTemplate(t testing.TB, helmPath, chartDir, valuesFilePath string) (string, error) {
+	// Ensure that valuesFilePath exists
+	if _, err := os.Stat(valuesFilePath); err != nil {
+		return "", xerrors.Errorf("values file %q does not exist: %w", valuesFilePath, err)
+	}
+
+	cmd := exec.Command(helmPath, "template", chartDir, "-f", valuesFilePath, "--namespace", "default")
+	t.Logf("exec command: %v", cmd.Args)
+	out, err := cmd.CombinedOutput()
+	return string(out), err
+}
+
+// lookupHelm ensures that Helm is available in $PATH and returns the path to the
+// Helm executable.
+func lookupHelm(t testing.TB) string {
+	helmPath, err := exec.LookPath("helm")
+	if err != nil {
+		t.Fatalf("helm not found in $PATH: %v", err)
+		return ""
+	}
+	t.Logf("Using helm at %q", helmPath)
+	return helmPath
+}
+
+func TestMain(m *testing.M) {
+	flag.Parse()
+	os.Exit(m.Run())
+}

helm/tests/testdata/default_values.golden

@@ -0,0 +1,161 @@
+---
+# Source: coder/templates/coder.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: "coder"
+  annotations: 
+    {}
+  labels:
+    helm.sh/chart: coder-0.1.0
+    app.kubernetes.io/name: coder
+    app.kubernetes.io/instance: release-name
+    app.kubernetes.io/part-of: coder
+    app.kubernetes.io/version: "0.1.0"
+    app.kubernetes.io/managed-by: Helm
+---
+# Source: coder/templates/rbac.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: coder-workspace-perms
+rules:
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["*"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["*"]
+---
+# Source: coder/templates/rbac.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: "coder"
+subjects:
+  - kind: ServiceAccount
+    name: "coder"
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: coder-workspace-perms
+---
+# Source: coder/templates/service.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  name: coder
+  labels:
+    helm.sh/chart: coder-0.1.0
+    app.kubernetes.io/name: coder
+    app.kubernetes.io/instance: release-name
+    app.kubernetes.io/part-of: coder
+    app.kubernetes.io/version: "0.1.0"
+    app.kubernetes.io/managed-by: Helm
+  annotations:
+    {}
+spec:
+  type: LoadBalancer
+  sessionAffinity: ClientIP
+  ports:
+    - name: "http"
+      port: 80
+      targetPort: "http"
+      protocol: TCP
+  externalTrafficPolicy: "Cluster"
+  selector:
+    app.kubernetes.io/name: coder
+    app.kubernetes.io/instance: release-name
+---
+# Source: coder/templates/coder.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: coder
+  labels:
+    helm.sh/chart: coder-0.1.0
+    app.kubernetes.io/name: coder
+    app.kubernetes.io/instance: release-name
+    app.kubernetes.io/part-of: coder
+    app.kubernetes.io/version: "0.1.0"
+    app.kubernetes.io/managed-by: Helm
+  annotations: 
+    {}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: coder
+      app.kubernetes.io/instance: release-name
+  template:
+    metadata:
+      labels:
+        helm.sh/chart: coder-0.1.0
+        app.kubernetes.io/name: coder
+        app.kubernetes.io/instance: release-name
+        app.kubernetes.io/part-of: coder
+        app.kubernetes.io/version: "0.1.0"
+        app.kubernetes.io/managed-by: Helm
+    spec:
+      serviceAccountName: "coder"
+      restartPolicy: Always
+      terminationGracePeriodSeconds: 60
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: app.kubernetes.io/instance
+                  operator: In
+                  values:
+                  - coder
+              topologyKey: kubernetes.io/hostname
+            weight: 1
+      containers:
+        - name: coder
+          image: "ghcr.io/coder/coder:latest"
+          imagePullPolicy: IfNotPresent
+          resources:
+            {}
+          env:
+            - name: CODER_HTTP_ADDRESS
+              value: "0.0.0.0:8080"
+            - name: CODER_PROMETHEUS_ADDRESS
+              value: "0.0.0.0:2112"
+            # Set the default access URL so a `helm apply` works by default.
+            # See: https://github.com/coder/coder/issues/5024
+            - name: CODER_ACCESS_URL
+              value: "http://coder.default.svc.cluster.local"
+            # Used for inter-pod communication with high-availability.
+            - name: KUBE_POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: CODER_DERP_SERVER_RELAY_URL
+              value: "http://$(KUBE_POD_IP):8080"
+            
+          ports:
+            - name: "http"
+              containerPort: 8080
+              protocol: TCP
+          securityContext: 
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: null
+            runAsGroup: 1000
+            runAsNonRoot: true
+            runAsUser: 1000
+            seccompProfile:
+              type: RuntimeDefault
+          readinessProbe:
+            httpGet:
+              path: /api/v2/buildinfo
+              port: "http"
+              scheme: "HTTP"
+          livenessProbe:
+            httpGet:
+              path: /api/v2/buildinfo
+              port: "http"
+              scheme: "HTTP"
+          volumeMounts: []
+      volumes: []

helm/tests/testdata/default_values.yaml

@@ -0,0 +1,3 @@
+coder:
+  image:
+    tag: latest

helm/tests/testdata/tls.golden

@@ -0,0 +1,183 @@
+---
+# Source: coder/templates/coder.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: "coder"
+  annotations: 
+    {}
+  labels:
+    helm.sh/chart: coder-0.1.0
+    app.kubernetes.io/name: coder
+    app.kubernetes.io/instance: release-name
+    app.kubernetes.io/part-of: coder
+    app.kubernetes.io/version: "0.1.0"
+    app.kubernetes.io/managed-by: Helm
+---
+# Source: coder/templates/rbac.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: coder-workspace-perms
+rules:
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["*"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["*"]
+---
+# Source: coder/templates/rbac.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: "coder"
+subjects:
+  - kind: ServiceAccount
+    name: "coder"
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: coder-workspace-perms
+---
+# Source: coder/templates/service.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  name: coder
+  labels:
+    helm.sh/chart: coder-0.1.0
+    app.kubernetes.io/name: coder
+    app.kubernetes.io/instance: release-name
+    app.kubernetes.io/part-of: coder
+    app.kubernetes.io/version: "0.1.0"
+    app.kubernetes.io/managed-by: Helm
+  annotations:
+    {}
+spec:
+  type: LoadBalancer
+  sessionAffinity: ClientIP
+  ports:
+    - name: "http"
+      port: 80
+      targetPort: "http"
+      protocol: TCP
+    - name: "https"
+      port: 443
+      targetPort: "https"
+      protocol: TCP
+  externalTrafficPolicy: "Cluster"
+  selector:
+    app.kubernetes.io/name: coder
+    app.kubernetes.io/instance: release-name
+---
+# Source: coder/templates/coder.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: coder
+  labels:
+    helm.sh/chart: coder-0.1.0
+    app.kubernetes.io/name: coder
+    app.kubernetes.io/instance: release-name
+    app.kubernetes.io/part-of: coder
+    app.kubernetes.io/version: "0.1.0"
+    app.kubernetes.io/managed-by: Helm
+  annotations: 
+    {}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: coder
+      app.kubernetes.io/instance: release-name
+  template:
+    metadata:
+      labels:
+        helm.sh/chart: coder-0.1.0
+        app.kubernetes.io/name: coder
+        app.kubernetes.io/instance: release-name
+        app.kubernetes.io/part-of: coder
+        app.kubernetes.io/version: "0.1.0"
+        app.kubernetes.io/managed-by: Helm
+    spec:
+      serviceAccountName: "coder"
+      restartPolicy: Always
+      terminationGracePeriodSeconds: 60
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: app.kubernetes.io/instance
+                  operator: In
+                  values:
+                  - coder
+              topologyKey: kubernetes.io/hostname
+            weight: 1
+      containers:
+        - name: coder
+          image: "ghcr.io/coder/coder:latest"
+          imagePullPolicy: IfNotPresent
+          resources:
+            {}
+          env:
+            - name: CODER_HTTP_ADDRESS
+              value: "0.0.0.0:8080"
+            - name: CODER_PROMETHEUS_ADDRESS
+              value: "0.0.0.0:2112"
+            # Set the default access URL so a `helm apply` works by default.
+            # See: https://github.com/coder/coder/issues/5024
+            - name: CODER_ACCESS_URL
+              value: "https://coder.default.svc.cluster.local"
+            # Used for inter-pod communication with high-availability.
+            - name: KUBE_POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: CODER_DERP_SERVER_RELAY_URL
+              value: "http://$(KUBE_POD_IP):8080"
+            
+            - name: CODER_TLS_ENABLE
+              value: "true"
+            - name: CODER_TLS_ADDRESS
+              value: "0.0.0.0:8443"
+            - name: CODER_TLS_CERT_FILE
+              value: "/etc/ssl/certs/coder/coder-tls/tls.crt"
+            - name: CODER_TLS_KEY_FILE
+              value: "/etc/ssl/certs/coder/coder-tls/tls.key"
+          ports:
+            - name: "http"
+              containerPort: 8080
+              protocol: TCP
+            - name: "https"
+              containerPort: 8443
+              protocol: TCP
+          securityContext: 
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: null
+            runAsGroup: 1000
+            runAsNonRoot: true
+            runAsUser: 1000
+            seccompProfile:
+              type: RuntimeDefault
+          readinessProbe:
+            httpGet:
+              path: /api/v2/buildinfo
+              port: "http"
+              scheme: "HTTP"
+          livenessProbe:
+            httpGet:
+              path: /api/v2/buildinfo
+              port: "http"
+              scheme: "HTTP"
+          volumeMounts:
+          - name: "tls-coder-tls"
+            mountPath: "/etc/ssl/certs/coder/coder-tls"
+            readOnly: true
+          
+      volumes:
+      - name: "tls-coder-tls"
+        secret:
+          secretName: "coder-tls"

helm/tests/testdata/tls.yaml

@@ -0,0 +1,6 @@
+coder:
+  image:
+    tag: latest
+  tls:
+    secretNames:
+      - coder-tls

site/.eslintignore

@@ -33,6 +33,7 @@ playwright-report/*
 
 # Make target for updating golden files.
 ../cli/testdata/.gen-golden
+../helm/tests/testdata/.gen-golden
 
 # Build
 ../build/

site/.prettierignore

@@ -33,6 +33,7 @@ playwright-report/*
 
 # Make target for updating golden files.
 ../cli/testdata/.gen-golden
+../helm/tests/testdata/.gen-golden
 
 # Build
 ../build/