refactor(scaletest/terraform): break up infra creation and k8s resource provisioning (#9824)

Breaks up scaletest/terraform into {infra, k8s}.

scaletest/terraform/infra/gcp_cluster.tf

@@ -3,7 +3,9 @@ data "google_compute_default_service_account" "default" {
 }
 
 locals {
-  cluster_kubeconfig_path = "${abspath(path.module)}/../.coderv2/${var.name}-cluster.kubeconfig"
+  abs_module_path         = abspath(path.module)
+  rel_kubeconfig_path     = "../../.coderv2/${var.name}-cluster.kubeconfig"
+  cluster_kubeconfig_path = abspath("${local.abs_module_path}/${local.rel_kubeconfig_path}")
 }
 
 resource "google_container_cluster" "primary" {

scaletest/terraform/infra/gcp_db.tf

@@ -42,6 +42,14 @@ resource "google_sql_database" "coder" {
   deletion_policy = "ABANDON"
 }
 
+resource "random_password" "coder-postgres-password" {
+  length = 12
+}
+
+resource "random_password" "prometheus-postgres-password" {
+  length = 12
+}
+
 resource "google_sql_user" "coder" {
   project  = var.project_id
   instance = google_sql_database_instance.db.id
@@ -61,3 +69,7 @@ resource "google_sql_user" "prometheus" {
   # required for postgres, otherwise user fails to delete
   deletion_policy = "ABANDON"
 }
+
+locals {
+  coder_db_url = "postgres://${google_sql_user.coder.name}:${urlencode(random_password.coder-postgres-password.result)}@${google_sql_database_instance.db.private_ip_address}/${google_sql_database.coder.name}?sslmode=disable"
+}

scaletest/terraform/infra/gcp_project.tf

@@ -1,8 +1,3 @@
-provider "google" {
-  region  = var.region
-  project = var.project_id
-}
-
 locals {
   project_apis = [
     "cloudtrace",

scaletest/terraform/infra/main.tf

@@ -0,0 +1,20 @@
+terraform {
+  required_providers {
+    google = {
+      source  = "hashicorp/google"
+      version = "~> 4.36"
+    }
+
+    random = {
+      source  = "hashicorp/random"
+      version = "~> 3.5"
+    }
+  }
+
+  required_version = "~> 1.5.0"
+}
+
+provider "google" {
+  region  = var.region
+  project = var.project_id
+}

scaletest/terraform/infra/outputs.tf

@@ -0,0 +1,73 @@
+output "coder_db_url" {
+  description = "URL of the database for Coder."
+  value       = local.coder_db_url
+  sensitive   = true
+}
+
+output "coder_address" {
+  description = "IP address to use for the Coder service."
+  value       = google_compute_address.coder.address
+}
+
+output "kubernetes_kubeconfig_path" {
+  description = "Kubeconfig path."
+  value       = local.cluster_kubeconfig_path
+}
+
+output "kubernetes_nodepool_coder" {
+  description = "Name of the nodepool on which to run Coder."
+  value       = google_container_node_pool.coder.name
+}
+
+output "kubernetes_nodepool_misc" {
+  description = "Name of the nodepool on which to run everything else."
+  value       = google_container_node_pool.misc.name
+}
+
+output "kubernetes_nodepool_workspaces" {
+  description = "Name of the nodepool on which to run workspaces."
+  value       = google_container_node_pool.workspaces.name
+}
+
+output "prometheus_external_label_cluster" {
+  description = "Value for the Prometheus external label named cluster."
+  value       = google_container_cluster.primary.name
+}
+
+output "prometheus_postgres_dbname" {
+  description = "Name of the database for Prometheus to monitor."
+  value       = google_sql_database.coder.name
+}
+
+output "prometheus_postgres_host" {
+  description = "Hostname of the database for Prometheus to connect to."
+  value       = google_sql_database_instance.db.private_ip_address
+}
+
+output "prometheus_postgres_password" {
+  description = "Postgres password for Prometheus."
+  value       = random_password.prometheus-postgres-password.result
+  sensitive   = true
+}
+
+output "prometheus_postgres_user" {
+  description = "Postgres username for Prometheus."
+  value       = google_sql_user.prometheus.name
+}
+
+resource "local_file" "outputs" {
+  filename = "${path.module}/../../.coderv2/infra_outputs.tfvars"
+  content  = <<EOF
+  coder_db_url = "${local.coder_db_url}"
+  coder_address = "${google_compute_address.coder.address}"
+  kubernetes_kubeconfig_path = "${local.cluster_kubeconfig_path}"
+  kubernetes_nodepool_coder = "${google_container_node_pool.coder.name}"
+  kubernetes_nodepool_misc = "${google_container_node_pool.misc.name}"
+  kubernetes_nodepool_workspaces = "${google_container_node_pool.workspaces.name}"
+  prometheus_external_label_cluster = "${google_container_cluster.primary.name}"
+  prometheus_postgres_dbname = "${google_sql_database.coder.name}"
+  prometheus_postgres_host = "${google_sql_database_instance.db.private_ip_address}"
+  prometheus_postgres_password = "${random_password.prometheus-postgres-password.result}"
+  prometheus_postgres_user = "${google_sql_user.prometheus.name}"
+EOF
+}

scaletest/terraform/infra/vars.tf

@@ -0,0 +1,102 @@
+variable "state" {
+  description = "The state of the cluster. Valid values are 'started', and 'stopped'."
+  validation {
+    condition     = contains(["started", "stopped"], var.state)
+    error_message = "value must be one of 'started' or 'stopped'"
+  }
+  default = "started"
+}
+
+variable "project_id" {
+  description = "The project in which to provision resources"
+}
+
+variable "name" {
+  description = "Adds a prefix to resources."
+}
+
+variable "region" {
+  description = "GCP region in which to provision resources."
+  default     = "us-east1"
+}
+
+variable "zone" {
+  description = "GCP zone in which to provision resources."
+  default     = "us-east1-c"
+}
+
+variable "k8s_version" {
+  description = "Kubernetes version to provision."
+  default     = "1.24"
+}
+
+variable "node_disk_size_gb" {
+  description = "Size of the root disk for cluster nodes."
+  default     = 100
+}
+
+variable "node_image_type" {
+  description = "Image type to use for cluster nodes."
+  default     = "cos_containerd"
+}
+
+// Preemptible nodes are way cheaper, but can be pulled out
+// from under you at any time. Caveat emptor.
+variable "node_preemptible" {
+  description = "Use preemptible nodes."
+  default     = false
+}
+
+// We create three nodepools:
+// - One for the Coder control plane
+// - One for workspaces
+// - One for everything else (for example, load generation)
+
+// These variables control the node pool dedicated to Coder.
+variable "nodepool_machine_type_coder" {
+  description = "Machine type to use for Coder control plane nodepool."
+  default     = "t2d-standard-4"
+}
+
+variable "nodepool_size_coder" {
+  description = "Number of cluster nodes for the Coder control plane nodepool."
+  default     = 1
+}
+
+// These variables control the node pool dedicated to workspaces.
+variable "nodepool_machine_type_workspaces" {
+  description = "Machine type to use for the workspaces nodepool."
+  default     = "t2d-standard-4"
+}
+
+variable "nodepool_size_workspaces" {
+  description = "Number of cluster nodes for the workspaces nodepool."
+  default     = 1
+}
+
+// These variables control the node pool for everything else.
+variable "nodepool_machine_type_misc" {
+  description = "Machine type to use for the misc nodepool."
+  default     = "t2d-standard-4"
+}
+
+variable "nodepool_size_misc" {
+  description = "Number of cluster nodes for the misc nodepool."
+  default     = 1
+}
+
+// These variables control the size of the database to be used by Coder.
+variable "cloudsql_version" {
+  description = "CloudSQL version to provision"
+  default     = "POSTGRES_14"
+}
+
+variable "cloudsql_tier" {
+  description = "CloudSQL database tier."
+  default     = "db-f1-micro"
+}
+
+variable "cloudsql_max_connections" {
+  description = "CloudSQL database max_connections"
+  default     = 500
+}

scaletest/terraform/k8s/coder.tf

@@ -7,32 +7,14 @@ locals {
   coder_namespace    = "coder-${var.name}"
   coder_admin_email  = "admin@coder.com"
   coder_admin_user   = "coder"
-  coder_address      = google_compute_address.coder.address
-  coder_url          = "http://${google_compute_address.coder.address}"
-}
-
-provider "kubernetes" {
-  host                   = "https://${google_container_cluster.primary.endpoint}"
-  cluster_ca_certificate = base64decode(google_container_cluster.primary.master_auth.0.cluster_ca_certificate)
-  token                  = data.google_client_config.default.access_token
-}
-
-provider "helm" {
-  kubernetes {
-    host                   = "https://${google_container_cluster.primary.endpoint}"
-    cluster_ca_certificate = base64decode(google_container_cluster.primary.master_auth.0.cluster_ca_certificate)
-    token                  = data.google_client_config.default.access_token
-  }
+  coder_access_url   = "http://${var.coder_address}"
 }
 
 resource "null_resource" "coder_namespace" {
   triggers = {
     namespace       = local.coder_namespace
-    kubeconfig_path = local.cluster_kubeconfig_path
+    kubeconfig_path = var.kubernetes_kubeconfig_path
   }
-  depends_on = [
-    google_container_node_pool.coder
-  ]
   provisioner "local-exec" {
     when    = create
     command = <<EOF
@@ -45,14 +27,6 @@ resource "null_resource" "coder_namespace" {
   }
 }
 
-resource "random_password" "coder-postgres-password" {
-  length = 12
-}
-
-resource "random_password" "prometheus-postgres-password" {
-  length = 12
-}
-
 resource "kubernetes_secret" "coder-db" {
   type = "Opaque"
   metadata {
@@ -61,7 +35,7 @@ resource "kubernetes_secret" "coder-db" {
   }
   depends_on = [null_resource.coder_namespace]
   data = {
-    url = "postgres://${google_sql_user.coder.name}:${urlencode(random_password.coder-postgres-password.result)}@${google_sql_database_instance.db.private_ip_address}/${google_sql_database.coder.name}?sslmode=disable"
+    url = var.coder_db_url
   }
 }
 
@@ -72,7 +46,6 @@ resource "helm_release" "coder-chart" {
   version    = var.coder_chart_version
   namespace  = local.coder_namespace
   depends_on = [
-    google_container_node_pool.coder,
     null_resource.coder_namespace
   ]
   values = [<<EOF
@@ -84,7 +57,7 @@ coder:
         - matchExpressions:
           - key: "cloud.google.com/gke-nodepool"
             operator: "In"
-            values: ["${google_container_node_pool.coder.name}"]
+            values: ["${var.kubernetes_nodepool_coder}"]
     podAntiAffinity:
       preferredDuringSchedulingIgnoredDuringExecution:
       - weight: 1
@@ -97,7 +70,7 @@ coder:
               values:   ["${local.coder_release_name}"]
   env:
     - name: "CODER_ACCESS_URL"
-      value: "${local.coder_url}"
+      value: "${local.coder_access_url}"
     - name: "CODER_CACHE_DIRECTORY"
       value: "/tmp/coder"
     - name: "CODER_ENABLE_TELEMETRY"
@@ -144,7 +117,7 @@ coder:
   service:
     enable: true
     sessionAffinity: None
-    loadBalancerIP: "${local.coder_address}"
+    loadBalancerIP: "${var.coder_address}"
   volumeMounts:
   - mountPath: "/tmp"
     name: cache
@@ -234,7 +207,7 @@ resource "local_file" "kubernetes_template" {
                 match_expressions {
                   key = "cloud.google.com/gke-nodepool"
                   operator = "In"
-                  values = ["${google_container_node_pool.workspaces.name}"]
+                  values = ["${var.kubernetes_nodepool_workspaces}"]
                 }
               }
             }
@@ -284,7 +257,7 @@ spec:
               - key: cloud.google.com/gke-nodepool
                 operator: In
                 values:
-                - ${google_container_node_pool.coder.name}
+                - ${var.kubernetes_nodepool_coder}
         podAntiAffinity:
           preferredDuringSchedulingIgnoredDuringExecution:
           - podAffinityTerm:
@@ -307,7 +280,7 @@ spec:
         - name: CODER_PROMETHEUS_ADDRESS
           value: 0.0.0.0:2112
         - name: CODER_ACCESS_URL
-          value: ${local.coder_url}
+          value: ${local.coder_access_url}
         - name: CODER_CACHE_DIRECTORY
           value: /tmp/coder
         - name: CODER_ENABLE_TELEMETRY
@@ -395,9 +368,9 @@ spec:
 }
 
 resource "null_resource" "provisionerd_deployment_apply" {
-  depends_on = [helm_release.coder-chart, local_file.provisionerd_deployment, null_resource.cluster_kubeconfig]
+  depends_on = [helm_release.coder-chart, local_file.provisionerd_deployment]
   triggers = {
-    kubeconfig_path = local.cluster_kubeconfig_path
+    kubeconfig_path = var.kubernetes_kubeconfig_path
     manifest_path   = local_file.provisionerd_deployment.filename
   }
   provisioner "local-exec" {
@@ -408,11 +381,11 @@ resource "null_resource" "provisionerd_deployment_apply" {
 }
 
 resource "local_file" "output_vars" {
-  filename = "${path.module}/../.coderv2/url"
-  content  = local.coder_url
+  filename = "${path.module}/../../.coderv2/url"
+  content  = local.coder_access_url
 }
 
 output "coder_url" {
   description = "URL of the Coder deployment"
-  value       = local.coder_url
+  value       = local.coder_access_url
 }

scaletest/terraform/k8s/main.tf

@@ -1,10 +1,5 @@
 terraform {
   required_providers {
-    google = {
-      source  = "hashicorp/google"
-      version = "~> 4.36"
-    }
-
     kubernetes = {
       source  = "hashicorp/kubernetes"
       version = "~> 2.20"
@@ -24,12 +19,17 @@ terraform {
       source  = "hashicorp/tls"
       version = "~> 4.0"
     }
-
-    docker = {
-      source  = "kreuzwerker/docker"
-      version = "~> 3.0"
-    }
   }
 
   required_version = "~> 1.5.0"
 }
+
+provider "kubernetes" {
+  config_path = var.kubernetes_kubeconfig_path
+}
+
+provider "helm" {
+  kubernetes {
+    config_path = var.kubernetes_kubeconfig_path
+  }
+}

scaletest/terraform/k8s/prometheus.tf

@@ -13,11 +13,9 @@ locals {
 resource "null_resource" "prometheus_namespace" {
   triggers = {
     namespace       = local.prometheus_namespace
-    kubeconfig_path = local.cluster_kubeconfig_path
+    kubeconfig_path = var.kubernetes_kubeconfig_path
   }
-  depends_on = [
-    google_container_node_pool.misc
-  ]
+  depends_on = []
   provisioner "local-exec" {
     when    = create
     command = <<EOF
@@ -64,7 +62,7 @@ blackboxExporter:
         - matchExpressions:
           - key: "cloud.google.com/gke-nodepool"
             operator: "In"
-            values: ["${google_container_node_pool.misc.name}"]
+            values: ["${var.kubernetes_nodepool_misc}"]
 operator:
   affinity:
     nodeAffinity:
@@ -73,7 +71,7 @@ operator:
         - matchExpressions:
           - key: "cloud.google.com/gke-nodepool"
             operator: "In"
-            values: ["${google_container_node_pool.misc.name}"]
+            values: ["${var.kubernetes_nodepool_misc}"]
 prometheus:
   affinity:
     nodeAffinity:
@@ -82,9 +80,9 @@ prometheus:
         - matchExpressions:
           - key: "cloud.google.com/gke-nodepool"
             operator: "In"
-            values: ["${google_container_node_pool.misc.name}"]
+            values: ["${var.kubernetes_nodepool_misc}"]
   externalLabels:
-    cluster: "${google_container_cluster.primary.name}"
+    cluster: "${var.prometheus_external_label_cluster}"
   persistence:
     enabled: true
     storageClass: standard
@@ -119,8 +117,8 @@ resource "kubernetes_secret" "prometheus-postgres-password" {
   }
   depends_on = [null_resource.prometheus_namespace]
   data = {
-    username = google_sql_user.prometheus.name
-    password = google_sql_user.prometheus.password
+    username = var.prometheus_postgres_user
+    password = var.prometheus_postgres_password
   }
 }
 
@@ -139,12 +137,12 @@ affinity:
     - matchExpressions:
       - key: "cloud.google.com/gke-nodepool"
         operator: "In"
-        values: ["${google_container_node_pool.misc.name}"]
+        values: ["${var.kubernetes_nodepool_misc}"]
 config:
   datasource:
-    host: "${google_sql_database_instance.db.private_ip_address}"
-    user: "${google_sql_user.prometheus.name}"
-    database: "${google_sql_database.coder.name}"
+    host: "${var.prometheus_postgres_host}"
+    user: "${var.prometheus_postgres_user}"
+    database: "${var.prometheus_postgres_dbname}"
     passwordSecret:
       name: "${kubernetes_secret.prometheus-postgres-password.metadata.0.name}"
       key: password
@@ -182,7 +180,7 @@ resource "null_resource" "coder-monitoring-manifest_apply" {
   provisioner "local-exec" {
     working_dir = "${abspath(path.module)}/../.coderv2"
     command     = <<EOF
-KUBECONFIG=${local.cluster_kubeconfig_path} kubectl apply -f ${abspath(local_file.coder-monitoring-manifest.filename)}
+KUBECONFIG=${var.kubernetes_kubeconfig_path} kubectl apply -f ${abspath(local_file.coder-monitoring-manifest.filename)}
     EOF
   }
   depends_on = [helm_release.prometheus-chart]

scaletest/terraform/k8s/vars.tf

@@ -7,98 +7,24 @@ variable "state" {
   default = "started"
 }
 
-variable "project_id" {
-  description = "The project in which to provision resources"
-}
-
 variable "name" {
   description = "Adds a prefix to resources."
 }
 
-variable "region" {
-  description = "GCP region in which to provision resources."
-  default     = "us-east1"
+variable "kubernetes_kubeconfig_path" {
+  description = "Path to kubeconfig to use to provision resources."
 }
 
-variable "zone" {
-  description = "GCP zone in which to provision resources."
-  default     = "us-east1-c"
+variable "kubernetes_nodepool_coder" {
+  description = "Name of the nodepool on which to run Coder."
 }
 
-variable "k8s_version" {
-  description = "Kubernetes vversion to provision."
-  default     = "1.24"
+variable "kubernetes_nodepool_workspaces" {
+  description = "Name of the nodepool on which to run workspaces."
 }
 
-variable "node_disk_size_gb" {
-  description = "Size of the root disk for cluster nodes."
-  default     = 100
-}
-
-variable "node_image_type" {
-  description = "Image type to use for cluster nodes."
-  default     = "cos_containerd"
-}
-
-// Preemptible nodes are way cheaper, but can be pulled out
-// from under you at any time. Caveat emptor.
-variable "node_preemptible" {
-  description = "Use preemptible nodes."
-  default     = false
-}
-
-// We create three nodepools:
-// - One for the Coder control plane
-// - One for workspaces
-// - One for everything else (for example, load generation)
-
-// These variables control the node pool dedicated to Coder.
-variable "nodepool_machine_type_coder" {
-  description = "Machine type to use for Coder control plane nodepool."
-  default     = "t2d-standard-4"
-}
-
-variable "nodepool_size_coder" {
-  description = "Number of cluster nodes for the Coder control plane nodepool."
-  default     = 1
-}
-
-// These variables control the node pool dedicated to workspaces.
-variable "nodepool_machine_type_workspaces" {
-  description = "Machine type to use for the workspaces nodepool."
-  default     = "t2d-standard-4"
-}
-
-variable "nodepool_size_workspaces" {
-  description = "Number of cluster nodes for the workspaces nodepool."
-  default     = 1
-}
-
-// These variables control the node pool for everything else.
-variable "nodepool_machine_type_misc" {
-  description = "Machine type to use for the misc nodepool."
-  default     = "t2d-standard-4"
-}
-
-variable "nodepool_size_misc" {
-  description = "Number of cluster nodes for the misc nodepool."
-  default     = 1
-}
-
-// These variables control the size of the database to be used by Coder.
-variable "cloudsql_version" {
-  description = "CloudSQL version to provision"
-  default     = "POSTGRES_14"
-}
-
-variable "cloudsql_tier" {
-  description = "CloudSQL database tier."
-  default     = "db-f1-micro"
-}
-
-variable "cloudsql_max_connections" {
-  description = "CloudSQL database max_connections"
-  default     = 500
+variable "kubernetes_nodepool_misc" {
+  description = "Name of the nodepool on which to run everything else."
 }
 
 // These variables control the Coder deployment.
@@ -107,6 +33,15 @@ variable "coder_replicas" {
   default     = 1
 }
 
+variable "coder_address" {
+  description = "IP address to use for Coder service."
+}
+
+variable "coder_db_url" {
+  description = "URL of the database for Coder to use."
+  sensitive   = true
+}
+
 // Ensure that requests allow for at least two replicas to be scheduled
 // on a single node temporarily, otherwise deployments may fail due to
 // lack of resources.
@@ -208,6 +143,27 @@ variable "workspace_mem_limit" {
 }
 
 // These variables control the Prometheus deployment.
+variable "prometheus_external_label_cluster" {
+  description = "Value for the Prometheus external label named cluster."
+}
+
+variable "prometheus_postgres_dbname" {
+  description = "Database for Postgres to monitor."
+}
+
+variable "prometheus_postgres_host" {
+  description = "Database hostname for Prometheus."
+}
+
+variable "prometheus_postgres_password" {
+  description = "Postgres password for Prometheus."
+  sensitive   = true
+}
+
+variable "prometheus_postgres_user" {
+  description = "Postgres username for Prometheus."
+}
+
 variable "prometheus_remote_write_user" {
   description = "Username for Prometheus remote write."
   default     = ""
@@ -216,6 +172,7 @@ variable "prometheus_remote_write_user" {
 variable "prometheus_remote_write_password" {
   description = "Password for Prometheus remote write."
   default     = ""
+  sensitive   = true
 }
 
 variable "prometheus_remote_write_url" {