Auto import kubernetes template in Helm charts (#3550)

This commit is contained in:
Dean Sheather 2022-08-26 05:32:35 +10:00 committed by GitHub
parent 94e96fa40b
commit 14a9576b77
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 423 additions and 41 deletions

1
.gitignore vendored
View File

@ -34,6 +34,7 @@ dist/
site/out/
*.tfstate
*.tfstate.backup
*.tfplan
*.lock.hcl
.terraform/

View File

@ -108,6 +108,7 @@ func Server(newAPI func(*coderd.Options) *coderd.API) *cobra.Command {
trace bool
secureAuthCookie bool
sshKeygenAlgorithmRaw string
autoImportTemplates []string
spooky bool
verbose bool
)
@ -284,6 +285,28 @@ func Server(newAPI func(*coderd.Options) *coderd.API) *cobra.Command {
URLs: []string{stunServer},
})
}
// Validate provided auto-import templates.
var (
validatedAutoImportTemplates = make([]coderd.AutoImportTemplate, len(autoImportTemplates))
seenValidatedAutoImportTemplates = make(map[coderd.AutoImportTemplate]struct{}, len(autoImportTemplates))
)
for i, autoImportTemplate := range autoImportTemplates {
var v coderd.AutoImportTemplate
switch autoImportTemplate {
case "kubernetes":
v = coderd.AutoImportTemplateKubernetes
default:
return xerrors.Errorf("auto import template %q is not supported", autoImportTemplate)
}
if _, ok := seenValidatedAutoImportTemplates[v]; ok {
return xerrors.Errorf("auto import template %q is specified more than once", v)
}
seenValidatedAutoImportTemplates[v] = struct{}{}
validatedAutoImportTemplates[i] = v
}
options := &coderd.Options{
AccessURL: accessURLParsed,
ICEServers: iceServers,
@ -297,6 +320,7 @@ func Server(newAPI func(*coderd.Options) *coderd.API) *cobra.Command {
TURNServer: turnServer,
TracerProvider: tracerProvider,
Telemetry: telemetry.NewNoop(),
AutoImportTemplates: validatedAutoImportTemplates,
}
if oauth2GithubClientSecret != "" {
@ -744,6 +768,7 @@ func Server(newAPI func(*coderd.Options) *coderd.API) *cobra.Command {
cliflag.BoolVarP(root.Flags(), &secureAuthCookie, "secure-auth-cookie", "", "CODER_SECURE_AUTH_COOKIE", false, "Specifies if the 'Secure' property is set on browser session cookies")
cliflag.StringVarP(root.Flags(), &sshKeygenAlgorithmRaw, "ssh-keygen-algorithm", "", "CODER_SSH_KEYGEN_ALGORITHM", "ed25519", "Specifies the algorithm to use for generating ssh keys. "+
`Accepted values are "ed25519", "ecdsa", or "rsa4096"`)
cliflag.StringArrayVarP(root.Flags(), &autoImportTemplates, "auto-import-template", "", "CODER_TEMPLATE_AUTOIMPORT", []string{}, "Which templates to auto-import. Available auto-importable templates are: kubernetes")
cliflag.BoolVarP(root.Flags(), &spooky, "spooky", "", "", false, "Specifies spookiness level")
cliflag.BoolVarP(root.Flags(), &verbose, "verbose", "v", "CODER_VERBOSE", false, "Enables verbose logging.")
_ = root.Flags().MarkHidden("spooky")

View File

@ -66,6 +66,7 @@ type Options struct {
Telemetry telemetry.Reporter
TURNServer *turnconn.Server
TracerProvider *sdktrace.TracerProvider
AutoImportTemplates []AutoImportTemplate
LicenseHandler http.Handler
}

View File

@ -68,6 +68,7 @@ type Options struct {
GoogleTokenValidator *idtoken.Validator
SSHKeygenAlgorithm gitsshkey.Algorithm
APIRateLimit int
AutoImportTemplates []coderd.AutoImportTemplate
AutobuildTicker <-chan time.Time
AutobuildStats chan<- executor.Stats
@ -210,6 +211,7 @@ func newWithAPI(t *testing.T, options *Options) (*codersdk.Client, io.Closer, *c
APIRateLimit: options.APIRateLimit,
Authorizer: options.Authorizer,
Telemetry: telemetry.NewNoop(),
AutoImportTemplates: options.AutoImportTemplates,
})
t.Cleanup(func() {
_ = coderAPI.Close()

View File

@ -2,7 +2,9 @@ package coderd
import (
"context"
"crypto/sha256"
"database/sql"
"encoding/hex"
"errors"
"fmt"
"net/http"
@ -10,6 +12,7 @@ import (
"github.com/go-chi/chi/v5"
"github.com/google/uuid"
"github.com/moby/moby/pkg/namesgenerator"
"golang.org/x/xerrors"
"github.com/coder/coder/coderd/database"
@ -26,6 +29,14 @@ var (
minAutostartIntervalDefault = time.Hour
)
// Auto-importable templates. These can be auto-imported after the first user
// has been created.
type AutoImportTemplate string
const (
AutoImportTemplateKubernetes AutoImportTemplate = "kubernetes"
)
// Returns a single template.
func (api *API) template(rw http.ResponseWriter, r *http.Request) {
template := httpmw.TemplateParam(r)
@ -508,6 +519,146 @@ func (api *API) patchTemplateMeta(rw http.ResponseWriter, r *http.Request) {
httpapi.Write(rw, http.StatusOK, convertTemplate(updated, count, createdByNameMap[updated.ID.String()]))
}
type autoImportTemplateOpts struct {
name string
archive []byte
params map[string]string
userID uuid.UUID
orgID uuid.UUID
}
func (api *API) autoImportTemplate(ctx context.Context, opts autoImportTemplateOpts) (database.Template, error) {
var template database.Template
err := api.Database.InTx(func(s database.Store) error {
// Insert the archive into the files table.
var (
hash = sha256.Sum256(opts.archive)
now = database.Now()
)
file, err := s.InsertFile(ctx, database.InsertFileParams{
Hash: hex.EncodeToString(hash[:]),
CreatedAt: now,
CreatedBy: opts.userID,
Mimetype: "application/x-tar",
Data: opts.archive,
})
if err != nil {
return xerrors.Errorf("insert auto-imported template archive into files table: %w", err)
}
jobID := uuid.New()
// Insert parameters
for key, value := range opts.params {
_, err = s.InsertParameterValue(ctx, database.InsertParameterValueParams{
ID: uuid.New(),
Name: key,
CreatedAt: now,
UpdatedAt: now,
Scope: database.ParameterScopeImportJob,
ScopeID: jobID,
SourceScheme: database.ParameterSourceSchemeData,
SourceValue: value,
DestinationScheme: database.ParameterDestinationSchemeProvisionerVariable,
})
if err != nil {
return xerrors.Errorf("insert job-scoped parameter %q with value %q: %w", key, value, err)
}
}
// Create provisioner job
job, err := s.InsertProvisionerJob(ctx, database.InsertProvisionerJobParams{
ID: jobID,
CreatedAt: now,
UpdatedAt: now,
OrganizationID: opts.orgID,
InitiatorID: opts.userID,
Provisioner: database.ProvisionerTypeTerraform,
StorageMethod: database.ProvisionerStorageMethodFile,
StorageSource: file.Hash,
Type: database.ProvisionerJobTypeTemplateVersionImport,
Input: []byte{'{', '}'},
})
if err != nil {
return xerrors.Errorf("insert provisioner job: %w", err)
}
// Create template version
templateVersion, err := s.InsertTemplateVersion(ctx, database.InsertTemplateVersionParams{
ID: uuid.New(),
TemplateID: uuid.NullUUID{
UUID: uuid.Nil,
Valid: false,
},
OrganizationID: opts.orgID,
CreatedAt: now,
UpdatedAt: now,
Name: namesgenerator.GetRandomName(1),
Readme: "",
JobID: job.ID,
CreatedBy: uuid.NullUUID{
UUID: opts.userID,
Valid: true,
},
})
if err != nil {
return xerrors.Errorf("insert template version: %w", err)
}
// Create template
template, err = s.InsertTemplate(ctx, database.InsertTemplateParams{
ID: uuid.New(),
CreatedAt: now,
UpdatedAt: now,
OrganizationID: opts.orgID,
Name: opts.name,
Provisioner: job.Provisioner,
ActiveVersionID: templateVersion.ID,
Description: "This template was auto-imported by Coder.",
MaxTtl: int64(maxTTLDefault),
MinAutostartInterval: int64(minAutostartIntervalDefault),
CreatedBy: opts.userID,
})
if err != nil {
return xerrors.Errorf("insert template: %w", err)
}
// Update template version with template ID
err = s.UpdateTemplateVersionByID(ctx, database.UpdateTemplateVersionByIDParams{
ID: templateVersion.ID,
TemplateID: uuid.NullUUID{
UUID: template.ID,
Valid: true,
},
})
if err != nil {
return xerrors.Errorf("update template version to set template ID: %s", err)
}
// Insert parameters at the template scope
for key, value := range opts.params {
_, err = s.InsertParameterValue(ctx, database.InsertParameterValueParams{
ID: uuid.New(),
Name: key,
CreatedAt: now,
UpdatedAt: now,
Scope: database.ParameterScopeTemplate,
ScopeID: template.ID,
SourceScheme: database.ParameterSourceSchemeData,
SourceValue: value,
DestinationScheme: database.ParameterDestinationSchemeProvisionerVariable,
})
if err != nil {
return xerrors.Errorf("insert template-scoped parameter %q with value %q: %w", key, value, err)
}
}
return nil
})
return template, err
}
func getCreatedByNamesByTemplateIDs(ctx context.Context, db database.Store, templates []database.Template) (map[string]string, error) {
creators := make(map[string]string, len(templates))
for _, template := range templates {

View File

@ -1,6 +1,7 @@
package coderd
import (
"bytes"
"context"
"crypto/sha256"
"database/sql"
@ -9,6 +10,7 @@ import (
"net"
"net/http"
"net/url"
"os"
"strings"
"time"
@ -18,6 +20,8 @@ import (
"github.com/tabbed/pqtype"
"golang.org/x/xerrors"
"cdr.dev/slog"
"github.com/coder/coder/coderd/database"
"github.com/coder/coder/coderd/gitsshkey"
"github.com/coder/coder/coderd/httpapi"
@ -27,6 +31,7 @@ import (
"github.com/coder/coder/coderd/userpassword"
"github.com/coder/coder/codersdk"
"github.com/coder/coder/cryptorand"
"github.com/coder/coder/examples"
)
// Returns whether the initial user has been created or not.
@ -82,6 +87,8 @@ func (api *API) postFirstUser(rw http.ResponseWriter, r *http.Request) {
Email: createUser.Email,
Username: createUser.Username,
Password: createUser.Password,
// Create an org for the first user.
OrganizationID: uuid.Nil,
},
LoginType: database.LoginTypePassword,
})
@ -116,6 +123,60 @@ func (api *API) postFirstUser(rw http.ResponseWriter, r *http.Request) {
return
}
// Auto-import any designated templates into the new organization.
for _, template := range api.AutoImportTemplates {
archive, err := examples.Archive(string(template))
if err != nil {
httpapi.Write(rw, http.StatusInternalServerError, codersdk.Response{
Message: "Internal error importing template.",
Detail: xerrors.Errorf("load template archive for %q: %w", template, err).Error(),
})
return
}
// Determine which parameter values to use.
parameters := map[string]string{}
switch template {
case AutoImportTemplateKubernetes:
// Determine the current namespace we're in.
const namespaceFile = "/var/run/secrets/kubernetes.io/serviceaccount/namespace"
namespace, err := os.ReadFile(namespaceFile)
if err != nil {
parameters["use_kubeconfig"] = "true" // use ~/.config/kubeconfig
parameters["namespace"] = "coder-workspaces"
} else {
parameters["use_kubeconfig"] = "false" // use SA auth
parameters["namespace"] = string(bytes.TrimSpace(namespace))
}
default:
httpapi.Write(rw, http.StatusInternalServerError, codersdk.Response{
Message: "Internal error importing template.",
Detail: fmt.Sprintf("cannot auto-import %q template", template),
})
return
}
tpl, err := api.autoImportTemplate(r.Context(), autoImportTemplateOpts{
name: string(template),
archive: archive,
params: parameters,
userID: user.ID,
orgID: organizationID,
})
if err != nil {
api.Logger.Warn(r.Context(), "failed to auto-import template", slog.F("template", template), slog.F("parameters", parameters), slog.Error(err))
httpapi.Write(rw, http.StatusInternalServerError, codersdk.Response{
Message: "Internal error importing template.",
Detail: xerrors.Errorf("failed to import template %q: %w", template, err).Error(),
})
return
}
api.Logger.Info(r.Context(), "auto-imported template", slog.F("id", tpl.ID), slog.F("template", template), slog.F("parameters", parameters))
}
httpapi.Write(rw, http.StatusCreated, codersdk.CreateFirstUserResponse{
UserID: user.ID,
OrganizationID: organizationID,

View File

@ -13,6 +13,7 @@ import (
"github.com/stretchr/testify/require"
"golang.org/x/sync/errgroup"
"github.com/coder/coder/coderd"
"github.com/coder/coder/coderd/coderdtest"
"github.com/coder/coder/coderd/rbac"
"github.com/coder/coder/codersdk"
@ -56,6 +57,77 @@ func TestFirstUser(t *testing.T) {
client := coderdtest.New(t, nil)
_ = coderdtest.CreateFirstUser(t, client)
})
t.Run("AutoImportsTemplates", func(t *testing.T) {
t.Parallel()
// All available auto import templates should be added to this list, and
// also added to the switch statement below.
autoImportTemplates := []coderd.AutoImportTemplate{
coderd.AutoImportTemplateKubernetes,
}
client := coderdtest.New(t, &coderdtest.Options{
AutoImportTemplates: autoImportTemplates,
})
u := coderdtest.CreateFirstUser(t, client)
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
defer cancel()
templates, err := client.TemplatesByOrganization(ctx, u.OrganizationID)
require.NoError(t, err, "list templates")
require.Len(t, templates, len(autoImportTemplates), "listed templates count does not match")
require.ElementsMatch(t, autoImportTemplates, []coderd.AutoImportTemplate{
coderd.AutoImportTemplate(templates[0].Name),
}, "template names don't match")
for _, template := range templates {
// Check template parameters.
templateParams, err := client.Parameters(ctx, codersdk.ParameterTemplate, template.ID)
require.NoErrorf(t, err, "get template parameters for %q", template.Name)
// Ensure all template parameters are present.
expectedParams := map[string]bool{}
switch template.Name {
case "kubernetes":
expectedParams["use_kubeconfig"] = false
expectedParams["namespace"] = false
default:
t.Fatalf("unexpected template name %q", template.Name)
}
for _, v := range templateParams {
if _, ok := expectedParams[v.Name]; !ok {
t.Fatalf("unexpected template parameter %q in template %q", v.Name, template.Name)
}
expectedParams[v.Name] = true
}
for k, v := range expectedParams {
if !v {
t.Fatalf("missing template parameter %q in template %q", k, template.Name)
}
}
// Ensure template version is legit
templateVersion, err := client.TemplateVersion(ctx, template.ActiveVersionID)
require.NoErrorf(t, err, "get template version for %q", template.Name)
// Compare job parameters to template parameters.
jobParams, err := client.Parameters(ctx, codersdk.ParameterImportJob, templateVersion.Job.ID)
require.NoErrorf(t, err, "get template import job parameters for %q", template.Name)
for _, v := range jobParams {
if _, ok := expectedParams[v.Name]; !ok {
t.Fatalf("unexpected job parameter %q for template %q", v.Name, template.Name)
}
// Change it back to false so we can reuse the map
expectedParams[v.Name] = false
}
for k, v := range expectedParams {
if v {
t.Fatalf("missing job parameter %q for template %q", k, template.Name)
}
}
}
})
}
func TestPostLogin(t *testing.T) {

View File

@ -195,6 +195,12 @@ You will also need to have a Kubernetes cluster running K8s 1.19+.
name: coder-db-url
key: url
# This env variable controls whether or not to auto-import the
# "kubernetes" template on first startup. This will not work unless
# coder.serviceAccount.workspacePerms is true.
- name: CODER_TEMPLATE_AUTOIMPORT
value: "kubernetes"
tls:
secretName: my-tls-secret-name
```

View File

@ -37,6 +37,7 @@ var ValidMethods = []string{"EdDSA"}
// key20220812 is the Coder license public key with id 2022-08-12 used to validate licenses signed
// by our signing infrastructure
//
//go:embed keys/2022-08-12
var key20220812 []byte
@ -134,12 +135,12 @@ func (a *licenseAPI) handler() http.Handler {
// postLicense adds a new Enterprise license to the cluster. We allow multiple different licenses
// in the cluster at one time for several reasons:
//
// 1. Upgrades --- if the license format changes from one version of Coder to the next, during a
// rolling update you will have different Coder servers that need different licenses to function.
// 2. Avoid abrupt feature breakage --- when an admin uploads a new license with different features
// we generally don't want the old features to immediately break without warning. With a grace
// period on the license, features will continue to work from the old license until its grace
// period, then the users will get a warning allowing them to gracefully stop using the feature.
// 1. Upgrades --- if the license format changes from one version of Coder to the next, during a
// rolling update you will have different Coder servers that need different licenses to function.
// 2. Avoid abrupt feature breakage --- when an admin uploads a new license with different features
// we generally don't want the old features to immediately break without warning. With a grace
// period on the license, features will continue to work from the old license until its grace
// period, then the users will get a warning allowing them to gracefully stop using the feature.
func (a *licenseAPI) postLicense(rw http.ResponseWriter, r *http.Request) {
if !a.auth.Authorize(r, rbac.ActionCreate, rbac.ResourceLicense) {
httpapi.Forbidden(rw)

View File

@ -1,14 +1,16 @@
---
name: Develop multiple services in Kubernetes
name: Develop in Kubernetes
description: Get started with Kubernetes development.
tags: [cloud, kubernetes]
---
# Getting started
This template creates a pod running the `codercom/enterprise-base:ubuntu` image.
## RBAC
The Coder provisioner requires permission to administer pods to use this template. The template
The Coder provisioner requires permission to administer pods to use this template. The template
creates workspaces in a single Kubernetes namespace, using the `workspaces_namespace` parameter set
while creating the template.
@ -20,15 +22,15 @@ kind: Role
metadata:
name: coder
rules:
- apiGroups: [""]
resources: ["pods"]
verbs: ["*"]
- apiGroups: [""]
resources: ["pods"]
verbs: ["*"]
```
## Authentication
This template can authenticate using in-cluster authentication, or using a kubeconfig local to the
Coder host. For additional authentication options, consult the [Kubernetes provider
Coder host. For additional authentication options, consult the [Kubernetes provider
documentation](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs).
### kubeconfig on Coder host
@ -46,8 +48,8 @@ you can use in-cluster authentication.
To use this authentication, set the parameter `use_kubeconfig` to false.
The Terraform provisioner will automatically use the service account associated with the pod to
authenticate to Kubernetes. Be sure to bind a [role with appropriate permission](#rbac) to the
service account. For example, assuming the Coder host runs in the same namespace as you intend
authenticate to Kubernetes. Be sure to bind a [role with appropriate permission](#rbac) to the
service account. For example, assuming the Coder host runs in the same namespace as you intend
to create workspaces:
```yaml

View File

@ -25,17 +25,21 @@ variable "use_kubeconfig" {
EOF
}
variable "coder_namespace" {
variable "namespace" {
type = string
sensitive = true
description = "The namespace to create workspaces in (must exist prior to creating workspaces)"
default = "coder-namespace"
default = "coder-workspaces"
}
variable "disk_size" {
type = number
description = "Disk size (__ GB)"
variable "home_disk_size" {
type = number
description = "How large would you like your home volume to be (in GB)?"
default = 10
validation {
condition = var.home_disk_size >= 1
error_message = "Value must be greater than or equal to 1."
}
}
provider "kubernetes" {
@ -46,8 +50,8 @@ provider "kubernetes" {
data "coder_workspace" "me" {}
resource "coder_agent" "main" {
os = "linux"
arch = "amd64"
os = "linux"
arch = "amd64"
startup_script = <<EOT
#!/bin/bash
@ -66,11 +70,26 @@ resource "coder_app" "code-server" {
relative_path = true
}
resource "kubernetes_persistent_volume_claim" "home" {
metadata {
name = "coder-${data.coder_workspace.me.owner}-${data.coder_workspace.me.name}-home"
namespace = var.namespace
}
spec {
access_modes = ["ReadWriteOnce"]
resources {
requests = {
storage = "${var.home_disk_size}Gi"
}
}
}
}
resource "kubernetes_pod" "main" {
count = data.coder_workspace.me.start_count
metadata {
name = "coder-${data.coder_workspace.me.owner}-${data.coder_workspace.me.name}"
namespace = var.coder_namespace
namespace = var.namespace
}
spec {
security_context {
@ -90,28 +109,16 @@ resource "kubernetes_pod" "main" {
}
volume_mount {
mount_path = "/home/coder"
name = "home-directory"
name = "home"
read_only = false
}
}
volume {
name = "home-directory"
persistent_volume_claim {
claim_name = kubernetes_persistent_volume_claim.home-directory.metadata.0.name
}
}
}
}
resource "kubernetes_persistent_volume_claim" "home-directory" {
metadata {
name = "home-coder-java-${data.coder_workspace.me.owner}-${data.coder_workspace.me.name}"
namespace = var.coder_namespace
}
spec {
access_modes = ["ReadWriteOnce"]
resources {
requests = {
storage = "${var.disk_size}Gi"
volume {
name = "home"
persistent_volume_claim {
claim_name = kubernetes_persistent_volume_claim.home.metadata.0.name
read_only = false
}
}
}

View File

@ -44,6 +44,12 @@ coder:
name: coder-db-url
key: url
# This env variable controls whether or not to auto-import the "kubernetes"
# template on first startup. This will not work unless
# coder.serviceAccount.workspacePerms is true.
- name: CODER_TEMPLATE_AUTOIMPORT
value: "kubernetes"
tls:
secretName: my-tls-secret-name
```

View File

@ -1,3 +1,10 @@
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: coder
---
apiVersion: apps/v1
kind: Deployment
metadata:
@ -17,6 +24,7 @@ spec:
labels:
{{- include "coder.selectorLabels" . | nindent 8 }}
spec:
serviceAccountName: coder
restartPolicy: Always
terminationGracePeriodSeconds: 60
containers:

27
helm/templates/rbac.yaml Normal file
View File

@ -0,0 +1,27 @@
{{- if .Values.coder.serviceAccount.workspacePerms }}
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: coder-workspace-perms
rules:
- apiGroups: [""]
resources: ["pods"]
verbs: ["*"]
- apiGroups: [""]
resources: ["persistentvolumeclaims"]
verbs: ["*"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: coder
subjects:
- kind: ServiceAccount
name: coder
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: coder-workspace-perms
{{- end }}

View File

@ -16,6 +16,18 @@ coder:
# https://kubernetes.io/docs/concepts/containers/images/#image-pull-policy
pullPolicy: IfNotPresent
# coder.serviceAccount -- Configuration for the automatically created service
# account. Creation of the service account cannot be disabled.
serviceAccount:
# coder.serviceAccount.workspacePerms -- Whether or not to grant the coder
# service account permissions to manage workspaces. This includes
# permission to manage pods and persistent volume claims in the deployment
# namespace.
#
# It is recommended to keep this on if you are using Kubernetes templates
# within Coder.
workspacePerms: true
# coder.env -- The environment variables to set for Coder. These can be used
# to configure all aspects of `coder server`. Please see `coder server --help`
# for information about what environment variables can be set.