diff --git a/coderd/provisionerdserver/acquirer_test.go b/coderd/provisionerdserver/acquirer_test.go index 088fb68a5a..6d213f082d 100644 --- a/coderd/provisionerdserver/acquirer_test.go +++ b/coderd/provisionerdserver/acquirer_test.go @@ -4,6 +4,8 @@ import ( "context" "database/sql" "encoding/json" + "fmt" + "strings" "sync" "testing" "time" @@ -54,7 +56,7 @@ func TestAcquirer_Single(t *testing.T) { workerID := uuid.New() pt := []database.ProvisionerType{database.ProvisionerTypeEcho} tags := provisionerdserver.Tags{ - "foo": "bar", + "environment": "on-prem", } acquiree := newTestAcquiree(t, workerID, pt, tags) jobID := uuid.New() @@ -82,7 +84,7 @@ func TestAcquirer_MultipleSameDomain(t *testing.T) { workerIDs := make(map[uuid.UUID]bool) pt := []database.ProvisionerType{database.ProvisionerTypeEcho} tags := provisionerdserver.Tags{ - "foo": "bar", + "environment": "on-prem", } for i := 0; i < 10; i++ { wID := uuid.New() @@ -125,7 +127,7 @@ func TestAcquirer_WaitsOnNoJobs(t *testing.T) { workerID := uuid.New() pt := []database.ProvisionerType{database.ProvisionerTypeEcho} tags := provisionerdserver.Tags{ - "foo": "bar", + "environment": "on-prem", } acquiree := newTestAcquiree(t, workerID, pt, tags) jobID := uuid.New() @@ -147,10 +149,10 @@ func TestAcquirer_WaitsOnNoJobs(t *testing.T) { "strong": "bad", }) postJob(t, ps, database.ProvisionerTypeEcho, provisionerdserver.Tags{ - "foo": "fighters", + "environment": "fighters", }) postJob(t, ps, database.ProvisionerTypeTerraform, provisionerdserver.Tags{ - "foo": "bar", + "environment": "on-prem", }) acquiree.requireBlocked() @@ -176,7 +178,7 @@ func TestAcquirer_RetriesPending(t *testing.T) { workerID := uuid.New() pt := []database.ProvisionerType{database.ProvisionerTypeEcho} tags := provisionerdserver.Tags{ - "foo": "bar", + "environment": "on-prem", } acquiree := newTestAcquiree(t, workerID, pt, tags) jobID := uuid.New() @@ -268,7 +270,7 @@ func TestAcquirer_BackupPoll(t *testing.T) { workerID := uuid.New() pt := []database.ProvisionerType{database.ProvisionerTypeEcho} tags := provisionerdserver.Tags{ - "foo": "bar", + "environment": "on-prem", } acquiree := newTestAcquiree(t, workerID, pt, tags) jobID := uuid.New() @@ -294,7 +296,7 @@ func TestAcquirer_UnblockOnCancel(t *testing.T) { pt := []database.ProvisionerType{database.ProvisionerTypeEcho} worker0 := uuid.New() tags := provisionerdserver.Tags{ - "foo": "bar", + "environment": "on-prem", } acquiree0 := newTestAcquiree(t, worker0, pt, tags) worker1 := uuid.New() @@ -324,10 +326,7 @@ func TestAcquirer_MatchTags(t *testing.T) { t.Skip("skipping this test due to -short") } - someID := uuid.NewString() - someOtherID := uuid.NewString() - - for _, tt := range []struct { + testCases := []struct { name string provisionerJobTags map[string]string acquireJobTags map[string]string @@ -339,72 +338,126 @@ func TestAcquirer_MatchTags(t *testing.T) { acquireJobTags: map[string]string{"scope": "organization", "owner": ""}, expectAcquire: true, }, + { + name: "tagged provisioner and tagged job", + provisionerJobTags: map[string]string{"scope": "organization", "owner": "", "environment": "on-prem"}, + acquireJobTags: map[string]string{"scope": "organization", "owner": "", "environment": "on-prem"}, + expectAcquire: true, + }, + { + name: "double-tagged provisioner and tagged job", + provisionerJobTags: map[string]string{"scope": "organization", "owner": "", "environment": "on-prem"}, + acquireJobTags: map[string]string{"scope": "organization", "owner": "", "environment": "on-prem", "datacenter": "chicago"}, + expectAcquire: true, + }, + { + name: "double-tagged provisioner and double-tagged job", + provisionerJobTags: map[string]string{"scope": "organization", "owner": "", "environment": "on-prem", "datacenter": "chicago"}, + acquireJobTags: map[string]string{"scope": "organization", "owner": "", "environment": "on-prem", "datacenter": "chicago"}, + expectAcquire: true, + }, + { + name: "user-scoped provisioner and user-scoped job", + provisionerJobTags: map[string]string{"scope": "user", "owner": "aaa"}, + acquireJobTags: map[string]string{"scope": "user", "owner": "aaa"}, + expectAcquire: true, + }, + { + name: "user-scoped provisioner with tags and user-scoped job", + provisionerJobTags: map[string]string{"scope": "user", "owner": "aaa"}, + acquireJobTags: map[string]string{"scope": "user", "owner": "aaa", "environment": "on-prem"}, + expectAcquire: true, + }, + { + name: "user-scoped provisioner with tags and user-scoped job with tags", + provisionerJobTags: map[string]string{"scope": "user", "owner": "aaa", "environment": "on-prem"}, + acquireJobTags: map[string]string{"scope": "user", "owner": "aaa", "environment": "on-prem"}, + expectAcquire: true, + }, + { + name: "user-scoped provisioner with multiple tags and user-scoped job with tags", + provisionerJobTags: map[string]string{"scope": "user", "owner": "aaa", "environment": "on-prem"}, + acquireJobTags: map[string]string{"scope": "user", "owner": "aaa", "environment": "on-prem", "datacenter": "chicago"}, + expectAcquire: true, + }, + { + name: "user-scoped provisioner with multiple tags and user-scoped job with multiple tags", + provisionerJobTags: map[string]string{"scope": "user", "owner": "aaa", "environment": "on-prem", "datacenter": "chicago"}, + acquireJobTags: map[string]string{"scope": "user", "owner": "aaa", "environment": "on-prem", "datacenter": "chicago"}, + expectAcquire: true, + }, { name: "untagged provisioner and tagged job", - provisionerJobTags: map[string]string{"scope": "organization", "owner": "", "foo": "bar"}, + provisionerJobTags: map[string]string{"scope": "organization", "owner": "", "environment": "on-prem"}, acquireJobTags: map[string]string{"scope": "organization", "owner": ""}, expectAcquire: false, }, { name: "tagged provisioner and untagged job", provisionerJobTags: map[string]string{"scope": "organization", "owner": ""}, - acquireJobTags: map[string]string{"scope": "organization", "owner": "", "foo": "bar"}, + acquireJobTags: map[string]string{"scope": "organization", "owner": "", "environment": "on-prem"}, expectAcquire: false, }, - { - name: "tagged provisioner and tagged job", - provisionerJobTags: map[string]string{"scope": "organization", "owner": "", "foo": "bar"}, - acquireJobTags: map[string]string{"scope": "organization", "owner": "", "foo": "bar"}, - expectAcquire: true, - }, { name: "tagged provisioner and double-tagged job", - provisionerJobTags: map[string]string{"scope": "organization", "owner": "", "foo": "bar", "baz": "zap"}, - acquireJobTags: map[string]string{"scope": "organization", "owner": "", "foo": "bar"}, + provisionerJobTags: map[string]string{"scope": "organization", "owner": "", "environment": "on-prem", "datacenter": "chicago"}, + acquireJobTags: map[string]string{"scope": "organization", "owner": "", "environment": "on-prem"}, expectAcquire: false, }, { - name: "double-tagged provisioner and tagged job", - provisionerJobTags: map[string]string{"scope": "organization", "owner": "", "foo": "bar"}, - acquireJobTags: map[string]string{"scope": "organization", "owner": "", "foo": "bar", "baz": "zap"}, - expectAcquire: true, + name: "double-tagged provisioner and double-tagged job with differing tags", + provisionerJobTags: map[string]string{"scope": "organization", "owner": "", "environment": "on-prem", "datacenter": "chicago"}, + acquireJobTags: map[string]string{"scope": "organization", "owner": "", "environment": "on-prem", "datacenter": "new_york"}, + expectAcquire: false, }, { - name: "double-tagged provisioner and double-tagged job", - provisionerJobTags: map[string]string{"scope": "organization", "owner": "", "foo": "bar", "baz": "zap"}, - acquireJobTags: map[string]string{"scope": "organization", "owner": "", "foo": "bar", "baz": "zap"}, - expectAcquire: true, - }, - { - name: "owner-scoped provisioner and untagged job", + name: "user-scoped provisioner and untagged job", provisionerJobTags: map[string]string{"scope": "organization", "owner": ""}, - acquireJobTags: map[string]string{"scope": "owner", "owner": someID}, + acquireJobTags: map[string]string{"scope": "user", "owner": "aaa"}, expectAcquire: false, }, { - name: "owner-scoped provisioner and owner-scoped job", - provisionerJobTags: map[string]string{"scope": "owner", "owner": someID}, - acquireJobTags: map[string]string{"scope": "owner", "owner": someID}, - expectAcquire: true, - }, - { - name: "owner-scoped provisioner and different owner-scoped job", - provisionerJobTags: map[string]string{"scope": "owner", "owner": someOtherID}, - acquireJobTags: map[string]string{"scope": "owner", "owner": someID}, + name: "user-scoped provisioner and different user-scoped job", + provisionerJobTags: map[string]string{"scope": "user", "owner": "bbb"}, + acquireJobTags: map[string]string{"scope": "user", "owner": "aaa"}, expectAcquire: false, }, { - name: "org-scoped provisioner and owner-scoped job", - provisionerJobTags: map[string]string{"scope": "owner", "owner": someID}, + name: "org-scoped provisioner and user-scoped job", + provisionerJobTags: map[string]string{"scope": "user", "owner": "aaa"}, acquireJobTags: map[string]string{"scope": "organization", "owner": ""}, expectAcquire: false, }, - } { + { + name: "user-scoped provisioner and org-scoped job with tags", + provisionerJobTags: map[string]string{"scope": "user", "owner": "aaa", "environment": "on-prem"}, + acquireJobTags: map[string]string{"scope": "organization", "owner": ""}, + expectAcquire: false, + }, + { + name: "user-scoped provisioner and user-scoped job with tags", + provisionerJobTags: map[string]string{"scope": "user", "owner": "aaa", "environment": "on-prem"}, + acquireJobTags: map[string]string{"scope": "user", "owner": "aaa"}, + expectAcquire: false, + }, + { + name: "user-scoped provisioner with tags and user-scoped job with multiple tags", + provisionerJobTags: map[string]string{"scope": "user", "owner": "aaa", "environment": "on-prem", "datacenter": "chicago"}, + acquireJobTags: map[string]string{"scope": "user", "owner": "aaa", "environment": "on-prem"}, + expectAcquire: false, + }, + { + name: "user-scoped provisioner with tags and user-scoped job with differing tags", + provisionerJobTags: map[string]string{"scope": "user", "owner": "aaa", "environment": "on-prem", "datacenter": "new_york"}, + acquireJobTags: map[string]string{"scope": "user", "owner": "aaa", "environment": "on-prem", "datacenter": "chicago"}, + expectAcquire: false, + }, + } + for _, tt := range testCases { tt := tt t.Run(tt.name, func(t *testing.T) { t.Parallel() - - ctx := testutil.Context(t, testutil.WaitShort/2) + ctx := testutil.Context(t, testutil.WaitShort) // NOTE: explicitly not using fake store for this test. db, ps := dbtestutil.NewDB(t) log := slogtest.Make(t, nil).Leveled(slog.LevelDebug) @@ -443,6 +496,37 @@ func TestAcquirer_MatchTags(t *testing.T) { } }) } + + t.Run("GenTable", func(t *testing.T) { + t.Parallel() + // Generate a table that can be copy-pasted into docs/admin/provisioners.md + lines := []string{ + "\n", + "| Provisioner Tags | Job Tags | Can Run Job? |", + "|------------------|----------|--------------|", + } + // turn the JSON map into k=v for readability + kvs := func(m map[string]string) string { + ss := make([]string, 0, len(m)) + // ensure consistent ordering of tags + for _, k := range []string{"scope", "owner", "environment", "datacenter"} { + if v, found := m[k]; found { + ss = append(ss, k+"="+v) + } + } + return strings.Join(ss, " ") + } + for _, tt := range testCases { + acquire := "✅" + if !tt.expectAcquire { + acquire = "❌" + } + s := fmt.Sprintf("| %s | %s | %s |", kvs(tt.acquireJobTags), kvs(tt.provisionerJobTags), acquire) + lines = append(lines, s) + } + t.Logf("You can paste this into docs/admin/provisioners.md") + t.Logf(strings.Join(lines, "\n")) + }) } func postJob(t *testing.T, ps pubsub.Pubsub, pt database.ProvisionerType, tags provisionerdserver.Tags) { diff --git a/docs/admin/provisioners.md b/docs/admin/provisioners.md index e20b86f5a0..22f1eccdf1 100644 --- a/docs/admin/provisioners.md +++ b/docs/admin/provisioners.md @@ -47,13 +47,25 @@ the [Helm example](#example-running-an-external-provisioner-with-helm) below. ## Types of provisioners -> Provisioners have two important tags: `scope` and `owner`. Coder sets these -> tags automatically. +Provisioners can broadly be categorized by scope: `organization` or `user`. The +scope of a provisioner can be specified with +[`-tag=scope=`](../cli/provisionerd_start.md#t---tag) when starting the +provisioner daemon. Only users with at least the +[Template Admin](../admin/users.md#roles) role or higher may create +organization-scoped provisioner daemons. + +There are two exceptions: + +- [Built-in provisioners](../cli/server.md#provisioner-daemons) are always + organization-scoped. +- External provisioners started using a + [pre-shared key (PSK)](../cli/provisionerd_start.md#psk) are always + organization-scoped. ### Organization-Scoped Provisioners **Organization-scoped Provisioners** can pick up build jobs created by any user. -These provisioners always have tags `scope=organization owner=""`. +These provisioners always have the implicit tags `scope=organization owner=""`. ```shell coder provisionerd start @@ -62,9 +74,8 @@ coder provisionerd start ### User-scoped Provisioners **User-scoped Provisioners** can only pick up build jobs created from -user-tagged templates. User-scoped provisioners always have tags -`scope=owner owner=`. Unlike the other provisioner types, any Coder user -can run user provisioners, but they have no impact unless there is at least one +user-tagged templates. Unlike the other provisioner types, any Coder user can +run user provisioners, but they have no impact unless there exists at least one template with the `scope=user` provisioner tag. ```shell @@ -80,58 +91,86 @@ coder templates push on-prem \ ### Provisioner Tags You can use **provisioner tags** to control which provisioners can pick up build -jobs from templates (and corresponding workspaces) with matching tags. +jobs from templates (and corresponding workspaces) with matching explicit tags. + +Provisioners have two implicit tags: `scope` and `owner`. Coder sets these tags +automatically. + +- Organization-scoped provisioners always have the implicit tags + `scope=organization owner=""` +- User-scoped provisioners always have the implicit tags + `scope=user owner=` + +For example: ```shell +# Start a provisioner with the explicit tags +# environment=on_prem and datacenter=chicago coder provisionerd start \ --tag environment=on_prem \ - --tag data_center=chicago + --tag datacenter=chicago # In another terminal, create/push -# a template that requires this provisioner +# a template that requires the explicit +# tag environment=on_prem coder templates push on-prem \ --provisioner-tag environment=on_prem -# Or, match the provisioner exactly +# Or, match the provisioner's explicit tags exactly coder templates push on-prem-chicago \ --provisioner-tag environment=on_prem \ - --provisioner-tag data_center=chicago + --provisioner-tag datacenter=chicago ``` A provisioner can run a given build job if one of the below is true: -1. The provisioner and job tags are both organization-scoped and both have no - additional tags set, -1. The set of tags of the build job is a subset of the set of tags of the - provisioner. - -This is illustrated in the below table: - -| Provisioner Tags | Job Tags | Can run job? | -| ------------------------------------------------------------------------------------- | ------------------------------------------------------------------------- | ------------ | -| `{"owner":"","scope":"organization"}` | `{"owner":"","scope":"organization"}` | true | -| `{"owner":"","scope":"organization"}` | `{"environment":"on_prem","owner":"","scope":"organization"}` | false | -| `{"environment":"on_prem","owner":"","scope":"organization"}` | `{"owner":"","scope":"organization"}` | false | -| `{"environment":"on_prem","owner":"","scope":"organization"}` | `{"foo":"bar","owner":"","scope":"organization"}` | true | -| `{"environment":"on_prem","owner":"","scope":"organization"}` | `{"data_center":"chicago","foo":"bar","owner":"","scope":"organization"}` | false | -| `{"data_center":"chicago","environment":"on_prem","owner":"","scope":"organization"}` | `{"foo":"bar","owner":"","scope":"organization"}` | true | -| `{"data_center":"chicago","environment":"on_prem","owner":"","scope":"organization"}` | `{"data_center":"chicago","foo":"bar","owner":"","scope":"organization"}` | true | -| `{"owner":"aaa","scope":"owner"}` | `{"owner":"","scope":"organization"}` | false | -| `{"owner":"aaa","scope":"owner"}` | `{"owner":"aaa","scope":"owner"}` | true | -| `{"owner":"aaa","scope":"owner"}` | `{"owner":"bbb","scope":"owner"}` | false | -| `{"owner":"","scope":"organization"}` | `{"owner":"aaa","scope":"owner"}` | false | +1. A job with no explicit tags can only be run on a provisioner with no explicit + tags. This way you can introduce tagging into your deployment without + disrupting existing provisioners and jobs. +1. If a job has any explicit tags, it can only run on a provisioner with those + explicit tags (the provisioner could have additional tags). The external provisioner in the above example can run build jobs with tags: - `environment=on_prem` -- `data_center=chicago` +- `datacenter=chicago` - `environment=on_prem datacenter=chicago` -- `environment=cloud datacenter=chicago` -- `environment=on_prem datacenter=new_york` However, it will not pick up any build jobs that do not have either of the `environment` or `datacenter` tags set. It will also not pick up any build jobs -from templates with the `user` tag set. +from templates with the tag `scope=user` set. + +This is illustrated in the below table: + +| Provisioner Tags | Job Tags | Can Run Job? | +| ----------------------------------------------------------------- | ---------------------------------------------------------------- | ------------ | +| scope=organization owner= | scope=organization owner= | ✅ | +| scope=organization owner= environment=on-prem | scope=organization owner= environment=on-prem | ✅ | +| scope=organization owner= environment=on-prem datacenter=chicago | scope=organization owner= environment=on-prem | ✅ | +| scope=organization owner= environment=on-prem datacenter=chicago | scope=organization owner= environment=on-prem datacenter=chicago | ✅ | +| scope=user owner=aaa | scope=user owner=aaa | ✅ | +| scope=user owner=aaa environment=on-prem | scope=user owner=aaa | ✅ | +| scope=user owner=aaa environment=on-prem | scope=user owner=aaa environment=on-prem | ✅ | +| scope=user owner=aaa environment=on-prem datacenter=chicago | scope=user owner=aaa environment=on-prem | ✅ | +| scope=user owner=aaa environment=on-prem datacenter=chicago | scope=user owner=aaa environment=on-prem datacenter=chicago | ✅ | +| scope=organization owner= | scope=organization owner= environment=on-prem | ❌ | +| scope=organization owner= environment=on-prem | scope=organization owner= | ❌ | +| scope=organization owner= environment=on-prem | scope=organization owner= environment=on-prem datacenter=chicago | ❌ | +| scope=organization owner= environment=on-prem datacenter=new_york | scope=organization owner= environment=on-prem datacenter=chicago | ❌ | +| scope=user owner=aaa | scope=organization owner= | ❌ | +| scope=user owner=aaa | scope=user owner=bbb | ❌ | +| scope=organization owner= | scope=user owner=aaa | ❌ | +| scope=organization owner= | scope=user owner=aaa environment=on-prem | ❌ | +| scope=user owner=aaa | scope=user owner=aaa environment=on-prem | ❌ | +| scope=user owner=aaa environment=on-prem | scope=user owner=aaa environment=on-prem datacenter=chicago | ❌ | +| scope=user owner=aaa environment=on-prem datacenter=chicago | scope=user owner=aaa environment=on-prem datacenter=new_york | ❌ | + +> **Note to maintainers:** to generate this table, run the following command and +> copy the output: +> +> ``` +> go test -v -count=1 ./coderd/provisionerdserver/ -test.run='^TestAcquirer_MatchTags/GenTable$' +> ``` ## Example: Running an external provisioner with Helm