feat: add auto-locking/deleting workspace based on template config (#8240)

This commit is contained in:
Jon Ayers 2023-07-02 21:29:52 -05:00 committed by GitHub
parent 818c4a7f23
commit 4a9c8f407a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 726 additions and 70 deletions

2
coderd/apidoc/docs.go generated
View File

@ -6972,7 +6972,7 @@ const docTemplate = `{
"type": "integer"
},
"locked_ttl_ms": {
"description": "LockedTTL allows optionally specifying the max lifetime before Coder\npermanently deletes locked workspaces created from this template.",
"description": "LockedTTLMillis allows optionally specifying the max lifetime before Coder\npermanently deletes locked workspaces created from this template.",
"type": "integer"
},
"max_ttl_ms": {

View File

@ -6210,7 +6210,7 @@
"type": "integer"
},
"locked_ttl_ms": {
"description": "LockedTTL allows optionally specifying the max lifetime before Coder\npermanently deletes locked workspaces created from this template.",
"description": "LockedTTLMillis allows optionally specifying the max lifetime before Coder\npermanently deletes locked workspaces created from this template.",
"type": "integer"
},
"max_ttl_ms": {

View File

@ -160,23 +160,65 @@ func (e *Executor) runOnce(t time.Time) Stats {
return nil
}
builder := wsbuilder.New(ws, nextTransition).
SetLastWorkspaceBuildInTx(&latestBuild).
SetLastWorkspaceBuildJobInTx(&latestJob).
Reason(reason)
if nextTransition != "" {
builder := wsbuilder.New(ws, nextTransition).
SetLastWorkspaceBuildInTx(&latestBuild).
SetLastWorkspaceBuildJobInTx(&latestJob).
Reason(reason)
if _, _, err := builder.Build(e.ctx, tx, nil); err != nil {
log.Error(e.ctx, "workspace build error",
slog.F("transition", nextTransition),
slog.Error(err),
if _, _, err := builder.Build(e.ctx, tx, nil); err != nil {
log.Error(e.ctx, "unable to transition workspace",
slog.F("transition", nextTransition),
slog.Error(err),
)
return nil
}
}
// Lock the workspace if it has breached the template's
// threshold for inactivity.
if reason == database.BuildReasonAutolock {
err = tx.UpdateWorkspaceLockedAt(e.ctx, database.UpdateWorkspaceLockedAtParams{
ID: ws.ID,
LockedAt: sql.NullTime{
Time: database.Now(),
Valid: true,
},
})
if err != nil {
log.Error(e.ctx, "unable to lock workspace",
slog.F("transition", nextTransition),
slog.Error(err),
)
return nil
}
log.Info(e.ctx, "locked workspace",
slog.F("last_used_at", ws.LastUsedAt),
slog.F("inactivity_ttl", templateSchedule.InactivityTTL),
slog.F("since_last_used_at", time.Since(ws.LastUsedAt)),
)
}
if reason == database.BuildReasonAutodelete {
log.Info(e.ctx, "deleted workspace",
slog.F("locked_at", ws.LockedAt.Time),
slog.F("locked_ttl", templateSchedule.LockedTTL),
)
}
if nextTransition == "" {
return nil
}
statsMu.Lock()
stats.Transitions[ws.ID] = nextTransition
statsMu.Unlock()
log.Info(e.ctx, "scheduling workspace transition", slog.F("transition", nextTransition))
log.Info(e.ctx, "scheduling workspace transition",
slog.F("transition", nextTransition),
slog.F("reason", reason),
)
return nil
@ -199,6 +241,12 @@ func (e *Executor) runOnce(t time.Time) Stats {
return stats
}
// getNextTransition returns the next eligible transition for the workspace
// as well as the reason for why it is transitioning. It is possible
// for this function to return a nil error as well as an empty transition.
// In such cases it means no provisioning should occur but the workspace
// may be "transitioning" to a new state (such as an inactive, stopped
// workspace transitioning to the locked state).
func getNextTransition(
ws database.Workspace,
latestBuild database.WorkspaceBuild,
@ -211,12 +259,23 @@ func getNextTransition(
error,
) {
switch {
case isEligibleForAutostop(latestBuild, latestJob, currentTick):
case isEligibleForAutostop(ws, latestBuild, latestJob, currentTick):
return database.WorkspaceTransitionStop, database.BuildReasonAutostop, nil
case isEligibleForAutostart(ws, latestBuild, latestJob, templateSchedule, currentTick):
return database.WorkspaceTransitionStart, database.BuildReasonAutostart, nil
case isEligibleForFailedStop(latestBuild, latestJob, templateSchedule):
case isEligibleForFailedStop(latestBuild, latestJob, templateSchedule, currentTick):
return database.WorkspaceTransitionStop, database.BuildReasonAutostop, nil
case isEligibleForLockedStop(ws, templateSchedule, currentTick):
// Only stop started workspaces.
if latestBuild.Transition == database.WorkspaceTransitionStart {
return database.WorkspaceTransitionStop, database.BuildReasonAutolock, nil
}
// We shouldn't transition the workspace but we should still
// lock it.
return "", database.BuildReasonAutolock, nil
case isEligibleForDelete(ws, templateSchedule, currentTick):
return database.WorkspaceTransitionDelete, database.BuildReasonAutodelete, nil
default:
return "", "", xerrors.Errorf("last transition not valid for autostart or autostop")
}
@ -225,7 +284,12 @@ func getNextTransition(
// isEligibleForAutostart returns true if the workspace should be autostarted.
func isEligibleForAutostart(ws database.Workspace, build database.WorkspaceBuild, job database.ProvisionerJob, templateSchedule schedule.TemplateScheduleOptions, currentTick time.Time) bool {
// Don't attempt to autostart failed workspaces.
if !job.CompletedAt.Valid || job.Error.String != "" {
if db2sdk.ProvisionerJobStatus(job) == codersdk.ProvisionerJobFailed {
return false
}
// If the workspace is locked we should not autostart it.
if ws.LockedAt.Valid {
return false
}
@ -253,9 +317,13 @@ func isEligibleForAutostart(ws database.Workspace, build database.WorkspaceBuild
}
// isEligibleForAutostart returns true if the workspace should be autostopped.
func isEligibleForAutostop(build database.WorkspaceBuild, job database.ProvisionerJob, currentTick time.Time) bool {
// Don't attempt to autostop failed workspaces.
if !job.CompletedAt.Valid || job.Error.String != "" {
func isEligibleForAutostop(ws database.Workspace, build database.WorkspaceBuild, job database.ProvisionerJob, currentTick time.Time) bool {
if db2sdk.ProvisionerJobStatus(job) == codersdk.ProvisionerJobFailed {
return false
}
// If the workspace is locked we should not autostop it.
if ws.LockedAt.Valid {
return false
}
@ -266,14 +334,35 @@ func isEligibleForAutostop(build database.WorkspaceBuild, job database.Provision
!currentTick.Before(build.Deadline)
}
// isEligibleForLockedStop returns true if the workspace should be locked
// for breaching the inactivity threshold of the template.
func isEligibleForLockedStop(ws database.Workspace, templateSchedule schedule.TemplateScheduleOptions, currentTick time.Time) bool {
// Only attempt to lock workspaces not already locked.
return !ws.LockedAt.Valid &&
// The template must specify an inactivity TTL.
templateSchedule.InactivityTTL > 0 &&
// The workspace must breach the inactivity TTL.
currentTick.Sub(ws.LastUsedAt) > templateSchedule.InactivityTTL
}
func isEligibleForDelete(ws database.Workspace, templateSchedule schedule.TemplateScheduleOptions, currentTick time.Time) bool {
// Only attempt to delete locked workspaces.
return ws.LockedAt.Valid &&
// Locked workspaces should only be deleted if a locked_ttl is specified.
templateSchedule.LockedTTL > 0 &&
// The workspace must breach the locked_ttl.
currentTick.Sub(ws.LockedAt.Time) > templateSchedule.LockedTTL
}
// isEligibleForFailedStop returns true if the workspace is eligible to be stopped
// due to a failed build.
func isEligibleForFailedStop(build database.WorkspaceBuild, job database.ProvisionerJob, templateSchedule schedule.TemplateScheduleOptions) bool {
func isEligibleForFailedStop(build database.WorkspaceBuild, job database.ProvisionerJob, templateSchedule schedule.TemplateScheduleOptions, currentTick time.Time) bool {
// If the template has specified a failure TLL.
return templateSchedule.FailureTTL > 0 &&
// And the job resulted in failure.
db2sdk.ProvisionerJobStatus(job) == codersdk.ProvisionerJobFailed &&
build.Transition == database.WorkspaceTransitionStart &&
// And sufficient time has elapsed since the job has completed.
job.CompletedAt.Valid && database.Now().Sub(job.CompletedAt.Time) > templateSchedule.FailureTTL
job.CompletedAt.Valid &&
currentTick.Sub(job.CompletedAt.Time) > templateSchedule.FailureTTL
}

View File

@ -21,7 +21,6 @@ import (
"github.com/coder/coder/codersdk"
"github.com/coder/coder/provisioner/echo"
"github.com/coder/coder/provisionersdk/proto"
"github.com/coder/coder/testutil"
)
func TestExecutorAutostartOK(t *testing.T) {
@ -651,8 +650,9 @@ func TestExecutorAutostartTemplateDisabled(t *testing.T) {
assert.Len(t, stats.Transitions, 0)
}
// TesetExecutorFailedWorkspace tests that failed workspaces that breach
// their template failed_ttl threshold trigger a stop job.
// TestExecutorFailedWorkspace test AGPL functionality which mainly
// ensures that autostop actions as a result of a failed workspace
// build do not trigger.
// For enterprise functionality see enterprise/coderd/workspaces_test.go
func TestExecutorFailedWorkspace(t *testing.T) {
t.Parallel()
@ -693,12 +693,57 @@ func TestExecutorFailedWorkspace(t *testing.T) {
ws := coderdtest.CreateWorkspace(t, client, user.OrganizationID, template.ID)
build := coderdtest.AwaitWorkspaceBuildJob(t, client, ws.LatestBuild.ID)
require.Equal(t, codersdk.WorkspaceStatusFailed, build.Status)
require.Eventually(t,
func() bool {
return database.Now().Sub(*build.Job.CompletedAt) > failureTTL
},
testutil.IntervalMedium, testutil.IntervalFast)
ticker <- time.Now()
ticker <- build.Job.CompletedAt.Add(failureTTL * 2)
stats := <-statCh
// Expect no transitions since we're using AGPL.
require.Len(t, stats.Transitions, 0)
})
}
// TestExecutorInactiveWorkspace test AGPL functionality which mainly
// ensures that autostop actions as a result of an inactive workspace
// do not trigger.
// For enterprise functionality see enterprise/coderd/workspaces_test.go
func TestExecutorInactiveWorkspace(t *testing.T) {
t.Parallel()
// Test that an AGPL TemplateScheduleStore properly disables
// functionality.
t.Run("OK", func(t *testing.T) {
t.Parallel()
var (
ticker = make(chan time.Time)
statCh = make(chan autobuild.Stats)
logger = slogtest.Make(t, &slogtest.Options{
// We ignore errors here since we expect to fail
// builds.
IgnoreErrors: true,
})
inactiveTTL = time.Millisecond
client = coderdtest.New(t, &coderdtest.Options{
Logger: &logger,
AutobuildTicker: ticker,
IncludeProvisionerDaemon: true,
AutobuildStats: statCh,
TemplateScheduleStore: schedule.NewAGPLTemplateScheduleStore(),
})
)
user := coderdtest.CreateFirstUser(t, client)
version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{
Parse: echo.ParseComplete,
ProvisionPlan: echo.ProvisionComplete,
ProvisionApply: echo.ProvisionComplete,
})
template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID, func(ctr *codersdk.CreateTemplateRequest) {
ctr.InactivityTTLMillis = ptr.Ref[int64](inactiveTTL.Milliseconds())
})
coderdtest.AwaitTemplateVersionJob(t, client, version.ID)
ws := coderdtest.CreateWorkspace(t, client, user.OrganizationID, template.ID)
build := coderdtest.AwaitWorkspaceBuildJob(t, client, ws.LatestBuild.ID)
require.Equal(t, codersdk.WorkspaceStatusRunning, build.Status)
ticker <- ws.LastUsedAt.Add(inactiveTTL * 2)
stats := <-statCh
// Expect no transitions since we're using AGPL.
require.Len(t, stats.Transitions, 0)

View File

@ -3495,12 +3495,17 @@ func (q *fakeQuerier) GetWorkspacesEligibleForTransition(ctx context.Context, no
return nil, err
}
if build.Transition == database.WorkspaceTransitionStart && !build.Deadline.IsZero() && build.Deadline.Before(now) {
if build.Transition == database.WorkspaceTransitionStart &&
!build.Deadline.IsZero() &&
build.Deadline.Before(now) &&
!workspace.LockedAt.Valid {
workspaces = append(workspaces, workspace)
continue
}
if build.Transition == database.WorkspaceTransitionStop && workspace.AutostartSchedule.Valid {
if build.Transition == database.WorkspaceTransitionStop &&
workspace.AutostartSchedule.Valid &&
!workspace.LockedAt.Valid {
workspaces = append(workspaces, workspace)
continue
}
@ -3513,6 +3518,19 @@ func (q *fakeQuerier) GetWorkspacesEligibleForTransition(ctx context.Context, no
workspaces = append(workspaces, workspace)
continue
}
template, err := q.GetTemplateByID(ctx, workspace.TemplateID)
if err != nil {
return nil, xerrors.Errorf("get template by ID: %w", err)
}
if !workspace.LockedAt.Valid && template.InactivityTTL > 0 {
workspaces = append(workspaces, workspace)
continue
}
if workspace.LockedAt.Valid && template.LockedTTL > 0 {
workspaces = append(workspaces, workspace)
continue
}
}
return workspaces, nil
@ -4702,6 +4720,7 @@ func (q *fakeQuerier) UpdateTemplateScheduleByID(_ context.Context, arg database
tpl.MaxTTL = arg.MaxTTL
tpl.FailureTTL = arg.FailureTTL
tpl.InactivityTTL = arg.InactivityTTL
tpl.LockedTTL = arg.LockedTTL
q.templates[idx] = tpl
return tpl.DeepCopy(), nil
}
@ -5245,6 +5264,7 @@ func (q *fakeQuerier) UpdateWorkspaceLockedAt(_ context.Context, arg database.Up
continue
}
workspace.LockedAt = arg.LockedAt
workspace.LastUsedAt = database.Now()
q.workspaces[index] = workspace
return nil
}

View File

@ -25,7 +25,10 @@ CREATE TYPE audit_action AS ENUM (
CREATE TYPE build_reason AS ENUM (
'initiator',
'autostart',
'autostop'
'autostop',
'autolock',
'failedstop',
'autodelete'
);
CREATE TYPE log_level AS ENUM (

View File

@ -0,0 +1 @@
-- It's not possible to delete enum values.

View File

@ -0,0 +1,5 @@
BEGIN;
ALTER TYPE build_reason ADD VALUE IF NOT EXISTS 'autolock';
ALTER TYPE build_reason ADD VALUE IF NOT EXISTS 'failedstop';
ALTER TYPE build_reason ADD VALUE IF NOT EXISTS 'autodelete';
COMMIT;

View File

@ -214,9 +214,12 @@ func AllAuditActionValues() []AuditAction {
type BuildReason string
const (
BuildReasonInitiator BuildReason = "initiator"
BuildReasonAutostart BuildReason = "autostart"
BuildReasonAutostop BuildReason = "autostop"
BuildReasonInitiator BuildReason = "initiator"
BuildReasonAutostart BuildReason = "autostart"
BuildReasonAutostop BuildReason = "autostop"
BuildReasonAutolock BuildReason = "autolock"
BuildReasonFailedstop BuildReason = "failedstop"
BuildReasonAutodelete BuildReason = "autodelete"
)
func (e *BuildReason) Scan(src interface{}) error {
@ -258,7 +261,10 @@ func (e BuildReason) Valid() bool {
switch e {
case BuildReasonInitiator,
BuildReasonAutostart,
BuildReasonAutostop:
BuildReasonAutostop,
BuildReasonAutolock,
BuildReasonFailedstop,
BuildReasonAutodelete:
return true
}
return false
@ -269,6 +275,9 @@ func AllBuildReasonValues() []BuildReason {
BuildReasonInitiator,
BuildReasonAutostart,
BuildReasonAutostop,
BuildReasonAutolock,
BuildReasonFailedstop,
BuildReasonAutodelete,
}
}

View File

@ -8663,6 +8663,8 @@ LEFT JOIN
workspace_builds ON workspace_builds.workspace_id = workspaces.id
INNER JOIN
provisioner_jobs ON workspace_builds.job_id = provisioner_jobs.id
INNER JOIN
templates ON workspaces.template_id = templates.id
WHERE
workspace_builds.build_number = (
SELECT
@ -8700,6 +8702,20 @@ WHERE
provisioner_jobs.error IS NOT NULL AND
provisioner_jobs.error != '' AND
workspace_builds.transition = 'start'::workspace_transition
) OR
-- If the workspace's template has an inactivity_ttl set
-- it may be eligible for locking.
(
templates.inactivity_ttl > 0 AND
workspaces.locked_at IS NULL
) OR
-- If the workspace's template has a locked_ttl set
-- and the workspace is already locked
(
templates.locked_ttl > 0 AND
workspaces.locked_at IS NOT NULL
)
) AND workspaces.deleted = 'false'
`
@ -8899,7 +8915,8 @@ const updateWorkspaceLockedAt = `-- name: UpdateWorkspaceLockedAt :exec
UPDATE
workspaces
SET
locked_at = $2
locked_at = $2,
last_used_at = now() at time zone 'utc'
WHERE
id = $1
`

View File

@ -414,6 +414,8 @@ LEFT JOIN
workspace_builds ON workspace_builds.workspace_id = workspaces.id
INNER JOIN
provisioner_jobs ON workspace_builds.job_id = provisioner_jobs.id
INNER JOIN
templates ON workspaces.template_id = templates.id
WHERE
workspace_builds.build_number = (
SELECT
@ -451,6 +453,20 @@ WHERE
provisioner_jobs.error IS NOT NULL AND
provisioner_jobs.error != '' AND
workspace_builds.transition = 'start'::workspace_transition
) OR
-- If the workspace's template has an inactivity_ttl set
-- it may be eligible for locking.
(
templates.inactivity_ttl > 0 AND
workspaces.locked_at IS NULL
) OR
-- If the workspace's template has a locked_ttl set
-- and the workspace is already locked
(
templates.locked_ttl > 0 AND
workspaces.locked_at IS NOT NULL
)
) AND workspaces.deleted = 'false';
@ -458,6 +474,7 @@ WHERE
UPDATE
workspaces
SET
locked_at = $2
locked_at = $2,
last_used_at = now() at time zone 'utc'
WHERE
id = $1;

View File

@ -227,6 +227,7 @@ func (api *API) postTemplateByOrganization(rw http.ResponseWriter, r *http.Reque
maxTTL time.Duration
failureTTL time.Duration
inactivityTTL time.Duration
lockedTTL time.Duration
)
if createTemplate.DefaultTTLMillis != nil {
defaultTTL = time.Duration(*createTemplate.DefaultTTLMillis) * time.Millisecond
@ -240,6 +241,9 @@ func (api *API) postTemplateByOrganization(rw http.ResponseWriter, r *http.Reque
if createTemplate.InactivityTTLMillis != nil {
inactivityTTL = time.Duration(*createTemplate.InactivityTTLMillis) * time.Millisecond
}
if createTemplate.LockedTTLMillis != nil {
lockedTTL = time.Duration(*createTemplate.LockedTTLMillis) * time.Millisecond
}
var validErrs []codersdk.ValidationError
if defaultTTL < 0 {
@ -257,6 +261,10 @@ func (api *API) postTemplateByOrganization(rw http.ResponseWriter, r *http.Reque
if inactivityTTL < 0 {
validErrs = append(validErrs, codersdk.ValidationError{Field: "inactivity_ttl_ms", Detail: "Must be a positive integer."})
}
if lockedTTL < 0 {
validErrs = append(validErrs, codersdk.ValidationError{Field: "locked_ttl_ms", Detail: "Must be a positive integer."})
}
if len(validErrs) > 0 {
httpapi.Write(ctx, rw, http.StatusBadRequest, codersdk.Response{
Message: "Invalid create template request.",
@ -312,6 +320,7 @@ func (api *API) postTemplateByOrganization(rw http.ResponseWriter, r *http.Reque
MaxTTL: maxTTL,
FailureTTL: failureTTL,
InactivityTTL: inactivityTTL,
LockedTTL: lockedTTL,
})
if err != nil {
return xerrors.Errorf("set template schedule options: %s", err)
@ -525,7 +534,7 @@ func (api *API) patchTemplateMeta(rw http.ResponseWriter, r *http.Request) {
req.MaxTTLMillis == time.Duration(template.MaxTTL).Milliseconds() &&
req.FailureTTLMillis == time.Duration(template.FailureTTL).Milliseconds() &&
req.InactivityTTLMillis == time.Duration(template.InactivityTTL).Milliseconds() &&
req.FailureTTLMillis == time.Duration(template.LockedTTL).Milliseconds() {
req.LockedTTLMillis == time.Duration(template.LockedTTL).Milliseconds() {
return nil
}

View File

@ -662,7 +662,7 @@ func TestPatchTemplateMeta(t *testing.T) {
template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID, func(ctr *codersdk.CreateTemplateRequest) {
ctr.FailureTTLMillis = ptr.Ref(0 * time.Hour.Milliseconds())
ctr.InactivityTTLMillis = ptr.Ref(0 * time.Hour.Milliseconds())
ctr.LockedTTL = ptr.Ref(0 * time.Hour.Milliseconds())
ctr.LockedTTLMillis = ptr.Ref(0 * time.Hour.Milliseconds())
})
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
@ -697,7 +697,7 @@ func TestPatchTemplateMeta(t *testing.T) {
template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID, func(ctr *codersdk.CreateTemplateRequest) {
ctr.FailureTTLMillis = ptr.Ref(0 * time.Hour.Milliseconds())
ctr.InactivityTTLMillis = ptr.Ref(0 * time.Hour.Milliseconds())
ctr.LockedTTL = ptr.Ref(0 * time.Hour.Milliseconds())
ctr.LockedTTLMillis = ptr.Ref(0 * time.Hour.Milliseconds())
})
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)

View File

@ -2399,11 +2399,12 @@ func TestWorkspaceLock(t *testing.T) {
})
require.NoError(t, err)
workspace, err = client.Workspace(ctx, workspace.ID)
workspace = coderdtest.MustWorkspace(t, client, workspace.ID)
require.NoError(t, err, "fetch provisioned workspace")
require.NotNil(t, workspace.LockedAt)
require.WithinRange(t, *workspace.LockedAt, time.Now().Add(-time.Second*10), time.Now())
lastUsedAt := workspace.LastUsedAt
err = client.UpdateWorkspaceLock(ctx, workspace.ID, codersdk.UpdateWorkspaceLock{
Lock: false,
})
@ -2412,6 +2413,7 @@ func TestWorkspaceLock(t *testing.T) {
workspace, err = client.Workspace(ctx, workspace.ID)
require.NoError(t, err, "fetch provisioned workspace")
require.Nil(t, workspace.LockedAt)
require.True(t, workspace.LastUsedAt.After(lastUsedAt))
})
t.Run("CannotStart", func(t *testing.T) {

View File

@ -108,9 +108,9 @@ type CreateTemplateRequest struct {
// InactivityTTLMillis allows optionally specifying the max lifetime before Coder
// locks inactive workspaces created from this template.
InactivityTTLMillis *int64 `json:"inactivity_ttl_ms,omitempty"`
// LockedTTL allows optionally specifying the max lifetime before Coder
// LockedTTLMillis allows optionally specifying the max lifetime before Coder
// permanently deletes locked workspaces created from this template.
LockedTTL *int64 `json:"locked_ttl_ms,omitempty"`
LockedTTLMillis *int64 `json:"locked_ttl_ms,omitempty"`
// DisableEveryoneGroupAccess allows optionally disabling the default
// behavior of granting the 'everyone' group access to use the template.

View File

@ -112,6 +112,11 @@ type LicenseOptions struct {
Features license.Features
}
// AddFullLicense generates a license with all features enabled.
func AddFullLicense(t *testing.T, client *codersdk.Client) codersdk.License {
return AddLicense(t, client, LicenseOptions{AllFeatures: true})
}
// AddLicense generates a new license with the options provided and inserts it.
func AddLicense(t *testing.T, client *codersdk.Client, options LicenseOptions) codersdk.License {
l, err := client.AddLicense(context.Background(), codersdk.AddLicenseRequest{

View File

@ -206,6 +206,55 @@ func TestTemplates(t *testing.T) {
require.NoError(t, err)
require.EqualValues(t, exp, *ws.TTLMillis)
})
t.Run("CleanupTTLs", func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitMedium)
client := coderdenttest.New(t, &coderdenttest.Options{
Options: &coderdtest.Options{
IncludeProvisionerDaemon: true,
},
})
user := coderdtest.CreateFirstUser(t, client)
_ = coderdenttest.AddFullLicense(t, client)
version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, nil)
coderdtest.AwaitTemplateVersionJob(t, client, version.ID)
template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID)
require.EqualValues(t, 0, template.InactivityTTLMillis)
require.EqualValues(t, 0, template.FailureTTLMillis)
require.EqualValues(t, 0, template.LockedTTLMillis)
var (
failureTTL int64 = 1
inactivityTTL int64 = 2
lockedTTL int64 = 3
)
updated, err := client.UpdateTemplateMeta(ctx, template.ID, codersdk.UpdateTemplateMeta{
Name: template.Name,
DisplayName: template.DisplayName,
Description: template.Description,
Icon: template.Icon,
AllowUserCancelWorkspaceJobs: template.AllowUserCancelWorkspaceJobs,
InactivityTTLMillis: inactivityTTL,
FailureTTLMillis: failureTTL,
LockedTTLMillis: lockedTTL,
})
require.NoError(t, err)
require.Equal(t, failureTTL, updated.FailureTTLMillis)
require.Equal(t, inactivityTTL, updated.InactivityTTLMillis)
require.Equal(t, lockedTTL, updated.LockedTTLMillis)
// Validate fetching the template returns the same values as updating
// the template.
template, err = client.Template(ctx, template.ID)
require.NoError(t, err)
require.Equal(t, failureTTL, updated.FailureTTLMillis)
require.Equal(t, inactivityTTL, updated.InactivityTTLMillis)
require.Equal(t, lockedTTL, updated.LockedTTLMillis)
})
}
func TestTemplateACL(t *testing.T) {

View File

@ -15,6 +15,7 @@ import (
"github.com/coder/coder/coderd/autobuild"
"github.com/coder/coder/coderd/coderdtest"
"github.com/coder/coder/coderd/database"
"github.com/coder/coder/coderd/schedule"
"github.com/coder/coder/coderd/util/ptr"
"github.com/coder/coder/codersdk"
"github.com/coder/coder/enterprise/coderd"
@ -93,7 +94,7 @@ func TestWorkspaceAutobuild(t *testing.T) {
// builds.
IgnoreErrors: true,
})
failureTTL = time.Millisecond
failureTTL = time.Minute
client = coderdenttest.New(t, &coderdenttest.Options{
Options: &coderdtest.Options{
@ -106,11 +107,7 @@ func TestWorkspaceAutobuild(t *testing.T) {
})
)
user := coderdtest.CreateFirstUser(t, client)
_ = coderdenttest.AddLicense(t, client, coderdenttest.LicenseOptions{
Features: license.Features{
codersdk.FeatureAdvancedTemplateScheduling: 1,
},
})
_ = coderdenttest.AddFullLicense(t, client)
version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{
Parse: echo.ParseComplete,
@ -124,12 +121,7 @@ func TestWorkspaceAutobuild(t *testing.T) {
ws := coderdtest.CreateWorkspace(t, client, user.OrganizationID, template.ID)
build := coderdtest.AwaitWorkspaceBuildJob(t, client, ws.LatestBuild.ID)
require.Equal(t, codersdk.WorkspaceStatusFailed, build.Status)
require.Eventually(t,
func() bool {
return database.Now().Sub(*build.Job.CompletedAt) > failureTTL
},
testutil.IntervalMedium, testutil.IntervalFast)
ticker <- time.Now()
ticker <- build.Job.CompletedAt.Add(failureTTL * 2)
stats := <-statCh
// Expect workspace to transition to stopped state for breaching
// failure TTL.
@ -161,11 +153,7 @@ func TestWorkspaceAutobuild(t *testing.T) {
})
)
user := coderdtest.CreateFirstUser(t, client)
_ = coderdenttest.AddLicense(t, client, coderdenttest.LicenseOptions{
Features: license.Features{
codersdk.FeatureAdvancedTemplateScheduling: 1,
},
})
_ = coderdenttest.AddFullLicense(t, client)
version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{
Parse: echo.ParseComplete,
ProvisionPlan: echo.ProvisionComplete,
@ -178,13 +166,16 @@ func TestWorkspaceAutobuild(t *testing.T) {
ws := coderdtest.CreateWorkspace(t, client, user.OrganizationID, template.ID)
build := coderdtest.AwaitWorkspaceBuildJob(t, client, ws.LatestBuild.ID)
require.Equal(t, codersdk.WorkspaceStatusFailed, build.Status)
ticker <- time.Now()
// Make it impossible to trigger the failure TTL.
ticker <- build.Job.CompletedAt.Add(-failureTTL * 2)
stats := <-statCh
// Expect no transitions since not enough time has elapsed.
require.Len(t, stats.Transitions, 0)
})
t.Run("FailureTTLUnset", func(t *testing.T) {
// This just provides a baseline that no actions are being taken
// against a workspace when none of the TTL fields are set.
t.Run("TemplateTTLsUnset", func(t *testing.T) {
t.Parallel()
var (
@ -207,25 +198,419 @@ func TestWorkspaceAutobuild(t *testing.T) {
})
)
user := coderdtest.CreateFirstUser(t, client)
_ = coderdenttest.AddLicense(t, client, coderdenttest.LicenseOptions{
Features: license.Features{
codersdk.FeatureAdvancedTemplateScheduling: 1,
},
})
_ = coderdenttest.AddFullLicense(t, client)
version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{
Parse: echo.ParseComplete,
ProvisionPlan: echo.ProvisionComplete,
ProvisionApply: echo.ProvisionFailed,
ProvisionApply: echo.ProvisionComplete,
})
// Create a template without setting a failure_ttl.
template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID)
require.Zero(t, template.InactivityTTLMillis)
require.Zero(t, template.FailureTTLMillis)
require.Zero(t, template.LockedTTLMillis)
coderdtest.AwaitTemplateVersionJob(t, client, version.ID)
ws := coderdtest.CreateWorkspace(t, client, user.OrganizationID, template.ID)
build := coderdtest.AwaitWorkspaceBuildJob(t, client, ws.LatestBuild.ID)
require.Equal(t, codersdk.WorkspaceStatusFailed, build.Status)
require.Equal(t, codersdk.WorkspaceStatusRunning, build.Status)
ticker <- time.Now()
stats := <-statCh
// Expect no transitions since the field is unset on the template.
// Expect no transitions since the fields are unset on the template.
require.Len(t, stats.Transitions, 0)
})
t.Run("InactiveTTLOK", func(t *testing.T) {
t.Parallel()
var (
ctx = testutil.Context(t, testutil.WaitMedium)
ticker = make(chan time.Time)
statCh = make(chan autobuild.Stats)
inactiveTTL = time.Minute
client = coderdenttest.New(t, &coderdenttest.Options{
Options: &coderdtest.Options{
AutobuildTicker: ticker,
IncludeProvisionerDaemon: true,
AutobuildStats: statCh,
TemplateScheduleStore: &coderd.EnterpriseTemplateScheduleStore{},
},
})
)
user := coderdtest.CreateFirstUser(t, client)
_ = coderdenttest.AddFullLicense(t, client)
version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{
Parse: echo.ParseComplete,
ProvisionPlan: echo.ProvisionComplete,
ProvisionApply: echo.ProvisionComplete,
})
template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID, func(ctr *codersdk.CreateTemplateRequest) {
ctr.InactivityTTLMillis = ptr.Ref[int64](inactiveTTL.Milliseconds())
})
coderdtest.AwaitTemplateVersionJob(t, client, version.ID)
ws := coderdtest.CreateWorkspace(t, client, user.OrganizationID, template.ID)
build := coderdtest.AwaitWorkspaceBuildJob(t, client, ws.LatestBuild.ID)
require.Equal(t, codersdk.WorkspaceStatusRunning, build.Status)
// Simulate being inactive.
ticker <- ws.LastUsedAt.Add(inactiveTTL * 2)
stats := <-statCh
// Expect workspace to transition to stopped state for breaching
// failure TTL.
require.Len(t, stats.Transitions, 1)
require.Equal(t, stats.Transitions[ws.ID], database.WorkspaceTransitionStop)
// The workspace should be locked.
ws = coderdtest.MustWorkspace(t, client, ws.ID)
require.NotNil(t, ws.LockedAt)
lastUsedAt := ws.LastUsedAt
err := client.UpdateWorkspaceLock(ctx, ws.ID, codersdk.UpdateWorkspaceLock{Lock: false})
require.NoError(t, err)
// Assert that we updated our last_used_at so that we don't immediately
// retrigger another lock action.
ws = coderdtest.MustWorkspace(t, client, ws.ID)
require.True(t, ws.LastUsedAt.After(lastUsedAt))
})
t.Run("InactiveTTLTooEarly", func(t *testing.T) {
t.Parallel()
var (
ticker = make(chan time.Time)
statCh = make(chan autobuild.Stats)
inactiveTTL = time.Minute
client = coderdenttest.New(t, &coderdenttest.Options{
Options: &coderdtest.Options{
AutobuildTicker: ticker,
IncludeProvisionerDaemon: true,
AutobuildStats: statCh,
TemplateScheduleStore: &coderd.EnterpriseTemplateScheduleStore{},
},
})
)
user := coderdtest.CreateFirstUser(t, client)
_ = coderdenttest.AddFullLicense(t, client)
version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{
Parse: echo.ParseComplete,
ProvisionPlan: echo.ProvisionComplete,
ProvisionApply: echo.ProvisionComplete,
})
template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID, func(ctr *codersdk.CreateTemplateRequest) {
ctr.InactivityTTLMillis = ptr.Ref[int64](inactiveTTL.Milliseconds())
})
coderdtest.AwaitTemplateVersionJob(t, client, version.ID)
ws := coderdtest.CreateWorkspace(t, client, user.OrganizationID, template.ID)
build := coderdtest.AwaitWorkspaceBuildJob(t, client, ws.LatestBuild.ID)
require.Equal(t, codersdk.WorkspaceStatusRunning, build.Status)
// Make it impossible to trigger the inactive ttl.
ticker <- ws.LastUsedAt.Add(-inactiveTTL)
stats := <-statCh
// Expect no transitions since not enough time has elapsed.
require.Len(t, stats.Transitions, 0)
})
// This is kind of a dumb test but it exists to offer some marginal
// confidence that a bug in the auto-deletion logic doesn't delete running
// workspaces.
t.Run("UnlockedWorkspacesNotDeleted", func(t *testing.T) {
t.Parallel()
var (
ticker = make(chan time.Time)
statCh = make(chan autobuild.Stats)
lockedTTL = time.Minute
client = coderdenttest.New(t, &coderdenttest.Options{
Options: &coderdtest.Options{
AutobuildTicker: ticker,
IncludeProvisionerDaemon: true,
AutobuildStats: statCh,
TemplateScheduleStore: &coderd.EnterpriseTemplateScheduleStore{},
},
})
)
user := coderdtest.CreateFirstUser(t, client)
_ = coderdenttest.AddFullLicense(t, client)
version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{
Parse: echo.ParseComplete,
ProvisionPlan: echo.ProvisionComplete,
ProvisionApply: echo.ProvisionComplete,
})
template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID, func(ctr *codersdk.CreateTemplateRequest) {
ctr.LockedTTLMillis = ptr.Ref[int64](lockedTTL.Milliseconds())
})
coderdtest.AwaitTemplateVersionJob(t, client, version.ID)
ws := coderdtest.CreateWorkspace(t, client, user.OrganizationID, template.ID)
build := coderdtest.AwaitWorkspaceBuildJob(t, client, ws.LatestBuild.ID)
require.Nil(t, ws.LockedAt)
require.Equal(t, codersdk.WorkspaceStatusRunning, build.Status)
ticker <- ws.LastUsedAt.Add(lockedTTL * 2)
stats := <-statCh
// Expect no transitions since workspace is unlocked.
require.Len(t, stats.Transitions, 0)
})
// Assert that a stopped workspace that breaches the inactivity threshold
// does not trigger a build transition but is still placed in the
// lock state.
t.Run("InactiveStoppedWorkspaceNoTransition", func(t *testing.T) {
t.Parallel()
var (
ticker = make(chan time.Time)
statCh = make(chan autobuild.Stats)
inactiveTTL = time.Minute
client = coderdenttest.New(t, &coderdenttest.Options{
Options: &coderdtest.Options{
AutobuildTicker: ticker,
IncludeProvisionerDaemon: true,
AutobuildStats: statCh,
TemplateScheduleStore: &coderd.EnterpriseTemplateScheduleStore{},
},
})
)
user := coderdtest.CreateFirstUser(t, client)
_ = coderdenttest.AddFullLicense(t, client)
version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{
Parse: echo.ParseComplete,
ProvisionPlan: echo.ProvisionComplete,
ProvisionApply: echo.ProvisionComplete,
})
template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID, func(ctr *codersdk.CreateTemplateRequest) {
ctr.InactivityTTLMillis = ptr.Ref[int64](inactiveTTL.Milliseconds())
})
coderdtest.AwaitTemplateVersionJob(t, client, version.ID)
ws := coderdtest.CreateWorkspace(t, client, user.OrganizationID, template.ID)
build := coderdtest.AwaitWorkspaceBuildJob(t, client, ws.LatestBuild.ID)
require.Equal(t, codersdk.WorkspaceStatusRunning, build.Status)
// Stop the workspace so we can assert autobuild does nothing
// if we breach our inactivity threshold.
ws = coderdtest.MustTransitionWorkspace(t, client, ws.ID, database.WorkspaceTransitionStart, database.WorkspaceTransitionStop)
// Simulate not having accessed the workspace in a while.
ticker <- ws.LastUsedAt.Add(2 * inactiveTTL)
stats := <-statCh
// Expect no transitions since workspace is stopped.
require.Len(t, stats.Transitions, 0)
ws = coderdtest.MustWorkspace(t, client, ws.ID)
// The workspace should still be locked even though we didn't
// transition the workspace.
require.NotNil(t, ws.LockedAt)
})
// Test the flow of a workspace transitioning from
// inactive -> locked -> deleted.
t.Run("WorkspaceInactiveDeleteTransition", func(t *testing.T) {
t.Parallel()
var (
ticker = make(chan time.Time)
statCh = make(chan autobuild.Stats)
transitionTTL = time.Minute
client = coderdenttest.New(t, &coderdenttest.Options{
Options: &coderdtest.Options{
AutobuildTicker: ticker,
IncludeProvisionerDaemon: true,
AutobuildStats: statCh,
TemplateScheduleStore: &coderd.EnterpriseTemplateScheduleStore{},
},
})
)
user := coderdtest.CreateFirstUser(t, client)
_ = coderdenttest.AddFullLicense(t, client)
version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{
Parse: echo.ParseComplete,
ProvisionPlan: echo.ProvisionComplete,
ProvisionApply: echo.ProvisionComplete,
})
template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID, func(ctr *codersdk.CreateTemplateRequest) {
ctr.InactivityTTLMillis = ptr.Ref[int64](transitionTTL.Milliseconds())
ctr.LockedTTLMillis = ptr.Ref[int64](transitionTTL.Milliseconds())
})
coderdtest.AwaitTemplateVersionJob(t, client, version.ID)
ws := coderdtest.CreateWorkspace(t, client, user.OrganizationID, template.ID)
build := coderdtest.AwaitWorkspaceBuildJob(t, client, ws.LatestBuild.ID)
require.Equal(t, codersdk.WorkspaceStatusRunning, build.Status)
// Simulate not having accessed the workspace in a while.
ticker <- ws.LastUsedAt.Add(2 * transitionTTL)
stats := <-statCh
// Expect workspace to transition to stopped state for breaching
// inactive TTL.
require.Len(t, stats.Transitions, 1)
require.Equal(t, stats.Transitions[ws.ID], database.WorkspaceTransitionStop)
ws = coderdtest.MustWorkspace(t, client, ws.ID)
// The workspace should be locked.
require.NotNil(t, ws.LockedAt)
// Wait for the autobuilder to stop the workspace.
_ = coderdtest.AwaitWorkspaceBuildJob(t, client, ws.LatestBuild.ID)
// Simulate the workspace being locked beyond the threshold.
ticker <- ws.LockedAt.Add(2 * transitionTTL)
stats = <-statCh
require.Len(t, stats.Transitions, 1)
// The workspace should be scheduled for deletion.
require.Equal(t, stats.Transitions[ws.ID], database.WorkspaceTransitionDelete)
// Wait for the workspace to be deleted.
ws = coderdtest.MustWorkspace(t, client, ws.ID)
_ = coderdtest.AwaitWorkspaceBuildJob(t, client, ws.LatestBuild.ID)
// Assert that the workspace is actually deleted.
_, err := client.Workspace(testutil.Context(t, testutil.WaitShort), ws.ID)
require.Error(t, err)
cerr, ok := codersdk.AsError(err)
require.True(t, ok)
require.Equal(t, http.StatusGone, cerr.StatusCode())
})
t.Run("LockedTTTooEarly", func(t *testing.T) {
t.Parallel()
var (
ticker = make(chan time.Time)
statCh = make(chan autobuild.Stats)
lockedTTL = time.Minute
client = coderdenttest.New(t, &coderdenttest.Options{
Options: &coderdtest.Options{
AutobuildTicker: ticker,
IncludeProvisionerDaemon: true,
AutobuildStats: statCh,
TemplateScheduleStore: &coderd.EnterpriseTemplateScheduleStore{},
},
})
)
user := coderdtest.CreateFirstUser(t, client)
_ = coderdenttest.AddFullLicense(t, client)
version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{
Parse: echo.ParseComplete,
ProvisionPlan: echo.ProvisionComplete,
ProvisionApply: echo.ProvisionComplete,
})
template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID, func(ctr *codersdk.CreateTemplateRequest) {
ctr.LockedTTLMillis = ptr.Ref[int64](lockedTTL.Milliseconds())
})
coderdtest.AwaitTemplateVersionJob(t, client, version.ID)
ws := coderdtest.CreateWorkspace(t, client, user.OrganizationID, template.ID)
build := coderdtest.AwaitWorkspaceBuildJob(t, client, ws.LatestBuild.ID)
require.Equal(t, codersdk.WorkspaceStatusRunning, build.Status)
ctx := testutil.Context(t, testutil.WaitMedium)
err := client.UpdateWorkspaceLock(ctx, ws.ID, codersdk.UpdateWorkspaceLock{
Lock: true,
})
require.NoError(t, err)
ws = coderdtest.MustWorkspace(t, client, ws.ID)
require.NotNil(t, ws.LockedAt)
// Ensure we haven't breached our threshold.
ticker <- ws.LockedAt.Add(-lockedTTL * 2)
stats := <-statCh
// Expect no transitions since not enough time has elapsed.
require.Len(t, stats.Transitions, 0)
_, err = client.UpdateTemplateMeta(ctx, template.ID, codersdk.UpdateTemplateMeta{
LockedTTLMillis: lockedTTL.Milliseconds(),
})
require.NoError(t, err)
// Simlute the workspace breaching the threshold.
ticker <- ws.LockedAt.Add(lockedTTL * 2)
stats = <-statCh
require.Len(t, stats.Transitions, 1)
require.Equal(t, database.WorkspaceTransitionDelete, stats.Transitions[ws.ID])
})
// Assert that a locked workspace does not autostart.
t.Run("LockedNoAutostart", func(t *testing.T) {
t.Parallel()
var (
ctx = testutil.Context(t, testutil.WaitMedium)
tickCh = make(chan time.Time)
statsCh = make(chan autobuild.Stats)
client = coderdenttest.New(t, &coderdenttest.Options{
Options: &coderdtest.Options{
AutobuildTicker: tickCh,
IncludeProvisionerDaemon: true,
AutobuildStats: statsCh,
TemplateScheduleStore: &coderd.EnterpriseTemplateScheduleStore{},
},
})
inactiveTTL = time.Minute
)
user := coderdtest.CreateFirstUser(t, client)
_ = coderdenttest.AddFullLicense(t, client)
version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{
Parse: echo.ParseComplete,
ProvisionPlan: echo.ProvisionComplete,
ProvisionApply: echo.ProvisionComplete,
})
coderdtest.AwaitTemplateVersionJob(t, client, version.ID)
template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID)
sched, err := schedule.Weekly("CRON_TZ=UTC 0 * * * *")
require.NoError(t, err)
ws := coderdtest.CreateWorkspace(t, client, user.OrganizationID, template.ID, func(cwr *codersdk.CreateWorkspaceRequest) {
cwr.AutostartSchedule = ptr.Ref(sched.String())
})
coderdtest.AwaitWorkspaceBuildJob(t, client, ws.LatestBuild.ID)
coderdtest.MustTransitionWorkspace(t, client, ws.ID, database.WorkspaceTransitionStart, database.WorkspaceTransitionStop)
// Assert that autostart works when the workspace isn't locked..
tickCh <- sched.Next(ws.LatestBuild.CreatedAt)
stats := <-statsCh
require.NoError(t, stats.Error)
require.Len(t, stats.Transitions, 1)
require.Contains(t, stats.Transitions, ws.ID)
require.Equal(t, database.WorkspaceTransitionStart, stats.Transitions[ws.ID])
ws = coderdtest.MustWorkspace(t, client, ws.ID)
coderdtest.AwaitWorkspaceBuildJob(t, client, ws.LatestBuild.ID)
// Now that we've validated that the workspace is eligible for autostart
// lets cause it to become locked.
_, err = client.UpdateTemplateMeta(ctx, template.ID, codersdk.UpdateTemplateMeta{
InactivityTTLMillis: inactiveTTL.Milliseconds(),
})
require.NoError(t, err)
// We should see the workspace get stopped now.
tickCh <- ws.LastUsedAt.Add(inactiveTTL * 2)
stats = <-statsCh
require.NoError(t, stats.Error)
require.Len(t, stats.Transitions, 1)
require.Contains(t, stats.Transitions, ws.ID)
require.Equal(t, database.WorkspaceTransitionStop, stats.Transitions[ws.ID])
// The workspace should be locked now.
ws = coderdtest.MustWorkspace(t, client, ws.ID)
coderdtest.AwaitWorkspaceBuildJob(t, client, ws.LatestBuild.ID)
require.NotNil(t, ws.LockedAt)
// Assert that autostart is no longer triggered since workspace is locked.
tickCh <- sched.Next(ws.LatestBuild.CreatedAt)
stats = <-statsCh
require.Len(t, stats.Transitions, 0)
})
}