selfhosted
/
coder
mirror of https://github.com/coder/coder.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

feat(coderd/database): use `template_usage_stats` in `GetTemplateInsights` query (#12666) 

This PR updates the `GetTemplateInsights` query to use rolled up `template_usage_stats` instead of raw agent and app stats.
This commit is contained in:
Mathias Fredriksson 2024-03-25 15:33:31 +02:00 committed by GitHub
parent f34592f45d
commit 35d08434a9
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
17 changed files with 329 additions and 223 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

195
coderd/database/dbmem/dbmem.go
View File

@ -16,6 +16,7 @@ import (
"github.com/google/uuid"
"github.com/lib/pq"
"golang.org/x/exp/constraints"
"golang.org/x/exp/maps"
"golang.org/x/exp/slices"
"golang.org/x/xerrors"
@ -791,6 +792,13 @@ func tagsSubset(m1, m2 map[string]string) bool {
// default tags when no tag is specified for a provisioner or job
var tagsUntagged = provisionersdk.MutateTags(uuid.Nil, nil)
func least[T constraints.Ordered](a, b T) T {
if a < b {
return a
}
return b
}
func (*FakeQuerier) AcquireLock(_ context.Context, _ int64) error {
return xerrors.New("AcquireLock must only be called within a transaction")
}
@ -3237,71 +3245,166 @@ func (q *FakeQuerier) GetTemplateInsights(_ context.Context, arg database.GetTem
return database.GetTemplateInsightsRow{}, err
}
templateIDSet := make(map[uuid.UUID]struct{})
appUsageIntervalsByUser := make(map[uuid.UUID]map[time.Time]*database.GetTemplateInsightsRow)
q.mutex.RLock()
defer q.mutex.RUnlock()
for _, s := range q.workspaceAgentStats {
if s.CreatedAt.Before(arg.StartTime) || s.CreatedAt.Equal(arg.EndTime) || s.CreatedAt.After(arg.EndTime) {
/*
WITH
*/
/*
insights AS (
SELECT
user_id,
-- See motivation in GetTemplateInsights for LEAST(SUM(n), 30).
LEAST(SUM(usage_mins), 30) AS usage_mins,
LEAST(SUM(ssh_mins), 30) AS ssh_mins,
LEAST(SUM(sftp_mins), 30) AS sftp_mins,
LEAST(SUM(reconnecting_pty_mins), 30) AS reconnecting_pty_mins,
LEAST(SUM(vscode_mins), 30) AS vscode_mins,
LEAST(SUM(jetbrains_mins), 30) AS jetbrains_mins
FROM
template_usage_stats
WHERE
start_time >= @start_time::timestamptz
AND end_time <= @end_time::timestamptz
AND CASE WHEN COALESCE(array_length(@template_ids::uuid[], 1), 0) > 0 THEN template_id = ANY(@template_ids::uuid[]) ELSE TRUE END
GROUP BY
start_time, user_id
),
*/
type insightsGroupBy struct {
StartTime time.Time
UserID uuid.UUID
}
type insightsRow struct {
insightsGroupBy
UsageMins int16
SSHMins int16
SFTPMins int16
ReconnectingPTYMins int16
VSCodeMins int16
JetBrainsMins int16
}
insights := make(map[insightsGroupBy]insightsRow)
for _, stat := range q.templateUsageStats {
if stat.StartTime.Before(arg.StartTime) || stat.EndTime.After(arg.EndTime) {
continue
}
if len(arg.TemplateIDs) > 0 && !slices.Contains(arg.TemplateIDs, s.TemplateID) {
if len(arg.TemplateIDs) > 0 && !slices.Contains(arg.TemplateIDs, stat.TemplateID) {
continue
}
if s.ConnectionCount == 0 {
key := insightsGroupBy{
StartTime: stat.StartTime,
UserID: stat.UserID,
}
row, ok := insights[key]
if !ok {
row = insightsRow{
insightsGroupBy: key,
}
}
row.UsageMins = least(row.UsageMins+stat.UsageMins, 30)
row.SSHMins = least(row.SSHMins+stat.SshMins, 30)
row.SFTPMins = least(row.SFTPMins+stat.SftpMins, 30)
row.ReconnectingPTYMins = least(row.ReconnectingPTYMins+stat.ReconnectingPtyMins, 30)
row.VSCodeMins = least(row.VSCodeMins+stat.VscodeMins, 30)
row.JetBrainsMins = least(row.JetBrainsMins+stat.JetbrainsMins, 30)
insights[key] = row
}
/*
templates AS (
SELECT
array_agg(DISTINCT template_id) AS template_ids,
array_agg(DISTINCT template_id) FILTER (WHERE ssh_mins > 0) AS ssh_template_ids,
array_agg(DISTINCT template_id) FILTER (WHERE sftp_mins > 0) AS sftp_template_ids,
array_agg(DISTINCT template_id) FILTER (WHERE reconnecting_pty_mins > 0) AS reconnecting_pty_template_ids,
array_agg(DISTINCT template_id) FILTER (WHERE vscode_mins > 0) AS vscode_template_ids,
array_agg(DISTINCT template_id) FILTER (WHERE jetbrains_mins > 0) AS jetbrains_template_ids
FROM
template_usage_stats
WHERE
start_time >= @start_time::timestamptz
AND end_time <= @end_time::timestamptz
AND CASE WHEN COALESCE(array_length(@template_ids::uuid[], 1), 0) > 0 THEN template_id = ANY(@template_ids::uuid[]) ELSE TRUE END
)
*/
type templateRow struct {
TemplateIDs []uuid.UUID
SSHTemplateIDs []uuid.UUID
SFTPTemplateIDs []uuid.UUID
ReconnectingPTYIDs []uuid.UUID
VSCodeTemplateIDs []uuid.UUID
JetBrainsTemplateIDs []uuid.UUID
}
templates := templateRow{}
for _, stat := range q.templateUsageStats {
if stat.StartTime.Before(arg.StartTime) || stat.EndTime.After(arg.EndTime) {
continue
}
templateIDSet[s.TemplateID] = struct{}{}
if appUsageIntervalsByUser[s.UserID] == nil {
appUsageIntervalsByUser[s.UserID] = make(map[time.Time]*database.GetTemplateInsightsRow)
if len(arg.TemplateIDs) > 0 && !slices.Contains(arg.TemplateIDs, stat.TemplateID) {
continue
}
t := s.CreatedAt.Truncate(time.Minute)
if _, ok := appUsageIntervalsByUser[s.UserID][t]; !ok {
appUsageIntervalsByUser[s.UserID][t] = &database.GetTemplateInsightsRow{}
templates.TemplateIDs = append(templates.TemplateIDs, stat.TemplateID)
if stat.SshMins > 0 {
templates.SSHTemplateIDs = append(templates.SSHTemplateIDs, stat.TemplateID)
}
if s.SessionCountJetBrains > 0 {
appUsageIntervalsByUser[s.UserID][t].UsageJetbrainsSeconds = 60
if stat.SftpMins > 0 {
templates.SFTPTemplateIDs = append(templates.SFTPTemplateIDs, stat.TemplateID)
}
if s.SessionCountVSCode > 0 {
appUsageIntervalsByUser[s.UserID][t].UsageVscodeSeconds = 60
if stat.ReconnectingPtyMins > 0 {
templates.ReconnectingPTYIDs = append(templates.ReconnectingPTYIDs, stat.TemplateID)
}
if s.SessionCountReconnectingPTY > 0 {
appUsageIntervalsByUser[s.UserID][t].UsageReconnectingPtySeconds = 60
if stat.VscodeMins > 0 {
templates.VSCodeTemplateIDs = append(templates.VSCodeTemplateIDs, stat.TemplateID)
}
if s.SessionCountSSH > 0 {
appUsageIntervalsByUser[s.UserID][t].UsageSshSeconds = 60
if stat.JetbrainsMins > 0 {
templates.JetBrainsTemplateIDs = append(templates.JetBrainsTemplateIDs, stat.TemplateID)
}
}
templateIDs := make([]uuid.UUID, 0, len(templateIDSet))
for templateID := range templateIDSet {
templateIDs = append(templateIDs, templateID)
}
slices.SortFunc(templateIDs, func(a, b uuid.UUID) int {
return slice.Ascending(a.String(), b.String())
})
activeUserIDs := make([]uuid.UUID, 0, len(appUsageIntervalsByUser))
for userID := range appUsageIntervalsByUser {
activeUserIDs = append(activeUserIDs, userID)
}
/*
SELECT
COALESCE((SELECT template_ids FROM templates), '{}')::uuid[] AS template_ids, -- Includes app usage.
COALESCE((SELECT ssh_template_ids FROM templates), '{}')::uuid[] AS ssh_template_ids,
COALESCE((SELECT sftp_template_ids FROM templates), '{}')::uuid[] AS sftp_template_ids,
COALESCE((SELECT reconnecting_pty_template_ids FROM templates), '{}')::uuid[] AS reconnecting_pty_template_ids,
COALESCE((SELECT vscode_template_ids FROM templates), '{}')::uuid[] AS vscode_template_ids,
COALESCE((SELECT jetbrains_template_ids FROM templates), '{}')::uuid[] AS jetbrains_template_ids,
COALESCE(COUNT(DISTINCT user_id), 0)::bigint AS active_users, -- Includes app usage.
COALESCE(SUM(usage_mins) * 60, 0)::bigint AS usage_total_seconds, -- Includes app usage.
COALESCE(SUM(ssh_mins) * 60, 0)::bigint AS usage_ssh_seconds,
COALESCE(SUM(sftp_mins) * 60, 0)::bigint AS usage_sftp_seconds,
COALESCE(SUM(reconnecting_pty_mins) * 60, 0)::bigint AS usage_reconnecting_pty_seconds,
COALESCE(SUM(vscode_mins) * 60, 0)::bigint AS usage_vscode_seconds,
COALESCE(SUM(jetbrains_mins) * 60, 0)::bigint AS usage_jetbrains_seconds
FROM
insights;
*/
result := database.GetTemplateInsightsRow{
TemplateIDs: templateIDs,
ActiveUserIDs: activeUserIDs,
var row database.GetTemplateInsightsRow
row.TemplateIDs = uniqueSortedUUIDs(templates.TemplateIDs)
row.SshTemplateIds = uniqueSortedUUIDs(templates.SSHTemplateIDs)
row.SftpTemplateIds = uniqueSortedUUIDs(templates.SFTPTemplateIDs)
row.ReconnectingPtyTemplateIds = uniqueSortedUUIDs(templates.ReconnectingPTYIDs)
row.VscodeTemplateIds = uniqueSortedUUIDs(templates.VSCodeTemplateIDs)
row.JetbrainsTemplateIds = uniqueSortedUUIDs(templates.JetBrainsTemplateIDs)
activeUserIDs := make(map[uuid.UUID]struct{})
for _, insight := range insights {
activeUserIDs[insight.UserID] = struct{}{}
row.UsageTotalSeconds += int64(insight.UsageMins) * 60
row.UsageSshSeconds += int64(insight.SSHMins) * 60
row.UsageSftpSeconds += int64(insight.SFTPMins) * 60
row.UsageReconnectingPtySeconds += int64(insight.ReconnectingPTYMins) * 60
row.UsageVscodeSeconds += int64(insight.VSCodeMins) * 60
row.UsageJetbrainsSeconds += int64(insight.JetBrainsMins) * 60
}
for _, intervals := range appUsageIntervalsByUser {
for _, interval := range intervals {
result.UsageJetbrainsSeconds += interval.UsageJetbrainsSeconds
result.UsageVscodeSeconds += interval.UsageVscodeSeconds
result.UsageReconnectingPtySeconds += interval.UsageReconnectingPtySeconds
result.UsageSshSeconds += interval.UsageSshSeconds
}
}
return result, nil
row.ActiveUsers = int64(len(activeUserIDs))
return row, nil
}
func (q *FakeQuerier) GetTemplateInsightsByInterval(ctx context.Context, arg database.GetTemplateInsightsByIntervalParams) ([]database.GetTemplateInsightsByIntervalRow, error) {

12
coderd/database/querier.go
View File

@ -178,9 +178,15 @@ type sqlcQuerier interface {
GetTemplateByID(ctx context.Context, id uuid.UUID) (Template, error)
GetTemplateByOrganizationAndName(ctx context.Context, arg GetTemplateByOrganizationAndNameParams) (Template, error)
GetTemplateDAUs(ctx context.Context, arg GetTemplateDAUsParams) ([]GetTemplateDAUsRow, error)
// GetTemplateInsights has a granularity of 5 minutes where if a session/app was
// in use during a minute, we will add 5 minutes to the total usage for that
// session/app (per user).
// GetTemplateInsights returns the aggregate user-produced usage of all
// workspaces in a given timeframe. The template IDs, active users, and
// usage_seconds all reflect any usage in the template, including apps.
//
// When combining data from multiple templates, we must make a guess at
// how the user behaved for the 30 minute interval. In this case we make
// the assumption that if the user used two workspaces for 15 minutes,
// they did so sequentially, thus we sum the usage up to a maximum of
// 30 minutes with LEAST(SUM(n), 30).
GetTemplateInsights(ctx context.Context, arg GetTemplateInsightsParams) (GetTemplateInsightsRow, error)
// GetTemplateInsightsByInterval returns all intervals between start and end
// time, if end time is a partial interval, it will be included in the results and

117
coderd/database/queries.sql.go
View File

@ -1894,37 +1894,58 @@ func (q *sqlQuerier) GetTemplateAppInsightsByTemplate(ctx context.Context, arg G
}
const getTemplateInsights = `-- name: GetTemplateInsights :one
WITH agent_stats_by_interval_and_user AS (
SELECT
date_trunc('minute', was.created_at),
was.user_id,
array_agg(was.template_id) AS template_ids,
CASE WHEN SUM(was.session_count_vscode) > 0 THEN 60 ELSE 0 END AS usage_vscode_seconds,
CASE WHEN SUM(was.session_count_jetbrains) > 0 THEN 60 ELSE 0 END AS usage_jetbrains_seconds,
CASE WHEN SUM(was.session_count_reconnecting_pty) > 0 THEN 60 ELSE 0 END AS usage_reconnecting_pty_seconds,
CASE WHEN SUM(was.session_count_ssh) > 0 THEN 60 ELSE 0 END AS usage_ssh_seconds
FROM workspace_agent_stats was
WHERE
was.created_at >= $1::timestamptz
AND was.created_at < $2::timestamptz
AND was.connection_count > 0
AND CASE WHEN COALESCE(array_length($3::uuid[], 1), 0) > 0 THEN was.template_id = ANY($3::uuid[]) ELSE TRUE END
GROUP BY date_trunc('minute', was.created_at), was.user_id
), template_ids AS (
SELECT array_agg(DISTINCT template_id) AS ids
FROM agent_stats_by_interval_and_user, unnest(template_ids) template_id
WHERE template_id IS NOT NULL
)
WITH
insights AS (
SELECT
user_id,
-- See motivation in GetTemplateInsights for LEAST(SUM(n), 30).
LEAST(SUM(usage_mins), 30) AS usage_mins,
LEAST(SUM(ssh_mins), 30) AS ssh_mins,
LEAST(SUM(sftp_mins), 30) AS sftp_mins,
LEAST(SUM(reconnecting_pty_mins), 30) AS reconnecting_pty_mins,
LEAST(SUM(vscode_mins), 30) AS vscode_mins,
LEAST(SUM(jetbrains_mins), 30) AS jetbrains_mins
FROM
template_usage_stats
WHERE
start_time >= $1::timestamptz
AND end_time <= $2::timestamptz
AND CASE WHEN COALESCE(array_length($3::uuid[], 1), 0) > 0 THEN template_id = ANY($3::uuid[]) ELSE TRUE END
GROUP BY
start_time, user_id
),
templates AS (
SELECT
array_agg(DISTINCT template_id) AS template_ids,
array_agg(DISTINCT template_id) FILTER (WHERE ssh_mins > 0) AS ssh_template_ids,
array_agg(DISTINCT template_id) FILTER (WHERE sftp_mins > 0) AS sftp_template_ids,
array_agg(DISTINCT template_id) FILTER (WHERE reconnecting_pty_mins > 0) AS reconnecting_pty_template_ids,
array_agg(DISTINCT template_id) FILTER (WHERE vscode_mins > 0) AS vscode_template_ids,
array_agg(DISTINCT template_id) FILTER (WHERE jetbrains_mins > 0) AS jetbrains_template_ids
FROM
template_usage_stats
WHERE
start_time >= $1::timestamptz
AND end_time <= $2::timestamptz
AND CASE WHEN COALESCE(array_length($3::uuid[], 1), 0) > 0 THEN template_id = ANY($3::uuid[]) ELSE TRUE END
)
SELECT
COALESCE((SELECT ids FROM template_ids), '{}')::uuid[] AS template_ids,
-- Return IDs so we can combine this with GetTemplateAppInsights.
COALESCE(array_agg(DISTINCT user_id), '{}')::uuid[] AS active_user_ids,
COALESCE(SUM(usage_vscode_seconds), 0)::bigint AS usage_vscode_seconds,
COALESCE(SUM(usage_jetbrains_seconds), 0)::bigint AS usage_jetbrains_seconds,
COALESCE(SUM(usage_reconnecting_pty_seconds), 0)::bigint AS usage_reconnecting_pty_seconds,
COALESCE(SUM(usage_ssh_seconds), 0)::bigint AS usage_ssh_seconds
FROM agent_stats_by_interval_and_user
COALESCE((SELECT template_ids FROM templates), '{}')::uuid[] AS template_ids, -- Includes app usage.
COALESCE((SELECT ssh_template_ids FROM templates), '{}')::uuid[] AS ssh_template_ids,
COALESCE((SELECT sftp_template_ids FROM templates), '{}')::uuid[] AS sftp_template_ids,
COALESCE((SELECT reconnecting_pty_template_ids FROM templates), '{}')::uuid[] AS reconnecting_pty_template_ids,
COALESCE((SELECT vscode_template_ids FROM templates), '{}')::uuid[] AS vscode_template_ids,
COALESCE((SELECT jetbrains_template_ids FROM templates), '{}')::uuid[] AS jetbrains_template_ids,
COALESCE(COUNT(DISTINCT user_id), 0)::bigint AS active_users, -- Includes app usage.
COALESCE(SUM(usage_mins) * 60, 0)::bigint AS usage_total_seconds, -- Includes app usage.
COALESCE(SUM(ssh_mins) * 60, 0)::bigint AS usage_ssh_seconds,
COALESCE(SUM(sftp_mins) * 60, 0)::bigint AS usage_sftp_seconds,
COALESCE(SUM(reconnecting_pty_mins) * 60, 0)::bigint AS usage_reconnecting_pty_seconds,
COALESCE(SUM(vscode_mins) * 60, 0)::bigint AS usage_vscode_seconds,
COALESCE(SUM(jetbrains_mins) * 60, 0)::bigint AS usage_jetbrains_seconds
FROM
insights
`
type GetTemplateInsightsParams struct {
@ -1935,26 +1956,46 @@ type GetTemplateInsightsParams struct {
type GetTemplateInsightsRow struct {
TemplateIDs []uuid.UUID `db:"template_ids" json:"template_ids"`
ActiveUserIDs []uuid.UUID `db:"active_user_ids" json:"active_user_ids"`
SshTemplateIds []uuid.UUID `db:"ssh_template_ids" json:"ssh_template_ids"`
SftpTemplateIds []uuid.UUID `db:"sftp_template_ids" json:"sftp_template_ids"`
ReconnectingPtyTemplateIds []uuid.UUID `db:"reconnecting_pty_template_ids" json:"reconnecting_pty_template_ids"`
VscodeTemplateIds []uuid.UUID `db:"vscode_template_ids" json:"vscode_template_ids"`
JetbrainsTemplateIds []uuid.UUID `db:"jetbrains_template_ids" json:"jetbrains_template_ids"`
ActiveUsers int64 `db:"active_users" json:"active_users"`
UsageTotalSeconds int64 `db:"usage_total_seconds" json:"usage_total_seconds"`
UsageSshSeconds int64 `db:"usage_ssh_seconds" json:"usage_ssh_seconds"`
UsageSftpSeconds int64 `db:"usage_sftp_seconds" json:"usage_sftp_seconds"`
UsageReconnectingPtySeconds int64 `db:"usage_reconnecting_pty_seconds" json:"usage_reconnecting_pty_seconds"`
UsageVscodeSeconds int64 `db:"usage_vscode_seconds" json:"usage_vscode_seconds"`
UsageJetbrainsSeconds int64 `db:"usage_jetbrains_seconds" json:"usage_jetbrains_seconds"`
UsageReconnectingPtySeconds int64 `db:"usage_reconnecting_pty_seconds" json:"usage_reconnecting_pty_seconds"`
UsageSshSeconds int64 `db:"usage_ssh_seconds" json:"usage_ssh_seconds"`
}
// GetTemplateInsights has a granularity of 5 minutes where if a session/app was
// in use during a minute, we will add 5 minutes to the total usage for that
// session/app (per user).
// GetTemplateInsights returns the aggregate user-produced usage of all
// workspaces in a given timeframe. The template IDs, active users, and
// usage_seconds all reflect any usage in the template, including apps.
//
// When combining data from multiple templates, we must make a guess at
// how the user behaved for the 30 minute interval. In this case we make
// the assumption that if the user used two workspaces for 15 minutes,
// they did so sequentially, thus we sum the usage up to a maximum of
// 30 minutes with LEAST(SUM(n), 30).
func (q *sqlQuerier) GetTemplateInsights(ctx context.Context, arg GetTemplateInsightsParams) (GetTemplateInsightsRow, error) {
row := q.db.QueryRowContext(ctx, getTemplateInsights, arg.StartTime, arg.EndTime, pq.Array(arg.TemplateIDs))
var i GetTemplateInsightsRow
err := row.Scan(
pq.Array(&i.TemplateIDs),
pq.Array(&i.ActiveUserIDs),
pq.Array(&i.SshTemplateIds),
pq.Array(&i.SftpTemplateIds),
pq.Array(&i.ReconnectingPtyTemplateIds),
pq.Array(&i.VscodeTemplateIds),
pq.Array(&i.JetbrainsTemplateIds),
&i.ActiveUsers,
&i.UsageTotalSeconds,
&i.UsageSshSeconds,
&i.UsageSftpSeconds,
&i.UsageReconnectingPtySeconds,
&i.UsageVscodeSeconds,
&i.UsageJetbrainsSeconds,
&i.UsageReconnectingPtySeconds,
&i.UsageSshSeconds,
)
return i, err
}

91
coderd/database/queries/insights.sql
View File

@ -99,40 +99,67 @@ GROUP BY users.id, username, avatar_url
ORDER BY user_id ASC;
-- name: GetTemplateInsights :one
-- GetTemplateInsights has a granularity of 5 minutes where if a session/app was
-- in use during a minute, we will add 5 minutes to the total usage for that
-- session/app (per user).
WITH agent_stats_by_interval_and_user AS (
SELECT
date_trunc('minute', was.created_at),
was.user_id,
array_agg(was.template_id) AS template_ids,
CASE WHEN SUM(was.session_count_vscode) > 0 THEN 60 ELSE 0 END AS usage_vscode_seconds,
CASE WHEN SUM(was.session_count_jetbrains) > 0 THEN 60 ELSE 0 END AS usage_jetbrains_seconds,
CASE WHEN SUM(was.session_count_reconnecting_pty) > 0 THEN 60 ELSE 0 END AS usage_reconnecting_pty_seconds,
CASE WHEN SUM(was.session_count_ssh) > 0 THEN 60 ELSE 0 END AS usage_ssh_seconds
FROM workspace_agent_stats was
WHERE
was.created_at >= @start_time::timestamptz
AND was.created_at < @end_time::timestamptz
AND was.connection_count > 0
AND CASE WHEN COALESCE(array_length(@template_ids::uuid[], 1), 0) > 0 THEN was.template_id = ANY(@template_ids::uuid[]) ELSE TRUE END
GROUP BY date_trunc('minute', was.created_at), was.user_id
), template_ids AS (
SELECT array_agg(DISTINCT template_id) AS ids
FROM agent_stats_by_interval_and_user, unnest(template_ids) template_id
WHERE template_id IS NOT NULL
)
-- GetTemplateInsights returns the aggregate user-produced usage of all
-- workspaces in a given timeframe. The template IDs, active users, and
-- usage_seconds all reflect any usage in the template, including apps.
--
-- When combining data from multiple templates, we must make a guess at
-- how the user behaved for the 30 minute interval. In this case we make
-- the assumption that if the user used two workspaces for 15 minutes,
-- they did so sequentially, thus we sum the usage up to a maximum of
-- 30 minutes with LEAST(SUM(n), 30).
WITH
insights AS (
SELECT
user_id,
-- See motivation in GetTemplateInsights for LEAST(SUM(n), 30).
LEAST(SUM(usage_mins), 30) AS usage_mins,
LEAST(SUM(ssh_mins), 30) AS ssh_mins,
LEAST(SUM(sftp_mins), 30) AS sftp_mins,
LEAST(SUM(reconnecting_pty_mins), 30) AS reconnecting_pty_mins,
LEAST(SUM(vscode_mins), 30) AS vscode_mins,
LEAST(SUM(jetbrains_mins), 30) AS jetbrains_mins
FROM
template_usage_stats
WHERE
start_time >= @start_time::timestamptz
AND end_time <= @end_time::timestamptz
AND CASE WHEN COALESCE(array_length(@template_ids::uuid[], 1), 0) > 0 THEN template_id = ANY(@template_ids::uuid[]) ELSE TRUE END
GROUP BY
start_time, user_id
),
templates AS (
SELECT
array_agg(DISTINCT template_id) AS template_ids,
array_agg(DISTINCT template_id) FILTER (WHERE ssh_mins > 0) AS ssh_template_ids,
array_agg(DISTINCT template_id) FILTER (WHERE sftp_mins > 0) AS sftp_template_ids,
array_agg(DISTINCT template_id) FILTER (WHERE reconnecting_pty_mins > 0) AS reconnecting_pty_template_ids,
array_agg(DISTINCT template_id) FILTER (WHERE vscode_mins > 0) AS vscode_template_ids,
array_agg(DISTINCT template_id) FILTER (WHERE jetbrains_mins > 0) AS jetbrains_template_ids
FROM
template_usage_stats
WHERE
start_time >= @start_time::timestamptz
AND end_time <= @end_time::timestamptz
AND CASE WHEN COALESCE(array_length(@template_ids::uuid[], 1), 0) > 0 THEN template_id = ANY(@template_ids::uuid[]) ELSE TRUE END
)
SELECT
COALESCE((SELECT ids FROM template_ids), '{}')::uuid[] AS template_ids,
-- Return IDs so we can combine this with GetTemplateAppInsights.
COALESCE(array_agg(DISTINCT user_id), '{}')::uuid[] AS active_user_ids,
COALESCE(SUM(usage_vscode_seconds), 0)::bigint AS usage_vscode_seconds,
COALESCE(SUM(usage_jetbrains_seconds), 0)::bigint AS usage_jetbrains_seconds,
COALESCE(SUM(usage_reconnecting_pty_seconds), 0)::bigint AS usage_reconnecting_pty_seconds,
COALESCE(SUM(usage_ssh_seconds), 0)::bigint AS usage_ssh_seconds
FROM agent_stats_by_interval_and_user;
COALESCE((SELECT template_ids FROM templates), '{}')::uuid[] AS template_ids, -- Includes app usage.
COALESCE((SELECT ssh_template_ids FROM templates), '{}')::uuid[] AS ssh_template_ids,
COALESCE((SELECT sftp_template_ids FROM templates), '{}')::uuid[] AS sftp_template_ids,
COALESCE((SELECT reconnecting_pty_template_ids FROM templates), '{}')::uuid[] AS reconnecting_pty_template_ids,
COALESCE((SELECT vscode_template_ids FROM templates), '{}')::uuid[] AS vscode_template_ids,
COALESCE((SELECT jetbrains_template_ids FROM templates), '{}')::uuid[] AS jetbrains_template_ids,
COALESCE(COUNT(DISTINCT user_id), 0)::bigint AS active_users, -- Includes app usage.
COALESCE(SUM(usage_mins) * 60, 0)::bigint AS usage_total_seconds, -- Includes app usage.
COALESCE(SUM(ssh_mins) * 60, 0)::bigint AS usage_ssh_seconds,
COALESCE(SUM(sftp_mins) * 60, 0)::bigint AS usage_sftp_seconds,
COALESCE(SUM(reconnecting_pty_mins) * 60, 0)::bigint AS usage_reconnecting_pty_seconds,
COALESCE(SUM(vscode_mins) * 60, 0)::bigint AS usage_vscode_seconds,
COALESCE(SUM(jetbrains_mins) * 60, 0)::bigint AS usage_jetbrains_seconds
FROM
insights;
-- name: GetTemplateInsightsByTemplate :many
WITH agent_stats_by_interval_and_user AS (

45
coderd/insights.go
View File

@ -395,8 +395,8 @@ func (api *API) insightsTemplates(rw http.ResponseWriter, r *http.Request) {
resp.Report = &codersdk.TemplateInsightsReport{
StartTime: startTime,
EndTime: endTime,
TemplateIDs: convertTemplateInsightsTemplateIDs(usage, appUsage),
ActiveUsers: convertTemplateInsightsActiveUsers(usage, appUsage),
TemplateIDs: usage.TemplateIDs,
ActiveUsers: usage.ActiveUsers,
AppsUsage: convertTemplateInsightsApps(usage, appUsage),
ParametersUsage: parametersUsage,
}
@ -416,39 +416,6 @@ func (api *API) insightsTemplates(rw http.ResponseWriter, r *http.Request) {
httpapi.Write(ctx, rw, http.StatusOK, resp)
}
func convertTemplateInsightsTemplateIDs(usage database.GetTemplateInsightsRow, appUsage []database.GetTemplateAppInsightsRow) []uuid.UUID {
templateIDSet := make(map[uuid.UUID]struct{})
for _, id := range usage.TemplateIDs {
templateIDSet[id] = struct{}{}
}
for _, app := range appUsage {
for _, id := range app.TemplateIDs {
templateIDSet[id] = struct{}{}
}
}
templateIDs := make([]uuid.UUID, 0, len(templateIDSet))
for id := range templateIDSet {
templateIDs = append(templateIDs, id)
}
slices.SortFunc(templateIDs, func(a, b uuid.UUID) int {
return slice.Ascending(a.String(), b.String())
})
return templateIDs
}
func convertTemplateInsightsActiveUsers(usage database.GetTemplateInsightsRow, appUsage []database.GetTemplateAppInsightsRow) int64 {
activeUserIDSet := make(map[uuid.UUID]struct{})
for _, id := range usage.ActiveUserIDs {
activeUserIDSet[id] = struct{}{}
}
for _, app := range appUsage {
for _, id := range app.ActiveUserIDs {
activeUserIDSet[id] = struct{}{}
}
}
return int64(len(activeUserIDSet))
}
// convertTemplateInsightsApps builds the list of builtin apps and template apps
// from the provided database rows, builtin apps are implicitly a part of all
// templates.
@ -456,7 +423,7 @@ func convertTemplateInsightsApps(usage database.GetTemplateInsightsRow, appUsage
// Builtin apps.
apps := []codersdk.TemplateAppUsage{
{
TemplateIDs: usage.TemplateIDs,
TemplateIDs: usage.VscodeTemplateIds,
Type: codersdk.TemplateAppsTypeBuiltin,
DisplayName: codersdk.TemplateBuiltinAppDisplayNameVSCode,
Slug: "vscode",
@ -464,7 +431,7 @@ func convertTemplateInsightsApps(usage database.GetTemplateInsightsRow, appUsage
Seconds: usage.UsageVscodeSeconds,
},
{
TemplateIDs: usage.TemplateIDs,
TemplateIDs: usage.JetbrainsTemplateIds,
Type: codersdk.TemplateAppsTypeBuiltin,
DisplayName: codersdk.TemplateBuiltinAppDisplayNameJetBrains,
Slug: "jetbrains",
@ -478,7 +445,7 @@ func convertTemplateInsightsApps(usage database.GetTemplateInsightsRow, appUsage
// condition finding the corresponding app entry in appUsage is:
// !app.IsApp && app.AccessMethod == "terminal" && app.SlugOrPort == ""
{
TemplateIDs: usage.TemplateIDs,
TemplateIDs: usage.ReconnectingPtyTemplateIds,
Type: codersdk.TemplateAppsTypeBuiltin,
DisplayName: codersdk.TemplateBuiltinAppDisplayNameWebTerminal,
Slug: "reconnecting-pty",
@ -486,7 +453,7 @@ func convertTemplateInsightsApps(usage database.GetTemplateInsightsRow, appUsage
Seconds: usage.UsageReconnectingPtySeconds,
},
{
TemplateIDs: usage.TemplateIDs,
TemplateIDs: usage.SshTemplateIds,
Type: codersdk.TemplateAppsTypeBuiltin,
DisplayName: codersdk.TemplateBuiltinAppDisplayNameSSH,
Slug: "ssh",

20
coderd/insights_test.go
View File

@ -1238,6 +1238,11 @@ func TestTemplateInsights_Golden(t *testing.T) {
templates, users, testData := prepareFixtureAndTestData(t, tt.makeFixture, tt.makeTestData)
client, events := prepare(t, templates, users, testData)
// Drain two events, the first one resumes rolluper
// operation and the second one waits for the rollup
// to complete.
_, _ = <-events, <-events
for _, req := range tt.requests {
req := req
t.Run(req.name, func(t *testing.T) {
@ -1245,11 +1250,6 @@ func TestTemplateInsights_Golden(t *testing.T) {
ctx := testutil.Context(t, testutil.WaitMedium)
// Drain two events, the first one resumes rolluper
// operation and the second one waits for the rollup
// to complete.
_, _ = <-events, <-events
report, err := client.TemplateInsights(ctx, req.makeRequest(templates))
require.NoError(t, err, "want no error getting template insights")
@ -2024,6 +2024,11 @@ func TestUserActivityInsights_Golden(t *testing.T) {
templates, users, testData := prepareFixtureAndTestData(t, tt.makeFixture, tt.makeTestData)
client, events := prepare(t, templates, users, testData)
// Drain two events, the first one resumes rolluper
// operation and the second one waits for the rollup
// to complete.
_, _ = <-events, <-events
for _, req := range tt.requests {
req := req
t.Run(req.name, func(t *testing.T) {
@ -2031,11 +2036,6 @@ func TestUserActivityInsights_Golden(t *testing.T) {
ctx := testutil.Context(t, testutil.WaitMedium)
// Drain two events, the first one resumes rolluper
// operation and the second one waits for the rollup
// to complete.
_, _ = <-events, <-events
report, err := client.UserActivityInsights(ctx, req.makeRequest(templates))
require.NoError(t, err, "want no error getting template insights")

4
coderd/testdata/insights/template/multiple_users_and_workspaces_three_weeks_second_template.json.golden vendored
View File

@ -18,9 +18,7 @@
"seconds": 3600
},
{
"template_ids": [
"00000000-0000-0000-0000-000000000002"
],
"template_ids": [],
"type": "builtin",
"display_name": "JetBrains",
"slug": "jetbrains",

4
coderd/testdata/insights/template/multiple_users_and_workspaces_three_weeks_second_template_only_report.json.golden vendored
View File

@ -18,9 +18,7 @@
"seconds": 3600
},
{
"template_ids": [
"00000000-0000-0000-0000-000000000002"
],
"template_ids": [],
"type": "builtin",
"display_name": "JetBrains",
"slug": "jetbrains",

9
coderd/testdata/insights/template/multiple_users_and_workspaces_week_all_templates.json.golden vendored
View File

@ -12,8 +12,7 @@
{
"template_ids": [
"00000000-0000-0000-0000-000000000001",
"00000000-0000-0000-0000-000000000002",
"00000000-0000-0000-0000-000000000003"
"00000000-0000-0000-0000-000000000002"
],
"type": "builtin",
"display_name": "Visual Studio Code",
@ -23,9 +22,7 @@
},
{
"template_ids": [
"00000000-0000-0000-0000-000000000001",
"00000000-0000-0000-0000-000000000002",
"00000000-0000-0000-0000-000000000003"
"00000000-0000-0000-0000-000000000001"
],
"type": "builtin",
"display_name": "JetBrains",
@ -35,8 +32,6 @@
},
{
"template_ids": [
"00000000-0000-0000-0000-000000000001",
"00000000-0000-0000-0000-000000000002",
"00000000-0000-0000-0000-000000000003"
],
"type": "builtin",

9
coderd/testdata/insights/template/multiple_users_and_workspaces_week_deployment_wide.json.golden vendored
View File

@ -12,8 +12,7 @@
{
"template_ids": [
"00000000-0000-0000-0000-000000000001",
"00000000-0000-0000-0000-000000000002",
"00000000-0000-0000-0000-000000000003"
"00000000-0000-0000-0000-000000000002"
],
"type": "builtin",
"display_name": "Visual Studio Code",
@ -23,9 +22,7 @@
},
{
"template_ids": [
"00000000-0000-0000-0000-000000000001",
"00000000-0000-0000-0000-000000000002",
"00000000-0000-0000-0000-000000000003"
"00000000-0000-0000-0000-000000000001"
],
"type": "builtin",
"display_name": "JetBrains",
@ -35,8 +32,6 @@
},
{
"template_ids": [
"00000000-0000-0000-0000-000000000001",
"00000000-0000-0000-0000-000000000002",
"00000000-0000-0000-0000-000000000003"
],
"type": "builtin",

4
coderd/testdata/insights/template/multiple_users_and_workspaces_week_first_template.json.golden vendored
View File

@ -28,9 +28,7 @@
"seconds": 120
},
{
"template_ids": [
"00000000-0000-0000-0000-000000000001"
],
"template_ids": [],
"type": "builtin",
"display_name": "Web Terminal",
"slug": "reconnecting-pty",

4
coderd/testdata/insights/template/multiple_users_and_workspaces_week_other_timezone_(São_Paulo).json.golden vendored
View File

@ -30,9 +30,7 @@
"seconds": 120
},
{
"template_ids": [
"00000000-0000-0000-0000-000000000001"
],
"template_ids": [],
"type": "builtin",
"display_name": "Web Terminal",
"slug": "reconnecting-pty",

8
coderd/testdata/insights/template/multiple_users_and_workspaces_week_second_template.json.golden vendored
View File

@ -18,9 +18,7 @@
"seconds": 3600
},
{
"template_ids": [
"00000000-0000-0000-0000-000000000002"
],
"template_ids": [],
"type": "builtin",
"display_name": "JetBrains",
"slug": "jetbrains",
@ -28,9 +26,7 @@
"seconds": 0
},
{
"template_ids": [
"00000000-0000-0000-0000-000000000002"
],
"template_ids": [],
"type": "builtin",
"display_name": "Web Terminal",
"slug": "reconnecting-pty",

8
coderd/testdata/insights/template/multiple_users_and_workspaces_week_third_template.json.golden vendored
View File

@ -8,9 +8,7 @@
"active_users": 1,
"apps_usage": [
{
"template_ids": [
"00000000-0000-0000-0000-000000000003"
],
"template_ids": [],
"type": "builtin",
"display_name": "Visual Studio Code",
"slug": "vscode",
@ -18,9 +16,7 @@
"seconds": 0
},
{
"template_ids": [
"00000000-0000-0000-0000-000000000003"
],
"template_ids": [],
"type": "builtin",
"display_name": "JetBrains",
"slug": "jetbrains",

9
coderd/testdata/insights/template/multiple_users_and_workspaces_weekly_aggregated_deployment_wide.json.golden vendored
View File

@ -12,8 +12,7 @@
{
"template_ids": [
"00000000-0000-0000-0000-000000000001",
"00000000-0000-0000-0000-000000000002",
"00000000-0000-0000-0000-000000000003"
"00000000-0000-0000-0000-000000000002"
],
"type": "builtin",
"display_name": "Visual Studio Code",
@ -23,9 +22,7 @@
},
{
"template_ids": [
"00000000-0000-0000-0000-000000000001",
"00000000-0000-0000-0000-000000000002",
"00000000-0000-0000-0000-000000000003"
"00000000-0000-0000-0000-000000000001"
],
"type": "builtin",
"display_name": "JetBrains",
@ -35,8 +32,6 @@
},
{
"template_ids": [
"00000000-0000-0000-0000-000000000001",
"00000000-0000-0000-0000-000000000002",
"00000000-0000-0000-0000-000000000003"
],
"type": "builtin",

4
coderd/testdata/insights/template/multiple_users_and_workspaces_weekly_aggregated_first_template.json.golden vendored
View File

@ -28,9 +28,7 @@
"seconds": 120
},
{
"template_ids": [
"00000000-0000-0000-0000-000000000001"
],
"template_ids": [],
"type": "builtin",
"display_name": "Web Terminal",
"slug": "reconnecting-pty",

9
coderd/testdata/insights/template/multiple_users_and_workspaces_weekly_aggregated_templates.json.golden vendored
View File

@ -12,8 +12,7 @@
{
"template_ids": [
"00000000-0000-0000-0000-000000000001",
"00000000-0000-0000-0000-000000000002",
"00000000-0000-0000-0000-000000000003"
"00000000-0000-0000-0000-000000000002"
],
"type": "builtin",
"display_name": "Visual Studio Code",
@ -23,9 +22,7 @@
},
{
"template_ids": [
"00000000-0000-0000-0000-000000000001",
"00000000-0000-0000-0000-000000000002",
"00000000-0000-0000-0000-000000000003"
"00000000-0000-0000-0000-000000000001"
],
"type": "builtin",
"display_name": "JetBrains",
@ -35,8 +32,6 @@
},
{
"template_ids": [
"00000000-0000-0000-0000-000000000001",
"00000000-0000-0000-0000-000000000002",
"00000000-0000-0000-0000-000000000003"
],
"type": "builtin",