selfhosted
/
coder
mirror of https://github.com/coder/coder.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Roll it up

This commit is contained in:
Kyle Carberry 2024-05-01 21:33:32 +00:00
parent ef93844f56
commit 74f84b41e4
12 changed files with 385 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cli/intel.go
View File

@ -86,7 +86,7 @@ func (r *RootCmd) intelDaemonStart() *serpent.Command {
srv := inteld.New(inteld.Options{
Dialer: func(ctx context.Context, hostInfo codersdk.IntelDaemonHostInfo) (proto.DRPCIntelDaemonClient, error) {
return client.ServeIntelDaemon(ctx, codersdk.ServeIntelDaemonRequest{
return client.ServeIntelDaemon(ctx, uuid.Nil, codersdk.ServeIntelDaemonRequest{
InstanceID: instanceID,
IntelDaemonHostInfo: hostInfo,
})

8
coderd/coderd.go
View File

@ -204,6 +204,10 @@ type Options struct {
DatabaseRolluper *dbrollup.Rolluper
// WorkspaceUsageTracker tracks workspace usage by the CLI.
WorkspaceUsageTracker *workspaceusage.Tracker
// IntelServerInvocationFlushInterval is the interval at which the intel
// server flushes invocations to the database.
IntelServerInvocationFlushInterval time.Duration
}
// @title Coder API
@ -824,6 +828,8 @@ func New(options *Options) *API {
})
r.Route("/intel", func(r chi.Router) {
r.Get("/serve", api.intelDaemonServe)
r.Get("/report", api.intelReport)
r.Post("/report", api.postIntelReport)
r.Get("/machines", api.intelMachines)
r.Route("/cohorts", func(r chi.Router) {
r.Get("/", api.intelCohorts)
@ -836,8 +842,6 @@ func New(options *Options) *API {
r.Delete("/", api.deleteIntelCohort)
})
})
r.Get("/report", api.intelReport)
})
})
})

10
coderd/coderdtest/coderdtest.go
View File

@ -124,10 +124,11 @@ type Options struct {
FilesRateLimit int
// IncludeProvisionerDaemon when true means to start an in-memory provisionerD
IncludeProvisionerDaemon bool
MetricsCacheRefreshInterval time.Duration
AgentStatsRefreshInterval time.Duration
DeploymentValues *codersdk.DeploymentValues
IncludeProvisionerDaemon bool
MetricsCacheRefreshInterval time.Duration
AgentStatsRefreshInterval time.Duration
IntelServerInvocationFlushInterval time.Duration
DeploymentValues *codersdk.DeploymentValues
// Set update check options to enable update check.
UpdateCheckOptions *updatecheck.Options
@ -476,6 +477,7 @@ func NewOptions(t testing.TB, options *Options) (func(http.Handler), context.Can
TLSCertificates: options.TLSCertificates,
TrialGenerator: options.TrialGenerator,
RefreshEntitlements: options.RefreshEntitlements,
IntelServerInvocationFlushInterval: options.IntelServerInvocationFlushInterval,
TailnetCoordinator: options.Coordinator,
BaseDERPMap: derpMap,
DERPMapUpdateFrequency: 150 * time.Millisecond,

17
coderd/database/dbrollup/dbrollup.go
View File

@ -21,8 +21,9 @@ const (
)
type Event struct {
Init bool `json:"-"`
TemplateUsageStats bool `json:"template_usage_stats"`
Init bool `json:"-"`
TemplateUsageStats bool `json:"template_usage_stats"`
IntelInvocationSummaries bool `json:"intel_invocation_summaries"`
}
type Rolluper struct {
@ -107,7 +108,17 @@ func (r *Rolluper) start(ctx context.Context) {
}
ev.TemplateUsageStats = true
return tx.UpsertTemplateUsageStats(ctx)
err = tx.UpsertTemplateUsageStats(ctx)
if err != nil {
return err
}
ev.IntelInvocationSummaries = true
err = tx.UpsertIntelInvocationSummaries(ctx)
if err != nil {
return err
}
return nil
}, nil)
})

32
coderd/intel.go
View File

@ -130,7 +130,11 @@ func (api *API) intelReport(rw http.ResponseWriter, r *http.Request) {
command, ok := reportByBinary[binaryID]
if !ok {
command = codersdk.IntelReportCommand{
BinaryName: row.BinaryName,
BinaryName: row.BinaryName,
ExitCodes: map[int]int64{},
GitRemoteURLs: map[string]int64{},
WorkingDirectories: map[string]int64{},
BinaryPaths: map[string]int64{},
}
err = json.Unmarshal(row.BinaryArgs, &command.BinaryArgs)
if err != nil {
@ -213,6 +217,21 @@ func (api *API) intelReport(rw http.ResponseWriter, r *http.Request) {
httpapi.Write(ctx, rw, http.StatusOK, report)
}
// postIntelReport updates intel invocation summaries which will
// enable the intel report to be up-to-date.
func (api *API) postIntelReport(rw http.ResponseWriter, r *http.Request) {
ctx := r.Context()
err := api.Database.UpsertIntelInvocationSummaries(ctx)
if err != nil {
httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{
Message: "Internal error upserting intel invocation summaries.",
Detail: err.Error(),
})
return
}
httpapi.Write(ctx, rw, http.StatusNoContent, nil)
}
// intelMachines returns all machines that match the given filters.
//
// @Summary List intel machines
@ -477,11 +496,12 @@ func (api *API) intelDaemonServe(rw http.ResponseWriter, r *http.Request) {
logger := api.Logger
srv, err := inteldserver.New(srvCtx, inteldserver.Options{
Database: api.Database,
Pubsub: api.Pubsub,
Logger: logger.Named("intel_server"),
MachineID: machine.ID,
UserID: apiKey.UserID,
Database: api.Database,
Pubsub: api.Pubsub,
Logger: logger.Named("inteldserver"),
MachineID: machine.ID,
UserID: apiKey.UserID,
InvocationFlushInterval: api.IntelServerInvocationFlushInterval,
})
if err != nil {
if !xerrors.Is(err, context.Canceled) {

56
coderd/intel_test.go
View File

@ -4,12 +4,14 @@ import (
"context"
"fmt"
"testing"
"time"
"github.com/google/uuid"
"github.com/stretchr/testify/require"
"github.com/coder/coder/v2/coderd/coderdtest"
"github.com/coder/coder/v2/codersdk"
"github.com/coder/coder/v2/inteld/proto"
)
func TestIntelMachines(t *testing.T) {
@ -97,14 +99,64 @@ func TestIntelReport(t *testing.T) {
user := coderdtest.CreateFirstUser(t, client)
ctx := context.Background()
cohort, err := client.CreateIntelCohort(ctx, user.OrganizationID, codersdk.CreateIntelCohortRequest{
Name: "example",
Name: "Everyone",
})
require.NoError(t, err)
report, err := client.IntelReport(ctx, user.OrganizationID, codersdk.IntelReportRequest{
CohortIDs: []uuid.UUID{cohort.ID},
})
require.NoError(t, err)
// require.Equal(t, )
// No invocations of course!
require.Equal(t, int64(0), report.Invocations)
})
t.Run("Basic", func(t *testing.T) {
t.Parallel()
flushInterval := 25 * time.Millisecond
client := coderdtest.New(t, &coderdtest.Options{
IntelServerInvocationFlushInterval: flushInterval,
})
user := coderdtest.CreateFirstUser(t, client)
ctx := context.Background()
cohort, err := client.CreateIntelCohort(ctx, user.OrganizationID, codersdk.CreateIntelCohortRequest{
Name: "Everyone",
})
require.NoError(t, err)
firstClient, err := client.ServeIntelDaemon(ctx, user.OrganizationID, codersdk.ServeIntelDaemonRequest{
IntelDaemonHostInfo: codersdk.IntelDaemonHostInfo{
OperatingSystem: "linux",
},
InstanceID: "test",
})
require.NoError(t, err)
defer firstClient.DRPCConn().Close()
_, err = firstClient.RecordInvocation(ctx, &proto.RecordInvocationRequest{
Invocations: []*proto.Invocation{{
Executable: &proto.Executable{
Hash: "hash",
Basename: "go",
Path: "/usr/bin/go",
Version: "1.0.0",
},
Arguments: []string{"run", "main.go"},
DurationMs: 354,
ExitCode: 1,
WorkingDirectory: "/home/coder",
GitRemoteUrl: "https://github.com/coder/coder",
}},
})
require.NoError(t, err)
// TODO: @kylecarbs this is obviously a racey piece of code...
// Wait for invocations to flush
<-time.After(flushInterval * 2)
err = client.RefreshIntelReport(ctx, user.OrganizationID)
require.NoError(t, err)
report, err := client.IntelReport(ctx, user.OrganizationID, codersdk.IntelReportRequest{
CohortIDs: []uuid.UUID{cohort.ID},
})
require.NoError(t, err)
require.Equal(t, int64(1), report.Invocations)
fmt.Printf("%+v\n", report)
})
}

17
codersdk/intel.go
View File

@ -314,3 +314,20 @@ func (c *Client) IntelReport(ctx context.Context, organizationID uuid.UUID, req
var report IntelReport
return report, json.NewDecoder(res.Body).Decode(&report)
}
// RefreshIntelReport refreshes the intel report for an organization.
func (c *Client) RefreshIntelReport(ctx context.Context, organizationID uuid.UUID) error {
orgParam := organizationID.String()
if organizationID == uuid.Nil {
orgParam = DefaultOrganization
}
res, err := c.Request(ctx, http.MethodPost, fmt.Sprintf("/api/v2/organizations/%s/intel/report", orgParam), nil)
if err != nil {
return err
}
defer res.Body.Close()
if res.StatusCode != http.StatusNoContent {
return ReadBodyAsError(res)
}
return nil
}

7
site/src/api/api.ts
View File

@ -1861,3 +1861,10 @@ export const getJFrogXRayScan = async (options: GetJFrogXRayScanParams) => {
}
}
};
export const getIntelCohorts = async (organizationId: string) => {
const response = await axiosInstance.get<TypesGen.IntelCohort[]>(
`/api/v2/organizations/${organizationId}/intel/cohorts`,
);
return response.data;
}

12
site/src/api/queries/intel.ts Normal file
View File

@ -0,0 +1,12 @@
import type { UseQueryOptions } from "react-query"
import * as API from "api/api"
import type { IntelCohort } from "api/typesGenerated"
const intelCohortsKey = ["intel", "cohorts"]
export const intelCohorts = (organizationId: string): UseQueryOptions<IntelCohort[]> => {
return {
queryKey: intelCohortsKey,
queryFn: () => API.getIntelCohorts(organizationId),
}
}

155
site/src/pages/CreateIntelCohortPage/CreateIntelCohortPage.tsx
View File

@ -1,5 +1,156 @@
import { css } from "@emotion/react";
import { Button, FormHelperText, TextField } from "@mui/material";
import { Margins } from "components/Margins/Margins";
import {
PageHeader,
PageHeaderSubtitle,
PageHeaderTitle,
} from "components/PageHeader/PageHeader";
import { Stack } from "components/Stack/Stack";
import { FormikContextType, useFormik } from "formik";
import * as TypesGen from "api/typesGenerated";
import {
getFormHelpers,
nameValidator,
onChangeTrimmed,
} from "utils/formUtils";
import { FormFields, FormSection, HorizontalForm } from "components/Form/Form";
import * as Yup from "yup";
import { IconField } from "components/IconField/IconField";
const CreateIntelCohortPage = () => {
const form: FormikContextType<TypesGen.CreateIntelCohortRequest> =
useFormik<TypesGen.CreateIntelCohortRequest>({
initialValues: {
name: "",
description: "",
icon: "",
tracked_executables: [],
regex_filters: {
architecture: ".*",
instance_id: ".*",
operating_system: ".*",
operating_system_platform: ".*",
operating_system_version: ".*",
},
},
validationSchema: Yup.object({
name: nameValidator("Cohort Name"),
}),
enableReinitialize: true,
onSubmit: (request) => {
console.log("submit", request);
},
});
const isSubmitting = false;
const getFieldHelpers = getFormHelpers(form);
}
return (
<Margins size="medium">
<PageHeader actions={<Button>Cancel</Button>}>
<PageHeaderTitle>Create an Intel Cohort</PageHeaderTitle>
export default CreateIntelCohortPage
<PageHeaderSubtitle condensed>
<div
css={css`
max-width: 700px;
`}
>
Define filters to monitor command invocations, detect redundant
tools, identify time-consuming processes, and check for version
inconsistencies in development environments.
</div>
</PageHeaderSubtitle>
</PageHeader>
<HorizontalForm onSubmit={form.handleSubmit}>
<FormSection
title="Display"
description="The Cohort will be visible to everyone. Provide lots of details on which machines it should target!"
>
<FormFields>
<TextField
{...getFieldHelpers("name")}
disabled={isSubmitting}
// resetMutation facilitates the clearing of validation errors
onChange={onChangeTrimmed(form)}
fullWidth
label="Name"
/>
<TextField
{...getFieldHelpers("description", {
maxLength: 128,
})}
disabled={isSubmitting}
rows={1}
fullWidth
label="Description"
/>
<IconField
{...getFieldHelpers("icon")}
disabled={isSubmitting}
onChange={onChangeTrimmed(form)}
fullWidth
onPickEmoji={(value) => form.setFieldValue("icon", value)}
/>
</FormFields>
</FormSection>
<FormSection
title="Machines"
description="The Cohort will target all registered machines by default."
>
<FormFields>
<Stack direction="row">
<TextField
{...getFieldHelpers("regex_filters.operating_system")}
disabled={isSubmitting}
// resetMutation facilitates the clearing of validation errors
onChange={onChangeTrimmed(form)}
fullWidth
label="Operating System"
helperText="e.g: linux, darwin, windows"
/>
<TextField
{...getFieldHelpers("regex_filters.operating_system_platform")}
disabled={isSubmitting}
// resetMutation facilitates the clearing of validation errors
onChange={onChangeTrimmed(form)}
fullWidth
label="Operating System Platform"
helperText="e.g: 22.02"
/>
</Stack>
<Stack direction="row">
<TextField
{...getFieldHelpers("regex_filters.operating_system_version")}
disabled={isSubmitting}
// resetMutation facilitates the clearing of validation errors
onChange={onChangeTrimmed(form)}
fullWidth
label="Operating System Version"
helperText="e.g: 22.02"
/>
<TextField
{...getFieldHelpers("regex_filters.architecture")}
disabled={isSubmitting}
// resetMutation facilitates the clearing of validation errors
onChange={onChangeTrimmed(form)}
fullWidth
label="Architecture"
helperText="e.g: arm, arm64, 386, amd64"
/>
</Stack>
</FormFields>
</FormSection>
</HorizontalForm>
</Margins>
);
};
export default CreateIntelCohortPage;

97
site/src/pages/IntelPage/IntelLayout.tsx
View File

@ -1,4 +1,4 @@
import type { FC, PropsWithChildren } from "react";
import { useEffect, useMemo, useRef, useState, type FC, type PropsWithChildren } from "react";
import { Helmet } from "react-helmet-async";
import { Outlet, useLocation } from "react-router-dom";
import { FilterSearchMenu, OptionItem } from "components/Filter/filter";
@ -7,18 +7,31 @@ import { Margins } from "components/Margins/Margins";
import { PageHeader, PageHeaderTitle } from "components/PageHeader/PageHeader";
import { TabLink, Tabs, TabsList } from "components/Tabs/Tabs";
import { pageTitle } from "utils/page";
import { useQuery } from "react-query";
import { intelCohorts } from "api/queries/intel";
import { useAuthenticated } from "contexts/auth/RequireAuth";
import { Button, Menu, MenuItem, MenuList } from "@mui/material";
import { KeyboardArrowDown } from "@mui/icons-material";
import { css } from "@emotion/react";
import { IntelCohort } from "api/typesGenerated";
const IntelLayout: FC<PropsWithChildren> = ({ children = <Outlet /> }) => {
const location = useLocation();
const paths = location.pathname.split("/");
const activeTab = paths[2] ?? "summary";
const { organizationId } = useAuthenticated();
const cohortsQuery = useQuery(intelCohorts(organizationId));
const cohortFilter = useFilterMenu({
onChange: () => undefined,
value: "Create Cohort",
value: "All Cohorts",
id: "cohort",
getOptions: async (_) => {
return [];
return (
cohortsQuery.data?.map((cohort) => ({
label: cohort.name,
value: cohort.id,
})) ?? []
);
},
getSelectedOption: async () => {
return null;
@ -26,6 +39,10 @@ const IntelLayout: FC<PropsWithChildren> = ({ children = <Outlet /> }) => {
enabled: true,
});
if (cohortsQuery.isLoading) {
return <div>Loading...</div>;
}
return (
<Margins>
<Helmet>
@ -34,15 +51,7 @@ const IntelLayout: FC<PropsWithChildren> = ({ children = <Outlet /> }) => {
<PageHeader
actions={
<>
<FilterSearchMenu menu={cohortFilter} label={cohortFilter.selectedOption ? cohortFilter.selectedOption : "Create Cohorts"} id="Cohort">
{(itemProps) => (
<OptionItem
option={itemProps.option}
isSelected={itemProps.isSelected}
left={<>{"Something"}</>}
/>
)}
</FilterSearchMenu>
<CohortSelector />
</>
}
>
@ -69,4 +78,66 @@ const IntelLayout: FC<PropsWithChildren> = ({ children = <Outlet /> }) => {
);
};
const CohortSelector = () => {
const { organizationId } = useAuthenticated();
const cohortsQuery = useQuery(intelCohorts(organizationId))
const cohortByID = useMemo(() => {
if (!cohortsQuery.data) {
return
}
const cohortByID: Record<string, IntelCohort> = {}
cohortsQuery.data.forEach(cohort => {
cohortByID[cohort.id] = cohort
})
return cohortByID
}, [cohortsQuery.data])
const buttonRef = useRef<HTMLButtonElement>(null);
const [isMenuOpen, setIsMenuOpen] = useState(false);
const [selectedCohorts, setSelectedCohorts] = useState<string[]>([]);
useEffect(() => {
if (cohortByID) {
setSelectedCohorts(Object.keys(cohortByID))
}
}, [cohortByID])
if (cohortsQuery.isLoading) {
return null
}
return (
<>
<Button
ref={buttonRef}
endIcon={<KeyboardArrowDown />}
css={css`
border-radius: 6px;
justify-content: space-between;
line-height: 120%;
`}
onClick={() => setIsMenuOpen(true)}
>
Select a Cohort
</Button>
<Menu
open={isMenuOpen}
onClose={() => setIsMenuOpen(false)}
anchorEl={buttonRef.current}
>
<MenuList>
{selectedCohorts.map(cohortID => {
const cohort = cohortByID?.[cohortID]
return (
<MenuItem key={cohortID} onClick={() => {
setSelectedCohorts(selectedCohorts.filter(id => id !== cohortID))
}}>
{cohort?.name}
</MenuItem>
)
})}
</MenuList>
</Menu>
</>
);
};
export default IntelLayout;

7
site/src/router.tsx
View File

@ -253,6 +253,9 @@ const InsightsCommandsPage = lazy(
const InsightsEditorsPage = lazy(
() => import("./pages/IntelPage/IntelEditorsPage"),
);
const CreateIntelCohortPage = lazy(
() => import("./pages/CreateIntelCohortPage/CreateIntelCohortPage")
)
const RoutesWithSuspense = () => {
return (
@ -318,13 +321,13 @@ export const router = createBrowserRouter(
</Route>
</Route>
<Route path="/intel" element={<IntelLayout />}>
<Route path="/intel">
<Route index element={<InsightsSummaryPage />} />
<Route path="tools" element={<InsightsToolsPage />} />
<Route path="commands" element={<InsightsCommandsPage />} />
<Route path="editors" element={<InsightsEditorsPage />} />
<Route path="cohorts">
<Route path="new" element={<div>hi</div>} />
<Route path="new" element={<CreateIntelCohortPage />} />
<Route path=":cohort">
<Route index element={<div>hi</div>} />
<Route path="edit" element={<div>hi</div>} />