selfhosted
/
coder
mirror of https://github.com/coder/coder.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

refactor(cli)!: move scaletest to exp/scaletest (#8339) 

* refactor(cli): mv scaletest exp/scaletest

* make gen
This commit is contained in:
Cian Johnston 2023-07-07 09:10:14 +01:00 committed by GitHub
parent ccea595b39
commit 435c67ab75
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 24 additions and 531 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
cli/exp.go Normal file
View File

@ -0,0 +1,18 @@
package cli
import "github.com/coder/coder/cli/clibase"
func (r *RootCmd) expCmd() *clibase.Cmd {
cmd := &clibase.Cmd{
Use: "exp",
Short: "Internal commands for testing and experimentation. These are prone to breaking changes with no notice.",
Handler: func(i *clibase.Invocation) error {
return i.Command.HelpHandler(i)
},
Hidden: true,
Children: []*clibase.Cmd{
r.scaletestCmd(),
},
}
return cmd
}

2
cli/scaletest.go → cli/exp_scaletest.go
View File

@ -38,7 +38,7 @@ import (
const scaletestTracerName = "coder_scaletest"
func (r *RootCmd) scaletest() *clibase.Cmd {
func (r *RootCmd) scaletestCmd() *clibase.Cmd {
cmd := &clibase.Cmd{
Use: "scaletest",
Short: "Run a scale test against the Coder API",

4
cli/scaletest_test.go → cli/exp_scaletest_test.go
View File

@ -29,7 +29,7 @@ func TestScaleTestCreateWorkspaces(t *testing.T) {
tDir := t.TempDir()
outputFile := filepath.Join(tDir, "output.json")
inv, root := clitest.New(t, "scaletest", "create-workspaces",
inv, root := clitest.New(t, "exp", "scaletest", "create-workspaces",
"--count", "2",
"--template", "doesnotexist",
"--no-cleanup",
@ -63,7 +63,7 @@ func TestScaleTestWorkspaceTraffic(t *testing.T) {
client := coderdtest.New(t, nil)
_ = coderdtest.CreateFirstUser(t, client)
inv, root := clitest.New(t, "scaletest", "workspace-traffic",
inv, root := clitest.New(t, "exp", "scaletest", "workspace-traffic",
"--timeout", "1s",
"--bytes-per-tick", "1024",
"--tick-interval", "100ms",

2
cli/root.go
View File

@ -95,7 +95,6 @@ func (r *RootCmd) Core() []*clibase.Cmd {
r.list(),
r.ping(),
r.rename(),
r.scaletest(),
r.schedules(),
r.show(),
r.speedtest(),
@ -111,6 +110,7 @@ func (r *RootCmd) Core() []*clibase.Cmd {
r.netcheck(),
r.vscodeSSH(),
r.workspaceAgent(),
r.expCmd(),
}
}

1
cli/testdata/coder_--help.golden vendored
View File

@ -26,7 +26,6 @@ Coder v0.0.0-devel — A tool for provisioning self-hosted development environme
reset-password Directly connect to the database to reset a user's
password
restart Restart a workspace
scaletest Run a scale test against the Coder API
schedule Schedule automated start and stop times for workspaces
server Start a Coder server
show Display details of a workspace's resources and agents

1
docs/cli.md
View File

@ -42,7 +42,6 @@ Coder — A tool for provisioning self-hosted development environments with Terr
| [<code>rename</code>](./cli/rename.md) | Rename a workspace |
| [<code>reset-password</code>](./cli/reset-password.md) | Directly connect to the database to reset a user's password |
| [<code>restart</code>](./cli/restart.md) | Restart a workspace |
| [<code>scaletest</code>](./cli/scaletest.md) | Run a scale test against the Coder API |
| [<code>schedule</code>](./cli/schedule.md) | Schedule automated start and stop times for workspaces |
| [<code>server</code>](./cli/server.md) | Start a Coder server |
| [<code>show</code>](./cli/show.md) | Display details of a workspace's resources and agents |

19
docs/cli/scaletest.md
View File

@ -1,19 +0,0 @@
<!-- DO NOT EDIT | GENERATED CONTENT -->
# scaletest
Run a scale test against the Coder API
## Usage
```console
coder scaletest
```
## Subcommands
| Name | Purpose |
| ------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| [<code>cleanup</code>](./scaletest_cleanup.md) | Cleanup scaletest workspaces, then cleanup scaletest users. |
| [<code>create-workspaces</code>](./scaletest_create-workspaces.md) | Creates many users, then creates a workspace for each user and waits for them finish building and fully come online. Optionally runs a command inside each workspace, and connects to the workspace over WireGuard. |
| [<code>workspace-traffic</code>](./scaletest_workspace-traffic.md) | Generate traffic to scaletest workspaces through coderd |

49
docs/cli/scaletest_cleanup.md
View File

@ -1,49 +0,0 @@
<!-- DO NOT EDIT | GENERATED CONTENT -->
# scaletest cleanup
Cleanup scaletest workspaces, then cleanup scaletest users.
## Usage
```console
coder scaletest cleanup [flags]
```
## Description
```console
The strategy flags will apply to each stage of the cleanup process.
```
## Options
### --cleanup-concurrency
| | |
| ----------- | ------------------------------------------------- |
| Type | <code>int</code> |
| Environment | <code>$CODER_SCALETEST_CLEANUP_CONCURRENCY</code> |
| Default | <code>1</code> |
Number of concurrent cleanup jobs to run. 0 means unlimited.
### --cleanup-job-timeout
| | |
| ----------- | ------------------------------------------------- |
| Type | <code>duration</code> |
| Environment | <code>$CODER_SCALETEST_CLEANUP_JOB_TIMEOUT</code> |
| Default | <code>5m</code> |
Timeout per job. Jobs may take longer to complete under higher concurrency limits.
### --cleanup-timeout
| | |
| ----------- | --------------------------------------------- |
| Type | <code>duration</code> |
| Environment | <code>$CODER_SCALETEST_CLEANUP_TIMEOUT</code> |
| Default | <code>30m</code> |
Timeout for the entire cleanup run. 0 means unlimited.

276
docs/cli/scaletest_create-workspaces.md
View File

@ -1,276 +0,0 @@
<!-- DO NOT EDIT | GENERATED CONTENT -->
# scaletest create-workspaces
Creates many users, then creates a workspace for each user and waits for them finish building and fully come online. Optionally runs a command inside each workspace, and connects to the workspace over WireGuard.
## Usage
```console
coder scaletest create-workspaces [flags]
```
## Description
```console
It is recommended that all rate limits are disabled on the server before running this scaletest. This test generates many login events which will be rate limited against the (most likely single) IP.
```
## Options
### --cleanup-concurrency
| | |
| ----------- | ------------------------------------------------- |
| Type | <code>int</code> |
| Environment | <code>$CODER_SCALETEST_CLEANUP_CONCURRENCY</code> |
| Default | <code>1</code> |
Number of concurrent cleanup jobs to run. 0 means unlimited.
### --cleanup-job-timeout
| | |
| ----------- | ------------------------------------------------- |
| Type | <code>duration</code> |
| Environment | <code>$CODER_SCALETEST_CLEANUP_JOB_TIMEOUT</code> |
| Default | <code>5m</code> |
Timeout per job. Jobs may take longer to complete under higher concurrency limits.
### --cleanup-timeout
| | |
| ----------- | --------------------------------------------- |
| Type | <code>duration</code> |
| Environment | <code>$CODER_SCALETEST_CLEANUP_TIMEOUT</code> |
| Default | <code>30m</code> |
Timeout for the entire cleanup run. 0 means unlimited.
### --concurrency
| | |
| ----------- | ----------------------------------------- |
| Type | <code>int</code> |
| Environment | <code>$CODER_SCALETEST_CONCURRENCY</code> |
| Default | <code>1</code> |
Number of concurrent jobs to run. 0 means unlimited.
### --connect-hold
| | |
| ----------- | ------------------------------------------ |
| Type | <code>duration</code> |
| Environment | <code>$CODER_SCALETEST_CONNECT_HOLD</code> |
| Default | <code>30s</code> |
How long to hold the WireGuard connection open for.
### --connect-interval
| | |
| ----------- | ---------------------------------------------- |
| Type | <code>duration</code> |
| Environment | <code>$CODER_SCALETEST_CONNECT_INTERVAL</code> |
| Default | <code>1s</code> |
How long to wait between making requests to the --connect-url once the connection is established.
### --connect-mode
| | |
| ----------- | ------------------------------------------ | -------------- |
| Type | <code>enum[derp | direct]</code> |
| Environment | <code>$CODER_SCALETEST_CONNECT_MODE</code> |
| Default | <code>derp</code> |
Mode to use for connecting to the workspace.
### --connect-timeout
| | |
| ----------- | --------------------------------------------- |
| Type | <code>duration</code> |
| Environment | <code>$CODER_SCALETEST_CONNECT_TIMEOUT</code> |
| Default | <code>5s</code> |
Timeout for each request to the --connect-url.
### --connect-url
| | |
| ----------- | ----------------------------------------- |
| Type | <code>string</code> |
| Environment | <code>$CODER_SCALETEST_CONNECT_URL</code> |
URL to connect to inside the the workspace over WireGuard. If not specified, no connections will be made over WireGuard.
### -c, --count
| | |
| ----------- | ----------------------------------- |
| Type | <code>int</code> |
| Environment | <code>$CODER_SCALETEST_COUNT</code> |
| Default | <code>1</code> |
Required: Number of workspaces to create.
### --job-timeout
| | |
| ----------- | ----------------------------------------- |
| Type | <code>duration</code> |
| Environment | <code>$CODER_SCALETEST_JOB_TIMEOUT</code> |
| Default | <code>5m</code> |
Timeout per job. Jobs may take longer to complete under higher concurrency limits.
### --no-cleanup
| | |
| ----------- | ---------------------------------------- |
| Type | <code>bool</code> |
| Environment | <code>$CODER_SCALETEST_NO_CLEANUP</code> |
Do not clean up resources after the test completes. You can cleanup manually using coder scaletest cleanup.
### --no-plan
| | |
| ----------- | ------------------------------------- |
| Type | <code>bool</code> |
| Environment | <code>$CODER_SCALETEST_NO_PLAN</code> |
Skip the dry-run step to plan the workspace creation. This step ensures that the given parameters are valid for the given template.
### --no-wait-for-agents
| | |
| ----------- | ------------------------------------------------ |
| Type | <code>bool</code> |
| Environment | <code>$CODER_SCALETEST_NO_WAIT_FOR_AGENTS</code> |
Do not wait for agents to start before marking the test as succeeded. This can be useful if you are running the test against a template that does not start the agent quickly.
### --output
| | |
| ----------- | ------------------------------------- |
| Type | <code>string-array</code> |
| Environment | <code>$CODER_SCALETEST_OUTPUTS</code> |
| Default | <code>text</code> |
Output format specs in the format "<format>[:<path>]". Not specifying a path will default to stdout. Available formats: text, json.
### --run-command
| | |
| ----------- | ----------------------------------------- |
| Type | <code>string</code> |
| Environment | <code>$CODER_SCALETEST_RUN_COMMAND</code> |
Command to run inside each workspace using reconnecting-pty (i.e. web terminal protocol). If not specified, no command will be run.
### --run-expect-output
| | |
| ----------- | ----------------------------------------------- |
| Type | <code>string</code> |
| Environment | <code>$CODER_SCALETEST_RUN_EXPECT_OUTPUT</code> |
Expect the command to output the given string (on a single line). If the command does not output the given string, it will be marked as failed.
### --run-expect-timeout
| | |
| ----------- | ------------------------------------------------ |
| Type | <code>bool</code> |
| Environment | <code>$CODER_SCALETEST_RUN_EXPECT_TIMEOUT</code> |
Expect the command to timeout. If the command does not finish within the given --run-timeout, it will be marked as succeeded. If the command finishes before the timeout, it will be marked as failed.
### --run-log-output
| | |
| ----------- | -------------------------------------------- |
| Type | <code>bool</code> |
| Environment | <code>$CODER_SCALETEST_RUN_LOG_OUTPUT</code> |
Log the output of the command to the test logs. This should be left off unless you expect small amounts of output. Large amounts of output will cause high memory usage.
### --run-timeout
| | |
| ----------- | ----------------------------------------- |
| Type | <code>duration</code> |
| Environment | <code>$CODER_SCALETEST_RUN_TIMEOUT</code> |
| Default | <code>5s</code> |
Timeout for the command to complete.
### -t, --template
| | |
| ----------- | -------------------------------------- |
| Type | <code>string</code> |
| Environment | <code>$CODER_SCALETEST_TEMPLATE</code> |
Required: Name or ID of the template to use for workspaces.
### --timeout
| | |
| ----------- | ------------------------------------- |
| Type | <code>duration</code> |
| Environment | <code>$CODER_SCALETEST_TIMEOUT</code> |
| Default | <code>30m</code> |
Timeout for the entire test run. 0 means unlimited.
### --trace
| | |
| ----------- | ----------------------------------- |
| Type | <code>bool</code> |
| Environment | <code>$CODER_SCALETEST_TRACE</code> |
Whether application tracing data is collected. It exports to a backend configured by environment variables. See: https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/exporter.md.
### --trace-coder
| | |
| ----------- | ----------------------------------------- |
| Type | <code>bool</code> |
| Environment | <code>$CODER_SCALETEST_TRACE_CODER</code> |
Whether opentelemetry traces are sent to Coder. We recommend keeping this disabled unless we advise you to enable it.
### --trace-honeycomb-api-key
| | |
| ----------- | ----------------------------------------------------- |
| Type | <code>string</code> |
| Environment | <code>$CODER_SCALETEST_TRACE_HONEYCOMB_API_KEY</code> |
Enables trace exporting to Honeycomb.io using the provided API key.
### --trace-propagate
| | |
| ----------- | --------------------------------------------- |
| Type | <code>bool</code> |
| Environment | <code>$CODER_SCALETEST_TRACE_PROPAGATE</code> |
Enables trace propagation to the Coder backend, which will be used to correlate server-side spans with client-side spans. Only enable this if the server is configured with the exact same tracing configuration as the client.
### --use-host-login
| | |
| ----------- | -------------------------------------------- |
| Type | <code>bool</code> |
| Environment | <code>$CODER_SCALETEST_USE_HOST_LOGIN</code> |
| Default | <code>false</code> |
Use the use logged in on the host machine, instead of creating users.

159
docs/cli/scaletest_workspace-traffic.md
View File

@ -1,159 +0,0 @@
<!-- DO NOT EDIT | GENERATED CONTENT -->
# scaletest workspace-traffic
Generate traffic to scaletest workspaces through coderd
## Usage
```console
coder scaletest workspace-traffic [flags]
```
## Options
### --bytes-per-tick
| | |
| ----------- | -------------------------------------------------------------- |
| Type | <code>int</code> |
| Environment | <code>$CODER_SCALETEST_WORKSPACE_TRAFFIC_BYTES_PER_TICK</code> |
| Default | <code>1024</code> |
How much traffic to generate per tick.
### --cleanup-concurrency
| | |
| ----------- | ------------------------------------------------- |
| Type | <code>int</code> |
| Environment | <code>$CODER_SCALETEST_CLEANUP_CONCURRENCY</code> |
| Default | <code>1</code> |
Number of concurrent cleanup jobs to run. 0 means unlimited.
### --cleanup-job-timeout
| | |
| ----------- | ------------------------------------------------- |
| Type | <code>duration</code> |
| Environment | <code>$CODER_SCALETEST_CLEANUP_JOB_TIMEOUT</code> |
| Default | <code>5m</code> |
Timeout per job. Jobs may take longer to complete under higher concurrency limits.
### --cleanup-timeout
| | |
| ----------- | --------------------------------------------- |
| Type | <code>duration</code> |
| Environment | <code>$CODER_SCALETEST_CLEANUP_TIMEOUT</code> |
| Default | <code>30m</code> |
Timeout for the entire cleanup run. 0 means unlimited.
### --concurrency
| | |
| ----------- | ----------------------------------------- |
| Type | <code>int</code> |
| Environment | <code>$CODER_SCALETEST_CONCURRENCY</code> |
| Default | <code>1</code> |
Number of concurrent jobs to run. 0 means unlimited.
### --job-timeout
| | |
| ----------- | ----------------------------------------- |
| Type | <code>duration</code> |
| Environment | <code>$CODER_SCALETEST_JOB_TIMEOUT</code> |
| Default | <code>5m</code> |
Timeout per job. Jobs may take longer to complete under higher concurrency limits.
### --output
| | |
| ----------- | ------------------------------------- |
| Type | <code>string-array</code> |
| Environment | <code>$CODER_SCALETEST_OUTPUTS</code> |
| Default | <code>text</code> |
Output format specs in the format "<format>[:<path>]". Not specifying a path will default to stdout. Available formats: text, json.
### --scaletest-prometheus-address
| | |
| ----------- | ------------------------------------------------ |
| Type | <code>string</code> |
| Environment | <code>$CODER_SCALETEST_PROMETHEUS_ADDRESS</code> |
| Default | <code>0.0.0.0:21112</code> |
Address on which to expose scaletest Prometheus metrics.
### --scaletest-prometheus-wait
| | |
| ----------- | --------------------------------------------- |
| Type | <code>duration</code> |
| Environment | <code>$CODER_SCALETEST_PROMETHEUS_WAIT</code> |
| Default | <code>5s</code> |
How long to wait before exiting in order to allow Prometheus metrics to be scraped.
### --tick-interval
| | |
| ----------- | ------------------------------------------------------------- |
| Type | <code>duration</code> |
| Environment | <code>$CODER_SCALETEST_WORKSPACE_TRAFFIC_TICK_INTERVAL</code> |
| Default | <code>100ms</code> |
How often to send traffic.
### --timeout
| | |
| ----------- | ------------------------------------- |
| Type | <code>duration</code> |
| Environment | <code>$CODER_SCALETEST_TIMEOUT</code> |
| Default | <code>30m</code> |
Timeout for the entire test run. 0 means unlimited.
### --trace
| | |
| ----------- | ----------------------------------- |
| Type | <code>bool</code> |
| Environment | <code>$CODER_SCALETEST_TRACE</code> |
Whether application tracing data is collected. It exports to a backend configured by environment variables. See: https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/exporter.md.
### --trace-coder
| | |
| ----------- | ----------------------------------------- |
| Type | <code>bool</code> |
| Environment | <code>$CODER_SCALETEST_TRACE_CODER</code> |
Whether opentelemetry traces are sent to Coder. We recommend keeping this disabled unless we advise you to enable it.
### --trace-honeycomb-api-key
| | |
| ----------- | ----------------------------------------------------- |
| Type | <code>string</code> |
| Environment | <code>$CODER_SCALETEST_TRACE_HONEYCOMB_API_KEY</code> |
Enables trace exporting to Honeycomb.io using the provided API key.
### --trace-propagate
| | |
| ----------- | --------------------------------------------- |
| Type | <code>bool</code> |
| Environment | <code>$CODER_SCALETEST_TRACE_PROPAGATE</code> |
Enables trace propagation to the Coder backend, which will be used to correlate server-side spans with client-side spans. Only enable this if the server is configured with the exact same tracing configuration as the client.

20
docs/manifest.json
View File

@ -632,26 +632,6 @@
"description": "Restart a workspace",
"path": "cli/restart.md"
},
{
"title": "scaletest",
"description": "Run a scale test against the Coder API",
"path": "cli/scaletest.md"
},
{
"title": "scaletest cleanup",
"description": "Cleanup scaletest workspaces, then cleanup scaletest users.",
"path": "cli/scaletest_cleanup.md"
},
{
"title": "scaletest create-workspaces",
"description": "Creates many users, then creates a workspace for each user and waits for them finish building and fully come online. Optionally runs a command inside each workspace, and connects to the workspace over WireGuard.",
"path": "cli/scaletest_create-workspaces.md"
},
{
"title": "scaletest workspace-traffic",
"description": "Generate traffic to scaletest workspaces through coderd",
"path": "cli/scaletest_workspace-traffic.md"
},
{
"title": "schedule",
"description": "Schedule automated start and stop times for workspaces",

2
scaletest/lib/coder_workspacetraffic.sh
View File

@ -80,7 +80,7 @@ spec:
- command:
- sh
- -c
- "curl -fsSL $CODER_URL/bin/coder-linux-amd64 -o /tmp/coder && chmod +x /tmp/coder && /tmp/coder --verbose --url=$CODER_URL --token=$CODER_TOKEN scaletest workspace-traffic --concurrency=0 --bytes-per-tick=${SCALETEST_TRAFFIC_BYTES_PER_TICK} --tick-interval=${SCALETEST_TRAFFIC_TICK_INTERVAL} --scaletest-prometheus-wait=60s"
- "curl -fsSL $CODER_URL/bin/coder-linux-amd64 -o /tmp/coder && chmod +x /tmp/coder && /tmp/coder --verbose --url=$CODER_URL --token=$CODER_TOKEN exp scaletest workspace-traffic --concurrency=0 --bytes-per-tick=${SCALETEST_TRAFFIC_BYTES_PER_TICK} --tick-interval=${SCALETEST_TRAFFIC_TICK_INTERVAL} --scaletest-prometheus-wait=60s"
env:
- name: CODER_URL
value: $CODER_URL

2
scaletest/scaletest.sh
View File

@ -181,7 +181,7 @@ if [[ -n "${SCALETEST_CODER_LICENSE}" ]]; then
fi
echo "Creating ${SCALETEST_NUM_WORKSPACES} workspaces."
DRY_RUN="$DRY_RUN" "${PROJECT_ROOT}/scaletest/lib/coder_shim.sh" scaletest create-workspaces \
DRY_RUN="$DRY_RUN" "${PROJECT_ROOT}/scaletest/lib/coder_shim.sh" exp scaletest create-workspaces \
--count "${SCALETEST_NUM_WORKSPACES}" \
--template=kubernetes \
--concurrency "${SCALETEST_CREATE_CONCURRENCY}" \