mirror of https://github.com/coder/coder.git
fix: use flag to enable Prometheus (#12345)
This commit is contained in:
parent
bedd2c5922
commit
30d9d84758
|
@ -141,6 +141,10 @@ func extractTar(t *testing.T, data []byte, directory string) {
|
|||
// Start runs the command in a goroutine and cleans it up when the test
|
||||
// completed.
|
||||
func Start(t *testing.T, inv *clibase.Invocation) {
|
||||
StartWithAssert(t, inv, nil)
|
||||
}
|
||||
|
||||
func StartWithAssert(t *testing.T, inv *clibase.Invocation, assertCallback func(t *testing.T, err error)) { //nolint:revive
|
||||
t.Helper()
|
||||
|
||||
closeCh := make(chan struct{})
|
||||
|
@ -155,6 +159,12 @@ func Start(t *testing.T, inv *clibase.Invocation) {
|
|||
go func() {
|
||||
defer close(closeCh)
|
||||
err := waiter.Wait()
|
||||
|
||||
if assertCallback != nil {
|
||||
assertCallback(t, err)
|
||||
return
|
||||
}
|
||||
|
||||
switch {
|
||||
case errors.Is(err, context.Canceled):
|
||||
return
|
||||
|
|
|
@ -229,3 +229,17 @@ This can be disabled with a server-wide
|
|||
```shell
|
||||
coder server --provisioner-daemons=0
|
||||
```
|
||||
|
||||
## Prometheus metrics
|
||||
|
||||
Coder provisioner daemon exports metrics via the HTTP endpoint, which can be
|
||||
enabled using either the environment variable `CODER_PROMETHEUS_ENABLE` or the
|
||||
flag `--prometheus-enable`.
|
||||
|
||||
The Prometheus endpoint address is `http://localhost:2112/` by default. You can
|
||||
use either the environment variable `CODER_PROMETHEUS_ADDRESS` or the flag
|
||||
`--prometheus-address <network-interface>:<port>` to select a different listen
|
||||
address.
|
||||
|
||||
If you have provisioners daemons deployed as pods, it is advised to monitor them
|
||||
separately.
|
||||
|
|
|
@ -184,3 +184,14 @@ goes offline, the session will fall back to the primary proxy. This could take
|
|||
up to 60 seconds.
|
||||
|
||||
![Workspace proxy picker](../images/admin/workspace-proxy-picker.png)
|
||||
|
||||
## Step 3: Observability
|
||||
|
||||
Coder workspace proxy exports metrics via the HTTP endpoint, which can be
|
||||
enabled using either the environment variable `CODER_PROMETHEUS_ENABLE` or the
|
||||
flag `--prometheus-enable`.
|
||||
|
||||
The Prometheus endpoint address is `http://localhost:2112/` by default. You can
|
||||
use either the environment variable `CODER_PROMETHEUS_ADDRESS` or the flag
|
||||
`--prometheus-address <network-interface>:<port>` to select a different listen
|
||||
address.
|
||||
|
|
|
@ -98,6 +98,16 @@ Deprecated and ignored.
|
|||
|
||||
The bind address to serve prometheus metrics.
|
||||
|
||||
### --prometheus-enable
|
||||
|
||||
| | |
|
||||
| ----------- | ------------------------------------- |
|
||||
| Type | <code>bool</code> |
|
||||
| Environment | <code>$CODER_PROMETHEUS_ENABLE</code> |
|
||||
| Default | <code>false</code> |
|
||||
|
||||
Serve prometheus metrics on the address defined by prometheus address.
|
||||
|
||||
### --psk
|
||||
|
||||
| | |
|
||||
|
|
|
@ -71,6 +71,7 @@ func (r *RootCmd) provisionerDaemonStart() *clibase.Cmd {
|
|||
preSharedKey string
|
||||
verbose bool
|
||||
|
||||
prometheusEnable bool
|
||||
prometheusAddress string
|
||||
)
|
||||
client := new(codersdk.Client)
|
||||
|
@ -171,7 +172,7 @@ func (r *RootCmd) provisionerDaemonStart() *clibase.Cmd {
|
|||
}()
|
||||
|
||||
var metrics *provisionerd.Metrics
|
||||
if prometheusAddress != "" {
|
||||
if prometheusEnable {
|
||||
logger.Info(ctx, "starting Prometheus endpoint", slog.F("address", prometheusAddress))
|
||||
|
||||
prometheusRegistry := prometheus.NewRegistry()
|
||||
|
@ -315,6 +316,13 @@ func (r *RootCmd) provisionerDaemonStart() *clibase.Cmd {
|
|||
Value: clibase.StringArrayOf(&logFilter),
|
||||
Default: "",
|
||||
},
|
||||
{
|
||||
Flag: "prometheus-enable",
|
||||
Env: "CODER_PROMETHEUS_ENABLE",
|
||||
Description: "Serve prometheus metrics on the address defined by prometheus address.",
|
||||
Value: clibase.BoolOf(&prometheusEnable),
|
||||
Default: "false",
|
||||
},
|
||||
{
|
||||
Flag: "prometheus-address",
|
||||
Env: "CODER_PROMETHEUS_ADDRESS",
|
||||
|
|
|
@ -170,15 +170,6 @@ func TestProvisionerDaemon_SessionToken(t *testing.T) {
|
|||
t.Run("PrometheusEnabled", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
// Helper function to find a free random port
|
||||
randomPort := func(t *testing.T) int {
|
||||
random, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
require.NoError(t, err)
|
||||
_ = random.Close()
|
||||
tcpAddr, valid := random.Addr().(*net.TCPAddr)
|
||||
require.True(t, valid)
|
||||
return tcpAddr.Port
|
||||
}
|
||||
prometheusPort := randomPort(t)
|
||||
|
||||
// Configure CLI client
|
||||
|
@ -191,7 +182,7 @@ func TestProvisionerDaemon_SessionToken(t *testing.T) {
|
|||
},
|
||||
})
|
||||
anotherClient, _ := coderdtest.CreateAnotherUser(t, client, admin.OrganizationID, rbac.RoleTemplateAdmin())
|
||||
inv, conf := newCLI(t, "provisionerd", "start", "--name", "daemon-with-prometheus", "--prometheus-address", fmt.Sprintf("127.0.0.1:%d", prometheusPort))
|
||||
inv, conf := newCLI(t, "provisionerd", "start", "--name", "daemon-with-prometheus", "--prometheus-enable", "--prometheus-address", fmt.Sprintf("127.0.0.1:%d", prometheusPort))
|
||||
clitest.SetupConfig(t, anotherClient, conf)
|
||||
pty := ptytest.New(t).Attach(inv)
|
||||
ctx, cancel := context.WithTimeout(inv.Context(), testutil.WaitLong)
|
||||
|
@ -251,3 +242,13 @@ func TestProvisionerDaemon_SessionToken(t *testing.T) {
|
|||
require.True(t, hasPromHTTP, "Prometheus HTTP metrics are missing")
|
||||
})
|
||||
}
|
||||
|
||||
// randomPort is a helper function to find a free random port, for instance to spawn Prometheus endpoint.
|
||||
func randomPort(t *testing.T) int {
|
||||
random, err := net.Listen("tcp", "127.0.0.1:0")
|
||||
require.NoError(t, err)
|
||||
_ = random.Close()
|
||||
tcpAddr, valid := random.Addr().(*net.TCPAddr)
|
||||
require.True(t, valid)
|
||||
return tcpAddr.Port
|
||||
}
|
||||
|
|
|
@ -1,16 +1,23 @@
|
|||
package cli_test
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/coder/coder/v2/cli/clitest"
|
||||
"github.com/coder/coder/v2/pty/ptytest"
|
||||
"github.com/coder/coder/v2/testutil"
|
||||
)
|
||||
|
||||
func Test_Headers(t *testing.T) {
|
||||
|
@ -52,3 +59,88 @@ func Test_Headers(t *testing.T) {
|
|||
|
||||
assert.EqualValues(t, 1, atomic.LoadInt64(&called))
|
||||
}
|
||||
|
||||
func TestWorkspaceProxy_Server_PrometheusEnabled(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
prometheusPort := randomPort(t)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(1)
|
||||
|
||||
// Start fake coderd
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/api/v2/workspaceproxies/me/register" {
|
||||
// Give fake app_security_key (96 bytes)
|
||||
w.WriteHeader(http.StatusCreated)
|
||||
_, _ = w.Write([]byte(`{"app_security_key": "012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789123456012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789123456"}`))
|
||||
return
|
||||
}
|
||||
if r.URL.Path == "/api/v2/workspaceproxies/me/coordinate" {
|
||||
// Slow down proxy registration, so that test runner can check if Prometheus endpoint is exposed.
|
||||
wg.Wait()
|
||||
|
||||
// Does not matter, we are not going to implement a real workspace proxy.
|
||||
w.WriteHeader(http.StatusNotImplemented)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Add("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{}`)) // build info can be ignored
|
||||
}))
|
||||
defer srv.Close()
|
||||
defer wg.Done()
|
||||
|
||||
// Configure CLI client
|
||||
inv, _ := newCLI(t, "wsproxy", "server",
|
||||
"--primary-access-url", srv.URL,
|
||||
"--proxy-session-token", "test-token",
|
||||
"--access-url", "http://foobar:3001",
|
||||
"--http-address", fmt.Sprintf("127.0.0.1:%d", randomPort(t)),
|
||||
"--prometheus-enable",
|
||||
"--prometheus-address", fmt.Sprintf("127.0.0.1:%d", prometheusPort),
|
||||
)
|
||||
pty := ptytest.New(t).Attach(inv)
|
||||
|
||||
ctx, cancel := context.WithTimeout(inv.Context(), testutil.WaitLong)
|
||||
defer cancel()
|
||||
|
||||
// Start "wsproxy server" command
|
||||
clitest.StartWithAssert(t, inv, func(t *testing.T, err error) {
|
||||
assert.Error(t, err)
|
||||
assert.False(t, errors.Is(err, context.Canceled), "error was expected, but context was canceled")
|
||||
})
|
||||
pty.ExpectMatchContext(ctx, "Started HTTP listener at")
|
||||
|
||||
// Fetch metrics from Prometheus endpoint
|
||||
var res *http.Response
|
||||
require.Eventually(t, func() bool {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("http://127.0.0.1:%d", prometheusPort), nil)
|
||||
assert.NoError(t, err)
|
||||
// nolint:bodyclose
|
||||
res, err = http.DefaultClient.Do(req)
|
||||
return err == nil
|
||||
}, testutil.WaitShort, testutil.IntervalFast)
|
||||
defer res.Body.Close()
|
||||
|
||||
// Scan for metric patterns
|
||||
scanner := bufio.NewScanner(res.Body)
|
||||
hasGoStats := false
|
||||
hasPromHTTP := false
|
||||
for scanner.Scan() {
|
||||
if strings.HasPrefix(scanner.Text(), "go_goroutines") {
|
||||
hasGoStats = true
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(scanner.Text(), "promhttp_metric_handler_requests_total") {
|
||||
hasPromHTTP = true
|
||||
continue
|
||||
}
|
||||
t.Logf("scanned %s", scanner.Text())
|
||||
}
|
||||
require.NoError(t, scanner.Err())
|
||||
|
||||
// Verify patterns
|
||||
require.True(t, hasGoStats, "Go stats are missing")
|
||||
require.True(t, hasPromHTTP, "Prometheus HTTP metrics are missing")
|
||||
}
|
||||
|
|
|
@ -35,6 +35,9 @@ OPTIONS:
|
|||
--prometheus-address string, $CODER_PROMETHEUS_ADDRESS (default: 127.0.0.1:2112)
|
||||
The bind address to serve prometheus metrics.
|
||||
|
||||
--prometheus-enable bool, $CODER_PROMETHEUS_ENABLE (default: false)
|
||||
Serve prometheus metrics on the address defined by prometheus address.
|
||||
|
||||
--psk string, $CODER_PROVISIONER_DAEMON_PSK
|
||||
Pre-shared key to authenticate with Coder server.
|
||||
|
||||
|
|
Loading…
Reference in New Issue