feat: enable Prometheus endpoint for external provisioner (#12320)

docs/cli/provisionerd_start.md

@@ -88,6 +88,16 @@ Deprecated and ignored.
 
 Deprecated and ignored.
 
+### --prometheus-address
+
+|             |                                        |
+| ----------- | -------------------------------------- |
+| Type        | <code>string</code>                    |
+| Environment | <code>$CODER_PROMETHEUS_ADDRESS</code> |
+| Default     | <code>127.0.0.1:2112</code>            |
+
+The bind address to serve prometheus metrics.
+
 ### --psk
 
 |             |                                            |

enterprise/cli/provisionerdaemons.go

@@ -10,6 +10,9 @@ import (
 	"time"
 
 	"github.com/google/uuid"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/collectors"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"golang.org/x/xerrors"
 
 	"cdr.dev/slog"
@@ -67,6 +70,8 @@ func (r *RootCmd) provisionerDaemonStart() *clibase.Cmd {
 		pollJitter     time.Duration
 		preSharedKey   string
 		verbose        bool
+
+		prometheusAddress string
 	)
 	client := new(codersdk.Client)
 	cmd := &clibase.Cmd{
@@ -165,6 +170,24 @@ func (r *RootCmd) provisionerDaemonStart() *clibase.Cmd {
 				}
 			}()
 
+			var metrics *provisionerd.Metrics
+			if prometheusAddress != "" {
+				logger.Info(ctx, "starting Prometheus endpoint", slog.F("address", prometheusAddress))
+
+				prometheusRegistry := prometheus.NewRegistry()
+				prometheusRegistry.MustRegister(collectors.NewGoCollector())
+				prometheusRegistry.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}))
+
+				m := provisionerd.NewMetrics(prometheusRegistry)
+				m.Runner.NumDaemons.Set(float64(1)) // Set numDaemons to 1 as this is standalone mode.
+				metrics = &m
+
+				closeFunc := agpl.ServeHandler(ctx, logger, promhttp.InstrumentMetricHandler(
+					prometheusRegistry, promhttp.HandlerFor(prometheusRegistry, promhttp.HandlerOpts{}),
+				), prometheusAddress, "prometheus")
+				defer closeFunc()
+			}
+
 			logger.Info(ctx, "starting provisioner daemon", slog.F("tags", tags), slog.F("name", name))
 
 			connector := provisionerd.LocalProvisioners{
@@ -185,6 +208,7 @@ func (r *RootCmd) provisionerDaemonStart() *clibase.Cmd {
 				Logger:         logger,
 				UpdateInterval: 500 * time.Millisecond,
 				Connector:      connector,
+				Metrics:        metrics,
 			})
 
 			var exitErr error
@@ -291,6 +315,13 @@ func (r *RootCmd) provisionerDaemonStart() *clibase.Cmd {
 			Value:       clibase.StringArrayOf(&logFilter),
 			Default:     "",
 		},
+		{
+			Flag:        "prometheus-address",
+			Env:         "CODER_PROMETHEUS_ADDRESS",
+			Description: "The bind address to serve prometheus metrics.",
+			Value:       clibase.StringOf(&prometheusAddress),
+			Default:     "127.0.0.1:2112",
+		},
 	}
 
 	return cmd

enterprise/cli/provisionerdaemons_test.go

@@ -1,7 +1,12 @@
 package cli_test
 
 import (
+	"bufio"
 	"context"
+	"fmt"
+	"net"
+	"net/http"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -161,4 +166,88 @@ func TestProvisionerDaemon_SessionToken(t *testing.T) {
 		assert.Equal(t, buildinfo.Version(), daemons[0].Version)
 		assert.Equal(t, proto.CurrentVersion.String(), daemons[0].APIVersion)
 	})
+
+	t.Run("PrometheusEnabled", func(t *testing.T) {
+		t.Parallel()
+
+		// Helper function to find a free random port
+		randomPort := func(t *testing.T) int {
+			random, err := net.Listen("tcp", "127.0.0.1:0")
+			require.NoError(t, err)
+			_ = random.Close()
+			tcpAddr, valid := random.Addr().(*net.TCPAddr)
+			require.True(t, valid)
+			return tcpAddr.Port
+		}
+		prometheusPort := randomPort(t)
+
+		// Configure CLI client
+		client, admin := coderdenttest.New(t, &coderdenttest.Options{
+			ProvisionerDaemonPSK: "provisionersftw",
+			LicenseOptions: &coderdenttest.LicenseOptions{
+				Features: license.Features{
+					codersdk.FeatureExternalProvisionerDaemons: 1,
+				},
+			},
+		})
+		anotherClient, _ := coderdtest.CreateAnotherUser(t, client, admin.OrganizationID, rbac.RoleTemplateAdmin())
+		inv, conf := newCLI(t, "provisionerd", "start", "--name", "daemon-with-prometheus", "--prometheus-address", fmt.Sprintf("127.0.0.1:%d", prometheusPort))
+		clitest.SetupConfig(t, anotherClient, conf)
+		pty := ptytest.New(t).Attach(inv)
+		ctx, cancel := context.WithTimeout(inv.Context(), testutil.WaitLong)
+		defer cancel()
+
+		// Start "provisionerd" command
+		clitest.Start(t, inv)
+		pty.ExpectMatchContext(ctx, "starting provisioner daemon")
+
+		var daemons []codersdk.ProvisionerDaemon
+		var err error
+		require.Eventually(t, func() bool {
+			daemons, err = client.ProvisionerDaemons(ctx)
+			if err != nil {
+				return false
+			}
+			return len(daemons) == 1
+		}, testutil.WaitShort, testutil.IntervalSlow)
+		require.Equal(t, "daemon-with-prometheus", daemons[0].Name)
+
+		// Fetch metrics from Prometheus endpoint
+		var res *http.Response
+		require.Eventually(t, func() bool {
+			req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("http://127.0.0.1:%d", prometheusPort), nil)
+			assert.NoError(t, err)
+			// nolint:bodyclose
+			res, err = http.DefaultClient.Do(req)
+			return err == nil
+		}, testutil.WaitShort, testutil.IntervalFast)
+		defer res.Body.Close()
+
+		// Scan for metric patterns
+		scanner := bufio.NewScanner(res.Body)
+		hasOneDaemon := false
+		hasGoStats := false
+		hasPromHTTP := false
+		for scanner.Scan() {
+			if strings.HasPrefix(scanner.Text(), "coderd_provisionerd_num_daemons 1") {
+				hasOneDaemon = true
+				continue
+			}
+			if strings.HasPrefix(scanner.Text(), "go_goroutines") {
+				hasGoStats = true
+				continue
+			}
+			if strings.HasPrefix(scanner.Text(), "promhttp_metric_handler_requests_total") {
+				hasPromHTTP = true
+				continue
+			}
+			t.Logf("scanned %s", scanner.Text())
+		}
+		require.NoError(t, scanner.Err())
+
+		// Verify patterns
+		require.True(t, hasOneDaemon, "should be one daemon running")
+		require.True(t, hasGoStats, "Go stats are missing")
+		require.True(t, hasPromHTTP, "Prometheus HTTP metrics are missing")
+	})
 }

enterprise/cli/testdata/coder_provisionerd_start_--help.golden

@@ -32,6 +32,9 @@ OPTIONS:
       --poll-jitter duration, $CODER_PROVISIONERD_POLL_JITTER (default: 100ms)
           Deprecated and ignored.
 
+      --prometheus-address string, $CODER_PROMETHEUS_ADDRESS (default: 127.0.0.1:2112)
+          The bind address to serve prometheus metrics.
+
       --psk string, $CODER_PROVISIONER_DAEMON_PSK
           Pre-shared key to authenticate with Coder server.