feat(coderd/healthcheck): add health check for proxy (#10846)

Adds a health check for workspace proxies: - Healthy iff all proxies are healthy and the same version, - Warning if some proxies are unhealthy, - Error if all proxies are unhealthy, or do not all have the same version.
2023-11-24 15:06:51 +00:00 · 2023-11-24 15:06:51 +00:00 · 411ce46442
parent b501046cf9
commit 411ce46442
15 changed files with 865 additions and 35 deletions
--- a/coderd/apidoc/docs.go
+++ b/coderd/apidoc/docs.go
@ -12388,6 +12388,9 @@ const docTemplate = `{
                },
                "websocket": {
                    "$ref": "#/definitions/healthcheck.WebsocketReport"
+                },
+                "workspace_proxy": {
+                    "$ref": "#/definitions/healthcheck.WorkspaceProxyReport"
                }
            }
        },
@ -12427,6 +12430,29 @@ const docTemplate = `{
                }
            }
        },
+        "healthcheck.WorkspaceProxyReport": {
+            "type": "object",
+            "properties": {
+                "error": {
+                    "type": "string"
+                },
+                "healthy": {
+                    "type": "boolean"
+                },
+                "severity": {
+                    "$ref": "#/definitions/health.Severity"
+                },
+                "warnings": {
+                    "type": "array",
+                    "items": {
+                        "type": "string"
+                    }
+                },
+                "workspace_proxies": {
+                    "$ref": "#/definitions/codersdk.RegionsResponse-codersdk_WorkspaceProxy"
+                }
+            }
+        },
        "netcheck.Report": {
            "type": "object",
            "properties": {
--- a/coderd/apidoc/swagger.json
+++ b/coderd/apidoc/swagger.json
@ -11277,6 +11277,9 @@
        },
        "websocket": {
          "$ref": "#/definitions/healthcheck.WebsocketReport"
+        },
+        "workspace_proxy": {
+          "$ref": "#/definitions/healthcheck.WorkspaceProxyReport"
        }
      }
    },
@ -11312,6 +11315,29 @@
        }
      }
    },
+    "healthcheck.WorkspaceProxyReport": {
+      "type": "object",
+      "properties": {
+        "error": {
+          "type": "string"
+        },
+        "healthy": {
+          "type": "boolean"
+        },
+        "severity": {
+          "$ref": "#/definitions/health.Severity"
+        },
+        "warnings": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        },
+        "workspace_proxies": {
+          "$ref": "#/definitions/codersdk.RegionsResponse-codersdk_WorkspaceProxy"
+        }
+      }
+    },
    "netcheck.Report": {
      "type": "object",
      "properties": {
--- a/coderd/coderd.go
+++ b/coderd/coderd.go
@ -135,10 +135,12 @@ type Options struct {
 	AccessControlStore          *atomic.Pointer[dbauthz.AccessControlStore]
 	// AppSecurityKey is the crypto key used to sign and encrypt tokens related to
 	// workspace applications. It consists of both a signing and encryption key.
-	AppSecurityKey     workspaceapps.SecurityKey
-	HealthcheckFunc    func(ctx context.Context, apiKey string) *healthcheck.Report
-	HealthcheckTimeout time.Duration
-	HealthcheckRefresh time.Duration
+	AppSecurityKey workspaceapps.SecurityKey
+
+	HealthcheckFunc              func(ctx context.Context, apiKey string) *healthcheck.Report
+	HealthcheckTimeout           time.Duration
+	HealthcheckRefresh           time.Duration
+	WorkspaceProxiesFetchUpdater *atomic.Pointer[healthcheck.WorkspaceProxiesFetchUpdater]

 	// OAuthSigningKey is the crypto key used to sign and encrypt state strings
 	// related to OAuth. This is a symmetric secret key using hmac to sign payloads.
@ -396,6 +398,13 @@ func New(options *Options) *API {
 			*options.UpdateCheckOptions,
 		)
 	}
+
+	if options.WorkspaceProxiesFetchUpdater == nil {
+		options.WorkspaceProxiesFetchUpdater = &atomic.Pointer[healthcheck.WorkspaceProxiesFetchUpdater]{}
+		var wpfu healthcheck.WorkspaceProxiesFetchUpdater = &healthcheck.AGPLWorkspaceProxiesFetchUpdater{}
+		options.WorkspaceProxiesFetchUpdater.Store(&wpfu)
+	}
+
 	if options.HealthcheckFunc == nil {
 		options.HealthcheckFunc = func(ctx context.Context, apiKey string) *healthcheck.Report {
 			return healthcheck.Run(ctx, &healthcheck.ReportOptions{
@ -413,9 +422,14 @@ func New(options *Options) *API {
 				DerpHealth: derphealth.ReportOptions{
 					DERPMap: api.DERPMap(),
 				},
+				WorkspaceProxy: healthcheck.WorkspaceProxyReportOptions{
+					CurrentVersion:               buildinfo.Version(),
+					WorkspaceProxiesFetchUpdater: *(options.WorkspaceProxiesFetchUpdater).Load(),
+				},
 			})
 		}
 	}
+
 	if options.HealthcheckTimeout == 0 {
 		options.HealthcheckTimeout = 30 * time.Second
 	}
--- a/coderd/healthcheck/healthcheck.go
+++ b/coderd/healthcheck/healthcheck.go
@ -13,10 +13,11 @@ import (
 )

 const (
-	SectionDERP      string = "DERP"
-	SectionAccessURL string = "AccessURL"
-	SectionWebsocket string = "Websocket"
-	SectionDatabase  string = "Database"
+	SectionDERP           string = "DERP"
+	SectionAccessURL      string = "AccessURL"
+	SectionWebsocket      string = "Websocket"
+	SectionDatabase       string = "Database"
+	SectionWorkspaceProxy string = "WorkspaceProxy"
 )

 type Checker interface {
@ -24,6 +25,7 @@ type Checker interface {
 	AccessURL(ctx context.Context, opts *AccessURLReportOptions) AccessURLReport
 	Websocket(ctx context.Context, opts *WebsocketReportOptions) WebsocketReport
 	Database(ctx context.Context, opts *DatabaseReportOptions) DatabaseReport
+	WorkspaceProxy(ctx context.Context, opts *WorkspaceProxyReportOptions) WorkspaceProxyReport
 }

 // @typescript-generate Report
@ -38,20 +40,22 @@ type Report struct {
 	// FailingSections is a list of sections that have failed their healthcheck.
 	FailingSections []string `json:"failing_sections"`

-	DERP      derphealth.Report `json:"derp"`
-	AccessURL AccessURLReport   `json:"access_url"`
-	Websocket WebsocketReport   `json:"websocket"`
-	Database  DatabaseReport    `json:"database"`
+	DERP           derphealth.Report    `json:"derp"`
+	AccessURL      AccessURLReport      `json:"access_url"`
+	Websocket      WebsocketReport      `json:"websocket"`
+	Database       DatabaseReport       `json:"database"`
+	WorkspaceProxy WorkspaceProxyReport `json:"workspace_proxy"`

 	// The Coder version of the server that the report was generated on.
 	CoderVersion string `json:"coder_version"`
 }

 type ReportOptions struct {
-	AccessURL  AccessURLReportOptions
-	Database   DatabaseReportOptions
-	DerpHealth derphealth.ReportOptions
-	Websocket  WebsocketReportOptions
+	AccessURL      AccessURLReportOptions
+	Database       DatabaseReportOptions
+	DerpHealth     derphealth.ReportOptions
+	Websocket      WebsocketReportOptions
+	WorkspaceProxy WorkspaceProxyReportOptions

 	Checker Checker
 }
@ -78,6 +82,11 @@ func (defaultChecker) Database(ctx context.Context, opts *DatabaseReportOptions)
 	return report
 }

+func (defaultChecker) WorkspaceProxy(ctx context.Context, opts *WorkspaceProxyReportOptions) (report WorkspaceProxyReport) {
+	report.Run(ctx, opts)
+	return report
+}
+
 func Run(ctx context.Context, opts *ReportOptions) *Report {
 	var (
 		wg     sync.WaitGroup
@ -136,6 +145,18 @@ func Run(ctx context.Context, opts *ReportOptions) *Report {
 		report.Database = opts.Checker.Database(ctx, &opts.Database)
 	}()

+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		defer func() {
+			if err := recover(); err != nil {
+				report.WorkspaceProxy.Error = ptr.Ref(fmt.Sprint(err))
+			}
+		}()
+
+		report.WorkspaceProxy = opts.Checker.WorkspaceProxy(ctx, &opts.WorkspaceProxy)
+	}()
+
 	report.CoderVersion = buildinfo.Version()
 	wg.Wait()

@ -153,6 +174,9 @@ func Run(ctx context.Context, opts *ReportOptions) *Report {
 	if !report.Database.Healthy {
 		report.FailingSections = append(report.FailingSections, SectionDatabase)
 	}
+	if !report.WorkspaceProxy.Healthy {
+		report.FailingSections = append(report.FailingSections, SectionWorkspaceProxy)
+	}

 	report.Healthy = len(report.FailingSections) == 0

@ -171,6 +195,9 @@ func Run(ctx context.Context, opts *ReportOptions) *Report {
 	if report.Database.Severity.Value() > report.Severity.Value() {
 		report.Severity = report.Database.Severity
 	}
+	if report.WorkspaceProxy.Severity.Value() > report.Severity.Value() {
+		report.Severity = report.WorkspaceProxy.Severity
+	}
 	return &report
 }

--- a/coderd/healthcheck/healthcheck_test.go
+++ b/coderd/healthcheck/healthcheck_test.go
@ -12,10 +12,11 @@ import (
 )

 type testChecker struct {
-	DERPReport      derphealth.Report
-	AccessURLReport healthcheck.AccessURLReport
-	WebsocketReport healthcheck.WebsocketReport
-	DatabaseReport  healthcheck.DatabaseReport
+	DERPReport           derphealth.Report
+	AccessURLReport      healthcheck.AccessURLReport
+	WebsocketReport      healthcheck.WebsocketReport
+	DatabaseReport       healthcheck.DatabaseReport
+	WorkspaceProxyReport healthcheck.WorkspaceProxyReport
 }

 func (c *testChecker) DERP(context.Context, *derphealth.ReportOptions) derphealth.Report {
@ -34,6 +35,10 @@ func (c *testChecker) Database(context.Context, *healthcheck.DatabaseReportOptio
 	return c.DatabaseReport
 }

+func (c *testChecker) WorkspaceProxy(context.Context, *healthcheck.WorkspaceProxyReportOptions) healthcheck.WorkspaceProxyReport {
+	return c.WorkspaceProxyReport
+}
+
 func TestHealthcheck(t *testing.T) {
 	t.Parallel()

@ -62,6 +67,10 @@ func TestHealthcheck(t *testing.T) {
 				Healthy:  true,
 				Severity: health.SeverityOK,
 			},
+			WorkspaceProxyReport: healthcheck.WorkspaceProxyReport{
+				Healthy:  true,
+				Severity: health.SeverityOK,
+			},
 		},
 		healthy:         true,
 		severity:        health.SeverityOK,
@ -85,6 +94,10 @@ func TestHealthcheck(t *testing.T) {
 				Healthy:  true,
 				Severity: health.SeverityOK,
 			},
+			WorkspaceProxyReport: healthcheck.WorkspaceProxyReport{
+				Healthy:  true,
+				Severity: health.SeverityOK,
+			},
 		},
 		healthy:         false,
 		severity:        health.SeverityError,
@ -109,6 +122,10 @@ func TestHealthcheck(t *testing.T) {
 				Healthy:  true,
 				Severity: health.SeverityOK,
 			},
+			WorkspaceProxyReport: healthcheck.WorkspaceProxyReport{
+				Healthy:  true,
+				Severity: health.SeverityOK,
+			},
 		},
 		healthy:         true,
 		severity:        health.SeverityWarning,
@ -132,6 +149,10 @@ func TestHealthcheck(t *testing.T) {
 				Healthy:  true,
 				Severity: health.SeverityOK,
 			},
+			WorkspaceProxyReport: healthcheck.WorkspaceProxyReport{
+				Healthy:  true,
+				Severity: health.SeverityOK,
+			},
 		},
 		healthy:         false,
 		severity:        health.SeverityWarning,
@ -155,6 +176,10 @@ func TestHealthcheck(t *testing.T) {
 				Healthy:  true,
 				Severity: health.SeverityOK,
 			},
+			WorkspaceProxyReport: healthcheck.WorkspaceProxyReport{
+				Healthy:  true,
+				Severity: health.SeverityOK,
+			},
 		},
 		healthy:         false,
 		severity:        health.SeverityError,
@ -178,12 +203,44 @@ func TestHealthcheck(t *testing.T) {
 				Healthy:  false,
 				Severity: health.SeverityError,
 			},
+			WorkspaceProxyReport: healthcheck.WorkspaceProxyReport{
+				Healthy:  true,
+				Severity: health.SeverityOK,
+			},
 		},
 		healthy:         false,
 		severity:        health.SeverityError,
 		failingSections: []string{healthcheck.SectionDatabase},
 	}, {
-		name: "AllFail",
+		name: "ProxyFail",
+		checker: &testChecker{
+			DERPReport: derphealth.Report{
+				Healthy:  true,
+				Severity: health.SeverityOK,
+			},
+			AccessURLReport: healthcheck.AccessURLReport{
+				Healthy:  true,
+				Severity: health.SeverityOK,
+			},
+			WebsocketReport: healthcheck.WebsocketReport{
+				Healthy:  true,
+				Severity: health.SeverityOK,
+			},
+			DatabaseReport: healthcheck.DatabaseReport{
+				Healthy:  true,
+				Severity: health.SeverityOK,
+			},
+			WorkspaceProxyReport: healthcheck.WorkspaceProxyReport{
+				Healthy:  false,
+				Severity: health.SeverityError,
+			},
+		},
+		severity:        health.SeverityError,
+		healthy:         false,
+		failingSections: []string{healthcheck.SectionWorkspaceProxy},
+	}, {
+		name:    "AllFail",
+		healthy: false,
 		checker: &testChecker{
 			DERPReport: derphealth.Report{
 				Healthy:  false,
@ -201,14 +258,18 @@ func TestHealthcheck(t *testing.T) {
 				Healthy:  false,
 				Severity: health.SeverityError,
 			},
+			WorkspaceProxyReport: healthcheck.WorkspaceProxyReport{
+				Healthy:  false,
+				Severity: health.SeverityError,
+			},
 		},
-		healthy:  false,
 		severity: health.SeverityError,
 		failingSections: []string{
 			healthcheck.SectionDERP,
 			healthcheck.SectionAccessURL,
 			healthcheck.SectionWebsocket,
 			healthcheck.SectionDatabase,
+			healthcheck.SectionWorkspaceProxy,
 		},
 	}} {
 		c := c
@ -228,6 +289,8 @@ func TestHealthcheck(t *testing.T) {
 			assert.Equal(t, c.checker.AccessURLReport.Healthy, report.AccessURL.Healthy)
 			assert.Equal(t, c.checker.AccessURLReport.Severity, report.AccessURL.Severity)
 			assert.Equal(t, c.checker.WebsocketReport.Healthy, report.Websocket.Healthy)
+			assert.Equal(t, c.checker.WorkspaceProxyReport.Healthy, report.WorkspaceProxy.Healthy)
+			assert.Equal(t, c.checker.WorkspaceProxyReport.Warnings, report.WorkspaceProxy.Warnings)
 			assert.Equal(t, c.checker.WebsocketReport.Severity, report.Websocket.Severity)
 			assert.Equal(t, c.checker.DatabaseReport.Healthy, report.Database.Healthy)
 			assert.Equal(t, c.checker.DatabaseReport.Severity, report.Database.Severity)
--- a/coderd/healthcheck/workspaceproxy.go
+++ b/coderd/healthcheck/workspaceproxy.go
@ -0,0 +1,146 @@
+package healthcheck
+
+import (
+	"context"
+	"errors"
+	"sort"
+
+	"golang.org/x/xerrors"
+
+	"github.com/coder/coder/v2/buildinfo"
+	"github.com/coder/coder/v2/coderd/healthcheck/health"
+	"github.com/coder/coder/v2/coderd/util/ptr"
+	"github.com/coder/coder/v2/codersdk"
+)
+
+type WorkspaceProxyReportOptions struct {
+	// CurrentVersion is the current server version.
+	// We pass this in to make it easier to test.
+	CurrentVersion               string
+	WorkspaceProxiesFetchUpdater WorkspaceProxiesFetchUpdater
+}
+
+// @typescript-generate WorkspaceProxyReport
+type WorkspaceProxyReport struct {
+	Healthy  bool            `json:"healthy"`
+	Severity health.Severity `json:"severity"`
+	Warnings []string        `json:"warnings"`
+	Error    *string         `json:"error"`
+
+	WorkspaceProxies codersdk.RegionsResponse[codersdk.WorkspaceProxy] `json:"workspace_proxies"`
+}
+
+type WorkspaceProxiesFetchUpdater interface {
+	Fetch(context.Context) (codersdk.RegionsResponse[codersdk.WorkspaceProxy], error)
+	Update(context.Context) error
+}
+
+// AGPLWorkspaceProxiesFetchUpdater implements WorkspaceProxiesFetchUpdater
+// to the extent required by AGPL code. Which isn't that much.
+type AGPLWorkspaceProxiesFetchUpdater struct{}
+
+func (*AGPLWorkspaceProxiesFetchUpdater) Fetch(context.Context) (codersdk.RegionsResponse[codersdk.WorkspaceProxy], error) {
+	return codersdk.RegionsResponse[codersdk.WorkspaceProxy]{}, nil
+}
+
+func (*AGPLWorkspaceProxiesFetchUpdater) Update(context.Context) error {
+	return nil
+}
+
+func (r *WorkspaceProxyReport) Run(ctx context.Context, opts *WorkspaceProxyReportOptions) {
+	r.Healthy = true
+	r.Severity = health.SeverityOK
+	r.Warnings = []string{}
+
+	if opts.WorkspaceProxiesFetchUpdater == nil {
+		opts.WorkspaceProxiesFetchUpdater = &AGPLWorkspaceProxiesFetchUpdater{}
+	}
+
+	// If this fails, just mark it as a warning. It is still updated in the background.
+	if err := opts.WorkspaceProxiesFetchUpdater.Update(ctx); err != nil {
+		r.Severity = health.SeverityWarning
+		r.Warnings = append(r.Warnings, xerrors.Errorf("update proxy health: %w", err).Error())
+		return
+	}
+
+	proxies, err := opts.WorkspaceProxiesFetchUpdater.Fetch(ctx)
+	if err != nil {
+		r.Healthy = false
+		r.Severity = health.SeverityError
+		r.Error = ptr.Ref(err.Error())
+		return
+	}
+
+	r.WorkspaceProxies = proxies
+	// Stable sort based on create timestamp.
+	sort.Slice(r.WorkspaceProxies.Regions, func(i int, j int) bool {
+		return r.WorkspaceProxies.Regions[i].CreatedAt.Before(r.WorkspaceProxies.Regions[j].CreatedAt)
+	})
+
+	var total, healthy int
+	for _, proxy := range r.WorkspaceProxies.Regions {
+		total++
+		if proxy.Healthy {
+			healthy++
+		}
+
+		if len(proxy.Status.Report.Errors) > 0 {
+			for _, err := range proxy.Status.Report.Errors {
+				r.appendError(xerrors.New(err))
+			}
+		}
+	}
+
+	r.Severity = calculateSeverity(total, healthy)
+	r.Healthy = r.Severity.Value() < health.SeverityError.Value()
+
+	// Versions _must_ match. Perform this check last. This will clobber any other severity.
+	for _, proxy := range r.WorkspaceProxies.Regions {
+		if vErr := checkVersion(proxy, opts.CurrentVersion); vErr != nil {
+			r.Healthy = false
+			r.Severity = health.SeverityError
+			r.appendError(vErr)
+		}
+	}
+}
+
+// appendError appends errs onto r.Error.
+// We only have one error, so multiple errors need to be squashed in there.
+func (r *WorkspaceProxyReport) appendError(errs ...error) {
+	if len(errs) == 0 {
+		return
+	}
+	if r.Error != nil {
+		errs = append([]error{xerrors.New(*r.Error)}, errs...)
+	}
+	r.Error = ptr.Ref(errors.Join(errs...).Error())
+}
+
+func checkVersion(proxy codersdk.WorkspaceProxy, currentVersion string) error {
+	if proxy.Version == "" {
+		return nil // may have not connected yet, this is OK
+	}
+	if buildinfo.VersionsMatch(proxy.Version, currentVersion) {
+		return nil
+	}
+
+	return xerrors.Errorf("proxy %q version %q does not match primary server version %q",
+		proxy.Name,
+		proxy.Version,
+		currentVersion,
+	)
+}
+
+// calculateSeverity returns:
+// health.SeverityError if all proxies are unhealthy,
+// health.SeverityOK if all proxies are healthy,
+// health.SeverityWarning otherwise.
+func calculateSeverity(total, healthy int) health.Severity {
+	if total == 0 || total == healthy {
+		return health.SeverityOK
+	}
+	if total-healthy == total {
+		return health.SeverityError
+	}
+	return health.SeverityWarning
+}
--- a/coderd/healthcheck/workspaceproxy_internal_test.go
+++ b/coderd/healthcheck/workspaceproxy_internal_test.go
@ -0,0 +1,95 @@
+package healthcheck
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/coder/coder/v2/coderd/healthcheck/health"
+	"github.com/coder/coder/v2/coderd/util/ptr"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"golang.org/x/xerrors"
+)
+
+func Test_WorkspaceProxyReport_appendErrors(t *testing.T) {
+	t.Parallel()
+
+	for _, tt := range []struct {
+		name     string
+		expected string
+		prevErr  string
+		errs     []error
+	}{
+		{
+			name: "nil",
+			errs: nil,
+		},
+		{
+			name:     "one error",
+			expected: assert.AnError.Error(),
+			errs:     []error{assert.AnError},
+		},
+		{
+			name:     "one error, one prev",
+			prevErr:  "previous error",
+			expected: "previous error\n" + assert.AnError.Error(),
+			errs:     []error{assert.AnError},
+		},
+		{
+			name:     "two errors",
+			expected: assert.AnError.Error() + "\nanother error",
+			errs:     []error{assert.AnError, xerrors.Errorf("another error")},
+		},
+		{
+			name:     "two errors, one prev",
+			prevErr:  "previous error",
+			expected: "previous error\n" + assert.AnError.Error() + "\nanother error",
+			errs:     []error{assert.AnError, xerrors.Errorf("another error")},
+		},
+	} {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			var rpt WorkspaceProxyReport
+			if tt.prevErr != "" {
+				rpt.Error = ptr.Ref(tt.prevErr)
+			}
+			rpt.appendError(tt.errs...)
+			if tt.expected == "" {
+				require.Nil(t, rpt.Error)
+			} else {
+				require.NotNil(t, rpt.Error)
+				require.Equal(t, tt.expected, *rpt.Error)
+			}
+		})
+	}
+}
+
+func Test_calculateSeverity(t *testing.T) {
+	t.Parallel()
+
+	for _, tt := range []struct {
+		total    int
+		healthy  int
+		expected health.Severity
+	}{
+		{0, 0, health.SeverityOK},
+		{1, 1, health.SeverityOK},
+		{1, 0, health.SeverityError},
+		{2, 2, health.SeverityOK},
+		{2, 1, health.SeverityWarning},
+		{2, 0, health.SeverityError},
+	} {
+		tt := tt
+		name := fmt.Sprintf("%d total, %d healthy -> %s", tt.total, tt.healthy, tt.expected)
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+
+			actual := calculateSeverity(tt.total, tt.healthy)
+			assert.Equal(t, tt.expected, actual)
+		})
+	}
+}
--- a/coderd/healthcheck/workspaceproxy_test.go
+++ b/coderd/healthcheck/workspaceproxy_test.go
@ -0,0 +1,238 @@
+package healthcheck_test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/coder/coder/v2/coderd/healthcheck"
+	"github.com/coder/coder/v2/coderd/healthcheck/health"
+	"github.com/coder/coder/v2/codersdk"
+)
+
+func TestWorkspaceProxies(t *testing.T) {
+	t.Parallel()
+
+	var (
+		newerPatchVersion = "v2.34.6"
+		currentVersion    = "v2.34.5"
+		olderVersion      = "v2.33.0"
+	)
+
+	for _, tt := range []struct {
+		name                  string
+		fetchWorkspaceProxies func(context.Context) (codersdk.RegionsResponse[codersdk.WorkspaceProxy], error)
+		updateProxyHealth     func(context.Context) error
+		expectedHealthy       bool
+		expectedError         string
+		expectedSeverity      health.Severity
+	}{
+		{
+			name:             "NotEnabled",
+			expectedHealthy:  true,
+			expectedSeverity: health.SeverityOK,
+		},
+		{
+			name:                  "Enabled/NoProxies",
+			fetchWorkspaceProxies: fakeFetchWorkspaceProxies(),
+			updateProxyHealth:     fakeUpdateProxyHealth(nil),
+			expectedHealthy:       true,
+			expectedSeverity:      health.SeverityOK,
+		},
+		{
+			name:                  "Enabled/OneHealthy",
+			fetchWorkspaceProxies: fakeFetchWorkspaceProxies(fakeWorkspaceProxy("alpha", true, currentVersion)),
+			updateProxyHealth:     fakeUpdateProxyHealth(nil),
+			expectedHealthy:       true,
+			expectedSeverity:      health.SeverityOK,
+		},
+		{
+			name:                  "Enabled/OneUnhealthy",
+			fetchWorkspaceProxies: fakeFetchWorkspaceProxies(fakeWorkspaceProxy("alpha", false, currentVersion)),
+			updateProxyHealth:     fakeUpdateProxyHealth(nil),
+			expectedHealthy:       false,
+			expectedSeverity:      health.SeverityError,
+		},
+		{
+			name: "Enabled/OneUnreachable",
+			fetchWorkspaceProxies: func(ctx context.Context) (codersdk.RegionsResponse[codersdk.WorkspaceProxy], error) {
+				return codersdk.RegionsResponse[codersdk.WorkspaceProxy]{
+					Regions: []codersdk.WorkspaceProxy{
+						{
+							Region: codersdk.Region{
+								Name:    "gone",
+								Healthy: false,
+							},
+							Version: currentVersion,
+							Status: codersdk.WorkspaceProxyStatus{
+								Status: codersdk.ProxyUnreachable,
+								Report: codersdk.ProxyHealthReport{
+									Errors: []string{
+										"request to proxy failed: Get \"http://127.0.0.1:3001/healthz-report\": dial tcp 127.0.0.1:3001: connect: connection refused",
+									},
+								},
+							},
+						},
+					},
+				}, nil
+			},
+			updateProxyHealth: fakeUpdateProxyHealth(nil),
+			expectedHealthy:   false,
+			expectedSeverity:  health.SeverityError,
+			expectedError:     "connect: connection refused",
+		},
+		{
+			name: "Enabled/AllHealthy",
+			fetchWorkspaceProxies: fakeFetchWorkspaceProxies(
+				fakeWorkspaceProxy("alpha", true, currentVersion),
+				fakeWorkspaceProxy("beta", true, currentVersion),
+			),
+			updateProxyHealth: func(ctx context.Context) error {
+				return nil
+			},
+			expectedHealthy:  true,
+			expectedSeverity: health.SeverityOK,
+		},
+		{
+			name: "Enabled/OneHealthyOneUnhealthy",
+			fetchWorkspaceProxies: fakeFetchWorkspaceProxies(
+				fakeWorkspaceProxy("alpha", false, currentVersion),
+				fakeWorkspaceProxy("beta", true, currentVersion),
+			),
+			updateProxyHealth: fakeUpdateProxyHealth(nil),
+			expectedHealthy:   true,
+			expectedSeverity:  health.SeverityWarning,
+		},
+		{
+			name: "Enabled/AllUnhealthy",
+			fetchWorkspaceProxies: fakeFetchWorkspaceProxies(
+				fakeWorkspaceProxy("alpha", false, currentVersion),
+				fakeWorkspaceProxy("beta", false, currentVersion),
+			),
+			updateProxyHealth: fakeUpdateProxyHealth(nil),
+			expectedHealthy:   false,
+			expectedSeverity:  health.SeverityError,
+		},
+		{
+			name: "Enabled/OneOutOfDate",
+			fetchWorkspaceProxies: fakeFetchWorkspaceProxies(
+				fakeWorkspaceProxy("alpha", true, currentVersion),
+				fakeWorkspaceProxy("beta", true, olderVersion),
+			),
+			updateProxyHealth: fakeUpdateProxyHealth(nil),
+			expectedHealthy:   false,
+			expectedSeverity:  health.SeverityError,
+			expectedError:     `proxy "beta" version "v2.33.0" does not match primary server version "v2.34.5"`,
+		},
+		{
+			name: "Enabled/OneSlightlyNewerButStillOK",
+			fetchWorkspaceProxies: fakeFetchWorkspaceProxies(
+				fakeWorkspaceProxy("alpha", true, currentVersion),
+				fakeWorkspaceProxy("beta", true, newerPatchVersion),
+			),
+			updateProxyHealth: fakeUpdateProxyHealth(nil),
+			expectedHealthy:   true,
+			expectedSeverity:  health.SeverityOK,
+		},
+		{
+			name: "Enabled/NotConnectedYet",
+			fetchWorkspaceProxies: fakeFetchWorkspaceProxies(
+				fakeWorkspaceProxy("slowpoke", true, ""),
+			),
+			updateProxyHealth: fakeUpdateProxyHealth(nil),
+			expectedHealthy:   true,
+			expectedSeverity:  health.SeverityOK,
+		},
+		{
+			name:                  "Enabled/ErrFetchWorkspaceProxy",
+			fetchWorkspaceProxies: fakeFetchWorkspaceProxiesErr(assert.AnError),
+			updateProxyHealth:     fakeUpdateProxyHealth(nil),
+			expectedHealthy:       false,
+			expectedSeverity:      health.SeverityError,
+			expectedError:         assert.AnError.Error(),
+		},
+		{
+			name:                  "Enabled/ErrUpdateProxyHealth",
+			fetchWorkspaceProxies: fakeFetchWorkspaceProxies(fakeWorkspaceProxy("alpha", true, currentVersion)),
+			updateProxyHealth:     fakeUpdateProxyHealth(assert.AnError),
+			expectedHealthy:       true,
+			expectedSeverity:      health.SeverityWarning,
+		},
+	} {
+		tt := tt
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+			var rpt healthcheck.WorkspaceProxyReport
+			var opts healthcheck.WorkspaceProxyReportOptions
+			opts.CurrentVersion = currentVersion
+			if tt.fetchWorkspaceProxies != nil && tt.updateProxyHealth != nil {
+				opts.WorkspaceProxiesFetchUpdater = &fakeWorkspaceProxyFetchUpdater{
+					fetchFunc:  tt.fetchWorkspaceProxies,
+					updateFunc: tt.updateProxyHealth,
+				}
+			} else {
+				opts.WorkspaceProxiesFetchUpdater = &healthcheck.AGPLWorkspaceProxiesFetchUpdater{}
+			}
+
+			rpt.Run(context.Background(), &opts)
+
+			assert.Equal(t, tt.expectedHealthy, rpt.Healthy)
+			assert.Equal(t, tt.expectedSeverity, rpt.Severity)
+			if tt.expectedError != "" {
+				assert.NotNil(t, rpt.Error)
+				assert.Contains(t, *rpt.Error, tt.expectedError)
+			} else {
+				if !assert.Nil(t, rpt.Error) {
+					assert.Empty(t, *rpt.Error)
+				}
+			}
+		})
+	}
+}
+
+// yet another implementation of the thing
+type fakeWorkspaceProxyFetchUpdater struct {
+	fetchFunc  func(context.Context) (codersdk.RegionsResponse[codersdk.WorkspaceProxy], error)
+	updateFunc func(context.Context) error
+}
+
+func (u *fakeWorkspaceProxyFetchUpdater) Fetch(ctx context.Context) (codersdk.RegionsResponse[codersdk.WorkspaceProxy], error) {
+	return u.fetchFunc(ctx)
+}
+
+func (u *fakeWorkspaceProxyFetchUpdater) Update(ctx context.Context) error {
+	return u.updateFunc(ctx)
+}
+
+func fakeWorkspaceProxy(name string, healthy bool, version string) codersdk.WorkspaceProxy {
+	return codersdk.WorkspaceProxy{
+		Region: codersdk.Region{
+			Name:    name,
+			Healthy: healthy,
+		},
+		Version: version,
+	}
+}
+
+func fakeFetchWorkspaceProxies(ps ...codersdk.WorkspaceProxy) func(context.Context) (codersdk.RegionsResponse[codersdk.WorkspaceProxy], error) {
+	return func(context.Context) (codersdk.RegionsResponse[codersdk.WorkspaceProxy], error) {
+		return codersdk.RegionsResponse[codersdk.WorkspaceProxy]{
+			Regions: ps,
+		}, nil
+	}
+}
+
+func fakeFetchWorkspaceProxiesErr(err error) func(context.Context) (codersdk.RegionsResponse[codersdk.WorkspaceProxy], error) {
+	return func(context.Context) (codersdk.RegionsResponse[codersdk.WorkspaceProxy], error) {
+		return codersdk.RegionsResponse[codersdk.WorkspaceProxy]{
+			Regions: []codersdk.WorkspaceProxy{},
+		}, err
+	}
+}
+
+func fakeUpdateProxyHealth(err error) func(context.Context) error {
+	return func(context.Context) error {
+		return err
+	}
+}
--- a/docs/api/debug.md
+++ b/docs/api/debug.md
@ -253,6 +253,39 @@ curl -X GET http://coder-server:8080/api/v2/debug/health \
    "healthy": true,
    "severity": "ok",
    "warnings": ["string"]
+  },
+  "workspace_proxy": {
+    "error": "string",
+    "healthy": true,
+    "severity": "ok",
+    "warnings": ["string"],
+    "workspace_proxies": {
+      "regions": [
+        {
+          "created_at": "2019-08-24T14:15:22Z",
+          "deleted": true,
+          "derp_enabled": true,
+          "derp_only": true,
+          "display_name": "string",
+          "healthy": true,
+          "icon_url": "string",
+          "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
+          "name": "string",
+          "path_app_url": "string",
+          "status": {
+            "checked_at": "2019-08-24T14:15:22Z",
+            "report": {
+              "errors": ["string"],
+              "warnings": ["string"]
+            },
+            "status": "ok"
+          },
+          "updated_at": "2019-08-24T14:15:22Z",
+          "version": "string",
+          "wildcard_hostname": "string"
+        }
+      ]
+    }
  }
 }
 ```
--- a/docs/api/schemas.md
+++ b/docs/api/schemas.md
@ -7789,23 +7789,57 @@ If the schedule is empty, the user will be updated to use the default schedule.|
    "healthy": true,
    "severity": "ok",
    "warnings": ["string"]
+  },
+  "workspace_proxy": {
+    "error": "string",
+    "healthy": true,
+    "severity": "ok",
+    "warnings": ["string"],
+    "workspace_proxies": {
+      "regions": [
+        {
+          "created_at": "2019-08-24T14:15:22Z",
+          "deleted": true,
+          "derp_enabled": true,
+          "derp_only": true,
+          "display_name": "string",
+          "healthy": true,
+          "icon_url": "string",
+          "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
+          "name": "string",
+          "path_app_url": "string",
+          "status": {
+            "checked_at": "2019-08-24T14:15:22Z",
+            "report": {
+              "errors": ["string"],
+              "warnings": ["string"]
+            },
+            "status": "ok"
+          },
+          "updated_at": "2019-08-24T14:15:22Z",
+          "version": "string",
+          "wildcard_hostname": "string"
+        }
+      ]
+    }
  }
 }
 ```

 ### Properties

-| Name               | Type                                                       | Required | Restrictions | Description                                                                         |
-| ------------------ | ---------------------------------------------------------- | -------- | ------------ | ----------------------------------------------------------------------------------- |
-| `access_url`       | [healthcheck.AccessURLReport](#healthcheckaccessurlreport) | false    |              |                                                                                     |
-| `coder_version`    | string                                                     | false    |              | The Coder version of the server that the report was generated on.                   |
-| `database`         | [healthcheck.DatabaseReport](#healthcheckdatabasereport)   | false    |              |                                                                                     |
-| `derp`             | [derphealth.Report](#derphealthreport)                     | false    |              |                                                                                     |
-| `failing_sections` | array of string                                            | false    |              | Failing sections is a list of sections that have failed their healthcheck.          |
-| `healthy`          | boolean                                                    | false    |              | Healthy is true if the report returns no errors. Deprecated: use `Severity` instead |
-| `severity`         | [health.Severity](#healthseverity)                         | false    |              | Severity indicates the status of Coder health.                                      |
-| `time`             | string                                                     | false    |              | Time is the time the report was generated at.                                       |
-| `websocket`        | [healthcheck.WebsocketReport](#healthcheckwebsocketreport) | false    |              |                                                                                     |
+| Name               | Type                                                                 | Required | Restrictions | Description                                                                         |
+| ------------------ | -------------------------------------------------------------------- | -------- | ------------ | ----------------------------------------------------------------------------------- |
+| `access_url`       | [healthcheck.AccessURLReport](#healthcheckaccessurlreport)           | false    |              |                                                                                     |
+| `coder_version`    | string                                                               | false    |              | The Coder version of the server that the report was generated on.                   |
+| `database`         | [healthcheck.DatabaseReport](#healthcheckdatabasereport)             | false    |              |                                                                                     |
+| `derp`             | [derphealth.Report](#derphealthreport)                               | false    |              |                                                                                     |
+| `failing_sections` | array of string                                                      | false    |              | Failing sections is a list of sections that have failed their healthcheck.          |
+| `healthy`          | boolean                                                              | false    |              | Healthy is true if the report returns no errors. Deprecated: use `Severity` instead |
+| `severity`         | [health.Severity](#healthseverity)                                   | false    |              | Severity indicates the status of Coder health.                                      |
+| `time`             | string                                                               | false    |              | Time is the time the report was generated at.                                       |
+| `websocket`        | [healthcheck.WebsocketReport](#healthcheckwebsocketreport)           | false    |              |                                                                                     |
+| `workspace_proxy`  | [healthcheck.WorkspaceProxyReport](#healthcheckworkspaceproxyreport) | false    |              |                                                                                     |

 #### Enumerated Values

@ -7847,6 +7881,54 @@ If the schedule is empty, the user will be updated to use the default schedule.|
 | `severity` | `warning` |
 | `severity` | `error`   |

+## healthcheck.WorkspaceProxyReport
+
+```json
+{
+  "error": "string",
+  "healthy": true,
+  "severity": "ok",
+  "warnings": ["string"],
+  "workspace_proxies": {
+    "regions": [
+      {
+        "created_at": "2019-08-24T14:15:22Z",
+        "deleted": true,
+        "derp_enabled": true,
+        "derp_only": true,
+        "display_name": "string",
+        "healthy": true,
+        "icon_url": "string",
+        "id": "497f6eca-6276-4993-bfeb-53cbbbba6f08",
+        "name": "string",
+        "path_app_url": "string",
+        "status": {
+          "checked_at": "2019-08-24T14:15:22Z",
+          "report": {
+            "errors": ["string"],
+            "warnings": ["string"]
+          },
+          "status": "ok"
+        },
+        "updated_at": "2019-08-24T14:15:22Z",
+        "version": "string",
+        "wildcard_hostname": "string"
+      }
+    ]
+  }
+}
+```
+
+### Properties
+
+| Name                | Type                                                                                                 | Required | Restrictions | Description |
+| ------------------- | ---------------------------------------------------------------------------------------------------- | -------- | ------------ | ----------- |
+| `error`             | string                                                                                               | false    |              |             |
+| `healthy`           | boolean                                                                                              | false    |              |             |
+| `severity`          | [health.Severity](#healthseverity)                                                                   | false    |              |             |
+| `warnings`          | array of string                                                                                      | false    |              |             |
+| `workspace_proxies` | [codersdk.RegionsResponse-codersdk_WorkspaceProxy](#codersdkregionsresponse-codersdk_workspaceproxy) | false    |              |             |
+
 ## netcheck.Report

 ```json
--- a/enterprise/coderd/coderd.go
+++ b/enterprise/coderd/coderd.go
@ -25,6 +25,7 @@ import (
 	"github.com/coder/coder/v2/coderd"
 	agplaudit "github.com/coder/coder/v2/coderd/audit"
 	agpldbauthz "github.com/coder/coder/v2/coderd/database/dbauthz"
+	"github.com/coder/coder/v2/coderd/healthcheck"
 	"github.com/coder/coder/v2/coderd/httpapi"
 	"github.com/coder/coder/v2/coderd/httpmw"
 	"github.com/coder/coder/v2/coderd/rbac"
@ -374,6 +375,13 @@ func New(ctx context.Context, options *Options) (_ *API, err error) {
 		// Use proxy health to return the healthy workspace proxy hostnames.
 		f := api.ProxyHealth.ProxyHosts
 		api.AGPL.WorkspaceProxyHostsFn.Store(&f)
+
+		// Wire this up to healthcheck.
+		var fetchUpdater healthcheck.WorkspaceProxiesFetchUpdater = &workspaceProxiesFetchUpdater{
+			fetchFunc:  api.fetchWorkspaceProxies,
+			updateFunc: api.ProxyHealth.ForceUpdate,
+		}
+		api.AGPL.WorkspaceProxiesFetchUpdater.Store(&fetchUpdater)
 	}

 	err = api.PrometheusRegistry.Register(&api.licenseMetricsCollector)
@ -552,8 +560,8 @@ func (api *API) updateEntitlements(ctx context.Context) error {
 				Log:      api.Logger.Named("quota_committer"),
 				Database: api.Database,
 			}
-			ptr := proto.QuotaCommitter(&committer)
-			api.AGPL.QuotaCommitter.Store(&ptr)
+			qcPtr := proto.QuotaCommitter(&committer)
+			api.AGPL.QuotaCommitter.Store(&qcPtr)
 		} else {
 			api.AGPL.QuotaCommitter.Store(nil)
 		}
--- a/enterprise/coderd/workspaceproxy.go
+++ b/enterprise/coderd/workspaceproxy.go
@ -960,3 +960,20 @@ func convertProxy(p database.WorkspaceProxy, status proxyhealth.ProxyStatus) cod
 		},
 	}
 }
+
+// workspaceProxiesFetchUpdater implements healthcheck.WorkspaceProxyFetchUpdater
+// in an actually useful and meaningful way.
+type workspaceProxiesFetchUpdater struct {
+	fetchFunc  func(context.Context) (codersdk.RegionsResponse[codersdk.WorkspaceProxy], error)
+	updateFunc func(context.Context) error
+}
+
+func (w *workspaceProxiesFetchUpdater) Fetch(ctx context.Context) (codersdk.RegionsResponse[codersdk.WorkspaceProxy], error) {
+	//nolint:gocritic // Need perms to read all workspace proxies.
+	authCtx := dbauthz.AsSystemRestricted(ctx)
+	return w.fetchFunc(authCtx)
+}
+
+func (w *workspaceProxiesFetchUpdater) Update(ctx context.Context) error {
+	return w.updateFunc(ctx)
+}
--- a/scripts/apitypings/main.go
+++ b/scripts/apitypings/main.go
@ -867,6 +867,10 @@ func (g *Generator) typescriptType(ty types.Type) (TypescriptType, error) {
 			return TypescriptType{ValueType: "Record<string, string>"}, nil
 		case "github.com/coder/coder/v2/cli/clibase.URL":
 			return TypescriptType{ValueType: "string"}, nil
+		// XXX: For some reason, the type generator generates this as `any`
+		//      so explicitly specifying the correct generic TS type.
+		case "github.com/coder/coder/v2/codersdk.RegionsResponse[github.com/coder/coder/v2/codersdk.WorkspaceProxy]":
+			return TypescriptType{ValueType: "RegionsResponse<WorkspaceProxy>"}, nil
 		}

 		// Some hard codes are a bit trickier.
--- a/site/src/api/typesGenerated.ts
+++ b/site/src/api/typesGenerated.ts
@ -2115,6 +2115,7 @@ export interface HealthcheckReport {
  readonly access_url: HealthcheckAccessURLReport;
  readonly websocket: HealthcheckWebsocketReport;
  readonly database: HealthcheckDatabaseReport;
+  readonly workspace_proxy: HealthcheckWorkspaceProxyReport;
  readonly coder_version: string;
 }

@ -2128,6 +2129,15 @@ export interface HealthcheckWebsocketReport {
  readonly error?: string;
 }

+// From healthcheck/workspaceproxy.go
+export interface HealthcheckWorkspaceProxyReport {
+  readonly healthy: boolean;
+  readonly severity: HealthSeverity;
+  readonly warnings: string[];
+  readonly error?: string;
+  readonly workspace_proxies: RegionsResponse<WorkspaceProxy>;
+}
+
 // The code below is generated from cli/clibase.

 // From clibase/clibase.go
--- a/site/src/testHelpers/entities.ts
+++ b/site/src/testHelpers/entities.ts
@ -2827,6 +2827,14 @@ export const MockHealth: TypesGen.HealthcheckReport = {
    latency_ms: 92570,
    threshold_ms: 92570,
  },
+  workspace_proxy: {
+    healthy: true,
+    severity: "ok",
+    warnings: [],
+    workspace_proxies: {
+      regions: [],
+    },
+  },
  coder_version: "v0.27.1-devel+c575292",
 };

@ -2877,4 +2885,37 @@ export const DeploymentHealthUnhealthy: TypesGen.HealthcheckReport = {
    body: "",
    code: 0,
  },
+  workspace_proxy: {
+    healthy: false,
+    error: "some error",
+    severity: "error",
+    warnings: [],
+    workspace_proxies: {
+      regions: [
+        {
+          id: "df7e4b2b-2d40-47e5-a021-e5d08b219c77",
+          name: "unhealthy",
+          display_name: "unhealthy",
+          icon_url: "/emojis/1f5fa.png",
+          healthy: false,
+          path_app_url: "http://127.0.0.1:3001",
+          wildcard_hostname: "",
+          derp_enabled: true,
+          derp_only: false,
+          status: {
+            status: "unreachable",
+            report: {
+              errors: ["some error"],
+              warnings: [],
+            },
+            checked_at: "2023-11-24T12:14:05.743303497Z",
+          },
+          created_at: "2023-11-23T15:37:25.513213Z",
+          updated_at: "2023-11-23T18:09:19.734747Z",
+          deleted: false,
+          version: "v2.4.0-devel+89bae7eff",
+        },
+      ],
+    },
+  },
 };