2023-01-25 21:27:36 +00:00
|
|
|
package coderd
|
|
|
|
|
2023-04-03 06:28:42 +00:00
|
|
|
import (
|
2023-11-28 17:15:17 +00:00
|
|
|
"bytes"
|
2023-04-03 06:28:42 +00:00
|
|
|
"context"
|
2023-11-29 16:37:40 +00:00
|
|
|
"database/sql"
|
2023-11-28 17:15:17 +00:00
|
|
|
"encoding/json"
|
2023-09-25 22:55:50 +00:00
|
|
|
"fmt"
|
2023-04-03 06:28:42 +00:00
|
|
|
"net/http"
|
|
|
|
"time"
|
|
|
|
|
2023-11-29 16:37:40 +00:00
|
|
|
"github.com/google/uuid"
|
2023-11-28 17:15:17 +00:00
|
|
|
"golang.org/x/exp/slices"
|
|
|
|
"golang.org/x/xerrors"
|
|
|
|
|
2023-11-29 16:37:40 +00:00
|
|
|
"cdr.dev/slog"
|
2023-11-28 17:15:17 +00:00
|
|
|
"github.com/coder/coder/v2/coderd/audit"
|
|
|
|
"github.com/coder/coder/v2/coderd/database"
|
2023-08-18 18:55:43 +00:00
|
|
|
"github.com/coder/coder/v2/coderd/httpapi"
|
|
|
|
"github.com/coder/coder/v2/coderd/httpmw"
|
2023-11-28 17:15:17 +00:00
|
|
|
"github.com/coder/coder/v2/coderd/rbac"
|
2023-08-18 18:55:43 +00:00
|
|
|
"github.com/coder/coder/v2/codersdk"
|
2024-03-26 17:44:31 +00:00
|
|
|
"github.com/coder/coder/v2/codersdk/healthsdk"
|
2023-04-03 06:28:42 +00:00
|
|
|
)
|
2023-01-25 21:27:36 +00:00
|
|
|
|
|
|
|
// @Summary Debug Info Wireguard Coordinator
|
|
|
|
// @ID debug-info-wireguard-coordinator
|
|
|
|
// @Security CoderSessionToken
|
|
|
|
// @Produce text/html
|
|
|
|
// @Tags Debug
|
|
|
|
// @Success 200
|
|
|
|
// @Router /debug/coordinator [get]
|
|
|
|
func (api *API) debugCoordinator(rw http.ResponseWriter, r *http.Request) {
|
|
|
|
(*api.TailnetCoordinator.Load()).ServeHTTPDebug(rw, r)
|
|
|
|
}
|
2023-04-03 06:28:42 +00:00
|
|
|
|
2023-11-13 23:14:12 +00:00
|
|
|
// @Summary Debug Info Tailnet
|
|
|
|
// @ID debug-info-tailnet
|
|
|
|
// @Security CoderSessionToken
|
|
|
|
// @Produce text/html
|
|
|
|
// @Tags Debug
|
|
|
|
// @Success 200
|
|
|
|
// @Router /debug/tailnet [get]
|
|
|
|
func (api *API) debugTailnet(rw http.ResponseWriter, r *http.Request) {
|
|
|
|
api.agentProvider.ServeHTTPDebug(rw, r)
|
|
|
|
}
|
|
|
|
|
2023-04-03 06:28:42 +00:00
|
|
|
// @Summary Debug Info Deployment Health
|
|
|
|
// @ID debug-info-deployment-health
|
|
|
|
// @Security CoderSessionToken
|
|
|
|
// @Produce json
|
|
|
|
// @Tags Debug
|
2024-03-26 17:44:31 +00:00
|
|
|
// @Success 200 {object} healthsdk.HealthcheckReport
|
2023-04-03 06:28:42 +00:00
|
|
|
// @Router /debug/health [get]
|
2023-11-15 15:54:15 +00:00
|
|
|
// @Param force query boolean false "Force a healthcheck to run"
|
2023-04-03 06:28:42 +00:00
|
|
|
func (api *API) debugDeploymentHealth(rw http.ResponseWriter, r *http.Request) {
|
2023-05-30 19:22:32 +00:00
|
|
|
apiKey := httpmw.APITokenFromRequest(r)
|
2023-11-13 14:14:43 +00:00
|
|
|
ctx, cancel := context.WithTimeout(r.Context(), api.Options.HealthcheckTimeout)
|
2023-04-03 06:28:42 +00:00
|
|
|
defer cancel()
|
|
|
|
|
2023-12-06 08:38:03 +00:00
|
|
|
// Load sections previously marked as dismissed.
|
|
|
|
// We hydrate this here as we cache the healthcheck and hydrating in the
|
|
|
|
// healthcheck function itself can lead to stale results.
|
|
|
|
dismissed := loadDismissedHealthchecks(ctx, api.Database, api.Logger)
|
|
|
|
|
2023-11-15 15:54:15 +00:00
|
|
|
// Check if the forced query parameter is set.
|
|
|
|
forced := r.URL.Query().Get("force") == "true"
|
|
|
|
|
|
|
|
// Get cached report if it exists and the requester did not force a refresh.
|
|
|
|
if !forced {
|
|
|
|
if report := api.healthCheckCache.Load(); report != nil {
|
|
|
|
if time.Since(report.Time) < api.Options.HealthcheckRefresh {
|
2023-12-06 08:38:03 +00:00
|
|
|
formatHealthcheck(ctx, rw, r, *report, dismissed...)
|
2023-11-15 15:54:15 +00:00
|
|
|
return
|
|
|
|
}
|
2023-05-30 19:22:32 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2024-03-26 17:44:31 +00:00
|
|
|
resChan := api.healthCheckGroup.DoChan("", func() (*healthsdk.HealthcheckReport, error) {
|
2023-05-30 19:22:32 +00:00
|
|
|
// Create a new context not tied to the request.
|
2023-11-13 14:14:43 +00:00
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), api.Options.HealthcheckTimeout)
|
2023-05-30 19:22:32 +00:00
|
|
|
defer cancel()
|
|
|
|
|
2023-06-02 00:21:24 +00:00
|
|
|
report := api.HealthcheckFunc(ctx, apiKey)
|
|
|
|
api.healthCheckCache.Store(report)
|
|
|
|
return report, nil
|
2023-04-03 06:28:42 +00:00
|
|
|
})
|
|
|
|
|
|
|
|
select {
|
|
|
|
case <-ctx.Done():
|
|
|
|
httpapi.Write(ctx, rw, http.StatusNotFound, codersdk.Response{
|
|
|
|
Message: "Healthcheck is in progress and did not complete in time. Try again in a few seconds.",
|
|
|
|
})
|
|
|
|
return
|
|
|
|
case res := <-resChan:
|
2023-12-06 08:38:03 +00:00
|
|
|
report := res.Val
|
|
|
|
if report == nil {
|
|
|
|
httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{
|
|
|
|
Message: "There was an unknown error completing the healthcheck.",
|
|
|
|
Detail: "nil report from healthcheck result channel",
|
|
|
|
})
|
|
|
|
return
|
|
|
|
}
|
|
|
|
formatHealthcheck(ctx, rw, r, *report, dismissed...)
|
2023-04-03 06:28:42 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
}
|
2023-05-30 19:22:32 +00:00
|
|
|
|
2024-03-26 17:44:31 +00:00
|
|
|
func formatHealthcheck(ctx context.Context, rw http.ResponseWriter, r *http.Request, hc healthsdk.HealthcheckReport, dismissed ...healthsdk.HealthSection) {
|
2023-12-06 08:38:03 +00:00
|
|
|
// Mark any sections previously marked as dismissed.
|
|
|
|
for _, d := range dismissed {
|
|
|
|
switch d {
|
2024-03-26 17:44:31 +00:00
|
|
|
case healthsdk.HealthSectionAccessURL:
|
2023-12-06 08:38:03 +00:00
|
|
|
hc.AccessURL.Dismissed = true
|
2024-03-26 17:44:31 +00:00
|
|
|
case healthsdk.HealthSectionDERP:
|
2023-12-06 08:38:03 +00:00
|
|
|
hc.DERP.Dismissed = true
|
2024-03-26 17:44:31 +00:00
|
|
|
case healthsdk.HealthSectionDatabase:
|
2023-12-06 08:38:03 +00:00
|
|
|
hc.Database.Dismissed = true
|
2024-03-26 17:44:31 +00:00
|
|
|
case healthsdk.HealthSectionWebsocket:
|
2023-12-06 08:38:03 +00:00
|
|
|
hc.Websocket.Dismissed = true
|
2024-03-26 17:44:31 +00:00
|
|
|
case healthsdk.HealthSectionWorkspaceProxy:
|
2023-12-06 08:38:03 +00:00
|
|
|
hc.WorkspaceProxy.Dismissed = true
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-09-25 22:55:50 +00:00
|
|
|
format := r.URL.Query().Get("format")
|
|
|
|
switch format {
|
|
|
|
case "text":
|
|
|
|
rw.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
|
|
|
rw.WriteHeader(http.StatusOK)
|
|
|
|
|
|
|
|
_, _ = fmt.Fprintln(rw, "time:", hc.Time.Format(time.RFC3339))
|
|
|
|
_, _ = fmt.Fprintln(rw, "healthy:", hc.Healthy)
|
|
|
|
_, _ = fmt.Fprintln(rw, "derp:", hc.DERP.Healthy)
|
|
|
|
_, _ = fmt.Fprintln(rw, "access_url:", hc.AccessURL.Healthy)
|
|
|
|
_, _ = fmt.Fprintln(rw, "websocket:", hc.Websocket.Healthy)
|
|
|
|
_, _ = fmt.Fprintln(rw, "database:", hc.Database.Healthy)
|
|
|
|
|
|
|
|
case "", "json":
|
|
|
|
httpapi.WriteIndent(ctx, rw, http.StatusOK, hc)
|
|
|
|
|
|
|
|
default:
|
|
|
|
httpapi.Write(ctx, rw, http.StatusBadRequest, codersdk.Response{
|
|
|
|
Message: fmt.Sprintf("Invalid format option %q.", format),
|
|
|
|
Detail: "Allowed values are: \"json\", \"simple\".",
|
|
|
|
})
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-11-28 17:15:17 +00:00
|
|
|
// @Summary Get health settings
|
|
|
|
// @ID get-health-settings
|
|
|
|
// @Security CoderSessionToken
|
|
|
|
// @Produce json
|
|
|
|
// @Tags Debug
|
2024-03-26 17:44:31 +00:00
|
|
|
// @Success 200 {object} healthsdk.HealthSettings
|
2023-11-28 17:15:17 +00:00
|
|
|
// @Router /debug/health/settings [get]
|
|
|
|
func (api *API) deploymentHealthSettings(rw http.ResponseWriter, r *http.Request) {
|
|
|
|
settingsJSON, err := api.Database.GetHealthSettings(r.Context())
|
|
|
|
if err != nil {
|
|
|
|
httpapi.Write(r.Context(), rw, http.StatusInternalServerError, codersdk.Response{
|
|
|
|
Message: "Failed to fetch health settings.",
|
|
|
|
Detail: err.Error(),
|
|
|
|
})
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2024-03-26 17:44:31 +00:00
|
|
|
var settings healthsdk.HealthSettings
|
2023-11-28 17:15:17 +00:00
|
|
|
err = json.Unmarshal([]byte(settingsJSON), &settings)
|
|
|
|
if err != nil {
|
|
|
|
httpapi.Write(r.Context(), rw, http.StatusInternalServerError, codersdk.Response{
|
|
|
|
Message: "Failed to unmarshal health settings.",
|
|
|
|
Detail: err.Error(),
|
|
|
|
})
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
if len(settings.DismissedHealthchecks) == 0 {
|
2024-03-26 17:44:31 +00:00
|
|
|
settings.DismissedHealthchecks = []healthsdk.HealthSection{}
|
2023-11-28 17:15:17 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
httpapi.Write(r.Context(), rw, http.StatusOK, settings)
|
|
|
|
}
|
|
|
|
|
|
|
|
// @Summary Update health settings
|
|
|
|
// @ID update-health-settings
|
|
|
|
// @Security CoderSessionToken
|
|
|
|
// @Accept json
|
|
|
|
// @Produce json
|
|
|
|
// @Tags Debug
|
2024-03-26 17:44:31 +00:00
|
|
|
// @Param request body healthsdk.UpdateHealthSettings true "Update health settings"
|
|
|
|
// @Success 200 {object} healthsdk.UpdateHealthSettings
|
2023-11-28 17:15:17 +00:00
|
|
|
// @Router /debug/health/settings [put]
|
|
|
|
func (api *API) putDeploymentHealthSettings(rw http.ResponseWriter, r *http.Request) {
|
|
|
|
ctx := r.Context()
|
|
|
|
|
|
|
|
if !api.Authorize(r, rbac.ActionUpdate, rbac.ResourceDeploymentValues) {
|
|
|
|
httpapi.Write(ctx, rw, http.StatusForbidden, codersdk.Response{
|
|
|
|
Message: "Insufficient permissions to update health settings.",
|
|
|
|
})
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
2024-03-26 17:44:31 +00:00
|
|
|
var settings healthsdk.HealthSettings
|
2023-11-28 17:15:17 +00:00
|
|
|
if !httpapi.Read(ctx, rw, r, &settings) {
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
err := validateHealthSettings(settings)
|
|
|
|
if err != nil {
|
|
|
|
httpapi.Write(r.Context(), rw, http.StatusInternalServerError, codersdk.Response{
|
|
|
|
Message: "Failed to validate health settings.",
|
|
|
|
Detail: err.Error(),
|
|
|
|
})
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
settingsJSON, err := json.Marshal(&settings)
|
|
|
|
if err != nil {
|
|
|
|
httpapi.Write(r.Context(), rw, http.StatusInternalServerError, codersdk.Response{
|
|
|
|
Message: "Failed to marshal health settings.",
|
|
|
|
Detail: err.Error(),
|
|
|
|
})
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
currentSettingsJSON, err := api.Database.GetHealthSettings(r.Context())
|
|
|
|
if err != nil {
|
|
|
|
httpapi.Write(r.Context(), rw, http.StatusInternalServerError, codersdk.Response{
|
|
|
|
Message: "Failed to fetch current health settings.",
|
|
|
|
Detail: err.Error(),
|
|
|
|
})
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
if bytes.Equal(settingsJSON, []byte(currentSettingsJSON)) {
|
2023-12-05 19:06:56 +00:00
|
|
|
// See: https://www.rfc-editor.org/rfc/rfc7231#section-6.3.5
|
|
|
|
httpapi.Write(r.Context(), rw, http.StatusNoContent, nil)
|
2023-11-28 17:15:17 +00:00
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
auditor := api.Auditor.Load()
|
|
|
|
aReq, commitAudit := audit.InitRequest[database.HealthSettings](rw, &audit.RequestParams{
|
|
|
|
Audit: *auditor,
|
|
|
|
Log: api.Logger,
|
|
|
|
Request: r,
|
|
|
|
Action: database.AuditActionWrite,
|
|
|
|
})
|
|
|
|
defer commitAudit()
|
2023-12-05 20:00:27 +00:00
|
|
|
|
2023-11-28 17:15:17 +00:00
|
|
|
aReq.New = database.HealthSettings{
|
|
|
|
ID: uuid.New(),
|
|
|
|
DismissedHealthchecks: settings.DismissedHealthchecks,
|
|
|
|
}
|
|
|
|
|
|
|
|
err = api.Database.UpsertHealthSettings(ctx, string(settingsJSON))
|
|
|
|
if err != nil {
|
|
|
|
httpapi.Write(r.Context(), rw, http.StatusInternalServerError, codersdk.Response{
|
|
|
|
Message: "Failed to update health settings.",
|
|
|
|
Detail: err.Error(),
|
|
|
|
})
|
|
|
|
return
|
|
|
|
}
|
|
|
|
|
|
|
|
httpapi.Write(r.Context(), rw, http.StatusOK, settings)
|
|
|
|
}
|
|
|
|
|
2024-03-26 17:44:31 +00:00
|
|
|
func validateHealthSettings(settings healthsdk.HealthSettings) error {
|
2023-11-28 17:15:17 +00:00
|
|
|
for _, dismissed := range settings.DismissedHealthchecks {
|
2024-03-26 17:44:31 +00:00
|
|
|
ok := slices.Contains(healthsdk.HealthSections, dismissed)
|
2023-11-28 17:15:17 +00:00
|
|
|
if !ok {
|
|
|
|
return xerrors.Errorf("unknown healthcheck section: %s", dismissed)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return nil
|
|
|
|
}
|
|
|
|
|
2023-05-30 19:22:32 +00:00
|
|
|
// For some reason the swagger docs need to be attached to a function.
|
feat: expose DERP server debug metrics (#12135)
Adds some debug endpoints for looking into the DERP server.
The `api/v2/debug/derp/traffic` endpoint requires the `ss` utility to be present in order to function. I have *not* added the `iproute2` package to our base image as it adds 11MB, so this endpoint won't be useful by default. However, in a debugging situation, we could exec into the container and then `apk add iproute2`, or build a special debug image.
The `api/v2/debug/expvar` handler contains DERP metrics as well as commandline and memstats.
Example:
```
{
"alert_failed": 0,
"alert_generated": 0,
"cmdline": ["/Users/spike/repos/coder/build/coder_darwin_arm64","--global-config","/Users/spike/repos/coder/.coderv2","server","--http-address","0.0.0.0:3000","--swagger-enable","--access-url","http://127.0.0.1:3000","--dangerous-allow-cors-requests=true"],
"derp": {"accepts": 1, "average_queue_duration_ms": 0, "bytes_received": 0, "bytes_sent": 0, "counter_packets_dropped_reason": {"gone_disconnected": 0, "gone_not_here": 0, "queue_head": 0, "queue_tail": 0, "unknown_dest": 0, "unknown_dest_on_fwd": 0, "write_error": 0}, "counter_packets_dropped_type": {"disco": 0, "other": 0}, "counter_packets_received_kind": {"disco": 0, "other": 0}, "counter_tcp_rtt": {}, "counter_total_dup_client_conns": 0, "gauge_clients_local": 1, "gauge_clients_remote": 0, "gauge_clients_total": 1, "gauge_current_connections": 1, "gauge_current_dup_client_conns": 0, "gauge_current_dup_client_keys": 0, "gauge_current_file_descriptors": 0, "gauge_current_home_connections": 1, "gauge_memstats_sys0": 20874504, "gauge_watchers": 0, "got_ping": 0, "home_moves_in": 0, "home_moves_out": 0, "multiforwarder_created": 0, "multiforwarder_deleted": 0, "packet_forwarder_delete_other_value": 0, "packets_dropped": 0, "packets_forwarded_in": 0, "packets_forwarded_out": 0, "packets_received": 0, "packets_sent": 0, "peer_gone_disconnected_frames": 0, "peer_gone_not_here_frames": 0, "sent_pong": 0, "unknown_frames": 0, "version": "1.47.0-dev20240214-t64db8c604"},
"memstats": {"Alloc":286506256,"TotalAlloc":297594632,"Sys":310621512,"Lookups":0,"Mallocs":304204,"Frees":171570,"HeapAlloc":286506256,"HeapSys":294060032,"HeapIdle":3694592,"HeapInuse":290365440,"HeapReleased":3620864,"HeapObjects":132634,"StackInuse":3735552,"StackSys":3735552,"MSpanInuse":347256,"MSpanSys":358512,"MCacheInuse":9600,"MCacheSys":15600,"BuckHashSys":1469877,"GCSys":9434896,"OtherSys":1547043,"NextGC":551867656,"LastGC":1707892877408883000,"PauseTotalNs":1247000,"PauseNs":[200333,229375,239875,209542,106958,203792,57125,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"PauseEnd":[1707892876217481000,1707892876219726000,1707892876222273000,1707892876226151000,1707892876234815000,1707892877398146000,1707892877408883000,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"NumGC":7,"NumForcedGC":0,"GCCPUFraction":0.0022425810335762954,"EnableGC":true,"DebugGC":false,"BySize":[{"Size":0,"Mallocs":0,"Frees":0},{"Size":8,"Mallocs":14396,"Frees":9143},{"Size":16,"Mallocs":89090,"Frees":50507},{"Size":24,"Mallocs":40839,"Frees":24456},{"Size":32,"Mallocs":22404,"Frees":12379},{"Size":48,"Mallocs":51174,"Frees":23718},{"Size":64,"Mallocs":15406,"Frees":3501},{"Size":80,"Mallocs":6688,"Frees":2352},{"Size":96,"Mallocs":2567,"Frees":374},{"Size":112,"Mallocs":19371,"Frees":16883},{"Size":128,"Mallocs":2873,"Frees":1061},{"Size":144,"Mallocs":5600,"Frees":2742},{"Size":160,"Mallocs":2159,"Frees":622},{"Size":176,"Mallocs":454,"Frees":86},{"Size":192,"Mallocs":227,"Frees":128},{"Size":208,"Mallocs":1407,"Frees":732},{"Size":224,"Mallocs":1365,"Frees":1090},{"Size":240,"Mallocs":82,"Frees":48},{"Size":256,"Mallocs":310,"Frees":162},{"Size":288,"Mallocs":1945,"Frees":562},{"Size":320,"Mallocs":1200,"Frees":458},{"Size":352,"Mallocs":133,"Frees":33},{"Size":384,"Mallocs":582,"Frees":51},{"Size":416,"Mallocs":747,"Frees":200},{"Size":448,"Mallocs":113,"Frees":22},{"Size":480,"Mallocs":34,"Frees":21},{"Size":512,"Mallocs":951,"Frees":91},{"Size":576,"Mallocs":364,"Frees":122},{"Size":640,"Mallocs":532,"Frees":270},{"Size":704,"Mallocs":93,"Frees":39},{"Size":768,"Mallocs":83,"Frees":35},{"Size":896,"Mallocs":308,"Frees":175},{"Size":1024,"Mallocs":226,"Frees":122},{"Size":1152,"Mallocs":198,"Frees":100},{"Size":1280,"Mallocs":314,"Frees":171},{"Size":1408,"Mallocs":77,"Frees":47},{"Size":1536,"Mallocs":80,"Frees":54},{"Size":1792,"Mallocs":199,"Frees":107},{"Size":2048,"Mallocs":112,"Frees":48},{"Size":2304,"Mallocs":71,"Frees":32},{"Size":2688,"Mallocs":206,"Frees":81},{"Size":3072,"Mallocs":39,"Frees":15},{"Size":3200,"Mallocs":16,"Frees":7},{"Size":3456,"Mallocs":44,"Frees":29},{"Size":4096,"Mallocs":192,"Frees":83},{"Size":4864,"Mallocs":44,"Frees":25},{"Size":5376,"Mallocs":105,"Frees":43},{"Size":6144,"Mallocs":25,"Frees":5},{"Size":6528,"Mallocs":22,"Frees":7},{"Size":6784,"Mallocs":3,"Frees":0},{"Size":6912,"Mallocs":4,"Frees":2},{"Size":8192,"Mallocs":59,"Frees":10},{"Size":9472,"Mallocs":31,"Frees":12},{"Size":9728,"Mallocs":5,"Frees":2},{"Size":10240,"Mallocs":5,"Frees":0},{"Size":10880,"Mallocs":27,"Frees":11},{"Size":12288,"Mallocs":4,"Frees":1},{"Size":13568,"Mallocs":4,"Frees":2},{"Size":14336,"Mallocs":9,"Frees":2},{"Size":16384,"Mallocs":10,"Frees":2},{"Size":18432,"Mallocs":4,"Frees":2}]},
"warning_failed": 0,
"warning_generated": 0
}
```
If we find the DERP metrics useful we could consider how to include them in Prometheus scrapes based on the tailnet `varz` package. That's for a later PR if at all.
2024-02-14 11:11:45 +00:00
|
|
|
|
2023-05-30 19:22:32 +00:00
|
|
|
// @Summary Debug Info Websocket Test
|
|
|
|
// @ID debug-info-websocket-test
|
|
|
|
// @Security CoderSessionToken
|
|
|
|
// @Produce json
|
|
|
|
// @Tags Debug
|
|
|
|
// @Success 201 {object} codersdk.Response
|
|
|
|
// @Router /debug/ws [get]
|
|
|
|
// @x-apidocgen {"skip": true}
|
|
|
|
func _debugws(http.ResponseWriter, *http.Request) {} //nolint:unused
|
2023-11-29 16:37:40 +00:00
|
|
|
|
feat: expose DERP server debug metrics (#12135)
Adds some debug endpoints for looking into the DERP server.
The `api/v2/debug/derp/traffic` endpoint requires the `ss` utility to be present in order to function. I have *not* added the `iproute2` package to our base image as it adds 11MB, so this endpoint won't be useful by default. However, in a debugging situation, we could exec into the container and then `apk add iproute2`, or build a special debug image.
The `api/v2/debug/expvar` handler contains DERP metrics as well as commandline and memstats.
Example:
```
{
"alert_failed": 0,
"alert_generated": 0,
"cmdline": ["/Users/spike/repos/coder/build/coder_darwin_arm64","--global-config","/Users/spike/repos/coder/.coderv2","server","--http-address","0.0.0.0:3000","--swagger-enable","--access-url","http://127.0.0.1:3000","--dangerous-allow-cors-requests=true"],
"derp": {"accepts": 1, "average_queue_duration_ms": 0, "bytes_received": 0, "bytes_sent": 0, "counter_packets_dropped_reason": {"gone_disconnected": 0, "gone_not_here": 0, "queue_head": 0, "queue_tail": 0, "unknown_dest": 0, "unknown_dest_on_fwd": 0, "write_error": 0}, "counter_packets_dropped_type": {"disco": 0, "other": 0}, "counter_packets_received_kind": {"disco": 0, "other": 0}, "counter_tcp_rtt": {}, "counter_total_dup_client_conns": 0, "gauge_clients_local": 1, "gauge_clients_remote": 0, "gauge_clients_total": 1, "gauge_current_connections": 1, "gauge_current_dup_client_conns": 0, "gauge_current_dup_client_keys": 0, "gauge_current_file_descriptors": 0, "gauge_current_home_connections": 1, "gauge_memstats_sys0": 20874504, "gauge_watchers": 0, "got_ping": 0, "home_moves_in": 0, "home_moves_out": 0, "multiforwarder_created": 0, "multiforwarder_deleted": 0, "packet_forwarder_delete_other_value": 0, "packets_dropped": 0, "packets_forwarded_in": 0, "packets_forwarded_out": 0, "packets_received": 0, "packets_sent": 0, "peer_gone_disconnected_frames": 0, "peer_gone_not_here_frames": 0, "sent_pong": 0, "unknown_frames": 0, "version": "1.47.0-dev20240214-t64db8c604"},
"memstats": {"Alloc":286506256,"TotalAlloc":297594632,"Sys":310621512,"Lookups":0,"Mallocs":304204,"Frees":171570,"HeapAlloc":286506256,"HeapSys":294060032,"HeapIdle":3694592,"HeapInuse":290365440,"HeapReleased":3620864,"HeapObjects":132634,"StackInuse":3735552,"StackSys":3735552,"MSpanInuse":347256,"MSpanSys":358512,"MCacheInuse":9600,"MCacheSys":15600,"BuckHashSys":1469877,"GCSys":9434896,"OtherSys":1547043,"NextGC":551867656,"LastGC":1707892877408883000,"PauseTotalNs":1247000,"PauseNs":[200333,229375,239875,209542,106958,203792,57125,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"PauseEnd":[1707892876217481000,1707892876219726000,1707892876222273000,1707892876226151000,1707892876234815000,1707892877398146000,1707892877408883000,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"NumGC":7,"NumForcedGC":0,"GCCPUFraction":0.0022425810335762954,"EnableGC":true,"DebugGC":false,"BySize":[{"Size":0,"Mallocs":0,"Frees":0},{"Size":8,"Mallocs":14396,"Frees":9143},{"Size":16,"Mallocs":89090,"Frees":50507},{"Size":24,"Mallocs":40839,"Frees":24456},{"Size":32,"Mallocs":22404,"Frees":12379},{"Size":48,"Mallocs":51174,"Frees":23718},{"Size":64,"Mallocs":15406,"Frees":3501},{"Size":80,"Mallocs":6688,"Frees":2352},{"Size":96,"Mallocs":2567,"Frees":374},{"Size":112,"Mallocs":19371,"Frees":16883},{"Size":128,"Mallocs":2873,"Frees":1061},{"Size":144,"Mallocs":5600,"Frees":2742},{"Size":160,"Mallocs":2159,"Frees":622},{"Size":176,"Mallocs":454,"Frees":86},{"Size":192,"Mallocs":227,"Frees":128},{"Size":208,"Mallocs":1407,"Frees":732},{"Size":224,"Mallocs":1365,"Frees":1090},{"Size":240,"Mallocs":82,"Frees":48},{"Size":256,"Mallocs":310,"Frees":162},{"Size":288,"Mallocs":1945,"Frees":562},{"Size":320,"Mallocs":1200,"Frees":458},{"Size":352,"Mallocs":133,"Frees":33},{"Size":384,"Mallocs":582,"Frees":51},{"Size":416,"Mallocs":747,"Frees":200},{"Size":448,"Mallocs":113,"Frees":22},{"Size":480,"Mallocs":34,"Frees":21},{"Size":512,"Mallocs":951,"Frees":91},{"Size":576,"Mallocs":364,"Frees":122},{"Size":640,"Mallocs":532,"Frees":270},{"Size":704,"Mallocs":93,"Frees":39},{"Size":768,"Mallocs":83,"Frees":35},{"Size":896,"Mallocs":308,"Frees":175},{"Size":1024,"Mallocs":226,"Frees":122},{"Size":1152,"Mallocs":198,"Frees":100},{"Size":1280,"Mallocs":314,"Frees":171},{"Size":1408,"Mallocs":77,"Frees":47},{"Size":1536,"Mallocs":80,"Frees":54},{"Size":1792,"Mallocs":199,"Frees":107},{"Size":2048,"Mallocs":112,"Frees":48},{"Size":2304,"Mallocs":71,"Frees":32},{"Size":2688,"Mallocs":206,"Frees":81},{"Size":3072,"Mallocs":39,"Frees":15},{"Size":3200,"Mallocs":16,"Frees":7},{"Size":3456,"Mallocs":44,"Frees":29},{"Size":4096,"Mallocs":192,"Frees":83},{"Size":4864,"Mallocs":44,"Frees":25},{"Size":5376,"Mallocs":105,"Frees":43},{"Size":6144,"Mallocs":25,"Frees":5},{"Size":6528,"Mallocs":22,"Frees":7},{"Size":6784,"Mallocs":3,"Frees":0},{"Size":6912,"Mallocs":4,"Frees":2},{"Size":8192,"Mallocs":59,"Frees":10},{"Size":9472,"Mallocs":31,"Frees":12},{"Size":9728,"Mallocs":5,"Frees":2},{"Size":10240,"Mallocs":5,"Frees":0},{"Size":10880,"Mallocs":27,"Frees":11},{"Size":12288,"Mallocs":4,"Frees":1},{"Size":13568,"Mallocs":4,"Frees":2},{"Size":14336,"Mallocs":9,"Frees":2},{"Size":16384,"Mallocs":10,"Frees":2},{"Size":18432,"Mallocs":4,"Frees":2}]},
"warning_failed": 0,
"warning_generated": 0
}
```
If we find the DERP metrics useful we could consider how to include them in Prometheus scrapes based on the tailnet `varz` package. That's for a later PR if at all.
2024-02-14 11:11:45 +00:00
|
|
|
// @Summary Debug DERP traffic
|
|
|
|
// @ID debug-derp-traffic
|
|
|
|
// @Security CoderSessionToken
|
|
|
|
// @Produce json
|
|
|
|
// @Success 200 {array} derp.BytesSentRecv
|
|
|
|
// @Tags Debug
|
|
|
|
// @Router /debug/derp/traffic [get]
|
|
|
|
// @x-apidocgen {"skip": true}
|
|
|
|
func _debugDERPTraffic(http.ResponseWriter, *http.Request) {} //nolint:unused
|
|
|
|
|
|
|
|
// @Summary Debug expvar
|
|
|
|
// @ID debug-expvar
|
|
|
|
// @Security CoderSessionToken
|
|
|
|
// @Produce json
|
|
|
|
// @Tags Debug
|
|
|
|
// @Success 200 {object} map[string]any
|
|
|
|
// @Router /debug/expvar [get]
|
|
|
|
// @x-apidocgen {"skip": true}
|
|
|
|
func _debugExpVar(http.ResponseWriter, *http.Request) {} //nolint:unused
|
|
|
|
|
2024-03-26 17:44:31 +00:00
|
|
|
func loadDismissedHealthchecks(ctx context.Context, db database.Store, logger slog.Logger) []healthsdk.HealthSection {
|
|
|
|
dismissedHealthchecks := []healthsdk.HealthSection{}
|
2023-11-29 16:37:40 +00:00
|
|
|
settingsJSON, err := db.GetHealthSettings(ctx)
|
|
|
|
if err == nil {
|
2024-03-26 17:44:31 +00:00
|
|
|
var settings healthsdk.HealthSettings
|
2023-11-29 16:37:40 +00:00
|
|
|
err = json.Unmarshal([]byte(settingsJSON), &settings)
|
|
|
|
if len(settings.DismissedHealthchecks) > 0 {
|
|
|
|
dismissedHealthchecks = settings.DismissedHealthchecks
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if err != nil && !xerrors.Is(err, sql.ErrNoRows) {
|
2023-12-06 08:38:03 +00:00
|
|
|
logger.Error(ctx, "unable to fetch health settings", slog.Error(err))
|
2023-11-29 16:37:40 +00:00
|
|
|
}
|
|
|
|
return dismissedHealthchecks
|
|
|
|
}
|