From 5a0d240bc3999c88b288aad41fe0bdd41dc1d8f7 Mon Sep 17 00:00:00 2001 From: Spike Curtis Date: Wed, 14 Feb 2024 15:11:45 +0400 Subject: [PATCH] feat: expose DERP server debug metrics (#12135) Adds some debug endpoints for looking into the DERP server. The `api/v2/debug/derp/traffic` endpoint requires the `ss` utility to be present in order to function. I have *not* added the `iproute2` package to our base image as it adds 11MB, so this endpoint won't be useful by default. However, in a debugging situation, we could exec into the container and then `apk add iproute2`, or build a special debug image. The `api/v2/debug/expvar` handler contains DERP metrics as well as commandline and memstats. Example: ``` { "alert_failed": 0, "alert_generated": 0, "cmdline": ["/Users/spike/repos/coder/build/coder_darwin_arm64","--global-config","/Users/spike/repos/coder/.coderv2","server","--http-address","0.0.0.0:3000","--swagger-enable","--access-url","http://127.0.0.1:3000","--dangerous-allow-cors-requests=true"], "derp": {"accepts": 1, "average_queue_duration_ms": 0, "bytes_received": 0, "bytes_sent": 0, "counter_packets_dropped_reason": {"gone_disconnected": 0, "gone_not_here": 0, "queue_head": 0, "queue_tail": 0, "unknown_dest": 0, "unknown_dest_on_fwd": 0, "write_error": 0}, "counter_packets_dropped_type": {"disco": 0, "other": 0}, "counter_packets_received_kind": {"disco": 0, "other": 0}, "counter_tcp_rtt": {}, "counter_total_dup_client_conns": 0, "gauge_clients_local": 1, "gauge_clients_remote": 0, "gauge_clients_total": 1, "gauge_current_connections": 1, "gauge_current_dup_client_conns": 0, "gauge_current_dup_client_keys": 0, "gauge_current_file_descriptors": 0, "gauge_current_home_connections": 1, "gauge_memstats_sys0": 20874504, "gauge_watchers": 0, "got_ping": 0, "home_moves_in": 0, "home_moves_out": 0, "multiforwarder_created": 0, "multiforwarder_deleted": 0, "packet_forwarder_delete_other_value": 0, "packets_dropped": 0, "packets_forwarded_in": 0, "packets_forwarded_out": 0, "packets_received": 0, "packets_sent": 0, "peer_gone_disconnected_frames": 0, "peer_gone_not_here_frames": 0, "sent_pong": 0, "unknown_frames": 0, "version": "1.47.0-dev20240214-t64db8c604"}, "memstats": {"Alloc":286506256,"TotalAlloc":297594632,"Sys":310621512,"Lookups":0,"Mallocs":304204,"Frees":171570,"HeapAlloc":286506256,"HeapSys":294060032,"HeapIdle":3694592,"HeapInuse":290365440,"HeapReleased":3620864,"HeapObjects":132634,"StackInuse":3735552,"StackSys":3735552,"MSpanInuse":347256,"MSpanSys":358512,"MCacheInuse":9600,"MCacheSys":15600,"BuckHashSys":1469877,"GCSys":9434896,"OtherSys":1547043,"NextGC":551867656,"LastGC":1707892877408883000,"PauseTotalNs":1247000,"PauseNs":[200333,229375,239875,209542,106958,203792,57125,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"PauseEnd":[1707892876217481000,1707892876219726000,1707892876222273000,1707892876226151000,1707892876234815000,1707892877398146000,1707892877408883000,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"NumGC":7,"NumForcedGC":0,"GCCPUFraction":0.0022425810335762954,"EnableGC":true,"DebugGC":false,"BySize":[{"Size":0,"Mallocs":0,"Frees":0},{"Size":8,"Mallocs":14396,"Frees":9143},{"Size":16,"Mallocs":89090,"Frees":50507},{"Size":24,"Mallocs":40839,"Frees":24456},{"Size":32,"Mallocs":22404,"Frees":12379},{"Size":48,"Mallocs":51174,"Frees":23718},{"Size":64,"Mallocs":15406,"Frees":3501},{"Size":80,"Mallocs":6688,"Frees":2352},{"Size":96,"Mallocs":2567,"Frees":374},{"Size":112,"Mallocs":19371,"Frees":16883},{"Size":128,"Mallocs":2873,"Frees":1061},{"Size":144,"Mallocs":5600,"Frees":2742},{"Size":160,"Mallocs":2159,"Frees":622},{"Size":176,"Mallocs":454,"Frees":86},{"Size":192,"Mallocs":227,"Frees":128},{"Size":208,"Mallocs":1407,"Frees":732},{"Size":224,"Mallocs":1365,"Frees":1090},{"Size":240,"Mallocs":82,"Frees":48},{"Size":256,"Mallocs":310,"Frees":162},{"Size":288,"Mallocs":1945,"Frees":562},{"Size":320,"Mallocs":1200,"Frees":458},{"Size":352,"Mallocs":133,"Frees":33},{"Size":384,"Mallocs":582,"Frees":51},{"Size":416,"Mallocs":747,"Frees":200},{"Size":448,"Mallocs":113,"Frees":22},{"Size":480,"Mallocs":34,"Frees":21},{"Size":512,"Mallocs":951,"Frees":91},{"Size":576,"Mallocs":364,"Frees":122},{"Size":640,"Mallocs":532,"Frees":270},{"Size":704,"Mallocs":93,"Frees":39},{"Size":768,"Mallocs":83,"Frees":35},{"Size":896,"Mallocs":308,"Frees":175},{"Size":1024,"Mallocs":226,"Frees":122},{"Size":1152,"Mallocs":198,"Frees":100},{"Size":1280,"Mallocs":314,"Frees":171},{"Size":1408,"Mallocs":77,"Frees":47},{"Size":1536,"Mallocs":80,"Frees":54},{"Size":1792,"Mallocs":199,"Frees":107},{"Size":2048,"Mallocs":112,"Frees":48},{"Size":2304,"Mallocs":71,"Frees":32},{"Size":2688,"Mallocs":206,"Frees":81},{"Size":3072,"Mallocs":39,"Frees":15},{"Size":3200,"Mallocs":16,"Frees":7},{"Size":3456,"Mallocs":44,"Frees":29},{"Size":4096,"Mallocs":192,"Frees":83},{"Size":4864,"Mallocs":44,"Frees":25},{"Size":5376,"Mallocs":105,"Frees":43},{"Size":6144,"Mallocs":25,"Frees":5},{"Size":6528,"Mallocs":22,"Frees":7},{"Size":6784,"Mallocs":3,"Frees":0},{"Size":6912,"Mallocs":4,"Frees":2},{"Size":8192,"Mallocs":59,"Frees":10},{"Size":9472,"Mallocs":31,"Frees":12},{"Size":9728,"Mallocs":5,"Frees":2},{"Size":10240,"Mallocs":5,"Frees":0},{"Size":10880,"Mallocs":27,"Frees":11},{"Size":12288,"Mallocs":4,"Frees":1},{"Size":13568,"Mallocs":4,"Frees":2},{"Size":14336,"Mallocs":9,"Frees":2},{"Size":16384,"Mallocs":10,"Frees":2},{"Size":18432,"Mallocs":4,"Frees":2}]}, "warning_failed": 0, "warning_generated": 0 } ``` If we find the DERP metrics useful we could consider how to include them in Prometheus scrapes based on the tailnet `varz` package. That's for a later PR if at all. --- coderd/apidoc/docs.go | 82 ++++++++++++++++++++++++++++++++++++++ coderd/apidoc/swagger.json | 74 ++++++++++++++++++++++++++++++++++ coderd/coderd.go | 17 ++++++++ coderd/debug.go | 22 +++++++++- docs/api/schemas.md | 28 +++++++++++++ 5 files changed, 222 insertions(+), 1 deletion(-) diff --git a/coderd/apidoc/docs.go b/coderd/apidoc/docs.go index a4e792067c..f49b91832f 100644 --- a/coderd/apidoc/docs.go +++ b/coderd/apidoc/docs.go @@ -398,6 +398,66 @@ const docTemplate = `{ } } }, + "/debug/derp/traffic": { + "get": { + "security": [ + { + "CoderSessionToken": [] + } + ], + "produces": [ + "application/json" + ], + "tags": [ + "Debug" + ], + "summary": "Debug DERP traffic", + "operationId": "debug-derp-traffic", + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/derp.BytesSentRecv" + } + } + } + }, + "x-apidocgen": { + "skip": true + } + } + }, + "/debug/expvar": { + "get": { + "security": [ + { + "CoderSessionToken": [] + } + ], + "produces": [ + "application/json" + ], + "tags": [ + "Debug" + ], + "summary": "Debug expvar", + "operationId": "debug-expvar", + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "object", + "additionalProperties": true + } + } + }, + "x-apidocgen": { + "skip": true + } + } + }, "/debug/health": { "get": { "security": [ @@ -13266,6 +13326,25 @@ const docTemplate = `{ } } }, + "derp.BytesSentRecv": { + "type": "object", + "properties": { + "key": { + "description": "Key is the public key of the client which sent/received these bytes.", + "allOf": [ + { + "$ref": "#/definitions/key.NodePublic" + } + ] + }, + "recv": { + "type": "integer" + }, + "sent": { + "type": "integer" + } + } + }, "derp.ServerInfoMessage": { "type": "object", "properties": { @@ -13771,6 +13850,9 @@ const docTemplate = `{ } } }, + "key.NodePublic": { + "type": "object" + }, "netcheck.Report": { "type": "object", "properties": { diff --git a/coderd/apidoc/swagger.json b/coderd/apidoc/swagger.json index 1fd4acd85d..1852fd9b5a 100644 --- a/coderd/apidoc/swagger.json +++ b/coderd/apidoc/swagger.json @@ -338,6 +338,58 @@ } } }, + "/debug/derp/traffic": { + "get": { + "security": [ + { + "CoderSessionToken": [] + } + ], + "produces": ["application/json"], + "tags": ["Debug"], + "summary": "Debug DERP traffic", + "operationId": "debug-derp-traffic", + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/derp.BytesSentRecv" + } + } + } + }, + "x-apidocgen": { + "skip": true + } + } + }, + "/debug/expvar": { + "get": { + "security": [ + { + "CoderSessionToken": [] + } + ], + "produces": ["application/json"], + "tags": ["Debug"], + "summary": "Debug expvar", + "operationId": "debug-expvar", + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "object", + "additionalProperties": true + } + } + }, + "x-apidocgen": { + "skip": true + } + } + }, "/debug/health": { "get": { "security": [ @@ -12079,6 +12131,25 @@ } } }, + "derp.BytesSentRecv": { + "type": "object", + "properties": { + "key": { + "description": "Key is the public key of the client which sent/received these bytes.", + "allOf": [ + { + "$ref": "#/definitions/key.NodePublic" + } + ] + }, + "recv": { + "type": "integer" + }, + "sent": { + "type": "integer" + } + } + }, "derp.ServerInfoMessage": { "type": "object", "properties": { @@ -12548,6 +12619,9 @@ } } }, + "key.NodePublic": { + "type": "object" + }, "netcheck.Report": { "type": "object", "properties": { diff --git a/coderd/coderd.go b/coderd/coderd.go index c76b9d92a1..5a8c8d828d 100644 --- a/coderd/coderd.go +++ b/coderd/coderd.go @@ -5,6 +5,7 @@ import ( "crypto/tls" "crypto/x509" "database/sql" + "expvar" "flag" "fmt" "io" @@ -85,6 +86,8 @@ func init() { globalHTTPSwaggerHandler = httpSwagger.Handler(httpSwagger.URL("/swagger/doc.json")) } +var expDERPOnce = sync.Once{} + // Options are requires parameters for Coder to start. type Options struct { AccessURL *url.URL @@ -561,6 +564,16 @@ func New(options *Options) *API { derpHandler := derphttp.Handler(api.DERPServer) derpHandler, api.derpCloseFunc = tailnet.WithWebsocketSupport(api.DERPServer, derpHandler) + // Register DERP on expvar HTTP handler, which we serve below in the router, c.f. expvar.Handler() + // These are the metrics the DERP server exposes. + // TODO: export via prometheus + expDERPOnce.Do(func() { + // We need to do this via a global Once because expvar registry is global and panics if we + // register multiple times. In production there is only one Coderd and one DERP server per + // process, but in testing, we create multiple of both, so the Once protects us from + // panicking. + expvar.Publish("derp", api.DERPServer.ExpVar()) + }) cors := httpmw.Cors(options.DeploymentValues.Dangerous.AllowAllCors.Value()) prometheusMW := httpmw.Prometheus(options.PrometheusRegistry) @@ -1038,6 +1051,10 @@ func New(options *Options) *API { r.Use(httpmw.ExtractUserParam(options.Database)) r.Get("/debug-link", api.userDebugOIDC) }) + r.Route("/derp", func(r chi.Router) { + r.Get("/traffic", options.DERPServer.ServeDebugTraffic) + }) + r.Method("GET", "/expvar", expvar.Handler()) // contains DERP metrics as well as cmdline and memstats }) }) diff --git a/coderd/debug.go b/coderd/debug.go index 0e9bb6acbb..40e3b10b82 100644 --- a/coderd/debug.go +++ b/coderd/debug.go @@ -276,7 +276,7 @@ func validateHealthSettings(settings codersdk.HealthSettings) error { } // For some reason the swagger docs need to be attached to a function. -// + // @Summary Debug Info Websocket Test // @ID debug-info-websocket-test // @Security CoderSessionToken @@ -287,6 +287,26 @@ func validateHealthSettings(settings codersdk.HealthSettings) error { // @x-apidocgen {"skip": true} func _debugws(http.ResponseWriter, *http.Request) {} //nolint:unused +// @Summary Debug DERP traffic +// @ID debug-derp-traffic +// @Security CoderSessionToken +// @Produce json +// @Success 200 {array} derp.BytesSentRecv +// @Tags Debug +// @Router /debug/derp/traffic [get] +// @x-apidocgen {"skip": true} +func _debugDERPTraffic(http.ResponseWriter, *http.Request) {} //nolint:unused + +// @Summary Debug expvar +// @ID debug-expvar +// @Security CoderSessionToken +// @Produce json +// @Tags Debug +// @Success 200 {object} map[string]any +// @Router /debug/expvar [get] +// @x-apidocgen {"skip": true} +func _debugExpVar(http.ResponseWriter, *http.Request) {} //nolint:unused + func loadDismissedHealthchecks(ctx context.Context, db database.Store, logger slog.Logger) []codersdk.HealthSection { dismissedHealthchecks := []codersdk.HealthSection{} settingsJSON, err := db.GetHealthSettings(ctx) diff --git a/docs/api/schemas.md b/docs/api/schemas.md index 364b943fc3..2ea43f292b 100644 --- a/docs/api/schemas.md +++ b/docs/api/schemas.md @@ -7517,6 +7517,24 @@ If the schedule is empty, the user will be updated to use the default schedule.| | `count` | integer | false | | | | `workspaces` | array of [codersdk.Workspace](#codersdkworkspace) | false | | | +## derp.BytesSentRecv + +```json +{ + "key": {}, + "recv": 0, + "sent": 0 +} +``` + +### Properties + +| Name | Type | Required | Restrictions | Description | +| ------ | -------------------------------- | -------- | ------------ | -------------------------------------------------------------------- | +| `key` | [key.NodePublic](#keynodepublic) | false | | Key is the public key of the client which sent/received these bytes. | +| `recv` | integer | false | | | +| `sent` | integer | false | | | + ## derp.ServerInfoMessage ```json @@ -8630,6 +8648,16 @@ If the schedule is empty, the user will be updated to use the default schedule.| | `warnings` | array of [health.Message](#healthmessage) | false | | | | `workspace_proxies` | [codersdk.RegionsResponse-codersdk_WorkspaceProxy](#codersdkregionsresponse-codersdk_workspaceproxy) | false | | | +## key.NodePublic + +```json +{} +``` + +### Properties + +_None_ + ## netcheck.Report ```json