feat: expose DERP server debug metrics (#12135)

Adds some debug endpoints for looking into the DERP server. The `api/v2/debug/derp/traffic` endpoint requires the `ss` utility to be present in order to function. I have *not* added the `iproute2` package to our base image as it adds 11MB, so this endpoint won't be useful by default. However, in a debugging situation, we could exec into the container and then `apk add iproute2`, or build a special debug image. The `api/v2/debug/expvar` handler contains DERP metrics as well as commandline and memstats. Example: ``` { "alert_failed": 0, "alert_generated": 0, "cmdline": ["/Users/spike/repos/coder/build/coder_darwin_arm64","--global-config","/Users/spike/repos/coder/.coderv2","server","--http-address","0.0.0.0:3000","--swagger-enable","--access-url","http://127.0.0.1:3000","--dangerous-allow-cors-requests=true"], "derp": {"accepts": 1, "average_queue_duration_ms": 0, "bytes_received": 0, "bytes_sent": 0, "counter_packets_dropped_reason": {"gone_disconnected": 0, "gone_not_here": 0, "queue_head": 0, "queue_tail": 0, "unknown_dest": 0, "unknown_dest_on_fwd": 0, "write_error": 0}, "counter_packets_dropped_type": {"disco": 0, "other": 0}, "counter_packets_received_kind": {"disco": 0, "other": 0}, "counter_tcp_rtt": {}, "counter_total_dup_client_conns": 0, "gauge_clients_local": 1, "gauge_clients_remote": 0, "gauge_clients_total": 1, "gauge_current_connections": 1, "gauge_current_dup_client_conns": 0, "gauge_current_dup_client_keys": 0, "gauge_current_file_descriptors": 0, "gauge_current_home_connections": 1, "gauge_memstats_sys0": 20874504, "gauge_watchers": 0, "got_ping": 0, "home_moves_in": 0, "home_moves_out": 0, "multiforwarder_created": 0, "multiforwarder_deleted": 0, "packet_forwarder_delete_other_value": 0, "packets_dropped": 0, "packets_forwarded_in": 0, "packets_forwarded_out": 0, "packets_received": 0, "packets_sent": 0, "peer_gone_disconnected_frames": 0, "peer_gone_not_here_frames": 0, "sent_pong": 0, "unknown_frames": 0, "version": "1.47.0-dev20240214-t64db8c604"}, "memstats": {"Alloc":286506256,"TotalAlloc":297594632,"Sys":310621512,"Lookups":0,"Mallocs":304204,"Frees":171570,"HeapAlloc":286506256,"HeapSys":294060032,"HeapIdle":3694592,"HeapInuse":290365440,"HeapReleased":3620864,"HeapObjects":132634,"StackInuse":3735552,"StackSys":3735552,"MSpanInuse":347256,"MSpanSys":358512,"MCacheInuse":9600,"MCacheSys":15600,"BuckHashSys":1469877,"GCSys":9434896,"OtherSys":1547043,"NextGC":551867656,"LastGC":1707892877408883000,"PauseTotalNs":1247000,"PauseNs":[200333,229375,239875,209542,106958,203792,57125,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"PauseEnd":[1707892876217481000,1707892876219726000,1707892876222273000,1707892876226151000,1707892876234815000,1707892877398146000,1707892877408883000,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"NumGC":7,"NumForcedGC":0,"GCCPUFraction":0.0022425810335762954,"EnableGC":true,"DebugGC":false,"BySize":[{"Size":0,"Mallocs":0,"Frees":0},{"Size":8,"Mallocs":14396,"Frees":9143},{"Size":16,"Mallocs":89090,"Frees":50507},{"Size":24,"Mallocs":40839,"Frees":24456},{"Size":32,"Mallocs":22404,"Frees":12379},{"Size":48,"Mallocs":51174,"Frees":23718},{"Size":64,"Mallocs":15406,"Frees":3501},{"Size":80,"Mallocs":6688,"Frees":2352},{"Size":96,"Mallocs":2567,"Frees":374},{"Size":112,"Mallocs":19371,"Frees":16883},{"Size":128,"Mallocs":2873,"Frees":1061},{"Size":144,"Mallocs":5600,"Frees":2742},{"Size":160,"Mallocs":2159,"Frees":622},{"Size":176,"Mallocs":454,"Frees":86},{"Size":192,"Mallocs":227,"Frees":128},{"Size":208,"Mallocs":1407,"Frees":732},{"Size":224,"Mallocs":1365,"Frees":1090},{"Size":240,"Mallocs":82,"Frees":48},{"Size":256,"Mallocs":310,"Frees":162},{"Size":288,"Mallocs":1945,"Frees":562},{"Size":320,"Mallocs":1200,"Frees":458},{"Size":352,"Mallocs":133,"Frees":33},{"Size":384,"Mallocs":582,"Frees":51},{"Size":416,"Mallocs":747,"Frees":200},{"Size":448,"Mallocs":113,"Frees":22},{"Size":480,"Mallocs":34,"Frees":21},{"Size":512,"Mallocs":951,"Frees":91},{"Size":576,"Mallocs":364,"Frees":122},{"Size":640,"Mallocs":532,"Frees":270},{"Size":704,"Mallocs":93,"Frees":39},{"Size":768,"Mallocs":83,"Frees":35},{"Size":896,"Mallocs":308,"Frees":175},{"Size":1024,"Mallocs":226,"Frees":122},{"Size":1152,"Mallocs":198,"Frees":100},{"Size":1280,"Mallocs":314,"Frees":171},{"Size":1408,"Mallocs":77,"Frees":47},{"Size":1536,"Mallocs":80,"Frees":54},{"Size":1792,"Mallocs":199,"Frees":107},{"Size":2048,"Mallocs":112,"Frees":48},{"Size":2304,"Mallocs":71,"Frees":32},{"Size":2688,"Mallocs":206,"Frees":81},{"Size":3072,"Mallocs":39,"Frees":15},{"Size":3200,"Mallocs":16,"Frees":7},{"Size":3456,"Mallocs":44,"Frees":29},{"Size":4096,"Mallocs":192,"Frees":83},{"Size":4864,"Mallocs":44,"Frees":25},{"Size":5376,"Mallocs":105,"Frees":43},{"Size":6144,"Mallocs":25,"Frees":5},{"Size":6528,"Mallocs":22,"Frees":7},{"Size":6784,"Mallocs":3,"Frees":0},{"Size":6912,"Mallocs":4,"Frees":2},{"Size":8192,"Mallocs":59,"Frees":10},{"Size":9472,"Mallocs":31,"Frees":12},{"Size":9728,"Mallocs":5,"Frees":2},{"Size":10240,"Mallocs":5,"Frees":0},{"Size":10880,"Mallocs":27,"Frees":11},{"Size":12288,"Mallocs":4,"Frees":1},{"Size":13568,"Mallocs":4,"Frees":2},{"Size":14336,"Mallocs":9,"Frees":2},{"Size":16384,"Mallocs":10,"Frees":2},{"Size":18432,"Mallocs":4,"Frees":2}]}, "warning_failed": 0, "warning_generated": 0 } ``` If we find the DERP metrics useful we could consider how to include them in Prometheus scrapes based on the tailnet `varz` package. That's for a later PR if at all.
2024-02-14 15:11:45 +04:00 · 2024-02-14 15:11:45 +04:00 · 5a0d240bc3
parent 53c55439be
commit 5a0d240bc3
5 changed files with 222 additions and 1 deletions
--- a/coderd/apidoc/docs.go
+++ b/coderd/apidoc/docs.go
@ -398,6 +398,66 @@ const docTemplate = `{
                }
            }
        },
+        "/debug/derp/traffic": {
+            "get": {
+                "security": [
+                    {
+                        "CoderSessionToken": []
+                    }
+                ],
+                "produces": [
+                    "application/json"
+                ],
+                "tags": [
+                    "Debug"
+                ],
+                "summary": "Debug DERP traffic",
+                "operationId": "debug-derp-traffic",
+                "responses": {
+                    "200": {
+                        "description": "OK",
+                        "schema": {
+                            "type": "array",
+                            "items": {
+                                "$ref": "#/definitions/derp.BytesSentRecv"
+                            }
+                        }
+                    }
+                },
+                "x-apidocgen": {
+                    "skip": true
+                }
+            }
+        },
+        "/debug/expvar": {
+            "get": {
+                "security": [
+                    {
+                        "CoderSessionToken": []
+                    }
+                ],
+                "produces": [
+                    "application/json"
+                ],
+                "tags": [
+                    "Debug"
+                ],
+                "summary": "Debug expvar",
+                "operationId": "debug-expvar",
+                "responses": {
+                    "200": {
+                        "description": "OK",
+                        "schema": {
+                            "type": "object",
+                            "additionalProperties": true
+                        }
+                    }
+                },
+                "x-apidocgen": {
+                    "skip": true
+                }
+            }
+        },
        "/debug/health": {
            "get": {
                "security": [
@ -13266,6 +13326,25 @@ const docTemplate = `{
                }
            }
        },
+        "derp.BytesSentRecv": {
+            "type": "object",
+            "properties": {
+                "key": {
+                    "description": "Key is the public key of the client which sent/received these bytes.",
+                    "allOf": [
+                        {
+                            "$ref": "#/definitions/key.NodePublic"
+                        }
+                    ]
+                },
+                "recv": {
+                    "type": "integer"
+                },
+                "sent": {
+                    "type": "integer"
+                }
+            }
+        },
        "derp.ServerInfoMessage": {
            "type": "object",
            "properties": {
@ -13771,6 +13850,9 @@ const docTemplate = `{
                }
            }
        },
+        "key.NodePublic": {
+            "type": "object"
+        },
        "netcheck.Report": {
            "type": "object",
            "properties": {
--- a/coderd/apidoc/swagger.json
+++ b/coderd/apidoc/swagger.json
@ -338,6 +338,58 @@
        }
      }
    },
+    "/debug/derp/traffic": {
+      "get": {
+        "security": [
+          {
+            "CoderSessionToken": []
+          }
+        ],
+        "produces": ["application/json"],
+        "tags": ["Debug"],
+        "summary": "Debug DERP traffic",
+        "operationId": "debug-derp-traffic",
+        "responses": {
+          "200": {
+            "description": "OK",
+            "schema": {
+              "type": "array",
+              "items": {
+                "$ref": "#/definitions/derp.BytesSentRecv"
+              }
+            }
+          }
+        },
+        "x-apidocgen": {
+          "skip": true
+        }
+      }
+    },
+    "/debug/expvar": {
+      "get": {
+        "security": [
+          {
+            "CoderSessionToken": []
+          }
+        ],
+        "produces": ["application/json"],
+        "tags": ["Debug"],
+        "summary": "Debug expvar",
+        "operationId": "debug-expvar",
+        "responses": {
+          "200": {
+            "description": "OK",
+            "schema": {
+              "type": "object",
+              "additionalProperties": true
+            }
+          }
+        },
+        "x-apidocgen": {
+          "skip": true
+        }
+      }
+    },
    "/debug/health": {
      "get": {
        "security": [
@ -12079,6 +12131,25 @@
        }
      }
    },
+    "derp.BytesSentRecv": {
+      "type": "object",
+      "properties": {
+        "key": {
+          "description": "Key is the public key of the client which sent/received these bytes.",
+          "allOf": [
+            {
+              "$ref": "#/definitions/key.NodePublic"
+            }
+          ]
+        },
+        "recv": {
+          "type": "integer"
+        },
+        "sent": {
+          "type": "integer"
+        }
+      }
+    },
    "derp.ServerInfoMessage": {
      "type": "object",
      "properties": {
@ -12548,6 +12619,9 @@
        }
      }
    },
+    "key.NodePublic": {
+      "type": "object"
+    },
    "netcheck.Report": {
      "type": "object",
      "properties": {
--- a/coderd/coderd.go
+++ b/coderd/coderd.go
@ -5,6 +5,7 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"database/sql"
+	"expvar"
 	"flag"
 	"fmt"
 	"io"
@ -85,6 +86,8 @@ func init() {
 	globalHTTPSwaggerHandler = httpSwagger.Handler(httpSwagger.URL("/swagger/doc.json"))
 }

+var expDERPOnce = sync.Once{}
+
 // Options are requires parameters for Coder to start.
 type Options struct {
 	AccessURL *url.URL
@ -561,6 +564,16 @@ func New(options *Options) *API {

 	derpHandler := derphttp.Handler(api.DERPServer)
 	derpHandler, api.derpCloseFunc = tailnet.WithWebsocketSupport(api.DERPServer, derpHandler)
+	// Register DERP on expvar HTTP handler, which we serve below in the router, c.f. expvar.Handler()
+	// These are the metrics the DERP server exposes.
+	// TODO: export via prometheus
+	expDERPOnce.Do(func() {
+		// We need to do this via a global Once because expvar registry is global and panics if we
+		// register multiple times.  In production there is only one Coderd and one DERP server per
+		// process, but in testing, we create multiple of both, so the Once protects us from
+		// panicking.
+		expvar.Publish("derp", api.DERPServer.ExpVar())
+	})
 	cors := httpmw.Cors(options.DeploymentValues.Dangerous.AllowAllCors.Value())
 	prometheusMW := httpmw.Prometheus(options.PrometheusRegistry)

@ -1038,6 +1051,10 @@ func New(options *Options) *API {
 				r.Use(httpmw.ExtractUserParam(options.Database))
 				r.Get("/debug-link", api.userDebugOIDC)
 			})
+			r.Route("/derp", func(r chi.Router) {
+				r.Get("/traffic", options.DERPServer.ServeDebugTraffic)
+			})
+			r.Method("GET", "/expvar", expvar.Handler()) // contains DERP metrics as well as cmdline and memstats
 		})
 	})

--- a/coderd/debug.go
+++ b/coderd/debug.go
@ -276,7 +276,7 @@ func validateHealthSettings(settings codersdk.HealthSettings) error {
 }

 // For some reason the swagger docs need to be attached to a function.
-//
+
 // @Summary Debug Info Websocket Test
 // @ID debug-info-websocket-test
 // @Security CoderSessionToken
@ -287,6 +287,26 @@ func validateHealthSettings(settings codersdk.HealthSettings) error {
 // @x-apidocgen {"skip": true}
 func _debugws(http.ResponseWriter, *http.Request) {} //nolint:unused

+// @Summary Debug DERP traffic
+// @ID debug-derp-traffic
+// @Security CoderSessionToken
+// @Produce json
+// @Success 200 {array} derp.BytesSentRecv
+// @Tags Debug
+// @Router /debug/derp/traffic [get]
+// @x-apidocgen {"skip": true}
+func _debugDERPTraffic(http.ResponseWriter, *http.Request) {} //nolint:unused
+
+// @Summary Debug expvar
+// @ID debug-expvar
+// @Security CoderSessionToken
+// @Produce json
+// @Tags Debug
+// @Success 200 {object} map[string]any
+// @Router /debug/expvar [get]
+// @x-apidocgen {"skip": true}
+func _debugExpVar(http.ResponseWriter, *http.Request) {} //nolint:unused
+
 func loadDismissedHealthchecks(ctx context.Context, db database.Store, logger slog.Logger) []codersdk.HealthSection {
 	dismissedHealthchecks := []codersdk.HealthSection{}
 	settingsJSON, err := db.GetHealthSettings(ctx)
--- a/docs/api/schemas.md
+++ b/docs/api/schemas.md
@ -7517,6 +7517,24 @@ If the schedule is empty, the user will be updated to use the default schedule.|
 | `count`      | integer                                           | false    |              |             |
 | `workspaces` | array of [codersdk.Workspace](#codersdkworkspace) | false    |              |             |

+## derp.BytesSentRecv
+
+```json
+{
+  "key": {},
+  "recv": 0,
+  "sent": 0
+}
+```
+
+### Properties
+
+| Name   | Type                             | Required | Restrictions | Description                                                          |
+| ------ | -------------------------------- | -------- | ------------ | -------------------------------------------------------------------- |
+| `key`  | [key.NodePublic](#keynodepublic) | false    |              | Key is the public key of the client which sent/received these bytes. |
+| `recv` | integer                          | false    |              |                                                                      |
+| `sent` | integer                          | false    |              |                                                                      |
+
 ## derp.ServerInfoMessage

 ```json
@ -8630,6 +8648,16 @@ If the schedule is empty, the user will be updated to use the default schedule.|
 | `warnings`          | array of [health.Message](#healthmessage)                                                            | false    |              |             |
 | `workspace_proxies` | [codersdk.RegionsResponse-codersdk_WorkspaceProxy](#codersdkregionsresponse-codersdk_workspaceproxy) | false    |              |             |

+## key.NodePublic
+
+```json
+{}
+```
+
+### Properties
+
+_None_
+
 ## netcheck.Report

 ```json