mirror of https://github.com/coder/coder.git
feat: add pprof and prometheus metrics to `coder server` (#1266)
This commit is contained in:
Colin Adler
GitHub
parent
5dcaf940b6
commit
55ad97bbd7
|
|
@ -12,6 +12,7 @@ import (
|
|||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/pprof"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/signal"
|
||||
|
|
@ -23,6 +24,7 @@ import (
|
|||
"github.com/google/go-github/v43/github"
|
||||
"github.com/pion/turn/v2"
|
||||
"github.com/pion/webrtc/v3"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
"github.com/spf13/cobra"
|
||||
"golang.org/x/oauth2"
|
||||
xgithub "golang.org/x/oauth2/github"
|
||||
|
|
@ -55,6 +57,10 @@ func server() *cobra.Command {
|
|||
var (
|
||||
accessURL string
|
||||
address string
|
||||
promEnabled bool
|
||||
promAddress string
|
||||
pprofEnabled bool
|
||||
pprofAddress string
|
||||
cacheDir string
|
||||
dev bool
|
||||
devUserEmail string
|
||||
|
|
@ -245,6 +251,17 @@ func server() *cobra.Command {
|
|||
}
|
||||
}
|
||||
|
||||
// This prevents the pprof import from being accidentally deleted.
|
||||
var _ = pprof.Handler
|
||||
if pprofEnabled {
|
||||
//nolint:revive
|
||||
defer serveHandler(cmd.Context(), logger, nil, pprofAddress, "pprof")()
|
||||
}
|
||||
if promEnabled {
|
||||
//nolint:revive
|
||||
defer serveHandler(cmd.Context(), logger, promhttp.Handler(), promAddress, "prometheus")()
|
||||
}
|
||||
|
||||
errCh := make(chan error, 1)
|
||||
provisionerDaemons := make([]*provisionerd.Server, 0)
|
||||
for i := 0; uint8(i) < provisionerDaemonCount; i++ {
|
||||
|
|
@ -400,8 +417,12 @@ func server() *cobra.Command {
|
|||
},
|
||||
}
|
||||
|
||||
cliflag.StringVarP(root.Flags(), &accessURL, "access-url", "", "CODER_ACCESS_URL", "", "Specifies the external URL to access Coder")
|
||||
cliflag.StringVarP(root.Flags(), &address, "address", "a", "CODER_ADDRESS", "127.0.0.1:3000", "The address to serve the API and dashboard")
|
||||
cliflag.StringVarP(root.Flags(), &accessURL, "access-url", "", "CODER_ACCESS_URL", "", "Specifies the external URL to access Coder.")
|
||||
cliflag.StringVarP(root.Flags(), &address, "address", "a", "CODER_ADDRESS", "127.0.0.1:3000", "The address to serve the API and dashboard.")
|
||||
cliflag.BoolVarP(root.Flags(), &promEnabled, "prometheus-enable", "", "CODER_PROMETHEUS_ENABLE", false, "Enable serving prometheus metrics on the addressdefined by --prometheus-address.")
|
||||
cliflag.StringVarP(root.Flags(), &promAddress, "prometheus-address", "", "CODER_PROMETHEUS_ADDRESS", "127.0.0.1:2112", "The address to serve prometheus metrics.")
|
||||
cliflag.BoolVarP(root.Flags(), &promEnabled, "pprof-enable", "", "CODER_PPROF_ENABLE", false, "Enable serving pprof metrics on the address defined by --pprof-address.")
|
||||
cliflag.StringVarP(root.Flags(), &pprofAddress, "pprof-address", "", "CODER_PPROF_ADDRESS", "127.0.0.1:6060", "The address to serve pprof.")
|
||||
// systemd uses the CACHE_DIRECTORY environment variable!
|
||||
cliflag.StringVarP(root.Flags(), &cacheDir, "cache-dir", "", "CACHE_DIRECTORY", filepath.Join(os.TempDir(), "coder-cache"), "Specifies a directory to cache binaries for provision operations.")
|
||||
cliflag.BoolVarP(root.Flags(), &dev, "dev", "", "CODER_DEV_MODE", false, "Serve Coder in dev mode for tinkering")
|
||||
|
|
@ -661,6 +682,20 @@ func configureGithubOAuth2(accessURL *url.URL, clientID, clientSecret string, al
|
|||
}, nil
|
||||
}
|
||||
|
||||
func serveHandler(ctx context.Context, logger slog.Logger, handler http.Handler, addr, name string) (closeFunc func()) {
|
||||
logger.Debug(ctx, "http server listening", slog.F("addr", addr), slog.F("name", name))
|
||||
|
||||
srv := &http.Server{Addr: addr, Handler: handler}
|
||||
go func() {
|
||||
err := srv.ListenAndServe()
|
||||
if err != nil && !xerrors.Is(err, http.ErrServerClosed) {
|
||||
logger.Error(ctx, "http server listen", slog.F("name", name), slog.Error(err))
|
||||
}
|
||||
}()
|
||||
|
||||
return func() { _ = srv.Close() }
|
||||
}
|
||||
|
||||
type datadogLogger struct {
|
||||
logger slog.Logger
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import (
|
|||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/go-chi/chi/v5/middleware"
|
||||
"github.com/pion/webrtc/v3"
|
||||
"google.golang.org/api/idtoken"
|
||||
|
||||
|
|
@ -68,9 +69,18 @@ func New(options *Options) (http.Handler, func()) {
|
|||
})
|
||||
|
||||
r := chi.NewRouter()
|
||||
r.Use(
|
||||
func(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
next.ServeHTTP(middleware.NewWrapResponseWriter(w, r.ProtoMajor), r)
|
||||
})
|
||||
},
|
||||
httpmw.Prometheus,
|
||||
chitrace.Middleware(),
|
||||
)
|
||||
|
||||
r.Route("/api/v2", func(r chi.Router) {
|
||||
r.Use(
|
||||
chitrace.Middleware(),
|
||||
// Specific routes can specify smaller limits.
|
||||
httpmw.RateLimitPerMinute(options.APIRateLimit),
|
||||
debugLogRequest(api.Logger),
|
||||
|
|
|
|||
|
|
@ -0,0 +1,110 @@
|
|||
package httpmw
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
chimw "github.com/go-chi/chi/v5/middleware"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promauto"
|
||||
)
|
||||
|
||||
var (
|
||||
requestsProcessed = promauto.NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: "coderd",
|
||||
Subsystem: "api",
|
||||
Name: "requests_processed_total",
|
||||
Help: "The total number of processed API requests",
|
||||
}, []string{"code", "method", "path"})
|
||||
requestsConcurrent = promauto.NewGauge(prometheus.GaugeOpts{
|
||||
Namespace: "coderd",
|
||||
Subsystem: "api",
|
||||
Name: "concurrent_requests",
|
||||
Help: "The number of concurrent API requests",
|
||||
})
|
||||
websocketsConcurrent = promauto.NewGauge(prometheus.GaugeOpts{
|
||||
Namespace: "coderd",
|
||||
Subsystem: "api",
|
||||
Name: "concurrent_websockets",
|
||||
Help: "The total number of concurrent API websockets",
|
||||
})
|
||||
websocketsDist = promauto.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Namespace: "coderd",
|
||||
Subsystem: "api",
|
||||
Name: "websocket_durations_ms",
|
||||
Help: "Websocket duration distribution of requests in milliseconds",
|
||||
Buckets: []float64{
|
||||
durationToFloatMs(01 * time.Millisecond),
|
||||
durationToFloatMs(01 * time.Second),
|
||||
durationToFloatMs(01 * time.Minute),
|
||||
durationToFloatMs(01 * time.Hour),
|
||||
durationToFloatMs(15 * time.Hour),
|
||||
durationToFloatMs(30 * time.Hour),
|
||||
},
|
||||
}, []string{"path"})
|
||||
requestsDist = promauto.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Namespace: "coderd",
|
||||
Subsystem: "api",
|
||||
Name: "request_latencies_ms",
|
||||
Help: "Latency distribution of requests in milliseconds",
|
||||
Buckets: []float64{1, 5, 10, 25, 50, 100, 500, 1000, 5000, 10000, 30000},
|
||||
}, []string{"method", "path"})
|
||||
)
|
||||
|
||||
func durationToFloatMs(d time.Duration) float64 {
|
||||
return float64(d.Milliseconds())
|
||||
}
|
||||
|
||||
func Prometheus(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
var (
|
||||
start = time.Now()
|
||||
method = r.Method
|
||||
rctx = chi.RouteContext(r.Context())
|
||||
)
|
||||
sw, ok := w.(chimw.WrapResponseWriter)
|
||||
if !ok {
|
||||
panic("dev error: http.ResponseWriter is not chimw.WrapResponseWriter")
|
||||
}
|
||||
|
||||
var (
|
||||
dist *prometheus.HistogramVec
|
||||
distOpts []string
|
||||
)
|
||||
// We want to count websockets separately.
|
||||
if isWebsocketUpgrade(r) {
|
||||
websocketsConcurrent.Inc()
|
||||
defer websocketsConcurrent.Dec()
|
||||
|
||||
dist = websocketsDist
|
||||
} else {
|
||||
requestsConcurrent.Inc()
|
||||
defer requestsConcurrent.Dec()
|
||||
|
||||
dist = requestsDist
|
||||
distOpts = []string{method}
|
||||
}
|
||||
|
||||
next.ServeHTTP(w, r)
|
||||
|
||||
path := rctx.RoutePattern()
|
||||
distOpts = append(distOpts, path)
|
||||
statusStr := strconv.Itoa(sw.Status())
|
||||
|
||||
requestsProcessed.WithLabelValues(statusStr, method, path).Inc()
|
||||
dist.WithLabelValues(distOpts...).Observe(float64(time.Since(start)) / 1e6)
|
||||
})
|
||||
}
|
||||
|
||||
func isWebsocketUpgrade(r *http.Request) bool {
|
||||
vs := r.Header.Values("Upgrade")
|
||||
for _, v := range vs {
|
||||
if v == "websocket" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
6
go.mod
6
go.mod
|
|
@ -90,6 +90,7 @@ require (
|
|||
github.com/pion/webrtc/v3 v3.1.34
|
||||
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8
|
||||
github.com/pkg/sftp v1.13.4
|
||||
github.com/prometheus/client_golang v1.12.1
|
||||
github.com/quasilyte/go-ruleguard/dsl v0.3.19
|
||||
github.com/robfig/cron/v3 v3.0.1
|
||||
github.com/spf13/cobra v1.4.0
|
||||
|
|
@ -135,6 +136,7 @@ require (
|
|||
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect
|
||||
github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect
|
||||
github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 // indirect
|
||||
github.com/beorn7/perks v1.0.1 // indirect
|
||||
github.com/cenkalti/backoff/v4 v4.1.2 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.1.2 // indirect
|
||||
github.com/charmbracelet/bubbles v0.10.3 // indirect
|
||||
|
|
@ -184,6 +186,7 @@ require (
|
|||
github.com/mailru/easyjson v0.7.7 // indirect
|
||||
github.com/mattn/go-colorable v0.1.12 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.13 // indirect
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
|
||||
github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect
|
||||
github.com/miekg/dns v1.1.45 // indirect
|
||||
github.com/mitchellh/go-wordwrap v1.0.1 // indirect
|
||||
|
|
@ -213,6 +216,9 @@ require (
|
|||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.0 // indirect
|
||||
github.com/pquerna/cachecontrol v0.1.0 // indirect
|
||||
github.com/prometheus/client_model v0.2.0 // indirect
|
||||
github.com/prometheus/common v0.32.1 // indirect
|
||||
github.com/prometheus/procfs v0.7.3 // indirect
|
||||
github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect
|
||||
github.com/rivo/uniseg v0.2.0 // indirect
|
||||
github.com/sirupsen/logrus v1.8.1 // indirect
|
||||
|
|
|
|||
|
|
@ -91,6 +91,7 @@ export interface CreateWorkspaceBuildRequest {
|
|||
// This is likely an enum in an external package ("github.com/coder/coder/coderd/database.WorkspaceTransition")
|
||||
readonly transition: string
|
||||
readonly dry_run: boolean
|
||||
readonly state: string
|
||||
}
|
||||
|
||||
// From codersdk/organizations.go:52:6
|
||||
|
|
@ -265,12 +266,12 @@ export interface UpdateUserProfileRequest {
|
|||
readonly username: string
|
||||
}
|
||||
|
||||
// From codersdk/workspaces.go:94:6
|
||||
// From codersdk/workspaces.go:95:6
|
||||
export interface UpdateWorkspaceAutostartRequest {
|
||||
readonly schedule: string
|
||||
}
|
||||
|
||||
// From codersdk/workspaces.go:114:6
|
||||
// From codersdk/workspaces.go:115:6
|
||||
export interface UpdateWorkspaceAutostopRequest {
|
||||
readonly schedule: string
|
||||
}
|
||||
|
|
@ -366,7 +367,7 @@ export interface WorkspaceAgentResourceMetadata {
|
|||
readonly cpu_mhz: number
|
||||
}
|
||||
|
||||
// From codersdk/workspacebuilds.go:17:6
|
||||
// From codersdk/workspacebuilds.go:18:6
|
||||
export interface WorkspaceBuild {
|
||||
readonly id: string
|
||||
readonly created_at: string
|
||||
|
|
|
|||
Loading…
Reference in New Issue