mirror of https://github.com/coder/coder.git
551 lines
16 KiB
Go
551 lines
16 KiB
Go
package wsproxy_test
|
|
|
|
import (
|
|
"fmt"
|
|
"net"
|
|
"testing"
|
|
|
|
"github.com/davecgh/go-spew/spew"
|
|
"github.com/google/uuid"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
"tailscale.com/derp/derphttp"
|
|
"tailscale.com/tailcfg"
|
|
"tailscale.com/types/key"
|
|
|
|
"cdr.dev/slog"
|
|
"cdr.dev/slog/sloggers/slogtest"
|
|
"github.com/coder/coder/v2/agent/agenttest"
|
|
"github.com/coder/coder/v2/cli/clibase"
|
|
"github.com/coder/coder/v2/coderd"
|
|
"github.com/coder/coder/v2/coderd/coderdtest"
|
|
"github.com/coder/coder/v2/coderd/healthcheck/derphealth"
|
|
"github.com/coder/coder/v2/coderd/httpmw"
|
|
"github.com/coder/coder/v2/coderd/workspaceapps/apptest"
|
|
"github.com/coder/coder/v2/codersdk"
|
|
"github.com/coder/coder/v2/enterprise/coderd/coderdenttest"
|
|
"github.com/coder/coder/v2/enterprise/coderd/license"
|
|
"github.com/coder/coder/v2/provisioner/echo"
|
|
"github.com/coder/coder/v2/testutil"
|
|
)
|
|
|
|
func TestDERPOnly(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
deploymentValues := coderdtest.DeploymentValues(t)
|
|
deploymentValues.Experiments = []string{
|
|
string(codersdk.ExperimentMoons),
|
|
"*",
|
|
}
|
|
|
|
client, closer, api, _ := coderdenttest.NewWithAPI(t, &coderdenttest.Options{
|
|
Options: &coderdtest.Options{
|
|
DeploymentValues: deploymentValues,
|
|
AppHostname: "*.primary.test.coder.com",
|
|
IncludeProvisionerDaemon: true,
|
|
RealIPConfig: &httpmw.RealIPConfig{
|
|
TrustedOrigins: []*net.IPNet{{
|
|
IP: net.ParseIP("127.0.0.1"),
|
|
Mask: net.CIDRMask(8, 32),
|
|
}},
|
|
TrustedHeaders: []string{
|
|
"CF-Connecting-IP",
|
|
},
|
|
},
|
|
},
|
|
LicenseOptions: &coderdenttest.LicenseOptions{
|
|
Features: license.Features{
|
|
codersdk.FeatureWorkspaceProxy: 1,
|
|
},
|
|
},
|
|
})
|
|
t.Cleanup(func() {
|
|
_ = closer.Close()
|
|
})
|
|
|
|
// Create an external proxy.
|
|
_ = coderdenttest.NewWorkspaceProxy(t, api, client, &coderdenttest.ProxyOptions{
|
|
Name: "best-proxy",
|
|
DerpOnly: true,
|
|
})
|
|
|
|
// Should not show up in the regions list.
|
|
ctx := testutil.Context(t, testutil.WaitLong)
|
|
regions, err := client.Regions(ctx)
|
|
require.NoError(t, err)
|
|
require.Len(t, regions, 1)
|
|
require.Equal(t, api.Options.AccessURL.String(), regions[0].PathAppURL)
|
|
}
|
|
|
|
func TestDERP(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
deploymentValues := coderdtest.DeploymentValues(t)
|
|
deploymentValues.Experiments = []string{
|
|
string(codersdk.ExperimentMoons),
|
|
"*",
|
|
}
|
|
|
|
client, closer, api, user := coderdenttest.NewWithAPI(t, &coderdenttest.Options{
|
|
Options: &coderdtest.Options{
|
|
DeploymentValues: deploymentValues,
|
|
AppHostname: "*.primary.test.coder.com",
|
|
IncludeProvisionerDaemon: true,
|
|
RealIPConfig: &httpmw.RealIPConfig{
|
|
TrustedOrigins: []*net.IPNet{{
|
|
IP: net.ParseIP("127.0.0.1"),
|
|
Mask: net.CIDRMask(8, 32),
|
|
}},
|
|
TrustedHeaders: []string{
|
|
"CF-Connecting-IP",
|
|
},
|
|
},
|
|
},
|
|
LicenseOptions: &coderdenttest.LicenseOptions{
|
|
Features: license.Features{
|
|
codersdk.FeatureWorkspaceProxy: 1,
|
|
},
|
|
},
|
|
})
|
|
t.Cleanup(func() {
|
|
_ = closer.Close()
|
|
})
|
|
|
|
// Create two running external proxies.
|
|
proxyAPI1 := coderdenttest.NewWorkspaceProxy(t, api, client, &coderdenttest.ProxyOptions{
|
|
Name: "best-proxy",
|
|
})
|
|
proxyAPI2 := coderdenttest.NewWorkspaceProxy(t, api, client, &coderdenttest.ProxyOptions{
|
|
Name: "worst-proxy",
|
|
})
|
|
|
|
// Create a running external proxy with DERP disabled.
|
|
proxyAPI3 := coderdenttest.NewWorkspaceProxy(t, api, client, &coderdenttest.ProxyOptions{
|
|
Name: "no-derp-proxy",
|
|
DerpDisabled: true,
|
|
})
|
|
|
|
// Create a proxy that is never started.
|
|
createProxyCtx := testutil.Context(t, testutil.WaitLong)
|
|
_, err := client.CreateWorkspaceProxy(createProxyCtx, codersdk.CreateWorkspaceProxyRequest{
|
|
Name: "never-started-proxy",
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
// Wait for both running proxies to become healthy.
|
|
require.Eventually(t, func() bool {
|
|
healthCtx := testutil.Context(t, testutil.WaitLong)
|
|
err := api.ProxyHealth.ForceUpdate(healthCtx)
|
|
if !assert.NoError(t, err) {
|
|
return false
|
|
}
|
|
|
|
regions, err := client.Regions(healthCtx)
|
|
if !assert.NoError(t, err) {
|
|
return false
|
|
}
|
|
if !assert.Len(t, regions, 5) {
|
|
return false
|
|
}
|
|
|
|
// The first 3 regions should be healthy.
|
|
for _, r := range regions[:4] {
|
|
if !r.Healthy {
|
|
return false
|
|
}
|
|
}
|
|
|
|
// The last region should never be healthy.
|
|
assert.False(t, regions[4].Healthy)
|
|
return true
|
|
}, testutil.WaitLong, testutil.IntervalMedium)
|
|
|
|
// Create a workspace + apps
|
|
authToken := uuid.NewString()
|
|
version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{
|
|
Parse: echo.ParseComplete,
|
|
ProvisionApply: echo.ProvisionApplyWithAgent(authToken),
|
|
})
|
|
template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID)
|
|
coderdtest.AwaitTemplateVersionJobCompleted(t, client, version.ID)
|
|
workspace := coderdtest.CreateWorkspace(t, client, user.OrganizationID, template.ID)
|
|
build := coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, workspace.LatestBuild.ID)
|
|
workspace.LatestBuild = build
|
|
|
|
agentID := uuid.Nil
|
|
resourceLoop:
|
|
for _, res := range build.Resources {
|
|
for _, agnt := range res.Agents {
|
|
agentID = agnt.ID
|
|
break resourceLoop
|
|
}
|
|
}
|
|
require.NotEqual(t, uuid.Nil, agentID)
|
|
|
|
// Connect an agent to the workspace
|
|
_ = agenttest.New(t, client.URL, authToken)
|
|
_ = coderdtest.AwaitWorkspaceAgents(t, client, workspace.ID)
|
|
|
|
t.Run("ReturnedInDERPMap", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
ctx := testutil.Context(t, testutil.WaitLong)
|
|
connInfo, err := client.WorkspaceAgentConnectionInfo(ctx, agentID)
|
|
require.NoError(t, err)
|
|
|
|
// There should be three DERP regions in the map: the primary, and each
|
|
// of the two running proxies. Also the STUN-only regions.
|
|
require.NotNil(t, connInfo.DERPMap)
|
|
require.Len(t, connInfo.DERPMap.Regions, 3+len(api.DeploymentValues.DERP.Server.STUNAddresses.Value()))
|
|
|
|
var (
|
|
primaryRegion *tailcfg.DERPRegion
|
|
proxy1Region *tailcfg.DERPRegion
|
|
proxy2Region *tailcfg.DERPRegion
|
|
)
|
|
for _, r := range connInfo.DERPMap.Regions {
|
|
if r.EmbeddedRelay {
|
|
primaryRegion = r
|
|
continue
|
|
}
|
|
if r.RegionName == "best-proxy" {
|
|
proxy1Region = r
|
|
continue
|
|
}
|
|
if r.RegionName == "worst-proxy" {
|
|
proxy2Region = r
|
|
continue
|
|
}
|
|
// The no-derp-proxy shouldn't show up in the map.
|
|
// The last region is never started, which means it's never healthy,
|
|
// which means it's never added to the DERP map.
|
|
|
|
if len(r.Nodes) == 1 && r.Nodes[0].STUNOnly {
|
|
// Skip STUN-only regions.
|
|
continue
|
|
}
|
|
|
|
t.Fatalf("unexpected region: %+v", r)
|
|
}
|
|
|
|
// The primary region:
|
|
require.Equal(t, "Coder Embedded Relay", primaryRegion.RegionName)
|
|
require.Equal(t, "coder", primaryRegion.RegionCode)
|
|
require.Equal(t, 999, primaryRegion.RegionID)
|
|
require.True(t, primaryRegion.EmbeddedRelay)
|
|
|
|
// The first proxy region:
|
|
require.Equal(t, "best-proxy", proxy1Region.RegionName)
|
|
require.Equal(t, "coder_best-proxy", proxy1Region.RegionCode)
|
|
require.Equal(t, 10001, proxy1Region.RegionID)
|
|
require.False(t, proxy1Region.EmbeddedRelay)
|
|
require.Len(t, proxy1Region.Nodes, 1)
|
|
require.Equal(t, "10001a", proxy1Region.Nodes[0].Name)
|
|
require.Equal(t, 10001, proxy1Region.Nodes[0].RegionID)
|
|
require.Equal(t, proxyAPI1.Options.AccessURL.Hostname(), proxy1Region.Nodes[0].HostName)
|
|
require.Equal(t, proxyAPI1.Options.AccessURL.Port(), fmt.Sprint(proxy1Region.Nodes[0].DERPPort))
|
|
require.Equal(t, proxyAPI1.Options.AccessURL.Scheme == "http", proxy1Region.Nodes[0].ForceHTTP)
|
|
|
|
// The second proxy region:
|
|
require.Equal(t, "worst-proxy", proxy2Region.RegionName)
|
|
require.Equal(t, "coder_worst-proxy", proxy2Region.RegionCode)
|
|
require.Equal(t, 10002, proxy2Region.RegionID)
|
|
require.False(t, proxy2Region.EmbeddedRelay)
|
|
require.Len(t, proxy2Region.Nodes, 1)
|
|
require.Equal(t, "10002a", proxy2Region.Nodes[0].Name)
|
|
require.Equal(t, 10002, proxy2Region.Nodes[0].RegionID)
|
|
require.Equal(t, proxyAPI2.Options.AccessURL.Hostname(), proxy2Region.Nodes[0].HostName)
|
|
require.Equal(t, proxyAPI2.Options.AccessURL.Port(), fmt.Sprint(proxy2Region.Nodes[0].DERPPort))
|
|
require.Equal(t, proxyAPI2.Options.AccessURL.Scheme == "http", proxy2Region.Nodes[0].ForceHTTP)
|
|
})
|
|
|
|
t.Run("ConnectDERP", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
connInfo, err := client.WorkspaceAgentConnectionInfo(testutil.Context(t, testutil.WaitLong), agentID)
|
|
require.NoError(t, err)
|
|
require.NotNil(t, connInfo.DERPMap)
|
|
require.Len(t, connInfo.DERPMap.Regions, 3+len(api.DeploymentValues.DERP.Server.STUNAddresses.Value()))
|
|
|
|
// Connect to each region.
|
|
for _, r := range connInfo.DERPMap.Regions {
|
|
r := r
|
|
if len(r.Nodes) == 1 && r.Nodes[0].STUNOnly {
|
|
// Skip STUN-only regions.
|
|
continue
|
|
}
|
|
|
|
t.Run(r.RegionName, func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
derpMap := &tailcfg.DERPMap{
|
|
Regions: map[int]*tailcfg.DERPRegion{
|
|
r.RegionID: r,
|
|
},
|
|
OmitDefaultRegions: true,
|
|
}
|
|
|
|
ctx := testutil.Context(t, testutil.WaitLong)
|
|
report := derphealth.Report{}
|
|
report.Run(ctx, &derphealth.ReportOptions{
|
|
DERPMap: derpMap,
|
|
})
|
|
|
|
t.Log("healthcheck report: " + spew.Sdump(&report))
|
|
require.True(t, report.Healthy, "healthcheck failed, see report dump")
|
|
})
|
|
}
|
|
})
|
|
|
|
t.Run("DERPDisabled", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
// Try to connect to the DERP server on the no-derp-proxy region.
|
|
client, err := derphttp.NewClient(key.NewNode(), proxyAPI3.Options.AccessURL.String(), func(format string, args ...any) {})
|
|
require.NoError(t, err)
|
|
|
|
ctx := testutil.Context(t, testutil.WaitLong)
|
|
err = client.Connect(ctx)
|
|
require.Error(t, err)
|
|
})
|
|
}
|
|
|
|
func TestDERPEndToEnd(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
deploymentValues := coderdtest.DeploymentValues(t)
|
|
deploymentValues.Experiments = []string{
|
|
string(codersdk.ExperimentMoons),
|
|
"*",
|
|
}
|
|
|
|
client, closer, api, user := coderdenttest.NewWithAPI(t, &coderdenttest.Options{
|
|
Options: &coderdtest.Options{
|
|
DeploymentValues: deploymentValues,
|
|
AppHostname: "*.primary.test.coder.com",
|
|
IncludeProvisionerDaemon: true,
|
|
RealIPConfig: &httpmw.RealIPConfig{
|
|
TrustedOrigins: []*net.IPNet{{
|
|
IP: net.ParseIP("127.0.0.1"),
|
|
Mask: net.CIDRMask(8, 32),
|
|
}},
|
|
TrustedHeaders: []string{
|
|
"CF-Connecting-IP",
|
|
},
|
|
},
|
|
},
|
|
LicenseOptions: &coderdenttest.LicenseOptions{
|
|
Features: license.Features{
|
|
codersdk.FeatureWorkspaceProxy: 1,
|
|
},
|
|
},
|
|
})
|
|
t.Cleanup(func() {
|
|
_ = closer.Close()
|
|
})
|
|
|
|
coderdenttest.NewWorkspaceProxy(t, api, client, &coderdenttest.ProxyOptions{
|
|
Name: "best-proxy",
|
|
})
|
|
|
|
// Wait for the proxy to become healthy.
|
|
require.Eventually(t, func() bool {
|
|
healthCtx := testutil.Context(t, testutil.WaitLong)
|
|
err := api.ProxyHealth.ForceUpdate(healthCtx)
|
|
if !assert.NoError(t, err) {
|
|
return false
|
|
}
|
|
|
|
regions, err := client.Regions(healthCtx)
|
|
if !assert.NoError(t, err) {
|
|
return false
|
|
}
|
|
if !assert.Len(t, regions, 2) {
|
|
return false
|
|
}
|
|
for _, r := range regions {
|
|
if !r.Healthy {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}, testutil.WaitLong, testutil.IntervalMedium)
|
|
|
|
// Swap out the DERPMapper for a fake one that only returns the proxy. This
|
|
// allows us to force the agent to pick the proxy as its preferred region.
|
|
oldDERPMapper := *api.AGPL.DERPMapper.Load()
|
|
newDERPMapper := func(derpMap *tailcfg.DERPMap) *tailcfg.DERPMap {
|
|
derpMap = oldDERPMapper(derpMap)
|
|
// Strip everything but the proxy, which is region ID 10001.
|
|
derpMap.Regions = map[int]*tailcfg.DERPRegion{
|
|
10001: derpMap.Regions[10001],
|
|
}
|
|
derpMap.OmitDefaultRegions = true
|
|
return derpMap
|
|
}
|
|
api.AGPL.DERPMapper.Store(&newDERPMapper)
|
|
|
|
// Create a workspace + apps
|
|
authToken := uuid.NewString()
|
|
version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{
|
|
Parse: echo.ParseComplete,
|
|
ProvisionApply: echo.ProvisionApplyWithAgent(authToken),
|
|
})
|
|
template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID)
|
|
coderdtest.AwaitTemplateVersionJobCompleted(t, client, version.ID)
|
|
workspace := coderdtest.CreateWorkspace(t, client, user.OrganizationID, template.ID)
|
|
build := coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, workspace.LatestBuild.ID)
|
|
workspace.LatestBuild = build
|
|
|
|
agentID := uuid.Nil
|
|
resourceLoop:
|
|
for _, res := range build.Resources {
|
|
for _, agnt := range res.Agents {
|
|
agentID = agnt.ID
|
|
break resourceLoop
|
|
}
|
|
}
|
|
require.NotEqual(t, uuid.Nil, agentID)
|
|
|
|
// Connect an agent to the workspace
|
|
_ = agenttest.New(t, client.URL, authToken)
|
|
_ = coderdtest.AwaitWorkspaceAgents(t, client, workspace.ID)
|
|
|
|
// Connect to the workspace agent.
|
|
ctx := testutil.Context(t, testutil.WaitLong)
|
|
conn, err := client.DialWorkspaceAgent(ctx, agentID, &codersdk.DialWorkspaceAgentOptions{
|
|
Logger: slogtest.Make(t, &slogtest.Options{
|
|
IgnoreErrors: true,
|
|
}).Named("client").Leveled(slog.LevelDebug),
|
|
// Force DERP.
|
|
BlockEndpoints: true,
|
|
})
|
|
require.NoError(t, err)
|
|
t.Cleanup(func() {
|
|
err := conn.Close()
|
|
assert.NoError(t, err)
|
|
})
|
|
|
|
ok := conn.AwaitReachable(ctx)
|
|
require.True(t, ok)
|
|
|
|
_, p2p, _, err := conn.Ping(ctx)
|
|
require.NoError(t, err)
|
|
require.False(t, p2p)
|
|
}
|
|
|
|
func TestWorkspaceProxyWorkspaceApps_Wsconncache(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
apptest.Run(t, false, func(t *testing.T, opts *apptest.DeploymentOptions) *apptest.Deployment {
|
|
deploymentValues := coderdtest.DeploymentValues(t)
|
|
deploymentValues.DisablePathApps = clibase.Bool(opts.DisablePathApps)
|
|
deploymentValues.Dangerous.AllowPathAppSharing = clibase.Bool(opts.DangerousAllowPathAppSharing)
|
|
deploymentValues.Dangerous.AllowPathAppSiteOwnerAccess = clibase.Bool(opts.DangerousAllowPathAppSiteOwnerAccess)
|
|
deploymentValues.Experiments = []string{
|
|
string(codersdk.ExperimentMoons),
|
|
"*",
|
|
}
|
|
|
|
client, closer, api, user := coderdenttest.NewWithAPI(t, &coderdenttest.Options{
|
|
Options: &coderdtest.Options{
|
|
DeploymentValues: deploymentValues,
|
|
AppHostname: "*.primary.test.coder.com",
|
|
IncludeProvisionerDaemon: true,
|
|
RealIPConfig: &httpmw.RealIPConfig{
|
|
TrustedOrigins: []*net.IPNet{{
|
|
IP: net.ParseIP("127.0.0.1"),
|
|
Mask: net.CIDRMask(8, 32),
|
|
}},
|
|
TrustedHeaders: []string{
|
|
"CF-Connecting-IP",
|
|
},
|
|
},
|
|
WorkspaceAppsStatsCollectorOptions: opts.StatsCollectorOptions,
|
|
},
|
|
LicenseOptions: &coderdenttest.LicenseOptions{
|
|
Features: license.Features{
|
|
codersdk.FeatureWorkspaceProxy: 1,
|
|
},
|
|
},
|
|
})
|
|
t.Cleanup(func() {
|
|
_ = closer.Close()
|
|
})
|
|
|
|
// Create the external proxy
|
|
if opts.DisableSubdomainApps {
|
|
opts.AppHost = ""
|
|
}
|
|
proxyAPI := coderdenttest.NewWorkspaceProxy(t, api, client, &coderdenttest.ProxyOptions{
|
|
Name: "best-proxy",
|
|
AppHostname: opts.AppHost,
|
|
DisablePathApps: opts.DisablePathApps,
|
|
})
|
|
|
|
return &apptest.Deployment{
|
|
Options: opts,
|
|
SDKClient: client,
|
|
FirstUser: user,
|
|
PathAppBaseURL: proxyAPI.Options.AccessURL,
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestWorkspaceProxyWorkspaceApps_SingleTailnet(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
apptest.Run(t, false, func(t *testing.T, opts *apptest.DeploymentOptions) *apptest.Deployment {
|
|
deploymentValues := coderdtest.DeploymentValues(t)
|
|
deploymentValues.DisablePathApps = clibase.Bool(opts.DisablePathApps)
|
|
deploymentValues.Dangerous.AllowPathAppSharing = clibase.Bool(opts.DangerousAllowPathAppSharing)
|
|
deploymentValues.Dangerous.AllowPathAppSiteOwnerAccess = clibase.Bool(opts.DangerousAllowPathAppSiteOwnerAccess)
|
|
deploymentValues.Experiments = []string{
|
|
string(codersdk.ExperimentMoons),
|
|
string(codersdk.ExperimentSingleTailnet),
|
|
"*",
|
|
}
|
|
|
|
client, _, api, user := coderdenttest.NewWithAPI(t, &coderdenttest.Options{
|
|
Options: &coderdtest.Options{
|
|
DeploymentValues: deploymentValues,
|
|
AppHostname: "*.primary.test.coder.com",
|
|
IncludeProvisionerDaemon: true,
|
|
RealIPConfig: &httpmw.RealIPConfig{
|
|
TrustedOrigins: []*net.IPNet{{
|
|
IP: net.ParseIP("127.0.0.1"),
|
|
Mask: net.CIDRMask(8, 32),
|
|
}},
|
|
TrustedHeaders: []string{
|
|
"CF-Connecting-IP",
|
|
},
|
|
},
|
|
WorkspaceAppsStatsCollectorOptions: opts.StatsCollectorOptions,
|
|
},
|
|
LicenseOptions: &coderdenttest.LicenseOptions{
|
|
Features: license.Features{
|
|
codersdk.FeatureWorkspaceProxy: 1,
|
|
},
|
|
},
|
|
})
|
|
|
|
// Create the external proxy
|
|
if opts.DisableSubdomainApps {
|
|
opts.AppHost = ""
|
|
}
|
|
proxyAPI := coderdenttest.NewWorkspaceProxy(t, api, client, &coderdenttest.ProxyOptions{
|
|
Name: "best-proxy",
|
|
Experiments: coderd.ReadExperiments(api.Logger, deploymentValues.Experiments.Value()),
|
|
AppHostname: opts.AppHost,
|
|
DisablePathApps: opts.DisablePathApps,
|
|
})
|
|
|
|
return &apptest.Deployment{
|
|
Options: opts,
|
|
SDKClient: client,
|
|
FirstUser: user,
|
|
PathAppBaseURL: proxyAPI.Options.AccessURL,
|
|
}
|
|
})
|
|
}
|