mirror of https://github.com/coder/coder.git
1124 lines
34 KiB
Go
1124 lines
34 KiB
Go
package wsproxy_test
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"net/url"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/davecgh/go-spew/spew"
|
|
"github.com/google/uuid"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
"tailscale.com/derp"
|
|
"tailscale.com/derp/derphttp"
|
|
"tailscale.com/tailcfg"
|
|
"tailscale.com/types/key"
|
|
|
|
"cdr.dev/slog"
|
|
"cdr.dev/slog/sloggers/slogtest"
|
|
"github.com/coder/coder/v2/agent/agenttest"
|
|
"github.com/coder/coder/v2/buildinfo"
|
|
"github.com/coder/coder/v2/coderd/coderdtest"
|
|
"github.com/coder/coder/v2/coderd/healthcheck/derphealth"
|
|
"github.com/coder/coder/v2/coderd/httpmw"
|
|
"github.com/coder/coder/v2/coderd/workspaceapps/apptest"
|
|
"github.com/coder/coder/v2/codersdk"
|
|
"github.com/coder/coder/v2/codersdk/workspacesdk"
|
|
"github.com/coder/coder/v2/cryptorand"
|
|
"github.com/coder/coder/v2/enterprise/coderd/coderdenttest"
|
|
"github.com/coder/coder/v2/enterprise/coderd/license"
|
|
"github.com/coder/coder/v2/enterprise/wsproxy/wsproxysdk"
|
|
"github.com/coder/coder/v2/provisioner/echo"
|
|
"github.com/coder/coder/v2/testutil"
|
|
"github.com/coder/serpent"
|
|
)
|
|
|
|
func TestDERPOnly(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
deploymentValues := coderdtest.DeploymentValues(t)
|
|
deploymentValues.Experiments = []string{
|
|
"*",
|
|
}
|
|
|
|
client, closer, api, _ := coderdenttest.NewWithAPI(t, &coderdenttest.Options{
|
|
Options: &coderdtest.Options{
|
|
DeploymentValues: deploymentValues,
|
|
AppHostname: "*.primary.test.coder.com",
|
|
IncludeProvisionerDaemon: true,
|
|
RealIPConfig: &httpmw.RealIPConfig{
|
|
TrustedOrigins: []*net.IPNet{{
|
|
IP: net.ParseIP("127.0.0.1"),
|
|
Mask: net.CIDRMask(8, 32),
|
|
}},
|
|
TrustedHeaders: []string{
|
|
"CF-Connecting-IP",
|
|
},
|
|
},
|
|
},
|
|
LicenseOptions: &coderdenttest.LicenseOptions{
|
|
Features: license.Features{
|
|
codersdk.FeatureWorkspaceProxy: 1,
|
|
},
|
|
},
|
|
})
|
|
t.Cleanup(func() {
|
|
_ = closer.Close()
|
|
})
|
|
|
|
// Create an external proxy.
|
|
_ = coderdenttest.NewWorkspaceProxyReplica(t, api, client, &coderdenttest.ProxyOptions{
|
|
Name: "best-proxy",
|
|
DerpOnly: true,
|
|
})
|
|
|
|
// Should not show up in the regions list.
|
|
ctx := testutil.Context(t, testutil.WaitLong)
|
|
regions, err := client.Regions(ctx)
|
|
require.NoError(t, err)
|
|
require.Len(t, regions, 1)
|
|
require.Equal(t, api.Options.AccessURL.String(), regions[0].PathAppURL)
|
|
}
|
|
|
|
func TestDERP(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
deploymentValues := coderdtest.DeploymentValues(t)
|
|
deploymentValues.Experiments = []string{
|
|
"*",
|
|
}
|
|
|
|
client, closer, api, user := coderdenttest.NewWithAPI(t, &coderdenttest.Options{
|
|
Options: &coderdtest.Options{
|
|
DeploymentValues: deploymentValues,
|
|
AppHostname: "*.primary.test.coder.com",
|
|
IncludeProvisionerDaemon: true,
|
|
RealIPConfig: &httpmw.RealIPConfig{
|
|
TrustedOrigins: []*net.IPNet{{
|
|
IP: net.ParseIP("127.0.0.1"),
|
|
Mask: net.CIDRMask(8, 32),
|
|
}},
|
|
TrustedHeaders: []string{
|
|
"CF-Connecting-IP",
|
|
},
|
|
},
|
|
},
|
|
LicenseOptions: &coderdenttest.LicenseOptions{
|
|
Features: license.Features{
|
|
codersdk.FeatureWorkspaceProxy: 1,
|
|
},
|
|
},
|
|
})
|
|
t.Cleanup(func() {
|
|
_ = closer.Close()
|
|
})
|
|
|
|
// Create two running external proxies.
|
|
proxyAPI1 := coderdenttest.NewWorkspaceProxyReplica(t, api, client, &coderdenttest.ProxyOptions{
|
|
Name: "best-proxy",
|
|
})
|
|
proxyAPI2 := coderdenttest.NewWorkspaceProxyReplica(t, api, client, &coderdenttest.ProxyOptions{
|
|
Name: "worst-proxy",
|
|
})
|
|
|
|
// Create a running external proxy with DERP disabled.
|
|
proxyAPI3 := coderdenttest.NewWorkspaceProxyReplica(t, api, client, &coderdenttest.ProxyOptions{
|
|
Name: "no-derp-proxy",
|
|
DerpDisabled: true,
|
|
})
|
|
|
|
// Create a proxy that is never started.
|
|
ctx := testutil.Context(t, testutil.WaitLong)
|
|
_, err := client.CreateWorkspaceProxy(ctx, codersdk.CreateWorkspaceProxyRequest{
|
|
Name: "never-started-proxy",
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
// Wait for both running proxies to become healthy.
|
|
require.Eventually(t, func() bool {
|
|
err := api.ProxyHealth.ForceUpdate(ctx)
|
|
if !assert.NoError(t, err) {
|
|
return false
|
|
}
|
|
|
|
regions, err := client.Regions(ctx)
|
|
if !assert.NoError(t, err) {
|
|
return false
|
|
}
|
|
if !assert.Len(t, regions, 5) {
|
|
return false
|
|
}
|
|
|
|
// The first 3 regions should be healthy.
|
|
for _, r := range regions[:4] {
|
|
if !r.Healthy {
|
|
return false
|
|
}
|
|
}
|
|
|
|
// The last region should never be healthy.
|
|
assert.False(t, regions[4].Healthy)
|
|
return true
|
|
}, testutil.WaitLong, testutil.IntervalMedium)
|
|
|
|
// Create a workspace + apps
|
|
authToken := uuid.NewString()
|
|
version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{
|
|
Parse: echo.ParseComplete,
|
|
ProvisionApply: echo.ProvisionApplyWithAgent(authToken),
|
|
})
|
|
template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID)
|
|
coderdtest.AwaitTemplateVersionJobCompleted(t, client, version.ID)
|
|
workspace := coderdtest.CreateWorkspace(t, client, user.OrganizationID, template.ID)
|
|
build := coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, workspace.LatestBuild.ID)
|
|
workspace.LatestBuild = build
|
|
|
|
agentID := uuid.Nil
|
|
resourceLoop:
|
|
for _, res := range build.Resources {
|
|
for _, agnt := range res.Agents {
|
|
agentID = agnt.ID
|
|
break resourceLoop
|
|
}
|
|
}
|
|
require.NotEqual(t, uuid.Nil, agentID)
|
|
|
|
// Connect an agent to the workspace
|
|
_ = agenttest.New(t, client.URL, authToken)
|
|
_ = coderdtest.AwaitWorkspaceAgents(t, client, workspace.ID)
|
|
|
|
t.Run("ReturnedInDERPMap", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
ctx := testutil.Context(t, testutil.WaitLong)
|
|
connInfo, err := workspacesdk.New(client).AgentConnectionInfo(ctx, agentID)
|
|
require.NoError(t, err)
|
|
|
|
// There should be three DERP regions in the map: the primary, and each
|
|
// of the two running proxies. Also the STUN-only regions.
|
|
require.NotNil(t, connInfo.DERPMap)
|
|
require.Len(t, connInfo.DERPMap.Regions, 3+len(api.DeploymentValues.DERP.Server.STUNAddresses.Value()))
|
|
|
|
var (
|
|
primaryRegion *tailcfg.DERPRegion
|
|
proxy1Region *tailcfg.DERPRegion
|
|
proxy2Region *tailcfg.DERPRegion
|
|
)
|
|
for _, r := range connInfo.DERPMap.Regions {
|
|
if r.EmbeddedRelay {
|
|
primaryRegion = r
|
|
continue
|
|
}
|
|
if r.RegionName == "best-proxy" {
|
|
proxy1Region = r
|
|
continue
|
|
}
|
|
if r.RegionName == "worst-proxy" {
|
|
proxy2Region = r
|
|
continue
|
|
}
|
|
// The no-derp-proxy shouldn't show up in the map.
|
|
// The last region is never started, which means it's never healthy,
|
|
// which means it's never added to the DERP map.
|
|
|
|
if len(r.Nodes) == 1 && r.Nodes[0].STUNOnly {
|
|
// Skip STUN-only regions.
|
|
continue
|
|
}
|
|
|
|
t.Fatalf("unexpected region: %+v", r)
|
|
}
|
|
|
|
// The primary region:
|
|
require.Equal(t, "Coder Embedded Relay", primaryRegion.RegionName)
|
|
require.Equal(t, "coder", primaryRegion.RegionCode)
|
|
require.Equal(t, 999, primaryRegion.RegionID)
|
|
require.True(t, primaryRegion.EmbeddedRelay)
|
|
|
|
// The first proxy region:
|
|
require.Equal(t, "best-proxy", proxy1Region.RegionName)
|
|
require.Equal(t, "coder_best-proxy", proxy1Region.RegionCode)
|
|
require.Equal(t, 10001, proxy1Region.RegionID)
|
|
require.False(t, proxy1Region.EmbeddedRelay)
|
|
require.Len(t, proxy1Region.Nodes, 1)
|
|
require.Equal(t, "10001a", proxy1Region.Nodes[0].Name)
|
|
require.Equal(t, 10001, proxy1Region.Nodes[0].RegionID)
|
|
require.Equal(t, proxyAPI1.Options.AccessURL.Hostname(), proxy1Region.Nodes[0].HostName)
|
|
require.Equal(t, proxyAPI1.Options.AccessURL.Port(), fmt.Sprint(proxy1Region.Nodes[0].DERPPort))
|
|
require.Equal(t, proxyAPI1.Options.AccessURL.Scheme == "http", proxy1Region.Nodes[0].ForceHTTP)
|
|
|
|
// The second proxy region:
|
|
require.Equal(t, "worst-proxy", proxy2Region.RegionName)
|
|
require.Equal(t, "coder_worst-proxy", proxy2Region.RegionCode)
|
|
require.Equal(t, 10002, proxy2Region.RegionID)
|
|
require.False(t, proxy2Region.EmbeddedRelay)
|
|
require.Len(t, proxy2Region.Nodes, 1)
|
|
require.Equal(t, "10002a", proxy2Region.Nodes[0].Name)
|
|
require.Equal(t, 10002, proxy2Region.Nodes[0].RegionID)
|
|
require.Equal(t, proxyAPI2.Options.AccessURL.Hostname(), proxy2Region.Nodes[0].HostName)
|
|
require.Equal(t, proxyAPI2.Options.AccessURL.Port(), fmt.Sprint(proxy2Region.Nodes[0].DERPPort))
|
|
require.Equal(t, proxyAPI2.Options.AccessURL.Scheme == "http", proxy2Region.Nodes[0].ForceHTTP)
|
|
})
|
|
|
|
t.Run("ConnectDERP", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
ctx := testutil.Context(t, testutil.WaitLong)
|
|
connInfo, err := workspacesdk.New(client).AgentConnectionInfo(ctx, agentID)
|
|
require.NoError(t, err)
|
|
require.NotNil(t, connInfo.DERPMap)
|
|
require.Len(t, connInfo.DERPMap.Regions, 3+len(api.DeploymentValues.DERP.Server.STUNAddresses.Value()))
|
|
|
|
// Connect to each region.
|
|
for _, r := range connInfo.DERPMap.Regions {
|
|
r := r
|
|
if len(r.Nodes) == 1 && r.Nodes[0].STUNOnly {
|
|
// Skip STUN-only regions.
|
|
continue
|
|
}
|
|
|
|
t.Run(r.RegionName, func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
derpMap := &tailcfg.DERPMap{
|
|
Regions: map[int]*tailcfg.DERPRegion{
|
|
r.RegionID: r,
|
|
},
|
|
OmitDefaultRegions: true,
|
|
}
|
|
|
|
report := derphealth.Report{}
|
|
report.Run(ctx, &derphealth.ReportOptions{
|
|
DERPMap: derpMap,
|
|
})
|
|
|
|
t.Log("healthcheck report: " + spew.Sdump(&report))
|
|
require.True(t, report.Healthy, "healthcheck failed, see report dump")
|
|
})
|
|
}
|
|
})
|
|
|
|
t.Run("DERPDisabled", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
// Try to connect to the DERP server on the no-derp-proxy region.
|
|
client, err := derphttp.NewClient(key.NewNode(), proxyAPI3.Options.AccessURL.String(), func(format string, args ...any) {})
|
|
require.NoError(t, err)
|
|
|
|
ctx := testutil.Context(t, testutil.WaitLong)
|
|
err = client.Connect(ctx)
|
|
require.Error(t, err)
|
|
})
|
|
}
|
|
|
|
func TestDERPEndToEnd(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
deploymentValues := coderdtest.DeploymentValues(t)
|
|
deploymentValues.Experiments = []string{
|
|
"*",
|
|
}
|
|
|
|
client, closer, api, user := coderdenttest.NewWithAPI(t, &coderdenttest.Options{
|
|
Options: &coderdtest.Options{
|
|
DeploymentValues: deploymentValues,
|
|
AppHostname: "*.primary.test.coder.com",
|
|
IncludeProvisionerDaemon: true,
|
|
RealIPConfig: &httpmw.RealIPConfig{
|
|
TrustedOrigins: []*net.IPNet{{
|
|
IP: net.ParseIP("127.0.0.1"),
|
|
Mask: net.CIDRMask(8, 32),
|
|
}},
|
|
TrustedHeaders: []string{
|
|
"CF-Connecting-IP",
|
|
},
|
|
},
|
|
},
|
|
LicenseOptions: &coderdenttest.LicenseOptions{
|
|
Features: license.Features{
|
|
codersdk.FeatureWorkspaceProxy: 1,
|
|
},
|
|
},
|
|
})
|
|
t.Cleanup(func() {
|
|
_ = closer.Close()
|
|
})
|
|
|
|
coderdenttest.NewWorkspaceProxyReplica(t, api, client, &coderdenttest.ProxyOptions{
|
|
Name: "best-proxy",
|
|
})
|
|
|
|
// Wait for the proxy to become healthy.
|
|
ctx := testutil.Context(t, testutil.WaitLong)
|
|
require.Eventually(t, func() bool {
|
|
err := api.ProxyHealth.ForceUpdate(ctx)
|
|
if !assert.NoError(t, err) {
|
|
return false
|
|
}
|
|
|
|
regions, err := client.Regions(ctx)
|
|
if !assert.NoError(t, err) {
|
|
return false
|
|
}
|
|
if !assert.Len(t, regions, 2) {
|
|
return false
|
|
}
|
|
for _, r := range regions {
|
|
if !r.Healthy {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}, testutil.WaitLong, testutil.IntervalMedium)
|
|
|
|
// Wait until the proxy appears in the DERP map, and then swap out the DERP
|
|
// map for one that only contains the proxy region. This allows us to force
|
|
// the agent to pick the proxy as its preferred region.
|
|
var proxyOnlyDERPMap *tailcfg.DERPMap
|
|
require.Eventually(t, func() bool {
|
|
derpMap := api.AGPL.DERPMap()
|
|
if derpMap == nil {
|
|
return false
|
|
}
|
|
if _, ok := derpMap.Regions[10001]; !ok {
|
|
return false
|
|
}
|
|
|
|
// Make a DERP map that only contains the proxy region.
|
|
proxyOnlyDERPMap = derpMap.Clone()
|
|
proxyOnlyDERPMap.Regions = map[int]*tailcfg.DERPRegion{
|
|
10001: proxyOnlyDERPMap.Regions[10001],
|
|
}
|
|
proxyOnlyDERPMap.OmitDefaultRegions = true
|
|
return true
|
|
}, testutil.WaitLong, testutil.IntervalMedium)
|
|
newDERPMapper := func(derpMap *tailcfg.DERPMap) *tailcfg.DERPMap {
|
|
return proxyOnlyDERPMap
|
|
}
|
|
api.AGPL.DERPMapper.Store(&newDERPMapper)
|
|
|
|
// Create a workspace + apps
|
|
authToken := uuid.NewString()
|
|
version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{
|
|
Parse: echo.ParseComplete,
|
|
ProvisionApply: echo.ProvisionApplyWithAgent(authToken),
|
|
})
|
|
template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID)
|
|
coderdtest.AwaitTemplateVersionJobCompleted(t, client, version.ID)
|
|
workspace := coderdtest.CreateWorkspace(t, client, user.OrganizationID, template.ID)
|
|
build := coderdtest.AwaitWorkspaceBuildJobCompleted(t, client, workspace.LatestBuild.ID)
|
|
workspace.LatestBuild = build
|
|
|
|
agentID := uuid.Nil
|
|
resourceLoop:
|
|
for _, res := range build.Resources {
|
|
for _, agnt := range res.Agents {
|
|
agentID = agnt.ID
|
|
break resourceLoop
|
|
}
|
|
}
|
|
require.NotEqual(t, uuid.Nil, agentID)
|
|
|
|
// Connect an agent to the workspace
|
|
_ = agenttest.New(t, client.URL, authToken)
|
|
_ = coderdtest.AwaitWorkspaceAgents(t, client, workspace.ID)
|
|
|
|
// Connect to the workspace agent.
|
|
conn, err := workspacesdk.New(client).
|
|
DialAgent(ctx, agentID, &workspacesdk.DialAgentOptions{
|
|
Logger: slogtest.Make(t, &slogtest.Options{
|
|
IgnoreErrors: true,
|
|
}).Named("client").Leveled(slog.LevelDebug),
|
|
// Force DERP.
|
|
BlockEndpoints: true,
|
|
})
|
|
require.NoError(t, err)
|
|
t.Cleanup(func() {
|
|
err := conn.Close()
|
|
assert.NoError(t, err)
|
|
})
|
|
|
|
ok := conn.AwaitReachable(ctx)
|
|
require.True(t, ok)
|
|
|
|
_, p2p, _, err := conn.Ping(ctx)
|
|
require.NoError(t, err)
|
|
require.False(t, p2p)
|
|
}
|
|
|
|
// TestDERPMesh spawns 6 workspace proxy replicas and tries to connect to a
|
|
// single DERP peer via every single one.
|
|
func TestDERPMesh(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
deploymentValues := coderdtest.DeploymentValues(t)
|
|
deploymentValues.Experiments = []string{
|
|
"*",
|
|
}
|
|
|
|
client, closer, api, _ := coderdenttest.NewWithAPI(t, &coderdenttest.Options{
|
|
Options: &coderdtest.Options{
|
|
DeploymentValues: deploymentValues,
|
|
AppHostname: "*.primary.test.coder.com",
|
|
IncludeProvisionerDaemon: true,
|
|
RealIPConfig: &httpmw.RealIPConfig{
|
|
TrustedOrigins: []*net.IPNet{{
|
|
IP: net.ParseIP("127.0.0.1"),
|
|
Mask: net.CIDRMask(8, 32),
|
|
}},
|
|
TrustedHeaders: []string{
|
|
"CF-Connecting-IP",
|
|
},
|
|
},
|
|
},
|
|
LicenseOptions: &coderdenttest.LicenseOptions{
|
|
Features: license.Features{
|
|
codersdk.FeatureWorkspaceProxy: 1,
|
|
},
|
|
},
|
|
})
|
|
t.Cleanup(func() {
|
|
_ = closer.Close()
|
|
})
|
|
|
|
proxyURL, err := url.Parse("https://proxy.test.coder.com")
|
|
require.NoError(t, err)
|
|
|
|
// Create 6 proxy replicas.
|
|
const count = 6
|
|
var (
|
|
sessionToken = ""
|
|
proxies = [count]coderdenttest.WorkspaceProxy{}
|
|
derpURLs = [count]string{}
|
|
)
|
|
for i := range proxies {
|
|
proxies[i] = coderdenttest.NewWorkspaceProxyReplica(t, api, client, &coderdenttest.ProxyOptions{
|
|
Name: "best-proxy",
|
|
Token: sessionToken,
|
|
ProxyURL: proxyURL,
|
|
})
|
|
if i == 0 {
|
|
sessionToken = proxies[i].Options.ProxySessionToken
|
|
}
|
|
|
|
derpURL := *proxies[i].ServerURL
|
|
derpURL.Path = "/derp"
|
|
derpURLs[i] = derpURL.String()
|
|
}
|
|
|
|
// Force all proxies to re-register immediately. This ensures the DERP mesh
|
|
// is up-to-date. In production this will happen automatically after about
|
|
// 15 seconds.
|
|
for i, proxy := range proxies {
|
|
err := proxy.RegisterNow()
|
|
require.NoErrorf(t, err, "failed to force proxy %d to re-register", i)
|
|
}
|
|
|
|
// Generate cases. We have a case for:
|
|
// - Each proxy to itself.
|
|
// - Each proxy to each other proxy (one way, no duplicates).
|
|
cases := [][2]string{}
|
|
for i, derpURL := range derpURLs {
|
|
cases = append(cases, [2]string{derpURL, derpURL})
|
|
for j := i + 1; j < len(derpURLs); j++ {
|
|
cases = append(cases, [2]string{derpURL, derpURLs[j]})
|
|
}
|
|
}
|
|
require.Len(t, cases, (count*(count+1))/2) // triangle number
|
|
|
|
for i, c := range cases {
|
|
i, c := i, c
|
|
t.Run(fmt.Sprintf("Proxy%d", i), func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
t.Logf("derp1=%s, derp2=%s", c[0], c[1])
|
|
ctx := testutil.Context(t, testutil.WaitLong)
|
|
client1, client1Recv := createDERPClient(t, ctx, "client1", c[0])
|
|
client2, client2Recv := createDERPClient(t, ctx, "client2", c[1])
|
|
|
|
// Send a packet from client 1 to client 2.
|
|
testDERPSend(t, ctx, client2.SelfPublicKey(), client2Recv, client1)
|
|
|
|
// Send a packet from client 2 to client 1.
|
|
testDERPSend(t, ctx, client1.SelfPublicKey(), client1Recv, client2)
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestWorkspaceProxyDERPMeshProbe ensures that each replica pings every other
|
|
// replica in the same region as itself periodically.
|
|
func TestWorkspaceProxyDERPMeshProbe(t *testing.T) {
|
|
t.Parallel()
|
|
createProxyRegion := func(ctx context.Context, t *testing.T, client *codersdk.Client, name string) codersdk.UpdateWorkspaceProxyResponse {
|
|
t.Helper()
|
|
proxyRes, err := client.CreateWorkspaceProxy(ctx, codersdk.CreateWorkspaceProxyRequest{
|
|
Name: name,
|
|
Icon: "/emojis/flag.png",
|
|
})
|
|
require.NoError(t, err, "failed to create workspace proxy")
|
|
return proxyRes
|
|
}
|
|
|
|
registerBrokenProxy := func(ctx context.Context, t *testing.T, primaryAccessURL *url.URL, accessURL, token string) uuid.UUID {
|
|
t.Helper()
|
|
// Create a HTTP server that always replies with 500.
|
|
srv := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
|
|
rw.WriteHeader(http.StatusInternalServerError)
|
|
}))
|
|
t.Cleanup(srv.Close)
|
|
|
|
// Register a proxy.
|
|
wsproxyClient := wsproxysdk.New(primaryAccessURL)
|
|
wsproxyClient.SetSessionToken(token)
|
|
hostname, err := cryptorand.String(6)
|
|
require.NoError(t, err)
|
|
replicaID := uuid.New()
|
|
_, err = wsproxyClient.RegisterWorkspaceProxy(ctx, wsproxysdk.RegisterWorkspaceProxyRequest{
|
|
AccessURL: accessURL,
|
|
WildcardHostname: "",
|
|
DerpEnabled: true,
|
|
DerpOnly: false,
|
|
ReplicaID: replicaID,
|
|
ReplicaHostname: hostname,
|
|
ReplicaError: "",
|
|
ReplicaRelayAddress: srv.URL,
|
|
Version: buildinfo.Version(),
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
return replicaID
|
|
}
|
|
|
|
t.Run("ProbeOK", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
deploymentValues := coderdtest.DeploymentValues(t)
|
|
deploymentValues.Experiments = []string{
|
|
"*",
|
|
}
|
|
|
|
client, closer, api, _ := coderdenttest.NewWithAPI(t, &coderdenttest.Options{
|
|
Options: &coderdtest.Options{
|
|
DeploymentValues: deploymentValues,
|
|
AppHostname: "*.primary.test.coder.com",
|
|
IncludeProvisionerDaemon: true,
|
|
RealIPConfig: &httpmw.RealIPConfig{
|
|
TrustedOrigins: []*net.IPNet{{
|
|
IP: net.ParseIP("127.0.0.1"),
|
|
Mask: net.CIDRMask(8, 32),
|
|
}},
|
|
TrustedHeaders: []string{
|
|
"CF-Connecting-IP",
|
|
},
|
|
},
|
|
},
|
|
LicenseOptions: &coderdenttest.LicenseOptions{
|
|
Features: license.Features{
|
|
codersdk.FeatureWorkspaceProxy: 1,
|
|
},
|
|
},
|
|
})
|
|
t.Cleanup(func() {
|
|
_ = closer.Close()
|
|
})
|
|
|
|
// Register but don't start a proxy in a different region. This
|
|
// shouldn't affect the mesh since it's in a different region.
|
|
ctx := testutil.Context(t, testutil.WaitLong)
|
|
fakeProxyRes := createProxyRegion(ctx, t, client, "fake-proxy")
|
|
registerBrokenProxy(ctx, t, api.AccessURL, "https://fake-proxy.test.coder.com", fakeProxyRes.ProxyToken)
|
|
|
|
proxyURL, err := url.Parse("https://proxy1.test.coder.com")
|
|
require.NoError(t, err)
|
|
|
|
// Create 6 proxy replicas.
|
|
const count = 6
|
|
var (
|
|
sessionToken = ""
|
|
proxies = [count]coderdenttest.WorkspaceProxy{}
|
|
replicaPingDone = [count]bool{}
|
|
)
|
|
for i := range proxies {
|
|
i := i
|
|
proxies[i] = coderdenttest.NewWorkspaceProxyReplica(t, api, client, &coderdenttest.ProxyOptions{
|
|
Name: "proxy-1",
|
|
Token: sessionToken,
|
|
ProxyURL: proxyURL,
|
|
ReplicaPingCallback: func(replicas []codersdk.Replica, err string) {
|
|
if len(replicas) != count-1 {
|
|
// Still warming up...
|
|
return
|
|
}
|
|
replicaPingDone[i] = true
|
|
assert.Emptyf(t, err, "replica %d ping callback error", i)
|
|
},
|
|
})
|
|
if i == 0 {
|
|
sessionToken = proxies[i].Options.ProxySessionToken
|
|
}
|
|
}
|
|
|
|
// Force all proxies to re-register immediately. This ensures the DERP
|
|
// mesh is up-to-date. In production this will happen automatically
|
|
// after about 15 seconds.
|
|
for i, proxy := range proxies {
|
|
err := proxy.RegisterNow()
|
|
require.NoErrorf(t, err, "failed to force proxy %d to re-register", i)
|
|
}
|
|
|
|
// Ensure that all proxies have pinged.
|
|
require.Eventually(t, func() bool {
|
|
ok := true
|
|
for i := range proxies {
|
|
if !replicaPingDone[i] {
|
|
t.Logf("replica %d has not pinged yet", i)
|
|
ok = false
|
|
}
|
|
}
|
|
return ok
|
|
}, testutil.WaitLong, testutil.IntervalSlow)
|
|
t.Log("all replicas have pinged")
|
|
|
|
// Check they're all healthy according to /healthz-report.
|
|
for _, proxy := range proxies {
|
|
// GET /healthz-report
|
|
u := proxy.ServerURL.ResolveReference(&url.URL{Path: "/healthz-report"})
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
|
|
require.NoError(t, err)
|
|
resp, err := http.DefaultClient.Do(req)
|
|
require.NoError(t, err)
|
|
|
|
var respJSON codersdk.ProxyHealthReport
|
|
err = json.NewDecoder(resp.Body).Decode(&respJSON)
|
|
resp.Body.Close()
|
|
require.NoError(t, err)
|
|
|
|
require.Empty(t, respJSON.Errors, "proxy is not healthy")
|
|
}
|
|
})
|
|
|
|
// Register one proxy, then pretend to register 5 others. This should cause
|
|
// the mesh to fail and return an error.
|
|
t.Run("ProbeFail", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
deploymentValues := coderdtest.DeploymentValues(t)
|
|
deploymentValues.Experiments = []string{
|
|
"*",
|
|
}
|
|
|
|
client, closer, api, _ := coderdenttest.NewWithAPI(t, &coderdenttest.Options{
|
|
Options: &coderdtest.Options{
|
|
DeploymentValues: deploymentValues,
|
|
AppHostname: "*.primary.test.coder.com",
|
|
IncludeProvisionerDaemon: true,
|
|
RealIPConfig: &httpmw.RealIPConfig{
|
|
TrustedOrigins: []*net.IPNet{{
|
|
IP: net.ParseIP("127.0.0.1"),
|
|
Mask: net.CIDRMask(8, 32),
|
|
}},
|
|
TrustedHeaders: []string{
|
|
"CF-Connecting-IP",
|
|
},
|
|
},
|
|
},
|
|
LicenseOptions: &coderdenttest.LicenseOptions{
|
|
Features: license.Features{
|
|
codersdk.FeatureWorkspaceProxy: 1,
|
|
},
|
|
},
|
|
})
|
|
t.Cleanup(func() {
|
|
_ = closer.Close()
|
|
})
|
|
|
|
proxyURL, err := url.Parse("https://proxy2.test.coder.com")
|
|
require.NoError(t, err)
|
|
|
|
// Create 1 real proxy replica.
|
|
const fakeCount = 5
|
|
replicaPingErr := make(chan string, 4)
|
|
proxy := coderdenttest.NewWorkspaceProxyReplica(t, api, client, &coderdenttest.ProxyOptions{
|
|
Name: "proxy-2",
|
|
ProxyURL: proxyURL,
|
|
ReplicaPingCallback: func(replicas []codersdk.Replica, err string) {
|
|
if len(replicas) != fakeCount {
|
|
// Still warming up...
|
|
return
|
|
}
|
|
replicaPingErr <- err
|
|
},
|
|
})
|
|
|
|
// Register (but don't start wsproxy.Server) 5 other proxies in the same
|
|
// region. Since they registered recently they should be included in the
|
|
// mesh. We create a HTTP server on the relay address that always
|
|
// responds with 500 so probes fail.
|
|
ctx := testutil.Context(t, testutil.WaitLong)
|
|
for i := 0; i < fakeCount; i++ {
|
|
registerBrokenProxy(ctx, t, api.AccessURL, proxyURL.String(), proxy.Options.ProxySessionToken)
|
|
}
|
|
|
|
// Force the proxy to re-register immediately.
|
|
err = proxy.RegisterNow()
|
|
require.NoError(t, err, "failed to force proxy to re-register")
|
|
|
|
// Wait for the ping to fail.
|
|
replicaErr := testutil.RequireRecvCtx(ctx, t, replicaPingErr)
|
|
require.NotEmpty(t, replicaErr, "replica ping error")
|
|
|
|
// GET /healthz-report
|
|
u := proxy.ServerURL.ResolveReference(&url.URL{Path: "/healthz-report"})
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
|
|
require.NoError(t, err)
|
|
resp, err := http.DefaultClient.Do(req)
|
|
require.NoError(t, err)
|
|
|
|
var respJSON codersdk.ProxyHealthReport
|
|
err = json.NewDecoder(resp.Body).Decode(&respJSON)
|
|
resp.Body.Close()
|
|
require.NoError(t, err)
|
|
|
|
require.Len(t, respJSON.Warnings, 1, "proxy is healthy")
|
|
require.Contains(t, respJSON.Warnings[0], "High availability networking")
|
|
})
|
|
|
|
// This test catches a regression we detected on dogfood which caused
|
|
// proxies to remain unhealthy after a mesh failure if they dropped to zero
|
|
// siblings after the failure.
|
|
t.Run("HealthyZero", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
deploymentValues := coderdtest.DeploymentValues(t)
|
|
deploymentValues.Experiments = []string{
|
|
"*",
|
|
}
|
|
|
|
client, closer, api, _ := coderdenttest.NewWithAPI(t, &coderdenttest.Options{
|
|
Options: &coderdtest.Options{
|
|
DeploymentValues: deploymentValues,
|
|
AppHostname: "*.primary.test.coder.com",
|
|
IncludeProvisionerDaemon: true,
|
|
RealIPConfig: &httpmw.RealIPConfig{
|
|
TrustedOrigins: []*net.IPNet{{
|
|
IP: net.ParseIP("127.0.0.1"),
|
|
Mask: net.CIDRMask(8, 32),
|
|
}},
|
|
TrustedHeaders: []string{
|
|
"CF-Connecting-IP",
|
|
},
|
|
},
|
|
},
|
|
LicenseOptions: &coderdenttest.LicenseOptions{
|
|
Features: license.Features{
|
|
codersdk.FeatureWorkspaceProxy: 1,
|
|
},
|
|
},
|
|
})
|
|
t.Cleanup(func() {
|
|
_ = closer.Close()
|
|
})
|
|
|
|
proxyURL, err := url.Parse("https://proxy2.test.coder.com")
|
|
require.NoError(t, err)
|
|
|
|
// Create 1 real proxy replica.
|
|
replicaPingErr := make(chan string, 4)
|
|
proxy := coderdenttest.NewWorkspaceProxyReplica(t, api, client, &coderdenttest.ProxyOptions{
|
|
Name: "proxy-2",
|
|
ProxyURL: proxyURL,
|
|
ReplicaPingCallback: func(replicas []codersdk.Replica, err string) {
|
|
replicaPingErr <- err
|
|
},
|
|
})
|
|
|
|
ctx := testutil.Context(t, testutil.WaitLong)
|
|
otherReplicaID := registerBrokenProxy(ctx, t, api.AccessURL, proxyURL.String(), proxy.Options.ProxySessionToken)
|
|
|
|
// Force the proxy to re-register immediately.
|
|
err = proxy.RegisterNow()
|
|
require.NoError(t, err, "failed to force proxy to re-register")
|
|
|
|
// Wait for the ping to fail.
|
|
for {
|
|
replicaErr := testutil.RequireRecvCtx(ctx, t, replicaPingErr)
|
|
t.Log("replica ping error:", replicaErr)
|
|
if replicaErr != "" {
|
|
break
|
|
}
|
|
}
|
|
|
|
// GET /healthz-report
|
|
u := proxy.ServerURL.ResolveReference(&url.URL{Path: "/healthz-report"})
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
|
|
require.NoError(t, err)
|
|
resp, err := http.DefaultClient.Do(req)
|
|
require.NoError(t, err)
|
|
var respJSON codersdk.ProxyHealthReport
|
|
err = json.NewDecoder(resp.Body).Decode(&respJSON)
|
|
resp.Body.Close()
|
|
require.NoError(t, err)
|
|
require.Len(t, respJSON.Warnings, 1, "proxy is healthy")
|
|
require.Contains(t, respJSON.Warnings[0], "High availability networking")
|
|
|
|
// Deregister the other replica.
|
|
wsproxyClient := wsproxysdk.New(api.AccessURL)
|
|
wsproxyClient.SetSessionToken(proxy.Options.ProxySessionToken)
|
|
err = wsproxyClient.DeregisterWorkspaceProxy(ctx, wsproxysdk.DeregisterWorkspaceProxyRequest{
|
|
ReplicaID: otherReplicaID,
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
// Force the proxy to re-register immediately.
|
|
err = proxy.RegisterNow()
|
|
require.NoError(t, err, "failed to force proxy to re-register")
|
|
|
|
// Wait for the ping to be skipped.
|
|
for {
|
|
replicaErr := testutil.RequireRecvCtx(ctx, t, replicaPingErr)
|
|
t.Log("replica ping error:", replicaErr)
|
|
// Should be empty because there are no more peers. This was where
|
|
// the regression was.
|
|
if replicaErr == "" {
|
|
break
|
|
}
|
|
}
|
|
|
|
// GET /healthz-report
|
|
req, err = http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
|
|
require.NoError(t, err)
|
|
resp, err = http.DefaultClient.Do(req)
|
|
require.NoError(t, err)
|
|
err = json.NewDecoder(resp.Body).Decode(&respJSON)
|
|
resp.Body.Close()
|
|
require.NoError(t, err)
|
|
require.Len(t, respJSON.Warnings, 0, "proxy is unhealthy")
|
|
})
|
|
}
|
|
|
|
func TestWorkspaceProxyWorkspaceApps(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
apptest.Run(t, false, func(t *testing.T, opts *apptest.DeploymentOptions) *apptest.Deployment {
|
|
deploymentValues := coderdtest.DeploymentValues(t)
|
|
deploymentValues.DisablePathApps = serpent.Bool(opts.DisablePathApps)
|
|
deploymentValues.Dangerous.AllowPathAppSharing = serpent.Bool(opts.DangerousAllowPathAppSharing)
|
|
deploymentValues.Dangerous.AllowPathAppSiteOwnerAccess = serpent.Bool(opts.DangerousAllowPathAppSiteOwnerAccess)
|
|
deploymentValues.Experiments = []string{
|
|
"*",
|
|
}
|
|
|
|
proxyStatsCollectorFlushCh := make(chan chan<- struct{}, 1)
|
|
flushStats := func() {
|
|
proxyStatsCollectorFlushDone := make(chan struct{}, 1)
|
|
proxyStatsCollectorFlushCh <- proxyStatsCollectorFlushDone
|
|
<-proxyStatsCollectorFlushDone
|
|
}
|
|
|
|
if opts.PrimaryAppHost == "" {
|
|
opts.PrimaryAppHost = "*.primary.test.coder.com"
|
|
}
|
|
client, closer, api, user := coderdenttest.NewWithAPI(t, &coderdenttest.Options{
|
|
Options: &coderdtest.Options{
|
|
DeploymentValues: deploymentValues,
|
|
AppHostname: opts.PrimaryAppHost,
|
|
IncludeProvisionerDaemon: true,
|
|
RealIPConfig: &httpmw.RealIPConfig{
|
|
TrustedOrigins: []*net.IPNet{{
|
|
IP: net.ParseIP("127.0.0.1"),
|
|
Mask: net.CIDRMask(8, 32),
|
|
}},
|
|
TrustedHeaders: []string{
|
|
"CF-Connecting-IP",
|
|
},
|
|
},
|
|
WorkspaceAppsStatsCollectorOptions: opts.StatsCollectorOptions,
|
|
},
|
|
LicenseOptions: &coderdenttest.LicenseOptions{
|
|
Features: license.Features{
|
|
codersdk.FeatureWorkspaceProxy: 1,
|
|
},
|
|
},
|
|
})
|
|
t.Cleanup(func() {
|
|
_ = closer.Close()
|
|
})
|
|
|
|
// Create the external proxy
|
|
if opts.DisableSubdomainApps {
|
|
opts.AppHost = ""
|
|
}
|
|
proxyAPI := coderdenttest.NewWorkspaceProxyReplica(t, api, client, &coderdenttest.ProxyOptions{
|
|
Name: "best-proxy",
|
|
AppHostname: opts.AppHost,
|
|
DisablePathApps: opts.DisablePathApps,
|
|
FlushStats: proxyStatsCollectorFlushCh,
|
|
})
|
|
|
|
return &apptest.Deployment{
|
|
Options: opts,
|
|
SDKClient: client,
|
|
FirstUser: user,
|
|
PathAppBaseURL: proxyAPI.Options.AccessURL,
|
|
FlushStats: flushStats,
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestWorkspaceProxyWorkspaceApps_BlockDirect(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
apptest.Run(t, false, func(t *testing.T, opts *apptest.DeploymentOptions) *apptest.Deployment {
|
|
deploymentValues := coderdtest.DeploymentValues(t)
|
|
deploymentValues.DisablePathApps = serpent.Bool(opts.DisablePathApps)
|
|
deploymentValues.Dangerous.AllowPathAppSharing = serpent.Bool(opts.DangerousAllowPathAppSharing)
|
|
deploymentValues.Dangerous.AllowPathAppSiteOwnerAccess = serpent.Bool(opts.DangerousAllowPathAppSiteOwnerAccess)
|
|
deploymentValues.Experiments = []string{
|
|
"*",
|
|
}
|
|
|
|
proxyStatsCollectorFlushCh := make(chan chan<- struct{}, 1)
|
|
flushStats := func() {
|
|
proxyStatsCollectorFlushDone := make(chan struct{}, 1)
|
|
proxyStatsCollectorFlushCh <- proxyStatsCollectorFlushDone
|
|
<-proxyStatsCollectorFlushDone
|
|
}
|
|
|
|
if opts.PrimaryAppHost == "" {
|
|
opts.PrimaryAppHost = "*.primary.test.coder.com"
|
|
}
|
|
client, closer, api, user := coderdenttest.NewWithAPI(t, &coderdenttest.Options{
|
|
Options: &coderdtest.Options{
|
|
DeploymentValues: deploymentValues,
|
|
AppHostname: opts.PrimaryAppHost,
|
|
IncludeProvisionerDaemon: true,
|
|
RealIPConfig: &httpmw.RealIPConfig{
|
|
TrustedOrigins: []*net.IPNet{{
|
|
IP: net.ParseIP("127.0.0.1"),
|
|
Mask: net.CIDRMask(8, 32),
|
|
}},
|
|
TrustedHeaders: []string{
|
|
"CF-Connecting-IP",
|
|
},
|
|
},
|
|
WorkspaceAppsStatsCollectorOptions: opts.StatsCollectorOptions,
|
|
},
|
|
LicenseOptions: &coderdenttest.LicenseOptions{
|
|
Features: license.Features{
|
|
codersdk.FeatureWorkspaceProxy: 1,
|
|
},
|
|
},
|
|
})
|
|
t.Cleanup(func() {
|
|
_ = closer.Close()
|
|
})
|
|
|
|
// Create the external proxy
|
|
if opts.DisableSubdomainApps {
|
|
opts.AppHost = ""
|
|
}
|
|
proxyAPI := coderdenttest.NewWorkspaceProxyReplica(t, api, client, &coderdenttest.ProxyOptions{
|
|
Name: "best-proxy",
|
|
AppHostname: opts.AppHost,
|
|
DisablePathApps: opts.DisablePathApps,
|
|
FlushStats: proxyStatsCollectorFlushCh,
|
|
BlockDirect: true,
|
|
})
|
|
|
|
return &apptest.Deployment{
|
|
Options: opts,
|
|
SDKClient: client,
|
|
FirstUser: user,
|
|
PathAppBaseURL: proxyAPI.Options.AccessURL,
|
|
FlushStats: flushStats,
|
|
}
|
|
})
|
|
}
|
|
|
|
// createDERPClient creates a DERP client and spawns a goroutine that reads from
|
|
// the client and sends the received packets to a channel.
|
|
//
|
|
//nolint:revive
|
|
func createDERPClient(t *testing.T, ctx context.Context, name string, derpURL string) (*derphttp.Client, <-chan derp.ReceivedPacket) {
|
|
t.Helper()
|
|
|
|
client, err := derphttp.NewClient(key.NewNode(), derpURL, func(format string, args ...any) {
|
|
t.Logf(name+": "+format, args...)
|
|
})
|
|
require.NoError(t, err, "create client")
|
|
t.Cleanup(func() {
|
|
_ = client.Close()
|
|
})
|
|
err = client.Connect(ctx)
|
|
require.NoError(t, err, "connect to DERP server")
|
|
|
|
ch := make(chan derp.ReceivedPacket, 1)
|
|
go func() {
|
|
defer close(ch)
|
|
for {
|
|
msg, err := client.Recv()
|
|
if err != nil {
|
|
t.Logf("Recv error: %v", err)
|
|
return
|
|
}
|
|
switch msg := msg.(type) {
|
|
case derp.ReceivedPacket:
|
|
ch <- msg
|
|
return
|
|
default:
|
|
// We don't care about other messages.
|
|
}
|
|
}
|
|
}()
|
|
|
|
return client, ch
|
|
}
|
|
|
|
// testDERPSend sends a message from src to dstKey and waits for it to be
|
|
// received on dstCh.
|
|
//
|
|
// If the packet doesn't arrive within 500ms, it will try to send it again until
|
|
// testutil.WaitLong is reached.
|
|
//
|
|
//nolint:revive
|
|
func testDERPSend(t *testing.T, ctx context.Context, dstKey key.NodePublic, dstCh <-chan derp.ReceivedPacket, src *derphttp.Client) {
|
|
t.Helper()
|
|
|
|
// The prefix helps identify where the packet starts if you get garbled data
|
|
// in logs.
|
|
const msgStrPrefix = "test_packet_"
|
|
msgStr, err := cryptorand.String(64 - len(msgStrPrefix))
|
|
require.NoError(t, err, "generate random msg string")
|
|
msg := []byte(msgStrPrefix + msgStr)
|
|
|
|
err = src.Send(dstKey, msg)
|
|
require.NoError(t, err, "send message via DERP")
|
|
|
|
ticker := time.NewTicker(time.Millisecond * 500)
|
|
defer ticker.Stop()
|
|
for {
|
|
select {
|
|
case pkt := <-dstCh:
|
|
require.Equal(t, src.SelfPublicKey(), pkt.Source, "packet came from wrong source")
|
|
require.Equal(t, msg, pkt.Data, "packet data is wrong")
|
|
return
|
|
case <-ctx.Done():
|
|
t.Fatal("timed out waiting for packet")
|
|
return
|
|
case <-ticker.C:
|
|
}
|
|
|
|
// Send another packet. Since we're sending packets immediately
|
|
// after opening the clients, they might not be meshed together
|
|
// properly yet.
|
|
err = src.Send(dstKey, msg)
|
|
require.NoError(t, err, "send message via DERP")
|
|
}
|
|
}
|