test: Use a template to prevent migrations from running for every test (#2462)

* test: Use a template to prevent migrations from running for every test

* Create a single makefile target

* Fix built-in race

* Extend timeout of built-in PostgreSQL fetch

.github/workflows/coder.yaml

@@ -267,30 +267,8 @@ jobs:
           terraform_version: 1.1.9
           terraform_wrapper: false
 
-      - name: Start PostgreSQL Database
-        env:
-          POSTGRES_PASSWORD: postgres
-          POSTGRES_USER: postgres
-          POSTGRES_DB: postgres
-          PGDATA: /tmp
-        run: |
-          docker run \
-            -e POSTGRES_PASSWORD=postgres \
-            -e POSTGRES_USER=postgres \
-            -e POSTGRES_DB=postgres \
-            -e PGDATA=/tmp \
-            -p 5432:5432 \
-            -d postgres:11 \
-            -c shared_buffers=1GB \
-            -c max_connections=1000
-          while ! pg_isready -h 127.0.0.1
-          do
-              echo "$(date) - waiting for database to start"
-              sleep 0.5
-          done
-
       - name: Test with PostgreSQL Database
-        run: "make test-postgres"
+        run: make test-postgres
 
       - name: Upload DataDog Trace
         if: always() && github.actor != 'dependabot[bot]' && !github.event.pull_request.head.repo.fork

.gitignore

@@ -13,6 +13,8 @@ node_modules
 vendor
 .eslintcache
 yarn-error.log
+gotests.xml
+gotests.coverage
 .idea
 .DS_Store
 

Makefile

@@ -171,14 +171,15 @@ test: test-clean
 	gotestsum -- -v -short ./...
 .PHONY: test
 
-test-postgres: test-clean
-	DB=ci gotestsum --junitfile="gotests.xml" --packages="./..." -- \
-          -covermode=atomic -coverprofile="gotests.coverage" -timeout=30m \
-          -coverpkg=./...,github.com/coder/coder/codersdk \
-          -count=1 -race -failfast
+test-postgres: test-clean test-postgres-docker
+	DB_FROM=$(shell go run scripts/migrate-ci/main.go) gotestsum --junitfile="gotests.xml" --packages="./..." -- \
+		-covermode=atomic -coverprofile="gotests.coverage" -timeout=30m \
+		-coverpkg=./...,github.com/coder/coder/codersdk \
+		-count=2 -race -failfast
 .PHONY: test-postgres
 
 test-postgres-docker:
+	docker rm -f test-postgres-docker || true
 	docker run \
 		--env POSTGRES_PASSWORD=postgres \
 		--env POSTGRES_USER=postgres \
@@ -189,12 +190,17 @@ test-postgres-docker:
 		--name test-postgres-docker \
 		--restart no \
 		--detach \
-		postgres:11 \
+		postgres:13 \
 		-c shared_buffers=1GB \
 		-c max_connections=1000 \
 		-c fsync=off \
 		-c synchronous_commit=off \
 		-c full_page_writes=off
+	while ! pg_isready -h 127.0.0.1
+	do
+		echo "$(date) - waiting for database to start"
+		sleep 0.5
+	done
 .PHONY: test-postgres-docker
 
 test-clean:

cli/server_test.go

@@ -83,9 +83,9 @@ func TestServer(t *testing.T) {
 			errC <- root.ExecuteContext(ctx)
 		}()
 		require.Eventually(t, func() bool {
-			_, err := cfg.URL().Read()
-			return err == nil
-		}, time.Minute, 25*time.Millisecond)
+			accessURLRaw, err := cfg.URL().Read()
+			return accessURLRaw != "" && err == nil
+		}, 3*time.Minute, 250*time.Millisecond)
 		cancelFunc()
 		require.ErrorIs(t, <-errC, context.Canceled)
 	})

coderd/coderdtest/coderdtest.go

@@ -113,8 +113,6 @@ func NewWithAPI(t *testing.T, options *Options) (*codersdk.Client, *coderd.API)
 		t.Cleanup(func() {
 			_ = sqlDB.Close()
 		})
-		err = database.MigrateUp(sqlDB)
-		require.NoError(t, err)
 		db = database.New(sqlDB)
 
 		pubsub, err = database.NewPubsub(context.Background(), sqlDB, connectionURL)

coderd/database/postgres/postgres.go

@@ -13,6 +13,7 @@ import (
 	"github.com/ory/dockertest/v3/docker"
 	"golang.org/x/xerrors"
 
+	"github.com/coder/coder/coderd/database"
 	"github.com/coder/coder/cryptorand"
 )
 
@@ -22,7 +23,7 @@ var openPortMutex sync.Mutex
 
 // Open creates a new PostgreSQL server using a Docker container.
 func Open() (string, func(), error) {
-	if os.Getenv("DB") == "ci" {
+	if os.Getenv("DB_FROM") != "" {
 		// In CI, creating a Docker container for each test is slow.
 		// This expects a PostgreSQL instance with the hardcoded credentials
 		// available.
@@ -39,9 +40,9 @@ func Open() (string, func(), error) {
 		}
 
 		dbName = "ci" + dbName
-		_, err = db.Exec("CREATE DATABASE " + dbName)
+		_, err = db.Exec("CREATE DATABASE " + dbName + " WITH TEMPLATE " + os.Getenv("DB_FROM"))
 		if err != nil {
-			return "", nil, xerrors.Errorf("create db: %w", err)
+			return "", nil, xerrors.Errorf("create db with template: %w", err)
 		}
 
 		deleteDB := func() {
@@ -74,7 +75,7 @@ func Open() (string, func(), error) {
 
 	resource, err := pool.RunWithOptions(&dockertest.RunOptions{
 		Repository: "postgres",
-		Tag:        "11",
+		Tag:        "13",
 		Env: []string{
 			"POSTGRES_PASSWORD=postgres",
 			"POSTGRES_USER=postgres",
@@ -133,6 +134,10 @@ func Open() (string, func(), error) {
 		if err != nil {
 			return xerrors.Errorf("ping postgres: %w", err)
 		}
+		err = database.MigrateUp(db)
+		if err != nil {
+			return xerrors.Errorf("migrate db: %w", err)
+		}
 
 		return nil
 	})

scripts/migrate-ci/main.go

@@ -0,0 +1,36 @@
+package main
+
+import (
+	"database/sql"
+	"fmt"
+
+	"github.com/coder/coder/coderd/database"
+	"github.com/coder/coder/cryptorand"
+)
+
+func main() {
+	dbURL := "postgres://postgres:postgres@127.0.0.1:5432/postgres?sslmode=disable"
+	db, err := sql.Open("postgres", dbURL)
+	if err != nil {
+		panic(err)
+	}
+	defer db.Close()
+
+	dbName, err := cryptorand.StringCharset(cryptorand.Lower, 10)
+	if err != nil {
+		panic(err)
+	}
+
+	dbName = "ci" + dbName
+	_, err = db.Exec("CREATE DATABASE " + dbName)
+	if err != nil {
+		panic(err)
+	}
+
+	err = database.MigrateUp(db)
+	if err != nil {
+		panic(err)
+	}
+
+	_, _ = fmt.Println(dbName)
+}