chore: Initial database scaffolding (#2)

* chore: Initial database scaffolding

This implements migrations and code generation for interfacing with a PostgreSQL database.

A dependency is added for the "postgres" binary on the host, but that seems like an acceptable requirement considering it's our primary database.

An in-memory database object can be created for simple cross-OS and fast testing.

* Run tests in CI

* Use Docker instead of binaries on the host

* Skip database tests on non-Linux operating systems

* chore: Add golangci-lint and codecov

* Use consistent file names

.eslintignore

@@ -1,4 +1,5 @@
 ###############################################################################
 # COPY PASTA OF .gitignore
 ###############################################################################
-node_modules
+node_modules
+vendor

.gitignore

@@ -10,4 +10,5 @@
 ###############################################################################
 
 node_modules
+vendor
 .eslintcache

.prettierignore

@@ -4,4 +4,5 @@
 # https://github.com/prettier/prettier/issues/8506
 # https://github.com/prettier/prettier/issues/8679
 ###############################################################################
-node_modules
+node_modules
+vendor

Makefile

@@ -1,3 +1,14 @@
+# Runs migrations to output a dump of the database.
+database/dump.sql: $(wildcard database/migrations/*.sql)
+	go run database/dump/main.go
+
+# Generates Go code for querying the database.
+.PHONY: database/generate
+database/generate: database/dump.sql database/query.sql
+	cd database && sqlc generate && rm db_tmp.go
+	cd database && gofmt -w -r 'Querier -> querier' *.go
+	cd database && gofmt -w -r 'Queries -> sqlQuerier' *.go
+
 fmt/prettier:
 	@echo "--- prettier"
 # Avoid writing files in CI to reduce file write activity
@@ -9,4 +20,4 @@ endif
 .PHONY: fmt/prettier
 
 fmt: fmt/prettier
-.PHONY: fmt
+.PHONY: fmt

database/db.go

@@ -0,0 +1,75 @@
+// Package database connects to external services for stateful storage.
+//
+// Query functions are generated using sqlc.
+//
+// To modify the database schema:
+// 1. Add a new migration using "create_migration.sh" in database/migrations/
+// 2. Run "make database/generate" in the root to generate models.
+// 3. Add/Edit queries in "query.sql" and run "make database/generate" to create Go code.
+package database
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+
+	"golang.org/x/xerrors"
+)
+
+// Store contains all queryable database functions.
+// It extends the generated interface to add transaction support.
+type Store interface {
+	querier
+
+	InTx(context.Context, func(Store) error) error
+}
+
+// DBTX represents a database connection or transaction.
+type DBTX interface {
+	ExecContext(context.Context, string, ...interface{}) (sql.Result, error)
+	PrepareContext(context.Context, string) (*sql.Stmt, error)
+	QueryContext(context.Context, string, ...interface{}) (*sql.Rows, error)
+	QueryRowContext(context.Context, string, ...interface{}) *sql.Row
+}
+
+// New creates a new database store using a SQL database connection.
+func New(sdb *sql.DB) Store {
+	return &sqlQuerier{
+		db:  sdb,
+		sdb: sdb,
+	}
+}
+
+type sqlQuerier struct {
+	sdb *sql.DB
+	db  DBTX
+}
+
+// InTx performs database operations inside a transaction.
+func (q *sqlQuerier) InTx(ctx context.Context, fn func(Store) error) error {
+	if q.sdb == nil {
+		return nil
+	}
+	tx, err := q.sdb.Begin()
+	if err != nil {
+		return xerrors.Errorf("begin transaction: %w", err)
+	}
+	defer func() {
+		rerr := tx.Rollback()
+		if rerr == nil || errors.Is(rerr, sql.ErrTxDone) {
+			// no need to do anything, tx committed successfully
+			return
+		}
+		// couldn't roll back for some reason, extend returned error
+		err = xerrors.Errorf("defer (%s): %w", rerr.Error(), err)
+	}()
+	err = fn(&sqlQuerier{db: tx})
+	if err != nil {
+		return xerrors.Errorf("execute transaction: %w", err)
+	}
+	err = tx.Commit()
+	if err != nil {
+		return xerrors.Errorf("commit transaction: %w", err)
+	}
+	return nil
+}

database/db_memory.go

@@ -0,0 +1,19 @@
+package database
+
+import "context"
+
+// NewInMemory returns an in-memory store of the database.
+func NewInMemory() Store {
+	return &memoryQuerier{}
+}
+
+type memoryQuerier struct{}
+
+// InTx doesn't rollback data properly for in-memory yet.
+func (q *memoryQuerier) InTx(ctx context.Context, fn func(Store) error) error {
+	return fn(q)
+}
+
+func (q *memoryQuerier) ExampleQuery(ctx context.Context) error {
+	return nil
+}

database/dump.sql

@@ -0,0 +1,2 @@
+-- Code generated by 'make database/generate'. DO NOT EDIT.
+

database/dump/main.go

@@ -0,0 +1,89 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"database/sql"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+
+	"github.com/coder/coder/database"
+	"github.com/coder/coder/database/postgres"
+)
+
+func main() {
+	connection, closeFn, err := postgres.Open()
+	if err != nil {
+		panic(err)
+	}
+	defer closeFn()
+	db, err := sql.Open("postgres", connection)
+	if err != nil {
+		panic(err)
+	}
+	err = database.Migrate(context.Background(), "postgres", db)
+	if err != nil {
+		panic(err)
+	}
+	cmd := exec.Command(
+		"pg_dump",
+		"--schema-only",
+		connection,
+		"--no-privileges",
+		"--no-owner",
+		"--no-comments",
+
+		// We never want to manually generate
+		// queries executing against this table.
+		"--exclude-table=schema_migrations",
+	)
+	cmd.Env = []string{
+		"PGTZ=UTC",
+		"PGCLIENTENCODING=UTF8",
+	}
+	var output bytes.Buffer
+	cmd.Stdout = &output
+	cmd.Stderr = os.Stderr
+	err = cmd.Run()
+	if err != nil {
+		panic(err)
+	}
+
+	for _, sed := range []string{
+		// Remove all comments.
+		"/^--/d",
+		// Public is implicit in the schema.
+		"s/ public\\./ /",
+		// Remove database settings.
+		"s/SET.*;//g",
+		// Remove select statements. These aren't useful
+		// to a reader of the dump.
+		"s/SELECT.*;//g",
+		// Removes multiple newlines.
+		"/^$/N;/^\\n$/D",
+	} {
+		cmd := exec.Command("sed", "-e", sed)
+		cmd.Stdin = bytes.NewReader(output.Bytes())
+		output = bytes.Buffer{}
+		cmd.Stdout = &output
+		cmd.Stderr = os.Stderr
+		err = cmd.Run()
+		if err != nil {
+			panic(err)
+		}
+	}
+
+	dump := fmt.Sprintf("-- Code generated by 'make database/generate'. DO NOT EDIT.\n%s", output.Bytes())
+	_, mainPath, _, ok := runtime.Caller(0)
+	if !ok {
+		panic("couldn't get caller path")
+	}
+	err = ioutil.WriteFile(filepath.Join(mainPath, "..", "..", "dump.sql"), []byte(dump), 0644)
+	if err != nil {
+		panic(err)
+	}
+}

database/migrate.go

@@ -0,0 +1,48 @@
+package database
+
+import (
+	"context"
+	"database/sql"
+	"embed"
+	"errors"
+
+	"github.com/golang-migrate/migrate/v4"
+	"github.com/golang-migrate/migrate/v4/database/postgres"
+	"github.com/golang-migrate/migrate/v4/source/iofs"
+	"golang.org/x/xerrors"
+)
+
+//go:embed migrations/*.sql
+var migrations embed.FS
+
+// Migrate runs SQL migrations to ensure the database schema is up-to-date.
+func Migrate(ctx context.Context, dbName string, db *sql.DB) error {
+	sourceDriver, err := iofs.New(migrations, "migrations")
+	if err != nil {
+		return xerrors.Errorf("create iofs: %w", err)
+	}
+	dbDriver, err := postgres.WithInstance(db, &postgres.Config{})
+	if err != nil {
+		return xerrors.Errorf("wrap postgres connection: %w", err)
+	}
+	m, err := migrate.NewWithInstance("", sourceDriver, dbName, dbDriver)
+	if err != nil {
+		return xerrors.Errorf("migrate: %w", err)
+	}
+	err = m.Up()
+	if err != nil {
+		if errors.Is(err, migrate.ErrNoChange) {
+			// It's OK if no changes happened!
+			return nil
+		}
+		return xerrors.Errorf("up: %w", err)
+	}
+	srcErr, dbErr := m.Close()
+	if srcErr != nil {
+		return xerrors.Errorf("close source: %w", err)
+	}
+	if dbErr != nil {
+		return xerrors.Errorf("close database: %w", err)
+	}
+	return nil
+}

database/migrate_test.go

@@ -0,0 +1,30 @@
+//go:build linux
+
+package database_test
+
+import (
+	"context"
+	"database/sql"
+	"testing"
+
+	"github.com/coder/coder/database"
+	"github.com/coder/coder/database/postgres"
+	"github.com/stretchr/testify/require"
+	"go.uber.org/goleak"
+)
+
+func TestMain(m *testing.M) {
+	goleak.VerifyTestMain(m)
+}
+
+func TestMigrate(t *testing.T) {
+	t.Parallel()
+
+	connection, closeFn, err := postgres.Open()
+	require.NoError(t, err)
+	defer closeFn()
+	db, err := sql.Open("postgres", connection)
+	require.NoError(t, err)
+	err = database.Migrate(context.Background(), "postgres", db)
+	require.NoError(t, err)
+}

database/migrations/create_migration.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+cd "$(dirname "$0")"
+
+if [ -z "$1" ]; then
+    echo "First argument is the migration name!"
+    exit 1
+fi
+
+migrate create -ext sql -dir . -seq $1
+
+echo "After making adjustments, run \"make database/generate\" to generate models."

database/models.go

@@ -0,0 +1,5 @@
+// Code generated by sqlc. DO NOT EDIT.
+
+package database
+
+import ()

database/postgres/postgres.go

@@ -0,0 +1,59 @@
+package postgres
+
+import (
+	"database/sql"
+	"fmt"
+	"log"
+	"time"
+
+	"github.com/ory/dockertest/v3"
+	"github.com/ory/dockertest/v3/docker"
+	"golang.org/x/xerrors"
+)
+
+// Open creates a new PostgreSQL server using a Docker container.
+func Open() (string, func(), error) {
+	pool, err := dockertest.NewPool("")
+	if err != nil {
+		return "", nil, xerrors.Errorf("create pool: %w", err)
+	}
+	resource, err := pool.RunWithOptions(&dockertest.RunOptions{
+		Repository: "postgres",
+		Tag:        "11",
+		Env: []string{
+			"POSTGRES_PASSWORD=postgres",
+			"POSTGRES_USER=postgres",
+			"POSTGRES_DB=postgres",
+			"listen_addresses = '*'",
+		},
+	}, func(config *docker.HostConfig) {
+		// set AutoRemove to true so that stopped container goes away by itself
+		config.AutoRemove = true
+		config.RestartPolicy = docker.RestartPolicy{Name: "no"}
+	})
+	if err != nil {
+		log.Fatalf("Could not start resource: %s", err)
+	}
+	hostAndPort := resource.GetHostPort("5432/tcp")
+	dbURL := fmt.Sprintf("postgres://postgres:postgres@%s/postgres?sslmode=disable", hostAndPort)
+
+	// Docker should hard-kill the container after 120 seconds.
+	resource.Expire(120)
+
+	pool.MaxWait = 120 * time.Second
+	err = pool.Retry(func() error {
+		db, err := sql.Open("postgres", dbURL)
+		if err != nil {
+			return err
+		}
+		err = db.Ping()
+		_ = db.Close()
+		return err
+	})
+	if err != nil {
+		return "", nil, err
+	}
+	return dbURL, func() {
+		_ = pool.Purge(resource)
+	}, nil
+}

database/postgres/postgres_test.go

@@ -0,0 +1,32 @@
+//go:build linux
+
+package postgres_test
+
+import (
+	"database/sql"
+	"testing"
+
+	"github.com/coder/coder/database/postgres"
+	"github.com/stretchr/testify/require"
+	"go.uber.org/goleak"
+
+	_ "github.com/lib/pq"
+)
+
+func TestMain(m *testing.M) {
+	goleak.VerifyTestMain(m)
+}
+
+func TestPostgres(t *testing.T) {
+	t.Parallel()
+
+	connect, close, err := postgres.Open()
+	require.NoError(t, err)
+	defer close()
+	db, err := sql.Open("postgres", connect)
+	require.NoError(t, err)
+	err = db.Ping()
+	require.NoError(t, err)
+	err = db.Close()
+	require.NoError(t, err)
+}

database/querier.go

@@ -0,0 +1,13 @@
+// Code generated by sqlc. DO NOT EDIT.
+
+package database
+
+import (
+	"context"
+)
+
+type querier interface {
+	ExampleQuery(ctx context.Context) error
+}
+
+var _ querier = (*sqlQuerier)(nil)

database/query.sql

@@ -0,0 +1,2 @@
+-- name: ExampleQuery :exec
+SELECT 'example query';

database/query.sql.go

@@ -0,0 +1,17 @@
+// Code generated by sqlc. DO NOT EDIT.
+// source: query.sql
+
+package database
+
+import (
+	"context"
+)
+
+const exampleQuery = `-- name: ExampleQuery :exec
+SELECT 'example query'
+`
+
+func (q *sqlQuerier) ExampleQuery(ctx context.Context) error {
+	_, err := q.db.ExecContext(ctx, exampleQuery)
+	return err
+}

database/sqlc.yaml

@@ -0,0 +1,20 @@
+# sqlc is used to generate types from sql schema language.
+# It was chosen to ensure type-safety when interacting with
+# the database.
+version: "1"
+packages:
+  - name: "database"
+    path: "."
+    queries: "./query.sql"
+    schema: "./dump.sql"
+    engine: "postgresql"
+    emit_interface: true
+    emit_json_tags: true
+    emit_db_tags: true
+    # We replace the generated db file with our own
+    # to add support for transactions. This file is
+    # deleted after generation.
+    output_db_file_name: db_tmp.go
+overrides:
+  - db_type: citext
+    go_type: string

go.mod

@@ -1,3 +1,47 @@
 module github.com/coder/coder
 
 go 1.17
+
+require (
+	github.com/golang-migrate/migrate/v4 v4.15.1
+	github.com/lib/pq v1.10.4
+	github.com/ory/dockertest/v3 v3.8.1
+	github.com/stretchr/testify v1.7.0
+	go.uber.org/goleak v1.1.12
+	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1
+)
+
+require (
+	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
+	github.com/Microsoft/go-winio v0.5.1 // indirect
+	github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect
+	github.com/cenkalti/backoff/v4 v4.1.2 // indirect
+	github.com/containerd/continuity v0.1.0 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/dhui/dktest v0.3.8 // indirect
+	github.com/docker/cli v20.10.11+incompatible // indirect
+	github.com/docker/docker v20.10.12+incompatible // indirect
+	github.com/docker/go-connections v0.4.0 // indirect
+	github.com/docker/go-units v0.4.0 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
+	github.com/hashicorp/errwrap v1.0.0 // indirect
+	github.com/hashicorp/go-multierror v1.1.0 // indirect
+	github.com/imdario/mergo v0.3.12 // indirect
+	github.com/mitchellh/mapstructure v1.4.1 // indirect
+	github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.0.2 // indirect
+	github.com/opencontainers/runc v1.0.2 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/sirupsen/logrus v1.8.1 // indirect
+	github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
+	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
+	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
+	go.uber.org/atomic v1.7.0 // indirect
+	golang.org/x/net v0.0.0-20211013171255-e13a2654a71e // indirect
+	golang.org/x/sys v0.0.0-20211013075003-97ac67df715c // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+)

yarn.lock

@@ -2,7 +2,7 @@
 # yarn lockfile v1
 
 
-prettier@^2.5.1:
+prettier@2.5.1:
   version "2.5.1"
   resolved "https://registry.yarnpkg.com/prettier/-/prettier-2.5.1.tgz#fff75fa9d519c54cf0fce328c1017d94546bc56a"
   integrity sha512-vBZcPRUR5MZJwoyi3ZoyQlc1rXeEck8KgeC9AwwOn+exuxLxq5toTRDTSaVrXHxelDMHy9zlicw8u66yxoSUFg==