implement SQLite storage driver

This commit is contained in:
Lukas Schulte Pelkum 2023-06-13 01:22:05 +02:00
parent 6260f20fc4
commit b9a6a81821
No known key found for this signature in database
GPG Key ID: AB3985CECFAFC962
17 changed files with 265 additions and 8 deletions

View File

@ -7,6 +7,7 @@ import (
"github.com/lus/pasty/internal/meta"
"github.com/lus/pasty/internal/storage"
"github.com/lus/pasty/internal/storage/postgres"
"github.com/lus/pasty/internal/storage/sqlite"
"github.com/lus/pasty/internal/web"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
@ -50,6 +51,9 @@ func main() {
case "postgres":
driver = postgres.New(cfg.Postgres.DSN)
break
case "sqlite":
driver = sqlite.New(cfg.SQLite.File)
break
default:
log.Fatal().Str("driver_name", cfg.StorageDriver).Msg("An invalid storage driver name was given.")
return

16
go.mod
View File

@ -10,20 +10,36 @@ require (
github.com/joho/godotenv v1.5.1
github.com/kelseyhightower/envconfig v1.4.0
github.com/rs/zerolog v1.29.1
modernc.org/sqlite v1.23.1
)
require (
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
github.com/jackc/puddle/v2 v2.2.0 // indirect
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
github.com/lib/pq v1.10.9 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.19 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
go.uber.org/atomic v1.11.0 // indirect
golang.org/x/crypto v0.9.0 // indirect
golang.org/x/mod v0.10.0 // indirect
golang.org/x/sync v0.2.0 // indirect
golang.org/x/sys v0.8.0 // indirect
golang.org/x/text v0.9.0 // indirect
golang.org/x/tools v0.9.1 // indirect
lukechampine.com/uint128 v1.2.0 // indirect
modernc.org/cc/v3 v3.40.0 // indirect
modernc.org/ccgo/v3 v3.16.13 // indirect
modernc.org/libc v1.22.5 // indirect
modernc.org/mathutil v1.5.0 // indirect
modernc.org/memory v1.5.0 // indirect
modernc.org/opt v0.1.3 // indirect
modernc.org/strutil v1.1.3 // indirect
modernc.org/token v1.0.1 // indirect
)

38
go.sum
View File

@ -10,12 +10,18 @@ github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m3
github.com/docker/docker v20.10.24+incompatible h1:Ugvxm7a8+Gz6vqQYQQ2W7GYq5EUPaAiuPgIfVyI3dYE=
github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/go-chi/chi/v5 v5.0.8 h1:lD+NLqFcAi1ovnVZpsnObHGW4xb4J8lNmoYVfECH1Y0=
github.com/go-chi/chi/v5 v5.0.8/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/golang-migrate/migrate/v4 v4.16.1 h1:O+0C55RbMN66pWm5MjO6mw0px6usGpY0+bkSGW9zCo0=
github.com/golang-migrate/migrate/v4 v4.16.1/go.mod h1:qXiwa/3Zeqaltm1MxOCZDYysW/F6folYiBgBG03l9hc=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ=
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
@ -31,6 +37,8 @@ github.com/jackc/puddle/v2 v2.2.0 h1:RdcDk92EJBuBS55nQMMYFXTxwstHug4jkhT5pq8VxPk
github.com/jackc/puddle/v2 v2.2.0/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8=
github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg=
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
@ -42,6 +50,7 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y=
github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
@ -50,6 +59,9 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/rs/zerolog v1.29.1 h1:cO+d60CHkknCbvzEWxP0S9K6KqyTjrCNUy1LdQLCGPc=
github.com/rs/zerolog v1.29.1/go.mod h1:Le6ESbR7hc+DP6Lt1THiV8CQSdkkNrd3R0XbEgp3ZBU=
@ -69,6 +81,7 @@ golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk=
golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
@ -108,7 +121,32 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo=
golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
lukechampine.com/uint128 v1.2.0 h1:mBi/5l91vocEN8otkC5bDLhi2KdCticRiwbdB0O+rjI=
lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk=
modernc.org/cc/v3 v3.40.0 h1:P3g79IUS/93SYhtoeaHW+kRCIrYaxJ27MFPv+7kaTOw=
modernc.org/cc/v3 v3.40.0/go.mod h1:/bTg4dnWkSXowUO6ssQKnOV0yMVxDYNIsIrzqTFDGH0=
modernc.org/ccgo/v3 v3.16.13 h1:Mkgdzl46i5F/CNR/Kj80Ri59hC8TKAhZrYSaqvkwzUw=
modernc.org/ccgo/v3 v3.16.13/go.mod h1:2Quk+5YgpImhPjv2Qsob1DnZ/4som1lJTodubIcoUkY=
modernc.org/ccorpus v1.11.6 h1:J16RXiiqiCgua6+ZvQot4yUuUy8zxgqbqEEUuGPlISk=
modernc.org/httpfs v1.0.6 h1:AAgIpFZRXuYnkjftxTAZwMIiwEqAfk8aVB2/oA6nAeM=
modernc.org/libc v1.22.5 h1:91BNch/e5B0uPbJFgqbxXuOnxBQjlS//icfQEGmvyjE=
modernc.org/libc v1.22.5/go.mod h1:jj+Z7dTNX8fBScMVNRAYZ/jF91K8fdT2hYMThc3YjBY=
modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ=
modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
modernc.org/memory v1.5.0 h1:N+/8c5rE6EqugZwHii4IFsaJ7MUhoWX07J5tC/iI5Ds=
modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU=
modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4=
modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
modernc.org/sqlite v1.23.1 h1:nrSBg4aRQQwq59JpvGEQ15tNxoO5pX/kUjcRNwSAGQM=
modernc.org/sqlite v1.23.1/go.mod h1:OrDj17Mggn6MhE+iPbBNf7RGKODDE9NFT0f3EwDzJqk=
modernc.org/strutil v1.1.3 h1:fNMm+oJklMGYfU9Ylcywl0CO5O6nTfaowNsh2wpPjzY=
modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw=
modernc.org/tcl v1.15.2 h1:C4ybAYCGJw968e+Me18oW55kD/FexcHbqH2xak1ROSY=
modernc.org/token v1.0.1 h1:A3qvTqOwexpfZZeyI0FeGPDlSWX5pjZu9hF4lU+EKWg=
modernc.org/token v1.0.1/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
modernc.org/z v1.7.3 h1:zDJf6iHjrnB+WRD88stbXokugjyc0/pB91ri1gO6LZY=

View File

@ -22,6 +22,7 @@ type Config struct {
AutoDelete *AutoDeleteConfig `split_words:"true"`
Reports *ReportConfig
Postgres *PostgresConfig
SQLite *SQLiteConfig
}
type AutoDeleteConfig struct {
@ -40,6 +41,10 @@ type PostgresConfig struct {
DSN string `default:"postgres://pasty:pasty@localhost/pasty"`
}
type SQLiteConfig struct {
File string `default:":memory:"`
}
func Load() (*Config, error) {
_ = godotenv.Overload()
cfg := new(Config)

View File

@ -65,6 +65,7 @@ func (driver *Driver) Initialize(ctx context.Context) error {
func (driver *Driver) Close() error {
driver.pastes = nil
driver.connPool.Close()
driver.connPool = nil
return nil
}

View File

@ -1,5 +1,5 @@
begin;
drop table if exists "pastes";
drop table "pastes";
commit;

View File

@ -1,6 +1,6 @@
begin;
create table if not exists "pastes" (
create table "pastes" (
"id" text not null,
"content" text not null,
"deletionToken" text not null,

View File

@ -1,5 +1,5 @@
begin;
alter table if exists "pastes" rename column "modificationToken" to "deletionToken";
alter table "pastes" rename column "modificationToken" to "deletionToken";
commit;

View File

@ -1,5 +1,5 @@
begin;
alter table if exists "pastes" rename column "deletionToken" to "modificationToken";
alter table "pastes" rename column "deletionToken" to "modificationToken";
commit;

View File

@ -1,5 +1,5 @@
begin;
alter table if exists "pastes" drop column "metadata";
alter table "pastes" drop column "metadata";
commit;

View File

@ -1,5 +1,5 @@
begin;
alter table if exists "pastes" add column "metadata" jsonb;
alter table "pastes" add column "metadata" jsonb;
commit;

View File

@ -1,5 +1,5 @@
begin;
alter table if exists "pastes" add column "autoDelete" boolean;
alter table "pastes" add column "autoDelete" boolean;
commit;

View File

@ -1,5 +1,5 @@
begin;
alter table if exists "pastes" drop column "autoDelete";
alter table "pastes" drop column "autoDelete";
commit;

View File

@ -0,0 +1,87 @@
package sqlite
import (
"context"
"database/sql"
"embed"
"errors"
"github.com/golang-migrate/migrate/v4"
"github.com/golang-migrate/migrate/v4/database/sqlite"
"github.com/golang-migrate/migrate/v4/source/iofs"
"github.com/lus/pasty/internal/pastes"
"github.com/lus/pasty/internal/storage"
"github.com/rs/zerolog/log"
_ "modernc.org/sqlite"
)
//go:embed migrations/*.sql
var migrations embed.FS
type Driver struct {
filePath string
connPool *sql.DB
pastes *pasteRepository
}
var _ storage.Driver = (*Driver)(nil)
func New(filePath string) *Driver {
return &Driver{
filePath: filePath,
}
}
func (driver *Driver) Initialize(ctx context.Context) error {
db, err := sql.Open("sqlite", driver.filePath)
if err != nil {
return err
}
if err := db.PingContext(ctx); err != nil {
return err
}
log.Info().Msg("Performing SQLite database migrations...")
source, err := iofs.New(migrations, "migrations")
if err != nil {
_ = db.Close()
return err
}
defer func() {
_ = source.Close()
}()
migrateDriver, err := sqlite.WithInstance(db, &sqlite.Config{
MigrationsTable: sqlite.DefaultMigrationsTable,
DatabaseName: driver.filePath,
NoTxWrap: false,
})
if err != nil {
_ = db.Close()
return err
}
migrator, err := migrate.NewWithInstance("iofs", source, "sqlite", migrateDriver)
if err != nil {
_ = db.Close()
return err
}
if err := migrator.Up(); err != nil && !errors.Is(err, migrate.ErrNoChange) {
_ = db.Close()
return err
}
driver.connPool = db
driver.pastes = &pasteRepository{
connPool: db,
}
return nil
}
func (driver *Driver) Close() error {
driver.pastes = nil
_ = driver.connPool.Close()
driver.connPool = nil
return nil
}
func (driver *Driver) Pastes() pastes.Repository {
return driver.pastes
}

View File

@ -0,0 +1 @@
DROP TABLE "pastes";

View File

@ -0,0 +1,8 @@
CREATE TABLE "pastes" (
"id" TEXT NOT NULL,
"content" TEXT NOT NULL,
"modification_token" TEXT NOT NULL,
"created" BIGINT NOT NULL,
"metadata" TEXT NOT NULL,
PRIMARY KEY ("id")
);

View File

@ -0,0 +1,97 @@
package sqlite
import (
"context"
"database/sql"
"encoding/json"
"errors"
"github.com/lus/pasty/internal/pastes"
"time"
)
type pasteRepository struct {
connPool *sql.DB
}
var _ pastes.Repository = (*pasteRepository)(nil)
func (repo *pasteRepository) ListIDs(ctx context.Context) ([]string, error) {
rows, err := repo.connPool.QueryContext(ctx, "SELECT id FROM pastes")
if err != nil {
return nil, err
}
defer func() {
_ = rows.Close()
}()
ids := make([]string, 0)
for rows.Next() {
var id string
if err := rows.Scan(&id); err != nil {
return nil, err
}
ids = append(ids, id)
}
return ids, nil
}
func (repo *pasteRepository) FindByID(ctx context.Context, id string) (*pastes.Paste, error) {
row := repo.connPool.QueryRowContext(ctx, "SELECT * FROM pastes WHERE id = ?", id)
obj := new(pastes.Paste)
var rawMetadata string
if err := row.Scan(&obj.ID, &obj.Content, &obj.ModificationToken, &obj.Created, &rawMetadata); err != nil {
if errors.Is(err, sql.ErrNoRows) {
return nil, nil
}
return nil, err
}
var metadata map[string]any
if err := json.Unmarshal([]byte(rawMetadata), &metadata); err != nil {
return nil, err
}
obj.Metadata = metadata
return obj, nil
}
func (repo *pasteRepository) Upsert(ctx context.Context, paste *pastes.Paste) error {
const query = `
INSERT INTO pastes
VALUES (?, ?, ?, ?, ?)
ON CONFLICT (id) DO UPDATE
SET content = excluded.content,
modification_token = excluded.modification_token,
metadata = excluded.metadata
`
rawMetadata, err := json.Marshal(paste.Metadata)
if err != nil {
return err
}
_, err = repo.connPool.ExecContext(ctx, query, paste.ID, paste.Content, paste.ModificationToken, paste.Created, rawMetadata)
return err
}
func (repo *pasteRepository) DeleteByID(ctx context.Context, id string) error {
_, err := repo.connPool.ExecContext(ctx, "DELETE FROM pastes WHERE id = ?", id)
return err
}
func (repo *pasteRepository) DeleteOlderThan(ctx context.Context, age time.Duration) (int, error) {
result, err := repo.connPool.ExecContext(ctx, "DELETE FROM pastes WHERE created < ?", time.Now().Add(-age).Unix())
if err != nil {
return 0, err
}
affected, err := result.RowsAffected()
if err != nil {
return -1, nil
}
return int(affected), nil
}