diff --git a/README.md b/README.md index a95c79f..724895e 100644 --- a/README.md +++ b/README.md @@ -11,9 +11,17 @@ Pasty is a fast and lightweight code pasting server | `PASTY_DELETION_TOKEN_LENGTH` | `12` | `number` | Defines the length of the deletion token of a paste | | `PASTY_RATE_LIMIT` | `30-M` | `string` | Defines the rate limit of the API (see https://github.com/ulule/limiter#usage) | +## AutoDelete +Pasty provides an intuitive system to automatically delete pastes after a specific amount of time. You can configure it with the following variables: + ## Storage types Pasty supports multiple storage types, defined using the `PASTY_STORAGE_TYPE` environment variable (use the value behind the corresponding title in this README). Every single one of them has its own configuration variables: +| Environment Variable | Default Value | Type | Description | +|----------------------------------|---------------|----------|--------------------------------------------------------------------------------| +| `PASTY_AUTODELETE` | `false` | `bool` | Defines whether or not the AutoDelete system should be enabled | +| `PASTY_AUTODELETE_LIFETIME` | `720h` | `string` | Defines the duration a paste should live until it gets deleted | +| `PASTY_AUTODELETE_TASK_INTERVAL` | `5m` | `string` | Defines the interval in which the AutoDelete task should clean up the database | ### File (`file`) | Environment Variable | Default Value | Type | Description | diff --git a/cmd/pasty/main.go b/cmd/pasty/main.go index 7ed39ac..ac9f1a2 100644 --- a/cmd/pasty/main.go +++ b/cmd/pasty/main.go @@ -5,6 +5,7 @@ import ( "github.com/Lukaesebrot/pasty/internal/storage" "github.com/Lukaesebrot/pasty/internal/web" "log" + "time" ) func main() { @@ -26,6 +27,24 @@ func main() { } }() + // Schedule the AutoDelete task + if env.Bool("AUTODELETE", false) { + log.Println("Scheduling the AutoDelete task...") + go func() { + for { + // Run the cleanup sequence + deleted, err := storage.Current.Cleanup() + if err != nil { + log.Fatalln(err) + } + log.Printf("AutoDelete: Deleted %d expired pastes", deleted) + + // Wait until the process should repeat + time.Sleep(env.Duration("AUTODELETE_TASK_INTERVAL", 5*time.Minute)) + } + }() + } + // Serve the web resources log.Println("Serving the web resources...") panic(web.Serve()) diff --git a/internal/env/env.go b/internal/env/env.go index ec3c817..60837c0 100644 --- a/internal/env/env.go +++ b/internal/env/env.go @@ -5,6 +5,7 @@ import ( "github.com/joho/godotenv" "os" "strconv" + "time" ) // Load loads an optional .env file @@ -26,3 +27,9 @@ func Bool(key string, fallback bool) bool { parsed, _ := strconv.ParseBool(Get(key, strconv.FormatBool(fallback))) return parsed } + +// Duration uses Get and parses it into a duration +func Duration(key string, fallback time.Duration) time.Duration { + parsed, _ := time.ParseDuration(Get(key, fallback.String())) + return parsed +} diff --git a/internal/pastes/paste.go b/internal/pastes/paste.go index bfc66e3..9f83034 100644 --- a/internal/pastes/paste.go +++ b/internal/pastes/paste.go @@ -1,7 +1,9 @@ package pastes import ( + "github.com/Lukaesebrot/pasty/internal/env" "github.com/alexedwards/argon2id" + "time" ) // Paste represents a saved paste @@ -10,6 +12,8 @@ type Paste struct { Content string `json:"content" bson:"content"` SuggestedSyntaxType string `json:"suggestedSyntaxType" bson:"suggestedSyntaxType"` DeletionToken string `json:"deletionToken" bson:"deletionToken"` + Created int64 `json:"created" bson:"created"` + AutoDelete bool `json:"autoDelete" bson:"autoDelete"` } // Create creates a new paste object using the given content @@ -29,6 +33,8 @@ func Create(id, content string) (*Paste, error) { Content: content, SuggestedSyntaxType: suggestedSyntaxType, DeletionToken: deletionToken, + Created: time.Now().Unix(), + AutoDelete: env.Bool("AUTODELETE", false), }, nil } diff --git a/internal/storage/driver.go b/internal/storage/driver.go index 21bac74..a5ab35c 100644 --- a/internal/storage/driver.go +++ b/internal/storage/driver.go @@ -18,6 +18,7 @@ type Driver interface { Get(id string) (*pastes.Paste, error) Save(paste *pastes.Paste) error Delete(id string) error + Cleanup() (int, error) } // Load loads the current storage driver diff --git a/internal/storage/file_driver.go b/internal/storage/file_driver.go index ca5f01a..ca32486 100644 --- a/internal/storage/file_driver.go +++ b/internal/storage/file_driver.go @@ -9,6 +9,7 @@ import ( "os" "path/filepath" "strings" + "time" ) // FileDriver represents the file storage driver @@ -104,3 +105,35 @@ func (driver *FileDriver) Delete(id string) error { id = base64.StdEncoding.EncodeToString([]byte(id)) return os.Remove(filepath.Join(driver.filePath, id+".json")) } + +// Cleanup cleans up the expired pastes +func (driver *FileDriver) Cleanup() (int, error) { + // Retrieve all paste IDs + ids, err := driver.ListIDs() + if err != nil { + return 0, err + } + + // Define the amount of deleted items + deleted := 0 + + // Loop through all pastes + for _, id := range ids { + // Retrieve the paste object + paste, err := driver.Get(id) + if err != nil { + return 0, err + } + + // Delete the paste if it is expired + lifetime := env.Duration("AUTODELETE_LIFETIME", 30*24*time.Hour) + if paste.AutoDelete && paste.Created+int64(lifetime.Seconds()) < time.Now().Unix() { + err = driver.Delete(id) + if err != nil { + return 0, err + } + deleted++ + } + } + return deleted, nil +} diff --git a/internal/storage/mongodb_driver.go b/internal/storage/mongodb_driver.go index 7aaa36b..cb4c225 100644 --- a/internal/storage/mongodb_driver.go +++ b/internal/storage/mongodb_driver.go @@ -101,7 +101,10 @@ func (driver *MongoDBDriver) Get(id string) (*pastes.Paste, error) { // Return the retrieved paste object paste := new(pastes.Paste) err = result.Decode(paste) - return paste, err + if err != nil { + return nil, err + } + return paste, nil } // Save saves a paste @@ -132,3 +135,35 @@ func (driver *MongoDBDriver) Delete(id string) error { _, err := collection.DeleteOne(ctx, filter) return err } + +// Cleanup cleans up the expired pastes +func (driver *MongoDBDriver) Cleanup() (int, error) { + // Retrieve all paste IDs + ids, err := driver.ListIDs() + if err != nil { + return 0, err + } + + // Define the amount of deleted items + deleted := 0 + + // Loop through all pastes + for _, id := range ids { + // Retrieve the paste object + paste, err := driver.Get(id) + if err != nil { + return 0, err + } + + // Delete the paste if it is expired + lifetime := env.Duration("AUTODELETE_LIFETIME", 30*24*time.Hour) + if paste.AutoDelete && paste.Created+int64(lifetime.Seconds()) < time.Now().Unix() { + err = driver.Delete(id) + if err != nil { + return 0, err + } + deleted++ + } + } + return deleted, nil +} diff --git a/internal/storage/s3_driver.go b/internal/storage/s3_driver.go index 80143a9..b0a82f8 100644 --- a/internal/storage/s3_driver.go +++ b/internal/storage/s3_driver.go @@ -10,6 +10,7 @@ import ( "github.com/minio/minio-go/v7/pkg/credentials" "io/ioutil" "strings" + "time" ) // S3Driver represents the AWS S3 storage driver @@ -100,3 +101,35 @@ func (driver *S3Driver) Save(paste *pastes.Paste) error { func (driver *S3Driver) Delete(id string) error { return driver.client.RemoveObject(context.Background(), driver.bucket, id+".json", minio.RemoveObjectOptions{}) } + +// Cleanup cleans up the expired pastes +func (driver *S3Driver) Cleanup() (int, error) { + // Retrieve all paste IDs + ids, err := driver.ListIDs() + if err != nil { + return 0, err + } + + // Define the amount of deleted items + deleted := 0 + + // Loop through all pastes + for _, id := range ids { + // Retrieve the paste object + paste, err := driver.Get(id) + if err != nil { + return 0, err + } + + // Delete the paste if it is expired + lifetime := env.Duration("AUTODELETE_LIFETIME", 30*24*time.Hour) + if paste.AutoDelete && paste.Created+int64(lifetime.Seconds()) < time.Now().Unix() { + err = driver.Delete(id) + if err != nil { + return 0, err + } + deleted++ + } + } + return deleted, nil +} diff --git a/internal/storage/sql_driver.go b/internal/storage/sql_driver.go index d57c466..ba3965d 100644 --- a/internal/storage/sql_driver.go +++ b/internal/storage/sql_driver.go @@ -7,6 +7,7 @@ import ( _ "github.com/go-sql-driver/mysql" _ "github.com/lib/pq" _ "github.com/mattn/go-sqlite3" + "time" ) // SQLDriver represents the SQL storage driver @@ -36,7 +37,9 @@ func (driver *SQLDriver) Initialize() error { id varchar NOT NULL PRIMARY KEY, content varchar NOT NULL, suggestedSyntaxType varchar NOT NULL, - deletionToken varchar NOT NULL + deletionToken varchar NOT NULL, + created bigint NOT NULL, + autoDelete bool NOT NULL ); `, table) if err != nil { @@ -96,7 +99,7 @@ func (driver *SQLDriver) Get(id string) (*pastes.Paste, error) { // Save saves a paste func (driver *SQLDriver) Save(paste *pastes.Paste) error { // Execute an INSERT statement to create the paste - _, err := driver.database.Exec("INSERT INTO ? (?, ?, ?, ?)", driver.table, paste.ID, paste.Content, paste.SuggestedSyntaxType, paste.DeletionToken) + _, err := driver.database.Exec("INSERT INTO ? (?, ?, ?, ?, ?, ?)", driver.table, paste.ID, paste.Content, paste.SuggestedSyntaxType, paste.DeletionToken, paste.Created, paste.AutoDelete) return err } @@ -106,3 +109,35 @@ func (driver *SQLDriver) Delete(id string) error { _, err := driver.database.Exec("DELETE FROM ? WHERE id = ?", driver.table, id) return err } + +// Cleanup cleans up the expired pastes +func (driver *SQLDriver) Cleanup() (int, error) { + // Retrieve all paste IDs + ids, err := driver.ListIDs() + if err != nil { + return 0, err + } + + // Define the amount of deleted items + deleted := 0 + + // Loop through all pastes + for _, id := range ids { + // Retrieve the paste object + paste, err := driver.Get(id) + if err != nil { + return 0, err + } + + // Delete the paste if it is expired + lifetime := env.Duration("AUTODELETE_LIFETIME", 30*24*time.Hour) + if paste.AutoDelete && paste.Created+int64(lifetime.Seconds()) < time.Now().Unix() { + err = driver.Delete(id) + if err != nil { + return 0, err + } + deleted++ + } + } + return deleted, nil +}