ShareX
/
aqua
mirror of https://github.com/Superioz/aqua.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add cleanup process for expired files

This commit is contained in:
Tobias B 2021-11-14 14:21:18 +01:00
parent 0b30282cb5
commit ae1a5b8a21
No known key found for this signature in database
GPG Key ID: 5EF4C92355A3B53D
5 changed files with 150 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
cmd/aqua/main.go
View File

@ -9,7 +9,8 @@ import (
"time"
)
// TODO Add cleanup process, to delete all images that are not in the sqlite or that are expired
// TODO when uploading, if expiration given, change expiration
// TODO Schedule cleanup process for every x minutes and on startup
func main() {
err := godotenv.Load()
@ -27,6 +28,10 @@ func main() {
// handler for receiving uploaded files
uh := handler.NewUploadHandler()
r.POST("/upload", uh.Upload)
err = uh.FileStorage.Cleanup()
if err != nil {
klog.Errorln(err)
}
r.GET("/healthz", func(c *gin.Context) {
c.JSON(200, gin.H{"status": "UP"})

64
internal/handler/upload.go
View File

@ -1,9 +1,7 @@
package handler
import (
"errors"
"github.com/gin-gonic/gin"
"github.com/google/uuid"
"github.com/superioz/aqua/internal/config"
"github.com/superioz/aqua/internal/storage"
"github.com/superioz/aqua/pkg/env"
@ -11,7 +9,6 @@ import (
"mime/multipart"
"net/http"
"strings"
"time"
)
const (
@ -30,9 +27,8 @@ var (
)
type UploadHandler struct {
authConfig *config.AuthConfig
fileMetaDb storage.FileMetaDatabase
fileStorage storage.FileStorage
AuthConfig *config.AuthConfig
FileStorage *storage.FileStorage
}
func NewUploadHandler() *UploadHandler {
@ -44,17 +40,13 @@ func NewUploadHandler() *UploadHandler {
// this is not good, but the system still works.
// nobody can upload a file though.
klog.Warningf("Could not open auth config at %s: %v", path, err)
handler.authConfig = config.NewEmptyAuthConfig()
handler.AuthConfig = config.NewEmptyAuthConfig()
} else {
klog.Infof("Loaded %d valid tokens", len(ac.ValidTokens))
handler.authConfig = ac
handler.AuthConfig = ac
}
metaDbFilePath := env.StringOrDefault("FILE_META_DB_FILE", "./files.db")
handler.fileMetaDb = storage.NewSqliteFileMetaDatabase(metaDbFilePath)
fileStoragePath := env.StringOrDefault("FILE_STORAGE_PATH", "/var/lib/aqua/")
handler.fileStorage = storage.NewLocalFileStorage(fileStoragePath)
handler.FileStorage = storage.NewFileStorage()
return handler
}
@ -64,7 +56,7 @@ func (u *UploadHandler) Upload(c *gin.Context) {
token := getToken(c)
klog.Infof("Checking authentication for token=%s", token)
if !u.authConfig.HasToken(token) {
if !u.AuthConfig.HasToken(token) {
c.JSON(http.StatusUnauthorized, gin.H{"msg": "the token is not valid"})
return
}
@ -98,7 +90,7 @@ func (u *UploadHandler) Upload(c *gin.Context) {
return
}
if !u.authConfig.CanUpload(token, ct) {
if !u.AuthConfig.CanUpload(token, ct) {
c.JSON(http.StatusForbidden, gin.H{"msg": "you can not upload a file with this content type"})
return
}
@ -111,29 +103,12 @@ func (u *UploadHandler) Upload(c *gin.Context) {
}
defer of.Close()
name, err := getRandomFileName(env.IntOrDefault("FILE_NAME_LENGTH", 8))
sf, err := u.FileStorage.StoreFile(of, storage.ExpireNever)
if err != nil {
klog.Error(err)
c.JSON(http.StatusInternalServerError, gin.H{"msg": "could not generate random name"})
return
c.JSON(http.StatusInternalServerError, gin.H{"msg": "could not store file"})
}
_, err = u.fileStorage.CreateFile(of, name)
if err != nil {
klog.Error(err)
c.JSON(http.StatusInternalServerError, gin.H{"msg": "could not save file"})
}
t := time.Now()
sf := &storage.StoredFile{
Id: name,
UploadedAt: t.Unix(),
ExpiresAt: t.Add(time.Duration(storage.ExpireNever)).Unix(),
}
// write to meta database
u.fileMetaDb.WriteFile(sf)
c.JSON(http.StatusOK, gin.H{"id": sf.Id})
}
@ -170,24 +145,3 @@ func getToken(c *gin.Context) string {
spl := strings.Split(bearerToken, "Bearer ")
return spl[1]
}
// getRandomFileName returns a random string with a fixed size
// that is generated from an uuid.
// It also checks, that no file with that name already exists,
// if that is the case, it generates a new one.
func getRandomFileName(size int) (string, error) {
if size <= 1 {
return "", errors.New("size must be greater than 1")
}
id, err := uuid.NewRandom()
if err != nil {
return "", err
}
// strip '-' from uuid
str := strings.ReplaceAll(id.String(), "-", "")
if size >= len(str) {
return str, nil
}
return str[:size], nil
}

20
internal/storage/files.go
View File

@ -6,17 +6,17 @@ import (
"os"
)
type FileStorage interface {
type FileSystem interface {
CreateFile(mf multipart.File, name string) (bool, error)
DeleteFile(id string) error
GetFile(id string) (*os.File, error)
}
type LocalFileStorage struct {
type LocalFileSystem struct {
FolderPath string
}
func (l LocalFileStorage) CreateFile(mf multipart.File, name string) (bool, error) {
func (l LocalFileSystem) CreateFile(mf multipart.File, name string) (bool, error) {
err := os.MkdirAll(l.FolderPath, os.ModePerm)
if err != nil {
return false, err
@ -37,14 +37,14 @@ func (l LocalFileStorage) CreateFile(mf multipart.File, name string) (bool, erro
return true, nil
}
func (l LocalFileStorage) DeleteFile(id string) error {
func (l LocalFileSystem) DeleteFile(id string) error {
return os.Remove(l.FolderPath + id)
}
func (l LocalFileSystem) GetFile(id string) (*os.File, error) {
panic("implement me")
}
func (l LocalFileStorage) GetFile(id string) (*os.File, error) {
panic("implement me")
}
func NewLocalFileStorage(path string) *LocalFileStorage {
return &LocalFileStorage{FolderPath: path}
func NewLocalFileStorage(path string) *LocalFileSystem {
return &LocalFileSystem{FolderPath: path}
}

16
internal/storage/meta.go
View File

@ -64,8 +64,20 @@ func (s *SqliteFileMetaDatabase) WriteFile(sf *StoredFile) error {
}
func (s *SqliteFileMetaDatabase) DeleteFile(id string) error {
db, err := sql.Open("sqlite3", s.DbFilePath)
if err != nil {
return err
}
defer db.Close()
return nil
stmt, err := db.Prepare(`delete from files where id = ?`)
if err != nil {
return err
}
defer stmt.Close()
_, err = stmt.Exec(id)
return err
}
func (s *SqliteFileMetaDatabase) GetFile(id string) (*StoredFile, error) {
@ -150,7 +162,7 @@ func (s *SqliteFileMetaDatabase) GetAllExpired() ([]*StoredFile, error) {
}
defer db.Close()
stmt, err := db.Prepare(`select * from files where expired_at < ?`)
stmt, err := db.Prepare(`select * from files where expires_at > 0 and expires_at <= ?`)
if err != nil {
return nil, err
}

111
internal/storage/storage.go
View File

@ -1,7 +1,13 @@
package storage
import (
"errors"
"fmt"
"github.com/google/uuid"
"github.com/superioz/aqua/pkg/env"
"k8s.io/klog"
"mime/multipart"
"strings"
"time"
_ "github.com/mattn/go-sqlite3"
@ -20,3 +26,108 @@ type StoredFile struct {
func (sf *StoredFile) String() string {
return fmt.Sprintf("StoredFile<%s, %s>", sf.Id, time.Unix(sf.UploadedAt, 0).String())
}
// FileStorage is the abstraction layer for storing uploaded files.
// It consists of a file system where the physical files are written to
// and a seperate database, where it stores metadata for each file.
type FileStorage struct {
fileMetaDb FileMetaDatabase
fileSystem FileSystem
}
func NewFileStorage() *FileStorage {
metaDbFilePath := env.StringOrDefault("FILE_META_DB_FILE", "./files.db")
fileMetaDb := NewSqliteFileMetaDatabase(metaDbFilePath)
fileStoragePath := env.StringOrDefault("FILE_STORAGE_PATH", "/var/lib/aqua/")
fileSystem := NewLocalFileStorage(fileStoragePath)
return &FileStorage{
fileMetaDb: fileMetaDb,
fileSystem: fileSystem,
}
}
// Cleanup uses the meta database to check for all files
// that have expired and deletes them accordingly.
func (fs *FileStorage) Cleanup() error {
klog.Infoln("Cleanup expired files")
expiredFiles, err := fs.fileMetaDb.GetAllExpired()
if err != nil {
return err
}
if len(expiredFiles) == 0 {
klog.Infoln("No expired files found.")
return nil
}
for _, file := range expiredFiles {
if file.ExpiresAt < 0 {
continue
}
klog.Infof("Delete file: %s", file)
// delete this file
err = fs.fileSystem.DeleteFile(file.Id)
if err != nil {
return errors.New(fmt.Sprintf("Could not delete file with id=%s: %v", file.Id, err))
}
err = fs.fileMetaDb.DeleteFile(file.Id)
if err != nil {
return errors.New(fmt.Sprintf("Could not delete file with id=%s: %v", file.Id, err))
}
}
return nil
}
func (fs *FileStorage) StoreFile(of multipart.File, expiration int64) (*StoredFile, error) {
name, err := getRandomFileName(env.IntOrDefault("FILE_NAME_LENGTH", 8))
if err != nil {
return nil, errors.New("could not generate random name")
}
_, err = fs.fileSystem.CreateFile(of, name)
if err != nil {
klog.Error(err)
return nil, errors.New("could not save file to system")
}
t := time.Now()
expAt := t.Add(time.Duration(expiration)).Unix()
if expiration == ExpireNever {
expAt = ExpireNever
}
sf := &StoredFile{
Id: name,
UploadedAt: t.Unix(),
ExpiresAt: expAt,
}
// write to meta database
fs.fileMetaDb.WriteFile(sf)
return sf, nil
}
// getRandomFileName returns a random string with a fixed size
// that is generated from an uuid.
// It also checks, that no file with that name already exists,
// if that is the case, it generates a new one.
func getRandomFileName(size int) (string, error) {
if size <= 1 {
return "", errors.New("size must be greater than 1")
}
id, err := uuid.NewRandom()
if err != nil {
return "", err
}
// strip '-' from uuid
str := strings.ReplaceAll(id.String(), "-", "")
if size >= len(str) {
return str, nil
}
return str[:size], nil
}