mirror of https://github.com/Superioz/aqua.git
Add automatic expired file cleanup
This commit is contained in:
Tobias B
parent
9f2b8b3e4d
commit
25160e21f6
|
|
@ -1,4 +1,5 @@
|
|||
AUTH_CONFIG_PATH=
|
||||
FILE_STORAGE_PATH=
|
||||
FILE_NAME_LENGTH=
|
||||
FILE_META_DB_FILE=
|
||||
FILE_META_DB_FILE=
|
||||
FILE_EXPIRATION_CYCLE=
|
||||
|
|
@ -2,16 +2,15 @@ package main
|
|||
|
||||
import (
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/go-co-op/gocron"
|
||||
"github.com/joho/godotenv"
|
||||
"github.com/superioz/aqua/internal/handler"
|
||||
"github.com/superioz/aqua/internal/middleware"
|
||||
"github.com/superioz/aqua/pkg/env"
|
||||
"k8s.io/klog"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TODO when uploading, if expiration given, change expiration
|
||||
// TODO Schedule cleanup process for every x minutes and on startup
|
||||
|
||||
func main() {
|
||||
err := godotenv.Load()
|
||||
if err != nil {
|
||||
|
|
@ -28,14 +27,19 @@ func main() {
|
|||
// handler for receiving uploaded files
|
||||
uh := handler.NewUploadHandler()
|
||||
r.POST("/upload", uh.Upload)
|
||||
err = uh.FileStorage.Cleanup()
|
||||
if err != nil {
|
||||
klog.Errorln(err)
|
||||
}
|
||||
|
||||
// scheduler to do the cleanup every x minutes
|
||||
s := gocron.NewScheduler(time.UTC)
|
||||
s.Every(env.IntOrDefault("FILE_EXPIRATION_CYCLE", 15)).Minutes().StartImmediately().Do(func() {
|
||||
err = uh.FileStorage.Cleanup()
|
||||
if err != nil {
|
||||
klog.Errorln(err)
|
||||
}
|
||||
})
|
||||
s.StartAsync()
|
||||
|
||||
r.GET("/healthz", func(c *gin.Context) {
|
||||
c.JSON(200, gin.H{"status": "UP"})
|
||||
})
|
||||
|
||||
_ = r.Run(":8765")
|
||||
}
|
||||
|
|
|
|||
3
go.mod
3
go.mod
|
|
@ -9,6 +9,7 @@ require (
|
|||
|
||||
require (
|
||||
github.com/gin-contrib/sse v0.1.0 // indirect
|
||||
github.com/go-co-op/gocron v1.9.0 // indirect
|
||||
github.com/go-playground/locales v0.13.0 // indirect
|
||||
github.com/go-playground/universal-translator v0.17.0 // indirect
|
||||
github.com/go-playground/validator/v10 v10.4.1 // indirect
|
||||
|
|
@ -21,8 +22,10 @@ require (
|
|||
github.com/mattn/go-sqlite3 v1.14.9 // indirect
|
||||
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect
|
||||
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 // indirect
|
||||
github.com/robfig/cron/v3 v3.0.1 // indirect
|
||||
github.com/ugorji/go/codec v1.1.7 // indirect
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 // indirect
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
|
||||
golang.org/x/sys v0.0.0-20200116001909-b77594299b42 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
)
|
||||
|
|
|
|||
9
go.sum
9
go.sum
|
|
@ -5,6 +5,8 @@ github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE
|
|||
github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
|
||||
github.com/gin-gonic/gin v1.7.4 h1:QmUZXrvJ9qZ3GfWvQ+2wnW/1ePrTEJqPKMYEU3lD/DM=
|
||||
github.com/gin-gonic/gin v1.7.4/go.mod h1:jD2toBW3GZUr5UMcdrwQA10I7RuaFOl/SGeDjXkfUtY=
|
||||
github.com/go-co-op/gocron v1.9.0 h1:+V+DDenw3ryB7B+tK1bAIC5p0ruw4oX9IqAsdRnGIf0=
|
||||
github.com/go-co-op/gocron v1.9.0/go.mod h1:DbJm9kdgr1sEvWpHCA7dFFs/PGHPMil9/97EXCRPr4k=
|
||||
github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
|
||||
github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A=
|
||||
github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
|
||||
|
|
@ -35,10 +37,13 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLD
|
|||
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
|
||||
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
|
||||
github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
|
||||
github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs=
|
||||
|
|
@ -47,6 +52,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
|
|||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200116001909-b77594299b42 h1:vEOn+mP2zCOVzKckCZy6YsCtDblrpj/w7B9nxGNELpg=
|
||||
|
|
@ -61,5 +68,7 @@ gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
|
|||
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
|
||||
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
|
||||
k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
package handler
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/superioz/aqua/internal/config"
|
||||
"github.com/superioz/aqua/internal/storage"
|
||||
|
|
@ -24,8 +25,14 @@ var (
|
|||
"text/csv",
|
||||
"text/plain",
|
||||
}
|
||||
|
||||
emptyRequestMetadata = &RequestMetadata{Expiration: storage.ExpireNever}
|
||||
)
|
||||
|
||||
type RequestMetadata struct {
|
||||
Expiration int64 `json:"expiration"`
|
||||
}
|
||||
|
||||
type UploadHandler struct {
|
||||
AuthConfig *config.AuthConfig
|
||||
FileStorage *storage.FileStorage
|
||||
|
|
@ -103,7 +110,8 @@ func (u *UploadHandler) Upload(c *gin.Context) {
|
|||
}
|
||||
defer of.Close()
|
||||
|
||||
sf, err := u.FileStorage.StoreFile(of, storage.ExpireNever)
|
||||
metadata := getMetadata(form)
|
||||
sf, err := u.FileStorage.StoreFile(of, metadata.Expiration)
|
||||
if err != nil {
|
||||
klog.Error(err)
|
||||
c.JSON(http.StatusInternalServerError, gin.H{"msg": "could not store file"})
|
||||
|
|
@ -112,6 +120,21 @@ func (u *UploadHandler) Upload(c *gin.Context) {
|
|||
c.JSON(http.StatusOK, gin.H{"id": sf.Id})
|
||||
}
|
||||
|
||||
func getMetadata(form *multipart.Form) *RequestMetadata {
|
||||
metaRawList := form.Value["metadata"]
|
||||
if len(metaRawList) == 0 {
|
||||
return emptyRequestMetadata
|
||||
}
|
||||
metaRaw := metaRawList[0]
|
||||
|
||||
var metadata *RequestMetadata
|
||||
err := json.Unmarshal([]byte(metaRaw), &metadata)
|
||||
if err != nil {
|
||||
return emptyRequestMetadata
|
||||
}
|
||||
return metadata
|
||||
}
|
||||
|
||||
// workaround for file Content-Type headers
|
||||
// which contain multiple values such as "...; charset=utf-8"
|
||||
func getContentType(f *multipart.FileHeader) string {
|
||||
|
|
|
|||
|
|
@ -66,7 +66,8 @@ func (fs *FileStorage) Cleanup() error {
|
|||
continue
|
||||
}
|
||||
|
||||
klog.Infof("Delete file: %s", file)
|
||||
klog.Infof("Delete file %s (expired at %s)", file.Id, time.Unix(file.ExpiresAt, 0).String())
|
||||
|
||||
// delete this file
|
||||
err = fs.fileSystem.DeleteFile(file.Id)
|
||||
if err != nil {
|
||||
|
|
@ -93,15 +94,15 @@ func (fs *FileStorage) StoreFile(of multipart.File, expiration int64) (*StoredFi
|
|||
return nil, errors.New("could not save file to system")
|
||||
}
|
||||
|
||||
t := time.Now()
|
||||
expAt := t.Add(time.Duration(expiration)).Unix()
|
||||
currentTime := time.Now().Unix()
|
||||
expAt := currentTime + expiration
|
||||
if expiration == ExpireNever {
|
||||
expAt = ExpireNever
|
||||
}
|
||||
|
||||
sf := &StoredFile{
|
||||
Id: name,
|
||||
UploadedAt: t.Unix(),
|
||||
UploadedAt: currentTime,
|
||||
ExpiresAt: expAt,
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue