Add metadata sqlite backend

auth.yml

@@ -1,2 +1,5 @@
 validTokens:
-- token: 71a4c056ab9b0fb965063344cd6616bc
+- token: 71a4c056ab9b0fb965063344cd6616bc
+  fileTypes:
+  - image/png
+  - image/jpeg

cmd/aqua/main.go

@@ -9,6 +9,8 @@ import (
 	"time"
 )
 
+// TODO Add cleanup process, to delete all images that are not in the sqlite or that are expired
+
 func main() {
 	err := godotenv.Load()
 	if err != nil {

go.mod

@@ -18,6 +18,7 @@ require (
 	github.com/json-iterator/go v1.1.9 // indirect
 	github.com/leodido/go-urn v1.2.0 // indirect
 	github.com/mattn/go-isatty v0.0.12 // indirect
+	github.com/mattn/go-sqlite3 v1.14.9 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect
 	github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 // indirect
 	github.com/ugorji/go/codec v1.1.7 // indirect

go.sum

@@ -27,6 +27,8 @@ github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
 github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
 github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
+github.com/mattn/go-sqlite3 v1.14.9 h1:10HX2Td0ocZpYEjhilsuo6WWtUqttj2Kb0KtD86/KYA=
+github.com/mattn/go-sqlite3 v1.14.9/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg=

internal/config/config.go

@@ -12,6 +12,10 @@ type AuthConfig struct {
 
 type TokenConfig struct {
 	Token string
+
+	// All file types that one can upload via this token.
+	// If empty, all file types are allowed.
+	ValidFileTypes []string `yaml:"fileTypes"`
 }
 
 func NewEmptyAuthConfig() *AuthConfig {
@@ -51,3 +55,21 @@ func (ac *AuthConfig) HasToken(token string) bool {
 	}
 	return false
 }
+
+func (ac *AuthConfig) CanUpload(token string, filetype string) bool {
+	for _, validToken := range ac.ValidTokens {
+		if validToken.Token == token {
+			ft := validToken.ValidFileTypes
+			if len(ft) == 0 {
+				return true
+			}
+
+			for _, s := range ft {
+				if s == filetype {
+					return true
+				}
+			}
+		}
+	}
+	return false
+}

internal/handler/upload.go

@@ -2,14 +2,12 @@ package handler
 
 import (
 	"github.com/gin-gonic/gin"
-	"github.com/google/uuid"
 	"github.com/superioz/aqua/internal/config"
+	"github.com/superioz/aqua/internal/storage"
 	"github.com/superioz/aqua/pkg/env"
-	"io"
 	"k8s.io/klog"
 	"mime/multipart"
 	"net/http"
-	"os"
 	"strings"
 )
 
@@ -28,6 +26,7 @@ var (
 	}
 
 	authConfig *config.AuthConfig
+	metaDb     storage.FileMetaDatabase
 )
 
 func Initialize() {
@@ -43,6 +42,9 @@ func Initialize() {
 		klog.Infof("Loaded %d valid tokens", len(ac.ValidTokens))
 	}
 	authConfig = ac
+
+	metaDbFilePath := env.StringOrDefault("FILE_META_DB_FILE", "./files.db")
+	metaDb = storage.NewSqliteFileMetaDatabase(metaDbFilePath)
 }
 
 func Upload(c *gin.Context) {
@@ -85,6 +87,11 @@ func Upload(c *gin.Context) {
 		return
 	}
 
+	if !authConfig.CanUpload(token, ct) {
+		c.JSON(http.StatusForbidden, gin.H{"msg": "you can not upload a file with this content type"})
+		return
+	}
+
 	of, err := file.Open()
 	if err != nil {
 		klog.Error(err)
@@ -93,54 +100,17 @@ func Upload(c *gin.Context) {
 	}
 	defer of.Close()
 
-	name, err := getRandomFileName(8)
+	sf, err := storage.StoreFile(of, storage.ExpireNever)
 	if err != nil {
 		klog.Error(err)
-		c.JSON(http.StatusInternalServerError, gin.H{"msg": "could not generate id of file"})
+		c.JSON(http.StatusInternalServerError, gin.H{"msg": "could not store file"})
 		return
 	}
 
-	path := env.StringOrDefault("FILE_STORAGE_PATH", "/var/lib/aqua/")
+	// write to meta database
+	metaDb.WriteFile(sf)
 
-	err = os.MkdirAll(path, os.ModePerm)
-	if err != nil {
-		klog.Error(err)
-		c.JSON(http.StatusInternalServerError, gin.H{"msg": "could not create file directory"})
-		return
-	}
-
-	f, err := os.Create(name)
-	if err != nil {
-		klog.Error(err)
-		c.JSON(http.StatusInternalServerError, gin.H{"msg": "could not create file"})
-		return
-	}
-	defer f.Close()
-
-	// use io.Copy so that we don't have to load all the image into the memory.
-	// they get copied in smaller 32kb chunks.
-	_, err = io.Copy(f, of)
-	if err != nil {
-		klog.Error(err)
-		c.JSON(http.StatusInternalServerError, gin.H{"msg": "could not copy content to file"})
-		return
-	}
-
-	c.JSON(http.StatusOK, gin.H{"id": name})
-}
-
-func getRandomFileName(size int) (string, error) {
-	id, err := uuid.NewRandom()
-	if err != nil {
-		return "", err
-	}
-
-	// strip '-' from uuid
-	str := strings.ReplaceAll(id.String(), "-", "")
-	if size >= len(str) {
-		return str, nil
-	}
-	return str[:size], nil
+	c.JSON(http.StatusOK, gin.H{"id": sf.Id})
 }
 
 // workaround for file Content-Type headers

internal/storage/meta.go

@@ -0,0 +1,188 @@
+package storage
+
+import (
+	"database/sql"
+	"time"
+)
+
+// FileMetaDatabase is for storing additional meta information
+// on each file, e.g. the time a file has been uploaded
+// or more imporantly when the file should be expired.
+//
+// On startup, we check for every entry that is expired
+// and delete it accordingly.
+type FileMetaDatabase interface {
+	Connect() error
+	WriteFile(sf *StoredFile) error
+	GetFile(id string) (*StoredFile, error)
+	GetAllFiles() ([]*StoredFile, error)
+	GetAllExpired() ([]*StoredFile, error)
+	DeleteFile(id string) error
+}
+
+type SqliteFileMetaDatabase struct {
+	DbFilePath string
+}
+
+func NewSqliteFileMetaDatabase(filePath string) *SqliteFileMetaDatabase {
+	return &SqliteFileMetaDatabase{
+		DbFilePath: filePath,
+	}
+}
+
+func (s *SqliteFileMetaDatabase) Connect() error {
+	return nil
+}
+
+func (s *SqliteFileMetaDatabase) WriteFile(sf *StoredFile) error {
+	db, err := sql.Open("sqlite3", s.DbFilePath)
+	if err != nil {
+		return err
+	}
+	defer db.Close()
+
+	_, err = db.Exec(`create table if not exists files (
+		id text not null primary key, 
+		uploaded_at integer,
+		expires_at integer
+	);`)
+	if err != nil {
+		return err
+	}
+
+	stmt, err := db.Prepare(`insert into files(id, uploaded_at, expires_at) values(?, ?, ?)`)
+	if err != nil {
+		return err
+	}
+	defer stmt.Close()
+
+	_, err = stmt.Exec(sf.Id, sf.UploadedAt, sf.ExpiresAt)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (s *SqliteFileMetaDatabase) DeleteFile(id string) error {
+
+	return nil
+}
+
+func (s *SqliteFileMetaDatabase) GetFile(id string) (*StoredFile, error) {
+	db, err := sql.Open("sqlite3", s.DbFilePath)
+	if err != nil {
+		return nil, err
+	}
+	defer db.Close()
+
+	stmt, err := db.Prepare(`select * from files where id = ?`)
+	if err != nil {
+		return nil, err
+	}
+	defer stmt.Close()
+
+	rows, err := stmt.Query(id)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	if !rows.Next() {
+		return nil, nil
+	}
+
+	var uploadedAt int
+	var expiresAt int
+
+	err = rows.Scan(&id, &uploadedAt, &expiresAt)
+	if err != nil {
+		return nil, err
+	}
+	sf := &StoredFile{
+		Id:         id,
+		UploadedAt: int64(uploadedAt),
+		ExpiresAt:  int64(expiresAt),
+	}
+	return sf, nil
+}
+
+func (s *SqliteFileMetaDatabase) GetAllFiles() ([]*StoredFile, error) {
+	db, err := sql.Open("sqlite3", s.DbFilePath)
+	if err != nil {
+		return nil, err
+	}
+	defer db.Close()
+
+	rows, err := db.Query(`select * from files`)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	var sfs []*StoredFile
+	for rows.Next() {
+		var id string
+		var uploadedAt int
+		var expiresAt int
+
+		err = rows.Scan(&id, &uploadedAt, &expiresAt)
+		if err != nil {
+			return nil, err
+		}
+		sf := &StoredFile{
+			Id:         id,
+			UploadedAt: int64(uploadedAt),
+			ExpiresAt:  int64(expiresAt),
+		}
+		sfs = append(sfs, sf)
+	}
+	err = rows.Err()
+	if err != nil {
+		return nil, err
+	}
+	return sfs, nil
+}
+
+func (s *SqliteFileMetaDatabase) GetAllExpired() ([]*StoredFile, error) {
+	db, err := sql.Open("sqlite3", s.DbFilePath)
+	if err != nil {
+		return nil, err
+	}
+	defer db.Close()
+
+	stmt, err := db.Prepare(`select * from files where expired_at < ?`)
+	if err != nil {
+		return nil, err
+	}
+	defer stmt.Close()
+
+	now := time.Now().Unix()
+	rows, err := stmt.Query(now)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	var sfs []*StoredFile
+	for rows.Next() {
+		var id string
+		var uploadedAt int
+		var expiresAt int
+
+		err = rows.Scan(&id, &uploadedAt, &expiresAt)
+		if err != nil {
+			return nil, err
+		}
+		sf := &StoredFile{
+			Id:         id,
+			UploadedAt: int64(uploadedAt),
+			ExpiresAt:  int64(expiresAt),
+		}
+		sfs = append(sfs, sf)
+	}
+	err = rows.Err()
+	if err != nil {
+		return nil, err
+	}
+	return sfs, nil
+}

internal/storage/storage.go

@@ -0,0 +1,85 @@
+package storage
+
+import (
+	"errors"
+	"fmt"
+	"github.com/google/uuid"
+	"github.com/superioz/aqua/pkg/env"
+	"io"
+	"mime/multipart"
+	"os"
+	"strings"
+	"time"
+
+	_ "github.com/mattn/go-sqlite3"
+)
+
+const (
+	ExpireNever = -1
+)
+
+type StoredFile struct {
+	Id         string
+	UploadedAt int64
+	ExpiresAt  int64
+}
+
+func (sf *StoredFile) String() string {
+	return fmt.Sprintf("StoredFile<%s, %s>", sf.Id, time.Unix(sf.UploadedAt, 0).String())
+}
+
+func StoreFile(mf multipart.File, expiration int64) (*StoredFile, error) {
+	name, err := getRandomFileName(env.IntOrDefault("FILE_NAME_LENGTH", 8))
+	if err != nil {
+		return nil, err
+	}
+	path := env.StringOrDefault("FILE_STORAGE_PATH", "/var/lib/aqua/")
+
+	err = os.MkdirAll(path, os.ModePerm)
+	if err != nil {
+		return nil, err
+	}
+
+	f, err := os.Create(path + name)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	// use io.Copy so that we don't have to load all the image into the memory.
+	// they get copied in smaller 32kb chunks.
+	_, err = io.Copy(f, mf)
+	if err != nil {
+		return nil, err
+	}
+
+	t := time.Now()
+	sf := &StoredFile{
+		Id:         name,
+		UploadedAt: t.Unix(),
+		ExpiresAt:  t.Add(time.Duration(expiration)).Unix(),
+	}
+
+	return sf, nil
+}
+
+// getRandomFileName returns a random string with a fixed size
+// that is generated from an uuid.
+// It also checks, that no file with that name already exists,
+// if that is the case, it generates a new one.
+func getRandomFileName(size int) (string, error) {
+	if size <= 1 {
+		return "", errors.New("size must be greater than 1")
+	}
+	id, err := uuid.NewRandom()
+	if err != nil {
+		return "", err
+	}
+
+	// strip '-' from uuid
+	str := strings.ReplaceAll(id.String(), "-", "")
+	if size >= len(str) {
+		return str, nil
+	}
+	return str[:size], nil
+}