Many small improvements

README.md

@@ -28,14 +28,35 @@ This is only for those people, that are still living in the 90s or are not comfo
 
 For further instructions like creating a service that can easily be started with `service aqua start`, please refer to other pages (there are a bunch that explain this) - I won't.
 
-## Docker Compose
+## Docker (Compose)
 
-Before following the steps, make sure you have [Docker](https://docs.docker.com/get-docker/) and [Docker Compose](https://docs.docker.com/compose/install/) installed on the machine.
+Before following the steps, make sure you have [Docker](https://docs.docker.com/get-docker/) installed. And if you want to use the preconfigured `docker-compose` file, install [Docker Compose](https://docs.docker.com/compose/install/) as well.
 
-1. Clone the repository to a directory of your liking with `git clone git@github.com:Superioz/aqua.git`
+Before you can start anything, you need the basis configuration:
-2. Edit the `auth.yml` with your custom auth tokens and settings.
+
-3. Rename the `.env.dist` to `.env` and edit it as well.
+1. Download and edit the `auth.yml` with your custom auth tokens and settings.
-4. `docker-compose up` and you should see it up and running.
+2. Download and rename the `.env.dist` to `.env` and edit it as well.
+
+For a quick start with Docker, you can use the following command:
+
+```sh
+docker run --rm                    \
+  --env-file ./.env                \
+  -v ./auth.yml:/etc/aqua/auth.yml \
+  -v ./files:/var/lib/aqua/        \
+  -p 8765:8765                     \
+  ghcr.io/superioz/aqua:latest
+```
+
+If you are on Windows, you might need to add `MSYS_NO_PATHCONV=1` so that the paths are correctly parsed. Also, to refer to the current directory, use `"$(pwd)"` instead of `./`, because that sometimes makes problems in Windows as well.
+
+For Docker Compose it's easier:
+
+```sh
+docker-compose up
+```
+
+If you want to build the image yourself instead, you can of course `git clone git@github.com:Superioz/aqua.git` and then execute `docker-compose up --build`.
 
 ## Kubernetes
 

internal/aqcli/aqcli.go

@@ -9,12 +9,50 @@ import (
 	"github.com/urfave/cli/v2"
 	"io"
 	"io/ioutil"
+	"math/rand"
 	"net/http"
 	"os"
 	"strings"
 	"time"
 )
 
+var GenerateCommand = &cli.Command{
+	Name:    "generate",
+	Aliases: []string{"gen"},
+	Usage:   "Generates a possible auth token",
+	Flags: []cli.Flag{
+		&cli.IntFlag{
+			Name:    "length",
+			Aliases: []string{"l"},
+			Value:   32,
+			Usage:   "Length of the token",
+		},
+	},
+	Action: func(c *cli.Context) error {
+		size := c.Int("length")
+		if size <= 1 {
+			return cli.Exit("You cannot generate a token with this length. Must be >=2.", 1)
+		}
+
+		fmt.Println(generateToken(size))
+		return nil
+	},
+}
+
+const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-+?!#$&%"
+
+// generateToken generates a random `size` long string from
+// a predefined hexadecimal charset.
+func generateToken(size int) string {
+	rand.Seed(time.Now().UnixNano())
+
+	b := make([]byte, size)
+	for i := range b {
+		b[i] = charset[rand.Intn(len(charset))]
+	}
+	return string(b)
+}
+
 var UploadCommand = &cli.Command{
 	Name:      "upload",
 	Usage:     "Uploads a file to the aqua server",

internal/aqcli/generate.go

@@ -1,45 +0,0 @@
-package aqcli
-
-import (
-	"fmt"
-	"github.com/urfave/cli/v2"
-	"math/rand"
-	"time"
-)
-
-var GenerateCommand = &cli.Command{
-	Name:    "generate",
-	Aliases: []string{"gen"},
-	Usage:   "Generates a possible auth token",
-	Flags: []cli.Flag{
-		&cli.IntFlag{
-			Name:    "length",
-			Aliases: []string{"l"},
-			Value:   32,
-			Usage:   "Length of the token",
-		},
-	},
-	Action: func(c *cli.Context) error {
-		size := c.Int("length")
-		if size <= 1 {
-			return cli.Exit("You cannot generate a token with this length. Must be >=2.", 1)
-		}
-
-		fmt.Println(generateToken(size))
-		return nil
-	},
-}
-
-const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-+?!#$&%"
-
-// generateToken generates a random `size` long string from
-// a predefined hexadecimal charset.
-func generateToken(size int) string {
-	rand.Seed(time.Now().UnixNano())
-
-	b := make([]byte, size)
-	for i := range b {
-		b[i] = charset[rand.Intn(len(charset))]
-	}
-	return string(b)
-}

internal/handler/handler.go

@@ -6,6 +6,7 @@ import (
 	"github.com/gin-gonic/gin"
 	"github.com/superioz/aqua/internal/config"
 	"github.com/superioz/aqua/internal/metrics"
+	"github.com/superioz/aqua/internal/mime"
 	"github.com/superioz/aqua/internal/storage"
 	"github.com/superioz/aqua/pkg/env"
 	"k8s.io/klog"
@@ -20,33 +21,13 @@ const (
 )
 
 var (
-	// validMimeTypes is a whitelist of all supported
-	// mime types. Taken from https://developer.mozilla.org/
-	validMimeTypes = []string{
-		"application/pdf",
-		"application/json",
-		"application/gzip",
-		"application/vnd.rar",
-		"application/zip",
-		"application/x-7z-compressed",
-		"image/png",
-		"image/jpeg",
-		"image/gif",
-		"image/svg+xml",
-		"text/csv",
-		"text/plain",
-		"audio/mpeg",
-		"audio/ogg",
-		"audio/opus",
-		"audio/webm",
-		"video/mp4",
-		"video/mpeg",
-		"video/webm",
-	}
-
 	emptyRequestMetadata = &RequestMetadata{Expiration: storage.ExpireNever}
 )
 
+// TODO when accessing: `/N2YwODUx.mp4` => `/N2YwODUx`
+
+// RequestFormFile is the metadata we get from the file
+// which is requested to be uploaded.
 type RequestFormFile struct {
 	File          multipart.File
 	ContentType   string
@@ -116,13 +97,13 @@ func (h *UploadHandler) Upload(c *gin.Context) {
 		c.Status(http.StatusLengthRequired)
 		return
 	}
-	if c.Request.ContentLength > 50*SizeMegaByte {
+	if c.Request.ContentLength > int64(env.IntOrDefault("FILE_MAX_SIZE", 100))*SizeMegaByte {
 		c.JSON(http.StatusRequestEntityTooLarge, gin.H{"msg": "content size must not exceed 50mb"})
 		return
 	}
 
 	ct := getContentType(file)
-	if !isContentTypeValid(ct) {
+	if !mime.IsValid(ct) {
 		c.JSON(http.StatusBadRequest, gin.H{"msg": "content type of file is not valid"})
 		return
 	}
@@ -144,7 +125,13 @@ func (h *UploadHandler) Upload(c *gin.Context) {
 	defer of.Close()
 
 	metadata := getMetadata(form)
-	sf, err := h.FileStorage.StoreFile(of, metadata.Expiration)
+	rff := &RequestFormFile{
+		File:          of,
+		ContentType:   ct,
+		ContentLength: c.Request.ContentLength,
+	}
+
+	sf, err := h.FileStorage.StoreFile(rff, metadata.Expiration)
 	if err != nil {
 		klog.Error(err)
 		c.JSON(http.StatusInternalServerError, gin.H{"msg": "could not store file"})
@@ -188,15 +175,6 @@ func getContentType(f *multipart.FileHeader) string {
 	return c
 }
 
-func isContentTypeValid(ct string) bool {
-	for _, mt := range validMimeTypes {
-		if mt == ct {
-			return true
-		}
-	}
-	return false
-}
-
 func getToken(c *gin.Context) string {
 	// try to get the Bearer token, because it's the standard
 	// for authorization

internal/mime/mime.go

@@ -0,0 +1,46 @@
+package mime
+
+var (
+	// Types is a whitelist of all supported
+	// mime types. Taken from https://developer.mozilla.org/
+	Types = map[string]string{
+		"application/pdf":             "pdf",
+		"application/json":            "json",
+		"application/gzip":            "gz",
+		"application/vnd.rar":         "rar",
+		"application/zip":             "zip",
+		"application/x-7z-compressed": "7z",
+		"image/png":                   "png",
+		"image/jpeg":                  "jpg",
+		"image/gif":                   "gif",
+		"image/svg+xml":               "svg",
+		"text/csv":                    "csv",
+		"text/plain":                  "txt",
+		"audio/mpeg":                  "mp3",
+		"audio/ogg":                   "ogg",
+		"audio/opus":                  "opus",
+		"audio/webm":                  "weba",
+		"video/mp4":                   "mp4",
+		"video/mpeg":                  "mpeg",
+		"video/webm":                  "webm",
+	}
+)
+
+// IsValid checks if given type is inside Types map
+func IsValid(t string) bool {
+	for _, mt := range Types {
+		if mt == t {
+			return true
+		}
+	}
+	return false
+}
+
+// GetExtension returns an extension for the given MIME type
+func GetExtension(t string) string {
+	ext, ok := Types[t]
+	if !ok {
+		return "application/octet-stream"
+	}
+	return ext
+}

internal/storage/filemeta.go

@@ -48,7 +48,9 @@ func (s *SqliteFileMetaDatabase) Connect() error {
 	_, err = db.Exec(`create table if not exists files (
 		id text not null primary key, 
 		uploaded_at integer,
-		expires_at integer
+		expires_at integer,
+		mime_type varchar,
+		size integer
 	);`)
 	if err != nil {
 		return err
@@ -63,13 +65,13 @@ func (s *SqliteFileMetaDatabase) WriteFile(sf *StoredFile) error {
 	}
 	defer db.Close()
 
-	stmt, err := db.Prepare(`insert into files(id, uploaded_at, expires_at) values(?, ?, ?)`)
+	stmt, err := db.Prepare(`insert into files(id, uploaded_at, expires_at, mime_type, size) values(?, ?, ?, ?, ?)`)
 	if err != nil {
 		return err
 	}
 	defer stmt.Close()
 
-	_, err = stmt.Exec(sf.Id, sf.UploadedAt, sf.ExpiresAt)
+	_, err = stmt.Exec(sf.Id, sf.UploadedAt, sf.ExpiresAt, sf.MimeType, sf.Size)
 	if err != nil {
 		return err
 	}
@@ -116,18 +118,10 @@ func (s *SqliteFileMetaDatabase) GetFile(id string) (*StoredFile, error) {
 		return nil, nil
 	}
 
-	var uploadedAt int
+	sf, err := getFromRows(rows)
-	var expiresAt int
-
-	err = rows.Scan(&id, &uploadedAt, &expiresAt)
 	if err != nil {
 		return nil, err
 	}
-	sf := &StoredFile{
-		Id:         id,
-		UploadedAt: int64(uploadedAt),
-		ExpiresAt:  int64(expiresAt),
-	}
 	return sf, nil
 }
 
@@ -146,19 +140,11 @@ func (s *SqliteFileMetaDatabase) GetAllFiles() ([]*StoredFile, error) {
 
 	var sfs []*StoredFile
 	for rows.Next() {
-		var id string
+		sf, err := getFromRows(rows)
-		var uploadedAt int
-		var expiresAt int
-
-		err = rows.Scan(&id, &uploadedAt, &expiresAt)
 		if err != nil {
 			return nil, err
 		}
-		sf := &StoredFile{
+
-			Id:         id,
-			UploadedAt: int64(uploadedAt),
-			ExpiresAt:  int64(expiresAt),
-		}
 		sfs = append(sfs, sf)
 	}
 	err = rows.Err()
@@ -190,19 +176,11 @@ func (s *SqliteFileMetaDatabase) GetAllExpired() ([]*StoredFile, error) {
 
 	var sfs []*StoredFile
 	for rows.Next() {
-		var id string
+		sf, err := getFromRows(rows)
-		var uploadedAt int
-		var expiresAt int
-
-		err = rows.Scan(&id, &uploadedAt, &expiresAt)
 		if err != nil {
 			return nil, err
 		}
-		sf := &StoredFile{
+
-			Id:         id,
-			UploadedAt: int64(uploadedAt),
-			ExpiresAt:  int64(expiresAt),
-		}
 		sfs = append(sfs, sf)
 	}
 	err = rows.Err()
@@ -211,3 +189,24 @@ func (s *SqliteFileMetaDatabase) GetAllExpired() ([]*StoredFile, error) {
 	}
 	return sfs, nil
 }
+
+func getFromRows(rows *sql.Rows) (*StoredFile, error) {
+	var id string
+	var uploadedAt int
+	var expiresAt int
+	var mimeType string
+	var size int
+
+	err := rows.Scan(&id, &uploadedAt, &expiresAt, &mimeType, &size)
+	if err != nil {
+		return nil, err
+	}
+	sf := &StoredFile{
+		Id:         id,
+		UploadedAt: int64(uploadedAt),
+		ExpiresAt:  int64(expiresAt),
+		MimeType:   mimeType,
+		Size:       int64(size),
+	}
+	return sf, nil
+}

internal/storage/filesys.go

@@ -2,12 +2,15 @@ package storage
 
 import (
 	"io"
-	"mime/multipart"
 	"os"
 )
 
 type FileSystem interface {
-	CreateFile(mf multipart.File, name string) (bool, error)
+	// CreateFile writes the content of given reader to a file
+	// with given name.
+	// Always returns if the file was partly written to disk.
+	CreateFile(r io.Reader, name string) (bool, error)
+	
 	DeleteFile(id string) error
 	GetFile(id string) (*os.File, error)
 	Exists(id string) (bool, error)
@@ -17,7 +20,7 @@ type LocalFileSystem struct {
 	FolderPath string
 }
 
-func (l LocalFileSystem) CreateFile(mf multipart.File, name string) (bool, error) {
+func (l LocalFileSystem) CreateFile(r io.Reader, name string) (bool, error) {
 	err := os.MkdirAll(l.FolderPath, os.ModePerm)
 	if err != nil {
 		return false, err
@@ -31,7 +34,7 @@ func (l LocalFileSystem) CreateFile(mf multipart.File, name string) (bool, error
 
 	// use io.Copy so that we don't have to load all the image into the memory.
 	// they get copied in smaller 32kb chunks.
-	_, err = io.Copy(f, mf)
+	_, err = io.Copy(f, r)
 	if err != nil {
 		return true, err
 	}

internal/storage/storage.go

@@ -5,10 +5,10 @@ import (
 	"errors"
 	"fmt"
 	"github.com/google/uuid"
+	"github.com/superioz/aqua/internal/handler"
 	"github.com/superioz/aqua/internal/metrics"
 	"github.com/superioz/aqua/pkg/env"
 	"k8s.io/klog"
-	"mime/multipart"
 	"strings"
 	"time"
 
@@ -26,6 +26,8 @@ type StoredFile struct {
 	Id         string
 	UploadedAt int64
 	ExpiresAt  int64
+	MimeType   string
+	Size       int64
 }
 
 func (sf *StoredFile) String() string {
@@ -100,13 +102,13 @@ func (fs *FileStorage) Cleanup() error {
 	return nil
 }
 
-func (fs *FileStorage) StoreFile(of multipart.File, expiration int64) (*StoredFile, error) {
+func (fs *FileStorage) StoreFile(rff *handler.RequestFormFile, expiration int64) (*StoredFile, error) {
 	name, err := getRandomFileName(env.IntOrDefault("FILE_NAME_LENGTH", 8))
 	if err != nil {
 		return nil, errors.New("could not generate random name")
 	}
 
-	_, err = fs.fileSystem.CreateFile(of, name)
+	_, err = fs.fileSystem.CreateFile(rff.File, name)
 	if err != nil {
 		klog.Error(err)
 		return nil, errors.New("could not save file to system")
@@ -122,6 +124,8 @@ func (fs *FileStorage) StoreFile(of multipart.File, expiration int64) (*StoredFi
 		Id:         name,
 		UploadedAt: currentTime,
 		ExpiresAt:  expAt,
+		MimeType:   rff.ContentType,
+		Size:       rff.ContentLength,
 	}
 
 	// write to meta database

run.sh

@@ -15,4 +15,4 @@ fi
 # we only need MSYS_NO_PATHCONV when running this script in Windows
 # because otherwise Mingw will try to convert
 # the given paths and that breaks everything.
-MSYS_NO_PATHCONV=1 docker run -it --rm -p 8765:8765 $IMAGE
+MSYS_NO_PATHCONV=1 docker run -it --rm -p 8765:8765 $IMAGE