From 1a2dbf266de2739c6917fbe52185811efff98eb4 Mon Sep 17 00:00:00 2001 From: Tobias B Date: Thu, 18 Nov 2021 01:14:48 +0100 Subject: [PATCH] Many small improvements --- README.md | 33 +++++++++--- internal/aqcli/{upload.go => aqcli.go} | 38 ++++++++++++++ internal/aqcli/generate.go | 45 ---------------- internal/handler/handler.go | 50 +++++------------- internal/mime/mime.go | 46 +++++++++++++++++ internal/storage/{meta.go => filemeta.go} | 63 +++++++++++------------ internal/storage/{files.go => filesys.go} | 11 ++-- internal/storage/storage.go | 10 ++-- run.sh | 2 +- 9 files changed, 171 insertions(+), 127 deletions(-) rename internal/aqcli/{upload.go => aqcli.go} (73%) delete mode 100644 internal/aqcli/generate.go create mode 100644 internal/mime/mime.go rename internal/storage/{meta.go => filemeta.go} (85%) rename internal/storage/{files.go => filesys.go} (78%) diff --git a/README.md b/README.md index 33a5747..2fd8ba8 100644 --- a/README.md +++ b/README.md @@ -28,14 +28,35 @@ This is only for those people, that are still living in the 90s or are not comfo For further instructions like creating a service that can easily be started with `service aqua start`, please refer to other pages (there are a bunch that explain this) - I won't. -## Docker Compose +## Docker (Compose) -Before following the steps, make sure you have [Docker](https://docs.docker.com/get-docker/) and [Docker Compose](https://docs.docker.com/compose/install/) installed on the machine. +Before following the steps, make sure you have [Docker](https://docs.docker.com/get-docker/) installed. And if you want to use the preconfigured `docker-compose` file, install [Docker Compose](https://docs.docker.com/compose/install/) as well. -1. Clone the repository to a directory of your liking with `git clone git@github.com:Superioz/aqua.git` -2. Edit the `auth.yml` with your custom auth tokens and settings. -3. Rename the `.env.dist` to `.env` and edit it as well. -4. `docker-compose up` and you should see it up and running. +Before you can start anything, you need the basis configuration: + +1. Download and edit the `auth.yml` with your custom auth tokens and settings. +2. Download and rename the `.env.dist` to `.env` and edit it as well. + +For a quick start with Docker, you can use the following command: + +```sh +docker run --rm \ + --env-file ./.env \ + -v ./auth.yml:/etc/aqua/auth.yml \ + -v ./files:/var/lib/aqua/ \ + -p 8765:8765 \ + ghcr.io/superioz/aqua:latest +``` + +If you are on Windows, you might need to add `MSYS_NO_PATHCONV=1` so that the paths are correctly parsed. Also, to refer to the current directory, use `"$(pwd)"` instead of `./`, because that sometimes makes problems in Windows as well. + +For Docker Compose it's easier: + +```sh +docker-compose up +``` + +If you want to build the image yourself instead, you can of course `git clone git@github.com:Superioz/aqua.git` and then execute `docker-compose up --build`. ## Kubernetes diff --git a/internal/aqcli/upload.go b/internal/aqcli/aqcli.go similarity index 73% rename from internal/aqcli/upload.go rename to internal/aqcli/aqcli.go index 615692d..e4bbc08 100644 --- a/internal/aqcli/upload.go +++ b/internal/aqcli/aqcli.go @@ -9,12 +9,50 @@ import ( "github.com/urfave/cli/v2" "io" "io/ioutil" + "math/rand" "net/http" "os" "strings" "time" ) +var GenerateCommand = &cli.Command{ + Name: "generate", + Aliases: []string{"gen"}, + Usage: "Generates a possible auth token", + Flags: []cli.Flag{ + &cli.IntFlag{ + Name: "length", + Aliases: []string{"l"}, + Value: 32, + Usage: "Length of the token", + }, + }, + Action: func(c *cli.Context) error { + size := c.Int("length") + if size <= 1 { + return cli.Exit("You cannot generate a token with this length. Must be >=2.", 1) + } + + fmt.Println(generateToken(size)) + return nil + }, +} + +const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-+?!#$&%" + +// generateToken generates a random `size` long string from +// a predefined hexadecimal charset. +func generateToken(size int) string { + rand.Seed(time.Now().UnixNano()) + + b := make([]byte, size) + for i := range b { + b[i] = charset[rand.Intn(len(charset))] + } + return string(b) +} + var UploadCommand = &cli.Command{ Name: "upload", Usage: "Uploads a file to the aqua server", diff --git a/internal/aqcli/generate.go b/internal/aqcli/generate.go deleted file mode 100644 index b3f780e..0000000 --- a/internal/aqcli/generate.go +++ /dev/null @@ -1,45 +0,0 @@ -package aqcli - -import ( - "fmt" - "github.com/urfave/cli/v2" - "math/rand" - "time" -) - -var GenerateCommand = &cli.Command{ - Name: "generate", - Aliases: []string{"gen"}, - Usage: "Generates a possible auth token", - Flags: []cli.Flag{ - &cli.IntFlag{ - Name: "length", - Aliases: []string{"l"}, - Value: 32, - Usage: "Length of the token", - }, - }, - Action: func(c *cli.Context) error { - size := c.Int("length") - if size <= 1 { - return cli.Exit("You cannot generate a token with this length. Must be >=2.", 1) - } - - fmt.Println(generateToken(size)) - return nil - }, -} - -const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-+?!#$&%" - -// generateToken generates a random `size` long string from -// a predefined hexadecimal charset. -func generateToken(size int) string { - rand.Seed(time.Now().UnixNano()) - - b := make([]byte, size) - for i := range b { - b[i] = charset[rand.Intn(len(charset))] - } - return string(b) -} diff --git a/internal/handler/handler.go b/internal/handler/handler.go index d045601..8e0ddfd 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -6,6 +6,7 @@ import ( "github.com/gin-gonic/gin" "github.com/superioz/aqua/internal/config" "github.com/superioz/aqua/internal/metrics" + "github.com/superioz/aqua/internal/mime" "github.com/superioz/aqua/internal/storage" "github.com/superioz/aqua/pkg/env" "k8s.io/klog" @@ -20,33 +21,13 @@ const ( ) var ( - // validMimeTypes is a whitelist of all supported - // mime types. Taken from https://developer.mozilla.org/ - validMimeTypes = []string{ - "application/pdf", - "application/json", - "application/gzip", - "application/vnd.rar", - "application/zip", - "application/x-7z-compressed", - "image/png", - "image/jpeg", - "image/gif", - "image/svg+xml", - "text/csv", - "text/plain", - "audio/mpeg", - "audio/ogg", - "audio/opus", - "audio/webm", - "video/mp4", - "video/mpeg", - "video/webm", - } - emptyRequestMetadata = &RequestMetadata{Expiration: storage.ExpireNever} ) +// TODO when accessing: `/N2YwODUx.mp4` => `/N2YwODUx` + +// RequestFormFile is the metadata we get from the file +// which is requested to be uploaded. type RequestFormFile struct { File multipart.File ContentType string @@ -116,13 +97,13 @@ func (h *UploadHandler) Upload(c *gin.Context) { c.Status(http.StatusLengthRequired) return } - if c.Request.ContentLength > 50*SizeMegaByte { + if c.Request.ContentLength > int64(env.IntOrDefault("FILE_MAX_SIZE", 100))*SizeMegaByte { c.JSON(http.StatusRequestEntityTooLarge, gin.H{"msg": "content size must not exceed 50mb"}) return } ct := getContentType(file) - if !isContentTypeValid(ct) { + if !mime.IsValid(ct) { c.JSON(http.StatusBadRequest, gin.H{"msg": "content type of file is not valid"}) return } @@ -144,7 +125,13 @@ func (h *UploadHandler) Upload(c *gin.Context) { defer of.Close() metadata := getMetadata(form) - sf, err := h.FileStorage.StoreFile(of, metadata.Expiration) + rff := &RequestFormFile{ + File: of, + ContentType: ct, + ContentLength: c.Request.ContentLength, + } + + sf, err := h.FileStorage.StoreFile(rff, metadata.Expiration) if err != nil { klog.Error(err) c.JSON(http.StatusInternalServerError, gin.H{"msg": "could not store file"}) @@ -188,15 +175,6 @@ func getContentType(f *multipart.FileHeader) string { return c } -func isContentTypeValid(ct string) bool { - for _, mt := range validMimeTypes { - if mt == ct { - return true - } - } - return false -} - func getToken(c *gin.Context) string { // try to get the Bearer token, because it's the standard // for authorization diff --git a/internal/mime/mime.go b/internal/mime/mime.go new file mode 100644 index 0000000..883d3b7 --- /dev/null +++ b/internal/mime/mime.go @@ -0,0 +1,46 @@ +package mime + +var ( + // Types is a whitelist of all supported + // mime types. Taken from https://developer.mozilla.org/ + Types = map[string]string{ + "application/pdf": "pdf", + "application/json": "json", + "application/gzip": "gz", + "application/vnd.rar": "rar", + "application/zip": "zip", + "application/x-7z-compressed": "7z", + "image/png": "png", + "image/jpeg": "jpg", + "image/gif": "gif", + "image/svg+xml": "svg", + "text/csv": "csv", + "text/plain": "txt", + "audio/mpeg": "mp3", + "audio/ogg": "ogg", + "audio/opus": "opus", + "audio/webm": "weba", + "video/mp4": "mp4", + "video/mpeg": "mpeg", + "video/webm": "webm", + } +) + +// IsValid checks if given type is inside Types map +func IsValid(t string) bool { + for _, mt := range Types { + if mt == t { + return true + } + } + return false +} + +// GetExtension returns an extension for the given MIME type +func GetExtension(t string) string { + ext, ok := Types[t] + if !ok { + return "application/octet-stream" + } + return ext +} diff --git a/internal/storage/meta.go b/internal/storage/filemeta.go similarity index 85% rename from internal/storage/meta.go rename to internal/storage/filemeta.go index 80effd9..8b4a97d 100644 --- a/internal/storage/meta.go +++ b/internal/storage/filemeta.go @@ -48,7 +48,9 @@ func (s *SqliteFileMetaDatabase) Connect() error { _, err = db.Exec(`create table if not exists files ( id text not null primary key, uploaded_at integer, - expires_at integer + expires_at integer, + mime_type varchar, + size integer );`) if err != nil { return err @@ -63,13 +65,13 @@ func (s *SqliteFileMetaDatabase) WriteFile(sf *StoredFile) error { } defer db.Close() - stmt, err := db.Prepare(`insert into files(id, uploaded_at, expires_at) values(?, ?, ?)`) + stmt, err := db.Prepare(`insert into files(id, uploaded_at, expires_at, mime_type, size) values(?, ?, ?, ?, ?)`) if err != nil { return err } defer stmt.Close() - _, err = stmt.Exec(sf.Id, sf.UploadedAt, sf.ExpiresAt) + _, err = stmt.Exec(sf.Id, sf.UploadedAt, sf.ExpiresAt, sf.MimeType, sf.Size) if err != nil { return err } @@ -116,18 +118,10 @@ func (s *SqliteFileMetaDatabase) GetFile(id string) (*StoredFile, error) { return nil, nil } - var uploadedAt int - var expiresAt int - - err = rows.Scan(&id, &uploadedAt, &expiresAt) + sf, err := getFromRows(rows) if err != nil { return nil, err } - sf := &StoredFile{ - Id: id, - UploadedAt: int64(uploadedAt), - ExpiresAt: int64(expiresAt), - } return sf, nil } @@ -146,19 +140,11 @@ func (s *SqliteFileMetaDatabase) GetAllFiles() ([]*StoredFile, error) { var sfs []*StoredFile for rows.Next() { - var id string - var uploadedAt int - var expiresAt int - - err = rows.Scan(&id, &uploadedAt, &expiresAt) + sf, err := getFromRows(rows) if err != nil { return nil, err } - sf := &StoredFile{ - Id: id, - UploadedAt: int64(uploadedAt), - ExpiresAt: int64(expiresAt), - } + sfs = append(sfs, sf) } err = rows.Err() @@ -190,19 +176,11 @@ func (s *SqliteFileMetaDatabase) GetAllExpired() ([]*StoredFile, error) { var sfs []*StoredFile for rows.Next() { - var id string - var uploadedAt int - var expiresAt int - - err = rows.Scan(&id, &uploadedAt, &expiresAt) + sf, err := getFromRows(rows) if err != nil { return nil, err } - sf := &StoredFile{ - Id: id, - UploadedAt: int64(uploadedAt), - ExpiresAt: int64(expiresAt), - } + sfs = append(sfs, sf) } err = rows.Err() @@ -211,3 +189,24 @@ func (s *SqliteFileMetaDatabase) GetAllExpired() ([]*StoredFile, error) { } return sfs, nil } + +func getFromRows(rows *sql.Rows) (*StoredFile, error) { + var id string + var uploadedAt int + var expiresAt int + var mimeType string + var size int + + err := rows.Scan(&id, &uploadedAt, &expiresAt, &mimeType, &size) + if err != nil { + return nil, err + } + sf := &StoredFile{ + Id: id, + UploadedAt: int64(uploadedAt), + ExpiresAt: int64(expiresAt), + MimeType: mimeType, + Size: int64(size), + } + return sf, nil +} diff --git a/internal/storage/files.go b/internal/storage/filesys.go similarity index 78% rename from internal/storage/files.go rename to internal/storage/filesys.go index e476c4e..3caa89f 100644 --- a/internal/storage/files.go +++ b/internal/storage/filesys.go @@ -2,12 +2,15 @@ package storage import ( "io" - "mime/multipart" "os" ) type FileSystem interface { - CreateFile(mf multipart.File, name string) (bool, error) + // CreateFile writes the content of given reader to a file + // with given name. + // Always returns if the file was partly written to disk. + CreateFile(r io.Reader, name string) (bool, error) + DeleteFile(id string) error GetFile(id string) (*os.File, error) Exists(id string) (bool, error) @@ -17,7 +20,7 @@ type LocalFileSystem struct { FolderPath string } -func (l LocalFileSystem) CreateFile(mf multipart.File, name string) (bool, error) { +func (l LocalFileSystem) CreateFile(r io.Reader, name string) (bool, error) { err := os.MkdirAll(l.FolderPath, os.ModePerm) if err != nil { return false, err @@ -31,7 +34,7 @@ func (l LocalFileSystem) CreateFile(mf multipart.File, name string) (bool, error // use io.Copy so that we don't have to load all the image into the memory. // they get copied in smaller 32kb chunks. - _, err = io.Copy(f, mf) + _, err = io.Copy(f, r) if err != nil { return true, err } diff --git a/internal/storage/storage.go b/internal/storage/storage.go index 93ac18f..054e824 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -5,10 +5,10 @@ import ( "errors" "fmt" "github.com/google/uuid" + "github.com/superioz/aqua/internal/handler" "github.com/superioz/aqua/internal/metrics" "github.com/superioz/aqua/pkg/env" "k8s.io/klog" - "mime/multipart" "strings" "time" @@ -26,6 +26,8 @@ type StoredFile struct { Id string UploadedAt int64 ExpiresAt int64 + MimeType string + Size int64 } func (sf *StoredFile) String() string { @@ -100,13 +102,13 @@ func (fs *FileStorage) Cleanup() error { return nil } -func (fs *FileStorage) StoreFile(of multipart.File, expiration int64) (*StoredFile, error) { +func (fs *FileStorage) StoreFile(rff *handler.RequestFormFile, expiration int64) (*StoredFile, error) { name, err := getRandomFileName(env.IntOrDefault("FILE_NAME_LENGTH", 8)) if err != nil { return nil, errors.New("could not generate random name") } - _, err = fs.fileSystem.CreateFile(of, name) + _, err = fs.fileSystem.CreateFile(rff.File, name) if err != nil { klog.Error(err) return nil, errors.New("could not save file to system") @@ -122,6 +124,8 @@ func (fs *FileStorage) StoreFile(of multipart.File, expiration int64) (*StoredFi Id: name, UploadedAt: currentTime, ExpiresAt: expAt, + MimeType: rff.ContentType, + Size: rff.ContentLength, } // write to meta database diff --git a/run.sh b/run.sh index c935f8a..2eaddee 100644 --- a/run.sh +++ b/run.sh @@ -15,4 +15,4 @@ fi # we only need MSYS_NO_PATHCONV when running this script in Windows # because otherwise Mingw will try to convert # the given paths and that breaks everything. -MSYS_NO_PATHCONV=1 docker run -it --rm -p 8765:8765 $IMAGE \ No newline at end of file +MSYS_NO_PATHCONV=1 docker run -it --rm -p 8765:8765 $IMAGE