From 800e3dfaaf5c411653018f4902a46e059f9bb95e Mon Sep 17 00:00:00 2001 From: Tobias B Date: Wed, 17 Nov 2021 23:14:54 +0100 Subject: [PATCH 1/5] Fix no file in form panic --- internal/handler/handler.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/internal/handler/handler.go b/internal/handler/handler.go index f59eaab..d045601 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -47,6 +47,12 @@ var ( emptyRequestMetadata = &RequestMetadata{Expiration: storage.ExpireNever} ) +type RequestFormFile struct { + File multipart.File + ContentType string + ContentLength int64 +} + type RequestMetadata struct { Expiration int64 `json:"expiration"` } @@ -100,6 +106,10 @@ func (h *UploadHandler) Upload(c *gin.Context) { c.JSON(http.StatusBadRequest, gin.H{"msg": "too many files in form"}) return } + if len(files) == 0 { + c.JSON(http.StatusBadRequest, gin.H{"msg": "no file in form"}) + return + } file := files[0] if c.Request.Header.Get("Content-Length") == "" { From ec65d88252707aaf0eb2246ccdede29a949ffbbb Mon Sep 17 00:00:00 2001 From: Tobias B Date: Wed, 17 Nov 2021 23:15:07 +0100 Subject: [PATCH 2/5] Increment version --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index a918a2a..ee6cdce 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.6.0 +0.6.1 From 1a2dbf266de2739c6917fbe52185811efff98eb4 Mon Sep 17 00:00:00 2001 From: Tobias B Date: Thu, 18 Nov 2021 01:14:48 +0100 Subject: [PATCH 3/5] Many small improvements --- README.md | 33 +++++++++--- internal/aqcli/{upload.go => aqcli.go} | 38 ++++++++++++++ internal/aqcli/generate.go | 45 ---------------- internal/handler/handler.go | 50 +++++------------- internal/mime/mime.go | 46 +++++++++++++++++ internal/storage/{meta.go => filemeta.go} | 63 +++++++++++------------ internal/storage/{files.go => filesys.go} | 11 ++-- internal/storage/storage.go | 10 ++-- run.sh | 2 +- 9 files changed, 171 insertions(+), 127 deletions(-) rename internal/aqcli/{upload.go => aqcli.go} (73%) delete mode 100644 internal/aqcli/generate.go create mode 100644 internal/mime/mime.go rename internal/storage/{meta.go => filemeta.go} (85%) rename internal/storage/{files.go => filesys.go} (78%) diff --git a/README.md b/README.md index 33a5747..2fd8ba8 100644 --- a/README.md +++ b/README.md @@ -28,14 +28,35 @@ This is only for those people, that are still living in the 90s or are not comfo For further instructions like creating a service that can easily be started with `service aqua start`, please refer to other pages (there are a bunch that explain this) - I won't. -## Docker Compose +## Docker (Compose) -Before following the steps, make sure you have [Docker](https://docs.docker.com/get-docker/) and [Docker Compose](https://docs.docker.com/compose/install/) installed on the machine. +Before following the steps, make sure you have [Docker](https://docs.docker.com/get-docker/) installed. And if you want to use the preconfigured `docker-compose` file, install [Docker Compose](https://docs.docker.com/compose/install/) as well. -1. Clone the repository to a directory of your liking with `git clone git@github.com:Superioz/aqua.git` -2. Edit the `auth.yml` with your custom auth tokens and settings. -3. Rename the `.env.dist` to `.env` and edit it as well. -4. `docker-compose up` and you should see it up and running. +Before you can start anything, you need the basis configuration: + +1. Download and edit the `auth.yml` with your custom auth tokens and settings. +2. Download and rename the `.env.dist` to `.env` and edit it as well. + +For a quick start with Docker, you can use the following command: + +```sh +docker run --rm \ + --env-file ./.env \ + -v ./auth.yml:/etc/aqua/auth.yml \ + -v ./files:/var/lib/aqua/ \ + -p 8765:8765 \ + ghcr.io/superioz/aqua:latest +``` + +If you are on Windows, you might need to add `MSYS_NO_PATHCONV=1` so that the paths are correctly parsed. Also, to refer to the current directory, use `"$(pwd)"` instead of `./`, because that sometimes makes problems in Windows as well. + +For Docker Compose it's easier: + +```sh +docker-compose up +``` + +If you want to build the image yourself instead, you can of course `git clone git@github.com:Superioz/aqua.git` and then execute `docker-compose up --build`. ## Kubernetes diff --git a/internal/aqcli/upload.go b/internal/aqcli/aqcli.go similarity index 73% rename from internal/aqcli/upload.go rename to internal/aqcli/aqcli.go index 615692d..e4bbc08 100644 --- a/internal/aqcli/upload.go +++ b/internal/aqcli/aqcli.go @@ -9,12 +9,50 @@ import ( "github.com/urfave/cli/v2" "io" "io/ioutil" + "math/rand" "net/http" "os" "strings" "time" ) +var GenerateCommand = &cli.Command{ + Name: "generate", + Aliases: []string{"gen"}, + Usage: "Generates a possible auth token", + Flags: []cli.Flag{ + &cli.IntFlag{ + Name: "length", + Aliases: []string{"l"}, + Value: 32, + Usage: "Length of the token", + }, + }, + Action: func(c *cli.Context) error { + size := c.Int("length") + if size <= 1 { + return cli.Exit("You cannot generate a token with this length. Must be >=2.", 1) + } + + fmt.Println(generateToken(size)) + return nil + }, +} + +const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-+?!#$&%" + +// generateToken generates a random `size` long string from +// a predefined hexadecimal charset. +func generateToken(size int) string { + rand.Seed(time.Now().UnixNano()) + + b := make([]byte, size) + for i := range b { + b[i] = charset[rand.Intn(len(charset))] + } + return string(b) +} + var UploadCommand = &cli.Command{ Name: "upload", Usage: "Uploads a file to the aqua server", diff --git a/internal/aqcli/generate.go b/internal/aqcli/generate.go deleted file mode 100644 index b3f780e..0000000 --- a/internal/aqcli/generate.go +++ /dev/null @@ -1,45 +0,0 @@ -package aqcli - -import ( - "fmt" - "github.com/urfave/cli/v2" - "math/rand" - "time" -) - -var GenerateCommand = &cli.Command{ - Name: "generate", - Aliases: []string{"gen"}, - Usage: "Generates a possible auth token", - Flags: []cli.Flag{ - &cli.IntFlag{ - Name: "length", - Aliases: []string{"l"}, - Value: 32, - Usage: "Length of the token", - }, - }, - Action: func(c *cli.Context) error { - size := c.Int("length") - if size <= 1 { - return cli.Exit("You cannot generate a token with this length. Must be >=2.", 1) - } - - fmt.Println(generateToken(size)) - return nil - }, -} - -const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-+?!#$&%" - -// generateToken generates a random `size` long string from -// a predefined hexadecimal charset. -func generateToken(size int) string { - rand.Seed(time.Now().UnixNano()) - - b := make([]byte, size) - for i := range b { - b[i] = charset[rand.Intn(len(charset))] - } - return string(b) -} diff --git a/internal/handler/handler.go b/internal/handler/handler.go index d045601..8e0ddfd 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -6,6 +6,7 @@ import ( "github.com/gin-gonic/gin" "github.com/superioz/aqua/internal/config" "github.com/superioz/aqua/internal/metrics" + "github.com/superioz/aqua/internal/mime" "github.com/superioz/aqua/internal/storage" "github.com/superioz/aqua/pkg/env" "k8s.io/klog" @@ -20,33 +21,13 @@ const ( ) var ( - // validMimeTypes is a whitelist of all supported - // mime types. Taken from https://developer.mozilla.org/ - validMimeTypes = []string{ - "application/pdf", - "application/json", - "application/gzip", - "application/vnd.rar", - "application/zip", - "application/x-7z-compressed", - "image/png", - "image/jpeg", - "image/gif", - "image/svg+xml", - "text/csv", - "text/plain", - "audio/mpeg", - "audio/ogg", - "audio/opus", - "audio/webm", - "video/mp4", - "video/mpeg", - "video/webm", - } - emptyRequestMetadata = &RequestMetadata{Expiration: storage.ExpireNever} ) +// TODO when accessing: `/N2YwODUx.mp4` => `/N2YwODUx` + +// RequestFormFile is the metadata we get from the file +// which is requested to be uploaded. type RequestFormFile struct { File multipart.File ContentType string @@ -116,13 +97,13 @@ func (h *UploadHandler) Upload(c *gin.Context) { c.Status(http.StatusLengthRequired) return } - if c.Request.ContentLength > 50*SizeMegaByte { + if c.Request.ContentLength > int64(env.IntOrDefault("FILE_MAX_SIZE", 100))*SizeMegaByte { c.JSON(http.StatusRequestEntityTooLarge, gin.H{"msg": "content size must not exceed 50mb"}) return } ct := getContentType(file) - if !isContentTypeValid(ct) { + if !mime.IsValid(ct) { c.JSON(http.StatusBadRequest, gin.H{"msg": "content type of file is not valid"}) return } @@ -144,7 +125,13 @@ func (h *UploadHandler) Upload(c *gin.Context) { defer of.Close() metadata := getMetadata(form) - sf, err := h.FileStorage.StoreFile(of, metadata.Expiration) + rff := &RequestFormFile{ + File: of, + ContentType: ct, + ContentLength: c.Request.ContentLength, + } + + sf, err := h.FileStorage.StoreFile(rff, metadata.Expiration) if err != nil { klog.Error(err) c.JSON(http.StatusInternalServerError, gin.H{"msg": "could not store file"}) @@ -188,15 +175,6 @@ func getContentType(f *multipart.FileHeader) string { return c } -func isContentTypeValid(ct string) bool { - for _, mt := range validMimeTypes { - if mt == ct { - return true - } - } - return false -} - func getToken(c *gin.Context) string { // try to get the Bearer token, because it's the standard // for authorization diff --git a/internal/mime/mime.go b/internal/mime/mime.go new file mode 100644 index 0000000..883d3b7 --- /dev/null +++ b/internal/mime/mime.go @@ -0,0 +1,46 @@ +package mime + +var ( + // Types is a whitelist of all supported + // mime types. Taken from https://developer.mozilla.org/ + Types = map[string]string{ + "application/pdf": "pdf", + "application/json": "json", + "application/gzip": "gz", + "application/vnd.rar": "rar", + "application/zip": "zip", + "application/x-7z-compressed": "7z", + "image/png": "png", + "image/jpeg": "jpg", + "image/gif": "gif", + "image/svg+xml": "svg", + "text/csv": "csv", + "text/plain": "txt", + "audio/mpeg": "mp3", + "audio/ogg": "ogg", + "audio/opus": "opus", + "audio/webm": "weba", + "video/mp4": "mp4", + "video/mpeg": "mpeg", + "video/webm": "webm", + } +) + +// IsValid checks if given type is inside Types map +func IsValid(t string) bool { + for _, mt := range Types { + if mt == t { + return true + } + } + return false +} + +// GetExtension returns an extension for the given MIME type +func GetExtension(t string) string { + ext, ok := Types[t] + if !ok { + return "application/octet-stream" + } + return ext +} diff --git a/internal/storage/meta.go b/internal/storage/filemeta.go similarity index 85% rename from internal/storage/meta.go rename to internal/storage/filemeta.go index 80effd9..8b4a97d 100644 --- a/internal/storage/meta.go +++ b/internal/storage/filemeta.go @@ -48,7 +48,9 @@ func (s *SqliteFileMetaDatabase) Connect() error { _, err = db.Exec(`create table if not exists files ( id text not null primary key, uploaded_at integer, - expires_at integer + expires_at integer, + mime_type varchar, + size integer );`) if err != nil { return err @@ -63,13 +65,13 @@ func (s *SqliteFileMetaDatabase) WriteFile(sf *StoredFile) error { } defer db.Close() - stmt, err := db.Prepare(`insert into files(id, uploaded_at, expires_at) values(?, ?, ?)`) + stmt, err := db.Prepare(`insert into files(id, uploaded_at, expires_at, mime_type, size) values(?, ?, ?, ?, ?)`) if err != nil { return err } defer stmt.Close() - _, err = stmt.Exec(sf.Id, sf.UploadedAt, sf.ExpiresAt) + _, err = stmt.Exec(sf.Id, sf.UploadedAt, sf.ExpiresAt, sf.MimeType, sf.Size) if err != nil { return err } @@ -116,18 +118,10 @@ func (s *SqliteFileMetaDatabase) GetFile(id string) (*StoredFile, error) { return nil, nil } - var uploadedAt int - var expiresAt int - - err = rows.Scan(&id, &uploadedAt, &expiresAt) + sf, err := getFromRows(rows) if err != nil { return nil, err } - sf := &StoredFile{ - Id: id, - UploadedAt: int64(uploadedAt), - ExpiresAt: int64(expiresAt), - } return sf, nil } @@ -146,19 +140,11 @@ func (s *SqliteFileMetaDatabase) GetAllFiles() ([]*StoredFile, error) { var sfs []*StoredFile for rows.Next() { - var id string - var uploadedAt int - var expiresAt int - - err = rows.Scan(&id, &uploadedAt, &expiresAt) + sf, err := getFromRows(rows) if err != nil { return nil, err } - sf := &StoredFile{ - Id: id, - UploadedAt: int64(uploadedAt), - ExpiresAt: int64(expiresAt), - } + sfs = append(sfs, sf) } err = rows.Err() @@ -190,19 +176,11 @@ func (s *SqliteFileMetaDatabase) GetAllExpired() ([]*StoredFile, error) { var sfs []*StoredFile for rows.Next() { - var id string - var uploadedAt int - var expiresAt int - - err = rows.Scan(&id, &uploadedAt, &expiresAt) + sf, err := getFromRows(rows) if err != nil { return nil, err } - sf := &StoredFile{ - Id: id, - UploadedAt: int64(uploadedAt), - ExpiresAt: int64(expiresAt), - } + sfs = append(sfs, sf) } err = rows.Err() @@ -211,3 +189,24 @@ func (s *SqliteFileMetaDatabase) GetAllExpired() ([]*StoredFile, error) { } return sfs, nil } + +func getFromRows(rows *sql.Rows) (*StoredFile, error) { + var id string + var uploadedAt int + var expiresAt int + var mimeType string + var size int + + err := rows.Scan(&id, &uploadedAt, &expiresAt, &mimeType, &size) + if err != nil { + return nil, err + } + sf := &StoredFile{ + Id: id, + UploadedAt: int64(uploadedAt), + ExpiresAt: int64(expiresAt), + MimeType: mimeType, + Size: int64(size), + } + return sf, nil +} diff --git a/internal/storage/files.go b/internal/storage/filesys.go similarity index 78% rename from internal/storage/files.go rename to internal/storage/filesys.go index e476c4e..3caa89f 100644 --- a/internal/storage/files.go +++ b/internal/storage/filesys.go @@ -2,12 +2,15 @@ package storage import ( "io" - "mime/multipart" "os" ) type FileSystem interface { - CreateFile(mf multipart.File, name string) (bool, error) + // CreateFile writes the content of given reader to a file + // with given name. + // Always returns if the file was partly written to disk. + CreateFile(r io.Reader, name string) (bool, error) + DeleteFile(id string) error GetFile(id string) (*os.File, error) Exists(id string) (bool, error) @@ -17,7 +20,7 @@ type LocalFileSystem struct { FolderPath string } -func (l LocalFileSystem) CreateFile(mf multipart.File, name string) (bool, error) { +func (l LocalFileSystem) CreateFile(r io.Reader, name string) (bool, error) { err := os.MkdirAll(l.FolderPath, os.ModePerm) if err != nil { return false, err @@ -31,7 +34,7 @@ func (l LocalFileSystem) CreateFile(mf multipart.File, name string) (bool, error // use io.Copy so that we don't have to load all the image into the memory. // they get copied in smaller 32kb chunks. - _, err = io.Copy(f, mf) + _, err = io.Copy(f, r) if err != nil { return true, err } diff --git a/internal/storage/storage.go b/internal/storage/storage.go index 93ac18f..054e824 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -5,10 +5,10 @@ import ( "errors" "fmt" "github.com/google/uuid" + "github.com/superioz/aqua/internal/handler" "github.com/superioz/aqua/internal/metrics" "github.com/superioz/aqua/pkg/env" "k8s.io/klog" - "mime/multipart" "strings" "time" @@ -26,6 +26,8 @@ type StoredFile struct { Id string UploadedAt int64 ExpiresAt int64 + MimeType string + Size int64 } func (sf *StoredFile) String() string { @@ -100,13 +102,13 @@ func (fs *FileStorage) Cleanup() error { return nil } -func (fs *FileStorage) StoreFile(of multipart.File, expiration int64) (*StoredFile, error) { +func (fs *FileStorage) StoreFile(rff *handler.RequestFormFile, expiration int64) (*StoredFile, error) { name, err := getRandomFileName(env.IntOrDefault("FILE_NAME_LENGTH", 8)) if err != nil { return nil, errors.New("could not generate random name") } - _, err = fs.fileSystem.CreateFile(of, name) + _, err = fs.fileSystem.CreateFile(rff.File, name) if err != nil { klog.Error(err) return nil, errors.New("could not save file to system") @@ -122,6 +124,8 @@ func (fs *FileStorage) StoreFile(of multipart.File, expiration int64) (*StoredFi Id: name, UploadedAt: currentTime, ExpiresAt: expAt, + MimeType: rff.ContentType, + Size: rff.ContentLength, } // write to meta database diff --git a/run.sh b/run.sh index c935f8a..2eaddee 100644 --- a/run.sh +++ b/run.sh @@ -15,4 +15,4 @@ fi # we only need MSYS_NO_PATHCONV when running this script in Windows # because otherwise Mingw will try to convert # the given paths and that breaks everything. -MSYS_NO_PATHCONV=1 docker run -it --rm -p 8765:8765 $IMAGE \ No newline at end of file +MSYS_NO_PATHCONV=1 docker run -it --rm -p 8765:8765 $IMAGE From a81a33c17a0f283d7c0858df4a9f60cf425119ed Mon Sep 17 00:00:00 2001 From: Tobias B Date: Thu, 18 Nov 2021 02:35:04 +0100 Subject: [PATCH 4/5] Add extensions to support Discord embeds --- .env.dist | 4 ++- README.md | 6 ++-- VERSION | 2 +- cmd/aqua/main.go | 2 +- internal/config/config.go | 7 ++++ internal/handler/handler.go | 68 ++++++++++++++++++------------------- internal/mime/mime.go | 2 +- internal/request/request.go | 38 +++++++++++++++++++++ internal/storage/storage.go | 20 ++++------- pkg/env/env.go | 14 ++++++++ 10 files changed, 109 insertions(+), 54 deletions(-) create mode 100644 internal/request/request.go diff --git a/.env.dist b/.env.dist index 9e2d4c1..c638ff8 100644 --- a/.env.dist +++ b/.env.dist @@ -4,5 +4,7 @@ FILE_NAME_LENGTH=12 FILE_MAX_SIZE=100 FILE_META_DB_PATH= FILE_EXPIRATION_CYCLE=10 -METRICS_ENABLED=true FILE_SERVING_ENABLED=true +FILE_EXTENSIONS_RESPONSE=true +FILE_EXTENSIONS_EXCLUDED=image/png,image/jpeg +METRICS_ENABLED=true diff --git a/README.md b/README.md index 2fd8ba8..f90fa0f 100644 --- a/README.md +++ b/README.md @@ -82,8 +82,10 @@ For examplary usage of the environment variables, have a look at the `.env.dist` | `FILE_MAX_SIZE` | Maximum size for uploaded files in Megabytes. | | `FILE_META_DB_PATH` | Path to the directory, where the sqlite database for file metadata should be stored. Recommended to not be the same folder as `FILE_STORAGE_PATH` to prevent overlapping. | | `FILE_EXPIRATION_CYCLE` | Determines the interval of the expiration cycle. `5` means that every 5 seconds the files will be checked for expiration. | +| `FILE_SERVING_ENABLED` | Defaults to `true`, if `false`, the server won't serve the stored files. | +| `FILE_EXTENSIONS_RESPONSE` | Defaults to `true`. if the file name returned will have its extension added to it. | +| `FILE_EXTENSIONS_EXCLUDED` | Comma-seperated list of MIME types, that should be excluded from the extension response rule above. Defaults to `image/png,image/jpeg` | | `METRICS_ENABLED` | Is normally set to true, but otherwise disables the Prometheus metrics publishing. | -| `FILE_SERVING_ENABLED` | Defaults to true, when `false`, then the server won't serve the stored files. | ## Tokens @@ -182,7 +184,7 @@ After that you can import it to your custom upload goals in the ShareX UI. "metadata": "{ \"expiration\": 3600 }" }, "FileFormName": "file", - "URL": "https://your-domain.com/$json:id$" + "URL": "https://your-domain.com/$json:fileName$" } ``` diff --git a/VERSION b/VERSION index ee6cdce..b616048 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.6.1 +0.6.2 diff --git a/cmd/aqua/main.go b/cmd/aqua/main.go index 5da0a9c..dffc439 100644 --- a/cmd/aqua/main.go +++ b/cmd/aqua/main.go @@ -15,7 +15,7 @@ import ( func main() { err := godotenv.Load() if err != nil { - klog.Warningf("Error loading .env file: %v", err) + klog.Warningf("Could not load .env file: %v", err) } klog.Infoln("Hello World!") diff --git a/internal/config/config.go b/internal/config/config.go index f2ff5dd..74fa2b7 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -6,6 +6,13 @@ import ( "os" ) +const ( + ExpireNever = -1 + + EnvDefaultFileStoragePath = "/var/lib/aqua/files/" + EnvDefaultMetaDbPath = "/var/lib/aqua/" +) + type AuthConfig struct { ValidTokens []*TokenConfig `yaml:"validTokens"` } diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 8e0ddfd..4caeb9e 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -1,12 +1,12 @@ package handler import ( - "encoding/json" "fmt" "github.com/gin-gonic/gin" "github.com/superioz/aqua/internal/config" "github.com/superioz/aqua/internal/metrics" "github.com/superioz/aqua/internal/mime" + "github.com/superioz/aqua/internal/request" "github.com/superioz/aqua/internal/storage" "github.com/superioz/aqua/pkg/env" "k8s.io/klog" @@ -20,27 +20,13 @@ const ( SizeMegaByte = 1 << (10 * 2) ) -var ( - emptyRequestMetadata = &RequestMetadata{Expiration: storage.ExpireNever} -) - -// TODO when accessing: `/N2YwODUx.mp4` => `/N2YwODUx` - -// RequestFormFile is the metadata we get from the file -// which is requested to be uploaded. -type RequestFormFile struct { - File multipart.File - ContentType string - ContentLength int64 -} - -type RequestMetadata struct { - Expiration int64 `json:"expiration"` -} - type UploadHandler struct { AuthConfig *config.AuthConfig FileStorage *storage.FileStorage + + // excluded as per defined by the environment variable + // FILE_EXTENSIONS_EXCEPT + exclMimeTypes []string } func NewUploadHandler() *UploadHandler { @@ -48,6 +34,7 @@ func NewUploadHandler() *UploadHandler { handler.ReloadAuthConfig() handler.FileStorage = storage.NewFileStorage() + handler.exclMimeTypes = env.ListOrDefault("FILE_EXTENSIONS_EXCLUDED", []string{"image/png", "image/jpeg"}) return handler } @@ -124,8 +111,8 @@ func (h *UploadHandler) Upload(c *gin.Context) { } defer of.Close() - metadata := getMetadata(form) - rff := &RequestFormFile{ + metadata := request.GetMetadata(form) + rff := &request.RequestFormFile{ File: of, ContentType: ct, ContentLength: c.Request.ContentLength, @@ -145,22 +132,26 @@ func (h *UploadHandler) Upload(c *gin.Context) { klog.Infof("Stored file %s (expiresIn: %s)", sf.Id, expiresIn) metrics.IncFilesUploaded() - c.JSON(http.StatusOK, gin.H{"id": sf.Id}) + // add extension to file name if it is enabled and + // the extension is not excluded + storedName := sf.Id + if env.BoolOrDefault("FILE_EXTENSIONS_RESPONSE", true) && !h.isExtensionExcluded(sf.MimeType) { + storedName = fmt.Sprintf("%s.%s", storedName, mime.GetExtension(sf.MimeType)) + } + + c.JSON(http.StatusOK, gin.H{"fileName": storedName}) } -func getMetadata(form *multipart.Form) *RequestMetadata { - metaRawList := form.Value["metadata"] - if len(metaRawList) == 0 { - return emptyRequestMetadata +// isExtensionExcluded returns if the given mime type +// should be excluded by the extension response rule, which states if +// an extension should be appended to the file name when responding. +func (h *UploadHandler) isExtensionExcluded(mimeType string) bool { + for _, exclMimeType := range h.exclMimeTypes { + if exclMimeType == mimeType { + return true + } } - metaRaw := metaRawList[0] - - var metadata *RequestMetadata - err := json.Unmarshal([]byte(metaRaw), &metadata) - if err != nil { - return emptyRequestMetadata - } - return metadata + return false } // workaround for file Content-Type headers @@ -191,10 +182,17 @@ func getToken(c *gin.Context) string { // HandleStaticFiles takes the files inside the configured file storage // path and serves them to the client. func HandleStaticFiles() gin.HandlerFunc { - fileStoragePath := env.StringOrDefault("FILE_STORAGE_PATH", storage.EnvDefaultFileStoragePath) + fileStoragePath := env.StringOrDefault("FILE_STORAGE_PATH", config.EnvDefaultFileStoragePath) return func(c *gin.Context) { fileName := c.Param("file") + + // the file name could contain the extension + // we split it and ship it. + if strings.Contains(fileName, ".") { + fileName = strings.Split(fileName, ".")[0] + } + fullPath := fileStoragePath + fileName f, err := os.Open(fullPath) diff --git a/internal/mime/mime.go b/internal/mime/mime.go index 883d3b7..2809eaf 100644 --- a/internal/mime/mime.go +++ b/internal/mime/mime.go @@ -28,7 +28,7 @@ var ( // IsValid checks if given type is inside Types map func IsValid(t string) bool { - for _, mt := range Types { + for mt := range Types { if mt == t { return true } diff --git a/internal/request/request.go b/internal/request/request.go new file mode 100644 index 0000000..2b14817 --- /dev/null +++ b/internal/request/request.go @@ -0,0 +1,38 @@ +package request + +import ( + "encoding/json" + "github.com/superioz/aqua/internal/config" + "mime/multipart" +) + +var ( + emptyRequestMetadata = &RequestMetadata{Expiration: config.ExpireNever} +) + +// RequestFormFile is the metadata we get from the file +// which is requested to be uploaded. +type RequestFormFile struct { + File multipart.File + ContentType string + ContentLength int64 +} + +type RequestMetadata struct { + Expiration int64 `json:"expiration"` +} + +func GetMetadata(form *multipart.Form) *RequestMetadata { + metaRawList := form.Value["metadata"] + if len(metaRawList) == 0 { + return emptyRequestMetadata + } + metaRaw := metaRawList[0] + + var metadata *RequestMetadata + err := json.Unmarshal([]byte(metaRaw), &metadata) + if err != nil { + return emptyRequestMetadata + } + return metadata +} diff --git a/internal/storage/storage.go b/internal/storage/storage.go index 054e824..3ae4dbc 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -5,8 +5,9 @@ import ( "errors" "fmt" "github.com/google/uuid" - "github.com/superioz/aqua/internal/handler" + "github.com/superioz/aqua/internal/config" "github.com/superioz/aqua/internal/metrics" + "github.com/superioz/aqua/internal/request" "github.com/superioz/aqua/pkg/env" "k8s.io/klog" "strings" @@ -15,13 +16,6 @@ import ( _ "modernc.org/sqlite" ) -const ( - ExpireNever = -1 - - EnvDefaultFileStoragePath = "/var/lib/aqua/files/" - EnvDefaultMetaDbPath = "/var/lib/aqua/" -) - type StoredFile struct { Id string UploadedAt int64 @@ -43,14 +37,14 @@ type FileStorage struct { } func NewFileStorage() *FileStorage { - metaDbFilePath := env.StringOrDefault("FILE_META_DB_PATH", EnvDefaultMetaDbPath) + metaDbFilePath := env.StringOrDefault("FILE_META_DB_PATH", config.EnvDefaultMetaDbPath) fileMetaDb := NewSqliteFileMetaDatabase(metaDbFilePath) err := fileMetaDb.Connect() if err != nil { klog.Errorf("Could not connect to file meta db: %v", err) } - fileStoragePath := env.StringOrDefault("FILE_STORAGE_PATH", EnvDefaultFileStoragePath) + fileStoragePath := env.StringOrDefault("FILE_STORAGE_PATH", config.EnvDefaultFileStoragePath) fileSystem := NewLocalFileStorage(fileStoragePath) return &FileStorage{ @@ -102,7 +96,7 @@ func (fs *FileStorage) Cleanup() error { return nil } -func (fs *FileStorage) StoreFile(rff *handler.RequestFormFile, expiration int64) (*StoredFile, error) { +func (fs *FileStorage) StoreFile(rff *request.RequestFormFile, expiration int64) (*StoredFile, error) { name, err := getRandomFileName(env.IntOrDefault("FILE_NAME_LENGTH", 8)) if err != nil { return nil, errors.New("could not generate random name") @@ -116,8 +110,8 @@ func (fs *FileStorage) StoreFile(rff *handler.RequestFormFile, expiration int64) currentTime := time.Now().Unix() expAt := currentTime + expiration - if expiration == ExpireNever { - expAt = ExpireNever + if expiration == config.ExpireNever { + expAt = config.ExpireNever } sf := &StoredFile{ diff --git a/pkg/env/env.go b/pkg/env/env.go index 873269c..b1d3b13 100644 --- a/pkg/env/env.go +++ b/pkg/env/env.go @@ -3,6 +3,7 @@ package env import ( "os" "strconv" + "strings" ) func String(name string) (string, bool) { @@ -57,3 +58,16 @@ func BoolOrDefault(name string, def bool) bool { } return b } + +func ListOrDefault(name string, def []string) []string { + s, ok := String(name) + if !ok { + return def + } + + spl := strings.Split(s, ",") + if len(spl) == 0 { + return def + } + return spl +} From 0bb5618177f231d75158b44485fc4fca2c0ef96c Mon Sep 17 00:00:00 2001 From: Tobias B Date: Thu, 18 Nov 2021 02:36:42 +0100 Subject: [PATCH 5/5] Fix compile error --- internal/aqcli/aqcli.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/aqcli/aqcli.go b/internal/aqcli/aqcli.go index e4bbc08..a091138 100644 --- a/internal/aqcli/aqcli.go +++ b/internal/aqcli/aqcli.go @@ -4,7 +4,7 @@ import ( "bytes" "encoding/json" "fmt" - "github.com/superioz/aqua/internal/handler" + "github.com/superioz/aqua/internal/request" "github.com/superioz/aqua/pkg/shttp" "github.com/urfave/cli/v2" "io" @@ -98,7 +98,7 @@ var UploadCommand = &cli.Command{ return fmt.Errorf("could not open file: %v", err) } - id, err := doPostRequest(host, token, file, &handler.RequestMetadata{ + id, err := doPostRequest(host, token, file, &request.RequestMetadata{ Expiration: int64(expires), }) if err != nil { @@ -118,7 +118,7 @@ type postResponse struct { Id string } -func doPostRequest(host string, token string, file *os.File, metadata *handler.RequestMetadata) (string, error) { +func doPostRequest(host string, token string, file *os.File, metadata *request.RequestMetadata) (string, error) { md, err := json.Marshal(metadata) if err != nil { return "", err