mirror of https://github.com/Superioz/aqua.git
Merge pull request #1 from Superioz/feature/discord-compatible
Feature/discord compatible
This commit is contained in:
commit
69e9c0a0b5
|
@ -4,5 +4,7 @@ FILE_NAME_LENGTH=12
|
||||||
FILE_MAX_SIZE=100
|
FILE_MAX_SIZE=100
|
||||||
FILE_META_DB_PATH=
|
FILE_META_DB_PATH=
|
||||||
FILE_EXPIRATION_CYCLE=10
|
FILE_EXPIRATION_CYCLE=10
|
||||||
METRICS_ENABLED=true
|
|
||||||
FILE_SERVING_ENABLED=true
|
FILE_SERVING_ENABLED=true
|
||||||
|
FILE_EXTENSIONS_RESPONSE=true
|
||||||
|
FILE_EXTENSIONS_EXCLUDED=image/png,image/jpeg
|
||||||
|
METRICS_ENABLED=true
|
||||||
|
|
39
README.md
39
README.md
|
@ -28,14 +28,35 @@ This is only for those people, that are still living in the 90s or are not comfo
|
||||||
|
|
||||||
For further instructions like creating a service that can easily be started with `service aqua start`, please refer to other pages (there are a bunch that explain this) - I won't.
|
For further instructions like creating a service that can easily be started with `service aqua start`, please refer to other pages (there are a bunch that explain this) - I won't.
|
||||||
|
|
||||||
## Docker Compose
|
## Docker (Compose)
|
||||||
|
|
||||||
Before following the steps, make sure you have [Docker](https://docs.docker.com/get-docker/) and [Docker Compose](https://docs.docker.com/compose/install/) installed on the machine.
|
Before following the steps, make sure you have [Docker](https://docs.docker.com/get-docker/) installed. And if you want to use the preconfigured `docker-compose` file, install [Docker Compose](https://docs.docker.com/compose/install/) as well.
|
||||||
|
|
||||||
1. Clone the repository to a directory of your liking with `git clone git@github.com:Superioz/aqua.git`
|
Before you can start anything, you need the basis configuration:
|
||||||
2. Edit the `auth.yml` with your custom auth tokens and settings.
|
|
||||||
3. Rename the `.env.dist` to `.env` and edit it as well.
|
1. Download and edit the `auth.yml` with your custom auth tokens and settings.
|
||||||
4. `docker-compose up` and you should see it up and running.
|
2. Download and rename the `.env.dist` to `.env` and edit it as well.
|
||||||
|
|
||||||
|
For a quick start with Docker, you can use the following command:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
docker run --rm \
|
||||||
|
--env-file ./.env \
|
||||||
|
-v ./auth.yml:/etc/aqua/auth.yml \
|
||||||
|
-v ./files:/var/lib/aqua/ \
|
||||||
|
-p 8765:8765 \
|
||||||
|
ghcr.io/superioz/aqua:latest
|
||||||
|
```
|
||||||
|
|
||||||
|
If you are on Windows, you might need to add `MSYS_NO_PATHCONV=1` so that the paths are correctly parsed. Also, to refer to the current directory, use `"$(pwd)"` instead of `./`, because that sometimes makes problems in Windows as well.
|
||||||
|
|
||||||
|
For Docker Compose it's easier:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
docker-compose up
|
||||||
|
```
|
||||||
|
|
||||||
|
If you want to build the image yourself instead, you can of course `git clone git@github.com:Superioz/aqua.git` and then execute `docker-compose up --build`.
|
||||||
|
|
||||||
## Kubernetes
|
## Kubernetes
|
||||||
|
|
||||||
|
@ -61,8 +82,10 @@ For examplary usage of the environment variables, have a look at the `.env.dist`
|
||||||
| `FILE_MAX_SIZE` | Maximum size for uploaded files in Megabytes. |
|
| `FILE_MAX_SIZE` | Maximum size for uploaded files in Megabytes. |
|
||||||
| `FILE_META_DB_PATH` | Path to the directory, where the sqlite database for file metadata should be stored. Recommended to not be the same folder as `FILE_STORAGE_PATH` to prevent overlapping. |
|
| `FILE_META_DB_PATH` | Path to the directory, where the sqlite database for file metadata should be stored. Recommended to not be the same folder as `FILE_STORAGE_PATH` to prevent overlapping. |
|
||||||
| `FILE_EXPIRATION_CYCLE` | Determines the interval of the expiration cycle. `5` means that every 5 seconds the files will be checked for expiration. |
|
| `FILE_EXPIRATION_CYCLE` | Determines the interval of the expiration cycle. `5` means that every 5 seconds the files will be checked for expiration. |
|
||||||
|
| `FILE_SERVING_ENABLED` | Defaults to `true`, if `false`, the server won't serve the stored files. |
|
||||||
|
| `FILE_EXTENSIONS_RESPONSE` | Defaults to `true`. if the file name returned will have its extension added to it. |
|
||||||
|
| `FILE_EXTENSIONS_EXCLUDED` | Comma-seperated list of MIME types, that should be excluded from the extension response rule above. Defaults to `image/png,image/jpeg` |
|
||||||
| `METRICS_ENABLED` | Is normally set to true, but otherwise disables the Prometheus metrics publishing. |
|
| `METRICS_ENABLED` | Is normally set to true, but otherwise disables the Prometheus metrics publishing. |
|
||||||
| `FILE_SERVING_ENABLED` | Defaults to true, when `false`, then the server won't serve the stored files. |
|
|
||||||
|
|
||||||
## Tokens
|
## Tokens
|
||||||
|
|
||||||
|
@ -161,7 +184,7 @@ After that you can import it to your custom upload goals in the ShareX UI.
|
||||||
"metadata": "{ \"expiration\": 3600 }"
|
"metadata": "{ \"expiration\": 3600 }"
|
||||||
},
|
},
|
||||||
"FileFormName": "file",
|
"FileFormName": "file",
|
||||||
"URL": "https://your-domain.com/$json:id$"
|
"URL": "https://your-domain.com/$json:fileName$"
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
|
@ -15,7 +15,7 @@ import (
|
||||||
func main() {
|
func main() {
|
||||||
err := godotenv.Load()
|
err := godotenv.Load()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
klog.Warningf("Error loading .env file: %v", err)
|
klog.Warningf("Could not load .env file: %v", err)
|
||||||
}
|
}
|
||||||
klog.Infoln("Hello World!")
|
klog.Infoln("Hello World!")
|
||||||
|
|
||||||
|
|
|
@ -4,17 +4,55 @@ import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"github.com/superioz/aqua/internal/handler"
|
"github.com/superioz/aqua/internal/request"
|
||||||
"github.com/superioz/aqua/pkg/shttp"
|
"github.com/superioz/aqua/pkg/shttp"
|
||||||
"github.com/urfave/cli/v2"
|
"github.com/urfave/cli/v2"
|
||||||
"io"
|
"io"
|
||||||
"io/ioutil"
|
"io/ioutil"
|
||||||
|
"math/rand"
|
||||||
"net/http"
|
"net/http"
|
||||||
"os"
|
"os"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
var GenerateCommand = &cli.Command{
|
||||||
|
Name: "generate",
|
||||||
|
Aliases: []string{"gen"},
|
||||||
|
Usage: "Generates a possible auth token",
|
||||||
|
Flags: []cli.Flag{
|
||||||
|
&cli.IntFlag{
|
||||||
|
Name: "length",
|
||||||
|
Aliases: []string{"l"},
|
||||||
|
Value: 32,
|
||||||
|
Usage: "Length of the token",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
Action: func(c *cli.Context) error {
|
||||||
|
size := c.Int("length")
|
||||||
|
if size <= 1 {
|
||||||
|
return cli.Exit("You cannot generate a token with this length. Must be >=2.", 1)
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Println(generateToken(size))
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-+?!#$&%"
|
||||||
|
|
||||||
|
// generateToken generates a random `size` long string from
|
||||||
|
// a predefined hexadecimal charset.
|
||||||
|
func generateToken(size int) string {
|
||||||
|
rand.Seed(time.Now().UnixNano())
|
||||||
|
|
||||||
|
b := make([]byte, size)
|
||||||
|
for i := range b {
|
||||||
|
b[i] = charset[rand.Intn(len(charset))]
|
||||||
|
}
|
||||||
|
return string(b)
|
||||||
|
}
|
||||||
|
|
||||||
var UploadCommand = &cli.Command{
|
var UploadCommand = &cli.Command{
|
||||||
Name: "upload",
|
Name: "upload",
|
||||||
Usage: "Uploads a file to the aqua server",
|
Usage: "Uploads a file to the aqua server",
|
||||||
|
@ -60,7 +98,7 @@ var UploadCommand = &cli.Command{
|
||||||
return fmt.Errorf("could not open file: %v", err)
|
return fmt.Errorf("could not open file: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
id, err := doPostRequest(host, token, file, &handler.RequestMetadata{
|
id, err := doPostRequest(host, token, file, &request.RequestMetadata{
|
||||||
Expiration: int64(expires),
|
Expiration: int64(expires),
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -80,7 +118,7 @@ type postResponse struct {
|
||||||
Id string
|
Id string
|
||||||
}
|
}
|
||||||
|
|
||||||
func doPostRequest(host string, token string, file *os.File, metadata *handler.RequestMetadata) (string, error) {
|
func doPostRequest(host string, token string, file *os.File, metadata *request.RequestMetadata) (string, error) {
|
||||||
md, err := json.Marshal(metadata)
|
md, err := json.Marshal(metadata)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
|
@ -1,45 +0,0 @@
|
||||||
package aqcli
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"github.com/urfave/cli/v2"
|
|
||||||
"math/rand"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
var GenerateCommand = &cli.Command{
|
|
||||||
Name: "generate",
|
|
||||||
Aliases: []string{"gen"},
|
|
||||||
Usage: "Generates a possible auth token",
|
|
||||||
Flags: []cli.Flag{
|
|
||||||
&cli.IntFlag{
|
|
||||||
Name: "length",
|
|
||||||
Aliases: []string{"l"},
|
|
||||||
Value: 32,
|
|
||||||
Usage: "Length of the token",
|
|
||||||
},
|
|
||||||
},
|
|
||||||
Action: func(c *cli.Context) error {
|
|
||||||
size := c.Int("length")
|
|
||||||
if size <= 1 {
|
|
||||||
return cli.Exit("You cannot generate a token with this length. Must be >=2.", 1)
|
|
||||||
}
|
|
||||||
|
|
||||||
fmt.Println(generateToken(size))
|
|
||||||
return nil
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-+?!#$&%"
|
|
||||||
|
|
||||||
// generateToken generates a random `size` long string from
|
|
||||||
// a predefined hexadecimal charset.
|
|
||||||
func generateToken(size int) string {
|
|
||||||
rand.Seed(time.Now().UnixNano())
|
|
||||||
|
|
||||||
b := make([]byte, size)
|
|
||||||
for i := range b {
|
|
||||||
b[i] = charset[rand.Intn(len(charset))]
|
|
||||||
}
|
|
||||||
return string(b)
|
|
||||||
}
|
|
|
@ -6,6 +6,13 @@ import (
|
||||||
"os"
|
"os"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
ExpireNever = -1
|
||||||
|
|
||||||
|
EnvDefaultFileStoragePath = "/var/lib/aqua/files/"
|
||||||
|
EnvDefaultMetaDbPath = "/var/lib/aqua/"
|
||||||
|
)
|
||||||
|
|
||||||
type AuthConfig struct {
|
type AuthConfig struct {
|
||||||
ValidTokens []*TokenConfig `yaml:"validTokens"`
|
ValidTokens []*TokenConfig `yaml:"validTokens"`
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,11 +1,12 @@
|
||||||
package handler
|
package handler
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"github.com/gin-gonic/gin"
|
"github.com/gin-gonic/gin"
|
||||||
"github.com/superioz/aqua/internal/config"
|
"github.com/superioz/aqua/internal/config"
|
||||||
"github.com/superioz/aqua/internal/metrics"
|
"github.com/superioz/aqua/internal/metrics"
|
||||||
|
"github.com/superioz/aqua/internal/mime"
|
||||||
|
"github.com/superioz/aqua/internal/request"
|
||||||
"github.com/superioz/aqua/internal/storage"
|
"github.com/superioz/aqua/internal/storage"
|
||||||
"github.com/superioz/aqua/pkg/env"
|
"github.com/superioz/aqua/pkg/env"
|
||||||
"k8s.io/klog"
|
"k8s.io/klog"
|
||||||
|
@ -19,41 +20,13 @@ const (
|
||||||
SizeMegaByte = 1 << (10 * 2)
|
SizeMegaByte = 1 << (10 * 2)
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
|
||||||
// validMimeTypes is a whitelist of all supported
|
|
||||||
// mime types. Taken from https://developer.mozilla.org/
|
|
||||||
validMimeTypes = []string{
|
|
||||||
"application/pdf",
|
|
||||||
"application/json",
|
|
||||||
"application/gzip",
|
|
||||||
"application/vnd.rar",
|
|
||||||
"application/zip",
|
|
||||||
"application/x-7z-compressed",
|
|
||||||
"image/png",
|
|
||||||
"image/jpeg",
|
|
||||||
"image/gif",
|
|
||||||
"image/svg+xml",
|
|
||||||
"text/csv",
|
|
||||||
"text/plain",
|
|
||||||
"audio/mpeg",
|
|
||||||
"audio/ogg",
|
|
||||||
"audio/opus",
|
|
||||||
"audio/webm",
|
|
||||||
"video/mp4",
|
|
||||||
"video/mpeg",
|
|
||||||
"video/webm",
|
|
||||||
}
|
|
||||||
|
|
||||||
emptyRequestMetadata = &RequestMetadata{Expiration: storage.ExpireNever}
|
|
||||||
)
|
|
||||||
|
|
||||||
type RequestMetadata struct {
|
|
||||||
Expiration int64 `json:"expiration"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type UploadHandler struct {
|
type UploadHandler struct {
|
||||||
AuthConfig *config.AuthConfig
|
AuthConfig *config.AuthConfig
|
||||||
FileStorage *storage.FileStorage
|
FileStorage *storage.FileStorage
|
||||||
|
|
||||||
|
// excluded as per defined by the environment variable
|
||||||
|
// FILE_EXTENSIONS_EXCEPT
|
||||||
|
exclMimeTypes []string
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewUploadHandler() *UploadHandler {
|
func NewUploadHandler() *UploadHandler {
|
||||||
|
@ -61,6 +34,7 @@ func NewUploadHandler() *UploadHandler {
|
||||||
handler.ReloadAuthConfig()
|
handler.ReloadAuthConfig()
|
||||||
|
|
||||||
handler.FileStorage = storage.NewFileStorage()
|
handler.FileStorage = storage.NewFileStorage()
|
||||||
|
handler.exclMimeTypes = env.ListOrDefault("FILE_EXTENSIONS_EXCLUDED", []string{"image/png", "image/jpeg"})
|
||||||
return handler
|
return handler
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -100,19 +74,23 @@ func (h *UploadHandler) Upload(c *gin.Context) {
|
||||||
c.JSON(http.StatusBadRequest, gin.H{"msg": "too many files in form"})
|
c.JSON(http.StatusBadRequest, gin.H{"msg": "too many files in form"})
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if len(files) == 0 {
|
||||||
|
c.JSON(http.StatusBadRequest, gin.H{"msg": "no file in form"})
|
||||||
|
return
|
||||||
|
}
|
||||||
file := files[0]
|
file := files[0]
|
||||||
|
|
||||||
if c.Request.Header.Get("Content-Length") == "" {
|
if c.Request.Header.Get("Content-Length") == "" {
|
||||||
c.Status(http.StatusLengthRequired)
|
c.Status(http.StatusLengthRequired)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if c.Request.ContentLength > 50*SizeMegaByte {
|
if c.Request.ContentLength > int64(env.IntOrDefault("FILE_MAX_SIZE", 100))*SizeMegaByte {
|
||||||
c.JSON(http.StatusRequestEntityTooLarge, gin.H{"msg": "content size must not exceed 50mb"})
|
c.JSON(http.StatusRequestEntityTooLarge, gin.H{"msg": "content size must not exceed 50mb"})
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
ct := getContentType(file)
|
ct := getContentType(file)
|
||||||
if !isContentTypeValid(ct) {
|
if !mime.IsValid(ct) {
|
||||||
c.JSON(http.StatusBadRequest, gin.H{"msg": "content type of file is not valid"})
|
c.JSON(http.StatusBadRequest, gin.H{"msg": "content type of file is not valid"})
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
@ -133,8 +111,14 @@ func (h *UploadHandler) Upload(c *gin.Context) {
|
||||||
}
|
}
|
||||||
defer of.Close()
|
defer of.Close()
|
||||||
|
|
||||||
metadata := getMetadata(form)
|
metadata := request.GetMetadata(form)
|
||||||
sf, err := h.FileStorage.StoreFile(of, metadata.Expiration)
|
rff := &request.RequestFormFile{
|
||||||
|
File: of,
|
||||||
|
ContentType: ct,
|
||||||
|
ContentLength: c.Request.ContentLength,
|
||||||
|
}
|
||||||
|
|
||||||
|
sf, err := h.FileStorage.StoreFile(rff, metadata.Expiration)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
klog.Error(err)
|
klog.Error(err)
|
||||||
c.JSON(http.StatusInternalServerError, gin.H{"msg": "could not store file"})
|
c.JSON(http.StatusInternalServerError, gin.H{"msg": "could not store file"})
|
||||||
|
@ -148,22 +132,26 @@ func (h *UploadHandler) Upload(c *gin.Context) {
|
||||||
klog.Infof("Stored file %s (expiresIn: %s)", sf.Id, expiresIn)
|
klog.Infof("Stored file %s (expiresIn: %s)", sf.Id, expiresIn)
|
||||||
metrics.IncFilesUploaded()
|
metrics.IncFilesUploaded()
|
||||||
|
|
||||||
c.JSON(http.StatusOK, gin.H{"id": sf.Id})
|
// add extension to file name if it is enabled and
|
||||||
|
// the extension is not excluded
|
||||||
|
storedName := sf.Id
|
||||||
|
if env.BoolOrDefault("FILE_EXTENSIONS_RESPONSE", true) && !h.isExtensionExcluded(sf.MimeType) {
|
||||||
|
storedName = fmt.Sprintf("%s.%s", storedName, mime.GetExtension(sf.MimeType))
|
||||||
|
}
|
||||||
|
|
||||||
|
c.JSON(http.StatusOK, gin.H{"fileName": storedName})
|
||||||
}
|
}
|
||||||
|
|
||||||
func getMetadata(form *multipart.Form) *RequestMetadata {
|
// isExtensionExcluded returns if the given mime type
|
||||||
metaRawList := form.Value["metadata"]
|
// should be excluded by the extension response rule, which states if
|
||||||
if len(metaRawList) == 0 {
|
// an extension should be appended to the file name when responding.
|
||||||
return emptyRequestMetadata
|
func (h *UploadHandler) isExtensionExcluded(mimeType string) bool {
|
||||||
|
for _, exclMimeType := range h.exclMimeTypes {
|
||||||
|
if exclMimeType == mimeType {
|
||||||
|
return true
|
||||||
|
}
|
||||||
}
|
}
|
||||||
metaRaw := metaRawList[0]
|
return false
|
||||||
|
|
||||||
var metadata *RequestMetadata
|
|
||||||
err := json.Unmarshal([]byte(metaRaw), &metadata)
|
|
||||||
if err != nil {
|
|
||||||
return emptyRequestMetadata
|
|
||||||
}
|
|
||||||
return metadata
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// workaround for file Content-Type headers
|
// workaround for file Content-Type headers
|
||||||
|
@ -178,15 +166,6 @@ func getContentType(f *multipart.FileHeader) string {
|
||||||
return c
|
return c
|
||||||
}
|
}
|
||||||
|
|
||||||
func isContentTypeValid(ct string) bool {
|
|
||||||
for _, mt := range validMimeTypes {
|
|
||||||
if mt == ct {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
func getToken(c *gin.Context) string {
|
func getToken(c *gin.Context) string {
|
||||||
// try to get the Bearer token, because it's the standard
|
// try to get the Bearer token, because it's the standard
|
||||||
// for authorization
|
// for authorization
|
||||||
|
@ -203,10 +182,17 @@ func getToken(c *gin.Context) string {
|
||||||
// HandleStaticFiles takes the files inside the configured file storage
|
// HandleStaticFiles takes the files inside the configured file storage
|
||||||
// path and serves them to the client.
|
// path and serves them to the client.
|
||||||
func HandleStaticFiles() gin.HandlerFunc {
|
func HandleStaticFiles() gin.HandlerFunc {
|
||||||
fileStoragePath := env.StringOrDefault("FILE_STORAGE_PATH", storage.EnvDefaultFileStoragePath)
|
fileStoragePath := env.StringOrDefault("FILE_STORAGE_PATH", config.EnvDefaultFileStoragePath)
|
||||||
|
|
||||||
return func(c *gin.Context) {
|
return func(c *gin.Context) {
|
||||||
fileName := c.Param("file")
|
fileName := c.Param("file")
|
||||||
|
|
||||||
|
// the file name could contain the extension
|
||||||
|
// we split it and ship it.
|
||||||
|
if strings.Contains(fileName, ".") {
|
||||||
|
fileName = strings.Split(fileName, ".")[0]
|
||||||
|
}
|
||||||
|
|
||||||
fullPath := fileStoragePath + fileName
|
fullPath := fileStoragePath + fileName
|
||||||
|
|
||||||
f, err := os.Open(fullPath)
|
f, err := os.Open(fullPath)
|
||||||
|
|
|
@ -0,0 +1,46 @@
|
||||||
|
package mime
|
||||||
|
|
||||||
|
var (
|
||||||
|
// Types is a whitelist of all supported
|
||||||
|
// mime types. Taken from https://developer.mozilla.org/
|
||||||
|
Types = map[string]string{
|
||||||
|
"application/pdf": "pdf",
|
||||||
|
"application/json": "json",
|
||||||
|
"application/gzip": "gz",
|
||||||
|
"application/vnd.rar": "rar",
|
||||||
|
"application/zip": "zip",
|
||||||
|
"application/x-7z-compressed": "7z",
|
||||||
|
"image/png": "png",
|
||||||
|
"image/jpeg": "jpg",
|
||||||
|
"image/gif": "gif",
|
||||||
|
"image/svg+xml": "svg",
|
||||||
|
"text/csv": "csv",
|
||||||
|
"text/plain": "txt",
|
||||||
|
"audio/mpeg": "mp3",
|
||||||
|
"audio/ogg": "ogg",
|
||||||
|
"audio/opus": "opus",
|
||||||
|
"audio/webm": "weba",
|
||||||
|
"video/mp4": "mp4",
|
||||||
|
"video/mpeg": "mpeg",
|
||||||
|
"video/webm": "webm",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
// IsValid checks if given type is inside Types map
|
||||||
|
func IsValid(t string) bool {
|
||||||
|
for mt := range Types {
|
||||||
|
if mt == t {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetExtension returns an extension for the given MIME type
|
||||||
|
func GetExtension(t string) string {
|
||||||
|
ext, ok := Types[t]
|
||||||
|
if !ok {
|
||||||
|
return "application/octet-stream"
|
||||||
|
}
|
||||||
|
return ext
|
||||||
|
}
|
|
@ -0,0 +1,38 @@
|
||||||
|
package request
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"github.com/superioz/aqua/internal/config"
|
||||||
|
"mime/multipart"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
emptyRequestMetadata = &RequestMetadata{Expiration: config.ExpireNever}
|
||||||
|
)
|
||||||
|
|
||||||
|
// RequestFormFile is the metadata we get from the file
|
||||||
|
// which is requested to be uploaded.
|
||||||
|
type RequestFormFile struct {
|
||||||
|
File multipart.File
|
||||||
|
ContentType string
|
||||||
|
ContentLength int64
|
||||||
|
}
|
||||||
|
|
||||||
|
type RequestMetadata struct {
|
||||||
|
Expiration int64 `json:"expiration"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func GetMetadata(form *multipart.Form) *RequestMetadata {
|
||||||
|
metaRawList := form.Value["metadata"]
|
||||||
|
if len(metaRawList) == 0 {
|
||||||
|
return emptyRequestMetadata
|
||||||
|
}
|
||||||
|
metaRaw := metaRawList[0]
|
||||||
|
|
||||||
|
var metadata *RequestMetadata
|
||||||
|
err := json.Unmarshal([]byte(metaRaw), &metadata)
|
||||||
|
if err != nil {
|
||||||
|
return emptyRequestMetadata
|
||||||
|
}
|
||||||
|
return metadata
|
||||||
|
}
|
|
@ -48,7 +48,9 @@ func (s *SqliteFileMetaDatabase) Connect() error {
|
||||||
_, err = db.Exec(`create table if not exists files (
|
_, err = db.Exec(`create table if not exists files (
|
||||||
id text not null primary key,
|
id text not null primary key,
|
||||||
uploaded_at integer,
|
uploaded_at integer,
|
||||||
expires_at integer
|
expires_at integer,
|
||||||
|
mime_type varchar,
|
||||||
|
size integer
|
||||||
);`)
|
);`)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
|
@ -63,13 +65,13 @@ func (s *SqliteFileMetaDatabase) WriteFile(sf *StoredFile) error {
|
||||||
}
|
}
|
||||||
defer db.Close()
|
defer db.Close()
|
||||||
|
|
||||||
stmt, err := db.Prepare(`insert into files(id, uploaded_at, expires_at) values(?, ?, ?)`)
|
stmt, err := db.Prepare(`insert into files(id, uploaded_at, expires_at, mime_type, size) values(?, ?, ?, ?, ?)`)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
defer stmt.Close()
|
defer stmt.Close()
|
||||||
|
|
||||||
_, err = stmt.Exec(sf.Id, sf.UploadedAt, sf.ExpiresAt)
|
_, err = stmt.Exec(sf.Id, sf.UploadedAt, sf.ExpiresAt, sf.MimeType, sf.Size)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
@ -116,18 +118,10 @@ func (s *SqliteFileMetaDatabase) GetFile(id string) (*StoredFile, error) {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
var uploadedAt int
|
sf, err := getFromRows(rows)
|
||||||
var expiresAt int
|
|
||||||
|
|
||||||
err = rows.Scan(&id, &uploadedAt, &expiresAt)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
sf := &StoredFile{
|
|
||||||
Id: id,
|
|
||||||
UploadedAt: int64(uploadedAt),
|
|
||||||
ExpiresAt: int64(expiresAt),
|
|
||||||
}
|
|
||||||
return sf, nil
|
return sf, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -146,19 +140,11 @@ func (s *SqliteFileMetaDatabase) GetAllFiles() ([]*StoredFile, error) {
|
||||||
|
|
||||||
var sfs []*StoredFile
|
var sfs []*StoredFile
|
||||||
for rows.Next() {
|
for rows.Next() {
|
||||||
var id string
|
sf, err := getFromRows(rows)
|
||||||
var uploadedAt int
|
|
||||||
var expiresAt int
|
|
||||||
|
|
||||||
err = rows.Scan(&id, &uploadedAt, &expiresAt)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
sf := &StoredFile{
|
|
||||||
Id: id,
|
|
||||||
UploadedAt: int64(uploadedAt),
|
|
||||||
ExpiresAt: int64(expiresAt),
|
|
||||||
}
|
|
||||||
sfs = append(sfs, sf)
|
sfs = append(sfs, sf)
|
||||||
}
|
}
|
||||||
err = rows.Err()
|
err = rows.Err()
|
||||||
|
@ -190,19 +176,11 @@ func (s *SqliteFileMetaDatabase) GetAllExpired() ([]*StoredFile, error) {
|
||||||
|
|
||||||
var sfs []*StoredFile
|
var sfs []*StoredFile
|
||||||
for rows.Next() {
|
for rows.Next() {
|
||||||
var id string
|
sf, err := getFromRows(rows)
|
||||||
var uploadedAt int
|
|
||||||
var expiresAt int
|
|
||||||
|
|
||||||
err = rows.Scan(&id, &uploadedAt, &expiresAt)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
sf := &StoredFile{
|
|
||||||
Id: id,
|
|
||||||
UploadedAt: int64(uploadedAt),
|
|
||||||
ExpiresAt: int64(expiresAt),
|
|
||||||
}
|
|
||||||
sfs = append(sfs, sf)
|
sfs = append(sfs, sf)
|
||||||
}
|
}
|
||||||
err = rows.Err()
|
err = rows.Err()
|
||||||
|
@ -211,3 +189,24 @@ func (s *SqliteFileMetaDatabase) GetAllExpired() ([]*StoredFile, error) {
|
||||||
}
|
}
|
||||||
return sfs, nil
|
return sfs, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func getFromRows(rows *sql.Rows) (*StoredFile, error) {
|
||||||
|
var id string
|
||||||
|
var uploadedAt int
|
||||||
|
var expiresAt int
|
||||||
|
var mimeType string
|
||||||
|
var size int
|
||||||
|
|
||||||
|
err := rows.Scan(&id, &uploadedAt, &expiresAt, &mimeType, &size)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
sf := &StoredFile{
|
||||||
|
Id: id,
|
||||||
|
UploadedAt: int64(uploadedAt),
|
||||||
|
ExpiresAt: int64(expiresAt),
|
||||||
|
MimeType: mimeType,
|
||||||
|
Size: int64(size),
|
||||||
|
}
|
||||||
|
return sf, nil
|
||||||
|
}
|
|
@ -2,12 +2,15 @@ package storage
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"io"
|
"io"
|
||||||
"mime/multipart"
|
|
||||||
"os"
|
"os"
|
||||||
)
|
)
|
||||||
|
|
||||||
type FileSystem interface {
|
type FileSystem interface {
|
||||||
CreateFile(mf multipart.File, name string) (bool, error)
|
// CreateFile writes the content of given reader to a file
|
||||||
|
// with given name.
|
||||||
|
// Always returns if the file was partly written to disk.
|
||||||
|
CreateFile(r io.Reader, name string) (bool, error)
|
||||||
|
|
||||||
DeleteFile(id string) error
|
DeleteFile(id string) error
|
||||||
GetFile(id string) (*os.File, error)
|
GetFile(id string) (*os.File, error)
|
||||||
Exists(id string) (bool, error)
|
Exists(id string) (bool, error)
|
||||||
|
@ -17,7 +20,7 @@ type LocalFileSystem struct {
|
||||||
FolderPath string
|
FolderPath string
|
||||||
}
|
}
|
||||||
|
|
||||||
func (l LocalFileSystem) CreateFile(mf multipart.File, name string) (bool, error) {
|
func (l LocalFileSystem) CreateFile(r io.Reader, name string) (bool, error) {
|
||||||
err := os.MkdirAll(l.FolderPath, os.ModePerm)
|
err := os.MkdirAll(l.FolderPath, os.ModePerm)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return false, err
|
||||||
|
@ -31,7 +34,7 @@ func (l LocalFileSystem) CreateFile(mf multipart.File, name string) (bool, error
|
||||||
|
|
||||||
// use io.Copy so that we don't have to load all the image into the memory.
|
// use io.Copy so that we don't have to load all the image into the memory.
|
||||||
// they get copied in smaller 32kb chunks.
|
// they get copied in smaller 32kb chunks.
|
||||||
_, err = io.Copy(f, mf)
|
_, err = io.Copy(f, r)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return true, err
|
return true, err
|
||||||
}
|
}
|
|
@ -5,27 +5,23 @@ import (
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
|
"github.com/superioz/aqua/internal/config"
|
||||||
"github.com/superioz/aqua/internal/metrics"
|
"github.com/superioz/aqua/internal/metrics"
|
||||||
|
"github.com/superioz/aqua/internal/request"
|
||||||
"github.com/superioz/aqua/pkg/env"
|
"github.com/superioz/aqua/pkg/env"
|
||||||
"k8s.io/klog"
|
"k8s.io/klog"
|
||||||
"mime/multipart"
|
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
_ "modernc.org/sqlite"
|
_ "modernc.org/sqlite"
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
|
||||||
ExpireNever = -1
|
|
||||||
|
|
||||||
EnvDefaultFileStoragePath = "/var/lib/aqua/files/"
|
|
||||||
EnvDefaultMetaDbPath = "/var/lib/aqua/"
|
|
||||||
)
|
|
||||||
|
|
||||||
type StoredFile struct {
|
type StoredFile struct {
|
||||||
Id string
|
Id string
|
||||||
UploadedAt int64
|
UploadedAt int64
|
||||||
ExpiresAt int64
|
ExpiresAt int64
|
||||||
|
MimeType string
|
||||||
|
Size int64
|
||||||
}
|
}
|
||||||
|
|
||||||
func (sf *StoredFile) String() string {
|
func (sf *StoredFile) String() string {
|
||||||
|
@ -41,14 +37,14 @@ type FileStorage struct {
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewFileStorage() *FileStorage {
|
func NewFileStorage() *FileStorage {
|
||||||
metaDbFilePath := env.StringOrDefault("FILE_META_DB_PATH", EnvDefaultMetaDbPath)
|
metaDbFilePath := env.StringOrDefault("FILE_META_DB_PATH", config.EnvDefaultMetaDbPath)
|
||||||
fileMetaDb := NewSqliteFileMetaDatabase(metaDbFilePath)
|
fileMetaDb := NewSqliteFileMetaDatabase(metaDbFilePath)
|
||||||
err := fileMetaDb.Connect()
|
err := fileMetaDb.Connect()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
klog.Errorf("Could not connect to file meta db: %v", err)
|
klog.Errorf("Could not connect to file meta db: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
fileStoragePath := env.StringOrDefault("FILE_STORAGE_PATH", EnvDefaultFileStoragePath)
|
fileStoragePath := env.StringOrDefault("FILE_STORAGE_PATH", config.EnvDefaultFileStoragePath)
|
||||||
fileSystem := NewLocalFileStorage(fileStoragePath)
|
fileSystem := NewLocalFileStorage(fileStoragePath)
|
||||||
|
|
||||||
return &FileStorage{
|
return &FileStorage{
|
||||||
|
@ -100,13 +96,13 @@ func (fs *FileStorage) Cleanup() error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (fs *FileStorage) StoreFile(of multipart.File, expiration int64) (*StoredFile, error) {
|
func (fs *FileStorage) StoreFile(rff *request.RequestFormFile, expiration int64) (*StoredFile, error) {
|
||||||
name, err := getRandomFileName(env.IntOrDefault("FILE_NAME_LENGTH", 8))
|
name, err := getRandomFileName(env.IntOrDefault("FILE_NAME_LENGTH", 8))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, errors.New("could not generate random name")
|
return nil, errors.New("could not generate random name")
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err = fs.fileSystem.CreateFile(of, name)
|
_, err = fs.fileSystem.CreateFile(rff.File, name)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
klog.Error(err)
|
klog.Error(err)
|
||||||
return nil, errors.New("could not save file to system")
|
return nil, errors.New("could not save file to system")
|
||||||
|
@ -114,14 +110,16 @@ func (fs *FileStorage) StoreFile(of multipart.File, expiration int64) (*StoredFi
|
||||||
|
|
||||||
currentTime := time.Now().Unix()
|
currentTime := time.Now().Unix()
|
||||||
expAt := currentTime + expiration
|
expAt := currentTime + expiration
|
||||||
if expiration == ExpireNever {
|
if expiration == config.ExpireNever {
|
||||||
expAt = ExpireNever
|
expAt = config.ExpireNever
|
||||||
}
|
}
|
||||||
|
|
||||||
sf := &StoredFile{
|
sf := &StoredFile{
|
||||||
Id: name,
|
Id: name,
|
||||||
UploadedAt: currentTime,
|
UploadedAt: currentTime,
|
||||||
ExpiresAt: expAt,
|
ExpiresAt: expAt,
|
||||||
|
MimeType: rff.ContentType,
|
||||||
|
Size: rff.ContentLength,
|
||||||
}
|
}
|
||||||
|
|
||||||
// write to meta database
|
// write to meta database
|
||||||
|
|
|
@ -3,6 +3,7 @@ package env
|
||||||
import (
|
import (
|
||||||
"os"
|
"os"
|
||||||
"strconv"
|
"strconv"
|
||||||
|
"strings"
|
||||||
)
|
)
|
||||||
|
|
||||||
func String(name string) (string, bool) {
|
func String(name string) (string, bool) {
|
||||||
|
@ -57,3 +58,16 @@ func BoolOrDefault(name string, def bool) bool {
|
||||||
}
|
}
|
||||||
return b
|
return b
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func ListOrDefault(name string, def []string) []string {
|
||||||
|
s, ok := String(name)
|
||||||
|
if !ok {
|
||||||
|
return def
|
||||||
|
}
|
||||||
|
|
||||||
|
spl := strings.Split(s, ",")
|
||||||
|
if len(spl) == 0 {
|
||||||
|
return def
|
||||||
|
}
|
||||||
|
return spl
|
||||||
|
}
|
||||||
|
|
2
run.sh
2
run.sh
|
@ -15,4 +15,4 @@ fi
|
||||||
# we only need MSYS_NO_PATHCONV when running this script in Windows
|
# we only need MSYS_NO_PATHCONV when running this script in Windows
|
||||||
# because otherwise Mingw will try to convert
|
# because otherwise Mingw will try to convert
|
||||||
# the given paths and that breaks everything.
|
# the given paths and that breaks everything.
|
||||||
MSYS_NO_PATHCONV=1 docker run -it --rm -p 8765:8765 $IMAGE
|
MSYS_NO_PATHCONV=1 docker run -it --rm -p 8765:8765 $IMAGE
|
||||||
|
|
Loading…
Reference in New Issue