ShareX
/
aqua
mirror of https://github.com/Superioz/aqua.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1 from Superioz/feature/discord-compatible 

Feature/discord compatible
This commit is contained in:
Tobias B 2021-11-18 02:42:32 +01:00 committed by GitHub
parent bbba0db935 0bb5618177
commit 69e9c0a0b5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 279 additions and 170 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.env.dist
View File

@ -4,5 +4,7 @@ FILE_NAME_LENGTH=12
FILE_MAX_SIZE=100
FILE_META_DB_PATH=
FILE_EXPIRATION_CYCLE=10
METRICS_ENABLED=true
FILE_SERVING_ENABLED=true
FILE_EXTENSIONS_RESPONSE=true
FILE_EXTENSIONS_EXCLUDED=image/png,image/jpeg
METRICS_ENABLED=true

39
README.md
View File

@ -28,14 +28,35 @@ This is only for those people, that are still living in the 90s or are not comfo
For further instructions like creating a service that can easily be started with `service aqua start`, please refer to other pages (there are a bunch that explain this) - I won't.
## Docker Compose
## Docker (Compose)
Before following the steps, make sure you have [Docker](https://docs.docker.com/get-docker/) and [Docker Compose](https://docs.docker.com/compose/install/) installed on the machine.
Before following the steps, make sure you have [Docker](https://docs.docker.com/get-docker/) installed. And if you want to use the preconfigured `docker-compose` file, install [Docker Compose](https://docs.docker.com/compose/install/) as well.
1. Clone the repository to a directory of your liking with `git clone git@github.com:Superioz/aqua.git`
2. Edit the `auth.yml` with your custom auth tokens and settings.
3. Rename the `.env.dist` to `.env` and edit it as well.
4. `docker-compose up` and you should see it up and running.
Before you can start anything, you need the basis configuration:
1. Download and edit the `auth.yml` with your custom auth tokens and settings.
2. Download and rename the `.env.dist` to `.env` and edit it as well.
For a quick start with Docker, you can use the following command:
```sh
docker run --rm \
--env-file ./.env \
-v ./auth.yml:/etc/aqua/auth.yml \
-v ./files:/var/lib/aqua/ \
-p 8765:8765 \
ghcr.io/superioz/aqua:latest
```
If you are on Windows, you might need to add `MSYS_NO_PATHCONV=1` so that the paths are correctly parsed. Also, to refer to the current directory, use `"$(pwd)"` instead of `./`, because that sometimes makes problems in Windows as well.
For Docker Compose it's easier:
```sh
docker-compose up
```
If you want to build the image yourself instead, you can of course `git clone git@github.com:Superioz/aqua.git` and then execute `docker-compose up --build`.
## Kubernetes
@ -61,8 +82,10 @@ For examplary usage of the environment variables, have a look at the `.env.dist`
| `FILE_MAX_SIZE` | Maximum size for uploaded files in Megabytes. |
| `FILE_META_DB_PATH` | Path to the directory, where the sqlite database for file metadata should be stored. Recommended to not be the same folder as `FILE_STORAGE_PATH` to prevent overlapping. |
| `FILE_EXPIRATION_CYCLE` | Determines the interval of the expiration cycle. `5` means that every 5 seconds the files will be checked for expiration. |
| `FILE_SERVING_ENABLED` | Defaults to `true`, if `false`, the server won't serve the stored files. |
| `FILE_EXTENSIONS_RESPONSE` | Defaults to `true`. if the file name returned will have its extension added to it. |
| `FILE_EXTENSIONS_EXCLUDED` | Comma-seperated list of MIME types, that should be excluded from the extension response rule above. Defaults to `image/png,image/jpeg` |
| `METRICS_ENABLED` | Is normally set to true, but otherwise disables the Prometheus metrics publishing. |
| `FILE_SERVING_ENABLED` | Defaults to true, when `false`, then the server won't serve the stored files. |
## Tokens
@ -161,7 +184,7 @@ After that you can import it to your custom upload goals in the ShareX UI.
"metadata": "{ \"expiration\": 3600 }"
},
"FileFormName": "file",
"URL": "https://your-domain.com/$json:id$"
"URL": "https://your-domain.com/$json:fileName$"
}
```

2
VERSION
View File

@ -1 +1 @@
0.6.0
0.6.2

2
cmd/aqua/main.go
View File

@ -15,7 +15,7 @@ import (
func main() {
err := godotenv.Load()
if err != nil {
klog.Warningf("Error loading .env file: %v", err)
klog.Warningf("Could not load .env file: %v", err)
}
klog.Infoln("Hello World!")

44
internal/aqcli/upload.go → internal/aqcli/aqcli.go
View File

@ -4,17 +4,55 @@ import (
"bytes"
"encoding/json"
"fmt"
"github.com/superioz/aqua/internal/handler"
"github.com/superioz/aqua/internal/request"
"github.com/superioz/aqua/pkg/shttp"
"github.com/urfave/cli/v2"
"io"
"io/ioutil"
"math/rand"
"net/http"
"os"
"strings"
"time"
)
var GenerateCommand = &cli.Command{
Name: "generate",
Aliases: []string{"gen"},
Usage: "Generates a possible auth token",
Flags: []cli.Flag{
&cli.IntFlag{
Name: "length",
Aliases: []string{"l"},
Value: 32,
Usage: "Length of the token",
},
},
Action: func(c *cli.Context) error {
size := c.Int("length")
if size <= 1 {
return cli.Exit("You cannot generate a token with this length. Must be >=2.", 1)
}
fmt.Println(generateToken(size))
return nil
},
}
const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-+?!#$&%"
// generateToken generates a random `size` long string from
// a predefined hexadecimal charset.
func generateToken(size int) string {
rand.Seed(time.Now().UnixNano())
b := make([]byte, size)
for i := range b {
b[i] = charset[rand.Intn(len(charset))]
}
return string(b)
}
var UploadCommand = &cli.Command{
Name: "upload",
Usage: "Uploads a file to the aqua server",
@ -60,7 +98,7 @@ var UploadCommand = &cli.Command{
return fmt.Errorf("could not open file: %v", err)
}
id, err := doPostRequest(host, token, file, &handler.RequestMetadata{
id, err := doPostRequest(host, token, file, &request.RequestMetadata{
Expiration: int64(expires),
})
if err != nil {
@ -80,7 +118,7 @@ type postResponse struct {
Id string
}
func doPostRequest(host string, token string, file *os.File, metadata *handler.RequestMetadata) (string, error) {
func doPostRequest(host string, token string, file *os.File, metadata *request.RequestMetadata) (string, error) {
md, err := json.Marshal(metadata)
if err != nil {
return "", err

45
internal/aqcli/generate.go
View File

@ -1,45 +0,0 @@
package aqcli
import (
"fmt"
"github.com/urfave/cli/v2"
"math/rand"
"time"
)
var GenerateCommand = &cli.Command{
Name: "generate",
Aliases: []string{"gen"},
Usage: "Generates a possible auth token",
Flags: []cli.Flag{
&cli.IntFlag{
Name: "length",
Aliases: []string{"l"},
Value: 32,
Usage: "Length of the token",
},
},
Action: func(c *cli.Context) error {
size := c.Int("length")
if size <= 1 {
return cli.Exit("You cannot generate a token with this length. Must be >=2.", 1)
}
fmt.Println(generateToken(size))
return nil
},
}
const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-+?!#$&%"
// generateToken generates a random `size` long string from
// a predefined hexadecimal charset.
func generateToken(size int) string {
rand.Seed(time.Now().UnixNano())
b := make([]byte, size)
for i := range b {
b[i] = charset[rand.Intn(len(charset))]
}
return string(b)
}

7
internal/config/config.go
View File

@ -6,6 +6,13 @@ import (
"os"
)
const (
ExpireNever = -1
EnvDefaultFileStoragePath = "/var/lib/aqua/files/"
EnvDefaultMetaDbPath = "/var/lib/aqua/"
)
type AuthConfig struct {
ValidTokens []*TokenConfig `yaml:"validTokens"`
}

106
internal/handler/handler.go
View File

@ -1,11 +1,12 @@
package handler
import (
"encoding/json"
"fmt"
"github.com/gin-gonic/gin"
"github.com/superioz/aqua/internal/config"
"github.com/superioz/aqua/internal/metrics"
"github.com/superioz/aqua/internal/mime"
"github.com/superioz/aqua/internal/request"
"github.com/superioz/aqua/internal/storage"
"github.com/superioz/aqua/pkg/env"
"k8s.io/klog"
@ -19,41 +20,13 @@ const (
SizeMegaByte = 1 << (10 * 2)
)
var (
// validMimeTypes is a whitelist of all supported
// mime types. Taken from https://developer.mozilla.org/
validMimeTypes = []string{
"application/pdf",
"application/json",
"application/gzip",
"application/vnd.rar",
"application/zip",
"application/x-7z-compressed",
"image/png",
"image/jpeg",
"image/gif",
"image/svg+xml",
"text/csv",
"text/plain",
"audio/mpeg",
"audio/ogg",
"audio/opus",
"audio/webm",
"video/mp4",
"video/mpeg",
"video/webm",
}
emptyRequestMetadata = &RequestMetadata{Expiration: storage.ExpireNever}
)
type RequestMetadata struct {
Expiration int64 `json:"expiration"`
}
type UploadHandler struct {
AuthConfig *config.AuthConfig
FileStorage *storage.FileStorage
// excluded as per defined by the environment variable
// FILE_EXTENSIONS_EXCEPT
exclMimeTypes []string
}
func NewUploadHandler() *UploadHandler {
@ -61,6 +34,7 @@ func NewUploadHandler() *UploadHandler {
handler.ReloadAuthConfig()
handler.FileStorage = storage.NewFileStorage()
handler.exclMimeTypes = env.ListOrDefault("FILE_EXTENSIONS_EXCLUDED", []string{"image/png", "image/jpeg"})
return handler
}
@ -100,19 +74,23 @@ func (h *UploadHandler) Upload(c *gin.Context) {
c.JSON(http.StatusBadRequest, gin.H{"msg": "too many files in form"})
return
}
if len(files) == 0 {
c.JSON(http.StatusBadRequest, gin.H{"msg": "no file in form"})
return
}
file := files[0]
if c.Request.Header.Get("Content-Length") == "" {
c.Status(http.StatusLengthRequired)
return
}
if c.Request.ContentLength > 50*SizeMegaByte {
if c.Request.ContentLength > int64(env.IntOrDefault("FILE_MAX_SIZE", 100))*SizeMegaByte {
c.JSON(http.StatusRequestEntityTooLarge, gin.H{"msg": "content size must not exceed 50mb"})
return
}
ct := getContentType(file)
if !isContentTypeValid(ct) {
if !mime.IsValid(ct) {
c.JSON(http.StatusBadRequest, gin.H{"msg": "content type of file is not valid"})
return
}
@ -133,8 +111,14 @@ func (h *UploadHandler) Upload(c *gin.Context) {
}
defer of.Close()
metadata := getMetadata(form)
sf, err := h.FileStorage.StoreFile(of, metadata.Expiration)
metadata := request.GetMetadata(form)
rff := &request.RequestFormFile{
File: of,
ContentType: ct,
ContentLength: c.Request.ContentLength,
}
sf, err := h.FileStorage.StoreFile(rff, metadata.Expiration)
if err != nil {
klog.Error(err)
c.JSON(http.StatusInternalServerError, gin.H{"msg": "could not store file"})
@ -148,22 +132,26 @@ func (h *UploadHandler) Upload(c *gin.Context) {
klog.Infof("Stored file %s (expiresIn: %s)", sf.Id, expiresIn)
metrics.IncFilesUploaded()
c.JSON(http.StatusOK, gin.H{"id": sf.Id})
// add extension to file name if it is enabled and
// the extension is not excluded
storedName := sf.Id
if env.BoolOrDefault("FILE_EXTENSIONS_RESPONSE", true) && !h.isExtensionExcluded(sf.MimeType) {
storedName = fmt.Sprintf("%s.%s", storedName, mime.GetExtension(sf.MimeType))
}
c.JSON(http.StatusOK, gin.H{"fileName": storedName})
}
func getMetadata(form *multipart.Form) *RequestMetadata {
metaRawList := form.Value["metadata"]
if len(metaRawList) == 0 {
return emptyRequestMetadata
// isExtensionExcluded returns if the given mime type
// should be excluded by the extension response rule, which states if
// an extension should be appended to the file name when responding.
func (h *UploadHandler) isExtensionExcluded(mimeType string) bool {
for _, exclMimeType := range h.exclMimeTypes {
if exclMimeType == mimeType {
return true
}
}
metaRaw := metaRawList[0]
var metadata *RequestMetadata
err := json.Unmarshal([]byte(metaRaw), &metadata)
if err != nil {
return emptyRequestMetadata
}
return metadata
return false
}
// workaround for file Content-Type headers
@ -178,15 +166,6 @@ func getContentType(f *multipart.FileHeader) string {
return c
}
func isContentTypeValid(ct string) bool {
for _, mt := range validMimeTypes {
if mt == ct {
return true
}
}
return false
}
func getToken(c *gin.Context) string {
// try to get the Bearer token, because it's the standard
// for authorization
@ -203,10 +182,17 @@ func getToken(c *gin.Context) string {
// HandleStaticFiles takes the files inside the configured file storage
// path and serves them to the client.
func HandleStaticFiles() gin.HandlerFunc {
fileStoragePath := env.StringOrDefault("FILE_STORAGE_PATH", storage.EnvDefaultFileStoragePath)
fileStoragePath := env.StringOrDefault("FILE_STORAGE_PATH", config.EnvDefaultFileStoragePath)
return func(c *gin.Context) {
fileName := c.Param("file")
// the file name could contain the extension
// we split it and ship it.
if strings.Contains(fileName, ".") {
fileName = strings.Split(fileName, ".")[0]
}
fullPath := fileStoragePath + fileName
f, err := os.Open(fullPath)

46
internal/mime/mime.go Normal file
View File

@ -0,0 +1,46 @@
package mime
var (
// Types is a whitelist of all supported
// mime types. Taken from https://developer.mozilla.org/
Types = map[string]string{
"application/pdf": "pdf",
"application/json": "json",
"application/gzip": "gz",
"application/vnd.rar": "rar",
"application/zip": "zip",
"application/x-7z-compressed": "7z",
"image/png": "png",
"image/jpeg": "jpg",
"image/gif": "gif",
"image/svg+xml": "svg",
"text/csv": "csv",
"text/plain": "txt",
"audio/mpeg": "mp3",
"audio/ogg": "ogg",
"audio/opus": "opus",
"audio/webm": "weba",
"video/mp4": "mp4",
"video/mpeg": "mpeg",
"video/webm": "webm",
}
)
// IsValid checks if given type is inside Types map
func IsValid(t string) bool {
for mt := range Types {
if mt == t {
return true
}
}
return false
}
// GetExtension returns an extension for the given MIME type
func GetExtension(t string) string {
ext, ok := Types[t]
if !ok {
return "application/octet-stream"
}
return ext
}

38
internal/request/request.go Normal file
View File

@ -0,0 +1,38 @@
package request
import (
"encoding/json"
"github.com/superioz/aqua/internal/config"
"mime/multipart"
)
var (
emptyRequestMetadata = &RequestMetadata{Expiration: config.ExpireNever}
)
// RequestFormFile is the metadata we get from the file
// which is requested to be uploaded.
type RequestFormFile struct {
File multipart.File
ContentType string
ContentLength int64
}
type RequestMetadata struct {
Expiration int64 `json:"expiration"`
}
func GetMetadata(form *multipart.Form) *RequestMetadata {
metaRawList := form.Value["metadata"]
if len(metaRawList) == 0 {
return emptyRequestMetadata
}
metaRaw := metaRawList[0]
var metadata *RequestMetadata
err := json.Unmarshal([]byte(metaRaw), &metadata)
if err != nil {
return emptyRequestMetadata
}
return metadata
}

63
internal/storage/meta.go → internal/storage/filemeta.go
View File

@ -48,7 +48,9 @@ func (s *SqliteFileMetaDatabase) Connect() error {
_, err = db.Exec(`create table if not exists files (
id text not null primary key,
uploaded_at integer,
expires_at integer
expires_at integer,
mime_type varchar,
size integer
);`)
if err != nil {
return err
@ -63,13 +65,13 @@ func (s *SqliteFileMetaDatabase) WriteFile(sf *StoredFile) error {
}
defer db.Close()
stmt, err := db.Prepare(`insert into files(id, uploaded_at, expires_at) values(?, ?, ?)`)
stmt, err := db.Prepare(`insert into files(id, uploaded_at, expires_at, mime_type, size) values(?, ?, ?, ?, ?)`)
if err != nil {
return err
}
defer stmt.Close()
_, err = stmt.Exec(sf.Id, sf.UploadedAt, sf.ExpiresAt)
_, err = stmt.Exec(sf.Id, sf.UploadedAt, sf.ExpiresAt, sf.MimeType, sf.Size)
if err != nil {
return err
}
@ -116,18 +118,10 @@ func (s *SqliteFileMetaDatabase) GetFile(id string) (*StoredFile, error) {
return nil, nil
}
var uploadedAt int
var expiresAt int
err = rows.Scan(&id, &uploadedAt, &expiresAt)
sf, err := getFromRows(rows)
if err != nil {
return nil, err
}
sf := &StoredFile{
Id: id,
UploadedAt: int64(uploadedAt),
ExpiresAt: int64(expiresAt),
}
return sf, nil
}
@ -146,19 +140,11 @@ func (s *SqliteFileMetaDatabase) GetAllFiles() ([]*StoredFile, error) {
var sfs []*StoredFile
for rows.Next() {
var id string
var uploadedAt int
var expiresAt int
err = rows.Scan(&id, &uploadedAt, &expiresAt)
sf, err := getFromRows(rows)
if err != nil {
return nil, err
}
sf := &StoredFile{
Id: id,
UploadedAt: int64(uploadedAt),
ExpiresAt: int64(expiresAt),
}
sfs = append(sfs, sf)
}
err = rows.Err()
@ -190,19 +176,11 @@ func (s *SqliteFileMetaDatabase) GetAllExpired() ([]*StoredFile, error) {
var sfs []*StoredFile
for rows.Next() {
var id string
var uploadedAt int
var expiresAt int
err = rows.Scan(&id, &uploadedAt, &expiresAt)
sf, err := getFromRows(rows)
if err != nil {
return nil, err
}
sf := &StoredFile{
Id: id,
UploadedAt: int64(uploadedAt),
ExpiresAt: int64(expiresAt),
}
sfs = append(sfs, sf)
}
err = rows.Err()
@ -211,3 +189,24 @@ func (s *SqliteFileMetaDatabase) GetAllExpired() ([]*StoredFile, error) {
}
return sfs, nil
}
func getFromRows(rows *sql.Rows) (*StoredFile, error) {
var id string
var uploadedAt int
var expiresAt int
var mimeType string
var size int
err := rows.Scan(&id, &uploadedAt, &expiresAt, &mimeType, &size)
if err != nil {
return nil, err
}
sf := &StoredFile{
Id: id,
UploadedAt: int64(uploadedAt),
ExpiresAt: int64(expiresAt),
MimeType: mimeType,
Size: int64(size),
}
return sf, nil
}

11
internal/storage/files.go → internal/storage/filesys.go
View File

@ -2,12 +2,15 @@ package storage
import (
"io"
"mime/multipart"
"os"
)
type FileSystem interface {
CreateFile(mf multipart.File, name string) (bool, error)
// CreateFile writes the content of given reader to a file
// with given name.
// Always returns if the file was partly written to disk.
CreateFile(r io.Reader, name string) (bool, error)
DeleteFile(id string) error
GetFile(id string) (*os.File, error)
Exists(id string) (bool, error)
@ -17,7 +20,7 @@ type LocalFileSystem struct {
FolderPath string
}
func (l LocalFileSystem) CreateFile(mf multipart.File, name string) (bool, error) {
func (l LocalFileSystem) CreateFile(r io.Reader, name string) (bool, error) {
err := os.MkdirAll(l.FolderPath, os.ModePerm)
if err != nil {
return false, err
@ -31,7 +34,7 @@ func (l LocalFileSystem) CreateFile(mf multipart.File, name string) (bool, error
// use io.Copy so that we don't have to load all the image into the memory.
// they get copied in smaller 32kb chunks.
_, err = io.Copy(f, mf)
_, err = io.Copy(f, r)
if err != nil {
return true, err
}

26
internal/storage/storage.go
View File

@ -5,27 +5,23 @@ import (
"errors"
"fmt"
"github.com/google/uuid"
"github.com/superioz/aqua/internal/config"
"github.com/superioz/aqua/internal/metrics"
"github.com/superioz/aqua/internal/request"
"github.com/superioz/aqua/pkg/env"
"k8s.io/klog"
"mime/multipart"
"strings"
"time"
_ "modernc.org/sqlite"
)
const (
ExpireNever = -1
EnvDefaultFileStoragePath = "/var/lib/aqua/files/"
EnvDefaultMetaDbPath = "/var/lib/aqua/"
)
type StoredFile struct {
Id string
UploadedAt int64
ExpiresAt int64
MimeType string
Size int64
}
func (sf *StoredFile) String() string {
@ -41,14 +37,14 @@ type FileStorage struct {
}
func NewFileStorage() *FileStorage {
metaDbFilePath := env.StringOrDefault("FILE_META_DB_PATH", EnvDefaultMetaDbPath)
metaDbFilePath := env.StringOrDefault("FILE_META_DB_PATH", config.EnvDefaultMetaDbPath)
fileMetaDb := NewSqliteFileMetaDatabase(metaDbFilePath)
err := fileMetaDb.Connect()
if err != nil {
klog.Errorf("Could not connect to file meta db: %v", err)
}
fileStoragePath := env.StringOrDefault("FILE_STORAGE_PATH", EnvDefaultFileStoragePath)
fileStoragePath := env.StringOrDefault("FILE_STORAGE_PATH", config.EnvDefaultFileStoragePath)
fileSystem := NewLocalFileStorage(fileStoragePath)
return &FileStorage{
@ -100,13 +96,13 @@ func (fs *FileStorage) Cleanup() error {
return nil
}
func (fs *FileStorage) StoreFile(of multipart.File, expiration int64) (*StoredFile, error) {
func (fs *FileStorage) StoreFile(rff *request.RequestFormFile, expiration int64) (*StoredFile, error) {
name, err := getRandomFileName(env.IntOrDefault("FILE_NAME_LENGTH", 8))
if err != nil {
return nil, errors.New("could not generate random name")
}
_, err = fs.fileSystem.CreateFile(of, name)
_, err = fs.fileSystem.CreateFile(rff.File, name)
if err != nil {
klog.Error(err)
return nil, errors.New("could not save file to system")
@ -114,14 +110,16 @@ func (fs *FileStorage) StoreFile(of multipart.File, expiration int64) (*StoredFi
currentTime := time.Now().Unix()
expAt := currentTime + expiration
if expiration == ExpireNever {
expAt = ExpireNever
if expiration == config.ExpireNever {
expAt = config.ExpireNever
}
sf := &StoredFile{
Id: name,
UploadedAt: currentTime,
ExpiresAt: expAt,
MimeType: rff.ContentType,
Size: rff.ContentLength,
}
// write to meta database

14
pkg/env/env.go vendored
View File

@ -3,6 +3,7 @@ package env
import (
"os"
"strconv"
"strings"
)
func String(name string) (string, bool) {
@ -57,3 +58,16 @@ func BoolOrDefault(name string, def bool) bool {
}
return b
}
func ListOrDefault(name string, def []string) []string {
s, ok := String(name)
if !ok {
return def
}
spl := strings.Split(s, ",")
if len(spl) == 0 {
return def
}
return spl
}

2
run.sh
View File

@ -15,4 +15,4 @@ fi
# we only need MSYS_NO_PATHCONV when running this script in Windows
# because otherwise Mingw will try to convert
# the given paths and that breaks everything.
MSYS_NO_PATHCONV=1 docker run -it --rm -p 8765:8765 $IMAGE
MSYS_NO_PATHCONV=1 docker run -it --rm -p 8765:8765 $IMAGE