ShareX
/
aqua
mirror of https://github.com/Superioz/aqua.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Many small improvements

This commit is contained in:
Tobias B 2021-11-18 01:14:48 +01:00
parent ec65d88252
commit 1a2dbf266d
No known key found for this signature in database
GPG Key ID: 5EF4C92355A3B53D
9 changed files with 171 additions and 127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
README.md
View File

@ -28,14 +28,35 @@ This is only for those people, that are still living in the 90s or are not comfo
For further instructions like creating a service that can easily be started with `service aqua start`, please refer to other pages (there are a bunch that explain this) - I won't.
## Docker Compose
## Docker (Compose)
Before following the steps, make sure you have [Docker](https://docs.docker.com/get-docker/) and [Docker Compose](https://docs.docker.com/compose/install/) installed on the machine.
Before following the steps, make sure you have [Docker](https://docs.docker.com/get-docker/) installed. And if you want to use the preconfigured `docker-compose` file, install [Docker Compose](https://docs.docker.com/compose/install/) as well.
1. Clone the repository to a directory of your liking with `git clone git@github.com:Superioz/aqua.git`
2. Edit the `auth.yml` with your custom auth tokens and settings.
3. Rename the `.env.dist` to `.env` and edit it as well.
4. `docker-compose up` and you should see it up and running.
Before you can start anything, you need the basis configuration:
1. Download and edit the `auth.yml` with your custom auth tokens and settings.
2. Download and rename the `.env.dist` to `.env` and edit it as well.
For a quick start with Docker, you can use the following command:
```sh
docker run --rm \
--env-file ./.env \
-v ./auth.yml:/etc/aqua/auth.yml \
-v ./files:/var/lib/aqua/ \
-p 8765:8765 \
ghcr.io/superioz/aqua:latest
```
If you are on Windows, you might need to add `MSYS_NO_PATHCONV=1` so that the paths are correctly parsed. Also, to refer to the current directory, use `"$(pwd)"` instead of `./`, because that sometimes makes problems in Windows as well.
For Docker Compose it's easier:
```sh
docker-compose up
```
If you want to build the image yourself instead, you can of course `git clone git@github.com:Superioz/aqua.git` and then execute `docker-compose up --build`.
## Kubernetes

38
internal/aqcli/upload.go → internal/aqcli/aqcli.go
View File

@ -9,12 +9,50 @@ import (
"github.com/urfave/cli/v2"
"io"
"io/ioutil"
"math/rand"
"net/http"
"os"
"strings"
"time"
)
var GenerateCommand = &cli.Command{
Name: "generate",
Aliases: []string{"gen"},
Usage: "Generates a possible auth token",
Flags: []cli.Flag{
&cli.IntFlag{
Name: "length",
Aliases: []string{"l"},
Value: 32,
Usage: "Length of the token",
},
},
Action: func(c *cli.Context) error {
size := c.Int("length")
if size <= 1 {
return cli.Exit("You cannot generate a token with this length. Must be >=2.", 1)
}
fmt.Println(generateToken(size))
return nil
},
}
const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-+?!#$&%"
// generateToken generates a random `size` long string from
// a predefined hexadecimal charset.
func generateToken(size int) string {
rand.Seed(time.Now().UnixNano())
b := make([]byte, size)
for i := range b {
b[i] = charset[rand.Intn(len(charset))]
}
return string(b)
}
var UploadCommand = &cli.Command{
Name: "upload",
Usage: "Uploads a file to the aqua server",

45
internal/aqcli/generate.go
View File

@ -1,45 +0,0 @@
package aqcli
import (
"fmt"
"github.com/urfave/cli/v2"
"math/rand"
"time"
)
var GenerateCommand = &cli.Command{
Name: "generate",
Aliases: []string{"gen"},
Usage: "Generates a possible auth token",
Flags: []cli.Flag{
&cli.IntFlag{
Name: "length",
Aliases: []string{"l"},
Value: 32,
Usage: "Length of the token",
},
},
Action: func(c *cli.Context) error {
size := c.Int("length")
if size <= 1 {
return cli.Exit("You cannot generate a token with this length. Must be >=2.", 1)
}
fmt.Println(generateToken(size))
return nil
},
}
const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-+?!#$&%"
// generateToken generates a random `size` long string from
// a predefined hexadecimal charset.
func generateToken(size int) string {
rand.Seed(time.Now().UnixNano())
b := make([]byte, size)
for i := range b {
b[i] = charset[rand.Intn(len(charset))]
}
return string(b)
}

50
internal/handler/handler.go
View File

@ -6,6 +6,7 @@ import (
"github.com/gin-gonic/gin"
"github.com/superioz/aqua/internal/config"
"github.com/superioz/aqua/internal/metrics"
"github.com/superioz/aqua/internal/mime"
"github.com/superioz/aqua/internal/storage"
"github.com/superioz/aqua/pkg/env"
"k8s.io/klog"
@ -20,33 +21,13 @@ const (
)
var (
// validMimeTypes is a whitelist of all supported
// mime types. Taken from https://developer.mozilla.org/
validMimeTypes = []string{
"application/pdf",
"application/json",
"application/gzip",
"application/vnd.rar",
"application/zip",
"application/x-7z-compressed",
"image/png",
"image/jpeg",
"image/gif",
"image/svg+xml",
"text/csv",
"text/plain",
"audio/mpeg",
"audio/ogg",
"audio/opus",
"audio/webm",
"video/mp4",
"video/mpeg",
"video/webm",
}
emptyRequestMetadata = &RequestMetadata{Expiration: storage.ExpireNever}
)
// TODO when accessing: `/N2YwODUx.mp4` => `/N2YwODUx`
// RequestFormFile is the metadata we get from the file
// which is requested to be uploaded.
type RequestFormFile struct {
File multipart.File
ContentType string
@ -116,13 +97,13 @@ func (h *UploadHandler) Upload(c *gin.Context) {
c.Status(http.StatusLengthRequired)
return
}
if c.Request.ContentLength > 50*SizeMegaByte {
if c.Request.ContentLength > int64(env.IntOrDefault("FILE_MAX_SIZE", 100))*SizeMegaByte {
c.JSON(http.StatusRequestEntityTooLarge, gin.H{"msg": "content size must not exceed 50mb"})
return
}
ct := getContentType(file)
if !isContentTypeValid(ct) {
if !mime.IsValid(ct) {
c.JSON(http.StatusBadRequest, gin.H{"msg": "content type of file is not valid"})
return
}
@ -144,7 +125,13 @@ func (h *UploadHandler) Upload(c *gin.Context) {
defer of.Close()
metadata := getMetadata(form)
sf, err := h.FileStorage.StoreFile(of, metadata.Expiration)
rff := &RequestFormFile{
File: of,
ContentType: ct,
ContentLength: c.Request.ContentLength,
}
sf, err := h.FileStorage.StoreFile(rff, metadata.Expiration)
if err != nil {
klog.Error(err)
c.JSON(http.StatusInternalServerError, gin.H{"msg": "could not store file"})
@ -188,15 +175,6 @@ func getContentType(f *multipart.FileHeader) string {
return c
}
func isContentTypeValid(ct string) bool {
for _, mt := range validMimeTypes {
if mt == ct {
return true
}
}
return false
}
func getToken(c *gin.Context) string {
// try to get the Bearer token, because it's the standard
// for authorization

46
internal/mime/mime.go Normal file
View File

@ -0,0 +1,46 @@
package mime
var (
// Types is a whitelist of all supported
// mime types. Taken from https://developer.mozilla.org/
Types = map[string]string{
"application/pdf": "pdf",
"application/json": "json",
"application/gzip": "gz",
"application/vnd.rar": "rar",
"application/zip": "zip",
"application/x-7z-compressed": "7z",
"image/png": "png",
"image/jpeg": "jpg",
"image/gif": "gif",
"image/svg+xml": "svg",
"text/csv": "csv",
"text/plain": "txt",
"audio/mpeg": "mp3",
"audio/ogg": "ogg",
"audio/opus": "opus",
"audio/webm": "weba",
"video/mp4": "mp4",
"video/mpeg": "mpeg",
"video/webm": "webm",
}
)
// IsValid checks if given type is inside Types map
func IsValid(t string) bool {
for _, mt := range Types {
if mt == t {
return true
}
}
return false
}
// GetExtension returns an extension for the given MIME type
func GetExtension(t string) string {
ext, ok := Types[t]
if !ok {
return "application/octet-stream"
}
return ext
}

63
internal/storage/meta.go → internal/storage/filemeta.go
View File

@ -48,7 +48,9 @@ func (s *SqliteFileMetaDatabase) Connect() error {
_, err = db.Exec(`create table if not exists files (
id text not null primary key,
uploaded_at integer,
expires_at integer
expires_at integer,
mime_type varchar,
size integer
);`)
if err != nil {
return err
@ -63,13 +65,13 @@ func (s *SqliteFileMetaDatabase) WriteFile(sf *StoredFile) error {
}
defer db.Close()
stmt, err := db.Prepare(`insert into files(id, uploaded_at, expires_at) values(?, ?, ?)`)
stmt, err := db.Prepare(`insert into files(id, uploaded_at, expires_at, mime_type, size) values(?, ?, ?, ?, ?)`)
if err != nil {
return err
}
defer stmt.Close()
_, err = stmt.Exec(sf.Id, sf.UploadedAt, sf.ExpiresAt)
_, err = stmt.Exec(sf.Id, sf.UploadedAt, sf.ExpiresAt, sf.MimeType, sf.Size)
if err != nil {
return err
}
@ -116,18 +118,10 @@ func (s *SqliteFileMetaDatabase) GetFile(id string) (*StoredFile, error) {
return nil, nil
}
var uploadedAt int
var expiresAt int
err = rows.Scan(&id, &uploadedAt, &expiresAt)
sf, err := getFromRows(rows)
if err != nil {
return nil, err
}
sf := &StoredFile{
Id: id,
UploadedAt: int64(uploadedAt),
ExpiresAt: int64(expiresAt),
}
return sf, nil
}
@ -146,19 +140,11 @@ func (s *SqliteFileMetaDatabase) GetAllFiles() ([]*StoredFile, error) {
var sfs []*StoredFile
for rows.Next() {
var id string
var uploadedAt int
var expiresAt int
err = rows.Scan(&id, &uploadedAt, &expiresAt)
sf, err := getFromRows(rows)
if err != nil {
return nil, err
}
sf := &StoredFile{
Id: id,
UploadedAt: int64(uploadedAt),
ExpiresAt: int64(expiresAt),
}
sfs = append(sfs, sf)
}
err = rows.Err()
@ -190,19 +176,11 @@ func (s *SqliteFileMetaDatabase) GetAllExpired() ([]*StoredFile, error) {
var sfs []*StoredFile
for rows.Next() {
var id string
var uploadedAt int
var expiresAt int
err = rows.Scan(&id, &uploadedAt, &expiresAt)
sf, err := getFromRows(rows)
if err != nil {
return nil, err
}
sf := &StoredFile{
Id: id,
UploadedAt: int64(uploadedAt),
ExpiresAt: int64(expiresAt),
}
sfs = append(sfs, sf)
}
err = rows.Err()
@ -211,3 +189,24 @@ func (s *SqliteFileMetaDatabase) GetAllExpired() ([]*StoredFile, error) {
}
return sfs, nil
}
func getFromRows(rows *sql.Rows) (*StoredFile, error) {
var id string
var uploadedAt int
var expiresAt int
var mimeType string
var size int
err := rows.Scan(&id, &uploadedAt, &expiresAt, &mimeType, &size)
if err != nil {
return nil, err
}
sf := &StoredFile{
Id: id,
UploadedAt: int64(uploadedAt),
ExpiresAt: int64(expiresAt),
MimeType: mimeType,
Size: int64(size),
}
return sf, nil
}

11
internal/storage/files.go → internal/storage/filesys.go
View File

@ -2,12 +2,15 @@ package storage
import (
"io"
"mime/multipart"
"os"
)
type FileSystem interface {
CreateFile(mf multipart.File, name string) (bool, error)
// CreateFile writes the content of given reader to a file
// with given name.
// Always returns if the file was partly written to disk.
CreateFile(r io.Reader, name string) (bool, error)
DeleteFile(id string) error
GetFile(id string) (*os.File, error)
Exists(id string) (bool, error)
@ -17,7 +20,7 @@ type LocalFileSystem struct {
FolderPath string
}
func (l LocalFileSystem) CreateFile(mf multipart.File, name string) (bool, error) {
func (l LocalFileSystem) CreateFile(r io.Reader, name string) (bool, error) {
err := os.MkdirAll(l.FolderPath, os.ModePerm)
if err != nil {
return false, err
@ -31,7 +34,7 @@ func (l LocalFileSystem) CreateFile(mf multipart.File, name string) (bool, error
// use io.Copy so that we don't have to load all the image into the memory.
// they get copied in smaller 32kb chunks.
_, err = io.Copy(f, mf)
_, err = io.Copy(f, r)
if err != nil {
return true, err
}

10
internal/storage/storage.go
View File

@ -5,10 +5,10 @@ import (
"errors"
"fmt"
"github.com/google/uuid"
"github.com/superioz/aqua/internal/handler"
"github.com/superioz/aqua/internal/metrics"
"github.com/superioz/aqua/pkg/env"
"k8s.io/klog"
"mime/multipart"
"strings"
"time"
@ -26,6 +26,8 @@ type StoredFile struct {
Id string
UploadedAt int64
ExpiresAt int64
MimeType string
Size int64
}
func (sf *StoredFile) String() string {
@ -100,13 +102,13 @@ func (fs *FileStorage) Cleanup() error {
return nil
}
func (fs *FileStorage) StoreFile(of multipart.File, expiration int64) (*StoredFile, error) {
func (fs *FileStorage) StoreFile(rff *handler.RequestFormFile, expiration int64) (*StoredFile, error) {
name, err := getRandomFileName(env.IntOrDefault("FILE_NAME_LENGTH", 8))
if err != nil {
return nil, errors.New("could not generate random name")
}
_, err = fs.fileSystem.CreateFile(of, name)
_, err = fs.fileSystem.CreateFile(rff.File, name)
if err != nil {
klog.Error(err)
return nil, errors.New("could not save file to system")
@ -122,6 +124,8 @@ func (fs *FileStorage) StoreFile(of multipart.File, expiration int64) (*StoredFi
Id: name,
UploadedAt: currentTime,
ExpiresAt: expAt,
MimeType: rff.ContentType,
Size: rff.ContentLength,
}
// write to meta database

2
run.sh
View File

@ -15,4 +15,4 @@ fi
# we only need MSYS_NO_PATHCONV when running this script in Windows
# because otherwise Mingw will try to convert
# the given paths and that breaks everything.
MSYS_NO_PATHCONV=1 docker run -it --rm -p 8765:8765 $IMAGE
MSYS_NO_PATHCONV=1 docker run -it --rm -p 8765:8765 $IMAGE