Add file uploading functionality

.env.dist

@@ -0,0 +1,2 @@
+AUTH_CONFIG_PATH=
+FILE_STORAGE_PATH=

.github/assets/aqua_structure.drawio

@@ -0,0 +1 @@
+<mxfile host="Electron" modified="2021-11-12T13:11:21.799Z" agent="5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) draw.io/13.6.2 Chrome/83.0.4103.122 Electron/9.2.0 Safari/537.36" etag="ilHPzfhKgGRDgH4Ut-0b" version="13.6.2" type="device"><diagram id="elm1hSbFkwVyf44fW0mf" name="Page-1">zZdNb4IwGIB/DcclLeXzKChuybaLhyW7VXkVYqGkVtH9+tVRUGZMtkSsF9I+fdvSp+RtsUhc7KeCVtkbT4FZNkr3Fhlbth0GWD2P4NAA1/MbsBJ52iB8ArP8CzREmm7zFDa9QMk5k3nVhwtelrCQPUaF4HU/bMlZf9aKruACzBaUXdKPPJVZQwPbP/FnyFdZOzP2wqaloG2wXskmoymvzxCZWCQWnMumVOxjYEd3rZemX3KltXsxAaX8Swf+WaxR+QLM26GPzfb9dT6tntxmlB1lW71g/bLy0BoQfFumcBwEWSSqs1zCrKKLY2uttlyxTBZM1bAqbqTga4g540KRkpcqLFrmjLXIsokfxkkcKq7nBiFhf3VRuFOlPjHgBUhxUCG6g6PlHvrV+myrNMrOdqkNo/rjWHXjnvypglb4D52eAZ3OeOSEySA6A8M6fQM6cexHvj+ITmwb9hkY8ImQ67h4GJ+eYZ+hAZ+jcYwSchufncAHSZ/toX/f4ygJxtFAQk0nUIwNCPWQFwU3yqC/hRpPodg2YJSMHOTaAxk1nUSxiUsTIc7cudEdtNP1KFnUxLUpdOeYwq2O+XsJVdXTD9hP29lfLJl8Aw==</diagram></mxfile>

auth.yml

@@ -0,0 +1,2 @@
+validTokens:
+- token: 71a4c056ab9b0fb965063344cd6616bc

cmd/aqua/main.go

@@ -2,18 +2,32 @@ package main
 
 import (
 	"github.com/gin-gonic/gin"
+	"github.com/joho/godotenv"
+	"github.com/superioz/aqua/internal/handler"
 	"github.com/superioz/aqua/internal/middleware"
 	"k8s.io/klog"
 	"time"
 )
 
 func main() {
+	err := godotenv.Load()
+	if err != nil {
+		klog.Fatal("Error loading .env file")
+	}
 	klog.Infoln("Hello World!")
 
+	// init some stuff for the handler
+	// like config etc.
+	handler.Initialize()
+
 	r := gin.New()
 	r.Use(middleware.Logger(3 * time.Second))
+	// restrict to max 100mb
+	r.Use(middleware.RestrictBodySize(100 * handler.SizeMegaByte))
 	r.Use(gin.Recovery())
 
+	r.POST("/upload", handler.Upload)
+
 	r.GET("/healthz", func(c *gin.Context) {
 		c.JSON(200, gin.H{"status": "UP"})
 	})

go.mod

@@ -13,6 +13,8 @@ require (
 	github.com/go-playground/universal-translator v0.17.0 // indirect
 	github.com/go-playground/validator/v10 v10.4.1 // indirect
 	github.com/golang/protobuf v1.3.3 // indirect
+	github.com/google/uuid v1.3.0 // indirect
+	github.com/joho/godotenv v1.4.0 // indirect
 	github.com/json-iterator/go v1.1.9 // indirect
 	github.com/leodido/go-urn v1.2.0 // indirect
 	github.com/mattn/go-isatty v0.0.12 // indirect
@@ -21,5 +23,5 @@ require (
 	github.com/ugorji/go/codec v1.1.7 // indirect
 	golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 // indirect
 	golang.org/x/sys v0.0.0-20200116001909-b77594299b42 // indirect
-	gopkg.in/yaml.v2 v2.2.8 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
 )

go.sum

@@ -17,6 +17,10 @@ github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn
 github.com/golang/protobuf v1.3.3 h1:gyjaxf+svBWX08ZjK86iN9geUJF0H6gp2IRKX6Nf6/I=
 github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
+github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg=
+github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
 github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns=
 github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
@@ -53,5 +57,7 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
 k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=

internal/config/config.go

@@ -0,0 +1,53 @@
+package config
+
+import (
+	"gopkg.in/yaml.v2"
+	"io/ioutil"
+	"os"
+)
+
+type AuthConfig struct {
+	ValidTokens []*TokenConfig `yaml:"validTokens"`
+}
+
+type TokenConfig struct {
+	Token string
+}
+
+func NewEmptyAuthConfig() *AuthConfig {
+	return &AuthConfig{
+		ValidTokens: []*TokenConfig{},
+	}
+}
+
+func FromData(data []byte) (*AuthConfig, error) {
+	var ac AuthConfig
+	err := yaml.Unmarshal(data, &ac)
+	if err != nil {
+		return nil, err
+	}
+	return &ac, nil
+}
+
+func FromLocalFile(path string) (*AuthConfig, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+
+	data, err := ioutil.ReadAll(f)
+	if err != nil {
+		return nil, err
+	}
+	return FromData(data)
+}
+
+func (ac *AuthConfig) HasToken(token string) bool {
+	for _, validToken := range ac.ValidTokens {
+		if validToken.Token == token {
+			return true
+		}
+	}
+	return false
+}

internal/handler/upload.go

@@ -0,0 +1,178 @@
+package handler
+
+import (
+	"github.com/gin-gonic/gin"
+	"github.com/google/uuid"
+	"github.com/superioz/aqua/internal/config"
+	"github.com/superioz/aqua/pkg/env"
+	"io"
+	"k8s.io/klog"
+	"mime/multipart"
+	"net/http"
+	"os"
+	"strings"
+)
+
+const (
+	SizeMegaByte = 1 << (10 * 2)
+)
+
+var (
+	validMimeTypes = []string{
+		"application/pdf",
+		"application/json",
+		"image/png",
+		"image/jpeg",
+		"text/csv",
+		"text/plain",
+	}
+
+	authConfig *config.AuthConfig
+)
+
+func Initialize() {
+	path := env.StringOrDefault("AUTH_CONFIG_PATH", "/etc/aqua/auth.yml")
+	ac, err := config.FromLocalFile(path)
+	if err != nil {
+		// this is not good, but the system still works.
+		// nobody can upload a file though.
+		klog.Warningf("Could not open auth config at %s: %v", path, err)
+		authConfig = config.NewEmptyAuthConfig()
+		return
+	} else {
+		klog.Infof("Loaded %d valid tokens", len(ac.ValidTokens))
+	}
+	authConfig = ac
+}
+
+func Upload(c *gin.Context) {
+	// get token for auth
+	// empty string, if not given
+	token := getToken(c)
+
+	klog.Infof("Checking authentication for token=%s", token)
+	if !authConfig.HasToken(token) {
+		c.JSON(http.StatusUnauthorized, gin.H{"msg": "the token is not valid"})
+		return
+	}
+
+	form, err := c.MultipartForm()
+	if err != nil {
+		_ = c.Error(err)
+		return
+	}
+
+	files := form.File["file"]
+	if len(files) > 1 {
+		c.JSON(http.StatusBadRequest, gin.H{"msg": "too many files in form"})
+		return
+	}
+	file := files[0]
+
+	if c.Request.Header.Get("Content-Length") == "" {
+		c.Status(http.StatusLengthRequired)
+		return
+	}
+	if c.Request.ContentLength > 50*SizeMegaByte {
+		c.JSON(http.StatusRequestEntityTooLarge, gin.H{"msg": "content size must not exceed 50mb"})
+		return
+	}
+
+	ct := getContentType(file)
+	klog.Infof("Detected content type: %s", ct)
+	if !isContentTypeValid(ct) {
+		c.JSON(http.StatusBadRequest, gin.H{"msg": "content type of file is not valid"})
+		return
+	}
+
+	of, err := file.Open()
+	if err != nil {
+		klog.Error(err)
+		c.JSON(http.StatusInternalServerError, gin.H{"msg": "could not open file"})
+		return
+	}
+	defer of.Close()
+
+	name, err := getRandomFileName(8)
+	if err != nil {
+		klog.Error(err)
+		c.JSON(http.StatusInternalServerError, gin.H{"msg": "could not generate id of file"})
+		return
+	}
+
+	path := env.StringOrDefault("FILE_STORAGE_PATH", "/var/lib/aqua/")
+
+	err = os.MkdirAll(path, os.ModePerm)
+	if err != nil {
+		klog.Error(err)
+		c.JSON(http.StatusInternalServerError, gin.H{"msg": "could not create file directory"})
+		return
+	}
+
+	f, err := os.Create(name)
+	if err != nil {
+		klog.Error(err)
+		c.JSON(http.StatusInternalServerError, gin.H{"msg": "could not create file"})
+		return
+	}
+	defer f.Close()
+
+	// use io.Copy so that we don't have to load all the image into the memory.
+	// they get copied in smaller 32kb chunks.
+	_, err = io.Copy(f, of)
+	if err != nil {
+		klog.Error(err)
+		c.JSON(http.StatusInternalServerError, gin.H{"msg": "could not copy content to file"})
+		return
+	}
+
+	c.JSON(http.StatusOK, gin.H{"id": name})
+}
+
+func getRandomFileName(size int) (string, error) {
+	id, err := uuid.NewRandom()
+	if err != nil {
+		return "", err
+	}
+
+	// strip '-' from uuid
+	str := strings.ReplaceAll(id.String(), "-", "")
+	if size >= len(str) {
+		return str, nil
+	}
+	return str[:size], nil
+}
+
+// workaround for file Content-Type headers
+// which contain multiple values such as "...; charset=utf-8"
+func getContentType(f *multipart.FileHeader) string {
+	c := f.Header.Get("Content-Type")
+	if strings.Contains(c, ";") {
+		c = strings.Split(c, ";")[0]
+	} else if strings.Contains(c, ",") {
+		c = strings.Split(c, ",")[0]
+	}
+	return c
+}
+
+func isContentTypeValid(ct string) bool {
+	for _, mt := range validMimeTypes {
+		if mt == ct {
+			return true
+		}
+	}
+	return false
+}
+
+func getToken(c *gin.Context) string {
+	// try to get the Bearer token, because it's the standard
+	// for authorization
+	bearerToken := c.Request.Header.Get("Authorization")
+	if !strings.HasPrefix(bearerToken, "Bearer ") {
+		// is not a Bearer token, so we don't want it
+		return ""
+	}
+
+	spl := strings.Split(bearerToken, "Bearer ")
+	return spl[1]
+}

internal/middleware/maxbytes.go

@@ -0,0 +1,15 @@
+package middleware
+
+import (
+	"github.com/gin-gonic/gin"
+	"net/http"
+)
+
+func RestrictBodySize(max int64) gin.HandlerFunc {
+	return func(c *gin.Context) {
+		var w http.ResponseWriter = c.Writer
+		c.Request.Body = http.MaxBytesReader(w, c.Request.Body, max)
+
+		c.Next()
+	}
+}

pkg/env/env.go

@@ -0,0 +1,59 @@
+package env
+
+import (
+	"os"
+	"strconv"
+)
+
+func String(name string) (string, bool) {
+	return os.LookupEnv(name)
+}
+
+func StringOrDefault(name string, def string) string {
+	s, ok := String(name)
+	if !ok {
+		return def
+	}
+	return s
+}
+
+func Int(name string) (int, bool) {
+	s, ok := String(name)
+	if !ok {
+		return 0, false
+	}
+
+	i, err := strconv.Atoi(s)
+	if err != nil {
+		return 0, false
+	}
+	return i, true
+}
+
+func IntOrDefault(name string, def int) int {
+	i, ok := Int(name)
+	if !ok {
+		return def
+	}
+	return i
+}
+
+func Bool(name string) (bool, bool) {
+	s, ok := String(name)
+	if !ok {
+		return false, false
+	}
+
+	if s == "true" || s == "enabled" {
+		return true, true
+	}
+	return false, true
+}
+
+func BoolOrDefault(name string, def bool) bool {
+	b, ok := Bool(name)
+	if !ok {
+		return def
+	}
+	return b
+}