selfhosted
/
pasty
mirror of https://github.com/lus/pasty.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Switch from snowflake to random string ID system

This commit is contained in:
Lukas SP 2020-08-24 20:22:53 +02:00
parent a1cd915759
commit 5585a26ab0
13 changed files with 97 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
README.md
View File

@ -7,6 +7,7 @@ Pasty is a fast and lightweight code pasting server
| `PASTY_WEB_ADDRESS` | `:8080` | `string` | Defines the address the web server listens to |
| `PASTY_STORAGE_TYPE` | `file` | `string` | Defines the storage type the pastes are saved to |
| `PASTY_HASTEBIN_SUPPORT` | `false` | `bool` | Defines whether or not the `POST /documents` endpoint should be enabled, as known from the hastebin servers |
| `PASTY_ID_LENGTH` | `6` | `number` | Defines the length of the ID of a paste |
| `PASTY_DELETION_TOKEN_LENGTH` | `12` | `number` | Defines the length of the deletion token of a paste |
| `PASTY_RATE_LIMIT` | `30-M` | `string` | Defines the rate limit of the API (see https://github.com/ulule/limiter#usage) |
@ -19,6 +20,8 @@ Every single one of them has its own configuration variables:
|---------------------------|---------------|----------|-----------------------------------------------------------|
| `PASTY_STORAGE_FILE_PATH` | `./data` | `string` | Defines the file path the paste files are being saved to |
---
### S3 (`s3`)
| Environment Variable | Default Value | Type | Description |
|--------------------------------|---------------|----------|-------------------------------------------------------------------------------------------|
@ -30,6 +33,8 @@ Every single one of them has its own configuration variables:
| `STORAGE_S3_REGION` | `<empty>` | `string` | Defines the region of the S3 storage |
| `STORAGE_S3_BUCKET` | `pasty` | `string` | Defines the name of the S3 bucket (has to be created before setup) |
---
### MongoDB (`mongodb`)
| Environment Variable | Default Value | Type | Description |
|-------------------------------------|--------------------------------------------|----------|-----------------------------------------------------------------|

1
go.mod
View File

@ -4,7 +4,6 @@ go 1.15
require (
github.com/alexedwards/argon2id v0.0.0-20200802152012-2464efd3196b
github.com/bwmarrin/snowflake v0.3.0
github.com/fasthttp/router v1.2.4
github.com/joho/godotenv v1.3.0
github.com/klauspost/compress v1.10.11 // indirect

2
go.sum
View File

@ -5,8 +5,6 @@ github.com/andybalholm/brotli v1.0.0 h1:7UCwP93aiSfvWpapti8g88vVVGp2qqtGyePsSuDa
github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
github.com/aws/aws-sdk-go v1.29.15 h1:0ms/213murpsujhsnxnNKNeVouW60aJqSd992Ks3mxs=
github.com/aws/aws-sdk-go v1.29.15/go.mod h1:1KvfttTE3SPKMpo8g2c6jL3ZKfXtFvKscTgahTma5Xg=
github.com/bwmarrin/snowflake v0.3.0 h1:xm67bEhkKh6ij1790JB83OujPR5CzNe8QuQqAgISZN0=
github.com/bwmarrin/snowflake v0.3.0/go.mod h1:NdZxfVWX+oR6y2K0o6qAYv6gIOP9rjG0/E9WsDpxqwE=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

11
internal/pastes/deletion_token.go
View File

@ -2,13 +2,10 @@ package pastes
import (
"github.com/Lukaesebrot/pasty/internal/env"
"math/rand"
"github.com/Lukaesebrot/pasty/internal/utils"
"strconv"
)
// deletionTokenContents represents the characters a deletion token may contain
const deletionTokenContents = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789#+-.,"
// generateDeletionToken generates a new deletion token
func generateDeletionToken() (string, error) {
// Read the deletion token length
@ -19,9 +16,5 @@ func generateDeletionToken() (string, error) {
}
// Generate the deletion token
bytes := make([]byte, length)
for i := range bytes {
bytes[i] = deletionTokenContents[rand.Int63()%int64(len(deletionTokenContents))]
}
return string(bytes), nil
return utils.RandomString(length), nil
}

20
internal/pastes/paste.go
View File

@ -2,26 +2,18 @@ package pastes
import (
"github.com/alexedwards/argon2id"
"github.com/bwmarrin/snowflake"
)
func init() {
snowflakeNode, _ = snowflake.NewNode(1)
}
// snowflakeNode holds the current snowflake node
var snowflakeNode *snowflake.Node
// Paste represents a saved paste
type Paste struct {
ID snowflake.ID `json:"id" bson:"_id"`
Content string `json:"content" bson:"content"`
SuggestedSyntaxType string `json:"suggestedSyntaxType" bson:"suggestedSyntaxType"`
DeletionToken string `json:"deletionToken" bson:"deletionToken"`
ID string `json:"id" bson:"_id"`
Content string `json:"content" bson:"content"`
SuggestedSyntaxType string `json:"suggestedSyntaxType" bson:"suggestedSyntaxType"`
DeletionToken string `json:"deletionToken" bson:"deletionToken"`
}
// Create creates a new paste object using the given content
func Create(content string) (*Paste, error) {
func Create(id, content string) (*Paste, error) {
// TODO: Generate the suggested syntax type
suggestedSyntaxType := ""
@ -33,7 +25,7 @@ func Create(content string) (*Paste, error) {
// Return the paste object
return &Paste{
ID: snowflakeNode.Generate(),
ID: id,
Content: content,
SuggestedSyntaxType: suggestedSyntaxType,
DeletionToken: deletionToken,

5
internal/storage/driver.go
View File

@ -4,7 +4,6 @@ import (
"fmt"
"github.com/Lukaesebrot/pasty/internal/env"
"github.com/Lukaesebrot/pasty/internal/pastes"
"github.com/bwmarrin/snowflake"
"strings"
)
@ -15,9 +14,9 @@ var Current Driver
type Driver interface {
Initialize() error
Terminate() error
Get(id snowflake.ID) (*pastes.Paste, error)
Get(id string) (*pastes.Paste, error)
Save(paste *pastes.Paste) error
Delete(id snowflake.ID) error
Delete(id string) error
}
// Load loads the current storage driver

11
internal/storage/file_driver.go
View File

@ -4,7 +4,6 @@ import (
"encoding/json"
"github.com/Lukaesebrot/pasty/internal/env"
"github.com/Lukaesebrot/pasty/internal/pastes"
"github.com/bwmarrin/snowflake"
"io/ioutil"
"os"
"path/filepath"
@ -27,9 +26,9 @@ func (driver *FileDriver) Terminate() error {
}
// Get loads a paste
func (driver *FileDriver) Get(id snowflake.ID) (*pastes.Paste, error) {
func (driver *FileDriver) Get(id string) (*pastes.Paste, error) {
// Read the file
data, err := ioutil.ReadFile(filepath.Join(driver.filePath, id.String()+".json"))
data, err := ioutil.ReadFile(filepath.Join(driver.filePath, id+".json"))
if err != nil {
if os.IsNotExist(err) {
return nil, nil
@ -55,7 +54,7 @@ func (driver *FileDriver) Save(paste *pastes.Paste) error {
}
// Create the file to save the paste to
file, err := os.Create(filepath.Join(driver.filePath, paste.ID.String()+".json"))
file, err := os.Create(filepath.Join(driver.filePath, paste.ID+".json"))
if err != nil {
return err
}
@ -67,6 +66,6 @@ func (driver *FileDriver) Save(paste *pastes.Paste) error {
}
// Delete deletes a paste
func (driver *FileDriver) Delete(id snowflake.ID) error {
return os.Remove(filepath.Join(driver.filePath, id.String()+".json"))
func (driver *FileDriver) Delete(id string) error {
return os.Remove(filepath.Join(driver.filePath, id+".json"))
}

29
internal/storage/id_generation.go Normal file
View File

@ -0,0 +1,29 @@
package storage
import (
"github.com/Lukaesebrot/pasty/internal/env"
"github.com/Lukaesebrot/pasty/internal/utils"
"strconv"
)
// AcquireID generates a new unique ID
func AcquireID() (string, error) {
// Read the ID length
rawLength := env.Get("ID_LENGTH", "6")
length, err := strconv.Atoi(rawLength)
if err != nil {
return "", err
}
// Generate the unique ID
for {
id := utils.RandomString(length)
paste, err := Current.Get(id)
if err != nil {
return "", err
}
if paste == nil {
return id, nil
}
}
}

9
internal/storage/mongodb_driver.go
View File

@ -4,7 +4,6 @@ import (
"context"
"github.com/Lukaesebrot/pasty/internal/env"
"github.com/Lukaesebrot/pasty/internal/pastes"
"github.com/bwmarrin/snowflake"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
@ -50,7 +49,7 @@ func (driver *MongoDBDriver) Terminate() error {
}
// Get loads a paste
func (driver *MongoDBDriver) Get(id snowflake.ID) (*pastes.Paste, error) {
func (driver *MongoDBDriver) Get(id string) (*pastes.Paste, error) {
// Define the collection to use for this database operation
collection := driver.client.Database(driver.database).Collection(driver.collection)
@ -59,7 +58,7 @@ func (driver *MongoDBDriver) Get(id snowflake.ID) (*pastes.Paste, error) {
defer cancel()
// Try to retrieve the corresponding paste document
filter := bson.M{"_id": id.String()}
filter := bson.M{"_id": id}
result := collection.FindOne(ctx, filter)
err := result.Err()
if err != nil {
@ -90,7 +89,7 @@ func (driver *MongoDBDriver) Save(paste *pastes.Paste) error {
}
// Delete deletes a paste
func (driver *MongoDBDriver) Delete(id snowflake.ID) error {
func (driver *MongoDBDriver) Delete(id string) error {
// Define the collection to use for this database operation
collection := driver.client.Database(driver.database).Collection(driver.collection)
@ -99,7 +98,7 @@ func (driver *MongoDBDriver) Delete(id snowflake.ID) error {
defer cancel()
// Delete the document
filter := bson.M{"_id": id.String()}
filter := bson.M{"_id": id}
_, err := collection.DeleteOne(ctx, filter)
return err
}

11
internal/storage/s3_driver.go
View File

@ -6,7 +6,6 @@ import (
"encoding/json"
"github.com/Lukaesebrot/pasty/internal/env"
"github.com/Lukaesebrot/pasty/internal/pastes"
"github.com/bwmarrin/snowflake"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"io/ioutil"
@ -39,9 +38,9 @@ func (driver *S3Driver) Terminate() error {
}
// Get loads a paste
func (driver *S3Driver) Get(id snowflake.ID) (*pastes.Paste, error) {
func (driver *S3Driver) Get(id string) (*pastes.Paste, error) {
// Read the object
object, err := driver.client.GetObject(context.Background(), driver.bucket, id.String()+".json", minio.GetObjectOptions{})
object, err := driver.client.GetObject(context.Background(), driver.bucket, id+".json", minio.GetObjectOptions{})
if err != nil {
return nil, err
}
@ -69,13 +68,13 @@ func (driver *S3Driver) Save(paste *pastes.Paste) error {
// Put the object
reader := bytes.NewReader(jsonBytes)
_, err = driver.client.PutObject(context.Background(), driver.bucket, paste.ID.String()+".json", reader, reader.Size(), minio.PutObjectOptions{
_, err = driver.client.PutObject(context.Background(), driver.bucket, paste.ID+".json", reader, reader.Size(), minio.PutObjectOptions{
ContentType: "application/json",
})
return err
}
// Delete deletes a paste
func (driver *S3Driver) Delete(id snowflake.ID) error {
return driver.client.RemoveObject(context.Background(), driver.bucket, id.String()+".json", minio.RemoveObjectOptions{})
func (driver *S3Driver) Delete(id string) error {
return driver.client.RemoveObject(context.Background(), driver.bucket, id+".json", minio.RemoveObjectOptions{})
}

15
internal/utils/randomString.go Normal file
View File

@ -0,0 +1,15 @@
package utils
import "math/rand"
// stringContents holds the chars a random string can contain
const stringContents = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
// RandomString returns a random string with the given length
func RandomString(length int) string {
bytes := make([]byte, length)
for i := range bytes {
bytes[i] = stringContents[rand.Int63()%int64(len(stringContents))]
}
return string(bytes)
}

12
internal/web/controllers/v1/hastebin_support.go
View File

@ -24,8 +24,16 @@ func HastebinSupportHandler(ctx *fasthttp.RequestCtx) {
return
}
// Acquire the paste ID
id, err := storage.AcquireID()
if err != nil {
ctx.SetStatusCode(fasthttp.StatusInternalServerError)
ctx.SetBodyString(err.Error())
return
}
// Create the paste object
paste, err := pastes.Create(content)
paste, err := pastes.Create(id, content)
if err != nil {
ctx.SetStatusCode(fasthttp.StatusInternalServerError)
ctx.SetBodyString(err.Error())
@ -50,7 +58,7 @@ func HastebinSupportHandler(ctx *fasthttp.RequestCtx) {
// Respond with the paste key
jsonData, _ := json.Marshal(map[string]string{
"key": paste.ID.String(),
"key": paste.ID,
})
ctx.SetBody(jsonData)
}

31
internal/web/controllers/v1/pastes.go
View File

@ -4,7 +4,6 @@ import (
"encoding/json"
"github.com/Lukaesebrot/pasty/internal/pastes"
"github.com/Lukaesebrot/pasty/internal/storage"
"github.com/bwmarrin/snowflake"
"github.com/fasthttp/router"
limitFasthttp "github.com/ulule/limiter/v3/drivers/middleware/fasthttp"
"github.com/valyala/fasthttp"
@ -19,13 +18,8 @@ func InitializePastesController(group *router.Group, rateLimiterMiddleware *limi
// v1GetPaste handles the 'GET /v1/pastes/{id}' endpoint
func v1GetPaste(ctx *fasthttp.RequestCtx) {
// Parse the ID
id, err := snowflake.ParseString(ctx.UserValue("id").(string))
if err != nil {
ctx.SetStatusCode(fasthttp.StatusBadRequest)
ctx.SetBodyString("invalid ID format")
return
}
// Read the ID
id := ctx.UserValue("id").(string)
// Retrieve the paste
paste, err := storage.Current.Get(id)
@ -68,8 +62,16 @@ func v1PostPaste(ctx *fasthttp.RequestCtx) {
return
}
// Acquire the paste ID
id, err := storage.AcquireID()
if err != nil {
ctx.SetStatusCode(fasthttp.StatusInternalServerError)
ctx.SetBodyString(err.Error())
return
}
// Create the paste object
paste, err := pastes.Create(values["content"])
paste, err := pastes.Create(id, values["content"])
if err != nil {
ctx.SetStatusCode(fasthttp.StatusInternalServerError)
ctx.SetBodyString(err.Error())
@ -105,17 +107,12 @@ func v1PostPaste(ctx *fasthttp.RequestCtx) {
// v1DeletePaste handles the 'DELETE /v1/pastes/{id}'
func v1DeletePaste(ctx *fasthttp.RequestCtx) {
// Parse the ID
id, err := snowflake.ParseString(ctx.UserValue("id").(string))
if err != nil {
ctx.SetStatusCode(fasthttp.StatusBadRequest)
ctx.SetBodyString("invalid ID format")
return
}
// Read the ID
id := ctx.UserValue("id").(string)
// Unmarshal the body
values := make(map[string]string)
err = json.Unmarshal(ctx.PostBody(), &values)
err := json.Unmarshal(ctx.PostBody(), &values)
if err != nil {
ctx.SetStatusCode(fasthttp.StatusBadRequest)
ctx.SetBodyString("invalid request body")