From 5585a26ab0a517f47369906aa09fe89b0b52f030 Mon Sep 17 00:00:00 2001 From: Lukas SP Date: Mon, 24 Aug 2020 20:22:53 +0200 Subject: [PATCH] Switch from snowflake to random string ID system --- README.md | 5 +++ go.mod | 1 - go.sum | 2 -- internal/pastes/deletion_token.go | 11 ++----- internal/pastes/paste.go | 20 ++++-------- internal/storage/driver.go | 5 ++- internal/storage/file_driver.go | 11 +++---- internal/storage/id_generation.go | 29 +++++++++++++++++ internal/storage/mongodb_driver.go | 9 +++--- internal/storage/s3_driver.go | 11 +++---- internal/utils/randomString.go | 15 +++++++++ .../web/controllers/v1/hastebin_support.go | 12 +++++-- internal/web/controllers/v1/pastes.go | 31 +++++++++---------- 13 files changed, 97 insertions(+), 65 deletions(-) create mode 100644 internal/storage/id_generation.go create mode 100644 internal/utils/randomString.go diff --git a/README.md b/README.md index 4457618..db6aeb0 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ Pasty is a fast and lightweight code pasting server | `PASTY_WEB_ADDRESS` | `:8080` | `string` | Defines the address the web server listens to | | `PASTY_STORAGE_TYPE` | `file` | `string` | Defines the storage type the pastes are saved to | | `PASTY_HASTEBIN_SUPPORT` | `false` | `bool` | Defines whether or not the `POST /documents` endpoint should be enabled, as known from the hastebin servers | +| `PASTY_ID_LENGTH` | `6` | `number` | Defines the length of the ID of a paste | | `PASTY_DELETION_TOKEN_LENGTH` | `12` | `number` | Defines the length of the deletion token of a paste | | `PASTY_RATE_LIMIT` | `30-M` | `string` | Defines the rate limit of the API (see https://github.com/ulule/limiter#usage) | @@ -19,6 +20,8 @@ Every single one of them has its own configuration variables: |---------------------------|---------------|----------|-----------------------------------------------------------| | `PASTY_STORAGE_FILE_PATH` | `./data` | `string` | Defines the file path the paste files are being saved to | +--- + ### S3 (`s3`) | Environment Variable | Default Value | Type | Description | |--------------------------------|---------------|----------|-------------------------------------------------------------------------------------------| @@ -30,6 +33,8 @@ Every single one of them has its own configuration variables: | `STORAGE_S3_REGION` | `` | `string` | Defines the region of the S3 storage | | `STORAGE_S3_BUCKET` | `pasty` | `string` | Defines the name of the S3 bucket (has to be created before setup) | +--- + ### MongoDB (`mongodb`) | Environment Variable | Default Value | Type | Description | |-------------------------------------|--------------------------------------------|----------|-----------------------------------------------------------------| diff --git a/go.mod b/go.mod index ec693fd..67e7e19 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,6 @@ go 1.15 require ( github.com/alexedwards/argon2id v0.0.0-20200802152012-2464efd3196b - github.com/bwmarrin/snowflake v0.3.0 github.com/fasthttp/router v1.2.4 github.com/joho/godotenv v1.3.0 github.com/klauspost/compress v1.10.11 // indirect diff --git a/go.sum b/go.sum index df18c61..04e41fa 100644 --- a/go.sum +++ b/go.sum @@ -5,8 +5,6 @@ github.com/andybalholm/brotli v1.0.0 h1:7UCwP93aiSfvWpapti8g88vVVGp2qqtGyePsSuDa github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/aws/aws-sdk-go v1.29.15 h1:0ms/213murpsujhsnxnNKNeVouW60aJqSd992Ks3mxs= github.com/aws/aws-sdk-go v1.29.15/go.mod h1:1KvfttTE3SPKMpo8g2c6jL3ZKfXtFvKscTgahTma5Xg= -github.com/bwmarrin/snowflake v0.3.0 h1:xm67bEhkKh6ij1790JB83OujPR5CzNe8QuQqAgISZN0= -github.com/bwmarrin/snowflake v0.3.0/go.mod h1:NdZxfVWX+oR6y2K0o6qAYv6gIOP9rjG0/E9WsDpxqwE= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/internal/pastes/deletion_token.go b/internal/pastes/deletion_token.go index c9fb2a6..a22a7cb 100644 --- a/internal/pastes/deletion_token.go +++ b/internal/pastes/deletion_token.go @@ -2,13 +2,10 @@ package pastes import ( "github.com/Lukaesebrot/pasty/internal/env" - "math/rand" + "github.com/Lukaesebrot/pasty/internal/utils" "strconv" ) -// deletionTokenContents represents the characters a deletion token may contain -const deletionTokenContents = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789#+-.," - // generateDeletionToken generates a new deletion token func generateDeletionToken() (string, error) { // Read the deletion token length @@ -19,9 +16,5 @@ func generateDeletionToken() (string, error) { } // Generate the deletion token - bytes := make([]byte, length) - for i := range bytes { - bytes[i] = deletionTokenContents[rand.Int63()%int64(len(deletionTokenContents))] - } - return string(bytes), nil + return utils.RandomString(length), nil } diff --git a/internal/pastes/paste.go b/internal/pastes/paste.go index 1fb6ec0..bfc66e3 100644 --- a/internal/pastes/paste.go +++ b/internal/pastes/paste.go @@ -2,26 +2,18 @@ package pastes import ( "github.com/alexedwards/argon2id" - "github.com/bwmarrin/snowflake" ) -func init() { - snowflakeNode, _ = snowflake.NewNode(1) -} - -// snowflakeNode holds the current snowflake node -var snowflakeNode *snowflake.Node - // Paste represents a saved paste type Paste struct { - ID snowflake.ID `json:"id" bson:"_id"` - Content string `json:"content" bson:"content"` - SuggestedSyntaxType string `json:"suggestedSyntaxType" bson:"suggestedSyntaxType"` - DeletionToken string `json:"deletionToken" bson:"deletionToken"` + ID string `json:"id" bson:"_id"` + Content string `json:"content" bson:"content"` + SuggestedSyntaxType string `json:"suggestedSyntaxType" bson:"suggestedSyntaxType"` + DeletionToken string `json:"deletionToken" bson:"deletionToken"` } // Create creates a new paste object using the given content -func Create(content string) (*Paste, error) { +func Create(id, content string) (*Paste, error) { // TODO: Generate the suggested syntax type suggestedSyntaxType := "" @@ -33,7 +25,7 @@ func Create(content string) (*Paste, error) { // Return the paste object return &Paste{ - ID: snowflakeNode.Generate(), + ID: id, Content: content, SuggestedSyntaxType: suggestedSyntaxType, DeletionToken: deletionToken, diff --git a/internal/storage/driver.go b/internal/storage/driver.go index 21c2377..c64fe72 100644 --- a/internal/storage/driver.go +++ b/internal/storage/driver.go @@ -4,7 +4,6 @@ import ( "fmt" "github.com/Lukaesebrot/pasty/internal/env" "github.com/Lukaesebrot/pasty/internal/pastes" - "github.com/bwmarrin/snowflake" "strings" ) @@ -15,9 +14,9 @@ var Current Driver type Driver interface { Initialize() error Terminate() error - Get(id snowflake.ID) (*pastes.Paste, error) + Get(id string) (*pastes.Paste, error) Save(paste *pastes.Paste) error - Delete(id snowflake.ID) error + Delete(id string) error } // Load loads the current storage driver diff --git a/internal/storage/file_driver.go b/internal/storage/file_driver.go index 96a6ae7..2f30abb 100644 --- a/internal/storage/file_driver.go +++ b/internal/storage/file_driver.go @@ -4,7 +4,6 @@ import ( "encoding/json" "github.com/Lukaesebrot/pasty/internal/env" "github.com/Lukaesebrot/pasty/internal/pastes" - "github.com/bwmarrin/snowflake" "io/ioutil" "os" "path/filepath" @@ -27,9 +26,9 @@ func (driver *FileDriver) Terminate() error { } // Get loads a paste -func (driver *FileDriver) Get(id snowflake.ID) (*pastes.Paste, error) { +func (driver *FileDriver) Get(id string) (*pastes.Paste, error) { // Read the file - data, err := ioutil.ReadFile(filepath.Join(driver.filePath, id.String()+".json")) + data, err := ioutil.ReadFile(filepath.Join(driver.filePath, id+".json")) if err != nil { if os.IsNotExist(err) { return nil, nil @@ -55,7 +54,7 @@ func (driver *FileDriver) Save(paste *pastes.Paste) error { } // Create the file to save the paste to - file, err := os.Create(filepath.Join(driver.filePath, paste.ID.String()+".json")) + file, err := os.Create(filepath.Join(driver.filePath, paste.ID+".json")) if err != nil { return err } @@ -67,6 +66,6 @@ func (driver *FileDriver) Save(paste *pastes.Paste) error { } // Delete deletes a paste -func (driver *FileDriver) Delete(id snowflake.ID) error { - return os.Remove(filepath.Join(driver.filePath, id.String()+".json")) +func (driver *FileDriver) Delete(id string) error { + return os.Remove(filepath.Join(driver.filePath, id+".json")) } diff --git a/internal/storage/id_generation.go b/internal/storage/id_generation.go new file mode 100644 index 0000000..9e2571d --- /dev/null +++ b/internal/storage/id_generation.go @@ -0,0 +1,29 @@ +package storage + +import ( + "github.com/Lukaesebrot/pasty/internal/env" + "github.com/Lukaesebrot/pasty/internal/utils" + "strconv" +) + +// AcquireID generates a new unique ID +func AcquireID() (string, error) { + // Read the ID length + rawLength := env.Get("ID_LENGTH", "6") + length, err := strconv.Atoi(rawLength) + if err != nil { + return "", err + } + + // Generate the unique ID + for { + id := utils.RandomString(length) + paste, err := Current.Get(id) + if err != nil { + return "", err + } + if paste == nil { + return id, nil + } + } +} diff --git a/internal/storage/mongodb_driver.go b/internal/storage/mongodb_driver.go index 7b02126..d5a8ba9 100644 --- a/internal/storage/mongodb_driver.go +++ b/internal/storage/mongodb_driver.go @@ -4,7 +4,6 @@ import ( "context" "github.com/Lukaesebrot/pasty/internal/env" "github.com/Lukaesebrot/pasty/internal/pastes" - "github.com/bwmarrin/snowflake" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" @@ -50,7 +49,7 @@ func (driver *MongoDBDriver) Terminate() error { } // Get loads a paste -func (driver *MongoDBDriver) Get(id snowflake.ID) (*pastes.Paste, error) { +func (driver *MongoDBDriver) Get(id string) (*pastes.Paste, error) { // Define the collection to use for this database operation collection := driver.client.Database(driver.database).Collection(driver.collection) @@ -59,7 +58,7 @@ func (driver *MongoDBDriver) Get(id snowflake.ID) (*pastes.Paste, error) { defer cancel() // Try to retrieve the corresponding paste document - filter := bson.M{"_id": id.String()} + filter := bson.M{"_id": id} result := collection.FindOne(ctx, filter) err := result.Err() if err != nil { @@ -90,7 +89,7 @@ func (driver *MongoDBDriver) Save(paste *pastes.Paste) error { } // Delete deletes a paste -func (driver *MongoDBDriver) Delete(id snowflake.ID) error { +func (driver *MongoDBDriver) Delete(id string) error { // Define the collection to use for this database operation collection := driver.client.Database(driver.database).Collection(driver.collection) @@ -99,7 +98,7 @@ func (driver *MongoDBDriver) Delete(id snowflake.ID) error { defer cancel() // Delete the document - filter := bson.M{"_id": id.String()} + filter := bson.M{"_id": id} _, err := collection.DeleteOne(ctx, filter) return err } diff --git a/internal/storage/s3_driver.go b/internal/storage/s3_driver.go index ded771d..ddfcbe6 100644 --- a/internal/storage/s3_driver.go +++ b/internal/storage/s3_driver.go @@ -6,7 +6,6 @@ import ( "encoding/json" "github.com/Lukaesebrot/pasty/internal/env" "github.com/Lukaesebrot/pasty/internal/pastes" - "github.com/bwmarrin/snowflake" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" "io/ioutil" @@ -39,9 +38,9 @@ func (driver *S3Driver) Terminate() error { } // Get loads a paste -func (driver *S3Driver) Get(id snowflake.ID) (*pastes.Paste, error) { +func (driver *S3Driver) Get(id string) (*pastes.Paste, error) { // Read the object - object, err := driver.client.GetObject(context.Background(), driver.bucket, id.String()+".json", minio.GetObjectOptions{}) + object, err := driver.client.GetObject(context.Background(), driver.bucket, id+".json", minio.GetObjectOptions{}) if err != nil { return nil, err } @@ -69,13 +68,13 @@ func (driver *S3Driver) Save(paste *pastes.Paste) error { // Put the object reader := bytes.NewReader(jsonBytes) - _, err = driver.client.PutObject(context.Background(), driver.bucket, paste.ID.String()+".json", reader, reader.Size(), minio.PutObjectOptions{ + _, err = driver.client.PutObject(context.Background(), driver.bucket, paste.ID+".json", reader, reader.Size(), minio.PutObjectOptions{ ContentType: "application/json", }) return err } // Delete deletes a paste -func (driver *S3Driver) Delete(id snowflake.ID) error { - return driver.client.RemoveObject(context.Background(), driver.bucket, id.String()+".json", minio.RemoveObjectOptions{}) +func (driver *S3Driver) Delete(id string) error { + return driver.client.RemoveObject(context.Background(), driver.bucket, id+".json", minio.RemoveObjectOptions{}) } diff --git a/internal/utils/randomString.go b/internal/utils/randomString.go new file mode 100644 index 0000000..a77029d --- /dev/null +++ b/internal/utils/randomString.go @@ -0,0 +1,15 @@ +package utils + +import "math/rand" + +// stringContents holds the chars a random string can contain +const stringContents = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + +// RandomString returns a random string with the given length +func RandomString(length int) string { + bytes := make([]byte, length) + for i := range bytes { + bytes[i] = stringContents[rand.Int63()%int64(len(stringContents))] + } + return string(bytes) +} diff --git a/internal/web/controllers/v1/hastebin_support.go b/internal/web/controllers/v1/hastebin_support.go index 9673c1d..b801a2b 100644 --- a/internal/web/controllers/v1/hastebin_support.go +++ b/internal/web/controllers/v1/hastebin_support.go @@ -24,8 +24,16 @@ func HastebinSupportHandler(ctx *fasthttp.RequestCtx) { return } + // Acquire the paste ID + id, err := storage.AcquireID() + if err != nil { + ctx.SetStatusCode(fasthttp.StatusInternalServerError) + ctx.SetBodyString(err.Error()) + return + } + // Create the paste object - paste, err := pastes.Create(content) + paste, err := pastes.Create(id, content) if err != nil { ctx.SetStatusCode(fasthttp.StatusInternalServerError) ctx.SetBodyString(err.Error()) @@ -50,7 +58,7 @@ func HastebinSupportHandler(ctx *fasthttp.RequestCtx) { // Respond with the paste key jsonData, _ := json.Marshal(map[string]string{ - "key": paste.ID.String(), + "key": paste.ID, }) ctx.SetBody(jsonData) } diff --git a/internal/web/controllers/v1/pastes.go b/internal/web/controllers/v1/pastes.go index 65df8e3..f8f7171 100644 --- a/internal/web/controllers/v1/pastes.go +++ b/internal/web/controllers/v1/pastes.go @@ -4,7 +4,6 @@ import ( "encoding/json" "github.com/Lukaesebrot/pasty/internal/pastes" "github.com/Lukaesebrot/pasty/internal/storage" - "github.com/bwmarrin/snowflake" "github.com/fasthttp/router" limitFasthttp "github.com/ulule/limiter/v3/drivers/middleware/fasthttp" "github.com/valyala/fasthttp" @@ -19,13 +18,8 @@ func InitializePastesController(group *router.Group, rateLimiterMiddleware *limi // v1GetPaste handles the 'GET /v1/pastes/{id}' endpoint func v1GetPaste(ctx *fasthttp.RequestCtx) { - // Parse the ID - id, err := snowflake.ParseString(ctx.UserValue("id").(string)) - if err != nil { - ctx.SetStatusCode(fasthttp.StatusBadRequest) - ctx.SetBodyString("invalid ID format") - return - } + // Read the ID + id := ctx.UserValue("id").(string) // Retrieve the paste paste, err := storage.Current.Get(id) @@ -68,8 +62,16 @@ func v1PostPaste(ctx *fasthttp.RequestCtx) { return } + // Acquire the paste ID + id, err := storage.AcquireID() + if err != nil { + ctx.SetStatusCode(fasthttp.StatusInternalServerError) + ctx.SetBodyString(err.Error()) + return + } + // Create the paste object - paste, err := pastes.Create(values["content"]) + paste, err := pastes.Create(id, values["content"]) if err != nil { ctx.SetStatusCode(fasthttp.StatusInternalServerError) ctx.SetBodyString(err.Error()) @@ -105,17 +107,12 @@ func v1PostPaste(ctx *fasthttp.RequestCtx) { // v1DeletePaste handles the 'DELETE /v1/pastes/{id}' func v1DeletePaste(ctx *fasthttp.RequestCtx) { - // Parse the ID - id, err := snowflake.ParseString(ctx.UserValue("id").(string)) - if err != nil { - ctx.SetStatusCode(fasthttp.StatusBadRequest) - ctx.SetBodyString("invalid ID format") - return - } + // Read the ID + id := ctx.UserValue("id").(string) // Unmarshal the body values := make(map[string]string) - err = json.Unmarshal(ctx.PostBody(), &values) + err := json.Unmarshal(ctx.PostBody(), &values) if err != nil { ctx.SetStatusCode(fasthttp.StatusBadRequest) ctx.SetBodyString("invalid request body")