Updated skeleton generator to work with automated tests.

This commit is contained in:
Nick Gerakines 2020-04-10 11:58:35 -04:00
parent 0535b947ce
commit 05355d7828
No known key found for this signature in database
GPG Key ID: 33D43D854F96B2E4
1 changed files with 265 additions and 172 deletions

View File

@ -1,10 +1,17 @@
package janitor
import (
"bytes"
"crypto/md5"
"crypto/rand"
"crypto/rsa"
"crypto/x509"
"encoding/hex"
"encoding/pem"
"fmt"
"io/ioutil"
"math/big"
"os"
"path/filepath"
"strings"
"time"
@ -22,16 +29,6 @@ var SkeletonCommand = cli.Command{
Usage: "The location to write files to.",
Value: "./",
},
&cli.StringFlag{
Name: "prefix",
Usage: "Set the prefix of the site.",
Value: "",
},
&cli.StringFlag{
Name: "domain",
Usage: "Set the domain of the site.",
Required: true,
},
&cli.StringFlag{
Name: "tag",
Usage: "The container tag to use.",
@ -42,123 +39,103 @@ var SkeletonCommand = cli.Command{
Usage: "The port to run web on",
Value: 9000,
},
&cli.IntFlag{
Name: "publisher-port",
Usage: "The port to run web on",
Value: 9200,
&cli.StringSliceFlag{
Name: "domain",
Usage: "Set the domains of the sites.",
Required: true,
},
&cli.IntFlag{
Name: "svger-port",
Usage: "The port to run svger on",
Value: 9100,
&cli.StringSliceFlag{
Name: "web-name",
Usage: "The names of the sites.",
Required: false,
Value: cli.NewStringSlice("web"),
},
&cli.StringSliceFlag{
Name: "web-user",
Usage: "The admin users of the sites. The email will be the @tvrn.dev suffix",
Required: false,
},
&cli.BoolFlag{
Name: "pem",
Usage: "Generate a PEM for each site.",
Value: true,
},
&cli.BoolFlag{
Name: "mount-assets",
Usage: "Mount the assets directory locally.",
Value: false,
},
&cli.BoolFlag{
Name: "mount-db",
Usage: "Mount the db directory locally.",
Value: false,
},
},
Action: skeletonCommandAction,
}
const (
svgerPort = 9100
publisherPort = 9200
)
func skeletonCommandAction(cliCtx *cli.Context) error {
prefix := cliCtx.String("prefix")
domains := cliCtx.StringSlice("domain")
webNames := cliCtx.StringSlice("web-name")
dbEnvFile := fmt.Sprintf("%sdb.env", prefix)
webEnvFile := fmt.Sprintf("%sweb.env", prefix)
initSqlFile := fmt.Sprintf("%sinit.sql", prefix)
dockerComposeFile := fmt.Sprintf("%sdocker-compose.yml", prefix)
if len(domains) != len(webNames) {
return fmt.Errorf("equal number of web-name and domain values expected")
}
webPort := cliCtx.Int("web-port")
publisherPort := cliCtx.Int("publisher-port")
svgerPort := cliCtx.Int("svger-port")
internalNetwork := fmt.Sprintf("%sinternal_network", prefix)
externalNetwork := fmt.Sprintf("%sexternal_network", prefix)
dbServiceName := "db"
svgerServiceName := "svger"
webServiceName := "web"
publisherServiceName := "publisher"
dbPassword := secretGen("database secret")
dockerComposeConfig := make(map[string]interface{})
services := make(map[string]interface{})
dbServiceConfig := make(map[string]interface{})
dbServiceConfig["restart"] = "on-failure"
dbServiceConfig["image"] = "postgres:12-alpine"
dbServiceConfig["networks"] = []string{internalNetwork}
dbServiceConfig["volumes"] = []string{
fmt.Sprintf("./%s:/docker-entrypoint-initdb.d/10-init.sql", initSqlFile),
fmt.Sprintf("./%spostgres:/var/lib/postgresql/data", prefix),
}
dbServiceConfig["env_file"] = []string{
fmt.Sprintf("./%s", dbEnvFile),
dbServiceConfig, err := dockerComposeDB(cliCtx, dbPassword)
if err != nil {
return err
}
services[dbServiceName] = dbServiceConfig
services["db"] = dbServiceConfig
svgerServiceConfig := make(map[string]interface{})
svgerServiceConfig["restart"] = "on-failure"
svgerServiceConfig["image"] = "ngerakines/svger:1.1.0"
svgerServiceConfig["networks"] = []string{
internalNetwork,
services["svger"] = map[string]interface{}{
"restart": "on-failure",
"image": "ngerakines/svger:1.1.0",
"networks": []string{"internal_network"},
"ports": []string{fmt.Sprintf("%d:%d", svgerPort, svgerPort)},
"environment": []string{
fmt.Sprintf("PORT=%d", svgerPort),
},
}
svgerServiceConfig["ports"] = []string{
fmt.Sprintf("%d:%d", svgerPort, svgerPort),
}
svgerServiceConfig["environment"] = []string{
fmt.Sprintf("PORT=%d", svgerPort),
services["publisher"] = map[string]interface{}{
"restart": "on-failure",
"image": fmt.Sprintf("ngerakines/tavern:%s", cliCtx.String("tag")),
"networks": []string{"internal_network", "external_network"},
"command": "publisher",
"ports": []string{fmt.Sprintf("%d:%d", publisherPort, publisherPort)},
"environment": []string{
fmt.Sprintf("LISTEN=0.0.0.0:%d", publisherPort),
fmt.Sprintf("PUBLISHER_CALLBACK=http://%s:%d/webhooks/publisher", webNames[0], cliCtx.Int("web-port")),
},
}
services[svgerServiceName] = svgerServiceConfig
for i, webName := range webNames {
webServiceConfig := make(map[string]interface{})
webServiceConfig["restart"] = "on-failure"
webServiceConfig["image"] = fmt.Sprintf("ngerakines/tavern:%s", cliCtx.String("tag"))
webServiceConfig["networks"] = []string{
externalNetwork,
internalNetwork,
}
webServiceConfig["ports"] = []string{
fmt.Sprintf("%d:%d", webPort, webPort),
}
webServiceConfig["depends_on"] = []string{
dbServiceName,
svgerServiceName,
publisherServiceName,
}
webServiceConfig["env_file"] = []string{
fmt.Sprintf("./%s", webEnvFile),
}
webServiceConfig["volumes"] = []string{
fmt.Sprintf("./%sassets:/assets", prefix),
}
webServiceConfig, err := dockerComposeWeb(cliCtx, i, webName, domains[i], dbPassword)
if err != nil {
return err
}
services[webServiceName] = webServiceConfig
publisherServiceConfig := make(map[string]interface{})
publisherServiceConfig["restart"] = "on-failure"
publisherServiceConfig["image"] = fmt.Sprintf("ngerakines/tavern:%s", cliCtx.String("tag"))
publisherServiceConfig["networks"] = []string{
internalNetwork,
externalNetwork,
services[webName] = webServiceConfig
}
publisherServiceConfig["command"] = []string{
"publisher",
}
publisherServiceConfig["ports"] = []string{
fmt.Sprintf("%d:%d", publisherPort, publisherPort),
}
publisherServiceConfig["environment"] = []string{
fmt.Sprintf("LISTEN=0.0.0.0:%d", publisherPort),
fmt.Sprintf("PUBLISHER_CALLBACK=http://web:%d/webhooks/publisher", webPort),
}
services[publisherServiceName] = publisherServiceConfig
dockerComposeConfig["services"] = services
dockerComposeConfig["version"] = "3"
dockerComposeConfig["networks"] = map[string]interface{}{
externalNetwork: nil,
internalNetwork: map[string]interface{}{
"external_network": map[string]interface{}{},
"internal_network": map[string]interface{}{
"internal": true,
},
}
@ -168,20 +145,126 @@ func skeletonCommandAction(cliCtx *cli.Context) error {
return err
}
dbPasswordH := md5.New()
dbPasswordH.Write([]byte(time.Now().String()))
dbPasswordH.Write([]byte("database password"))
var runFileContent strings.Builder
runFileContent.WriteString("#!/bin/sh\n")
runFileContent.WriteString("docker-compose -f docker-compose.yml up -d db svger publisher\n")
runFileContent.WriteString("sleep 15\n")
for _, name := range webNames {
runFileContent.WriteString(fmt.Sprintf("docker-compose -f docker-compose.yml run %s migrate\n", name))
runFileContent.WriteString("sleep 5\n")
}
for _, name := range webNames {
user, email, err := webUser(cliCtx, name)
if err != nil {
return err
}
runFileContent.WriteString(fmt.Sprintf("docker-compose -f docker-compose.yml run %s init-service --pem /service.pem\n", name))
runFileContent.WriteString(fmt.Sprintf("docker-compose -f docker-compose.yml run %s init-admin --admin-email=%s --admin-password=password --admin-name=%s\n", name, email, user))
}
runFileContent.WriteString("docker-compose -f docker-compose.yml up -d\n")
runFileContent.WriteString("docker-compose -f docker-compose.yml logs --tail=50 -f\n")
webSecretH := md5.New()
webSecretH.Write([]byte(time.Now().String()))
webSecretH.Write([]byte("web secret"))
var restartFileContent strings.Builder
restartFileContent.WriteString("#!/bin/sh\n")
restartFileContent.WriteString("docker-compose -f docker-compose.yml up -d\n")
restartFileContent.WriteString("docker-compose -f docker-compose.yml logs --tail=50 -f\n")
dbPassword := hex.EncodeToString(dbPasswordH.Sum(nil))
webSecret := hex.EncodeToString(webSecretH.Sum(nil))
if err = writeStringToFile(cliCtx.String("destination"), "run.sh", runFileContent.String()); err != nil {
return err
}
if err = writeStringToFile(cliCtx.String("destination"), "restart.sh", restartFileContent.String()); err != nil {
return err
}
if err = writeBytesToFile(cliCtx.String("destination"), "docker-compose.yml", d); err != nil {
return err
}
for _, fileName := range []string{"run.sh", "restart.sh"} {
if err := os.Chmod(filepath.Join(cliCtx.String("destination"), fileName), 0744); err != nil {
return err
}
}
return nil
}
func dockerComposeDB(cliCtx *cli.Context, dbPassword string) (map[string]interface{}, error) {
volumes := make([]string, 0)
initSqlFile := fmt.Sprintf("init.sql")
volumes = append(volumes, "./init.sql:/docker-entrypoint-initdb.d/10-init.sql")
if cliCtx.Bool("mount-db") {
volumes = append(volumes, "./postgres:/var/lib/postgresql/data")
}
dbServiceConfig := make(map[string]interface{})
dbServiceConfig["restart"] = "on-failure"
dbServiceConfig["image"] = "postgres:12-alpine"
dbServiceConfig["networks"] = []string{"internal_network"}
dbServiceConfig["volumes"] = volumes
dbServiceConfig["env_file"] = []string{
fmt.Sprintf("./%s", "db.env"),
}
var dbEnvFileContent strings.Builder
dbEnvFileContent.WriteString(fmt.Sprintf("POSTGRES_PASSWORD=%s\n", dbPassword))
if err := writeStringToFile(cliCtx.String("destination"), "db.env", dbEnvFileContent.String()); err != nil {
return nil, err
}
var initSqlFileContent strings.Builder
initSqlFileContent.WriteString(`CREATE EXTENSION IF NOT EXISTS "uuid-ossp";` + "\n")
for _, database := range cliCtx.StringSlice("web-name") {
initSqlFileContent.WriteString(fmt.Sprintf("CREATE DATABASE %s;\n", database))
}
if err := writeStringToFile(cliCtx.String("destination"), initSqlFile, initSqlFileContent.String()); err != nil {
return nil, err
}
return dbServiceConfig, nil
}
func dockerComposeWeb(cliCtx *cli.Context, pos int, name, domain, dbPassword string) (map[string]interface{}, error) {
tag := cliCtx.String("tag")
webPort := cliCtx.Int("web-port") + pos
destination := cliCtx.String("destination")
user, _, err := webUser(cliCtx, name)
if err != nil {
return nil, err
}
webSecret := secretGen(name, "web secret")
webEnvFile := fmt.Sprintf("%s.env", name)
volumes := make([]string, 0)
if cliCtx.Bool("mount-assets") {
volumes = append(volumes, fmt.Sprintf("./web_%d_assets:/assets", pos))
}
if cliCtx.Bool("pem") {
volumes = append(volumes, fmt.Sprintf("./%s.pem:/service.pem", name))
}
webServiceConfig := make(map[string]interface{})
webServiceConfig["restart"] = "on-failure"
webServiceConfig["image"] = fmt.Sprintf("ngerakines/tavern:%s", tag)
webServiceConfig["networks"] = []string{"internal_network", "external_network"}
webServiceConfig["ports"] = []string{
fmt.Sprintf("%d:%d", webPort, webPort),
}
webServiceConfig["depends_on"] = []string{"db", "svger", "publisher"}
webServiceConfig["env_file"] = []string{
fmt.Sprintf("./%s", webEnvFile),
}
webServiceConfig["volumes"] = volumes
var webEnvFileContent strings.Builder
for _, line := range []string{
"ENABLE_SENTRY=false",
@ -189,14 +272,14 @@ func skeletonCommandAction(cliCtx *cli.Context) error {
"ENABLE_PUBLISHER=true",
"ENABLE_GROUPS=true",
fmt.Sprintf("SECRET=%s", webSecret),
fmt.Sprintf("DOMAIN=%s", cliCtx.String("domain")),
fmt.Sprintf("DATABASE=postgresql://postgres:%s@%s:5432/tavern?sslmode=disable", dbPassword, dbServiceName),
fmt.Sprintf("SVGER=http://%s:%d/", svgerServiceName, svgerPort),
fmt.Sprintf("DOMAIN=%s", domain),
fmt.Sprintf("DATABASE=postgresql://postgres:%s@db:5432/%s?sslmode=disable", dbPassword, name),
fmt.Sprintf("SVGER=http://svger:%d/", svgerPort),
fmt.Sprintf("LISTEN=0.0.0.0:%d", webPort),
"ASSET_STORAGE_FILE_BASE=/assets/",
"ASSET_STORAGE_REMOTE_DENY=*",
"ALLOW_REPLY_COLLECTION_UPDATES=true",
fmt.Sprintf("PUBLISHER=http://%s:%d/", publisherServiceName, publisherPort),
fmt.Sprintf("PUBLISHER=http://publisher:%d/", publisherPort),
"ALLOW_OBJECT_FOLLOW=true",
"ALLOW_AUTO_ACCEPT_FOLLOWERS=true",
"ALLOW_INBOX_FORWARDING=true",
@ -204,73 +287,83 @@ func skeletonCommandAction(cliCtx *cli.Context) error {
"ALLOW_AUTO_ACCEPT_GROUP_FOLLOWERS=true",
"ALLOW_REMOTE_GROUP_FOLLOWERS=true",
"DEFAULT_GROUP_MEMBER_ROLE=1",
fmt.Sprintf("ADMIN_NAME=%s", user),
} {
webEnvFileContent.WriteString(line)
webEnvFileContent.WriteString("\n")
}
var initSqlFileContent strings.Builder
for _, line := range []string{
`CREATE DATABASE tavern;`,
`CREATE EXTENSION IF NOT EXISTS "uuid-ossp";`,
} {
initSqlFileContent.WriteString(line)
initSqlFileContent.WriteString("\n")
if err := writeStringToFile(destination, webEnvFile, webEnvFileContent.String()); err != nil {
return nil, err
}
var runFileContent strings.Builder
for _, line := range []string{
"#!/bin/sh",
"",
fmt.Sprintf(`docker-compose -f %s up -d db svger publisher`, dockerComposeFile),
"sleep 30",
fmt.Sprintf(`docker-compose -f %s run web migrate`, dockerComposeFile),
"sleep 5",
fmt.Sprintf(`docker-compose -f %s run web init --admin-email=nick.gerakines@gmail.com --admin-password=password --admin-name=nick`, dockerComposeFile),
fmt.Sprintf(`docker-compose -f %s up -d`, dockerComposeFile),
fmt.Sprintf(`docker-compose -f %s logs --tail=50 -f`, dockerComposeFile),
} {
runFileContent.WriteString(line)
runFileContent.WriteString("\n")
if cliCtx.Bool("pem") {
privateKey, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
return nil, err
}
n := privateKey.PublicKey.N.Bytes()
e := big.NewInt(int64(privateKey.PublicKey.E)).Bytes()
h := md5.New()
h.Write(n)
h.Write(e)
fingerprint := hex.EncodeToString(h.Sum(nil))
privateKeyBytes := x509.MarshalPKCS1PrivateKey(privateKey)
var privateKeyBuffer bytes.Buffer
if err := pem.Encode(&privateKeyBuffer, &pem.Block{
Type: "PRIVATE KEY",
Bytes: privateKeyBytes,
Headers: map[string]string{
"id": fmt.Sprintf("https://%s/server#%s", domain, fingerprint),
},
}); err != nil {
return nil, err
}
if err := writeBytesToFile(destination, fmt.Sprintf("%s.pem", name), privateKeyBuffer.Bytes()); err != nil {
return nil, err
}
}
var restartFileContent strings.Builder
for _, line := range []string{
"#!/bin/sh",
"",
fmt.Sprintf(`docker-compose -f %s up -d`, dockerComposeFile),
fmt.Sprintf(`docker-compose -f %s logs --tail=50 -f`, dockerComposeFile),
} {
restartFileContent.WriteString(line)
restartFileContent.WriteString("\n")
}
runLoc := filepath.Join(cliCtx.String("destination"), fmt.Sprintf("%srun.sh", prefix))
restartLoc := filepath.Join(cliCtx.String("destination"), fmt.Sprintf("%srestart.sh", prefix))
dbEnvLoc := filepath.Join(cliCtx.String("destination"), dbEnvFile)
webEnvLoc := filepath.Join(cliCtx.String("destination"), webEnvFile)
initSqlLoc := filepath.Join(cliCtx.String("destination"), initSqlFile)
dcLoc := filepath.Join(cliCtx.String("destination"), dockerComposeFile)
err = ioutil.WriteFile(runLoc, []byte(runFileContent.String()), 0744)
if err != nil {
return err
}
err = ioutil.WriteFile(restartLoc, []byte(restartFileContent.String()), 0744)
if err != nil {
return err
}
err = ioutil.WriteFile(dbEnvLoc, []byte(dbEnvFileContent.String()), 0644)
if err != nil {
return err
}
err = ioutil.WriteFile(webEnvLoc, []byte(webEnvFileContent.String()), 0644)
if err != nil {
return err
}
err = ioutil.WriteFile(initSqlLoc, []byte(initSqlFileContent.String()), 0644)
if err != nil {
return err
}
return ioutil.WriteFile(dcLoc, d, 0644)
return webServiceConfig, nil
}
func writeStringToFile(destination, fileName, content string) error {
return writeBytesToFile(destination, fileName, []byte(content))
}
func writeBytesToFile(destination, fileName string, content []byte) error {
initSqlLoc := filepath.Join(destination, fileName)
return ioutil.WriteFile(initSqlLoc, content, 0644)
}
func secretGen(inputs ...string) string {
h := md5.New()
h.Write([]byte(time.Now().String()))
for _, input := range inputs {
h.Write([]byte(input))
}
return hex.EncodeToString(h.Sum(nil))
}
func webUser(cliCtx *cli.Context, webName string) (string, string, error) {
domains := cliCtx.StringSlice("domain")
webs := cliCtx.StringSlice("web-name")
users := cliCtx.StringSlice("web-user")
for i, web := range webs {
if web == webName {
if cliCtx.IsSet("web-user") {
user := users[i]
return user, fmt.Sprintf("%s@%s", user, domains[i]), nil
}
return "nick", fmt.Sprintf("nick@%s", domains[i]), nil
}
}
return "", "", fmt.Errorf("invalid domain, web-name, and web-user config")
}