selfhosted
/
coder
mirror of https://github.com/coder/coder.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

feat: support template bundles as zip archives (#11839)

This commit is contained in:
Marcin Tojek 2024-01-31 14:49:55 +01:00 committed by GitHub
parent b25deaae20
commit 13cbca679e
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 248 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
coderd/apidoc/docs.go generated
View File

@ -920,7 +920,7 @@ const docTemplate = `{
{
"type": "string",
"default": "application/x-tar",
"description": "Content-Type must be ` + "`" + `application/x-tar` + "`" + `",
"description": "Content-Type must be ` + "`" + `application/x-tar` + "`" + ` or ` + "`" + `application/zip` + "`" + `",
"name": "Content-Type",
"in": "header",
"required": true

2
coderd/apidoc/swagger.json generated
View File

@ -788,7 +788,7 @@
{
"type": "string",
"default": "application/x-tar",
"description": "Content-Type must be `application/x-tar`",
"description": "Content-Type must be `application/x-tar` or `application/zip`",
"name": "Content-Type",
"in": "header",
"required": true

69
coderd/files.go
View File

@ -1,6 +1,9 @@
package coderd
import (
"archive/tar"
"archive/zip"
"bytes"
"crypto/sha256"
"database/sql"
"encoding/hex"
@ -9,6 +12,7 @@ import (
"io"
"net/http"
"cdr.dev/slog"
"github.com/go-chi/chi/v5"
"github.com/google/uuid"
@ -21,6 +25,9 @@ import (
const (
tarMimeType = "application/x-tar"
zipMimeType = "application/zip"
httpFileMaxBytes = 10 * (10 << 20)
)
// @Summary Upload file
@ -30,7 +37,7 @@ const (
// @Produce json
// @Accept application/x-tar
// @Tags Files
// @Param Content-Type header string true "Content-Type must be `application/x-tar`" default(application/x-tar)
// @Param Content-Type header string true "Content-Type must be `application/x-tar` or `application/zip`" default(application/x-tar)
// @Param file formData file true "File to be uploaded"
// @Success 201 {object} codersdk.UploadResponse
// @Router /files [post]
@ -39,9 +46,8 @@ func (api *API) postFile(rw http.ResponseWriter, r *http.Request) {
apiKey := httpmw.APIKey(r)
contentType := r.Header.Get("Content-Type")
switch contentType {
case tarMimeType:
case tarMimeType, zipMimeType:
default:
httpapi.Write(ctx, rw, http.StatusBadRequest, codersdk.Response{
Message: fmt.Sprintf("Unsupported content type header %q.", contentType),
@ -49,7 +55,7 @@ func (api *API) postFile(rw http.ResponseWriter, r *http.Request) {
return
}
r.Body = http.MaxBytesReader(rw, r.Body, 10*(10<<20))
r.Body = http.MaxBytesReader(rw, r.Body, httpFileMaxBytes)
data, err := io.ReadAll(r.Body)
if err != nil {
httpapi.Write(ctx, rw, http.StatusBadRequest, codersdk.Response{
@ -58,6 +64,28 @@ func (api *API) postFile(rw http.ResponseWriter, r *http.Request) {
})
return
}
if contentType == zipMimeType {
zipReader, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
if err != nil {
httpapi.Write(ctx, rw, http.StatusBadRequest, codersdk.Response{
Message: "Incomplete .zip archive file.",
Detail: err.Error(),
})
return
}
data, err = CreateTarFromZip(zipReader)
if err != nil {
httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{
Message: "Internal error processing .zip archive.",
Detail: err.Error(),
})
return
}
contentType = tarMimeType
}
hashBytes := sha256.Sum256(data)
hash := hex.EncodeToString(hashBytes[:])
file, err := api.Database.GetFileByHashAndCreator(ctx, database.GetFileByHashAndCreatorParams{
@ -108,7 +136,10 @@ func (api *API) postFile(rw http.ResponseWriter, r *http.Request) {
// @Success 200
// @Router /files/{fileID} [get]
func (api *API) fileByID(rw http.ResponseWriter, r *http.Request) {
ctx := r.Context()
var (
ctx = r.Context()
format = r.URL.Query().Get("format")
)
fileID := chi.URLParam(r, "fileID")
if fileID == "" {
@ -139,7 +170,29 @@ func (api *API) fileByID(rw http.ResponseWriter, r *http.Request) {
return
}
rw.Header().Set("Content-Type", file.Mimetype)
rw.WriteHeader(http.StatusOK)
_, _ = rw.Write(file.Data)
switch format {
case codersdk.FormatZip:
if file.Mimetype != codersdk.ContentTypeTar {
httpapi.Write(ctx, rw, http.StatusBadRequest, codersdk.Response{
Message: "Only .tar files can be converted to .zip format",
Detail: err.Error(),
})
return
}
rw.Header().Set("Content-Type", codersdk.ContentTypeZip)
rw.WriteHeader(http.StatusOK)
err = WriteZipArchive(rw, tar.NewReader(bytes.NewReader(file.Data)))
if err != nil {
api.Logger.Error(ctx, "invalid .zip archive", slog.F("file_id", fileID), slog.F("mimetype", file.Mimetype), slog.Error(err))
}
case "": // no format? no conversion
rw.Header().Set("Content-Type", file.Mimetype)
_, _ = rw.Write(file.Data)
default:
httpapi.Write(ctx, rw, http.StatusBadRequest, codersdk.Response{
Message: "Unsupported conversion format.",
Detail: err.Error(),
})
}
}

69
coderd/files_test.go
View File

@ -1,6 +1,7 @@
package coderd_test
import (
"archive/tar"
"bytes"
"context"
"net/http"
@ -9,8 +10,10 @@ import (
"github.com/google/uuid"
"github.com/stretchr/testify/require"
"github.com/coder/coder/v2/coderd"
"github.com/coder/coder/v2/coderd/coderdtest"
"github.com/coder/coder/v2/codersdk"
"github.com/coder/coder/v2/provisioner/echo"
"github.com/coder/coder/v2/testutil"
)
@ -72,19 +75,83 @@ func TestDownload(t *testing.T) {
require.Equal(t, http.StatusNotFound, apiErr.StatusCode())
})
t.Run("Insert", func(t *testing.T) {
t.Run("InsertTar_DownloadTar", func(t *testing.T) {
t.Parallel()
client := coderdtest.New(t, nil)
_ = coderdtest.CreateFirstUser(t, client)
// given
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
defer cancel()
// when
resp, err := client.Upload(ctx, codersdk.ContentTypeTar, bytes.NewReader(make([]byte, 1024)))
require.NoError(t, err)
data, contentType, err := client.Download(ctx, resp.ID)
require.NoError(t, err)
// then
require.Len(t, data, 1024)
require.Equal(t, codersdk.ContentTypeTar, contentType)
})
t.Run("InsertZip_DownloadTar", func(t *testing.T) {
t.Parallel()
client := coderdtest.New(t, nil)
_ = coderdtest.CreateFirstUser(t, client)
// given
tarball, err := echo.Tar(&echo.Responses{
Parse: echo.ParseComplete,
ProvisionApply: echo.ApplyComplete,
})
require.NoError(t, err)
tarReader := tar.NewReader(bytes.NewReader(tarball))
zipContent, err := coderd.CreateZipFromTar(tarReader)
require.NoError(t, err)
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
defer cancel()
// when
resp, err := client.Upload(ctx, codersdk.ContentTypeZip, bytes.NewReader(zipContent))
require.NoError(t, err)
data, contentType, err := client.Download(ctx, resp.ID)
require.NoError(t, err)
// then
require.Equal(t, codersdk.ContentTypeTar, contentType)
require.Equal(t, tarball, data)
})
t.Run("InsertTar_DownloadZip", func(t *testing.T) {
t.Parallel()
client := coderdtest.New(t, nil)
_ = coderdtest.CreateFirstUser(t, client)
// given
tarball, err := echo.Tar(&echo.Responses{
Parse: echo.ParseComplete,
ProvisionApply: echo.ApplyComplete,
})
require.NoError(t, err)
tarReader := tar.NewReader(bytes.NewReader(tarball))
expectedZip, err := coderd.CreateZipFromTar(tarReader)
require.NoError(t, err)
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
defer cancel()
// when
resp, err := client.Upload(ctx, codersdk.ContentTypeTar, bytes.NewReader(tarball))
require.NoError(t, err)
data, contentType, err := client.DownloadWithFormat(ctx, resp.ID, codersdk.FormatZip)
require.NoError(t, err)
// then
require.Equal(t, codersdk.ContentTypeZip, contentType)
require.Equal(t, expectedZip, data)
})
}

101
coderd/fileszip.go Normal file
View File

@ -0,0 +1,101 @@
package coderd
import (
"archive/tar"
"archive/zip"
"bytes"
"errors"
"io"
"log"
)
func CreateTarFromZip(zipReader *zip.Reader) ([]byte, error) {
var tarBuffer bytes.Buffer
err := writeTarArchive(&tarBuffer, zipReader)
if err != nil {
return nil, err
}
return tarBuffer.Bytes(), nil
}
func writeTarArchive(w io.Writer, zipReader *zip.Reader) error {
tarWriter := tar.NewWriter(w)
defer tarWriter.Close()
for _, file := range zipReader.File {
err := processFileInZipArchive(file, tarWriter)
if err != nil {
return err
}
}
return nil
}
func processFileInZipArchive(file *zip.File, tarWriter *tar.Writer) error {
fileReader, err := file.Open()
if err != nil {
return err
}
defer fileReader.Close()
err = tarWriter.WriteHeader(&tar.Header{
Name: file.Name,
Size: file.FileInfo().Size(),
Mode: 0o644,
})
if err != nil {
return err
}
n, err := io.CopyN(tarWriter, fileReader, httpFileMaxBytes)
log.Println(file.Name, n, err)
if errors.Is(err, io.EOF) {
err = nil
}
return err
}
func CreateZipFromTar(tarReader *tar.Reader) ([]byte, error) {
var zipBuffer bytes.Buffer
err := WriteZipArchive(&zipBuffer, tarReader)
if err != nil {
return nil, err
}
return zipBuffer.Bytes(), nil
}
func WriteZipArchive(w io.Writer, tarReader *tar.Reader) error {
zipWriter := zip.NewWriter(w)
defer zipWriter.Close()
for {
tarHeader, err := tarReader.Next()
if errors.Is(err, io.EOF) {
break
}
if err != nil {
return err
}
zipHeader, err := zip.FileInfoHeader(tarHeader.FileInfo())
if err != nil {
return err
}
zipHeader.Name = tarHeader.Name
zipEntry, err := zipWriter.CreateHeader(zipHeader)
if err != nil {
return err
}
_, err = io.CopyN(zipEntry, tarReader, httpFileMaxBytes)
if errors.Is(err, io.EOF) {
err = nil
}
if err != nil {
return err
}
}
return nil // don't need to flush as we call `writer.Close()`
}

10
codersdk/files.go
View File

@ -12,6 +12,9 @@ import (
const (
ContentTypeTar = "application/x-tar"
ContentTypeZip = "application/zip"
FormatZip = "zip"
)
// UploadResponse contains the hash to reference the uploaded file.
@ -38,7 +41,12 @@ func (c *Client) Upload(ctx context.Context, contentType string, rd io.Reader) (
// Download fetches a file by uploaded hash.
func (c *Client) Download(ctx context.Context, id uuid.UUID) ([]byte, string, error) {
res, err := c.Request(ctx, http.MethodGet, fmt.Sprintf("/api/v2/files/%s", id.String()), nil)
return c.DownloadWithFormat(ctx, id, "")
}
// Download fetches a file by uploaded hash, but it forces format conversion.
func (c *Client) DownloadWithFormat(ctx context.Context, id uuid.UUID, format string) ([]byte, string, error) {
res, err := c.Request(ctx, http.MethodGet, fmt.Sprintf("/api/v2/files/%s?format=%s", id.String(), format), nil)
if err != nil {
return nil, "", err
}

10
docs/api/files.md generated
View File

@ -22,11 +22,11 @@ file: string
### Parameters
| Name | In | Type | Required | Description |
| -------------- | ------ | ------ | -------- | ---------------------------------------- |
| `Content-Type` | header | string | true | Content-Type must be `application/x-tar` |
| `body` | body | object | true | |
| `» file` | body | binary | true | File to be uploaded |
| Name | In | Type | Required | Description |
| -------------- | ------ | ------ | -------- | ------------------------------------------------------------- |
| `Content-Type` | header | string | true | Content-Type must be `application/x-tar` or `application/zip` |
| `body` | body | object | true | |
| `» file` | body | binary | true | File to be uploaded |
### Example responses

3
provisioner/echo/serve.go
View File

@ -306,7 +306,8 @@ func TarWithOptions(ctx context.Context, logger slog.Logger, responses *Response
}
}
}
err := writer.Flush()
// `writer.Close()` function flushes the writer buffer, and adds extra padding to create a legal tarball.
err := writer.Close()
if err != nil {
return nil, err
}