diff --git a/coderd/apidoc/docs.go b/coderd/apidoc/docs.go index 4f7a7d3ebc..e4bb0c628d 100644 --- a/coderd/apidoc/docs.go +++ b/coderd/apidoc/docs.go @@ -920,7 +920,7 @@ const docTemplate = `{ { "type": "string", "default": "application/x-tar", - "description": "Content-Type must be ` + "`" + `application/x-tar` + "`" + `", + "description": "Content-Type must be ` + "`" + `application/x-tar` + "`" + ` or ` + "`" + `application/zip` + "`" + `", "name": "Content-Type", "in": "header", "required": true diff --git a/coderd/apidoc/swagger.json b/coderd/apidoc/swagger.json index c7696a4157..0e8460d4ed 100644 --- a/coderd/apidoc/swagger.json +++ b/coderd/apidoc/swagger.json @@ -788,7 +788,7 @@ { "type": "string", "default": "application/x-tar", - "description": "Content-Type must be `application/x-tar`", + "description": "Content-Type must be `application/x-tar` or `application/zip`", "name": "Content-Type", "in": "header", "required": true diff --git a/coderd/files.go b/coderd/files.go index a04ba1eace..d5379c4d8b 100644 --- a/coderd/files.go +++ b/coderd/files.go @@ -1,6 +1,9 @@ package coderd import ( + "archive/tar" + "archive/zip" + "bytes" "crypto/sha256" "database/sql" "encoding/hex" @@ -9,6 +12,7 @@ import ( "io" "net/http" + "cdr.dev/slog" "github.com/go-chi/chi/v5" "github.com/google/uuid" @@ -21,6 +25,9 @@ import ( const ( tarMimeType = "application/x-tar" + zipMimeType = "application/zip" + + httpFileMaxBytes = 10 * (10 << 20) ) // @Summary Upload file @@ -30,7 +37,7 @@ const ( // @Produce json // @Accept application/x-tar // @Tags Files -// @Param Content-Type header string true "Content-Type must be `application/x-tar`" default(application/x-tar) +// @Param Content-Type header string true "Content-Type must be `application/x-tar` or `application/zip`" default(application/x-tar) // @Param file formData file true "File to be uploaded" // @Success 201 {object} codersdk.UploadResponse // @Router /files [post] @@ -39,9 +46,8 @@ func (api *API) postFile(rw http.ResponseWriter, r *http.Request) { apiKey := httpmw.APIKey(r) contentType := r.Header.Get("Content-Type") - switch contentType { - case tarMimeType: + case tarMimeType, zipMimeType: default: httpapi.Write(ctx, rw, http.StatusBadRequest, codersdk.Response{ Message: fmt.Sprintf("Unsupported content type header %q.", contentType), @@ -49,7 +55,7 @@ func (api *API) postFile(rw http.ResponseWriter, r *http.Request) { return } - r.Body = http.MaxBytesReader(rw, r.Body, 10*(10<<20)) + r.Body = http.MaxBytesReader(rw, r.Body, httpFileMaxBytes) data, err := io.ReadAll(r.Body) if err != nil { httpapi.Write(ctx, rw, http.StatusBadRequest, codersdk.Response{ @@ -58,6 +64,28 @@ func (api *API) postFile(rw http.ResponseWriter, r *http.Request) { }) return } + + if contentType == zipMimeType { + zipReader, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) + if err != nil { + httpapi.Write(ctx, rw, http.StatusBadRequest, codersdk.Response{ + Message: "Incomplete .zip archive file.", + Detail: err.Error(), + }) + return + } + + data, err = CreateTarFromZip(zipReader) + if err != nil { + httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{ + Message: "Internal error processing .zip archive.", + Detail: err.Error(), + }) + return + } + contentType = tarMimeType + } + hashBytes := sha256.Sum256(data) hash := hex.EncodeToString(hashBytes[:]) file, err := api.Database.GetFileByHashAndCreator(ctx, database.GetFileByHashAndCreatorParams{ @@ -108,7 +136,10 @@ func (api *API) postFile(rw http.ResponseWriter, r *http.Request) { // @Success 200 // @Router /files/{fileID} [get] func (api *API) fileByID(rw http.ResponseWriter, r *http.Request) { - ctx := r.Context() + var ( + ctx = r.Context() + format = r.URL.Query().Get("format") + ) fileID := chi.URLParam(r, "fileID") if fileID == "" { @@ -139,7 +170,29 @@ func (api *API) fileByID(rw http.ResponseWriter, r *http.Request) { return } - rw.Header().Set("Content-Type", file.Mimetype) - rw.WriteHeader(http.StatusOK) - _, _ = rw.Write(file.Data) + switch format { + case codersdk.FormatZip: + if file.Mimetype != codersdk.ContentTypeTar { + httpapi.Write(ctx, rw, http.StatusBadRequest, codersdk.Response{ + Message: "Only .tar files can be converted to .zip format", + Detail: err.Error(), + }) + return + } + + rw.Header().Set("Content-Type", codersdk.ContentTypeZip) + rw.WriteHeader(http.StatusOK) + err = WriteZipArchive(rw, tar.NewReader(bytes.NewReader(file.Data))) + if err != nil { + api.Logger.Error(ctx, "invalid .zip archive", slog.F("file_id", fileID), slog.F("mimetype", file.Mimetype), slog.Error(err)) + } + case "": // no format? no conversion + rw.Header().Set("Content-Type", file.Mimetype) + _, _ = rw.Write(file.Data) + default: + httpapi.Write(ctx, rw, http.StatusBadRequest, codersdk.Response{ + Message: "Unsupported conversion format.", + Detail: err.Error(), + }) + } } diff --git a/coderd/files_test.go b/coderd/files_test.go index 1a3f407a6e..ff0eb60585 100644 --- a/coderd/files_test.go +++ b/coderd/files_test.go @@ -1,6 +1,7 @@ package coderd_test import ( + "archive/tar" "bytes" "context" "net/http" @@ -9,8 +10,10 @@ import ( "github.com/google/uuid" "github.com/stretchr/testify/require" + "github.com/coder/coder/v2/coderd" "github.com/coder/coder/v2/coderd/coderdtest" "github.com/coder/coder/v2/codersdk" + "github.com/coder/coder/v2/provisioner/echo" "github.com/coder/coder/v2/testutil" ) @@ -72,19 +75,83 @@ func TestDownload(t *testing.T) { require.Equal(t, http.StatusNotFound, apiErr.StatusCode()) }) - t.Run("Insert", func(t *testing.T) { + t.Run("InsertTar_DownloadTar", func(t *testing.T) { t.Parallel() client := coderdtest.New(t, nil) _ = coderdtest.CreateFirstUser(t, client) + // given ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong) defer cancel() + // when resp, err := client.Upload(ctx, codersdk.ContentTypeTar, bytes.NewReader(make([]byte, 1024))) require.NoError(t, err) data, contentType, err := client.Download(ctx, resp.ID) require.NoError(t, err) + + // then require.Len(t, data, 1024) require.Equal(t, codersdk.ContentTypeTar, contentType) }) + + t.Run("InsertZip_DownloadTar", func(t *testing.T) { + t.Parallel() + client := coderdtest.New(t, nil) + _ = coderdtest.CreateFirstUser(t, client) + + // given + tarball, err := echo.Tar(&echo.Responses{ + Parse: echo.ParseComplete, + ProvisionApply: echo.ApplyComplete, + }) + require.NoError(t, err) + + tarReader := tar.NewReader(bytes.NewReader(tarball)) + zipContent, err := coderd.CreateZipFromTar(tarReader) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong) + defer cancel() + + // when + resp, err := client.Upload(ctx, codersdk.ContentTypeZip, bytes.NewReader(zipContent)) + require.NoError(t, err) + data, contentType, err := client.Download(ctx, resp.ID) + require.NoError(t, err) + + // then + require.Equal(t, codersdk.ContentTypeTar, contentType) + require.Equal(t, tarball, data) + }) + + t.Run("InsertTar_DownloadZip", func(t *testing.T) { + t.Parallel() + client := coderdtest.New(t, nil) + _ = coderdtest.CreateFirstUser(t, client) + + // given + tarball, err := echo.Tar(&echo.Responses{ + Parse: echo.ParseComplete, + ProvisionApply: echo.ApplyComplete, + }) + require.NoError(t, err) + + tarReader := tar.NewReader(bytes.NewReader(tarball)) + expectedZip, err := coderd.CreateZipFromTar(tarReader) + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong) + defer cancel() + + // when + resp, err := client.Upload(ctx, codersdk.ContentTypeTar, bytes.NewReader(tarball)) + require.NoError(t, err) + data, contentType, err := client.DownloadWithFormat(ctx, resp.ID, codersdk.FormatZip) + require.NoError(t, err) + + // then + require.Equal(t, codersdk.ContentTypeZip, contentType) + require.Equal(t, expectedZip, data) + }) } diff --git a/coderd/fileszip.go b/coderd/fileszip.go new file mode 100644 index 0000000000..b001958912 --- /dev/null +++ b/coderd/fileszip.go @@ -0,0 +1,101 @@ +package coderd + +import ( + "archive/tar" + "archive/zip" + "bytes" + "errors" + "io" + "log" +) + +func CreateTarFromZip(zipReader *zip.Reader) ([]byte, error) { + var tarBuffer bytes.Buffer + err := writeTarArchive(&tarBuffer, zipReader) + if err != nil { + return nil, err + } + return tarBuffer.Bytes(), nil +} + +func writeTarArchive(w io.Writer, zipReader *zip.Reader) error { + tarWriter := tar.NewWriter(w) + defer tarWriter.Close() + + for _, file := range zipReader.File { + err := processFileInZipArchive(file, tarWriter) + if err != nil { + return err + } + } + return nil +} + +func processFileInZipArchive(file *zip.File, tarWriter *tar.Writer) error { + fileReader, err := file.Open() + if err != nil { + return err + } + defer fileReader.Close() + + err = tarWriter.WriteHeader(&tar.Header{ + Name: file.Name, + Size: file.FileInfo().Size(), + Mode: 0o644, + }) + if err != nil { + return err + } + + n, err := io.CopyN(tarWriter, fileReader, httpFileMaxBytes) + log.Println(file.Name, n, err) + if errors.Is(err, io.EOF) { + err = nil + } + return err +} + +func CreateZipFromTar(tarReader *tar.Reader) ([]byte, error) { + var zipBuffer bytes.Buffer + err := WriteZipArchive(&zipBuffer, tarReader) + if err != nil { + return nil, err + } + return zipBuffer.Bytes(), nil +} + +func WriteZipArchive(w io.Writer, tarReader *tar.Reader) error { + zipWriter := zip.NewWriter(w) + defer zipWriter.Close() + + for { + tarHeader, err := tarReader.Next() + if errors.Is(err, io.EOF) { + break + } + + if err != nil { + return err + } + + zipHeader, err := zip.FileInfoHeader(tarHeader.FileInfo()) + if err != nil { + return err + } + zipHeader.Name = tarHeader.Name + + zipEntry, err := zipWriter.CreateHeader(zipHeader) + if err != nil { + return err + } + + _, err = io.CopyN(zipEntry, tarReader, httpFileMaxBytes) + if errors.Is(err, io.EOF) { + err = nil + } + if err != nil { + return err + } + } + return nil // don't need to flush as we call `writer.Close()` +} diff --git a/codersdk/files.go b/codersdk/files.go index 3525e9d785..a14f2ca73d 100644 --- a/codersdk/files.go +++ b/codersdk/files.go @@ -12,6 +12,9 @@ import ( const ( ContentTypeTar = "application/x-tar" + ContentTypeZip = "application/zip" + + FormatZip = "zip" ) // UploadResponse contains the hash to reference the uploaded file. @@ -38,7 +41,12 @@ func (c *Client) Upload(ctx context.Context, contentType string, rd io.Reader) ( // Download fetches a file by uploaded hash. func (c *Client) Download(ctx context.Context, id uuid.UUID) ([]byte, string, error) { - res, err := c.Request(ctx, http.MethodGet, fmt.Sprintf("/api/v2/files/%s", id.String()), nil) + return c.DownloadWithFormat(ctx, id, "") +} + +// Download fetches a file by uploaded hash, but it forces format conversion. +func (c *Client) DownloadWithFormat(ctx context.Context, id uuid.UUID, format string) ([]byte, string, error) { + res, err := c.Request(ctx, http.MethodGet, fmt.Sprintf("/api/v2/files/%s?format=%s", id.String(), format), nil) if err != nil { return nil, "", err } diff --git a/docs/api/files.md b/docs/api/files.md index 81d93479ae..936be96cc5 100644 --- a/docs/api/files.md +++ b/docs/api/files.md @@ -22,11 +22,11 @@ file: string ### Parameters -| Name | In | Type | Required | Description | -| -------------- | ------ | ------ | -------- | ---------------------------------------- | -| `Content-Type` | header | string | true | Content-Type must be `application/x-tar` | -| `body` | body | object | true | | -| `» file` | body | binary | true | File to be uploaded | +| Name | In | Type | Required | Description | +| -------------- | ------ | ------ | -------- | ------------------------------------------------------------- | +| `Content-Type` | header | string | true | Content-Type must be `application/x-tar` or `application/zip` | +| `body` | body | object | true | | +| `» file` | body | binary | true | File to be uploaded | ### Example responses diff --git a/provisioner/echo/serve.go b/provisioner/echo/serve.go index 3196afdaa5..53ec286b3c 100644 --- a/provisioner/echo/serve.go +++ b/provisioner/echo/serve.go @@ -306,7 +306,8 @@ func TarWithOptions(ctx context.Context, logger slog.Logger, responses *Response } } } - err := writer.Flush() + // `writer.Close()` function flushes the writer buffer, and adds extra padding to create a legal tarball. + err := writer.Close() if err != nil { return nil, err }