selfhosted
/
coder
mirror of https://github.com/coder/coder.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

feat: Read params from file for template/workspace creation (#1541) 

* Read params from file for template/workspace creation

* Use os.ReadFile

* Refactor reading params into a separate module

* Add comments and unit tests

* Rename variable

* Uncomment and fix unit test

* Fix comment

* Refactor tests

* Fix unit tests for windows

* Fix unit tests for Windows

* Add comments for the hotfix
This commit is contained in:
Abhineet Jain 2022-05-20 11:29:10 -04:00 committed by GitHub
parent d0fd0d7040
commit 7c3e1a5d97
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 431 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
cli/create.go
View File

@ -17,6 +17,7 @@ func create() *cobra.Command {
var (
workspaceName string
templateName string
parameterFile string
)
cmd := &cobra.Command{
Annotations: workspaceCommand,
@ -116,23 +117,33 @@ func create() *cobra.Command {
return err
}
printed := false
// parameterMapFromFile can be nil if parameter file is not specified
var parameterMapFromFile map[string]string
if parameterFile != "" {
_, _ = fmt.Fprintln(cmd.OutOrStdout(), cliui.Styles.Paragraph.Render("Attempting to read the variables from the parameter file.")+"\r\n")
parameterMapFromFile, err = createParameterMapFromFile(parameterFile)
if err != nil {
return err
}
}
disclaimerPrinted := false
parameters := make([]codersdk.CreateParameterRequest, 0)
for _, parameterSchema := range parameterSchemas {
if !parameterSchema.AllowOverrideSource {
continue
}
if !printed {
if !disclaimerPrinted {
_, _ = fmt.Fprintln(cmd.OutOrStdout(), cliui.Styles.Paragraph.Render("This template has customizable parameters. Values can be changed after create, but may have unintended side effects (like data loss).")+"\r\n")
printed = true
disclaimerPrinted = true
}
value, err := cliui.ParameterSchema(cmd, parameterSchema)
parameterValue, err := getParameterValueFromMapOrInput(cmd, parameterMapFromFile, parameterSchema)
if err != nil {
return err
}
parameters = append(parameters, codersdk.CreateParameterRequest{
Name: parameterSchema.Name,
SourceValue: value,
SourceValue: parameterValue,
SourceScheme: codersdk.ParameterSourceSchemeData,
DestinationScheme: parameterSchema.DefaultDestinationScheme,
})
@ -194,5 +205,6 @@ func create() *cobra.Command {
}
cliflag.StringVarP(cmd.Flags(), &templateName, "template", "t", "CODER_TEMPLATE_NAME", "", "Specify a template name.")
cliflag.StringVarP(cmd.Flags(), &parameterFile, "parameter-file", "", "CODER_PARAMETER_FILE", "", "Specify a file path with parameter values.")
return cmd
}

144
cli/create_test.go
View File

@ -2,6 +2,7 @@ package cli_test
import (
"fmt"
"os"
"testing"
"github.com/stretchr/testify/require"
@ -113,39 +114,7 @@ func TestCreate(t *testing.T) {
defaultValue := "something"
version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{
Parse: []*proto.Parse_Response{{
Type: &proto.Parse_Response_Complete{
Complete: &proto.Parse_Complete{
ParameterSchemas: []*proto.ParameterSchema{
{
AllowOverrideSource: true,
Name: "region",
Description: "description 1",
DefaultSource: &proto.ParameterSource{
Scheme: proto.ParameterSource_DATA,
Value: defaultValue,
},
DefaultDestination: &proto.ParameterDestination{
Scheme: proto.ParameterDestination_PROVISIONER_VARIABLE,
},
},
{
AllowOverrideSource: true,
Name: "username",
Description: "description 2",
DefaultSource: &proto.ParameterSource{
Scheme: proto.ParameterSource_DATA,
// No default value
Value: "",
},
DefaultDestination: &proto.ParameterDestination{
Scheme: proto.ParameterDestination_PROVISIONER_VARIABLE,
},
},
},
},
},
}},
Parse: createTestParseResponseWithDefault(defaultValue),
Provision: echo.ProvisionComplete,
ProvisionDryRun: echo.ProvisionComplete,
})
@ -178,4 +147,113 @@ func TestCreate(t *testing.T) {
}
<-doneChan
})
t.Run("WithParameterFileContainingTheValue", func(t *testing.T) {
t.Parallel()
client := coderdtest.New(t, &coderdtest.Options{IncludeProvisionerD: true})
user := coderdtest.CreateFirstUser(t, client)
defaultValue := "something"
version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{
Parse: createTestParseResponseWithDefault(defaultValue),
Provision: echo.ProvisionComplete,
ProvisionDryRun: echo.ProvisionComplete,
})
coderdtest.AwaitTemplateVersionJob(t, client, version.ID)
_ = coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID)
tempDir := t.TempDir()
parameterFile, _ := os.CreateTemp(tempDir, "testParameterFile*.yaml")
_, _ = parameterFile.WriteString("region: \"bingo\"\nusername: \"boingo\"")
cmd, root := clitest.New(t, "create", "", "--parameter-file", parameterFile.Name())
clitest.SetupConfig(t, client, root)
doneChan := make(chan struct{})
pty := ptytest.New(t)
cmd.SetIn(pty.Input())
cmd.SetOut(pty.Output())
go func() {
defer close(doneChan)
err := cmd.Execute()
require.NoError(t, err)
}()
matches := []string{
"Specify a name", "my-workspace",
"Confirm create?", "yes",
}
for i := 0; i < len(matches); i += 2 {
match := matches[i]
value := matches[i+1]
pty.ExpectMatch(match)
pty.WriteLine(value)
}
<-doneChan
removeTmpDirUntilSuccess(t, tempDir)
})
t.Run("WithParameterFileNotContainingTheValue", func(t *testing.T) {
t.Parallel()
client := coderdtest.New(t, &coderdtest.Options{IncludeProvisionerD: true})
user := coderdtest.CreateFirstUser(t, client)
defaultValue := "something"
version := coderdtest.CreateTemplateVersion(t, client, user.OrganizationID, &echo.Responses{
Parse: createTestParseResponseWithDefault(defaultValue),
Provision: echo.ProvisionComplete,
ProvisionDryRun: echo.ProvisionComplete,
})
coderdtest.AwaitTemplateVersionJob(t, client, version.ID)
template := coderdtest.CreateTemplate(t, client, user.OrganizationID, version.ID)
tempDir := t.TempDir()
parameterFile, _ := os.CreateTemp(tempDir, "testParameterFile*.yaml")
_, _ = parameterFile.WriteString("zone: \"bananas\"")
cmd, root := clitest.New(t, "create", "my-workspace", "--template", template.Name, "--parameter-file", parameterFile.Name())
clitest.SetupConfig(t, client, root)
doneChan := make(chan struct{})
pty := ptytest.New(t)
cmd.SetIn(pty.Input())
cmd.SetOut(pty.Output())
go func() {
defer close(doneChan)
err := cmd.Execute()
require.EqualError(t, err, "Parameter value absent in parameter file for \"region\"!")
}()
<-doneChan
removeTmpDirUntilSuccess(t, tempDir)
})
}
func createTestParseResponseWithDefault(defaultValue string) []*proto.Parse_Response {
return []*proto.Parse_Response{{
Type: &proto.Parse_Response_Complete{
Complete: &proto.Parse_Complete{
ParameterSchemas: []*proto.ParameterSchema{
{
AllowOverrideSource: true,
Name: "region",
Description: "description 1",
DefaultSource: &proto.ParameterSource{
Scheme: proto.ParameterSource_DATA,
Value: defaultValue,
},
DefaultDestination: &proto.ParameterDestination{
Scheme: proto.ParameterDestination_PROVISIONER_VARIABLE,
},
},
{
AllowOverrideSource: true,
Name: "username",
Description: "description 2",
DefaultSource: &proto.ParameterSource{
Scheme: proto.ParameterSource_DATA,
// No default value
Value: "",
},
DefaultDestination: &proto.ParameterDestination{
Scheme: proto.ParameterDestination_PROVISIONER_VARIABLE,
},
},
},
},
},
}}
}

56
cli/parameter.go Normal file
View File

@ -0,0 +1,56 @@
package cli
import (
"os"
"golang.org/x/xerrors"
"gopkg.in/yaml.v3"
"github.com/coder/coder/cli/cliui"
"github.com/coder/coder/codersdk"
"github.com/spf13/cobra"
)
// Reads a YAML file and populates a string -> string map.
// Throws an error if the file name is empty.
func createParameterMapFromFile(parameterFile string) (map[string]string, error) {
if parameterFile != "" {
parameterMap := make(map[string]string)
parameterFileContents, err := os.ReadFile(parameterFile)
if err != nil {
return nil, err
}
err = yaml.Unmarshal(parameterFileContents, &parameterMap)
if err != nil {
return nil, err
}
return parameterMap, nil
}
return nil, xerrors.Errorf("Parameter file name is not specified")
}
// Returns a parameter value from a given map, if the map exists, else takes input from the user.
// Throws an error if the map exists but does not include a value for the parameter.
func getParameterValueFromMapOrInput(cmd *cobra.Command, parameterMap map[string]string, parameterSchema codersdk.ParameterSchema) (string, error) {
var parameterValue string
if parameterMap != nil {
var ok bool
parameterValue, ok = parameterMap[parameterSchema.Name]
if !ok {
return "", xerrors.Errorf("Parameter value absent in parameter file for %q!", parameterSchema.Name)
}
} else {
var err error
parameterValue, err = cliui.ParameterSchema(cmd, parameterSchema)
if err != nil {
return "", err
}
}
return parameterValue, nil
}

79
cli/parameter_internal_test.go Normal file
View File

@ -0,0 +1,79 @@
package cli
import (
"os"
"runtime"
"testing"
"github.com/stretchr/testify/assert"
)
func TestCreateParameterMapFromFile(t *testing.T) {
t.Parallel()
t.Run("CreateParameterMapFromFile", func(t *testing.T) {
t.Parallel()
tempDir := t.TempDir()
parameterFile, _ := os.CreateTemp(tempDir, "testParameterFile*.yaml")
_, _ = parameterFile.WriteString("region: \"bananas\"\ndisk: \"20\"\n")
parameterMapFromFile, err := createParameterMapFromFile(parameterFile.Name())
expectedMap := map[string]string{
"region": "bananas",
"disk": "20",
}
assert.Equal(t, expectedMap, parameterMapFromFile)
assert.Nil(t, err)
removeTmpDirUntilSuccess(t, tempDir)
})
t.Run("WithEmptyFilename", func(t *testing.T) {
t.Parallel()
parameterMapFromFile, err := createParameterMapFromFile("")
assert.Nil(t, parameterMapFromFile)
assert.EqualError(t, err, "Parameter file name is not specified")
})
t.Run("WithInvalidFilename", func(t *testing.T) {
t.Parallel()
parameterMapFromFile, err := createParameterMapFromFile("invalidFile.yaml")
assert.Nil(t, parameterMapFromFile)
// On Unix based systems, it is: `open invalidFile.yaml: no such file or directory`
// On Windows, it is `open invalidFile.yaml: The system cannot find the file specified.`
if runtime.GOOS == "windows" {
assert.EqualError(t, err, "open invalidFile.yaml: The system cannot find the file specified.")
} else {
assert.EqualError(t, err, "open invalidFile.yaml: no such file or directory")
}
})
t.Run("WithInvalidYAML", func(t *testing.T) {
t.Parallel()
tempDir := t.TempDir()
parameterFile, _ := os.CreateTemp(tempDir, "testParameterFile*.yaml")
_, _ = parameterFile.WriteString("region = \"bananas\"\ndisk = \"20\"\n")
parameterMapFromFile, err := createParameterMapFromFile(parameterFile.Name())
assert.Nil(t, parameterMapFromFile)
assert.EqualError(t, err, "yaml: unmarshal errors:\n line 1: cannot unmarshal !!str `region ...` into map[string]string")
removeTmpDirUntilSuccess(t, tempDir)
})
}
// Need this for Windows because of a known issue with Go:
// https://github.com/golang/go/issues/52986
func removeTmpDirUntilSuccess(t *testing.T, tempDir string) {
t.Helper()
t.Cleanup(func() {
err := os.RemoveAll(tempDir)
for err != nil {
err = os.RemoveAll(tempDir)
}
})
}

31
cli/templatecreate.go
View File

@ -21,9 +21,10 @@ import (
func templateCreate() *cobra.Command {
var (
yes bool
directory string
provisioner string
yes bool
directory string
provisioner string
parameterFile string
)
cmd := &cobra.Command{
Use: "create [name]",
@ -79,7 +80,7 @@ func templateCreate() *cobra.Command {
}
spin.Stop()
job, parameters, err := createValidTemplateVersion(cmd, client, organization, database.ProvisionerType(provisioner), resp.Hash)
job, parameters, err := createValidTemplateVersion(cmd, client, organization, database.ProvisionerType(provisioner), resp.Hash, parameterFile)
if err != nil {
return err
}
@ -116,6 +117,7 @@ func templateCreate() *cobra.Command {
currentDirectory, _ := os.Getwd()
cmd.Flags().StringVarP(&directory, "directory", "d", currentDirectory, "Specify the directory to create from")
cmd.Flags().StringVarP(&provisioner, "test.provisioner", "", "terraform", "Customize the provisioner backend")
cmd.Flags().StringVarP(&parameterFile, "parameter-file", "", "", "Specify a file path with parameter values.")
// This is for testing!
err := cmd.Flags().MarkHidden("test.provisioner")
if err != nil {
@ -125,7 +127,7 @@ func templateCreate() *cobra.Command {
return cmd
}
func createValidTemplateVersion(cmd *cobra.Command, client *codersdk.Client, organization codersdk.Organization, provisioner database.ProvisionerType, hash string, parameters ...codersdk.CreateParameterRequest) (*codersdk.TemplateVersion, []codersdk.CreateParameterRequest, error) {
func createValidTemplateVersion(cmd *cobra.Command, client *codersdk.Client, organization codersdk.Organization, provisioner database.ProvisionerType, hash string, parameterFile string, parameters ...codersdk.CreateParameterRequest) (*codersdk.TemplateVersion, []codersdk.CreateParameterRequest, error) {
before := time.Now()
version, err := client.CreateTemplateVersion(cmd.Context(), organization.ID, codersdk.CreateTemplateVersionRequest{
StorageMethod: codersdk.ProvisionerStorageMethodFile,
@ -184,20 +186,33 @@ func createValidTemplateVersion(cmd *cobra.Command, client *codersdk.Client, org
missingSchemas = append(missingSchemas, parameterSchema)
}
_, _ = fmt.Fprintln(cmd.OutOrStdout(), cliui.Styles.Paragraph.Render("This template has required variables! They are scoped to the template, and not viewable after being set.")+"\r\n")
// parameterMapFromFile can be nil if parameter file is not specified
var parameterMapFromFile map[string]string
if parameterFile != "" {
_, _ = fmt.Fprintln(cmd.OutOrStdout(), cliui.Styles.Paragraph.Render("Attempting to read the variables from the parameter file.")+"\r\n")
parameterMapFromFile, err = createParameterMapFromFile(parameterFile)
if err != nil {
return nil, nil, err
}
}
for _, parameterSchema := range missingSchemas {
value, err := cliui.ParameterSchema(cmd, parameterSchema)
parameterValue, err := getParameterValueFromMapOrInput(cmd, parameterMapFromFile, parameterSchema)
if err != nil {
return nil, nil, err
}
parameters = append(parameters, codersdk.CreateParameterRequest{
Name: parameterSchema.Name,
SourceValue: value,
SourceValue: parameterValue,
SourceScheme: codersdk.ParameterSourceSchemeData,
DestinationScheme: parameterSchema.DefaultDestinationScheme,
})
_, _ = fmt.Fprintln(cmd.OutOrStdout())
}
return createValidTemplateVersion(cmd, client, organization, provisioner, hash, parameters...)
// This recursion is only 1 level deep in practice.
// The first pass populates the missing parameters, so it does not enter this `if` block again.
return createValidTemplateVersion(cmd, client, organization, provisioner, hash, parameterFile, parameters...)
}
if version.Job.Status != codersdk.ProvisionerJobSucceeded {

144
cli/templatecreate_test.go
View File

@ -1,6 +1,7 @@
package cli_test
import (
"os"
"testing"
"github.com/stretchr/testify/require"
@ -9,6 +10,7 @@ import (
"github.com/coder/coder/coderd/coderdtest"
"github.com/coder/coder/coderd/database"
"github.com/coder/coder/provisioner/echo"
"github.com/coder/coder/provisionersdk/proto"
"github.com/coder/coder/pty/ptytest"
)
@ -47,4 +49,146 @@ func TestTemplateCreate(t *testing.T) {
require.NoError(t, <-execDone)
})
t.Run("WithParameter", func(t *testing.T) {
t.Parallel()
client := coderdtest.New(t, &coderdtest.Options{IncludeProvisionerD: true})
coderdtest.CreateFirstUser(t, client)
source := clitest.CreateTemplateVersionSource(t, &echo.Responses{
Parse: createTestParseResponse(),
Provision: echo.ProvisionComplete,
ProvisionDryRun: echo.ProvisionComplete,
})
cmd, root := clitest.New(t, "templates", "create", "my-template", "--directory", source, "--test.provisioner", string(database.ProvisionerTypeEcho))
clitest.SetupConfig(t, client, root)
pty := ptytest.New(t)
cmd.SetIn(pty.Input())
cmd.SetOut(pty.Output())
execDone := make(chan error)
go func() {
execDone <- cmd.Execute()
}()
matches := []struct {
match string
write string
}{
{match: "Create and upload", write: "yes"},
{match: "Enter a value:", write: "bananas"},
{match: "Confirm create?", write: "yes"},
}
for _, m := range matches {
pty.ExpectMatch(m.match)
pty.WriteLine(m.write)
}
require.NoError(t, <-execDone)
})
t.Run("WithParameterFileContainingTheValue", func(t *testing.T) {
t.Parallel()
client := coderdtest.New(t, &coderdtest.Options{IncludeProvisionerD: true})
coderdtest.CreateFirstUser(t, client)
source := clitest.CreateTemplateVersionSource(t, &echo.Responses{
Parse: createTestParseResponse(),
Provision: echo.ProvisionComplete,
ProvisionDryRun: echo.ProvisionComplete,
})
tempDir := t.TempDir()
parameterFile, _ := os.CreateTemp(tempDir, "testParameterFile*.yaml")
_, _ = parameterFile.WriteString("region: \"bananas\"")
cmd, root := clitest.New(t, "templates", "create", "my-template", "--directory", source, "--test.provisioner", string(database.ProvisionerTypeEcho), "--parameter-file", parameterFile.Name())
clitest.SetupConfig(t, client, root)
pty := ptytest.New(t)
cmd.SetIn(pty.Input())
cmd.SetOut(pty.Output())
execDone := make(chan error)
go func() {
execDone <- cmd.Execute()
}()
matches := []struct {
match string
write string
}{
{match: "Create and upload", write: "yes"},
{match: "Confirm create?", write: "yes"},
}
for _, m := range matches {
pty.ExpectMatch(m.match)
pty.WriteLine(m.write)
}
require.NoError(t, <-execDone)
removeTmpDirUntilSuccess(t, tempDir)
})
t.Run("WithParameterFileNotContainingTheValue", func(t *testing.T) {
t.Parallel()
client := coderdtest.New(t, &coderdtest.Options{IncludeProvisionerD: true})
coderdtest.CreateFirstUser(t, client)
source := clitest.CreateTemplateVersionSource(t, &echo.Responses{
Parse: createTestParseResponse(),
Provision: echo.ProvisionComplete,
ProvisionDryRun: echo.ProvisionComplete,
})
tempDir := t.TempDir()
parameterFile, _ := os.CreateTemp(tempDir, "testParameterFile*.yaml")
_, _ = parameterFile.WriteString("zone: \"bananas\"")
cmd, root := clitest.New(t, "templates", "create", "my-template", "--directory", source, "--test.provisioner", string(database.ProvisionerTypeEcho), "--parameter-file", parameterFile.Name())
clitest.SetupConfig(t, client, root)
pty := ptytest.New(t)
cmd.SetIn(pty.Input())
cmd.SetOut(pty.Output())
execDone := make(chan error)
go func() {
execDone <- cmd.Execute()
}()
matches := []struct {
match string
write string
}{
{match: "Create and upload", write: "yes"},
}
for _, m := range matches {
pty.ExpectMatch(m.match)
pty.WriteLine(m.write)
}
require.EqualError(t, <-execDone, "Parameter value absent in parameter file for \"region\"!")
removeTmpDirUntilSuccess(t, tempDir)
})
}
func createTestParseResponse() []*proto.Parse_Response {
return []*proto.Parse_Response{{
Type: &proto.Parse_Response_Complete{
Complete: &proto.Parse_Complete{
ParameterSchemas: []*proto.ParameterSchema{{
AllowOverrideSource: true,
Name: "region",
Description: "description",
DefaultDestination: &proto.ParameterDestination{
Scheme: proto.ParameterDestination_PROVISIONER_VARIABLE,
},
}},
},
},
}}
}
// Need this for Windows because of a known issue with Go:
// https://github.com/golang/go/issues/52986
func removeTmpDirUntilSuccess(t *testing.T, tempDir string) {
t.Helper()
t.Cleanup(func() {
err := os.RemoveAll(tempDir)
for err != nil {
err = os.RemoveAll(tempDir)
}
})
}

2
go.mod
View File

@ -120,6 +120,7 @@ require (
google.golang.org/api v0.79.0
google.golang.org/protobuf v1.28.0
gopkg.in/natefinch/lumberjack.v2 v2.0.0
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9
nhooyr.io/websocket v1.8.7
storj.io/drpc v0.0.30
@ -250,5 +251,4 @@ require (
gopkg.in/ini.v1 v1.62.0 // indirect
gopkg.in/square/go-jose.v2 v2.6.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
)