From cb77f041046d1750107c9bcbff36c7ebf5dde44a Mon Sep 17 00:00:00 2001 From: Marcin Tojek Date: Fri, 12 Jan 2024 15:08:23 +0100 Subject: [PATCH] feat: load variables from tfvars files (#11549) --- cli/templatecreate.go | 13 +++ cli/templatepush.go | 13 +++ cli/templatevariables.go | 180 +++++++++++++++++++++++++++++++++- cli/templatevariables_test.go | 178 +++++++++++++++++++++++++++++++++ 4 files changed, 382 insertions(+), 2 deletions(-) create mode 100644 cli/templatevariables_test.go diff --git a/cli/templatecreate.go b/cli/templatecreate.go index c23f34dca5..4cc92e95b8 100644 --- a/cli/templatecreate.go +++ b/cli/templatecreate.go @@ -95,6 +95,18 @@ func (r *RootCmd) templateCreate() *clibase.Cmd { message := uploadFlags.templateMessage(inv) + var varsFiles []string + if !uploadFlags.stdin() { + varsFiles, err = DiscoverVarsFiles(uploadFlags.directory) + if err != nil { + return err + } + + if len(varsFiles) > 0 { + _, _ = fmt.Fprintln(inv.Stdout, "Auto-discovered Terraform tfvars files. Make sure to review and clean up any unused files.") + } + } + // Confirm upload of the directory. resp, err := uploadFlags.upload(inv, client) if err != nil { @@ -107,6 +119,7 @@ func (r *RootCmd) templateCreate() *clibase.Cmd { } userVariableValues, err := ParseUserVariableValues( + varsFiles, variablesFile, commandLineVariables) if err != nil { diff --git a/cli/templatepush.go b/cli/templatepush.go index 26e3aa9472..c1099a67bd 100644 --- a/cli/templatepush.go +++ b/cli/templatepush.go @@ -78,6 +78,18 @@ func (r *RootCmd) templatePush() *clibase.Cmd { message := uploadFlags.templateMessage(inv) + var varsFiles []string + if !uploadFlags.stdin() { + varsFiles, err = DiscoverVarsFiles(uploadFlags.directory) + if err != nil { + return err + } + + if len(varsFiles) > 0 { + _, _ = fmt.Fprintln(inv.Stdout, "Auto-discovered Terraform tfvars files. Make sure to review and clean up any unused files.") + } + } + resp, err := uploadFlags.upload(inv, client) if err != nil { return err @@ -89,6 +101,7 @@ func (r *RootCmd) templatePush() *clibase.Cmd { } userVariableValues, err := ParseUserVariableValues( + varsFiles, variablesFile, commandLineVariables) if err != nil { diff --git a/cli/templatevariables.go b/cli/templatevariables.go index d284d5cbd8..889c632991 100644 --- a/cli/templatevariables.go +++ b/cli/templatevariables.go @@ -1,16 +1,65 @@ package cli import ( + "encoding/json" + "fmt" "os" + "path/filepath" + "sort" "strings" "golang.org/x/xerrors" "gopkg.in/yaml.v3" + "github.com/hashicorp/hcl/v2/hclparse" + "github.com/zclconf/go-cty/cty" + "github.com/coder/coder/v2/codersdk" ) -func ParseUserVariableValues(variablesFile string, commandLineVariables []string) ([]codersdk.VariableValue, error) { +/** + * DiscoverVarsFiles function loads vars files in a predefined order: + * 1. terraform.tfvars + * 2. terraform.tfvars.json + * 3. *.auto.tfvars + * 4. *.auto.tfvars.json + */ +func DiscoverVarsFiles(workDir string) ([]string, error) { + var found []string + + fi, err := os.Stat(filepath.Join(workDir, "terraform.tfvars")) + if err == nil { + found = append(found, filepath.Join(workDir, fi.Name())) + } else if !os.IsNotExist(err) { + return nil, err + } + + fi, err = os.Stat(filepath.Join(workDir, "terraform.tfvars.json")) + if err == nil { + found = append(found, filepath.Join(workDir, fi.Name())) + } else if !os.IsNotExist(err) { + return nil, err + } + + dirEntries, err := os.ReadDir(workDir) + if err != nil { + return nil, err + } + + for _, dirEntry := range dirEntries { + if strings.HasSuffix(dirEntry.Name(), ".auto.tfvars") || strings.HasSuffix(dirEntry.Name(), ".auto.tfvars.json") { + found = append(found, filepath.Join(workDir, dirEntry.Name())) + } + } + return found, nil +} + +func ParseUserVariableValues(varsFiles []string, variablesFile string, commandLineVariables []string) ([]codersdk.VariableValue, error) { + fromVars, err := parseVariableValuesFromVarsFiles(varsFiles) + if err != nil { + return nil, err + } + fromFile, err := parseVariableValuesFromFile(variablesFile) if err != nil { return nil, err @@ -21,7 +70,131 @@ func ParseUserVariableValues(variablesFile string, commandLineVariables []string return nil, err } - return combineVariableValues(fromFile, fromCommandLine), nil + return combineVariableValues(fromVars, fromFile, fromCommandLine), nil +} + +func parseVariableValuesFromVarsFiles(varsFiles []string) ([]codersdk.VariableValue, error) { + var parsed []codersdk.VariableValue + for _, varsFile := range varsFiles { + content, err := os.ReadFile(varsFile) + if err != nil { + return nil, err + } + + var t []codersdk.VariableValue + ext := filepath.Ext(varsFile) + switch ext { + case ".tfvars": + t, err = parseVariableValuesFromHCL(content) + if err != nil { + return nil, xerrors.Errorf("unable to parse HCL content: %w", err) + } + case ".json": + t, err = parseVariableValuesFromJSON(content) + if err != nil { + return nil, xerrors.Errorf("unable to parse JSON content: %w", err) + } + default: + return nil, xerrors.Errorf("unexpected tfvars format: %s", ext) + } + + parsed = append(parsed, t...) + } + return parsed, nil +} + +func parseVariableValuesFromHCL(content []byte) ([]codersdk.VariableValue, error) { + parser := hclparse.NewParser() + hclFile, diags := parser.ParseHCL(content, "file.hcl") + if diags.HasErrors() { + return nil, diags + } + + attrs, diags := hclFile.Body.JustAttributes() + if diags.HasErrors() { + return nil, diags + } + + stringData := map[string]string{} + for _, attribute := range attrs { + ctyValue, diags := attribute.Expr.Value(nil) + if diags.HasErrors() { + return nil, diags + } + + ctyType := ctyValue.Type() + if ctyType.Equals(cty.String) { + stringData[attribute.Name] = ctyValue.AsString() + } else if ctyType.Equals(cty.Number) { + stringData[attribute.Name] = ctyValue.AsBigFloat().String() + } else if ctyType.IsTupleType() { + // In case of tuples, Coder only supports the list(string) type. + var items []string + var err error + _ = ctyValue.ForEachElement(func(key, val cty.Value) (stop bool) { + if !val.Type().Equals(cty.String) { + err = xerrors.Errorf("unsupported tuple item type: %s ", val.GoString()) + return true + } + items = append(items, val.AsString()) + return false + }) + if err != nil { + return nil, err + } + + m, err := json.Marshal(items) + if err != nil { + return nil, err + } + stringData[attribute.Name] = string(m) + } else { + return nil, xerrors.Errorf("unsupported value type (name: %s): %s", attribute.Name, ctyType.GoString()) + } + } + + return convertMapIntoVariableValues(stringData), nil +} + +// parseVariableValuesFromJSON converts the .tfvars.json content into template variables. +// The function visits only root-level properties as template variables do not support nested +// structures. +func parseVariableValuesFromJSON(content []byte) ([]codersdk.VariableValue, error) { + var data map[string]interface{} + err := json.Unmarshal(content, &data) + if err != nil { + return nil, err + } + + stringData := map[string]string{} + for key, value := range data { + switch value.(type) { + case string, int, bool: + stringData[key] = fmt.Sprintf("%v", value) + default: + m, err := json.Marshal(value) + if err != nil { + return nil, err + } + stringData[key] = string(m) + } + } + + return convertMapIntoVariableValues(stringData), nil +} + +func convertMapIntoVariableValues(m map[string]string) []codersdk.VariableValue { + var parsed []codersdk.VariableValue + for key, value := range m { + parsed = append(parsed, codersdk.VariableValue{ + Name: key, + Value: value, + }) + } + sort.Slice(parsed, func(i, j int) bool { + return parsed[i].Name < parsed[j].Name + }) + return parsed } func parseVariableValuesFromFile(variablesFile string) ([]codersdk.VariableValue, error) { @@ -94,5 +267,8 @@ func combineVariableValues(valuesSets ...[]codersdk.VariableValue) []codersdk.Va result = append(result, codersdk.VariableValue{Name: name, Value: value}) } + sort.Slice(result, func(i, j int) bool { + return result[i].Name < result[j].Name + }) return result } diff --git a/cli/templatevariables_test.go b/cli/templatevariables_test.go new file mode 100644 index 0000000000..4b84f55778 --- /dev/null +++ b/cli/templatevariables_test.go @@ -0,0 +1,178 @@ +package cli_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/coder/coder/v2/cli" + "github.com/coder/coder/v2/codersdk" +) + +func TestDiscoverVarsFiles(t *testing.T) { + t.Parallel() + + // Given + tempDir, err := os.MkdirTemp(os.TempDir(), "test-discover-vars-files-*") + require.NoError(t, err) + + t.Cleanup(func() { + _ = os.RemoveAll(tempDir) + }) + + testFiles := []string{ + "terraform.tfvars", // ok + "terraform.tfvars.json", // ok + "aaa.tf", // not Terraform vars + "bbb.tf", // not Terraform vars + "example.auto.tfvars", // ok + "example.auto.tfvars.bak", // not Terraform vars + "example.auto.tfvars.json", // ok + "example.auto.tfvars.json.bak", // not Terraform vars + "other_file.txt", // not Terraform vars + "random_file1.tfvars", // should be .auto.tfvars, otherwise ignored + "random_file2.tf", // not Terraform vars + "random_file2.tfvars.json", // should be .auto.tfvars.json, otherwise ignored + "random_file3.auto.tfvars", // ok + "random_file3.tf", // not Terraform vars + "random_file4.auto.tfvars.json", // ok + } + + for _, file := range testFiles { + filePath := filepath.Join(tempDir, file) + err := os.WriteFile(filePath, []byte(""), 0o600) + require.NoError(t, err) + } + + // When + found, err := cli.DiscoverVarsFiles(tempDir) + require.NoError(t, err) + + // Then + expected := []string{ + filepath.Join(tempDir, "terraform.tfvars"), + filepath.Join(tempDir, "terraform.tfvars.json"), + filepath.Join(tempDir, "example.auto.tfvars"), + filepath.Join(tempDir, "example.auto.tfvars.json"), + filepath.Join(tempDir, "random_file3.auto.tfvars"), + filepath.Join(tempDir, "random_file4.auto.tfvars.json"), + } + require.EqualValues(t, expected, found) +} + +func TestParseVariableValuesFromVarsFiles(t *testing.T) { + t.Parallel() + + // Given + const ( + hclFilename1 = "file1.tfvars" + hclFilename2 = "file2.tfvars" + jsonFilename3 = "file3.tfvars.json" + jsonFilename4 = "file4.tfvars.json" + + hclContent1 = `region = "us-east-1" +cores = 2` + hclContent2 = `region = "us-west-2" +go_image = ["1.19","1.20","1.21"]` + jsonContent3 = `{"cat": "foobar", "cores": 3}` + jsonContent4 = `{"dog": 4, "go_image": "[\"1.19\",\"1.20\"]"}` + ) + + // Prepare the .tfvars files + tempDir, err := os.MkdirTemp(os.TempDir(), "test-parse-variable-values-from-vars-files-*") + require.NoError(t, err) + t.Cleanup(func() { + _ = os.RemoveAll(tempDir) + }) + + err = os.WriteFile(filepath.Join(tempDir, hclFilename1), []byte(hclContent1), 0o600) + require.NoError(t, err) + err = os.WriteFile(filepath.Join(tempDir, hclFilename2), []byte(hclContent2), 0o600) + require.NoError(t, err) + err = os.WriteFile(filepath.Join(tempDir, jsonFilename3), []byte(jsonContent3), 0o600) + require.NoError(t, err) + err = os.WriteFile(filepath.Join(tempDir, jsonFilename4), []byte(jsonContent4), 0o600) + require.NoError(t, err) + + // When + actual, err := cli.ParseUserVariableValues([]string{ + filepath.Join(tempDir, hclFilename1), + filepath.Join(tempDir, hclFilename2), + filepath.Join(tempDir, jsonFilename3), + filepath.Join(tempDir, jsonFilename4), + }, "", nil) + require.NoError(t, err) + + // Then + expected := []codersdk.VariableValue{ + {Name: "cat", Value: "foobar"}, + {Name: "cores", Value: "3"}, + {Name: "dog", Value: "4"}, + {Name: "go_image", Value: "[\"1.19\",\"1.20\"]"}, + {Name: "region", Value: "us-west-2"}, + } + require.Equal(t, expected, actual) +} + +func TestParseVariableValuesFromVarsFiles_InvalidJSON(t *testing.T) { + t.Parallel() + + // Given + const ( + jsonFilename = "file.tfvars.json" + jsonContent = `{"cat": "foobar", cores: 3}` // invalid content: no quotes around "cores" + ) + + // Prepare the .tfvars files + tempDir, err := os.MkdirTemp(os.TempDir(), "test-parse-variable-values-from-vars-files-invalid-json-*") + require.NoError(t, err) + t.Cleanup(func() { + _ = os.RemoveAll(tempDir) + }) + + err = os.WriteFile(filepath.Join(tempDir, jsonFilename), []byte(jsonContent), 0o600) + require.NoError(t, err) + + // When + actual, err := cli.ParseUserVariableValues([]string{ + filepath.Join(tempDir, jsonFilename), + }, "", nil) + + // Then + require.Nil(t, actual) + require.Error(t, err) + require.Contains(t, err.Error(), "unable to parse JSON content") +} + +func TestParseVariableValuesFromVarsFiles_InvalidHCL(t *testing.T) { + t.Parallel() + + // Given + const ( + hclFilename = "file.tfvars" + hclContent = `region = "us-east-1" +cores: 2` + ) + + // Prepare the .tfvars files + tempDir, err := os.MkdirTemp(os.TempDir(), "test-parse-variable-values-from-vars-files-invalid-hcl-*") + require.NoError(t, err) + t.Cleanup(func() { + _ = os.RemoveAll(tempDir) + }) + + err = os.WriteFile(filepath.Join(tempDir, hclFilename), []byte(hclContent), 0o600) + require.NoError(t, err) + + // When + actual, err := cli.ParseUserVariableValues([]string{ + filepath.Join(tempDir, hclFilename), + }, "", nil) + + // Then + require.Nil(t, actual) + require.Error(t, err) + require.Contains(t, err.Error(), `use the equals sign "=" to introduce the argument value`) +}