diff --git a/internal/dotenv/LICENSE b/internal/dotenv/LICENSE new file mode 100644 index 0000000000..e7ddd51be9 --- /dev/null +++ b/internal/dotenv/LICENSE @@ -0,0 +1,23 @@ +Copyright (c) 2013 John Barton + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + diff --git a/internal/dotenv/dotenv.go b/internal/dotenv/dotenv.go new file mode 100644 index 0000000000..b2851626d9 --- /dev/null +++ b/internal/dotenv/dotenv.go @@ -0,0 +1,22 @@ +// Package dotenv reads environment files in declaration order. +package dotenv + +import ( + "os" + + "github.com/elliotchance/orderedmap/v3" +) + +// Read reads a dotenv file, preserving the order of its variable declarations. +// Repeated keys retain their original position and take their last value. +func Read(filename string) (*orderedmap.OrderedMap[string, string], error) { + data, err := os.ReadFile(filename) + if err != nil { + return nil, err + } + vars := orderedmap.NewOrderedMap[string, string]() + if err := parseBytes(data, vars); err != nil { + return nil, err + } + return vars, nil +} diff --git a/internal/dotenv/dotenv_test.go b/internal/dotenv/dotenv_test.go new file mode 100644 index 0000000000..8b8999c393 --- /dev/null +++ b/internal/dotenv/dotenv_test.go @@ -0,0 +1,50 @@ +package dotenv_test + +import ( + "maps" + "os" + "path/filepath" + "slices" + "testing" + + "github.com/joho/godotenv" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/go-task/task/v3/internal/dotenv" +) + +func TestRead(t *testing.T) { + t.Parallel() + + tests := map[string]string{ + "syntax": "# comment\r\nexport Z=first\r\nA=${Z}/nested # comment\r\nM: yaml\r\nZ=last\r\nEMPTY=\r\n", + "quotes": "BASE=/home/user\nSINGLE='$BASE'\nDOUBLE=\"${BASE}/nested\\nnext\"\nMULTI='line one\nline two'\nESCAPED=\\$BASE\nTEMPLATE={{.BASE}}/nested", + "invalid key": "BAD-KEY=value\n", + "unterminated quote": "VALUE=\"unfinished\n", + } + for name, content := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + path := filepath.Join(t.TempDir(), ".env") + require.NoError(t, os.WriteFile(path, []byte(content), 0o600)) + expected, expectedErr := godotenv.Unmarshal(content) + actual, err := dotenv.Read(path) + if expectedErr != nil { + require.EqualError(t, err, expectedErr.Error()) + return + } + require.NoError(t, err) + assert.Equal(t, expected, maps.Collect(actual.AllFromFront())) + if name == "syntax" { + assert.Equal(t, []string{"Z", "A", "M", "EMPTY"}, slices.Collect(actual.Keys())) + } + }) + } +} + +func TestReadMissingFile(t *testing.T) { + t.Parallel() + _, err := dotenv.Read(filepath.Join(t.TempDir(), "missing.env")) + require.ErrorIs(t, err, os.ErrNotExist) +} diff --git a/internal/dotenv/parser.go b/internal/dotenv/parser.go new file mode 100644 index 0000000000..06b08a6a4f --- /dev/null +++ b/internal/dotenv/parser.go @@ -0,0 +1,276 @@ +// Adapted from github.com/joho/godotenv v1.5.1 to preserve declaration order. +// See LICENSE for the original MIT license. +package dotenv + +import ( + "bytes" + "fmt" + "regexp" + "strings" + "unicode" + + "github.com/elliotchance/orderedmap/v3" + + "github.com/go-task/task/v3/errors" +) + +const ( + charComment = '#' + prefixSingleQuote = '\'' + prefixDoubleQuote = '"' + + exportPrefix = "export" +) + +func parseBytes(src []byte, out *orderedmap.OrderedMap[string, string]) error { + src = bytes.ReplaceAll(src, []byte("\r\n"), []byte("\n")) + cutset := src + for { + cutset = getStatementStart(cutset) + if cutset == nil { + // reached end of file + break + } + + key, left, err := locateKeyName(cutset) + if err != nil { + return err + } + + value, left, err := extractVarValue(left, out) + if err != nil { + return err + } + + out.Set(key, value) + cutset = left + } + + return nil +} + +// getStatementPosition returns position of statement begin. +// +// It skips any comment line or non-whitespace character. +func getStatementStart(src []byte) []byte { + pos := indexOfNonSpaceChar(src) + if pos == -1 { + return nil + } + + src = src[pos:] + if src[0] != charComment { + return src + } + + // skip comment section + pos = bytes.IndexFunc(src, isCharFunc('\n')) + if pos == -1 { + return nil + } + + return getStatementStart(src[pos:]) +} + +// locateKeyName locates and parses key name and returns rest of slice +func locateKeyName(src []byte) (key string, cutset []byte, err error) { + // trim "export" and space at beginning + src = bytes.TrimLeftFunc(src, isSpace) + if trimmed, found := bytes.CutPrefix(src, []byte(exportPrefix)); found { + if bytes.IndexFunc(trimmed, isSpace) == 0 { + src = bytes.TrimLeftFunc(trimmed, isSpace) + } + } + + // locate key name end and validate it in single loop + offset := 0 +loop: + for i, char := range src { + rchar := rune(char) + if isSpace(rchar) { + continue + } + + switch char { + case '=', ':': + // library also supports yaml-style value declaration + key = string(src[0:i]) + offset = i + 1 + break loop + case '_': + default: + // variable name should match [A-Za-z0-9_.] + if unicode.IsLetter(rchar) || unicode.IsNumber(rchar) || rchar == '.' { + continue + } + + return "", nil, fmt.Errorf( + `unexpected character %q in variable name near %q`, + string(char), string(src)) + } + } + + if len(src) == 0 { + return "", nil, errors.New("zero length string") + } + + // trim whitespace + key = strings.TrimRightFunc(key, unicode.IsSpace) + cutset = bytes.TrimLeftFunc(src[offset:], isSpace) + return key, cutset, nil +} + +// extractVarValue extracts variable value and returns rest of slice +func extractVarValue(src []byte, vars *orderedmap.OrderedMap[string, string]) (value string, rest []byte, err error) { + quote, hasPrefix := hasQuotePrefix(src) + if !hasPrefix { + // unquoted value - read until end of line + endOfLine := bytes.IndexFunc(src, isLineEnd) + + // Hit EOF without a trailing newline + if endOfLine == -1 { + endOfLine = len(src) + + if endOfLine == 0 { + return "", nil, nil + } + } + + // Convert line to rune away to do accurate countback of runes + line := []rune(string(src[0:endOfLine])) + + // Assume end of line is end of var + endOfVar := len(line) + if endOfVar == 0 { + return "", src[endOfLine:], nil + } + + // Work backwards to check if the line ends in whitespace then + // a comment (ie asdasd # some comment) + for i := endOfVar - 1; i >= 0; i-- { + if line[i] == charComment && i > 0 { + if isSpace(line[i-1]) { + endOfVar = i + break + } + } + } + + trimmed := strings.TrimFunc(string(line[0:endOfVar]), isSpace) + + return expandVariables(trimmed, vars), src[endOfLine:], nil + } + + // lookup quoted string terminator + for i := 1; i < len(src); i++ { + if char := src[i]; char != quote { + continue + } + + // skip escaped quote symbol (\" or \', depends on quote) + if prevChar := src[i-1]; prevChar == '\\' { + continue + } + + // trim quotes + trimFunc := isCharFunc(rune(quote)) + value = string(bytes.TrimLeftFunc(bytes.TrimRightFunc(src[0:i], trimFunc), trimFunc)) + if quote == prefixDoubleQuote { + // unescape newlines for double quote (this is compat feature) + // and expand environment variables + value = expandVariables(expandEscapes(value), vars) + } + + return value, src[i+1:], nil + } + + // return formatted error if quoted string is not terminated + valEndIndex := bytes.IndexFunc(src, isCharFunc('\n')) + if valEndIndex == -1 { + valEndIndex = len(src) + } + + return "", nil, fmt.Errorf("unterminated quoted value %s", src[:valEndIndex]) +} + +func expandEscapes(str string) string { + out := escapeRegex.ReplaceAllStringFunc(str, func(match string) string { + c := strings.TrimPrefix(match, `\`) + switch c { + case "n": + return "\n" + case "r": + return "\r" + default: + return match + } + }) + return unescapeCharsRegex.ReplaceAllString(out, "$1") +} + +func indexOfNonSpaceChar(src []byte) int { + return bytes.IndexFunc(src, func(r rune) bool { + return !unicode.IsSpace(r) + }) +} + +// hasQuotePrefix reports whether charset starts with single or double quote and returns quote character +func hasQuotePrefix(src []byte) (prefix byte, isQuored bool) { + if len(src) == 0 { + return 0, false + } + + switch prefix := src[0]; prefix { + case prefixDoubleQuote, prefixSingleQuote: + return prefix, true + default: + return 0, false + } +} + +func isCharFunc(char rune) func(rune) bool { + return func(v rune) bool { + return v == char + } +} + +// isSpace reports whether the rune is a space character but not line break character +// +// this differs from unicode.IsSpace, which also applies line break as space +func isSpace(r rune) bool { + switch r { + case '\t', '\v', '\f', '\r', ' ', 0x85, 0xA0: + return true + } + return false +} + +func isLineEnd(r rune) bool { + if r == '\n' || r == '\r' { + return true + } + return false +} + +var ( + escapeRegex = regexp.MustCompile(`\\.`) + expandVarRegex = regexp.MustCompile(`(\\)?(\$)(\()?\{?([A-Z0-9_]+)?\}?`) + unescapeCharsRegex = regexp.MustCompile(`\\([^$])`) +) + +func expandVariables(v string, m *orderedmap.OrderedMap[string, string]) string { + return expandVarRegex.ReplaceAllStringFunc(v, func(s string) string { + submatch := expandVarRegex.FindStringSubmatch(s) + + if submatch == nil { + return s + } + if submatch[1] == "\\" || submatch[2] == "(" { + return submatch[0][1:] + } else if submatch[4] != "" { + value, _ := m.Get(submatch[4]) + return value + } + return s + }) +} diff --git a/task_test.go b/task_test.go index 540e7ba8de..6f42c4fb8a 100644 --- a/task_test.go +++ b/task_test.go @@ -2125,6 +2125,41 @@ func TestShortTaskNotation(t *testing.T) { assert.Equal(t, "string-slice-1\nstring-slice-2\nstring\n", buff.String()) } +func TestDotenvNestedTemplates(t *testing.T) { + t.Parallel() + + for range 50 { + var output bytes.Buffer + e := task.NewExecutor( + task.WithDir("testdata/dotenv/nested"), + task.WithStdout(&output), + task.WithStderr(io.Discard), + ) + require.NoError(t, e.Setup()) + require.NoError(t, e.Run(t.Context(), &task.Call{Task: "default"})) + assert.Equal(t, "/home/user/nested/deeper/file.txt|/home/user/nested/deeper/file.txt\n", output.String()) + } +} + +func TestTaskDotenvDeclarationOrder(t *testing.T) { + t.Parallel() + + for range 50 { + e := task.NewExecutor( + task.WithDir("testdata/dotenv/nested"), + task.WithEntrypoint("testdata/dotenv/nested/Taskfile.local.yml"), + ) + require.NoError(t, e.Setup()) + + compiled, err := e.CompiledTask(&task.Call{Task: "local"}) + require.NoError(t, err) + assert.Equal(t, []string{"BASE_DIR", "MID_DIR", "FINAL_DIR", "FULL_PATH", "MID_DIR_2", "FINAL_DIR_2", "FULL_PATH_2", "EXTRA"}, slices.Collect(compiled.Env.Keys())) + base, ok := compiled.Env.Get("BASE_DIR") + require.True(t, ok) + assert.Equal(t, "/home/user", base.Value) + } +} + func TestDotenvShouldIncludeAllEnvFiles(t *testing.T) { t.Parallel() diff --git a/taskfile/dotenv.go b/taskfile/dotenv.go index a86a9eed1e..94c582df37 100644 --- a/taskfile/dotenv.go +++ b/taskfile/dotenv.go @@ -4,8 +4,7 @@ import ( "fmt" "os" - "github.com/joho/godotenv" - + "github.com/go-task/task/v3/internal/dotenv" "github.com/go-task/task/v3/internal/filepathext" "github.com/go-task/task/v3/internal/templater" "github.com/go-task/task/v3/taskfile/ast" @@ -26,11 +25,11 @@ func Dotenv(vars *ast.Vars, tf *ast.Taskfile, dir string) (*ast.Vars, error) { continue } - envs, err := godotenv.Read(dotEnvPath) + envs, err := dotenv.Read(dotEnvPath) if err != nil { return nil, fmt.Errorf("error reading env file %s: %w", dotEnvPath, err) } - for key, value := range envs { + for key, value := range envs.AllFromFront() { if _, ok := env.Get(key); !ok { env.Set(key, ast.Var{Value: value}) } diff --git a/testdata/dotenv/nested/Taskfile.local.yml b/testdata/dotenv/nested/Taskfile.local.yml new file mode 100644 index 0000000000..0cdfb986c4 --- /dev/null +++ b/testdata/dotenv/nested/Taskfile.local.yml @@ -0,0 +1,6 @@ +version: '3' + +tasks: + local: + dotenv: [nested.env, fallback.env] + cmds: [] diff --git a/testdata/dotenv/nested/Taskfile.yml b/testdata/dotenv/nested/Taskfile.yml new file mode 100644 index 0000000000..ea088f9394 --- /dev/null +++ b/testdata/dotenv/nested/Taskfile.yml @@ -0,0 +1,9 @@ +version: '3' + +dotenv: [nested.env, fallback.env] + +tasks: + default: + silent: true + cmds: + - echo "$FULL_PATH|$FULL_PATH_2" diff --git a/testdata/dotenv/nested/fallback.env b/testdata/dotenv/nested/fallback.env new file mode 100644 index 0000000000..e87bf1a9e7 --- /dev/null +++ b/testdata/dotenv/nested/fallback.env @@ -0,0 +1,2 @@ +BASE_DIR=/wrong +EXTRA=from_fallback diff --git a/testdata/dotenv/nested/nested.env b/testdata/dotenv/nested/nested.env new file mode 100644 index 0000000000..195427d756 --- /dev/null +++ b/testdata/dotenv/nested/nested.env @@ -0,0 +1,7 @@ +BASE_DIR=/home/user +MID_DIR=$BASE_DIR/nested +FINAL_DIR=$MID_DIR/deeper +FULL_PATH=$FINAL_DIR/file.txt +MID_DIR_2={{.BASE_DIR}}/nested +FINAL_DIR_2={{.MID_DIR_2}}/deeper +FULL_PATH_2={{.FINAL_DIR_2}}/file.txt diff --git a/variables.go b/variables.go index 2742b26312..1ba314be70 100644 --- a/variables.go +++ b/variables.go @@ -7,10 +7,9 @@ import ( "path/filepath" "strings" - "github.com/joho/godotenv" - "github.com/go-task/task/v3/errors" "github.com/go-task/task/v3/internal/deepcopy" + "github.com/go-task/task/v3/internal/dotenv" "github.com/go-task/task/v3/internal/env" "github.com/go-task/task/v3/internal/execext" "github.com/go-task/task/v3/internal/filepathext" @@ -179,11 +178,11 @@ func (e *Executor) compiledTask(call *Call, evaluateShVars bool) (*ast.Task, err if _, err := os.Stat(dotEnvPath); os.IsNotExist(err) { continue } - envs, err := godotenv.Read(dotEnvPath) + envs, err := dotenv.Read(dotEnvPath) if err != nil { return nil, err } - for key, value := range envs { + for key, value := range envs.AllFromFront() { if _, ok := dotenvEnvs.Get(key); !ok { dotenvEnvs.Set(key, ast.Var{Value: value}) }