fix: request completions with "__complete" so a double dash can't run the command

.github/workflows/test.yml

@@ -34,6 +34,12 @@ jobs:
       - name: Set PATH
         run: echo "${GITHUB_WORKSPACE}/.local/bin" >>"${GITHUB_PATH}"
 
+      # The completion scripts are run in these to check what they send. Without
+      # them those tests skip, which is silent: CLI_SHELL_TESTS_REQUIRED below names
+      # them so that a skip is a failure and the coverage cannot go away unnoticed.
+      - if: matrix.os == 'ubuntu-24.04'
+        run: sudo apt-get update && sudo apt-get install -y bash-completion zsh fish
+
       - if: matrix.go == 'stable' && matrix.os == 'ubuntu-24.04'
         run: make ensure-goimports
 
@@ -42,6 +48,11 @@ jobs:
 
       - run: make vet
       - run: make test
+        env:
+          # The shells installed above, and only those: pwsh comes with the
+          # runner image, so requiring it would turn a change of image into a
+          # failure here.
+          CLI_SHELL_TESTS_REQUIRED: ${{ matrix.os == 'ubuntu-24.04' && 'bash,zsh,fish' || '' }}
       - run: make check-binary-size
 
       - if: matrix.go == 'stable' && matrix.os == 'ubuntu-24.04'

autocomplete/bash_autocomplete

@@ -11,15 +11,84 @@ __%[1]s_init_completion() {
   fi
 }
 
-__%[1]s_build_completion_request() {
-  local -a words_before_cursor=("${COMP_WORDS[@]:0:${COMP_CWORD}}")
-  local current_word="${COMP_WORDS[COMP_CWORD]}"
+# Remove one level of shell quoting from a word, the way the shell does before it
+# hands a word to a command, and leave the result in __cli_completion_dequoted.
+#
+# The variables here hold no application name, unlike the functions: a name is free to
+# hold a "-" or a ".", which a function name may and a variable name may not. They are
+# declared local by the entry point below, so two applications never share one.
+#
+# Doing it here rather than with eval is what keeps a command line holding $(...) or
+# `...` from being executed by pressing the tab key.
+__%[1]s_dequote() {
+  local s="$1" out="" c n quote=""
+  local i=0 len=${#1}
+
+  while (( i < len )); do
+    c="${s:i:1}"
+    if [[ "${quote}" == "'" ]]; then
+      if [[ "${c}" == "'" ]]; then quote=""; else out="${out}${c}"; fi
+    elif [[ "${quote}" == '"' ]]; then
+      if [[ "${c}" == '"' ]]; then
+        quote=""
+      elif [[ "${c}" == "\\" ]]; then
+        i=$(( i + 1 ))
+        n="${s:i:1}"
+        # Inside double quotes a backslash only escapes these.
+        case "${n}" in
+          '"' | "\\" | '$' | '`') out="${out}${n}" ;;
+          *) out="${out}\\${n}" ;;
+        esac
+      else
+        out="${out}${c}"
+      fi
+    else
+      case "${c}" in
+        "'" | '"') quote="${c}" ;;
+        "\\") i=$(( i + 1 )); out="${out}${s:i:1}" ;;
+        *) out="${out}${c}" ;;
+      esac
+    fi
+    i=$(( i + 1 ))
+  done
 
-  if [[ "${current_word}" == "-"* ]]; then
-    printf '%%s %%s --generate-shell-completion' "${words_before_cursor[*]}" "${current_word}"
-  else
-    printf '%%s --generate-shell-completion' "${words_before_cursor[*]}"
+  __cli_completion_dequoted="${out}"
+}
+
+# The request names the completion in its first argument, where a "--" typed on the
+# command line cannot turn it into a positional argument of whatever the command runs.
+# The word under the cursor is sent as the last argument, empty or not, so that
+# "cmd --<TAB>" and "cmd -- <TAB>" can be told apart.
+#
+# It is built as an array rather than as a string to eval, so that a word holding a
+# space reaches the command as the single word it is.
+#
+# The words come from words/cword rather than from COMP_WORDS/COMP_CWORD: bash splits
+# the line on COMP_WORDBREAKS, so "--opt=value" is three words in COMP_WORDS, while
+# __%[1]s_init_completion puts it back together. The candidates are filtered against
+# cur, which comes from there too, so a request built from anything else would ask the
+# command about a different word than the one being completed.
+__%[1]s_build_completion_request() {
+  local i cmd
+
+  __%[1]s_dequote "${words[0]}"
+  cmd="${__cli_completion_dequoted}"
+  # A command typed as "~/bin/app" has to be run as the path it stands for. eval used
+  # to do that as a side effect of re-parsing the line, along with everything else on
+  # it; this is the one expansion worth keeping, and it needs nothing evaluated. The
+  # raw word decides, because a quoted "~" is not a home directory to the shell either.
+  if [[ "${words[0]}" == "~" || "${words[0]}" == "~/"* ]]; then
+    cmd="${HOME}${cmd:1}"
   fi
+  __cli_completion_request=("${cmd}" "__complete")
+
+  for (( i = 1; i < cword; i++ )); do
+    __%[1]s_dequote "${words[i]}"
+    __cli_completion_request+=("${__cli_completion_dequoted}")
+  done
+
+  __%[1]s_dequote "${words[cword]-}"
+  __cli_completion_request+=("${__cli_completion_dequoted}")
 }
 
 # Keep Bash 3 compatibility: associative arrays require Bash 4+, so
@@ -42,23 +111,23 @@ __%[1]s_bash_autocomplete() {
   local words=("${COMP_WORDS[@]}")
 
   if [[ "${words[0]}" != "source" ]]; then
-    local cur opts
+    local cur prev opts
     local cword="${COMP_CWORD}"
-    local request_comp
+    local __cli_completion_dequoted=""
+    local -a __cli_completion_request=()
+    local -a __cli_completion_tokens=()
+    local -a __cli_completion_descriptions=()
 
     COMPREPLY=()
     cur="${words[$cword]}"
 
     __%[1]s_init_completion -n "=:" || return
 
-    request_comp="$(__%[1]s_build_completion_request)"
-    opts=$(eval "${request_comp}" 2>/dev/null)
+    __%[1]s_build_completion_request
+    opts=$("${__cli_completion_request[@]}" 2>/dev/null)
 
     # Completion output lines use "token:description" format.
     # Keep token/description in parallel arrays for Bash 3 compatibility.
-    __cli_completion_tokens=()
-    __cli_completion_descriptions=()
-
     local line
     local longest=0
     while IFS=$'\n' read -r line; do

autocomplete/fish_autocomplete

@@ -3,15 +3,31 @@
 function __%[1]s_perform_completion
     # Extract all args except the last one
     set -l args (commandline -opc)
-    # Extract the last arg (partial input)
-    set -l lastArg (commandline -ct)
-
-    if string match -q -- "-*" $lastArg
-        set results ($args[1] $args[2..-1] $lastArg --generate-shell-completion 2> /dev/null)
-    else
-        set results ($args[1] $args[2..-1] --generate-shell-completion 2> /dev/null)
+    # Extract the last arg (partial input), with one level of quoting taken off the
+    # way the shell would take it off before handing a word to a command. The words
+    # before it come tokenized, which does that already.
+    set -l rawArg (commandline -ct)
+    # string unescape answers with nothing for a word it cannot read, such as one
+    # ending in a lone backslash. The word as typed is a better answer than no word,
+    # which would be read as a fresh one.
+    set -l lastArg (string unescape -- $rawArg; or printf '%%s' $rawArg)
+
+    # The request names the completion in its first argument, where a "--" typed on
+    # the command line cannot turn it into a positional argument of whatever the
+    # command runs. The word under the cursor is sent as the last argument, quoted so
+    # that an empty one is still an argument, which tells "cmd --<TAB>" from
+    # "cmd -- <TAB>".
+    # A command typed as "~/bin/app" has to be run as the path it stands for, which
+    # nothing else here does: the words are taken apart, not evaluated. The word has
+    # been through the tokenizer by now, so a quoted "~" is expanded too, where the
+    # shell would leave it alone.
+    set -l cmd $args[1]
+    if string match -q -- '~' $cmd; or string match -q -- '~/*' $cmd
+        set cmd $HOME(string sub -s 2 -- $cmd)
     end
 
+    set results ($cmd __complete $args[2..-1] "$lastArg" 2> /dev/null)
+
     # Remove trailing empty lines
     for line in $results[-1..1]
         if test (string trim -- $line) = ""

autocomplete/powershell_autocomplete.ps1

@@ -1,9 +1,66 @@
 $fn = $($MyInvocation.MyCommand.Name)
 $name = $fn -replace "(.*)\.ps1$", '$1'
 Register-ArgumentCompleter -Native -CommandName $name -ScriptBlock {
-    param($commandName, $wordToComplete, $cursorPosition)
-    $other = "$wordToComplete --generate-shell-completion"
-    Invoke-Expression $other | ForEach-Object {
+    param($wordToComplete, $commandAst, $cursorPosition)
+
+    # One level of quoting is taken off each word, the way the shell would before
+    # handing it to a command. The value is not expanded: nothing on the command line
+    # is evaluated to answer a completion, so a "$(...)" reaches the command as the
+    # text it is rather than being run by pressing the tab key.
+    function __cliCompletionText($element) {
+        if ($element -is [System.Management.Automation.Language.StringConstantExpressionAst] -or
+            $element -is [System.Management.Automation.Language.ExpandableStringExpressionAst]) {
+            return $element.Value
+        }
+        return $element.Extent.Text
+    }
+
+    $elements = $commandAst.CommandElements
+    if ($elements.Count -eq 0) {
+        return
+    }
+
+    # The command name itself is the shell's to complete, not the command's.
+    if ($cursorPosition -le $elements[0].Extent.EndOffset) {
+        return
+    }
+
+    # The request names the completion in its first argument, where a "--" typed on
+    # the command line cannot turn it into a positional argument of whatever the
+    # command runs. The word under the cursor is sent as the last argument, empty or
+    # not, so that "cmd --<TAB>" and "cmd -- <TAB>" can be told apart.
+    #
+    # Which word that is comes from the cursor rather than from a comparison with
+    # $wordToComplete, which PowerShell hands over normalized: an unfinished "hello
+    # arrives here as "hello", matches no element as written, and would be sent both
+    # as a word of its own and as the word being completed. Reading the cursor also
+    # leaves out what follows it, so completing in the middle of a line asks about
+    # the line up to that point.
+    # A command typed as "~/bin/app" needs nothing done to it here: PowerShell resolves
+    # the tilde when it looks the command up, where the three other shells pass the
+    # word on as written and never find the command.
+    $command = __cliCompletionText $elements[0]
+    $words = @()
+    $word = ''
+    for ($i = 1; $i -lt $elements.Count; $i++) {
+        $extent = $elements[$i].Extent
+        if ($cursorPosition -gt $extent.StartOffset -and $cursorPosition -le $extent.EndOffset) {
+            $word = __cliCompletionText $elements[$i]
+        } elseif ($extent.EndOffset -lt $cursorPosition) {
+            $words += __cliCompletionText $elements[$i]
+        }
+    }
+
+    # The word being completed is the last argument, empty or not, which needs the
+    # argument passing mode PowerShell 7.3 made the default: before it, on Windows, an
+    # empty argument is dropped on the way to a native command and the request arrives
+    # a word short, which reads as a different command line rather than as an error.
+    # Windows PowerShell 5.1 has no such mode and cannot be helped from here.
+    if (Get-Variable -Name PSNativeCommandArgumentPassing -ErrorAction Ignore) {
+        $PSNativeCommandArgumentPassing = 'Standard'
+    }
+
+    & $command __complete @words $word 2>$null | ForEach-Object {
         $parts = $_.Split(':', 2)
         if ($parts.Count -eq 2) {
             $completion = $parts[0].Trim()
@@ -13,4 +70,4 @@ Register-ArgumentCompleter -Native -CommandName $name -ScriptBlock {
             [System.Management.Automation.CompletionResult]::new($_, $_, 'ParameterValue', $_)
         }
     }
-}
+}

autocomplete/zsh_autocomplete

@@ -5,15 +5,32 @@ compdef _%[1]s %[1]s
 
 _%[1]s() {
 	local -a opts # Declare a local array
-	local current
-	current=${words[-1]} # -1 means "the last element"
-	if [[ "$current" == "-"* ]]; then
-		# Current word starts with a hyphen, so complete flags/options
-		opts=("${(@f)$(${words[@]:0:#words[@]-1} ${current} --generate-shell-completion)}")
-	else
-		# Current word does not start with a hyphen, so complete subcommands
-		opts=("${(@f)$(${words[@]:0:#words[@]-1} --generate-shell-completion)}")
+	local -a request
+	# The request names the completion in its first argument, where a "--" typed on
+	# the command line cannot turn it into a positional argument of whatever the
+	# command runs. The word under the cursor is sent as the last argument, empty or
+	# not, so that "cmd --<TAB>" and "cmd -- <TAB>" can be told apart.
+	# (Q) takes one level of quoting off each word, the way the shell would before
+	# handing it to a command, so that a quoted word reaches the command as the word
+	# it is rather than with its quotes.
+	local raw="${words[CURRENT]}"
+	local current="${(Q)words[CURRENT]}"
+	# A word whose quote is still open has no closing quote to take off with it, so
+	# (Q) leaves it alone. Dropping the opening quote asks the command about the word
+	# being typed rather than about one starting with a quote character, which is what
+	# bash and PowerShell send for the same line.
+	if [[ "$current" == "$raw" && "$raw" == [\"\']* ]]; then
+		current="${current#[\"\']}"
+	fi
+	# A command typed as "~/bin/app" has to be run as the path it stands for, which
+	# nothing else here does: the words are taken apart, not evaluated. The raw word
+	# decides, because a quoted "~" is not a home directory to the shell either.
+	local cmd="${(Q)words[1]}"
+	if [[ "${words[1]}" == "~" || "${words[1]}" == "~/"* ]]; then
+		cmd="${HOME}${cmd#\~}"
 	fi
+	request=("$cmd" "__complete" "${(@Q)words[2,CURRENT-1]}" "$current")
+	opts=("${(@f)$("${request[@]}" 2>/dev/null)}")
 
 	if [[ "${opts[1]}" != "" ]]; then
 		_describe 'values' opts

command.go

@@ -157,6 +157,11 @@ type Command struct {
 	didSetupDefaults bool
 	// whether in shell completion mode
 	shellCompletion bool
+	// what the shell completion request being answered says about the word being
+	// completed, or nil when this run is answering none. Only the root command holds
+	// it, and every run replaces it, so a Command answering several requests never
+	// carries one request's state into the next.
+	completion *completionRequest
 	// whether global help flag was added
 	globaHelpFlagAdded bool
 	// whether global version flag was added

command_run.go

@@ -118,20 +118,21 @@ func (cmd *Command) run(ctx context.Context, osArgs []string) (_ context.Context
 				osArgs = append(osArgs, args...)
 			}
 		}
-		// handle the completion flag separately from the flagset since
+		// handle the completion request separately from the flagset since
 		// completion could be attempted after a flag, but before its value was put
 		// on the command line. this causes the flagset to interpret the completion
-		// flag name as the value of the flag before it which is undesirable
+		// request as the value of the flag before it which is undesirable
 		// note that we can only do this because the shell autocomplete function
-		// always appends the completion flag at the end of the command
+		// sends the request in a place the flagset never reaches: the first argument,
+		// or, for a script generated before that change, the last one
 		tracef("checking osArgs %v (cmd=%[2]q)", osArgs, cmd.Name)
-		cmd.shellCompletion, osArgs = checkShellCompleteFlag(cmd, osArgs)
+		cmd.shellCompletion, osArgs = parseShellCompleteRequest(cmd, osArgs)
 
-		tracef("setting cmd.shellCompletion=%[1]v from checkShellCompleteFlag (cmd=%[2]q)", cmd.shellCompletion && cmd.EnableShellCompletion, cmd.Name)
+		tracef("setting cmd.shellCompletion=%[1]v from parseShellCompleteRequest (cmd=%[2]q)", cmd.shellCompletion && cmd.EnableShellCompletion, cmd.Name)
 		cmd.shellCompletion = cmd.EnableShellCompletion && cmd.shellCompletion
 	}
 
-	tracef("using post-checkShellCompleteFlag arguments %[1]q (cmd=%[2]q)", osArgs, cmd.Name)
+	tracef("using post-parseShellCompleteRequest arguments %[1]q (cmd=%[2]q)", osArgs, cmd.Name)
 
 	tracef("setting self as cmd in context (cmd=%[1]q)", cmd.Name)
 	ctx = context.WithValue(ctx, commandContextKey, cmd)
@@ -164,11 +165,17 @@ func (cmd *Command) run(ctx context.Context, osArgs []string) (_ context.Context
 	tracef("using post-parse arguments %[1]q (cmd=%[2]q)", args, cmd.Name)
 
 	if shouldRunCompletion(cmd) {
-		var beforeErr error
-		if ctx, beforeErr = runBefore(ctx, commandChain(cmd)); beforeErr != nil {
-			return ctx, beforeErr
+		// Everything after "--" is a positional argument of whatever the command runs,
+		// so there is no completion to run and nothing to prepare for one: a Before
+		// with a side effect would otherwise fire on every tab key past the
+		// terminator, for an answer that is always empty.
+		if !cmd.Root().completionTerminated() {
+			var beforeErr error
+			if ctx, beforeErr = runBefore(ctx, commandChain(cmd)); beforeErr != nil {
+				return ctx, beforeErr
+			}
+			runCompletion(ctx, cmd)
 		}
-		runCompletion(ctx, cmd)
 		return ctx, nil
 	}
 

completion.go

@@ -11,7 +11,18 @@ const (
 	completionCommandName = "completion"
 
 	// This flag is supposed to only be used by the completion script itself to generate completions on the fly.
+	//
+	// What is deprecated is the request form, not this constant: a request appended to
+	// the end of the command line is indistinguishable from a positional argument
+	// after "--", so the scripts name it with completionCommandRequest instead. The
+	// flag is still understood, so that scripts generated before that change keep
+	// working, and this stays as the name they send.
 	completionFlag = "--generate-shell-completion"
+
+	// This argument is supposed to only be used by the completion script itself to
+	// generate completions on the fly. It is the first argument of the request, where
+	// "--" cannot turn it into a positional argument.
+	completionCommandRequest = "__complete"
 )
 
 type renderCompletion func(cmd *Command, appName string) (string, error)