Fix Newt #471

.gitignore

@@ -1,2 +1,3 @@
 .DS_Store
 bin/
+third_party/wintun/

Dockerfile

@@ -18,7 +18,8 @@ COPY . .
 # Build the application
 ARG VERSION=dev
 RUN NEWT_VERSION=$(go list -m -f '{{.Version}}' github.com/fosrl/newt | sed 's/^v//') && \
-    CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w -X github.com/fosrl/cli/internal/version.Version=${VERSION} -X github.com/fosrl/cli/internal/version.NewtVersionOverride=${NEWT_VERSION}" -o /pangolin-cli
+    OLM_VERSION=$(go list -m -f '{{.Version}}' github.com/fosrl/olm | sed 's/^v//') && \
+    CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w -X github.com/fosrl/cli/internal/version.Version=${VERSION} -X github.com/fosrl/cli/internal/version.NewtVersionOverride=${NEWT_VERSION} -X github.com/fosrl/cli/internal/version.OlmVersionOverride=${OLM_VERSION}" -o /pangolin-cli
 
 FROM public.ecr.aws/docker/library/alpine:3.23 AS runner
 

Makefile

@@ -3,9 +3,10 @@
 BINARY_NAME=pangolin
 OUTPUT_DIR=bin
 
-VERSION ?= 0.18.0
+VERSION ?= 0.19.0
 NEWT_VERSION := $(shell go list -m -f '{{.Version}}' github.com/fosrl/newt | sed 's/^v//')
-LDFLAGS = -s -w -X github.com/fosrl/cli/internal/version.Version=$(VERSION) -X github.com/fosrl/cli/internal/version.NewtVersionOverride=$(NEWT_VERSION)
+OLM_VERSION := $(shell go list -m -f '{{.Version}}' github.com/fosrl/olm | sed 's/^v//')
+LDFLAGS = -s -w -X github.com/fosrl/cli/internal/version.Version=$(VERSION) -X github.com/fosrl/cli/internal/version.NewtVersionOverride=$(NEWT_VERSION) -X github.com/fosrl/cli/internal/version.OlmVersionOverride=$(OLM_VERSION)
 
 all: clean build
 

cmd/up/client/client.go

@@ -771,6 +771,7 @@
 		HTTPAddr:   opts.HTTPAddr,
 		Version:    versionpkg.Version,
 		Agent:      defaultAgent,
+		OlmVersion: versionpkg.OlmVersionOverride,
 		// Spawn the pangolin binary itself in watchdog mode after a DNS
 		// override is installed. The watchdog will reset DNS if this
 		// process dies before it can restore the original configuration.

go.mod

@@ -2,6 +2,8 @@ module github.com/fosrl/cli
 
 go 1.26.0
 
+toolchain go1.26.10
+
 require (
 	github.com/Masterminds/semver/v3 v3.5.0
 	github.com/Microsoft/go-winio v0.6.3

internal/svcmgr/crashwatch.go

@@ -0,0 +1,83 @@
+package svcmgr
+
+import (
+	"bytes"
+	"io"
+	"sync"
+)
+
+// crashMarkers are line prefixes the Go runtime prints when a process dies
+// from an unrecovered panic or a fatal runtime error (on Windows, "Exception"
+// precedes the dump for access violations and other hardware exceptions).
+var crashMarkers = [][]byte{
+	[]byte("panic: "),
+	[]byte("fatal error: "),
+	[]byte("Exception 0x"),
+}
+
+const maxCrashMarkerLen = len("fatal error: ")
+
+// crashWatcher is the supervised child's stdout/stderr. It passes everything
+// through to out unchanged and closes Crashed the first time a line starts
+// with one of crashMarkers.
+//
+// The supervisor needs this because a crashing Go process does not always
+// exit: if a second thread faults, or the traceback itself faults, the
+// runtime can park forever in its crash handler. The process then keeps its
+// sockets open (so the site still looks connected) while forwarding nothing,
+// and the supervisor, which only reacts to process exit, never restarts it.
+type crashWatcher struct {
+	out     io.Writer
+	line    []byte // start of the current line, at most maxCrashMarkerLen bytes
+	crashed chan struct{}
+	once    sync.Once
+}
+
+func newCrashWatcher(out io.Writer) *crashWatcher {
+	return &crashWatcher{
+		out:     out,
+		line:    make([]byte, 0, maxCrashMarkerLen),
+		crashed: make(chan struct{}),
+	}
+}
+
+// Crashed is closed once the child has started printing a crash.
+func (w *crashWatcher) Crashed() <-chan struct{} {
+	return w.crashed
+}
+
+// Write never fails or short-writes: an error here would make the child's
+// own log writes fail or block, which is worse than losing log output.
+func (w *crashWatcher) Write(p []byte) (int, error) {
+	if w.out != nil {
+		_, _ = w.out.Write(p)
+	}
+
+	rest := p
+	for len(rest) > 0 {
+		chunk := rest
+		nl := bytes.IndexByte(rest, '\n')
+		if nl >= 0 {
+			chunk = rest[:nl]
+		}
+		if room := maxCrashMarkerLen - len(w.line); room > 0 {
+			w.line = append(w.line, chunk[:min(room, len(chunk))]...)
+		}
+		w.check()
+		if nl < 0 {
+			break
+		}
+		w.line = w.line[:0]
+		rest = rest[nl+1:]
+	}
+	return len(p), nil
+}
+
+func (w *crashWatcher) check() {
+	for _, m := range crashMarkers {
+		if bytes.HasPrefix(w.line, m) {
+			w.once.Do(func() { close(w.crashed) })
+			return
+		}
+	}
+}

internal/svcmgr/crashwatch_test.go

@@ -0,0 +1,66 @@
+package svcmgr
+
+import (
+	"bytes"
+	"testing"
+)
+
+func crashedNow(w *crashWatcher) bool {
+	select {
+	case <-w.Crashed():
+		return true
+	default:
+		return false
+	}
+}
+
+func TestCrashWatcherPassesOutputThrough(t *testing.T) {
+	var buf bytes.Buffer
+	w := newCrashWatcher(&buf)
+	in := "INFO: 2026/10/24 07:13:46 UDP Forwarder: started\nfatal error: unexpected signal\n"
+	n, err := w.Write([]byte(in))
+	if err != nil || n != len(in) {
+		t.Fatalf("Write = %d, %v; want %d, nil", n, err, len(in))
+	}
+	if buf.String() != in {
+		t.Fatalf("output = %q, want %q", buf.String(), in)
+	}
+}
+
+func TestCrashWatcherDetectsMarkers(t *testing.T) {
+	for _, line := range []string{
+		"panic: runtime error: invalid memory address or nil pointer dereference\n",
+		"fatal error: unexpected signal during runtime execution\n",
+		"Exception 0xc0000005 0x0 0xe8 0x7ff75a258cc5\n",
+	} {
+		w := newCrashWatcher(nil)
+		w.Write([]byte("INFO: normal line\n"))
+		if crashedNow(w) {
+			t.Fatalf("flagged a crash before %q", line)
+		}
+		w.Write([]byte(line))
+		if !crashedNow(w) {
+			t.Errorf("did not detect %q", line)
+		}
+	}
+}
+
+func TestCrashWatcherMarkerSplitAcrossWrites(t *testing.T) {
+	w := newCrashWatcher(nil)
+	for _, part := range []string{"INFO: x\nfat", "al er", "ror: fault\n"} {
+		w.Write([]byte(part))
+	}
+	if !crashedNow(w) {
+		t.Fatal("did not detect a marker split across writes")
+	}
+}
+
+func TestCrashWatcherIgnoresMarkerMidLine(t *testing.T) {
+	w := newCrashWatcher(nil)
+	w.Write([]byte("INFO: handler recovered from panic: boom\nERROR: fatal error: nope\n"))
+	w.Write([]byte("INFO: a very long line that keeps going past the marker length "))
+	w.Write([]byte("panic: still the same line\n"))
+	if crashedNow(w) {
+		t.Fatal("flagged a marker that was not at the start of a line")
+	}
+}

internal/svcmgr/olmclient_windows.go

@@ -138,6 +138,7 @@ func runOlmClient(ctx context.Context, serviceName string, creds ClientCreds) er
 		SocketPath: olm.ServiceSocketPath,
 		Version:    versionpkg.Version,
 		Agent:      "Pangolin Service",
+		OlmVersion: versionpkg.OlmVersionOverride,
 		// No WatchdogSubcommand: `pangolin up client` sets one (see
 		// cmd/watchdog) as a DNS-cleanup safety net for its own process,
 		// but the CLI's watchdog subcommand isn't implemented on Windows

internal/svcmgr/svcmgr_windows.go

@@ -5,6 +5,7 @@ package svcmgr
 import (
 	"errors"
 	"fmt"
+	"io"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -334,9 +335,14 @@ func (h *hostedService) Execute(_ []string, r <-chan svc.ChangeRequest, changes
 		defer logFile.Close()
 	}
 
+	var warn func(string)
+	if elogErr == nil {
+		warn = func(msg string) { _ = elog.Warning(1, msg) }
+	}
+
 	stopCh := make(chan struct{})
 	doneCh := make(chan struct{})
-	go h.supervise(logFile, stopCh, doneCh)
+	go h.supervise(logFile, warn, stopCh, doneCh)
 
 	changes <- svc.Status{State: svc.Running, Accepts: accepted}
 
@@ -364,29 +370,47 @@ func (h *hostedService) Execute(_ []string, r <-chan svc.ChangeRequest, changes
 	}
 }
 
+// crashExitTimeout is how long a child that has started printing a Go crash
+// gets to exit on its own before supervise kills it. A healthy crash exits
+// as soon as the traceback is written, which takes well under a second even
+// for thousands of goroutines.
+const crashExitTimeout = 30 * time.Second
+
 // supervise runs the actual `pangolin up ...` child process, restarting it
 // (after a short backoff) if it exits unexpectedly, until stopCh is closed.
 // This is the Windows equivalent of systemd's Restart=always / launchd's
-// KeepAlive.
-func (h *hostedService) supervise(logFile *os.File, stopCh <-chan struct{}, doneCh chan<- struct{}) {
+// KeepAlive. warn, if non-nil, reports to the Windows Event Log.
+func (h *hostedService) supervise(logFile *os.File, warn func(string), stopCh <-chan struct{}, doneCh chan<- struct{}) {
 	defer close(doneCh)
 
+	if warn == nil {
+		warn = func(string) {}
+	}
+
 	exePath, err := os.Executable()
 	if err != nil {
 		return
 	}
 
 	for {
 		cmd := exec.Command(exePath, h.upArgs...)
+		var out io.Writer
 		if logFile != nil {
-			cmd.Stdout = logFile
-			cmd.Stderr = logFile
+			out = logFile
 		}
+		watcher := newCrashWatcher(out)
+		cmd.Stdout = watcher
+		cmd.Stderr = watcher
+		// Output now goes through a pipe; don't let a handle inherited by
+		// a grandchild keep Wait from returning after the child is gone.
+		cmd.WaitDelay = 5 * time.Second
 		// Its own process group so a CTRL_BREAK_EVENT sent below targets
 		// only this child, not the service host itself.
 		cmd.SysProcAttr = &syscall.SysProcAttr{CreationFlags: windows.CREATE_NEW_PROCESS_GROUP}
 
-		if err := cmd.Start(); err == nil {
+		if err := cmd.Start(); err != nil {
+			warn(fmt.Sprintf("%s: failed to start worker: %v", h.name, err))
+		} else {
 			exitCh := make(chan struct{})
 			go func() {
 				_ = cmd.Wait()
@@ -395,18 +419,24 @@ func (h *hostedService) supervise(logFile *os.File, stopCh <-chan struct{}, done
 
 			select {
 			case <-stopCh:
-				// Ask nicely first (equivalent of SIGTERM), then force-kill
-				// if it doesn't exit in time.
-				_ = windows.GenerateConsoleCtrlEvent(windows.CTRL_BREAK_EVENT, uint32(cmd.Process.Pid))
+				stopChild(cmd, exitCh)
+				return
+			case <-exitCh:
+			case <-watcher.Crashed():
+				// The Go runtime can wedge in its crash handler instead of
+				// exiting (see crashWatcher), so don't wait on it forever.
 				select {
 				case <-exitCh:
-				case <-time.After(10 * time.Second):
+				case <-stopCh:
+					stopChild(cmd, exitCh)
+					return
+				case <-time.After(crashExitTimeout):
+					warn(fmt.Sprintf("%s: worker (pid %d) crashed but did not exit within %s; killing it", h.name, cmd.Process.Pid, crashExitTimeout))
 					_ = cmd.Process.Kill()
 					<-exitCh
 				}
-				return
-			case <-exitCh:
 			}
+			warn(fmt.Sprintf("%s: worker (pid %d) exited unexpectedly (%v); restarting", h.name, cmd.Process.Pid, cmd.ProcessState))
 		}
 
 		select {
@@ -416,3 +446,15 @@ func (h *hostedService) supervise(logFile *os.File, stopCh <-chan struct{}, done
 		}
 	}
 }
+
+// stopChild asks the child to exit (the equivalent of SIGTERM), then
+// force-kills it if it doesn't exit in time.
+func stopChild(cmd *exec.Cmd, exitCh <-chan struct{}) {
+	_ = windows.GenerateConsoleCtrlEvent(windows.CTRL_BREAK_EVENT, uint32(cmd.Process.Pid))
+	select {
+	case <-exitCh:
+	case <-time.After(10 * time.Second):
+		_ = cmd.Process.Kill()
+		<-exitCh
+	}
+}

internal/version/consts.go

@@ -4,4 +4,4 @@ package version
 // This value can be overridden at build time using ldflags:
 //
 //	go build -ldflags "-X github.com/fosrl/cli/internal/version.Version=<version>"
-var Version = "0.18.0"
+var Version = "0.19.0"

internal/version/olm.go

@@ -0,0 +1,12 @@
+package version
+
+// OlmVersionOverride is the version of the github.com/fosrl/olm module this
+// binary was built against (with any "v" prefix stripped), as read from
+// go.mod by the build process.
+//
+// It's set via -ldflags, the same way NewtVersionOverride is (see Makefile /
+// Dockerfile), and passed to olm as OlmConfig.OlmVersion so the server sees
+// the olm version separately from the CLI's own Version. If it wasn't set -
+// e.g. a plain `go build` - olm falls back to reading its module version
+// from the binary's embedded build info.
+var OlmVersionOverride string

pangolin-cli.wxs

@@ -2,7 +2,7 @@
 <Wix xmlns="http://wixtoolset.org/schemas/v4/wxs">
   <Package Name="Pangolin CLI"
            Language="1033"
-           Version="0.18.0"
+           Version="0.19.0"
            Manufacturer="Fossorial"
            UpgradeCode="35A1E3C4-C273-4334-9DF3-57408E83012E"
            Compressed="yes">
@@ -24,6 +24,12 @@
                        Action="set"
                        System="yes" />
         </Component>
+        <Component Id="WintunDll" Guid="E3F4A5B6-C7D8-4901-1234-56789ABCDEF0">
+          <File Id="WintunDll"
+                Source="third_party/wintun/amd64/wintun.dll"
+                Name="wintun.dll"
+                KeyPath="yes" />
+        </Component>
       </Directory>
     </StandardDirectory>
 
@@ -48,6 +54,7 @@
 
     <Feature Id="ProductFeature" Title="pangolin-cli" Level="1">
       <ComponentRef Id="PangolinCli" />
+      <ComponentRef Id="WintunDll" />
       <ComponentRef Id="StartMenuShortcut" />
     </Feature>
 

scripts/build-msi.bat

@@ -19,6 +19,11 @@ if not exist "%BUILDDIR%\pangolin-cli_windows_amd64.exe" (
   exit /b 1
 )
 
+if not exist "third_party\wintun\amd64\wintun.dll" (
+  echo ERROR: third_party\wintun\amd64\wintun.dll not found. Download Wintun from https://www.wintun.net/ 1>&2
+  exit /b 1
+)
+
 wix build -arch x64 -define "BuildDir=%BUILDDIR%" -o "%BUILDDIR%\pangolin-cli_windows_installer.msi" "pangolin-cli.wxs"
 if errorlevel 1 exit /b %errorlevel%
 echo Created "%BUILDDIR%\pangolin-cli_windows_installer.msi"