diff --git a/.gitignore b/.gitignore index b5731b3..68360f8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ .DS_Store bin/ +third_party/wintun/ diff --git a/Dockerfile b/Dockerfile index 4880121..67d52e3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,7 +18,8 @@ COPY . . # Build the application ARG VERSION=dev RUN NEWT_VERSION=$(go list -m -f '{{.Version}}' github.com/fosrl/newt | sed 's/^v//') && \ - CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w -X github.com/fosrl/cli/internal/version.Version=${VERSION} -X github.com/fosrl/cli/internal/version.NewtVersionOverride=${NEWT_VERSION}" -o /pangolin-cli + OLM_VERSION=$(go list -m -f '{{.Version}}' github.com/fosrl/olm | sed 's/^v//') && \ + CGO_ENABLED=0 GOOS=linux go build -ldflags="-s -w -X github.com/fosrl/cli/internal/version.Version=${VERSION} -X github.com/fosrl/cli/internal/version.NewtVersionOverride=${NEWT_VERSION} -X github.com/fosrl/cli/internal/version.OlmVersionOverride=${OLM_VERSION}" -o /pangolin-cli FROM public.ecr.aws/docker/library/alpine:3.23 AS runner diff --git a/Makefile b/Makefile index 639ec12..ce94ea9 100644 --- a/Makefile +++ b/Makefile @@ -3,9 +3,10 @@ BINARY_NAME=pangolin OUTPUT_DIR=bin -VERSION ?= 0.18.0 +VERSION ?= 0.19.0 NEWT_VERSION := $(shell go list -m -f '{{.Version}}' github.com/fosrl/newt | sed 's/^v//') -LDFLAGS = -s -w -X github.com/fosrl/cli/internal/version.Version=$(VERSION) -X github.com/fosrl/cli/internal/version.NewtVersionOverride=$(NEWT_VERSION) +OLM_VERSION := $(shell go list -m -f '{{.Version}}' github.com/fosrl/olm | sed 's/^v//') +LDFLAGS = -s -w -X github.com/fosrl/cli/internal/version.Version=$(VERSION) -X github.com/fosrl/cli/internal/version.NewtVersionOverride=$(NEWT_VERSION) -X github.com/fosrl/cli/internal/version.OlmVersionOverride=$(OLM_VERSION) all: clean build diff --git a/cmd/up/client/client.go b/cmd/up/client/client.go index 2ce6c2f..76ec986 100644 --- a/cmd/up/client/client.go +++ b/cmd/up/client/client.go @@ -771,6 +771,7 @@ func clientUpMain(cmd *cobra.Command, opts *ClientUpCmdOpts, extraArgs []string) HTTPAddr: opts.HTTPAddr, Version: versionpkg.Version, Agent: defaultAgent, + OlmVersion: versionpkg.OlmVersionOverride, // Spawn the pangolin binary itself in watchdog mode after a DNS // override is installed. The watchdog will reset DNS if this // process dies before it can restore the original configuration. diff --git a/go.mod b/go.mod index a2f02be..6ba2514 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,8 @@ module github.com/fosrl/cli go 1.26.0 +toolchain go1.26.10 + require ( github.com/Masterminds/semver/v3 v3.5.0 github.com/Microsoft/go-winio v0.6.3 diff --git a/internal/svcmgr/crashwatch.go b/internal/svcmgr/crashwatch.go new file mode 100644 index 0000000..1603c9b --- /dev/null +++ b/internal/svcmgr/crashwatch.go @@ -0,0 +1,83 @@ +package svcmgr + +import ( + "bytes" + "io" + "sync" +) + +// crashMarkers are line prefixes the Go runtime prints when a process dies +// from an unrecovered panic or a fatal runtime error (on Windows, "Exception" +// precedes the dump for access violations and other hardware exceptions). +var crashMarkers = [][]byte{ + []byte("panic: "), + []byte("fatal error: "), + []byte("Exception 0x"), +} + +const maxCrashMarkerLen = len("fatal error: ") + +// crashWatcher is the supervised child's stdout/stderr. It passes everything +// through to out unchanged and closes Crashed the first time a line starts +// with one of crashMarkers. +// +// The supervisor needs this because a crashing Go process does not always +// exit: if a second thread faults, or the traceback itself faults, the +// runtime can park forever in its crash handler. The process then keeps its +// sockets open (so the site still looks connected) while forwarding nothing, +// and the supervisor, which only reacts to process exit, never restarts it. +type crashWatcher struct { + out io.Writer + line []byte // start of the current line, at most maxCrashMarkerLen bytes + crashed chan struct{} + once sync.Once +} + +func newCrashWatcher(out io.Writer) *crashWatcher { + return &crashWatcher{ + out: out, + line: make([]byte, 0, maxCrashMarkerLen), + crashed: make(chan struct{}), + } +} + +// Crashed is closed once the child has started printing a crash. +func (w *crashWatcher) Crashed() <-chan struct{} { + return w.crashed +} + +// Write never fails or short-writes: an error here would make the child's +// own log writes fail or block, which is worse than losing log output. +func (w *crashWatcher) Write(p []byte) (int, error) { + if w.out != nil { + _, _ = w.out.Write(p) + } + + rest := p + for len(rest) > 0 { + chunk := rest + nl := bytes.IndexByte(rest, '\n') + if nl >= 0 { + chunk = rest[:nl] + } + if room := maxCrashMarkerLen - len(w.line); room > 0 { + w.line = append(w.line, chunk[:min(room, len(chunk))]...) + } + w.check() + if nl < 0 { + break + } + w.line = w.line[:0] + rest = rest[nl+1:] + } + return len(p), nil +} + +func (w *crashWatcher) check() { + for _, m := range crashMarkers { + if bytes.HasPrefix(w.line, m) { + w.once.Do(func() { close(w.crashed) }) + return + } + } +} diff --git a/internal/svcmgr/crashwatch_test.go b/internal/svcmgr/crashwatch_test.go new file mode 100644 index 0000000..791286a --- /dev/null +++ b/internal/svcmgr/crashwatch_test.go @@ -0,0 +1,66 @@ +package svcmgr + +import ( + "bytes" + "testing" +) + +func crashedNow(w *crashWatcher) bool { + select { + case <-w.Crashed(): + return true + default: + return false + } +} + +func TestCrashWatcherPassesOutputThrough(t *testing.T) { + var buf bytes.Buffer + w := newCrashWatcher(&buf) + in := "INFO: 2026/10/24 07:13:46 UDP Forwarder: started\nfatal error: unexpected signal\n" + n, err := w.Write([]byte(in)) + if err != nil || n != len(in) { + t.Fatalf("Write = %d, %v; want %d, nil", n, err, len(in)) + } + if buf.String() != in { + t.Fatalf("output = %q, want %q", buf.String(), in) + } +} + +func TestCrashWatcherDetectsMarkers(t *testing.T) { + for _, line := range []string{ + "panic: runtime error: invalid memory address or nil pointer dereference\n", + "fatal error: unexpected signal during runtime execution\n", + "Exception 0xc0000005 0x0 0xe8 0x7ff75a258cc5\n", + } { + w := newCrashWatcher(nil) + w.Write([]byte("INFO: normal line\n")) + if crashedNow(w) { + t.Fatalf("flagged a crash before %q", line) + } + w.Write([]byte(line)) + if !crashedNow(w) { + t.Errorf("did not detect %q", line) + } + } +} + +func TestCrashWatcherMarkerSplitAcrossWrites(t *testing.T) { + w := newCrashWatcher(nil) + for _, part := range []string{"INFO: x\nfat", "al er", "ror: fault\n"} { + w.Write([]byte(part)) + } + if !crashedNow(w) { + t.Fatal("did not detect a marker split across writes") + } +} + +func TestCrashWatcherIgnoresMarkerMidLine(t *testing.T) { + w := newCrashWatcher(nil) + w.Write([]byte("INFO: handler recovered from panic: boom\nERROR: fatal error: nope\n")) + w.Write([]byte("INFO: a very long line that keeps going past the marker length ")) + w.Write([]byte("panic: still the same line\n")) + if crashedNow(w) { + t.Fatal("flagged a marker that was not at the start of a line") + } +} diff --git a/internal/svcmgr/olmclient_windows.go b/internal/svcmgr/olmclient_windows.go index 668938c..ab92039 100644 --- a/internal/svcmgr/olmclient_windows.go +++ b/internal/svcmgr/olmclient_windows.go @@ -138,6 +138,7 @@ func runOlmClient(ctx context.Context, serviceName string, creds ClientCreds) er SocketPath: olm.ServiceSocketPath, Version: versionpkg.Version, Agent: "Pangolin Service", + OlmVersion: versionpkg.OlmVersionOverride, // No WatchdogSubcommand: `pangolin up client` sets one (see // cmd/watchdog) as a DNS-cleanup safety net for its own process, // but the CLI's watchdog subcommand isn't implemented on Windows diff --git a/internal/svcmgr/svcmgr_windows.go b/internal/svcmgr/svcmgr_windows.go index 06c5409..80dcfa8 100644 --- a/internal/svcmgr/svcmgr_windows.go +++ b/internal/svcmgr/svcmgr_windows.go @@ -5,6 +5,7 @@ package svcmgr import ( "errors" "fmt" + "io" "os" "os/exec" "path/filepath" @@ -334,9 +335,14 @@ func (h *hostedService) Execute(_ []string, r <-chan svc.ChangeRequest, changes defer logFile.Close() } + var warn func(string) + if elogErr == nil { + warn = func(msg string) { _ = elog.Warning(1, msg) } + } + stopCh := make(chan struct{}) doneCh := make(chan struct{}) - go h.supervise(logFile, stopCh, doneCh) + go h.supervise(logFile, warn, stopCh, doneCh) changes <- svc.Status{State: svc.Running, Accepts: accepted} @@ -364,13 +370,23 @@ func (h *hostedService) Execute(_ []string, r <-chan svc.ChangeRequest, changes } } +// crashExitTimeout is how long a child that has started printing a Go crash +// gets to exit on its own before supervise kills it. A healthy crash exits +// as soon as the traceback is written, which takes well under a second even +// for thousands of goroutines. +const crashExitTimeout = 30 * time.Second + // supervise runs the actual `pangolin up ...` child process, restarting it // (after a short backoff) if it exits unexpectedly, until stopCh is closed. // This is the Windows equivalent of systemd's Restart=always / launchd's -// KeepAlive. -func (h *hostedService) supervise(logFile *os.File, stopCh <-chan struct{}, doneCh chan<- struct{}) { +// KeepAlive. warn, if non-nil, reports to the Windows Event Log. +func (h *hostedService) supervise(logFile *os.File, warn func(string), stopCh <-chan struct{}, doneCh chan<- struct{}) { defer close(doneCh) + if warn == nil { + warn = func(string) {} + } + exePath, err := os.Executable() if err != nil { return @@ -378,15 +394,23 @@ func (h *hostedService) supervise(logFile *os.File, stopCh <-chan struct{}, done for { cmd := exec.Command(exePath, h.upArgs...) + var out io.Writer if logFile != nil { - cmd.Stdout = logFile - cmd.Stderr = logFile + out = logFile } + watcher := newCrashWatcher(out) + cmd.Stdout = watcher + cmd.Stderr = watcher + // Output now goes through a pipe; don't let a handle inherited by + // a grandchild keep Wait from returning after the child is gone. + cmd.WaitDelay = 5 * time.Second // Its own process group so a CTRL_BREAK_EVENT sent below targets // only this child, not the service host itself. cmd.SysProcAttr = &syscall.SysProcAttr{CreationFlags: windows.CREATE_NEW_PROCESS_GROUP} - if err := cmd.Start(); err == nil { + if err := cmd.Start(); err != nil { + warn(fmt.Sprintf("%s: failed to start worker: %v", h.name, err)) + } else { exitCh := make(chan struct{}) go func() { _ = cmd.Wait() @@ -395,18 +419,24 @@ func (h *hostedService) supervise(logFile *os.File, stopCh <-chan struct{}, done select { case <-stopCh: - // Ask nicely first (equivalent of SIGTERM), then force-kill - // if it doesn't exit in time. - _ = windows.GenerateConsoleCtrlEvent(windows.CTRL_BREAK_EVENT, uint32(cmd.Process.Pid)) + stopChild(cmd, exitCh) + return + case <-exitCh: + case <-watcher.Crashed(): + // The Go runtime can wedge in its crash handler instead of + // exiting (see crashWatcher), so don't wait on it forever. select { case <-exitCh: - case <-time.After(10 * time.Second): + case <-stopCh: + stopChild(cmd, exitCh) + return + case <-time.After(crashExitTimeout): + warn(fmt.Sprintf("%s: worker (pid %d) crashed but did not exit within %s; killing it", h.name, cmd.Process.Pid, crashExitTimeout)) _ = cmd.Process.Kill() <-exitCh } - return - case <-exitCh: } + warn(fmt.Sprintf("%s: worker (pid %d) exited unexpectedly (%v); restarting", h.name, cmd.Process.Pid, cmd.ProcessState)) } select { @@ -416,3 +446,15 @@ func (h *hostedService) supervise(logFile *os.File, stopCh <-chan struct{}, done } } } + +// stopChild asks the child to exit (the equivalent of SIGTERM), then +// force-kills it if it doesn't exit in time. +func stopChild(cmd *exec.Cmd, exitCh <-chan struct{}) { + _ = windows.GenerateConsoleCtrlEvent(windows.CTRL_BREAK_EVENT, uint32(cmd.Process.Pid)) + select { + case <-exitCh: + case <-time.After(10 * time.Second): + _ = cmd.Process.Kill() + <-exitCh + } +} diff --git a/internal/version/consts.go b/internal/version/consts.go index 505b181..ef3b698 100644 --- a/internal/version/consts.go +++ b/internal/version/consts.go @@ -4,4 +4,4 @@ package version // This value can be overridden at build time using ldflags: // // go build -ldflags "-X github.com/fosrl/cli/internal/version.Version=" -var Version = "0.18.0" +var Version = "0.19.0" diff --git a/internal/version/olm.go b/internal/version/olm.go new file mode 100644 index 0000000..4a5fa13 --- /dev/null +++ b/internal/version/olm.go @@ -0,0 +1,12 @@ +package version + +// OlmVersionOverride is the version of the github.com/fosrl/olm module this +// binary was built against (with any "v" prefix stripped), as read from +// go.mod by the build process. +// +// It's set via -ldflags, the same way NewtVersionOverride is (see Makefile / +// Dockerfile), and passed to olm as OlmConfig.OlmVersion so the server sees +// the olm version separately from the CLI's own Version. If it wasn't set - +// e.g. a plain `go build` - olm falls back to reading its module version +// from the binary's embedded build info. +var OlmVersionOverride string diff --git a/pangolin-cli.wxs b/pangolin-cli.wxs index 99a1b47..5a63827 100644 --- a/pangolin-cli.wxs +++ b/pangolin-cli.wxs @@ -2,7 +2,7 @@ @@ -24,6 +24,12 @@ Action="set" System="yes" /> + + + @@ -48,6 +54,7 @@ + diff --git a/scripts/build-msi.bat b/scripts/build-msi.bat index f6f40f4..c66a296 100644 --- a/scripts/build-msi.bat +++ b/scripts/build-msi.bat @@ -19,6 +19,11 @@ if not exist "%BUILDDIR%\pangolin-cli_windows_amd64.exe" ( exit /b 1 ) +if not exist "third_party\wintun\amd64\wintun.dll" ( + echo ERROR: third_party\wintun\amd64\wintun.dll not found. Download Wintun from https://www.wintun.net/ 1>&2 + exit /b 1 +) + wix build -arch x64 -define "BuildDir=%BUILDDIR%" -o "%BUILDDIR%\pangolin-cli_windows_installer.msi" "pangolin-cli.wxs" if errorlevel 1 exit /b %errorlevel% echo Created "%BUILDDIR%\pangolin-cli_windows_installer.msi"