diff --git a/.gitignore b/.gitignore index f38effa..482c642 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ labs/hw/license/ bundle/ /results/ out/ +labs/hw/kvm/bin/ diff --git a/docs/lab.md b/docs/lab.md index e7a3e01..e073392 100644 --- a/docs/lab.md +++ b/docs/lab.md @@ -23,6 +23,13 @@ The BMCs (IPMI, iDRAC) and two networked PDUs sit on the lab's management LAN an only through rackpi. `labs/hw/bringup-guarded.sh` uses them to recover a DUT whose NIC wedged during DPDK initialisation: first an IPMI power cycle, then a PDU power drain. +alice and bob have no BMC; each has a JetKVM on the same management LAN instead, driven by +`labs/hw/setup/kvm.sh {status|reboot|sysrq-reboot|on|off|cycle|reset}` (and by +`setup/ipmi.sh --target dut` for those profiles). `reboot` (Ctrl+Alt+Del) and `sysrq-reboot` +type on the host's keyboard, so they need a live kernel. Power on/off/cycle/reset press the +motherboard buttons and need the JetKVM ATX extension, which is not fitted yet. Every call takes +over the KVM session, signing out anyone using its web UI. + ## Cabling ``` diff --git a/labs/hw/bringup-guarded.sh b/labs/hw/bringup-guarded.sh index cb0096d..2eadb74 100755 --- a/labs/hw/bringup-guarded.sh +++ b/labs/hw/bringup-guarded.sh @@ -15,7 +15,6 @@ DUT_SEL="$DUT" # vars.sh profile name (server1|epyc) before DUT is reus DUT="$LAB_SSH_USER@$DUT_HOST" IPMI_PROXY="${IPMI_PROXY:-${LAB_PROXY:-}}" -ipmi() { $SSH "$IPMI_PROXY" "ipmitool -I lanplus -H '$DUT_IPMI_HOST' -U '$DUT_IPMI_USER' -P '$DUT_IPMI_PASS' $*"; } dut_alive() { timeout 10 $SSH -o ConnectTimeout=6 "$DUT" 'echo ok' 2>/dev/null | grep -q ok; } vpp_up() { timeout 14 $SSH -o ConnectTimeout=8 "$DUT" \ @@ -28,12 +27,13 @@ n_dut_containers() { 'sg docker -c "docker ps -q --filter name=hw-dut" | wc -l' 2>/dev/null | tr -d '[:space:]' } recover() { - echo ">> WEDGED — auto-recovering via IPMI cold cycle..." - ipmi power off >/dev/null 2>&1; sleep 20; ipmi power on >/dev/null 2>&1 - for _ in $(seq 1 18); do dut_alive && { echo ">> DUT back online."; return 0; }; sleep 15; done + echo ">> WEDGED — auto-recovering via an out-of-band cold cycle (IPMI, or JetKVM ATX)..." + if DUT="$DUT_SEL" bash "$SCRIPT_DIR/setup/ipmi.sh" --target dut cycle 2>&1 | sed 's/^/ /'; then + for _ in $(seq 1 18); do dut_alive && { echo ">> DUT back online."; return 0; }; sleep 15; done + fi local dut_name="${DUT_HOST%%.*}" - echo ">> IPMI cycle did not recover it — escalating to a PDU AC-drain (cx7-recover $dut_name)..." + echo ">> Power cycle did not recover it — escalating to a PDU AC-drain (cx7-recover $dut_name)..." if bash "$SCRIPT_DIR/setup/cx7-recover.sh" "$dut_name" --no-verify 2>&1 | sed 's/^/ /'; then for _ in $(seq 1 18); do dut_alive && { echo ">> DUT back online after AC drain."; return 0; }; sleep 15; done fi diff --git a/labs/hw/dut/start.sh b/labs/hw/dut/start.sh index 0b38420..0dc30a9 100755 --- a/labs/hw/dut/start.sh +++ b/labs/hw/dut/start.sh @@ -206,7 +206,8 @@ if [ "$NIC_STUCK" = "1" ]; then echo "" echo "FATAL: CX5 NIC firmware stuck in pre-init state." echo " Warm reset does NOT clear this. Fix: POWER CYCLE the server." - echo " Use: ./labs/hw/setup/ipmi.sh cycle (or power-off → power-on via IPMI)" + echo " Use: ./labs/hw/setup/ipmi.sh cycle (or power-off → power-on via IPMI;" + echo " alice/bob go through their JetKVM, which needs the ATX extension)" exit 1 fi fi diff --git a/labs/hw/kvm/jetkvm-rpc/go.mod b/labs/hw/kvm/jetkvm-rpc/go.mod new file mode 100644 index 0000000..59796b4 --- /dev/null +++ b/labs/hw/kvm/jetkvm-rpc/go.mod @@ -0,0 +1,32 @@ +module jetkvm-rpc + +go 1.25.0 + +require ( + github.com/coder/websocket v1.8.15 + github.com/pion/webrtc/v4 v4.2.22 +) + +require ( + github.com/google/uuid v1.6.0 // indirect + github.com/pion/datachannel v1.6.3 // indirect + github.com/pion/dtls/v3 v3.1.9 // indirect + github.com/pion/ice/v4 v4.4.4 // indirect + github.com/pion/interceptor v0.1.49 // indirect + github.com/pion/logging v0.2.4 // indirect + github.com/pion/mdns/v2 v2.2.1 // indirect + github.com/pion/randutil v0.1.0 // indirect + github.com/pion/rtcp v1.2.18 // indirect + github.com/pion/rtp v1.10.5 // indirect + github.com/pion/sctp v1.11.3 // indirect + github.com/pion/sdp/v3 v3.0.20 // indirect + github.com/pion/srtp/v3 v3.1.0 // indirect + github.com/pion/stun/v4 v4.0.1 // indirect + github.com/pion/transport/v5 v5.1.1 // indirect + github.com/pion/turn/v5 v5.1.2 // indirect + github.com/wlynxg/anet v0.0.5 // indirect + golang.org/x/crypto v0.52.0 // indirect + golang.org/x/net v0.54.0 // indirect + golang.org/x/sys v0.45.0 // indirect + golang.org/x/time v0.14.0 // indirect +) diff --git a/labs/hw/kvm/jetkvm-rpc/go.sum b/labs/hw/kvm/jetkvm-rpc/go.sum new file mode 100644 index 0000000..49638bc --- /dev/null +++ b/labs/hw/kvm/jetkvm-rpc/go.sum @@ -0,0 +1,50 @@ +github.com/coder/websocket v1.8.15 h1:6B2JPeOGlpff2Uz6vOEH1Vzpi0iUz20A+lPVhPHtNUA= +github.com/coder/websocket v1.8.15/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/pion/datachannel v1.6.3 h1:HZAYLpunVI0xi057ZiANl558d0uSpomcWw4UEPz3qXs= +github.com/pion/datachannel v1.6.3/go.mod h1:oRmkXVVXxkUbqa8MatgdNHW29kvtQY3HUdjJQsfv2GE= +github.com/pion/dtls/v3 v3.1.9 h1:rpeycmLIkc4krpk1IxP7+39o11QdCXbmV9+FGi9yZJ8= +github.com/pion/dtls/v3 v3.1.9/go.mod h1:iKFQNYrjsN2TiA2YKKMqB9MOZaFpjFULBI/A4sW0eyc= +github.com/pion/ice/v4 v4.4.4 h1:nPhqLkf8NoE9O8NBKd3Xo2NNc2PlBMdM07tWZ+IH/i8= +github.com/pion/ice/v4 v4.4.4/go.mod h1:vkrbLwF98P/DJuCKoAmOQY4FJ7dJPI5VcBFOO3kyYx0= +github.com/pion/interceptor v0.1.49 h1:iyBsNHRoLNNXZiJA/DdGvJgdyfS8YWDCzxIAlVjt5nY= +github.com/pion/interceptor v0.1.49/go.mod h1:MZ6PJkja/TCo350HAnBrs/rUIyad9mWjpcvytrf3ViQ= +github.com/pion/logging v0.2.4 h1:tTew+7cmQ+Mc1pTBLKH2puKsOvhm32dROumOZ655zB8= +github.com/pion/logging v0.2.4/go.mod h1:DffhXTKYdNZU+KtJ5pyQDjvOAh/GsNSyv1lbkFbe3so= +github.com/pion/mdns/v2 v2.2.1 h1:lQPWv6Mo5jG1sKwqkYgfeXGX7/seban4x7/j0n485/M= +github.com/pion/mdns/v2 v2.2.1/go.mod h1:ZX5f0AAH1D6TOjdjvcBORZZHaZuG0t9+br78lHEwiJ4= +github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA= +github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8= +github.com/pion/rtcp v1.2.18 h1:ZPgMZMCtBoReTixtFAyzcOkuJiebyB94qxtBAyvSooM= +github.com/pion/rtcp v1.2.18/go.mod h1:vX7Es6skWYm4oVvOSC5+4WHLYUTZHcbGNmk5wjzMGH4= +github.com/pion/rtp v1.10.5 h1:ip0HhO/wYZqQ4bKS+R99KnZh/GRCmIT0jDXikub7vlE= +github.com/pion/rtp v1.10.5/go.mod h1:Au8fc6cEByy8RLTwKTQTEeQqDB/SJDxwL4mZuxYA5Pk= +github.com/pion/sctp v1.11.3 h1:cFo/voNuKzdxEno6z96PWq/96xkAJeMB/eqB7GNfmlQ= +github.com/pion/sctp v1.11.3/go.mod h1:7v/MXGBROc8fu9g2TVwCXRasPISCMd3j8wSH+o45uCI= +github.com/pion/sdp/v3 v3.0.20 h1:TS6DViqcmp+49f0+mjw9anbr9xY3vJtsZewxAvlMCRQ= +github.com/pion/sdp/v3 v3.0.20/go.mod h1:slIMXDK5OKj0nhISwjfeN18AzTBCt2LYZq9uPw0cU5Q= +github.com/pion/srtp/v3 v3.1.0 h1:keugrBOxtvAv5Urf4lBMfso8OCAX3mm7Brdl26JSvik= +github.com/pion/srtp/v3 v3.1.0/go.mod h1:RlTlj08MtRReQiVm/PMmMFUDNjZkoKuKJMHkl9+M7a8= +github.com/pion/stun/v4 v4.0.1 h1:S2ggK4hsUJJKoZfpL90uIqIRTj6xJ0NOIEkXgcpQif0= +github.com/pion/stun/v4 v4.0.1/go.mod h1:byktbPA8U7HjJ2H8w3moP7nzq0Q+kYNulvMLPaubQL4= +github.com/pion/transport/v5 v5.1.1 h1:Sauq8YJqRopgRfIgnpfKpdpKG14g0N4jWlRgpWsEfN8= +github.com/pion/transport/v5 v5.1.1/go.mod h1:Qxw6fCEjFWQkRDZOhS4Vf+neJBcihauvA3uyEa1J1F0= +github.com/pion/turn/v5 v5.1.2 h1:acEOO+D5txJ52vKfcilmSMItfP23Q9K6pJ1LM5P2VV0= +github.com/pion/turn/v5 v5.1.2/go.mod h1:57Oadc8fHnac15VFyCd6nEaBTVZBxQe2POUMD4lsHRA= +github.com/pion/webrtc/v4 v4.2.22 h1:eoRTHzLKW5tI9o5iahw3s4KL4IHPESsfIF06zWi3JAo= +github.com/pion/webrtc/v4 v4.2.22/go.mod h1:JBu1ba8lUEX4jbYFKofglMQJG7f0rJk7UepKjI3GaLE= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= +github.com/wlynxg/anet v0.0.5 h1:J3VJGi1gvo0JwZ/P1/Yc/8p63SoW98B5dHkYDmpgvvU= +github.com/wlynxg/anet v0.0.5/go.mod h1:eay5PRQr7fIVAMbTbchTnO9gG65Hg/uYGdc7mguHxoA= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= +golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988= +golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc= +golang.org/x/net v0.54.0 h1:2zJIZAxAHV/OHCDTCOHAYehQzLfSXuf/5SoL/Dv6w/w= +golang.org/x/net v0.54.0/go.mod h1:Sj4oj8jK6XmHpBZU/zWHw3BV3abl4Kvi+Ut7cQcY+cQ= +golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= +golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= +golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= diff --git a/labs/hw/kvm/jetkvm-rpc/main.go b/labs/hw/kvm/jetkvm-rpc/main.go new file mode 100644 index 0000000..6356c51 --- /dev/null +++ b/labs/hw/kvm/jetkvm-rpc/main.go @@ -0,0 +1,174 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (c) 2026 FastNetMon (fastnetmon.com) + +// jetkvm-rpc runs JSON-RPC calls on a JetKVM: log in, open a WebRTC session +// through /webrtc/signaling/client and send each call on the "rpc" data channel. +package main + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/cookiejar" + "net/url" + "os" + "strings" + "time" + + "github.com/coder/websocket" + "github.com/coder/websocket/wsjson" + "github.com/pion/webrtc/v4" +) + +func die(f string, a ...any) { fmt.Fprintf(os.Stderr, "jetkvm-rpc: "+f+"\n", a...); os.Exit(1) } + +func main() { + if len(os.Args) < 3 { + die("usage: JETKVM_PASSWORD=... jetkvm-rpc [params-json] [ ...]") + } + host := os.Args[1] + type call struct{ method, params string } + var calls []call + for i := 2; i < len(os.Args); i += 2 { + c := call{os.Args[i], "{}"} + if i+1 < len(os.Args) { + c.params = os.Args[i+1] + } + if !json.Valid([]byte(c.params)) { + die("%s: params are not JSON: %s", c.method, c.params) + } + calls = append(calls, c) + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + jar, _ := cookiejar.New(nil) + hc := &http.Client{Jar: jar, Timeout: 10 * time.Second} + base := "http://" + host + body, _ := json.Marshal(map[string]string{"password": os.Getenv("JETKVM_PASSWORD")}) + resp, err := hc.Post(base+"/auth/login-local", "application/json", strings.NewReader(string(body))) + if err != nil { + die("login: %v", err) + } + io.Copy(io.Discard, resp.Body) + resp.Body.Close() + if resp.StatusCode != 200 { + die("login: HTTP %d", resp.StatusCode) + } + u, _ := url.Parse(base) + hdr := http.Header{} + for _, c := range jar.Cookies(u) { + hdr.Add("Cookie", c.Name+"="+c.Value) + } + ws, _, err := websocket.Dial(ctx, "ws://"+host+"/webrtc/signaling/client", &websocket.DialOptions{HTTPHeader: hdr}) + if err != nil { + die("signaling: %v", err) + } + defer ws.Close(websocket.StatusNormalClosure, "") + + pc, err := webrtc.NewPeerConnection(webrtc.Configuration{}) + if err != nil { + die("peer: %v", err) + } + defer pc.Close() + dc, err := pc.CreateDataChannel("rpc", nil) + if err != nil { + die("datachannel: %v", err) + } + result := make(chan json.RawMessage, 1) + next := 0 + send := func() { + req, _ := json.Marshal(map[string]any{"jsonrpc": "2.0", "id": next + 1, "method": calls[next].method, "params": json.RawMessage(calls[next].params)}) + dc.SendText(string(req)) + } + dc.OnOpen(send) + dc.OnMessage(func(m webrtc.DataChannelMessage) { + var r struct { + ID *int `json:"id"` + Result json.RawMessage `json:"result"` + Error json.RawMessage `json:"error"` + } + if json.Unmarshal(m.Data, &r) != nil || r.ID == nil || *r.ID != next+1 { + return + } + if len(r.Error) > 0 && string(r.Error) != "null" { + fmt.Fprintf(os.Stderr, "jetkvm-rpc: %s error: %s\n", calls[next].method, r.Error) + os.Exit(2) + } + fmt.Printf("%s: %s\n", calls[next].method, r.Result) + next++ + if next == len(calls) { + result <- r.Result + return + } + send() + }) + + offer, err := pc.CreateOffer(nil) + if err != nil { + die("offer: %v", err) + } + gathered := webrtc.GatheringCompletePromise(pc) + if err = pc.SetLocalDescription(offer); err != nil { + die("local description: %v", err) + } + <-gathered + ld, _ := json.Marshal(pc.LocalDescription()) + if err = wsjson.Write(ctx, ws, map[string]any{"type": "offer", "data": map[string]string{"sd": base64.StdEncoding.EncodeToString(ld)}}); err != nil { + die("send offer: %v", err) + } + + go func() { + for { + typ, msg, err := ws.Read(ctx) + if err != nil { + return + } + if typ != websocket.MessageText { + continue + } + var m struct { + Type string `json:"type"` + Data json.RawMessage `json:"data"` + Error json.RawMessage `json:"error"` + } + if json.Unmarshal(msg, &m) != nil { + continue + } + switch m.Type { + case "answer": + var s string + json.Unmarshal(m.Data, &s) + b, err := base64.StdEncoding.DecodeString(s) + if err != nil { + die("answer: %v", err) + } + var sd webrtc.SessionDescription + if err = json.Unmarshal(b, &sd); err != nil { + die("answer: %v", err) + } + if err = pc.SetRemoteDescription(sd); err != nil { + die("remote description: %v", err) + } + case "new-ice-candidate": + var c webrtc.ICECandidateInit + if json.Unmarshal(m.Data, &c) == nil && c.Candidate != "" { + pc.AddICECandidate(c) + } + default: + if len(m.Error) > 0 { + die("signaling error: %s", m.Error) + } + } + } + }() + + select { + case <-result: + case <-ctx.Done(): + die("%s: timed out", calls[next].method) + } +} diff --git a/labs/hw/lab.env.example b/labs/hw/lab.env.example index 91d1fe4..20c1c37 100644 --- a/labs/hw/lab.env.example +++ b/labs/hw/lab.env.example @@ -19,6 +19,10 @@ LAB_MAC_SENDER_alice="" LAB_MAC_RECEIVER_alice="" LAB_MAC_SENDER_bob="" LAB_MAC_RECEIVER_bob="" +LAB_KVM_HOST_alice="" +LAB_KVM_PASS_alice="" +LAB_KVM_HOST_bob="" +LAB_KVM_PASS_bob="" LAB_MAC_LEFT_server1="" LAB_MAC_RIGHT_server1="" LAB_IPMI_HOST_server1="" diff --git a/labs/hw/setup/ipmi.sh b/labs/hw/setup/ipmi.sh index f5ed9f6..cad3840 100755 --- a/labs/hw/setup/ipmi.sh +++ b/labs/hw/setup/ipmi.sh @@ -16,6 +16,15 @@ esac CMD="${1:-status}" +# Hosts without a BMC (alice, bob) are driven through their JetKVM instead, so +# every recovery path that calls this script reaches them unchanged. +if [ "$TARGET" = dut ] && [ -z "$DUT_IPMI_HOST" ] && [ -n "${DUT_KVM_HOST:-}" ]; then + case "$CMD" in + sol|sensors) echo "$CMD is not available for ${DUT_HOST%%.*}: use the JetKVM web UI"; exit 1 ;; + *) exec bash "$SCRIPT_DIR/kvm.sh" "$DUT" "$CMD" ;; + esac +fi + IPMI_PROXY="${IPMI_PROXY:-${LAB_PROXY:-}}" _ipmi() { diff --git a/labs/hw/setup/kvm.sh b/labs/hw/setup/kvm.sh new file mode 100755 index 0000000..5525778 --- /dev/null +++ b/labs/hw/setup/kvm.sh @@ -0,0 +1,120 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026 FastNetMon (fastnetmon.com) + +# Out-of-band control of a host through its JetKVM (alice, bob: no BMC). +# +# kvm.sh {status|reboot|sysrq-reboot|on|off|cycle|reset} +# +# reboot and sysrq-reboot type on the host's keyboard, so they need a live +# kernel: reboot sends Ctrl+Alt+Del, sysrq-reboot sends Alt+SysRq S, U, B for a +# host whose userspace hangs. on/off/cycle/reset press the motherboard buttons +# and need the JetKVM ATX extension wired and active; status shows whether it is. +# +# The JetKVM sits on the management LAN, so calls run on LAB_PROXY with the +# jetkvm-rpc client (labs/hw/kvm/jetkvm-rpc), built here for the proxy's +# architecture and copied over on first use. The password reaches the proxy on +# stdin only. Every call logs in afresh and takes over the KVM session, which +# signs out anyone using the JetKVM web UI at that moment. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +source "$SCRIPT_DIR/../vars.sh" + +NAME="${1:?usage: kvm.sh status|reboot|sysrq-reboot|on|off|cycle|reset}" +CMD="${2:-status}" +_h="LAB_KVM_HOST_$NAME"; _p="LAB_KVM_PASS_$NAME" +KVM_HOST="${!_h:-}"; KVM_PASS="${!_p:-}" +[ -n "$KVM_HOST" ] && [ -n "$KVM_PASS" ] || { echo "kvm.sh: no $_h / $_p in lab.env" >&2; exit 1; } +PROXY="${IPMI_PROXY:-${LAB_PROXY:-}}" +[ -n "$PROXY" ] || { echo "kvm.sh: LAB_PROXY is not set" >&2; exit 1; } +SSH="ssh $SSH_OPTS -o BatchMode=yes" +SRC="$SCRIPT_DIR/../kvm/jetkvm-rpc" +REMOTE_BIN='.cache/fastacl-testbench/jetkvm-rpc' + +ensure_client() { + local arch goarch bin + arch=$($SSH "$PROXY" uname -m) + case "$arch" in aarch64|arm64) goarch=arm64;; x86_64) goarch=amd64;; *) echo "kvm.sh: proxy arch $arch" >&2; return 1;; esac + bin="$SCRIPT_DIR/../kvm/bin/jetkvm-rpc-$goarch" + if [ ! -x "$bin" ] || [ -n "$(find "$SRC" -newer "$bin" -type f | head -1)" ]; then + mkdir -p "$(dirname "$bin")" + local log + log=$(docker run --rm -v "$SRC":/src:ro -v "$(dirname "$bin")":/out -w /src \ + -e GOARCH="$goarch" -e CGO_ENABLED=0 -e GOFLAGS=-modcacherw golang:1.24 \ + go build -trimpath -ldflags "-s -w" -o "/out/jetkvm-rpc-$goarch" . 2>&1) \ + || { echo "kvm.sh: building jetkvm-rpc failed:" >&2; echo "$log" >&2; return 1; } + fi + if [ "$($SSH "$PROXY" "sha256sum $REMOTE_BIN 2>/dev/null | cut -d' ' -f1")" != "$(sha256sum "$bin" | cut -d' ' -f1)" ]; then + $SSH "$PROXY" "mkdir -p \$(dirname $REMOTE_BIN)" + scp -q $SSH_OPTS "$bin" "$PROXY:$REMOTE_BIN" + fi +} + +rpc() { + local args="" a + for a in "$@"; do args="$args '$a'"; done + printf '%s\n' "$KVM_PASS" | $SSH "$PROXY" "read -r P; JETKVM_PASSWORD=\"\$P\" $REMOTE_BIN '$KVM_HOST'$args" +} + +key() { rpc keyboardReport "{\"modifier\":$1,\"keys\":[$2]}" keyboardReport '{"modifier":0,"keys":[]}' >/dev/null; } + +atx_active() { [ "$(rpc getActiveExtension | sed -n 's/^getActiveExtension: //p')" = '"atx-power"' ]; } +need_atx() { + atx_active && return 0 + echo "kvm.sh: $NAME's JetKVM has no active ATX extension -- it cannot press power/reset." >&2 + echo " Fit the ATX board and enable it, or use: kvm.sh $NAME reboot (needs a live kernel)." >&2 + exit 1 +} +powered() { rpc getATXState | grep -q '"power":true'; } +atx() { rpc setATXPowerAction "{\"action\":\"$1\"}" >/dev/null; } + +ensure_client +case "$CMD" in + status) + echo "=== $NAME JetKVM ($KVM_HOST) ===" + rpc getLocalVersion '{}' getActiveExtension '{}' getATXState '{}' | sed 's/^/ /' + atx_active || echo " (no ATX extension: power state above is not wired)" + ;; + reboot) + echo "=== $NAME: Ctrl+Alt+Del via JetKVM ===" + key 5 76 + ;; + sysrq-reboot) + echo "=== $NAME: Alt+SysRq S, U, B via JetKVM ===" + key 4 "70,22"; sleep 3 + key 4 "70,24"; sleep 3 + key 4 "70,5" + ;; + on) + need_atx + powered && { echo "$NAME is already on."; exit 0; } + atx power-short + ;; + off) + need_atx + powered || { echo "$NAME is already off."; exit 0; } + atx power-long + ;; + reset) + need_atx + atx reset + ;; + cycle) + need_atx + echo "=== $NAME hard power-cycle via JetKVM ATX ===" + powered && atx power-long + for _ in $(seq 1 20); do powered || break; sleep 2; done + sleep 5 + for _ in $(seq 1 3); do + atx power-short; sleep 6 + powered && { echo "Power-cycle complete. $NAME booting..."; exit 0; } + done + echo "WARNING: $NAME is still OFF after the cycle. Retry: $0 $NAME on" >&2 + exit 1 + ;; + *) + echo "Usage: $0 {status|reboot|sysrq-reboot|on|off|cycle|reset}" + exit 1 + ;; +esac diff --git a/labs/hw/setup/mellanox-init.sh b/labs/hw/setup/mellanox-init.sh index 6d0c8ce..cc46e81 100755 --- a/labs/hw/setup/mellanox-init.sh +++ b/labs/hw/setup/mellanox-init.sh @@ -84,7 +84,7 @@ fi if [ "$NEED_COLD_CYCLE" = "1" ]; then echo "mellanox-init: INTERNAL_CPU_MODEL changed -> COLD POWER CYCLE REQUIRED." echo " mlxfwreset does NOT activate this key. Run:" - echo " DUT=$DUT labs/hw/setup/ipmi.sh --target dut cycle" + echo " DUT=$DUT labs/hw/setup/ipmi.sh --target dut cycle (alice/bob: JetKVM, needs the ATX extension)" echo " then re-run this script to confirm the mode took effect." fi diff --git a/labs/hw/vars.sh b/labs/hw/vars.sh index c312703..defc410 100755 --- a/labs/hw/vars.sh +++ b/labs/hw/vars.sh @@ -40,6 +40,8 @@ case "$DUT" in DUT_LEFT_MAC_DEFAULT="${!_l:-}" DUT_RIGHT_MAC_DEFAULT="${!_r:-}" DUT_IPMI_HOST=""; DUT_IPMI_USER=""; DUT_IPMI_PASS="" + # No BMC: out-of-band control is the host's JetKVM (setup/kvm.sh). + _k="LAB_KVM_HOST_$DUT"; DUT_KVM_HOST="${!_k:-}" DUT_HUGEPAGES_1G=0 DUT_SKIP_OFED=1 ;;