Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -5,3 +5,4 @@ labs/hw/license/
bundle/
/results/
out/
labs/hw/kvm/bin/
7 changes: 7 additions & 0 deletions docs/lab.md
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,13 @@ The BMCs (IPMI, iDRAC) and two networked PDUs sit on the lab's management LAN an
only through rackpi. `labs/hw/bringup-guarded.sh` uses them to recover a DUT whose NIC wedged
during DPDK initialisation: first an IPMI power cycle, then a PDU power drain.

alice and bob have no BMC; each has a JetKVM on the same management LAN instead, driven by
`labs/hw/setup/kvm.sh <host> {status|reboot|sysrq-reboot|on|off|cycle|reset}` (and by
`setup/ipmi.sh --target dut` for those profiles). `reboot` (Ctrl+Alt+Del) and `sysrq-reboot`
type on the host's keyboard, so they need a live kernel. Power on/off/cycle/reset press the
motherboard buttons and need the JetKVM ATX extension, which is not fitted yet. Every call takes
over the KVM session, signing out anyone using its web UI.

## Cabling

```
Expand Down
10 changes: 5 additions & 5 deletions labs/hw/bringup-guarded.sh
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@ DUT_SEL="$DUT" # vars.sh profile name (server1|epyc) before DUT is reus
DUT="$LAB_SSH_USER@$DUT_HOST"
IPMI_PROXY="${IPMI_PROXY:-${LAB_PROXY:-}}"

ipmi() { $SSH "$IPMI_PROXY" "ipmitool -I lanplus -H '$DUT_IPMI_HOST' -U '$DUT_IPMI_USER' -P '$DUT_IPMI_PASS' $*"; }
dut_alive() { timeout 10 $SSH -o ConnectTimeout=6 "$DUT" 'echo ok' 2>/dev/null | grep -q ok; }
vpp_up() {
timeout 14 $SSH -o ConnectTimeout=8 "$DUT" \
Expand All @@ -28,12 +27,13 @@ n_dut_containers() {
'sg docker -c "docker ps -q --filter name=hw-dut" | wc -l' 2>/dev/null | tr -d '[:space:]'
}
recover() {
echo ">> WEDGED — auto-recovering via IPMI cold cycle..."
ipmi power off >/dev/null 2>&1; sleep 20; ipmi power on >/dev/null 2>&1
for _ in $(seq 1 18); do dut_alive && { echo ">> DUT back online."; return 0; }; sleep 15; done
echo ">> WEDGED — auto-recovering via an out-of-band cold cycle (IPMI, or JetKVM ATX)..."
if DUT="$DUT_SEL" bash "$SCRIPT_DIR/setup/ipmi.sh" --target dut cycle 2>&1 | sed 's/^/ /'; then
for _ in $(seq 1 18); do dut_alive && { echo ">> DUT back online."; return 0; }; sleep 15; done
fi

local dut_name="${DUT_HOST%%.*}"
echo ">> IPMI cycle did not recover it — escalating to a PDU AC-drain (cx7-recover $dut_name)..."
echo ">> Power cycle did not recover it — escalating to a PDU AC-drain (cx7-recover $dut_name)..."
if bash "$SCRIPT_DIR/setup/cx7-recover.sh" "$dut_name" --no-verify 2>&1 | sed 's/^/ /'; then
for _ in $(seq 1 18); do dut_alive && { echo ">> DUT back online after AC drain."; return 0; }; sleep 15; done
fi
Expand Down
3 changes: 2 additions & 1 deletion labs/hw/dut/start.sh
Original file line number Diff line number Diff line change
Expand Up @@ -206,7 +206,8 @@ if [ "$NIC_STUCK" = "1" ]; then
echo ""
echo "FATAL: CX5 NIC firmware stuck in pre-init state."
echo " Warm reset does NOT clear this. Fix: POWER CYCLE the server."
echo " Use: ./labs/hw/setup/ipmi.sh cycle (or power-off → power-on via IPMI)"
echo " Use: ./labs/hw/setup/ipmi.sh cycle (or power-off → power-on via IPMI;"
echo " alice/bob go through their JetKVM, which needs the ATX extension)"
exit 1
fi
fi
Expand Down
32 changes: 32 additions & 0 deletions labs/hw/kvm/jetkvm-rpc/go.mod
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
module jetkvm-rpc

go 1.25.0

require (
github.com/coder/websocket v1.8.15
github.com/pion/webrtc/v4 v4.2.22
)

require (
github.com/google/uuid v1.6.0 // indirect
github.com/pion/datachannel v1.6.3 // indirect
github.com/pion/dtls/v3 v3.1.9 // indirect
github.com/pion/ice/v4 v4.4.4 // indirect
github.com/pion/interceptor v0.1.49 // indirect
github.com/pion/logging v0.2.4 // indirect
github.com/pion/mdns/v2 v2.2.1 // indirect
github.com/pion/randutil v0.1.0 // indirect
github.com/pion/rtcp v1.2.18 // indirect
github.com/pion/rtp v1.10.5 // indirect
github.com/pion/sctp v1.11.3 // indirect
github.com/pion/sdp/v3 v3.0.20 // indirect
github.com/pion/srtp/v3 v3.1.0 // indirect
github.com/pion/stun/v4 v4.0.1 // indirect
github.com/pion/transport/v5 v5.1.1 // indirect
github.com/pion/turn/v5 v5.1.2 // indirect
github.com/wlynxg/anet v0.0.5 // indirect
golang.org/x/crypto v0.52.0 // indirect
golang.org/x/net v0.54.0 // indirect
golang.org/x/sys v0.45.0 // indirect
golang.org/x/time v0.14.0 // indirect
)
50 changes: 50 additions & 0 deletions labs/hw/kvm/jetkvm-rpc/go.sum
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
github.com/coder/websocket v1.8.15 h1:6B2JPeOGlpff2Uz6vOEH1Vzpi0iUz20A+lPVhPHtNUA=
github.com/coder/websocket v1.8.15/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/pion/datachannel v1.6.3 h1:HZAYLpunVI0xi057ZiANl558d0uSpomcWw4UEPz3qXs=
github.com/pion/datachannel v1.6.3/go.mod h1:oRmkXVVXxkUbqa8MatgdNHW29kvtQY3HUdjJQsfv2GE=
github.com/pion/dtls/v3 v3.1.9 h1:rpeycmLIkc4krpk1IxP7+39o11QdCXbmV9+FGi9yZJ8=
github.com/pion/dtls/v3 v3.1.9/go.mod h1:iKFQNYrjsN2TiA2YKKMqB9MOZaFpjFULBI/A4sW0eyc=
github.com/pion/ice/v4 v4.4.4 h1:nPhqLkf8NoE9O8NBKd3Xo2NNc2PlBMdM07tWZ+IH/i8=
github.com/pion/ice/v4 v4.4.4/go.mod h1:vkrbLwF98P/DJuCKoAmOQY4FJ7dJPI5VcBFOO3kyYx0=
github.com/pion/interceptor v0.1.49 h1:iyBsNHRoLNNXZiJA/DdGvJgdyfS8YWDCzxIAlVjt5nY=
github.com/pion/interceptor v0.1.49/go.mod h1:MZ6PJkja/TCo350HAnBrs/rUIyad9mWjpcvytrf3ViQ=
github.com/pion/logging v0.2.4 h1:tTew+7cmQ+Mc1pTBLKH2puKsOvhm32dROumOZ655zB8=
github.com/pion/logging v0.2.4/go.mod h1:DffhXTKYdNZU+KtJ5pyQDjvOAh/GsNSyv1lbkFbe3so=
github.com/pion/mdns/v2 v2.2.1 h1:lQPWv6Mo5jG1sKwqkYgfeXGX7/seban4x7/j0n485/M=
github.com/pion/mdns/v2 v2.2.1/go.mod h1:ZX5f0AAH1D6TOjdjvcBORZZHaZuG0t9+br78lHEwiJ4=
github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA=
github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8=
github.com/pion/rtcp v1.2.18 h1:ZPgMZMCtBoReTixtFAyzcOkuJiebyB94qxtBAyvSooM=
github.com/pion/rtcp v1.2.18/go.mod h1:vX7Es6skWYm4oVvOSC5+4WHLYUTZHcbGNmk5wjzMGH4=
github.com/pion/rtp v1.10.5 h1:ip0HhO/wYZqQ4bKS+R99KnZh/GRCmIT0jDXikub7vlE=
github.com/pion/rtp v1.10.5/go.mod h1:Au8fc6cEByy8RLTwKTQTEeQqDB/SJDxwL4mZuxYA5Pk=
github.com/pion/sctp v1.11.3 h1:cFo/voNuKzdxEno6z96PWq/96xkAJeMB/eqB7GNfmlQ=
github.com/pion/sctp v1.11.3/go.mod h1:7v/MXGBROc8fu9g2TVwCXRasPISCMd3j8wSH+o45uCI=
github.com/pion/sdp/v3 v3.0.20 h1:TS6DViqcmp+49f0+mjw9anbr9xY3vJtsZewxAvlMCRQ=
github.com/pion/sdp/v3 v3.0.20/go.mod h1:slIMXDK5OKj0nhISwjfeN18AzTBCt2LYZq9uPw0cU5Q=
github.com/pion/srtp/v3 v3.1.0 h1:keugrBOxtvAv5Urf4lBMfso8OCAX3mm7Brdl26JSvik=
github.com/pion/srtp/v3 v3.1.0/go.mod h1:RlTlj08MtRReQiVm/PMmMFUDNjZkoKuKJMHkl9+M7a8=
github.com/pion/stun/v4 v4.0.1 h1:S2ggK4hsUJJKoZfpL90uIqIRTj6xJ0NOIEkXgcpQif0=
github.com/pion/stun/v4 v4.0.1/go.mod h1:byktbPA8U7HjJ2H8w3moP7nzq0Q+kYNulvMLPaubQL4=
github.com/pion/transport/v5 v5.1.1 h1:Sauq8YJqRopgRfIgnpfKpdpKG14g0N4jWlRgpWsEfN8=
github.com/pion/transport/v5 v5.1.1/go.mod h1:Qxw6fCEjFWQkRDZOhS4Vf+neJBcihauvA3uyEa1J1F0=
github.com/pion/turn/v5 v5.1.2 h1:acEOO+D5txJ52vKfcilmSMItfP23Q9K6pJ1LM5P2VV0=
github.com/pion/turn/v5 v5.1.2/go.mod h1:57Oadc8fHnac15VFyCd6nEaBTVZBxQe2POUMD4lsHRA=
github.com/pion/webrtc/v4 v4.2.22 h1:eoRTHzLKW5tI9o5iahw3s4KL4IHPESsfIF06zWi3JAo=
github.com/pion/webrtc/v4 v4.2.22/go.mod h1:JBu1ba8lUEX4jbYFKofglMQJG7f0rJk7UepKjI3GaLE=
github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
github.com/wlynxg/anet v0.0.5 h1:J3VJGi1gvo0JwZ/P1/Yc/8p63SoW98B5dHkYDmpgvvU=
github.com/wlynxg/anet v0.0.5/go.mod h1:eay5PRQr7fIVAMbTbchTnO9gG65Hg/uYGdc7mguHxoA=
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988=
golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc=
golang.org/x/net v0.54.0 h1:2zJIZAxAHV/OHCDTCOHAYehQzLfSXuf/5SoL/Dv6w/w=
golang.org/x/net v0.54.0/go.mod h1:Sj4oj8jK6XmHpBZU/zWHw3BV3abl4Kvi+Ut7cQcY+cQ=
golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY=
golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
174 changes: 174 additions & 0 deletions labs/hw/kvm/jetkvm-rpc/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,174 @@
// SPDX-License-Identifier: Apache-2.0
// Copyright (c) 2026 FastNetMon (fastnetmon.com)

// jetkvm-rpc runs JSON-RPC calls on a JetKVM: log in, open a WebRTC session
// through /webrtc/signaling/client and send each call on the "rpc" data channel.
package main

import (
"context"
"encoding/base64"
"encoding/json"
"fmt"
"io"
"net/http"
"net/http/cookiejar"
"net/url"
"os"
"strings"
"time"

"github.com/coder/websocket"
"github.com/coder/websocket/wsjson"
"github.com/pion/webrtc/v4"
)

func die(f string, a ...any) { fmt.Fprintf(os.Stderr, "jetkvm-rpc: "+f+"\n", a...); os.Exit(1) }

func main() {
if len(os.Args) < 3 {
die("usage: JETKVM_PASSWORD=... jetkvm-rpc <host> <method> [params-json] [<method> <params-json> ...]")
}
host := os.Args[1]
type call struct{ method, params string }
var calls []call
for i := 2; i < len(os.Args); i += 2 {
c := call{os.Args[i], "{}"}
if i+1 < len(os.Args) {
c.params = os.Args[i+1]
}
if !json.Valid([]byte(c.params)) {
die("%s: params are not JSON: %s", c.method, c.params)
}
calls = append(calls, c)
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()

jar, _ := cookiejar.New(nil)
hc := &http.Client{Jar: jar, Timeout: 10 * time.Second}
base := "http://" + host
body, _ := json.Marshal(map[string]string{"password": os.Getenv("JETKVM_PASSWORD")})
resp, err := hc.Post(base+"/auth/login-local", "application/json", strings.NewReader(string(body)))
if err != nil {
die("login: %v", err)
}
io.Copy(io.Discard, resp.Body)
resp.Body.Close()
if resp.StatusCode != 200 {
die("login: HTTP %d", resp.StatusCode)
}
u, _ := url.Parse(base)
hdr := http.Header{}
for _, c := range jar.Cookies(u) {
hdr.Add("Cookie", c.Name+"="+c.Value)
}
ws, _, err := websocket.Dial(ctx, "ws://"+host+"/webrtc/signaling/client", &websocket.DialOptions{HTTPHeader: hdr})
if err != nil {
die("signaling: %v", err)
}
defer ws.Close(websocket.StatusNormalClosure, "")

pc, err := webrtc.NewPeerConnection(webrtc.Configuration{})
if err != nil {
die("peer: %v", err)
}
defer pc.Close()
dc, err := pc.CreateDataChannel("rpc", nil)
if err != nil {
die("datachannel: %v", err)
}
result := make(chan json.RawMessage, 1)
next := 0
send := func() {
req, _ := json.Marshal(map[string]any{"jsonrpc": "2.0", "id": next + 1, "method": calls[next].method, "params": json.RawMessage(calls[next].params)})
dc.SendText(string(req))
}
dc.OnOpen(send)
dc.OnMessage(func(m webrtc.DataChannelMessage) {
var r struct {
ID *int `json:"id"`
Result json.RawMessage `json:"result"`
Error json.RawMessage `json:"error"`
}
if json.Unmarshal(m.Data, &r) != nil || r.ID == nil || *r.ID != next+1 {
return
}
if len(r.Error) > 0 && string(r.Error) != "null" {
fmt.Fprintf(os.Stderr, "jetkvm-rpc: %s error: %s\n", calls[next].method, r.Error)
os.Exit(2)
}
fmt.Printf("%s: %s\n", calls[next].method, r.Result)
next++
if next == len(calls) {
result <- r.Result
return
}
send()
})

offer, err := pc.CreateOffer(nil)
if err != nil {
die("offer: %v", err)
}
gathered := webrtc.GatheringCompletePromise(pc)
if err = pc.SetLocalDescription(offer); err != nil {
die("local description: %v", err)
}
<-gathered
ld, _ := json.Marshal(pc.LocalDescription())
if err = wsjson.Write(ctx, ws, map[string]any{"type": "offer", "data": map[string]string{"sd": base64.StdEncoding.EncodeToString(ld)}}); err != nil {
die("send offer: %v", err)
}

go func() {
for {
typ, msg, err := ws.Read(ctx)
if err != nil {
return
}
if typ != websocket.MessageText {
continue
}
var m struct {
Type string `json:"type"`
Data json.RawMessage `json:"data"`
Error json.RawMessage `json:"error"`
}
if json.Unmarshal(msg, &m) != nil {
continue
}
switch m.Type {
case "answer":
var s string
json.Unmarshal(m.Data, &s)
b, err := base64.StdEncoding.DecodeString(s)
if err != nil {
die("answer: %v", err)
}
var sd webrtc.SessionDescription
if err = json.Unmarshal(b, &sd); err != nil {
die("answer: %v", err)
}
if err = pc.SetRemoteDescription(sd); err != nil {
die("remote description: %v", err)
}
case "new-ice-candidate":
var c webrtc.ICECandidateInit
if json.Unmarshal(m.Data, &c) == nil && c.Candidate != "" {
pc.AddICECandidate(c)
}
default:
if len(m.Error) > 0 {
die("signaling error: %s", m.Error)
}
}
}
}()

select {
case <-result:
case <-ctx.Done():
die("%s: timed out", calls[next].method)
}
}
4 changes: 4 additions & 0 deletions labs/hw/lab.env.example
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,10 @@ LAB_MAC_SENDER_alice="<mac_sender_alice>"
LAB_MAC_RECEIVER_alice="<mac_receiver_alice>"
LAB_MAC_SENDER_bob="<mac_sender_bob>"
LAB_MAC_RECEIVER_bob="<mac_receiver_bob>"
LAB_KVM_HOST_alice="<jetkvm_host_alice>"
LAB_KVM_PASS_alice="<jetkvm_pass_alice>"
LAB_KVM_HOST_bob="<jetkvm_host_bob>"
LAB_KVM_PASS_bob="<jetkvm_pass_bob>"
LAB_MAC_LEFT_server1="<mac_left_server1>"
LAB_MAC_RIGHT_server1="<mac_right_server1>"
LAB_IPMI_HOST_server1="<ipmi_host_server1>"
Expand Down
9 changes: 9 additions & 0 deletions labs/hw/setup/ipmi.sh
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,15 @@ esac

CMD="${1:-status}"

# Hosts without a BMC (alice, bob) are driven through their JetKVM instead, so
# every recovery path that calls this script reaches them unchanged.
if [ "$TARGET" = dut ] && [ -z "$DUT_IPMI_HOST" ] && [ -n "${DUT_KVM_HOST:-}" ]; then
case "$CMD" in
sol|sensors) echo "$CMD is not available for ${DUT_HOST%%.*}: use the JetKVM web UI"; exit 1 ;;
*) exec bash "$SCRIPT_DIR/kvm.sh" "$DUT" "$CMD" ;;
esac
fi

IPMI_PROXY="${IPMI_PROXY:-${LAB_PROXY:-}}"

_ipmi() {
Expand Down
Loading
Loading