feat(parakeet-cpp): VAD (Silero and the Moondream head), vad_model option and gallery entries

backend/go/parakeet-cpp/Makefile

@@ -1,6 +1,6 @@
 # parakeet-cpp backend Makefile.
 #
-# Upstream pin lives below as PARAKEET_VERSION?=11c1a0fc00556d390987e651217f9fc7caa4c4f7
+# Upstream pin lives below as PARAKEET_VERSION?=6165e3de4b10fd736ec6ec49bf6c5c26558de0dd
 # (.github/bump_deps.sh) can find and update it - matches the
 # whisper.cpp / ds4 / vibevoice-cpp convention.
 #
@@ -15,7 +15,7 @@
 # That's what the L0 smoke test uses. The default target below does the
 # proper clone-at-pin + cmake build so CI doesn't need a side-checkout.
 
-PARAKEET_VERSION?=11c1a0fc00556d390987e651217f9fc7caa4c4f7
+PARAKEET_VERSION?=6165e3de4b10fd736ec6ec49bf6c5c26558de0dd
 PARAKEET_REPO?=https://github.com/mudler/parakeet.cpp
 
 GOCMD?=go

backend/go/parakeet-cpp/goparakeetcpp.go

@@ -46,6 +46,21 @@ var (
 	// only in newer libparakeet.so (additive, no ABI bump); nil when absent.
 	CppTranscribePathJSONVad func(ctx uintptr, wavPath string, decoder int32) uintptr
 
+	// CppTranscribePathJSONVadWith is CppTranscribePathJSONVad with the speech
+	// probabilities taken from an external Silero VAD context (vadCtx), so an ASR
+	// model without a VAD head can cut long audio. vadCtx == 0 uses the model's
+	// own head. optionsJSON is "" for the defaults or a flat JSON object
+	// (threshold, min_pause, min_speech, max_segment). Additive; nil when absent.
+	CppTranscribePathJSONVadWith func(ctx, vadCtx uintptr, wavPath string, decoder int32, optionsJSON string) uintptr
+
+	// CppVadPcmJSON is the standalone VAD: it returns the speech regions of mono
+	// float PCM as a JSON document, for a Silero context or an ASR context with a
+	// VAD head. optionsJSON is "" for the defaults or a flat JSON object
+	// (threshold, min_pause, min_speech, speech_pad, max_segment, mode,
+	// probabilities). Returns 0 on error with the message in last_error.
+	// Additive; nil when absent.
+	CppVadPcmJSON func(ctx uintptr, samples []float32, nSamples int32, sampleRate int32, optionsJSON string) uintptr
+
 	// Batched JSON transcription: takes a concatenated float buffer of clips
 	// plus their per-clip sample counts (sum(nSamples)==len(samplesConcat))
 	// and returns a malloc'd char* JSON ARRAY of per-clip {"text","words",
@@ -250,6 +265,14 @@ type ParakeetCpp struct {
 	// point (vad:true model option). It bypasses the dynamic batcher, which has
 	// no VAD variant, and is not used for streaming.
 	vad bool
+	// vadCtx is the Silero VAD context: the primary when the model file is a
+	// Silero GGUF, or the vad_model: companion. 0 when none is loaded; the VAD
+	// RPC then falls back to the ASR context's own VAD head.
+	vadCtx uintptr
+	// vadOptions is the JSON object built from the vad_threshold, vad_min_pause,
+	// vad_min_speech, vad_speech_pad and vad_max_segment model options ("" when
+	// none is set, so the library picks the defaults of the detector in use).
+	vadOptions string
 }
 
 // Load is the LocalAI gRPC entry point for LoadModel: it calls
@@ -267,6 +290,19 @@ func (p *ParakeetCpp) Load(opts *pb.ModelOptions) error {
 	}
 	p.vad = vad
 
+	vadOpts, err := parseVADTuning(opts)
+	if err != nil {
+		return err
+	}
+	p.vadOptions = vadOpts
+	if optString(opts, "vad_model") != "" {
+		if CppTranscribePathJSONVadWith == nil {
+			return errors.New("parakeet-cpp: vad_model needs a libparakeet.so with parakeet_capi_transcribe_path_json_vad_with; rebuild the backend against a newer parakeet.cpp")
+		}
+		// vad_model implies vad: a Silero model is only useful to cut audio.
+		p.vad = true
+	}
+
 	if err := p.loadRoles(opts); err != nil {
 		return err
 	}
@@ -522,12 +558,18 @@ func (p *ParakeetCpp) AudioTranscription(ctx context.Context, opts *pb.Transcrip
 // It holds engineMu for the call: the engine is single-threaded and the batcher
 // is not involved on this path.
 func (p *ParakeetCpp) transcribePathDoc(path string) (transcriptJSON, error) {
-	fn, name := CppTranscribePathJSON, "transcribe_path_json"
-	if p.vad {
-		fn, name = CppTranscribePathJSONVad, "transcribe_path_json_vad"
+	call, name := func() uintptr { return CppTranscribePathJSON(p.ctxPtr, path, 0) }, "transcribe_path_json"
+	switch {
+	case p.vad && (p.vadCtx != 0 || p.vadOptions != "") && CppTranscribePathJSONVadWith != nil:
+		// An external Silero, or tuned segmenter options on the model's own head.
+		call, name = func() uintptr {
+			return CppTranscribePathJSONVadWith(p.ctxPtr, p.vadCtx, path, 0, p.vadOptions)
+		}, "transcribe_path_json_vad_with"
+	case p.vad:
+		call, name = func() uintptr { return CppTranscribePathJSONVad(p.ctxPtr, path, 0) }, "transcribe_path_json_vad"
 	}
 	p.engineMu.Lock()
-	cstr := fn(p.ctxPtr, path, 0)
+	cstr := call()
 	var lastErr string
 	if cstr == 0 {
 		lastErr = CppLastError(p.ctxPtr)
@@ -1069,7 +1111,7 @@ func (p *ParakeetCpp) Free() error {
 	// re-checks ctxPtr under the lock) can never feed into a freed ctx.
 	p.engineMu.Lock()
 	defer p.engineMu.Unlock()
-	for _, ctxField := range [...]*uintptr{&p.ctxPtr, &p.diarCtx, &p.tagCtx, &p.spkCtx} {
+	for _, ctxField := range [...]*uintptr{&p.ctxPtr, &p.diarCtx, &p.tagCtx, &p.spkCtx, &p.vadCtx} {
 		if *ctxField != 0 {
 			CppFree(*ctxField)
 			*ctxField = 0

backend/go/parakeet-cpp/main.go

@@ -78,6 +78,17 @@ func main() {
 		purego.RegisterLibFunc(&CppTranscribePathJSONVad, lib, "parakeet_capi_transcribe_path_json_vad")
 	}
 
+	// Silero VAD, the standalone VAD RPC and transcription with an external Silero
+	// (vad_model: option). Additive in the C-API (no ABI bump). Each symbol is
+	// probed on its own: a library without one still loads, and the feature that
+	// needs it fails with a clear message only when it is used.
+	if sym, err := purego.Dlsym(lib, "parakeet_capi_transcribe_path_json_vad_with"); err == nil && sym != 0 {
+		purego.RegisterLibFunc(&CppTranscribePathJSONVadWith, lib, "parakeet_capi_transcribe_path_json_vad_with")
+	}
+	if sym, err := purego.Dlsym(lib, "parakeet_capi_vad_pcm_json"); err == nil && sym != 0 {
+		purego.RegisterLibFunc(&CppVadPcmJSON, lib, "parakeet_capi_vad_pcm_json")
+	}
+
 	// Per-request language variants (multilingual nemotron). Same probe pattern:
 	// present only in libparakeet.so built with multilingual support, so the
 	// backend still loads against an older library and falls back to the

backend/go/parakeet-cpp/roles.go

@@ -18,6 +18,7 @@ const (
 	modelKindDiarization = 2
 	modelKindSound       = 3
 	modelKindSpeaker     = 4
+	modelKindVAD         = 5 // Silero VAD GGUF
 )
 
 // Diarization streaming latency modes (mirrors PARAKEET_DIAR_LATENCY_* in
@@ -41,6 +42,8 @@ func modelKindName(kind int32) string {
 		return "sound"
 	case modelKindSpeaker:
 		return "speaker"
+	case modelKindVAD:
+		return "VAD"
 	default:
 		return "unknown"
 	}
@@ -133,10 +136,11 @@ func (p *ParakeetCpp) loadRoles(opts *pb.ModelOptions) error {
 	asrModelOpt := optString(opts, "asr_model")
 	soundModelOpt := optString(opts, "sound_model")
 	speakerModelOpt := optString(opts, "speaker_model")
-	hasCompanionOpts := diarModelOpt != "" || asrModelOpt != "" || soundModelOpt != "" || speakerModelOpt != ""
+	vadModelOpt := optString(opts, "vad_model")
+	hasCompanionOpts := diarModelOpt != "" || asrModelOpt != "" || soundModelOpt != "" || speakerModelOpt != "" || vadModelOpt != ""
 
 	if hasCompanionOpts && CppModelKind == nil {
-		return errors.New("parakeet-cpp: asr_model/diarization_model/sound_model/speaker_model options need " +
+		return errors.New("parakeet-cpp: asr_model/diarization_model/sound_model/speaker_model/vad_model options need " +
 			"parakeet_capi_model_kind (ABI v8) to verify what they load; the loaded libparakeet.so " +
 			"is too old to report companion model roles")
 	}
@@ -184,7 +188,7 @@ func (p *ParakeetCpp) loadRoles(opts *pb.ModelOptions) error {
 		for _, c := range loaded {
 			CppFree(c)
 		}
-		p.ctxPtr, p.diarCtx, p.tagCtx, p.spkCtx = 0, 0, 0, 0
+		p.ctxPtr, p.diarCtx, p.tagCtx, p.spkCtx, p.vadCtx = 0, 0, 0, 0, 0
 		p.companions = nil
 	}
 
@@ -205,6 +209,8 @@ func (p *ParakeetCpp) loadRoles(opts *pb.ModelOptions) error {
 		freeLoaded()
 		return errors.New("parakeet-cpp: a speaker model cannot be the primary model; " +
 			"use it as speaker_model: next to a diarization model")
+	case modelKindVAD:
+		p.vadCtx = primary
 	default:
 		p.ctxPtr = primary
 	}
@@ -222,6 +228,9 @@ func (p *ParakeetCpp) loadRoles(opts *pb.ModelOptions) error {
 		{"speaker_model", speakerModelOpt, modelKindSpeaker,
 			func(pp *ParakeetCpp, c uintptr) { pp.spkCtx = c },
 			func(pp *ParakeetCpp) uintptr { return pp.spkCtx }},
+		{"vad_model", vadModelOpt, modelKindVAD,
+			func(pp *ParakeetCpp, c uintptr) { pp.vadCtx = c },
+			func(pp *ParakeetCpp) uintptr { return pp.vadCtx }},
 	}
 	for _, spec := range specs {
 		if spec.value == "" {
@@ -245,13 +254,18 @@ func (p *ParakeetCpp) loadRoles(opts *pb.ModelOptions) error {
 		loaded = append(loaded, cctx)
 		if gotKind := CppModelKind(cctx); gotKind != spec.wantKind {
 			freeLoaded()
-			return fmt.Errorf("parakeet-cpp: %s %q is a %s model, expected a %s model",
-				spec.optName, resolved, modelKindName(gotKind), modelKindName(spec.wantKind))
+			return fmt.Errorf("parakeet-cpp: %s %q is %s %s model, expected %s %s model",
+				spec.optName, resolved, indefiniteArticle(modelKindName(gotKind)), modelKindName(gotKind),
+				indefiniteArticle(modelKindName(spec.wantKind)), modelKindName(spec.wantKind))
 		}
 		spec.assign(p, cctx)
 		p.companions = append(p.companions, cctx)
 	}
 
+	if vadModelOpt != "" && p.ctxPtr == 0 {
+		freeLoaded()
+		return errors.New("parakeet-cpp: vad_model cuts audio for transcription and needs an ASR model (the primary or asr_model:)")
+	}
 	if p.spkCtx != 0 && p.diarCtx == 0 {
 		freeLoaded()
 		return errors.New("parakeet-cpp: speaker_model needs a diarization model (the primary or diarization_model:)")
@@ -275,6 +289,9 @@ func (p *ParakeetCpp) notASRError() error {
 	case p.tagCtx != 0:
 		return errors.New("parakeet-cpp: loaded model is a sound model, not ASR " +
 			"(use SoundDetection)")
+	case p.vadCtx != 0:
+		return errors.New("parakeet-cpp: loaded model is a Silero VAD model, not ASR " +
+			"(use the VAD endpoint, or load an ASR model with vad_model:)")
 	default:
 		return nil
 	}

backend/go/parakeet-cpp/vad.go

@@ -0,0 +1,115 @@
+package main
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"math"
+	"strconv"
+
+	pb "github.com/mudler/LocalAI/pkg/grpc/proto"
+)
+
+// vadSampleRate is the rate of VADRequest.audio: the VAD endpoint takes float32
+// PCM at 16 kHz, the rate the other VAD backends (silero-vad, whisper) assume.
+const vadSampleRate = 16000
+
+// vadTuning maps a model option to the key of the library's VAD options object.
+// All values are seconds except the threshold. An unset option leaves the
+// default of the detector in use (the VAD head and Silero differ), so no value
+// is repeated here.
+var vadTuning = []struct {
+	option, key string
+	// min and max bound the value; the library rejects the rest, but failing the
+	// load names the option instead of failing every request.
+	min, max float64
+	minOpen  bool // true when min itself is not allowed (the threshold is in (0, 1])
+}{
+	{"vad_threshold", "threshold", 0, 1, true},
+	{"vad_min_pause", "min_pause", 0, math.Inf(1), true},
+	{"vad_min_speech", "min_speech", 0, math.Inf(1), false},
+	{"vad_speech_pad", "speech_pad", 0, math.Inf(1), false},
+	{"vad_max_segment", "max_segment", 0, math.Inf(1), true},
+}
+
+// parseVADTuning reads the vad_threshold, vad_min_pause, vad_min_speech,
+// vad_speech_pad and vad_max_segment model options and returns them as the JSON
+// options object of the C-API, or "" when none is set. A value that does not
+// parse or is out of range fails the load.
+func parseVADTuning(opts *pb.ModelOptions) (string, error) {
+	obj := map[string]float64{}
+	for _, t := range vadTuning {
+		raw := optString(opts, t.option)
+		if raw == "" {
+			continue
+		}
+		v, err := strconv.ParseFloat(raw, 64)
+		if err != nil || math.IsNaN(v) || math.IsInf(v, 0) {
+			return "", fmt.Errorf("parakeet-cpp: option %s: %q is not a number", t.option, raw)
+		}
+		if v < t.min || (t.minOpen && v == t.min) || v > t.max {
+			return "", fmt.Errorf("parakeet-cpp: option %s: %v is out of range", t.option, v)
+		}
+		obj[t.key] = v
+	}
+	if len(obj) == 0 {
+		return "", nil
+	}
+	b, err := json.Marshal(obj)
+	if err != nil {
+		return "", err
+	}
+	return string(b), nil
+}
+
+// vadDocument is the part of the C-API VAD result the RPC needs.
+type vadDocument struct {
+	Segments []struct {
+		Start float64 `json:"start"`
+		End   float64 `json:"end"`
+	} `json:"segments"`
+}
+
+// VAD runs the standalone voice activity detection of libparakeet on the
+// request audio and returns the speech regions in seconds. It uses the Silero
+// context (the model itself or the vad_model: companion) when there is one, else
+// the VAD head of the ASR model. An ASR model without a head fails with the
+// library's message.
+func (p *ParakeetCpp) VAD(req *pb.VADRequest) (pb.VADResponse, error) {
+	if CppVadPcmJSON == nil {
+		return pb.VADResponse{}, errors.New("parakeet-cpp: VAD needs a libparakeet.so with parakeet_capi_vad_pcm_json; rebuild the backend against a newer parakeet.cpp")
+	}
+	p.engineMu.Lock()
+	ctx := p.vadCtx
+	if ctx == 0 {
+		ctx = p.ctxPtr
+	}
+	if ctx == 0 {
+		p.engineMu.Unlock()
+		if err := p.notASRError(); err != nil {
+			return pb.VADResponse{}, err
+		}
+		return pb.VADResponse{}, errors.New("parakeet-cpp: no model loaded for VAD")
+	}
+	cstr := CppVadPcmJSON(ctx, req.GetAudio(), int32(len(req.GetAudio())), vadSampleRate, p.vadOptions)
+	var lastErr string
+	if cstr == 0 {
+		lastErr = CppLastError(ctx)
+	}
+	p.engineMu.Unlock()
+	if cstr == 0 {
+		return pb.VADResponse{}, fmt.Errorf("parakeet-cpp: vad failed: %s", lastErr)
+	}
+	raw := goStringFromCPtr(cstr)
+	CppFreeString(cstr)
+
+	var doc vadDocument
+	if err := json.Unmarshal([]byte(raw), &doc); err != nil {
+		return pb.VADResponse{}, fmt.Errorf("parakeet-cpp: decode vad json: %w", err)
+	}
+	segs := make([]*pb.VADSegment, 0, len(doc.Segments))
+	for _, s := range doc.Segments {
+		segs = append(segs, &pb.VADSegment{Start: float32(s.Start), End: float32(s.End)})
+	}
+	return pb.VADResponse{Segments: segs}, nil
+}