diff --git a/backend/go/parakeet-cpp/Makefile b/backend/go/parakeet-cpp/Makefile index c813074a53db..da0584a3a1bd 100644 --- a/backend/go/parakeet-cpp/Makefile +++ b/backend/go/parakeet-cpp/Makefile @@ -1,6 +1,6 @@ # parakeet-cpp backend Makefile. # -# Upstream pin lives below as PARAKEET_VERSION?=11c1a0fc00556d390987e651217f9fc7caa4c4f7 +# Upstream pin lives below as PARAKEET_VERSION?=6165e3de4b10fd736ec6ec49bf6c5c26558de0dd # (.github/bump_deps.sh) can find and update it - matches the # whisper.cpp / ds4 / vibevoice-cpp convention. # @@ -15,7 +15,7 @@ # That's what the L0 smoke test uses. The default target below does the # proper clone-at-pin + cmake build so CI doesn't need a side-checkout. -PARAKEET_VERSION?=11c1a0fc00556d390987e651217f9fc7caa4c4f7 +PARAKEET_VERSION?=6165e3de4b10fd736ec6ec49bf6c5c26558de0dd PARAKEET_REPO?=https://github.com/mudler/parakeet.cpp GOCMD?=go diff --git a/backend/go/parakeet-cpp/goparakeetcpp.go b/backend/go/parakeet-cpp/goparakeetcpp.go index 97ea89edc828..931018326497 100644 --- a/backend/go/parakeet-cpp/goparakeetcpp.go +++ b/backend/go/parakeet-cpp/goparakeetcpp.go @@ -46,6 +46,21 @@ var ( // only in newer libparakeet.so (additive, no ABI bump); nil when absent. CppTranscribePathJSONVad func(ctx uintptr, wavPath string, decoder int32) uintptr + // CppTranscribePathJSONVadWith is CppTranscribePathJSONVad with the speech + // probabilities taken from an external Silero VAD context (vadCtx), so an ASR + // model without a VAD head can cut long audio. vadCtx == 0 uses the model's + // own head. optionsJSON is "" for the defaults or a flat JSON object + // (threshold, min_pause, min_speech, max_segment). Additive; nil when absent. + CppTranscribePathJSONVadWith func(ctx, vadCtx uintptr, wavPath string, decoder int32, optionsJSON string) uintptr + + // CppVadPcmJSON is the standalone VAD: it returns the speech regions of mono + // float PCM as a JSON document, for a Silero context or an ASR context with a + // VAD head. optionsJSON is "" for the defaults or a flat JSON object + // (threshold, min_pause, min_speech, speech_pad, max_segment, mode, + // probabilities). Returns 0 on error with the message in last_error. + // Additive; nil when absent. + CppVadPcmJSON func(ctx uintptr, samples []float32, nSamples int32, sampleRate int32, optionsJSON string) uintptr + // Batched JSON transcription: takes a concatenated float buffer of clips // plus their per-clip sample counts (sum(nSamples)==len(samplesConcat)) // and returns a malloc'd char* JSON ARRAY of per-clip {"text","words", @@ -250,6 +265,14 @@ type ParakeetCpp struct { // point (vad:true model option). It bypasses the dynamic batcher, which has // no VAD variant, and is not used for streaming. vad bool + // vadCtx is the Silero VAD context: the primary when the model file is a + // Silero GGUF, or the vad_model: companion. 0 when none is loaded; the VAD + // RPC then falls back to the ASR context's own VAD head. + vadCtx uintptr + // vadOptions is the JSON object built from the vad_threshold, vad_min_pause, + // vad_min_speech, vad_speech_pad and vad_max_segment model options ("" when + // none is set, so the library picks the defaults of the detector in use). + vadOptions string } // Load is the LocalAI gRPC entry point for LoadModel: it calls @@ -267,6 +290,19 @@ func (p *ParakeetCpp) Load(opts *pb.ModelOptions) error { } p.vad = vad + vadOpts, err := parseVADTuning(opts) + if err != nil { + return err + } + p.vadOptions = vadOpts + if optString(opts, "vad_model") != "" { + if CppTranscribePathJSONVadWith == nil { + return errors.New("parakeet-cpp: vad_model needs a libparakeet.so with parakeet_capi_transcribe_path_json_vad_with; rebuild the backend against a newer parakeet.cpp") + } + // vad_model implies vad: a Silero model is only useful to cut audio. + p.vad = true + } + if err := p.loadRoles(opts); err != nil { return err } @@ -522,12 +558,18 @@ func (p *ParakeetCpp) AudioTranscription(ctx context.Context, opts *pb.Transcrip // It holds engineMu for the call: the engine is single-threaded and the batcher // is not involved on this path. func (p *ParakeetCpp) transcribePathDoc(path string) (transcriptJSON, error) { - fn, name := CppTranscribePathJSON, "transcribe_path_json" - if p.vad { - fn, name = CppTranscribePathJSONVad, "transcribe_path_json_vad" + call, name := func() uintptr { return CppTranscribePathJSON(p.ctxPtr, path, 0) }, "transcribe_path_json" + switch { + case p.vad && (p.vadCtx != 0 || p.vadOptions != "") && CppTranscribePathJSONVadWith != nil: + // An external Silero, or tuned segmenter options on the model's own head. + call, name = func() uintptr { + return CppTranscribePathJSONVadWith(p.ctxPtr, p.vadCtx, path, 0, p.vadOptions) + }, "transcribe_path_json_vad_with" + case p.vad: + call, name = func() uintptr { return CppTranscribePathJSONVad(p.ctxPtr, path, 0) }, "transcribe_path_json_vad" } p.engineMu.Lock() - cstr := fn(p.ctxPtr, path, 0) + cstr := call() var lastErr string if cstr == 0 { lastErr = CppLastError(p.ctxPtr) @@ -1069,7 +1111,7 @@ func (p *ParakeetCpp) Free() error { // re-checks ctxPtr under the lock) can never feed into a freed ctx. p.engineMu.Lock() defer p.engineMu.Unlock() - for _, ctxField := range [...]*uintptr{&p.ctxPtr, &p.diarCtx, &p.tagCtx, &p.spkCtx} { + for _, ctxField := range [...]*uintptr{&p.ctxPtr, &p.diarCtx, &p.tagCtx, &p.spkCtx, &p.vadCtx} { if *ctxField != 0 { CppFree(*ctxField) *ctxField = 0 diff --git a/backend/go/parakeet-cpp/main.go b/backend/go/parakeet-cpp/main.go index 790af17b5c89..9022c0a9a357 100644 --- a/backend/go/parakeet-cpp/main.go +++ b/backend/go/parakeet-cpp/main.go @@ -78,6 +78,17 @@ func main() { purego.RegisterLibFunc(&CppTranscribePathJSONVad, lib, "parakeet_capi_transcribe_path_json_vad") } + // Silero VAD, the standalone VAD RPC and transcription with an external Silero + // (vad_model: option). Additive in the C-API (no ABI bump). Each symbol is + // probed on its own: a library without one still loads, and the feature that + // needs it fails with a clear message only when it is used. + if sym, err := purego.Dlsym(lib, "parakeet_capi_transcribe_path_json_vad_with"); err == nil && sym != 0 { + purego.RegisterLibFunc(&CppTranscribePathJSONVadWith, lib, "parakeet_capi_transcribe_path_json_vad_with") + } + if sym, err := purego.Dlsym(lib, "parakeet_capi_vad_pcm_json"); err == nil && sym != 0 { + purego.RegisterLibFunc(&CppVadPcmJSON, lib, "parakeet_capi_vad_pcm_json") + } + // Per-request language variants (multilingual nemotron). Same probe pattern: // present only in libparakeet.so built with multilingual support, so the // backend still loads against an older library and falls back to the diff --git a/backend/go/parakeet-cpp/roles.go b/backend/go/parakeet-cpp/roles.go index cecaae7411f9..42ff6353c644 100644 --- a/backend/go/parakeet-cpp/roles.go +++ b/backend/go/parakeet-cpp/roles.go @@ -18,6 +18,7 @@ const ( modelKindDiarization = 2 modelKindSound = 3 modelKindSpeaker = 4 + modelKindVAD = 5 // Silero VAD GGUF ) // Diarization streaming latency modes (mirrors PARAKEET_DIAR_LATENCY_* in @@ -41,6 +42,8 @@ func modelKindName(kind int32) string { return "sound" case modelKindSpeaker: return "speaker" + case modelKindVAD: + return "VAD" default: return "unknown" } @@ -133,10 +136,11 @@ func (p *ParakeetCpp) loadRoles(opts *pb.ModelOptions) error { asrModelOpt := optString(opts, "asr_model") soundModelOpt := optString(opts, "sound_model") speakerModelOpt := optString(opts, "speaker_model") - hasCompanionOpts := diarModelOpt != "" || asrModelOpt != "" || soundModelOpt != "" || speakerModelOpt != "" + vadModelOpt := optString(opts, "vad_model") + hasCompanionOpts := diarModelOpt != "" || asrModelOpt != "" || soundModelOpt != "" || speakerModelOpt != "" || vadModelOpt != "" if hasCompanionOpts && CppModelKind == nil { - return errors.New("parakeet-cpp: asr_model/diarization_model/sound_model/speaker_model options need " + + return errors.New("parakeet-cpp: asr_model/diarization_model/sound_model/speaker_model/vad_model options need " + "parakeet_capi_model_kind (ABI v8) to verify what they load; the loaded libparakeet.so " + "is too old to report companion model roles") } @@ -184,7 +188,7 @@ func (p *ParakeetCpp) loadRoles(opts *pb.ModelOptions) error { for _, c := range loaded { CppFree(c) } - p.ctxPtr, p.diarCtx, p.tagCtx, p.spkCtx = 0, 0, 0, 0 + p.ctxPtr, p.diarCtx, p.tagCtx, p.spkCtx, p.vadCtx = 0, 0, 0, 0, 0 p.companions = nil } @@ -205,6 +209,8 @@ func (p *ParakeetCpp) loadRoles(opts *pb.ModelOptions) error { freeLoaded() return errors.New("parakeet-cpp: a speaker model cannot be the primary model; " + "use it as speaker_model: next to a diarization model") + case modelKindVAD: + p.vadCtx = primary default: p.ctxPtr = primary } @@ -222,6 +228,9 @@ func (p *ParakeetCpp) loadRoles(opts *pb.ModelOptions) error { {"speaker_model", speakerModelOpt, modelKindSpeaker, func(pp *ParakeetCpp, c uintptr) { pp.spkCtx = c }, func(pp *ParakeetCpp) uintptr { return pp.spkCtx }}, + {"vad_model", vadModelOpt, modelKindVAD, + func(pp *ParakeetCpp, c uintptr) { pp.vadCtx = c }, + func(pp *ParakeetCpp) uintptr { return pp.vadCtx }}, } for _, spec := range specs { if spec.value == "" { @@ -245,13 +254,18 @@ func (p *ParakeetCpp) loadRoles(opts *pb.ModelOptions) error { loaded = append(loaded, cctx) if gotKind := CppModelKind(cctx); gotKind != spec.wantKind { freeLoaded() - return fmt.Errorf("parakeet-cpp: %s %q is a %s model, expected a %s model", - spec.optName, resolved, modelKindName(gotKind), modelKindName(spec.wantKind)) + return fmt.Errorf("parakeet-cpp: %s %q is %s %s model, expected %s %s model", + spec.optName, resolved, indefiniteArticle(modelKindName(gotKind)), modelKindName(gotKind), + indefiniteArticle(modelKindName(spec.wantKind)), modelKindName(spec.wantKind)) } spec.assign(p, cctx) p.companions = append(p.companions, cctx) } + if vadModelOpt != "" && p.ctxPtr == 0 { + freeLoaded() + return errors.New("parakeet-cpp: vad_model cuts audio for transcription and needs an ASR model (the primary or asr_model:)") + } if p.spkCtx != 0 && p.diarCtx == 0 { freeLoaded() return errors.New("parakeet-cpp: speaker_model needs a diarization model (the primary or diarization_model:)") @@ -275,6 +289,9 @@ func (p *ParakeetCpp) notASRError() error { case p.tagCtx != 0: return errors.New("parakeet-cpp: loaded model is a sound model, not ASR " + "(use SoundDetection)") + case p.vadCtx != 0: + return errors.New("parakeet-cpp: loaded model is a Silero VAD model, not ASR " + + "(use the VAD endpoint, or load an ASR model with vad_model:)") default: return nil } diff --git a/backend/go/parakeet-cpp/vad.go b/backend/go/parakeet-cpp/vad.go new file mode 100644 index 000000000000..ea1c48c5df2f --- /dev/null +++ b/backend/go/parakeet-cpp/vad.go @@ -0,0 +1,115 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "math" + "strconv" + + pb "github.com/mudler/LocalAI/pkg/grpc/proto" +) + +// vadSampleRate is the rate of VADRequest.audio: the VAD endpoint takes float32 +// PCM at 16 kHz, the rate the other VAD backends (silero-vad, whisper) assume. +const vadSampleRate = 16000 + +// vadTuning maps a model option to the key of the library's VAD options object. +// All values are seconds except the threshold. An unset option leaves the +// default of the detector in use (the VAD head and Silero differ), so no value +// is repeated here. +var vadTuning = []struct { + option, key string + // min and max bound the value; the library rejects the rest, but failing the + // load names the option instead of failing every request. + min, max float64 + minOpen bool // true when min itself is not allowed (the threshold is in (0, 1]) +}{ + {"vad_threshold", "threshold", 0, 1, true}, + {"vad_min_pause", "min_pause", 0, math.Inf(1), true}, + {"vad_min_speech", "min_speech", 0, math.Inf(1), false}, + {"vad_speech_pad", "speech_pad", 0, math.Inf(1), false}, + {"vad_max_segment", "max_segment", 0, math.Inf(1), true}, +} + +// parseVADTuning reads the vad_threshold, vad_min_pause, vad_min_speech, +// vad_speech_pad and vad_max_segment model options and returns them as the JSON +// options object of the C-API, or "" when none is set. A value that does not +// parse or is out of range fails the load. +func parseVADTuning(opts *pb.ModelOptions) (string, error) { + obj := map[string]float64{} + for _, t := range vadTuning { + raw := optString(opts, t.option) + if raw == "" { + continue + } + v, err := strconv.ParseFloat(raw, 64) + if err != nil || math.IsNaN(v) || math.IsInf(v, 0) { + return "", fmt.Errorf("parakeet-cpp: option %s: %q is not a number", t.option, raw) + } + if v < t.min || (t.minOpen && v == t.min) || v > t.max { + return "", fmt.Errorf("parakeet-cpp: option %s: %v is out of range", t.option, v) + } + obj[t.key] = v + } + if len(obj) == 0 { + return "", nil + } + b, err := json.Marshal(obj) + if err != nil { + return "", err + } + return string(b), nil +} + +// vadDocument is the part of the C-API VAD result the RPC needs. +type vadDocument struct { + Segments []struct { + Start float64 `json:"start"` + End float64 `json:"end"` + } `json:"segments"` +} + +// VAD runs the standalone voice activity detection of libparakeet on the +// request audio and returns the speech regions in seconds. It uses the Silero +// context (the model itself or the vad_model: companion) when there is one, else +// the VAD head of the ASR model. An ASR model without a head fails with the +// library's message. +func (p *ParakeetCpp) VAD(req *pb.VADRequest) (pb.VADResponse, error) { + if CppVadPcmJSON == nil { + return pb.VADResponse{}, errors.New("parakeet-cpp: VAD needs a libparakeet.so with parakeet_capi_vad_pcm_json; rebuild the backend against a newer parakeet.cpp") + } + p.engineMu.Lock() + ctx := p.vadCtx + if ctx == 0 { + ctx = p.ctxPtr + } + if ctx == 0 { + p.engineMu.Unlock() + if err := p.notASRError(); err != nil { + return pb.VADResponse{}, err + } + return pb.VADResponse{}, errors.New("parakeet-cpp: no model loaded for VAD") + } + cstr := CppVadPcmJSON(ctx, req.GetAudio(), int32(len(req.GetAudio())), vadSampleRate, p.vadOptions) + var lastErr string + if cstr == 0 { + lastErr = CppLastError(ctx) + } + p.engineMu.Unlock() + if cstr == 0 { + return pb.VADResponse{}, fmt.Errorf("parakeet-cpp: vad failed: %s", lastErr) + } + raw := goStringFromCPtr(cstr) + CppFreeString(cstr) + + var doc vadDocument + if err := json.Unmarshal([]byte(raw), &doc); err != nil { + return pb.VADResponse{}, fmt.Errorf("parakeet-cpp: decode vad json: %w", err) + } + segs := make([]*pb.VADSegment, 0, len(doc.Segments)) + for _, s := range doc.Segments { + segs = append(segs, &pb.VADSegment{Start: float32(s.Start), End: float32(s.End)}) + } + return pb.VADResponse{Segments: segs}, nil +} diff --git a/backend/go/parakeet-cpp/vad_rpc_test.go b/backend/go/parakeet-cpp/vad_rpc_test.go new file mode 100644 index 000000000000..d2cf827ab77d --- /dev/null +++ b/backend/go/parakeet-cpp/vad_rpc_test.go @@ -0,0 +1,309 @@ +package main + +import ( + "context" + "path/filepath" + "sync" + + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// The VAD RPC and vad_model specs run against stubbed C entry points, so they +// need no libparakeet.so. + +var _ = Describe("VAD RPC", func() { + var ( + savedPcm func(uintptr, []float32, int32, int32, string) uintptr + savedFree func(uintptr) + savedLastError func(uintptr) string + pool *diarizeCstrPool + ) + + BeforeEach(func() { + savedPcm, savedFree, savedLastError = CppVadPcmJSON, CppFreeString, CppLastError + pool = &diarizeCstrPool{} + CppFreeString = func(uintptr) {} + }) + AfterEach(func() { + CppVadPcmJSON, CppFreeString, CppLastError = savedPcm, savedFree, savedLastError + }) + + It("maps the library segments to the response in seconds", func() { + var gotCtx uintptr + var gotN, gotRate int32 + var gotOpts string + CppVadPcmJSON = func(ctx uintptr, s []float32, n, rate int32, o string) uintptr { + gotCtx, gotN, gotRate, gotOpts = ctx, n, rate, o + return pool.cstr(`{"mode":"speech","duration":3.0,"frame_sec":0.032,"backend":"cpu",` + + `"segments":[{"start":0.514,"end":1.5},{"start":2.0,"end":2.75}]}`) + } + p := &ParakeetCpp{ctxPtr: 7} + res, err := p.VAD(&pb.VADRequest{Audio: make([]float32, 480)}) + Expect(err).ToNot(HaveOccurred()) + Expect(gotCtx).To(Equal(uintptr(7))) + Expect(gotN).To(Equal(int32(480))) + Expect(gotRate).To(Equal(int32(16000))) + Expect(gotOpts).To(BeEmpty()) + Expect(res.Segments).To(HaveLen(2)) + Expect(res.Segments[0].Start).To(BeNumerically("~", 0.514, 1e-6)) + Expect(res.Segments[1].End).To(BeNumerically("~", 2.75, 1e-6)) + }) + + It("returns an empty, non-nil segment list when nothing is speech", func() { + CppVadPcmJSON = func(uintptr, []float32, int32, int32, string) uintptr { + return pool.cstr(`{"segments":[]}`) + } + res, err := (&ParakeetCpp{ctxPtr: 7}).VAD(&pb.VADRequest{}) + Expect(err).ToNot(HaveOccurred()) + Expect(res.Segments).ToNot(BeNil()) + Expect(res.Segments).To(BeEmpty()) + }) + + It("prefers the Silero context over the ASR head", func() { + var gotCtx uintptr + CppVadPcmJSON = func(ctx uintptr, _ []float32, _, _ int32, _ string) uintptr { + gotCtx = ctx + return pool.cstr(`{"segments":[]}`) + } + _, err := (&ParakeetCpp{ctxPtr: 7, vadCtx: 9}).VAD(&pb.VADRequest{}) + Expect(err).ToNot(HaveOccurred()) + Expect(gotCtx).To(Equal(uintptr(9))) + }) + + It("works on a Silero primary that has no ASR context", func() { + CppVadPcmJSON = func(ctx uintptr, _ []float32, _, _ int32, _ string) uintptr { + Expect(ctx).To(Equal(uintptr(9))) + return pool.cstr(`{"segments":[{"start":1,"end":2}]}`) + } + res, err := (&ParakeetCpp{vadCtx: 9}).VAD(&pb.VADRequest{}) + Expect(err).ToNot(HaveOccurred()) + Expect(res.Segments).To(HaveLen(1)) + }) + + It("passes the tuning options to the library", func() { + var gotOpts string + CppVadPcmJSON = func(_ uintptr, _ []float32, _, _ int32, o string) uintptr { + gotOpts = o + return pool.cstr(`{"segments":[]}`) + } + p := &ParakeetCpp{ctxPtr: 7, vadOptions: `{"threshold":0.6}`} + _, err := p.VAD(&pb.VADRequest{}) + Expect(err).ToNot(HaveOccurred()) + Expect(gotOpts).To(Equal(`{"threshold":0.6}`)) + }) + + It("surfaces the library message for a model without a VAD head", func() { + CppVadPcmJSON = func(uintptr, []float32, int32, int32, string) uintptr { return 0 } + CppLastError = func(uintptr) string { return "model has no VAD head" } + _, err := (&ParakeetCpp{ctxPtr: 7}).VAD(&pb.VADRequest{Audio: []float32{0}}) + Expect(err).To(MatchError("parakeet-cpp: vad failed: model has no VAD head")) + }) + + It("gives a clear error only when the symbol is missing and VAD is used", func() { + CppVadPcmJSON = nil + _, err := (&ParakeetCpp{ctxPtr: 7}).VAD(&pb.VADRequest{}) + Expect(err).To(MatchError(ContainSubstring("parakeet_capi_vad_pcm_json"))) + }) + + It("names the role when a diarization model is loaded", func() { + CppVadPcmJSON = func(uintptr, []float32, int32, int32, string) uintptr { + Fail("no C call expected") + return 0 + } + _, err := (&ParakeetCpp{diarCtx: 3}).VAD(&pb.VADRequest{}) + Expect(err).To(MatchError(ContainSubstring("diarization model"))) + }) + + It("reports no model when nothing is loaded", func() { + CppVadPcmJSON = func(uintptr, []float32, int32, int32, string) uintptr { return 0 } + _, err := (&ParakeetCpp{}).VAD(&pb.VADRequest{}) + Expect(err).To(MatchError(ContainSubstring("no model loaded"))) + }) + + It("serializes concurrent requests on the engine", func() { + var mu sync.Mutex + active, maxActive := 0, 0 + CppVadPcmJSON = func(uintptr, []float32, int32, int32, string) uintptr { + mu.Lock() + active++ + if active > maxActive { + maxActive = active + } + mu.Unlock() + mu.Lock() + active-- + mu.Unlock() + return pool.cstr(`{"segments":[]}`) + } + p := &ParakeetCpp{ctxPtr: 7} + var wg sync.WaitGroup + for i := 0; i < 4; i++ { + wg.Add(1) + go func() { + defer wg.Done() + defer GinkgoRecover() + _, err := p.VAD(&pb.VADRequest{Audio: []float32{0}}) + Expect(err).ToNot(HaveOccurred()) + }() + } + wg.Wait() + Expect(maxActive).To(Equal(1)) + }) +}) + +var _ = Describe("VAD tuning options", func() { + opts := func(o ...string) *pb.ModelOptions { return &pb.ModelOptions{Options: o} } + + It("is empty when no option is set", func() { + s, err := parseVADTuning(opts("vad:true")) + Expect(err).ToNot(HaveOccurred()) + Expect(s).To(BeEmpty()) + }) + + It("maps each option to the library key", func() { + s, err := parseVADTuning(opts("vad_threshold:0.6", "vad_min_pause:0.3", "vad_min_speech:0.2", + "vad_speech_pad:0.05", "vad_max_segment:20")) + Expect(err).ToNot(HaveOccurred()) + Expect(s).To(MatchJSON(`{"threshold":0.6,"min_pause":0.3,"min_speech":0.2,"speech_pad":0.05,"max_segment":20}`)) + }) + + It("allows a zero speech pad", func() { + s, err := parseVADTuning(opts("vad_speech_pad:0")) + Expect(err).ToNot(HaveOccurred()) + Expect(s).To(MatchJSON(`{"speech_pad":0}`)) + }) + + DescribeTable("rejects bad values at load", + func(opt, msg string) { + _, err := parseVADTuning(opts(opt)) + Expect(err).To(MatchError(ContainSubstring(msg))) + }, + Entry("not a number", "vad_threshold:high", "is not a number"), + Entry("threshold zero", "vad_threshold:0", "out of range"), + Entry("threshold above one", "vad_threshold:1.5", "out of range"), + Entry("negative pad", "vad_speech_pad:-1", "out of range"), + Entry("NaN", "vad_min_pause:NaN", "is not a number"), + ) +}) + +var _ = Describe("vad_model", func() { + var ( + restore func() + savedWith func(ctx, vadCtx uintptr, p string, d int32, o string) uintptr + savedFree func(uintptr) + savedLastError func(uintptr) string + savedVad, savedPlain func(uintptr, string, int32) uintptr + pool *diarizeCstrPool + gotCtx, gotVad uintptr + gotOpts, gotPath string + calledWith, calledHead bool + ) + + BeforeEach(func() { + savedWith, savedFree, savedLastError = CppTranscribePathJSONVadWith, CppFreeString, CppLastError + savedVad, savedPlain = CppTranscribePathJSONVad, CppTranscribePathJSON + pool = &diarizeCstrPool{} + CppFreeString = func(uintptr) {} + calledWith, calledHead = false, false + CppTranscribePathJSONVadWith = func(ctx, vad uintptr, path string, _ int32, o string) uintptr { + calledWith = true + gotCtx, gotVad, gotPath, gotOpts = ctx, vad, path, o + return pool.cstr(`{"text":"hi.","frame_sec":0.08,"words":[],"tokens":[]}`) + } + CppTranscribePathJSONVad = func(uintptr, string, int32) uintptr { + calledHead = true + return pool.cstr(`{"text":"head.","frame_sec":0.08,"words":[],"tokens":[]}`) + } + }) + AfterEach(func() { + if restore != nil { + restore() + restore = nil + } + CppTranscribePathJSONVadWith, CppFreeString, CppLastError = savedWith, savedFree, savedLastError + CppTranscribePathJSONVad, CppTranscribePathJSON = savedVad, savedPlain + }) + + load := func(f *fakeLib, o *pb.ModelOptions) (*ParakeetCpp, error) { + restore = f.install() + p := &ParakeetCpp{} + return p, p.Load(o) + } + + It("loads a Silero companion resolved against the models root, and implies vad", func() { + f := newFakeLib(). + withModel("asr.gguf", modelKindASR). + withModel(filepath.Join("/models", "silero.gguf"), modelKindVAD) + p, err := load(f, &pb.ModelOptions{ModelFile: "asr.gguf", ModelPath: "/models", + Options: []string{"vad_model:silero.gguf", "vad_threshold:0.4"}}) + Expect(err).ToNot(HaveOccurred()) + Expect(p.vad).To(BeTrue()) + Expect(p.vadCtx).ToNot(BeZero()) + Expect(p.ctxPtr).ToNot(BeZero()) + Expect(p.vadOptions).To(MatchJSON(`{"threshold":0.4}`)) + Expect(p.Free()).To(Succeed()) + Expect(f.freed).To(HaveLen(2)) + Expect(p.vadCtx).To(BeZero()) + }) + + It("routes transcription through the _with entry point with the Silero context", func() { + p := &ParakeetCpp{ctxPtr: 7, vadCtx: 9, vad: true, vadOptions: `{"threshold":0.4}`} + doc, err := p.transcribePathDoc("/x/long.wav") + Expect(err).ToNot(HaveOccurred()) + Expect(doc.Text).To(Equal("hi.")) + Expect(calledWith).To(BeTrue()) + Expect(calledHead).To(BeFalse()) + Expect([]any{gotCtx, gotVad, gotPath, gotOpts}).To(Equal([]any{uintptr(7), uintptr(9), "/x/long.wav", `{"threshold":0.4}`})) + }) + + It("keeps vad:true on the model's own head when there is no vad_model and no tuning", func() { + p := &ParakeetCpp{ctxPtr: 7, vad: true} + _, err := p.transcribePathDoc("/x/long.wav") + Expect(err).ToNot(HaveOccurred()) + Expect(calledHead).To(BeTrue()) + Expect(calledWith).To(BeFalse()) + }) + + It("passes tuning to the head through _with (null Silero context)", func() { + p := &ParakeetCpp{ctxPtr: 7, vad: true, vadOptions: `{"max_segment":20}`} + _, err := p.transcribePathDoc("/x/long.wav") + Expect(err).ToNot(HaveOccurred()) + Expect(calledWith).To(BeTrue()) + Expect(gotVad).To(BeZero()) + }) + + It("refuses vad_model on a library without the _with entry point", func() { + CppTranscribePathJSONVadWith = nil + f := newFakeLib().withModel("asr.gguf", modelKindASR).withModel("s.gguf", modelKindVAD) + _, err := load(f, &pb.ModelOptions{ModelFile: "asr.gguf", Options: []string{"vad_model:s.gguf"}}) + Expect(err).To(MatchError(ContainSubstring("parakeet_capi_transcribe_path_json_vad_with"))) + }) + + It("rejects a vad_model that is not a Silero model", func() { + f := newFakeLib().withModel("asr.gguf", modelKindASR).withModel("other.gguf", modelKindASR) + _, err := load(f, &pb.ModelOptions{ModelFile: "asr.gguf", Options: []string{"vad_model:other.gguf"}}) + Expect(err).To(MatchError(ContainSubstring("is an ASR model, expected a VAD model"))) + Expect(f.freed).To(HaveLen(2)) + }) + + It("rejects vad_model without an ASR model", func() { + f := newFakeLib().withModel("diar.gguf", modelKindDiarization).withModel("s.gguf", modelKindVAD) + _, err := load(f, &pb.ModelOptions{ModelFile: "diar.gguf", Options: []string{"vad_model:s.gguf"}}) + Expect(err).To(MatchError(ContainSubstring("needs an ASR model"))) + }) + + It("loads a Silero GGUF as the primary and refuses transcription with a clear error", func() { + f := newFakeLib().withModel("silero.gguf", modelKindVAD) + p, err := load(f, &pb.ModelOptions{ModelFile: "silero.gguf"}) + Expect(err).ToNot(HaveOccurred()) + Expect(p.vadCtx).ToNot(BeZero()) + Expect(p.ctxPtr).To(BeZero()) + Expect(p.vad).To(BeFalse()) + _, err = p.AudioTranscription(context.Background(), &pb.TranscriptRequest{Dst: "x.wav"}) + Expect(err).To(MatchError(ContainSubstring("Silero VAD model, not ASR"))) + Expect(p.Free()).To(Succeed()) + Expect(f.freed).To(HaveLen(1)) + }) +}) diff --git a/core/gallery/parakeet_vad_entries_test.go b/core/gallery/parakeet_vad_entries_test.go new file mode 100644 index 000000000000..cdf24f725737 --- /dev/null +++ b/core/gallery/parakeet_vad_entries_test.go @@ -0,0 +1,94 @@ +package gallery_test + +import ( + "path/filepath" + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/gallery" +) + +// The parakeet-cpp VAD entries serve the VAD endpoint through known_usecases +// and the vad_model option points at a file the same entry downloads. A typo in +// either one installs cleanly and fails at the first request. +var _ = Describe("gallery/index.yaml parakeet-cpp VAD entries", func() { + byName := func() map[string]gallery.GalleryModel { + entries, err := loadGalleryIndex() + Expect(err).ToNot(HaveOccurred()) + m := map[string]gallery.GalleryModel{} + for _, e := range entries { + m[e.Name] = e + } + return m + } + filenames := func(e gallery.GalleryModel) []string { + var out []string + for _, f := range e.AdditionalFiles { + out = append(out, f.Filename) + } + return out + } + + It("declares the VAD usecase and an existing model file on each VAD-only entry", func() { + entries := byName() + for _, name := range []string{ + "parakeet-cpp-silero-vad-f16", + "parakeet-cpp-vad-moondream-redux-packed", + "parakeet-cpp-vad-moondream-ultra-q8_0", + } { + e, ok := entries[name] + Expect(ok).To(BeTrue(), name) + Expect(e.Overrides["backend"]).To(Equal("parakeet-cpp"), name) + Expect(e.Overrides["known_usecases"]).To(ConsistOf("vad"), name) + params, _ := e.Overrides["parameters"].(map[string]any) + Expect(filenames(e)).To(ContainElement(params["model"]), name) + for _, f := range e.AdditionalFiles { + Expect(f.SHA256).To(HaveLen(64), name) + } + } + }) + + It("shares the VAD head files with the matching ASR entries", func() { + entries := byName() + pairs := map[string]string{ + "parakeet-cpp-vad-moondream-redux-packed": "parakeet-cpp-moondream-redux-packed", + "parakeet-cpp-vad-moondream-ultra-q8_0": "parakeet-cpp-moondream-ultra-q8_0", + } + for vad, asr := range pairs { + Expect(entries[vad].AdditionalFiles[0].Filename).To(Equal(entries[asr].AdditionalFiles[0].Filename)) + Expect(entries[vad].AdditionalFiles[0].SHA256).To(Equal(entries[asr].AdditionalFiles[0].SHA256)) + } + }) + + It("installs Silero from the parakeet-cpp-vad entry and declares no variants", func() { + // Variant ranking prefers the larger build that fits, and these are + // different detectors, so the entry must not offer a choice. + meta, ok := byName()["parakeet-cpp-vad"] + Expect(ok).To(BeTrue()) + Expect(meta.Variants).To(BeEmpty()) + Expect(filenames(meta)).To(ConsistOf("parakeet-cpp/silero-vad-f16.gguf")) + Expect(meta.Overrides["parameters"]).To(HaveKeyWithValue("model", "parakeet-cpp/silero-vad-f16.gguf")) + }) + + It("downloads the Silero file that the vad_model option of the v3 entry names", func() { + e, ok := byName()["parakeet-cpp-tdt-0.6b-v3-silero-vad"] + Expect(ok).To(BeTrue()) + opts, _ := e.Overrides["options"].([]any) + var vadModel string + for _, o := range opts { + if s, ok := o.(string); ok && strings.HasPrefix(s, "vad_model:") { + vadModel = strings.TrimPrefix(s, "vad_model:") + } + } + Expect(vadModel).ToNot(BeEmpty()) + Expect(filenames(e)).To(ContainElement(filepath.ToSlash(vadModel))) + }) + + It("leaves the existing Silero entries on their own backends", func() { + entries := byName() + Expect(entries["silero-vad-ggml"].Overrides["backend"]).To(Equal("whisper")) + Expect(entries["silero-vad"].Overrides["backend"]).To(Equal("silero-vad")) + }) +}) diff --git a/docs/content/features/audio-to-text.md b/docs/content/features/audio-to-text.md index 698418a0778a..b5a363afdf6c 100644 --- a/docs/content/features/audio-to-text.md +++ b/docs/content/features/audio-to-text.md @@ -270,6 +270,33 @@ options: `vad:true` applies to offline transcription only and bypasses dynamic batching, because the batched entry point has no VAD variant. Streaming is not affected. A model without a VAD head fails each request with `model has no VAD head`, and a `libparakeet.so` that is too old to export the VAD entry point fails the load. Remove the option for models that have no VAD head. +### Cutting long audio with Silero (`vad_model`) + +A model without a VAD head, such as `parakeet-cpp-tdt-0.6b-v3` or a Nemotron model, can cut long audio with [Silero VAD](https://github.com/snakers4/silero-vad) instead. Name a Silero GGUF in the `vad_model` option. The path is resolved against the models directory, like the other companion files. `vad_model` implies `vad`: + +```yaml +name: parakeet-v3-silero +backend: parakeet-cpp +parameters: + model: parakeet-cpp/tdt-0.6b-v3-f16.gguf +options: +- vad_model:parakeet-cpp/silero-vad-f16.gguf # Silero GGUF that cuts long audio at pauses +- vad_min_pause:0.3 # optional, seconds +``` + +The gallery entry `parakeet-cpp-tdt-0.6b-v3-silero-vad` installs both files with this configuration. Audio of 30 seconds or less is transcribed whole and the VAD does not run. `vad:true` alone keeps meaning "use the model's own head". With `vad_model` set, the Silero model is used even if the ASR model has a head. + +The segmenter options below apply to both `vad:true` and `vad_model`. Each is optional; an unset value keeps the default of the detector in use, and a bad value fails the load: + +| Option | Unit | Meaning | +|---|---|---| +| `vad_threshold` | 0 to 1 | A frame is speech when its probability is at least this | +| `vad_min_pause` | seconds | A silence this long separates two pieces | +| `vad_min_speech` | seconds | Shorter speech runs are dropped | +| `vad_max_segment` | seconds | Cap on the length of a piece (default 30) | + +`vad_speech_pad` (seconds) pads each region and only affects the [VAD endpoint]({{%relref "features/voice-activity-detection" %}}). `vad_model` needs a `libparakeet.so` that exports `parakeet_capi_transcribe_path_json_vad_with`; an older library fails the load with a message that names it. + ## See also - [Audio Transform]({{< relref "audio-transform.md" >}}) - clean up the audio (echo cancellation, noise suppression, dereverberation) before passing it to a transcription model. diff --git a/docs/content/features/voice-activity-detection.md b/docs/content/features/voice-activity-detection.md index 63f516f4fb9a..b868c7faa1d1 100644 --- a/docs/content/features/voice-activity-detection.md +++ b/docs/content/features/voice-activity-detection.md @@ -117,6 +117,38 @@ Malformed values, negative durations, and NaN thresholds are ignored; the defaul Reload the model (or restart LocalAI) after changing these options. +## parakeet-cpp backend + +The `parakeet-cpp` backend serves the same endpoint. It runs one of two detectors: + +- **Silero VAD** from a GGUF file (gallery entry `parakeet-cpp-silero-vad-f16`, 1.3 MB). One probability per 32 ms. +- **The VAD head** of a Moondream Ultra or Redux model (gallery entries `parakeet-cpp-vad-moondream-ultra-q8_0` and `parakeet-cpp-vad-moondream-redux-packed`). One probability per 80 ms. The packed Redux file runs on CPU only. + +The entry `parakeet-cpp-vad` installs Silero. The detectors differ and are not variants of one model, so install the entry of the VAD head by name if you want it. The request is the same as above: `audio` is 16 kHz mono float32 PCM, and the response lists `segments` with `start` and `end` in seconds. An ASR model that has no VAD head fails the request with `model has no VAD head`. + +```yaml +name: parakeet-vad +backend: parakeet-cpp +known_usecases: + - vad +parameters: + model: parakeet-cpp/silero-vad-f16.gguf +options: + - vad_threshold:0.5 + - vad_min_pause:0.1 +``` + +All options are optional. An unset value keeps the default of the detector in use (the library defaults differ between Silero and the head): + +| Option | Unit | Silero default | Head default | Description | +|--------|------|---------------:|-------------:|-------------| +| `vad_threshold` | 0 to 1 | `0.5` | `0.5` | Speech probability threshold | +| `vad_min_pause` | seconds | `0.1` | `0.2` | A silence this long separates two segments; shorter gaps merge | +| `vad_min_speech` | seconds | `0.25` | `0.1` | Shorter speech runs are dropped | +| `vad_speech_pad` | seconds | `0.03` | `0` | Padding added around each segment | + +Option names differ from the Silero backend above (`min_silence_duration_ms` and `speech_pad_ms` are in milliseconds there). The same options tune transcription with `vad:true` or `vad_model`; see [audio to text]({{%relref "features/audio-to-text" %}}). Requests on one loaded model run one at a time. + ## Detection Parameters The Silero VAD backend uses the following internal defaults (overridable via `options` above): diff --git a/gallery/index.yaml b/gallery/index.yaml index 74672c7199b2..f0df4fe6e0b5 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -53929,6 +53929,154 @@ - filename: parakeet-cpp/redux-q8_0.gguf uri: huggingface://mudler/parakeet-cpp-gguf/redux-q8_0.gguf sha256: 3429a4598946c406dbb4af5bb589d4dd307b069d550d98e24bef6779c4e6988f +- name: parakeet-cpp-silero-vad-f16 + url: github:mudler/LocalAI/gallery/virtual.yaml@master + urls: + - https://github.com/snakers4/silero-vad + - https://huggingface.co/mudler/parakeet-cpp-gguf + - https://github.com/mudler/parakeet.cpp + description: | + Silero VAD v6.2.3 as a GGUF (F16, about 1.3 MB) for the parakeet-cpp backend. It only detects speech: use it for the VAD endpoint, or as the vad_model: option of a parakeet-cpp ASR model that has no VAD head. + Converted from the official Silero VAD model, MIT licensed, copyright Silero Team: https://github.com/snakers4/silero-vad + license: mit + icon: https://github.com/snakers4/silero-models/raw/master/files/silero_logo.jpg + tags: + - silero + - vad + - voice-activity-detection + - parakeet-cpp + - gguf + - ggml + - speech + - cpu + overrides: + backend: parakeet-cpp + known_usecases: + - vad + name: parakeet-cpp-silero-vad-f16 + parameters: + model: parakeet-cpp/silero-vad-f16.gguf + files: + - filename: parakeet-cpp/silero-vad-f16.gguf + uri: https://huggingface.co/mudler/parakeet-cpp-gguf/resolve/main/silero-vad-f16.gguf + sha256: 8160489282352accc0e95925c2f6bf3d76fb8f7bccdce5c7e46808cb15e8443c +- name: parakeet-cpp-vad-moondream-redux-packed + url: github:mudler/LocalAI/gallery/virtual.yaml@master + urls: + - https://huggingface.co/mudler/parakeet-cpp-gguf + - https://github.com/mudler/parakeet.cpp + description: | + Voice activity detection with the VAD head of Moondream Redux, packed ternary weights (about 213 MB). CPU only. The file is the same as the ASR entry parakeet-cpp-moondream-redux-packed, so the two share it on disk. + Use it for the VAD endpoint. License CC-BY-4.0: credit Moondream and NVIDIA. + license: cc-by-4.0 + tags: + - parakeet + - parakeet-cpp + - moondream + - vad + - voice-activity-detection + - gguf + - ggml + - speech + - cpu + overrides: + backend: parakeet-cpp + known_usecases: + - vad + name: parakeet-cpp-vad-moondream-redux-packed + parameters: + model: parakeet-cpp/redux-packed.gguf + files: + - filename: parakeet-cpp/redux-packed.gguf + uri: https://huggingface.co/mudler/parakeet-cpp-gguf/resolve/main/redux-packed.gguf + sha256: 574614b9a4d9f72ab202877a7ad6a1f4bf819dd1d27b9d42dfe8cd429fcebdd5 +- name: parakeet-cpp-vad-moondream-ultra-q8_0 + url: github:mudler/LocalAI/gallery/virtual.yaml@master + urls: + - https://huggingface.co/mudler/parakeet-cpp-gguf + - https://github.com/mudler/parakeet.cpp + description: | + Voice activity detection with the VAD head of Moondream Ultra, Q8_0 (about 0.9 GB). Runs on CPU and GPU. The file is the same as the ASR entry parakeet-cpp-moondream-ultra-q8_0, so the two share it on disk. + Use it for the VAD endpoint. License CC-BY-4.0: credit Moondream and NVIDIA. + license: cc-by-4.0 + tags: + - parakeet + - parakeet-cpp + - moondream + - vad + - voice-activity-detection + - gguf + - ggml + - speech + overrides: + backend: parakeet-cpp + known_usecases: + - vad + name: parakeet-cpp-vad-moondream-ultra-q8_0 + parameters: + model: parakeet-cpp/ultra-q8_0.gguf + files: + - filename: parakeet-cpp/ultra-q8_0.gguf + uri: https://huggingface.co/mudler/parakeet-cpp-gguf/resolve/main/ultra-q8_0.gguf + sha256: c2fb452a9df468a141012b01c8c168a25ce93f710897c7de6e353c6cc250986a +- name: parakeet-cpp-vad + url: github:mudler/LocalAI/gallery/virtual.yaml@master + urls: + - https://huggingface.co/mudler/parakeet-cpp-gguf + - https://github.com/mudler/parakeet.cpp + description: | + Voice activity detection served by the parakeet-cpp backend, with Silero VAD (about 1.3 MB). To use the VAD head of Moondream Redux or Ultra instead, install parakeet-cpp-vad-moondream-redux-packed or parakeet-cpp-vad-moondream-ultra-q8_0. The detectors are different, not builds of the same weights, so this entry does not pick between them. + tags: + - vad + - voice-activity-detection + - parakeet-cpp + - cpu + overrides: + backend: parakeet-cpp + known_usecases: + - vad + parameters: + model: parakeet-cpp/silero-vad-f16.gguf + files: + - filename: parakeet-cpp/silero-vad-f16.gguf + uri: https://huggingface.co/mudler/parakeet-cpp-gguf/resolve/main/silero-vad-f16.gguf + sha256: 8160489282352accc0e95925c2f6bf3d76fb8f7bccdce5c7e46808cb15e8443c +- name: parakeet-cpp-tdt-0.6b-v3-silero-vad + url: github:mudler/LocalAI/gallery/virtual.yaml@master + urls: + - https://huggingface.co/mudler/parakeet-cpp-gguf + - https://github.com/mudler/parakeet.cpp + - https://github.com/snakers4/silero-vad + description: | + TDT FastConformer, 0.6B (v3, multilingual) with Silero VAD. The vad_model option makes the backend cut long audio at pauses found by Silero before it transcribes, so recordings of any length work. The model has no VAD head of its own, so Silero does the cutting. + Both files are GGUF for the parakeet-cpp backend. The Silero model is MIT licensed, copyright Silero Team. + license: cc-by-4.0 + tags: + - parakeet + - parakeet-cpp + - asr + - speech-recognition + - stt + - silero + - vad + - gguf + - ggml + overrides: + backend: parakeet-cpp + known_usecases: + - transcript + name: parakeet-cpp-tdt-0.6b-v3-silero-vad + options: + - vad_model:parakeet-cpp/silero-vad-f16.gguf + parameters: + model: parakeet-cpp/tdt-0.6b-v3-f16.gguf + files: + - filename: parakeet-cpp/tdt-0.6b-v3-f16.gguf + uri: huggingface://mudler/parakeet-cpp-gguf/tdt-0.6b-v3-f16.gguf + sha256: 8ba47343e1e919895aca90e099150a01ed203ee0942d8ed31e27295efc5abb22 + - filename: parakeet-cpp/silero-vad-f16.gguf + uri: https://huggingface.co/mudler/parakeet-cpp-gguf/resolve/main/silero-vad-f16.gguf + sha256: 8160489282352accc0e95925c2f6bf3d76fb8f7bccdce5c7e46808cb15e8443c - name: parakeet-cpp-nemotron-3-diarization url: github:mudler/LocalAI/gallery/virtual.yaml@master urls: