Skip to content

Include AAC audio in the native HLS fallback writer #669

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
matteius merged 2 commits into from
Oct 24, 2026
Merged

Include AAC audio in the native HLS fallback writer #669

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4e936b9
Include AAC audio in the native HLS fallback writer
matteius Oct 24, 2026
813fdaf
ci: run test_hls_cleanup in the Unity coverage job
matteius Oct 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion .github/workflows/integration-test.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,7 @@ jobs:
test_onvif_soap_fault \
test_faac_encoder \
test_audio_transcoder \
test_hls_cleanup \
test_stream_manager \
test_stream_state \
test_packet_buffer \
Expand All @@ -132,7 +133,7 @@ jobs:
run: |
cd build
ctest --output-on-failure -V \
-R "test_storage_pressure$|test_storage_pressure_extended|test_detection_result_structures|test_storage_retention_sqlite|test_storage_manager_retention|test_storage_target_pressure_cleanup|test_storage_migration|test_storage_archive_s3|test_storage_archive_operator|test_authorization|test_api_handlers_recordings_playback|test_db_storage_targets|test_db_storage_policies|test_api_handlers_storage_targets|test_api_handlers_storage_policies|test_config|test_logger|test_strings|test_memory|test_request_response|test_shutdown_coordinator|test_detection_config|test_detection_model_motion|test_db_streams|test_db_camera_favorites|test_api_handlers_camera_favorites|test_db_backup|test_mp4_failed_open_memory|test_db_recordings_extended|test_db_detections|test_db_zones|test_db_events|test_db_auth|test_db_transactions|test_db_maintenance|test_db_query_builder|test_logger_json|test_batch_delete_progress|test_db_recordings_sync|test_httpd_utils|test_zone_filter|test_onvif_soap_fault|test_faac_encoder|test_audio_transcoder|test_stream_manager|test_stream_state|test_packet_buffer|test_timestamp_manager|test_motion_trigger_parse|test_motion_max_area|test_url_utils"
-R "test_storage_pressure$|test_storage_pressure_extended|test_detection_result_structures|test_storage_retention_sqlite|test_storage_manager_retention|test_storage_target_pressure_cleanup|test_storage_migration|test_storage_archive_s3|test_storage_archive_operator|test_authorization|test_api_handlers_recordings_playback|test_db_storage_targets|test_db_storage_policies|test_api_handlers_storage_targets|test_api_handlers_storage_policies|test_config|test_logger|test_strings|test_memory|test_request_response|test_shutdown_coordinator|test_detection_config|test_detection_model_motion|test_db_streams|test_db_camera_favorites|test_api_handlers_camera_favorites|test_db_backup|test_mp4_failed_open_memory|test_db_recordings_extended|test_db_detections|test_db_zones|test_db_events|test_db_auth|test_db_transactions|test_db_maintenance|test_db_query_builder|test_logger_json|test_batch_delete_progress|test_db_recordings_sync|test_httpd_utils|test_zone_filter|test_onvif_soap_fault|test_faac_encoder|test_audio_transcoder|test_hls_cleanup|test_stream_manager|test_stream_state|test_packet_buffer|test_timestamp_manager|test_motion_trigger_parse|test_motion_max_area|test_url_utils"

- name: Generate coverage report
if: always()
Expand Down
3 changes: 3 additions & 0 deletions include/video/hls/hls_unified_thread.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,9 @@ typedef struct {
// Stream configuration
int protocol; // STREAM_PROTOCOL_TCP or STREAM_PROTOCOL_UDP
int segment_duration;
// Effective audio setting (record_audio && !audio_disabled); when 0 no
// audio is ever muxed into the HLS output
int audio_enabled;

// HLS writer (embedded directly instead of pointer)
hls_writer_t *writer;
Expand Down
22 changes: 21 additions & 1 deletion include/video/hls_writer.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,11 @@ typedef struct hls_writer_t {

// Per-stream DTS tracking
stream_dts_info_t dts_tracker;
stream_dts_info_t audio_dts_tracker;

// Output stream indices; audio_out_index is -1 when the output is video only
int video_out_index;
int audio_out_index;

// Counter for DTS jumps to detect stream issues
int dts_jump_count;
Expand All @@ -57,7 +62,22 @@ hls_writer_t *hls_writer_create(const char *output_dir, const char *stream_name,
int hls_writer_initialize(hls_writer_t *writer, const AVStream *input_stream);

/**
* Write packet to HLS stream
* Initialize HLS writer with a video stream and an optional audio stream.
* Only AAC audio is muxed (stream copy); any other codec, or a NULL
* audio_stream, yields video-only output. Callers must pass NULL when audio
* is disabled for the stream so no audio reaches the segments.
*/
int hls_writer_initialize_av(hls_writer_t *writer, const AVStream *video_stream,
const AVStream *audio_stream);

/**
* Whether hls_writer_initialize_av() can mux audio with these parameters (AAC only)
*/
int hls_writer_audio_supported(const AVCodecParameters *par);

/**
* Write packet to HLS stream. The packet is routed by the input stream's media
* type; audio packets are dropped when the writer has no audio output.
*/
int hls_writer_write_packet(hls_writer_t *writer, const AVPacket *pkt, const AVStream *input_stream);

Expand Down
56 changes: 37 additions & 19 deletions src/video/hls/hls_unified_thread.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -786,6 +786,7 @@ void *hls_unified_thread_func(void *arg) {
AVFormatContext *input_ctx = NULL;
AVPacket *pkt = NULL;
int video_stream_idx = -1;
int audio_stream_idx = -1;
int ret;
hls_thread_state_t thread_state = HLS_THREAD_INITIALIZING;
int reconnect_attempt = 0;
Expand Down Expand Up @@ -1111,8 +1112,9 @@ void *hls_unified_thread_func(void *arg) {
break;
}

// Find video stream
// Find video stream (and audio, only when audio is enabled for the stream)
video_stream_idx = find_video_stream_index(input_ctx);
audio_stream_idx = ctx->audio_enabled ? find_audio_stream_index(input_ctx) : -1;
if (video_stream_idx == -1) {
log_error("No video stream found for stream %s", stream_name);

Expand Down Expand Up @@ -1168,7 +1170,8 @@ void *hls_unified_thread_func(void *arg) {
if (!is_shutdown_initiated() && input_ctx->streams[video_stream_idx]) {
// CRITICAL FIX: Check if context is still valid before accessing
if (ctx && !is_context_pending_deletion(ctx) && !is_context_already_freed(ctx) && ctx->writer) {
ret = hls_writer_initialize(ctx->writer, input_ctx->streams[video_stream_idx]);
ret = hls_writer_initialize_av(ctx->writer, input_ctx->streams[video_stream_idx],
audio_stream_idx >= 0 ? input_ctx->streams[audio_stream_idx] : NULL);
} else {
log_warn("Context for stream %s is no longer valid or writer is NULL, skipping HLS writer initialization", stream_name);
// Set thread state to stopping to exit gracefully
Expand Down Expand Up @@ -1284,13 +1287,15 @@ void *hls_unified_thread_func(void *arg) {
continue;
}

// Process packets based on stream type
if (pkt->stream_index == video_stream_idx) {
// Process packets based on stream type. audio_stream_idx is -1
// when audio is disabled, so audio packets never reach the writer.
const bool is_audio_pkt = audio_stream_idx >= 0 && pkt->stream_index == audio_stream_idx;
if (pkt->stream_index == video_stream_idx || is_audio_pkt) {
// Record frame for telemetry metrics
metrics_record_frame_from_source(stream_name, pkt->size, true,
METRICS_SOURCE_HLS);

// This is a video packet - process it
if (!is_audio_pkt) {
metrics_record_frame_from_source(stream_name, pkt->size, true,
METRICS_SOURCE_HLS);
}

// CRITICAL FIX: Check if context is still valid before accessing
if (!ctx || is_context_pending_deletion(ctx) || is_context_already_freed(ctx)) {
Expand Down Expand Up @@ -1330,10 +1335,17 @@ void *hls_unified_thread_func(void *arg) {
continue;
}

// A writer recreated after reconnection is initialized here so
// that its audio output is set up before the header is written
if (!writer->initialized) {
hls_writer_initialize_av(writer, input_ctx->streams[video_stream_idx],
audio_stream_idx >= 0 ? input_ctx->streams[audio_stream_idx] : NULL);
}

ret = hls_writer_write_packet(writer, pkt, input_stream);

// Log key frames for debugging
bool is_key_frame = (pkt->flags & AV_PKT_FLAG_KEY) != 0;
bool is_key_frame = !is_audio_pkt && (pkt->flags & AV_PKT_FLAG_KEY) != 0;
if (is_key_frame && ret >= 0) {
log_debug("Processed video key frame for stream %s", stream_name);
}
Expand All @@ -1342,13 +1354,17 @@ void *hls_unified_thread_func(void *arg) {
if (ret < 0) {
char error_buf[AV_ERROR_MAX_STRING_SIZE] = {0};
av_strerror(ret, error_buf, AV_ERROR_MAX_STRING_SIZE);
log_warn("Error writing video packet to HLS for stream %s: %s", stream_name, error_buf);
log_warn("Error writing %s packet to HLS for stream %s: %s",
is_audio_pkt ? "audio" : "video", stream_name, error_buf);
} else {
// Successfully processed a packet
last_packet_time = time(NULL);
atomic_store(&ctx->last_packet_time, (int_fast64_t)last_packet_time);
atomic_store(&ctx->consecutive_failures, 0);
atomic_store(&ctx->connection_valid, 1);
// Successfully processed a packet. Only video refreshes the
// liveness timestamp so flowing audio cannot mask a video stall.
if (!is_audio_pkt) {
last_packet_time = time(NULL);
atomic_store(&ctx->last_packet_time, (int_fast64_t)last_packet_time);
atomic_store(&ctx->consecutive_failures, 0);
atomic_store(&ctx->connection_valid, 1);
}

// Mark that we've successfully written packets
// This is used to determine if we need to recreate the writer on reconnection
Expand All @@ -1358,9 +1374,7 @@ void *hls_unified_thread_func(void *arg) {
// Unlock the writer mutex
pthread_mutex_unlock(&writer->mutex);
} else {
// This is a non-video packet (likely audio)
// For now, we'll just log it and skip processing
// This prevents the "Invalid packet stream index" errors
// Non-video packet that is not muxed (audio disabled, extra tracks, data)
if (pkt->stream_index >= 0 && pkt->stream_index < input_ctx->nb_streams) {
const AVStream *stream = input_ctx->streams[pkt->stream_index];
const AVCodecParameters *codecpar = stream->codecpar;
Expand Down Expand Up @@ -1515,8 +1529,9 @@ void *hls_unified_thread_func(void *arg) {
break;
}

// Find video stream
// Find video stream (and audio, only when audio is enabled for the stream)
video_stream_idx = find_video_stream_index(input_ctx);
audio_stream_idx = ctx->audio_enabled ? find_audio_stream_index(input_ctx) : -1;
if (video_stream_idx == -1) {
log_error("No video stream found for stream %s during reconnection",
stream_name);
Expand Down Expand Up @@ -2214,6 +2229,9 @@ int start_hls_unified_stream(const char *stream_name) {
// Set protocol information in the context
ctx->protocol = config.protocol;

// Respect per-stream and instance-wide audio settings
ctx->audio_enabled = config.record_audio && !g_config.audio_disabled;

// Set HLS segment duration.
// config.segment_duration is the MP4/recording segment duration (often 900s = 15 min).
// For HLS, we need short segments (1-10s). Use 2s default unless the configured
Expand Down
Loading
Loading