fix(recordings): prepare playback outside the HTTP worker pool

Run serialized HEVC conversions in a bounded background worker and expose a retryable preparation response. Poll before loading media in recording, timeline, and investigation players; cancel stale polling and remove duplicate media loads. Abort active conversion during shutdown.

Validate with 23 C cases, 85 frontend unit tests, three browser integration tests, modal/timeline decoding smoke checks, and 30 concurrent requests against a two-worker server. Preserve the existing FFmpeg thread and memory limits.

Author: matteius

docs/API.md

@@ -836,7 +836,21 @@ Deletes a recording.
 GET /api/recordings/play/{id}
 ```
 
-Streams a recording for playback.
+Streams a recording for playback with byte-range support. HEVC recordings use
+a cached H.264 copy prepared by a single background transcode worker.
+
+Before loading the media URL, poll `GET /api/recordings/play/{id}?prepare=1`:
+
+- `200` with `{"status":"ready"}`: load the media URL without `prepare=1`.
+- `202` with `{"status":"preparing"}`: wait for the `Retry-After` interval and poll again.
+- `500`: preparation failed; the JSON `error` explains the failure. The original
+  recording remains available from the download endpoint.
+
+Preparation uses the same replay authorization as playback. A media request
+made while conversion is pending returns `503` with `Retry-After`; it does not
+hold an HTTP worker open for the conversion. H.264 recordings and completed
+cache entries are ready immediately. Clients should stop polling when playback
+is closed or another recording is selected.
 
 #### Download Recording
 

include/video/recording_transcode.h

@@ -44,4 +44,23 @@ bool recording_needs_hevc_transcode(const char *file_path);
  */
 int ensure_recording_transcode_cache(const char *original_path, const char *cache_path);
 
+typedef enum {
+    RECORDING_TRANSCODE_FAILED = -1,
+    RECORDING_TRANSCODE_READY = 0,
+    RECORDING_TRANSCODE_PENDING = 1
+} recording_transcode_status_t;
+
+/**
+ * Start or poll preparation on a dedicated thread, without waiting for FFmpeg
+ * in an HTTP worker. Only one background job is admitted; other recordings
+ * return PENDING and can retry later. Duplicate requests share the same job.
+ * Failed jobs return FAILED for 30 seconds to avoid a subprocess retry storm.
+ * Paths are copied before returning. Completed cache hits return READY.
+ */
+recording_transcode_status_t request_recording_transcode_cache(
+    const char *original_path, const char *cache_path);
+
+/** Stop admitting playback jobs, abort FFmpeg and join the background worker. */
+void shutdown_recording_transcode(void);
+
 #endif /* LIGHTNVR_RECORDING_TRANSCODE_H */

include/web/api_handlers_recordings_playback.h

@@ -7,6 +7,9 @@
  * @brief Backend-agnostic handler for GET /api/recordings/play/:id
  *
  * Serves a recording file for playback with range request support for seeking.
+ * With ?prepare=1, returns JSON: 200 when ready or 202 with Retry-After while
+ * a bounded background transcode is pending. Media requests made before the
+ * cache is ready return 503 with Retry-After instead of blocking an HTTP worker.
  *
  * @param req HTTP request
  * @param res HTTP response

src/core/main.c

@@ -43,6 +43,7 @@
 #include "video/streams.h"
 #include "video/hls_streaming.h"
 #include "video/mp4_recording.h"
+#include "video/recording_transcode.h"
 #include "video/stream_transcoding.h"
 #include "video/hls_writer.h"
 #include "video/detection_stream.h"
@@ -1408,6 +1409,7 @@ int main(int argc, char *argv[]) {
     // Cleanup
 cleanup:
     log_info("Starting cleanup process...");
+    shutdown_recording_transcode();
 
     // Stop request producers before any state they can inspect is dismantled.
     // Keeping this at the shared cleanup label also covers partial startup.

src/video/recording_transcode.c

@@ -6,11 +6,13 @@
 #include <fcntl.h>
 #include <inttypes.h>
 #include <pthread.h>
+#include <signal.h>
 #include <stdatomic.h>
 #include <stdio.h>
 #include <string.h>
 #include <sys/stat.h>
 #include <sys/wait.h>
+#include <time.h>
 #include <unistd.h>
 
 #include <libavformat/avformat.h>
@@ -28,10 +30,24 @@
  * PATH already covers both. */
 #define FFMPEG_BINARY "ffmpeg"
 
-/* Playback handlers run concurrently in the HTTP worker pool. Keep only one
- * recording transcode active across all recordings: even different clips can
- * exhaust the memory shared with recording and go2rtc on a small NVR. */
+/* Keep only one recording transcode active, including synchronous callers:
+ * different clips can exhaust memory shared with recording on a small NVR. */
 static pthread_mutex_t s_transcode_mutex = PTHREAD_MUTEX_INITIALIZER;
+static atomic_bool s_stopping = false;
+
+/* This mutex only protects job bookkeeping, never an encode or a slot wait.
+ * A separate thread keeps both FFmpeg and s_transcode_mutex waits out of
+ * libuv's shared pool (which also serves APIs and reads video file chunks). */
+static pthread_mutex_t s_job_mutex = PTHREAD_MUTEX_INITIALIZER;
+static struct {
+    pthread_t thread;
+    bool started;
+    bool active;
+    char original_path[512];
+    char cache_path[512];
+    int result;
+    struct timespec finished;
+} s_job;
 
 static bool file_exists_nonempty(const char *path) {
     struct stat st;
@@ -91,11 +107,20 @@ static int run_and_wait(char *const argv[]) {
     }
 
     int status = 0;
-    while (waitpid(pid, &status, 0) < 0) {
-        if (errno != EINTR) {
+    for (;;) {
+        pid_t result = waitpid(pid, &status, WNOHANG);
+        if (result == pid) break;
+        if (result < 0 && errno != EINTR) {
             log_error("recording_transcode: waitpid failed: %s", strerror(errno));
             return -1;
         }
+        if (atomic_load(&s_stopping)) {
+            kill(pid, SIGKILL);
+            while (waitpid(pid, &status, 0) < 0 && errno == EINTR) {}
+            return -1;
+        }
+        const struct timespec delay = { .tv_nsec = 100000000 };
+        nanosleep(&delay, NULL);
     }
 
     return (WIFEXITED(status) && WEXITSTATUS(status) == 0) ? 0 : -1;
@@ -105,6 +130,7 @@ static int run_and_wait(char *const argv[]) {
  * completed cache file. Recheck the cache because another request may have
  * produced it while this caller was waiting. */
 static int transcode_cache_locked(const char *original_path, const char *cache_path) {
+    if (atomic_load(&s_stopping)) return -1;
     if (file_exists_nonempty(cache_path)) {
         return 0;
     }
@@ -193,7 +219,7 @@ static int transcode_cache_locked(const char *original_path, const char *cache_p
             unlink(tmp_path);
         }
     }
-    if (rc != 0) {
+    if (rc != 0 && !atomic_load(&s_stopping)) {
         log_info("recording_transcode: transcoding %s -> %s via software libx264", original_path, cache_path);
         rc = run_and_wait(argv_software);
     }
@@ -234,3 +260,71 @@ int ensure_recording_transcode_cache(const char *original_path, const char *cach
     pthread_mutex_unlock(&s_transcode_mutex);
     return rc;
 }
+
+static void *transcode_worker(void *unused) {
+    (void)unused;
+    int result = ensure_recording_transcode_cache(s_job.original_path, s_job.cache_path);
+    pthread_mutex_lock(&s_job_mutex);
+    s_job.result = result;
+    clock_gettime(CLOCK_MONOTONIC, &s_job.finished);
+    s_job.active = false;
+    pthread_mutex_unlock(&s_job_mutex);
+    return NULL;
+}
+
+recording_transcode_status_t request_recording_transcode_cache(
+    const char *original_path, const char *cache_path) {
+    if (!original_path || !original_path[0] || !cache_path || !cache_path[0] ||
+        strlen(original_path) >= sizeof(s_job.original_path) ||
+        strlen(cache_path) >= sizeof(s_job.cache_path)) {
+        return RECORDING_TRANSCODE_FAILED;
+    }
+    if (file_exists_nonempty(cache_path)) return RECORDING_TRANSCODE_READY;
+
+    pthread_mutex_lock(&s_job_mutex);
+    recording_transcode_status_t status = RECORDING_TRANSCODE_PENDING;
+    if (atomic_load(&s_stopping)) {
+        status = RECORDING_TRANSCODE_FAILED;
+    } else if (file_exists_nonempty(cache_path)) {
+        status = RECORDING_TRANSCODE_READY;
+    } else if (!s_job.active) {
+        if (s_job.started) {
+            pthread_join(s_job.thread, NULL);
+            s_job.started = false;
+        }
+        struct timespec now;
+        clock_gettime(CLOCK_MONOTONIC, &now);
+        if (s_job.result != 0 &&
+            strcmp(original_path, s_job.original_path) == 0 &&
+            strcmp(cache_path, s_job.cache_path) == 0 &&
+            now.tv_sec - s_job.finished.tv_sec < 30) {
+            status = RECORDING_TRANSCODE_FAILED;
+        } else {
+            strcpy(s_job.original_path, original_path);
+            strcpy(s_job.cache_path, cache_path);
+            s_job.active = true;
+            int err = pthread_create(&s_job.thread, NULL, transcode_worker, NULL);
+            if (err != 0) {
+                log_error("recording_transcode: failed to start worker: %s", strerror(err));
+                s_job.active = false;
+                s_job.result = -1;
+                s_job.finished = now;
+                status = RECORDING_TRANSCODE_FAILED;
+            } else {
+                s_job.started = true;
+            }
+        }
+    }
+    pthread_mutex_unlock(&s_job_mutex);
+    return status;
+}
+
+void shutdown_recording_transcode(void) {
+    pthread_mutex_lock(&s_job_mutex);
+    atomic_store(&s_stopping, true);
+    bool join = s_job.started;
+    pthread_t thread = s_job.thread;
+    s_job.started = false;
+    pthread_mutex_unlock(&s_job_mutex);
+    if (join) pthread_join(thread, NULL);
+}

src/web/api_handlers_recordings_playback.c

@@ -81,25 +81,44 @@ void handle_recordings_playback(const http_request_t *req, http_response_t *res)
 
     log_info("Serving file for playback: %s (%ld bytes)", recording.file_path, st.st_size);
 
-    // Most browsers have no royalty-free HEVC decoder for an HTML5 <video>
-    // element, so an HEVC recording (e.g. FrontDoor, whose native stream is
-    // HEVC and whose recordings are a raw stream-copy of it) fails in-browser
-    // with "no video with supported format and MIME type found" even though
-    // it plays fine in a local player. Transparently swap in a cached H.264
-    // copy when that's the case; the original file on disk is untouched.
+    // The modal polls ?prepare=1 before attaching the media URL. Never wait
+    // for an encode in a libuv worker: enough duplicate/range requests would
+    // starve both other APIs and the filesystem reads used to serve videos.
+    char prepare[8];
+    bool prepare_only = http_request_get_query_param(req, "prepare", prepare,
+                                                     sizeof(prepare)) > 0 &&
+                        strcmp(prepare, "1") == 0;
+
     const char *serve_path = recording.file_path;
     char transcode_cache_path[MAX_PATH_LENGTH];
-    if (recording_needs_hevc_transcode(recording.file_path) &&
-        build_recording_transcode_cache_path(g_config.storage_path, id,
-                                             transcode_cache_path,
-                                             sizeof(transcode_cache_path)) == 0) {
-        if (ensure_recording_transcode_cache(recording.file_path, transcode_cache_path) == 0) {
-            serve_path = transcode_cache_path;
-        } else {
-            log_warn("Failed to prepare HEVC playback cache for recording %llu, "
-                     "serving original file (browser playback may fail)",
-                     (unsigned long long)id);
+    bool have_cache_path = build_recording_transcode_cache_path(
+        g_config.storage_path, id, transcode_cache_path,
+        sizeof(transcode_cache_path)) == 0;
+    struct stat cache_st;
+    if (have_cache_path && stat(transcode_cache_path, &cache_st) == 0 &&
+        S_ISREG(cache_st.st_mode) && cache_st.st_size > 0) {
+        serve_path = transcode_cache_path;
+    } else if (recording_needs_hevc_transcode(recording.file_path)) {
+        recording_transcode_status_t status = have_cache_path
+            ? request_recording_transcode_cache(recording.file_path, transcode_cache_path)
+            : RECORDING_TRANSCODE_FAILED;
+        if (status == RECORDING_TRANSCODE_PENDING) {
+            http_response_add_header(res, "Retry-After", "2");
+            http_response_set_json(res, prepare_only ? 202 : 503,
+                                   "{\"status\":\"preparing\"}");
+            return;
+        }
+        if (status == RECORDING_TRANSCODE_FAILED) {
+            http_response_set_json_error(res, 500,
+                "Unable to prepare recording for playback. Download the original or try again later.");
+            return;
         }
+        serve_path = transcode_cache_path;
+    }
+
+    if (prepare_only) {
+        http_response_set_json(res, 200, "{\"status\":\"ready\"}");
+        return;
     }
 
     // Determine content type based on file extension

tests/integration/specs/investigation.ui.spec.ts

@@ -68,7 +68,10 @@ test.describe('Investigation player regressions @ui @investigation', () => {
         },
       },
     }));
-    await page.route('**/api/recordings/play/568*', route => route.fulfill({
+    await page.route('**/api/recordings/play/568*', route => route.fulfill(
+      new URL(route.request().url()).searchParams.has('prepare')
+        ? { json: { status: 'ready' } }
+        : {
       status: 200,
       contentType: 'video/webm',
       headers: { 'Accept-Ranges': 'bytes' },

tests/integration/specs/recordings.ui.spec.ts

@@ -316,7 +316,19 @@ test.describe('Recordings Page @ui @recordings', () => {
         }],
         pagination: { total: 1, pages: 1, limit: 20 }
       } }));
-      await page.route('**/api/recordings/play/601*', route => route.fulfill({ status: 204, body: '' }));
+      let prepared = false;
+      let preparationRequests = 0;
+      await page.route('**/api/recordings/play/601*', route => {
+        if (new URL(route.request().url()).searchParams.has('prepare')) {
+          preparationRequests++;
+          return route.fulfill({
+            status: prepared ? 200 : 202,
+            headers: { 'Retry-After': '1' },
+            json: { status: prepared ? 'ready' : 'preparing' },
+          });
+        }
+        return route.fulfill({ status: 204, body: '' });
+      });
       await page.route('**/api/recordings/601', route => route.fulfill({ json: {
         id: 601,
         stream: 'cam1',
@@ -343,6 +355,17 @@ test.describe('Recordings Page @ui @recordings', () => {
       await expect(page.locator('#recordings-table')).toBeVisible();
       await page.locator('button[title="Play"]').first().click();
 
+      const modal = page.locator('#video-preview-modal');
+      await expect(modal.getByRole('status')).toHaveText('Preparing recording for playback…');
+      await expect(modal.locator('video')).not.toHaveAttribute('src');
+      await modal.locator('button.close').click();
+      const requestsAtClose = preparationRequests;
+      await page.waitForTimeout(1200);
+      expect(preparationRequests).toBe(requestsAtClose);
+      prepared = true;
+      await page.locator('button[title="Play"]').first().click();
+      await expect(modal.locator('video')).toHaveAttribute('src', '/api/recordings/play/601');
+
       await expect(page.locator('#video-preview-modal')).toBeVisible();
       await expect(page.locator('#recording-playback-position')).toHaveText('cam1 - 00:00:00');
 
@@ -417,7 +440,11 @@ test.describe('Recordings Page @ui @recordings', () => {
         contentType: 'image/svg+xml',
         body: '<svg xmlns="http://www.w3.org/2000/svg" width="320" height="180"><rect width="320" height="180" fill="#111827"/></svg>'
       }));
-      await page.route('**/api/recordings/play/602*', route => route.fulfill({ status: 204, body: '' }));
+      await page.route('**/api/recordings/play/602*', route => route.fulfill(
+        new URL(route.request().url()).searchParams.has('prepare')
+          ? { json: { status: 'ready' } }
+          : { status: 204, body: '' }
+      ));
       await page.route('**/api/recordings/602', route => route.fulfill({ json: {
         id: 602,
         stream: 'front-door-camera-with-a-long-name',

tests/unit/CMakeLists.txt

@@ -165,6 +165,14 @@ add_layer2_test(test_db_camera_collections)
 add_layer2_test(test_api_handlers_camera_collections)
 add_layer2_test(test_db_recordings_extended)
 add_layer2_test(test_recording_transcode)
+add_layer2_test(test_api_handlers_recordings_playback)
+target_link_options(test_api_handlers_recordings_playback PRIVATE
+    "-Wl,--wrap=get_recording_metadata_by_id"
+    "-Wl,--wrap=httpd_authorize_camera_identity_action_with_context"
+    "-Wl,--wrap=recording_needs_hevc_transcode"
+    "-Wl,--wrap=request_recording_transcode_cache"
+    "-Wl,--wrap=http_serve_file"
+)
 add_layer2_test(test_storage_manager_retention)
 add_layer2_test(test_storage_target_pressure_cleanup)
 add_layer2_test(test_storage_migration)