integration-test.yml

name: Integration Tests

on:
  push:
    branches:
      - main
      - 'feature/**'
  workflow_dispatch:

permissions:
  contents: read
  checks: write
  pull-requests: write

jobs:
  unity-tests:
    name: Unity C Tests (Layer 1 + Layer 2)
    runs-on: ubuntu-latest
    container: debian:sid-slim
    timeout-minutes: 10

    steps:
      - name: Install C build dependencies
        run: |
          apt-get update
          apt-get install -y \
            git \
            build-essential \
            cmake \
            pkg-config \
            libavcodec-dev \
            libavformat-dev \
            libavutil-dev \
            libswscale-dev \
            libsqlite3-dev \
            libuv1-dev \
            libllhttp-dev \
            libcurl4-openssl-dev \
            libcjson-dev \
            libmbedtls-dev \
            lcov \
            curl \
            gpg

      - name: Checkout
        uses: actions/checkout@v6
        with:
          submodules: false

      - name: Check embedded migration generation
        run: |
          python3 tests/unit/test_embedded_migration_generator.py
          python3 scripts/generate_embedded_migrations.py \
            --check db/migrations include/database/db_embedded_migrations.h

      - name: Build Unity tests
        run: |
          mkdir -p build
          cd build
          # Concurrent tests need atomic gcov counters to avoid corrupt coverage.
          cmake .. \
            -DCMAKE_BUILD_TYPE=Debug \
            -DBUILD_TESTS=ON \
            -DENABLE_SOD=OFF \
            -DENABLE_GO2RTC=OFF \
            -DENABLE_MQTT=OFF \
            -DCMAKE_C_FLAGS="--coverage -fprofile-update=atomic" \
            -DCMAKE_EXE_LINKER_FLAGS="--coverage"
          make -j$(nproc) \
            test_storage_pressure \
            test_storage_pressure_extended \
            test_detection_result_structures \
            test_storage_retention_sqlite \
            test_storage_manager_retention \
            test_storage_target_pressure_cleanup \
            test_storage_migration \
            test_storage_archive \
            test_authorization \
            test_api_handlers_recordings_playback \
            test_db_storage_targets \
            test_db_storage_policies \
            test_api_handlers_storage_targets \
            test_api_handlers_storage_policies \
            test_config \
            test_logger \
            test_strings \
            test_memory \
            test_request_response \
            test_shutdown_coordinator \
            test_detection_config \
            test_detection_model_motion \
            test_db_streams \
            test_db_backup \
            test_mp4_failed_open_memory \
            test_db_recordings_extended \
            test_db_detections \
            test_db_zones \
            test_db_events \
            test_db_auth \
            test_db_transactions \
            test_db_maintenance \
            test_db_query_builder \
            test_logger_json \
            test_batch_delete_progress \
            test_db_recordings_sync \
            test_httpd_utils \
            test_zone_filter \
            test_onvif_soap_fault \
            test_stream_manager \
            test_stream_state \
            test_packet_buffer \
            test_timestamp_manager \
            test_motion_trigger_parse \
            test_motion_max_area \
            test_url_utils
          echo "Unity test binaries built:"
          ls -la bin/test_*

      - name: Run Unity tests
        run: |
          cd build
          ctest --output-on-failure -V \
            -R "test_storage_pressure$|test_storage_pressure_extended|test_detection_result_structures|test_storage_retention_sqlite|test_storage_manager_retention|test_storage_target_pressure_cleanup|test_storage_migration|test_storage_archive_s3|test_storage_archive_operator|test_authorization|test_api_handlers_recordings_playback|test_db_storage_targets|test_db_storage_policies|test_api_handlers_storage_targets|test_api_handlers_storage_policies|test_config|test_logger|test_strings|test_memory|test_request_response|test_shutdown_coordinator|test_detection_config|test_detection_model_motion|test_db_streams|test_db_backup|test_mp4_failed_open_memory|test_db_recordings_extended|test_db_detections|test_db_zones|test_db_events|test_db_auth|test_db_transactions|test_db_maintenance|test_db_query_builder|test_logger_json|test_batch_delete_progress|test_db_recordings_sync|test_httpd_utils|test_zone_filter|test_onvif_soap_fault|test_stream_manager|test_stream_state|test_packet_buffer|test_timestamp_manager|test_motion_trigger_parse|test_motion_max_area|test_url_utils"

      - name: Generate coverage report
        if: always()
        run: |
          cd build
          # Collect coverage data from all .gcda files generated during the test run
          lcov --capture \
               --directory . \
               --output-file coverage.info \
               --ignore-errors mismatch

          # Strip noise: system headers, third-party code, and the test files themselves
          lcov --remove coverage.info \
               '/usr/*' \
               '*/third_party/*' \
               '*/external/*' \
               '*/tests/*' \
               --output-file coverage_filtered.info \
               --ignore-errors unused

          # Print a quick per-file summary to the job log
          lcov --list coverage_filtered.info

          # Render HTML
          genhtml coverage_filtered.info \
                  --output-directory ../coverage-report \
                  --title "LightNVR C Unit-Test Coverage" \
                  --show-details \
                  --legend

          echo "Coverage report generated at coverage-report/"

      - name: Upload coverage report
        uses: actions/upload-artifact@v7
        if: always()
        with:
          name: c-coverage-report
          path: coverage-report/
          retention-days: 30

      - name: Upload to Codecov
        uses: codecov/codecov-action@v7
        if: always()
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          files: build/coverage_filtered.info
          flags: unity-c
          name: lightnvr-c-coverage
          fail_ci_if_error: false

      - name: Upload test results on failure
        uses: actions/upload-artifact@v7
        if: failure()
        with:
          name: unity-test-results
          path: build/Testing/
          retention-days: 7

  system-health-tests:
    name: System Health Release Gates (P0/P1, P2/P4, optional P3)
    needs: unity-tests
    runs-on: ubuntu-latest
    container: debian:sid-slim
    timeout-minutes: 30

    steps:
      - name: Install build and broker dependencies
        run: |
          apt-get update
          apt-get install -y \
            git \
            build-essential \
            cmake \
            pkg-config \
            python3 \
            libavcodec-dev \
            libavformat-dev \
            libavutil-dev \
            libswscale-dev \
            libswresample-dev \
            libsqlite3-dev \
            libuv1-dev \
            libllhttp-dev \
            libcurl4-openssl-dev \
            libcjson-dev \
            libmbedtls-dev \
            libmosquitto-dev \
            mosquitto \
            mosquitto-clients \
            lcov \
            gpg \
            jq \
            shellcheck \
            ca-certificates \
            curl \
            xz-utils

      - name: Checkout
        uses: actions/checkout@v6
        with:
          submodules: false

      - name: Set up Node.js
        uses: actions/setup-node@v6
        with:
          node-version-file: .nvmrc
          cache: npm
          cache-dependency-path: |
            package-lock.json
            web/package-lock.json

      - name: Build MQTT-enabled health binaries
        run: |
          cmake -S . -B build/health-ci \
            -DCMAKE_BUILD_TYPE=Release \
            -DBUILD_TESTS=ON \
            -DENABLE_SOD=OFF \
            -DENABLE_GO2RTC=OFF \
            -DENABLE_LITERT=OFF \
            -DENABLE_MQTT=ON \
            -DCMAKE_C_FLAGS="--coverage -fprofile-update=atomic" \
            -DCMAKE_EXE_LINKER_FLAGS="--coverage"
          cmake --build build/health-ci -- -j2

      - name: Build MQTT-disabled variant
        run: |
          cmake -S . -B build/health-no-mqtt-ci \
            -DCMAKE_BUILD_TYPE=Release \
            -DBUILD_TESTS=ON \
            -DENABLE_SOD=OFF \
            -DENABLE_GO2RTC=OFF \
            -DENABLE_LITERT=OFF \
            -DENABLE_MQTT=OFF
          cmake --build build/health-no-mqtt-ci \
            --target lightnvr test_system_health_policy -- -j2

      - name: P0/P1 portable collection and incident gate
        run: |
          ctest --test-dir build/health-ci --output-on-failure \
            -R 'test_system_health_(types|proc|cgroup|filesystem|process|thermal|network|clock|policy|sampler|evaluator|storage)$|test_recording_io_metrics$|test_health_helper_runner$'

      - name: P2/P4 event, presence, API, and UI-unit gate
        run: |
          ctest --test-dir build/health-ci --output-on-failure \
            -R 'test_db_system_health_incidents$|test_mqtt_presence$|test_mqtt_destination_client$|test_api_handlers_(system_health|metrics)$|test_system_health_(self_observability|restart)$'
          npm --prefix web ci
          TZ=America/New_York npm --prefix web test -- --runInBand
          npm --prefix web run build

      - name: Optional P3 provider contract gate
        run: |
          ctest --test-dir build/health-ci --output-on-failure \
            -R 'test_system_health_hardware$|test_smartctl_provider$|test_kernel_log_provider$'

      - name: Deterministic replay and resource gates
        run: |
          build/health-ci/bin/test_system_health_benchmark \
            --samples 360 --heap-limit-bytes 524288 --cpu-ratio-limit 0.005
          SYSTEM_HEALTH_BENCHMARK="$GITHUB_WORKSPACE/build/health-ci/bin/test_system_health_benchmark" \
            T24_SKIP_DOCKER=1 scripts/run-system-health-replay.sh --all

      - name: Real-broker LWT and stale-heartbeat gate
        run: |
          LIGHTNVR_TEST_BIN="$GITHUB_WORKSPACE/build/health-ci/bin/lightnvr" \
            LIGHTNVR_TEST_CONFIG="$GITHUB_WORKSPACE/config/lightnvr-health-test.ini" \
            scripts/test-system-health-presence.sh

      - name: Health API, MQTT, and browser integration gate
        run: |
          npm ci
          npx playwright install --with-deps chromium
          LIGHTNVR_TEST_BIN="$GITHUB_WORKSPACE/build/health-ci/bin/lightnvr" \
            LIGHTNVR_TEST_CONFIG="$GITHUB_WORKSPACE/config/lightnvr-health-test.ini" \
            LIGHTNVR_SKIP_GO2RTC=1 \
            npx playwright test \
              tests/integration/specs/system-health.api.spec.ts \
              tests/integration/specs/system-health-mqtt.api.spec.ts \
              tests/integration/specs/system-health.ui.spec.ts \
              tests/integration/specs/system-health-settings.ui.spec.ts

      - name: Generate system health coverage report
        if: always()
        run: |
          lcov --capture \
               --directory build/health-ci \
               --output-file build/health-ci/coverage.info \
               --ignore-errors mismatch,inconsistent,negative
          lcov --remove build/health-ci/coverage.info \
               '/usr/*' \
               '*/third_party/*' \
               '*/external/*' \
               '*/tests/*' \
               --output-file build/health-ci/coverage_filtered.info \
               --ignore-errors unused,inconsistent,negative
          lcov --list build/health-ci/coverage_filtered.info \
               --ignore-errors inconsistent,negative

      - name: Upload system health coverage to Codecov
        uses: codecov/codecov-action@v7
        if: always()
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          files: build/health-ci/coverage_filtered.info
          flags: system-health
          name: lightnvr-system-health-coverage
          fail_ci_if_error: false

      - name: Upload health test artifacts
        uses: actions/upload-artifact@v7
        if: failure()
        with:
          name: system-health-test-results
          path: |
            build/health-ci/Testing/
            playwright-report/
            test-results/
          retention-days: 7

  go2rtc-unit-tests:
    name: go2rtc Unity Tests (T12 + T15)
    needs: unity-tests
    runs-on: ubuntu-latest
    container: debian:sid-slim
    timeout-minutes: 15

    steps:
      - name: Install dependencies
        run: |
          apt-get update
          apt-get install -y \
            git \
            build-essential \
            cmake \
            pkg-config \
            libavcodec-dev \
            libavformat-dev \
            libavutil-dev \
            libswscale-dev \
            libsqlite3-dev \
            libuv1-dev \
            libllhttp-dev \
            libcurl4-openssl-dev \
            libcjson-dev \
            libmbedtls-dev \
            libyaml-dev \
            lcov \
            golang-go \
            curl \
            ca-certificates \
            gpg

      - name: Checkout (with go2rtc submodule)
        uses: actions/checkout@v6
        with:
          submodules: recursive

      - name: Build go2rtc from vendored submodule
        run: |
          cd go2rtc
          CGO_ENABLED=0 go build -buildvcs=false -ldflags "-s -w" -trimpath -o go2rtc
          chmod +x go2rtc
          ./go2rtc --version
          echo "GO2RTC_TEST_BINARY=$GITHUB_WORKSPACE/go2rtc/go2rtc" >> "$GITHUB_ENV"

      - name: Configure with go2rtc enabled (+coverage)
        run: |
          mkdir -p build
          cd build
          # --coverage on both CFLAGS and linker so the new yaml/go2rtc
          # tests contribute to Codecov (previously the tests ran but
          # their coverage was never uploaded, pinning patch coverage at
          # ~0% for files only exercised here).
          # Use atomic gcov counters for code exercised by multiple threads.
          cmake .. \
            -DCMAKE_BUILD_TYPE=Debug \
            -DBUILD_TESTS=ON \
            -DENABLE_SOD=OFF \
            -DENABLE_GO2RTC=ON \
            -DENABLE_LITERT=OFF \
            -DENABLE_MQTT=OFF \
            -DCMAKE_C_FLAGS="--coverage -fprofile-update=atomic" \
            -DCMAKE_EXE_LINKER_FLAGS="--coverage"

      - name: Build go2rtc-related tests
        run: |
          cd build
          make -j$(nproc) \
            test_yaml_validate \
            test_yaml_redact \
            test_db_system_settings \
            test_go2rtc_process_detection \
            test_go2rtc_process_config_generation \
            test_go2rtc_binary_detection \
            test_go2rtc_two_config_merge \
            lightnvr-ffmpeg-guard \
            test_ffmpeg_guard_probe

      - name: Run go2rtc-related tests
        run: |
          cd build
          ctest --output-on-failure -V \
            -R "test_yaml_validate|test_yaml_redact|test_db_system_settings|test_go2rtc_process_detection|test_go2rtc_process_config_generation|test_go2rtc_binary_detection|test_go2rtc_two_config_merge|test_ffmpeg_guard"

      - name: Generate coverage report
        if: always()
        run: |
          cd build
          lcov --capture \
               --directory . \
               --output-file coverage.info \
               --ignore-errors mismatch

          lcov --remove coverage.info \
               '/usr/*' \
               '*/third_party/*' \
               '*/external/*' \
               '*/tests/*' \
               --output-file coverage_filtered.info \
               --ignore-errors unused

          lcov --list coverage_filtered.info

      - name: Upload coverage to Codecov
        uses: codecov/codecov-action@v7
        if: always()
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          files: build/coverage_filtered.info
          flags: go2rtc-unity
          name: lightnvr-go2rtc-coverage
          fail_ci_if_error: false

      - name: Upload test artifacts on failure
        uses: actions/upload-artifact@v7
        if: failure()
        with:
          name: go2rtc-unity-test-results
          path: build/Testing/
          retention-days: 7

  playwright-tests:
    name: Playwright Integration Tests (Layer 3, shard ${{ matrix.shard }}/2)
    needs: unity-tests
    runs-on: ubuntu-latest
    container: debian:sid-slim
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        # Keep one worker per LightNVR instance, but split the suite across
        # isolated jobs so shared database state cannot leak between workers.
        shard: [1, 2]

    steps:
      - name: Install system dependencies
        run: |
          apt-get update
          apt-get install -y \
            git \
            build-essential \
            cmake \
            pkg-config \
            libavcodec-dev \
            libavformat-dev \
            libavutil-dev \
            libswscale-dev \
            libsqlite3-dev \
            libuv1-dev \
            libllhttp-dev \
            libcurl4-openssl-dev \
            libcjson-dev \
            libmbedtls-dev \
            ffmpeg \
            curl \
            jq \
            golang-go \
            procps

      - name: Checkout
        uses: actions/checkout@v6

      - name: Checkout go2rtc submodule
        # LiteRT is disabled for this job and is much larger than go2rtc, so
        # avoid spending several minutes recursively cloning unused sources.
        run: |
          # actions/checkout records safe.directory outside the job container;
          # add it inside the container before invoking Git in a later step.
          git config --global --add safe.directory "$GITHUB_WORKSPACE"
          git submodule update --init --depth 1 go2rtc

      - name: Set up Node.js
        uses: actions/setup-node@v6
        with:
          node-version-file: .nvmrc
          cache: npm
          cache-dependency-path: |
            package-lock.json
            web/package-lock.json

      - name: Build go2rtc from source
        run: |
          # Build go2rtc from the opensensor fork submodule
          cd go2rtc
          echo "Building go2rtc from source..."
          CGO_ENABLED=0 go build -buildvcs=false -ldflags "-s -w" -trimpath -o go2rtc
          chmod +x go2rtc
          ./go2rtc --version || echo "go2rtc built successfully"
          echo "go2rtc binary size: $(stat -c%s go2rtc) bytes"
          # Binary is now at go2rtc/go2rtc - exactly where lightnvr-test.ini expects it

      - name: Build lightNVR
        run: |
          mkdir -p build
          cd build
          cmake .. \
            -DCMAKE_BUILD_TYPE=Release \
            -DENABLE_GO2RTC=ON \
            -DENABLE_LITERT=OFF \
            -DGO2RTC_BINARY_PATH=$(pwd)/../go2rtc/go2rtc
          make -j$(nproc)
          echo "lightNVR built successfully"
          ls -la bin/lightnvr

      - name: Build web frontend
        run: |
          cd web
          npm ci
          npm run build
          echo "Web frontend built successfully"
          ls -la dist/ | head -10

      - name: Install Playwright dependencies
        run: |
          npm ci
          npx playwright install --with-deps chromium

      - name: Run Playwright UI tests
        run: |
          # Clean any existing playwright reports to ensure fresh state
          rm -rf playwright-report test-results
          mkdir -p playwright-report test-results

          # Pre-flight checks before running tests
          echo "=== Pre-flight checks ==="
          echo "Checking if lightNVR binary exists..."
          ls -la build/bin/lightnvr || echo "lightNVR binary NOT found"
          echo ""
          echo "Checking if go2rtc binary exists..."
          ls -la go2rtc/go2rtc || echo "go2rtc binary NOT found"
          echo ""
          echo "Checking if web/dist exists..."
          ls -la web/dist/ | head -5 || echo "web/dist NOT found"
          echo ""
          echo "Checking if config file exists..."
          ls -la config/lightnvr-test.ini || echo "Config file NOT found"
          echo ""

          echo ""
          echo "=== Running Playwright shard ${{ matrix.shard }}/2 ==="

          # Stream output into the job log while preserving a copy for the
          # failure artifact. This also leaves useful progress visible if a
          # runner is cancelled externally.
          RESULT=0
          set +e
          npx playwright test --project=ui --shard=${{ matrix.shard }}/2 2>&1 | tee playwright-output.log
          RESULT=${PIPESTATUS[0]}
          set -e

          if [ $RESULT -ne 0 ]; then
            echo ""
            echo "=== Playwright tests failed ==="
            echo ""
            echo "Checking if lightNVR is still running..."
            pgrep -a lightnvr || echo "lightNVR is NOT running"
            pgrep -a go2rtc || echo "go2rtc is NOT running"
            echo ""
            echo "Checking lightNVR logs..."
            if [ -f /tmp/lightnvr-test/lightnvr.log ]; then
              echo "=== Last 200 lines of lightnvr.log ==="
              tail -200 /tmp/lightnvr-test/lightnvr.log
            else
              echo "No log file found at /tmp/lightnvr-test/lightnvr.log"
            fi
            echo ""
            echo "Checking for core dumps..."
            ls -la /tmp/core* 2>/dev/null || echo "No core dumps found"
            echo ""
            exit $RESULT
          fi
        shell: bash
        env:
          CI: true

      - name: Upload Playwright report
        uses: actions/upload-artifact@v7
        if: always()
        with:
          name: playwright-report-${{ matrix.shard }}
          path: playwright-report/
          retention-days: 30

      - name: Upload test screenshots
        uses: actions/upload-artifact@v7
        if: failure()
        with:
          name: test-screenshots-${{ matrix.shard }}
          path: test-results/
          retention-days: 7

      - name: Upload lightNVR logs
        uses: actions/upload-artifact@v7
        if: failure()
        with:
          name: lightnvr-logs-${{ matrix.shard }}
          path: |
            /tmp/lightnvr-test/
            playwright-output.log
          retention-days: 7

      - name: Cleanup
        if: always()
        run: |
          # Stop any remaining processes
          pkill -f lightnvr || true
          pkill -f go2rtc || true

          # Clean up test artifacts
          rm -rf /tmp/lightnvr-test