gfx: standalone fixes + the video pixel-format vocabulary

src/app/main.cpp

@@ -273,6 +273,9 @@ static void setup_x11(int argc, char** argv)
 
         helper_dylibs.run_under_x11 = true;
         helper_dylibs.xwayland = wayland;
+
+        // EGL is the only way to get zero-copy with dma-buf import
+        qputenv("QT_XCB_GL_INTEGRATION", "xcb_egl");
       }
     }
   };
@@ -518,6 +521,7 @@ static void setup_opengl(bool& enable_opengl_ui)
 
 #ifndef QT_NO_OPENGL
 #if (defined(__arm__) || defined(__aarch64__)) && !defined(_WIN32) && !defined(__APPLE__)
+  // Raspberry Pi & such
   QSurfaceFormat fmt = QSurfaceFormat::defaultFormat();
   fmt.setRenderableType(QSurfaceFormat::OpenGLES);
   fmt.setSwapInterval(1);
@@ -533,6 +537,7 @@ static void setup_opengl(bool& enable_opengl_ui)
   fmt.setDefaultFormat(fmt);
 #else
   {
+    // Desktop GL
     std::vector<std::pair<int, int>> versions_to_test
         = {{4, 6}, {4, 5}, {4, 4}, {4, 3}, {4, 2}, {4, 1}, {4, 0},
            {3, 3}, {3, 2}, {3, 1}, {3, 0}, {2, 1}, {2, 0}};
@@ -571,7 +576,9 @@ static void setup_opengl(bool& enable_opengl_ui)
 
     QSurfaceFormat fmt = QSurfaceFormat::defaultFormat();
     fmt.setProfile(QSurfaceFormat::CoreProfile);
+    fmt.setRenderableType(QSurfaceFormat::OpenGL);
     fmt.setSwapInterval(1);
+    fmt.setStencilBufferSize(8);
     bool ok = false;
     for(auto [maj, min] : versions_to_test)
     {

src/plugins/score-plugin-gfx/CMakeLists.txt

@@ -239,6 +239,20 @@ set(HDRS
     Gfx/Graph/encoders/UYVYCompute.hpp
     Gfx/Graph/encoders/BGRACompute.hpp
     Gfx/Graph/encoders/ComputeEncoder.hpp
+    Gfx/Graph/interop/VideoPixelFormatAV.hpp
+    Gfx/Graph/interop/VideoPixelFormatAV.cpp
+    Gfx/Graph/interop/VideoPixelFormat.hpp
+    Gfx/Graph/interop/V4L2PixelFormat.hpp
+    Gfx/Graph/interop/DirectShowPixelFormat.hpp
+    Gfx/Graph/interop/DrmPixelFormat.hpp
+    Gfx/Graph/interop/VideoPixelFormatQRhi.hpp
+    Gfx/Graph/interop/GStreamerPixelFormat.hpp
+    Gfx/Graph/interop/VideoPixelFormat.cpp
+    Gfx/Graph/interop/V4L2PixelFormat.cpp
+    Gfx/Graph/interop/DirectShowPixelFormat.cpp
+    Gfx/Graph/interop/DrmPixelFormat.cpp
+    Gfx/Graph/interop/VideoPixelFormatQRhi.cpp
+    Gfx/Graph/interop/GStreamerPixelFormat.cpp
 
     Gfx/Graph/decoders/GPUVideoDecoder.hpp
     Gfx/Graph/decoders/GPUVideoDecoderFactory.hpp

src/plugins/score-plugin-gfx/Gfx/CameraDevice.v4l2.cpp

@@ -30,99 +30,63 @@ void *v4l2_mmap(void *start, size_t length, int prot, int flags,
 int v4l2_munmap(void *_start, size_t length);
 }
 
+#include <Gfx/Graph/interop/V4L2PixelFormat.hpp>
+#include <Gfx/Graph/interop/VideoPixelFormatAV.hpp>
+
 namespace Gfx
 {
 namespace
 {
-// Imported from ffmpeg source code
-struct fmt_map
+// V4L2 exposes two kinds of fourcc: raw buffer layouts, and compressed
+// streams. The layouts are described once, in interop/V4L2PixelFormat, shared
+// with the DMA capture path -- this file used to carry a second, independently
+// maintained copy of that mapping. Only the codec side stays here, because a
+// codec is not a pixel format.
+struct compressed_fmt
 {
-  enum AVPixelFormat ff_fmt;
-  enum AVCodecID codec_id;
   uint32_t v4l2_fmt;
+  AVCodecID codec_id;
 };
 
-const struct fmt_map ff_fmt_conversion_table[] = {
-    //ff_fmt              codec_id              v4l2_fmt
-    {AV_PIX_FMT_YUV420P, AV_CODEC_ID_RAWVIDEO, V4L2_PIX_FMT_YUV420},
-    {AV_PIX_FMT_YUV420P, AV_CODEC_ID_RAWVIDEO, V4L2_PIX_FMT_YVU420},
-    {AV_PIX_FMT_YUV422P, AV_CODEC_ID_RAWVIDEO, V4L2_PIX_FMT_YUV422P},
-    {AV_PIX_FMT_YUYV422, AV_CODEC_ID_RAWVIDEO, V4L2_PIX_FMT_YUYV},
-    {AV_PIX_FMT_UYVY422, AV_CODEC_ID_RAWVIDEO, V4L2_PIX_FMT_UYVY},
-    {AV_PIX_FMT_YUV411P, AV_CODEC_ID_RAWVIDEO, V4L2_PIX_FMT_YUV411P},
-    {AV_PIX_FMT_YUV410P, AV_CODEC_ID_RAWVIDEO, V4L2_PIX_FMT_YUV410},
-    {AV_PIX_FMT_YUV410P, AV_CODEC_ID_RAWVIDEO, V4L2_PIX_FMT_YVU410},
-    {AV_PIX_FMT_RGB555LE, AV_CODEC_ID_RAWVIDEO, V4L2_PIX_FMT_RGB555},
-    {AV_PIX_FMT_RGB555BE, AV_CODEC_ID_RAWVIDEO, V4L2_PIX_FMT_RGB555X},
-    {AV_PIX_FMT_RGB565LE, AV_CODEC_ID_RAWVIDEO, V4L2_PIX_FMT_RGB565},
-    {AV_PIX_FMT_RGB565BE, AV_CODEC_ID_RAWVIDEO, V4L2_PIX_FMT_RGB565X},
-    {AV_PIX_FMT_BGR24, AV_CODEC_ID_RAWVIDEO, V4L2_PIX_FMT_BGR24},
-    {AV_PIX_FMT_RGB24, AV_CODEC_ID_RAWVIDEO, V4L2_PIX_FMT_RGB24},
-#ifdef V4L2_PIX_FMT_XBGR32
-    {AV_PIX_FMT_BGR0, AV_CODEC_ID_RAWVIDEO, V4L2_PIX_FMT_XBGR32},
-    {AV_PIX_FMT_0RGB, AV_CODEC_ID_RAWVIDEO, V4L2_PIX_FMT_XRGB32},
-    {AV_PIX_FMT_BGRA, AV_CODEC_ID_RAWVIDEO, V4L2_PIX_FMT_ABGR32},
-    {AV_PIX_FMT_ARGB, AV_CODEC_ID_RAWVIDEO, V4L2_PIX_FMT_ARGB32},
-#endif
-    {AV_PIX_FMT_BGR0, AV_CODEC_ID_RAWVIDEO, V4L2_PIX_FMT_BGR32},
-    {AV_PIX_FMT_0RGB, AV_CODEC_ID_RAWVIDEO, V4L2_PIX_FMT_RGB32},
-    {AV_PIX_FMT_GRAY8, AV_CODEC_ID_RAWVIDEO, V4L2_PIX_FMT_GREY},
-#ifdef V4L2_PIX_FMT_Y16
-    {AV_PIX_FMT_GRAY16LE, AV_CODEC_ID_RAWVIDEO, V4L2_PIX_FMT_Y16},
-#endif
-#ifdef V4L2_PIX_FMT_Z16
-    {AV_PIX_FMT_GRAY16LE, AV_CODEC_ID_RAWVIDEO, V4L2_PIX_FMT_Z16},
-#endif
-    {AV_PIX_FMT_NV12, AV_CODEC_ID_RAWVIDEO, V4L2_PIX_FMT_NV12},
-    {AV_PIX_FMT_NONE, AV_CODEC_ID_MJPEG, V4L2_PIX_FMT_MJPEG},
-    {AV_PIX_FMT_NONE, AV_CODEC_ID_MJPEG, V4L2_PIX_FMT_JPEG},
+const compressed_fmt compressed_formats[] = {
+    {V4L2_PIX_FMT_MJPEG, AV_CODEC_ID_MJPEG},
+    {V4L2_PIX_FMT_JPEG, AV_CODEC_ID_MJPEG},
 #ifdef V4L2_PIX_FMT_H264
-    {AV_PIX_FMT_NONE, AV_CODEC_ID_H264, V4L2_PIX_FMT_H264},
+    {V4L2_PIX_FMT_H264, AV_CODEC_ID_H264},
 #endif
 #ifdef V4L2_PIX_FMT_MPEG4
-    {AV_PIX_FMT_NONE, AV_CODEC_ID_MPEG4, V4L2_PIX_FMT_MPEG4},
+    {V4L2_PIX_FMT_MPEG4, AV_CODEC_ID_MPEG4},
 #endif
 #ifdef V4L2_PIX_FMT_CPIA1
-    {AV_PIX_FMT_NONE, AV_CODEC_ID_CPIA, V4L2_PIX_FMT_CPIA1},
-#endif
-#ifdef V4L2_PIX_FMT_SRGGB8
-    {AV_PIX_FMT_BAYER_BGGR8, AV_CODEC_ID_RAWVIDEO, V4L2_PIX_FMT_SBGGR8},
-    {AV_PIX_FMT_BAYER_GBRG8, AV_CODEC_ID_RAWVIDEO, V4L2_PIX_FMT_SGBRG8},
-    {AV_PIX_FMT_BAYER_GRBG8, AV_CODEC_ID_RAWVIDEO, V4L2_PIX_FMT_SGRBG8},
-    {AV_PIX_FMT_BAYER_RGGB8, AV_CODEC_ID_RAWVIDEO, V4L2_PIX_FMT_SRGGB8},
+    {V4L2_PIX_FMT_CPIA1, AV_CODEC_ID_CPIA},
 #endif
-    {AV_PIX_FMT_NONE, AV_CODEC_ID_NONE, 0},
 };
 
-enum AVPixelFormat ff_fmt_v4l2ff(uint32_t v4l2_fmt, enum AVCodecID codec_id)
+AVCodecID ff_fmt_v4l2codec(uint32_t v4l2_fmt)
 {
-  int i;
-
-  for(i = 0; ff_fmt_conversion_table[i].codec_id != AV_CODEC_ID_NONE; i++)
-  {
-    if(ff_fmt_conversion_table[i].v4l2_fmt == v4l2_fmt
-       && ff_fmt_conversion_table[i].codec_id == codec_id)
-    {
-      return ff_fmt_conversion_table[i].ff_fmt;
-    }
-  }
-
-  return AV_PIX_FMT_NONE;
+  for(const auto& c : compressed_formats)
+    if(c.v4l2_fmt == v4l2_fmt)
+      return c.codec_id;
+  return score::gfx::interop::fromV4L2PixelFormat(v4l2_fmt)
+                 != score::gfx::interop::VideoPixelFormat::Unknown
+             ? AV_CODEC_ID_RAWVIDEO
+             : AV_CODEC_ID_NONE;
 }
 
-enum AVCodecID ff_fmt_v4l2codec(uint32_t v4l2_fmt)
+AVPixelFormat ff_fmt_v4l2ff(uint32_t v4l2_fmt, AVCodecID codec_id)
 {
-  int i;
-
-  for(i = 0; ff_fmt_conversion_table[i].codec_id != AV_CODEC_ID_NONE; i++)
-  {
-    if(ff_fmt_conversion_table[i].v4l2_fmt == v4l2_fmt)
-    {
-      return ff_fmt_conversion_table[i].codec_id;
-    }
-  }
-
-  return AV_CODEC_ID_NONE;
+  if(codec_id != AV_CODEC_ID_RAWVIDEO)
+    return AV_PIX_FMT_NONE;
+
+  using namespace score::gfx::interop;
+  const auto layout = fromV4L2PixelFormat(v4l2_fmt);
+  if(const auto av = toAVPixelFormat(layout); av != AV_PIX_FMT_NONE)
+    return av;
+  // The V-before-U layouts have no AVPixelFormat of their own. Name the twin so
+  // the format stays offered, as it was before this mapping was shared; a
+  // consumer wanting correct chroma must exchange the U and V planes, which
+  // chromaSwappedTwin() is what records.
+  return toAVPixelFormat(chromaSwappedTwin(layout));
 }
 
 class libv4l2

src/plugins/score-plugin-gfx/Gfx/CameraDevice.win32.cpp

@@ -17,6 +17,9 @@ extern "C" {
 #include <wmcodecdsp.h>
 #include <oleauto.h>
 
+#include <Gfx/Graph/interop/DirectShowPixelFormat.hpp>
+#include <Gfx/Graph/interop/VideoPixelFormatAV.hpp>
+
 namespace Gfx
 {
 static constexpr const GUID MEDIASUBTYPE_Y800             = { 0x30303859, 0x0000, 0x0010, { 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71}};
@@ -38,8 +41,52 @@ static constexpr const GUID MEDIASUBTYPE_Y210             = { '012Y', 0x0000, 0x
 static constexpr const GUID MEDIASUBTYPE_Y216             = { '612Y', 0x0000, 0x0010, { 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71 }};
 static constexpr const GUID MEDIASUBTYPE_P408             = { '804P', 0x0000, 0x0010, { 0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71 }};
 
+// Every YUV MEDIASUBTYPE is {fourcc, 0x0000, 0x0010, {0x80,0,0,0xaa,0,0x38,0x9b,0x71}},
+// so the subtype reduces to the fourcc in Data1. Returns 0 for the RGB subtypes,
+// which are genuine SDK GUIDs and are matched below.
+static uint32_t subtypeFourcc(const GUID& subtype) noexcept
+{
+  static const uint8_t suffix[8]
+      = {0x80, 0x00, 0x00, 0xaa, 0x00, 0x38, 0x9b, 0x71};
+  if(subtype.Data2 != 0x0000 || subtype.Data3 != 0x0010)
+    return 0;
+  for(int i = 0; i < 8; ++i)
+    if(subtype.Data4[i] != suffix[i])
+      return 0;
+  return uint32_t(subtype.Data1);
+}
+
+// True when the subtype names a compressed stream. Previously this was inferred
+// from guidToPixelFormat happening to answer AV_PIX_FMT_YUVJ420P, which meant a
+// camera genuinely offering that layout would have been taken for MJPEG.
+static bool isCompressedSubtype(const GUID& subtype) noexcept
+{
+  const auto fourcc = subtypeFourcc(subtype);
+  return fourcc != 0
+         && score::gfx::interop::isDirectShowCompressedFourcc(fourcc);
+}
+
 static int guidToPixelFormat(const GUID& subtype)
 {
+  // The YUV layouts come from the shared, host-testable fourcc table, so they
+  // cannot drift and are not carrying the chroma swaps this function used to.
+  if(const auto fourcc = subtypeFourcc(subtype))
+  {
+    using namespace score::gfx::interop;
+    const auto layout = fromDirectShowFourcc(fourcc);
+    if(layout != VideoPixelFormat::Unknown)
+    {
+      if(const auto av = toAVPixelFormat(layout); av != AV_PIX_FMT_NONE)
+        return av;
+      // The V-before-U layouts have no AVPixelFormat of their own; name the twin
+      // so the format stays offered, and chromaSwappedTwin records that a
+      // consumer wanting correct chroma must exchange the U and V planes.
+      if(const auto twin = toAVPixelFormat(chromaSwappedTwin(layout));
+         twin != AV_PIX_FMT_NONE)
+        return twin;
+    }
+  }
+
   if(subtype == MEDIASUBTYPE_RGB24)
     return AV_PIX_FMT_BGR24;
   else if(subtype == MEDIASUBTYPE_RGB32)
@@ -51,6 +98,8 @@ static int guidToPixelFormat(const GUID& subtype)
   else if(subtype == MEDIASUBTYPE_RGB555)
     return AV_PIX_FMT_BGR555LE;
   else if(subtype == MEDIASUBTYPE_ARGB1555)
+    // FFmpeg has no 5:5:5 format carrying alpha, so the single alpha bit is
+    // dropped. The layout is otherwise identical to RGB555.
     return AV_PIX_FMT_BGR555LE;
   else if(subtype == MEDIASUBTYPE_RGB8)
     return AV_PIX_FMT_PAL8;
@@ -86,8 +135,6 @@ static int guidToPixelFormat(const GUID& subtype)
     return AV_PIX_FMT_YUV420P;
   else if(subtype == MEDIASUBTYPE_I420)
     return AV_PIX_FMT_YUV420P;
-  else if(subtype == MEDIASUBTYPE_YV12)
-    return AV_PIX_FMT_YUV420P;
   else if(subtype == MEDIASUBTYPE_NV12)
     return AV_PIX_FMT_NV12;
 //  else if(subtype == MEDIASUBTYPE_NV21)
@@ -114,8 +161,6 @@ static int guidToPixelFormat(const GUID& subtype)
     return AV_PIX_FMT_Y210LE;
   else if(subtype == MEDIASUBTYPE_Y216)
     return AV_PIX_FMT_Y216LE;
-  else if(subtype == MEDIASUBTYPE_V216) // not sure
-    return AV_PIX_FMT_Y216LE;
 
 /////   Not supported in dshow as of ffmpeg 8, causes hangs in avformat_find_stream_info
   else if(subtype == MEDIASUBTYPE_Y8) {
@@ -130,7 +175,7 @@ static int guidToPixelFormat(const GUID& subtype)
   else if(subtype == MEDIASUBTYPE_MICROSOFT_IR8) {
     return AV_PIX_FMT_GRAY8;
   }
-  else if (subtype == MEDIASUBTYPE_YVU9 || subtype == MEDIASUBTYPE_IF09)
+  else if(subtype == MEDIASUBTYPE_IF09)
     return AV_PIX_FMT_YUV410P;
 
   else if (subtype == MEDIASUBTYPE_Y411 || subtype == MEDIASUBTYPE_Y41P)
@@ -144,8 +189,6 @@ static int guidToPixelFormat(const GUID& subtype)
     return AV_PIX_FMT_YUVJ420P;
   else if(subtype == MEDIASUBTYPE_Plum)
     return AV_PIX_FMT_YUVJ420P;
-  else if(subtype == MEDIASUBTYPE_YVU9)
-    return AV_PIX_FMT_YUV410P;
   else
     return AV_PIX_FMT_NONE;
 }
@@ -163,7 +206,8 @@ static void enumerateCameraFormat(
   source.pixelformat = guidToPixelFormat(subtype);
 
   // Basic validation
-  if(width <= 0 || height <= 0 || source.pixelformat == AV_PIX_FMT_NONE)
+  if(width <= 0 || height <= 0
+     || (source.pixelformat == AV_PIX_FMT_NONE && !isCompressedSubtype(subtype)))
   {
     // OLECHAR* guidString;
     // StringFromCLSID(subtype, &guidString);
@@ -172,8 +216,7 @@ static void enumerateCameraFormat(
     return;
   }
 
-  // MJPEG special case
-  if(source.pixelformat == AV_PIX_FMT_YUVJ420P)
+  if(isCompressedSubtype(subtype))
   {
     source.codec = AV_CODEC_ID_MJPEG;
     source.pixelformat = AV_PIX_FMT_NONE;