diff --git a/cmake/ScoreCodeviewWindows.cmake b/cmake/ScoreCodeviewWindows.cmake index d40f97249c..07cbe37814 100644 --- a/cmake/ScoreCodeviewWindows.cmake +++ b/cmake/ScoreCodeviewWindows.cmake @@ -1,10 +1,23 @@ if(WIN32) + # Clang only. GCC's CodeView writer segfaults on any boost::container type, + # which this codebase uses in around thirty translation units, so -gcodeview + # makes a gcc Debug or RelWithDebInfo build impossible to complete: + # + # $ cat REPRO.cpp + # #include + # void f() { boost::container::vector v; (void)v; } + # $ g++ -gcodeview -c REPRO.cpp + # REPRO.cpp:2:54: internal compiler error: Segmentation fault + # + # Reproduced on gcc 16.2.0 (MSYS2 UCRT64) with boost 1.91. The flag alone is + # enough -- no optimisation level or standard setting is involved -- and DWARF + # is unaffected, as is clang. The crash is in the type-record emission reached + # from dwarf2out_finish, which is why every report points at the last line of + # the file. if(CMAKE_C_COMPILER_ID MATCHES "Clang") if(NOT (CMAKE_C_COMPILER_FRONTEND_VARIANT STREQUAL "MSVC")) set(SCORE_COMPILER_NEEDS_GCODEVIEW 1) endif() - elseif(CMAKE_CXX_COMPILER_ID MATCHES "GNU") - set(SCORE_COMPILER_NEEDS_GCODEVIEW 1) endif() if (SCORE_COMPILER_NEEDS_GCODEVIEW) diff --git a/cmake/ScoreTests.cmake b/cmake/ScoreTests.cmake index d831003b5d..e985bff6b1 100644 --- a/cmake/ScoreTests.cmake +++ b/cmake/ScoreTests.cmake @@ -121,11 +121,23 @@ function(score_add_test NAME) add_test(NAME ${NAME} COMMAND ${NAME}) endif() + # Catch2 exits with 4 when every test case in the binary was skipped + # (AllTestsSkippedExitCode, catch_session.cpp). A test that skips because its + # precondition is absent -- no display, no shader library, no capture device -- + # is not a defect, and counting it as one silently inflates the failure count. + set_tests_properties(${NAME} PROPERTIES SKIP_RETURN_CODE 4) + # App/integration tests rely on runtime dynamic-plugin discovery from # "/plugins": run them from the build root where /plugins lives. if(ARG_APP OR ARG_GUI) set_tests_properties(${NAME} PROPERTIES WORKING_DIRECTORY "${SCORE_ROOT_BINARY_DIR}") + + # ...and tell the fixture where that is, so running the executable by hand + # from some other directory boots the same application instead of one with + # no plug-ins at all. See prepare_test_environment(). + target_compile_definitions(${NAME} PRIVATE + "SCORE_TEST_BINARY_DIR=\"${SCORE_ROOT_BINARY_DIR}\"") endif() if(ARG_APP) diff --git a/src/app/Application.cpp b/src/app/Application.cpp index 77ca595e7c..2c9dfd6e7c 100644 --- a/src/app/Application.cpp +++ b/src/app/Application.cpp @@ -196,7 +196,8 @@ bool runningUnderAnUISession() noexcept if(qgetenv("XDG_SESSION_TYPE") != "tty") return true; if(platform.contains("gl") || platform.contains("vkkhr") - || platform.contains("linuxfb") || platform.contains("vnc")) + || platform.contains("linuxfb") || platform.contains("vnc") + || platform.contains("offscreen")) return true; return false; #endif diff --git a/src/app/main.cpp b/src/app/main.cpp index 05e996dd6d..f8dc64c2ff 100644 --- a/src/app/main.cpp +++ b/src/app/main.cpp @@ -247,8 +247,12 @@ static void setup_x11(int argc, char** argv) if(!x11 && !wayland) { - // Try eglfs - qputenv("QT_QPA_PLATFORM", "eglfs"); + // Try eglfs -- unless a platform was asked for. Overwriting it here sent + // QT_QPA_PLATFORM=offscreen to eglfs, which finds no display device, and + // Qt then qFatal()s on the first window for having no screens. A headless + // machine asking for offscreen has to get offscreen. + if(!has_platform) + qputenv("QT_QPA_PLATFORM", "eglfs"); return; } static constexpr auto setup_x11_error_handling = [] { diff --git a/src/lib/core/presenter/DocumentManager.cpp b/src/lib/core/presenter/DocumentManager.cpp index 269b1db377..24424730ef 100644 --- a/src/lib/core/presenter/DocumentManager.cpp +++ b/src/lib/core/presenter/DocumentManager.cpp @@ -242,8 +242,15 @@ void DocumentManager::setCurrentDocument( bool DocumentManager::closeDocument( const score::GUIApplicationContext& ctx, Document& doc) { - // Warn the user if he might loose data - if(!doc.commandStack().isAtSavedIndex()) + // Warn the user if he might loose data. Only when there is a user: with + // applicationSettings.gui false (headless, --script, offscreen QPA) nothing + // can answer a modal, and QMessageBox::exec() aborts instead of returning. + // Every score::MessageBox helper already guards on this same flag; this call + // site was the one raw QMessageBox left, which is why a scripted /exit on a + // modified document died in teardown with SIGABRT. No GUI means no one to + // save for, so proceed as Discard -- the same outcome forceExit() already + // produces, since it quits 500ms later whatever the answer would have been. + if(!doc.commandStack().isAtSavedIndex() && ctx.applicationSettings.gui) { QMessageBox msgBox; msgBox.setText(tr("The document has been modified.")); diff --git a/src/lib/score/application/ApplicationContext.hpp b/src/lib/score/application/ApplicationContext.hpp index 8e05f5013a..8bf36fe38d 100644 --- a/src/lib/score/application/ApplicationContext.hpp +++ b/src/lib/score/application/ApplicationContext.hpp @@ -39,17 +39,34 @@ struct SCORE_LIB_BASE_EXPORT ApplicationContext */ template T& settings() const + { + if(auto c = findSettings()) + return *c; + + SCORE_ABORT; + throw; + } + + /** + * @brief Access a Settings model instance, or null when its plug-in did not + * register one. + * + * settings() aborts the process in that case, which is the right answer for + * application code (a missing settings model means the plug-in it belongs to + * is not loaded, and nothing downstream can work). Callers that can report + * the situation themselves want the null instead. + */ + template + T* findSettings() const noexcept { for(auto& elt : this->m_settings) { if(auto c = dynamic_cast(elt.get())) { - return *c; + return c; } } - - SCORE_ABORT; - throw; + return nullptr; } const auto& allSettings() const noexcept { return m_settings; } diff --git a/src/lib/score/gfx/Vulkan.cpp b/src/lib/score/gfx/Vulkan.cpp index 800fb605cf..309daca8c3 100644 --- a/src/lib/score/gfx/Vulkan.cpp +++ b/src/lib/score/gfx/Vulkan.cpp @@ -60,9 +60,17 @@ QVulkanInstance* staticVulkanInstance(bool create) if(!instance.create()) { g_staticVulkanInstanceInvalid = true; + delete g_staticVulkanInstance; + g_staticVulkanInstance = nullptr; } }); + // Re-check: on the very first call, create() may just have failed inside + // call_once — returning the half-initialized instance would send callers + // (Graph's API-fallback check, QRhi::create) straight into a crash. + if(g_staticVulkanInstanceInvalid) + return nullptr; + return g_staticVulkanInstance; } } diff --git a/src/lib/score/tools/File.cpp b/src/lib/score/tools/File.cpp index 6bc11a9284..a84a6d1013 100644 --- a/src/lib/score/tools/File.cpp +++ b/src/lib/score/tools/File.cpp @@ -80,6 +80,18 @@ QString addUniqueSuffix(const QString& fileName) } } +QString locateFilePath(const QString& filename) noexcept +{ + if(filename.startsWith(":")) + { + QSettings set; + QString path = filename; + path.replace(":", set.value("Library/RootPath").toString() + "/"); + return QFileInfo{path}.absoluteFilePath(); + } + return filename; +} + QString locateFilePath(const QString& filename, const score::DocumentContext& ctx) noexcept { diff --git a/src/lib/score/tools/FilePath.hpp b/src/lib/score/tools/FilePath.hpp index 2b449ba6d5..fee5fa93c4 100644 --- a/src/lib/score/tools/FilePath.hpp +++ b/src/lib/score/tools/FilePath.hpp @@ -12,6 +12,13 @@ SCORE_LIB_BASE_EXPORT QString locateFilePath(const QString& filename, const score::DocumentContext& ctx) noexcept; +//! Same, for callers that run before any document exists -- a --script file is +//! read while the application is still starting up. : is a settings +//! lookup and resolves fine; : and relative paths have no document to +//! resolve against and are returned untouched. +SCORE_LIB_BASE_EXPORT +QString locateFilePath(const QString& filename) noexcept; + //! Will try to convert an absolute path //! in a relative path from the document's point of view SCORE_LIB_BASE_EXPORT diff --git a/src/plugins/score-plugin-gfx/3rdparty/libisf/src/isf.cpp b/src/plugins/score-plugin-gfx/3rdparty/libisf/src/isf.cpp index 66cb9114c9..cc2e208fa0 100644 --- a/src/plugins/score-plugin-gfx/3rdparty/libisf/src/isf.cpp +++ b/src/plugins/score-plugin-gfx/3rdparty/libisf/src/isf.cpp @@ -29,7 +29,7 @@ namespace { static constexpr struct glsl45_t { - static constexpr auto versionPrelude = R"_(#version 450 + static constexpr auto versionPrelude = R"_(#version 460 )_"; static constexpr auto vertexPrelude = R"_( @@ -3515,7 +3515,7 @@ void parser::parse_csf() m_fragment.clear(); // Add version - m_fragment += "#version 450\n\n"; + m_fragment += "#version 460\n\n"; // Add standard ProcessUBO uniforms (same as ISF/VSA) m_fragment += GLSL45.defaultUniforms; diff --git a/src/plugins/score-plugin-gfx/CMakeLists.txt b/src/plugins/score-plugin-gfx/CMakeLists.txt index fa670c0cf9..643785948c 100644 --- a/src/plugins/score-plugin-gfx/CMakeLists.txt +++ b/src/plugins/score-plugin-gfx/CMakeLists.txt @@ -287,6 +287,7 @@ set(HDRS Gfx/Window/CollapsibleSection.hpp Gfx/Window/DesktopLayout.hpp Gfx/Window/MultiWindowDevice.hpp + Gfx/Window/OffscreenDevice.hpp Gfx/Window/OutputMapping.hpp Gfx/Window/OutputPreview.hpp Gfx/Window/TestCard.hpp diff --git a/src/plugins/score-plugin-gfx/Gfx/GfxContext.cpp b/src/plugins/score-plugin-gfx/Gfx/GfxContext.cpp index 5fc416fffe..af0c3032ab 100644 --- a/src/plugins/score-plugin-gfx/Gfx/GfxContext.cpp +++ b/src/plugins/score-plugin-gfx/Gfx/GfxContext.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include @@ -10,6 +11,8 @@ #include #include + +#include #include #include @@ -492,6 +495,47 @@ void GfxContext::on_watchdog_timer(score::HighResolutionTimer* self) updateGraph(); } +void GfxContext::renderFrames(int frames) +{ + if(frames <= 0 || !m_graph) + return; + + const bool step = m_stepRate > 0.; + const int64_t frame_flicks + = step ? int64_t(std::llround(ossia::flicks_per_second / m_stepRate)) : 0; + // Held for the whole call so PROGRESS sweeps 0..1 across it rather than + // restarting on every frame. + const ossia::time_value span{frame_flicks * (m_stepFrame + frames)}; + + for(int i = 0; i < frames; i++) + { + // Same order as the timer-driven path: parameters first, then draw, so a + // value written by the script is visible in the frame that follows it. + updateGraph(); + + // After updateGraph, which would otherwise overwrite the UBO with the date + // the transport last sent. + if(step) + { + const score::gfx::Timings tk{ + .date = ossia::time_value{frame_flicks * m_stepFrame}, + .parent_duration = span}; + for(auto& [id, node] : nodes) + { + if(auto proc = dynamic_cast(node.get())) + proc->process(tk); + } + m_stepFrame++; + } + + for(auto output : m_graph->outputs()) + { + if(output && output->canRender()) + output->render(); + } + } +} + void GfxContext::on_manual_timer(score::HighResolutionTimer* self) { if(auto ptr = m_manualTimers.find(self); ptr != m_manualTimers.end()) diff --git a/src/plugins/score-plugin-gfx/Gfx/GfxContext.hpp b/src/plugins/score-plugin-gfx/Gfx/GfxContext.hpp index 7422bd1212..255c811139 100644 --- a/src/plugins/score-plugin-gfx/Gfx/GfxContext.hpp +++ b/src/plugins/score-plugin-gfx/Gfx/GfxContext.hpp @@ -79,6 +79,35 @@ class SCORE_PLUGIN_GFX_EXPORT GfxContext : public QObject void update_inputs(); void updateGraph(); + /** + * @brief Render exactly @p frames times, synchronously, and return. + * + * The normal path renders off wall-clock timers, so "how many frames have I + * drawn" depends on how long the caller happened to wait and how fast the + * machine is -- a harness that sleeps and then grabs gets a different frame + * on a Raspberry Pi than on a workstation, and an animated shader gives a + * different image every run. + * + * Stepping instead makes frame N mean the same thing everywhere, which is + * what lets a rendered frame be compared against a stored reference at all. + * + * Each step also hands every process node a synthetic token of + * `frame / stepRate()`, so TIME, TIMEDELTA, FRAMEINDEX and PROGRESS follow the + * counter rather than whatever the transport last delivered. Without it an + * animated shader keeps reading the execution clock and frame N is a + * different picture every run. + * + * Still on the execution clock: whatever a node takes from the transport + * itself rather than from its process UBO -- video decode position, + * automation -- so a graph built on those is only as reproducible as that + * clock is. + */ + void renderFrames(int frames); + + //! Step used by renderFrames(), in frames per second. + double stepRate() const noexcept { return m_stepRate; } + void setStepRate(double fps) noexcept { m_stepRate = fps; } + void send_message(score::gfx::Message&& msg) noexcept { tick_messages.enqueue(std::move(msg)); @@ -102,6 +131,9 @@ class SCORE_PLUGIN_GFX_EXPORT GfxContext : public QObject score::gfx::Graph* m_graph{}; QThread m_thread; + double m_stepRate{60.}; + int64_t m_stepFrame{}; + struct NodeCommand { enum diff --git a/src/plugins/score-plugin-gfx/Gfx/Graph/ScreenNode.cpp b/src/plugins/score-plugin-gfx/Gfx/Graph/ScreenNode.cpp index 1381b25136..b16bde7e45 100644 --- a/src/plugins/score-plugin-gfx/Gfx/Graph/ScreenNode.cpp +++ b/src/plugins/score-plugin-gfx/Gfx/Graph/ScreenNode.cpp @@ -175,9 +175,12 @@ createRenderState(GraphicsApi graphicsApi, QSize sz, QWindow* window) qCritical() << "createRenderState: QRhi::create(OpenGLES2) FAILED. " "This output will never render."; } - state.renderSize = sz; - populateCaps(state); - return st; + else + { + state.renderSize = sz; + populateCaps(state); + return st; + } } #endif @@ -204,6 +207,10 @@ createRenderState(GraphicsApi graphicsApi, QSize sz, QWindow* window) { params.inst = score::gfx::staticVulkanInstance(); } + // No instance (headless platform plugins cannot create one): bail to the + // null-rhi state instead of letting QRhi::create dereference it. + if(!params.inst) + return st; state.version = Gfx::Settings::shaderVersionForAPI(Vulkan); // Create shared VkDevice with video decode queues BEFORE QRhi. @@ -243,9 +250,12 @@ createRenderState(GraphicsApi graphicsApi, QSize sz, QWindow* window) if(!state.rhi) state.rhi = QRhi::create(QRhi::Vulkan, ¶ms, flags); - state.renderSize = sz; - populateCaps(state); - return st; + if(state.rhi) + { + state.renderSize = sz; + populateCaps(state); + return st; + } } #endif @@ -263,9 +273,12 @@ createRenderState(GraphicsApi graphicsApi, QSize sz, QWindow* window) // } state.version = Gfx::Settings::shaderVersionForAPI(D3D11); state.rhi = QRhi::create(QRhi::D3D11, ¶ms, flags); - state.renderSize = sz; - populateCaps(state); - return st; + if(state.rhi) + { + state.renderSize = sz; + populateCaps(state); + return st; + } } #if QT_VERSION >= QT_VERSION_CHECK(6, 9, 0) else if(graphicsApi == D3D12) @@ -281,9 +294,12 @@ createRenderState(GraphicsApi graphicsApi, QSize sz, QWindow* window) // } state.version = Gfx::Settings::shaderVersionForAPI(D3D12); state.rhi = QRhi::create(QRhi::D3D12, ¶ms, flags); - state.renderSize = sz; - populateCaps(state); - return st; + if(state.rhi) + { + state.renderSize = sz; + populateCaps(state); + return st; + } } #endif #endif @@ -294,9 +310,12 @@ createRenderState(GraphicsApi graphicsApi, QSize sz, QWindow* window) QRhiMetalInitParams params; state.version = Gfx::Settings::shaderVersionForAPI(Metal); state.rhi = QRhi::create(QRhi::Metal, ¶ms, flags); - state.renderSize = sz; - populateCaps(state); - return st; + if(state.rhi) + { + state.renderSize = sz; + populateCaps(state); + return st; + } } #endif diff --git a/src/plugins/score-plugin-gfx/Gfx/ShaderProgram.cpp b/src/plugins/score-plugin-gfx/Gfx/ShaderProgram.cpp index 5759c0d6bc..1afa6aea0b 100644 --- a/src/plugins/score-plugin-gfx/Gfx/ShaderProgram.cpp +++ b/src/plugins/score-plugin-gfx/Gfx/ShaderProgram.cpp @@ -412,7 +412,8 @@ ProgramCache::get(const ShaderSource& program) noexcept } ShaderSource -programFromISFFragmentShaderPath(const QString& fsFilename, QByteArray fsData) +programFromISFFragmentShaderPath( + const QString& fsFilename, QByteArray fsData, ShaderSource::ProgramType type) { // ISF works by storing a vertex shader next to the fragment shader. QString vertexName = fsFilename; @@ -454,7 +455,11 @@ programFromISFFragmentShaderPath(const QString& fsFilename, QByteArray fsData) resolveGLSLIncludes(fsData, shaderIncludePath, file.absolutePath(), 0); resolveGLSLIncludes(vertexData, shaderIncludePath, file.absolutePath(), 0); */ - return {ShaderSource::ProgramType::ISF, vertexData, fsData}; + // The MODE declared in the header decides how the pair is compiled: a + // RAW_RASTER_PIPELINE shader brings its own vertex stage and must not be given + // the ISF prelude, which does not declare `position`. Defaulted to ISF so the + // ISF process is unaffected. + return {type, vertexData, fsData}; } ShaderSource programFromVSAVertexShaderPath(const QString& vertexFilename, QByteArray vertexData) diff --git a/src/plugins/score-plugin-gfx/Gfx/ShaderProgram.hpp b/src/plugins/score-plugin-gfx/Gfx/ShaderProgram.hpp index d84195088c..ec4e38c125 100644 --- a/src/plugins/score-plugin-gfx/Gfx/ShaderProgram.hpp +++ b/src/plugins/score-plugin-gfx/Gfx/ShaderProgram.hpp @@ -115,7 +115,9 @@ struct SCORE_PLUGIN_GFX_EXPORT ShaderSource }; SCORE_PLUGIN_GFX_EXPORT ShaderSource -programFromISFFragmentShaderPath(const QString& fsFilename, QByteArray fsData); +programFromISFFragmentShaderPath( + const QString& fsFilename, QByteArray fsData, + ShaderSource::ProgramType type = ShaderSource::ProgramType::ISF); SCORE_PLUGIN_GFX_EXPORT ShaderSource programFromVSAVertexShaderPath(const QString& vertexFilename, QByteArray vertexData); } diff --git a/src/plugins/score-plugin-gfx/Gfx/Window/OffscreenDevice.hpp b/src/plugins/score-plugin-gfx/Gfx/Window/OffscreenDevice.hpp new file mode 100644 index 0000000000..20d1f6c46a --- /dev/null +++ b/src/plugins/score-plugin-gfx/Gfx/Window/OffscreenDevice.hpp @@ -0,0 +1,90 @@ +#pragma once + +#include +#include + +#include +#include +#include + +#include + +namespace Gfx +{ + +// Headless device used when SCORE_FORCE_OFFSCREEN_WINDOW selects this +// window device by name. Wraps a BackgroundNode — which already drives +// beginOffscreenFrame/endOffscreenFrame — without the ScenarioDocumentView +// dependency of background_device. Exposes only the parameters required by +// offscreen tests (size, rendersize) and holds the shared_readback used by +// WindowDevice::grabTo to write frames to disk. +class offscreen_device : public ossia::net::device_base +{ + score::gfx::BackgroundNode* m_node{}; + gfx_node_base m_root; + QObject m_qtContext; + + ossia::net::parameter_base* size_param{}; + ossia::net::parameter_base* rendersize_param{}; + +public: + offscreen_device(std::unique_ptr proto, std::string name) + : ossia::net::device_base{std::move(proto)} + , m_node{new score::gfx::BackgroundNode} + , m_root{*this, *static_cast(m_protocol.get()), m_node, name} + { + this->m_capabilities.change_tree = true; + m_node->shared_readback = std::make_shared(); + + { + auto size_node = std::make_unique("size", *this, m_root); + size_param = size_node->create_parameter(ossia::val_type::VEC2F); + size_param->push_value(ossia::vec2f{1280.f, 720.f}); + m_node->setSize(QSize{1280, 720}); + size_param->add_callback([this](const ossia::value& v) { + if(auto val = v.target()) + { + ossia::qt::run_async(&m_qtContext, [node = this->m_node, v = *val] { + node->setSize({(int)v[0], (int)v[1]}); + }); + } + }); + m_root.add_child(std::move(size_node)); + } + + { + auto size_node + = std::make_unique("rendersize", *this, m_root); + ossia::net::set_description( + *size_node, "Set to [0, 0] to use the viewport's size"); + rendersize_param = size_node->create_parameter(ossia::val_type::VEC2F); + rendersize_param->push_value(ossia::vec2f{0.f, 0.f}); + rendersize_param->add_callback([this](const ossia::value& v) { + if(auto val = v.target()) + { + ossia::qt::run_async(&m_qtContext, [node = this->m_node, v = *val] { + node->setRenderSize({(int)v[0], (int)v[1]}); + }); + } + }); + m_root.add_child(std::move(size_node)); + } + } + + ~offscreen_device() + { + // The graph owns the node: gfx_parameter_base hands it to register_node as + // a unique_ptr and gives it back in its own destructor, which + // clear_children() runs. Teardown order is the same as background_device's. + m_protocol->stop(); + m_root.clear_children(); + m_protocol.reset(); + } + + score::gfx::BackgroundNode* node() const noexcept { return m_node; } + + const gfx_node_base& get_root_node() const override { return m_root; } + gfx_node_base& get_root_node() override { return m_root; } +}; + +} diff --git a/src/plugins/score-plugin-gfx/Gfx/WindowDevice.cpp b/src/plugins/score-plugin-gfx/Gfx/WindowDevice.cpp index f6e1fe73bd..b03ef8c197 100644 --- a/src/plugins/score-plugin-gfx/Gfx/WindowDevice.cpp +++ b/src/plugins/score-plugin-gfx/Gfx/WindowDevice.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -11,6 +12,8 @@ #include #include +#include +#include #include @@ -19,6 +22,24 @@ W_OBJECT_IMPL(Gfx::WindowDevice) namespace Gfx { +// SCORE_FORCE_OFFSCREEN_WINDOW=Name1,Name2 forces any matching WindowDevice +// (whatever its Single/Background/MultiWindow mode) into a headless offscreen +// render path. Used by tests that need grabTo output but must not pop a +// platform window. +static bool shouldForceOffscreen(const QString& name) +{ + static const QByteArray env = qgetenv("SCORE_FORCE_OFFSCREEN_WINDOW"); + if(env.isEmpty()) + return false; + for(const auto& part : env.split(',')) + { + const auto trimmed = QString::fromUtf8(part).trimmed(); + if(!trimmed.isEmpty() && trimmed == name) + return true; + } + return false; +} + score::gfx::Window* WindowDevice::window() const noexcept { if(m_dev) @@ -75,6 +96,84 @@ void WindowDevice::disconnect() deviceChanged(prev.get(), nullptr); } +void WindowDevice::grabTo(const QString& path) const +{ + if(auto dev = dynamic_cast(m_dev.get())) + { + auto node = dev->node(); + if(!node || !node->shared_readback) + { + qWarning() << "grabTo: offscreen device has not rendered yet"; + return; + } + + const auto& rb = *node->shared_readback; + const int w = rb.pixelSize.width(); + const int h = rb.pixelSize.height(); + const int expected = w * h * 4; + + // BackgroundNode::render() clears the readback when its render list holds + // nothing but the output itself, which leaves the default-constructed + // QSize(-1, -1) here. + if(w <= 0 || h <= 0) + { + qWarning() << "grabTo: nothing rendered into" << m_settings.name + << "- no process is connected to this device's input"; + return; + } + if(rb.data.size() < expected) + { + qWarning() << "grabTo: readback is" << rb.data.size() << "bytes for" << w << "x" + << h << "- expected" << expected; + return; + } + + QImage img{ + reinterpret_cast(rb.data.constData()), w, h, w * 4, + QImage::Format_RGBA8888}; + if(!img.save(path)) + qWarning() << "grabTo: could not write" << path; + } + else if(auto win = this->window()) + { + // QScreen::grabWindow reads the framebuffer at the window's geometry, not + // the window's own buffer: anything drawn on top lands in the file. Valid + // to eyeball an interactive session, never valid as a reference image. + qWarning() << "grabTo: capturing the SCREEN at" << m_settings.name + << "geometry, not the rendered frame. Set SCORE_FORCE_OFFSCREEN_WINDOW=" + << m_settings.name << "for a real readback."; + auto grab = win->screen()->grabWindow(win->winId()); + if(!grab.save(path)) + qWarning() << "grabTo: could not write" << path; + } + else + { + qWarning() << "grabTo: device has no window and is not offscreen"; + } +} + +void WindowDevice::renderFrames(int frames) const +{ + if(auto plug = m_ctx.findPlugin()) + plug->context.renderFrames(frames); + else + qWarning() << "renderFrames: no gfx document plugin"; +} + +void WindowDevice::setStepRate(double fps) const +{ + if(auto plug = m_ctx.findPlugin()) + plug->context.setStepRate(fps); + else + qWarning() << "setStepRate: no gfx document plugin"; +} + +void WindowDevice::grabFrame(int frames, const QString& path) const +{ + renderFrames(frames); + grabTo(path); +} + bool WindowDevice::reconnect() { disconnect(); @@ -90,6 +189,18 @@ bool WindowDevice::reconnect() auto view = m_ctx.document.view(); auto main_view = view ? qobject_cast( &view->viewDelegate()) : nullptr; + + if(shouldForceOffscreen(m_settings.name)) + { + m_dev = std::make_unique( + std::unique_ptr(m_protocol), + m_settings.name.toStdString()); + + enableCallbacks(); + deviceChanged(nullptr, m_dev.get()); + return connected(); + } + switch(set.mode) { case WindowMode::Background: { diff --git a/src/plugins/score-plugin-gfx/Gfx/WindowDevice.hpp b/src/plugins/score-plugin-gfx/Gfx/WindowDevice.hpp index e0549f04e5..b385b36378 100644 --- a/src/plugins/score-plugin-gfx/Gfx/WindowDevice.hpp +++ b/src/plugins/score-plugin-gfx/Gfx/WindowDevice.hpp @@ -74,6 +74,28 @@ class SCORE_PLUGIN_GFX_EXPORT WindowDevice final : public GfxOutputDevice void disconnect() override; bool reconnect() override; + //! Write the current frame to @p path. On an offscreen device this reads the + //! render target back directly, which is what makes headless pixel testing + //! possible; on a real window it grabs the window. + void grabTo(const QString& path) const; + W_SLOT(grabTo) + + //! Render exactly @p frames times and return. See GfxContext::renderFrames. + //! Call it repeatedly to step frame by frame: the clock keeps counting across + //! calls, so renderFrames(1) sixty times is the timeline renderFrames(60) is. + void renderFrames(int frames) const; + W_SLOT(renderFrames) + + //! Frames per second the step clock advances by. 60 unless set. + void setStepRate(double fps) const; + W_SLOT(setStepRate) + + //! Render @p frames times, then write that frame. The point of naming the + //! frame rather than sleeping is that frame N is the same picture on every + //! machine, which is what a stored reference can be compared against. + void grabFrame(int frames, const QString& path) const; + W_SLOT(grabFrame) + private: gfx_protocol_base* m_protocol{}; mutable std::unique_ptr m_dev; diff --git a/src/plugins/score-plugin-js/JS/ApplicationPlugin.cpp b/src/plugins/score-plugin-js/JS/ApplicationPlugin.cpp index 59573ddd69..99a558e9fa 100644 --- a/src/plugins/score-plugin-js/JS/ApplicationPlugin.cpp +++ b/src/plugins/score-plugin-js/JS/ApplicationPlugin.cpp @@ -18,6 +18,8 @@ #include #include +#include +#include #if __has_include() #include @@ -33,6 +35,30 @@ namespace JS { +// Whether --script was given a program or the path of one. An existing file is +// always a path; anything with JS punctuation in it is a program. +static bool stringIsScript(const QString& input) +{ + if(input.isEmpty()) + return false; + + if(QFileInfo fileInfo{input}; fileInfo.exists() && fileInfo.isFile()) + return false; + + if(input.length() > 4096) + return true; + + for(QChar ch : input) + { + const char16_t c = ch.unicode(); + if(c == '\n' || c == '\r' || c == ';' || c == '{' || c == '}' || c == '(' + || c == ')') + return true; + } + + return true; +} + ApplicationPlugin::ApplicationPlugin(const score::GUIApplicationContext& ctx) : score::GUIApplicationPlugin{ctx} { @@ -45,10 +71,12 @@ ApplicationPlugin::ApplicationPlugin(const score::GUIApplicationContext& ctx) "Library", m_consoleEngine.newQObject(new JsLibrary)); m_consoleEngine.globalObject().setProperty("Device", m_consoleEngine.newQObject(new DeviceContext{m_consoleEngine})); m_consoleEngine.globalObject().setProperty("View", m_consoleEngine.newQObject(new JsViewContext)); - connect(&m_consoleEngine, &QQmlEngine::exit, this, [&] { + connect(&m_consoleEngine, &QQmlEngine::exit, this, [&](int retCode) { for(auto& doc : score::GUIAppContext().docManager.documents()) doc->commandStack().markCurrentIndexAsSaved(); - qApp->quit(); + // quit() is exit(0), which discarded the code Qt.exit() was given: a script + // could stop the app but never report that it had failed. + qApp->exit(retCode); QTimer::singleShot( 500, [] { score::GUIApplicationInterface::instance().forceExit(); }); }); @@ -83,7 +111,26 @@ ApplicationPlugin::ApplicationPlugin(const score::GUIApplicationContext& ctx) parser.addOption(script_opt); parser.parse(ctx.applicationSettings.arguments); - this->m_start_script = parser.value(script_opt); + const auto script = parser.value(script_opt); + if(stringIsScript(script)) + { + this->m_start_script = script; + } + else if(!script.isEmpty()) + { + QFile f{script}; + if(f.open(QIODevice::ReadOnly)) + { + this->m_start_script = f.readAll(); + this->m_start_script_name = script; + this->m_start_script_path = QFileInfo{f}.canonicalPath(); + } + else + { + qCritical() << "--script: cannot open" << script << ":" << f.errorString(); + this->m_start_script_failed = true; + } + } } void ApplicationPlugin::on_newDocument(score::Document& doc) @@ -126,9 +173,33 @@ void ApplicationPlugin::on_createdDocument(score::Document& doc) if(auto customData = doc.context().findPlugin(); !customData) score::addDocumentPlugin(doc); + if(m_start_script_failed) + { + qGuiApp->exit(2); + return; + } + if(!m_start_script.isEmpty()) { - QTimer::singleShot(100, this, [this] { m_consoleEngine.evaluate(m_start_script); }); + QTimer::singleShot(100, this, [this] { + if(!m_start_script_path.isEmpty()) + m_consoleEngine.addImportPath(m_start_script_path); + + // A --script that throws used to fail silently with exit code 0: an + // unresolvable readFile returns an empty string, eval("") is a no-op, and + // the process exits reporting success. A harness cannot tell that from a + // pass, so the whole run is unfalsifiable. + const auto res = m_consoleEngine.evaluate(m_start_script, m_start_script_name); + if(res.isError()) + { + qCritical().noquote() + << "--script:" + << (m_start_script_name.isEmpty() ? QStringLiteral("") + : m_start_script_name) + << "line" << res.property("lineNumber").toInt() << ":" << res.toString(); + qGuiApp->exit(3); + } + }); } } void ApplicationPlugin::afterStartup() diff --git a/src/plugins/score-plugin-js/JS/ApplicationPlugin.hpp b/src/plugins/score-plugin-js/JS/ApplicationPlugin.hpp index db4aae0d45..d9432ecfa0 100644 --- a/src/plugins/score-plugin-js/JS/ApplicationPlugin.hpp +++ b/src/plugins/score-plugin-js/JS/ApplicationPlugin.hpp @@ -45,5 +45,8 @@ class ApplicationPlugin final ossia::net::network_context_ptr m_asioContext; QString m_start_script; + QString m_start_script_name; //!< path as given, for error messages + QString m_start_script_path; //!< directory, added as a QML import path + bool m_start_script_failed{}; }; } diff --git a/src/plugins/score-plugin-js/JS/Qml/EditContext.scenario.cpp b/src/plugins/score-plugin-js/JS/Qml/EditContext.scenario.cpp index 122f5b914a..ad08ef2865 100644 --- a/src/plugins/score-plugin-js/JS/Qml/EditContext.scenario.cpp +++ b/src/plugins/score-plugin-js/JS/Qml/EditContext.scenario.cpp @@ -107,6 +107,12 @@ QObject* EditJsContext::createProcess(QObject* interval, QString name, QString d if(!doc) return nullptr; + // Processes that take a file open it themselves, and they do not know about + // the library prefixes. Without this a ":/..." shader silently loads + // as empty and the process is created with no content at all. + if(data.startsWith('<')) + data = locateFilePath(data); + std::optional maybe_uid; { if(name.trimmed().length() == 36) diff --git a/src/plugins/score-plugin-js/JS/Qml/Utils.cpp b/src/plugins/score-plugin-js/JS/Qml/Utils.cpp index c0bbeb4f4c..eb768167a3 100644 --- a/src/plugins/score-plugin-js/JS/Qml/Utils.cpp +++ b/src/plugins/score-plugin-js/JS/Qml/Utils.cpp @@ -71,12 +71,25 @@ bool JsUtils::canWriteFile(QString path) QByteArray JsUtils::readFile(QString path) { + const QString requested = path; if(auto doc = score::AppContext().currentDocument()) path = score::locateFilePath(path, *doc); + else + path = score::locateFilePath(path); QFile f(path); if(f.open(QIODevice::ReadOnly)) return f.readAll(); + + // An empty return is otherwise indistinguishable from an empty file, and a + // script doing eval(readFile(...)) on a path that does not resolve simply + // does nothing and reports success. Say which path was tried, resolved and + // unresolved: the two differ exactly when a :/: prefix is + // pointing somewhere unexpected. + if(requested == path) + qWarning().noquote() << "Score.readFile: cannot read" << path; + else + qWarning().noquote() << "Score.readFile: cannot read" << requested << "->" << path; return {}; } diff --git a/tests/fixtures/score_test/App.hpp b/tests/fixtures/score_test/App.hpp index 4c9b53e5d6..966aa13f64 100644 --- a/tests/fixtures/score_test/App.hpp +++ b/tests/fixtures/score_test/App.hpp @@ -40,6 +40,17 @@ namespace score::test /// no platform is set, forces the offscreen QPA platform. inline void prepare_test_environment(bool headless) { +#if defined(SCORE_TEST_BINARY_DIR) + // score::PluginLoader::pluginsDir() probes "/plugins", and the test + // executables do not live next to /plugins the way the application + // binary does. ctest gets this right through WORKING_DIRECTORY; a hand-run + // executable did not, and booted an application with zero plug-ins whose + // first ctx.settings() then hit SCORE_ABORT. Anchor + // ourselves so both invocations load the same plug-ins. + if(!QDir{QStringLiteral("plugins")}.exists()) + QDir::setCurrent(QStringLiteral(SCORE_TEST_BINARY_DIR)); +#endif + // WebAssembly only ever has the "wasm" platform: asking for another one // is a fatal error, and the page is headless anyway. #if !defined(__EMSCRIPTEN__) diff --git a/tests/fixtures/score_test/ModelInvariants.hpp b/tests/fixtures/score_test/ModelInvariants.hpp new file mode 100644 index 0000000000..b9a6b9dae6 --- /dev/null +++ b/tests/fixtures/score_test/ModelInvariants.hpp @@ -0,0 +1,87 @@ +#pragma once +// Catch2-native replacements for the two QtTest classes the suite used to reach +// for. The suite standardizes on Catch2 and links no QtTest anywhere; QtTest is +// also simply absent from some Qt builds (the macOS ossia-sdk static Qt ships +// 192 Qt6 modules without it), which made a QtTest dependency fail the whole +// configure rather than one target. + +#include + +#include +#include +#include + +#include + +#include + +namespace score::test +{ + +//! Counts emissions of a signal. Replaces QSignalSpy for the count()-only uses. +class SignalCounter +{ +public: + template + SignalCounter(Obj* obj, Signal sig) + { + m_conn = QObject::connect(obj, sig, [this] { ++m_count; }); + } + ~SignalCounter() { QObject::disconnect(m_conn); } + + SignalCounter(const SignalCounter&) = delete; + SignalCounter& operator=(const SignalCounter&) = delete; + + int count() const noexcept { return m_count; } + void clear() noexcept { m_count = 0; } + +private: + QMetaObject::Connection m_conn; + int m_count{}; +}; + +//! Walks the whole tree and checks the QAbstractItemModel contract, in place of +//! QAbstractItemModelTester. Covers what a tree model can realistically get +//! wrong: a child whose parent() does not round-trip, an index handed out for a +//! row/column outside the parent's counts, hasChildren() disagreeing with +//! rowCount(), a reused internal pointer aliasing two distinct indices, and +//! sibling/child accessors that contradict index(). +inline void +checkModelInvariants(const QAbstractItemModel& m, const QModelIndex& parent = {}) +{ + const int rows = m.rowCount(parent); + const int cols = m.columnCount(parent); + REQUIRE(rows >= 0); + REQUIRE(cols >= 0); + CHECK(m.hasChildren(parent) == (rows > 0 && cols > 0)); + + // Out-of-range requests must produce invalid indices, not crashes or + // fabricated ones. + CHECK(!m.index(rows, 0, parent).isValid()); + CHECK(!m.index(-1, 0, parent).isValid()); + CHECK(!m.index(0, cols, parent).isValid()); + + std::set seen; + for(int r = 0; r < rows; ++r) + { + for(int c = 0; c < cols; ++c) + { + const QModelIndex idx = m.index(r, c, parent); + REQUIRE(idx.isValid()); + CHECK(idx.row() == r); + CHECK(idx.column() == c); + CHECK(idx.model() == &m); + CHECK(m.parent(idx) == parent); + CHECK(m.sibling(r, c, idx) == idx); + } + + const QModelIndex first = m.index(r, 0, parent); + // Column 0 owns the children, so its internal pointer identifies the node. + if(void* p = first.internalPointer(); p != nullptr) + { + CHECK(seen.insert(p).second); + } + checkModelInvariants(m, first); + } +} +} diff --git a/tests/integration/ShaderSweep.hpp b/tests/integration/ShaderSweep.hpp index 376addc498..a147616b37 100644 --- a/tests/integration/ShaderSweep.hpp +++ b/tests/integration/ShaderSweep.hpp @@ -27,6 +27,7 @@ #include #include +#include #include #include #include @@ -265,6 +266,45 @@ struct Sweeper }; +//! Writes the last frame a shader produced, so that "it rendered" can be +//! checked against what the shader is supposed to draw rather than taken on +//! faith. Off unless SCORE_SHADER_SWEEP_DUMP_DIR names a directory. +void dumpFrame( + const QString& shader, const std::shared_ptr& rb_p) +{ + static const QString dir = qEnvironmentVariable("SCORE_SHADER_SWEEP_DUMP_DIR"); + if(dir.isEmpty() || !rb_p) + return; + + const auto& rb = *rb_p; + const auto px = rb.pixelSize.width() * rb.pixelSize.height(); + if(px <= 0 || rb.data.size() != px * 4) + return; + + QString name = shader; + name.replace('/', '_'); + + QDir{}.mkpath(dir); + const QImage img{ + reinterpret_cast(rb.data.constData()), rb.pixelSize.width(), + rb.pixelSize.height(), QImage::Format_RGBA8888}; + img.copy().save(dir + '/' + name + ".png"); +} + +//! The MODE declared in a shader's JSON header, or an empty string when it has +//! none (plain ISF). Which sweep owns a file is decided by this, not by its +//! extension: RAW_RASTER_PIPELINE, COMPUTE_SHADER and VERTEX_SHADER_ART shaders +//! are all written as .fs/.vs, and routing them into the ISF loader compiles +//! them against the wrong prelude -- 41 of the testers failed that way, while +//! the subsystem they were written for went entirely unexercised. +inline QString shaderMode(const QByteArray& data) +{ + static const QRegularExpression re{ + R"_("MODE"\s*:\s*"([A-Z_]+)")_"}; + const auto m = re.match(QString::fromUtf8(data.left(8192))); + return m.hasMatch() ? m.captured(1) : QString{}; +} + //! ProgramCache reports both ISF parsing and shader baking through one string. const char* programErrorKind(const QString& error) { @@ -308,10 +348,62 @@ report(const QString& shader, const std::map& kinds) << QString::fromStdString(message); } +//! The baseline records only the failure KINDS per file, so the test fails on +//! *new* failures rather than on a known-bad corpus. Shared by every sweep. +inline void diffAgainstBaseline( + const std::map>& failures, + const QString& baseline) +{ + QStringList current; + for(const auto& [file, kinds] : failures) + for(const auto& [kind, _] : kinds) + current.push_back(file + '\t' + QString::fromStdString(kind)); + current.sort(); + + if(qEnvironmentVariableIsSet("SCORE_SHADER_SWEEP_WRITE_BASELINE")) + { + QFile out{baseline}; + REQUIRE(out.open(QIODevice::WriteOnly | QIODevice::Text)); + QTextStream{&out} << current.join('\n') << '\n'; + return; + } + + QStringList known; + if(QFile in{baseline}; in.open(QIODevice::ReadOnly | QIODevice::Text)) + { + known = QString::fromUtf8(in.readAll()).split('\n', Qt::SkipEmptyParts); + known.sort(); + } + else + { + FAIL( + "no baseline at " << baseline.toStdString() + << ": run with SCORE_SHADER_SWEEP_WRITE_BASELINE to create it"); + } + + QStringList regressions; + for(const auto& entry : current) + if(!known.contains(entry)) + regressions.push_back(entry); + + INFO("new failures:\n" << regressions.join('\n').toStdString()); + CHECK(regressions.isEmpty()); +} + //! Runs one shader kind over the library and diffs against its baseline. +//! @p wantMode selects which files this sweep owns: an empty string means "no +//! MODE header at all", i.e. plain ISF. Files declaring another mode are skipped, +//! not failed. +//! @p blankIsFailure says whether "every pixel identical" means anything for this +//! kind of shader. It does for ISF, which draws a full-screen pass on its own. It +//! does NOT for a raster pipeline: those draw geometry, and this harness wires no +//! geometry producer, so they legitimately render nothing here. Counting that as a +//! failure would measure the harness, not the shader — pixel validation for raster +//! belongs to the JS-wiring harness, which assembles the whole scene chain. inline void sweepLibrary( const score::GUIApplicationContext& ctx, const QStringList& patterns, - ProgramLoader load, const QString& baseline) + ProgramLoader load, const QString& baseline, const QString& wantMode = {}, + bool blankIsFailure = true) { const QString root = libraryRoot(ctx); if(root.isEmpty() || !QFileInfo::exists(root)) @@ -328,24 +420,44 @@ inline void sweepLibrary( if(shaders.isEmpty()) SKIP("no shaders of this kind in the library"); + // The backend comes from the gfx settings model, not from the environment: + // Gfx::Settings::Model reads QSG_RHI_BACKEND at construction and unsets it + // straight away, so by the time we get here the environment no longer says + // anything. score::gfx::BackgroundNode reads the same model in its + // constructor, so there is no rendering to be had without it either. + const auto* gfx_settings = ctx.findSettings(); + if(!gfx_settings) + FAIL( + "score_plugin_gfx registered no settings model: the gfx plug-in was not " + "loaded. Plug-ins are discovered in /plugins -- run this from the " + "build root, as ctest does."); + g_previous = qInstallMessageHandler(capture); - Sweeper sweeper{ctx.settings().graphicsApiEnum()}; + Sweeper sweeper{gfx_settings->graphicsApiEnum()}; std::map> failures; for(const QString& path : shaders) { const QString rel = QDir{root}.relativeFilePath(path); - // Announce before rendering: on a backend that can hang or take the - // process down, the last line printed names the shader responsible. - qInfo().noquote() << "[sweep]" << rel; QFile f{path}; if(!f.open(QIODevice::ReadOnly | QIODevice::Text)) continue; + const QByteArray data = f.readAll(); + + // Skip, do not fail, a shader another sweep owns. Compiling a + // RAW_RASTER_PIPELINE against the ISF prelude only ever produces + // "'position' : undeclared identifier", which says nothing about the shader. + if(shaderMode(data) != wantMode) + continue; + + // Announce before rendering: on a backend that can hang or take the + // process down, the last line printed names the shader responsible. + qInfo().noquote() << "[sweep]" << rel; QString error; - const auto program = load(path, f.readAll(), error); + const auto program = load(path, data, error); if(!program) { failures[rel][programErrorKind(error)] @@ -354,7 +466,11 @@ inline void sweepLibrary( continue; } - if(auto res = sweeper.run(*program); !res.empty()) + auto res = sweeper.run(*program); + if(!blankIsFailure) + res.erase("blank"); + dumpFrame(rel, sweeper.output.shared_readback); + if(!res.empty()) { report(rel, res); failures[rel] = std::move(res); @@ -365,39 +481,6 @@ inline void sweepLibrary( INFO("swept " << shaders.size() << " shaders, " << failures.size() << " failing"); - QStringList current; - for(const auto& [file, kinds] : failures) - for(const auto& [kind, _] : kinds) - current.push_back(file + '\t' + QString::fromStdString(kind)); - current.sort(); - - if(qEnvironmentVariableIsSet("SCORE_SHADER_SWEEP_WRITE_BASELINE")) - { - QFile out{baseline}; - REQUIRE(out.open(QIODevice::WriteOnly | QIODevice::Text)); - QTextStream{&out} << current.join('\n') << '\n'; - return; - } - - QStringList known; - if(QFile in{baseline}; in.open(QIODevice::ReadOnly | QIODevice::Text)) - { - known = QString::fromUtf8(in.readAll()).split('\n', Qt::SkipEmptyParts); - known.sort(); - } - else - { - FAIL( - "no baseline at " << baseline.toStdString() - << ": run with SCORE_SHADER_SWEEP_WRITE_BASELINE to create it"); - } - - QStringList regressions; - for(const auto& entry : current) - if(!known.contains(entry)) - regressions.push_back(entry); - - INFO("new failures:\n" << regressions.join('\n').toStdString()); - CHECK(regressions.isEmpty()); + diffAgainstBaseline(failures, baseline); } } diff --git a/tests/integration/ShaderSweepCSF.cpp b/tests/integration/ShaderSweepCSF.cpp index 9f061e43a1..1e93eee45c 100644 --- a/tests/integration/ShaderSweepCSF.cpp +++ b/tests/integration/ShaderSweepCSF.cpp @@ -54,6 +54,9 @@ TEST_CASE("Every CSF shader in the library renders", "[integration][gfx][shaders { requestGlesContext(); score::test::run_in_gui_app([](const score::GUIApplicationContext& ctx) { - sweepLibrary(ctx, {"*.cs", "*.comp", "*.csf"}, &loadCSF, QStringLiteral(SCORE_SHADER_SWEEP_BASELINE_CSF)); + sweepLibrary( + ctx, {"*.cs", "*.comp", "*.csf"}, &loadCSF, + QStringLiteral(SCORE_SHADER_SWEEP_BASELINE_CSF), + QStringLiteral("COMPUTE_SHADER")); }); } diff --git a/tests/integration/ShaderSweepVSA.cpp b/tests/integration/ShaderSweepVSA.cpp index 9940614a14..80a5e7ca9b 100644 --- a/tests/integration/ShaderSweepVSA.cpp +++ b/tests/integration/ShaderSweepVSA.cpp @@ -26,6 +26,9 @@ TEST_CASE("Every VSA shader in the library renders", "[integration][gfx][shaders { requestGlesContext(); score::test::run_in_gui_app([](const score::GUIApplicationContext& ctx) { - sweepLibrary(ctx, {"*.vs", "*.vert"}, &loadVSA, QStringLiteral(SCORE_SHADER_SWEEP_BASELINE_VSA)); + sweepLibrary( + ctx, {"*.vs", "*.vert"}, &loadVSA, + QStringLiteral(SCORE_SHADER_SWEEP_BASELINE_VSA), + QStringLiteral("VERTEX_SHADER_ART")); }); } diff --git a/tests/integration/golden-render/cases-llvmpipe.txt b/tests/integration/golden-render/cases-llvmpipe.txt new file mode 100644 index 0000000000..fa77dc0e8b --- /dev/null +++ b/tests/integration/golden-render/cases-llvmpipe.txt @@ -0,0 +1,23 @@ +# Golden-render pinned case list — llvmpipe backend class. +# One tests-scene script basename per line (no .js). Chosen from the subset +# proven to render on llvmpipe (2026-07 sweep), spread across pipeline +# families: ISF basic/multi-input/multipass/MRT, raw-raster texture upload + +# strides, sampler state, storage images, float formats, CSF compute. +# Time-animated cases are rejected automatically by --update-refs +# self-consistency; permanently unstable ones land in refs//UNSTABLE.txt. +build-isf-solid-color +build-isf-image-passthrough +build-isf-two-images +build-2d-no-stride +build-2d-stride-xy +build-isf-nearest-filter +build-isf-three-pass +build-isf-multipass-size +build-isf-mrt-four-outputs +build-mrt-gbuffer +build-pass-override-state +build-binding-storage-image-fragment +build-isf-pass-format-rgba16f +build-output-format-rgba16f +build-csf-texture-sampling +build-isf-point3d-as-color diff --git a/tests/integration/golden-render/compare.py b/tests/integration/golden-render/compare.py new file mode 100644 index 0000000000..2bd578b299 --- /dev/null +++ b/tests/integration/golden-render/compare.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +"""Perceptual image comparison for the golden-render harness. + + compare.py [--profile strict|cross|loose] [--json] + +Metrics: PSNR (dB), SSIM (gaussian 11x11 sigma=1.5, standard Wang et al. +constants), mean absolute diff and max absolute diff (8-bit scale). + +Profiles (thresholds from the 2026-07 measurement session — same-machine +llvmpipe renders are bit-stable in practice, GL vs Vulkan frame means matched +to 4 decimal digits): + strict : same backend-class regression gate. + PSNR >= 40 dB AND SSIM >= 0.995 AND max_abs <= 8 + cross : GL vs Vulkan / driver-vs-driver informational check. + PSNR >= 30 dB AND SSIM >= 0.99 + loose : structural sanity only. SSIM >= 0.95 + self : ref-acceptance self-consistency (two renders of one case). + PSNR >= 45 dB AND max_abs <= 4 + +Exit codes: 0 pass, 1 fail, 2 usage/IO error. Identical files short-circuit +to PASS with psnr=inf. A size mismatch is always FAIL (never resampled: +resolution drift IS a regression). +""" +import argparse +import json +import sys + +import numpy as np +from PIL import Image +from scipy.ndimage import gaussian_filter + +PROFILES = { + "strict": dict(psnr=40.0, ssim=0.995, max_abs=8, mean_abs=None), + "cross": dict(psnr=30.0, ssim=0.99, max_abs=None, mean_abs=None), + "loose": dict(psnr=None, ssim=0.95, max_abs=None, mean_abs=None), + "self": dict(psnr=45.0, ssim=None, max_abs=4, mean_abs=None), +} + + +def load(path): + try: + img = Image.open(path).convert("RGB") + except Exception as e: # noqa: BLE001 + print(f"ERROR: cannot read {path}: {e}", file=sys.stderr) + sys.exit(2) + return np.asarray(img, dtype=np.float64) + + +def psnr(a, b): + mse = np.mean((a - b) ** 2) + if mse == 0: + return float("inf") + return 10.0 * np.log10(255.0**2 / mse) + + +def ssim(a, b, sigma=1.5): + """Mean SSIM over the luma plane, gaussian-windowed (Wang et al. 2004).""" + # ITU-R BT.601 luma; SSIM on luma is the standard single-channel variant. + la = a @ np.array([0.299, 0.587, 0.114]) + lb = b @ np.array([0.299, 0.587, 0.114]) + c1, c2 = (0.01 * 255) ** 2, (0.03 * 255) ** 2 + mu_a = gaussian_filter(la, sigma) + mu_b = gaussian_filter(lb, sigma) + mu_a2, mu_b2, mu_ab = mu_a * mu_a, mu_b * mu_b, mu_a * mu_b + var_a = gaussian_filter(la * la, sigma) - mu_a2 + var_b = gaussian_filter(lb * lb, sigma) - mu_b2 + cov = gaussian_filter(la * lb, sigma) - mu_ab + num = (2 * mu_ab + c1) * (2 * cov + c2) + den = (mu_a2 + mu_b2 + c1) * (var_a + var_b + c2) + return float(np.mean(num / den)) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("ref") + ap.add_argument("test") + ap.add_argument("--profile", choices=PROFILES, default="strict") + ap.add_argument("--json", action="store_true") + args = ap.parse_args() + + a, b = load(args.ref), load(args.test) + out = {"ref": args.ref, "test": args.test, "profile": args.profile} + + if a.shape != b.shape: + out.update(verdict="FAIL", reason=f"size mismatch {a.shape} vs {b.shape}") + print(json.dumps(out) if args.json else f"FAIL size-mismatch {a.shape} vs {b.shape}") + sys.exit(1) + + diff = np.abs(a - b) + m = { + "psnr": round(psnr(a, b), 3), + "ssim": round(ssim(a, b), 6), + "mean_abs": round(float(diff.mean()), 4), + "max_abs": float(diff.max()), + } + out.update(m) + + th = PROFILES[args.profile] + fails = [] + if th["psnr"] is not None and m["psnr"] < th["psnr"]: + fails.append(f"psnr {m['psnr']} < {th['psnr']}") + if th["ssim"] is not None and m["ssim"] < th["ssim"]: + fails.append(f"ssim {m['ssim']} < {th['ssim']}") + if th["max_abs"] is not None and m["max_abs"] > th["max_abs"]: + fails.append(f"max_abs {m['max_abs']} > {th['max_abs']}") + + out["verdict"] = "FAIL" if fails else "PASS" + if fails: + out["reason"] = "; ".join(fails) + + if args.json: + print(json.dumps(out)) + else: + line = f"{out['verdict']} psnr={m['psnr']} ssim={m['ssim']} mean_abs={m['mean_abs']} max_abs={m['max_abs']}" + if fails: + line += f" [{out['reason']}]" + print(line) + sys.exit(1 if fails else 0) + + +if __name__ == "__main__": + main() diff --git a/tests/integration/golden-render/frame-determinism.sh b/tests/integration/golden-render/frame-determinism.sh new file mode 100755 index 0000000000..f986e176f3 --- /dev/null +++ b/tests/integration/golden-render/frame-determinism.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +# Acceptance test for deterministic frame stepping. +# +# Renders the SAME tester to the SAME frame index in two separate processes and +# requires the two PNGs to be byte-identical. Two processes rather than two +# grabs in one, because anything cached in the process would hide exactly the +# nondeterminism this is looking for. +# +# If this fails, golden references cannot be pinned: frame N is not a stable +# picture, and every stored reference is a snapshot of one particular run. +# +# frame-determinism.sh [--score PATH] [--case NAME] [--frame N] +# +# Exit 0 iff the two renders agree. +set -uo pipefail + +SCORE="${OSSIA_SCORE:-}" +# Animated on purpose: this tester draws TIME, TIMEDELTA, PROGRESS and +# FRAMEINDEX as bars, so it fails if any of them still follows a wall clock. A +# static case would pass whether or not the step clock works. +CASE="build-isf-time-uniforms" +FRAME=30 +OUT="${TMPDIR:-/tmp}/score-frame-determinism.$$" + +while [ $# -gt 0 ]; do + case "$1" in + --score) SCORE="$2"; shift 2 ;; + --case) CASE="$2"; shift 2 ;; + --frame) FRAME="$2"; shift 2 ;; + *) echo "unknown argument: $1" >&2; exit 2 ;; + esac +done + +[ -n "$SCORE" ] || { echo "FATAL: set OSSIA_SCORE or pass --score"; exit 2; } +[ -x "$SCORE" ] || { echo "FATAL: not executable: $SCORE"; exit 2; } + +mkdir -p "$OUT" +trap 'rm -rf "$OUT"' EXIT + +# The tester scripts resolve their own paths through :, so the only +# thing this needs to know is the case name. --wait defers autoplay by N seconds +# for slow media to load; it is not an exit timer, so the script ends the run +# itself with Qt.exit and the timeout below is only a hang guard. +render() { + local n="$1" + # Score.play() first: the gfx nodes only exist while the score executes, so a + # grab on a stopped document finds nothing wired to the Window device. + # Qt.exit() last, so the run ends when the work is done and the exit code is + # the script's rather than the timeout's. + cat > "$OUT/run$n.js" <:/packages/csf-examples/csf-testers/tests-scene/scripts/${CASE}.js")); +Score.play(); +Score.device('Window').grabFrame(${FRAME}, "${OUT}/frame$n.png"); +Qt.exit(0); +JS + # Without this the Window device opens a real window and grabTo falls back to + # QScreen::grabWindow, which reads the framebuffer at the window's geometry -- + # i.e. the desktop, screensaver included. Checked again below, because a + # screenshot of a static desktop is byte-stable and would "pass". + SCORE_FORCE_OFFSCREEN_WINDOW=Window \ + SCORE_AUDIO_BACKEND=dummy SCORE_DISABLE_AUDIOPLUGINS=1 DISPLAY="${DISPLAY:-:0}" \ + timeout 120 "$SCORE" --no-gui --no-restore --script "$OUT/run$n.js" --wait 0 --autoplay \ + > "$OUT/run$n.log" 2>&1 + return $? +} + +echo "== rendering $CASE frame $FRAME, twice, in separate processes ==" +render 1; rc1=$? +render 2; rc2=$? + +fail=0 +for n in 1 2; do + if [ ! -s "$OUT/frame$n.png" ]; then + echo "FAIL: run $n produced no frame (exit $([ $n = 1 ] && echo $rc1 || echo $rc2))" + sed 's/^/ /' "$OUT/run$n.log" | tail -20 + fail=1 + fi +done +[ $fail -eq 1 ] && exit 1 + +# A screen grab of a quiet desktop is byte-identical between two runs, so this +# has to be fatal rather than cosmetic: it is the difference between comparing +# renders and comparing wallpaper. +for n in 1 2; do + if grep -q "capturing the SCREEN" "$OUT/run$n.log"; then + echo "FAIL: run $n grabbed the screen instead of the render." + echo " The offscreen device was not selected -- SCORE_FORCE_OFFSCREEN_WINDOW" + echo " must name the Window device, and this build must honour it." + exit 1 + fi + if grep -q "nothing rendered into" "$OUT/run$n.log"; then + echo "FAIL: run $n rendered nothing -- no process is wired to the Window device." + grep "nothing rendered into" "$OUT/run$n.log" | sed 's/^/ /' + exit 1 + fi +done + +# Identical blank frames are identical. Without this, a case that renders +# nothing at all is the easiest way to pass a determinism test, and the result +# would say the mechanism works when it has not been exercised. +blank=$(python3 - "$OUT/frame1.png" <<'PY' 2>/dev/null +import sys +try: + from PIL import Image + print(len(set(Image.open(sys.argv[1]).convert('RGB').getdata()))) +except Exception: + print("?") +PY +) +case "$blank" in + "?") echo "NOTE: cannot check for blankness (no PIL); result is weaker than it looks" ;; + 1) echo "FAIL: frame $FRAME is a single flat colour -- nothing was rendered." + echo " Two blank frames match trivially; this proves nothing about determinism." + exit 1 ;; + *) echo " frame has $blank distinct colours (not blank)" ;; +esac + +if cmp -s "$OUT/frame1.png" "$OUT/frame2.png"; then + echo "PASS: both runs produced byte-identical frames ($(stat -c%s "$OUT/frame1.png") bytes)" + exit 0 +fi + +echo "FAIL: frame $FRAME differs between two runs of the same case." +echo " run 1: $(stat -c%s "$OUT/frame1.png") bytes md5 $(md5sum < "$OUT/frame1.png" | cut -d' ' -f1)" +echo " run 2: $(stat -c%s "$OUT/frame2.png") bytes md5 $(md5sum < "$OUT/frame2.png" | cut -d' ' -f1)" +echo " Frame stepping is not deterministic yet; golden references cannot be pinned." +cp "$OUT/frame1.png" "$OUT/frame2.png" "${TMPDIR:-/tmp}/" 2>/dev/null && \ + echo " copies kept in ${TMPDIR:-/tmp}/frame1.png and frame2.png" +exit 1 diff --git a/tests/integration/golden-render/golden-render.sh b/tests/integration/golden-render/golden-render.sh new file mode 100755 index 0000000000..8f775791fd --- /dev/null +++ b/tests/integration/golden-render/golden-render.sh @@ -0,0 +1,194 @@ +#!/usr/bin/env bash +# Golden-image render regression harness. +# +# golden-render.sh [--backend llvmpipe|nvidia|vulkan-lavapipe|nvidia-vulkan] +# [--update-refs] [--cases "name name ..."] [--keep-going] +# +# Renders each pinned tests-scene pipeline (cases-.txt, falling +# back to cases-llvmpipe.txt) through the real ossia-score binary headless — +# same proven recipe as /tmp/verify-shader.sh / scene-js-sweep.sh — and +# compares the grabbed frame against refs//.png with compare.py +# (SSIM/PSNR/max-abs, profile "strict"). +# +# check mode (default) : ref must exist; verdict per case is +# PASS / FAIL / NOREF / NORENDER / SKIP-UNSTABLE. +# Exit 0 iff no FAIL/NOREF/NORENDER. +# --update-refs : renders each case TWICE and accepts the ref only if +# the two runs agree (compare.py --profile self) and +# are non-blank. Disagreeing cases are recorded in +# refs//UNSTABLE.txt (time-animated shaders +# self-reject here); blank ones in BLANK.txt. +# +# Backend classes: +# llvmpipe offscreen QPA + software GL (CI-able, fully headless) +# nvidia xcb on :0 + NVIDIA GLX (rig) +# vulkan-lavapipe/ nvidia-vulkan: GraphicsApi=Vulkan via an isolated +# XDG_CONFIG_HOME (score reads score_plugin_gfx/GraphicsApi from QSettings; +# there is no CLI flag). ALL runs use an isolated config home seeded from the +# user's score.conf with GraphicsApi pinned — a run must not depend on the +# user's live settings (they currently say Vulkan!) nor trip failsafe.bit. +# +# Serialization: each app run holds flock /tmp/score-harness.lock (OSC port +# 6666 is global). Do NOT wrap this whole script in that lock (see the +# EXHAUSTIVE-TEST-PLAN consolidation note on self-deadlock). +set -u + +HERE="$(cd "$(dirname "$0")" && pwd)" +SRCROOT="$(cd "$HERE/../../.." && pwd)" # tests/integration/golden-render -> repo root +BIN="${OSSIA_SCORE:-$SRCROOT/build-sanitizers/ossia-score}" +SCRIPTS="${SCRIPTS:-$HOME/Documents/ossia/score/packages/csf-examples/csf-testers/tests-scene/scripts}" +OSC=6666 +BLANK_MEAN="${BLANK_MEAN:-0.002}" +TIMEOUT="${TIMEOUT:-90}" +GRABTRIES="${GRABTRIES:-25}" # x2s poll for the grab (ASAN startup is slow) +ASAN="detect_leaks=0:halt_on_error=0:handle_segv=1:detect_odr_violation=0:protect_shadow_gap=0" + +BACKEND=llvmpipe +UPDATE=0 +KEEPGOING=0 +CASES_OVERRIDE="" +while [ $# -gt 0 ]; do + case "$1" in + --backend) BACKEND="$2"; shift 2 ;; + --update-refs) UPDATE=1; shift ;; + --cases) CASES_OVERRIDE="$2"; shift 2 ;; + --keep-going) KEEPGOING=1; shift ;; + *) echo "unknown arg: $1" >&2; exit 2 ;; + esac +done + +# Prerequisites -> ctest SKIP (return 77) rather than a hard failure. +command -v oscsend >/dev/null || { echo "SKIP: oscsend not found"; exit 77; } +command -v convert >/dev/null || { echo "SKIP: ImageMagick not found"; exit 77; } +[ -x "$BIN" ] || { echo "SKIP: $BIN not built"; exit 77; } +[ -d "$SCRIPTS" ] || { echo "SKIP: corpus missing ($SCRIPTS)"; exit 77; } + +REFS="$HERE/refs/$BACKEND" +OUT="${OUT:-/tmp/golden-render/$BACKEND}" +mkdir -p "$REFS" "$OUT" + +CASES_FILE="$HERE/cases-$BACKEND.txt" +[ -f "$CASES_FILE" ] || CASES_FILE="$HERE/cases-llvmpipe.txt" +if [ -n "$CASES_OVERRIDE" ]; then + read -r -a CASES <<< "$CASES_OVERRIDE" +else + mapfile -t CASES < <(grep -v '^\s*#' "$CASES_FILE" | grep -v '^\s*$') +fi + +# ---- isolated, pinned config home ------------------------------------------- +# GraphicsApi comes from QSettings (score_plugin_gfx/GraphicsApi, +# Gfx/Settings/Model.cpp:38) — pin it so runs are hermetic. +CFG="$OUT/config-home" +mkdir -p "$CFG/ossia" +case "$BACKEND" in + vulkan-*|*-vulkan) API=Vulkan ;; + *) API=OpenGL ;; +esac +python3 - "$HOME/.config/ossia/score.conf" "$CFG/ossia/score.conf" "$API" <<'EOF' +import re, sys, pathlib +src, dst, api = sys.argv[1], sys.argv[2], sys.argv[3] +try: + text = pathlib.Path(src).read_text() +except OSError: + text = "" +if "[score_plugin_gfx]" not in text: + text += "\n[score_plugin_gfx]\nGraphicsApi=%s\n" % api +elif re.search(r"^GraphicsApi=.*$", text, re.M): + text = re.sub(r"^GraphicsApi=.*$", "GraphicsApi=%s" % api, text, flags=re.M) +else: + text = text.replace("[score_plugin_gfx]", "[score_plugin_gfx]\nGraphicsApi=%s" % api) +pathlib.Path(dst).write_text(text) +EOF + +# Backend env (mirrors scene-js-sweep.sh; `env -u DISPLAY` for llvmpipe because +# a live DISPLAY makes offscreen-EGL negotiate a GL 2.0 context). +backend_env() { + case "$BACKEND" in + nvidia|nvidia-vulkan) + echo "DISPLAY=:0 QT_QPA_PLATFORM=xcb __GLX_VENDOR_LIBRARY_NAME=nvidia" ;; + vulkan-lavapipe) + echo "DISPLAY=:0 QT_QPA_PLATFORM=xcb VK_LOADER_DRIVERS_SELECT=lvp*" ;; + *) + echo "QT_QPA_PLATFORM=offscreen LIBGL_ALWAYS_SOFTWARE=1 GALLIUM_DRIVER=llvmpipe" ;; + esac +} + +pixel_mean() { convert "$1" -format '%[fx:mean]' info: 2>/dev/null || echo 0; } + +# ---- one full app run -> one PNG -------------------------------------------- +render_one() { # case_name out_png -> 0 ok, 2 no png + local name="$1" png="$2" js="$SCRIPTS/$1.js" log="$OUT/$1.log" + rm -f "$png" + [ -f "$js" ] || { echo " missing script $js" >&2; return 2; } + local benv; benv=$(backend_env) + ( + flock -w 300 9 || { echo " LOCK-TIMEOUT" >&2; exit 4; } + ( for _ in $(seq 1 "$GRABTRIES"); do + sleep 2 + oscsend 127.0.0.1 $OSC /script s "Score.device('Window').grabTo('$png')" 2>/dev/null + [ -s "$png" ] && break + done + sleep 0.5; oscsend 127.0.0.1 $OSC /stop; sleep 0.5; oscsend 127.0.0.1 $OSC /exit ) >/dev/null 2>&1 & + # shellcheck disable=SC2086 + env -u DISPLAY XDG_CONFIG_HOME="$CFG" \ + SCORE_AUDIO_BACKEND=dummy SCORE_DISABLE_AUDIOPLUGINS=1 \ + SCORE_FORCE_OFFSCREEN_WINDOW=Window \ + ASAN_OPTIONS="$ASAN" LLVM_PROFILE_FILE="$OUT/%p.profraw" \ + $benv \ + timeout --foreground "$TIMEOUT" "$BIN" --no-gui --no-restore \ + --script "$js" --wait 1 --autoplay >"$log" 2>&1 + wait 2>/dev/null + ) 9>/tmp/score-harness.lock + [ -s "$png" ] || return 2 + return 0 +} + +listed() { [ -f "$2" ] && grep -qx "$1" "$2"; } + +fails=0; passes=0; skips=0 +for name in "${CASES[@]}"; do + printf '%-42s' "$name" + if [ "$UPDATE" = 1 ]; then + if render_one "$name" "$OUT/$name.A.png" && render_one "$name" "$OUT/$name.B.png"; then + m=$(pixel_mean "$OUT/$name.A.png") + if ! awk "BEGIN{exit !($m > $BLANK_MEAN)}"; then + echo "BLANK mean=$m (not accepted as ref)"; grep -qx "$name" "$REFS/BLANK.txt" 2>/dev/null || echo "$name" >> "$REFS/BLANK.txt" + skips=$((skips+1)); continue + fi + if res=$(python3 "$HERE/compare.py" "$OUT/$name.A.png" "$OUT/$name.B.png" --profile self); then + cp "$OUT/$name.A.png" "$REFS/$name.png" + # no longer unstable/blank if it stabilized + sed -i "/^$name\$/d" "$REFS/UNSTABLE.txt" "$REFS/BLANK.txt" 2>/dev/null + echo "REF-UPDATED ($res)"; passes=$((passes+1)) + else + echo "UNSTABLE ($res) — excluded"; grep -qx "$name" "$REFS/UNSTABLE.txt" 2>/dev/null || echo "$name" >> "$REFS/UNSTABLE.txt" + skips=$((skips+1)) + fi + else + echo "NORENDER (see $OUT/$name.log)"; fails=$((fails+1)) + [ "$KEEPGOING" = 1 ] || true + fi + else + if listed "$name" "$REFS/UNSTABLE.txt"; then echo "SKIP-UNSTABLE"; skips=$((skips+1)); continue; fi + if listed "$name" "$REFS/BLANK.txt"; then echo "SKIP-BLANK"; skips=$((skips+1)); continue; fi + if [ ! -f "$REFS/$name.png" ]; then echo "NOREF (run --update-refs)"; fails=$((fails+1)); continue; fi + if render_one "$name" "$OUT/$name.png"; then + if res=$(python3 "$HERE/compare.py" "$REFS/$name.png" "$OUT/$name.png" --profile strict); then + echo "PASS ($res)"; passes=$((passes+1)) + else + echo "FAIL ($res) ref=$REFS/$name.png test=$OUT/$name.png"; fails=$((fails+1)) + fi + else + echo "NORENDER (see $OUT/$name.log)"; fails=$((fails+1)) + fi + fi +done + +echo "----" +echo "golden-render[$BACKEND]$([ "$UPDATE" = 1 ] && echo ' (update-refs)'): $passes ok, $fails failing, $skips skipped" +# In check mode, if there are no refs at all yet, SKIP (77) instead of failing — +# refs are generated once with --update-refs and committed alongside the branch. +if [ "$UPDATE" = 0 ] && [ "$passes" = 0 ] && [ "$fails" -gt 0 ] && ! ls "$REFS"/*.png >/dev/null 2>&1; then + echo "SKIP: no references present — run --update-refs once and commit refs/$BACKEND/"; exit 77 +fi +[ "$fails" = 0 ] diff --git a/tests/integration/golden-render/sweep.sh b/tests/integration/golden-render/sweep.sh new file mode 100755 index 0000000000..6fc64ce939 --- /dev/null +++ b/tests/integration/golden-render/sweep.sh @@ -0,0 +1,126 @@ +#!/usr/bin/env bash +# Runs every tests-scene tester and records what came out. +# +# One process per case: a case that crashes or hangs takes only itself down, and +# the document each tester builds starts from a clean application either way. +# +# sweep.sh [--score PATH] [--out DIR] [--frame N] [--filter GLOB] +# +# Writes DIR/results.tsv (case, verdict, colours, ms, note) and DIR/.png. +# Verdicts: +# RENDER frame written, more than one colour in it +# BLANK frame written, every pixel identical +# NORENDER nothing wired to the Window device, or no frame written +# SCREEN the grab fell back to capturing the screen -- result is worthless +# CRASH score died on a signal -- the per-case process is what contains it +# FAIL score exited nonzero, or was killed by the guard +set -uo pipefail + +SCORE="${OSSIA_SCORE:-}" +OUT="${SWEEP_OUT:-/tmp/score-sweep}" +FRAME=30 +FILTER='*' +SCRIPTS=":/packages/csf-examples/csf-testers/tests-scene/scripts" +SCRIPTS_DIR="${SWEEP_SCRIPTS_DIR:-$HOME/Documents/ossia/score/packages/csf-examples/csf-testers/tests-scene/scripts}" + +while [ $# -gt 0 ]; do + case "$1" in + --score) SCORE="$2"; shift 2 ;; + --out) OUT="$2"; shift 2 ;; + --frame) FRAME="$2"; shift 2 ;; + --filter) FILTER="$2"; shift 2 ;; + *) echo "unknown argument: $1" >&2; exit 2 ;; + esac +done + +[ -n "$SCORE" ] || { echo "FATAL: set OSSIA_SCORE or pass --score"; exit 2; } +[ -x "$SCORE" ] || { echo "FATAL: not executable: $SCORE"; exit 2; } +[ -d "$SCRIPTS_DIR" ] || { echo "FATAL: no scripts at $SCRIPTS_DIR"; exit 2; } + +mkdir -p "$OUT/logs" +: > "$OUT/results.tsv" + +colours() { + python3 - "$1" <<'PY' 2>/dev/null || echo "?" +import sys +try: + from PIL import Image + print(len(set(Image.open(sys.argv[1]).convert('RGB').getdata()))) +except Exception: + print("?") +PY +} + +total=0; render=0; blank=0; norender=0; failed=0; screen=0; crashed=0 +start_all=$(date +%s) + +for f in "$SCRIPTS_DIR"/build-$FILTER.js; do + [ -e "$f" ] || continue + name=$(basename "$f" .js) + total=$((total + 1)) + png="$OUT/$name.png" + log="$OUT/logs/$name.log" + rm -f "$png" + + cat > "$OUT/run.js" < "$log" 2>&1 + rc=$? + ms=$(( ($(date +%s%N) - t0) / 1000000 )) + + note="" + if grep -q "capturing the SCREEN" "$log"; then + verdict=SCREEN; screen=$((screen + 1)) + elif [ ! -s "$png" ]; then + if grep -q "nothing rendered into" "$log"; then + verdict=NORENDER; norender=$((norender + 1)) + elif [ $rc -ge 128 ]; then + # Killed by a signal. 124 is the timeout's own kill and is handled as + # FAIL below; anything else here died, most often on a SCORE_ASSERT in + # render-target creation. One process per case is what keeps that from + # ending the sweep. + verdict=CRASH; crashed=$((crashed + 1)) + note="signal $((rc - 128))" + site=$(grep -m1 -oE '[A-Za-z]+\.cpp:[0-9]+' "$log") + [ -n "$site" ] && note="$note at $site" + elif [ $rc -ne 0 ]; then + verdict=FAIL; failed=$((failed + 1)); note="exit $rc" + else + verdict=NORENDER; norender=$((norender + 1)); note="no frame written" + fi + else + c=$(colours "$png") + if [ "$c" = "1" ]; then verdict=BLANK; blank=$((blank + 1)) + else verdict=RENDER; render=$((render + 1)); fi + note="$c colours" + fi + + # First real complaint from the log, so the table says why without opening it. + if [ -z "$note" ] || [ "$verdict" = FAIL ] || [ "$verdict" = NORENDER ]; then + why=$(grep -m1 -E 'error|Error|failed|Failed|not supported|Missing' "$log" \ + | cut -c1-90 | tr -d '\t') + [ -n "$why" ] && note="${note:+$note; }$why" + fi + + printf '%s\t%s\t%s\t%s\n' "$name" "$verdict" "$ms" "$note" >> "$OUT/results.tsv" + printf ' %-46s %-9s %5s ms %s\n' "$name" "$verdict" "$ms" "$note" +done + +echo +echo "== $total cases in $(( $(date +%s) - start_all ))s ==" +printf ' RENDER %d BLANK %d NORENDER %d CRASH %d FAIL %d SCREEN %d\n' \ + "$render" "$blank" "$norender" "$crashed" "$failed" "$screen" +echo " results: $OUT/results.tsv" + +# SCREEN means the harness measured the desktop, which is never a usable result. +[ "$screen" -gt 0 ] && exit 1 +exit 0 diff --git a/tests/library/CMakeLists.txt b/tests/library/CMakeLists.txt index fba5efa72a..39f9790c26 100644 --- a/tests/library/CMakeLists.txt +++ b/tests/library/CMakeLists.txt @@ -1,11 +1,7 @@ # Tests for the process-library model population (staged subtree publish). # APP tests: they need the full headless application so that the settings # models and the process factories from every plugin are registered. -# QtTest is linked for QAbstractItemModelTester. -find_package(${QT_VERSION} REQUIRED COMPONENTS Test) - score_add_test(test_library_publish SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/ProcessLibraryPublishTest.cpp" APP - PLUGINS score_plugin_library score_lib_process - LIBS ${QT_PREFIX}::Test) + PLUGINS score_plugin_library score_lib_process) diff --git a/tests/library/ProcessLibraryPublishTest.cpp b/tests/library/ProcessLibraryPublishTest.cpp index e88aee6b54..90faaf7313 100644 --- a/tests/library/ProcessLibraryPublishTest.cpp +++ b/tests/library/ProcessLibraryPublishTest.cpp @@ -1,8 +1,8 @@ // Tests for the staged-subtree publish path of Library::ProcessesItemModel. // // Correctness: every mutation outside rescan() goes through publish() / -// replaceChildren() with exact QAbstractItemModel signals — verified by Qt's -// own QAbstractItemModelTester and by mirroring the model through a live +// replaceChildren() with exact QAbstractItemModel signals — verified by our +// own checkModelInvariants() walk and by mirroring the model through a live // QSortFilterProxyModel (the exact observer that silent mutation used to // corrupt). // @@ -21,12 +21,11 @@ #include #include -#include #include #include -#include #include +#include #include @@ -149,8 +148,6 @@ TEST_CASE("publish: model invariants and structure", "[library]") score::test::run_in_app([](const score::GUIApplicationContext& ctx) { primeLibrarySettings(ctx); Fixture f{ctx}; - QAbstractItemModelTester tester{ - &f.model, QAbstractItemModelTester::FailureReportingMode::Fatal}; // Deep new path, single entry f.model.publish(f.entry({"GIG", "orchestra"}, "violin")); @@ -194,6 +191,8 @@ TEST_CASE("publish: model invariants and structure", "[library]") f.model.publish(std::move(stray)); f.model.flushPending(); REQUIRE(f.model.rowCount(anchor) == before); + + score::test::checkModelInvariants(f.model); }); } @@ -202,9 +201,7 @@ TEST_CASE("publish: coalescing boundaries and signal counts", "[library]") score::test::run_in_app([](const score::GUIApplicationContext& ctx) { primeLibrarySettings(ctx); Fixture f{ctx}; - QAbstractItemModelTester tester{ - &f.model, QAbstractItemModelTester::FailureReportingMode::Fatal}; - QSignalSpy spy{&f.model, &QAbstractItemModel::rowsInserted}; + score::test::SignalCounter spy{&f.model, &QAbstractItemModel::rowsInserted}; // A whole new folder published in one flush: exactly one insert. for(int i = 0; i < 100; i++) @@ -237,6 +234,8 @@ TEST_CASE("publish: coalescing boundaries and signal counts", "[library]") REQUIRE(childNames(f.model, packB) == QStringList{"b0", "b1", "b2"}); const auto packA = f.model.index(0, 0, audio); REQUIRE(f.model.rowCount(packA) == 200); + + score::test::checkModelInvariants(f.model); }); } @@ -275,8 +274,6 @@ TEST_CASE("replaceChildren: exact ranges, proxy stays consistent", "[library]") score::test::run_in_app([](const score::GUIApplicationContext& ctx) { primeLibrarySettings(ctx); Fixture f{ctx}; - QAbstractItemModelTester tester{ - &f.model, QAbstractItemModelTester::FailureReportingMode::Fatal}; Library::ProcessFilterProxy proxy; proxy.setSourceModel(&f.model); @@ -319,6 +316,8 @@ TEST_CASE("replaceChildren: exact ranges, proxy stays consistent", "[library]") // Unknown key: no-op f.model.replaceChildren(Process::ProcessModelFactory::ConcreteKey{}, forest(1, 1)); REQUIRE(f.model.rowCount(f.model.find(f.key)) == 0); + + score::test::checkModelInvariants(f.model); }); } @@ -396,7 +395,7 @@ TEST_CASE("publish: 36k-entry storm, hot proxy mirrors the tree", "[library][ben proxy.hasChildren(i); REQUIRE(proxy.rowCount(proxy.mapFromSource(f.anchor)) == 1); - QSignalSpy spy{&f.model, &QAbstractItemModel::rowsInserted}; + score::test::SignalCounter spy{&f.model, &QAbstractItemModel::rowsInserted}; // Deliver in batches of 255 like RecursiveWatch; the flush timer runs in // the processEvents between batches. Timed per batch, flush included.