Load FreeVideo for offline environment packages; wider attention head groups on Linux GPUs with memory to spare (0.3.20)

freevideo_engine/desktop_shortcut.py

@@ -130,6 +130,20 @@ def method(obj, index, types, *values):
     return targets
 
 
+# IShellLinkW::SetPath fails (E_FAIL) for a target of MAX_PATH, 260 characters, or more.
+LINK_PATH_LIMIT = 259
+
+
+def copies_folder(engine, fingerprint):
+    """Where the launcher copy behind the shortcut goes: in the installation, unless the path
+    would be too long for a Windows shortcut, as in an offline installation in a deep folder."""
+    parent = Path(os.path.abspath(engine)) / 'launcher/application'
+    if len(str(parent / (fingerprint + '-' + 'f' * 8) / 'FreeVideo.exe')) <= LINK_PATH_LIMIT:
+        return parent
+    from .desktop_runtime import launcher_root
+    return Path(os.path.abspath(launcher_root())) / 'application'
+
+
 def _complete(root, fingerprint, folder):
     """A launcher copy the shortcut can start; one this account cannot read is not."""
     try:
@@ -150,7 +164,7 @@ def launcher_target(engine, source, portable_root=None):
         from .launcher_update import current_build
         folder = (current_build() or {}).get('packaging') == 'onedir'
         fingerprint = digest(executable)
-        parent = Path(engine) / 'launcher/application'
+        parent = copies_folder(engine, fingerprint)
         for root in [parent / fingerprint, *sorted(parent.glob(fingerprint + '-*'))]:
             if _complete(root, fingerprint, folder):
                 return root / 'FreeVideo.exe', [], root / 'FreeVideo.exe'

freevideo_engine/failure_details.py

@@ -107,6 +107,14 @@ def launcher_failure(value, *, zh=False):
          ('This folder holds an offline installation', '这个位置是离线包安装'),
          ('Open it from the launcher. For an automatic installation, go back to Setup and choose another folder.',
           '请在启动页直接打开这份安装。如需自动安装，请回到安装设置，换一个安装位置。')),
+        # Both carry ComfyUI's log, whose unrelated lines must not pick the card.
+        (r'ComfyUI is still starting or could not load FreeVideo', 'comfy-nodes',
+         ('ComfyUI did not load FreeVideo', 'ComfyUI 没有加载 FreeVideo'),
+         ('Click Retry. If it stops here again, the ComfyUI log in the details shows why the FreeVideo nodes did not load; export the report to send it.',
+          '请点击重试。如果仍停在这里，详情里的 ComfyUI 日志会显示 FreeVideo 节点没有加载的原因，可以导出报告反馈。')),
+        (r'ComfyUI is already running\. Restart it once', 'comfy-restart',
+         ('ComfyUI must restart to load FreeVideo', 'ComfyUI 需要重启才能加载 FreeVideo'),
+         ('Close the ComfyUI that is running now, then click Retry.', '请关闭正在运行的 ComfyUI，然后点击重试。')),
         # web/health.js lists the failed files and what to do on the page itself.
         (r'The browser page did not load FreeVideo completely|浏览器页面没有完整加载 FreeVideo', 'page',
          ('FreeVideo did not load completely in the browser', '浏览器里的 FreeVideo 没有完整加载'),

freevideo_engine/policy.py

@@ -180,6 +180,9 @@ def prefetch_for_budget(gpu_budget, *, ram_budget=None, twelve=False, ampere=Fal
                 and architecture == 'blackwell-rtx')
             or (twelve and ampere))
 
+# The transformer's attention heads; a group this wide is unchunked.
+ALL_HEADS = 56
+
 # Whole-device bytes a request needs beside the resident blocks, by head
 # group and token count, with the attention outputs on the GPU. A head-group
 # slice is tokens x heads x 128 x 2 bytes, so this scales with the request
@@ -209,6 +212,23 @@ def prefetch_for_budget(gpu_budget, *, ram_budget=None, twelve=False, ampere=Fal
     4: ((41472, 6.11), (72576, 7.69), (102816, 9.97)),
     8: ((41472, 6.11), (72576, 8.03), (102816, 10.53)),
     16: ((41472, 6.75), (72576, 10.05), (102816, 14.21)),
+    # Wider groups for cards with room to spare. Linux RTX PRO 6000, eight
+    # resident blocks, complete requests from 672x384 to 1920x1088, measured
+    # with the feed-forward unchunked, which needs a little more than the
+    # 2048-row tiles kept here (0.6 GiB at 72576 tokens): whole-device peak
+    # minus the blocks, times 1.05 because FP8 measured 4% above int8 (20.66
+    # against 19.82 GiB at 72576 tokens), plus 0.25 GiB. The same method gave
+    # 4.79 / 7.15 / 10.31 / 13.72 / 18.09 GiB for sixteen heads, close to the
+    # anchors above. With the 2048-row tiles kept, complete 1344x768x243
+    # requests against sixteen heads: all 56 took 7.7% less time per
+    # full-resolution step and 9.3% per first-pass step (int8; FP8 8.1 and
+    # 11.3%), 28 heads 6.4% at full resolution; a 16 GiB budget, which widens
+    # only the first pass, sampled 6% faster, 20 GiB 8.6% and 96 GiB 10.8%.
+    # The tiles stay: a pass with a different group changes them in place,
+    # and the unchunked feed-forward would cost that path special cases for
+    # 2 to 3%.
+    28: ((18144, 6.32), (41472, 9.85), (72576, 14.89), (102816, 21.08), (146880, 28.83)),
+    ALL_HEADS: ((18144, 8.42), (41472, 13.73), (72576, 21.06), (102816, 29.0), (146880, 39.54)),
 }
 
 
@@ -844,8 +864,14 @@ def held(group):
         # A fixed budget threshold cannot express this: 41472 tokens take
         # sixteen heads on an 8 GiB card and 102816 do not take eight on a
         # 10 GiB one, and the group is worth 13 to 51% where it fits.
+        # Groups wider than sixteen only with room left for the blocks the
+        # card holds anyway: the eight every larger card keeps, or more where
+        # RAM cannot take the rest (ram_wants below), so a wider group never
+        # moves weights to disk.
+        held = max(RESIDENT_TARGET, 50 - max(0, int((retention_budget - 2.5 * GiB) // block_bytes)))
         head = next((group for group in sorted(ACTIVATION_BY_TOKENS, reverse=True)
-                     if gpu_budget >= activation_bytes(group, effective_tokens)), 4)
+                     if gpu_budget >= activation_bytes(group, effective_tokens)
+                     + (held * block_bytes if group > 16 else 0)), 4)
     resident = (4 if ampere else 1) if twelve else 3
     residual_offload = False
     cpu_outputs = False
@@ -1039,11 +1065,18 @@ def held(group):
             # after two allocator retries; 1344x768x243 without blocks, 3.88 and
             # 4.52 with none. Every 0.3.2 Windows staging plan held no blocks.
             resident = 0
+    pinned_like = None
     if not small:
         # One measured requirement for the group in use, so the block count is
         # continuous across the band edges rather than restarting at each.
         resident = max(0, min(RESIDENT_VRAM_CEILING,
                               int((gpu_budget - activation_bytes(head, effective_tokens)) / block_bytes)))
+        # The pinned subset below keeps the size sixteen heads give it: a
+        # wider group's activations come out of the same budget, and a lower
+        # count here would only lock more host memory, which no run measured.
+        if head > 16:
+            pinned_like = max(0, min(RESIDENT_VRAM_CEILING,
+                                     int((gpu_budget - activation_bytes(16, effective_tokens)) / block_bytes)))
     # Placement alone is numerically neutral. Scale the activation allowance
     # with the actual token count before spending memory on resident weights.
     # The small staging path is anchored to the complete 243-frame Ada run;
@@ -1123,7 +1156,7 @@ def held(group):
         head, resident, prefetch = 4, 0, False
         cpu_outputs = residual_offload = True
     pin = max(0, min(22_000_000_000, retention_budget - HOST_WEIGHT_HEADROOM)) / 1e9
-    pin = min(pin, (50 - resident) * block_bytes / 1e9)
+    pin = min(pin, (50 - (resident if pinned_like is None else pinned_like)) * block_bytes / 1e9)
     if small:
         # Only locking every offloaded layer removes the offloader's own
         # staging buffers, so a partial pin pays full memory for both: a
@@ -1330,8 +1363,11 @@ def held(group):
     # Windows also avoids the whole-checkpoint CPU load before uploading the
     # resident prefix. Keep residency/headroom unchanged: this spends less
     # memory on the same placement, rather than filling the reclaimed space.
-    decoder_linear_cache = ((hardware.architecture == 'blackwell-rtx'
-                             and hardware.system == 'Linux')
+    # On Linux every architecture: the cache holds exactly the FP16 values
+    # autocast casts to on each call. On an RTX PRO 6000 a 1344x768x243
+    # decode measured 19.2 s without it and 15.4 s with it, 4.2 GiB lower,
+    # bit-identical; Hopper and Ampere cast the same way.
+    decoder_linear_cache = (hardware.system == 'Linux'
                             or (hardware.architecture in ('ada', 'blackwell-rtx')
                                 and hardware.system == 'Windows'))
     if decoder_offload:

freevideo_engine/portable.py

@@ -221,10 +221,23 @@ def initialize(root, progress=lambda **kw: None):
     return machine
 
 
-def connect(root, value, machine, progress=lambda **kw: None, cancelled=None, *, url='http://127.0.0.1:8188', on_controller=None):
-    """Use the existing ComfyUI connection/startup behavior without installing."""
-    from .comfy_launcher_runtime import Controller
+def connect(root, value, machine, progress=lambda **kw: None, cancelled=None, *, url='http://127.0.0.1:8188',
+            on_controller=None, previous=None):
+    """Use the existing ComfyUI connection/startup behavior without installing.
+
+    previous: the controller of an earlier attempt, whose ComfyUI may still run.
+    """
+    from .comfy_launcher_runtime import Controller, deploy, node_target
+    comfy = inside(root, value['comfy'])
+    if node_target(comfy).resolve() != Path(machine['source']).resolve():
+        # An environment package brings ComfyUI without the FreeVideo node: the
+        # launcher source it was imported with backs one, as in any installation.
+        deploy(comfy, machine['source'], machine['root'])
     controller = Controller(machine['source'], child_environment=lambda: environment(root, value))
+    if previous is not None and previous.owns_server():
+        # A ComfyUI this launcher started for an earlier attempt, e.g. one that
+        # came up without FreeVideo: connecting restarts it instead of asking to.
+        controller.server, controller.server_log = previous.server, previous.server_log
     controller.shortcut_root = root
     if not machine['gpu_uuid'].startswith(('GPU-', 'MIG-')):
         raise ValueError('The CUDA device did not provide a valid GPU identity; rerun the GPU check.')
@@ -235,7 +248,7 @@ def connect(root, value, machine, progress=lambda **kw: None, cancelled=None, *,
         controller.cancelled = cancelled
         if cancelled.is_set():
             raise RuntimeError('Startup cancelled')
-    controller.selection = dict(ready=True, root=str(inside(root, value['comfy'])), engine=machine['root'],
+    controller.selection = dict(ready=True, root=str(comfy), engine=machine['root'],
         source=machine['source'], python=machine['python'], url=url,
         portable=False, separate=False)
     controller.restore_terminal(machine['root'])

freevideo_engine/portable_launcher.py

@@ -130,7 +130,8 @@ def _launch(self, values):
         machine = json.loads((self.root/'engine/machine.json').read_text(encoding='utf-8'))
         self.progress(dict(stage='comfy', detail='Open FreeVideo workflow'))
         opened = portable.connect(self.root, self.bundle, machine, self.progress_keywords,
-                                  self.cancelled, url=url, on_controller=self.retain_comfy_controller)
+                                  self.cancelled, url=url, on_controller=self.retain_comfy_controller,
+                                  previous=self.comfy_controller)
         self.selection['ready'] = True
         self.ensure_shortcut()
         self.state = dict(self.state, status='open', url=opened, selection=dict(self.selection),

freevideo_engine/release_notes.json

@@ -1,21 +1,42 @@
 {
   "schema": 1,
-  "product_version": "0.3.19",
+  "product_version": "0.3.20",
   "en": {
-    "summary": "On Windows, GPUs with 8 or 10 GB of memory sample the first pass a little faster when a video uses reference images, a first or last frame, or reference audio.",
+    "summary": "Offline installations now open FreeVideo in ComfyUI, and on Linux, GPUs with memory to spare generate faster.",
     "changes": [
-      "On Windows GPUs with 8 or 10 GB of memory, the first pass of a video up to 10 seconds long now runs attention 8 heads at a time when the video uses reference images, a first or last frame, or reference audio, as text-only videos already did, instead of 4. With one reference image, a 10-second 1344×768 video took about 8% less time per first-pass step on an 8 GB budget, staying within the same video memory budget. The same seed can give a slightly different video than before, as it already does between GPUs with different memory.",
-      "On Windows, the model layers kept on the GPU between first-pass steps no longer shrink from step to step, so fewer layers are transferred again on each step."
+      "Offline installations imported from the environment package now open FreeVideo in ComfyUI: the launcher installs the FreeVideo node that the package does not carry. Before, ComfyUI started without FreeVideo and the launch stopped at \"could not load FreeVideo\". A retry now also restarts the ComfyUI that an earlier attempt left running, and the message no longer suggests repairing the environment.",
+      "Offline installations in a longer folder path now also get their desktop and Start menu shortcuts. Windows cannot create a shortcut to a program whose path is 260 characters or longer, and the folders of an offline installation could make the launcher copy's path that long; the copy now goes to a shorter folder in that case.",
+      "On Linux, when the GPU has memory to spare, attention now runs 28 or all 56 heads at a time instead of 16; how many model layers stay on the GPU, and what is kept in system memory, does not change. On a 96 GB GPU, sampling a 10-second 1344×768 video took about 11% less time; with a 16–20 GB budget the gain is about 6–9%. Smaller GPUs and Windows keep their current settings. The same seed can give a slightly different video than before, as it already does between GPUs with different memory.",
+      "On Linux, video decoding now uses its FP16 cache on every GPU (it was on for some only). Decoding a 10-second 1344×768 video on a 96 GB GPU took 15.4 s instead of 19.2 s and 4.2 GB less memory, with identical output."
     ]
   },
   "zh": {
-    "summary": "在 Windows 上，8 GB 和 10 GB 显存的显卡生成带参考图、首尾帧或参考音频的视频时，第一遍采样稍快一些。",
+    "summary": "离线安装可以在 ComfyUI 中打开 FreeVideo；在 Linux 上，显存有富余的显卡生成更快。",
     "changes": [
-      "在 8 GB 和 10 GB 显存的 Windows 显卡上，10 秒以内的视频如果带参考图、首帧或尾帧、参考音频，第一遍注意力改为每次计算 8 个头（原来是 4 个），与纯文本生成一致。带一张参考图生成 10 秒 1344×768 视频时，在 8 GB 显存预算下第一遍每步缩短约 8%，显存仍在原来的预算之内。同一个种子生成的视频可能与以前略有不同，就像在显存不同的显卡上本来就会有差异一样。",
-      "在 Windows 上，第一遍步与步之间保留在显卡上的模型层不再逐步减少，每步需要重新传输的层更少。"
+      "用环境包导入的离线安装现在可以在 ComfyUI 中打开 FreeVideo：启动器会安装环境包里没有带的 FreeVideo 节点。之前 ComfyUI 启动时没有加载 FreeVideo，启动停在“无法加载 FreeVideo”。重试时也会重启上一次留下的 ComfyUI，提示也不再让你去修复运行环境。",
+      "离线安装放在较长的文件夹路径下时，现在也能创建桌面和开始菜单快捷方式。Windows 无法为路径达到 260 个字符的程序创建快捷方式，而离线安装的目录层级可能让启动器副本的路径超过这个长度；这种情况下，副本会改放到路径较短的位置。",
+      "在 Linux 上，显存有富余时，注意力改为每次计算 28 个或全部 56 个头（原来每次 16 个）；留在显卡上的模型层数和放在内存里的内容都不变。在 96 GB 显卡上，生成 10 秒 1344×768 视频的采样时间缩短约 11%；显存预算在 16–20 GB 时约缩短 6–9%。显存较小的显卡和 Windows 保持现有设置。同一个种子生成的视频可能与以前略有不同，就像在显存不同的显卡上本来就会有差异一样。",
+      "在 Linux 上，所有显卡的视频解码都使用 FP16 缓存（以前只有部分显卡开启）。在 96 GB 显卡上解码 10 秒 1344×768 视频从 19.2 秒降到 15.4 秒，少占 4.2 GB 显存，输出完全一致。"
     ]
   },
   "history": [
+    {
+      "product_version": "0.3.19",
+      "en": {
+        "summary": "On Windows, GPUs with 8 or 10 GB of memory sample the first pass a little faster when a video uses reference images, a first or last frame, or reference audio.",
+        "changes": [
+          "On Windows GPUs with 8 or 10 GB of memory, the first pass of a video up to 10 seconds long now runs attention 8 heads at a time when the video uses reference images, a first or last frame, or reference audio, as text-only videos already did, instead of 4. With one reference image, a 10-second 1344×768 video took about 8% less time per first-pass step on an 8 GB budget, staying within the same video memory budget. The same seed can give a slightly different video than before, as it already does between GPUs with different memory.",
+          "On Windows, the model layers kept on the GPU between first-pass steps no longer shrink from step to step, so fewer layers are transferred again on each step."
+        ]
+      },
+      "zh": {
+        "summary": "在 Windows 上，8 GB 和 10 GB 显存的显卡生成带参考图、首尾帧或参考音频的视频时，第一遍采样稍快一些。",
+        "changes": [
+          "在 8 GB 和 10 GB 显存的 Windows 显卡上，10 秒以内的视频如果带参考图、首帧或尾帧、参考音频，第一遍注意力改为每次计算 8 个头（原来是 4 个），与纯文本生成一致。带一张参考图生成 10 秒 1344×768 视频时，在 8 GB 显存预算下第一遍每步缩短约 8%，显存仍在原来的预算之内。同一个种子生成的视频可能与以前略有不同，就像在显存不同的显卡上本来就会有差异一样。",
+          "在 Windows 上，第一遍步与步之间保留在显卡上的模型层不再逐步减少，每步需要重新传输的层更少。"
+        ]
+      }
+    },
     {
       "product_version": "0.3.18",
       "en": {
@@ -232,23 +253,6 @@
           "ComfyUI 内的安装面板会在按钮下方说明「安装／修复」暂不可用的原因：需要先「检测并预览」、勾选同意，或先解决计划中标红的问题；修改设置后会清除过期的计划。"
         ]
       }
-    },
-    {
-      "product_version": "0.3.7",
-      "en": {
-        "summary": "Installation no longer stops with \"Select at most 64 model libraries\" when ComfyUI has many model folders.",
-        "changes": [
-          "Fixed installation stopping with \"Select at most 64 model libraries\" when ComfyUI has more than ten model folders. Each folder now counts once, and when more than 64 are found, the first 64 are used and installation continues.",
-          "A saved model folder that has since been deleted, such as a removed offline package folder, is skipped instead of blocking installation or offline import."
-        ]
-      },
-      "zh": {
-        "summary": "修复 ComfyUI 模型目录较多时，安装提示「Select at most 64 model libraries」的问题。",
-        "changes": [
-          "修复 ComfyUI 模型目录超过十个时，安装停在「Select at most 64 model libraries」的问题。同一个目录现在只计一次；找到的目录超过 64 个时使用前 64 个，安装照常进行。",
-          "已保存的模型文件夹如果已被删除（例如删除了离线包目录），会自动跳过，不再阻止安装或离线导入。"
-        ]
-      }
     }
   ]
 }