From bc1d2143ca16319a9e6f5048b5cf37ef999ceaba Mon Sep 17 00:00:00 2001 From: dxqb <183307934+dxqb@users.noreply.github.com> Date: Sat, 18 Jul 2026 16:24:46 +0200 Subject: [PATCH 1/3] Quiet non-actionable startup warnings Suppress a handful of specific, noisy-but-harmless messages emitted while launching the UI and starting training: - diffusers/transformers logger.warning() lines (Modular Diffusers experimental notice, unexpected-config-attributes, unrecognized loss_type) via filters on the exact emitting loggers - huggingface_hub local_dir_use_symlinks deprecation and the torch.compile inductor performance notes via warnings/logger filters - Qt gnome portal dbus errors via QT_LOGGING_RULES - tensorboard subprocess banner/notices by discarding its stdout/stderr Each filter targets one specific message, so other warnings from the same libraries still come through. Co-Authored-By: Claude Opus 4.8 (1M context) --- modules/trainer/BaseTrainer.py | 8 +++++++- modules/ui/TrainUIController.py | 7 ++++++- modules/util/ui/pyside6_util.py | 9 +++++++++ scripts/util/import_util.py | 36 +++++++++++++++++++++++++++++++++ 4 files changed, 58 insertions(+), 2 deletions(-) diff --git a/modules/trainer/BaseTrainer.py b/modules/trainer/BaseTrainer.py index 20fc5eb7a..e4b7a3b29 100644 --- a/modules/trainer/BaseTrainer.py +++ b/modules/trainer/BaseTrainer.py @@ -97,7 +97,13 @@ def _start_tensorboard(self): if self.config.tensorboard_expose: tensorboard_args.append("--bind_all") - self.tensorboard_subprocess = subprocess.Popen(tensorboard_args) + # Discard the tensorboard child's stdout/stderr: the TF-not-found notice, the + # experimental-data-loading NOTE and the serving banner are all noise, and the + # UI already exposes the tensorboard URL. Popen still raises if the executable + # is missing, so a real launch failure is not hidden. + self.tensorboard_subprocess = subprocess.Popen( + tensorboard_args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + ) def _stop_tensorboard(self): self.tensorboard_subprocess.kill() diff --git a/modules/ui/TrainUIController.py b/modules/ui/TrainUIController.py index 72623def4..2c4896226 100644 --- a/modules/ui/TrainUIController.py +++ b/modules/ui/TrainUIController.py @@ -103,8 +103,13 @@ def _start_always_on_tensorboard(self): if self.train_config.tensorboard_expose: tensorboard_args.append("--bind_all") + # Discard the tensorboard child's stdout/stderr: the TF-not-found notice, the + # experimental-data-loading NOTE and the serving banner are all noise, and the + # UI already exposes the tensorboard URL. try: - self.always_on_tensorboard_subprocess = subprocess.Popen(tensorboard_args) + self.always_on_tensorboard_subprocess = subprocess.Popen( + tensorboard_args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + ) except Exception: self.always_on_tensorboard_subprocess = None diff --git a/modules/util/ui/pyside6_util.py b/modules/util/ui/pyside6_util.py index fd7b2ef8a..42b4a3e60 100644 --- a/modules/util/ui/pyside6_util.py +++ b/modules/util/ui/pyside6_util.py @@ -1,3 +1,4 @@ +import os import signal import sys from abc import ABCMeta @@ -19,6 +20,14 @@ def create_application() -> QApplication: # active and Ctrl+C would be ignored. signal.signal(signal.SIGINT, signal.SIG_DFL) + # On desktops without the xdg-desktop-portal Settings interface, Qt spams two + # "qt.qpa.theme.gnome: dbus reply error ... org.freedesktop.portal.Settings" + # lines while probing for the system color scheme. Silence just that category; + # the rules string is read when Qt's logging initializes at QApplication init. + _gnome_theme_rule = "qt.qpa.theme.gnome=false" + existing_rules = os.environ.get("QT_LOGGING_RULES") + os.environ["QT_LOGGING_RULES"] = f"{existing_rules};{_gnome_theme_rule}" if existing_rules else _gnome_theme_rule + app = QApplication(sys.argv) # Force Fusion everywhere: native styles (e.g. windowsvista) draw standard # controls via OS theme APIs, which breaks once an application stylesheet diff --git a/scripts/util/import_util.py b/scripts/util/import_util.py index 150df4d8a..a0c0a83b6 100644 --- a/scripts/util/import_util.py +++ b/scripts/util/import_util.py @@ -1,7 +1,9 @@ def script_imports(allow_zluda: bool = True): import logging import os + import re import sys + import warnings from pathlib import Path # Filter out the Triton warning on startup. @@ -10,6 +12,40 @@ def script_imports(allow_zluda: bool = True): .getLogger("xformers") \ .addFilter(lambda record: 'A matching Triton is not available' not in record.getMessage()) + # Silence specific non-actionable startup/compile warnings. A logger filter + # targets the exact emitting logger, since a parent logger's filter misses + # records from child loggers. + + # diffusers/transformers chatty logger.warning() lines at import/load time. + logging.getLogger("diffusers.modular_pipelines").addFilter( + lambda record: 'Modular Diffusers is currently an experimental feature' not in record.getMessage() + ) + # The subject of these two is interpolated into the message, so match the whole + # sentence with .* standing in for the runtime value. + logging.getLogger("diffusers.configuration_utils").addFilter( + lambda record: not re.search( + r"The config attributes .* were passed to .*, but are not expected and will be ignored", + record.getMessage(), + ) + ) + logging.getLogger("transformers.modeling_utils").addFilter( + lambda record: not re.search( + r"`loss_type=.*` was set in the config but it is unrecognized", record.getMessage() + ) + ) + + # A dependency still calls hf_hub_download with the removed local_dir_use_symlinks + # argument; the deprecation warning is not actionable. + warnings.filterwarnings("ignore", message=r".*local_dir_use_symlinks.*") + + # torch.compile emits performance notes when inductor falls back or can't use a + # fast path; harmless and noisy for normal runs. The SMs note is a logger.warning() + # on its exact emitting logger; the complex-operators note is a warnings.warn(). + warnings.filterwarnings("ignore", message=r".*does not support code generation for complex operators.*") + logging.getLogger("torch._inductor.utils").addFilter( + lambda record: 'Not enough SMs to use max_autotune_gemm mode' not in record.getMessage() + ) + # Insert ourselves as the highest-priority library path, so our modules are # always found without any risk of being shadowed by another import path. # 3 .parent calls to navigate from /scripts/util/import_util.py to the main directory From eb85620630a45b5616929ef913b028430bba2459 Mon Sep 17 00:00:00 2001 From: dxqb <183307934+dxqb@users.noreply.github.com> Date: Sun, 9 Aug 2026 18:56:16 +0200 Subject: [PATCH 2/3] Show compile progress in a progress bar instead of a line per compile A cold torch.compile cache announces every frame it compiles, which scrolls the progress bar off the screen. There is no knowable total to build a real progress bar from, so the announcement goes into the postfix of the innermost running bar instead, and is cleared again by that bar's next redraw or by its close. tqdm keeps its bars in an unordered WeakSet, so which of the nested bars is the innermost one cannot be recovered from it. modules/util/tqdm_util.py subclasses tqdm to track that itself and adds show_status() next to tqdm.write(); every tqdm import in the repo now comes from there. Bars owned by mgds are outside this and still draw as before. Also gates the warning filters on OT_DEBUG_WARNINGS, so setting it brings every suppressed message back, and silences the diffusers attention-backend experimental notice. Co-Authored-By: Claude Opus 5 (1M context) --- modules/modelSampler/AnimaSampler.py | 2 +- modules/modelSampler/ChromaSampler.py | 3 +- modules/modelSampler/ErnieSampler.py | 2 +- modules/modelSampler/Flux2Sampler.py | 2 +- modules/modelSampler/FluxSampler.py | 3 +- modules/modelSampler/HiDreamSampler.py | 3 +- modules/modelSampler/HunyuanVideoSampler.py | 2 +- modules/modelSampler/IdeogramSampler.py | 2 +- modules/modelSampler/Krea2Sampler.py | 3 +- modules/modelSampler/PixArtAlphaSampler.py | 3 +- modules/modelSampler/QwenSampler.py | 3 +- modules/modelSampler/SanaSampler.py | 3 +- .../modelSampler/StableDiffusion3Sampler.py | 3 +- .../modelSampler/StableDiffusionSampler.py | 3 +- .../modelSampler/StableDiffusionXLSampler.py | 3 +- modules/modelSampler/WuerstchenSampler.py | 2 +- modules/modelSampler/ZImageSampler.py | 3 +- modules/module/BaseImageCaptionModel.py | 2 +- modules/module/BaseImageMaskModel.py | 2 +- modules/module/GenerateLossesModel.py | 3 +- modules/trainer/BaseTrainer.py | 6 +- modules/trainer/GenericTrainer.py | 3 +- modules/ui/TrainUIController.py | 4 +- modules/util/compile_util.py | 5 +- modules/util/multi_gpu_util.py | 3 +- modules/util/quantization_util.py | 2 +- modules/util/tqdm_util.py | 49 ++++++++++++++++ scripts/util/import_util.py | 57 ++++++++++--------- 28 files changed, 108 insertions(+), 73 deletions(-) create mode 100644 modules/util/tqdm_util.py diff --git a/modules/modelSampler/AnimaSampler.py b/modules/modelSampler/AnimaSampler.py index d5de18ac1..56d5c523e 100644 --- a/modules/modelSampler/AnimaSampler.py +++ b/modules/modelSampler/AnimaSampler.py @@ -12,13 +12,13 @@ from modules.util.enum.ModelType import ModelType from modules.util.enum.NoiseScheduler import NoiseScheduler from modules.util.enum.VideoFormat import VideoFormat +from modules.util.tqdm_util import tqdm import torch from diffusers import VaeImageProcessor import numpy as np -from tqdm import tqdm @factory.register(BaseModelSampler, ModelType.ANIMA) diff --git a/modules/modelSampler/ChromaSampler.py b/modules/modelSampler/ChromaSampler.py index 23ef4aee5..b8325d297 100644 --- a/modules/modelSampler/ChromaSampler.py +++ b/modules/modelSampler/ChromaSampler.py @@ -12,11 +12,10 @@ from modules.util.enum.ModelType import ModelType from modules.util.enum.NoiseScheduler import NoiseScheduler from modules.util.enum.VideoFormat import VideoFormat +from modules.util.tqdm_util import tqdm import torch -from tqdm import tqdm - @factory.register(BaseModelSampler, ModelType.CHROMA_1) class ChromaSampler(BaseModelSampler): diff --git a/modules/modelSampler/ErnieSampler.py b/modules/modelSampler/ErnieSampler.py index a9cb57e0a..161390f17 100644 --- a/modules/modelSampler/ErnieSampler.py +++ b/modules/modelSampler/ErnieSampler.py @@ -11,12 +11,12 @@ from modules.util.enum.ModelType import ModelType from modules.util.enum.NoiseScheduler import NoiseScheduler from modules.util.enum.VideoFormat import VideoFormat +from modules.util.tqdm_util import tqdm import torch import numpy as np from PIL import Image as PILImage -from tqdm import tqdm @factory.register(BaseModelSampler, ModelType.ERNIE) diff --git a/modules/modelSampler/Flux2Sampler.py b/modules/modelSampler/Flux2Sampler.py index 7ecbd5c83..514e9648c 100644 --- a/modules/modelSampler/Flux2Sampler.py +++ b/modules/modelSampler/Flux2Sampler.py @@ -12,13 +12,13 @@ from modules.util.enum.ModelType import ModelType from modules.util.enum.NoiseScheduler import NoiseScheduler from modules.util.enum.VideoFormat import VideoFormat +from modules.util.tqdm_util import tqdm import torch from diffusers.pipelines.flux2.pipeline_flux2 import compute_empirical_mu import numpy as np -from tqdm import tqdm @factory.register(BaseModelSampler, ModelType.FLUX_2) diff --git a/modules/modelSampler/FluxSampler.py b/modules/modelSampler/FluxSampler.py index fc2b2e0c8..8a935be79 100644 --- a/modules/modelSampler/FluxSampler.py +++ b/modules/modelSampler/FluxSampler.py @@ -14,13 +14,12 @@ from modules.util.enum.NoiseScheduler import NoiseScheduler from modules.util.enum.VideoFormat import VideoFormat from modules.util.image_util import load_image +from modules.util.tqdm_util import tqdm import torch from torch import nn from torchvision.transforms import transforms -from tqdm import tqdm - @factory.register(BaseModelSampler, ModelType.FLUX_DEV_1) @factory.register(BaseModelSampler, ModelType.FLUX_FILL_DEV_1) diff --git a/modules/modelSampler/HiDreamSampler.py b/modules/modelSampler/HiDreamSampler.py index c5b30723b..e04209a00 100644 --- a/modules/modelSampler/HiDreamSampler.py +++ b/modules/modelSampler/HiDreamSampler.py @@ -12,11 +12,10 @@ from modules.util.enum.ModelType import ModelType from modules.util.enum.NoiseScheduler import NoiseScheduler from modules.util.enum.VideoFormat import VideoFormat +from modules.util.tqdm_util import tqdm import torch -from tqdm import tqdm - @factory.register(BaseModelSampler, ModelType.HI_DREAM_FULL) class HiDreamSampler(BaseModelSampler): diff --git a/modules/modelSampler/HunyuanVideoSampler.py b/modules/modelSampler/HunyuanVideoSampler.py index 10b22bfc9..49cb0b426 100644 --- a/modules/modelSampler/HunyuanVideoSampler.py +++ b/modules/modelSampler/HunyuanVideoSampler.py @@ -12,11 +12,11 @@ from modules.util.enum.ModelType import ModelType from modules.util.enum.NoiseScheduler import NoiseScheduler from modules.util.enum.VideoFormat import VideoFormat +from modules.util.tqdm_util import tqdm import torch from PIL import Image -from tqdm import tqdm @factory.register(BaseModelSampler, ModelType.HUNYUAN_VIDEO) diff --git a/modules/modelSampler/IdeogramSampler.py b/modules/modelSampler/IdeogramSampler.py index cb253db0e..2672837e7 100644 --- a/modules/modelSampler/IdeogramSampler.py +++ b/modules/modelSampler/IdeogramSampler.py @@ -11,6 +11,7 @@ from modules.util.enum.ModelType import ModelType from modules.util.enum.NoiseScheduler import NoiseScheduler from modules.util.enum.VideoFormat import VideoFormat +from modules.util.tqdm_util import tqdm import torch @@ -18,7 +19,6 @@ import numpy as np from PIL import Image as PILImage -from tqdm import tqdm @factory.register(BaseModelSampler, ModelType.IDEOGRAM_4) diff --git a/modules/modelSampler/Krea2Sampler.py b/modules/modelSampler/Krea2Sampler.py index b83205a93..eb0853bb8 100644 --- a/modules/modelSampler/Krea2Sampler.py +++ b/modules/modelSampler/Krea2Sampler.py @@ -13,13 +13,12 @@ from modules.util.enum.ModelType import ModelType from modules.util.enum.NoiseScheduler import NoiseScheduler from modules.util.enum.VideoFormat import VideoFormat +from modules.util.tqdm_util import tqdm import torch from diffusers import Krea2Pipeline -from tqdm import tqdm - @factory.register(BaseModelSampler, ModelType.KREA_2) class Krea2Sampler(BaseModelSampler): diff --git a/modules/modelSampler/PixArtAlphaSampler.py b/modules/modelSampler/PixArtAlphaSampler.py index f8c38f135..58db739ec 100644 --- a/modules/modelSampler/PixArtAlphaSampler.py +++ b/modules/modelSampler/PixArtAlphaSampler.py @@ -11,11 +11,10 @@ from modules.util.enum.ModelType import ModelType from modules.util.enum.NoiseScheduler import NoiseScheduler from modules.util.enum.VideoFormat import VideoFormat +from modules.util.tqdm_util import tqdm import torch -from tqdm import tqdm - @factory.register(BaseModelSampler, ModelType.PIXART_ALPHA) @factory.register(BaseModelSampler, ModelType.PIXART_SIGMA) diff --git a/modules/modelSampler/QwenSampler.py b/modules/modelSampler/QwenSampler.py index c18eece7e..07e66f202 100644 --- a/modules/modelSampler/QwenSampler.py +++ b/modules/modelSampler/QwenSampler.py @@ -13,11 +13,10 @@ from modules.util.enum.ModelType import ModelType from modules.util.enum.NoiseScheduler import NoiseScheduler from modules.util.enum.VideoFormat import VideoFormat +from modules.util.tqdm_util import tqdm import torch -from tqdm import tqdm - @factory.register(BaseModelSampler, ModelType.QWEN) class QwenSampler(BaseModelSampler): diff --git a/modules/modelSampler/SanaSampler.py b/modules/modelSampler/SanaSampler.py index f4089a222..ee7df291d 100644 --- a/modules/modelSampler/SanaSampler.py +++ b/modules/modelSampler/SanaSampler.py @@ -12,11 +12,10 @@ from modules.util.enum.ModelType import ModelType from modules.util.enum.NoiseScheduler import NoiseScheduler from modules.util.enum.VideoFormat import VideoFormat +from modules.util.tqdm_util import tqdm import torch -from tqdm import tqdm - @factory.register(BaseModelSampler, ModelType.SANA) class SanaSampler(BaseModelSampler): diff --git a/modules/modelSampler/StableDiffusion3Sampler.py b/modules/modelSampler/StableDiffusion3Sampler.py index f21f34627..67fef6baa 100644 --- a/modules/modelSampler/StableDiffusion3Sampler.py +++ b/modules/modelSampler/StableDiffusion3Sampler.py @@ -12,11 +12,10 @@ from modules.util.enum.ModelType import ModelType from modules.util.enum.NoiseScheduler import NoiseScheduler from modules.util.enum.VideoFormat import VideoFormat +from modules.util.tqdm_util import tqdm import torch -from tqdm import tqdm - @factory.register(BaseModelSampler, ModelType.STABLE_DIFFUSION_3) @factory.register(BaseModelSampler, ModelType.STABLE_DIFFUSION_35) diff --git a/modules/modelSampler/StableDiffusionSampler.py b/modules/modelSampler/StableDiffusionSampler.py index 791290edc..6f49ef737 100644 --- a/modules/modelSampler/StableDiffusionSampler.py +++ b/modules/modelSampler/StableDiffusionSampler.py @@ -12,13 +12,12 @@ from modules.util.enum.NoiseScheduler import NoiseScheduler from modules.util.enum.VideoFormat import VideoFormat from modules.util.image_util import load_image +from modules.util.tqdm_util import tqdm import torch from torch import nn from torchvision.transforms import transforms -from tqdm import tqdm - @factory.register(BaseModelSampler, ModelType.STABLE_DIFFUSION_15) @factory.register(BaseModelSampler, ModelType.STABLE_DIFFUSION_15_INPAINTING) diff --git a/modules/modelSampler/StableDiffusionXLSampler.py b/modules/modelSampler/StableDiffusionXLSampler.py index 93d9f23d3..603ef9be6 100644 --- a/modules/modelSampler/StableDiffusionXLSampler.py +++ b/modules/modelSampler/StableDiffusionXLSampler.py @@ -12,13 +12,12 @@ from modules.util.enum.NoiseScheduler import NoiseScheduler from modules.util.enum.VideoFormat import VideoFormat from modules.util.image_util import load_image +from modules.util.tqdm_util import tqdm import torch from torch import nn from torchvision.transforms import transforms -from tqdm import tqdm - @factory.register(BaseModelSampler, ModelType.STABLE_DIFFUSION_XL_10_BASE) @factory.register(BaseModelSampler, ModelType.STABLE_DIFFUSION_XL_10_BASE_INPAINTING) diff --git a/modules/modelSampler/WuerstchenSampler.py b/modules/modelSampler/WuerstchenSampler.py index a1e9d8ba0..d5c833031 100644 --- a/modules/modelSampler/WuerstchenSampler.py +++ b/modules/modelSampler/WuerstchenSampler.py @@ -11,11 +11,11 @@ from modules.util.enum.ModelType import ModelType from modules.util.enum.NoiseScheduler import NoiseScheduler from modules.util.enum.VideoFormat import VideoFormat +from modules.util.tqdm_util import tqdm import torch from PIL import Image -from tqdm import tqdm @factory.register(BaseModelSampler, ModelType.WUERSTCHEN_2) diff --git a/modules/modelSampler/ZImageSampler.py b/modules/modelSampler/ZImageSampler.py index 0e001df2a..371dd20c5 100644 --- a/modules/modelSampler/ZImageSampler.py +++ b/modules/modelSampler/ZImageSampler.py @@ -12,11 +12,10 @@ from modules.util.enum.ModelType import ModelType from modules.util.enum.NoiseScheduler import NoiseScheduler from modules.util.enum.VideoFormat import VideoFormat +from modules.util.tqdm_util import tqdm import torch -from tqdm import tqdm - @factory.register(BaseModelSampler, ModelType.Z_IMAGE) class ZImageSampler(BaseModelSampler): diff --git a/modules/module/BaseImageCaptionModel.py b/modules/module/BaseImageCaptionModel.py index 2dfcf4a87..39cf79703 100644 --- a/modules/module/BaseImageCaptionModel.py +++ b/modules/module/BaseImageCaptionModel.py @@ -6,9 +6,9 @@ from modules.util import path_util from modules.util.image_util import load_image +from modules.util.tqdm_util import tqdm from PIL import Image -from tqdm import tqdm class CaptionSample: diff --git a/modules/module/BaseImageMaskModel.py b/modules/module/BaseImageMaskModel.py index 017ec7dd3..f5cfbfabb 100644 --- a/modules/module/BaseImageMaskModel.py +++ b/modules/module/BaseImageMaskModel.py @@ -5,13 +5,13 @@ from modules.util import path_util from modules.util.image_util import load_image +from modules.util.tqdm_util import tqdm import torch from torch import Tensor from torchvision.transforms import transforms from PIL import Image -from tqdm import tqdm class MaskSample: diff --git a/modules/module/GenerateLossesModel.py b/modules/module/GenerateLossesModel.py index d4c821b74..ea90beddc 100644 --- a/modules/module/GenerateLossesModel.py +++ b/modules/module/GenerateLossesModel.py @@ -8,12 +8,11 @@ from modules.util import create from modules.util.config.TrainConfig import QuantizationConfig, TrainConfig from modules.util.torch_util import torch_gc +from modules.util.tqdm_util import tqdm from modules.util.TrainProgress import TrainProgress import torch -from tqdm import tqdm - class GenerateLossesModel: """Based on train args, writes a JSON instead of a model with filenames mapped to losses, diff --git a/modules/trainer/BaseTrainer.py b/modules/trainer/BaseTrainer.py index e4b7a3b29..87a96d7d0 100644 --- a/modules/trainer/BaseTrainer.py +++ b/modules/trainer/BaseTrainer.py @@ -97,10 +97,8 @@ def _start_tensorboard(self): if self.config.tensorboard_expose: tensorboard_args.append("--bind_all") - # Discard the tensorboard child's stdout/stderr: the TF-not-found notice, the - # experimental-data-loading NOTE and the serving banner are all noise, and the - # UI already exposes the tensorboard URL. Popen still raises if the executable - # is missing, so a real launch failure is not hidden. + # discard the child's banner and notices; the UI already shows the tensorboard URL. + # Popen still raises if the executable is missing. self.tensorboard_subprocess = subprocess.Popen( tensorboard_args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) diff --git a/modules/trainer/GenericTrainer.py b/modules/trainer/GenericTrainer.py index 509811d2c..cce3ec51d 100644 --- a/modules/trainer/GenericTrainer.py +++ b/modules/trainer/GenericTrainer.py @@ -32,6 +32,7 @@ from modules.util.profiling_util import PeakMemoryRecorder, TorchMemoryRecorder, TorchProfiler from modules.util.time_util import get_string_timestamp from modules.util.torch_util import torch_gc +from modules.util.tqdm_util import tqdm from modules.util.TrainProgress import TrainProgress import torch @@ -41,8 +42,6 @@ from torch.utils.tensorboard import SummaryWriter from torchvision.transforms.functional import pil_to_tensor -from tqdm import tqdm - class GenericTrainer(BaseTrainer): model_loader: BaseModelLoader diff --git a/modules/ui/TrainUIController.py b/modules/ui/TrainUIController.py index 4fa3fb688..895be42fa 100644 --- a/modules/ui/TrainUIController.py +++ b/modules/ui/TrainUIController.py @@ -108,9 +108,7 @@ def _start_always_on_tensorboard(self): if self.train_config.tensorboard_expose: tensorboard_args.append("--bind_all") - # Discard the tensorboard child's stdout/stderr: the TF-not-found notice, the - # experimental-data-loading NOTE and the serving banner are all noise, and the - # UI already exposes the tensorboard URL. + # discard the child's banner and notices; the UI already shows the tensorboard URL. try: self.always_on_tensorboard_subprocess = subprocess.Popen( tensorboard_args, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, diff --git a/modules/util/compile_util.py b/modules/util/compile_util.py index 0a09f1023..ffada3b2a 100644 --- a/modules/util/compile_util.py +++ b/modules/util/compile_util.py @@ -1,9 +1,10 @@ +from modules.util.tqdm_util import tqdm + import torch import torch._dynamo.callback import torch.utils._sympy.functions from sympy import S -from tqdm import tqdm #code from https://github.com/pytorch/pytorch/blob/ed82d5fcfd80110565f69130f286c7bfec6db2dc/torch/utils/_sympy/functions.py#L481 @@ -95,7 +96,7 @@ def init_compile(): def _on_compile_start(args: "torch._dynamo.callback.CallbackArgs") -> None: frame_id, _, frame_compile_id = args.compile_id.partition("/") direction = "backward" if args.callback_trigger == torch._dynamo.callback.CallbackTrigger.LAZY_BACKWARD else "forward" - tqdm.write(f"[torch.compile] compiling kernel {frame_id} {direction} (variant #{frame_compile_id or 0})...") + tqdm.show_status(f"compiling kernel {frame_id} {direction} (variant #{frame_compile_id or 0})...") torch._dynamo.callback.on_compile_start(_on_compile_start) diff --git a/modules/util/multi_gpu_util.py b/modules/util/multi_gpu_util.py index 962434b65..545a90ef8 100644 --- a/modules/util/multi_gpu_util.py +++ b/modules/util/multi_gpu_util.py @@ -3,11 +3,10 @@ from modules.util.bf16_stochastic_rounding import copy_stochastic_ from modules.util.commands.TrainCommands import TrainCommands from modules.util.enum.GradientReducePrecision import GradientReducePrecision +from modules.util.tqdm_util import tqdm import torch -from tqdm import tqdm - def is_enabled() -> bool: return torch.distributed.is_available() and torch.distributed.is_initialized() diff --git a/modules/util/quantization_util.py b/modules/util/quantization_util.py index d7229687d..d405802b0 100644 --- a/modules/util/quantization_util.py +++ b/modules/util/quantization_util.py @@ -9,6 +9,7 @@ from modules.util.config.TrainConfig import QuantizationConfig, TrainConfig from modules.util.enum.DataType import DataType from modules.util.ModuleFilter import ModuleFilter +from modules.util.tqdm_util import tqdm import torch from torch import Tensor, nn @@ -16,7 +17,6 @@ from diffusers.quantizers.gguf.utils import GGUFLinear, dequantize_gguf_tensor import accelerate -from tqdm import tqdm try: from modules.module.quantized.LinearNf4 import LinearNf4 diff --git a/modules/util/tqdm_util.py b/modules/util/tqdm_util.py new file mode 100644 index 000000000..838e38fee --- /dev/null +++ b/modules/util/tqdm_util.py @@ -0,0 +1,49 @@ +from tqdm import tqdm as _tqdm + +#progress bars created through the tqdm below, innermost last +_bars = [] + + +class tqdm(_tqdm): + _status = None + + @classmethod + def get_lock(cls): + #tqdm caches the terminal write lock on the class that first asks for it, so a subclass + #would get one of its own and stop serializing against bars drawn by tqdm itself. + return _tqdm.get_lock() + + @classmethod + def show_status(cls, message: str): + #status of long-running work - a compile, an autotune sweep - goes into the innermost bar's + #postfix rather than on a line of its own, and stands until the next postfix write replaces it. + bar = next((bar for bar in reversed(_bars) if not bar.disable), None) + if bar is None: + cls.write(message) + else: + bar.set_postfix_str(message) + bar._status = message + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + _bars.append(self) + + def _clear_status(self, refresh=True): + #anything written over the status - the training loop's loss - is left standing. + if self._status is not None and self.postfix == self._status: + self.set_postfix_str("", refresh=refresh) + self._status = None + + def update(self, n=1): + #the status describes work that was running while the bar stood still, so the step that + #follows it is the point where it stops being current. + self._clear_status(refresh=False) + return super().update(n) + + def close(self): + self._clear_status() + super().close() + #compared by identity: tqdm's __eq__ is by screen position, so a bar closed late by __del__ + #would drop whichever live bar has taken over its line. + global _bars + _bars = [bar for bar in _bars if bar is not self] diff --git a/scripts/util/import_util.py b/scripts/util/import_util.py index a0c0a83b6..e269a0505 100644 --- a/scripts/util/import_util.py +++ b/scripts/util/import_util.py @@ -14,37 +14,40 @@ def script_imports(allow_zluda: bool = True): # Silence specific non-actionable startup/compile warnings. A logger filter # targets the exact emitting logger, since a parent logger's filter misses - # records from child loggers. - - # diffusers/transformers chatty logger.warning() lines at import/load time. - logging.getLogger("diffusers.modular_pipelines").addFilter( - lambda record: 'Modular Diffusers is currently an experimental feature' not in record.getMessage() - ) - # The subject of these two is interpolated into the message, so match the whole - # sentence with .* standing in for the runtime value. - logging.getLogger("diffusers.configuration_utils").addFilter( - lambda record: not re.search( - r"The config attributes .* were passed to .*, but are not expected and will be ignored", - record.getMessage(), + # records from child loggers. Set OT_DEBUG_WARNINGS to see them all. + if not os.environ.get("OT_DEBUG_WARNINGS"): + # diffusers/transformers chatty logger.warning() lines at import/load time. + logging.getLogger("diffusers.modular_pipelines").addFilter( + lambda record: 'Modular Diffusers is currently an experimental feature' not in record.getMessage() + ) + # The subject of these two is interpolated into the message, so match the whole + # sentence with .* standing in for the runtime value. + logging.getLogger("diffusers.configuration_utils").addFilter( + lambda record: not re.search( + r"The config attributes .* were passed to .*, but are not expected and will be ignored", + record.getMessage(), + ) ) - ) - logging.getLogger("transformers.modeling_utils").addFilter( - lambda record: not re.search( - r"`loss_type=.*` was set in the config but it is unrecognized", record.getMessage() + logging.getLogger("diffusers.models.modeling_utils").addFilter( + lambda record: 'Attention backends are an experimental feature' not in record.getMessage() + ) + logging.getLogger("transformers.modeling_utils").addFilter( + lambda record: not re.search( + r"`loss_type=.*` was set in the config but it is unrecognized", record.getMessage() + ) ) - ) - # A dependency still calls hf_hub_download with the removed local_dir_use_symlinks - # argument; the deprecation warning is not actionable. - warnings.filterwarnings("ignore", message=r".*local_dir_use_symlinks.*") + # A dependency still calls hf_hub_download with the removed local_dir_use_symlinks + # argument; the deprecation warning is not actionable. + warnings.filterwarnings("ignore", message=r".*local_dir_use_symlinks.*") - # torch.compile emits performance notes when inductor falls back or can't use a - # fast path; harmless and noisy for normal runs. The SMs note is a logger.warning() - # on its exact emitting logger; the complex-operators note is a warnings.warn(). - warnings.filterwarnings("ignore", message=r".*does not support code generation for complex operators.*") - logging.getLogger("torch._inductor.utils").addFilter( - lambda record: 'Not enough SMs to use max_autotune_gemm mode' not in record.getMessage() - ) + # torch.compile emits performance notes when inductor falls back or can't use a + # fast path; harmless and noisy for normal runs. The SMs note is a logger.warning() + # on its exact emitting logger; the complex-operators note is a warnings.warn(). + warnings.filterwarnings("ignore", message=r".*does not support code generation for complex operators.*") + logging.getLogger("torch._inductor.utils").addFilter( + lambda record: 'Not enough SMs to use max_autotune_gemm mode' not in record.getMessage() + ) # Insert ourselves as the highest-priority library path, so our modules are # always found without any risk of being shadowed by another import path. From 174e3582bdbb6ab7d5907a87fd5329ff288077e5 Mon Sep 17 00:00:00 2001 From: dxqb <183307934+dxqb@users.noreply.github.com> Date: Thu, 20 Aug 2026 02:00:33 +0200 Subject: [PATCH 3/3] Drop the Modular Diffusers experimental-warning filter Upstream diffusers removed the warning itself, so the filter has nothing left to match. It still fires on the diffusers commit pinned in requirements-global.txt, which predates that removal, until the pin is bumped. Also picks up master, which had landed the profiler-steps and mxfp8 changes this branch was still showing as removals. Co-Authored-By: Claude Opus 5 (1M context) --- modules/trainer/GenericTrainer.py | 10 ++++++++-- modules/util/triton_mm_8bit.py | 24 ++++++++++++++++++++++-- scripts/util/import_util.py | 3 --- 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/modules/trainer/GenericTrainer.py b/modules/trainer/GenericTrainer.py index cce3ec51d..61579f314 100644 --- a/modules/trainer/GenericTrainer.py +++ b/modules/trainer/GenericTrainer.py @@ -42,6 +42,12 @@ from torch.utils.tensorboard import SummaryWriter from torchvision.transforms.functional import pil_to_tensor +# OT_DEBUG_PROFILES=1 dumps a CUDA memory snapshot for the first two steps, where the allocator is still +# growing, and a profiler trace at steps 10 and 40, past compilation and warmup. +_DEBUG_PROFILES = os.environ.get("OT_DEBUG_PROFILES") == "1" +_MEMORY_PROFILE_STEPS = (0, 1) if _DEBUG_PROFILES else () +_PROFILE_STEPS = (10, 11, 40, 41) if _DEBUG_PROFILES else () + class GenericTrainer(BaseTrainer): model_loader: BaseModelLoader @@ -724,8 +730,8 @@ def sample_commands_fun(): self.callbacks.on_update_status("Training ...") with ( - TorchMemoryRecorder(enabled=False, filename=f"memory-step{train_progress.global_step}-{get_string_timestamp()}.pickle"), - TorchProfiler (enabled=False, filename=f"profile-step{train_progress.global_step}-{get_string_timestamp()}.json"), + TorchMemoryRecorder(enabled=multi.is_master() and train_progress.global_step in _MEMORY_PROFILE_STEPS, filename=f"memory-step{train_progress.global_step}-{get_string_timestamp()}.pickle"), + TorchProfiler (enabled=multi.is_master() and train_progress.global_step in _PROFILE_STEPS, filename=f"profile-step{train_progress.global_step}-{get_string_timestamp()}.json"), ): step_seed = train_progress.global_step bf16_stochastic_rounding_set_seed(step_seed, train_device) diff --git a/modules/util/triton_mm_8bit.py b/modules/util/triton_mm_8bit.py index 522959d89..754d9ceda 100644 --- a/modules/util/triton_mm_8bit.py +++ b/modules/util/triton_mm_8bit.py @@ -17,6 +17,14 @@ import triton.language as tl +#Blackwell's block-scaled fp8 mma (mxf8f6f4) runs at the full 8-bit tensor core rate, the legacy +#fp8 mma only at half rate. Pre-Blackwell has no such instruction and triton emulates +#tl.dot_scaled with a bf16 mma, which is slower than plain tl.dot - so pick by compute capability. +#On ROCm the capability is the gfx arch number and RDNA4 reports 12, so require CUDA +def _prefer_mxfp8(device: torch.device) -> bool: + return torch.version.cuda is not None and torch.cuda.get_device_capability(device)[0] >= 12 + + @triton.autotune( configs=[ triton.Config({'BLOCK_SIZE_M': 128, 'BLOCK_SIZE_N': 128, 'BLOCK_SIZE_K': 128}, num_stages=4,num_warps=4), @@ -56,6 +64,7 @@ def _mm_kernel( BLOCK_SIZE_M: tl.constexpr, BLOCK_SIZE_N: tl.constexpr, BLOCK_SIZE_K: tl.constexpr, QUANTIZED_M, FLOAT: tl.constexpr, + MXFP8_MMA: tl.constexpr, ): pid_n = tl.program_id(axis=0) @@ -78,13 +87,23 @@ def _mm_kernel( accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32 if FLOAT else tl.int32) + #the mma multiplies each group of 32 elements along K by one ue8m0 scale. ue8m0 is a bare + #exponent with bias 127, so the value 127 means a scale of 1.0 and every element is left + #unchanged - the result is the same as an unscaled fp8 matmul + if MXFP8_MMA: + a_scale = tl.full((BLOCK_SIZE_M, BLOCK_SIZE_K // 32), 127, dtype=tl.uint8) + b_scale = tl.full((BLOCK_SIZE_N, BLOCK_SIZE_K // 32), 127, dtype=tl.uint8) + for k in range(tl.cdiv(K, BLOCK_SIZE_K)): a_mask = (offs_am[:, None] < M) & (offs_k[None, :] < K - k*BLOCK_SIZE_K) b_mask = (offs_bn[None, :] < N) & (offs_k[:, None] < K - k*BLOCK_SIZE_K) a = tl.load(a_ptrs, mask=a_mask, other=0.0) b = tl.load(b_ptrs, mask=b_mask, other=0.0) - accumulator = tl.dot(a, b, accumulator, out_dtype=tl.float32 if FLOAT else tl.int32) + if MXFP8_MMA: + accumulator = tl.dot_scaled(a, a_scale, "e4m3", b, b_scale, "e4m3", acc=accumulator) + else: + accumulator = tl.dot(a, b, accumulator, out_dtype=tl.float32 if FLOAT else tl.int32) a_ptrs += BLOCK_SIZE_K * stride_ak b_ptrs += BLOCK_SIZE_K * stride_bk @@ -116,6 +135,7 @@ def grid(META): b.stride(0), b.stride(1), c.stride(0), c.stride(1), QUANTIZED_M = M // 64, - FLOAT = FLOAT + FLOAT = FLOAT, + MXFP8_MMA = FLOAT and _prefer_mxfp8(a.device), ) return c diff --git a/scripts/util/import_util.py b/scripts/util/import_util.py index e269a0505..db213f2e5 100644 --- a/scripts/util/import_util.py +++ b/scripts/util/import_util.py @@ -17,9 +17,6 @@ def script_imports(allow_zluda: bool = True): # records from child loggers. Set OT_DEBUG_WARNINGS to see them all. if not os.environ.get("OT_DEBUG_WARNINGS"): # diffusers/transformers chatty logger.warning() lines at import/load time. - logging.getLogger("diffusers.modular_pipelines").addFilter( - lambda record: 'Modular Diffusers is currently an experimental feature' not in record.getMessage() - ) # The subject of these two is interpolated into the message, so match the whole # sentence with .* standing in for the runtime value. logging.getLogger("diffusers.configuration_utils").addFilter(