Bulk MultiQC reporting as a pipeline-installed Python module

assets/multiqc_config.yml

@@ -13,7 +13,32 @@ export_plots: true
 # Run only these modules
 run_modules:
   - custom_content
-  - run_alphafold2_pred
-  - colabfold_batch
+
+table_sample_merge:
+  "rank_0": "_rank_0"
+  "rank_1": "_rank_1"
+  "rank_2": "_rank_2"
+  "rank_3": "_rank_3"
+  "rank_4": "_rank_4"
+  "rank_5": "_rank_5"
+  "rank_6": "_rank_6"
+  "rank_7": "_rank_7"
+  "rank_8": "_rank_8"
+  "rank_9": "_rank_9"
+  "rank_10": "_rank_10"
+  "rank_11": "_rank_11"
+  "rank_12": "_rank_12"
+  "rank_13": "_rank_13"
+  "rank_14": "_rank_14"
+  "rank_15": "_rank_15"
+  "rank_16": "_rank_16"
+  "rank_17": "_rank_17"
+  "rank_18": "_rank_18"
+  "rank_19": "_rank_19"
+  "rank_20": "_rank_20"
+  "rank_21": "_rank_21"
+  "rank_22": "_rank_22"
+  "rank_23": "_rank_23"
+  "rank_24": "_rank_24"
 
 disable_version_detection: true

bin/extract_metrics.py

@@ -285,7 +285,7 @@ def extract_structs_plddt_to_tsv(name, structures):
     plddt_rows =  [["Positions"] + rank_names]
     res_id_col = list(range(len(plddt_cols[0])))
     plddt_rows.extend([list(row) for row in zip(res_id_col, *plddt_cols)])  # Combine lists column-wise to make rows
-    write_tsv(f"{name}_plddt_mqc.tsv", plddt_rows)
+    write_tsv(f"{name}_plddt.tsv", plddt_rows)
 
 def read_pkl(name, pkl_files, struct_files=None):
     """

bin/generate_comparison_report.py

@@ -63,7 +63,7 @@ def generate_output(plddt_data, name, out_dir, generate_tsv, pdb):
 
     fig = go.Figure()
     for idx, (model_name, value_plddt) in enumerate(plddt_per_model.items()):
-        rank_label = os.path.splitext(pdb[idx])[0]
+        rank_label = os.path.splitext(os.path.basename(pdb[idx]))[0]
         fig.add_trace(
             go.Scatter(
                 x=list(range(len(value_plddt))),

bin/generate_report.py

@@ -338,9 +338,12 @@ def align_structures(structures):
     return aligned_structures
 
 
+def natural_path_sort_key(path):
+    return [int(part) if part.isdigit() else part.lower() for part in re.split(r"(\d+)", str(path))]
+
+
 def pdb_to_lddt(struct_files, generate_tsv):
-    struct_files_sorted = struct_files
-    struct_files_sorted.sort()
+    struct_files_sorted = sorted(struct_files, key=natural_path_sort_key)
 
     output_lddt = []
     averages = []
@@ -521,8 +524,7 @@ def build_pair_score_matrices(score_maps):
 )
 
 print("generating html report...")
-structures = args.pdb
-structures.sort() #TODO: make sure sorting here doesnt break rank order
+structures = sorted(args.pdb, key=natural_path_sort_key)
 iptm_scores = read_ranked_score_tsv(args.iptm, len(structures))
 ipsae_scores = read_ranked_score_tsv(args.ipsae, len(structures))
 chainwise_iptm_scores = read_pair_score_tsv(args.chainwise_iptm, len(structures))
@@ -538,61 +540,53 @@ def build_pair_score_matrices(score_maps):
 aligned_structures[0] = ref_structure_path
 
 proteinfold_template = open(args.html_template, "r").read()
-proteinfold_template = proteinfold_template.replace("*sample_name*", args.name)
-proteinfold_template = proteinfold_template.replace(
-    "*prog_name*", model_name[args.in_type.lower()]
-)
 
 model_names = [
-    f"{os.path.splitext(model)[0]}.cif"
-    for model in structures
+    f"{os.path.splitext(os.path.basename(model))[0]}.cif" for model in structures
 ]
-args_pdb_array_js = ",\n".join([f'"{model}"' for model in model_names])
-proteinfold_template = re.sub(
-    r"const MODELS = \[.*?\];",  # Match the existing MODELS array in HTML template
-    f"const MODELS = [\n  {args_pdb_array_js}\n];",  # Replace with the new array
-    proteinfold_template,
-    flags=re.DOTALL,
-)
+models_data = [open(s, "r").read() for s in aligned_structures]
 
-averages_js_array = f"const LDDT_AVERAGES = {lddt_averages};"
-proteinfold_template = proteinfold_template.replace(
-    "const LDDT_AVERAGES = [];", averages_js_array
-)
 
-iptm_js_array = f"const IPTM_SCORES = {iptm_scores};"
-proteinfold_template = proteinfold_template.replace(
-    "const IPTM_SCORES = [];", iptm_js_array
-)
+def script_safe_json_dumps(obj):
+    """json.dumps with <, > and & escaped, so a payload containing "</script>"
+    cannot terminate the enclosing <script type="application/json"> element early."""
+    return (
+        json.dumps(obj)
+        .replace("<", "\\u003c")
+        .replace(">", "\\u003e")
+        .replace("&", "\\u0026")
+    )
 
-ipsae_js_array = f"const IPSAE_SCORES = {ipsae_scores};"
-proteinfold_template = proteinfold_template.replace(
-    "const IPSAE_SCORES = [];", ipsae_js_array
-)
 
-chainwise_iptm_js_array = f"const CHAINWISE_IPTM_SCORES = {json.dumps(chainwise_iptm_matrices)};"
-proteinfold_template = proteinfold_template.replace(
-    "const CHAINWISE_IPTM_SCORES = [];", chainwise_iptm_js_array
+report_config = {
+    "reportType": "standard",
+    "sampleName": args.name,
+    "programName": model_name[args.in_type.lower()],
+    "structFormat": "cif",
+    "models": model_names,
+    "models_data": models_data,
+    "lddt_averages": lddt_averages,
+    "iptm_scores": iptm_scores,
+    "ipsae_scores": ipsae_scores,
+    "chainwise_iptm": chainwise_iptm_matrices,
+    "chainwise_ipsae": chainwise_ipsae_matrices,
+}
+config_blob = (
+    '<script type="application/json" id="report-config">'
+    f"{script_safe_json_dumps(report_config)}</script>"
 )
-
-chainwise_ipsae_js_array = f"const CHAINWISE_IPSAE_SCORES = {json.dumps(chainwise_ipsae_matrices)};"
 proteinfold_template = proteinfold_template.replace(
-    "const CHAINWISE_IPSAE_SCORES = [];", chainwise_ipsae_js_array
+    "</head>", f"{config_blob}\n  </head>", 1
 )
 
-i = 0
-for structure in aligned_structures:
-    proteinfold_template = proteinfold_template.replace(
-        f"*_data_ranked_{i}.cif*", open(structure, "r").read().replace("\n", "\\n")
-    )
-    i += 1
-
 if not is_missing_input(args.msa):
     image_path = f"{args.output_dir}/{args.name}_{args.in_type}_seq_coverage.png"
     with open(image_path, "rb") as in_file:
+        data_uri = f"data:image/png;base64,{base64.b64encode(in_file.read()).decode('utf-8')}"
         proteinfold_template = proteinfold_template.replace(
-            "seq_coverage.png",
-            f"data:image/png;base64,{base64.b64encode(in_file.read()).decode('utf-8')}",
+            '<div id="seq_cov_placeholder"></div>',
+            f'<img src="{data_uri}" alt="Sequence coverage (MSA)" '
+            'class="w-full h-auto rounded" />',
         )
 else:
     pattern = r'<div id="seq_coverage_container".*?>.*?(<!--.*?-->.*?)*?</div>\s*</div>\s*</div>\s*</div>'

conf/containers_conda_lock_files_amd64.config

@@ -1 +1 @@
-process { withName: 'MULTIQC' { container = 'modules/nf-core/multiqc/.conda-lock/linux_amd64-bd-db7c73dae76bc9e6_1.txt' } }
+process { withName: 'MULTIQC' { container = 'modules/nf-core/multiqc/.conda-lock/linux_amd64-bd-c17fb751507e9dfc_1.txt' } }

conf/containers_conda_lock_files_arm64.config

@@ -1 +1 @@
-process { withName: 'MULTIQC' { container = 'modules/nf-core/multiqc/.conda-lock/linux_arm64-bd-d167b8012595a136_1.txt' } }
+process { withName: 'MULTIQC' { container = 'modules/nf-core/multiqc/.conda-lock/linux_arm64-bd-5c84a5000a226ab5_1.txt' } }

conf/containers_docker_amd64.config

@@ -1 +1 @@
-process { withName: 'MULTIQC' { container = 'community.wave.seqera.io/library/multiqc:1.34--db7c73dae76bc9e6' } }
+process { withName: 'MULTIQC' { container = 'community.wave.seqera.io/library/multiqc:1.35--c17fb751507e9dfc' } }

conf/containers_docker_arm64.config

@@ -1 +1 @@
-process { withName: 'MULTIQC' { container = 'community.wave.seqera.io/library/multiqc:1.34--d167b8012595a136' } }
+process { withName: 'MULTIQC' { container = 'community.wave.seqera.io/library/multiqc:1.35--5c84a5000a226ab5' } }

conf/containers_singularity_https_amd64.config

@@ -1 +1 @@
-process { withName: 'MULTIQC' { container = 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/1b/1bef8af6be88c5733461959c46ac8ef73d18f65277f62a1695d0e1633054f9c2/data' } }
+process { withName: 'MULTIQC' { container = 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/c8/c8e346f4f6080eadf1253505e6ff09ef004454fc18e8d672006fd7b222cc412e/data' } }

conf/containers_singularity_https_arm64.config

@@ -1 +1 @@
-process { withName: 'MULTIQC' { container = 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/9a/9a1fec9662a152683e6fcae440d0ce20920b3b89dc62d1e3a52e73f92eba0969/data' } }
+process { withName: 'MULTIQC' { container = 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/e4/e48aa28aebc881254a499b24c3e1ce77b8df1b85a5432699ed6f72eb17ac7fb5/data' } }

conf/containers_singularity_oras_amd64.config

@@ -1 +1 @@
-process { withName: 'MULTIQC' { container = 'oras://community.wave.seqera.io/library/multiqc:1.34--4fc8657c816047c0' } }
+process { withName: 'MULTIQC' { container = 'oras://community.wave.seqera.io/library/multiqc:1.35--c680f2aea25ccec2' } }

conf/containers_singularity_oras_arm64.config

@@ -1 +1 @@
-process { withName: 'MULTIQC' { container = 'oras://community.wave.seqera.io/library/multiqc:1.34--7fbd82d945c06726' } }
+process { withName: 'MULTIQC' { container = 'oras://community.wave.seqera.io/library/multiqc:1.35--c0468833d65b2f81' } }

conf/modules.config

@@ -44,7 +44,7 @@ process {
     }
 
     withName: 'MULTIQC' {
-        ext.prefix = { "${meta.model}" }
+        ext.prefix = { "${meta.model}_multiqc_report" }
         ext.args   = { params.multiqc_title ? "--title \"$params.multiqc_title\"" : '' }
         publishDir = [
             path: { "${params.outdir}/multiqc" },
@@ -53,6 +53,12 @@ process {
         ]
     }
 
+    withName: 'GENERATE_MULTIQC_CONTENTS' {
+        publishDir = [
+            enabled: false
+        ]
+    }
+
     withName: 'GENERATE_REPORT' {
         publishDir = [
             path: { "${params.outdir}/reports" },

conf/modules_alphafold2.config

@@ -120,8 +120,6 @@ process {
                 saveAs: { filename ->
                     if(filename.endsWith('_pae.tsv')){
                         "paes/$filename"
-                    } else if(filename.endsWith('_plddt_mqc.tsv')){
-                        filename.replaceAll(/_plddt_mqc\.tsv$/, '_plddt.tsv')
                     } else { filename }
                 },
                 pattern: '*.tsv'

conf/modules_alphafold3.config

@@ -101,8 +101,7 @@ process {
             [
                 path: { "${params.outdir}/alphafold3/${meta.id}" },
                 mode: 'copy',
-                saveAs: { filename -> filename.replaceAll(/_plddt_mqc\.tsv$/, '_plddt.tsv') },
-                pattern: '*_plddt_mqc.tsv'
+                pattern: '*_plddt.tsv'
             ],
             [
                 path: { "${params.outdir}/alphafold3/${meta.id}" },
@@ -128,7 +127,7 @@ process {
             [
                 path: { "${params.outdir}/alphafold3/${meta.id}" },
                 mode: 'copy',
-                pattern: '*_alphafold3_msa.tsv'
+                pattern: '*_msa.tsv'
             ],
             [
                 enabled: params.save_intermediates,

conf/modules_boltz.config

@@ -2,7 +2,6 @@
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
     Config file for defining DSL2 per module options and publishing paths
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-    Available keys to override module options:
         ext.args   = Additional arguments appended to command in module.
         ext.args2  = Second set of arguments appended to command in module (multi-tool modules).
         ext.args3  = Third set of arguments appended to command in module (multi-tool modules).
@@ -79,13 +78,12 @@ process {
             [
                 path: { "${params.outdir}/boltz/${meta.id}" },
                 mode: 'copy',
-                saveAs: { filename -> filename.replaceAll(/_plddt_mqc\.tsv$/, '_plddt.tsv') },
-                pattern: '*_plddt_mqc.tsv'
+                pattern: '*_plddt.tsv'
             ],
             [
                 path: { "${params.outdir}/boltz/${meta.id}" },
                 mode: 'copy',
-                pattern: '*_boltz_msa.tsv'
+                pattern: '*_msa.tsv'
             ],
             [
                 path: { "${params.outdir}/boltz/${meta.id}" },
@@ -95,7 +93,7 @@ process {
             [
                 path: { "${params.outdir}/boltz/${meta.id}/paes" },
                 mode: 'copy',
-                pattern: '*_[0-5]_pae.tsv'
+                pattern: '*_pae.tsv'
             ],
             [
                 path: { "${params.outdir}/boltz/top_ranked_structures" },

conf/modules_colabfold.config

@@ -41,14 +41,43 @@ process {
             [
                 path: { "${params.outdir}/colabfold/${meta.id}/" },
                 mode: 'copy',
-                saveAs: { filename ->
-                    if(filename.endsWith('_pae.tsv')){
-                        "paes/$filename"
-                    } else if(filename.endsWith('_plddt_mqc.tsv')){
-                        filename.replaceAll(/_plddt_mqc\.tsv$/, '_plddt.tsv')
-                    } else { filename }
-                },
-                pattern: '*.tsv'
+                pattern: '*_msa.tsv'
+            ],
+            [
+                path: { "${params.outdir}/colabfold/${meta.id}/" },
+                mode: 'copy',
+                pattern: '*_plddt.tsv'
+            ],
+            [
+                path: { "${params.outdir}/colabfold/${meta.id}/" },
+                mode: 'copy',
+                pattern: '*_ptm.tsv'
+            ],
+            [
+                path: { "${params.outdir}/colabfold/${meta.id}/" },
+                mode: 'copy',
+                pattern: '*_iptm.tsv'
+            ],
+            [
+                path: { "${params.outdir}/colabfold/${meta.id}/" },
+                mode: 'copy',
+                pattern: '*_ipsae.tsv'
+            ],
+            [
+                path: { "${params.outdir}/colabfold/${meta.id}/" },
+                mode: 'copy',
+                pattern: '*_chainwise_iptm.tsv'
+            ],
+            [
+                path: { "${params.outdir}/colabfold/${meta.id}/" },
+                mode: 'copy',
+                pattern: '*_chainwise_ipsae.tsv'
+            ],
+            [
+                path: { "${params.outdir}/colabfold/${meta.id}/" },
+                mode: 'copy',
+                saveAs: { filename -> "paes/$filename" },
+                pattern: '*_pae.tsv'
             ],
             [
                 enabled: params.save_intermediates,

conf/modules_esmfold.config

@@ -30,8 +30,7 @@ process {
             [
                 path: { "${params.outdir}/esmfold/${meta.id}" },
                 mode: 'copy',
-                saveAs: { filename -> filename.replaceAll(/_plddt_mqc\.tsv$/, '_plddt.tsv') },
-                pattern: '*_plddt_mqc.tsv'
+                pattern: '*_plddt.tsv'
             ],
             [
                 path: { "${params.outdir}/esmfold/top_ranked_structures" },