From 8d0c3cbde2b9bb5e9ab44887ed96274f0603beb9 Mon Sep 17 00:00:00 2001 From: "Juan J." Date: Wed, 4 Mar 2026 10:09:32 +0100 Subject: [PATCH 01/66] DFL trustworthiness and numerical datasets --- nebula/addons/trustworthiness/calculation.py | 406 ++++++++++- .../trustworthiness/configs/eval_metrics.json | 102 ++- .../configs/eval_metrics_dfl.json | 640 +++++++++++++++++ .../configs/factsheet_template.json | 7 +- .../configs/factsheet_template_dfl.json | 56 ++ nebula/addons/trustworthiness/dfl_local.py | 285 ++++++++ nebula/addons/trustworthiness/factsheet.py | 82 ++- nebula/addons/trustworthiness/graphics.py | 122 +++- nebula/addons/trustworthiness/metric.py | 70 +- .../trustworthiness/per_round_metrics.py | 174 +++++ .../addons/trustworthiness/trustworthiness.py | 246 +++++-- nebula/addons/trustworthiness/utils.py | 62 +- nebula/controller/scenarios.py | 48 ++ nebula/core/datasets/adultcensus/__init__.py | 0 .../core/datasets/adultcensus/adultcensus.py | 242 +++++++ .../core/datasets/breast_cancer/__init__.py | 0 .../datasets/breast_cancer/breast_cancer.py | 158 +++++ nebula/core/datasets/covtype/__init__.py | 0 nebula/core/datasets/covtype/covtype.py | 220 ++++++ nebula/core/datasets/nebuladataset.py | 6 + nebula/core/models/adultcensus/__init__.py | 0 nebula/core/models/adultcensus/mlp.py | 67 ++ nebula/core/models/breast_cancer/__init__.py | 0 nebula/core/models/breast_cancer/mlp.py | 55 ++ nebula/core/models/covtype/__init__.py | 0 nebula/core/models/covtype/mlp.py | 55 ++ nebula/core/node.py | 27 + .../static/js/deployment/help-content.js | 3 + nebula/frontend/static/js/deployment/main.js | 8 +- .../frontend/static/js/deployment/scenario.js | 132 +++- .../static/js/deployment/trustworthiness.js | 643 ++++++++++++------ nebula/frontend/templates/deployment.html | 509 +++++++++----- 32 files changed, 3937 insertions(+), 488 deletions(-) create mode 100755 nebula/addons/trustworthiness/configs/eval_metrics_dfl.json create mode 100755 nebula/addons/trustworthiness/configs/factsheet_template_dfl.json create mode 100644 nebula/addons/trustworthiness/dfl_local.py create mode 100644 nebula/addons/trustworthiness/per_round_metrics.py create mode 100755 nebula/core/datasets/adultcensus/__init__.py create mode 100644 nebula/core/datasets/adultcensus/adultcensus.py create mode 100755 nebula/core/datasets/breast_cancer/__init__.py create mode 100644 nebula/core/datasets/breast_cancer/breast_cancer.py create mode 100755 nebula/core/datasets/covtype/__init__.py create mode 100644 nebula/core/datasets/covtype/covtype.py create mode 100755 nebula/core/models/adultcensus/__init__.py create mode 100644 nebula/core/models/adultcensus/mlp.py create mode 100755 nebula/core/models/breast_cancer/__init__.py create mode 100644 nebula/core/models/breast_cancer/mlp.py create mode 100755 nebula/core/models/covtype/__init__.py create mode 100644 nebula/core/models/covtype/mlp.py diff --git a/nebula/addons/trustworthiness/calculation.py b/nebula/addons/trustworthiness/calculation.py index db3499f5d..251fe1e5a 100755 --- a/nebula/addons/trustworthiness/calculation.py +++ b/nebula/addons/trustworthiness/calculation.py @@ -12,10 +12,12 @@ import shap import torch.nn from art.estimators.classification import PyTorchClassifier -from art.metrics import clever_u +from art.metrics import clever_u, loss_sensitivity, empirical_robustness from codecarbon import EmissionsTracker from scipy.stats import variation from torch import nn, optim +import torch.nn.functional as F +import time from nebula.addons.trustworthiness.utils import read_csv @@ -286,6 +288,21 @@ def get_bytes_models(models_files): return avg_model_size +def get_bytes_model(model_file): + """ + Calculates the bytes of the final model of a node. + + Args: + model_file: Final model. + + Returns: + float: The bytes of the model. + """ + + model_size = os.path.getsize(model_file) + + return model_size + def get_bytes_sent_recv(scenario_name): """ @@ -309,7 +326,7 @@ def get_bytes_sent_recv(scenario_name): total_upload_bytes = int(data["bytes_sent"].sum()) total_download_bytes = int(data["bytes_recv"].sum()) - + avg_upload_bytes = total_upload_bytes / number_files avg_download_bytes = total_download_bytes / number_files @@ -330,15 +347,46 @@ def get_avg_loss_accuracy(scenario_name): total_accuracy = 0 total_loss = 0 + expected_nodes = 3 + """ + if os.path.exists(factsheet_file): + with open(factsheet_file, "r") as f: + fs = json.load(f) + # normalmente client_num viene como string, lo convierto + expected_nodes = int(fs.get("participants", {}).get("client_num", 0) or 0) + logger.info(f"nodes={expected_nodes}") + """ + + data_file = os.path.join(os.environ.get('NEBULA_LOGS_DIR'), scenario_name, "trustworthiness", "data_results.csv") + logger.info(f"FIRST 5 LINES:\n{open(data_file,'r').read().splitlines()[:5]}") + logger.info(f"LAST 5 LINES:\n{open(data_file,'r').read().splitlines()[-5:]}") + data = read_csv(data_file) + logger.info(f"shape={data.shape}") + logger.info(f"dtypes={data.dtypes.to_dict()}") + logger.info(f"accuracy sample raw={data['accuracy'].head(20).tolist()}") + logger.info(f"accuracy non-null={data['accuracy'].notna().sum()}") + number_files = len(data) + logger.info(f"number_files={number_files}") + + """ + while (number_files != expected_nodes): + logger.info("WAIT") + time.sleep(5) + data_file = os.path.join(os.environ.get('NEBULA_LOGS_DIR'), scenario_name, "trustworthiness", "data_results.csv") + data = read_csv(data_file) + number_files = len(data) + logger.info(f"number_files={number_files}") + logger.info(f"expected_nodes={expected_nodes}") + """ total_loss = data["loss"].sum() total_accuracy = data["accuracy"].sum() - + avg_loss = total_loss / number_files avg_accuracy = total_accuracy / number_files std_accuracy = statistics.stdev(data["accuracy"]) @@ -399,18 +447,46 @@ def get_clever_score(model, test_sample, nb_classes, learning_rate): float: The CLEVER score. """ + images, _ = test_sample - background = images[-1] + input_shape = None + + # Si por cualquier motivo llega sin batch, lo añadimos + if torch.is_tensor(images) and images.dim() >= 1 and images.shape[0] != 0: + pass + else: + raise ValueError("`test_sample[0]` debe ser un torch.Tensor no vacío.") + + if input_shape is None: + if images.dim() >= 2: + # (B, ...) -> input_shape = (...) + input_shape = tuple(images.shape[1:]) + else: + # (...) sin batch + input_shape = tuple(images.shape) + + # Escogemos un "background" (aquí el último del batch, como hacías tú) + background = images[-1] if images.dim() >= 2 else images + + # Convertir a numpy de forma segura (GPU-friendly) + x = background.detach().cpu().numpy() + + # Asegurar batch dimension para clever_u: (1, *input_shape) + if tuple(x.shape) == tuple(input_shape): + x = x.reshape((1,) + tuple(input_shape)) + criterion = nn.CrossEntropyLoss() optimizer = optim.Adam(model.parameters(), learning_rate) + + # Create the ART classifier classifier = PyTorchClassifier( model=model, loss=criterion, optimizer=optimizer, - input_shape=(1, 28, 28), + input_shape=input_shape, nb_classes=nb_classes, ) @@ -434,6 +510,7 @@ def stop_emissions_tracking_and_save( role: str, workload: str, sample_size: int = 0, + participant_idx=None, ): """ Stops emissions tracking object from CodeCarbon and saves relevant information to emissions.csv file. @@ -456,6 +533,7 @@ def stop_emissions_tracking_and_save( else: df = pd.DataFrame( columns=[ + "id", "role", "energy_grid", "emissions", @@ -470,6 +548,7 @@ def stop_emissions_tracking_and_save( [ df, pd.DataFrame({ + "id": participant_idx, "role": role, "energy_grid": [energy_grid], "emissions": [tracker.final_emissions_data.emissions], @@ -491,3 +570,320 @@ def stop_emissions_tracking_and_save( df.to_csv(emissions_file, encoding="utf-8", index=False) except Exception as e: logger.warning(e) + +def comm_efficiency(bytes_up: int, bytes_down: int, test_acc_avg: float, eps: float = 1e-12) -> float: + """ + Communication efficiency = total_bytes / final_accuracy. + Lower is better. + + Args: + bytes_up: total uploaded bytes + bytes_down: total downloaded bytes + final_accuracy: final test accuracy in [0,1] (or [0,100] if your factsheet uses %) + eps: small constant to avoid division by zero + + Returns: + float + """ + total_bytes = float(bytes_up) + float(bytes_down) + acc = float(test_acc_avg) + + # Si tu factsheet guarda accuracy como porcentaje (0-100), descomenta esto: + # if acc > 1.0: + # acc = acc / 100.0 + + if acc < eps: + acc = eps + + return total_bytes / acc + +def get_loss_sensitivity_score(model, test_sample, nb_classes, learning_rate): + + images, labels = test_sample + sample = images[-1].unsqueeze(0) + label = labels[-1].unsqueeze(0) + + label = F.one_hot(label, num_classes=nb_classes).float() + + criterion = nn.CrossEntropyLoss() + optimizer = optim.Adam(model.parameters(), learning_rate) + + # Create the ART classifier + classifier = PyTorchClassifier( + model=model, + loss=criterion, + optimizer=optimizer, + input_shape=sample.shape[1:], + nb_classes=nb_classes, + ) + + score = loss_sensitivity( + classifier, + sample.numpy(), + label.numpy(), + ) + return float(score) + +def compute_adversarial_accuracy_art( + model, + test_loader, + nb_classes, + learning_rate, + epsilon=0.03 +): + """ + Computes adversarial accuracy using ART FGSM attack. + """ + + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + model.eval() + model.to(device) + + criterion = nn.CrossEntropyLoss() + optimizer = optim.Adam(model.parameters(), lr=learning_rate) + + # Obtener shape dinámicamente + sample_batch = next(iter(test_loader)) + images, _ = sample_batch + input_shape = images.shape[1:] #CAMBIAR + + classifier = PyTorchClassifier( + model=model, + loss=criterion, + optimizer=optimizer, + input_shape=input_shape, + nb_classes=nb_classes, + ) + """ + from art.attack.evasion import FastGradientMethod + + attack = FastGradientMethod( + estimator=classifier, + eps=epsilon, + norm=np.inf + ) + """ + + correct = 0 + total = 0 + + for images, labels in test_loader: + images = images.to(device) + labels = labels.to(device) + + # Generar adversarios con FGSM puro + x_adv = fgsm_attack(model, images, labels, epsilon=epsilon) + + # Predicciones + with torch.no_grad(): + outputs = model(x_adv) + preds = outputs.argmax(dim=1) + + correct += (preds == labels).sum().item() + total += labels.size(0) + + return correct / total + +def get_empirical_robustness_score( + model: object, + test_sample: object, + nb_classes: int, + learning_rate: float, + attack_name: str = "fgsm", + attack_params: dict | None = None, + max_samples: int = 32, +) -> float: + """ + Calculates the Empirical Robustness score using Adversarial Robustness Toolbox (ART). + + Empirical robustness estimates the minimal relative perturbation required for a successful attack + on the provided samples. Higher is better (needs larger perturbation to fool the model). + + Args: + model (object): The model. + test_sample (object): A batch from the test dataloader (images, labels). + nb_classes (int): Number of classes. + learning_rate (float): LR used to build the ART classifier wrapper. + attack_name (str): Attack key supported by ART empirical_robustness (commonly "fgsm" or "hsj"). + attack_params (dict | None): Optional attack parameters. + max_samples (int): Max number of samples from the batch to use. + + Returns: + float: Empirical robustness score (>= 0.0). If it cannot be computed, returns 0.0. + """ + try: + images, _ = test_sample + + # Limit how many samples we use from the batch (keeps it lightweight) + batch_size: int = int(images.shape[0]) + n: int = int(min(max_samples, batch_size)) + x = images[:n].detach().cpu().numpy() + + # Infer input shape for ART (no batch dimension) + input_shape = tuple(images.shape[1:]) + + criterion = nn.CrossEntropyLoss() + optimizer = optim.Adam(model.parameters(), learning_rate) + + classifier = PyTorchClassifier( + model=model, + loss=criterion, + optimizer=optimizer, + input_shape=input_shape, + nb_classes=nb_classes, + ) + + score = empirical_robustness( + classifier=classifier, + x=x, + attack_name=attack_name, + attack_params=attack_params, + ) + + # ART may return ndarray depending on input; aggregate to scalar + if isinstance(score, np.ndarray): + score = float(np.mean(score)) + + if score is None or (isinstance(score, float) and math.isnan(score)): + return 0.0 + + return float(score) + + except Exception as exc: + logger.warning("Could not compute empirical robustness (ART). Returning 0.0") + logger.warning(exc) + return 0.0 + + + +def fgsm_attack(model, images, labels, epsilon=0.03): + """ + Genera ejemplos adversariales usando FGSM puro en PyTorch. Cuando se pueda meter los ataques de ART se podría cambiar + """ + images = images.clone().detach().to(images.device) + labels = labels.to(images.device) + images.requires_grad = True + + outputs = model(images) + loss = nn.CrossEntropyLoss()(outputs, labels) + model.zero_grad() + loss.backward() + + # FGSM: x_adv = x + epsilon * sign(grad) + perturbation = epsilon * images.grad.sign() + x_adv = images + perturbation + + # Limitar valores al rango [0,1] + #x_adv = torch.clamp(x_adv, 0, 1) + return x_adv.detach() + +def get_confidence_score( + model, + test_sample, + max_samples: int = 128, + use_true_label: bool = True, +) -> float: + """ + Confidence Score basado en probabilidades softmax. + + - Si use_true_label=True: devuelve la media de P(y_true | x). + - Si use_true_label=False: devuelve la media de max softmax prob (MSP). + + Args: + model (object): Modelo (torch.nn.Module). + test_sample (object): Batch del dataloader: (x, y). + max_samples (int): Máximo nº de muestras del batch a usar. + use_true_label (bool): Ver arriba. + + Returns: + float: Confidence score en [0, 1] (o 0.0 si falla). + """ + try: + if not isinstance(model, torch.nn.Module): + logger.warning("Model is not a torch.nn.Module") + return 0.0 + + x, y = test_sample + + # Recorta batch para que sea barato + if isinstance(x, torch.Tensor): + x = x[:max_samples] + if isinstance(y, torch.Tensor): + y = y[:max_samples] + + # Usa el device real del modelo + try: + device = next(model.parameters()).device + except Exception: + device = torch.device("cpu") + + model.eval() + with torch.no_grad(): + x = x.to(device) if isinstance(x, torch.Tensor) else x + out = model(x) + + # Por si el modelo devuelve tupla (logits, ...) + logits = out[0] if isinstance(out, (tuple, list)) else out + probs = torch.softmax(logits, dim=1) + + if use_true_label and isinstance(y, torch.Tensor): + # y puede venir como índices [B] o one-hot [B, C] + if y.ndim > 1: + y_idx = torch.argmax(y, dim=1) + else: + y_idx = y + y_idx = y_idx.to(device) + + # P(y_true|x) + true_probs = probs.gather(1, y_idx.view(-1, 1)).squeeze(1) + return float(true_probs.mean().detach().cpu().item()) + + # MSP: max_c P(c|x) + msp = probs.max(dim=1).values + return float(msp.mean().detach().cpu().item()) + + except Exception as e: + logger.warning("Could not compute confidence score") + logger.warning(e) + return 0.0 + +def attack_success_rate(model, test_sample,epsilon=0.03): + """ + Calcula ASR para un ataque untargeted. + + attack_fn debe recibir (model, images, labels) + y devolver imágenes adversariales. + """ + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + model.eval() + + images, labels = test_sample + images = images.to(device) + labels = labels.to(device) + + # 1️⃣ Predicciones originales + with torch.no_grad(): + outputs = model(images) + preds = outputs.argmax(dim=1) + + # Solo consideramos los originalmente correctos + correct_mask = preds.eq(labels) + num_correct = correct_mask.sum().item() + + if num_correct == 0: + return 0.0 # evitar división por cero + + # 2️⃣ Generar adversariales + x_adv = fgsm_attack(model, images, labels, epsilon=epsilon) + + # 3️⃣ Predicciones adversariales + with torch.no_grad(): + outputs_adv = model(x_adv) + preds_adv = outputs_adv.argmax(dim=1) + + # 4️⃣ Ataque exitoso = antes correcto y ahora incorrecto + successful_attacks = (correct_mask & preds_adv.ne(labels)).sum().item() + + asr = successful_attacks / num_correct + + return asr diff --git a/nebula/addons/trustworthiness/configs/eval_metrics.json b/nebula/addons/trustworthiness/configs/eval_metrics.json index 5ab1b3427..642efb262 100755 --- a/nebula/addons/trustworthiness/configs/eval_metrics.json +++ b/nebula/addons/trustworthiness/configs/eval_metrics.json @@ -14,7 +14,72 @@ "score_function": "get_range_score", "type": "true_score", "description": "Cross Lipschitz Extreme Value for network Robustness: attack-agnostic estimator of the lower bound βL", - "weight": 1 + "weight": 0.4 + }, + "loss_sensitivity": { + "inputs": [ + { + "source": "factsheet", + "field_path": "performance/test_loss_sensitivity" + } + ], + "operation": "get_value", + "score_function": "get_range_score", + "type": "true_score", + "description": "", + "weight": 0.2 + }, + "adversarial_accuracy": { + "inputs": [ + { + "source": "factsheet", + "field_path": "performance/test_adv_accuracy" + } + ], + "operation": "get_value", + "score_function": "get_range_score", + "type": "true_score", + "description": "", + "weight": 0.1 + }, + "emprical_robustness": { + "inputs": [ + { + "source": "factsheet", + "field_path": "performance/test_empirical_robustness" + } + ], + "operation": "get_value", + "score_function": "get_range_score", + "type": "true_score", + "description": "", + "weight": 0.1 + }, + "confidence_score": { + "inputs": [ + { + "source": "factsheet", + "field_path": "performance/test_confidence_score" + } + ], + "operation": "get_value", + "score_function": "get_range_score", + "type": "true_score", + "description": "", + "weight": 0.1 + }, + "attack_success_rate": { + "inputs": [ + { + "source": "factsheet", + "field_path": "performance/test_attack_success_rate" + } + ], + "operation": "get_value", + "score_function": "get_range_score", + "type": "true_score", + "description": "", + "weight": 0.1 } } }, @@ -388,6 +453,26 @@ { "source": "factsheet", "field_path": "performance/test_clever" + }, + { + "source": "factsheet", + "field_path": "performance/test_loss_sensitivity" + }, + { + "source": "factsheet", + "field_path": "performance/test_adv_accuracy" + }, + { + "source": "factsheet", + "field_path": "performance/test_empirical_robustness" + }, + { + "source": "factsheet", + "field_path": "performance/test_confidence_score" + }, + { + "source": "factsheet", + "field_path": "performance/test_attack_success_rate" } ], "operation": "check_properties", @@ -554,6 +639,19 @@ "federation_complexity": { "weight": 0.25, "metrics": { + "communication_efficiency": { + "inputs": [ + { "source": "factsheet", "field_path": "system/total_upload_bytes" }, + { "source": "factsheet", "field_path": "system/total_download_bytes" }, + { "source": "factsheet", "field_path": "performance/test_acc_avg" } + ], + "operation": "comm_efficiency", + "type": "ranges", + "direction": "low", + "ranges":[0.1, 10e2, 10e3,10e4, 10e5, 10e6,10e7,10e8,10e9,10e10,10e11], + "description": "Descripcion de la metrica", + "weight": 0.1 + }, "number_of_training_rounds": { "inputs": [ { @@ -566,7 +664,7 @@ "direction": "desc", "ranges": [5, 10, 15, 20, 25, 30, 35, 40, 45, 50], "description": "The total number of training rounds", - "weight": 0.16666666 + "weight": 0.06666666 }, "avg_model_size": { "inputs": [ diff --git a/nebula/addons/trustworthiness/configs/eval_metrics_dfl.json b/nebula/addons/trustworthiness/configs/eval_metrics_dfl.json new file mode 100755 index 000000000..fea2f70d3 --- /dev/null +++ b/nebula/addons/trustworthiness/configs/eval_metrics_dfl.json @@ -0,0 +1,640 @@ +{ + "robustness": { + "resilience_to_attacks": { + "weight": 0.4, + "metrics": { + "certified_robustness": { + "inputs": [ + { + "source": "factsheet", + "field_path": "performance/test_clever" + } + ], + "operation": "get_value", + "score_function": "get_range_score", + "type": "true_score", + "description": "Cross Lipschitz Extreme Value for network Robustness: attack-agnostic estimator of the lower bound βL", + "weight": 0.4 + }, + "loss_sensitivity": { + "inputs": [ + { + "source": "factsheet", + "field_path": "performance/test_loss_sensitivity" + } + ], + "operation": "get_value", + "score_function": "get_range_score", + "type": "true_score", + "description": "", + "weight": 0.2 + }, + "adversarial_accuracy": { + "inputs": [ + { + "source": "factsheet", + "field_path": "performance/test_adv_accuracy" + } + ], + "operation": "get_value", + "score_function": "get_range_score", + "type": "true_score", + "description": "", + "weight": 0.1 + }, + "emprical_robustness": { + "inputs": [ + { + "source": "factsheet", + "field_path": "performance/test_empirical_robustness" + } + ], + "operation": "get_value", + "score_function": "get_range_score", + "type": "true_score", + "description": "", + "weight": 0.1 + }, + "confidence_score": { + "inputs": [ + { + "source": "factsheet", + "field_path": "performance/test_confidence_score" + } + ], + "operation": "get_value", + "score_function": "get_range_score", + "type": "true_score", + "description": "", + "weight": 0.1 + }, + "attack_success_rate": { + "inputs": [ + { + "source": "factsheet", + "field_path": "performance/test_attack_success_rate" + } + ], + "operation": "get_value", + "score_function": "get_range_score", + "type": "true_score", + "description": "", + "weight": 0.1 + } + } + }, + "algorithm_robustness": { + "weight": 0.4, + "metrics": { + "personalization": { + "inputs": [ + { + "source": "factsheet", + "field_path": "configuration/personalization" + } + ], + "operation": "get_value", + "type": "true_score", + "description": "The use of personalized FL algorithm.", + "weight": 1.0 + } + } + }, + "client_reliability": { + "weight": 0.2, + "metrics": { + "scale": { + "inputs": [ + { + "source": "factsheet", + "field_path": "participants/client_num" + } + ], + "operation": "get_value", + "type": "ranges", + "direction": "desc", + "ranges": [5, 10, 15, 20, 25, 30, 35, 40, 45, 50], + "description": "The number of clients in the model.", + "weight": 1 + } + } + } + }, + "privacy": { + "technique": { + "weight": 0.2, + "metrics": { + "differential_privacy": { + "inputs": [ + { + "source": "factsheet", + "field_path": "configuration/differential_privacy" + } + ], + "operation": "get_value", + "type": "true_score", + "description": "The use of differential privacy.", + "weight": 1 + } + } + }, + "uncertainty": { + "weight": 0.6, + "metrics": { + "entropy": { + "inputs": [ + { + "source": "factsheet", + "field_path": "data/entropy_local" + } + ], + "operation": "get_value", + "type": "true_score", + "description": "The measure of uncertainty in identifying a client.", + "weight": 1 + } + } + }, + "indistinguishability": { + "weight": 0.2, + "metrics": { + "global_privacy_risk": { + "inputs": [ + { + "source": "factsheet", + "field_path": "configuration/differential_privacy" + }, + { + "source": "factsheet", + "field_path": "configuration/dp_epsilon" + }, + { + "source": "factsheet", + "field_path": "participants/client_num" + } + ], + "operation": "get_global_privacy_risk", + "type": "true_score", + "direction": "desc", + "description": "A worst-case approximation of the maximal risk for distinguishing two clients.", + "weight": 1 + } + } + } + }, + "fairness": { + "class_distribution": { + "weight": 1, + "metrics": { + "class_imbalance": { + "inputs": [ + { + "source": "factsheet", + "field_path": "fairness/class_imbalance" + } + ], + "operation": "get_value", + "type": "true_score", + "direction": "desc", + "description": "Variation of the sample size per class.", + "weight": 1 + } + } + } + }, + "explainability": { + "interpretability": { + "weight": 0.4, + "metrics": { + "algorithmic_transparency": { + "inputs": [ + { + "source": "factsheet", + "field_path": "configuration/training_model" + } + ], + "operation": "get_value", + "type": "score_mapping", + "score_map": { + "RandomForestClassifier": 4, + "KNeighborsClassifier": 3, + "SVC": 2, + "GaussianProcessClassifier": 3, + "DecisionTreeClassifier": 5, + "MLP": 1, + "AdaBoostClassifier": 3, + "GaussianNB": 3.5, + "QuadraticDiscriminantAnalysis": 3, + "LogisticRegression": 4, + "LinearRegression": 3.5, + "Sequential": 1, + "CNN": 1 + }, + "description": "Mapping of Learning techniques to the level of explainability based on on literature research and qualitative analysis of each learning technique.", + "weight": 0.6 + }, + "model_size": { + "inputs": [ + { + "source": "factsheet", + "field_path": "configuration/trainable_param_num" + } + ], + "operation": "get_value", + "type": "ranges", + "direction": "desc", + "ranges": [10e1, 10e2, 10e3, 10e4, 10e5, 10e6, 10e7, 10e8], + "description": "Ranges of how to map model size to a score from 1-5.", + "weight": 0.4 + } + } + }, + "post_hoc_methods": { + "weight": 0.6, + "metrics": { + "feature_importance": { + "inputs": [ + { + "source": "factsheet", + "field_path": "performance/test_feature_importance_cv" + } + ], + "operation": "get_value", + "type": "true_score", + "description": "Variation of feature importance scores of all the features.", + "weight": 0.5 + }, + "visualization": { + "inputs": [ + { + "source": "factsheet", + "field_path": "configuration/visualization" + } + ], + "operation": "get_value", + "type": "true_score", + "description": "The use of graphical capabilities to show the explainability.", + "weight": 0.5 + } + } + } + }, + "accountability": { + "factsheet_completeness": { + "weight": 1, + "metrics": { + "project_specs": { + "inputs": [ + { + "source": "factsheet", + "field_path": "project/overview" + }, + { + "source": "factsheet", + "field_path": "project/purpose" + }, + { + "source": "factsheet", + "field_path": "project/background" + } + ], + "operation": "check_properties", + "type": "property_check", + "description": "Specifications of the project.", + "weight": 0.1 + }, + "participants": { + "inputs": [ + { + "source": "factsheet", + "field_path": "participants/client_num" + }, + { + "source": "factsheet", + "field_path": "participants/sample_client_rate" + }, + { + "source": "factsheet", + "field_path": "participants/client_selector" + } + ], + "operation": "check_properties", + "type": "property_check", + "description": "Participants information.", + "weight": 0.1 + }, + "data": { + "inputs": [ + { + "source": "factsheet", + "field_path": "data/provenance" + }, + { + "source": "factsheet", + "field_path": "data/preprocessing" + }, + { + "source": "factsheet", + "field_path": "data/entropy_local" + } + ], + "operation": "check_properties", + "type": "property_check", + "description": "Meta data about the data.", + "weight": 0.2 + }, + "configuration": { + "inputs": [ + { + "source": "factsheet", + "field_path": "configuration/optimization_algorithm" + }, + { + "source": "factsheet", + "field_path": "configuration/training_model" + }, + { + "source": "factsheet", + "field_path": "configuration/personalization" + }, + { + "source": "factsheet", + "field_path": "configuration/differential_privacy" + }, + { + "source": "factsheet", + "field_path": "configuration/dp_epsilon" + }, + { + "source": "factsheet", + "field_path": "configuration/trainable_param_num" + }, + { + "source": "factsheet", + "field_path": "configuration/total_round_num" + }, + { + "source": "factsheet", + "field_path": "configuration/learning_rate" + }, + { + "source": "factsheet", + "field_path": "configuration/local_update_steps" + } + ], + "operation": "check_properties", + "type": "property_check", + "description": "FL model configurations.", + "weight": 0.2 + }, + "performance": { + "inputs": [ + { + "source": "factsheet", + "field_path": "performance/test_loss" + }, + { + "source": "factsheet", + "field_path": "performance/test_acc" + }, + { + "source": "factsheet", + "field_path": "performance/test_feature_importance_cv" + }, + { + "source": "factsheet", + "field_path": "performance/test_clever" + }, + { + "source": "factsheet", + "field_path": "performance/test_loss_sensitivity" + }, + { + "source": "factsheet", + "field_path": "performance/test_adv_accuracy" + }, + { + "source": "factsheet", + "field_path": "performance/test_empirical_robustness" + }, + { + "source": "factsheet", + "field_path": "performance/test_confidence_score" + }, + { + "source": "factsheet", + "field_path": "performance/test_attack_success_rate" + } + ], + "operation": "check_properties", + "type": "property_check", + "description": "Performance evaluation results.", + "weight": 0.2 + }, + "fairness": { + "inputs": [ + { + "source": "factsheet", + "field_path": "fairness/class_imbalance" + } + ], + "operation": "check_properties", + "type": "property_check", + "description": "Fairness metrics results.", + "weight": 0.1 + }, + "system": { + "inputs": [ + { + "source": "factsheet", + "field_path": "system/time_minutes" + }, + { + "source": "factsheet", + "field_path": "system/model_size" + }, + { + "source": "factsheet", + "field_path": "system/upload_bytes" + }, + { + "source": "factsheet", + "field_path": "system/download_bytes" + } + ], + "operation": "check_properties", + "type": "property_check", + "description": "System usage information.", + "weight": 0.1 + } + } + } + }, + "architectural_soundness": { + "client_management": { + "weight": 0.5, + "metrics": { + "client_selector": { + "inputs": [ + { + "source": "factsheet", + "field_path": "participants/client_selector" + } + ], + "operation": "check_properties", + "type": "property_check", + "description": "The use of a client selector.", + "weight": 1 + } + } + }, + "optimization": { + "weight": 0.5, + "metrics": { + "algorithm": { + "inputs": [ + { + "source": "factsheet", + "field_path": "configuration/aggregation_algorithm" + } + ], + "operation": "get_value", + "type": "score_map_value", + "score_map": { + "FedAvg": 0.9509, + "Krum": 0.9535, + "TrimmedMean": 0.9595, + "Median": 0.9461 + }, + "description": "The choice of a suitable aggregation algorithm.", + "weight": 1 + } + } + } + }, + "sustainability": { + "energy_source": { + "weight": 0.5, + "metrics": { + "carbon_intensity_clients": { + "inputs": [ + { + "source": "factsheet", + "field_path": "sustainability/carbon_intensity_local" + } + ], + "operation": "get_value", + "type": "scaled_score", + "direction": "desc", + "scale": [20, 795], + "description": "Carbon intensity of energy grid used by clients", + "weight": 1 + } + } + }, + "federation_complexity": { + "weight": 0.5, + "metrics": { + "communication_efficiency": { + "inputs": [ + { "source": "factsheet", "field_path": "system/upload_bytes" }, + { "source": "factsheet", "field_path": "system/download_bytes" }, + { "source": "factsheet", "field_path": "performance/test_acc" } + ], + "operation": "comm_efficiency", + "type": "ranges", + "direction": "low", + "ranges":[0.1, 10e2, 10e3,10e4, 10e5, 10e6,10e7,10e8,10e9,10e10,10e11], + "description": "Descripcion de la metrica", + "weight": 0.1 + }, + "number_of_training_rounds": { + "inputs": [ + { + "source": "factsheet", + "field_path": "configuration/total_round_num" + } + ], + "operation": "get_value", + "type": "ranges", + "direction": "desc", + "ranges": [5, 10, 15, 20, 25, 30, 35, 40, 45, 50], + "description": "The total number of training rounds", + "weight": 0.06666666 + }, + "avg_model_size": { + "inputs": [ + { + "source": "factsheet", + "field_path": "configuration/trainable_param_num" + } + ], + "operation": "get_value", + "type": "ranges", + "direction": "desc", + "ranges":[10e4, 10e5, 10e6,10e7,10e8,10e9,10e10,10e11], + "description": "The size of the model", + "weight": 0.16666666 + }, + "client_selection_rate": { + "inputs": [ + { + "source": "factsheet", + "field_path": "participants/sample_client_rate" + } + ], + "operation": "get_value", + "type": "scaled_score", + "direction": "asc", + "scale": [ + 0.1,1 + ], + "description": "The selection rate of clients for each training round", + "weight": 0.16666666 + }, + "number_of_clients": { + "inputs": [ + { + "source": "factsheet", + "field_path": "participants/client_num" + } + ], + "operation": "get_value", + "type": "ranges", + "direction": "desc", + "ranges": [5, 10, 15, 20, 25, 30, 35, 40, 45, 50], + "description": "The number of clients in the federation.", + "weight": 0.16666666 + }, + "local_training_rounds": { + "inputs": [ + { + "source": "factsheet", + "field_path": "configuration/local_update_steps" + } + ], + "operation": "get_value", + "type": "scaled_score", + "direction": "desc", + "scale": [1, 100], + "description": "The number of local training rounds.", + "weight": 0.16666666 + }, + "avg_dataset_size": { + "inputs": [ + { + "source": "factsheet", + "field_path": "participants/local_dataset_size" + } + ], + "operation": "get_value", + "type": "ranges", + "direction": "desc", + "ranges": [10e1, 10e2, 10e3, 10e4, 10e5], + "description": "The average number of training samples", + "weight": 0.16666666 + } + } + } + } + } diff --git a/nebula/addons/trustworthiness/configs/factsheet_template.json b/nebula/addons/trustworthiness/configs/factsheet_template.json index eeeaa7f67..b2369d7ea 100755 --- a/nebula/addons/trustworthiness/configs/factsheet_template.json +++ b/nebula/addons/trustworthiness/configs/factsheet_template.json @@ -31,7 +31,12 @@ "test_loss_avg": "", "test_acc_avg": "", "test_feature_importance_cv": "", - "test_clever": "" + "test_clever": "", + "test_loss_sensitivity": "", + "test_adv_accuracy": "", + "test_empirical_robustness": "", + "test_confidence_score": "", + "test_attack_success_rate": "" }, "fairness": { "test_acc_cv": "", diff --git a/nebula/addons/trustworthiness/configs/factsheet_template_dfl.json b/nebula/addons/trustworthiness/configs/factsheet_template_dfl.json new file mode 100755 index 000000000..e2efbce7d --- /dev/null +++ b/nebula/addons/trustworthiness/configs/factsheet_template_dfl.json @@ -0,0 +1,56 @@ +{ + "project": { + "overview": "", + "purpose": "", + "background": "" + }, + "data": { + "provenance": "", + "preprocessing": "", + "entropy_local": "" + }, + "participants": { + "client_num": "", + "sample_client_rate": "", + "client_selector": "", + "local_dataset_size": "" + }, + "configuration": { + "aggregation_algorithm": "", + "training_model": "", + "personalization": "", + "visualization": "", + "differential_privacy": "", + "dp_epsilon": "", + "trainable_param_num": "", + "total_round_num": "", + "learning_rate": "", + "local_update_steps": "" + }, + "performance": { + "test_loss": "", + "test_acc": "", + "test_feature_importance_cv": "", + "test_clever": "", + "test_loss_sensitivity": "", + "test_adv_accuracy": "", + "test_empirical_robustness": "", + "test_confidence_score": "", + "test_attack_success_rate": "" + }, + "fairness": { + "class_imbalance": "" + }, + "system": { + "time_minutes": "", + "model_size": "", + "upload_bytes": "", + "download_bytes":"" + }, + "sustainability": { + "carbon_intensity_local": "", + "emissions_training_local": "", + "energy_consumed_local": "", + "emissions_communication_local": "" + } +} diff --git a/nebula/addons/trustworthiness/dfl_local.py b/nebula/addons/trustworthiness/dfl_local.py new file mode 100644 index 000000000..ee24b3d58 --- /dev/null +++ b/nebula/addons/trustworthiness/dfl_local.py @@ -0,0 +1,285 @@ +# nebula/addons/trustworthiness/dfl_local.py +import json, os, shutil +from datetime import datetime +from nebula.addons.trustworthiness.metric import TrustMetricManager +import logging +import glob +import shutil +from json import JSONDecodeError +import pickle +import numpy as np +import pandas as pd +import time + +# from nebula.core.models.cifar10.cnn import CIFAR10ModelCNN +from nebula.core.models.mnist.mlp import MNISTModelMLP +from nebula.core.models.mnist.cnn import MNISTModelCNN +from nebula.core.models.covtype.mlp import CovtypeModelMLP +from nebula.core.models.adultcensus.mlp import AdultCensusModelMLP +from nebula.core.models.breast_cancer.mlp import BreastCancerModelMLP +from nebula.addons.trustworthiness.calculation import get_elapsed_time, get_bytes_models, get_bytes_sent_recv, get_avg_loss_accuracy, get_cv, get_clever_score, get_feature_importance_cv, get_loss_sensitivity_score, compute_adversarial_accuracy_art,get_empirical_robustness_score,get_confidence_score,attack_success_rate, get_bytes_model +from nebula.addons.trustworthiness.utils import count_all_class_samples, read_csv, check_field_filled, get_all_data_entropy + +dirname = os.path.dirname(__file__) +logger = logging.getLogger(__name__) + +def compute_trust_local_dfl(experiment_name, participant_idx, data, start_time, end_time): + trust_dir = os.path.join(os.environ.get("NEBULA_LOGS_DIR"), experiment_name, "trustworthiness") + os.makedirs(trust_dir, exist_ok=True) + + # 1) Factsheet por nodo + factsheet_name = f"factsheet_participant_{participant_idx}.json" + factsheet_path = os.path.join(trust_dir, factsheet_name) + + # Copia de template (la misma que usa Factsheet) :contentReference[oaicite:9]{index=9} + template_path = os.path.join(dirname, "configs", "factsheet_template_dfl.json") + if not os.path.exists(factsheet_path): + shutil.copyfile(template_path, factsheet_path) + + # Relleno mínimo: aquí pones valores LOCALES del nodo. + # (puedes ir ampliándolo) + with open(factsheet_path, "r+", encoding="utf-8") as f: + factsheet = {} + factsheet = json.load(f) + + # Pre-train básico desde data (usa federation, dataset, etc.) :contentReference[oaicite:10]{index=10} + logging.info("DFL FactSheet: Populating factsheet with pre training metrics") + + federation = data["federation"] + n_nodes = int(data["n_nodes"]) + dataset = data["dataset"] + algorithm = data["model"] + aggregation_algorithm = data["agg_algorithm"] + n_rounds = int(data["rounds"]) + attack = data["attack_params"]["attacks"] + if attack != "No Attack": + poisoned_node_percent = int(data["attack_params"]["poisoned_node_percent"]) + poisoned_sample_percent = int(data["attack_params"]["poisoned_sample_percent"]) + poisoned_noise_percent = int(data["attack_params"]["poisoned_noise_percent"]) + else: + poisoned_node_percent = 0 + poisoned_sample_percent = 0 + poisoned_noise_percent = 0 + with_reputation = data["reputation"]["enabled"] + is_dynamic_topology = False # data["is_dynamic_topology"] + is_dynamic_aggregation = False # data["is_dynamic_aggregation"] + target_aggregation = False # data["target_aggregation"] + + if attack != "No Attack" and with_reputation == True and is_dynamic_aggregation == True: + background = f"For the project setup, the most important aspects are the following: The federation architecture is {federation}, involving {n_nodes} clients, the dataset used is {dataset}, the learning algorithm is {algorithm}, the aggregation algorithm is {aggregation_algorithm} and the number of rounds is {n_rounds}. In addition, the type of attack used against the clients is {attack}, where the percentage of attacked nodes is {poisoned_node_percent}, the percentage of attacked samples of each node is {poisoned_sample_percent}, and the percent of poisoned noise is {poisoned_noise_percent}. A reputation-based defence with a dynamic aggregation based on the aggregation algorithm {target_aggregation} is used, and the trustworthiness of the project is desired." + + elif attack != "No Attack" and with_reputation == True and is_dynamic_topology == True: + background = f"For the project setup, the most important aspects are the following: The federation architecture is {federation}, involving {n_nodes} clients, the dataset used is {dataset}, the learning algorithm is {algorithm}, the aggregation algorithm is {aggregation_algorithm} and the number of rounds is {n_rounds}. In addition, the type of attack used against the clients is {attack}, where the percentage of attacked nodes is {poisoned_node_percent}, the percentage of attacked samples of each node is {poisoned_sample_percent}, and the percent of poisoned noise is {poisoned_noise_percent}. A reputation-based defence with a dynamic topology is used, and the trustworthiness of the project is desired." + + elif attack != "No Attack" and with_reputation == False: + background = f"For the project setup, the most important aspects are the following: The federation architecture is {federation}, involving {n_nodes} clients, the dataset used is {dataset}, the learning algorithm is {algorithm}, the aggregation algorithm is {aggregation_algorithm} and the number of rounds is {n_rounds}. In addition, the type of attack used against the clients is {attack}, where the percentage of attacked nodes is {poisoned_node_percent}, the percentage of attacked samples of each node is {poisoned_sample_percent}, and the percent of poisoned noise is {poisoned_noise_percent}. No defence mechanism is used, and the trustworthiness of the project is desired." + + elif attack == "No Attack": + background = f"For the project setup, the most important aspects are the following: The federation architecture is {federation}, involving {n_nodes} clients, the dataset used is {dataset}, the learning algorithm is {algorithm}, the aggregation algorithm is {aggregation_algorithm} and the number of rounds is {n_rounds}. No attacks against clients are used, and the trustworthiness of the project is desired." + + # Set project specifications + factsheet["project"]["overview"] = data["scenario_title"] + factsheet["project"]["purpose"] = data["scenario_description"] + factsheet["project"]["background"] = background + + # Set data specifications + factsheet["data"]["provenance"] = data["dataset"] + factsheet["data"]["preprocessing"] = data["topology"] + + # Set participants + factsheet["participants"]["client_num"] = data["n_nodes"] or "" + factsheet["participants"]["sample_client_rate"] = 1 + factsheet["participants"]["client_selector"] = "" + + # Set configuration + factsheet["configuration"]["aggregation_algorithm"] = data["agg_algorithm"] or "" + factsheet["configuration"]["training_model"] = data["model"] or "" + factsheet["configuration"]["personalization"] = False + factsheet["configuration"]["visualization"] = True + factsheet["configuration"]["total_round_num"] = n_rounds + + if poisoned_noise_percent != 0: + factsheet["configuration"]["differential_privacy"] = True + factsheet["configuration"]["dp_epsilon"] = poisoned_noise_percent + else: + factsheet["configuration"]["differential_privacy"] = False + factsheet["configuration"]["dp_epsilon"] = "" + + if dataset == "MNIST" and algorithm == "MLP": + model = MNISTModelMLP() + num_classes_temp = 10 + elif dataset == "MNIST" and algorithm == "CNN": + model = MNISTModelCNN() + num_classes_temp = 10 + elif dataset == "Covtype" and algorithm == "MLP": + model = CovtypeModelMLP() + num_classes_temp = 7 + elif dataset == "AdultCensus" and algorithm == "MLP": + model = AdultCensusModelMLP() + num_classes_temp = 2 + elif dataset == "BreastCancer" and algorithm == "MLP": + model = BreastCancerModelMLP() + num_classes_temp = 2 + + factsheet["configuration"]["learning_rate"] = model.get_learning_rate() + factsheet["configuration"]["trainable_param_num"] = model.count_parameters() + factsheet["configuration"]["local_update_steps"] = 1 + + files_dir = os.path.join(os.environ.get("NEBULA_LOGS_DIR"), experiment_name, "trustworthiness") + + final_model_file = os.path.join(files_dir, f"participant_{participant_idx}_final_model.pk") + train_model_file = os.path.join(files_dir, f"participant_{participant_idx}_train_model.pk") + test_dataloader_file = os.path.join(files_dir, f"participant_{participant_idx}_test_loader.pk") + emissions_file = os.path.join(files_dir, f"emissions.csv") + + with open(train_model_file, "rb") as t_file: + lightning_model = pickle.load(t_file) + + get_all_data_entropy(experiment_name) + + data_class_count_file = os.path.join(os.environ.get('NEBULA_LOGS_DIR'), experiment_name, "trustworthiness", f"{str(participant_idx)}_class_count.json") + + entropy_local = normalized_entropy_from_class_counts(data_class_count_file) + + factsheet["data"]["entropy_local"] = entropy_local + + df = load_round_metrics(experiment_name, participant_idx) + acc = df["accuracy"].astype(float).to_numpy() + loss = df["loss"].astype(float).to_numpy() + + final_acc = float(acc[-1]) + final_loss = float(loss[-1]) + + factsheet["performance"]["test_loss"] = float(final_loss) + factsheet["performance"]["test_acc"] = float(final_acc) + + bytes_sent, bytes_recv = get_bytes(experiment_name, participant_idx) + + model_file = os.path.join(files_dir, f"participant_{participant_idx}_final_model.pk") + factsheet["system"]["model_size"] = get_bytes_model(model_file) + + factsheet["system"]["upload_bytes"] = int(bytes_sent) + factsheet["system"]["download_bytes"] = int(bytes_recv) + + factsheet["system"]["time_minutes"] = get_elapsed_time(start_time, end_time) + + count_class_file = os.path.join(files_dir, f"{participant_idx}_class_count.json") + if os.path.exists(count_class_file): + with open(count_class_file, "r") as fs: + class_distribution = json.load(fs) + class_samples_sizes = list(class_distribution.values()) + class_imbalance = get_cv(list=class_samples_sizes) + factsheet["fairness"]["class_imbalance"] = 1 if class_imbalance > 1 else class_imbalance + else: + factsheet["fairness"]["class_imbalance"] = factsheet["fairness"].get("class_imbalance", 0.0) + + carbon_intensity_local, emissions_training_local, energy_consumed_local, sample_size = get_emissions(emissions_file, participant_idx) + + factsheet["sustainability"]["carbon_intensity_local"] = carbon_intensity_local + factsheet["sustainability"]["emissions_training_local"] = emissions_training_local + factsheet["sustainability"]["energy_consumed_local"] = energy_consumed_local + factsheet["participants"]["local_dataset_size"] = sample_size + + factsheet["sustainability"]["emissions_communication_local"] = (bytes_sent * 2.24e-10 * carbon_intensity_local)+(bytes_recv * 2.24e-10 * carbon_intensity_local) + + model.load_state_dict(lightning_model.state_dict()) + + with open(test_dataloader_file, "rb") as d_file: + test_dataloader = pickle.load(d_file) + + test_sample = next(iter(test_dataloader)) + + lr = factsheet["configuration"]["learning_rate"] + value_clever = get_clever_score(model, test_sample, num_classes_temp, lr) + + factsheet["performance"]["test_clever"] = 1 if value_clever > 1 else value_clever + + value_loss_sensitivity = get_loss_sensitivity_score(model, test_sample, num_classes_temp, lr) + + factsheet["performance"]["test_loss_sensitivity"] = 1 if value_loss_sensitivity > 1 else value_loss_sensitivity + + value_adv_accuracy = compute_adversarial_accuracy_art(model, test_dataloader, num_classes_temp, lr) + + factsheet["performance"]["test_adv_accuracy"] = 1 if value_adv_accuracy > 1 else value_adv_accuracy + + value_empirical_robustness = get_empirical_robustness_score(model, test_sample, num_classes_temp, lr) + + factsheet["performance"]["test_empirical_robustness"] = 1 if value_empirical_robustness > 1 else value_empirical_robustness + + value_confidence_score = get_confidence_score(model, test_sample) + + factsheet["performance"]["test_confidence_score"] = 1 if value_confidence_score > 1 else value_confidence_score + attack_success_rate + + value_attack_success_rate = attack_success_rate(model, test_sample) + + factsheet["performance"]["test_attack_success_rate"] = 1 if value_attack_success_rate > 1 else value_attack_success_rate + + feature_importance = get_feature_importance_cv(model, test_sample) + + factsheet["performance"]["test_feature_importance_cv"] = 1 if feature_importance > 1 else feature_importance + + f.seek(0) + f.truncate() + json.dump(factsheet, f, indent=4) + +def load_round_metrics(experiment_name: str, participant_idx: int): + files_dir = os.path.join(os.environ.get("NEBULA_LOGS_DIR"), experiment_name, "trustworthiness") + path = os.path.join(files_dir, f"round_metrics_participant_{participant_idx}.csv") + df = pd.read_csv(path) + + # Asegura orden + if "round" in df.columns: + df = df.sort_values("round") + + # Limpieza básica + df = df.dropna(subset=["loss", "accuracy"]) + return df + +def get_bytes(experiment_name: str, participant_idx: int): + data_file = os.path.join(os.environ.get('NEBULA_LOGS_DIR'), experiment_name, "trustworthiness", f"data_results_{participant_idx}.csv") + + data = read_csv(data_file) + + row = data[data["id"] == participant_idx] + + bytes_sent = row["bytes_sent"].iloc[0] + bytes_recv = row["bytes_recv"].iloc[0] + + return bytes_sent, bytes_recv + +def get_emissions(emissions_file, participant_idx: int): + data = read_csv(emissions_file) + + row = data[data["id"] == participant_idx] + + avg_carbon_intensity_clients = row["energy_grid"].iloc[0] + emissions_training = row["emissions"].iloc[0] + energy_consumed = row["energy_consumed"].iloc[0] + sample_size = row["sample_size"].iloc[0] + + return avg_carbon_intensity_clients, emissions_training, energy_consumed, sample_size + +def normalized_entropy_from_class_counts(count_class_file: str) -> float: + with open(count_class_file, "r") as f: + dist = json.load(f) + + counts = np.array(list(dist.values()), dtype=float) + total = counts.sum() + if total <= 0: + return 0.0 + + p = counts / total + + # Entropía (evita log(0)) + eps = 1e-12 + H = -float(np.sum(p * np.log(p + eps))) + + # Normalización por número de clases + K = len(p) + if K <= 1: + return 0.0 + + H_norm = H / float(np.log(K)) + # seguridad numérica + return max(0.0, min(1.0, H_norm)) diff --git a/nebula/addons/trustworthiness/factsheet.py b/nebula/addons/trustworthiness/factsheet.py index 3ffce970a..55aaa1bc2 100755 --- a/nebula/addons/trustworthiness/factsheet.py +++ b/nebula/addons/trustworthiness/factsheet.py @@ -7,15 +7,20 @@ import pickle import numpy as np import pandas as pd +import time # from nebula.core.models.cifar10.cnn import CIFAR10ModelCNN from nebula.core.models.mnist.mlp import MNISTModelMLP from nebula.core.models.mnist.cnn import MNISTModelCNN -from nebula.addons.trustworthiness.calculation import get_elapsed_time, get_bytes_models, get_bytes_sent_recv, get_avg_loss_accuracy, get_cv, get_clever_score, get_feature_importance_cv +from nebula.core.models.covtype.mlp import CovtypeModelMLP +from nebula.core.models.adultcensus.mlp import AdultCensusModelMLP +from nebula.core.models.breast_cancer.mlp import BreastCancerModelMLP +from nebula.addons.trustworthiness.calculation import get_elapsed_time, get_bytes_models, get_bytes_sent_recv, get_avg_loss_accuracy, get_cv, get_clever_score, get_feature_importance_cv, get_loss_sensitivity_score, compute_adversarial_accuracy_art,get_empirical_robustness_score,get_confidence_score,attack_success_rate from nebula.addons.trustworthiness.utils import count_all_class_samples, read_csv, check_field_filled, get_all_data_entropy # from nebula.core.models.syscall.mlp import SyscallModelMLP dirname = os.path.dirname(__file__) +logger = logging.getLogger(__name__) class Factsheet: def __init__(self): @@ -112,8 +117,19 @@ def populate_factsheet_pre_train(self, data, scenario_name): if dataset == "MNIST" and algorithm == "MLP": model = MNISTModelMLP() + num_classes_temp = 10 elif dataset == "MNIST" and algorithm == "CNN": model = MNISTModelCNN() + num_classes_temp = 10 + elif dataset == "Covtype" and algorithm == "MLP": + model = CovtypeModelMLP() + num_classes_temp = 7 + elif dataset == "AdultCensus" and algorithm == "MLP": + model = AdultCensusModelMLP() + num_classes_temp = 2 + elif dataset == "BreastCancer" and algorithm == "MLP": + model = BreastCancerModelMLP() + num_classes_temp = 2 # elif dataset == "Syscall" and algorithm == "MLP": # model = SyscallModelMLP() # else: @@ -147,6 +163,28 @@ def populate_factsheet_post_train(self, scenario_name, start_time, end_time): try: factsheet = json.load(f) + expected_total = int(factsheet.get("participants", {}).get("client_num", 0) or 0) + logging.info(f"[Factsheet] expected_total_nodes = {expected_total}") + + data_file = os.path.join(os.environ.get('NEBULA_LOGS_DIR'), scenario_name, "trustworthiness", "confirmation.csv") + + data = read_csv(data_file) + + number_files = len(data) + + logger.info(f"number_files={number_files}") + + while (number_files != expected_total): + logger.info("WAIT") + time.sleep(5) + data_file = os.path.join(os.environ.get('NEBULA_LOGS_DIR'), scenario_name, "trustworthiness", "confirmation.csv") + data = read_csv(data_file) + number_files = len(data) + logger.info(f"number_files={number_files}") + logger.info(f"expected_nodes={expected_total}") + + + dataset = factsheet["data"]["provenance"] model = factsheet["configuration"]["training_model"] @@ -165,7 +203,7 @@ def populate_factsheet_post_train(self, scenario_name, start_time, end_time): # dataloader = pickle.load(file) # get_entropy(i, scenario_name, dataloader) # i += 1 - + get_all_data_entropy(scenario_name) with open(f"{files_dir}/entropy.json", "r") as file: @@ -198,7 +236,7 @@ def populate_factsheet_post_train(self, scenario_name, start_time, end_time): factsheet["fairness"]["selection_cv"] = 1 count_all_class_samples(scenario_name) - + with open(f"{files_dir}/count_class.json", "r") as file: class_distribution = json.load(file) @@ -211,13 +249,24 @@ def populate_factsheet_post_train(self, scenario_name, start_time, end_time): if dataset == "MNIST" and model == "MLP": model = MNISTModelMLP() + num_classes_temp = 10 # CAMBIAR elif dataset == "MNIST" and model == "CNN": model = MNISTModelCNN() + num_classes_temp = 10 + elif dataset == "Covtype" and model == "MLP": + model = CovtypeModelMLP() + num_classes_temp = 7 + elif dataset == "AdultCensus" and model == "MLP": + model = AdultCensusModelMLP() + num_classes_temp = 2 + elif dataset == "BreastCancer" and algorithm == "MLP": + model = BreastCancerModelMLP() + num_classes_temp = 2 # elif dataset == "Syscall" and model == "MLP": # model = SyscallModelMLP() # else: # model = CIFAR10ModelCNN() - + model.load_state_dict(lightning_model.state_dict()) with open(test_dataloader_file, "rb") as file: @@ -226,10 +275,31 @@ def populate_factsheet_post_train(self, scenario_name, start_time, end_time): test_sample = next(iter(test_dataloader)) lr = factsheet["configuration"]["learning_rate"] - value_clever = get_clever_score(model, test_sample, 10, lr) + value_clever = get_clever_score(model, test_sample, num_classes_temp, lr) factsheet["performance"]["test_clever"] = 1 if value_clever > 1 else value_clever + value_loss_sensitivity = get_loss_sensitivity_score(model, test_sample, num_classes_temp, lr) + + factsheet["performance"]["test_loss_sensitivity"] = 1 if value_loss_sensitivity > 1 else value_loss_sensitivity + + value_adv_accuracy = compute_adversarial_accuracy_art(model, test_dataloader, num_classes_temp, lr) + + factsheet["performance"]["test_adv_accuracy"] = 1 if value_adv_accuracy > 1 else value_adv_accuracy + + value_empirical_robustness = get_empirical_robustness_score(model, test_sample, num_classes_temp, lr) + + factsheet["performance"]["test_empirical_robustness"] = 1 if value_empirical_robustness > 1 else value_empirical_robustness + + value_confidence_score = get_confidence_score(model, test_sample) + + factsheet["performance"]["test_confidence_score"] = 1 if value_confidence_score > 1 else value_confidence_score + attack_success_rate + + value_attack_success_rate = attack_success_rate(model, test_sample) + + factsheet["performance"]["test_attack_success_rate"] = 1 if value_attack_success_rate > 1 else value_attack_success_rate + feature_importance = get_feature_importance_cv(model, test_sample) factsheet["performance"]["test_feature_importance_cv"] = 1 if feature_importance > 1 else feature_importance @@ -278,4 +348,4 @@ def populate_factsheet_post_train(self, scenario_name, start_time, end_time): except JSONDecodeError as e: logging.info(f"{factsheet_file} is invalid") - logging.error(e) \ No newline at end of file + logging.error(e) diff --git a/nebula/addons/trustworthiness/graphics.py b/nebula/addons/trustworthiness/graphics.py index 9233db756..03239ac72 100644 --- a/nebula/addons/trustworthiness/graphics.py +++ b/nebula/addons/trustworthiness/graphics.py @@ -19,13 +19,17 @@ class Graphics(): def __init__( self, scenario_start_time, - scenario_name + scenario_name, + participant_id=None, ): self.scenario_start_time = scenario_start_time self.scenario_name = scenario_name log_dir = os.path.join(os.environ["NEBULA_LOGS_DIR"], scenario_name) - self.nebulalogger = NebulaTensorBoardLogger(scenario_start_time, f"{log_dir}", name="metrics", version=f"trust", log_graph=True) - + if participant_id==None: + self.nebulalogger = NebulaTensorBoardLogger(scenario_start_time, f"{log_dir}", name="metrics", version=f"trust", log_graph=True) + else: + self.nebulalogger = NebulaTensorBoardLogger(scenario_start_time, f"{log_dir}", name="metrics", version=f"trust_{participant_id}", log_graph=True) + def __log_figure(self, df, pillar, color, notion_y_pos = -0.4, figsize=(10,6)): filtered_df = df[df['Pillar'] == pillar].copy() @@ -36,7 +40,7 @@ def __log_figure(self, df, pillar, color, notion_y_pos = -0.4, figsize=(10,6)): filtered_df.loc[:, 'Notion'] = filtered_df['Notion'].apply(lambda x: str(x).title()) unique_notion_count = filtered_df['Notion'].nunique() - palette = [color] * unique_notion_count + palette = [color] * unique_notion_count plt.figure(figsize=figsize) ax = sns.barplot(data=filtered_df, x='Metric', y='Metric Score', hue='Notion', palette=palette, dodge=False) @@ -50,12 +54,12 @@ def __log_figure(self, df, pillar, color, notion_y_pos = -0.4, figsize=(10,6)): notion = row['Notion'] notion_score = row['Notion Score'] metric_score = row['Metric Score'] - + if notion not in notion_scores: metrics_for_notion = filtered_df[filtered_df['Notion'] == notion]['Metric'] start_pos = x_positions[i] end_pos = x_positions[i + len(metrics_for_notion) - 1] - + notion_x_pos = (start_pos + end_pos) / 2 ax.axhline(notion_score, ls='--', color='black', lw=0.5, xmin=start_pos/len(x_positions), xmax=(end_pos+1)/len(x_positions)) ax.text(notion_x_pos, notion_score + 0.01, f"{notion_score:.2f}", ha='center', va='bottom', fontsize=10, color='black') # Color negro @@ -70,15 +74,15 @@ def __log_figure(self, df, pillar, color, notion_y_pos = -0.4, figsize=(10,6)): metrics_for_notion = filtered_df[filtered_df['Notion'] == notion]['Metric'] start_pos = x_positions[i] end_pos = x_positions[i + len(metrics_for_notion) - 1] - + notion_x_pos = (start_pos + end_pos) / 2 - - ax.text(notion_x_pos, notion_y_pos, notion, ha='center', va='center', fontsize=10, color='black') - - seen_notions.add(notion) + + ax.text(notion_x_pos, notion_y_pos, notion, ha='center', va='center', fontsize=10, color='black') + + seen_notions.add(notion) for i, v in enumerate(filtered_df['Metric Score']): - ax.text(i, v + 0.01, f"{v:.2f}", ha='center', va='bottom', fontsize=10, color='black') + ax.text(i, v + 0.01, f"{v:.2f}", ha='center', va='bottom', fontsize=10, color='black') plt.xlabel('Metrics and notions', labelpad=35) plt.ylabel('Score') @@ -87,7 +91,7 @@ def __log_figure(self, df, pillar, color, notion_y_pos = -0.4, figsize=(10,6)): ax.legend_.remove() plt.tight_layout() - + self.nebulalogger.log_figure(ax.get_figure(), 0, f"Trust/Pillar/{pillar}") plt.close() @@ -179,4 +183,94 @@ def graphics(self): ax.set_xticklabels(name_labels, rotation=45) self.nebulalogger.log_figure(ax.get_figure(), 0, f"Trust/AllPillars") - plt.close() \ No newline at end of file + plt.close() + + def graphics_dfl(self,participant_id): + results_file = os.path.join(os.environ.get("NEBULA_LOGS_DIR"), self.scenario_name, "trustworthiness", f"nebula_trust_results_{participant_id}.json") + with open(results_file, 'r') as f: + results = json.load(f) + + pillars_list = [] + notion_names = [] + notion_scores = [] + metric_names = [] + metric_scores = [] + + for pillar in results["pillars"]: + for key, value in pillar.items(): + pillar_name = key + if "notions" in value: + for notion in value["notions"]: + for notion_key, notion_value in notion.items(): + notion_name = notion_key + notion_score = notion_value["score"] + for metric in notion_value["metrics"]: + for metric_key, metric_value in metric.items(): + metric_name = metric_key + metric_score = metric_value["score"] + + pillars_list.append(pillar_name) + notion_names.append(notion_name) + notion_scores.append(notion_score) + metric_names.append(metric_name) + metric_scores.append(metric_score) + + df = pd.DataFrame({ + "Pillar": pillars_list, + "Notion": notion_names, + "Notion Score": notion_scores, + "Metric": metric_names, + "Metric Score": metric_scores + }) + + self.__log_figure(df, 'robustness', "#F8D3DF") + self.__log_figure(df, "privacy", "#DA8D8B", -0.2) + self.__log_figure(df, "fairness", "#DDDDDD") + self.__log_figure(df, "explainability", "#FCEFC3") + self.__log_figure(df, "accountability", "#8FAADC", -0.3) + self.__log_figure(df, "architectural_soundness", "#DBB9FA", -0.3) + self.__log_figure(df, "sustainability", "#BBFDAF", -0.5, figsize=(12,8)) + + categories = [ + "robustness", + "privacy", + "fairness", + "explainability", + "accountability", + "architectural_soundness", + "sustainability" + ] + + scores = [results["pillars"][i][category]["score"] for i, category in enumerate(categories)] + + trust_score = results["trust_score"] + categories.append("trust_score") + scores.append(trust_score) + + palette = ["#F8D3DF", "#DA8D8B", "#DDDDDD", "#FCEFC3", "#8FAADC", "#DBB9FA", "#BBFDAF", "#BF9000"] + + plt.figure(figsize=(10, 8)) + ax = sns.barplot(x=categories, y=scores, palette=palette, hue=categories, legend=False) + ax.set_xlabel("Pillar") + ax.set_ylabel("Score") + ax.set_title("Pillars and trust scores") + + for i, v in enumerate(scores): + ax.text(i, v + 0.01, f"{v:.2f}", ha='center', va='bottom', fontsize=10) + + name_labels = [ + f"Robustness_{participant_id}", + f"Privacy_{participant_id}", + f"Fairness_{participant_id}", + f"Explainability_{participant_id}", + f"Accountability_{participant_id}", + f"Architectural Soundness_{participant_id}", + f"Sustainability_{participant_id}", + f"Trust Score_{participant_id}" + ] + + ax.set_xticks(range(len(categories))) + ax.set_xticklabels(name_labels, rotation=45) + + self.nebulalogger.log_figure(ax.get_figure(), 0, f"Trust/AllPillars_{participant_id}") + plt.close() diff --git a/nebula/addons/trustworthiness/metric.py b/nebula/addons/trustworthiness/metric.py index 0952576b3..da62568b7 100755 --- a/nebula/addons/trustworthiness/metric.py +++ b/nebula/addons/trustworthiness/metric.py @@ -16,11 +16,17 @@ class TrustMetricManager: Manager class to help store the output directory and handle calls from the FL framework. """ - def __init__(self, scenario_start_time): - self.factsheet_file_nm = "factsheet.json" - self.eval_metrics_file_nm = "eval_metrics.json" - self.nebula_trust_results_nm = "nebula_trust_results.json" - self.scenario_start_time = scenario_start_time + def __init__(self, scenario_start_time, federation, participant=None): + if federation == "DFL": + self.factsheet_file_nm = f"factsheet_participant_{participant}.json" # IDEA: Pasarle desde trustworthiness.py el id del participante, ponerlo a None para CFL + self.eval_metrics_file_nm = "eval_metrics_dfl.json" + self.nebula_trust_results_nm = f"nebula_trust_results_{participant}.json" + self.scenario_start_time = scenario_start_time + else: + self.factsheet_file_nm = "factsheet.json" + self.eval_metrics_file_nm = "eval_metrics.json" + self.nebula_trust_results_nm = "nebula_trust_results.json" + self.scenario_start_time = scenario_start_time def evaluate(self, experiment_name, weights, use_weights=False): """ @@ -64,6 +70,58 @@ def evaluate(self, experiment_name, weights, use_weights=False): final_score = round(final_score, 2) result_json["trust_score"] = final_score write_results_json(results_file, result_json) - + graphics = Graphics(self.scenario_start_time, scenario_name) graphics.graphics() + + def evaluate_participant(self, experiment_name, weights, participant_id, use_weights=False): + """ + Evaluates the trustworthiness score. + + Args: + scenario (object): The scenario in whith the trustworthiness will be calculated. + weights (dict): The desired weghts of the pillars. + use_weights (bool): True to turn on the weights in the metric config file, default to False. + """ + # Get scenario name + scenario_name = experiment_name + factsheet_file = os.path.join(os.environ.get('NEBULA_LOGS_DIR'), scenario_name, "trustworthiness", self.factsheet_file_nm) + metrics_cfg_file = os.path.join(dirname, "configs", self.eval_metrics_file_nm) + results_file = os.path.join(os.environ.get('NEBULA_LOGS_DIR'), scenario_name, "trustworthiness", self.nebula_trust_results_nm) + + if not os.path.exists(factsheet_file): + logger.error(f"{factsheet_file} is missing! Please check documentation.") + return + + if not os.path.exists(metrics_cfg_file): + logger.error(f"{metrics_cfg_file} is missing! Please check documentation.") + return + + with open(factsheet_file, "r") as f, open(metrics_cfg_file, "r") as m: + factsheet = json.load(f) + #metrics_cfg = json.load(m) + #metrics_cfg = replace_everywhere(metrics_cfg, "factsheet", f"factsheet_participant_{participant_id}") + + raw_metrics_cfg: str = m.read() + raw_metrics_cfg = raw_metrics_cfg.replace("factsheet", f"factsheet_participant_{participant_id}") + metrics_cfg = json.loads(raw_metrics_cfg) + + metrics = metrics_cfg.items() + input_docs = {f"factsheet_participant_{participant_id}": factsheet} + + result_json = {"trust_score": 0, "pillars": []} + final_score = 0 + result_print = [] + for key, value in metrics: + pillar = TrustPillar(key, value, input_docs, use_weights) + score, result = pillar.evaluate() + weight = weights.get(key) / 100 + final_score += weight * score + result_print.append([key, score]) + result_json["pillars"].append(result) + final_score = round(final_score, 2) + result_json["trust_score"] = final_score + write_results_json(results_file, result_json) + + graphics = Graphics(self.scenario_start_time, scenario_name, participant_id) + graphics.graphics_dfl(participant_id) diff --git a/nebula/addons/trustworthiness/per_round_metrics.py b/nebula/addons/trustworthiness/per_round_metrics.py new file mode 100644 index 000000000..086167065 --- /dev/null +++ b/nebula/addons/trustworthiness/per_round_metrics.py @@ -0,0 +1,174 @@ +# nebula/addons/trustworthiness/per_round_metrics.py +from __future__ import annotations + +import asyncio +import copy +import csv +import os +from dataclasses import dataclass, field +from typing import Any, Optional, Tuple + +import torch + +from nebula.addons.functions import print_msg_box +from nebula.addons.trustworthiness.calculation import get_feature_importance_cv + + +def _safe_get_round(engine) -> int: + trainer = getattr(engine, "trainer", None) + if trainer is None: + return -1 + + # Nebula suele exponer get_round() o el atributo round + try: + return int(trainer.get_round()) + except Exception: + return int(getattr(trainer, "round", -1)) + + +def _get_local_test_loader(engine): + trainer = getattr(engine, "trainer", None) + dm = getattr(trainer, "datamodule", None) + if dm is None: + return None + + try: + dm.setup(stage="test") + except Exception: + pass + + try: + tdl = dm.test_dataloader() + # En Nebula normalmente: [local_loader, global_loader] + if isinstance(tdl, (list, tuple)) and len(tdl) > 0: + return tdl[0] + return tdl + except Exception: + return None + + +def _build_test_sample_min_bs(test_loader, min_bs: int = 10) -> Optional[Tuple[Any, Any]]: + """ + Devuelve un batch (x, y) con batch_size >= min_bs si es posible. + así que min_bs=10 es lo ideal. + """ + if test_loader is None: + return None + + try: + it = iter(test_loader) + batch = next(it) + except Exception: + return None + + if not (isinstance(batch, (tuple, list)) and len(batch) >= 2): + return None + + x, y = batch[0], batch[1] + if not (isinstance(x, torch.Tensor) and isinstance(y, torch.Tensor)): + return None + + if x.size(0) >= min_bs: + return (x, y) + + xs = [x] + ys = [y] + cur = x.size(0) + + while cur < min_bs: + try: + b2 = next(it) + except Exception: + break + if not (isinstance(b2, (tuple, list)) and len(b2) >= 2): + break + x2, y2 = b2[0], b2[1] + if not (isinstance(x2, torch.Tensor) and isinstance(y2, torch.Tensor)): + break + xs.append(x2) + ys.append(y2) + cur += x2.size(0) + + x_cat = torch.cat(xs, dim=0) + y_cat = torch.cat(ys, dim=0) + return (x_cat, y_cat) + + +@dataclass +class PerRoundTrustMetrics: + experiment_name: str + participant_idx: int + trust_dir: str + role_label: str + + # Control + enable_print: bool = True + enable_csv: bool = True + + fi_every_n_rounds: int = 1 # pon 5 o 10 si quieres reducir coste + + # Estado interno + _csv_path: str = field(init=False) + _prev_acc: Optional[float] = field(default=None, init=False) + _test_loader: Any = field(default=None, init=False) + _lock: asyncio.Lock = field(default_factory=asyncio.Lock, init=False) + + async def setup(self, engine) -> None: + os.makedirs(self.trust_dir, exist_ok=True) + self._csv_path = os.path.join( + self.trust_dir, f"round_metrics_participant_{self.participant_idx}.csv" + ) + + if self.enable_csv and not os.path.exists(self._csv_path): + with open(self._csv_path, "w", newline="") as f: + w = csv.writer(f) + w.writerow([ + "round", + "participant", + "role", + "loss", + "accuracy", + "tw_stability", + ]) + + self._test_loader = _get_local_test_loader(engine) + + async def on_test_metrics(self, engine, loss: float, acc: float) -> None: + async with self._lock: + round_id = _safe_get_round(engine) + + # Métrica sencilla per-round (ejemplo): estabilidad de accuracy + if self._prev_acc is None: + tw_stability = 1.0 + else: + tw_stability = 1.0 - abs(acc - self._prev_acc) + tw_stability = max(0.0, min(1.0, tw_stability)) + self._prev_acc = acc + + fi_cv: Optional[float] = None + + if self.enable_csv: + with open(self._csv_path, "a", newline="") as f: + w = csv.writer(f) + w.writerow([ + round_id, + self.participant_idx, + self.role_label, + float(loss), + float(acc), + float(tw_stability), + None if fi_cv is None else float(fi_cv), + ]) + + if self.enable_print: + fi_txt = "NA" if fi_cv is None else f"{fi_cv:.4f}" + print_msg_box( + msg=( + f"Round: {round_id}\n" + f"Loss: {loss:.4f}\n" + f"Accuracy: {acc:.4f}\n" + f"TW/Stability: {tw_stability:.4f}\n" + ), + indent=2, + title=f"Trustworthiness (per-round) | {self.role_label} | Participant: {self.participant_idx}", + ) diff --git a/nebula/addons/trustworthiness/trustworthiness.py b/nebula/addons/trustworthiness/trustworthiness.py index 1eaa17c6a..ab860816f 100644 --- a/nebula/addons/trustworthiness/trustworthiness.py +++ b/nebula/addons/trustworthiness/trustworthiness.py @@ -8,8 +8,14 @@ from nebula.core.engine import Engine import pickle from nebula.addons.trustworthiness.calculation import stop_emissions_tracking_and_save -from nebula.addons.trustworthiness.utils import save_results_csv +from nebula.addons.trustworthiness.utils import save_results_csv, save_confirmation_csv from codecarbon import EmissionsTracker +from nebula.addons.trustworthiness.per_round_metrics import PerRoundTrustMetrics +from datetime import datetime +from nebula.addons.trustworthiness.factsheet import Factsheet +from nebula.addons.trustworthiness.metric import TrustMetricManager +from nebula.addons.trustworthiness.dfl_local import compute_trust_local_dfl +import json, os """ ############################## # TRUST WORKLOADS # @@ -23,23 +29,23 @@ class TrustWorkload(ABC): @abstractmethod async def init(self, experiment_name): raise NotImplementedError - + @abstractmethod def get_workload(self) -> str: raise NotImplementedError - + @abstractmethod def get_sample_size(self) -> float: raise NotImplementedError - + abstractmethod def get_metrics(self) -> tuple[float, float]: raise NotImplementedError - + @abstractmethod async def finish_experiment_role_pre_actions(self): raise NotImplementedError - + @abstractmethod async def finish_experiment_role_post_actions(self, trust_config, experiment_name): raise NotImplementedError @@ -55,14 +61,29 @@ def __init__(self, engine, idx, trust_files_route): self._current_loss = None self._current_accuracy = None self._experiment_name = "" - + self._per_round = None + self._start_time = datetime.now().strftime("%d/%m/%Y %H:%M:%S") + self._end_time = None + async def init(self, experiment_name): self._experiment_name = experiment_name await EventManager.get_instance().subscribe_node_event(RoundEndEvent, self._process_round_end_event) await EventManager.get_instance().subscribe_addonevent(TestMetricsEvent, self._process_test_metrics_event) await EventManager.get_instance().subscribe_node_event(ExperimentFinishEvent, self._process_experiment_finished_event) await self._create_pk_files(experiment_name) - + + self._per_round = PerRoundTrustMetrics( + experiment_name=experiment_name, + participant_idx=self._idx, + trust_dir=self._trust_files_route, + role_label="TRAINER", + enable_print=True, + enable_csv=True, + fi_every_n_rounds=1, # cambia a 5/10 si quieres menos coste + ) + await self._per_round.setup(self._engine) + + async def _create_pk_files(self, experiment_name): # Save data to local files to calculate the trustworthyness train_loader_filename = f"/nebula/app/logs/{experiment_name}/trustworthiness/participant_{self._idx}_train_loader.pk" @@ -71,54 +92,97 @@ async def _create_pk_files(self, experiment_name): train_loader = self._engine.trainer.datamodule.train_dataloader() self._engine.trainer.datamodule.setup(stage="test") test_loader = self._engine.trainer.datamodule.test_dataloader()[0] - + with open(train_loader_filename, 'wb') as f: pickle.dump(train_loader, f) f.close() with open(test_loader_filename, 'wb') as f: pickle.dump(test_loader, f) f.close() - + def get_workload(self): return self._workload - + def get_sample_size(self): return self._sample_size - + def get_metrics(self): return (self._current_loss, self._current_accuracy) - + async def finish_experiment_role_pre_actions(self): with open(self._train_loader_file, 'rb') as file: train_loader = pickle.load(file) self._sample_size = len(train_loader) - + async def finish_experiment_role_post_actions(self, trust_config, experiment_name): - pass - - async def _process_round_end_event(self, ree: RoundEndEvent): + federation = trust_config.get("federation") # "CFL" o "DFL" :contentReference[oaicite:13]{index=13} + + if federation == "DFL": + self._end_time = datetime.now().strftime("%d/%m/%Y %H:%M:%S") + data_file_path = os.path.join(os.environ.get('NEBULA_CONFIG_DIR'), experiment_name, "scenario.json") + with open(data_file_path, 'r') as data_file: + data = json.load(data_file) + + weights = { + "robustness": float(data["robustness_pillar"]), + "resilience_to_attacks": float(data["resilience_to_attacks"]), + "algorithm_robustness": float(data["algorithm_robustness"]), + "client_reliability": float(data["client_reliability"]), + "privacy": float(data["privacy_pillar"]), + "technique": float(data["technique"]), + "uncertainty": float(data["uncertainty"]), + "indistinguishability": float(data["indistinguishability"]), + "fairness": float(data["fairness_pillar"]), + "class_distribution": float(data["class_distribution"]), + "explainability": float(data["explainability_pillar"]), + "interpretability": float(data["interpretability"]), + "post_hoc_methods": float(data["post_hoc_methods"]), + "accountability": float(data["accountability_pillar"]), + "factsheet_completeness": float(data["factsheet_completeness"]), + "architectural_soundness": float(data["architectural_soundness_pillar"]), + "client_management": float(data["client_management"]), + "optimization": float(data["optimization"]), + "sustainability": float(data["sustainability_pillar"]), + "energy_source": float(data["energy_source"]), + "federation_complexity": float(data["federation_complexity"]) + } + # 1) calcula pesos (igual que ya hacías en el server, leyendo scenario.json) + # 2) cada nodo genera factsheet_participant_.json + results_participant_.json + compute_trust_local_dfl(experiment_name, self._idx, trust_config, self._start_time, self._end_time) + + trust_metric_manager = TrustMetricManager(self._start_time, federation, self._idx) + trust_metric_manager.evaluate_participant(experiment_name, weights, self._idx, use_weights=True) + elif federation == "SDFL": + pass + else: + pass + + async def _process_round_end_event(self, ree: RoundEndEvent): scenario_name = self._engine.config.participant["scenario_args"]["name"] train_model = f"/nebula/app/logs/{scenario_name}/trustworthiness/participant_{self._idx}_train_model.pk" # Save the train model in trustworthy dir with open(train_model, 'wb') as f: pickle.dump(self._engine.trainer.model, f) - + async def _process_test_metrics_event(self, tme: TestMetricsEvent): cur_loss, cur_acc = await tme.get_event_data() if cur_loss and cur_acc: self._current_loss, self._current_accuracy = cur_loss, cur_acc - - async def _process_experiment_finished_event(self, efe:ExperimentFinishEvent): + + if self._per_round is not None: + await self._per_round.on_test_metrics(self._engine, float(cur_loss), float(cur_acc)) + + async def _process_experiment_finished_event(self, efe:ExperimentFinishEvent): model_file = f"/nebula/app/logs/{self._experiment_name}/trustworthiness/participant_{self._engine.idx}_final_model.pk" - + + # Save model in trustworthy dir with open(model_file, 'wb') as f: pickle.dump(self._engine.trainer.model, f) - - - + + class TrustWorkloadServer(TrustWorkload): - + def __init__(self, engine: Engine, idx, trust_files_route): self._workload = 'aggregation' self._sample_size = 0 @@ -129,39 +193,54 @@ def __init__(self, engine: Engine, idx, trust_files_route): self._engine: Engine = engine self._end_time = None self._experiment_name = "" - + self._idx = idx + self._trust_files_route = trust_files_route + self._per_round = None + async def init(self, experiment_name): self._experiment_name = experiment_name - await EventManager.get_instance().subscribe_addonevent(TestMetricsEvent, self._process_test_metrics_event) + await EventManager.get_instance().subscribe_addonevent(TestMetricsEvent, self._process_test_metrics_event) await EventManager.get_instance().subscribe_node_event(ExperimentFinishEvent, self._process_experiment_finished_event) - + + self._per_round = PerRoundTrustMetrics( + experiment_name=experiment_name, + participant_idx=self._idx, + trust_dir=self._trust_files_route, + role_label="SERVER", + enable_print=True, + enable_csv=True, + fi_every_n_rounds=1, + ) + await self._per_round.setup(self._engine) + + def get_workload(self): return self._workload - + def get_sample_size(self): return self._sample_size - + def get_metrics(self): return (self._current_loss, self._current_accuracy) - + async def finish_experiment_role_pre_actions(self): pass - + async def finish_experiment_role_post_actions(self, trust_config, experiment_name): from datetime import datetime self._end_time = datetime.now().strftime("%d/%m/%Y %H:%M:%S") await self._generate_factsheet(trust_config, experiment_name) - + async def _generate_factsheet(self, trust_config, experiment_name): from nebula.addons.trustworthiness.factsheet import Factsheet from nebula.addons.trustworthiness.metric import TrustMetricManager import json import os - + factsheet = Factsheet() factsheet.populate_factsheet_pre_train(trust_config, experiment_name) factsheet.populate_factsheet_post_train(experiment_name, self._start_time, self._end_time) - + data_file_path = os.path.join(os.environ.get('NEBULA_CONFIG_DIR'), experiment_name, "scenario.json") with open(data_file_path, 'r') as data_file: data = json.load(data_file) @@ -192,18 +271,22 @@ async def _generate_factsheet(self, trust_config, experiment_name): "hardware_efficiency": float(data["hardware_efficiency"]), "federation_complexity": float(data["federation_complexity"]) } + federation = trust_config.get("federation") - trust_metric_manager = TrustMetricManager(self._start_time) + trust_metric_manager = TrustMetricManager(self._start_time, federation) trust_metric_manager.evaluate(experiment_name, weights, use_weights=True) - + async def _process_test_metrics_event(self, tme: TestMetricsEvent): cur_loss, cur_acc = await tme.get_event_data() if cur_loss and cur_acc: self._current_loss, self._current_accuracy = cur_loss, cur_acc - async def _process_experiment_finished_event(self, efe:ExperimentFinishEvent): + if self._per_round is not None: + await self._per_round.on_test_metrics(self._engine, float(cur_loss), float(cur_acc)) + + async def _process_experiment_finished_event(self, efe:ExperimentFinishEvent): model_file = f"/nebula/app/logs/{self._experiment_name}/trustworthiness/participant_{self._engine.idx}_final_model.pk" - + # Save model in trustworthy dir with open(model_file, 'wb') as f: pickle.dump(self._engine.trainer.model, f) @@ -228,55 +311,106 @@ def __init__(self, engine: Engine, config: Config): self._emissions_file = 'emissions.csv' self._role: Role = engine.rb.get_role() self._idx = self._config.participant["device_args"]["idx"] - self._trust_workload: TrustWorkload = self._factory_trust_workload(self._role, self._engine, self._idx, self._trust_dir_files) - + self._trust_workload: TrustWorkload = self._factory_trust_workload(self._role, self._engine, self._idx, self._trust_dir_files) + # EmissionsTracker from codecarbon to measure the emissions during the aggregation step in the server self._tracker= EmissionsTracker(tracking_mode='process', log_level='error', save_to_file=False) - + @property def tw(self): """TrustWorkload depending on the node Role""" return self._trust_workload - + async def start(self): await self._create_trustworthiness_directory() await self.tw.init(self._experiment_name) await EventManager.get_instance().subscribe_node_event(ExperimentFinishEvent, self._process_experiment_finish_event) self._tracker.start() - + async def _create_trustworthiness_directory(self): import os trust_dir = os.path.join(os.environ.get("NEBULA_LOGS_DIR"), self._experiment_name, "trustworthiness") # Create a directory to save files to calcutate trust os.makedirs(trust_dir, exist_ok=True) os.chmod(trust_dir, 0o777) - + async def _process_experiment_finish_event(self, efe: ExperimentFinishEvent): from nebula.addons.trustworthiness.utils import save_class_count_per_participant class_counter = self._engine.trainer.datamodule.get_samples_per_label() save_class_count_per_participant(self._experiment_name, class_counter, self._idx) - + await self.tw.finish_experiment_role_pre_actions() - + last_loss, last_accuracy = self.tw.get_metrics() - + # Get bytes send/received from reporter bytes_sent = self._engine.reporter.acc_bytes_sent bytes_recv = self._engine.reporter.acc_bytes_recv - + # Get TrustWorkload info workload = self.tw.get_workload() sample_size = self.tw.get_sample_size() - + # Last operations save_results_csv(self._experiment_name, self._idx, bytes_sent, bytes_recv, last_loss, last_accuracy) - stop_emissions_tracking_and_save(self._tracker, self._trust_dir_files, self._emissions_file, self._role.value, workload, sample_size) - + stop_emissions_tracking_and_save(self._tracker, self._trust_dir_files, self._emissions_file, self._role.value, workload, sample_size, self._idx) + save_confirmation_csv(self._experiment_name, self._idx) + """ + federation = self._trust_config.get("federation") # "CFL" o "DFL" :contentReference[oaicite:13]{index=13} + + if federation == "DFL": + data_file_path = os.path.join(os.environ.get('NEBULA_CONFIG_DIR'), self._experiment_name, "scenario.json") + with open(data_file_path, 'r') as data_file: + data = json.load(data_file) + + weights = { + "robustness": float(data["robustness_pillar"]), + "resilience_to_attacks": float(data["resilience_to_attacks"]), + "algorithm_robustness": float(data["algorithm_robustness"]), + "client_reliability": float(data["client_reliability"]), + "privacy": float(data["privacy_pillar"]), + "technique": float(data["technique"]), + "uncertainty": float(data["uncertainty"]), + "indistinguishability": float(data["indistinguishability"]), + "fairness": float(data["fairness_pillar"]), + "selection_fairness": float(data["selection_fairness"]), + "performance_fairness": float(data["performance_fairness"]), + "class_distribution": float(data["class_distribution"]), + "explainability": float(data["explainability_pillar"]), + "interpretability": float(data["interpretability"]), + "post_hoc_methods": float(data["post_hoc_methods"]), + "accountability": float(data["accountability_pillar"]), + "factsheet_completeness": float(data["factsheet_completeness"]), + "architectural_soundness": float(data["architectural_soundness_pillar"]), + "client_management": float(data["client_management"]), + "optimization": float(data["optimization"]), + "sustainability": float(data["sustainability_pillar"]), + "energy_source": float(data["energy_source"]), + "hardware_efficiency": float(data["hardware_efficiency"]), + "federation_complexity": float(data["federation_complexity"]) + } + # 1) calcula pesos (igual que ya hacías en el server, leyendo scenario.json) + # 2) cada nodo genera factsheet_participant_.json + results_participant_.json + compute_trust_local_dfl(self._experiment_name, self._idx, self._trust_config, weights) + + # y SALES sin tocar el camino CFL + return + + # Si NO es DFL => CFL (o lo que uses) sigue EXACTAMENTE IGUAL + + elif federation == "SDFL": + #SDFL + return + """ await self.tw.finish_experiment_role_post_actions(self._trust_config, self._experiment_name) - - def _factory_trust_workload(self, role: Role, engine: Engine, idx, trust_files_route) -> TrustWorkload: + + def _factory_trust_workload(self, role: Role, engine: Engine, idx, trust_files_route) -> TrustWorkload: trust_workloads = { - Role.TRAINER: TrustWorkloadTrainer, + Role.TRAINER: TrustWorkloadTrainer, + Role.AGGREGATOR: TrustWorkloadTrainer, + Role.PROXY: TrustWorkloadTrainer, + Role.IDLE: TrustWorkloadTrainer, + Role.TRAINER_AGGREGATOR: TrustWorkloadTrainer, Role.SERVER: TrustWorkloadServer } trust_workload = trust_workloads.get(role) @@ -284,5 +418,3 @@ def _factory_trust_workload(self, role: Role, engine: Engine, idx, trust_files_r return trust_workload(engine, idx, trust_files_route) else: raise TrustWorkloadException(f"Trustworthiness workload for role {role} not defined") - - \ No newline at end of file diff --git a/nebula/addons/trustworthiness/utils.py b/nebula/addons/trustworthiness/utils.py index e081fcafd..b4597c41f 100755 --- a/nebula/addons/trustworthiness/utils.py +++ b/nebula/addons/trustworthiness/utils.py @@ -59,7 +59,7 @@ def count_class_samples(scenario_name, dataloaders_files, class_counter: Counter result = {} dataloaders = [] - + if class_counter: result = {hashids.encode(int(class_id)): count for class_id, count in class_counter.items()} else: @@ -81,7 +81,7 @@ def count_class_samples(scenario_name, dataloaders_files, class_counter: Counter name_file = os.path.join(os.environ.get('NEBULA_LOGS_DIR'), scenario_name, "trustworthiness", "count_class.json") except: name_file = os.path.join("nebula", "app", "logs", scenario_name, "trustworthiness", "count_class.json") - + with open(name_file, "w") as f: json.dump(result, f) @@ -90,7 +90,7 @@ def get_all_data_entropy(experiment_name): participant_id = 0 data_class_count_file = os.path.join(os.environ.get('NEBULA_LOGS_DIR'), experiment_name, "trustworthiness", f"{str(participant_id)}_class_count.json") entropy_per_participant = {} - + while True: data_class_count_file = os.path.join(os.environ.get('NEBULA_LOGS_DIR'), experiment_name, "trustworthiness", f"{str(participant_id)}_class_count.json") @@ -109,12 +109,12 @@ def get_all_data_entropy(experiment_name): entropy_per_participant[str(participant_id)] = round(entropy_value, 6) participant_id += 1 - + name_file = os.path.join(os.environ.get('NEBULA_LOGS_DIR'),experiment_name, "trustworthiness", "entropy.json") with open(name_file, "w") as f: json.dump(entropy_per_participant, f, indent=2) - + def get_entropy(client_id, scenario_name, dataloader): """ Get the entropy of each client in the scenario. @@ -129,7 +129,7 @@ def get_entropy(client_id, scenario_name, dataloader): client_entropy = {} name_file = os.path.join(os.environ.get('NEBULA_LOGS_DIR'), scenario_name, "trustworthiness", "entropy.json") - + if os.path.exists(name_file): logging.info(f"entropy fiel already exists.. loading.") with open(name_file, "r") as f: @@ -274,18 +274,64 @@ def save_results_csv(scenario_name: str, id: int, bytes_sent: int, bytes_recv: i data_results_file = os.path.join(os.environ.get('NEBULA_LOGS_DIR'), scenario_name, "trustworthiness", "data_results.csv") except: data_results_file = os.path.join("nebula", "app", "logs", scenario_name, "trustworthiness", "data_results.csv") - + if exists(data_results_file): df = pd.read_csv(data_results_file) else: df = pd.DataFrame(columns=["id", "bytes_sent", "bytes_recv", "accuracy", "loss"]) - + try: # Add new entry to DataFrame new_data = pd.DataFrame({'id': [id], 'bytes_sent': [bytes_sent], 'bytes_recv': [bytes_recv], 'accuracy': [accuracy], 'loss': [loss]}) df = pd.concat([df, new_data], ignore_index=True) + logger.info(f"new_data={new_data}") + + df.to_csv(data_results_file, encoding='utf-8', index=False) + + except Exception as e: + logger.warning(e) + + try: + data_results_id_file = os.path.join(os.environ.get('NEBULA_LOGS_DIR'), scenario_name, "trustworthiness", f"data_results_{id}.csv") + except: + data_results_id_file = os.path.join("nebula", "app", "logs", scenario_name, "trustworthiness", f"data_results_{id}.csv") + + if exists(data_results_id_file): + df = pd.read_csv(data_results_id_file) + else: + df = pd.DataFrame(columns=["id", "bytes_sent", "bytes_recv", "accuracy", "loss"]) + + try: + # Add new entry to DataFrame + new_data = pd.DataFrame({'id': [id], 'bytes_sent': [bytes_sent], + 'bytes_recv': [bytes_recv], 'accuracy': [accuracy], + 'loss': [loss]}) + df = pd.concat([df, new_data], ignore_index=True) + logger.info(f"new_data={new_data}") + + df.to_csv(data_results_id_file, encoding='utf-8', index=False) + + except Exception as e: + logger.warning(e) + +def save_confirmation_csv(scenario_name: str, id: int): + try: + data_results_file = os.path.join(os.environ.get('NEBULA_LOGS_DIR'), scenario_name, "trustworthiness", "confirmation.csv") + except: + data_results_file = os.path.join("nebula", "app", "logs", scenario_name, "trustworthiness", "confirmation.csv") + + if exists(data_results_file): + df = pd.read_csv(data_results_file) + else: + df = pd.DataFrame(columns=["id", "OK"]) + + try: + # Add new entry to DataFrame + new_data = pd.DataFrame({'id': [id], 'OK': ["OK"]}) + df = pd.concat([df, new_data], ignore_index=True) + logger.info(f"new_data={new_data}") df.to_csv(data_results_file, encoding='utf-8', index=False) diff --git a/nebula/controller/scenarios.py b/nebula/controller/scenarios.py index bbfa8996c..e91e23e0e 100644 --- a/nebula/controller/scenarios.py +++ b/nebula/controller/scenarios.py @@ -23,6 +23,9 @@ from nebula.core.datasets.cifar100.cifar100 import CIFAR100Dataset from nebula.core.datasets.emnist.emnist import EMNISTDataset from nebula.core.datasets.fashionmnist.fashionmnist import FashionMNISTDataset +from nebula.core.datasets.covtype.covtype import CovtypeDataset +from nebula.core.datasets.adultcensus.adultcensus import AdultCensusDataset +from nebula.core.datasets.breast_cancer.breast_cancer import BreastCancerDataset from nebula.core.datasets.mnist.mnist import MNISTDataset from nebula.core.utils.certificate import generate_ca_certificate, generate_certificate from nebula.utils import DockerUtils, FileUtils @@ -988,9 +991,15 @@ async def load_configurations_and_start_nodes( if additional_participants: self.n_nodes += len(additional_participants) + + # Splitting dataset dataset_name = self.scenario.dataset dataset = None + + + logging.info(f"[DEBUG] dataset_name received: {dataset_name!r}") + logging.info("SALE YA") if dataset_name == "MNIST": dataset = MNISTDataset( num_classes=10, @@ -1011,6 +1020,36 @@ async def load_configurations_and_start_nodes( seed=42, config_dir=self.config_dir, ) + elif dataset_name == "Covtype": + dataset = CovtypeDataset( + num_classes=7, + partitions_number=self.n_nodes, + iid=self.scenario.iid, + partition=self.scenario.partition_selection, + partition_parameter=self.scenario.partition_parameter, + seed=42, + config_dir=self.config_dir, + ) + elif dataset_name == "AdultCensus": + dataset = AdultCensusDataset( + num_classes=2, + partitions_number=self.n_nodes, + iid=self.scenario.iid, + partition=self.scenario.partition_selection, + partition_parameter=self.scenario.partition_parameter, + seed=42, + config_dir=self.config_dir, + ) + elif dataset_name == "BreastCancer": + dataset = BreastCancerDataset( + num_classes=2, + partitions_number=self.n_nodes, + iid=self.scenario.iid, + partition=self.scenario.partition_selection, + partition_parameter=self.scenario.partition_parameter, + seed=42, + config_dir=self.config_dir, + ) elif dataset_name == "EMNIST": dataset = EMNISTDataset( num_classes=47, @@ -1046,6 +1085,15 @@ async def load_configurations_and_start_nodes( logging.info(f"Splitting {dataset_name} dataset...") dataset.initialize_dataset() + logging.info( + f"[DEBUG] train_set is None? {dataset.train_set is None} | " + f"test_set is None? {dataset.test_set is None}" + ) + + if dataset.train_set is not None and hasattr(dataset.train_set, "data"): + logging.info(f"[DEBUG] AdultCensus train_set.data.shape = {dataset.train_set.data.shape}") + else: + logging.info("[DEBUG] AdultCensus train_set has no .data yet (or train_set is None)") logging.info(f"Splitting {dataset_name} dataset... Done") if self.scenario.deployment in ["docker", "process", "physical"]: diff --git a/nebula/core/datasets/adultcensus/__init__.py b/nebula/core/datasets/adultcensus/__init__.py new file mode 100755 index 000000000..e69de29bb diff --git a/nebula/core/datasets/adultcensus/adultcensus.py b/nebula/core/datasets/adultcensus/adultcensus.py new file mode 100644 index 000000000..f85d472e9 --- /dev/null +++ b/nebula/core/datasets/adultcensus/adultcensus.py @@ -0,0 +1,242 @@ +# nebula/core/datasets/adultcensus/adultcensus.py + +import os +from typing import Tuple, Any + +import numpy as np +import torch +from torch.utils.data import Dataset + +from nebula.core.datasets.nebuladataset import NebulaDataset, NebulaPartitionHandler + + +class AdultCensusTorchDataset(Dataset): + """ + Torch Dataset wrapper for Adult Census Income dataset (tabular, already numeric). + x: float32 tensor (n_features,) + y: long scalar {0,1} where 1 means >50K + """ + def __init__(self, x: np.ndarray, y: np.ndarray): + if not isinstance(x, np.ndarray) or not isinstance(y, np.ndarray): + raise ValueError("x and y must be numpy arrays") + + if x.ndim != 2: + raise ValueError(f"x must be 2D (n_samples, n_features). Got shape={x.shape}") + + y_arr: np.ndarray = np.asarray(y).reshape(-1) + if x.shape[0] != y_arr.shape[0]: + raise ValueError(f"x and y must have same number of samples. Got {x.shape[0]} != {y_arr.shape[0]}") + + self.x: np.ndarray = x.astype(np.float32, copy=False) + self.y: np.ndarray = y_arr.astype(np.int64, copy=False) + + # Nebula conventions + self.data: np.ndarray = self.x + self.targets: np.ndarray = self.y + self.classes: list[str] = ["<=50K", ">50K"] + + def __len__(self) -> int: + return int(self.y.shape[0]) + + def __getitem__(self, idx: int) -> Tuple[torch.Tensor, torch.Tensor]: + x_i: torch.Tensor = torch.from_numpy(self.x[idx]) + y_i: torch.Tensor = torch.tensor(int(self.y[idx]), dtype=torch.long) + return x_i, y_i + + +class AdultCensusPartitionHandler(NebulaPartitionHandler): + """ + Partition handler for tabular data. + """ + def __init__(self, file_path: str, prefix: str, config: Any, empty: bool = False): + super().__init__(file_path, prefix, config, empty) + self.transform = None # no torchvision transforms for tabular + + def __getitem__(self, idx: int) -> Tuple[torch.Tensor, torch.Tensor]: + data, target = super().__getitem__(idx) + + # Some Nebula handlers may wrap data in tuples + if isinstance(data, tuple): + data = data[0] + + if isinstance(data, torch.Tensor): + x: torch.Tensor = data.to(dtype=torch.float32) + else: + x = torch.tensor(np.asarray(data), dtype=torch.float32) + + if isinstance(target, torch.Tensor): + y: torch.Tensor = target.to(dtype=torch.long) + else: + y = torch.tensor(int(target), dtype=torch.long) + + if self.target_transform is not None: + y = self.target_transform(y) + + return x, y + + +class AdultCensusDataset(NebulaDataset): + """ + Adult Census Income dataset integration for Nebula. + + - 2 classes: <=50K vs >50K + - mixed categorical + numerical -> numeric via preprocessing (impute + OHE + scale) + - deterministic stratified train/test split + """ + def __init__( + self, + num_classes: int = 2, + partitions_number: int = 1, + batch_size: int = 32, + num_workers: int = 4, + iid: bool = True, + partition: str = "dirichlet", + partition_parameter: float = 0.5, + seed: int = 42, + config_dir: str | None = None, + test_size: float = 0.2, + ): + super().__init__( + num_classes=num_classes, + partitions_number=partitions_number, + batch_size=batch_size, + num_workers=num_workers, + iid=iid, + partition=partition, + partition_parameter=partition_parameter, + seed=seed, + config_dir=config_dir, + ) + self.test_size: float = float(test_size) + + def initialize_dataset(self) -> None: + if self.train_set is None or self.test_set is None: + self.train_set, self.test_set = self.load_adult_census_dataset() + + self.data_partitioning(plot=True) + + @staticmethod + def _make_ohe_dense(): + """ + scikit-learn compatibility: + - older: OneHotEncoder(..., sparse=False) + - newer: OneHotEncoder(..., sparse_output=False) + """ + from sklearn.preprocessing import OneHotEncoder + + try: + return OneHotEncoder(handle_unknown="ignore", sparse_output=False) + except TypeError: + return OneHotEncoder(handle_unknown="ignore", sparse=False) + + def load_adult_census_dataset(self) -> Tuple[AdultCensusTorchDataset, AdultCensusTorchDataset]: + """ + Loads Adult dataset from OpenML and preprocesses to all-numeric features. + + Steps: + 1) fetch_openml(data_id=1590, as_frame=True) + 2) y = (target == '>50K').astype(int) + 3) replace '?' with NA for missing values + 4) ColumnTransformer: + - numeric: median impute + StandardScaler + - categorical: most_frequent impute + OneHotEncoder(dense) + 5) train/test split (stratified), fit preprocessing only on train (avoid leakage) + """ + data_dir: str = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data") + os.makedirs(data_dir, exist_ok=True) + + try: + import pandas as pd + from sklearn.datasets import fetch_openml + from sklearn.model_selection import train_test_split + from sklearn.compose import ColumnTransformer, make_column_selector as selector + from sklearn.pipeline import Pipeline + from sklearn.impute import SimpleImputer + from sklearn.preprocessing import StandardScaler + except Exception as e: + raise ImportError( + "AdultCensusDataset requires pandas + scikit-learn. Install them (e.g., pip install pandas scikit-learn)." + ) from e + + # 1) Load from OpenML + bunch = fetch_openml(data_id=1590, as_frame=True, data_home=data_dir) + X_df = bunch.data.copy() + y_raw = bunch.target + + # 2) Target -> {0,1} + # Normalize spaces to avoid variants like ' >50K' + y_str = y_raw.astype(str).str.strip() + y: np.ndarray = (y_str == ">50K").astype(np.int64).to_numpy() + + # 3) Replace '?' markers with NA (UCI Adult uses '?' for missing categorical values) + X_df = X_df.replace(r"^\s*\?\s*$", pd.NA, regex=True) + + # 4) Preprocess + numeric_selector = selector(dtype_exclude=["object", "category", "string"]) + categorical_selector = selector(dtype_include=["object", "category", "string"]) + + numeric_transformer = Pipeline( + steps=[ + ("impute", SimpleImputer(strategy="median")), + ("scaler", StandardScaler(with_mean=True, with_std=True)), + ] + ) + + categorical_transformer = Pipeline( + steps=[ + ("impute", SimpleImputer(strategy="most_frequent")), + ("ohe", self._make_ohe_dense()), + ] + ) + + preprocessor = ColumnTransformer( + transformers=[ + ("num", numeric_transformer, numeric_selector), + ("cat", categorical_transformer, categorical_selector), + ], + remainder="drop", + ) + + # 5) Split then fit on train + X_train_df, X_test_df, y_train, y_test = train_test_split( + X_df, + y, + test_size=self.test_size, + random_state=self.seed, + shuffle=True, + stratify=y, + ) + + X_train = preprocessor.fit_transform(X_train_df) + X_test = preprocessor.transform(X_test_df) + + # In case some sklearn path returns sparse matrices, densify safely + if hasattr(X_train, "toarray"): + X_train = X_train.toarray() + if hasattr(X_test, "toarray"): + X_test = X_test.toarray() + + X_train_np: np.ndarray = np.asarray(X_train, dtype=np.float32) + import logging + logging.getLogger().info(f"[AdultCensus] X_train shape = {X_train_np.shape}") + logging.getLogger().info(f"[AdultCensus] INPUT_DIM (post-OHE) = {int(X_train_np.shape[1])}") + X_test_np: np.ndarray = np.asarray(X_test, dtype=np.float32) + + train_ds = AdultCensusTorchDataset(X_train_np, np.asarray(y_train, dtype=np.int64)) + test_ds = AdultCensusTorchDataset(X_test_np, np.asarray(y_test, dtype=np.int64)) + + return train_ds, test_ds + + def generate_non_iid_map(self, dataset, partition: str = "dirichlet", partition_parameter: float = 0.5): + if partition == "dirichlet": + return self.dirichlet_partition(dataset, alpha=partition_parameter) + if partition == "percent": + return self.percentage_partition(dataset, percentage=partition_parameter) + raise ValueError(f"Partition {partition} is not supported for Non-IID map") + + def generate_iid_map(self, dataset, partition: str = "balancediid", partition_parameter: float = 2): + if partition == "balancediid": + return self.balanced_iid_partition(dataset) + if partition == "unbalancediid": + return self.unbalanced_iid_partition(dataset, imbalance_factor=partition_parameter) + raise ValueError(f"Partition {partition} is not supported for IID map") diff --git a/nebula/core/datasets/breast_cancer/__init__.py b/nebula/core/datasets/breast_cancer/__init__.py new file mode 100755 index 000000000..e69de29bb diff --git a/nebula/core/datasets/breast_cancer/breast_cancer.py b/nebula/core/datasets/breast_cancer/breast_cancer.py new file mode 100644 index 000000000..9181c1422 --- /dev/null +++ b/nebula/core/datasets/breast_cancer/breast_cancer.py @@ -0,0 +1,158 @@ +import os +from typing import Tuple, Any + +import numpy as np +import torch +from torch.utils.data import Dataset + +from nebula.core.datasets.nebuladataset import NebulaDataset, NebulaPartitionHandler + + +class BreastCancerTorchDataset(Dataset): + """ + Torch Dataset wrapper for sklearn breast cancer dataset (tabular). + x: float32 tensor (n_features,) + y: long scalar {0,1} + """ + def __init__(self, x: np.ndarray, y: np.ndarray): + if not isinstance(x, np.ndarray) or not isinstance(y, np.ndarray): + raise ValueError("x and y must be numpy arrays") + + if x.ndim != 2: + raise ValueError(f"x must be 2D (n_samples, n_features). Got shape={x.shape}") + + y = np.asarray(y).reshape(-1) + if x.shape[0] != y.shape[0]: + raise ValueError(f"x and y must have same number of samples. Got {x.shape[0]} != {y.shape[0]}") + + self.x = x.astype(np.float32, copy=False) + self.y = y.astype(np.int64, copy=False) + + # Nebula conventions (some utilities expect these) + self.data = self.x + self.targets = self.y + self.classes = ["0", "1"] + + def __len__(self) -> int: + return int(self.y.shape[0]) + + def __getitem__(self, idx: int) -> Tuple[torch.Tensor, torch.Tensor]: + x_i = torch.from_numpy(self.x[idx]) + y_i = torch.tensor(self.y[idx], dtype=torch.long) + return x_i, y_i + + +class BreastCancerPartitionHandler(NebulaPartitionHandler): + """ + Partition handler for tabular data. + """ + def __init__(self, file_path: str, prefix: str, config: Any, empty: bool = False): + super().__init__(file_path, prefix, config, empty) + self.transform = None # no torchvision transforms for tabular + + def __getitem__(self, idx: int): + data, target = super().__getitem__(idx) + + if isinstance(data, tuple): + data = data[0] + + if isinstance(data, torch.Tensor): + x = data.to(dtype=torch.float32) + else: + x = torch.tensor(np.asarray(data), dtype=torch.float32) + + if isinstance(target, torch.Tensor): + y = target.to(dtype=torch.long) + else: + y = torch.tensor(int(target), dtype=torch.long) + + if self.target_transform is not None: + y = self.target_transform(y) + + return x, y + + +class BreastCancerDataset(NebulaDataset): + """ + Breast Cancer Wisconsin (Diagnostic) dataset integration for Nebula. + + - 2 classes + - tabular features (30) + - deterministic stratified train/test split + """ + def __init__( + self, + num_classes: int = 2, + partitions_number: int = 1, + batch_size: int = 32, + num_workers: int = 4, + iid: bool = True, + partition: str = "dirichlet", + partition_parameter: float = 0.5, + seed: int = 42, + config_dir: str | None = None, + test_size: float = 0.2, + ): + super().__init__( + num_classes=num_classes, + partitions_number=partitions_number, + batch_size=batch_size, + num_workers=num_workers, + iid=iid, + partition=partition, + partition_parameter=partition_parameter, + seed=seed, + config_dir=config_dir, + ) + self.test_size = float(test_size) + + def initialize_dataset(self): + if self.train_set is None or self.test_set is None: + self.train_set, self.test_set = self.load_breast_cancer_dataset() + + self.data_partitioning(plot=True) + + def load_breast_cancer_dataset(self): + # Local cache directory (aunque load_breast_cancer no descarga, seguimos el patrón) + data_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data") + os.makedirs(data_dir, exist_ok=True) + + try: + from sklearn.datasets import load_breast_cancer + from sklearn.model_selection import train_test_split + except Exception as e: + raise ImportError( + "BreastCancerDataset requires scikit-learn. Install it (e.g., pip install scikit-learn)." + ) from e + + ds = load_breast_cancer() + x = np.asarray(ds.data) + y = np.asarray(ds.target).reshape(-1) # already 0/1 + + x_train, x_test, y_train, y_test = train_test_split( + x, + y, + test_size=self.test_size, + random_state=self.seed, + shuffle=True, + stratify=y, + ) + + train_ds = BreastCancerTorchDataset(x_train, y_train) + test_ds = BreastCancerTorchDataset(x_test, y_test) + + return train_ds, test_ds + + def generate_non_iid_map(self, dataset, partition: str = "dirichlet", partition_parameter: float = 0.5): + if partition == "dirichlet": + return self.dirichlet_partition(dataset, alpha=partition_parameter) + if partition == "percent": + return self.percentage_partition(dataset, percentage=partition_parameter) + raise ValueError(f"Partition {partition} is not supported for Non-IID map") + + def generate_iid_map(self, dataset, partition: str = "balancediid", partition_parameter: float = 2): + if partition == "balancediid": + return self.balanced_iid_partition(dataset) + if partition == "unbalancediid": + return self.unbalanced_iid_partition(dataset, imbalance_factor=partition_parameter) + raise ValueError(f"Partition {partition} is not supported for IID map") diff --git a/nebula/core/datasets/covtype/__init__.py b/nebula/core/datasets/covtype/__init__.py new file mode 100755 index 000000000..e69de29bb diff --git a/nebula/core/datasets/covtype/covtype.py b/nebula/core/datasets/covtype/covtype.py new file mode 100644 index 000000000..f46a6b289 --- /dev/null +++ b/nebula/core/datasets/covtype/covtype.py @@ -0,0 +1,220 @@ +# nebula/core/datasets/covtype/covtype.py + +import os +from typing import Tuple, Any + +import numpy as np +import torch +from torch.utils.data import Dataset + +from nebula.core.datasets.nebuladataset import NebulaDataset, NebulaPartitionHandler + + +class CovtypeTorchDataset(Dataset): + """ + Simple torch Dataset wrapper for tabular Covtype data. + + Returns: + x: torch.float32 tensor of shape (n_features,) + y: torch.long scalar in [0, num_classes-1] + """ + def __init__(self, x: np.ndarray, y: np.ndarray): + if not isinstance(x, np.ndarray) or not isinstance(y, np.ndarray): + raise ValueError("x and y must be numpy arrays") + + if x.ndim != 2: + raise ValueError(f"x must be 2D (n_samples, n_features). Got shape={x.shape}") + if y.ndim != 1: + y = y.reshape(-1) + + if x.shape[0] != y.shape[0]: + raise ValueError(f"x and y must have same number of samples. Got {x.shape[0]} != {y.shape[0]}") + + self.x = x.astype(np.float32, copy=False) + self.y = y.astype(np.int64, copy=False) + + self.data = self.x + self.targets = self.y + + n_classes = int(np.max(self.targets)) + 1 + self.classes = [str(i) for i in range(n_classes)] + + def __len__(self) -> int: + return int(self.y.shape[0]) + + def __getitem__(self, idx: int) -> Tuple[torch.Tensor, torch.Tensor]: + x_i = torch.from_numpy(self.x[idx]) + y_i = torch.tensor(self.y[idx], dtype=torch.long) + return x_i, y_i + + +class CovtypePartitionHandler(NebulaPartitionHandler): + """ + Partition handler for tabular datasets. + + NebulaPartitionHandler provides (data, target) from the partition storage. + For images, we usually convert to PIL and apply torchvision transforms. + Here we convert features to float32 torch tensors and targets to long. + """ + def __init__(self, file_path: str, prefix: str, config: Any, empty: bool = False): + super().__init__(file_path, prefix, config, empty) + + # For tabular data we typically don't apply torchvision transforms. + # If you later want normalization here, do it explicitly and carefully + # (train stats vs test stats, per-partition stats, etc.). + self.transform = None + + def __getitem__(self, idx: int): + data, target = super().__getitem__(idx) + + # Defensive: depending on how NebulaPartitionHandler stores/returns, + # "data" might be list/tuple/np.ndarray. Ensure we end up with 1D float32 tensor. + if isinstance(data, tuple): + # Some vision datasets store (img, meta). For tabular we ignore extras. + data = data[0] + + if isinstance(data, torch.Tensor): + x = data.to(dtype=torch.float32) + else: + x = torch.tensor(np.asarray(data), dtype=torch.float32) + + # Ensure target in [0..num_classes-1] and torch.long + if isinstance(target, torch.Tensor): + y = target.to(dtype=torch.long) + else: + y = torch.tensor(int(target), dtype=torch.long) + + if self.target_transform is not None: + y = self.target_transform(y) + + return x, y + + +class CovtypeDataset(NebulaDataset): + """ + Covtype (Forest CoverType) dataset integration for Nebula. + + Notes: + - Covtype has 7 classes. + - Features are tabular (54 features in the classic version). + - We provide a simple train/test split with fixed seed. + + Requirements: + - scikit-learn must be installed (for fetch_covtype + train_test_split). + """ + def __init__( + self, + num_classes: int = 7, + partitions_number: int = 1, + batch_size: int = 32, + num_workers: int = 4, + iid: bool = True, + partition: str = "dirichlet", + partition_parameter: float = 0.5, + seed: int = 42, + config_dir: str | None = None, + test_size: float = 0.2, + train_limit: int | None = 40000, + test_limit: int | None = 5000, + ): + super().__init__( + num_classes=num_classes, + partitions_number=partitions_number, + batch_size=batch_size, + num_workers=num_workers, + iid=iid, + partition=partition, + partition_parameter=partition_parameter, + seed=seed, + config_dir=config_dir, + ) + self.test_size = float(test_size) + self.train_limit = train_limit + self.test_limit = test_limit + + def initialize_dataset(self): + if self.train_set is None or self.test_set is None: + self.train_set, self.test_set = self.load_covtype_dataset() + + self.data_partitioning(plot=True) + + def load_covtype_dataset(self): + """ + Loads Covtype via sklearn, performs a deterministic train/test split, + and wraps into torch Datasets. + """ + # Local cache directory for sklearn dataset downloads + data_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data") + os.makedirs(data_dir, exist_ok=True) + + try: + from sklearn.datasets import fetch_covtype + from sklearn.model_selection import train_test_split + except Exception as e: + raise ImportError( + "CovtypeDataset requires scikit-learn. Install it (e.g., pip install scikit-learn)." + ) from e + + cov = fetch_covtype(data_home=data_dir, download_if_missing=True) + + x = cov.data + y = cov.target # commonly 1..7 in sklearn + + # Map labels to 0..6 (CrossEntropyLoss convention) + # If already 0..6, this is harmless for 1..7 only if we detect min. + y = np.asarray(y).reshape(-1) + if y.min() == 1: + y = y - 1 + + # Split "grande" + x_train, x_test, y_train, y_test = train_test_split( + x, y, + test_size=self.test_size, + random_state=self.seed, + shuffle=True, + stratify=y, + ) + + # Submuestreo estratificado (corto y determinista) + if self.train_limit is not None and len(y_train) > self.train_limit: + x_train, _, y_train, _ = train_test_split( + x_train, y_train, + train_size=self.train_limit, + random_state=self.seed, + shuffle=True, + stratify=y_train, + ) + + if self.test_limit is not None and len(y_test) > self.test_limit: + x_test, _, y_test, _ = train_test_split( + x_test, y_test, + train_size=self.test_limit, + random_state=self.seed, + shuffle=True, + stratify=y_test, + ) + + train_ds = CovtypeTorchDataset(x_train, y_train) + test_ds = CovtypeTorchDataset(x_test, y_test) + + return train_ds, test_ds + + def generate_non_iid_map(self, dataset, partition: str = "dirichlet", partition_parameter: float = 0.5): + if partition == "dirichlet": + partitions_map = self.dirichlet_partition(dataset, alpha=partition_parameter) + elif partition == "percent": + partitions_map = self.percentage_partition(dataset, percentage=partition_parameter) + else: + raise ValueError(f"Partition {partition} is not supported for Non-IID map") + + return partitions_map + + def generate_iid_map(self, dataset, partition: str = "balancediid", partition_parameter: float = 2): + if partition == "balancediid": + partitions_map = self.balanced_iid_partition(dataset) + elif partition == "unbalancediid": + partitions_map = self.unbalanced_iid_partition(dataset, imbalance_factor=partition_parameter) + else: + raise ValueError(f"Partition {partition} is not supported for IID map") + + return partitions_map diff --git a/nebula/core/datasets/nebuladataset.py b/nebula/core/datasets/nebuladataset.py index 0c2e03d8a..e42657989 100755 --- a/nebula/core/datasets/nebuladataset.py +++ b/nebula/core/datasets/nebuladataset.py @@ -1285,11 +1285,17 @@ def factory_nebuladataset(dataset, **config) -> NebulaDataset: from nebula.core.datasets.cifar100.cifar100 import CIFAR100Dataset from nebula.core.datasets.emnist.emnist import EMNISTDataset from nebula.core.datasets.fashionmnist.fashionmnist import FashionMNISTDataset + from nebula.core.datasets.covtype.covtype import CovtypeDataset + from nebula.core.datasets.adultcensus.adultcensus import AdultCensusDataset + from nebula.core.datasets.breast_cancer.breast_cancer import BreastCancerDataset from nebula.core.datasets.mnist.mnist import MNISTDataset options = { "MNIST": MNISTDataset, "FashionMNIST": FashionMNISTDataset, + "Covtype": CovtypeDataset, + "AdultCensus": AdultCensusDataset, + "BreastCancer": BreastCancerDataset, "EMNIST": EMNISTDataset, "CIFAR10": CIFAR10Dataset, "CIFAR100": CIFAR100Dataset, diff --git a/nebula/core/models/adultcensus/__init__.py b/nebula/core/models/adultcensus/__init__.py new file mode 100755 index 000000000..e69de29bb diff --git a/nebula/core/models/adultcensus/mlp.py b/nebula/core/models/adultcensus/mlp.py new file mode 100644 index 000000000..b2f33eacb --- /dev/null +++ b/nebula/core/models/adultcensus/mlp.py @@ -0,0 +1,67 @@ +# nebula/core/models/adultcensus/mlp.py + +import torch + +from nebula.core.models.nebulamodel import NebulaModel + + +class AdultCensusModelMLP(NebulaModel): + """ + Simple MLP for Adult Census (tabular). + - input_dim MUST match the number of features after preprocessing (OneHot + scaling). + - num_classes = 2 (<=50K vs >50K) + """ + def __init__( + self, + input_dim: int = 105, + num_classes: int = 2, + learning_rate: float = 1e-3, + metrics=None, + confusion_matrix=None, + seed=None, + hidden1: int = 256, + hidden2: int = 128, + dropout: float = 0.0, + ): + # NebulaModel expects something like input_channels first; for tabular we pass input_dim there. + super().__init__(input_dim, num_classes, learning_rate, metrics, confusion_matrix, seed) + + self.config = {"beta1": 0.9, "beta2": 0.999, "amsgrad": True} + + self.example_input_array = torch.rand(1, int(input_dim)) + self.learning_rate = float(learning_rate) + self.criterion = torch.nn.CrossEntropyLoss() + + self.l1 = torch.nn.Linear(int(input_dim), int(hidden1)) + self.l2 = torch.nn.Linear(int(hidden1), int(hidden2)) + self.l3 = torch.nn.Linear(int(hidden2), int(num_classes)) + + self.dropout = torch.nn.Dropout(float(dropout)) if float(dropout) > 0.0 else None + + def forward(self, x: torch.Tensor) -> torch.Tensor: + # Expected: (batch, input_dim). Sometimes: (batch, 1, input_dim) + if x.dim() == 3 and x.size(1) == 1: + x = x.squeeze(1) + + x = self.l1(x) + x = torch.relu(x) + if self.dropout is not None: + x = self.dropout(x) + + x = self.l2(x) + x = torch.relu(x) + if self.dropout is not None: + x = self.dropout(x) + + x = self.l3(x) + return x + + def configure_optimizers(self): + optimizer = torch.optim.Adam(self.parameters(), lr=self.learning_rate) + return optimizer + + def get_learning_rate(self) -> float: + return float(self.learning_rate) + + def count_parameters(self) -> int: + return int(sum(p.numel() for p in self.parameters() if p.requires_grad)) diff --git a/nebula/core/models/breast_cancer/__init__.py b/nebula/core/models/breast_cancer/__init__.py new file mode 100755 index 000000000..e69de29bb diff --git a/nebula/core/models/breast_cancer/mlp.py b/nebula/core/models/breast_cancer/mlp.py new file mode 100644 index 000000000..e84d099f2 --- /dev/null +++ b/nebula/core/models/breast_cancer/mlp.py @@ -0,0 +1,55 @@ +# nebula/core/models/covtype/mlp.py + +import torch + +from nebula.core.models.nebulamodel import NebulaModel + + +class BreastCancerModelMLP(NebulaModel): + def __init__( + self, + input_dim=30, + num_classes=2, + learning_rate=1e-3, + metrics=None, + confusion_matrix=None, + seed=None, + ): + # OJO: NebulaModel está pensado para imágenes (input_channels), + # pero en la práctica se usa ese primer argumento como "input shape info". + # Para tabular, pasamos input_dim en input_channels para mantener la firma. + super().__init__(input_dim, num_classes, learning_rate, metrics, confusion_matrix, seed) + + # Mantengo el mismo patrón que tu MLP de FashionMNIST. + self.config = {"beta1": 0.9, "beta2": 0.999, "amsgrad": True} + + self.example_input_array = torch.rand(1, input_dim) + self.learning_rate = learning_rate + self.criterion = torch.nn.CrossEntropyLoss() + + self.l1 = torch.nn.Linear(input_dim, 256) + self.l2 = torch.nn.Linear(256, 128) + self.l3 = torch.nn.Linear(128, num_classes) + + def forward(self, x): + # En tabular, x debe ser (batch, input_dim). + # A veces puede venir con dimensión extra (batch, 1, input_dim) por loaders. + if x.dim() == 3 and x.size(1) == 1: + x = x.squeeze(1) + + x = self.l1(x) + x = torch.relu(x) + x = self.l2(x) + x = torch.relu(x) + x = self.l3(x) + return x + + def configure_optimizers(self): + optimizer = torch.optim.Adam(self.parameters(), lr=self.learning_rate) + return optimizer + + def get_learning_rate(self) -> float: + return float(self.learning_rate) + + def count_parameters(self) -> int: + return int(sum(p.numel() for p in self.parameters() if p.requires_grad)) diff --git a/nebula/core/models/covtype/__init__.py b/nebula/core/models/covtype/__init__.py new file mode 100755 index 000000000..e69de29bb diff --git a/nebula/core/models/covtype/mlp.py b/nebula/core/models/covtype/mlp.py new file mode 100644 index 000000000..0399caa43 --- /dev/null +++ b/nebula/core/models/covtype/mlp.py @@ -0,0 +1,55 @@ +# nebula/core/models/covtype/mlp.py + +import torch + +from nebula.core.models.nebulamodel import NebulaModel + + +class CovtypeModelMLP(NebulaModel): + def __init__( + self, + input_dim=54, + num_classes=7, + learning_rate=1e-3, + metrics=None, + confusion_matrix=None, + seed=None, + ): + # OJO: NebulaModel está pensado para imágenes (input_channels), + # pero en la práctica se usa ese primer argumento como "input shape info". + # Para tabular, pasamos input_dim en input_channels para mantener la firma. + super().__init__(input_dim, num_classes, learning_rate, metrics, confusion_matrix, seed) + + # Mantengo el mismo patrón que tu MLP de FashionMNIST. + self.config = {"beta1": 0.9, "beta2": 0.999, "amsgrad": True} + + self.example_input_array = torch.rand(1, input_dim) + self.learning_rate = learning_rate + self.criterion = torch.nn.CrossEntropyLoss() + + self.l1 = torch.nn.Linear(input_dim, 256) + self.l2 = torch.nn.Linear(256, 128) + self.l3 = torch.nn.Linear(128, num_classes) + + def forward(self, x): + # En tabular, x debe ser (batch, input_dim). + # A veces puede venir con dimensión extra (batch, 1, input_dim) por loaders. + if x.dim() == 3 and x.size(1) == 1: + x = x.squeeze(1) + + x = self.l1(x) + x = torch.relu(x) + x = self.l2(x) + x = torch.relu(x) + x = self.l3(x) + return x + + def configure_optimizers(self): + optimizer = torch.optim.Adam(self.parameters(), lr=self.learning_rate) + return optimizer + + def get_learning_rate(self) -> float: + return float(self.learning_rate) + + def count_parameters(self) -> int: + return int(sum(p.numel() for p in self.parameters() if p.requires_grad)) diff --git a/nebula/core/node.py b/nebula/core/node.py index 86a73cc2a..c5f80843e 100755 --- a/nebula/core/node.py +++ b/nebula/core/node.py @@ -25,6 +25,9 @@ from nebula.core.datasets.datamodule import DataModule from nebula.core.datasets.emnist.emnist import EMNISTPartitionHandler from nebula.core.datasets.fashionmnist.fashionmnist import FashionMNISTPartitionHandler +from nebula.core.datasets.covtype.covtype import CovtypePartitionHandler +from nebula.core.datasets.adultcensus.adultcensus import AdultCensusPartitionHandler +from nebula.core.datasets.breast_cancer.breast_cancer import BreastCancerPartitionHandler from nebula.core.datasets.mnist.mnist import MNISTPartitionHandler from nebula.core.datasets.nebuladataset import NebulaPartition from nebula.core.models.cifar10.cnn import CIFAR10ModelCNN @@ -38,6 +41,9 @@ from nebula.core.models.emnist.mlp import EMNISTModelMLP from nebula.core.models.fashionmnist.cnn import FashionMNISTModelCNN from nebula.core.models.fashionmnist.mlp import FashionMNISTModelMLP +from nebula.core.models.covtype.mlp import CovtypeModelMLP +from nebula.core.models.adultcensus.mlp import AdultCensusModelMLP +from nebula.core.models.breast_cancer.mlp import BreastCancerModelMLP from nebula.core.models.mnist.cnn import MNISTModelCNN from nebula.core.models.mnist.mlp import MNISTModelMLP from nebula.core.engine import Engine @@ -112,6 +118,27 @@ async def main(config: Config): model = FashionMNISTModelCNN() else: raise ValueError(f"Model {model} not supported for dataset {dataset_name}") + elif dataset_name == "Covtype": + batch_size = 32 + handler = CovtypePartitionHandler + if model_name == "MLP": + model = CovtypeModelMLP() + else: + raise ValueError(f"Model {model} not supported for dataset {dataset_name}") + elif dataset_name == "AdultCensus": + batch_size = 32 + handler = AdultCensusPartitionHandler + if model_name == "MLP": + model = AdultCensusModelMLP() + else: + raise ValueError(f"Model {model} not supported for dataset {dataset_name}") + elif dataset_name == "BreastCancer": + batch_size = 32 + handler = BreastCancerPartitionHandler + if model_name == "MLP": + model = BreastCancerModelMLP() + else: + raise ValueError(f"Model {model} not supported for dataset {dataset_name}") elif dataset_name == "EMNIST": batch_size = 32 handler = EMNISTPartitionHandler diff --git a/nebula/frontend/static/js/deployment/help-content.js b/nebula/frontend/static/js/deployment/help-content.js index 673cae881..111d4e4a8 100644 --- a/nebula/frontend/static/js/deployment/help-content.js +++ b/nebula/frontend/static/js/deployment/help-content.js @@ -61,6 +61,9 @@ const HelpContent = (function() {
  • MNIST: The MNIST dataset
  • FashionMNIST: The FashionMNIST dataset
  • CIFAR10: The CIFAR10 dataset
    • +
  • Covtype: The Covtype dataset
    • +
  • AdultCensus: The AdultCensus dataset
    • +
  • BreastCancer: The BreastCancer dataset
    • `; diff --git a/nebula/frontend/static/js/deployment/main.js b/nebula/frontend/static/js/deployment/main.js index 3ec18a8ba..43b546f35 100644 --- a/nebula/frontend/static/js/deployment/main.js +++ b/nebula/frontend/static/js/deployment/main.js @@ -210,7 +210,7 @@ const DeploymentManager = (function() { datasetSelect.innerHTML = ""; // Add dataset options - const datasets = ['MNIST', 'FashionMNIST', 'EMNIST', 'CIFAR10', 'CIFAR100']; + const datasets = ['MNIST', 'FashionMNIST', 'EMNIST', 'CIFAR10', 'CIFAR100', 'Covtype', 'AdultCensus', 'BreastCancer']; datasets.forEach(dataset => { const option = document.createElement("option"); option.value = dataset; @@ -251,6 +251,12 @@ const DeploymentManager = (function() { return ['CNN', 'ResNet9', 'fastermobilenet', 'simplemobilenet', 'CNNv2', 'CNNv3']; case 'cifar100': return ['CNN']; + case 'covtype': + return ['MLP']; + case 'adultcensus': + return ['MLP']; + case 'breast_cancer': + return ['MLP']; default: return ['MLP', 'CNN']; } diff --git a/nebula/frontend/static/js/deployment/scenario.js b/nebula/frontend/static/js/deployment/scenario.js index feb5c978b..553c5211c 100644 --- a/nebula/frontend/static/js/deployment/scenario.js +++ b/nebula/frontend/static/js/deployment/scenario.js @@ -100,31 +100,115 @@ const ScenarioManager = (function () { sar_training: window.SaManager.getSaConfig().sar_training || false, sar_training_policy: window.SaManager.getSaConfig().sar_training_policy || "Broad-Propagation Strategy", random_topology_probability: document.getElementById("random-probability").value || 0.5, + // --- Trustworthiness (IDs distintos para CFL/DFL) --- with_trustworthiness: document.getElementById("TrustworthinessSwitch").checked ? true : false, - robustness_pillar: document.getElementById("robustness-pillar").value, - resilience_to_attacks: document.getElementById("robustness-notion-1").value, - algorithm_robustness: document.getElementById("robustness-notion-2").value, - client_reliability: document.getElementById("robustness-notion-3").value, - privacy_pillar: document.getElementById("privacy-pillar").value, - technique: document.getElementById("privacy-notion-1").value, - uncertainty: document.getElementById("privacy-notion-2").value, - indistinguishability: document.getElementById("privacy-notion-3").value, - fairness_pillar: document.getElementById("fairness-pillar").value, - selection_fairness: document.getElementById("fairness-notion-1").value, - performance_fairness: document.getElementById("fairness-notion-2").value, - class_distribution: document.getElementById("fairness-notion-3").value, - explainability_pillar: document.getElementById("explainability-pillar").value, - interpretability: document.getElementById("explainability-notion-1").value, - post_hoc_methods: document.getElementById("explainability-notion-2").value, - accountability_pillar: document.getElementById("accountability-pillar").value, - factsheet_completeness: document.getElementById("accountability-notion-1").value, - architectural_soundness_pillar: document.getElementById("architectural-soundness-pillar").value, - client_management: document.getElementById("architectural-soundness-notion-1").value, - optimization: document.getElementById("architectural-soundness-notion-2").value, - sustainability_pillar: document.getElementById("sustainability-pillar").value, - energy_source: document.getElementById("sustainability-notion-1").value, - hardware_efficiency: document.getElementById("sustainability-notion-2").value, - federation_complexity: document.getElementById("sustainability-notion-3").value, + + // Si no está activado, manda 0s para mantener el schema + ...(document.getElementById("TrustworthinessSwitch").checked + ? (() => { + const federationType = document.getElementById("federationArchitecture").value; + const useDFL = (federationType === "DFL" || federationType === "SDFL"); + + if (useDFL) { + // DFL (AJUSTA si tu DFL tiene otras nociones) + return { + robustness_pillar: document.getElementById("dfl-robustness-pillar")?.value || "0", + resilience_to_attacks: document.getElementById("dfl-robustness-notion-1")?.value || "0", + algorithm_robustness: document.getElementById("dfl-robustness-notion-2")?.value || "0", + client_reliability: document.getElementById("dfl-robustness-notion-3")?.value || "0", + + privacy_pillar: document.getElementById("dfl-privacy-pillar")?.value || "0", + technique: document.getElementById("dfl-privacy-notion-1")?.value || "0", + uncertainty: document.getElementById("dfl-privacy-notion-2")?.value || "0", + indistinguishability: document.getElementById("dfl-privacy-notion-3")?.value || "0", + + fairness_pillar: document.getElementById("dfl-fairness-pillar")?.value || "0", + // En DFL normalmente solo guardas class_distribution (notion-3) + selection_fairness: "0", + performance_fairness: "0", + class_distribution: document.getElementById("dfl-fairness-notion-3")?.value || "0", + + explainability_pillar: document.getElementById("dfl-explainability-pillar")?.value || "0", + interpretability: document.getElementById("dfl-explainability-notion-1")?.value || "0", + post_hoc_methods: document.getElementById("dfl-explainability-notion-2")?.value || "0", + + accountability_pillar: document.getElementById("dfl-accountability-pillar")?.value || "0", + factsheet_completeness: document.getElementById("dfl-accountability-notion-1")?.value || "100", + + architectural_soundness_pillar: document.getElementById("dfl-architectural-soundness-pillar")?.value || "0", + client_management: document.getElementById("dfl-architectural-soundness-notion-1")?.value || "0", + optimization: document.getElementById("dfl-architectural-soundness-notion-2")?.value || "0", + + sustainability_pillar: document.getElementById("dfl-sustainability-pillar")?.value || "0", + energy_source: document.getElementById("dfl-sustainability-notion-1")?.value || "0", + // Si en DFL no existe hardware_efficiency, lo dejamos a 0 + hardware_efficiency: "0", + // En DFL mapea federation_complexity a tu notion-3 (si es así) + federation_complexity: document.getElementById("dfl-sustainability-notion-3")?.value || "0", + }; + } + + // CFL + return { + robustness_pillar: document.getElementById("cfl-robustness-pillar")?.value || "0", + resilience_to_attacks: document.getElementById("cfl-robustness-notion-1")?.value || "0", + algorithm_robustness: document.getElementById("cfl-robustness-notion-2")?.value || "0", + client_reliability: document.getElementById("cfl-robustness-notion-3")?.value || "0", + + privacy_pillar: document.getElementById("cfl-privacy-pillar")?.value || "0", + technique: document.getElementById("cfl-privacy-notion-1")?.value || "0", + uncertainty: document.getElementById("cfl-privacy-notion-2")?.value || "0", + indistinguishability: document.getElementById("cfl-privacy-notion-3")?.value || "0", + + fairness_pillar: document.getElementById("cfl-fairness-pillar")?.value || "0", + selection_fairness: document.getElementById("cfl-fairness-notion-1")?.value || "0", + performance_fairness: document.getElementById("cfl-fairness-notion-2")?.value || "0", + class_distribution: document.getElementById("cfl-fairness-notion-3")?.value || "0", + + explainability_pillar: document.getElementById("cfl-explainability-pillar")?.value || "0", + interpretability: document.getElementById("cfl-explainability-notion-1")?.value || "0", + post_hoc_methods: document.getElementById("cfl-explainability-notion-2")?.value || "0", + + accountability_pillar: document.getElementById("cfl-accountability-pillar")?.value || "0", + factsheet_completeness: document.getElementById("cfl-accountability-notion-1")?.value || "100", + + architectural_soundness_pillar: document.getElementById("cfl-architectural-soundness-pillar")?.value || "0", + client_management: document.getElementById("cfl-architectural-soundness-notion-1")?.value || "0", + optimization: document.getElementById("cfl-architectural-soundness-notion-2")?.value || "0", + + sustainability_pillar: document.getElementById("cfl-sustainability-pillar")?.value || "0", + energy_source: document.getElementById("cfl-sustainability-notion-1")?.value || "0", + hardware_efficiency: document.getElementById("cfl-sustainability-notion-2")?.value || "0", + federation_complexity: document.getElementById("cfl-sustainability-notion-3")?.value || "0", + }; + })() + : { + robustness_pillar: "0", + resilience_to_attacks: "0", + algorithm_robustness: "0", + client_reliability: "0", + privacy_pillar: "0", + technique: "0", + uncertainty: "0", + indistinguishability: "0", + fairness_pillar: "0", + selection_fairness: "0", + performance_fairness: "0", + class_distribution: "0", + explainability_pillar: "0", + interpretability: "0", + post_hoc_methods: "0", + accountability_pillar: "0", + factsheet_completeness: "100", + architectural_soundness_pillar: "0", + client_management: "0", + optimization: "0", + sustainability_pillar: "0", + energy_source: "0", + hardware_efficiency: "0", + federation_complexity: "0", + }), + // --- /Trustworthiness --- network_subnet: "172.20.0.0/16", network_gateway: "172.20.0.1", additional_participants: window.MobilityManager.getMobilityConfig().additionalParticipants || [], diff --git a/nebula/frontend/static/js/deployment/trustworthiness.js b/nebula/frontend/static/js/deployment/trustworthiness.js index 7ba4d3f43..ec6ad92b0 100644 --- a/nebula/frontend/static/js/deployment/trustworthiness.js +++ b/nebula/frontend/static/js/deployment/trustworthiness.js @@ -2,268 +2,501 @@ const TrustworthinessManager = (function() { function initializeTrustworthinessSystem() { setupTrustworthinessSwitch(); + setupTrustworthinessFederationSwitch(); setupWeightValidation(); } - + + function isDFL() { + const ft = document.getElementById("federationArchitecture")?.value || "CFL"; + return (ft === "DFL" || ft === "SDFL"); + } + + function showTrustworthinessWeightsBlock() { + const cflBlock = document.getElementById("tw-cfl"); + const dflBlock = document.getElementById("tw-dfl"); + if (!cflBlock || !dflBlock) return; + + const use = isDFL(); + cflBlock.style.display = use ? "none" : "block"; + dflBlock.style.display = use ? "block" : "none"; + } + function setupTrustworthinessSwitch() { - document.getElementById("TrustworthinessSwitch").addEventListener("change", function() { + const sw = document.getElementById("TrustworthinessSwitch"); + if (!sw) return; + + sw.addEventListener("change", function() { const trustworthinessOptionsDiv = document.getElementById("trustworthiness-options"); - - if(this.checked){ - document.getElementById("federationArchitecture").value = "CFL"; - document.getElementById("federationArchitecture").dispatchEvent(new Event('change')); - document.getElementById("federationArchitecture").disabled = true; - trustworthinessOptionsDiv.style.display = "block" + if (!trustworthinessOptionsDiv) return; + + if (this.checked) { + trustworthinessOptionsDiv.style.display = "block"; + showTrustworthinessWeightsBlock(); + validateWeights(); } else { - document.getElementById("federationArchitecture").disabled = false; trustworthinessOptionsDiv.style.display = "none"; } }); } - + + function setupTrustworthinessFederationSwitch() { + const fed = document.getElementById("federationArchitecture"); + if (!fed) return; + + fed.addEventListener("change", function() { + const trustworthinessOptionsDiv = document.getElementById("trustworthiness-options"); + if (trustworthinessOptionsDiv?.style.display === "block") { + showTrustworthinessWeightsBlock(); + validateWeights(); + } + }); + } + function setupWeightValidation() { - const pillarIds = [ - "robustness-pillar", - "privacy-pillar", - "fairness-pillar", - "explainability-pillar", - "accountability-pillar", - "architectural-soundness-pillar", - "sustainability-pillar" + // IDs CFL + const cflPillarIds = [ + "cfl-robustness-pillar", + "cfl-privacy-pillar", + "cfl-fairness-pillar", + "cfl-explainability-pillar", + "cfl-accountability-pillar", + "cfl-architectural-soundness-pillar", + "cfl-sustainability-pillar" + ]; + const cflNotionIds = [ + "cfl-robustness-notion-1", + "cfl-robustness-notion-2", + "cfl-robustness-notion-3", + "cfl-privacy-notion-1", + "cfl-privacy-notion-2", + "cfl-privacy-notion-3", + "cfl-fairness-notion-1", + "cfl-fairness-notion-2", + "cfl-fairness-notion-3", + "cfl-explainability-notion-1", + "cfl-explainability-notion-2", + "cfl-accountability-notion-1", + "cfl-architectural-soundness-notion-1", + "cfl-architectural-soundness-notion-2", + "cfl-sustainability-notion-1", + "cfl-sustainability-notion-2", + "cfl-sustainability-notion-3" ]; - const notionIds = [ - "robustness-notion-1", - "robustness-notion-2", - "robustness-notion-3", - "privacy-notion-1", - "privacy-notion-2", - "privacy-notion-3", - "fairness-notion-1", - "fairness-notion-2", - "fairness-notion-3", - "explainability-notion-1", - "explainability-notion-2", - "architectural-soundness-notion-1", - "architectural-soundness-notion-2", - "sustainability-notion-1", - "sustainability-notion-2", - "sustainability-notion-3" + + // IDs DFL (AJUSTA si tu DFL tiene otras nociones) + const dflPillarIds = [ + "dfl-robustness-pillar", + "dfl-privacy-pillar", + "dfl-fairness-pillar", + "dfl-explainability-pillar", + "dfl-accountability-pillar", + "dfl-architectural-soundness-pillar", + "dfl-sustainability-pillar" ]; - - pillarIds.concat(notionIds).forEach(id => { + const dflNotionIds = [ + "dfl-robustness-notion-1", + "dfl-robustness-notion-2", + "dfl-robustness-notion-3", + "dfl-privacy-notion-1", + "dfl-privacy-notion-2", + "dfl-privacy-notion-3", + // DFL fairness reducido: + "dfl-fairness-notion-3", + "dfl-explainability-notion-1", + "dfl-explainability-notion-2", + "dfl-accountability-notion-1", + "dfl-architectural-soundness-notion-1", + "dfl-architectural-soundness-notion-2", + // DFL sustainability reducido: + "dfl-sustainability-notion-1", + "dfl-sustainability-notion-3" + ]; + + cflPillarIds.concat(cflNotionIds, dflPillarIds, dflNotionIds).forEach(id => { const input = document.getElementById(id); - if (input) { - input.addEventListener("input", validateWeights); - } + if (input) input.addEventListener("input", validateWeights); }); } - + function validateWeights() { - const robustnessPercent = parseFloat(document.getElementById("robustness-pillar").value) || 0; - const privacyPercent = parseFloat(document.getElementById("privacy-pillar").value) || 0; - const fairnessPercent = parseFloat(document.getElementById("fairness-pillar").value) || 0; - const explainabilityPercent = parseFloat(document.getElementById("explainability-pillar").value) || 0; - const accountabilityPercent = parseFloat(document.getElementById("accountability-pillar").value) || 0; - const architecturalSoundnessPercent = parseFloat(document.getElementById("architectural-soundness-pillar").value) || 0; - const sustainabilityPercent = parseFloat(document.getElementById("sustainability-pillar").value) || 0; - - const robustnessNotion1 = parseFloat(document.getElementById("robustness-notion-1").value) || 0; - const robustnessNotion2 = parseFloat(document.getElementById("robustness-notion-2").value) || 0; - const robustnessNotion3 = parseFloat(document.getElementById("robustness-notion-3").value) || 0; - const privacyNotion1 = parseFloat(document.getElementById("privacy-notion-1").value) || 0; - const privacyNotion2 = parseFloat(document.getElementById("privacy-notion-2").value) || 0; - const privacyNotion3 = parseFloat(document.getElementById("privacy-notion-3").value) || 0; - const fairnessNotion1 = parseFloat(document.getElementById("fairness-notion-1").value) || 0; - const fairnessNotion2 = parseFloat(document.getElementById("fairness-notion-2").value) || 0; - const fairnessNotion3 = parseFloat(document.getElementById("fairness-notion-3").value) || 0; - const explainabilityNotion1 = parseFloat(document.getElementById("explainability-notion-1").value) || 0; - const explainabilityNotion2 = parseFloat(document.getElementById("explainability-notion-2").value) || 0; - const architecturalSoundnessNotion1 = parseFloat(document.getElementById("architectural-soundness-notion-1").value) || 0; - const architecturalSoundnessNotion2 = parseFloat(document.getElementById("architectural-soundness-notion-2").value) || 0; - const sustainabilityNotion1 = parseFloat(document.getElementById("sustainability-notion-1").value) || 0; - const sustainabilityNotion2 = parseFloat(document.getElementById("sustainability-notion-2").value) || 0; - const sustainabilityNotion3 = parseFloat(document.getElementById("sustainability-notion-3").value) || 0; - + if (isDFL()) { + return validateWeightsDFL(); + } + return validateWeightsCFL(); + } + + function validateWeightsCFL() { + const robustnessPercent = parseFloat(document.getElementById("cfl-robustness-pillar").value) || 0; + const privacyPercent = parseFloat(document.getElementById("cfl-privacy-pillar").value) || 0; + const fairnessPercent = parseFloat(document.getElementById("cfl-fairness-pillar").value) || 0; + const explainabilityPercent = parseFloat(document.getElementById("cfl-explainability-pillar").value) || 0; + const accountabilityPercent = parseFloat(document.getElementById("cfl-accountability-pillar").value) || 0; + const architecturalSoundnessPercent = parseFloat(document.getElementById("cfl-architectural-soundness-pillar").value) || 0; + const sustainabilityPercent = parseFloat(document.getElementById("cfl-sustainability-pillar").value) || 0; + + const robustnessNotion1 = parseFloat(document.getElementById("cfl-robustness-notion-1").value) || 0; + const robustnessNotion2 = parseFloat(document.getElementById("cfl-robustness-notion-2").value) || 0; + const robustnessNotion3 = parseFloat(document.getElementById("cfl-robustness-notion-3").value) || 0; + + const privacyNotion1 = parseFloat(document.getElementById("cfl-privacy-notion-1").value) || 0; + const privacyNotion2 = parseFloat(document.getElementById("cfl-privacy-notion-2").value) || 0; + const privacyNotion3 = parseFloat(document.getElementById("cfl-privacy-notion-3").value) || 0; + + const fairnessNotion1 = parseFloat(document.getElementById("cfl-fairness-notion-1").value) || 0; + const fairnessNotion2 = parseFloat(document.getElementById("cfl-fairness-notion-2").value) || 0; + const fairnessNotion3 = parseFloat(document.getElementById("cfl-fairness-notion-3").value) || 0; + + const explainabilityNotion1 = parseFloat(document.getElementById("cfl-explainability-notion-1").value) || 0; + const explainabilityNotion2 = parseFloat(document.getElementById("cfl-explainability-notion-2").value) || 0; + + const architecturalSoundnessNotion1 = parseFloat(document.getElementById("cfl-architectural-soundness-notion-1").value) || 0; + const architecturalSoundnessNotion2 = parseFloat(document.getElementById("cfl-architectural-soundness-notion-2").value) || 0; + + const sustainabilityNotion1 = parseFloat(document.getElementById("cfl-sustainability-notion-1").value) || 0; + const sustainabilityNotion2 = parseFloat(document.getElementById("cfl-sustainability-notion-2").value) || 0; + const sustainabilityNotion3 = parseFloat(document.getElementById("cfl-sustainability-notion-3").value) || 0; + const totalPillar = - robustnessPercent + - privacyPercent + - fairnessPercent + - explainabilityPercent + - accountabilityPercent + - architecturalSoundnessPercent + - sustainabilityPercent; - + robustnessPercent + privacyPercent + fairnessPercent + explainabilityPercent + + accountabilityPercent + architecturalSoundnessPercent + sustainabilityPercent; + const totalRobustnessNotion = robustnessNotion1 + robustnessNotion2 + robustnessNotion3; const totalPrivacyNotion = privacyNotion1 + privacyNotion2 + privacyNotion3; const totalFairnessNotion = fairnessNotion1 + fairnessNotion2 + fairnessNotion3; const totalExplainabilityNotion = explainabilityNotion1 + explainabilityNotion2; const totalArchitecturalSoundnessNotion = architecturalSoundnessNotion1 + architecturalSoundnessNotion2; const totalSustainabilityNotion = sustainabilityNotion1 + sustainabilityNotion2 + sustainabilityNotion3; - - if (totalPillar !== 100) { - return "[Trustworthiness] Check pillars weights"; - } - if (totalRobustnessNotion !== 100) { - return "[Trustworthiness] Check robustness notions weights"; - } - if (totalPrivacyNotion !== 100) { - return "[Trustworthiness] Check privacy notions weights"; - } - if (totalFairnessNotion !== 100) { - return "[Trustworthiness] Check fairness notions weights"; - } - if (totalExplainabilityNotion !== 100) { - return "[Trustworthiness] Check explainability notions weights"; - } - if (totalArchitecturalSoundnessNotion !== 100) { - return "[Trustworthiness] Check architectural soundness notions weights"; - } - if (totalSustainabilityNotion !== 100) { - return "[Trustworthiness] Check sustainability notions weights"; - } + + if (totalPillar !== 100) return "[Trustworthiness] Check pillars weights"; + if (totalRobustnessNotion !== 100) return "[Trustworthiness] Check robustness notions weights"; + if (totalPrivacyNotion !== 100) return "[Trustworthiness] Check privacy notions weights"; + if (totalFairnessNotion !== 100) return "[Trustworthiness] Check fairness notions weights"; + if (totalExplainabilityNotion !== 100) return "[Trustworthiness] Check explainability notions weights"; + if (totalArchitecturalSoundnessNotion !== 100) return "[Trustworthiness] Check architectural soundness notions weights"; + if (totalSustainabilityNotion !== 100) return "[Trustworthiness] Check sustainability notions weights"; } - + + function validateWeightsDFL() { + const robustnessPercent = parseFloat(document.getElementById("dfl-robustness-pillar").value) || 0; + const privacyPercent = parseFloat(document.getElementById("dfl-privacy-pillar").value) || 0; + const fairnessPercent = parseFloat(document.getElementById("dfl-fairness-pillar").value) || 0; + const explainabilityPercent = parseFloat(document.getElementById("dfl-explainability-pillar").value) || 0; + const accountabilityPercent = parseFloat(document.getElementById("dfl-accountability-pillar").value) || 0; + const architecturalSoundnessPercent = parseFloat(document.getElementById("dfl-architectural-soundness-pillar").value) || 0; + const sustainabilityPercent = parseFloat(document.getElementById("dfl-sustainability-pillar").value) || 0; + + const robustnessNotion1 = parseFloat(document.getElementById("dfl-robustness-notion-1").value) || 0; + const robustnessNotion2 = parseFloat(document.getElementById("dfl-robustness-notion-2").value) || 0; + const robustnessNotion3 = parseFloat(document.getElementById("dfl-robustness-notion-3").value) || 0; + + const privacyNotion1 = parseFloat(document.getElementById("dfl-privacy-notion-1").value) || 0; + const privacyNotion2 = parseFloat(document.getElementById("dfl-privacy-notion-2").value) || 0; + const privacyNotion3 = parseFloat(document.getElementById("dfl-privacy-notion-3").value) || 0; + + // DFL fairness reducido (AJUSTA si corresponde) + const fairnessNotion3 = parseFloat(document.getElementById("dfl-fairness-notion-3").value) || 0; + + const explainabilityNotion1 = parseFloat(document.getElementById("dfl-explainability-notion-1").value) || 0; + const explainabilityNotion2 = parseFloat(document.getElementById("dfl-explainability-notion-2").value) || 0; + + const architecturalSoundnessNotion1 = parseFloat(document.getElementById("dfl-architectural-soundness-notion-1").value) || 0; + const architecturalSoundnessNotion2 = parseFloat(document.getElementById("dfl-architectural-soundness-notion-2").value) || 0; + + // DFL sustainability reducido (AJUSTA si corresponde) + const sustainabilityNotion1 = parseFloat(document.getElementById("dfl-sustainability-notion-1").value) || 0; + const sustainabilityNotion3 = parseFloat(document.getElementById("dfl-sustainability-notion-3").value) || 0; + + const totalPillar = + robustnessPercent + privacyPercent + fairnessPercent + explainabilityPercent + + accountabilityPercent + architecturalSoundnessPercent + sustainabilityPercent; + + const totalRobustnessNotion = robustnessNotion1 + robustnessNotion2 + robustnessNotion3; + const totalPrivacyNotion = privacyNotion1 + privacyNotion2 + privacyNotion3; + const totalFairnessNotion = fairnessNotion3; + const totalExplainabilityNotion = explainabilityNotion1 + explainabilityNotion2; + const totalArchitecturalSoundnessNotion = architecturalSoundnessNotion1 + architecturalSoundnessNotion2; + const totalSustainabilityNotion = sustainabilityNotion1 + sustainabilityNotion3; + + if (totalPillar !== 100) return "[Trustworthiness] Check pillars weights"; + if (totalRobustnessNotion !== 100) return "[Trustworthiness] Check robustness notions weights"; + if (totalPrivacyNotion !== 100) return "[Trustworthiness] Check privacy notions weights"; + if (totalFairnessNotion !== 100) return "[Trustworthiness] Check fairness notions weights"; + if (totalExplainabilityNotion !== 100) return "[Trustworthiness] Check explainability notions weights"; + if (totalArchitecturalSoundnessNotion !== 100) return "[Trustworthiness] Check architectural soundness notions weights"; + if (totalSustainabilityNotion !== 100) return "[Trustworthiness] Check sustainability notions weights"; + } + function getTrustworthinessConfig() { const enabled = document.getElementById("trustworthiness-options").style.display === "block"; const federationArchitecture = document.getElementById("federationArchitecture").value; - + + if (isDFL()) return getTrustworthinessConfigDFL(enabled, federationArchitecture); + return getTrustworthinessConfigCFL(enabled, federationArchitecture); + } + + function getTrustworthinessConfigCFL(enabled, federationArchitecture) { const pillars = { - robustness: parseFloat(document.getElementById("robustness-pillar").value) || 0, - privacy: parseFloat(document.getElementById("privacy-pillar").value) || 0, - fairness: parseFloat(document.getElementById("fairness-pillar").value) || 0, - explainability: parseFloat(document.getElementById("explainability-pillar").value) || 0, - accountability: parseFloat(document.getElementById("accountability-pillar").value) || 0, - architecturalSoundness: parseFloat(document.getElementById("architectural-soundness-pillar").value) || 0, - sustainability: parseFloat(document.getElementById("sustainability-pillar").value) || 0 + robustness: parseFloat(document.getElementById("cfl-robustness-pillar").value) || 0, + privacy: parseFloat(document.getElementById("cfl-privacy-pillar").value) || 0, + fairness: parseFloat(document.getElementById("cfl-fairness-pillar").value) || 0, + explainability: parseFloat(document.getElementById("cfl-explainability-pillar").value) || 0, + accountability: parseFloat(document.getElementById("cfl-accountability-pillar").value) || 0, + architecturalSoundness: parseFloat(document.getElementById("cfl-architectural-soundness-pillar").value) || 0, + sustainability: parseFloat(document.getElementById("cfl-sustainability-pillar").value) || 0 }; - + const notions = { robustness: [ - parseFloat(document.getElementById("robustness-notion-1").value) || 0, - parseFloat(document.getElementById("robustness-notion-2").value) || 0, - parseFloat(document.getElementById("robustness-notion-3").value) || 0 + parseFloat(document.getElementById("cfl-robustness-notion-1").value) || 0, + parseFloat(document.getElementById("cfl-robustness-notion-2").value) || 0, + parseFloat(document.getElementById("cfl-robustness-notion-3").value) || 0 ], privacy: [ - parseFloat(document.getElementById("privacy-notion-1").value) || 0, - parseFloat(document.getElementById("privacy-notion-2").value) || 0, - parseFloat(document.getElementById("privacy-notion-3").value) || 0 + parseFloat(document.getElementById("cfl-privacy-notion-1").value) || 0, + parseFloat(document.getElementById("cfl-privacy-notion-2").value) || 0, + parseFloat(document.getElementById("cfl-privacy-notion-3").value) || 0 ], fairness: [ - parseFloat(document.getElementById("fairness-notion-1").value) || 0, - parseFloat(document.getElementById("fairness-notion-2").value) || 0, - parseFloat(document.getElementById("fairness-notion-3").value) || 0 + parseFloat(document.getElementById("cfl-fairness-notion-1").value) || 0, + parseFloat(document.getElementById("cfl-fairness-notion-2").value) || 0, + parseFloat(document.getElementById("cfl-fairness-notion-3").value) || 0 ], explainability: [ - parseFloat(document.getElementById("explainability-notion-1").value) || 0, - parseFloat(document.getElementById("explainability-notion-2").value) || 0 + parseFloat(document.getElementById("cfl-explainability-notion-1").value) || 0, + parseFloat(document.getElementById("cfl-explainability-notion-2").value) || 0 + ], + accountability: [ + parseFloat(document.getElementById("cfl-accountability-notion-1")?.value) || 100 ], architecturalSoundness: [ - parseFloat(document.getElementById("architectural-soundness-notion-1").value) || 0, - parseFloat(document.getElementById("architectural-soundness-notion-2").value) || 0 + parseFloat(document.getElementById("cfl-architectural-soundness-notion-1").value) || 0, + parseFloat(document.getElementById("cfl-architectural-soundness-notion-2").value) || 0 ], sustainability: [ - parseFloat(document.getElementById("sustainability-notion-1").value) || 0, - parseFloat(document.getElementById("sustainability-notion-2").value) || 0, - parseFloat(document.getElementById("sustainability-notion-3").value) || 0 + parseFloat(document.getElementById("cfl-sustainability-notion-1").value) || 0, + parseFloat(document.getElementById("cfl-sustainability-notion-2").value) || 0, + parseFloat(document.getElementById("cfl-sustainability-notion-3").value) || 0 ] }; - - return { - enabled, - federationArchitecture, - pillars, - notions + + return { enabled, federationArchitecture, pillars, notions }; + } + + function getTrustworthinessConfigDFL(enabled, federationArchitecture) { + const pillars = { + robustness: parseFloat(document.getElementById("dfl-robustness-pillar").value) || 0, + privacy: parseFloat(document.getElementById("dfl-privacy-pillar").value) || 0, + fairness: parseFloat(document.getElementById("dfl-fairness-pillar").value) || 0, + explainability: parseFloat(document.getElementById("dfl-explainability-pillar").value) || 0, + accountability: parseFloat(document.getElementById("dfl-accountability-pillar").value) || 0, + architecturalSoundness: parseFloat(document.getElementById("dfl-architectural-soundness-pillar").value) || 0, + sustainability: parseFloat(document.getElementById("dfl-sustainability-pillar").value) || 0 + }; + + const notions = { + robustness: [ + parseFloat(document.getElementById("dfl-robustness-notion-1").value) || 0, + parseFloat(document.getElementById("dfl-robustness-notion-2").value) || 0, + parseFloat(document.getElementById("dfl-robustness-notion-3").value) || 0 + ], + privacy: [ + parseFloat(document.getElementById("dfl-privacy-notion-1").value) || 0, + parseFloat(document.getElementById("dfl-privacy-notion-2").value) || 0, + parseFloat(document.getElementById("dfl-privacy-notion-3").value) || 0 + ], + // DFL fairness reducido (AJUSTA si corresponde) + fairness: [ + parseFloat(document.getElementById("dfl-fairness-notion-3").value) || 0 + ], + explainability: [ + parseFloat(document.getElementById("dfl-explainability-notion-1").value) || 0, + parseFloat(document.getElementById("dfl-explainability-notion-2").value) || 0 + ], + accountability: [ + parseFloat(document.getElementById("dfl-accountability-notion-1")?.value) || 100 + ], + architecturalSoundness: [ + parseFloat(document.getElementById("dfl-architectural-soundness-notion-1").value) || 0, + parseFloat(document.getElementById("dfl-architectural-soundness-notion-2").value) || 0 + ], + // DFL sustainability reducido (AJUSTA si corresponde) + sustainability: [ + parseFloat(document.getElementById("dfl-sustainability-notion-1").value) || 0, + parseFloat(document.getElementById("dfl-sustainability-notion-3").value) || 0 + ] }; + + return { enabled, federationArchitecture, pillars, notions }; } - + function setTrustworthinessConfig(config) { if (!config) return; - - // Set pillar weights + + if (isDFL()) setTrustworthinessConfigDFL(config); + else setTrustworthinessConfigCFL(config); + + validateWeights(); + } + + function setTrustworthinessConfigCFL(config) { if (config.pillars) { - document.getElementById("robustness-pillar").value = config.pillars.robustness || 0; - document.getElementById("privacy-pillar").value = config.pillars.privacy || 0; - document.getElementById("fairness-pillar").value = config.pillars.fairness || 0; - document.getElementById("explainability-pillar").value = config.pillars.explainability || 0; - document.getElementById("accountability-pillar").value = config.pillars.accountability || 0; - document.getElementById("architectural-soundness-pillar").value = config.pillars.architecturalSoundness || 0; - document.getElementById("sustainability-pillar").value = config.pillars.sustainability || 0; + document.getElementById("cfl-robustness-pillar").value = config.pillars.robustness || 0; + document.getElementById("cfl-privacy-pillar").value = config.pillars.privacy || 0; + document.getElementById("cfl-fairness-pillar").value = config.pillars.fairness || 0; + document.getElementById("cfl-explainability-pillar").value = config.pillars.explainability || 0; + document.getElementById("cfl-accountability-pillar").value = config.pillars.accountability || 0; + document.getElementById("cfl-architectural-soundness-pillar").value = config.pillars.architecturalSoundness || 0; + document.getElementById("cfl-sustainability-pillar").value = config.pillars.sustainability || 0; } - - // Set notion weights + if (config.notions) { - const rNotions = config.notions.robustness || [0, 0, 0]; - document.getElementById("robustness-notion-1").value = rNotions[0]; - document.getElementById("robustness-notion-2").value = rNotions[1]; - document.getElementById("robustness-notion-3").value = rNotions[2]; - - const pNotions = config.notions.privacy || [0, 0, 0]; - document.getElementById("privacy-notion-1").value = pNotions[0]; - document.getElementById("privacy-notion-2").value = pNotions[1]; - document.getElementById("privacy-notion-3").value = pNotions[2]; - - const fNotions = config.notions.fairness || [0, 0, 0]; - document.getElementById("fairness-notion-1").value = fNotions[0]; - document.getElementById("fairness-notion-2").value = fNotions[1]; - document.getElementById("fairness-notion-3").value = fNotions[2]; - - const eNotions = config.notions.explainability || [0, 0]; - document.getElementById("explainability-notion-1").value = eNotions[0]; - document.getElementById("explainability-notion-2").value = eNotions[1]; - - const aNotions = config.notions.architecturalSoundness || [0, 0]; - document.getElementById("architectural-soundness-notion-1").value = aNotions[0]; - document.getElementById("architectural-soundness-notion-2").value = aNotions[1]; - - const sNotions = config.notions.sustainability || [0, 0, 0]; - document.getElementById("sustainability-notion-1").value = sNotions[0]; - document.getElementById("sustainability-notion-2").value = sNotions[1]; - document.getElementById("sustainability-notion-3").value = sNotions[2]; + const r = config.notions.robustness || [0, 0, 0]; + document.getElementById("cfl-robustness-notion-1").value = r[0]; + document.getElementById("cfl-robustness-notion-2").value = r[1]; + document.getElementById("cfl-robustness-notion-3").value = r[2]; + + const p = config.notions.privacy || [0, 0, 0]; + document.getElementById("cfl-privacy-notion-1").value = p[0]; + document.getElementById("cfl-privacy-notion-2").value = p[1]; + document.getElementById("cfl-privacy-notion-3").value = p[2]; + + const f = config.notions.fairness || [0, 0, 0]; + document.getElementById("cfl-fairness-notion-1").value = f[0]; + document.getElementById("cfl-fairness-notion-2").value = f[1]; + document.getElementById("cfl-fairness-notion-3").value = f[2]; + + const e = config.notions.explainability || [0, 0]; + document.getElementById("cfl-explainability-notion-1").value = e[0]; + document.getElementById("cfl-explainability-notion-2").value = e[1]; + + const a = config.notions.architecturalSoundness || [0, 0]; + document.getElementById("cfl-architectural-soundness-notion-1").value = a[0]; + document.getElementById("cfl-architectural-soundness-notion-2").value = a[1]; + + const s = config.notions.sustainability || [0, 0, 0]; + document.getElementById("cfl-sustainability-notion-1").value = s[0]; + document.getElementById("cfl-sustainability-notion-2").value = s[1]; + document.getElementById("cfl-sustainability-notion-3").value = s[2]; } - - // Perform a weight validation check to update any warnings if needed - validateWeights(); } - + + function setTrustworthinessConfigDFL(config) { + if (config.pillars) { + document.getElementById("dfl-robustness-pillar").value = config.pillars.robustness || 0; + document.getElementById("dfl-privacy-pillar").value = config.pillars.privacy || 0; + document.getElementById("dfl-fairness-pillar").value = config.pillars.fairness || 0; + document.getElementById("dfl-explainability-pillar").value = config.pillars.explainability || 0; + document.getElementById("dfl-accountability-pillar").value = config.pillars.accountability || 0; + document.getElementById("dfl-architectural-soundness-pillar").value = config.pillars.architecturalSoundness || 0; + document.getElementById("dfl-sustainability-pillar").value = config.pillars.sustainability || 0; + } + + if (config.notions) { + const r = config.notions.robustness || [0, 0, 0]; + document.getElementById("dfl-robustness-notion-1").value = r[0]; + document.getElementById("dfl-robustness-notion-2").value = r[1]; + document.getElementById("dfl-robustness-notion-3").value = r[2]; + + const p = config.notions.privacy || [0, 0, 0]; + document.getElementById("dfl-privacy-notion-1").value = p[0]; + document.getElementById("dfl-privacy-notion-2").value = p[1]; + document.getElementById("dfl-privacy-notion-3").value = p[2]; + + // DFL fairness reducido (AJUSTA si corresponde) + const f = config.notions.fairness || [0]; + document.getElementById("dfl-fairness-notion-3").value = f[0]; + + const e = config.notions.explainability || [0, 0]; + document.getElementById("dfl-explainability-notion-1").value = e[0]; + document.getElementById("dfl-explainability-notion-2").value = e[1]; + + const a = config.notions.architecturalSoundness || [0, 0]; + document.getElementById("dfl-architectural-soundness-notion-1").value = a[0]; + document.getElementById("dfl-architectural-soundness-notion-2").value = a[1]; + + // DFL sustainability reducido (AJUSTA si corresponde) + const s = config.notions.sustainability || [0, 0]; + document.getElementById("dfl-sustainability-notion-1").value = s[0]; + document.getElementById("dfl-sustainability-notion-3").value = s[1]; + } + } + function resetTrustworthinessConfig() { const trustworthinessOptionsDiv = document.getElementById("trustworthiness-options"); const fedArchElement = document.getElementById("federationArchitecture"); - - // Hide options and re-enable federationArchitecture + trustworthinessOptionsDiv.style.display = "none"; fedArchElement.disabled = false; - - // Reset pillars to 0 - document.getElementById("robustness-pillar").value = "0"; - document.getElementById("privacy-pillar").value = "0"; - document.getElementById("fairness-pillar").value = "0"; - document.getElementById("explainability-pillar").value = "0"; - document.getElementById("accountability-pillar").value = "0"; - document.getElementById("architectural-soundness-pillar").value = "0"; - document.getElementById("sustainability-pillar").value = "0"; - - // Reset notions to 0 - document.getElementById("robustness-notion-1").value = "0"; - document.getElementById("robustness-notion-2").value = "0"; - document.getElementById("robustness-notion-3").value = "0"; - document.getElementById("privacy-notion-1").value = "0"; - document.getElementById("privacy-notion-2").value = "0"; - document.getElementById("privacy-notion-3").value = "0"; - document.getElementById("fairness-notion-1").value = "0"; - document.getElementById("fairness-notion-2").value = "0"; - document.getElementById("fairness-notion-3").value = "0"; - document.getElementById("explainability-notion-1").value = "0"; - document.getElementById("explainability-notion-2").value = "0"; - document.getElementById("architectural-soundness-notion-1").value = "0"; - document.getElementById("architectural-soundness-notion-2").value = "0"; - document.getElementById("sustainability-notion-1").value = "0"; - document.getElementById("sustainability-notion-2").value = "0"; - document.getElementById("sustainability-notion-3").value = "0"; - - // Re-validate weights after reset + + if (isDFL()) resetTrustworthinessConfigDFL(); + else resetTrustworthinessConfigCFL(); + validateWeights(); } - + + function resetTrustworthinessConfigCFL() { + document.getElementById("cfl-robustness-pillar").value = "0"; + document.getElementById("cfl-privacy-pillar").value = "0"; + document.getElementById("cfl-fairness-pillar").value = "0"; + document.getElementById("cfl-explainability-pillar").value = "0"; + document.getElementById("cfl-accountability-pillar").value = "0"; + document.getElementById("cfl-architectural-soundness-pillar").value = "0"; + document.getElementById("cfl-sustainability-pillar").value = "0"; + + document.getElementById("cfl-robustness-notion-1").value = "0"; + document.getElementById("cfl-robustness-notion-2").value = "0"; + document.getElementById("cfl-robustness-notion-3").value = "0"; + + document.getElementById("cfl-privacy-notion-1").value = "0"; + document.getElementById("cfl-privacy-notion-2").value = "0"; + document.getElementById("cfl-privacy-notion-3").value = "0"; + + document.getElementById("cfl-fairness-notion-1").value = "0"; + document.getElementById("cfl-fairness-notion-2").value = "0"; + document.getElementById("cfl-fairness-notion-3").value = "0"; + + document.getElementById("cfl-explainability-notion-1").value = "0"; + document.getElementById("cfl-explainability-notion-2").value = "0"; + + document.getElementById("cfl-architectural-soundness-notion-1").value = "0"; + document.getElementById("cfl-architectural-soundness-notion-2").value = "0"; + + document.getElementById("cfl-sustainability-notion-1").value = "0"; + document.getElementById("cfl-sustainability-notion-2").value = "0"; + document.getElementById("cfl-sustainability-notion-3").value = "0"; + } + + function resetTrustworthinessConfigDFL() { + document.getElementById("dfl-robustness-pillar").value = "0"; + document.getElementById("dfl-privacy-pillar").value = "0"; + document.getElementById("dfl-fairness-pillar").value = "0"; + document.getElementById("dfl-explainability-pillar").value = "0"; + document.getElementById("dfl-accountability-pillar").value = "0"; + document.getElementById("dfl-architectural-soundness-pillar").value = "0"; + document.getElementById("dfl-sustainability-pillar").value = "0"; + + document.getElementById("dfl-robustness-notion-1").value = "0"; + document.getElementById("dfl-robustness-notion-2").value = "0"; + document.getElementById("dfl-robustness-notion-3").value = "0"; + + document.getElementById("dfl-privacy-notion-1").value = "0"; + document.getElementById("dfl-privacy-notion-2").value = "0"; + document.getElementById("dfl-privacy-notion-3").value = "0"; + + // DFL fairness reducido (AJUSTA si corresponde) + document.getElementById("dfl-fairness-notion-3").value = "0"; + + document.getElementById("dfl-explainability-notion-1").value = "0"; + document.getElementById("dfl-explainability-notion-2").value = "0"; + + document.getElementById("dfl-architectural-soundness-notion-1").value = "0"; + document.getElementById("dfl-architectural-soundness-notion-2").value = "0"; + + // DFL sustainability reducido (AJUSTA si corresponde) + document.getElementById("dfl-sustainability-notion-1").value = "0"; + document.getElementById("dfl-sustainability-notion-3").value = "0"; + } + return { initializeTrustworthinessSystem, getTrustworthinessConfig, @@ -271,5 +504,5 @@ const TrustworthinessManager = (function() { resetTrustworthinessConfig }; })(); - -export default TrustworthinessManager; \ No newline at end of file + +export default TrustworthinessManager; diff --git a/nebula/frontend/templates/deployment.html b/nebula/frontend/templates/deployment.html index 8572403ff..18d2f2e42 100755 --- a/nebula/frontend/templates/deployment.html +++ b/nebula/frontend/templates/deployment.html @@ -143,6 +143,18 @@
    Dataset FashionMNIST + + +
    - Epsilon and bounds use the dataset input scale; image datasets convert pixel scale to normalized tensors. + Image datasets use FGSM/PGD. AdultCensus uses CAA for tabular adversarial training. From ed8e6088869c2d8d919903ff207c3a76875438c3 Mon Sep 17 00:00:00 2001 From: "Juan J." Date: Fri, 29 May 2026 13:28:15 +0200 Subject: [PATCH 54/66] Refactor: Feature Squeezing, factsheets, graphics and trustworthiness --- nebula/addons/defenses/feature_squeezing.py | 23 +- .../addons/trustworthiness/dfl_factsheet.py | 48 +-- .../trustworthiness/factsheet_populators.py | 38 +- nebula/addons/trustworthiness/graphics.py | 376 ++++++++++-------- .../addons/trustworthiness/trustworthiness.py | 345 +++++++++------- 5 files changed, 481 insertions(+), 349 deletions(-) diff --git a/nebula/addons/defenses/feature_squeezing.py b/nebula/addons/defenses/feature_squeezing.py index 6ff592d12..683cb9cce 100644 --- a/nebula/addons/defenses/feature_squeezing.py +++ b/nebula/addons/defenses/feature_squeezing.py @@ -6,7 +6,6 @@ import torch from PIL import Image -IMAGE_DATASETS = {"MNIST", "FashionMNIST", "EMNIST", "CIFAR10", "CIFAR100"} PIL_IMAGE_MODES = {"1", "L", "P", "RGB", "RGBA", "CMYK", "YCbCr"} @@ -34,6 +33,7 @@ class FeatureSqueezingDefense: """Dataset-level feature squeezing for image Nebula datasets.""" def __init__(self, config: FeatureSqueezingConfig): + # Validate the number of quantization levels requested by the scenario. if not isinstance(config.bit_depth, int) or not 1 <= config.bit_depth <= 64: raise ValueError("feature_squeezing.bit_depth must be an integer in [1, 64]") @@ -42,6 +42,7 @@ def __init__(self, config: FeatureSqueezingConfig): @classmethod def from_participant_config(cls, participant_config: dict[str, Any]) -> "FeatureSqueezingDefense | None": + # Build the defense only when feature squeezing is enabled in the participant config. raw = participant_config.get("defense_args", {}).get("feature_squeezing", {}) if not raw or not raw.get("enabled", False): return None @@ -58,18 +59,12 @@ def from_participant_config(cls, participant_config: dict[str, Any]) -> "Feature ) def apply_to_partition(self, partition) -> None: + # Apply the defense to each enabled split in the participant partition. train_set = getattr(partition, "train_set", None) if train_set is None: logging.warning("[FeatureSqueezingDefense] No train set found; skipping defense") return - if self.config.dataset_name not in IMAGE_DATASETS: - logging.info( - "[FeatureSqueezingDefense] Skipping feature squeezing: dataset is not image-supported | dataset=%s", - self.config.dataset_name, - ) - return - logging.info( "[FeatureSqueezingDefense] Applying feature squeezing | dataset=%s | bit_depth=%s", self.config.dataset_name, @@ -86,6 +81,7 @@ def apply_to_partition(self, partition) -> None: self._transform_dataset(dataset, name, seen_data) def _transform_dataset(self, dataset, name: str, seen_data: set[int]) -> None: + # Transform all samples in one dataset split, avoiding duplicated shared data. data = getattr(dataset, "data", None) if dataset is None or data is None: return @@ -105,6 +101,7 @@ def _transform_dataset(self, dataset, name: str, seen_data: set[int]) -> None: self._log_check(data, name, status="transformed", before=before) def _transform_sample(self, sample): + # Transform only the input image and keep labels or metadata unchanged. if isinstance(sample, tuple) and sample: return (self._squeeze_image(sample[0]), *sample[1:]) return self._squeeze_image(sample) @@ -114,6 +111,7 @@ def _transform_sample(self, sample): # ------------------------------------------------------------------ def _squeeze_image(self, value): + # Quantize PIL images, tensors, and arrays while preserving the original container type. if isinstance(value, Image.Image): image = value if value.mode in PIL_IMAGE_MODES else value.convert("RGB") arr = np.asarray(image) @@ -124,6 +122,7 @@ def _squeeze_image(self, value): return self._restore_type(value, squeezed) def _squeeze_image_array(self, arr: np.ndarray) -> np.ndarray: + # Normalize values to [0, 1], quantize them, and map them back to the original range. arr_float = arr.astype(np.float32, copy=False) if np.issubdtype(arr.dtype, np.integer): info = np.iinfo(arr.dtype) @@ -143,9 +142,11 @@ def _squeeze_image_array(self, arr: np.ndarray) -> np.ndarray: # ------------------------------------------------------------------ def _quantize01(self, arr: np.ndarray) -> np.ndarray: + # Reduce normalized values to the discrete levels defined by bit_depth. return np.rint(np.clip(arr, 0.0, 1.0) * self.levels) / self.levels def _log_check(self, data, name: str, status: str, before: str | None = None) -> None: + # Log a compact before/after summary to verify that squeezing was applied. if not len(data): logging.info("[FeatureSqueezingDefense] Verification %s | status=%s | empty dataset", name, status) return @@ -174,6 +175,7 @@ def _log_check(self, data, name: str, status: str, before: str | None = None) -> ) def _summary(self, sample) -> str: + # Create a short numeric summary of one sample for diagnostics. arr = self._as_numpy(self._unwrap(sample)) if arr.size == 0: return f"shape={arr.shape}, empty=True" @@ -187,6 +189,7 @@ def _summary(self, sample) -> str: ) def _as_numpy(self, value) -> np.ndarray: + # Convert supported image containers to numpy for quantization and logging. if isinstance(value, torch.Tensor): return value.detach().cpu().numpy() if isinstance(value, Image.Image): @@ -194,6 +197,7 @@ def _as_numpy(self, value) -> np.ndarray: return np.asarray(value) def _restore_type(self, original, arr: np.ndarray): + # Return squeezed data with the same high-level type as the original sample. if isinstance(original, torch.Tensor): return torch.as_tensor(arr, dtype=original.dtype, device=original.device) if isinstance(original, np.ndarray): @@ -201,9 +205,11 @@ def _restore_type(self, original, arr: np.ndarray): return arr def _unwrap(self, sample): + # Extract the image from common dataset samples shaped as (image, label, ...). return sample[0] if isinstance(sample, tuple) and sample else sample def _fmt(self, value) -> str: + # Format numbers in logs without unnecessary trailing decimals. try: number = float(value) except (TypeError, ValueError): @@ -212,6 +218,7 @@ def _fmt(self, value) -> str: def apply_feature_squeezing_if_enabled(partition, participant_config: dict[str, Any]) -> None: + # Public entrypoint used by the node startup flow. defense = FeatureSqueezingDefense.from_participant_config(participant_config) if defense is not None: defense.apply_to_partition(partition) diff --git a/nebula/addons/trustworthiness/dfl_factsheet.py b/nebula/addons/trustworthiness/dfl_factsheet.py index f4c78d4aa..3f32e8b9e 100644 --- a/nebula/addons/trustworthiness/dfl_factsheet.py +++ b/nebula/addons/trustworthiness/dfl_factsheet.py @@ -74,8 +74,6 @@ def populate_factsheet_dfl( files_dir = get_trustworthiness_dir(scenario_name) - emissions_file = os.path.join(files_dir, f"emissions_{participant_idx}.csv") - get_all_data_entropy(scenario_name) factsheet["data"]["entropy_local"] = get_local_normalized_entropy(scenario_name, participant_idx) @@ -90,7 +88,7 @@ def populate_factsheet_dfl( factsheet["performance"]["test_loss"] = float(final_loss) factsheet["performance"]["test_acc"] = float(final_acc) - bytes_sent, bytes_recv = get_bytes(scenario_name, participant_idx) + bytes_sent, bytes_recv, *_ = load_data_results_participant(scenario_name, participant_idx) factsheet["system"]["model_size"] = get_bytes_model(model) @@ -110,8 +108,19 @@ def populate_factsheet_dfl( populate_participation(factsheet, participation_summary) - carbon_intensity_local, emissions_training_local, energy_consumed_local, sample_size = get_emissions( - emissions_file, + ( + role, + carbon_intensity_local, + emissions_training_local, + workload, + cpu_model, + gpu_model, + cpu_used, + gpu_used, + energy_consumed_local, + sample_size, + ) = load_emissions_participant( + scenario_name, participant_idx, ) @@ -155,32 +164,3 @@ def load_round_metrics(scenario_name, participant_idx): df = df.dropna(subset=["loss", "accuracy"]) return df - - -def get_bytes(scenario_name, participant_idx): - data_file = os.path.join( - get_trustworthiness_dir(scenario_name), - f"data_results_{participant_idx}.csv", - ) - - data = read_csv(data_file) - - row = data[data["id"] == participant_idx] - - bytes_sent = row["bytes_sent"].iloc[0] - bytes_recv = row["bytes_recv"].iloc[0] - - return bytes_sent, bytes_recv - - -def get_emissions(emissions_file, participant_idx): - data = read_csv(emissions_file) - - row = data[data["id"] == participant_idx] - - avg_carbon_intensity_clients = row["energy_grid"].iloc[0] - emissions_training = row["emissions"].iloc[0] - energy_consumed = row["energy_consumed"].iloc[0] - sample_size = row["sample_size"].iloc[0] - - return avg_carbon_intensity_clients, emissions_training, energy_consumed, sample_size diff --git a/nebula/addons/trustworthiness/factsheet_populators.py b/nebula/addons/trustworthiness/factsheet_populators.py index 1fa8ea6f8..d5cce3371 100644 --- a/nebula/addons/trustworthiness/factsheet_populators.py +++ b/nebula/addons/trustworthiness/factsheet_populators.py @@ -34,6 +34,7 @@ def get_federation_profile(federation): + # Group SDFL with DFL because both use decentralized factsheet profiles. return FEDERATION_DFL if str(federation).upper() in {"DFL", "SDFL"} else FEDERATION_CFL @@ -45,9 +46,10 @@ def populate_profile_metrics( test_loader, test_accuracy, ): + # Select the profile-specific populator, falling back to the shared metric set. federation_profile = get_federation_profile(federation) data_type = get_normalized_model_data_type(model) - populator = PROFILE_POPULATORS.get((federation_profile, data_type), populate_default_metrics) + populator = PROFILE_POPULATORS.get((federation_profile, data_type), populate_common_profile_metrics) populator( factsheet=factsheet, @@ -59,23 +61,28 @@ def populate_profile_metrics( def populate_cfl_images_metrics(factsheet, model, train_loader, test_loader, test_accuracy): - populate_default_metrics(factsheet, model, train_loader, test_loader, test_accuracy) + # Populate the current shared metrics for CFL image factsheets. + populate_common_profile_metrics(factsheet, model, train_loader, test_loader, test_accuracy) def populate_cfl_tabular_metrics(factsheet, model, train_loader, test_loader, test_accuracy): - populate_default_metrics(factsheet, model, train_loader, test_loader, test_accuracy) + # Populate the current shared metrics for CFL tabular factsheets. + populate_common_profile_metrics(factsheet, model, train_loader, test_loader, test_accuracy) def populate_dfl_images_metrics(factsheet, model, train_loader, test_loader, test_accuracy): - populate_default_metrics(factsheet, model, train_loader, test_loader, test_accuracy) + # Populate the current shared metrics for DFL/SDFL image factsheets. + populate_common_profile_metrics(factsheet, model, train_loader, test_loader, test_accuracy) def populate_dfl_tabular_metrics(factsheet, model, train_loader, test_loader, test_accuracy): - populate_default_metrics(factsheet, model, train_loader, test_loader, test_accuracy) + # Populate the current shared metrics for DFL/SDFL tabular factsheets. + populate_common_profile_metrics(factsheet, model, train_loader, test_loader, test_accuracy) -def populate_default_metrics(factsheet, model, train_loader, test_loader, test_accuracy): - """Current shared metric set used by every factsheet profile.""" +def populate_common_profile_metrics(factsheet, model, train_loader, test_loader, test_accuracy): + # Current shared metric set used by every factsheet profile. + # Reuse one test batch for sample-based metrics and compute summary explainability once. test_sample = next(iter(test_loader)) explainability_metrics = get_explainability_metrics_summary(model, test_loader) @@ -87,8 +94,8 @@ def populate_default_metrics(factsheet, model, train_loader, test_loader, test_a test_accuracy, test_sample, ) - populate_explainability_metrics(factsheet, explainability_metrics) - populate_image_robustness_metrics(factsheet, model, test_loader, test_sample) + populate_common_explainability_metrics(factsheet, explainability_metrics) + populate_common_robustness_metrics(factsheet, model, test_loader, test_sample) def populate_common_model_quality_metrics( @@ -99,13 +106,16 @@ def populate_common_model_quality_metrics( test_accuracy, test_sample, ): + # Populate model quality, privacy, and fairness metrics shared by all profiles. factsheet["performance"]["test_macro_f1"] = get_macro_f1_score(model, test_loader) + # Privacy metrics derived from train/test behavior. factsheet["privacy"]["epsilon_star"] = get_epsilon_star(model, train_loader, test_loader) factsheet["privacy"]["inverse_epsilon_star"] = inverse_score(factsheet["privacy"]["epsilon_star"]) factsheet["privacy"]["mia_auc"] = get_mia_auc(model, train_loader, test_loader) factsheet["privacy"]["mia_auc_score"] = 1 - 2 * abs(factsheet["privacy"]["mia_auc"] - 0.5) + # Fairness and calibration metrics expressed as inverse scores. overfitting_value = get_overfitting_score(model, train_loader, test_accuracy) factsheet["fairness"]["inverse_overfitting"] = inverse_score(overfitting_value) @@ -121,11 +131,13 @@ def populate_common_model_quality_metrics( coefficient_of_variation_value = get_coefficient_of_variation(model, test_loader) factsheet["fairness"]["inverse_coefficient_of_variation"] = inverse_score(coefficient_of_variation_value) + # Confidence is capped so factsheet scores stay within the expected range. value_confidence_score = get_confidence_score(model, test_sample) factsheet["performance"]["clipped_test_confidence_score"] = cap_score(value_confidence_score) -def populate_explainability_metrics(factsheet, explainability_metrics): +def populate_common_explainability_metrics(factsheet, explainability_metrics): + # Copy explainability summary metrics into the factsheet schema. factsheet["explainability"]["alpha_score"] = explainability_metrics["alpha_score"] factsheet["explainability"]["spread_ratio"] = explainability_metrics["spread_ratio"] factsheet["explainability"]["spread_divergence"] = explainability_metrics["spread_divergence"] @@ -134,16 +146,19 @@ def populate_explainability_metrics(factsheet, explainability_metrics): factsheet["performance"]["clipped_test_feature_importance_cv"] = cap_score(feature_importance) -def populate_image_robustness_metrics(factsheet, model, test_loader, test_sample): +def populate_common_robustness_metrics(factsheet, model, test_loader, test_sample): + # Populate adversarial robustness metrics shared by the current factsheet profiles. lr = factsheet["configuration"]["learning_rate"] num_classes = model.get_num_classes() + # Sample-based robustness scores. value_clever = get_clever_score(model, test_sample, num_classes, lr) factsheet["performance"]["clipped_test_clever"] = cap_score(value_clever) value_loss_sensitivity = get_loss_sensitivity_score(model, test_sample, num_classes, lr) factsheet["performance"]["inverse_test_loss_sensitivity"] = inverse_score(value_loss_sensitivity) + # Loader-based adversarial accuracy. value_adv_accuracy = compute_adversarial_accuracy_art(model, test_loader, num_classes, lr) factsheet["performance"]["clipped_test_adv_accuracy"] = cap_score(value_adv_accuracy) @@ -155,6 +170,7 @@ def populate_image_robustness_metrics(factsheet, model, test_loader, test_sample ) factsheet["performance"]["clipped_test_empirical_robustness"] = cap_score(value_empirical_robustness) + # Attack success is inverted so higher remains better in the factsheet. value_attack_success_rate = attack_success_rate( model, test_sample, diff --git a/nebula/addons/trustworthiness/graphics.py b/nebula/addons/trustworthiness/graphics.py index e2f339eb3..13743680e 100644 --- a/nebula/addons/trustworthiness/graphics.py +++ b/nebula/addons/trustworthiness/graphics.py @@ -1,163 +1,220 @@ -from abc import ABC +import json import logging -import torch import os -import pickle -import lightning as pl -from torchmetrics.classification import MulticlassAccuracy, MulticlassRecall, MulticlassPrecision, MulticlassF1Score, MulticlassConfusionMatrix -from torchmetrics import MetricCollection -import seaborn as sns + import matplotlib.pyplot as plt -import json import pandas as pd +import seaborn as sns from nebula.core.utils.nebulalogger_tensorboard import NebulaTensorBoardLogger + logging.basicConfig(level=logging.INFO) -class Graphics(): +PILLAR_CONFIGS = [ + ("robustness", "#F8D3DF", -0.4, (10, 6), "Robustness"), + ("privacy", "#DA8D8B", -0.2, (10, 6), "Privacy"), + ("fairness", "#DDDDDD", -0.4, (10, 6), "Fairness"), + ("explainability", "#FCEFC3", -0.4, (10, 6), "Explainability"), + ("accountability", "#8FAADC", -0.3, (10, 6), "Accountability"), + ("architectural_soundness", "#DBB9FA", -0.3, (10, 6), "Architectural Soundness"), + ("sustainability", "#BBFDAF", -0.5, (12, 8), "Sustainability"), +] +TRUST_SCORE_COLOR = "#BF9000" + + +class Graphics: def __init__( self, scenario_start_time, scenario_name, participant_id=None, ): + # Configure the TensorBoard logger used to store trustworthiness figures. self.scenario_start_time = scenario_start_time self.scenario_name = scenario_name log_dir = os.path.join(os.environ["NEBULA_LOGS_DIR"], scenario_name) - if participant_id==None: - self.nebulalogger = NebulaTensorBoardLogger(scenario_start_time, f"{log_dir}", name="metrics", version=f"trust", log_graph=True) - else: - self.nebulalogger = NebulaTensorBoardLogger(scenario_start_time, f"{log_dir}", name="metrics", version=f"trust_{participant_id}", log_graph=True) + version = "trust" if participant_id is None else f"trust_{participant_id}" + self.nebulalogger = NebulaTensorBoardLogger( + scenario_start_time, + f"{log_dir}", + name="metrics", + version=version, + log_graph=True, + ) + + def _trustworthiness_dir(self): + # Return the directory where trustworthiness JSON reports are stored. + return os.path.join(os.environ.get("NEBULA_LOGS_DIR"), self.scenario_name, "trustworthiness") + + def _trust_report_path(self, file_name): + # Build the absolute path for one trustworthiness report file. + return os.path.join(self._trustworthiness_dir(), file_name) - def __log_figure(self, df, pillar, color, tag_root, notion_y_pos = -0.4, figsize=(10,6)): - filtered_df = df[df['Pillar'] == pillar].copy() + def _load_trust_results(self, results_file): + # Load one trustworthiness JSON report from disk. + with open(results_file, "r") as f: + return json.load(f) - filtered_df.loc[:, 'Metric'] = filtered_df['Metric'].astype(str).str.replace('_', ' ') - filtered_df.loc[:, 'Metric'] = filtered_df['Metric'].apply(lambda x: str(x).title()) + def _log_report_from_file(self, results_file, tag_root, all_pillars_tag, label_suffix=""): + # Load a report and log all figures generated from it. + results = self._load_trust_results(results_file) + self._log_trust_report(results, tag_root, all_pillars_tag, label_suffix=label_suffix) - filtered_df.loc[:, 'Notion'] = filtered_df['Notion'].astype(str).str.replace('_', ' ') - filtered_df.loc[:, 'Notion'] = filtered_df['Notion'].apply(lambda x: str(x).title()) + def _format_report_dataframe(self, df, pillar): + # Keep one pillar and format metric/notion names for plot labels. + filtered_df = df[df["Pillar"] == pillar].copy() - unique_notion_count = filtered_df['Notion'].nunique() - palette = [color] * unique_notion_count + filtered_df.loc[:, "Metric"] = filtered_df["Metric"].astype(str).str.replace("_", " ") + filtered_df.loc[:, "Metric"] = filtered_df["Metric"].apply(lambda x: str(x).title()) - plt.figure(figsize=figsize) - ax = sns.barplot(data=filtered_df, x='Metric', y='Metric Score', hue='Notion', palette=palette, dodge=False) + filtered_df.loc[:, "Notion"] = filtered_df["Notion"].astype(str).str.replace("_", " ") + filtered_df.loc[:, "Notion"] = filtered_df["Notion"].apply(lambda x: str(x).title()) + return filtered_df + def _notion_ranges(self, filtered_df): + # Compute the x-axis range occupied by each notion in a pillar plot. + ranges = [] x_positions = range(len(filtered_df)) + seen_notions = set() - notion_scores = {} - - for i in range(len(filtered_df)): - row = filtered_df.iloc[i] - notion = row['Notion'] - notion_score = row['Notion Score'] - metric_score = row['Metric Score'] - - if notion not in notion_scores: - metrics_for_notion = filtered_df[filtered_df['Notion'] == notion]['Metric'] - start_pos = x_positions[i] - end_pos = x_positions[i + len(metrics_for_notion) - 1] - - notion_x_pos = (start_pos + end_pos) / 2 - ax.axhline(notion_score, ls='--', color='black', lw=0.5, xmin=start_pos/len(x_positions), xmax=(end_pos+1)/len(x_positions)) - ax.text(notion_x_pos, notion_score + 0.01, f"{notion_score:.2f}", ha='center', va='bottom', fontsize=10, color='black') # Color negro - notion_scores[notion] = notion_score + for i, notion in enumerate(filtered_df["Notion"]): + if notion in seen_notions: + continue + + metrics_for_notion = filtered_df[filtered_df["Notion"] == notion]["Metric"] + start_pos = x_positions[i] + end_pos = x_positions[i + len(metrics_for_notion) - 1] + notion_x_pos = (start_pos + end_pos) / 2 + + ranges.append((notion, start_pos, end_pos, notion_x_pos)) + seen_notions.add(notion) + + return ranges + + def _draw_notion_score_lines(self, ax, filtered_df): + # Draw dashed horizontal notion score lines over the metrics they group. + x_count = len(filtered_df) + if x_count == 0: + return + + for notion, start_pos, end_pos, notion_x_pos in self._notion_ranges(filtered_df): + notion_score = filtered_df[filtered_df["Notion"] == notion]["Notion Score"].iloc[0] + ax.axhline( + notion_score, + ls="--", + color="black", + lw=0.5, + xmin=start_pos / x_count, + xmax=(end_pos + 1) / x_count, + ) + ax.text( + notion_x_pos, + notion_score + 0.01, + f"{notion_score:.2f}", + ha="center", + va="bottom", + fontsize=10, + color="black", + ) - ax.set_xticks(x_positions) - ax.set_xticklabels(filtered_df['Metric'], rotation=45, ha='right', fontsize=10) + def _draw_notion_labels(self, ax, filtered_df, notion_y_pos): + # Add notion labels below the metric labels. + for notion, _, _, notion_x_pos in self._notion_ranges(filtered_df): + ax.text( + notion_x_pos, + notion_y_pos, + notion, + ha="center", + va="center", + fontsize=10, + color="black", + ) - seen_notions = set() - for i, (metric, notion) in enumerate(zip(filtered_df['Metric'], filtered_df['Notion'])): - if notion not in seen_notions: - metrics_for_notion = filtered_df[filtered_df['Notion'] == notion]['Metric'] - start_pos = x_positions[i] - end_pos = x_positions[i + len(metrics_for_notion) - 1] + def _draw_metric_score_labels(self, ax, filtered_df): + # Add numeric metric scores above each bar. + for i, value in enumerate(filtered_df["Metric Score"]): + ax.text(i, value + 0.01, f"{value:.2f}", ha="center", va="bottom", fontsize=10, color="black") - notion_x_pos = (start_pos + end_pos) / 2 + def _log_pillar_figure(self, df, pillar, color, tag_root, notion_y_pos=-0.4, figsize=(10, 6)): + # Generate and log the metric/notion bar chart for one pillar. + filtered_df = self._format_report_dataframe(df, pillar) + unique_notion_count = filtered_df["Notion"].nunique() + palette = [color] * unique_notion_count - ax.text(notion_x_pos, notion_y_pos, notion, ha='center', va='center', fontsize=10, color='black') + plt.figure(figsize=figsize) + ax = sns.barplot(data=filtered_df, x="Metric", y="Metric Score", hue="Notion", palette=palette, dodge=False) - seen_notions.add(notion) + x_positions = range(len(filtered_df)) + ax.set_xticks(x_positions) + ax.set_xticklabels(filtered_df["Metric"], rotation=45, ha="right", fontsize=10) - for i, v in enumerate(filtered_df['Metric Score']): - ax.text(i, v + 0.01, f"{v:.2f}", ha='center', va='bottom', fontsize=10, color='black') + self._draw_notion_score_lines(ax, filtered_df) + self._draw_notion_labels(ax, filtered_df, notion_y_pos) + self._draw_metric_score_labels(ax, filtered_df) - plt.xlabel('Metrics and notions', labelpad=35) - plt.ylabel('Score') - plt.title(f'Metrics and notion scores for the {pillar} pillar') + plt.xlabel("Metrics and notions", labelpad=35) + plt.ylabel("Score") + plt.title(f"Metrics and notion scores for the {pillar} pillar") - ax.legend_.remove() + if ax.legend_ is not None: + ax.legend_.remove() plt.tight_layout() self.nebulalogger.log_figure(ax.get_figure(), 0, f"{tag_root}/Pillar/{pillar}") plt.close() - def _load_trust_results(self, results_file): - with open(results_file, 'r') as f: - return json.load(f) - - def _log_trust_report(self, results, tag_root, all_pillars_tag, label_suffix=""): - pillars_list = [] - notion_names = [] - notion_scores = [] - metric_names = [] - metric_scores = [] - + def _trust_report_rows(self, results): + # Flatten the nested trust report into rows that pandas can plot. + rows = [] for pillar in results["pillars"]: - for key, value in pillar.items(): - pillar_name = key - if "notions" in value: - for notion in value["notions"]: - for notion_key, notion_value in notion.items(): - notion_name = notion_key - notion_score = notion_value["score"] - for metric in notion_value["metrics"]: - for metric_key, metric_value in metric.items(): - metric_name = metric_key - metric_score = metric_value["score"] - - pillars_list.append(pillar_name) - notion_names.append(notion_name) - notion_scores.append(notion_score) - metric_names.append(metric_name) - metric_scores.append(metric_score) - - df = pd.DataFrame({ - "Pillar": pillars_list, - "Notion": notion_names, - "Notion Score": notion_scores, - "Metric": metric_names, - "Metric Score": metric_scores - }) - - self.__log_figure(df, 'robustness', "#F8D3DF", tag_root) - self.__log_figure(df, "privacy", "#DA8D8B", tag_root, -0.2) - self.__log_figure(df, "fairness", "#DDDDDD", tag_root) - self.__log_figure(df, "explainability", "#FCEFC3", tag_root) - self.__log_figure(df, "accountability", "#8FAADC", tag_root, -0.3) - self.__log_figure(df, "architectural_soundness", "#DBB9FA", tag_root, -0.3) - self.__log_figure(df, "sustainability", "#BBFDAF", tag_root, -0.5, figsize=(12,8)) - - categories = [ - "robustness", - "privacy", - "fairness", - "explainability", - "accountability", - "architectural_soundness", - "sustainability" - ] - + for pillar_name, pillar_value in pillar.items(): + if "notions" not in pillar_value: + continue + + for notion in pillar_value["notions"]: + for notion_name, notion_value in notion.items(): + for metric in notion_value["metrics"]: + for metric_name, metric_value in metric.items(): + rows.append( + { + "Pillar": pillar_name, + "Notion": notion_name, + "Notion Score": notion_value["score"], + "Metric": metric_name, + "Metric Score": metric_value["score"], + } + ) + return rows + + def _build_trust_report_dataframe(self, results): + # Convert flattened report rows into a DataFrame for pillar plots. + return pd.DataFrame( + self._trust_report_rows(results), + columns=["Pillar", "Notion", "Notion Score", "Metric", "Metric Score"], + ) + + def _pillar_scores(self, results): + # Read pillar scores in the same order used by the all-pillars chart. + categories = [config[0] for config in PILLAR_CONFIGS] scores = [results["pillars"][i][category]["score"] for i, category in enumerate(categories)] + return categories, scores + + def _pillar_labels(self, label_suffix): + # Build human-readable labels for the all-pillars chart. + labels = [config[4] for config in PILLAR_CONFIGS] + labels.append("Trust Score") + return [f"{label}{label_suffix}" for label in labels] - trust_score = results["trust_score"] + def _log_all_pillars_figure(self, results, all_pillars_tag, label_suffix=""): + # Generate and log the summary chart with every pillar and the final trust score. + categories, scores = self._pillar_scores(results) categories.append("trust_score") - scores.append(trust_score) + scores.append(results["trust_score"]) - palette = ["#F8D3DF", "#DA8D8B", "#DDDDDD", "#FCEFC3", "#8FAADC", "#DBB9FA", "#BBFDAF", "#BF9000"] + palette = [config[1] for config in PILLAR_CONFIGS] + palette.append(TRUST_SCORE_COLOR) plt.figure(figsize=(10, 8)) ax = sns.barplot(x=categories, y=scores, palette=palette, hue=categories, legend=False) @@ -165,62 +222,55 @@ def _log_trust_report(self, results, tag_root, all_pillars_tag, label_suffix="") ax.set_ylabel("Score") ax.set_title("Pillars and trust scores") - for i, v in enumerate(scores): - ax.text(i, v + 0.01, f"{v:.2f}", ha='center', va='bottom', fontsize=10) - - name_labels = [ - f"Robustness{label_suffix}", - f"Privacy{label_suffix}", - f"Fairness{label_suffix}", - f"Explainability{label_suffix}", - f"Accountability{label_suffix}", - f"Architectural Soundness{label_suffix}", - f"Sustainability{label_suffix}", - f"Trust Score{label_suffix}" - ] + for i, value in enumerate(scores): + ax.text(i, value + 0.01, f"{value:.2f}", ha="center", va="bottom", fontsize=10) ax.set_xticks(range(len(categories))) - ax.set_xticklabels(name_labels, rotation=45) + ax.set_xticklabels(self._pillar_labels(label_suffix), rotation=45) self.nebulalogger.log_figure(ax.get_figure(), 0, all_pillars_tag) plt.close() - def graphics(self): - results_file = os.path.join(os.environ.get("NEBULA_LOGS_DIR"), self.scenario_name, "trustworthiness", "nebula_trust_results.json") - results = self._load_trust_results(results_file) - self._log_trust_report(results, "Trust", "Trust/AllPillars") + def _log_trust_report(self, results, tag_root, all_pillars_tag, label_suffix=""): + # Log each pillar chart plus the all-pillars summary for a trust report. + df = self._build_trust_report_dataframe(results) + + for pillar, color, notion_y_pos, figsize, _ in PILLAR_CONFIGS: + self._log_pillar_figure(df, pillar, color, tag_root, notion_y_pos, figsize=figsize) - def graphics_dfl(self,participant_id): - results_file = os.path.join(os.environ.get("NEBULA_LOGS_DIR"), self.scenario_name, "trustworthiness", f"nebula_trust_results_{participant_id}.json") - results = self._load_trust_results(results_file) - self._log_trust_report(results, "Trust", f"Trust/AllPillars_{participant_id}", label_suffix=f"_{participant_id}") + self._log_all_pillars_figure(results, all_pillars_tag, label_suffix=label_suffix) + + def graphics(self): + # Log centralized/global trustworthiness graphics. + results_file = self._trust_report_path("nebula_trust_results.json") + self._log_report_from_file(results_file, "Trust", "Trust/AllPillars") + + def graphics_dfl(self, participant_id): + # Log local DFL graphics for one participant. + results_file = self._trust_report_path(f"nebula_trust_results_{participant_id}.json") + self._log_report_from_file( + results_file, + "Trust", + f"Trust/AllPillars_{participant_id}", + label_suffix=f"_{participant_id}", + ) def graphics_dfl_global(self, participant_id): - results_file = os.path.join( - os.environ.get("NEBULA_LOGS_DIR"), - self.scenario_name, - "trustworthiness", - f"nebula_trust_results_{participant_id}_global.json", - ) - results = self._load_trust_results(results_file) - self._log_trust_report( - results, - "TrustGlobal", - f"TrustGlobal/AllPillars_{participant_id}", - label_suffix=f"_{participant_id}", - ) + # Log aggregated DFL global graphics for one participant. + results_file = self._trust_report_path(f"nebula_trust_results_{participant_id}_global.json") + self._log_report_from_file( + results_file, + "TrustGlobal", + f"TrustGlobal/AllPillars_{participant_id}", + label_suffix=f"_{participant_id}", + ) def graphics_sdfl_global(self, participant_id): - results_file = os.path.join( - os.environ.get("NEBULA_LOGS_DIR"), - self.scenario_name, - "trustworthiness", - "nebula_trust_results.json", - ) - results = self._load_trust_results(results_file) - self._log_trust_report( - results, - "TrustGlobal", - f"TrustGlobal/AllPillars_{participant_id}", - label_suffix=f"_{participant_id}", - ) + # Log SDFL global graphics from the shared global report. + results_file = self._trust_report_path("nebula_trust_results.json") + self._log_report_from_file( + results_file, + "TrustGlobal", + f"TrustGlobal/AllPillars_{participant_id}", + label_suffix=f"_{participant_id}", + ) diff --git a/nebula/addons/trustworthiness/trustworthiness.py b/nebula/addons/trustworthiness/trustworthiness.py index fb337da9b..b2a9ba2ad 100644 --- a/nebula/addons/trustworthiness/trustworthiness.py +++ b/nebula/addons/trustworthiness/trustworthiness.py @@ -32,30 +32,37 @@ class TrustWorkloadException(Exception): class TrustWorkload(ABC): @abstractmethod async def init(self, experiment_name): + # Initialize workload resources and event subscriptions. raise NotImplementedError @abstractmethod def get_workload(self) -> str: + # Return the workload label persisted in trustworthiness outputs. raise NotImplementedError @abstractmethod def get_sample_size(self) -> float: + # Return the local sample size used by the workload. raise NotImplementedError @abstractmethod def get_metrics(self) -> tuple[float, float]: + # Return the latest test loss and accuracy. raise NotImplementedError @abstractmethod async def finish_experiment_role_pre_actions(self): + # Run role-specific work before final metrics are persisted. raise NotImplementedError @abstractmethod async def finish_experiment_role_post_actions(self, trust_config, experiment_name): + # Run role-specific work after final metrics are persisted. raise NotImplementedError class BaseTrustWorkload(TrustWorkload): def __init__(self, engine: Engine, idx, trust_files_route, workload: str, role_label: str, sample_size=None, start_time=None): + # Store shared workload state used by trainers and servers. self._engine: Engine = engine self._workload = workload self._idx = idx @@ -77,6 +84,7 @@ def __init__(self, engine: Engine, idx, trust_files_route, workload: str, role_l self._timed_out_rounds_total = 0 async def init(self, experiment_name): + # Subscribe to the events needed to build final trust summaries. self._experiment_name = experiment_name await EventManager.get_instance().subscribe_node_event(AggregationEvent, self._process_aggregation_event) await EventManager.get_instance().subscribe_node_event(RoundStartEvent, self._process_round_start_event) @@ -94,26 +102,33 @@ async def init(self, experiment_name): await self._per_round.setup(self._engine) def get_workload(self): + # Return the workload name associated with this node role. return self._workload def get_sample_size(self): + # Return the sample size captured by the role pre-actions. return self._sample_size def get_metrics(self): + # Return the latest test metrics observed through events. return (self._current_loss, self._current_accuracy) def get_validation_metrics(self): + # Return the latest validation metrics observed through events. return (self._current_val_loss, self._current_val_accuracy) def _is_reputation_enabled(self) -> bool: + # Read the reputation toggle from the participant defense config. defense_args = self._engine.config.participant.get("defense_args", {}) reputation_config = defense_args.get("reputation", {}) return bool(reputation_config.get("enabled", False)) def _get_reputation_system(self): + # Return the reputation system attached to the engine, when present. return getattr(self._engine, "_reputation", None) def _get_reputation_trust_summary(self) -> dict: + # Build the reputation fields added to the trust factsheet. summary = { "reputation_enabled": self._is_reputation_enabled(), "avg_neighbor_reputation": 0.0, @@ -144,6 +159,7 @@ def _get_reputation_trust_summary(self) -> dict: return summary def _get_participation_trust_summary(self) -> dict: + # Build the participation variability fields added to the trust factsheet. total_clients = int(self._engine.config.participant["scenario_args"]["n_nodes"]) - 1 counts = list(self._round_participation_counts.values()) @@ -155,6 +171,7 @@ def _get_participation_trust_summary(self) -> dict: } def _get_system_reliability_summary(self) -> dict: + # Build dropout and timeout rates from aggregation events. dropout_rate = 0.0 if self._dropout_expected_total > 0: dropout_rate = self._dropout_missing_total / self._dropout_expected_total @@ -169,11 +186,13 @@ def _get_system_reliability_summary(self) -> dict: } async def _process_round_start_event(self, rse: RoundStartEvent): + # Track how often each peer is expected to participate. _, _, expected_nodes = await rse.get_event_data() for node_addr in expected_nodes: self._round_participation_counts[node_addr] = self._round_participation_counts.get(node_addr, 0) + 1 async def _process_aggregation_event(self, age: AggregationEvent): + # Track missing peers and timed-out aggregation rounds. _, expected_nodes, missing_nodes = await age.get_event_data() self_addr = self._engine.addr @@ -187,6 +206,7 @@ async def _process_aggregation_event(self, age: AggregationEvent): self._timed_out_rounds_total += 1 async def _process_test_metrics_event(self, tme: TestMetricsEvent): + # Cache final test metrics and forward them to per-round trust metrics. cur_loss, cur_acc = await tme.get_event_data() if cur_loss is not None and cur_acc is not None: self._current_loss, self._current_accuracy = cur_loss, cur_acc @@ -195,6 +215,7 @@ async def _process_test_metrics_event(self, tme: TestMetricsEvent): await self._per_round.on_test_metrics(self._engine, float(cur_loss), float(cur_acc)) async def _process_validation_metrics_event(self, vme: ValidationMetricsEvent): + # Cache final validation metrics for final trustworthiness outputs. cur_loss, cur_acc = await vme.get_event_data() if cur_loss is not None and cur_acc is not None: self._current_val_loss, self._current_val_accuracy = cur_loss, cur_acc @@ -206,6 +227,7 @@ class TrustWorkloadTrainer(BaseTrustWorkload): TRUSTSCORES_FORWARDING_GRACE_MARGIN_SECONDS = 1.0 def __init__(self, engine, idx, trust_files_route): + # Initialize trainer-side state for CFL reports and DFL/SDFL trustscores. super().__init__(engine, idx, trust_files_route, workload="training", role_label="TRAINER") self._expected_trustscores_sources = set() self._expected_trustscores_reports = int(self._engine.config.participant["scenario_args"]["n_nodes"]) - 1 @@ -218,97 +240,119 @@ def __init__(self, engine, idx, trust_files_route): self._trustscores_local_report_initialized = False async def init(self, experiment_name): + # Reset exchange state before subscribing to shared workload events. self._reset_trustscores_exchange_state() self._trustscores_wait_event = asyncio.Event() await super().init(experiment_name) async def finish_experiment_role_pre_actions(self): + # Capture the training sample size before final trust outputs are written. self._engine.trainer.datamodule.setup(stage="fit") train_loader = self._engine.trainer.datamodule.train_dataloader() self._sample_size = len(train_loader) async def finish_experiment_role_post_actions(self, trust_config, experiment_name): + # Finish with the report flow required by the selected federation type. federation = trust_config.get("federation") - if federation == "DFL" or federation == "SDFL": + if self._uses_trustscores_exchange(federation): await self._finish_trustscores_exchange(federation, trust_config, experiment_name) - else: - cm = CommunicationsManager.get_instance() - - server_addr = str(self._engine.config.participant["network_args"]["neighbors"]).strip() - - bytes_sent, bytes_recv, accuracy, loss, val_accuracy, dp_enabled, dp_epsilon = load_data_results_participant(experiment_name, self._idx) - - role, energy_grid, emissions, workload, cpu_model, gpu_model, cpu_used, gpu_used, energy_consumed, sample_size = load_emissions_participant(experiment_name, self._idx) - - class_imbalance = get_class_imbalance_local(self._idx, experiment_name) - - model_size = get_bytes_model(self._engine.trainer.model) - - local_entropy = get_local_entropy(self._idx, experiment_name) - - message = cm.create_message( - "trustworthiness", - action="report", - node_id=str(self._idx), - bytes_sent=bytes_sent, - bytes_recv=bytes_recv, - accuracy=accuracy, - loss=loss, - role=role, - energy_grid=energy_grid, - emissions=emissions, - workload=workload, - cpu_model=cpu_model, - gpu_model=gpu_model, - cpu_used=cpu_used, - gpu_used=gpu_used, - energy_consumed=energy_consumed, - sample_size=sample_size, - class_imbalance=class_imbalance, - model_size=model_size, - local_entropy=local_entropy, - val_accuracy=val_accuracy, - dp_enabled=dp_enabled, - dp_epsilon=dp_epsilon - ) + return - logging.info( - "[TW SEND] dest=%s node_id=%s bytes_sent=%s bytes_recv=%s " - "accuracy=%s loss=%s role=%s energy_grid=%s emissions=%s workload=%s " - "cpu_model=%s gpu_model=%s cpu_used=%s gpu_used=%s energy_consumed=%s sample_size=%s class_imbalance=%s model_size=%s local_entropy=%s val_accuracy=%s dp_enabled=%s dp_epsilon=%s", - server_addr, - str(self._idx), - bytes_sent, - bytes_recv, - accuracy, - loss, - role, - energy_grid, - emissions, - workload, - cpu_model, - gpu_model, - cpu_used, - gpu_used, - energy_consumed, - sample_size, - class_imbalance, - model_size, - local_entropy, - val_accuracy, - dp_enabled, - dp_epsilon - ) + await self._send_cfl_trustworthiness_report(experiment_name) - await cm.send_message( - server_addr, - message, - message_type="trustworthiness", - allow_after_learning_finished=True, - ) + def _uses_trustscores_exchange(self, federation: str | None) -> bool: + # DFL and SDFL share trust reports directly between participants. + return federation in {"DFL", "SDFL"} + + async def _send_cfl_trustworthiness_report(self, experiment_name: str): + # Send the participant trustworthiness report to the CFL server. + cm = CommunicationsManager.get_instance() + server_addr = str(self._engine.config.participant["network_args"]["neighbors"]).strip() + report = self._build_cfl_trustworthiness_report(experiment_name) + + message = cm.create_message( + "trustworthiness", + action="report", + node_id=str(self._idx), + **report, + ) + + self._log_cfl_trustworthiness_report(server_addr, report) + + await cm.send_message( + server_addr, + message, + message_type="trustworthiness", + allow_after_learning_finished=True, + ) + + def _build_cfl_trustworthiness_report(self, experiment_name: str) -> dict: + # Load local metrics and shape them as a trustworthiness message payload. + bytes_sent, bytes_recv, accuracy, loss, val_accuracy, dp_enabled, dp_epsilon = load_data_results_participant( + experiment_name, + self._idx, + ) + role, energy_grid, emissions, workload, cpu_model, gpu_model, cpu_used, gpu_used, energy_consumed, sample_size = load_emissions_participant( + experiment_name, + self._idx, + ) + + return { + "bytes_sent": bytes_sent, + "bytes_recv": bytes_recv, + "accuracy": accuracy, + "loss": loss, + "role": role, + "energy_grid": energy_grid, + "emissions": emissions, + "workload": workload, + "cpu_model": cpu_model, + "gpu_model": gpu_model, + "cpu_used": cpu_used, + "gpu_used": gpu_used, + "energy_consumed": energy_consumed, + "sample_size": sample_size, + "class_imbalance": get_class_imbalance_local(self._idx, experiment_name), + "model_size": get_bytes_model(self._engine.trainer.model), + "local_entropy": get_local_entropy(self._idx, experiment_name), + "val_accuracy": val_accuracy, + "dp_enabled": dp_enabled, + "dp_epsilon": dp_epsilon, + } + + def _log_cfl_trustworthiness_report(self, server_addr: str, report: dict): + # Log the CFL report with the same fields sent over the network. + logging.info( + "[TW SEND] dest=%s node_id=%s bytes_sent=%s bytes_recv=%s " + "accuracy=%s loss=%s role=%s energy_grid=%s emissions=%s workload=%s " + "cpu_model=%s gpu_model=%s cpu_used=%s gpu_used=%s energy_consumed=%s sample_size=%s class_imbalance=%s model_size=%s local_entropy=%s val_accuracy=%s dp_enabled=%s dp_epsilon=%s", + server_addr, + str(self._idx), + report["bytes_sent"], + report["bytes_recv"], + report["accuracy"], + report["loss"], + report["role"], + report["energy_grid"], + report["emissions"], + report["workload"], + report["cpu_model"], + report["gpu_model"], + report["cpu_used"], + report["gpu_used"], + report["energy_consumed"], + report["sample_size"], + report["class_imbalance"], + report["model_size"], + report["local_entropy"], + report["val_accuracy"], + report["dp_enabled"], + report["dp_epsilon"], + ) async def _finish_trustscores_exchange(self, federation, trust_config, experiment_name): + # Compute, share, wait for, and optionally aggregate DFL/SDFL trustscores. self._end_time = datetime.now().strftime("%d/%m/%Y %H:%M:%S") await self._prepare_trustscores_exchange(federation) @@ -332,11 +376,12 @@ async def _finish_trustscores_exchange(self, federation, trust_config, experimen await self._wait_for_trustscores_forwarding_drain(federation) if federation == "DFL": - self._finalize_trustscores_aggregation() + self._finalize_local_trustscores_aggregation() elif self._is_sdfl_aggregator_node(): self._finalize_sdfl_global_trustscores_aggregation() def _compute_local_trustscores_report(self, experiment_name, trust_config, weights, federation) -> str: + # Build the local DFL/SDFL factsheet and return its JSON report. factsheet = DflFactsheet() self._engine.trainer.datamodule.setup(stage="fit") train_loader = self._engine.trainer.datamodule.train_dataloader() @@ -362,10 +407,12 @@ def _compute_local_trustscores_report(self, experiment_name, trust_config, weigh return load_trust_report_json_dumped(experiment_name, self._idx) def _load_local_trustscores_weights(self, experiment_name: str) -> dict: + # Load trust metric weights for the active federation. federation = self._engine.config.participant["trust_args"]["scenario"].get("federation") return load_trust_weights(experiment_name, federation) def _reset_trustscores_exchange_state(self): + # Clear mutable state from any previous trustscores exchange. self._expected_trustscores_sources = set() self._received_trustscores_node_ids = set() self._trustscores_score_accumulator = {} @@ -375,6 +422,7 @@ def _reset_trustscores_exchange_state(self): self._trustscores_local_report_initialized = False def _get_trustscores_weight_for_source(self, source: str, node_id: int | str) -> float: + # Resolve the aggregation weight for a remote trust report. if not self._is_reputation_enabled(): return 0.5 @@ -399,6 +447,7 @@ def _get_trustscores_weight_for_source(self, source: str, node_id: int | str) -> return float(reputation_entry["reputation"]) def _get_trustscores_peer_weights_from_reputation(self) -> dict: + # Extract peer trustscores weights from the reputation system. if not self._is_reputation_enabled(): return {} @@ -415,9 +464,11 @@ def _get_trustscores_peer_weights_from_reputation(self) -> dict: return peer_weights def _get_trustscores_self_weight(self) -> float: + # Keep local reports fully trusted in the weighted aggregation. return 1.0 def _log_trustscores_node_weights(self, federation: str): + # Log the weights that will be used by trustscores aggregation. if not self._is_reputation_enabled(): logging.info( "[TW %s] Reputation system disabled. trustscores weights fallback to 0.5 for all nodes", @@ -451,19 +502,12 @@ def _log_trustscores_node_weights(self, federation: str): ) def _initialize_local_trustscores_aggregation(self, experiment_name: str): + # Initialize a DFL local aggregation copy with this node's own report. if self._trustscores_local_report_initialized: return trust_report_template, copy_path = create_local_trust_report_copy(experiment_name, self._idx) - self._trustscores_template_report = trust_report_template - self._trustscores_local_copy_path = copy_path - accumulate_weighted_trustscores( - report=trust_report_template, - weight=self._get_trustscores_self_weight(), - score_accumulator=self._trustscores_score_accumulator, - weight_accumulator=self._trustscores_weight_accumulator, - ) - self._trustscores_local_report_initialized = True + self._initialize_trustscores_accumulator(trust_report_template, copy_path, self._get_trustscores_self_weight()) logging.info( "[TW DFL] Local trustscores copy created at %s and accumulator initialized with local weight=%s", copy_path, @@ -471,6 +515,7 @@ def _initialize_local_trustscores_aggregation(self, experiment_name: str): ) async def _prepare_trustscores_exchange(self, federation: str): + # Discover direct neighbors and prepare the wait event for incoming reports. cm = CommunicationsManager.get_instance() self._expected_trustscores_sources = await cm.get_all_addrs_current_connections(only_direct=True) @@ -497,6 +542,7 @@ async def _prepare_trustscores_exchange(self, federation: str): self._log_trustscores_node_weights(federation) async def _share_trustscores_report(self, trust_report_json: str, federation: str): + # Broadcast the local trustscores report to direct neighbors. cm = CommunicationsManager.get_instance() neighbors = self._expected_trustscores_sources.copy() @@ -521,6 +567,7 @@ async def _share_trustscores_report(self, trust_report_json: str, federation: st ) async def _wait_for_trustscores_reports(self, federation: str): + # Wait until every expected report arrives or the exchange times out. if self._trustscores_wait_event is None: return @@ -545,6 +592,7 @@ async def _wait_for_trustscores_reports(self, federation: str): ) async def _wait_for_trustscores_forwarding_drain(self, federation: str): + # Give the forwarder a short grace period before shutdown. if not self._expected_trustscores_sources: return @@ -564,16 +612,24 @@ async def _wait_for_trustscores_forwarding_drain(self, federation: str): ) await asyncio.sleep(forwarding_grace) - def _finalize_trustscores_aggregation(self): + def _build_weighted_trustscores_report(self) -> dict | None: + # Build the weighted report when the aggregation template is available. if self._trustscores_template_report is None or self._trustscores_local_copy_path is None: - logging.warning("[TW DFL] Skipping weighted trustscores write because local copy/template is not available") - return + return None - aggregated_report = build_weighted_trustscores_report( + return build_weighted_trustscores_report( template_report=self._trustscores_template_report, score_accumulator=self._trustscores_score_accumulator, weight_accumulator=self._trustscores_weight_accumulator, ) + + def _finalize_local_trustscores_aggregation(self): + # Write the weighted DFL report and generate DFL graphics. + aggregated_report = self._build_weighted_trustscores_report() + if aggregated_report is None: + logging.warning("[TW DFL] Skipping weighted trustscores write because local copy/template is not available") + return + save_trust_report_json(self._trustscores_local_copy_path, aggregated_report) logging.info( "[TW DFL] Weighted trustscores written to local copy=%s", @@ -583,11 +639,29 @@ def _finalize_trustscores_aggregation(self): graphics = Graphics(self._start_time, self._experiment_name, self._idx) graphics.graphics_dfl_global(self._idx) + def _finalize_sdfl_global_trustscores_aggregation(self): + # Write the weighted SDFL global report and generate SDFL graphics. + aggregated_report = self._build_weighted_trustscores_report() + if aggregated_report is None: + logging.warning("[TW SDFL] Skipping global trustscores write because the template/output is not available") + return + + save_trust_report_json(self._trustscores_local_copy_path, aggregated_report) + logging.info( + "[TW SDFL] Global weighted trustscores written to %s", + self._trustscores_local_copy_path, + ) + + graphics = Graphics(self._start_time, self._experiment_name, self._idx) + graphics.graphics_sdfl_global(self._idx) + def _is_sdfl_aggregator_node(self) -> bool: + # Check whether this node should aggregate global SDFL trustscores. effective_role = self._engine.rb.get_role_name(True) return effective_role in {Role.AGGREGATOR.value, Role.TRAINER_AGGREGATOR.value} def _initialize_sdfl_global_trustscores_aggregation(self, experiment_name: str): + # Initialize the SDFL global aggregation output with this node's own report. if self._trustscores_local_report_initialized: return @@ -601,44 +675,31 @@ def _initialize_sdfl_global_trustscores_aggregation(self, experiment_name: str): ) save_trust_report_json(output_path, trust_report_template) - self._trustscores_template_report = trust_report_template - self._trustscores_local_copy_path = output_path - accumulate_weighted_trustscores( - report=trust_report_template, - weight=1.0, - score_accumulator=self._trustscores_score_accumulator, - weight_accumulator=self._trustscores_weight_accumulator, - ) - self._trustscores_local_report_initialized = True + self._initialize_trustscores_accumulator(trust_report_template, output_path, self._get_trustscores_self_weight()) logging.info( "[TW SDFL] Global trustscores accumulator initialized at %s with local weight=1.0", output_path, ) - def _finalize_sdfl_global_trustscores_aggregation(self): - if self._trustscores_template_report is None or self._trustscores_local_copy_path is None: - logging.warning("[TW SDFL] Skipping global trustscores write because the template/output is not available") - return - - aggregated_report = build_weighted_trustscores_report( - template_report=self._trustscores_template_report, + def _initialize_trustscores_accumulator(self, trust_report_template: dict, output_path: str, local_weight: float): + # Store the aggregation template and seed accumulators with the local report. + self._trustscores_template_report = trust_report_template + self._trustscores_local_copy_path = output_path + accumulate_weighted_trustscores( + report=trust_report_template, + weight=local_weight, score_accumulator=self._trustscores_score_accumulator, weight_accumulator=self._trustscores_weight_accumulator, ) - save_trust_report_json(self._trustscores_local_copy_path, aggregated_report) - logging.info( - "[TW SDFL] Global weighted trustscores written to %s", - self._trustscores_local_copy_path, - ) - - graphics = Graphics(self._start_time, self._experiment_name, self._idx) - graphics.graphics_sdfl_global(self._idx) + self._trustscores_local_report_initialized = True async def register_trustscores_report(self, source, message): + # Register a remote trustscores message using the active federation. federation = self._engine.config.participant["trust_args"]["scenario"].get("federation") await self._register_trustscores_report(source, message, federation) async def _register_trustscores_report(self, source, message, federation: str): + # Deduplicate, optionally accumulate, and mark remote trustscores as received. if str(message.node_id) == str(self._idx): logging.info("[TW %s] Ignoring own trustscores report from %s", federation, source) return @@ -691,6 +752,7 @@ class TrustWorkloadServer(BaseTrustWorkload): REPORTS_WAIT_TIMEOUT_SECONDS = 60 def __init__(self, engine: Engine, idx, trust_files_route): + # Initialize server-side state for collecting participant reports. server_start_time: ServerRoleBehavior = engine.rb super().__init__( engine, @@ -710,12 +772,15 @@ def __init__(self, engine: Engine, idx, trust_files_route): self._reports_wait_event.set() async def init(self, experiment_name): + # Reuse the shared workload event subscriptions. await super().init(experiment_name) async def finish_experiment_role_pre_actions(self): + # Server has no pre-save work because aggregation sample size is zero. pass async def finish_experiment_role_post_actions(self, trust_config, experiment_name): + # Wait for participant reports, save CSV data, and generate the CFL factsheet. self._end_time = datetime.now().strftime("%d/%m/%Y %H:%M:%S") self._trust_config = trust_config self._experiment_name = experiment_name @@ -723,27 +788,35 @@ async def finish_experiment_role_post_actions(self, trust_config, experiment_nam if self._csv_completed: logging.info("[TW SERVER] finish_experiment_role_post_actions called, trustworthiness reports OK, starting generate_factsheet") await self._save_local_server_report_and_generate_factsheet(trust_config, experiment_name) - else: - logging.info("[TW SERVER] finish_experiment_role_post_actions called, waiting for trustworthiness reports") - try: - await asyncio.wait_for( - self._reports_wait_event.wait(), - timeout=self.REPORTS_WAIT_TIMEOUT_SECONDS, - ) - except asyncio.TimeoutError: - logging.warning( - "[TW SERVER] Timeout waiting trustworthiness reports. Received=%s/%s", - len(self._trustworthiness_reports), - self._expected_reports, - ) - - if self._trustworthiness_reports is not None and not self._csv_completed: - save_trustworthiness_reports_csv(self._trustworthiness_reports, self._experiment_name) - self._csv_completed = True + return - await self._save_local_server_report_and_generate_factsheet(trust_config, experiment_name) + logging.info("[TW SERVER] finish_experiment_role_post_actions called, waiting for trustworthiness reports") + await self._wait_for_trustworthiness_reports() + self._save_trustworthiness_reports_once() + await self._save_local_server_report_and_generate_factsheet(trust_config, experiment_name) + + async def _wait_for_trustworthiness_reports(self): + # Wait until reports arrive or the server-side timeout expires. + try: + await asyncio.wait_for( + self._reports_wait_event.wait(), + timeout=self.REPORTS_WAIT_TIMEOUT_SECONDS, + ) + except asyncio.TimeoutError: + logging.warning( + "[TW SERVER] Timeout waiting trustworthiness reports. Received=%s/%s", + len(self._trustworthiness_reports), + self._expected_reports, + ) + + def _save_trustworthiness_reports_once(self): + # Persist received participant reports only once. + if self._trustworthiness_reports is not None and not self._csv_completed: + save_trustworthiness_reports_csv(self._trustworthiness_reports, self._experiment_name) + self._csv_completed = True async def _save_local_server_report_and_generate_factsheet(self, trust_config, experiment_name): + # Add the server's own local report and generate final trust artifacts. bytes_sent, bytes_recv, _, _, val_accuracy, dp_enabled, dp_epsilon = load_data_results_participant( self._experiment_name, self._idx, @@ -768,6 +841,7 @@ async def _save_local_server_report_and_generate_factsheet(self, trust_config, e await self._generate_factsheet(trust_config, experiment_name) async def register_trustworthiness_report(self, source, message): + # Store one participant trustworthiness report received by the server. self._trustworthiness_reports[message.node_id] = { "source": source, "node_id": message.node_id, @@ -801,13 +875,12 @@ async def register_trustworthiness_report(self, source, message): if (len(self._trustworthiness_reports) >= self._expected_reports): logging.info("[TW SERVER] all reports received, generating csv") - # Generate CSV files - save_trustworthiness_reports_csv(self._trustworthiness_reports, self._experiment_name) - self._csv_completed = True + self._save_trustworthiness_reports_once() self._reports_wait_event.set() logging.info(f"[TW SERVER] all reports received, waiting for finish post, csv_completed {self._csv_completed}") async def _generate_factsheet(self, trust_config, experiment_name): + # Generate the CFL factsheet and evaluate final trust metrics. factsheet = CflFactsheet() self._engine.trainer.datamodule.setup(stage="fit") train_loader = self._engine.trainer.datamodule.train_dataloader() @@ -840,6 +913,7 @@ async def _generate_factsheet(self, trust_config, experiment_name): class Trustworthiness(): def __init__(self, engine: Engine, config: Config): + # Select the workload implementation for this node and start emissions tracking. config.reset_logging_configuration() print_msg_box( msg=f"Name Trustworthiness Module\nRole: {engine.rb.get_role_name()}", @@ -864,15 +938,18 @@ def __init__(self, engine: Engine, config: Config): @property def tw(self): """TrustWorkload implementation chosen according to the node role.""" + # Expose the role-specific trust workload. return self._trust_workload async def start(self): + # Prepare output directories, subscribe to finish events, and start tracking emissions. await self._create_trustworthiness_directory() await self.tw.init(self._experiment_name) await EventManager.get_instance().subscribe_node_event(ExperimentFinishEvent, self._process_experiment_finish_event) self._tracker.start() async def _create_trustworthiness_directory(self): + # Ensure the experiment trustworthiness directory exists. logs_dir = os.environ.get("NEBULA_LOGS_DIR", os.path.join("nebula", "app", "logs")) trust_dir = os.path.join(logs_dir, self._experiment_name, "trustworthiness") # Create a directory to store files used to compute trust @@ -880,6 +957,7 @@ async def _create_trustworthiness_directory(self): os.chmod(trust_dir, 0o755) async def _process_experiment_finish_event(self, efe: ExperimentFinishEvent): + # Persist final local metrics and delegate role-specific finalization. class_counter = self._engine.trainer.datamodule.get_samples_per_label() save_class_count_per_participant(self._experiment_name, class_counter, self._idx) @@ -911,6 +989,7 @@ async def _process_experiment_finish_event(self, efe: ExperimentFinishEvent): await self.tw.finish_experiment_role_post_actions(self._trust_config, self._experiment_name) def _factory_trust_workload(self, role: Role, engine: Engine, idx, trust_files_route) -> TrustWorkload: + # Create the workload implementation associated with the node role. trust_workloads = { Role.TRAINER: TrustWorkloadTrainer, Role.AGGREGATOR: TrustWorkloadTrainer, From 7516058060bad373a97d05af2e41b73d26e0924e Mon Sep 17 00:00:00 2001 From: "Juan J." Date: Fri, 29 May 2026 17:08:52 +0200 Subject: [PATCH 55/66] Refactoring calculation and utils --- nebula/addons/trustworthiness/calculation.py | 2094 ----------------- .../addons/trustworthiness/dfl_factsheet.py | 18 +- nebula/addons/trustworthiness/factsheet.py | 15 +- .../trustworthiness/factsheet_populators.py | 26 +- .../trustworthiness/helpers/__init__.py | 1 + .../addons/trustworthiness/helpers/csv_io.py | 316 +++ .../helpers/data_distribution.py | 178 ++ .../trustworthiness/helpers/explainability.py | 407 ++++ .../helpers/factsheet_values.py | 108 + .../trustworthiness/helpers/model_quality.py | 371 +++ .../addons/trustworthiness/helpers/privacy.py | 209 ++ .../trustworthiness/helpers/robustness.py | 413 ++++ .../helpers/scenario_metrics.py | 350 +++ .../addons/trustworthiness/helpers/scoring.py | 190 ++ .../trustworthiness/helpers/trust_reports.py | 197 ++ nebula/addons/trustworthiness/metric.py | 2 +- .../trustworthiness/per_round_metrics.py | 1 - nebula/addons/trustworthiness/pillar.py | 20 +- .../addons/trustworthiness/trustworthiness.py | 27 +- nebula/addons/trustworthiness/utils.py | 656 ------ 20 files changed, 2818 insertions(+), 2781 deletions(-) delete mode 100755 nebula/addons/trustworthiness/calculation.py create mode 100644 nebula/addons/trustworthiness/helpers/__init__.py create mode 100644 nebula/addons/trustworthiness/helpers/csv_io.py create mode 100644 nebula/addons/trustworthiness/helpers/data_distribution.py create mode 100644 nebula/addons/trustworthiness/helpers/explainability.py create mode 100644 nebula/addons/trustworthiness/helpers/factsheet_values.py create mode 100644 nebula/addons/trustworthiness/helpers/model_quality.py create mode 100644 nebula/addons/trustworthiness/helpers/privacy.py create mode 100644 nebula/addons/trustworthiness/helpers/robustness.py create mode 100644 nebula/addons/trustworthiness/helpers/scenario_metrics.py create mode 100644 nebula/addons/trustworthiness/helpers/scoring.py create mode 100644 nebula/addons/trustworthiness/helpers/trust_reports.py delete mode 100755 nebula/addons/trustworthiness/utils.py diff --git a/nebula/addons/trustworthiness/calculation.py b/nebula/addons/trustworthiness/calculation.py deleted file mode 100755 index 6fcb60ea2..000000000 --- a/nebula/addons/trustworthiness/calculation.py +++ /dev/null @@ -1,2094 +0,0 @@ -import logging -import math -import numbers -import os.path -import statistics -import copy -import gc -from datetime import datetime -from math import e -from os.path import exists -import json - -import numpy as np -import pandas as pd -import shap -import torch -import torch.nn -from art.estimators.classification import PyTorchClassifier -from art.metrics import clever_u, loss_sensitivity, empirical_robustness -from codecarbon import EmissionsTracker -from scipy.spatial.distance import jensenshannon -from scipy.stats import entropy, variation -from sklearn.metrics import f1_score, roc_auc_score, roc_curve -from torch import nn, optim -import torch.nn.functional as F -import io - - -from nebula.addons.trustworthiness.utils import read_csv - -dirname = os.path.dirname(__file__) -logger = logging.getLogger(__name__) - -R_L1 = 40 -R_L2 = 2 -R_LI = 0.1 - - -# --------------------------------------------------------------------------- -# Generic score mapping helpers used by eval_metrics*.json -# --------------------------------------------------------------------------- - -def get_mapped_score(score_key, score_map): - """ - Finds the score by the score_key in the score_map. - - Args: - score_key (string): The key to look up in the score_map. - score_map (dict): The score map defined in the eval_metrics.json file. - - Returns: - float: The normalized score of [0, 1]. - """ - score = 0 - if score_map is None: - logger.warning("Score map is missing") - else: - keys = [key for key, value in score_map.items()] - scores = [value for key, value in score_map.items()] - normalized_scores = get_normalized_scores(scores) - normalized_score_map = dict(zip(keys, normalized_scores, strict=False)) - score = normalized_score_map.get(score_key, np.nan) - - return score - - -def get_normalized_scores(scores): - """ - Calculates the normalized scores of a list. - - Args: - scores (list): The values that will be normalized. - - Returns: - list: The normalized list. - """ - if scores is None or len(scores) == 0: - return [] - - min_score = np.min(scores) - max_score = np.max(scores) - if max_score == min_score: - return [1.0 for _ in scores] - - normalized = [(x - min_score) / (max_score - min_score) for x in scores] - return normalized - - -def get_range_score(value, ranges, direction="asc"): - """ - Maps the value to a range and gets the score by the range and direction. - - Args: - value (int): The input score. - ranges (list): The ranges defined. - direction (string): Asc means the higher the range the higher the score, desc means otherwise. - - Returns: - float: The normalized score of [0, 1]. - """ - - if not (type(value) == int or type(value) == float): - logger.warning("Input value is not a number") - logger.warning(f"{value}") - return 0 - else: - score = 0 - if ranges is None: - logger.warning("Score ranges are missing") - else: - total_bins = len(ranges) + 1 - bin = np.digitize(value, ranges, right=True) - score = 1 - (bin / total_bins) if direction == "desc" else bin / total_bins - return score - - -def get_map_value_score(score_key, score_map): - """ - Finds the score by the score_key in the score_map and returns the value. - - Args: - score_key (string): The key to look up in the score_map. - score_map (dict): The score map defined in the eval_metrics.json file. - - Returns: - float: The score obtained in the score_map. - """ - score = 0 - if score_map is None: - logger.warning("Score map is missing") - else: - score = score_map[score_key] - return score - - -def get_true_score(value, direction): - """ - Returns the negative of the value if direction is 'desc', otherwise returns value. - - Args: - value (int): The input score. - direction (string): Asc means the higher the range the higher the score, desc means otherwise. - - Returns: - float: The score obtained. - """ - - if value is True: - return 1 - elif value is False: - return 0 - else: - if not (type(value) == int or type(value) == float): - logger.warning("Input value is not a number") - logger.warning(f"{value}.") - return 0 - else: - if direction == "desc": - return 1 - value - else: - return value - - -def get_scaled_score(value, scale: list, direction: str): - """ - Maps a score of a specific scale into the scale between zero and one. - - Args: - value (int or float): The raw value of the metric. - scale (list): List containing the minimum and maximum value the value can fall in between. - - Returns: - float: The normalized score of [0, 1]. - """ - - score = 0 - try: - value_min, value_max = scale[0], scale[1] - except Exception: - logger.warning("Score minimum or score maximum is missing. The minimum has been set to 0 and the maximum to 1") - value_min, value_max = 0, 1 - if value is None or value == "": - logger.warning("Score value is missing. Set value to zero") - else: - low, high = 0, 1 - if value >= value_max: - score = 1 - elif value <= value_min: - score = 0 - else: - diff = value_max - value_min - diffScale = high - low - score = (float(value) - value_min) * (float(diffScale) / diff) + low - if direction == "desc": - score = high - score - - return score - - -def get_value(value): - """ - Get the value of a metric. - - Args: - value (float): The value of the metric. - - Returns: - float: The value of the metric. - """ - - return value - - -def check_properties(*args): - """ - Check if all the arguments have values. - - Args: - args (list): All the arguments. - - Returns: - float: The mean of arguments that have values. - """ - - result = map(lambda x: x is not None and x != "", args) - return np.mean(list(result)) - - -# --------------------------------------------------------------------------- -# Local/global data distribution and participation metrics -# --------------------------------------------------------------------------- - -def get_class_count_file(scenario_name, participant_id): - """ - Returns the class-count file path for a participant. - """ - return os.path.join( - os.environ.get("NEBULA_LOGS_DIR"), - scenario_name, - "trustworthiness", - f"{str(participant_id)}_class_count.json", - ) - - -def load_class_counts(scenario_name, participant_id): - """ - Loads the saved class-count distribution for a participant. - """ - with open(get_class_count_file(scenario_name, participant_id), "r") as file: - return json.load(file) - - -def get_class_imbalance_from_counts(class_counts): - """ - Calculates class imbalance as coefficient of variation over class counts. - - Higher values mean a more imbalanced local dataset. - """ - return get_cv(list=list(class_counts.values())) - - -def get_class_imbalance_score(class_imbalance): - """ - Converts class imbalance into a trust score. - - A score of 1 means balanced classes; higher imbalance lowers the score. - """ - return 1 / (1 + class_imbalance) - - -def get_class_imbalance_local(participant_id, experiment_name): - class_distribution = load_class_counts(experiment_name, participant_id) - return get_class_imbalance_from_counts(class_distribution) - - -def get_local_class_imbalance_score(scenario_name, participant_id): - """ - Calculates the class-imbalance trust score for a participant. - """ - return get_class_imbalance_score(get_class_imbalance_local(participant_id, scenario_name)) - - -def get_entropy_from_class_counts(class_counts, normalize=False): - """ - Calculates entropy from class counts. - - When normalized, returns a value in [0, 1] independent of class count. - """ - counts = np.array(list(class_counts.values()), dtype=float) - total = counts.sum() - if total <= 0: - return 0.0 - - probabilities = counts / total - entropy_value = entropy(probabilities, base=2) - - if not normalize: - return round(float(entropy_value), 6) - - class_count = len(probabilities) - if class_count <= 1: - return 0.0 - - normalized_entropy = float(entropy_value / np.log2(class_count)) - return max(0.0, min(1.0, normalized_entropy)) - - -def get_local_normalized_entropy(scenario_name, participant_id): - """ - Calculates normalized entropy from a participant's saved class counts. - """ - return get_entropy_from_class_counts( - load_class_counts(scenario_name, participant_id), - normalize=True, - ) - - -def get_cv(list=None, std=None, mean=None): - """ - Get the coefficient of variation. - - Args: - list (list): List in which the coefficient of variation will be calculated. - std (float): Standard deviation of a list. - mean (float): Mean of a list. - - Returns: - float: The coefficient of variation calculated. - """ - if std is not None and mean is not None: - if mean == 0: - return 0 - return std / mean - - if list is not None: - mean_value = np.mean(list) - if mean_value == 0: - return 0 - return np.std(list) / mean_value - - return 0 - - -def get_participation_variation_score(participation_counts): - """ - Convert participation-count dispersion into a trust-oriented score. - - Args: - participation_counts (list[float | int]): Number of participations per client. - - Returns: - float: Score in [0, 1] where 1 means equal participation. - """ - if not participation_counts: - return 1.0 - - counts = np.asarray(participation_counts, dtype=float) - mean_count = float(np.mean(counts)) - - if mean_count <= 0: - return 0.0 - - cv = get_cv(list=counts) - if not np.isfinite(cv): - return 0.0 - - return float(1 / (1 + cv)) - - -# --------------------------------------------------------------------------- -# Privacy metrics -# --------------------------------------------------------------------------- - - -def get_global_privacy_risk(dp, epsilon, n): - """ - Calculates the global privacy risk by epsilon and the number of clients. - - Args: - dp (bool): Indicates if differential privacy is used or not. - epsilon (int): The epsilon value. - n (int): The number of clients in the scenario. - - Returns: - float: The global privacy risk. - """ - - try: - epsilon = float(epsilon) - n = float(n) - except (TypeError, ValueError): - return 1 - - if dp is True and isinstance(epsilon, numbers.Number): - return 1 / (1 + (n - 1) * math.pow(e, -epsilon)) - else: - return 1 - -def get_global_privacy_risk_dfl(dp, epsilon, n): - """ - Calculates the global privacy risk by epsilon and the number of clients. - - Args: - dp (bool): Indicates if differential privacy is used or not. - epsilon (int): The epsilon value. - n (int): The number of neighbours. - - Returns: - float: The global privacy risk. - """ - - try: - epsilon = float(epsilon) - n = float(n) - except (TypeError, ValueError): - return 1 - - if dp is True and isinstance(epsilon, numbers.Number): - return 1 / (1 + (n + 1) * math.pow(e, -epsilon)) - else: - return 1 - - -def _collect_per_sample_losses(model, dataloader, max_samples=5000): - """ - Compute per-sample cross-entropy losses for a dataloader. - - Args: - model (torch.nn.Module): The model to evaluate. - dataloader: DataLoader providing (samples, labels). - max_samples (int): Maximum number of samples to process. - - Returns: - np.ndarray: Losses per sample. - """ - if not isinstance(model, torch.nn.Module) or dataloader is None: - return np.array([]) - - try: - device = next(model.parameters()).device - except Exception: - device = torch.device("cpu") - - criterion = nn.CrossEntropyLoss(reduction="none") - losses = [] - collected = 0 - - model.eval() - with torch.no_grad(): - for batch in dataloader: - if not isinstance(batch, (tuple, list)) or len(batch) < 2: - continue - - samples, labels = batch[0], batch[1] - if not torch.is_tensor(samples) or not torch.is_tensor(labels): - continue - - remaining = max_samples - collected - if remaining <= 0: - break - - samples = samples[:remaining].to(device) - labels = labels[:remaining] - - if labels.ndim > 1: - labels = torch.argmax(labels, dim=1) - - labels = labels.long().to(device) - - outputs = model(samples) - logits = outputs[0] if isinstance(outputs, (tuple, list)) else outputs - batch_losses = criterion(logits, labels) - - losses.append(batch_losses.detach().cpu().numpy()) - collected += int(batch_losses.shape[0]) - - if not losses: - return np.array([]) - - return np.concatenate(losses, axis=0) - - -def get_epsilon_star(model, train_dataloader, test_dataloader, max_samples=5000): - """ - Compute empirical epsilon* from train/test loss distributions. - - This follows the same core structure as privacy_metrics_core.epsilon_star, - adapted to PyTorch models and DataLoaders used in Nebula. - - Args: - model (torch.nn.Module): Model to evaluate. - train_dataloader: Training DataLoader. - test_dataloader: Test DataLoader. - max_samples (int): Maximum samples to evaluate per split. - - Returns: - float: Empirical epsilon* value. Returns 0.0 on failure. - """ - try: - loss_train = _collect_per_sample_losses(model, train_dataloader, max_samples=max_samples) - loss_test = _collect_per_sample_losses(model, test_dataloader, max_samples=max_samples) - - if loss_train.size == 0 or loss_test.size == 0: - return 0.0 - - scores = np.concatenate([-loss_train, -loss_test]) - y_true = np.concatenate([np.ones(len(loss_train)), np.zeros(len(loss_test))]) - - fpr, tpr, _ = roc_curve(y_true, scores) - - fpr = np.clip(fpr, 1e-10, 1 - 1e-10) - tpr = np.clip(tpr, 1e-10, 1 - 1e-10) - fnr = 1 - tpr - - delta = 1.0 / len(loss_train) if len(loss_train) > 0 else 1e-5 - - m1 = (1 - delta - fnr) / fpr - m2 = (1 - delta - fpr) / fnr - m3 = (fnr - delta) / (1 - fpr) - m4 = (fpr - delta) / (1 - fnr) - - epsilon_star_val = np.log( - np.nanmax(np.maximum.reduce([m1, m2, m3, m4, np.ones_like(m1)])) - ) - - if np.isnan(epsilon_star_val) or np.isinf(epsilon_star_val): - return 0.0 - - return float(max(0.0, epsilon_star_val)) - except Exception as exc: - logger.warning("Could not compute epsilon_star") - logger.warning(exc) - return 0.0 - - -def get_mia_auc(model, train_dataloader, test_dataloader, max_samples=5000): - """ - Compute membership inference attack AUC using per-sample loss as the attack score. - - Lower loss suggests a sample is more likely to be a training member, so the - attack score is defined as negative loss. - - Args: - model (torch.nn.Module): Model to evaluate. - train_dataloader: Training DataLoader. - test_dataloader: Test DataLoader. - max_samples (int): Maximum samples to evaluate per split. - - Returns: - float: ROC-AUC of the loss-threshold membership attack. Returns 0.5 on failure. - """ - try: - loss_train = _collect_per_sample_losses(model, train_dataloader, max_samples=max_samples) - loss_test = _collect_per_sample_losses(model, test_dataloader, max_samples=max_samples) - - if loss_train.size == 0 or loss_test.size == 0: - return 0.5 - - scores = np.concatenate([-loss_train, -loss_test]) - y_true = np.concatenate([np.ones(len(loss_train)), np.zeros(len(loss_test))]) - mia_auc = roc_auc_score(y_true, scores) - - if np.isnan(mia_auc) or np.isinf(mia_auc): - return 0.5 - - return float(np.clip(mia_auc, 0.0, 1.0)) - except Exception as exc: - logger.warning("Could not compute mia_auc") - logger.warning(exc) - return 0.5 - - -# --------------------------------------------------------------------------- -# Scenario report readers and aggregate system metrics -# --------------------------------------------------------------------------- - -def get_elapsed_time(start_time, end_time): - """ - Calculates the elapsed time during the execution of the scenario. - - Args: - start_time (datetime): Start datetime. - end_time (datetime): End datetime. - - Returns: - float: The elapsed time. - """ - start_date = datetime.strptime(start_time, "%d/%m/%Y %H:%M:%S") - end_date = datetime.strptime(end_time, "%d/%m/%Y %H:%M:%S") - - elapsed_time = (end_date - start_date).total_seconds() / 60 - - return elapsed_time - - -def _trustworthiness_dir(scenario_name): - return os.path.join(os.environ.get('NEBULA_LOGS_DIR'), scenario_name, "trustworthiness") - - -def _global_data_results_path(scenario_name): - return os.path.join(_trustworthiness_dir(scenario_name), "data_results.csv") - - -def _participant_data_results_path(scenario_name, participant_id): - return os.path.join(_trustworthiness_dir(scenario_name), f"data_results_{participant_id}.csv") - - -def _read_global_results(scenario_name): - return read_csv(_global_data_results_path(scenario_name)) - - -def _read_participant_results(scenario_name, participant_id): - return read_csv(_participant_data_results_path(scenario_name, participant_id)) - - -def _find_participant_row(data, participant_id, source_name): - row = data[data["id"] == participant_id] - - if row.empty: - try: - row = data[data["id"] == int(participant_id)] - except (TypeError, ValueError): - row = data.iloc[0:0] - - if row.empty: - raise ValueError(f"Participant {participant_id} not found in {source_name}") - - return row.iloc[0] - - -def get_bytes_model(model): - """ - Calculates the serialized size in bytes of a PyTorch model state_dict. - - Args: - model (nn.Module): PyTorch model. - - Returns: - int: Model size in bytes. - """ - buffer: io.BytesIO = io.BytesIO() - torch.save(model.state_dict(), buffer) - model_size: int = buffer.tell() - - return model_size - - -def get_bytes_sent_recv(scenario_name): - """ - Calculates the mean bytes sent and received of the nodes. - - Args: - bytes_sent_files (list): Files that contain the bytes sent of the nodes. - bytes_recv_files (list): Files that contain the bytes received of the nodes. - - Returns: - 4-tupla: The total bytes sent, the total bytes received, the mean bytes sent and the mean bytes received of the nodes. - """ - data = _read_global_results(scenario_name) - - number_files = len(data) - - total_upload_bytes = int(data["bytes_sent"].sum()) - total_download_bytes = int(data["bytes_recv"].sum()) - - avg_upload_bytes = total_upload_bytes / number_files - avg_download_bytes = total_download_bytes / number_files - - return total_upload_bytes, total_download_bytes, avg_upload_bytes, avg_download_bytes - - -def get_avg_loss_accuracy(scenario_name): - """ - Calculates the mean accuracy and loss models of the nodes. - - Args: - loss_files (list): Files that contain the loss of the models of the nodes. - accuracy_files (list): Files that contain the acurracies of the models of the nodes. - - Returns: - 3-tupla: The mean loss of the models, the mean accuracies of the models, the standard deviation of the accuracies of the models. - """ - data = _read_global_results(scenario_name) - - number_files = len(data) - - total_loss = data["loss"].sum() - total_accuracy = data["accuracy"].sum() - - denominator = max(1, number_files - 1) - avg_loss = total_loss / denominator - avg_accuracy = total_accuracy / denominator - std_accuracy = statistics.stdev(data["accuracy"]) if number_files > 1 else 0.0 - - return avg_loss, avg_accuracy, std_accuracy - -def get_underfitting_score(scenario_name, id): - """ - Calculates the mean val accuracy of the nodes. - """ - data = _read_global_results(scenario_name) - - number_files = len(data) - - total_val_accuracy = data["val_accuracy"].sum() - - avg_val_accuracy = total_val_accuracy / max(1, number_files - 1) - - return avg_val_accuracy - - -def get_participant_loss_accuracy(scenario_name, participant_id): - """ - Gets loss and accuracy for a specific participant from CFL aggregated results. - - Args: - scenario_name (str): Scenario name. - participant_id (int | str): Participant identifier. - - Returns: - tuple[float, float]: (loss, accuracy) - """ - data_file = _global_data_results_path(scenario_name) - row = _find_participant_row(read_csv(data_file), participant_id, data_file) - - loss = float(row["loss"]) - accuracy = float(row["accuracy"]) - return loss, accuracy - - -# --------------------------------------------------------------------------- -# Model performance metrics -# --------------------------------------------------------------------------- - - -def _get_model_accuracy(model, dataloader): - """ - Calculates model accuracy over a dataloader. - - Args: - model (torch.nn.Module): Model to evaluate. - dataloader (DataLoader): Dataloader with (x, y) batches. - - Returns: - float: Accuracy in [0, 1]. - """ - if not isinstance(model, torch.nn.Module): - logger.warning("Model is not a torch.nn.Module") - return 0.0 - - try: - device = next(model.parameters()).device - except Exception: - device = torch.device("cpu") - - model.eval() - correct = 0 - total = 0 - - with torch.no_grad(): - for x, y in dataloader: - x = x.to(device) - y = y.to(device) - - out = model(x) - logits = out[0] if isinstance(out, (tuple, list)) else out - preds = logits.argmax(dim=1) - - correct += (preds == y).sum().item() - total += y.size(0) - - return correct / total if total > 0 else 0.0 - - -def get_macro_f1_score(model, dataloader): - """ - Calculates macro F1 score over a dataloader. - - Args: - model (torch.nn.Module): Model to evaluate. - dataloader (DataLoader): Dataloader with (x, y) batches. - - Returns: - float: Macro F1 score in [0, 1]. - """ - if not isinstance(model, torch.nn.Module): - logger.warning("Model is not a torch.nn.Module") - return 0.0 - - try: - device = next(model.parameters()).device - except Exception: - device = torch.device("cpu") - - model.eval() - y_true = [] - y_pred = [] - - with torch.no_grad(): - for x, y in dataloader: - x = x.to(device) - y = y.to(device) - - out = model(x) - logits = out[0] if isinstance(out, (tuple, list)) else out - preds = logits.argmax(dim=1) - - y_true.extend(y.detach().cpu().numpy().tolist()) - y_pred.extend(preds.detach().cpu().numpy().tolist()) - - if not y_true: - return 0.0 - - return float(f1_score(y_true, y_pred, average="macro", zero_division=0)) - -def _extract_model_logits(model_output): - """ - Normalize the output returned by a model forward pass into a logits tensor. - - Some models may return tuples/lists; for trust metrics we always consume the - first element as the classification output. - """ - return model_output[0] if isinstance(model_output, (tuple, list)) else model_output - - -def _prepare_class_targets(y): - """ - Convert different target representations into a flat class-index tensor. - """ - if not torch.is_tensor(y): - y = torch.as_tensor(y) - - if y.ndim > 1: - if y.size(-1) > 1: - y = y.argmax(dim=-1) - else: - y = y.view(-1) - - return y.long().view(-1) - - -def _logits_to_probabilities(logits): - """ - Convert model outputs into a probability matrix of shape (N, C). - - Supports: - - multiclass logits/log-probabilities with shape (N, C) - - binary logits with shape (N,) or (N, 1) - - already-normalized probability matrices - """ - if not torch.is_tensor(logits): - logits = torch.as_tensor(logits) - - if logits.ndim == 0: - logits = logits.view(1, 1) - elif logits.ndim == 1: - logits = logits.view(-1, 1) - elif logits.ndim > 2: - logits = logits.reshape(logits.shape[0], -1) - - if logits.size(1) == 1: - pos_prob = torch.sigmoid(logits[:, 0]) - probs = torch.stack([1.0 - pos_prob, pos_prob], dim=1) - else: - row_sums = logits.sum(dim=1) - looks_like_probs = ( - torch.all(logits >= 0) - and torch.all(logits <= 1.0 + 1e-6) - and torch.allclose(row_sums, torch.ones_like(row_sums), atol=1e-4, rtol=1e-4) - ) - probs = logits if looks_like_probs else torch.softmax(logits, dim=1) - - probs = torch.clamp(probs, min=0.0, max=1.0) - probs = probs / probs.sum(dim=1, keepdim=True).clamp_min(1e-12) - return probs - - -def _collect_classification_statistics(model, dataloader): - """ - Collect prediction statistics required by calibration and inequality metrics. - - Returns: - tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]: - predicted labels, true labels, prediction confidences, correctness flags, - and probability assigned to the true class. - """ - if not isinstance(model, torch.nn.Module): - logger.warning("Model is not a torch.nn.Module") - empty = np.array([], dtype=float) - return empty, empty, empty, empty, empty - - try: - device = next(model.parameters()).device - except Exception: - device = torch.device("cpu") - - preds_all = [] - targets_all = [] - confidences_all = [] - correct_all = [] - true_probs_all = [] - - model.eval() - with torch.no_grad(): - for batch in dataloader: - if not isinstance(batch, (tuple, list)) or len(batch) < 2: - continue - - x, y = batch[0], batch[1] - if not (torch.is_tensor(x) and torch.is_tensor(y)): - continue - - x = x.to(device) - y = _prepare_class_targets(y).to(device) - - out = model(x) - logits = _extract_model_logits(out) - probs = _logits_to_probabilities(logits) - - if probs.ndim != 2 or probs.size(0) == 0: - continue - - if y.numel() != probs.size(0): - n = min(int(y.numel()), int(probs.size(0))) - if n == 0: - continue - y = y[:n] - probs = probs[:n] - - valid_mask = (y >= 0) & (y < probs.size(1)) - if not torch.any(valid_mask): - continue - - y = y[valid_mask] - probs = probs[valid_mask] - - conf, preds = probs.max(dim=1) - true_probs = probs.gather(1, y.view(-1, 1)).squeeze(1) - correct = preds.eq(y).float() - - preds_all.extend(preds.detach().cpu().numpy().tolist()) - targets_all.extend(y.detach().cpu().numpy().tolist()) - confidences_all.extend(conf.detach().cpu().numpy().tolist()) - correct_all.extend(correct.detach().cpu().numpy().tolist()) - true_probs_all.extend(true_probs.detach().cpu().numpy().tolist()) - - return ( - np.asarray(preds_all, dtype=int), - np.asarray(targets_all, dtype=int), - np.asarray(confidences_all, dtype=float), - np.asarray(correct_all, dtype=float), - np.asarray(true_probs_all, dtype=float), - ) - - - -def get_overfitting_score(model, train_dataloader, test_accuracy): - """ - Calculates overfitting as the positive train-test accuracy gap. - - Args: - model (torch.nn.Module): Model to evaluate on training data. - train_dataloader (DataLoader): Training dataloader. - test_accuracy (float): Test accuracy in [0, 1]. - - Returns: - float: Positive train-test accuracy gap. - """ - try: - train_accuracy = _get_model_accuracy(model, train_dataloader) - return max(0.0, float(train_accuracy) - float(test_accuracy)) - except Exception as exc: - logger.warning("Could not compute overfitting score") - logger.warning(exc) - return 0.0 - -def get_underfitting_score_local(scenario_name, id): - """ - Gets the local validation accuracy for a specific DFL/SDFL participant. - - Args: - scenario_name (str): Scenario name. - participant_id (int | str): Participant identifier. - - Returns: - float: Validation accuracy. - """ - data = _read_participant_results(scenario_name, id) - return float(data["val_accuracy"].iloc[0]) - -def get_dp_local(scenario_name, id): - """ - Gets the dp metrics for a specific DFL/SDFL participant. - - Args: - scenario_name (str): Scenario name. - participant_id (int | str): Participant identifier. - - Returns: - float: DP Enabled, Epsilon. - """ - data = _read_participant_results(scenario_name, id) - return data["dp_enabled"].iloc[0], float(data["dp_epsilon"].iloc[0]) - - -def get_dp_global(scenario_name): - """ - Gets the aggregated DP metrics for a CFL scenario, excluding the server node. - - Args: - scenario_name (str): Scenario name. - - Returns: - tuple[bool, float | str]: Whether DP is enabled, and the - average epsilon across client nodes. - """ - data = _read_global_results(scenario_name) - - if data["dp_enabled"].iloc[0] == False: - return False, 0.0 - - number_files = len(data) - - avg_epsilon = data["dp_epsilon"].sum() / max(1, number_files - 1) - - return True, avg_epsilon - - -# --------------------------------------------------------------------------- -# Fairness and calibration metrics -# --------------------------------------------------------------------------- - -def get_well_calibration_error(model, test_dataloader, n_bins=10): - """ - Calculates a well-calibration error style metric using prediction confidence. - - For multiclass models, confidence is taken as the max softmax probability and - the observed outcome is whether the prediction is correct. - - Args: - model (torch.nn.Module): Model to evaluate. - test_dataloader (DataLoader): Test dataloader. - n_bins (int): Number of quantile bins. - - Returns: - float: Calibration error in [0, 1] when computation succeeds. - """ - if not isinstance(model, torch.nn.Module): - logger.warning("Model is not a torch.nn.Module") - return 0.0 - - try: - n_bins = max(2, int(n_bins)) - except Exception: - n_bins = 10 - - _, _, confidences, correct, _ = _collect_classification_statistics(model, test_dataloader) - - if len(confidences) == 0 or len(correct) == 0: - return 0.0 - - confidences = np.clip(np.asarray(confidences, dtype=float), 0.0, 1.0) - correct = np.clip(np.asarray(correct, dtype=float), 0.0, 1.0) - - bin_edges = np.linspace(0.0, 1.0, n_bins + 1) - ece = 0.0 - total = float(len(confidences)) - - for idx in range(n_bins): - left = bin_edges[idx] - right = bin_edges[idx + 1] - if idx == n_bins - 1: - mask = (confidences >= left) & (confidences <= right) - else: - mask = (confidences >= left) & (confidences < right) - - if not np.any(mask): - continue - - bin_weight = float(mask.sum()) / total - bin_accuracy = float(correct[mask].mean()) - bin_confidence = float(confidences[mask].mean()) - ece += bin_weight * abs(bin_accuracy - bin_confidence) - - return float(np.clip(ece, 0.0, 1.0)) - - -def get_generalized_entropy_index(model, test_dataloader, alpha=2): - """ - Calculates generalized entropy index from model predictions. - - Args: - model (torch.nn.Module): Model to evaluate. - test_dataloader (DataLoader): Test dataloader. - alpha (float): GEI alpha parameter. - - Returns: - float: Generalized entropy index value. - """ - try: - _, _, _, _, true_class_probs = _collect_classification_statistics(model, test_dataloader) - if len(true_class_probs) == 0: - return 0.0 - - # Use the probability assigned to the true class as a continuous, positive - # benefit. This works consistently for multiclass neural models on both - # images and tabular data, and avoids collapsing the metric to a coarse - # correct/incorrect indicator. - eps = 1e-12 - b = np.clip(np.asarray(true_class_probs, dtype=float), eps, 1.0) - mu = float(np.mean(b)) - if mu <= 0: - return 0.0 - - ratio = np.clip(b / mu, eps, None) - - if alpha == 0: - val = float(np.mean(-np.log(ratio))) - elif alpha == 1: - val = float(np.mean(ratio * np.log(ratio))) - elif alpha == 2: - val = float(np.mean((ratio - 1.0) ** 2) / 2.0) - else: - val = float(np.mean(ratio**alpha - 1.0) / (alpha * (alpha - 1.0))) - - if math.isnan(val) or math.isinf(val): - return 0.0 - return max(0.0, val) - except Exception as exc: - logger.warning("Could not compute generalized entropy index") - logger.warning(exc) - return 0.0 - - -def get_theil_index(model, test_dataloader): - """ - Convenience wrapper for generalized entropy index with alpha=1. - """ - return get_generalized_entropy_index(model, test_dataloader, alpha=1) - - -def get_coefficient_of_variation(model, test_dataloader): - """ - Calculates coefficient of variation from GEI(alpha=2). - """ - try: - gei = get_generalized_entropy_index(model, test_dataloader, alpha=2) - return float(np.sqrt(2 * gei)) - except Exception as exc: - logger.warning("Could not compute coefficient of variation") - logger.warning(exc) - return 0.0 - - -def get_avg_class_imbalance_model_size(scenario_name): - """ - Calculates the mean class imbalance and model size of the nodes. - - Args: - data_results_files (list): Files that contain the class imbalance and model size of the nodes - - Returns: - 2-tupla: The mean class imbalance mean and model size mean of the nodes. - """ - data = _read_global_results(scenario_name) - - number_files = len(data) - - total_class_imbalance = data["class_imbalance"].sum() - total_model_size = data["model_size"].sum() - - avg_class_imbalance = total_class_imbalance / number_files - avg_model_size = total_model_size / number_files - - return avg_class_imbalance, avg_model_size - - -def get_entropy_list(scenario_name): - """ - Obtiene una lista con los valores de entropy de todos los nodos. - - Args: - scenario_name (str): Nombre del escenario. - - Returns: - list: Lista con los valores de entropy - """ - data = _read_global_results(scenario_name) - - entropy_list = data["local_entropy"].tolist() - - return entropy_list - - -# --------------------------------------------------------------------------- -# Explainability metrics -# --------------------------------------------------------------------------- - -def get_feature_importance_cv(model, test_sample): - """ - Calculates the coefficient of variation of the feature importance. - - Args: - model (object): The model. - test_sample (object): One test sample to calculate the feature importance. - - Returns: - float: The coefficient of variation of the feature importance. - """ - - try: - vals = np.asarray(_get_feature_importances(model, test_sample), dtype=float).reshape(-1) - vals = np.nan_to_num(vals, nan=0.0, posinf=0.0, neginf=0.0) - vals = vals[vals > 0] - - if len(vals) <= 1: - return 0.0 - - cv = float(variation(vals)) - if math.isnan(cv) or math.isinf(cv): - return 1.0 - return max(0.0, cv) - except Exception as exc: - logger.warning("Could not compute feature importance CV with shap") - logger.warning(exc) - return 1.0 - - -def _get_feature_importances(model, test_sample): - """ - Computes global feature importances from SHAP values. - - Args: - model (object): The model. - test_sample (object): One test sample batch. - - Returns: - np.ndarray: Global importances per feature. - """ - if not isinstance(model, torch.nn.Module): - logger.warning("Model is not a torch.nn.Module") - return np.array([]) - - def _clone_model(model_ref, device): - optimizer_attrs = ("_optimizer", "_optimizer_override") - optimizer_state = {} - try: - for attr in optimizer_attrs: - if hasattr(model_ref, attr): - optimizer_state[attr] = getattr(model_ref, attr) - setattr(model_ref, attr, None) - - model_clone = copy.deepcopy(model_ref) - for attr in optimizer_attrs: - if hasattr(model_clone, attr): - setattr(model_clone, attr, None) - - model_clone.to(device) - model_clone.eval() - return model_clone - except Exception as exc: - logger.warning("Could not clone model for SHAP, using original model") - logger.warning(exc) - model_ref.eval() - return model_ref - finally: - for attr, value in optimizer_state.items(): - setattr(model_ref, attr, value) - - def _prepare_shap_inputs(sample): - if not (isinstance(sample, (tuple, list)) and len(sample) >= 1): - return None, None, None - - batched_data = sample[0] - if not torch.is_tensor(batched_data) or batched_data.ndim == 0 or batched_data.size(0) == 0: - return None, None, None - - if not torch.is_floating_point(batched_data): - batched_data = batched_data.float() - - batch_size = int(batched_data.size(0)) - input_shape = tuple(int(dim) for dim in batched_data.shape[1:]) - - if batch_size == 1: - return batched_data[:1], batched_data[:1], input_shape - - background_size = min(max(8, batch_size // 4), 32, batch_size - 1) - explainable = batch_size - background_size - explain_size = min(max(4, explainable), 32, explainable) - - background = batched_data[:background_size] - test_data = batched_data[background_size:background_size + explain_size] - - if test_data.size(0) == 0: - test_data = batched_data[: min(batch_size, 32)] - - return background, test_data, input_shape - - def _compute_shap_values(model_ref, background, test_data): - explainer_errors = [] - - for explainer_name in ("DeepExplainer", "GradientExplainer"): - explainer = None - try: - if explainer_name == "DeepExplainer": - explainer = shap.DeepExplainer(model_ref, background) - return explainer.shap_values(test_data, check_additivity=False) - - explainer = shap.GradientExplainer(model_ref, background) - return explainer.shap_values(test_data) - except Exception as exc: - explainer_errors.append(f"{explainer_name}: {exc}") - finally: - # SHAP explainers may register autograd hooks. If we explain on the - # original model, those hooks can leak into later ART metrics. - del explainer - gc.collect() - - raise RuntimeError("; ".join(explainer_errors)) - - def _compute_gradient_importances(model_ref, test_data): - was_training = bool(getattr(model_ref, "training", False)) - model_ref.eval() - - try: - inputs = test_data.detach().clone().requires_grad_(True) - model_ref.zero_grad(set_to_none=True) - - outputs = model_ref(inputs) - if isinstance(outputs, (tuple, list)): - outputs = outputs[0] - - if outputs.ndim == 1: - score = outputs.sum() - else: - score = outputs.reshape(outputs.shape[0], -1).max(dim=1).values.sum() - - score.backward() - if inputs.grad is None: - return np.array([]) - - importances = torch.abs(inputs.grad * inputs).mean(dim=0) - importances = importances.detach().cpu().numpy().reshape(-1) - importances = np.nan_to_num(importances, nan=0.0, posinf=0.0, neginf=0.0) - return np.maximum(importances, 0.0) - finally: - if was_training: - model_ref.train() - - def _feature_axes_from_shape(arr_shape, input_shape, n_samples): - input_shape = tuple(input_shape) - input_rank = len(input_shape) - - if input_rank == 0 or len(arr_shape) < input_rank: - return None - - if len(arr_shape) >= input_rank + 1 and tuple(arr_shape[1:1 + input_rank]) == input_shape: - return tuple(range(1, 1 + input_rank)) - - if len(arr_shape) >= input_rank + 2 and arr_shape[1] == n_samples and tuple(arr_shape[2:2 + input_rank]) == input_shape: - return tuple(range(2, 2 + input_rank)) - - candidates = [] - for start in range(len(arr_shape) - input_rank + 1): - if tuple(arr_shape[start:start + input_rank]) == input_shape: - candidates.append(start) - - if not candidates: - return None - - # Prefer matches that do not consume the leading sample/output axes. - non_leading = [start for start in candidates if start > 0] - if non_leading: - candidates = non_leading - - if len(arr_shape) > 1 and arr_shape[1] == n_samples: - non_output_sample = [start for start in candidates if start > 1] - if non_output_sample: - candidates = non_output_sample - - start = candidates[0] - return tuple(range(start, start + input_rank)) - - try: - try: - device = next(model.parameters()).device - except Exception: - device = torch.device("cpu") - - background, test_data, input_shape = _prepare_shap_inputs(test_sample) - if background is None or test_data is None or input_shape is None: - return np.array([]) - - background = background.to(device) - test_data = test_data.to(device) - - shap_model = _clone_model(model, device) - try: - shap_values = _compute_shap_values(shap_model, background, test_data) - except Exception as exc: - logger.debug("Could not compute feature importances with SHAP, using gradient fallback: %s", exc) - del shap_model - gc.collect() - - gradient_model = _clone_model(model, device) - try: - return _compute_gradient_importances(gradient_model, test_data) - except Exception as fallback_exc: - logger.debug("Could not compute feature importances with gradient fallback: %s", fallback_exc) - return np.array([]) - finally: - del gradient_model - gc.collect() - finally: - if "shap_model" in locals(): - del shap_model - gc.collect() - - if shap_values is None: - return np.array([]) - - if isinstance(shap_values, (list, tuple)): - arrays = [np.asarray(val, dtype=float) for val in shap_values if val is not None] - if not arrays: - return np.array([]) - shap_arr = np.stack(arrays, axis=0) - else: - shap_arr = np.asarray(shap_values, dtype=float) - - if shap_arr.size == 0: - return np.array([]) - - shap_arr = np.nan_to_num(shap_arr, nan=0.0, posinf=0.0, neginf=0.0) - feature_axes = _feature_axes_from_shape(tuple(shap_arr.shape), input_shape, int(test_data.size(0))) - - if feature_axes is None: - # Conservative fallback: treat the first axis as samples when possible and - # flatten the remaining dimensions into features. - if shap_arr.ndim == 1: - importances = np.abs(shap_arr) - else: - aggregate_axes = (0,) - importances = np.mean(np.abs(shap_arr), axis=aggregate_axes) - else: - aggregate_axes = tuple(idx for idx in range(shap_arr.ndim) if idx not in feature_axes) - if aggregate_axes: - importances = np.mean(np.abs(shap_arr), axis=aggregate_axes) - else: - importances = np.abs(shap_arr) - - importances = np.asarray(importances, dtype=float).reshape(-1) - importances = np.nan_to_num(importances, nan=0.0, posinf=0.0, neginf=0.0) - return np.maximum(importances, 0.0) - except Exception as exc: - logger.debug("Could not compute feature importances") - logger.debug(exc) - return np.array([]) - - -def get_alpha_score(model, test_sample, alpha=0.8): - """ - Computes alpha score from global feature importances. - """ - try: - vals = np.asarray(_get_feature_importances(model, test_sample), dtype=float).reshape(-1) - vals = np.nan_to_num(vals, nan=0.0, posinf=0.0, neginf=0.0) - vals = np.maximum(vals, 0.0) - total_features = len(vals) - if total_features == 0 or np.sum(vals) <= 1e-12: - return 1.0 - - try: - alpha = float(alpha) - except Exception: - alpha = 0.8 - alpha = min(max(alpha, 0.0), 1.0) - - vals_sorted = np.sort(vals)[::-1] - cum_sum = np.cumsum(vals_sorted) - threshold = float(alpha) * np.sum(vals_sorted) - idx = np.searchsorted(cum_sum, threshold) - return float(min(total_features, idx + 1) / total_features) - except Exception as exc: - logger.warning("Could not compute alpha score") - logger.warning(exc) - return 1.0 - - -def _get_spread_base(model, test_sample, divergence=True): - vals = _get_feature_importances(model, test_sample) - tol = 1e-8 - - if len(vals) == 0 or np.sum(vals) < tol: - return 0.0 if divergence else 1.0 - if len(vals) == 1: - return 0.0 if divergence else 1.0 - - weights = vals / np.sum(vals) - equal_weights = np.ones(len(vals)) / len(vals) - - if divergence: - metric = jensenshannon(weights, equal_weights, base=2) - else: - denom = entropy(equal_weights) - metric = 0.0 if denom <= tol else entropy(weights) / denom - - if math.isnan(metric) or math.isinf(metric): - return 0.0 if divergence else 1.0 - return float(np.clip(metric, 0.0, 1.0)) - - -def get_spread_ratio(model, test_sample): - """ - Computes spread ratio from global feature importances. - """ - try: - return _get_spread_base(model, test_sample, divergence=False) - except Exception as exc: - logger.warning("Could not compute spread ratio") - logger.warning(exc) - return 1.0 - - -def get_spread_divergence(model, test_sample): - """ - Computes spread divergence from global feature importances. - """ - try: - return _get_spread_base(model, test_sample, divergence=True) - except Exception as exc: - logger.warning("Could not compute spread divergence") - logger.warning(exc) - return 0.0 - - -def get_explainability_metrics_summary(model, test_dataloader, max_batches=4): - """ - Computes explainability metrics over multiple test batches and returns - their mean values. - - Args: - model (object): The model. - test_dataloader: Test dataloader providing batches. - max_batches (int): Maximum number of batches to use. - - Returns: - dict: Mean values for feature_importance_cv, alpha_score, - spread_ratio and spread_divergence. - """ - summary = { - "feature_importance_cv": 1.0, - "alpha_score": 1.0, - "spread_ratio": 1.0, - "spread_divergence": 0.0, - } - - if test_dataloader is None: - return summary - - try: - max_batches = max(1, int(max_batches)) - except Exception: - max_batches = 4 - - fi_values = [] - alpha_values = [] - spread_ratio_values = [] - spread_divergence_values = [] - - try: - for batch_idx, test_sample in enumerate(test_dataloader): - if batch_idx >= max_batches: - break - - fi_values.append(float(get_feature_importance_cv(model, test_sample))) - alpha_values.append(float(get_alpha_score(model, test_sample))) - spread_ratio_values.append(float(get_spread_ratio(model, test_sample))) - spread_divergence_values.append(float(get_spread_divergence(model, test_sample))) - except Exception as exc: - logger.warning("Could not compute explainability metrics summary") - logger.warning(exc) - - if fi_values: - summary["feature_importance_cv"] = float(np.mean(fi_values)) - if alpha_values: - summary["alpha_score"] = float(np.mean(alpha_values)) - if spread_ratio_values: - summary["spread_ratio"] = float(np.mean(spread_ratio_values)) - if spread_divergence_values: - summary["spread_divergence"] = float(np.mean(spread_divergence_values)) - - return summary - - -# --------------------------------------------------------------------------- -# Robustness metrics based on ART estimators -# --------------------------------------------------------------------------- - -def _build_art_classifier(model, input_shape, nb_classes, learning_rate): - criterion = nn.CrossEntropyLoss() - optimizer = optim.Adam(model.parameters(), learning_rate) - - return PyTorchClassifier( - model=model, - loss=criterion, - optimizer=optimizer, - input_shape=tuple(input_shape), - nb_classes=nb_classes, - ) - - -def _validate_test_sample_tensors(test_sample): - if not (isinstance(test_sample, (tuple, list)) and len(test_sample) >= 2): - raise ValueError("`test_sample` must contain samples and labels.") - - samples, labels = test_sample[0], test_sample[1] - if not (torch.is_tensor(samples) and torch.is_tensor(labels) and samples.shape[0] > 0): - raise ValueError("`test_sample` must contain non-empty tensors for samples and labels.") - - return samples, labels - - -def _coerce_max_samples(max_samples, default=8): - try: - return max(1, int(max_samples)) - except Exception: - return default - - -def get_clever_score(model, test_sample, nb_classes, learning_rate, max_samples=8): - """ - Calculates the CLEVER score as the mean score over multiple samples. - - Args: - model (object): The model. - test_sample (object): A batch from the test dataloader. - nb_classes (int): The nb_classes of the model. - learning_rate (float): The learning rate of the model. - max_samples (int): Maximum number of samples from the batch to evaluate. - - Returns: - float: Mean CLEVER score across the selected samples. - """ - samples, _ = _validate_test_sample_tensors(test_sample) - - input_shape = tuple(samples.shape[1:]) if samples.dim() >= 2 else tuple(samples.shape) - - max_samples = _coerce_max_samples(max_samples) - n_samples = min(int(samples.shape[0]), max_samples) - - # Create the ART classifier once and reuse it for all selected samples. - classifier = _build_art_classifier(model, input_shape, nb_classes, learning_rate) - - clever_scores = [] - for idx in range(n_samples): - background = samples[idx].detach().cpu() - sample_np = background.numpy() - - try: - score_untargeted = clever_u( - classifier, - sample_np, - 10, - 5, - R_L2, - norm=2, - pool_factor=3, - verbose=False, - ) - if score_untargeted is not None and not math.isnan(float(score_untargeted)): - clever_scores.append(float(score_untargeted)) - except Exception as exc: - logger.warning("Could not compute CLEVER score for sample index %s", idx) - logger.warning(exc) - - if not clever_scores: - return 0.0 - - return float(np.mean(clever_scores)) - - -# --------------------------------------------------------------------------- -# Sustainability and communication metrics -# --------------------------------------------------------------------------- - -def stop_emissions_tracking_and_save( - tracker: EmissionsTracker, - outdir: str, - emissions_file: str, - role: str, - workload: str, - sample_size: int = 0, - participant_idx=None, -): - """ - Stops emissions tracking object from CodeCarbon and saves relevant information to emissions.csv file. - - Args: - tracker (object): The emissions tracker object holding information. - outdir (str): The path of the output directory of the experiment. - emissions_file (str): The path to the emissions file. - role (str): Either client or server depending on the role. - workload (str): Either aggregation or training depending on the workload. - sample_size (int): The number of samples used for training, if aggregation 0. - """ - - tracker.stop() - - emissions_file = os.path.join(outdir, emissions_file) - - if exists(emissions_file): - df = pd.read_csv(emissions_file) - else: - df = pd.DataFrame( - columns=[ - "id", - "role", - "energy_grid", - "emissions", - "workload", - "CPU_model", - "GPU_model", - ] - ) - try: - energy_grid = (tracker.final_emissions_data.emissions / tracker.final_emissions_data.energy_consumed) * 1000 - df = pd.concat( - [ - df, - pd.DataFrame({ - "id": participant_idx, - "role": role, - "energy_grid": [energy_grid], - "emissions": [tracker.final_emissions_data.emissions], - "workload": workload, - "CPU_model": tracker.final_emissions_data.cpu_model - if tracker.final_emissions_data.cpu_model - else "None", - "GPU_model": tracker.final_emissions_data.gpu_model - if tracker.final_emissions_data.gpu_model - else "None", - "CPU_used": True if tracker.final_emissions_data.cpu_energy else False, - "GPU_used": True if tracker.final_emissions_data.gpu_energy else False, - "energy_consumed": tracker.final_emissions_data.energy_consumed, - "sample_size": sample_size, - }), - ], - ignore_index=True, - ) - df.to_csv(emissions_file, encoding="utf-8", index=False) - except Exception as e: - logger.warning(e) - -def comm_efficiency(bytes_up: int, bytes_down: int, test_acc_avg: float, eps: float = 1e-12) -> float: - """ - Communication efficiency = total_bytes / final_accuracy. - Lower is better. - - Args: - bytes_up: total uploaded bytes - bytes_down: total downloaded bytes - final_accuracy: final test accuracy in [0,1] - eps: small constant to avoid division by zero - - Returns: - float - """ - total_bytes = float(bytes_up) + float(bytes_down) - acc = float(test_acc_avg) - - if acc < eps: - acc = eps - - return total_bytes / acc - - -# --------------------------------------------------------------------------- -# Additional robustness and adversarial metrics -# --------------------------------------------------------------------------- - -def get_loss_sensitivity_score(model, test_sample, nb_classes, learning_rate, max_samples=8): - - """ - Calculates the loss sensitivity score as the mean score over multiple samples. - - Args: - model (object): The model. - test_sample (object): A batch from the test dataloader. - nb_classes (int): The nb_classes of the model. - learning_rate (float): The learning rate of the model. - max_samples (int): Maximum number of samples from the batch to evaluate. - - Returns: - float: Mean loss sensitivity score across the selected samples. - """ - samples, labels = _validate_test_sample_tensors(test_sample) - - max_samples = _coerce_max_samples(max_samples) - n_samples = min(int(samples.shape[0]), max_samples) - - # Create the ART classifier once and reuse it for all selected samples. - classifier = _build_art_classifier(model, samples.shape[1:], nb_classes, learning_rate) - - sensitivity_scores = [] - for idx in range(n_samples): - sample = samples[idx].detach().cpu().unsqueeze(0) - label = labels[idx].detach().cpu().unsqueeze(0) - label = F.one_hot(label, num_classes=nb_classes).float() - - try: - score = loss_sensitivity( - classifier, - sample.numpy(), - label.numpy(), - ) - if score is not None and not math.isnan(float(score)): - sensitivity_scores.append(float(score)) - except Exception as exc: - logger.warning("Could not compute loss sensitivity for sample index %s", idx) - logger.warning(exc) - - if not sensitivity_scores: - return 0.0 - - return float(np.mean(sensitivity_scores)) - -def compute_adversarial_accuracy_art( - model, - test_loader, - nb_classes, - learning_rate, - epsilon=0.03 -): - """ - Computes adversarial accuracy using FGSM attack. - - Args: - model (object): The model. - test_loader (DataLoader): DataLoader providing test samples. - nb_classes (int): The nb_classes of the model. - learning_rate (float): The learning rate of the model. - epsilon (float): Maximum perturbation magnitude for the attacks. - - Returns: - float: The adversarial accuracy score. - """ - - device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - model.eval() - model.to(device) - - correct = 0 - total = 0 - - for samples, labels in test_loader: - samples = samples.to(device) - labels = labels.to(device) - - x_adv = fgsm_attack(model, samples, labels, epsilon=epsilon) - - with torch.no_grad(): - outputs = model(x_adv) - logits = outputs[0] if isinstance(outputs, (tuple, list)) else outputs - preds = logits.argmax(dim=1) - - correct += (preds == labels).sum().item() - total += labels.size(0) - - return correct / total if total > 0 else 0.0 - -def get_empirical_robustness_score( - model, - test_sample, - nb_classes, - learning_rate, - attack_name = "fgsm", - attack_params = None, - max_samples = 128, -): - """ - Calculates the Empirical Robustness score using Adversarial Robustness Toolbox (ART). - - Args: - model (object): The model. - test_sample (object): A batch from the test dataloader (samples, labels). - nb_classes (int): The nb_classes of the model. - learning_rate (float): The learning rate of the model. - attack_name (str): Attack key supported by ART empirical_robustness. - attack_params (dict | None): Optional attack parameters. - max_samples (int): Max number of samples from the batch to use. - - Returns: - float: Empirical robustness score (>= 0.0). If it cannot be computed, returns 0.0. - """ - try: - samples, _ = _validate_test_sample_tensors(test_sample) - - batch_size: int = int(samples.shape[0]) - n: int = int(min(max_samples, batch_size)) - x = samples[:n].detach().cpu().numpy() - - classifier = _build_art_classifier(model, samples.shape[1:], nb_classes, learning_rate) - - score = empirical_robustness( - classifier=classifier, - x=x, - attack_name=attack_name, - attack_params=attack_params, - ) - - if isinstance(score, np.ndarray): - score = float(np.mean(score)) - - if score is None or (isinstance(score, float) and math.isnan(score)): - return 0.0 - - return float(score) - - except Exception as exc: - logger.warning("Could not compute empirical robustness (ART). Returning 0.0") - logger.warning(exc) - return 0.0 - - - -def _get_image_normalization_for_samples(samples): - if not isinstance(samples, torch.Tensor) or samples.ndim < 4: - return None - - channels = int(samples.shape[1]) - if channels == 1: - return (0.5,), (0.5,) - if channels == 3: - return (0.4914, 0.4822, 0.4465), (0.2471, 0.2435, 0.2616) - return None - - -def _channel_tensor(values, samples): - shape = [1, len(values)] + [1] * max(samples.dim() - 2, 0) - return torch.tensor(values, dtype=samples.dtype, device=samples.device).view(*shape) - - -def _fgsm_step_and_clamp(samples, grad, epsilon): - normalization = _get_image_normalization_for_samples(samples) - if normalization is None: - return samples + epsilon * grad.sign() - - mean, std = normalization - mean = _channel_tensor(mean, samples) - std = _channel_tensor(std, samples) - - normalized_epsilon = float(epsilon) / std - lower = (0.0 - mean) / std - upper = (1.0 - mean) / std - - x_adv = samples + normalized_epsilon * grad.sign() - x_adv = torch.max(torch.min(x_adv, samples + normalized_epsilon), samples - normalized_epsilon) - return torch.max(torch.min(x_adv, upper), lower) - - -def fgsm_attack(model, samples, labels, epsilon=0.03): - """ - Performs an FGSM (Fast Gradient Sign Method) adversarial attack on a batch of samples. - - Args: - model (torch.nn.Module): The PyTorch model to attack. - samples (torch.Tensor): Input samples to perturb, shape (B, ...). - labels (torch.Tensor): True labels corresponding to the samples. - epsilon (float, optional): Maximum perturbation magnitude for the attack. Defaults to 0.03. - - Returns: - torch.Tensor: Adversarially perturbed samples with the same shape as `samples`. - """ - try: - device = next(model.parameters()).device - except Exception: - device = samples.device - - samples = samples.clone().detach().to(device) - labels = labels.to(device) - samples.requires_grad = True - - outputs = model(samples) - logits = outputs[0] if isinstance(outputs, (tuple, list)) else outputs - loss = nn.CrossEntropyLoss()(logits, labels) - grad = torch.autograd.grad(loss, samples, only_inputs=True)[0] - x_adv = _fgsm_step_and_clamp(samples, grad, epsilon) - - return x_adv.detach() - -def get_confidence_score( - model, - test_sample, - max_samples = 128, - use_true_label = True, -): - """ - Calculates the confidence score. - - Args: - model (object): The model. - test_sample (object): A batch from the test dataloader (samples, labels). - max_samples (int): Max number of samples from the batch to use. - use_true_label (bool): Whether to compute confidence with respect to the true labels. Defaults to True. - - Returns: - float: Confidence score. - """ - try: - if not isinstance(model, torch.nn.Module): - logger.warning("Model is not a torch.nn.Module") - return 0.0 - - x, y = test_sample - - if isinstance(x, torch.Tensor): - x = x[:max_samples] - if isinstance(y, torch.Tensor): - y = y[:max_samples] - - try: - device = next(model.parameters()).device - except Exception: - device = torch.device("cpu") - - model.eval() - with torch.no_grad(): - x = x.to(device) if isinstance(x, torch.Tensor) else x - out = model(x) - - logits = out[0] if isinstance(out, (tuple, list)) else out - probs = torch.softmax(logits, dim=1) - - if use_true_label and isinstance(y, torch.Tensor): - if y.ndim > 1: - y_idx = torch.argmax(y, dim=1) - else: - y_idx = y - y_idx = y_idx.to(device) - - true_probs = probs.gather(1, y_idx.view(-1, 1)).squeeze(1) - return float(true_probs.mean().detach().cpu().item()) - - msp = probs.max(dim=1).values - return float(msp.mean().detach().cpu().item()) - - except Exception as e: - logger.warning("Could not compute confidence score") - logger.warning(e) - return 0.0 - -def attack_success_rate(model, test_sample,epsilon=0.03): - """ - Calculates the ASR. - - Args: - model (object): The model. - test_sample (object): A batch from the test dataloader (samples, labels). - epsilon (float): Maximum perturbation magnitude for the attacks. - - Returns: - float: The ASR. - """ - device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - model.eval() - model.to(device) - - images, labels = test_sample - images = images.to(device) - labels = labels.to(device) - - with torch.no_grad(): - outputs = model(images) - logits = outputs[0] if isinstance(outputs, (tuple, list)) else outputs - preds = logits.argmax(dim=1) - - correct_mask = preds.eq(labels) - num_correct = correct_mask.sum().item() - - if num_correct == 0: - return 0.0 - - x_adv = fgsm_attack(model, images, labels, epsilon=epsilon) - - with torch.no_grad(): - outputs_adv = model(x_adv) - logits_adv = outputs_adv[0] if isinstance(outputs_adv, (tuple, list)) else outputs_adv - preds_adv = logits_adv.argmax(dim=1) - - successful_attacks = (correct_mask & preds_adv.ne(labels)).sum().item() - - asr = successful_attacks / num_correct - - return asr diff --git a/nebula/addons/trustworthiness/dfl_factsheet.py b/nebula/addons/trustworthiness/dfl_factsheet.py index 3f32e8b9e..5be3ee012 100644 --- a/nebula/addons/trustworthiness/dfl_factsheet.py +++ b/nebula/addons/trustworthiness/dfl_factsheet.py @@ -2,13 +2,22 @@ import os import pandas as pd -from nebula.addons.trustworthiness.calculation import ( +from nebula.addons.trustworthiness.helpers.csv_io import ( + load_data_results_participant, + load_emissions_participant, +) +from nebula.addons.trustworthiness.helpers.data_distribution import ( + get_all_data_entropy, + get_local_class_imbalance_score, + get_local_normalized_entropy, +) +from nebula.addons.trustworthiness.helpers.privacy import ( + get_global_privacy_risk_dfl, +) +from nebula.addons.trustworthiness.helpers.scenario_metrics import ( get_bytes_model, get_dp_local, get_elapsed_time, - get_global_privacy_risk_dfl, - get_local_class_imbalance_score, - get_local_normalized_entropy, get_underfitting_score_local, ) from nebula.addons.trustworthiness.factsheet_common import ( @@ -24,7 +33,6 @@ write_factsheet, ) from nebula.addons.trustworthiness.factsheet_populators import populate_profile_metrics -from nebula.addons.trustworthiness.utils import read_csv, get_all_data_entropy logger = logging.getLogger(__name__) diff --git a/nebula/addons/trustworthiness/factsheet.py b/nebula/addons/trustworthiness/factsheet.py index 7c23f20c2..0417efdc2 100755 --- a/nebula/addons/trustworthiness/factsheet.py +++ b/nebula/addons/trustworthiness/factsheet.py @@ -4,16 +4,22 @@ import numpy as np import pandas as pd -from nebula.addons.trustworthiness.calculation import ( +from nebula.addons.trustworthiness.helpers.csv_io import read_csv +from nebula.addons.trustworthiness.helpers.data_distribution import ( + get_class_imbalance_score, + get_cv, +) +from nebula.addons.trustworthiness.helpers.factsheet_values import check_field_filled +from nebula.addons.trustworthiness.helpers.privacy import ( + get_global_privacy_risk, +) +from nebula.addons.trustworthiness.helpers.scenario_metrics import ( get_avg_class_imbalance_model_size, get_avg_loss_accuracy, get_bytes_sent_recv, - get_class_imbalance_score, - get_cv, get_dp_global, get_elapsed_time, get_entropy_list, - get_global_privacy_risk, get_participant_loss_accuracy, get_underfitting_score, ) @@ -31,7 +37,6 @@ write_factsheet, ) from nebula.addons.trustworthiness.factsheet_populators import populate_profile_metrics -from nebula.addons.trustworthiness.utils import read_csv, check_field_filled # from nebula.core.models.syscall.mlp import SyscallModelMLP logger = logging.getLogger(__name__) diff --git a/nebula/addons/trustworthiness/factsheet_populators.py b/nebula/addons/trustworthiness/factsheet_populators.py index d5cce3371..5ace6b034 100644 --- a/nebula/addons/trustworthiness/factsheet_populators.py +++ b/nebula/addons/trustworthiness/factsheet_populators.py @@ -2,23 +2,29 @@ import logging -from nebula.addons.trustworthiness.calculation import ( - attack_success_rate, - compute_adversarial_accuracy_art, - get_clever_score, - get_coefficient_of_variation, - get_confidence_score, - get_empirical_robustness_score, - get_epsilon_star, +from nebula.addons.trustworthiness.helpers.explainability import ( get_explainability_metrics_summary, +) +from nebula.addons.trustworthiness.helpers.model_quality import ( + get_coefficient_of_variation, get_generalized_entropy_index, - get_loss_sensitivity_score, get_macro_f1_score, - get_mia_auc, get_overfitting_score, get_theil_index, get_well_calibration_error, ) +from nebula.addons.trustworthiness.helpers.privacy import ( + get_epsilon_star, + get_mia_auc, +) +from nebula.addons.trustworthiness.helpers.robustness import ( + attack_success_rate, + compute_adversarial_accuracy_art, + get_clever_score, + get_confidence_score, + get_empirical_robustness_score, + get_loss_sensitivity_score, +) logger = logging.getLogger(__name__) from nebula.addons.trustworthiness.factsheet_common import ( diff --git a/nebula/addons/trustworthiness/helpers/__init__.py b/nebula/addons/trustworthiness/helpers/__init__.py new file mode 100644 index 000000000..4ef3ae023 --- /dev/null +++ b/nebula/addons/trustworthiness/helpers/__init__.py @@ -0,0 +1 @@ +"""Small helper modules for trustworthiness calculations and persistence.""" diff --git a/nebula/addons/trustworthiness/helpers/csv_io.py b/nebula/addons/trustworthiness/helpers/csv_io.py new file mode 100644 index 000000000..40bd7fda0 --- /dev/null +++ b/nebula/addons/trustworthiness/helpers/csv_io.py @@ -0,0 +1,316 @@ +import csv +import json +import logging +import os + +import pandas as pd + +logger = logging.getLogger(__name__) + +# CSV schemas used by trustworthiness outputs. Keeping column order centralized +# avoids subtle differences between append writes and full report exports. +DATA_RESULTS_COLUMNS = [ + "id", + "bytes_sent", + "bytes_recv", + "accuracy", + "loss", + "val_accuracy", + "dp_enabled", + "dp_epsilon", +] + +CFL_DATA_RESULTS_COLUMNS = [ + "id", + "bytes_sent", + "bytes_recv", + "accuracy", + "loss", + "class_imbalance", + "model_size", + "local_entropy", + "val_accuracy", + "dp_enabled", + "dp_epsilon", +] + +EMISSIONS_COLUMNS = [ + "id", + "role", + "energy_grid", + "emissions", + "workload", + "CPU_model", + "GPU_model", + "CPU_used", + "GPU_used", + "energy_consumed", + "sample_size", +] + + +def _logs_dir(): + # Prefer the runtime logs directory; keep the historical app path as fallback. + return os.environ.get("NEBULA_LOGS_DIR") or os.path.join("nebula", "app", "logs") + + +def _trustworthiness_dir(scenario_name: str) -> str: + # Every scenario stores trustworthiness artifacts in this subdirectory. + return os.path.join(_logs_dir(), scenario_name, "trustworthiness") + + +def _trustworthiness_path(scenario_name: str, filename: str) -> str: + # Build a concrete artifact path for a scenario. + return os.path.join(_trustworthiness_dir(scenario_name), filename) + + +def _ensure_parent_dir(file_path: str) -> None: + # Ensure CSV/JSON writes work even when the trust folder was not created yet. + directory = os.path.dirname(file_path) + if directory: + os.makedirs(directory, exist_ok=True) + + +def _read_first_csv_row(file_path: str) -> dict: + # Per-participant summary CSVs are expected to contain one current row. + if not os.path.exists(file_path): + raise FileNotFoundError(f"File not found: {file_path}") + + with open(file_path, "r", newline="") as csv_file: + rows = list(csv.DictReader(csv_file)) + + if not rows: + raise ValueError(f"No rows found in {file_path}") + + return rows[0] + + +def _read_or_empty_dataframe(file_path: str, columns: list[str]) -> pd.DataFrame: + # Append flows start from the existing CSV or from an empty schema. + if os.path.exists(file_path): + return pd.read_csv(file_path) + + return pd.DataFrame(columns=columns) + + +def _append_csv_row(file_path: str, columns: list[str], row: dict) -> None: + # Preserve the declared schema and ignore any unexpected keys in row. + _ensure_parent_dir(file_path) + df = _read_or_empty_dataframe(file_path, columns) + new_row = pd.DataFrame([{column: row.get(column) for column in columns}]) + pd.concat([df, new_row], ignore_index=True).to_csv(file_path, encoding="utf-8", index=False) + + +def _write_csv_rows(file_path: str, fieldnames: list[str], rows: list[dict]) -> None: + # Aggregate reports replace the previous CSV content in one write. + _ensure_parent_dir(file_path) + with open(file_path, "w", newline="") as csv_file: + writer = csv.DictWriter(csv_file, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(rows) + + +def _to_bool(value) -> bool: + # DictReader returns strings, while some tests/builders may pass booleans. + return str(value).strip().lower() == "true" + + +def read_csv(filename): + # Missing optional CSVs are represented as None for existing callers. + if os.path.exists(filename): + return pd.read_csv(filename) + + return None + + +def write_results_json(out_file, data): + # Trust metric evaluation appends one result object per evaluation call. + _ensure_parent_dir(out_file) + with open(out_file, "a", encoding="utf-8") as file: + json.dump(data, file, indent=4) + + +def load_data_results_participant(experiment_name: str, participant_id: int | str): + # Load the DFL/SDFL participant training summary written by save_results_csv. + row = _read_first_csv_row( + _trustworthiness_path(experiment_name, f"data_results_{participant_id}.csv") + ) + + return ( + int(float(row["bytes_sent"])), + int(float(row["bytes_recv"])), + float(row["accuracy"]), + float(row["loss"]), + float(row["val_accuracy"]), + _to_bool(row["dp_enabled"]), + float(row["dp_epsilon"]), + ) + + +def load_emissions_participant(experiment_name: str, participant_id: int | str): + # Load the DFL/SDFL participant CodeCarbon summary. + row = _read_first_csv_row( + _trustworthiness_path(experiment_name, f"emissions_{participant_id}.csv") + ) + + return ( + str(row["role"]), + float(row["energy_grid"]), + float(row["emissions"]), + str(row["workload"]), + str(row["CPU_model"]), + str(row["GPU_model"]), + _to_bool(row["CPU_used"]), + _to_bool(row["GPU_used"]), + float(row["energy_consumed"]), + int(float(row["sample_size"])), + ) + + +def save_trustworthiness_reports_csv( + reports: dict, + experiment_name: str, +) -> None: + # Server-side CFL flow exports one aggregate data CSV and one emissions CSV. + sorted_reports = sorted(reports.values(), key=lambda report: int(report["node_id"])) + + data_rows = [ + { + "id": report["node_id"], + "bytes_sent": report["bytes_sent"], + "bytes_recv": report["bytes_recv"], + "accuracy": report["accuracy"], + "loss": report["loss"], + "class_imbalance": report["class_imbalance"], + "model_size": report["model_size"], + "local_entropy": report["local_entropy"], + "val_accuracy": report["val_accuracy"], + "dp_enabled": report["dp_enabled"], + "dp_epsilon": report["dp_epsilon"], + } + for report in sorted_reports + ] + emissions_rows = [ + { + "id": report["node_id"], + "role": report["role"], + "energy_grid": report["energy_grid"], + "emissions": report["emissions"], + "workload": report["workload"], + "CPU_model": report["cpu_model"], + "GPU_model": report["gpu_model"], + "CPU_used": report["cpu_used"], + "GPU_used": report["gpu_used"], + "energy_consumed": report["energy_consumed"], + "sample_size": report["sample_size"], + } + for report in sorted_reports + ] + + data_results_path = _trustworthiness_path(experiment_name, "data_results.csv") + emissions_path = _trustworthiness_path(experiment_name, "emissions.csv") + + _write_csv_rows(data_results_path, CFL_DATA_RESULTS_COLUMNS, data_rows) + _write_csv_rows(emissions_path, EMISSIONS_COLUMNS, emissions_rows) + + logger.info( + "[TW SERVER] CSV files written correctly: %s, %s", + data_results_path, + emissions_path, + ) + + +def save_results_csv_cfl( + scenario_name: str, + id: int, + bytes_sent: int, + bytes_recv: int, + accuracy: float, + loss: float, + class_imbalance: float, + model_size: int, + local_entropy: float, + val_accuracy: float, + dp_enabled: bool, + dp_epsilon: float, +): + # Append one participant to the centralized data-results CSV. + _append_csv_row( + _trustworthiness_path(scenario_name, "data_results.csv"), + CFL_DATA_RESULTS_COLUMNS, + { + "id": id, + "bytes_sent": bytes_sent, + "bytes_recv": bytes_recv, + "accuracy": accuracy, + "loss": loss, + "class_imbalance": class_imbalance, + "model_size": model_size, + "local_entropy": local_entropy, + "val_accuracy": val_accuracy, + "dp_enabled": dp_enabled, + "dp_epsilon": dp_epsilon, + }, + ) + + +def save_emissions_csv_cfl( + scenario_name: str, + id: int, + role: str, + energy_grid: float, + emissions: float, + workload: str, + cpu_model: str, + gpu_model: str, + cpu_used: bool, + gpu_used: bool, + energy_consumed: float, + sample_size: int, +): + # Append one participant to the centralized emissions CSV. + _append_csv_row( + _trustworthiness_path(scenario_name, "emissions.csv"), + EMISSIONS_COLUMNS, + { + "id": id, + "role": role, + "energy_grid": energy_grid, + "emissions": emissions, + "workload": workload, + "CPU_model": cpu_model, + "GPU_model": gpu_model, + "CPU_used": cpu_used, + "GPU_used": gpu_used, + "energy_consumed": energy_consumed, + "sample_size": sample_size, + }, + ) + + +def save_results_csv( + scenario_name: str, + id: int, + bytes_sent: int, + bytes_recv: int, + accuracy: float, + loss: float, + val_accuracy: float, + dp_enabled: bool, + dp_epsilon: float, +): + # Local DFL/SDFL nodes persist their own data-results CSV before exchange. + _append_csv_row( + _trustworthiness_path(scenario_name, f"data_results_{id}.csv"), + DATA_RESULTS_COLUMNS, + { + "id": id, + "bytes_sent": bytes_sent, + "bytes_recv": bytes_recv, + "accuracy": accuracy, + "loss": loss, + "val_accuracy": val_accuracy, + "dp_enabled": dp_enabled, + "dp_epsilon": dp_epsilon, + }, + ) diff --git a/nebula/addons/trustworthiness/helpers/data_distribution.py b/nebula/addons/trustworthiness/helpers/data_distribution.py new file mode 100644 index 000000000..6a118019b --- /dev/null +++ b/nebula/addons/trustworthiness/helpers/data_distribution.py @@ -0,0 +1,178 @@ +import json +import os +from collections import Counter + +import numpy as np +from hashids import Hashids +from scipy.stats import entropy + +hashids = Hashids() + + +def _logs_dir(): + # Return the base logs directory used to read and write trust artifacts. + return os.environ.get("NEBULA_LOGS_DIR") or os.path.join("nebula", "app", "logs") + + +def _trustworthiness_dir(scenario_name: str) -> str: + # Return the trustworthiness directory for a scenario. + return os.path.join(_logs_dir(), scenario_name, "trustworthiness") + + +def _trustworthiness_path(scenario_name: str, filename: str) -> str: + # Return the path of a trustworthiness artifact for a scenario. + return os.path.join(_trustworthiness_dir(scenario_name), filename) + + +def _ensure_trustworthiness_dir(scenario_name: str) -> None: + # Create the scenario trustworthiness directory if it does not exist. + os.makedirs(_trustworthiness_dir(scenario_name), exist_ok=True) + + +def _encode_class_id(class_id) -> str: + # Convert a numeric class ID into the hash used in persisted JSON files. + return hashids.encode(int(class_id)) + + +def _class_counts_from_counter(class_counter: Counter) -> dict: + # Return hashed class counts from an in-memory Counter. + return { + _encode_class_id(class_id): int(count) + for class_id, count in class_counter.items() + } + + +def _write_json(scenario_name: str, filename: str, data: dict, indent=None) -> None: + # Write a JSON trust artifact inside the scenario trustworthiness directory. + _ensure_trustworthiness_dir(scenario_name) + with open(_trustworthiness_path(scenario_name, filename), "w") as file: + json.dump(data, file, indent=indent) + + +def _iter_participant_class_counts(experiment_name: str): + # Yield each consecutive participant ID and its saved class-count dictionary. + participant_id = 0 + while True: + file_path = get_class_count_file(experiment_name, participant_id) + if not os.path.exists(file_path): + break + + yield participant_id, load_class_counts(experiment_name, participant_id) + participant_id += 1 + + +def get_class_count_file(scenario_name, participant_id): + # Return the class-count JSON path for one participant. + return _trustworthiness_path(scenario_name, f"{str(participant_id)}_class_count.json") + + +def load_class_counts(scenario_name, participant_id): + # Load one participant's saved class-count dictionary. + with open(get_class_count_file(scenario_name, participant_id), "r") as file: + return json.load(file) + + +def get_class_imbalance_from_counts(class_counts): + # Calculate class imbalance as the coefficient of variation of class counts. + return get_cv(list=list(class_counts.values())) + + +def get_class_imbalance_score(class_imbalance): + # Convert class imbalance into a score where 1 means balanced classes. + return 1 / (1 + class_imbalance) + + +def get_class_imbalance_local(participant_id, experiment_name): + # Return the raw class-imbalance value for one participant. + return get_class_imbalance_from_counts(load_class_counts(experiment_name, participant_id)) + + +def get_local_class_imbalance_score(scenario_name, participant_id): + # Return the trust-oriented class-imbalance score for one participant. + return get_class_imbalance_score(get_class_imbalance_local(participant_id, scenario_name)) + + +def get_entropy_from_class_counts(class_counts, normalize=False): + # Calculate entropy from a class-count dictionary, optionally normalized to [0, 1]. + counts = np.array(list(class_counts.values()), dtype=float) + total = counts.sum() + if total <= 0: + return 0.0 + + probabilities = counts / total + entropy_value = entropy(probabilities, base=2) + + if not normalize: + return round(float(entropy_value), 6) + + class_count = len(probabilities) + if class_count <= 1: + return 0.0 + + normalized_entropy = float(entropy_value / np.log2(class_count)) + return float(np.clip(normalized_entropy, 0.0, 1.0)) + + +def get_local_normalized_entropy(scenario_name, participant_id): + # Return normalized entropy for one participant's saved class counts. + return get_entropy_from_class_counts( + load_class_counts(scenario_name, participant_id), + normalize=True, + ) + + +def get_cv(list=None, std=None, mean=None): + # Return the coefficient of variation from either values or precomputed std/mean. + if std is not None and mean is not None: + return 0 if mean == 0 else std / mean + + if list is None: + return 0 + + values = np.asarray(list, dtype=float) + mean_value = float(np.mean(values)) if values.size else 0.0 + if mean_value == 0: + return 0 + + return float(np.std(values) / mean_value) + + +def get_participation_variation_score(participation_counts): + # Convert participation-count dispersion into a score where 1 means equal participation. + if not participation_counts: + return 1.0 + + counts = np.asarray(participation_counts, dtype=float) + mean_count = float(np.mean(counts)) + if mean_count <= 0: + return 0.0 + + cv = get_cv(list=counts) + if not np.isfinite(cv): + return 0.0 + + return float(1 / (1 + cv)) + + +def save_class_count_per_participant(experiment_name, class_counter: Counter, idx): + # Save one participant's class-count dictionary as _class_count.json. + _write_json( + experiment_name, + f"{str(idx)}_class_count.json", + _class_counts_from_counter(class_counter), + ) + + +def get_all_data_entropy(experiment_name): + # Compute entropy for every participant class-count file and write entropy.json. + entropy_per_participant = { + str(participant_id): round(get_entropy_from_class_counts(class_count), 6) + for participant_id, class_count in _iter_participant_class_counts(experiment_name) + } + + _write_json(experiment_name, "entropy.json", entropy_per_participant, indent=2) + + +def get_local_entropy(id, experiment_name): + # Return non-normalized entropy for one participant's saved class counts. + return get_entropy_from_class_counts(load_class_counts(experiment_name, id)) diff --git a/nebula/addons/trustworthiness/helpers/explainability.py b/nebula/addons/trustworthiness/helpers/explainability.py new file mode 100644 index 000000000..ce9809c3e --- /dev/null +++ b/nebula/addons/trustworthiness/helpers/explainability.py @@ -0,0 +1,407 @@ +import copy +import gc +import logging +import math + +import numpy as np +import shap +import torch +from scipy.spatial.distance import jensenshannon +from scipy.stats import entropy, variation + +logger = logging.getLogger(__name__) + +def get_feature_importance_cv(model, test_sample): + """ + Calculates the coefficient of variation of the feature importance. + + Args: + model (object): The model. + test_sample (object): One test sample to calculate the feature importance. + + Returns: + float: The coefficient of variation of the feature importance. + """ + + try: + vals = np.asarray(_get_feature_importances(model, test_sample), dtype=float).reshape(-1) + vals = np.nan_to_num(vals, nan=0.0, posinf=0.0, neginf=0.0) + vals = vals[vals > 0] + + if len(vals) <= 1: + return 0.0 + + cv = float(variation(vals)) + if math.isnan(cv) or math.isinf(cv): + return 1.0 + return max(0.0, cv) + except Exception as exc: + logger.warning("Could not compute feature importance CV with shap") + logger.warning(exc) + return 1.0 + + +def _get_feature_importances(model, test_sample): + """ + Computes global feature importances from SHAP values. + + Args: + model (object): The model. + test_sample (object): One test sample batch. + + Returns: + np.ndarray: Global importances per feature. + """ + if not isinstance(model, torch.nn.Module): + logger.warning("Model is not a torch.nn.Module") + return np.array([]) + + def _clone_model(model_ref, device): + optimizer_attrs = ("_optimizer", "_optimizer_override") + optimizer_state = {} + try: + for attr in optimizer_attrs: + if hasattr(model_ref, attr): + optimizer_state[attr] = getattr(model_ref, attr) + setattr(model_ref, attr, None) + + model_clone = copy.deepcopy(model_ref) + for attr in optimizer_attrs: + if hasattr(model_clone, attr): + setattr(model_clone, attr, None) + + model_clone.to(device) + model_clone.eval() + return model_clone + except Exception as exc: + logger.warning("Could not clone model for SHAP, using original model") + logger.warning(exc) + model_ref.eval() + return model_ref + finally: + for attr, value in optimizer_state.items(): + setattr(model_ref, attr, value) + + def _prepare_shap_inputs(sample): + if not (isinstance(sample, (tuple, list)) and len(sample) >= 1): + return None, None, None + + batched_data = sample[0] + if not torch.is_tensor(batched_data) or batched_data.ndim == 0 or batched_data.size(0) == 0: + return None, None, None + + if not torch.is_floating_point(batched_data): + batched_data = batched_data.float() + + batch_size = int(batched_data.size(0)) + input_shape = tuple(int(dim) for dim in batched_data.shape[1:]) + + if batch_size == 1: + return batched_data[:1], batched_data[:1], input_shape + + background_size = min(max(8, batch_size // 4), 32, batch_size - 1) + explainable = batch_size - background_size + explain_size = min(max(4, explainable), 32, explainable) + + background = batched_data[:background_size] + test_data = batched_data[background_size:background_size + explain_size] + + if test_data.size(0) == 0: + test_data = batched_data[: min(batch_size, 32)] + + return background, test_data, input_shape + + def _compute_shap_values(model_ref, background, test_data): + explainer_errors = [] + + for explainer_name in ("DeepExplainer", "GradientExplainer"): + explainer = None + try: + if explainer_name == "DeepExplainer": + explainer = shap.DeepExplainer(model_ref, background) + return explainer.shap_values(test_data, check_additivity=False) + + explainer = shap.GradientExplainer(model_ref, background) + return explainer.shap_values(test_data) + except Exception as exc: + explainer_errors.append(f"{explainer_name}: {exc}") + finally: + # SHAP explainers may register autograd hooks. If we explain on the + # original model, those hooks can leak into later ART metrics. + del explainer + gc.collect() + + raise RuntimeError("; ".join(explainer_errors)) + + def _compute_gradient_importances(model_ref, test_data): + was_training = bool(getattr(model_ref, "training", False)) + model_ref.eval() + + try: + inputs = test_data.detach().clone().requires_grad_(True) + model_ref.zero_grad(set_to_none=True) + + outputs = model_ref(inputs) + if isinstance(outputs, (tuple, list)): + outputs = outputs[0] + + if outputs.ndim == 1: + score = outputs.sum() + else: + score = outputs.reshape(outputs.shape[0], -1).max(dim=1).values.sum() + + score.backward() + if inputs.grad is None: + return np.array([]) + + importances = torch.abs(inputs.grad * inputs).mean(dim=0) + importances = importances.detach().cpu().numpy().reshape(-1) + importances = np.nan_to_num(importances, nan=0.0, posinf=0.0, neginf=0.0) + return np.maximum(importances, 0.0) + finally: + if was_training: + model_ref.train() + + def _feature_axes_from_shape(arr_shape, input_shape, n_samples): + input_shape = tuple(input_shape) + input_rank = len(input_shape) + + if input_rank == 0 or len(arr_shape) < input_rank: + return None + + if len(arr_shape) >= input_rank + 1 and tuple(arr_shape[1:1 + input_rank]) == input_shape: + return tuple(range(1, 1 + input_rank)) + + if len(arr_shape) >= input_rank + 2 and arr_shape[1] == n_samples and tuple(arr_shape[2:2 + input_rank]) == input_shape: + return tuple(range(2, 2 + input_rank)) + + candidates = [] + for start in range(len(arr_shape) - input_rank + 1): + if tuple(arr_shape[start:start + input_rank]) == input_shape: + candidates.append(start) + + if not candidates: + return None + + # Prefer matches that do not consume the leading sample/output axes. + non_leading = [start for start in candidates if start > 0] + if non_leading: + candidates = non_leading + + if len(arr_shape) > 1 and arr_shape[1] == n_samples: + non_output_sample = [start for start in candidates if start > 1] + if non_output_sample: + candidates = non_output_sample + + start = candidates[0] + return tuple(range(start, start + input_rank)) + + try: + try: + device = next(model.parameters()).device + except Exception: + device = torch.device("cpu") + + background, test_data, input_shape = _prepare_shap_inputs(test_sample) + if background is None or test_data is None or input_shape is None: + return np.array([]) + + background = background.to(device) + test_data = test_data.to(device) + + shap_model = _clone_model(model, device) + try: + shap_values = _compute_shap_values(shap_model, background, test_data) + except Exception as exc: + logger.debug("Could not compute feature importances with SHAP, using gradient fallback: %s", exc) + shap_model = None + gc.collect() + + gradient_model = _clone_model(model, device) + try: + return _compute_gradient_importances(gradient_model, test_data) + except Exception as fallback_exc: + logger.debug("Could not compute feature importances with gradient fallback: %s", fallback_exc) + return np.array([]) + finally: + del gradient_model + gc.collect() + finally: + if shap_model is not None: + del shap_model + gc.collect() + + if shap_values is None: + return np.array([]) + + if isinstance(shap_values, (list, tuple)): + arrays = [np.asarray(val, dtype=float) for val in shap_values if val is not None] + if not arrays: + return np.array([]) + shap_arr = np.stack(arrays, axis=0) + else: + shap_arr = np.asarray(shap_values, dtype=float) + + if shap_arr.size == 0: + return np.array([]) + + shap_arr = np.nan_to_num(shap_arr, nan=0.0, posinf=0.0, neginf=0.0) + feature_axes = _feature_axes_from_shape(tuple(shap_arr.shape), input_shape, int(test_data.size(0))) + + if feature_axes is None: + # Conservative fallback: treat the first axis as samples when possible and + # flatten the remaining dimensions into features. + if shap_arr.ndim == 1: + importances = np.abs(shap_arr) + else: + aggregate_axes = (0,) + importances = np.mean(np.abs(shap_arr), axis=aggregate_axes) + else: + aggregate_axes = tuple(idx for idx in range(shap_arr.ndim) if idx not in feature_axes) + if aggregate_axes: + importances = np.mean(np.abs(shap_arr), axis=aggregate_axes) + else: + importances = np.abs(shap_arr) + + importances = np.asarray(importances, dtype=float).reshape(-1) + importances = np.nan_to_num(importances, nan=0.0, posinf=0.0, neginf=0.0) + return np.maximum(importances, 0.0) + except Exception as exc: + logger.debug("Could not compute feature importances") + logger.debug(exc) + return np.array([]) + + +def get_alpha_score(model, test_sample, alpha=0.8): + """ + Computes alpha score from global feature importances. + """ + try: + vals = np.asarray(_get_feature_importances(model, test_sample), dtype=float).reshape(-1) + vals = np.nan_to_num(vals, nan=0.0, posinf=0.0, neginf=0.0) + vals = np.maximum(vals, 0.0) + total_features = len(vals) + if total_features == 0 or np.sum(vals) <= 1e-12: + return 1.0 + + try: + alpha = float(alpha) + except Exception: + alpha = 0.8 + alpha = min(max(alpha, 0.0), 1.0) + + vals_sorted = np.sort(vals)[::-1] + cum_sum = np.cumsum(vals_sorted) + threshold = float(alpha) * np.sum(vals_sorted) + idx = np.searchsorted(cum_sum, threshold) + return float(min(total_features, idx + 1) / total_features) + except Exception as exc: + logger.warning("Could not compute alpha score") + logger.warning(exc) + return 1.0 + + +def _get_spread_base(model, test_sample, divergence=True): + vals = _get_feature_importances(model, test_sample) + tol = 1e-8 + + if len(vals) == 0 or np.sum(vals) < tol: + return 0.0 if divergence else 1.0 + if len(vals) == 1: + return 0.0 if divergence else 1.0 + + weights = vals / np.sum(vals) + equal_weights = np.ones(len(vals)) / len(vals) + + if divergence: + metric = jensenshannon(weights, equal_weights, base=2) + else: + denom = entropy(equal_weights) + metric = 0.0 if denom <= tol else entropy(weights) / denom + + if math.isnan(metric) or math.isinf(metric): + return 0.0 if divergence else 1.0 + return float(np.clip(metric, 0.0, 1.0)) + + +def get_spread_ratio(model, test_sample): + """ + Computes spread ratio from global feature importances. + """ + try: + return _get_spread_base(model, test_sample, divergence=False) + except Exception as exc: + logger.warning("Could not compute spread ratio") + logger.warning(exc) + return 1.0 + + +def get_spread_divergence(model, test_sample): + """ + Computes spread divergence from global feature importances. + """ + try: + return _get_spread_base(model, test_sample, divergence=True) + except Exception as exc: + logger.warning("Could not compute spread divergence") + logger.warning(exc) + return 0.0 + + +def get_explainability_metrics_summary(model, test_dataloader, max_batches=4): + """ + Computes explainability metrics over multiple test batches and returns + their mean values. + + Args: + model (object): The model. + test_dataloader: Test dataloader providing batches. + max_batches (int): Maximum number of batches to use. + + Returns: + dict: Mean values for feature_importance_cv, alpha_score, + spread_ratio and spread_divergence. + """ + summary = { + "feature_importance_cv": 1.0, + "alpha_score": 1.0, + "spread_ratio": 1.0, + "spread_divergence": 0.0, + } + + if test_dataloader is None: + return summary + + try: + max_batches = max(1, int(max_batches)) + except Exception: + max_batches = 4 + + fi_values = [] + alpha_values = [] + spread_ratio_values = [] + spread_divergence_values = [] + + try: + for batch_idx, test_sample in enumerate(test_dataloader): + if batch_idx >= max_batches: + break + + fi_values.append(float(get_feature_importance_cv(model, test_sample))) + alpha_values.append(float(get_alpha_score(model, test_sample))) + spread_ratio_values.append(float(get_spread_ratio(model, test_sample))) + spread_divergence_values.append(float(get_spread_divergence(model, test_sample))) + except Exception as exc: + logger.warning("Could not compute explainability metrics summary") + logger.warning(exc) + + if fi_values: + summary["feature_importance_cv"] = float(np.mean(fi_values)) + if alpha_values: + summary["alpha_score"] = float(np.mean(alpha_values)) + if spread_ratio_values: + summary["spread_ratio"] = float(np.mean(spread_ratio_values)) + if spread_divergence_values: + summary["spread_divergence"] = float(np.mean(spread_divergence_values)) + + return summary diff --git a/nebula/addons/trustworthiness/helpers/factsheet_values.py b/nebula/addons/trustworthiness/helpers/factsheet_values.py new file mode 100644 index 000000000..ee42940af --- /dev/null +++ b/nebula/addons/trustworthiness/helpers/factsheet_values.py @@ -0,0 +1,108 @@ +import logging +import math + +from nebula.addons.trustworthiness.helpers.privacy import ( + get_global_privacy_risk, + get_global_privacy_risk_dfl, +) +from nebula.addons.trustworthiness.helpers.scenario_metrics import comm_efficiency +from nebula.addons.trustworthiness.helpers.scoring import ( + check_properties, + get_value, +) + +logger = logging.getLogger(__name__) + +OPERATIONS = { + "check_properties": check_properties, + "comm_efficiency": comm_efficiency, + "get_global_privacy_risk": get_global_privacy_risk, + "get_global_privacy_risk_dfl": get_global_privacy_risk_dfl, + "get_value": get_value, +} + +def check_field_filled(factsheet_dict, factsheet_path, value, empty=""): + """ + Check if the field in the factsheet file is filled or not. + + Args: + factsheet_dict (dict): The factshett dict. + factsheet_path (list): The factsheet field to check. + value (float): The value to add in the field. + empty (string): If the value could not be appended, the empty string is returned. + + Returns: + float: The value added in the factsheet or empty if the value could not be appened + + """ + if factsheet_dict[factsheet_path[0]][factsheet_path[1]]: + return factsheet_dict[factsheet_path[0]][factsheet_path[1]] + elif value != "" and value != "nan": + if type(value) != str and type(value) != list: + if math.isnan(value): + return 0 + else: + return value + else: + return value + else: + return empty + + +def get_input_value(input_docs, inputs, operation): + """ + Gets the input value from input document and apply the metric operation on the value. + + Args: + inputs_docs (map): The input document map. + inputs (list): All the inputs. + operation (string): The metric operation. + + Returns: + float: The metric value + + """ + + input_value = None + args = [] + for i in inputs: + source = i.get("source", "") + field = i.get("field_path", "") + input_doc = input_docs.get(source, None) + if input_doc is None: + logger.warning(f"{source} is null") + else: + input = get_value_from_path(input_doc, field) + args.append(input) + try: + operationFn = OPERATIONS[operation] + input_value = operationFn(*args) + except KeyError: + logger.warning(f"{operation} is not valid") + except TypeError: + logger.warning(f"{operation} is not valid") + + return input_value + + +def get_value_from_path(input_doc, path): + """ + Gets the input value from input document by path. + + Args: + inputs_doc (map): The input document map. + path (string): The field name of the input value of interest. + + Returns: + float: The input value from the input document + + """ + + d = input_doc + for nested_key in path.split("/"): + temp = d.get(nested_key) + if isinstance(temp, dict): + d = d.get(nested_key) + else: + return temp + return None diff --git a/nebula/addons/trustworthiness/helpers/model_quality.py b/nebula/addons/trustworthiness/helpers/model_quality.py new file mode 100644 index 000000000..0b87937fe --- /dev/null +++ b/nebula/addons/trustworthiness/helpers/model_quality.py @@ -0,0 +1,371 @@ +import logging +import math + +import numpy as np +import torch +from sklearn.metrics import f1_score + +logger = logging.getLogger(__name__) + +def _get_model_accuracy(model, dataloader): + """ + Calculates model accuracy over a dataloader. + + Args: + model (torch.nn.Module): Model to evaluate. + dataloader (DataLoader): Dataloader with (x, y) batches. + + Returns: + float: Accuracy in [0, 1]. + """ + if not isinstance(model, torch.nn.Module): + logger.warning("Model is not a torch.nn.Module") + return 0.0 + + try: + device = next(model.parameters()).device + except Exception: + device = torch.device("cpu") + + model.eval() + correct = 0 + total = 0 + + with torch.no_grad(): + for x, y in dataloader: + x = x.to(device) + y = y.to(device) + + out = model(x) + logits = out[0] if isinstance(out, (tuple, list)) else out + preds = logits.argmax(dim=1) + + correct += (preds == y).sum().item() + total += y.size(0) + + return correct / total if total > 0 else 0.0 + + +def get_macro_f1_score(model, dataloader): + """ + Calculates macro F1 score over a dataloader. + + Args: + model (torch.nn.Module): Model to evaluate. + dataloader (DataLoader): Dataloader with (x, y) batches. + + Returns: + float: Macro F1 score in [0, 1]. + """ + if not isinstance(model, torch.nn.Module): + logger.warning("Model is not a torch.nn.Module") + return 0.0 + + try: + device = next(model.parameters()).device + except Exception: + device = torch.device("cpu") + + model.eval() + y_true = [] + y_pred = [] + + with torch.no_grad(): + for x, y in dataloader: + x = x.to(device) + y = y.to(device) + + out = model(x) + logits = out[0] if isinstance(out, (tuple, list)) else out + preds = logits.argmax(dim=1) + + y_true.extend(y.detach().cpu().numpy().tolist()) + y_pred.extend(preds.detach().cpu().numpy().tolist()) + + if not y_true: + return 0.0 + + return float(f1_score(y_true, y_pred, average="macro", zero_division=0)) + + +def _extract_model_logits(model_output): + """ + Normalize the output returned by a model forward pass into a logits tensor. + + Some models may return tuples/lists; for trust metrics we always consume the + first element as the classification output. + """ + return model_output[0] if isinstance(model_output, (tuple, list)) else model_output + + +def _prepare_class_targets(y): + """ + Convert different target representations into a flat class-index tensor. + """ + if not torch.is_tensor(y): + y = torch.as_tensor(y) + + if y.ndim > 1: + if y.size(-1) > 1: + y = y.argmax(dim=-1) + else: + y = y.view(-1) + + return y.long().view(-1) + + +def _logits_to_probabilities(logits): + """ + Convert model outputs into a probability matrix of shape (N, C). + + Supports: + - multiclass logits/log-probabilities with shape (N, C) + - binary logits with shape (N,) or (N, 1) + - already-normalized probability matrices + """ + if not torch.is_tensor(logits): + logits = torch.as_tensor(logits) + + if logits.ndim == 0: + logits = logits.view(1, 1) + elif logits.ndim == 1: + logits = logits.view(-1, 1) + elif logits.ndim > 2: + logits = logits.reshape(logits.shape[0], -1) + + if logits.size(1) == 1: + pos_prob = torch.sigmoid(logits[:, 0]) + probs = torch.stack([1.0 - pos_prob, pos_prob], dim=1) + else: + row_sums = logits.sum(dim=1) + looks_like_probs = ( + torch.all(logits >= 0) + and torch.all(logits <= 1.0 + 1e-6) + and torch.allclose(row_sums, torch.ones_like(row_sums), atol=1e-4, rtol=1e-4) + ) + probs = logits if looks_like_probs else torch.softmax(logits, dim=1) + + probs = torch.clamp(probs, min=0.0, max=1.0) + probs = probs / probs.sum(dim=1, keepdim=True).clamp_min(1e-12) + return probs + + +def _collect_classification_statistics(model, dataloader): + """ + Collect prediction statistics required by calibration and inequality metrics. + + Returns: + tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]: + predicted labels, true labels, prediction confidences, correctness flags, + and probability assigned to the true class. + """ + if not isinstance(model, torch.nn.Module): + logger.warning("Model is not a torch.nn.Module") + empty = np.array([], dtype=float) + return empty, empty, empty, empty, empty + + try: + device = next(model.parameters()).device + except Exception: + device = torch.device("cpu") + + preds_all = [] + targets_all = [] + confidences_all = [] + correct_all = [] + true_probs_all = [] + + model.eval() + with torch.no_grad(): + for batch in dataloader: + if not isinstance(batch, (tuple, list)) or len(batch) < 2: + continue + + x, y = batch[0], batch[1] + if not (torch.is_tensor(x) and torch.is_tensor(y)): + continue + + x = x.to(device) + y = _prepare_class_targets(y).to(device) + + out = model(x) + logits = _extract_model_logits(out) + probs = _logits_to_probabilities(logits) + + if probs.ndim != 2 or probs.size(0) == 0: + continue + + if y.numel() != probs.size(0): + n = min(int(y.numel()), int(probs.size(0))) + if n == 0: + continue + y = y[:n] + probs = probs[:n] + + valid_mask = (y >= 0) & (y < probs.size(1)) + if not torch.any(valid_mask): + continue + + y = y[valid_mask] + probs = probs[valid_mask] + + conf, preds = probs.max(dim=1) + true_probs = probs.gather(1, y.view(-1, 1)).squeeze(1) + correct = preds.eq(y).float() + + preds_all.extend(preds.detach().cpu().numpy().tolist()) + targets_all.extend(y.detach().cpu().numpy().tolist()) + confidences_all.extend(conf.detach().cpu().numpy().tolist()) + correct_all.extend(correct.detach().cpu().numpy().tolist()) + true_probs_all.extend(true_probs.detach().cpu().numpy().tolist()) + + return ( + np.asarray(preds_all, dtype=int), + np.asarray(targets_all, dtype=int), + np.asarray(confidences_all, dtype=float), + np.asarray(correct_all, dtype=float), + np.asarray(true_probs_all, dtype=float), + ) + + +def get_overfitting_score(model, train_dataloader, test_accuracy): + """ + Calculates overfitting as the positive train-test accuracy gap. + + Args: + model (torch.nn.Module): Model to evaluate on training data. + train_dataloader (DataLoader): Training dataloader. + test_accuracy (float): Test accuracy in [0, 1]. + + Returns: + float: Positive train-test accuracy gap. + """ + try: + train_accuracy = _get_model_accuracy(model, train_dataloader) + return max(0.0, float(train_accuracy) - float(test_accuracy)) + except Exception as exc: + logger.warning("Could not compute overfitting score") + logger.warning(exc) + return 0.0 + + +def get_well_calibration_error(model, test_dataloader, n_bins=10): + """ + Calculates a well-calibration error style metric using prediction confidence. + + For multiclass models, confidence is taken as the max softmax probability and + the observed outcome is whether the prediction is correct. + + Args: + model (torch.nn.Module): Model to evaluate. + test_dataloader (DataLoader): Test dataloader. + n_bins (int): Number of quantile bins. + + Returns: + float: Calibration error in [0, 1] when computation succeeds. + """ + if not isinstance(model, torch.nn.Module): + logger.warning("Model is not a torch.nn.Module") + return 0.0 + + try: + n_bins = max(2, int(n_bins)) + except Exception: + n_bins = 10 + + _, _, confidences, correct, _ = _collect_classification_statistics(model, test_dataloader) + + if len(confidences) == 0 or len(correct) == 0: + return 0.0 + + confidences = np.clip(np.asarray(confidences, dtype=float), 0.0, 1.0) + correct = np.clip(np.asarray(correct, dtype=float), 0.0, 1.0) + + bin_edges = np.linspace(0.0, 1.0, n_bins + 1) + ece = 0.0 + total = float(len(confidences)) + + for idx in range(n_bins): + left = bin_edges[idx] + right = bin_edges[idx + 1] + if idx == n_bins - 1: + mask = (confidences >= left) & (confidences <= right) + else: + mask = (confidences >= left) & (confidences < right) + + if not np.any(mask): + continue + + bin_weight = float(mask.sum()) / total + bin_accuracy = float(correct[mask].mean()) + bin_confidence = float(confidences[mask].mean()) + ece += bin_weight * abs(bin_accuracy - bin_confidence) + + return float(np.clip(ece, 0.0, 1.0)) + + +def get_generalized_entropy_index(model, test_dataloader, alpha=2): + """ + Calculates generalized entropy index from model predictions. + + Args: + model (torch.nn.Module): Model to evaluate. + test_dataloader (DataLoader): Test dataloader. + alpha (float): GEI alpha parameter. + + Returns: + float: Generalized entropy index value. + """ + try: + _, _, _, _, true_class_probs = _collect_classification_statistics(model, test_dataloader) + if len(true_class_probs) == 0: + return 0.0 + + # Use the probability assigned to the true class as a continuous, positive + # benefit. This works consistently for multiclass neural models on both + # images and tabular data, and avoids collapsing the metric to a coarse + # correct/incorrect indicator. + eps = 1e-12 + b = np.clip(np.asarray(true_class_probs, dtype=float), eps, 1.0) + mu = float(np.mean(b)) + if mu <= 0: + return 0.0 + + ratio = np.clip(b / mu, eps, None) + + if alpha == 0: + val = float(np.mean(-np.log(ratio))) + elif alpha == 1: + val = float(np.mean(ratio * np.log(ratio))) + elif alpha == 2: + val = float(np.mean((ratio - 1.0) ** 2) / 2.0) + else: + val = float(np.mean(ratio**alpha - 1.0) / (alpha * (alpha - 1.0))) + + if math.isnan(val) or math.isinf(val): + return 0.0 + return max(0.0, val) + except Exception as exc: + logger.warning("Could not compute generalized entropy index") + logger.warning(exc) + return 0.0 + + +def get_theil_index(model, test_dataloader): + """ + Convenience wrapper for generalized entropy index with alpha=1. + """ + return get_generalized_entropy_index(model, test_dataloader, alpha=1) + + +def get_coefficient_of_variation(model, test_dataloader): + """ + Calculates coefficient of variation from GEI(alpha=2). + """ + try: + gei = get_generalized_entropy_index(model, test_dataloader, alpha=2) + return float(np.sqrt(2 * gei)) + except Exception as exc: + logger.warning("Could not compute coefficient of variation") + logger.warning(exc) + return 0.0 diff --git a/nebula/addons/trustworthiness/helpers/privacy.py b/nebula/addons/trustworthiness/helpers/privacy.py new file mode 100644 index 000000000..f6ed327c1 --- /dev/null +++ b/nebula/addons/trustworthiness/helpers/privacy.py @@ -0,0 +1,209 @@ +import logging +import math +import numbers +from math import e + +import numpy as np +import torch +from sklearn.metrics import roc_auc_score, roc_curve +from torch import nn + +logger = logging.getLogger(__name__) + +def get_global_privacy_risk(dp, epsilon, n): + """ + Calculates the global privacy risk by epsilon and the number of clients. + + Args: + dp (bool): Indicates if differential privacy is used or not. + epsilon (int): The epsilon value. + n (int): The number of clients in the scenario. + + Returns: + float: The global privacy risk. + """ + + try: + epsilon = float(epsilon) + n = float(n) + except (TypeError, ValueError): + return 1 + + if dp is True and isinstance(epsilon, numbers.Number): + return 1 / (1 + (n - 1) * math.pow(e, -epsilon)) + else: + return 1 + + +def get_global_privacy_risk_dfl(dp, epsilon, n): + """ + Calculates the global privacy risk by epsilon and the number of clients. + + Args: + dp (bool): Indicates if differential privacy is used or not. + epsilon (int): The epsilon value. + n (int): The number of neighbours. + + Returns: + float: The global privacy risk. + """ + + try: + epsilon = float(epsilon) + n = float(n) + except (TypeError, ValueError): + return 1 + + if dp is True and isinstance(epsilon, numbers.Number): + return 1 / (1 + (n + 1) * math.pow(e, -epsilon)) + else: + return 1 + + +def _collect_per_sample_losses(model, dataloader, max_samples=5000): + """ + Compute per-sample cross-entropy losses for a dataloader. + + Args: + model (torch.nn.Module): The model to evaluate. + dataloader: DataLoader providing (samples, labels). + max_samples (int): Maximum number of samples to process. + + Returns: + np.ndarray: Losses per sample. + """ + if not isinstance(model, torch.nn.Module) or dataloader is None: + return np.array([]) + + try: + device = next(model.parameters()).device + except Exception: + device = torch.device("cpu") + + criterion = nn.CrossEntropyLoss(reduction="none") + losses = [] + collected = 0 + + model.eval() + with torch.no_grad(): + for batch in dataloader: + if not isinstance(batch, (tuple, list)) or len(batch) < 2: + continue + + samples, labels = batch[0], batch[1] + if not torch.is_tensor(samples) or not torch.is_tensor(labels): + continue + + remaining = max_samples - collected + if remaining <= 0: + break + + samples = samples[:remaining].to(device) + labels = labels[:remaining] + + if labels.ndim > 1: + labels = torch.argmax(labels, dim=1) + + labels = labels.long().to(device) + + outputs = model(samples) + logits = outputs[0] if isinstance(outputs, (tuple, list)) else outputs + batch_losses = criterion(logits, labels) + + losses.append(batch_losses.detach().cpu().numpy()) + collected += int(batch_losses.shape[0]) + + if not losses: + return np.array([]) + + return np.concatenate(losses, axis=0) + + +def get_epsilon_star(model, train_dataloader, test_dataloader, max_samples=5000): + """ + Compute empirical epsilon* from train/test loss distributions. + + This follows the same core structure as privacy_metrics_core.epsilon_star, + adapted to PyTorch models and DataLoaders used in Nebula. + + Args: + model (torch.nn.Module): Model to evaluate. + train_dataloader: Training DataLoader. + test_dataloader: Test DataLoader. + max_samples (int): Maximum samples to evaluate per split. + + Returns: + float: Empirical epsilon* value. Returns 0.0 on failure. + """ + try: + loss_train = _collect_per_sample_losses(model, train_dataloader, max_samples=max_samples) + loss_test = _collect_per_sample_losses(model, test_dataloader, max_samples=max_samples) + + if loss_train.size == 0 or loss_test.size == 0: + return 0.0 + + scores = np.concatenate([-loss_train, -loss_test]) + y_true = np.concatenate([np.ones(len(loss_train)), np.zeros(len(loss_test))]) + + fpr, tpr, _ = roc_curve(y_true, scores) + + fpr = np.clip(fpr, 1e-10, 1 - 1e-10) + tpr = np.clip(tpr, 1e-10, 1 - 1e-10) + fnr = 1 - tpr + + delta = 1.0 / len(loss_train) if len(loss_train) > 0 else 1e-5 + + m1 = (1 - delta - fnr) / fpr + m2 = (1 - delta - fpr) / fnr + m3 = (fnr - delta) / (1 - fpr) + m4 = (fpr - delta) / (1 - fnr) + + epsilon_star_val = np.log( + np.nanmax(np.maximum.reduce([m1, m2, m3, m4, np.ones_like(m1)])) + ) + + if np.isnan(epsilon_star_val) or np.isinf(epsilon_star_val): + return 0.0 + + return float(max(0.0, epsilon_star_val)) + except Exception as exc: + logger.warning("Could not compute epsilon_star") + logger.warning(exc) + return 0.0 + + +def get_mia_auc(model, train_dataloader, test_dataloader, max_samples=5000): + """ + Compute membership inference attack AUC using per-sample loss as the attack score. + + Lower loss suggests a sample is more likely to be a training member, so the + attack score is defined as negative loss. + + Args: + model (torch.nn.Module): Model to evaluate. + train_dataloader: Training DataLoader. + test_dataloader: Test DataLoader. + max_samples (int): Maximum samples to evaluate per split. + + Returns: + float: ROC-AUC of the loss-threshold membership attack. Returns 0.5 on failure. + """ + try: + loss_train = _collect_per_sample_losses(model, train_dataloader, max_samples=max_samples) + loss_test = _collect_per_sample_losses(model, test_dataloader, max_samples=max_samples) + + if loss_train.size == 0 or loss_test.size == 0: + return 0.5 + + scores = np.concatenate([-loss_train, -loss_test]) + y_true = np.concatenate([np.ones(len(loss_train)), np.zeros(len(loss_test))]) + mia_auc = roc_auc_score(y_true, scores) + + if np.isnan(mia_auc) or np.isinf(mia_auc): + return 0.5 + + return float(np.clip(mia_auc, 0.0, 1.0)) + except Exception as exc: + logger.warning("Could not compute mia_auc") + logger.warning(exc) + return 0.5 diff --git a/nebula/addons/trustworthiness/helpers/robustness.py b/nebula/addons/trustworthiness/helpers/robustness.py new file mode 100644 index 000000000..13611842b --- /dev/null +++ b/nebula/addons/trustworthiness/helpers/robustness.py @@ -0,0 +1,413 @@ +import logging +import math + +import numpy as np +import torch +import torch.nn.functional as F +from art.estimators.classification import PyTorchClassifier +from art.metrics import clever_u, empirical_robustness, loss_sensitivity +from torch import nn, optim + +logger = logging.getLogger(__name__) + +R_L2 = 2 + +def _build_art_classifier(model, input_shape, nb_classes, learning_rate): + criterion = nn.CrossEntropyLoss() + optimizer = optim.Adam(model.parameters(), learning_rate) + + return PyTorchClassifier( + model=model, + loss=criterion, + optimizer=optimizer, + input_shape=tuple(input_shape), + nb_classes=nb_classes, + ) + + +def _validate_test_sample_tensors(test_sample): + if not (isinstance(test_sample, (tuple, list)) and len(test_sample) >= 2): + raise ValueError("`test_sample` must contain samples and labels.") + + samples, labels = test_sample[0], test_sample[1] + if not (torch.is_tensor(samples) and torch.is_tensor(labels) and samples.shape[0] > 0): + raise ValueError("`test_sample` must contain non-empty tensors for samples and labels.") + + return samples, labels + + +def _coerce_max_samples(max_samples, default=8): + try: + return max(1, int(max_samples)) + except Exception: + return default + + +def get_clever_score(model, test_sample, nb_classes, learning_rate, max_samples=8): + """ + Calculates the CLEVER score as the mean score over multiple samples. + + Args: + model (object): The model. + test_sample (object): A batch from the test dataloader. + nb_classes (int): The nb_classes of the model. + learning_rate (float): The learning rate of the model. + max_samples (int): Maximum number of samples from the batch to evaluate. + + Returns: + float: Mean CLEVER score across the selected samples. + """ + samples, _ = _validate_test_sample_tensors(test_sample) + + input_shape = tuple(samples.shape[1:]) if samples.dim() >= 2 else tuple(samples.shape) + + max_samples = _coerce_max_samples(max_samples) + n_samples = min(int(samples.shape[0]), max_samples) + + # Create the ART classifier once and reuse it for all selected samples. + classifier = _build_art_classifier(model, input_shape, nb_classes, learning_rate) + + clever_scores = [] + for idx in range(n_samples): + background = samples[idx].detach().cpu() + sample_np = background.numpy() + + try: + score_untargeted = clever_u( + classifier, + sample_np, + 10, + 5, + R_L2, + norm=2, + pool_factor=3, + verbose=False, + ) + if score_untargeted is not None and not math.isnan(float(score_untargeted)): + clever_scores.append(float(score_untargeted)) + except Exception as exc: + logger.warning("Could not compute CLEVER score for sample index %s", idx) + logger.warning(exc) + + if not clever_scores: + return 0.0 + + return float(np.mean(clever_scores)) + +def get_loss_sensitivity_score(model, test_sample, nb_classes, learning_rate, max_samples=8): + + """ + Calculates the loss sensitivity score as the mean score over multiple samples. + + Args: + model (object): The model. + test_sample (object): A batch from the test dataloader. + nb_classes (int): The nb_classes of the model. + learning_rate (float): The learning rate of the model. + max_samples (int): Maximum number of samples from the batch to evaluate. + + Returns: + float: Mean loss sensitivity score across the selected samples. + """ + samples, labels = _validate_test_sample_tensors(test_sample) + + max_samples = _coerce_max_samples(max_samples) + n_samples = min(int(samples.shape[0]), max_samples) + + # Create the ART classifier once and reuse it for all selected samples. + classifier = _build_art_classifier(model, samples.shape[1:], nb_classes, learning_rate) + + sensitivity_scores = [] + for idx in range(n_samples): + sample = samples[idx].detach().cpu().unsqueeze(0) + label = labels[idx].detach().cpu().unsqueeze(0) + label = F.one_hot(label, num_classes=nb_classes).float() + + try: + score = loss_sensitivity( + classifier, + sample.numpy(), + label.numpy(), + ) + if score is not None and not math.isnan(float(score)): + sensitivity_scores.append(float(score)) + except Exception as exc: + logger.warning("Could not compute loss sensitivity for sample index %s", idx) + logger.warning(exc) + + if not sensitivity_scores: + return 0.0 + + return float(np.mean(sensitivity_scores)) + + +def compute_adversarial_accuracy_art( + model, + test_loader, + nb_classes, + learning_rate, + epsilon=0.03 +): + """ + Computes adversarial accuracy using FGSM attack. + + Args: + model (object): The model. + test_loader (DataLoader): DataLoader providing test samples. + nb_classes (int): The nb_classes of the model. + learning_rate (float): The learning rate of the model. + epsilon (float): Maximum perturbation magnitude for the attacks. + + Returns: + float: The adversarial accuracy score. + """ + + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + model.eval() + model.to(device) + + correct = 0 + total = 0 + + for samples, labels in test_loader: + samples = samples.to(device) + labels = labels.to(device) + + x_adv = fgsm_attack(model, samples, labels, epsilon=epsilon) + + with torch.no_grad(): + outputs = model(x_adv) + logits = outputs[0] if isinstance(outputs, (tuple, list)) else outputs + preds = logits.argmax(dim=1) + + correct += (preds == labels).sum().item() + total += labels.size(0) + + return correct / total if total > 0 else 0.0 + + +def get_empirical_robustness_score( + model, + test_sample, + nb_classes, + learning_rate, + attack_name = "fgsm", + attack_params = None, + max_samples = 128, +): + """ + Calculates the Empirical Robustness score using Adversarial Robustness Toolbox (ART). + + Args: + model (object): The model. + test_sample (object): A batch from the test dataloader (samples, labels). + nb_classes (int): The nb_classes of the model. + learning_rate (float): The learning rate of the model. + attack_name (str): Attack key supported by ART empirical_robustness. + attack_params (dict | None): Optional attack parameters. + max_samples (int): Max number of samples from the batch to use. + + Returns: + float: Empirical robustness score (>= 0.0). If it cannot be computed, returns 0.0. + """ + try: + samples, _ = _validate_test_sample_tensors(test_sample) + + batch_size: int = int(samples.shape[0]) + n: int = int(min(max_samples, batch_size)) + x = samples[:n].detach().cpu().numpy() + + classifier = _build_art_classifier(model, samples.shape[1:], nb_classes, learning_rate) + + score = empirical_robustness( + classifier=classifier, + x=x, + attack_name=attack_name, + attack_params=attack_params, + ) + + if isinstance(score, np.ndarray): + score = float(np.mean(score)) + + if score is None or (isinstance(score, float) and math.isnan(score)): + return 0.0 + + return float(score) + + except Exception as exc: + logger.warning("Could not compute empirical robustness (ART). Returning 0.0") + logger.warning(exc) + return 0.0 + + +def _get_image_normalization_for_samples(samples): + if not isinstance(samples, torch.Tensor) or samples.ndim < 4: + return None + + channels = int(samples.shape[1]) + if channels == 1: + return (0.5,), (0.5,) + if channels == 3: + return (0.4914, 0.4822, 0.4465), (0.2471, 0.2435, 0.2616) + return None + + +def _channel_tensor(values, samples): + shape = [1, len(values)] + [1] * max(samples.dim() - 2, 0) + return torch.tensor(values, dtype=samples.dtype, device=samples.device).view(*shape) + + +def _fgsm_step_and_clamp(samples, grad, epsilon): + normalization = _get_image_normalization_for_samples(samples) + if normalization is None: + return samples + epsilon * grad.sign() + + mean, std = normalization + mean = _channel_tensor(mean, samples) + std = _channel_tensor(std, samples) + + normalized_epsilon = float(epsilon) / std + lower = (0.0 - mean) / std + upper = (1.0 - mean) / std + + x_adv = samples + normalized_epsilon * grad.sign() + x_adv = torch.max(torch.min(x_adv, samples + normalized_epsilon), samples - normalized_epsilon) + return torch.max(torch.min(x_adv, upper), lower) + + +def fgsm_attack(model, samples, labels, epsilon=0.03): + """ + Performs an FGSM (Fast Gradient Sign Method) adversarial attack on a batch of samples. + + Args: + model (torch.nn.Module): The PyTorch model to attack. + samples (torch.Tensor): Input samples to perturb, shape (B, ...). + labels (torch.Tensor): True labels corresponding to the samples. + epsilon (float, optional): Maximum perturbation magnitude for the attack. Defaults to 0.03. + + Returns: + torch.Tensor: Adversarially perturbed samples with the same shape as `samples`. + """ + try: + device = next(model.parameters()).device + except Exception: + device = samples.device + + samples = samples.clone().detach().to(device) + labels = labels.to(device) + samples.requires_grad = True + + outputs = model(samples) + logits = outputs[0] if isinstance(outputs, (tuple, list)) else outputs + loss = nn.CrossEntropyLoss()(logits, labels) + grad = torch.autograd.grad(loss, samples, only_inputs=True)[0] + x_adv = _fgsm_step_and_clamp(samples, grad, epsilon) + + return x_adv.detach() + + +def get_confidence_score( + model, + test_sample, + max_samples = 128, + use_true_label = True, +): + """ + Calculates the confidence score. + + Args: + model (object): The model. + test_sample (object): A batch from the test dataloader (samples, labels). + max_samples (int): Max number of samples from the batch to use. + use_true_label (bool): Whether to compute confidence with respect to the true labels. Defaults to True. + + Returns: + float: Confidence score. + """ + try: + if not isinstance(model, torch.nn.Module): + logger.warning("Model is not a torch.nn.Module") + return 0.0 + + x, y = test_sample + + if isinstance(x, torch.Tensor): + x = x[:max_samples] + if isinstance(y, torch.Tensor): + y = y[:max_samples] + + try: + device = next(model.parameters()).device + except Exception: + device = torch.device("cpu") + + model.eval() + with torch.no_grad(): + x = x.to(device) if isinstance(x, torch.Tensor) else x + out = model(x) + + logits = out[0] if isinstance(out, (tuple, list)) else out + probs = torch.softmax(logits, dim=1) + + if use_true_label and isinstance(y, torch.Tensor): + if y.ndim > 1: + y_idx = torch.argmax(y, dim=1) + else: + y_idx = y + y_idx = y_idx.to(device) + + true_probs = probs.gather(1, y_idx.view(-1, 1)).squeeze(1) + return float(true_probs.mean().detach().cpu().item()) + + msp = probs.max(dim=1).values + return float(msp.mean().detach().cpu().item()) + + except Exception as e: + logger.warning("Could not compute confidence score") + logger.warning(e) + return 0.0 + + +def attack_success_rate(model, test_sample,epsilon=0.03): + """ + Calculates the ASR. + + Args: + model (object): The model. + test_sample (object): A batch from the test dataloader (samples, labels). + epsilon (float): Maximum perturbation magnitude for the attacks. + + Returns: + float: The ASR. + """ + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + model.eval() + model.to(device) + + images, labels = test_sample + images = images.to(device) + labels = labels.to(device) + + with torch.no_grad(): + outputs = model(images) + logits = outputs[0] if isinstance(outputs, (tuple, list)) else outputs + preds = logits.argmax(dim=1) + + correct_mask = preds.eq(labels) + num_correct = correct_mask.sum().item() + + if num_correct == 0: + return 0.0 + + x_adv = fgsm_attack(model, images, labels, epsilon=epsilon) + + with torch.no_grad(): + outputs_adv = model(x_adv) + logits_adv = outputs_adv[0] if isinstance(outputs_adv, (tuple, list)) else outputs_adv + preds_adv = logits_adv.argmax(dim=1) + + successful_attacks = (correct_mask & preds_adv.ne(labels)).sum().item() + + asr = successful_attacks / num_correct + + return asr diff --git a/nebula/addons/trustworthiness/helpers/scenario_metrics.py b/nebula/addons/trustworthiness/helpers/scenario_metrics.py new file mode 100644 index 000000000..d714e8523 --- /dev/null +++ b/nebula/addons/trustworthiness/helpers/scenario_metrics.py @@ -0,0 +1,350 @@ +import io +import logging +import os +import statistics +from datetime import datetime +from os.path import exists + +import pandas as pd +import torch +from codecarbon import EmissionsTracker + +from nebula.addons.trustworthiness.helpers.csv_io import read_csv + +logger = logging.getLogger(__name__) + +def get_elapsed_time(start_time, end_time): + """ + Calculates the elapsed time during the execution of the scenario. + + Args: + start_time (datetime): Start datetime. + end_time (datetime): End datetime. + + Returns: + float: The elapsed time. + """ + start_date = datetime.strptime(start_time, "%d/%m/%Y %H:%M:%S") + end_date = datetime.strptime(end_time, "%d/%m/%Y %H:%M:%S") + + elapsed_time = (end_date - start_date).total_seconds() / 60 + + return elapsed_time + + +def _trustworthiness_dir(scenario_name): + return os.path.join(os.environ.get('NEBULA_LOGS_DIR'), scenario_name, "trustworthiness") + + +def _global_data_results_path(scenario_name): + return os.path.join(_trustworthiness_dir(scenario_name), "data_results.csv") + + +def _participant_data_results_path(scenario_name, participant_id): + return os.path.join(_trustworthiness_dir(scenario_name), f"data_results_{participant_id}.csv") + + +def _read_global_results(scenario_name): + return read_csv(_global_data_results_path(scenario_name)) + + +def _read_participant_results(scenario_name, participant_id): + return read_csv(_participant_data_results_path(scenario_name, participant_id)) + + +def _find_participant_row(data, participant_id, source_name): + row = data[data["id"] == participant_id] + + if row.empty: + try: + row = data[data["id"] == int(participant_id)] + except (TypeError, ValueError): + row = data.iloc[0:0] + + if row.empty: + raise ValueError(f"Participant {participant_id} not found in {source_name}") + + return row.iloc[0] + + +def get_bytes_model(model): + """ + Calculates the serialized size in bytes of a PyTorch model state_dict. + + Args: + model (nn.Module): PyTorch model. + + Returns: + int: Model size in bytes. + """ + buffer: io.BytesIO = io.BytesIO() + torch.save(model.state_dict(), buffer) + model_size: int = buffer.tell() + + return model_size + + +def get_bytes_sent_recv(scenario_name): + """ + Calculates the mean bytes sent and received of the nodes. + + Args: + bytes_sent_files (list): Files that contain the bytes sent of the nodes. + bytes_recv_files (list): Files that contain the bytes received of the nodes. + + Returns: + 4-tupla: The total bytes sent, the total bytes received, the mean bytes sent and the mean bytes received of the nodes. + """ + data = _read_global_results(scenario_name) + + number_files = len(data) + + total_upload_bytes = int(data["bytes_sent"].sum()) + total_download_bytes = int(data["bytes_recv"].sum()) + + avg_upload_bytes = total_upload_bytes / number_files + avg_download_bytes = total_download_bytes / number_files + + return total_upload_bytes, total_download_bytes, avg_upload_bytes, avg_download_bytes + + +def get_avg_loss_accuracy(scenario_name): + """ + Calculates the mean accuracy and loss models of the nodes. + + Args: + loss_files (list): Files that contain the loss of the models of the nodes. + accuracy_files (list): Files that contain the acurracies of the models of the nodes. + + Returns: + 3-tupla: The mean loss of the models, the mean accuracies of the models, the standard deviation of the accuracies of the models. + """ + data = _read_global_results(scenario_name) + + number_files = len(data) + + total_loss = data["loss"].sum() + total_accuracy = data["accuracy"].sum() + + denominator = max(1, number_files - 1) + avg_loss = total_loss / denominator + avg_accuracy = total_accuracy / denominator + std_accuracy = statistics.stdev(data["accuracy"]) if number_files > 1 else 0.0 + + return avg_loss, avg_accuracy, std_accuracy + + +def get_underfitting_score(scenario_name, id): + """ + Calculates the mean val accuracy of the nodes. + """ + data = _read_global_results(scenario_name) + + number_files = len(data) + + total_val_accuracy = data["val_accuracy"].sum() + + avg_val_accuracy = total_val_accuracy / max(1, number_files - 1) + + return avg_val_accuracy + + +def get_participant_loss_accuracy(scenario_name, participant_id): + """ + Gets loss and accuracy for a specific participant from CFL aggregated results. + + Args: + scenario_name (str): Scenario name. + participant_id (int | str): Participant identifier. + + Returns: + tuple[float, float]: (loss, accuracy) + """ + data_file = _global_data_results_path(scenario_name) + row = _find_participant_row(read_csv(data_file), participant_id, data_file) + + loss = float(row["loss"]) + accuracy = float(row["accuracy"]) + return loss, accuracy + +def get_underfitting_score_local(scenario_name, id): + """ + Gets the local validation accuracy for a specific DFL/SDFL participant. + + Args: + scenario_name (str): Scenario name. + participant_id (int | str): Participant identifier. + + Returns: + float: Validation accuracy. + """ + data = _read_participant_results(scenario_name, id) + return float(data["val_accuracy"].iloc[0]) + + +def get_dp_local(scenario_name, id): + """ + Gets the dp metrics for a specific DFL/SDFL participant. + + Args: + scenario_name (str): Scenario name. + participant_id (int | str): Participant identifier. + + Returns: + float: DP Enabled, Epsilon. + """ + data = _read_participant_results(scenario_name, id) + return data["dp_enabled"].iloc[0], float(data["dp_epsilon"].iloc[0]) + + +def get_dp_global(scenario_name): + """ + Gets the aggregated DP metrics for a CFL scenario, excluding the server node. + + Args: + scenario_name (str): Scenario name. + + Returns: + tuple[bool, float | str]: Whether DP is enabled, and the + average epsilon across client nodes. + """ + data = _read_global_results(scenario_name) + + if data["dp_enabled"].iloc[0] == False: + return False, 0.0 + + number_files = len(data) + + avg_epsilon = data["dp_epsilon"].sum() / max(1, number_files - 1) + + return True, avg_epsilon + +def get_avg_class_imbalance_model_size(scenario_name): + """ + Calculates the mean class imbalance and model size of the nodes. + + Args: + data_results_files (list): Files that contain the class imbalance and model size of the nodes + + Returns: + 2-tupla: The mean class imbalance mean and model size mean of the nodes. + """ + data = _read_global_results(scenario_name) + + number_files = len(data) + + total_class_imbalance = data["class_imbalance"].sum() + total_model_size = data["model_size"].sum() + + avg_class_imbalance = total_class_imbalance / number_files + avg_model_size = total_model_size / number_files + + return avg_class_imbalance, avg_model_size + + +def get_entropy_list(scenario_name): + """ + Obtiene una lista con los valores de entropy de todos los nodos. + + Args: + scenario_name (str): Nombre del escenario. + + Returns: + list: Lista con los valores de entropy + """ + data = _read_global_results(scenario_name) + + entropy_list = data["local_entropy"].tolist() + + return entropy_list + +def stop_emissions_tracking_and_save( + tracker: EmissionsTracker, + outdir: str, + emissions_file: str, + role: str, + workload: str, + sample_size: int = 0, + participant_idx=None, +): + """ + Stops emissions tracking object from CodeCarbon and saves relevant information to emissions.csv file. + + Args: + tracker (object): The emissions tracker object holding information. + outdir (str): The path of the output directory of the experiment. + emissions_file (str): The path to the emissions file. + role (str): Either client or server depending on the role. + workload (str): Either aggregation or training depending on the workload. + sample_size (int): The number of samples used for training, if aggregation 0. + """ + + tracker.stop() + + emissions_file = os.path.join(outdir, emissions_file) + + if exists(emissions_file): + df = pd.read_csv(emissions_file) + else: + df = pd.DataFrame( + columns=[ + "id", + "role", + "energy_grid", + "emissions", + "workload", + "CPU_model", + "GPU_model", + ] + ) + try: + energy_grid = (tracker.final_emissions_data.emissions / tracker.final_emissions_data.energy_consumed) * 1000 + df = pd.concat( + [ + df, + pd.DataFrame({ + "id": participant_idx, + "role": role, + "energy_grid": [energy_grid], + "emissions": [tracker.final_emissions_data.emissions], + "workload": workload, + "CPU_model": tracker.final_emissions_data.cpu_model + if tracker.final_emissions_data.cpu_model + else "None", + "GPU_model": tracker.final_emissions_data.gpu_model + if tracker.final_emissions_data.gpu_model + else "None", + "CPU_used": True if tracker.final_emissions_data.cpu_energy else False, + "GPU_used": True if tracker.final_emissions_data.gpu_energy else False, + "energy_consumed": tracker.final_emissions_data.energy_consumed, + "sample_size": sample_size, + }), + ], + ignore_index=True, + ) + df.to_csv(emissions_file, encoding="utf-8", index=False) + except Exception as e: + logger.warning(e) + + +def comm_efficiency(bytes_up: int, bytes_down: int, test_acc_avg: float, eps: float = 1e-12) -> float: + """ + Communication efficiency = total_bytes / final_accuracy. + Lower is better. + + Args: + bytes_up: total uploaded bytes + bytes_down: total downloaded bytes + final_accuracy: final test accuracy in [0,1] + eps: small constant to avoid division by zero + + Returns: + float + """ + total_bytes = float(bytes_up) + float(bytes_down) + acc = float(test_acc_avg) + + if acc < eps: + acc = eps + + return total_bytes / acc diff --git a/nebula/addons/trustworthiness/helpers/scoring.py b/nebula/addons/trustworthiness/helpers/scoring.py new file mode 100644 index 000000000..5103626c8 --- /dev/null +++ b/nebula/addons/trustworthiness/helpers/scoring.py @@ -0,0 +1,190 @@ +import logging + +import numpy as np + +logger = logging.getLogger(__name__) + +def get_mapped_score(score_key, score_map): + """ + Finds the score by the score_key in the score_map. + + Args: + score_key (string): The key to look up in the score_map. + score_map (dict): The score map defined in the eval_metrics.json file. + + Returns: + float: The normalized score of [0, 1]. + """ + score = 0 + if score_map is None: + logger.warning("Score map is missing") + else: + keys = [key for key, value in score_map.items()] + scores = [value for key, value in score_map.items()] + normalized_scores = get_normalized_scores(scores) + normalized_score_map = dict(zip(keys, normalized_scores, strict=False)) + score = normalized_score_map.get(score_key, np.nan) + + return score + + +def get_normalized_scores(scores): + """ + Calculates the normalized scores of a list. + + Args: + scores (list): The values that will be normalized. + + Returns: + list: The normalized list. + """ + if scores is None or len(scores) == 0: + return [] + + min_score = np.min(scores) + max_score = np.max(scores) + if max_score == min_score: + return [1.0 for _ in scores] + + normalized = [(x - min_score) / (max_score - min_score) for x in scores] + return normalized + + +def get_range_score(value, ranges, direction="asc"): + """ + Maps the value to a range and gets the score by the range and direction. + + Args: + value (int): The input score. + ranges (list): The ranges defined. + direction (string): Asc means the higher the range the higher the score, desc means otherwise. + + Returns: + float: The normalized score of [0, 1]. + """ + + if not (type(value) == int or type(value) == float): + logger.warning("Input value is not a number") + logger.warning(f"{value}") + return 0 + else: + score = 0 + if ranges is None: + logger.warning("Score ranges are missing") + else: + total_bins = len(ranges) + 1 + bin = np.digitize(value, ranges, right=True) + score = 1 - (bin / total_bins) if direction == "desc" else bin / total_bins + return score + + +def get_map_value_score(score_key, score_map): + """ + Finds the score by the score_key in the score_map and returns the value. + + Args: + score_key (string): The key to look up in the score_map. + score_map (dict): The score map defined in the eval_metrics.json file. + + Returns: + float: The score obtained in the score_map. + """ + score = 0 + if score_map is None: + logger.warning("Score map is missing") + else: + score = score_map[score_key] + return score + + +def get_true_score(value, direction): + """ + Returns the negative of the value if direction is 'desc', otherwise returns value. + + Args: + value (int): The input score. + direction (string): Asc means the higher the range the higher the score, desc means otherwise. + + Returns: + float: The score obtained. + """ + + if value is True: + return 1 + elif value is False: + return 0 + else: + if not (type(value) == int or type(value) == float): + logger.warning("Input value is not a number") + logger.warning(f"{value}.") + return 0 + else: + if direction == "desc": + return 1 - value + else: + return value + + +def get_scaled_score(value, scale: list, direction: str): + """ + Maps a score of a specific scale into the scale between zero and one. + + Args: + value (int or float): The raw value of the metric. + scale (list): List containing the minimum and maximum value the value can fall in between. + + Returns: + float: The normalized score of [0, 1]. + """ + + score = 0 + try: + value_min, value_max = scale[0], scale[1] + except Exception: + logger.warning("Score minimum or score maximum is missing. The minimum has been set to 0 and the maximum to 1") + value_min, value_max = 0, 1 + if value is None or value == "": + logger.warning("Score value is missing. Set value to zero") + else: + low, high = 0, 1 + if value >= value_max: + score = 1 + elif value <= value_min: + score = 0 + else: + diff = value_max - value_min + diffScale = high - low + score = (float(value) - value_min) * (float(diffScale) / diff) + low + if direction == "desc": + score = high - score + + return score + + +def get_value(value): + """ + Get the value of a metric. + + Args: + value (float): The value of the metric. + + Returns: + float: The value of the metric. + """ + + return value + + +def check_properties(*args): + """ + Check if all the arguments have values. + + Args: + args (list): All the arguments. + + Returns: + float: The mean of arguments that have values. + """ + + result = map(lambda x: x is not None and x != "", args) + return np.mean(list(result)) diff --git a/nebula/addons/trustworthiness/helpers/trust_reports.py b/nebula/addons/trustworthiness/helpers/trust_reports.py new file mode 100644 index 000000000..08d1798ec --- /dev/null +++ b/nebula/addons/trustworthiness/helpers/trust_reports.py @@ -0,0 +1,197 @@ +import copy +import json +import os + +def load_trust_report_json_dumped(scenario_name: str, participant_id: int) -> str: + """ + Read a participant trustworthiness JSON file and return it + serialized as a string with json.dumps(...). + + Args: + scenario_name (str): Scenario/experiment name. + participant_id (int): Participant ID. + + Returns: + str: JSON content serialized as a string. + + Raises: + FileNotFoundError: If the file does not exist. + ValueError: If the file content is not valid JSON. + """ + logs_dir = os.environ.get("NEBULA_LOGS_DIR") + if not logs_dir: + raise ValueError("The NEBULA_LOGS_DIR environment variable is not defined.") + + file_name = f"nebula_trust_results_{participant_id}.json" + file_path = os.path.join( + logs_dir, + scenario_name, + "trustworthiness", + file_name, + ) + + if not os.path.exists(file_path): + raise FileNotFoundError(f"The file does not exist: {file_path}") + + try: + with open(file_path, "r", encoding="utf-8") as f: + trust_report = json.load(f) + except json.JSONDecodeError as e: + raise ValueError(f"The file does not contain valid JSON: {file_path}") from e + + return json.dumps(trust_report) + + +def load_trust_report_json(scenario_name: str, participant_id: int | str) -> dict: + trust_report_json = load_trust_report_json_dumped(scenario_name, participant_id) + return json.loads(trust_report_json) + + +def create_local_trust_report_copy(scenario_name: str, participant_id: int | str, suffix: str = "global") -> tuple[dict, str]: + trust_report = load_trust_report_json(scenario_name, participant_id) + logs_dir = os.environ.get("NEBULA_LOGS_DIR") + if not logs_dir: + raise ValueError("The NEBULA_LOGS_DIR environment variable is not defined.") + + trust_dir = os.path.join(logs_dir, scenario_name, "trustworthiness") + os.makedirs(trust_dir, exist_ok=True) + + file_path = os.path.join(trust_dir, f"nebula_trust_results_{participant_id}_{suffix}.json") + with open(file_path, "w", encoding="utf-8") as f: + json.dump(trust_report, f, indent=4) + + return trust_report, file_path + + +def save_trust_report_json(file_path: str, trust_report: dict) -> str: + directory = os.path.dirname(file_path) + if directory: + os.makedirs(directory, exist_ok=True) + + with open(file_path, "w", encoding="utf-8") as f: + json.dump(trust_report, f, indent=4) + + return file_path + + +def accumulate_weighted_trustscores(report: dict, weight: float, score_accumulator: dict, weight_accumulator: dict): + if weight <= 0: + raise ValueError("The aggregation weight must be greater than 0.") + + _accumulate_weighted_trustscores_recursive( + obj=report, + weight=float(weight), + path=(), + score_accumulator=score_accumulator, + weight_accumulator=weight_accumulator, + ) + + +def build_weighted_trustscores_report(template_report: dict, score_accumulator: dict, weight_accumulator: dict) -> dict: + aggregated_report = copy.deepcopy(template_report) + _apply_weighted_trustscores_recursive( + obj=aggregated_report, + path=(), + score_accumulator=score_accumulator, + weight_accumulator=weight_accumulator, + ) + return aggregated_report + + +def _accumulate_weighted_trustscores_recursive(obj, weight: float, path: tuple, score_accumulator: dict, weight_accumulator: dict): + if isinstance(obj, dict): + structural_named_entry = _get_structural_named_entry(obj) + if structural_named_entry is not None: + _, nested_value = structural_named_entry + _accumulate_weighted_trustscores_recursive( + obj=nested_value, + weight=weight, + path=path + ("__named_entry__",), + score_accumulator=score_accumulator, + weight_accumulator=weight_accumulator, + ) + return + + for key, value in obj.items(): + if key in {"trust_score", "score"} and _is_numeric_score(value): + score_path = path + (key,) + score_accumulator[score_path] = score_accumulator.get(score_path, 0.0) + (float(value) * weight) + weight_accumulator[score_path] = weight_accumulator.get(score_path, 0.0) + weight + continue + + _accumulate_weighted_trustscores_recursive( + obj=value, + weight=weight, + path=path + (key,), + score_accumulator=score_accumulator, + weight_accumulator=weight_accumulator, + ) + return + + if isinstance(obj, list): + for index, item in enumerate(obj): + _accumulate_weighted_trustscores_recursive( + obj=item, + weight=weight, + path=path + (index,), + score_accumulator=score_accumulator, + weight_accumulator=weight_accumulator, + ) + + +def _apply_weighted_trustscores_recursive(obj, path: tuple, score_accumulator: dict, weight_accumulator: dict): + if isinstance(obj, dict): + structural_named_entry = _get_structural_named_entry(obj) + if structural_named_entry is not None: + entry_key, nested_value = structural_named_entry + obj[entry_key] = _apply_weighted_trustscores_recursive( + obj=nested_value, + path=path + ("__named_entry__",), + score_accumulator=score_accumulator, + weight_accumulator=weight_accumulator, + ) + return obj + + for key, value in obj.items(): + if key in {"trust_score", "score"} and _is_numeric_score(value): + score_path = path + (key,) + total_weight = weight_accumulator.get(score_path) + if total_weight: + obj[key] = round(score_accumulator[score_path] / total_weight, 6) + continue + + obj[key] = _apply_weighted_trustscores_recursive( + obj=value, + path=path + (key,), + score_accumulator=score_accumulator, + weight_accumulator=weight_accumulator, + ) + return obj + + if isinstance(obj, list): + for index, item in enumerate(obj): + obj[index] = _apply_weighted_trustscores_recursive( + obj=item, + path=path + (index,), + score_accumulator=score_accumulator, + weight_accumulator=weight_accumulator, + ) + return obj + + +def _get_structural_named_entry(obj: dict): + if len(obj) != 1: + return None + + entry_key, nested_value = next(iter(obj.items())) + if not isinstance(nested_value, dict): + return None + + if any(key in nested_value for key in ("score", "metrics", "notions", "pillars")): + return entry_key, nested_value + + return None + + +def _is_numeric_score(value): + return isinstance(value, (int, float)) and not isinstance(value, bool) diff --git a/nebula/addons/trustworthiness/metric.py b/nebula/addons/trustworthiness/metric.py index f9e24e72d..f1e453235 100755 --- a/nebula/addons/trustworthiness/metric.py +++ b/nebula/addons/trustworthiness/metric.py @@ -4,7 +4,7 @@ from nebula.addons.trustworthiness.graphics import Graphics from nebula.addons.trustworthiness.pillar import TrustPillar -from nebula.addons.trustworthiness.utils import write_results_json +from nebula.addons.trustworthiness.helpers.csv_io import write_results_json dirname = os.path.dirname(__file__) diff --git a/nebula/addons/trustworthiness/per_round_metrics.py b/nebula/addons/trustworthiness/per_round_metrics.py index e8104befd..ea5a2ff3d 100644 --- a/nebula/addons/trustworthiness/per_round_metrics.py +++ b/nebula/addons/trustworthiness/per_round_metrics.py @@ -2,7 +2,6 @@ from __future__ import annotations import asyncio -import copy import csv import os from dataclasses import dataclass, field diff --git a/nebula/addons/trustworthiness/pillar.py b/nebula/addons/trustworthiness/pillar.py index a57ec1abb..ecd15cf7e 100755 --- a/nebula/addons/trustworthiness/pillar.py +++ b/nebula/addons/trustworthiness/pillar.py @@ -1,7 +1,13 @@ import logging -from nebula.addons.trustworthiness import calculation -from nebula.addons.trustworthiness.utils import get_input_value +from nebula.addons.trustworthiness.helpers.factsheet_values import get_input_value +from nebula.addons.trustworthiness.helpers.scoring import ( + get_map_value_score, + get_mapped_score, + get_range_score, + get_scaled_score, + get_true_score, +) logger = logging.getLogger(__name__) @@ -96,15 +102,15 @@ def get_metric_score(self, result, name, metric): logger.warning(f"{name} input value is null") else: if score_type == "true_score": - score = calculation.get_true_score(input_value, metric.get("direction")) + score = get_true_score(input_value, metric.get("direction")) elif score_type == "score_mapping": - score = calculation.get_mapped_score(input_value, metric.get("score_map")) + score = get_mapped_score(input_value, metric.get("score_map")) elif score_type == "ranges": - score = calculation.get_range_score(input_value, metric.get("ranges"), metric.get("direction")) + score = get_range_score(input_value, metric.get("ranges"), metric.get("direction")) elif score_type == "score_map_value": - score = calculation.get_map_value_score(input_value, metric.get("score_map")) + score = get_map_value_score(input_value, metric.get("score_map")) elif score_type == "scaled_score": - score = calculation.get_scaled_score(input_value, metric.get("scale"), metric.get("direction")) + score = get_scaled_score(input_value, metric.get("scale"), metric.get("direction")) elif score_type == "property_check": score = 0 if input_value is None else input_value diff --git a/nebula/addons/trustworthiness/trustworthiness.py b/nebula/addons/trustworthiness/trustworthiness.py index b2a9ba2ad..17b9a4ef8 100644 --- a/nebula/addons/trustworthiness/trustworthiness.py +++ b/nebula/addons/trustworthiness/trustworthiness.py @@ -7,8 +7,31 @@ from abc import ABC, abstractmethod from nebula.config.config import Config from nebula.core.engine import Engine -from nebula.addons.trustworthiness.calculation import stop_emissions_tracking_and_save, get_bytes_model, get_class_imbalance_local, get_participation_variation_score -from nebula.addons.trustworthiness.utils import save_results_csv, save_trustworthiness_reports_csv, load_emissions_participant, load_data_results_participant, save_results_csv_cfl, save_emissions_csv_cfl, save_class_count_per_participant, get_local_entropy, load_trust_report_json_dumped, create_local_trust_report_copy, accumulate_weighted_trustscores, build_weighted_trustscores_report, save_trust_report_json +from nebula.addons.trustworthiness.helpers.csv_io import ( + load_data_results_participant, + load_emissions_participant, + save_emissions_csv_cfl, + save_results_csv, + save_results_csv_cfl, + save_trustworthiness_reports_csv, +) +from nebula.addons.trustworthiness.helpers.data_distribution import ( + get_class_imbalance_local, + get_local_entropy, + get_participation_variation_score, + save_class_count_per_participant, +) +from nebula.addons.trustworthiness.helpers.scenario_metrics import ( + get_bytes_model, + stop_emissions_tracking_and_save, +) +from nebula.addons.trustworthiness.helpers.trust_reports import ( + accumulate_weighted_trustscores, + build_weighted_trustscores_report, + create_local_trust_report_copy, + load_trust_report_json_dumped, + save_trust_report_json, +) from codecarbon import EmissionsTracker from nebula.addons.trustworthiness.per_round_metrics import PerRoundTrustMetrics from datetime import datetime diff --git a/nebula/addons/trustworthiness/utils.py b/nebula/addons/trustworthiness/utils.py deleted file mode 100755 index 62dfe5f08..000000000 --- a/nebula/addons/trustworthiness/utils.py +++ /dev/null @@ -1,656 +0,0 @@ -import json -import csv -import logging -import math -import os -import pickle -from os.path import exists -import copy - -import pandas as pd -from hashids import Hashids -from scipy.stats import entropy - -from nebula.addons.trustworthiness import calculation -from collections import Counter - -hashids = Hashids() -logger = logging.getLogger(__name__) -dirname = os.path.dirname(__file__) - - -def save_class_count_per_participant(experiment_name, class_counter: Counter, idx): - class_count = os.path.join(os.environ.get('NEBULA_LOGS_DIR'), experiment_name, "trustworthiness", f"{str(idx)}_class_count.json") - result = {hashids.encode(int(class_id)): count for class_id, count in class_counter.items()} - with open(class_count, "w") as f: - json.dump(result, f) - -def count_all_class_samples(experiment_name): - participant_id = 0 - global_class_count = {} - - while True: - data_class_count_file = os.path.join(os.environ.get('NEBULA_LOGS_DIR'), experiment_name, "trustworthiness", f"{str(participant_id)}_class_count.json") - - if not os.path.exists(data_class_count_file): - break - - with open(data_class_count_file, "r") as f: - class_count = json.load(f) - - for class_hash, count in class_count.items(): - global_class_count[class_hash] = global_class_count.get(class_hash, 0) + count - - participant_id += 1 - - # Save the total class count into class_count.json - output_file = os.path.join(os.environ.get('NEBULA_LOGS_DIR'),experiment_name, "trustworthiness", "count_class.json") - - with open(output_file, "w") as f: - json.dump(global_class_count, f, indent=2) - -def count_class_samples(scenario_name, dataloaders_files, class_counter: Counter = None): - """ - Counts the number of samples by class. - - Args: - scenario_name (string): Name of the scenario. - dataloaders_files (list): Files that contain the dataloaders. - - """ - - result = {} - dataloaders = [] - - if class_counter: - result = {hashids.encode(int(class_id)): count for class_id, count in class_counter.items()} - else: - for file in dataloaders_files: - with open(file, "rb") as f: - dataloader = pickle.load(f) - dataloaders.append(dataloader) - - for dataloader in dataloaders: - for batch, labels in dataloader: - for b, label in zip(batch, labels): - l = hashids.encode(label.item()) - if l in result: - result[l] += 1 - else: - result[l] = 1 - - try: - name_file = os.path.join(os.environ.get('NEBULA_LOGS_DIR'), scenario_name, "trustworthiness", "count_class.json") - except: - name_file = os.path.join("nebula", "app", "logs", scenario_name, "trustworthiness", "count_class.json") - - with open(name_file, "w") as f: - json.dump(result, f) - - -def get_all_data_entropy(experiment_name): - participant_id = 0 - data_class_count_file = os.path.join(os.environ.get('NEBULA_LOGS_DIR'), experiment_name, "trustworthiness", f"{str(participant_id)}_class_count.json") - entropy_per_participant = {} - - while True: - data_class_count_file = os.path.join(os.environ.get('NEBULA_LOGS_DIR'), experiment_name, "trustworthiness", f"{str(participant_id)}_class_count.json") - - if not os.path.exists(data_class_count_file): - break - - with open(data_class_count_file, "r") as f: - class_count = json.load(f) - - entropy_value = calculation.get_entropy_from_class_counts(class_count) - - entropy_per_participant[str(participant_id)] = round(entropy_value, 6) - participant_id += 1 - - name_file = os.path.join(os.environ.get('NEBULA_LOGS_DIR'),experiment_name, "trustworthiness", "entropy.json") - - with open(name_file, "w") as f: - json.dump(entropy_per_participant, f, indent=2) - -def get_local_entropy(id, experiment_name): - data_class_count_file = os.path.join(os.environ.get('NEBULA_LOGS_DIR'), experiment_name, "trustworthiness", f"{str(id)}_class_count.json") - - with open(data_class_count_file, "r") as f: - class_count = json.load(f) - - return calculation.get_entropy_from_class_counts(class_count) - -def get_entropy(client_id, scenario_name, dataloader): - """ - Get the entropy of each client in the scenario. - - Args: - client_id (int): The client id. - scenario_name (string): Name of the scenario. - dataloaders_files (list): Files that contain the dataloaders. - - """ - result = {} - client_entropy = {} - - name_file = os.path.join(os.environ.get('NEBULA_LOGS_DIR'), scenario_name, "trustworthiness", "entropy.json") - - if os.path.exists(name_file): - logging.info(f"entropy fiel already exists.. loading.") - with open(name_file, "r") as f: - client_entropy = json.load(f) - - client_id_hash = hashids.encode(client_id) - - for batch, labels in dataloader: - for b, label in zip(batch, labels): - l = hashids.encode(label.item()) - if l in result: - result[l] += 1 - else: - result[l] = 1 - - n = len(dataloader) - entropy_value = entropy([x / n for x in result.values()], base=2) - client_entropy[client_id_hash] = entropy_value - with open(name_file, "w") as f: - json.dump(client_entropy, f) - - -def read_csv(filename): - """ - Read a CSV file. - - Args: - filename (string): Name of the file. - - Returns: - object: The CSV readed. - - """ - if exists(filename): - return pd.read_csv(filename) - - -def check_field_filled(factsheet_dict, factsheet_path, value, empty=""): - """ - Check if the field in the factsheet file is filled or not. - - Args: - factsheet_dict (dict): The factshett dict. - factsheet_path (list): The factsheet field to check. - value (float): The value to add in the field. - empty (string): If the value could not be appended, the empty string is returned. - - Returns: - float: The value added in the factsheet or empty if the value could not be appened - - """ - if factsheet_dict[factsheet_path[0]][factsheet_path[1]]: - return factsheet_dict[factsheet_path[0]][factsheet_path[1]] - elif value != "" and value != "nan": - if type(value) != str and type(value) != list: - if math.isnan(value): - return 0 - else: - return value - else: - return value - else: - return empty - - -def get_input_value(input_docs, inputs, operation): - """ - Gets the input value from input document and apply the metric operation on the value. - - Args: - inputs_docs (map): The input document map. - inputs (list): All the inputs. - operation (string): The metric operation. - - Returns: - float: The metric value - - """ - - input_value = None - args = [] - for i in inputs: - source = i.get("source", "") - field = i.get("field_path", "") - input_doc = input_docs.get(source, None) - if input_doc is None: - logger.warning(f"{source} is null") - else: - input = get_value_from_path(input_doc, field) - args.append(input) - try: - operationFn = getattr(calculation, operation) - input_value = operationFn(*args) - except TypeError: - logger.warning(f"{operation} is not valid") - - return input_value - - -def get_value_from_path(input_doc, path): - """ - Gets the input value from input document by path. - - Args: - inputs_doc (map): The input document map. - path (string): The field name of the input value of interest. - - Returns: - float: The input value from the input document - - """ - - d = input_doc - for nested_key in path.split("/"): - temp = d.get(nested_key) - if isinstance(temp, dict): - d = d.get(nested_key) - else: - return temp - return None - - -def write_results_json(out_file, dict): - """ - Writes the result to JSON. - - Args: - out_file (string): The output file. - dict (dict): The object to be witten into JSON. - - Returns: - float: The input value from the input document - - """ - - with open(out_file, "a") as f: - json.dump(dict, f, indent=4) - -def load_data_results_participant(experiment_name: str, participant_id: int | str): - data_results_path = os.path.join(os.environ.get('NEBULA_LOGS_DIR'), experiment_name, "trustworthiness", f"data_results_{participant_id}.csv") - - if not os.path.exists(data_results_path): - raise FileNotFoundError(f"File not found: {data_results_path}") - - with open(data_results_path, "r", newline="") as csv_file: - reader = csv.DictReader(csv_file) - rows = list(reader) - - if len(rows) == 0: - raise ValueError(f"No rows found in {data_results_path}") - - row = rows[0] - - bytes_sent = int(float(row["bytes_sent"])) - bytes_recv = int(float(row["bytes_recv"])) - accuracy = float(row["accuracy"]) - loss = float(row["loss"]) - val_accuracy = float(row["val_accuracy"]) - dp_enabled = row["dp_enabled"].lower() == "true" - dp_epsilon = float(row["dp_epsilon"]) - - return bytes_sent, bytes_recv, accuracy, loss, val_accuracy, dp_enabled, dp_epsilon - - -def load_emissions_participant(experiment_name: str, participant_id: int | str): - emissions_path = os.path.join(os.environ.get('NEBULA_LOGS_DIR'), experiment_name, "trustworthiness", f"emissions_{participant_id}.csv") - - if not os.path.exists(emissions_path): - raise FileNotFoundError(f"File not found: {emissions_path}") - - with open(emissions_path, "r", newline="") as csv_file: - reader = csv.DictReader(csv_file) - rows = list(reader) - - if len(rows) == 0: - raise ValueError(f"No rows found in {emissions_path}") - - row = rows[0] - - role = str(row["role"]) - energy_grid = float(row["energy_grid"]) - emissions = float(row["emissions"]) - workload = str(row["workload"]) - cpu_model = str(row["CPU_model"]) - gpu_model = str(row["GPU_model"]) - cpu_used = str(row["CPU_used"]).strip().lower() == "true" - gpu_used = str(row["GPU_used"]).strip().lower() == "true" - energy_consumed = float(row["energy_consumed"]) - sample_size = int(float(row["sample_size"])) - - return role, energy_grid, emissions, workload, cpu_model, gpu_model, cpu_used, gpu_used, energy_consumed, sample_size - -def save_trustworthiness_reports_csv( - reports: dict, - experiment_name: str, -) -> None: - - data_results_path = os.path.join("nebula", "app", "logs", experiment_name, "trustworthiness", "data_results.csv") - emissions_path = os.path.join("nebula", "app", "logs", experiment_name, "trustworthiness", "emissions.csv") - - sorted_reports = sorted( - reports.values(), - key=lambda report: int(report["node_id"]) - ) - - with open(data_results_path, "w", newline="") as csv_file: - writer = csv.DictWriter( - csv_file, - fieldnames=["id", "bytes_sent", "bytes_recv", "accuracy", "loss", "class_imbalance", "model_size", "local_entropy", "val_accuracy", "dp_enabled", "dp_epsilon"], - ) - writer.writeheader() - - for report in sorted_reports: - writer.writerow({ - "id": report["node_id"], - "bytes_sent": report["bytes_sent"], - "bytes_recv": report["bytes_recv"], - "accuracy": report["accuracy"], - "loss": report["loss"], - "class_imbalance": report["class_imbalance"], - "model_size": report["model_size"], - "local_entropy": report["local_entropy"], - "val_accuracy": report["val_accuracy"], - "dp_enabled": report["dp_enabled"], - "dp_epsilon": report["dp_epsilon"], - }) - - with open(emissions_path, "w", newline="") as csv_file: - writer = csv.DictWriter( - csv_file, - fieldnames=["id", "role", "energy_grid", "emissions", "workload", "CPU_model", "GPU_model", "CPU_used", "GPU_used", "energy_consumed", "sample_size"], - ) - writer.writeheader() - - for report in sorted_reports: - writer.writerow({ - "id": report["node_id"], - "role": report["role"], - "energy_grid": report["energy_grid"], - "emissions": report["emissions"], - "workload": report["workload"], - "CPU_model": report["cpu_model"], - "GPU_model": report["gpu_model"], - "CPU_used": report["cpu_used"], - "GPU_used": report["gpu_used"], - "energy_consumed": report["energy_consumed"], - "sample_size": report["sample_size"], - }) - - logging.info( - "[TW SERVER] CSV files written correctly: %s, %s", - data_results_path, - emissions_path, - ) - -def save_results_csv_cfl(scenario_name: str, id: int, bytes_sent: int, bytes_recv: int, accuracy: float, loss: float, class_imbalance: float, model_size: int, local_entropy: float, val_accuracy: float, dp_enabled: bool, dp_epsilon: float): - try: - data_results_file = os.path.join(os.environ.get('NEBULA_LOGS_DIR'), scenario_name, "trustworthiness", "data_results.csv") - except: - data_results_file = os.path.join("nebula", "app", "logs", scenario_name, "trustworthiness", "data_results.csv") - - if exists(data_results_file): - df = pd.read_csv(data_results_file) - else: - df = pd.DataFrame(columns=["id", "bytes_sent", "bytes_recv", "accuracy", "loss", "class_imbalance", "model_size", "local_entropy", "val_accuracy", "dp_enabled", "dp_epsilon"]) - - try: - # Add new entry to DataFrame - new_data = pd.DataFrame({'id': [id], 'bytes_sent': [bytes_sent], - 'bytes_recv': [bytes_recv], 'accuracy': [accuracy], - 'loss': [loss], 'class_imbalance': [class_imbalance], 'model_size': [model_size], 'local_entropy': [local_entropy], 'val_accuracy': [val_accuracy], 'dp_enabled': [dp_enabled], 'dp_epsilon': [dp_epsilon]}) - df = pd.concat([df, new_data], ignore_index=True) - - df.to_csv(data_results_file, encoding='utf-8', index=False) - - except Exception as e: - logger.warning(e) - -def save_emissions_csv_cfl(scenario_name: str, id: int, role: str, energy_grid: float, emissions: float, workload: str, cpu_model: str, gpu_model: str, cpu_used: bool, gpu_used: bool, energy_consumed: float, sample_size: int): - try: - data_results_file = os.path.join(os.environ.get('NEBULA_LOGS_DIR'), scenario_name, "trustworthiness", "emissions.csv") - except: - data_results_file = os.path.join("nebula", "app", "logs", scenario_name, "trustworthiness", "emissions.csv") - - if exists(data_results_file): - df = pd.read_csv(data_results_file) - else: - df = pd.DataFrame(columns=["id", "role", "energy_grid", "emissions", "workload", "CPU_model", "GPU_model", "CPU_used", "GPU_used", "energy_consumed", "sample_size"]) - - try: - # Add new entry to DataFrame - new_data = pd.DataFrame({'id': [id], 'role': [role], 'energy_grid': [energy_grid], - 'emissions': [emissions], 'workload': [workload], 'CPU_model': [cpu_model], 'GPU_model': [gpu_model], 'CPU_used': [cpu_used], 'GPU_used': [gpu_used], 'energy_consumed': [energy_consumed], - 'sample_size': [sample_size]}) - df = pd.concat([df, new_data], ignore_index=True) - - df.to_csv(data_results_file, encoding='utf-8', index=False) - - except Exception as e: - logger.warning(e) - - -def save_results_csv(scenario_name: str, id: int, bytes_sent: int, bytes_recv: int, accuracy: float, loss: float, val_accuracy: float, dp_enabled: bool, dp_epsilon: float): - - try: - data_results_id_file = os.path.join(os.environ.get('NEBULA_LOGS_DIR'), scenario_name, "trustworthiness", f"data_results_{id}.csv") - except: - data_results_id_file = os.path.join("nebula", "app", "logs", scenario_name, "trustworthiness", f"data_results_{id}.csv") - - if exists(data_results_id_file): - df = pd.read_csv(data_results_id_file) - else: - df = pd.DataFrame(columns=["id", "bytes_sent", "bytes_recv", "accuracy", "loss", "val_accuracy", "dp_enabled", "dp_epsilon"]) - - try: - # Add new entry to DataFrame - new_data = pd.DataFrame({'id': [id], 'bytes_sent': [bytes_sent], - 'bytes_recv': [bytes_recv], 'accuracy': [accuracy], - 'loss': [loss], 'val_accuracy': [val_accuracy], 'dp_enabled': [dp_enabled], 'dp_epsilon': [dp_epsilon]}) - df = pd.concat([df, new_data], ignore_index=True) - - df.to_csv(data_results_id_file, encoding='utf-8', index=False) - - except Exception as e: - logger.warning(e) - -def load_trust_report_json_dumped(scenario_name: str, participant_id: int) -> str: - """ - Read a participant trustworthiness JSON file and return it - serialized as a string with json.dumps(...). - - Args: - scenario_name (str): Scenario/experiment name. - participant_id (int): Participant ID. - - Returns: - str: JSON content serialized as a string. - - Raises: - FileNotFoundError: If the file does not exist. - ValueError: If the file content is not valid JSON. - """ - logs_dir = os.environ.get("NEBULA_LOGS_DIR") - if not logs_dir: - raise ValueError("The NEBULA_LOGS_DIR environment variable is not defined.") - - file_name = f"nebula_trust_results_{participant_id}.json" - file_path = os.path.join( - logs_dir, - scenario_name, - "trustworthiness", - file_name, - ) - - if not os.path.exists(file_path): - raise FileNotFoundError(f"The file does not exist: {file_path}") - - try: - with open(file_path, "r", encoding="utf-8") as f: - trust_report = json.load(f) - except json.JSONDecodeError as e: - raise ValueError(f"The file does not contain valid JSON: {file_path}") from e - - return json.dumps(trust_report) - - -def load_trust_report_json(scenario_name: str, participant_id: int | str) -> dict: - trust_report_json = load_trust_report_json_dumped(scenario_name, participant_id) - return json.loads(trust_report_json) - - -def create_local_trust_report_copy(scenario_name: str, participant_id: int | str, suffix: str = "global") -> tuple[dict, str]: - trust_report = load_trust_report_json(scenario_name, participant_id) - logs_dir = os.environ.get("NEBULA_LOGS_DIR") - if not logs_dir: - raise ValueError("The NEBULA_LOGS_DIR environment variable is not defined.") - - trust_dir = os.path.join(logs_dir, scenario_name, "trustworthiness") - os.makedirs(trust_dir, exist_ok=True) - - file_path = os.path.join(trust_dir, f"nebula_trust_results_{participant_id}_{suffix}.json") - with open(file_path, "w", encoding="utf-8") as f: - json.dump(trust_report, f, indent=4) - - return trust_report, file_path - - -def save_trust_report_json(file_path: str, trust_report: dict) -> str: - directory = os.path.dirname(file_path) - if directory: - os.makedirs(directory, exist_ok=True) - - with open(file_path, "w", encoding="utf-8") as f: - json.dump(trust_report, f, indent=4) - - return file_path - - -def accumulate_weighted_trustscores(report: dict, weight: float, score_accumulator: dict, weight_accumulator: dict): - if weight <= 0: - raise ValueError("The aggregation weight must be greater than 0.") - - _accumulate_weighted_trustscores_recursive( - obj=report, - weight=float(weight), - path=(), - score_accumulator=score_accumulator, - weight_accumulator=weight_accumulator, - ) - - -def build_weighted_trustscores_report(template_report: dict, score_accumulator: dict, weight_accumulator: dict) -> dict: - aggregated_report = copy.deepcopy(template_report) - _apply_weighted_trustscores_recursive( - obj=aggregated_report, - path=(), - score_accumulator=score_accumulator, - weight_accumulator=weight_accumulator, - ) - return aggregated_report - - -def _accumulate_weighted_trustscores_recursive(obj, weight: float, path: tuple, score_accumulator: dict, weight_accumulator: dict): - if isinstance(obj, dict): - structural_named_entry = _get_structural_named_entry(obj) - if structural_named_entry is not None: - _, nested_value = structural_named_entry - _accumulate_weighted_trustscores_recursive( - obj=nested_value, - weight=weight, - path=path + ("__named_entry__",), - score_accumulator=score_accumulator, - weight_accumulator=weight_accumulator, - ) - return - - for key, value in obj.items(): - if key in {"trust_score", "score"} and _is_numeric_score(value): - score_path = path + (key,) - score_accumulator[score_path] = score_accumulator.get(score_path, 0.0) + (float(value) * weight) - weight_accumulator[score_path] = weight_accumulator.get(score_path, 0.0) + weight - continue - - _accumulate_weighted_trustscores_recursive( - obj=value, - weight=weight, - path=path + (key,), - score_accumulator=score_accumulator, - weight_accumulator=weight_accumulator, - ) - return - - if isinstance(obj, list): - for index, item in enumerate(obj): - _accumulate_weighted_trustscores_recursive( - obj=item, - weight=weight, - path=path + (index,), - score_accumulator=score_accumulator, - weight_accumulator=weight_accumulator, - ) - - -def _apply_weighted_trustscores_recursive(obj, path: tuple, score_accumulator: dict, weight_accumulator: dict): - if isinstance(obj, dict): - structural_named_entry = _get_structural_named_entry(obj) - if structural_named_entry is not None: - entry_key, nested_value = structural_named_entry - obj[entry_key] = _apply_weighted_trustscores_recursive( - obj=nested_value, - path=path + ("__named_entry__",), - score_accumulator=score_accumulator, - weight_accumulator=weight_accumulator, - ) - return obj - - for key, value in obj.items(): - if key in {"trust_score", "score"} and _is_numeric_score(value): - score_path = path + (key,) - total_weight = weight_accumulator.get(score_path) - if total_weight: - obj[key] = round(score_accumulator[score_path] / total_weight, 6) - continue - - obj[key] = _apply_weighted_trustscores_recursive( - obj=value, - path=path + (key,), - score_accumulator=score_accumulator, - weight_accumulator=weight_accumulator, - ) - return obj - - if isinstance(obj, list): - for index, item in enumerate(obj): - obj[index] = _apply_weighted_trustscores_recursive( - obj=item, - path=path + (index,), - score_accumulator=score_accumulator, - weight_accumulator=weight_accumulator, - ) - return obj - - -def _get_structural_named_entry(obj: dict): - if len(obj) != 1: - return None - - entry_key, nested_value = next(iter(obj.items())) - if not isinstance(nested_value, dict): - return None - - if any(key in nested_value for key in ("score", "metrics", "notions", "pillars")): - return entry_key, nested_value - - return None - - -def _is_numeric_score(value): - return isinstance(value, (int, float)) and not isinstance(value, bool) From 23fec3593ca182be874aeddddb15ba3c289d5628 Mon Sep 17 00:00:00 2001 From: "Juan J." Date: Mon, 1 Jun 2026 13:37:25 +0200 Subject: [PATCH 56/66] Refactoring scoring, trust_reports, scenario_metrics, factsheet_values, and many functionality commented, other changes --- nebula/addons/reputation/reputation.py | 16 + .../{factsheet.py => cfl_factsheet.py} | 19 +- .../addons/trustworthiness/dfl_factsheet.py | 15 +- .../trustworthiness/factsheet_common.py | 33 +- .../helpers/factsheet_values.py | 109 +++--- .../helpers/scenario_metrics.py | 329 ++++++------------ .../addons/trustworthiness/helpers/scoring.py | 205 ++++------- .../trustworthiness/helpers/trust_reports.py | 137 ++++---- .../addons/trustworthiness/trustworthiness.py | 3 +- .../updatehandlers/sdflupdatehandler.py | 10 + nebula/core/engine.py | 4 + nebula/core/models/nebulamodel.py | 1 + nebula/core/network/forwarder.py | 2 + nebula/core/network/messages.py | 4 + nebula/core/node.py | 1 + nebula/core/noderole.py | 12 + nebula/core/training/dp.py | 10 + nebula/core/training/lightning.py | 1 + nebula/core/training/lightning_dp.py | 13 + 19 files changed, 410 insertions(+), 514 deletions(-) rename nebula/addons/trustworthiness/{factsheet.py => cfl_factsheet.py} (89%) diff --git a/nebula/addons/reputation/reputation.py b/nebula/addons/reputation/reputation.py index 19b4e9159..25ce5a770 100644 --- a/nebula/addons/reputation/reputation.py +++ b/nebula/addons/reputation/reputation.py @@ -1861,6 +1861,7 @@ async def calculate_reputation(self, ae: AggregationEvent): async def calculate_sdfl_reputation(self, _ree: RoundEndEvent): """Calculate SDFL reputation at round end for trainers and aggregators.""" + # SDFL shares reputation tables instead of direct feedback messages at round end. await self.calculate_and_send_sdfl_reputation_table() async def calculate_and_send_sdfl_reputation_table(self): @@ -1875,6 +1876,7 @@ async def calculate_and_send_sdfl_reputation_table(self): await self._log_reputation_calculation_start() + # Each node computes direct-neighbor reputation from locally observed metrics. neighbors = set(await self._engine._cm.get_addrs_current_connections(only_direct=True)) await self._process_neighbor_metrics(neighbors) await self._calculate_reputation_by_factor(neighbors) @@ -1965,6 +1967,7 @@ async def _finalize_reputation_calculation(self, updates, neighbors): await self.update_process_aggregation(updates) federation = self._engine.config.participant["scenario_args"].get("federation") if federation == "SDFL": + # SDFL forwards compact reputation tables so the aggregator can infer non-neighbor trust. await self.send_reputation_table_to_neighbors(neighbors) elif federation != "CFL": await self.send_reputation_to_neighbors(neighbors) @@ -1975,6 +1978,7 @@ async def get_local_reputation_table(self, round_num: int = None): round_num = await self._engine.get_round() direct_neighbors = set(await self._engine.cm.get_addrs_current_connections(only_direct=True, myself=False)) + # Only export scores observed locally for this round; indirect scores are not re-shared. return { node_id: float(data["reputation"]) for node_id, data in self.reputation.items() @@ -1985,6 +1989,7 @@ async def get_local_reputation_table(self, round_num: int = None): async def register_reputation_table(self, node_id: str, round_num: int, reputation_table: dict, received_from: str = None): """Store a reputation table received for a round.""" + # Normalize table payloads at the boundary so aggregation uses numeric scores only. normalized_table = {} for neighbor, score in reputation_table.items(): try: @@ -2006,6 +2011,7 @@ async def register_reputation_table(self, node_id: str, round_num: int, reputati expected = self._reputation_tables_expected.get(round_num) event = self._reputation_tables_events.get(round_num) if expected and event and expected.issubset(self.reputation_tables[round_num].keys()): + # Wake any aggregator task blocked waiting for all expected reputation tables. event.set() async def wait_reputation_tables(self, expected_nodes, round_num: int, timeout: float): @@ -2014,6 +2020,7 @@ async def wait_reputation_tables(self, expected_nodes, round_num: int, timeout: self._reputation_tables_expected[round_num] = expected_nodes event = self._reputation_tables_events.setdefault(round_num, asyncio.Event()) + # The table may have arrived before the wait was registered. if expected_nodes.issubset(self.reputation_tables.get(round_num, {}).keys()): event.set() @@ -2035,6 +2042,7 @@ def start_reputation_tables_collection(self, expected_nodes, round_num: int, tim if round_num in self._reputation_tables_wait_tasks: return + # Keep collecting in the background so late tables are visible before aggregation. async def _wait_and_log(): tables, missing = await self.wait_reputation_tables(expected_nodes, round_num, timeout) logging.info( @@ -2056,6 +2064,7 @@ async def calculate_indirect_reputation_for_non_neighbors( ): """Calculate indirect SDFL reputation for non-neighbor nodes from received tables.""" direct_neighbors = set(await self._engine.cm.get_addrs_current_connections(only_direct=True, myself=False)) + # The aggregator already has direct scores for neighbors; tables fill the non-neighbor gap. target_nodes = set(target_nodes) - direct_neighbors - {self._addr} expected_table_nodes = set(expected_table_nodes) @@ -2076,6 +2085,7 @@ async def calculate_indirect_reputation_for_non_neighbors( indirect_reputations = {} for node_id in target_nodes: + # Average all tables that contain the target node to estimate indirect reputation. scores = [ float(table[node_id]) for table in tables.values() @@ -2097,6 +2107,7 @@ async def calculate_indirect_reputation_for_non_neighbors( indirect_reputations[node_id] = reputation if reputation < self.REPUTATION_THRESHOLD and round_num > 0: + # Rejections based on indirect reputation affect aggregation weights for this round. self.rejected_nodes.add(node_id) logging.info(f"SDFL reputation | Indirect reputation rejected node {node_id} at round {round_num}") @@ -2110,9 +2121,11 @@ async def send_reputation_table_to_neighbors(self, neighbors): """Send the local SDFL reputation table through the forwarding channel.""" round_num = await self._engine.get_round() reputation_table = await self.get_local_reputation_table(round_num) + # Register our own table locally so local aggregation paths see the same state as receivers. await self.register_reputation_table(self._addr, round_num, reputation_table, received_from=self._addr) if self._engine.rb.get_role_name(True) == "aggregator": + # Aggregators start waiting early because trainer tables may arrive before aggregation. expected_nodes = self._engine.get_sdfl_expected_trainers() timeout = float( self._config.participant["defense_args"] @@ -2130,6 +2143,7 @@ async def send_reputation_table_to_neighbors(self, neighbors): ) for neighbor in neighbors: + # Reputation tables are forwarded by the network layer in SDFL. await self._engine.cm.send_message(neighbor, message) logging.info( @@ -2477,6 +2491,7 @@ async def recollect_number_message(self, source, message): async def recollect_duplicated_number_message(self, dme: DuplicatedMessageEvent): """Record a duplicated message event.""" if self._engine.config.participant["scenario_args"].get("federation") == "SDFL": + # SDFL forwards model/table messages, so duplicates are not a reliable reputation signal. return event_data = await dme.get_event_data() @@ -2490,6 +2505,7 @@ async def _record_message_data(self, source: str): """Record message data for the given source if it's not the current address.""" if source != self._addr: if self._engine.config.participant["scenario_args"].get("federation") == "SDFL": + # In SDFL, message-count reputation is only meaningful for direct neighbors. direct_neighbors = await self._engine.cm.get_addrs_current_connections(only_direct=True, myself=False) if source not in direct_neighbors: return diff --git a/nebula/addons/trustworthiness/factsheet.py b/nebula/addons/trustworthiness/cfl_factsheet.py similarity index 89% rename from nebula/addons/trustworthiness/factsheet.py rename to nebula/addons/trustworthiness/cfl_factsheet.py index 0417efdc2..88eedab28 100755 --- a/nebula/addons/trustworthiness/factsheet.py +++ b/nebula/addons/trustworthiness/cfl_factsheet.py @@ -43,9 +43,7 @@ class CflFactsheet: def __init__(self): - """ - Manager class to populate the FactSheet - """ + # Manage the single CFL factsheet populated from server-side aggregation. self.factsheet_file_nm = "factsheet.json" self.factsheet_template_file_nm = "factsheet_template_cfl.json" @@ -64,6 +62,7 @@ def populate_factsheet_cfl( reliability_summary=None, ): + # Resolve the output factsheet and template for federation/data type. factsheet_file = get_factsheet_path(scenario_name, self.factsheet_file_nm) factsheet_template_file_nm = get_factsheet_template_name( data["federation"], @@ -82,10 +81,12 @@ def populate_factsheet_cfl( populate_common_pre_train_sections(factsheet, data, model) + # CFL reads aggregate CSV artifacts from the scenario trust directory. files_dir = get_trustworthiness_dir(scenario_name) emissions_file = os.path.join(files_dir, "emissions.csv") + # Aggregate class imbalance, entropy and model size across participants. avg_class_imbalance, avg_model_size = get_avg_class_imbalance_model_size(scenario_name) entropy_distribution = get_entropy_list (scenario_name) @@ -97,7 +98,7 @@ def populate_factsheet_cfl( factsheet["data"]["avg_entropy"] = avg_entropy - # Set performance data + # Set global performance and fairness metrics from aggregate results. result_avg_loss_accuracy = get_avg_loss_accuracy(scenario_name) factsheet["performance"]["test_loss_avg"] = result_avg_loss_accuracy[0] factsheet["performance"]["test_acc_avg"] = result_avg_loss_accuracy[1] @@ -105,6 +106,7 @@ def populate_factsheet_cfl( factsheet["fairness"]["test_acc_cv"] = 1 if test_acc_cv > 1 else test_acc_cv _, participant_test_acc = get_participant_loss_accuracy(scenario_name, participant_idx) + # Compute CFL privacy risk from aggregate DP settings and client count. dp_enabled, dp_epsilon = get_dp_global(scenario_name) set_dp_configuration(factsheet, dp_enabled, dp_epsilon) factsheet["privacy"]["privacy_risk"] = get_global_privacy_risk( @@ -113,6 +115,7 @@ def populate_factsheet_cfl( factsheet["participants"]["client_num"], ) + # Populate system timing, model-size and communication totals. factsheet["system"]["avg_time_minutes"] = get_elapsed_time(start_time, end_time) factsheet["system"]["avg_model_size"] = avg_model_size @@ -124,6 +127,7 @@ def populate_factsheet_cfl( populate_reliability(factsheet, reliability_summary) populate_participation(factsheet, participation_summary) + # Convert class imbalance and runtime summaries into factsheet fields. class_imbalance_score = get_class_imbalance_score(avg_class_imbalance) factsheet["fairness"]["class_imbalance"] = cap_score(class_imbalance_score) populate_reputation(factsheet, reputation_summary) @@ -131,6 +135,7 @@ def populate_factsheet_cfl( underfitting_score = get_underfitting_score(scenario_name, participant_idx) factsheet["fairness"]["underfitting"] = underfitting_score + # Add model/profile-specific metrics after base factsheet fields exist. populate_profile_metrics( factsheet, data["federation"], @@ -140,7 +145,7 @@ def populate_factsheet_cfl( participant_test_acc, ) - # Set emissions metrics + # Enrich CodeCarbon emissions with CPU/GPU benchmark metadata. emissions = None if emissions_file is None else read_csv(emissions_file) if emissions is not None: logging.info("FactSheet: Populating emissions") @@ -156,6 +161,7 @@ def populate_factsheet_cfl( emissions.drop("gpuName", axis=1, inplace=True) emissions["powerPerf"] = emissions["powerPerf"].astype(float) emissions["powerPerformance"] = emissions["powerPerformance"].astype(float) + # Trainer rows represent client-side training cost. client_emissions = emissions.loc[emissions["role"] == "trainer"] client_avg_carbon_intensity = round(client_emissions["energy_grid"].mean(), 2) factsheet["sustainability"]["avg_carbon_intensity_clients"] = check_field_filled(factsheet, ["sustainability", "avg_carbon_intensity_clients"], client_avg_carbon_intensity, "") @@ -166,6 +172,7 @@ def populate_factsheet_cfl( clients_power_performance = round(pd.concat([GPU_powerperf, CPU_powerperf]).mean(), 2) factsheet["sustainability"]["avg_power_performance_clients"] = check_field_filled(factsheet, ["sustainability", "avg_power_performance_clients"], clients_power_performance, "") + # Server rows represent aggregation cost. server_emissions = emissions.loc[emissions["role"] == "server"] server_avg_carbon_intensity = round(server_emissions["energy_grid"].mean(), 2) factsheet["sustainability"]["avg_carbon_intensity_server"] = check_field_filled(factsheet, ["sustainability", "avg_carbon_intensity_server"], server_avg_carbon_intensity, "") @@ -175,11 +182,13 @@ def populate_factsheet_cfl( server_power_performance = round(pd.concat([GPU_powerperf, CPU_powerperf]).mean(), 2) factsheet["sustainability"]["avg_power_performance_server"] = check_field_filled(factsheet, ["sustainability", "avg_power_performance_server"], server_power_performance, "") + # Estimate communication emissions from byte counts and carbon intensity. factsheet["sustainability"]["emissions_communication_uplink"] = check_field_filled(factsheet, ["sustainability", "emissions_communication_uplink"], factsheet["system"]["total_upload_bytes"] * 2.24e-10 * factsheet["sustainability"]["avg_carbon_intensity_clients"], "") factsheet["sustainability"]["emissions_communication_downlink"] = check_field_filled(factsheet, ["sustainability", "emissions_communication_downlink"], factsheet["system"]["total_download_bytes"] * 2.24e-10 * factsheet["sustainability"]["avg_carbon_intensity_server"], "") write_factsheet(factsheet_file, factsheet) except JSONDecodeError as e: + # Keep corrupted factsheet failures explicit in logs. logging.info(f"{factsheet_file} is invalid") logging.error(e) diff --git a/nebula/addons/trustworthiness/dfl_factsheet.py b/nebula/addons/trustworthiness/dfl_factsheet.py index 5be3ee012..2fb2fd115 100644 --- a/nebula/addons/trustworthiness/dfl_factsheet.py +++ b/nebula/addons/trustworthiness/dfl_factsheet.py @@ -38,9 +38,7 @@ class DflFactsheet: def __init__(self): - """ - Manager class to populate the FactSheet - """ + # Manage participant-specific DFL/SDFL factsheets. self.factsheet_template_file_nm = "factsheet_template_dfl.json" def populate_factsheet_dfl( @@ -58,6 +56,7 @@ def populate_factsheet_dfl( reliability_summary=None, ): + # Resolve participant-specific output and data-type-aware template. self.factsheet_file_nm = f"factsheet_participant_{participant_idx}.json" factsheet_template_file_nm = get_factsheet_template_name( data["federation"], @@ -77,15 +76,18 @@ def populate_factsheet_dfl( populate_common_pre_train_sections(factsheet, data, model) + # DP configuration is stored per participant in decentralized runs. dp_enabled, dp_epsilon = get_dp_local(scenario_name, participant_idx) set_dp_configuration(factsheet, dp_enabled, dp_epsilon) files_dir = get_trustworthiness_dir(scenario_name) + # Refresh entropy.json so participant-local entropy can be read consistently. get_all_data_entropy(scenario_name) factsheet["data"]["entropy_local"] = get_local_normalized_entropy(scenario_name, participant_idx) + # Use the final valid round metrics as participant test performance. df = load_round_metrics(scenario_name, participant_idx) acc = df["accuracy"].astype(float).to_numpy() loss = df["loss"].astype(float).to_numpy() @@ -96,6 +98,7 @@ def populate_factsheet_dfl( factsheet["performance"]["test_loss"] = float(final_loss) factsheet["performance"]["test_acc"] = float(final_acc) + # Load local communication and privacy values reported by the participant. bytes_sent, bytes_recv, *_ = load_data_results_participant(scenario_name, participant_idx) factsheet["system"]["model_size"] = get_bytes_model(model) @@ -107,6 +110,7 @@ def populate_factsheet_dfl( factsheet["system"]["time_minutes"] = get_elapsed_time(start_time, end_time) + # Class imbalance can only be populated after local class-counts exist. count_class_file = os.path.join(files_dir, f"{participant_idx}_class_count.json") factsheet["fairness"]["class_imbalance"] = ( get_local_class_imbalance_score(scenario_name, participant_idx) @@ -116,6 +120,7 @@ def populate_factsheet_dfl( populate_participation(factsheet, participation_summary) + # Local CodeCarbon output feeds participant sustainability fields. ( role, carbon_intensity_local, @@ -138,17 +143,20 @@ def populate_factsheet_dfl( factsheet["participants"]["local_dataset_size"] = sample_size populate_reputation(factsheet, reputation_summary, include_neighbor_num=True) + # DFL privacy risk depends on local DP settings and neighbor count. factsheet["privacy"]["privacy_risk"] = get_global_privacy_risk_dfl( dp_enabled, dp_epsilon, factsheet["participants"]["neighbor_num"], ) + # Communication emissions are estimated from local bytes and carbon intensity. factsheet["sustainability"]["emissions_communication_local"] = ( (bytes_sent * 2.24e-10 * carbon_intensity_local) + (bytes_recv * 2.24e-10 * carbon_intensity_local) ) + # Populate model/profile metrics after final participant accuracy is known. factsheet["fairness"]["underfitting"] = get_underfitting_score_local(scenario_name, participant_idx) populate_profile_metrics( factsheet, @@ -163,6 +171,7 @@ def populate_factsheet_dfl( def load_round_metrics(scenario_name, participant_idx): + # Load participant per-round metrics and keep only rows with loss/accuracy. files_dir = get_trustworthiness_dir(scenario_name) path = os.path.join(files_dir, f"round_metrics_participant_{participant_idx}.csv") df = pd.read_csv(path) diff --git a/nebula/addons/trustworthiness/factsheet_common.py b/nebula/addons/trustworthiness/factsheet_common.py index 3aa972fa9..7cfbe11d9 100644 --- a/nebula/addons/trustworthiness/factsheet_common.py +++ b/nebula/addons/trustworthiness/factsheet_common.py @@ -1,5 +1,3 @@ -"""Shared helpers for trustworthiness factsheet generation.""" - import json import os import shutil @@ -7,12 +5,13 @@ dirname = os.path.dirname(__file__) +# Shared helpers for trustworthiness factsheet generation. DATA_TYPE_IMAGES = "images" DATA_TYPE_TABULAR = "tabular" def get_model_data_type(model): - """Returns the data type declared by the model, when available.""" + # Return the data type declared by the model, when available. if not hasattr(model, "get_data_type"): return "" @@ -27,10 +26,12 @@ def get_model_data_type(model): def get_normalized_model_data_type(model): + # Normalize the model data type before matching templates or profiles. return get_model_data_type(model).lower() def get_factsheet_template_name(federation, model, default_template_name): + # Select a data-type-specific template when one exists for the federation. federation_prefix = "dfl" if str(federation).upper() in {"DFL", "SDFL"} else "cfl" data_type = get_normalized_model_data_type(model) @@ -44,22 +45,22 @@ def get_factsheet_template_name(federation, model, default_template_name): def get_trustworthiness_dir(scenario_name): - """Returns the trustworthiness output directory for a scenario.""" + # Return the trustworthiness output directory for a scenario. return os.path.join(os.environ.get("NEBULA_LOGS_DIR"), scenario_name, "trustworthiness") def get_factsheet_path(scenario_name, factsheet_name): - """Returns the path to a factsheet inside the scenario trustworthiness directory.""" + # Return the path to a factsheet inside the scenario trustworthiness directory. return os.path.join(get_trustworthiness_dir(scenario_name), factsheet_name) def get_factsheet_template_path(template_name): - """Returns the path to a factsheet template bundled with the addon.""" + # Return the path to a factsheet template bundled with the addon. return os.path.join(dirname, "configs", template_name) def load_or_create_factsheet(scenario_name, factsheet_name, template_name): - """Loads a factsheet, creating it from its template if it does not exist.""" + # Load a factsheet, creating it from the selected template if needed. trustworthiness_dir = get_trustworthiness_dir(scenario_name) os.makedirs(trustworthiness_dir, exist_ok=True) @@ -74,23 +75,23 @@ def load_or_create_factsheet(scenario_name, factsheet_name, template_name): def write_factsheet(factsheet_path, factsheet): - """Writes a factsheet using the standard JSON formatting.""" + # Write a factsheet using readable standard JSON formatting. with open(factsheet_path, "w", encoding="utf-8") as factsheet_file: json.dump(factsheet, factsheet_file, indent=4) def cap_score(value, maximum=1): - """Caps a score to the maximum value expected by the factsheet.""" + # Cap a score to the maximum value expected by the factsheet. return maximum if value > maximum else value def inverse_score(value): - """Converts an error or risk value into a bounded inverse score.""" + # Convert an error or risk value into a bounded inverse score. return 1 / (1 + value) def build_project_background(data): - """Builds the natural-language scenario description used in factsheets.""" + # Build the natural-language scenario description used in factsheets. federation = data["federation"] n_nodes = int(data["n_nodes"]) dataset = data["dataset"] @@ -121,7 +122,7 @@ def build_project_background(data): def populate_common_pre_train_sections(factsheet, data, model): - """Populates project, data, participant and training configuration fields.""" + # Populate project, data, participant and training configuration fields. with_reputation = data["reputation"]["enabled"] factsheet["project"]["overview"] = data["scenario_title"] @@ -153,13 +154,13 @@ def populate_common_pre_train_sections(factsheet, data, model): def set_dp_configuration(factsheet, dp_enabled, dp_epsilon): - """Writes differential privacy configuration using the factsheet schema.""" + # Write differential privacy configuration using the factsheet schema. factsheet["configuration"]["differential_privacy"] = bool(dp_enabled) factsheet["configuration"]["dp_epsilon"] = dp_epsilon if dp_enabled else "" def populate_reliability(factsheet, reliability_summary): - """Writes dropout and timeout rates, defaulting to a fully reliable run.""" + # Write dropout and timeout rates, defaulting to a fully reliable run. factsheet["system"]["dropout_rate"] = ( reliability_summary.get("dropout_rate", 0.0) if reliability_summary is not None @@ -173,7 +174,7 @@ def populate_reliability(factsheet, reliability_summary): def populate_participation(factsheet, participation_summary): - """Writes participant selection dispersion, defaulting to full participation.""" + # Write participant selection dispersion, defaulting to full participation. factsheet["fairness"]["selection_cv"] = ( participation_summary.get("selection_cv", 1) if participation_summary is not None @@ -182,7 +183,7 @@ def populate_participation(factsheet, participation_summary): def populate_reputation(factsheet, reputation_summary, include_neighbor_num=False): - """Writes reputation information for centralized or decentralized factsheets.""" + # Write reputation information for centralized or decentralized factsheets. if reputation_summary is not None: factsheet["participants"]["avg_neighbor_reputation"] = reputation_summary.get( "avg_neighbor_reputation", diff --git a/nebula/addons/trustworthiness/helpers/factsheet_values.py b/nebula/addons/trustworthiness/helpers/factsheet_values.py index ee42940af..8faa2cf81 100644 --- a/nebula/addons/trustworthiness/helpers/factsheet_values.py +++ b/nebula/addons/trustworthiness/helpers/factsheet_values.py @@ -13,6 +13,7 @@ logger = logging.getLogger(__name__) +# Operations available from the eval_metrics JSON files. OPERATIONS = { "check_properties": check_properties, "comm_efficiency": comm_efficiency, @@ -21,88 +22,62 @@ "get_value": get_value, } + def check_field_filled(factsheet_dict, factsheet_path, value, empty=""): - """ - Check if the field in the factsheet file is filled or not. - - Args: - factsheet_dict (dict): The factshett dict. - factsheet_path (list): The factsheet field to check. - value (float): The value to add in the field. - empty (string): If the value could not be appended, the empty string is returned. - - Returns: - float: The value added in the factsheet or empty if the value could not be appened - - """ - if factsheet_dict[factsheet_path[0]][factsheet_path[1]]: - return factsheet_dict[factsheet_path[0]][factsheet_path[1]] - elif value != "" and value != "nan": - if type(value) != str and type(value) != list: - if math.isnan(value): - return 0 - else: - return value - else: - return value - else: + # Keep an existing factsheet value; otherwise return a clean fallback for empty or NaN values. + current_value = factsheet_dict[factsheet_path[0]][factsheet_path[1]] + if current_value: + return current_value + + if _is_empty_value(value): return empty + if _is_nan_number(value): + return 0 + + return value -def get_input_value(input_docs, inputs, operation): - """ - Gets the input value from input document and apply the metric operation on the value. - Args: - inputs_docs (map): The input document map. - inputs (list): All the inputs. - operation (string): The metric operation. +def _is_empty_value(value): + # Empty strings and the literal "nan" should not overwrite missing factsheet fields. + return value == "" or value == "nan" - Returns: - float: The metric value - """ +def _is_nan_number(value): + # Only numeric values can be checked with math.isnan safely. + return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isnan(value) - input_value = None + +def get_input_value(input_docs, inputs, operation): + # Collect metric inputs from their configured paths and apply the configured operation. args = [] - for i in inputs: - source = i.get("source", "") - field = i.get("field_path", "") - input_doc = input_docs.get(source, None) + for input_config in inputs: + source = input_config.get("source", "") + field = input_config.get("field_path", "") + input_doc = input_docs.get(source) if input_doc is None: logger.warning(f"{source} is null") - else: - input = get_value_from_path(input_doc, field) - args.append(input) + continue + + args.append(get_value_from_path(input_doc, field)) + try: - operationFn = OPERATIONS[operation] - input_value = operationFn(*args) - except KeyError: - logger.warning(f"{operation} is not valid") - except TypeError: + operation_fn = OPERATIONS[operation] + return operation_fn(*args) + except (KeyError, TypeError): logger.warning(f"{operation} is not valid") - - return input_value + return None def get_value_from_path(input_doc, path): - """ - Gets the input value from input document by path. - - Args: - inputs_doc (map): The input document map. - path (string): The field name of the input value of interest. - - Returns: - float: The input value from the input document + # Walk a slash-separated path through a nested dict and return the leaf value. + current_value = input_doc + for nested_key in path.split("/"): + if not isinstance(current_value, dict): + return None - """ + current_value = current_value.get(nested_key) + if current_value is None: + return None - d = input_doc - for nested_key in path.split("/"): - temp = d.get(nested_key) - if isinstance(temp, dict): - d = d.get(nested_key) - else: - return temp - return None + return current_value diff --git a/nebula/addons/trustworthiness/helpers/scenario_metrics.py b/nebula/addons/trustworthiness/helpers/scenario_metrics.py index d714e8523..1d1f35615 100644 --- a/nebula/addons/trustworthiness/helpers/scenario_metrics.py +++ b/nebula/addons/trustworthiness/helpers/scenario_metrics.py @@ -3,7 +3,6 @@ import os import statistics from datetime import datetime -from os.path import exists import pandas as pd import torch @@ -13,53 +12,46 @@ logger = logging.getLogger(__name__) -def get_elapsed_time(start_time, end_time): - """ - Calculates the elapsed time during the execution of the scenario. - - Args: - start_time (datetime): Start datetime. - end_time (datetime): End datetime. - - Returns: - float: The elapsed time. - """ - start_date = datetime.strptime(start_time, "%d/%m/%Y %H:%M:%S") - end_date = datetime.strptime(end_time, "%d/%m/%Y %H:%M:%S") +DATETIME_FORMAT = "%d/%m/%Y %H:%M:%S" - elapsed_time = (end_date - start_date).total_seconds() / 60 - return elapsed_time +def get_elapsed_time(start_time, end_time): + # Return scenario duration in minutes from the timestamps stored by the workload. + start_date = datetime.strptime(start_time, DATETIME_FORMAT) + end_date = datetime.strptime(end_time, DATETIME_FORMAT) + return (end_date - start_date).total_seconds() / 60 def _trustworthiness_dir(scenario_name): - return os.path.join(os.environ.get('NEBULA_LOGS_DIR'), scenario_name, "trustworthiness") + # All scenario metrics are stored under the scenario trustworthiness directory. + return os.path.join(os.environ.get("NEBULA_LOGS_DIR"), scenario_name, "trustworthiness") def _global_data_results_path(scenario_name): + # CFL/global metrics are written in the shared data_results.csv file. return os.path.join(_trustworthiness_dir(scenario_name), "data_results.csv") def _participant_data_results_path(scenario_name, participant_id): + # DFL/SDFL participant metrics are written in participant-specific CSV files. return os.path.join(_trustworthiness_dir(scenario_name), f"data_results_{participant_id}.csv") def _read_global_results(scenario_name): + # Load the aggregate scenario metrics once and let callers pick the columns they need. return read_csv(_global_data_results_path(scenario_name)) def _read_participant_results(scenario_name, participant_id): + # Load local metrics for one participant. return read_csv(_participant_data_results_path(scenario_name, participant_id)) def _find_participant_row(data, participant_id, source_name): + # Match both string and integer IDs because CSV typing can vary between runs. row = data[data["id"] == participant_id] - if row.empty: - try: - row = data[data["id"] == int(participant_id)] - except (TypeError, ValueError): - row = data.iloc[0:0] + row = _find_participant_row_by_int_id(data, participant_id) if row.empty: raise ValueError(f"Participant {participant_id} not found in {source_name}") @@ -67,36 +59,34 @@ def _find_participant_row(data, participant_id, source_name): return row.iloc[0] -def get_bytes_model(model): - """ - Calculates the serialized size in bytes of a PyTorch model state_dict. +def _find_participant_row_by_int_id(data, participant_id): + # Retry numeric participant IDs when pandas read the id column as integers. + try: + return data[data["id"] == int(participant_id)] + except (TypeError, ValueError): + return data.iloc[0:0] - Args: - model (nn.Module): PyTorch model. - Returns: - int: Model size in bytes. - """ - buffer: io.BytesIO = io.BytesIO() - torch.save(model.state_dict(), buffer) - model_size: int = buffer.tell() +def _client_count(data): + # Global CSVs include the server row, so client averages exclude one row. + return max(1, len(data) - 1) - return model_size +def _mean_client_column(data, column_name): + # Average a global metric across clients while keeping the historical server-row exclusion. + return data[column_name].sum() / _client_count(data) -def get_bytes_sent_recv(scenario_name): - """ - Calculates the mean bytes sent and received of the nodes. - Args: - bytes_sent_files (list): Files that contain the bytes sent of the nodes. - bytes_recv_files (list): Files that contain the bytes received of the nodes. +def get_bytes_model(model): + # Serialize the model state_dict to measure the bytes that would be transmitted. + buffer = io.BytesIO() + torch.save(model.state_dict(), buffer) + return buffer.tell() - Returns: - 4-tupla: The total bytes sent, the total bytes received, the mean bytes sent and the mean bytes received of the nodes. - """ - data = _read_global_results(scenario_name) +def get_bytes_sent_recv(scenario_name): + # Return total and average upload/download bytes from aggregate scenario results. + data = _read_global_results(scenario_name) number_files = len(data) total_upload_bytes = int(data["bytes_sent"].sum()) @@ -109,154 +99,67 @@ def get_bytes_sent_recv(scenario_name): def get_avg_loss_accuracy(scenario_name): - """ - Calculates the mean accuracy and loss models of the nodes. - - Args: - loss_files (list): Files that contain the loss of the models of the nodes. - accuracy_files (list): Files that contain the acurracies of the models of the nodes. - - Returns: - 3-tupla: The mean loss of the models, the mean accuracies of the models, the standard deviation of the accuracies of the models. - """ + # Return client-average test loss, test accuracy and accuracy standard deviation. data = _read_global_results(scenario_name) - number_files = len(data) - - total_loss = data["loss"].sum() - total_accuracy = data["accuracy"].sum() - - denominator = max(1, number_files - 1) - avg_loss = total_loss / denominator - avg_accuracy = total_accuracy / denominator - std_accuracy = statistics.stdev(data["accuracy"]) if number_files > 1 else 0.0 + avg_loss = _mean_client_column(data, "loss") + avg_accuracy = _mean_client_column(data, "accuracy") + std_accuracy = statistics.stdev(data["accuracy"]) if len(data) > 1 else 0.0 return avg_loss, avg_accuracy, std_accuracy -def get_underfitting_score(scenario_name, id): - """ - Calculates the mean val accuracy of the nodes. - """ +def get_underfitting_score(scenario_name, participant_id): + # CFL underfitting uses the average validation accuracy across client rows. data = _read_global_results(scenario_name) - - number_files = len(data) - - total_val_accuracy = data["val_accuracy"].sum() - - avg_val_accuracy = total_val_accuracy / max(1, number_files - 1) - - return avg_val_accuracy + return _mean_client_column(data, "val_accuracy") def get_participant_loss_accuracy(scenario_name, participant_id): - """ - Gets loss and accuracy for a specific participant from CFL aggregated results. - - Args: - scenario_name (str): Scenario name. - participant_id (int | str): Participant identifier. - - Returns: - tuple[float, float]: (loss, accuracy) - """ + # Read one participant's final CFL loss and accuracy from the aggregate CSV. data_file = _global_data_results_path(scenario_name) row = _find_participant_row(read_csv(data_file), participant_id, data_file) + return float(row["loss"]), float(row["accuracy"]) - loss = float(row["loss"]) - accuracy = float(row["accuracy"]) - return loss, accuracy - -def get_underfitting_score_local(scenario_name, id): - """ - Gets the local validation accuracy for a specific DFL/SDFL participant. - Args: - scenario_name (str): Scenario name. - participant_id (int | str): Participant identifier. - - Returns: - float: Validation accuracy. - """ - data = _read_participant_results(scenario_name, id) +def get_underfitting_score_local(scenario_name, participant_id): + # DFL/SDFL underfitting uses the participant-local validation accuracy. + data = _read_participant_results(scenario_name, participant_id) return float(data["val_accuracy"].iloc[0]) -def get_dp_local(scenario_name, id): - """ - Gets the dp metrics for a specific DFL/SDFL participant. - - Args: - scenario_name (str): Scenario name. - participant_id (int | str): Participant identifier. - - Returns: - float: DP Enabled, Epsilon. - """ - data = _read_participant_results(scenario_name, id) +def get_dp_local(scenario_name, participant_id): + # Return DP settings stored by a single DFL/SDFL participant. + data = _read_participant_results(scenario_name, participant_id) return data["dp_enabled"].iloc[0], float(data["dp_epsilon"].iloc[0]) def get_dp_global(scenario_name): - """ - Gets the aggregated DP metrics for a CFL scenario, excluding the server node. - - Args: - scenario_name (str): Scenario name. - - Returns: - tuple[bool, float | str]: Whether DP is enabled, and the - average epsilon across client nodes. - """ + # Return CFL DP settings, averaging epsilon across client rows when DP is enabled. data = _read_global_results(scenario_name) if data["dp_enabled"].iloc[0] == False: return False, 0.0 - number_files = len(data) - - avg_epsilon = data["dp_epsilon"].sum() / max(1, number_files - 1) + return True, _mean_client_column(data, "dp_epsilon") - return True, avg_epsilon def get_avg_class_imbalance_model_size(scenario_name): - """ - Calculates the mean class imbalance and model size of the nodes. - - Args: - data_results_files (list): Files that contain the class imbalance and model size of the nodes - - Returns: - 2-tupla: The mean class imbalance mean and model size mean of the nodes. - """ + # Return average class imbalance and model size across all global result rows. data = _read_global_results(scenario_name) - number_files = len(data) - total_class_imbalance = data["class_imbalance"].sum() - total_model_size = data["model_size"].sum() - - avg_class_imbalance = total_class_imbalance / number_files - avg_model_size = total_model_size / number_files + avg_class_imbalance = data["class_imbalance"].sum() / number_files + avg_model_size = data["model_size"].sum() / number_files return avg_class_imbalance, avg_model_size def get_entropy_list(scenario_name): - """ - Obtiene una lista con los valores de entropy de todos los nodos. - - Args: - scenario_name (str): Nombre del escenario. - - Returns: - list: Lista con los valores de entropy - """ + # Return local entropy values so callers can normalize the distribution. data = _read_global_results(scenario_name) + return data["local_entropy"].tolist() - entropy_list = data["local_entropy"].tolist() - - return entropy_list def stop_emissions_tracking_and_save( tracker: EmissionsTracker, @@ -267,84 +170,60 @@ def stop_emissions_tracking_and_save( sample_size: int = 0, participant_idx=None, ): - """ - Stops emissions tracking object from CodeCarbon and saves relevant information to emissions.csv file. - - Args: - tracker (object): The emissions tracker object holding information. - outdir (str): The path of the output directory of the experiment. - emissions_file (str): The path to the emissions file. - role (str): Either client or server depending on the role. - workload (str): Either aggregation or training depending on the workload. - sample_size (int): The number of samples used for training, if aggregation 0. - """ - + # Stop CodeCarbon tracking and append the final emissions row to emissions.csv. tracker.stop() - emissions_file = os.path.join(outdir, emissions_file) - - if exists(emissions_file): - df = pd.read_csv(emissions_file) - else: - df = pd.DataFrame( - columns=[ - "id", - "role", - "energy_grid", - "emissions", - "workload", - "CPU_model", - "GPU_model", - ] - ) + emissions_path = os.path.join(outdir, emissions_file) + df = _read_or_create_emissions_dataframe(emissions_path) + try: - energy_grid = (tracker.final_emissions_data.emissions / tracker.final_emissions_data.energy_consumed) * 1000 - df = pd.concat( - [ - df, - pd.DataFrame({ - "id": participant_idx, - "role": role, - "energy_grid": [energy_grid], - "emissions": [tracker.final_emissions_data.emissions], - "workload": workload, - "CPU_model": tracker.final_emissions_data.cpu_model - if tracker.final_emissions_data.cpu_model - else "None", - "GPU_model": tracker.final_emissions_data.gpu_model - if tracker.final_emissions_data.gpu_model - else "None", - "CPU_used": True if tracker.final_emissions_data.cpu_energy else False, - "GPU_used": True if tracker.final_emissions_data.gpu_energy else False, - "energy_consumed": tracker.final_emissions_data.energy_consumed, - "sample_size": sample_size, - }), - ], - ignore_index=True, - ) - df.to_csv(emissions_file, encoding="utf-8", index=False) + row = _build_emissions_row(tracker, role, workload, sample_size, participant_idx) + df = pd.concat([df, pd.DataFrame(row)], ignore_index=True) + df.to_csv(emissions_path, encoding="utf-8", index=False) except Exception as e: logger.warning(e) -def comm_efficiency(bytes_up: int, bytes_down: int, test_acc_avg: float, eps: float = 1e-12) -> float: - """ - Communication efficiency = total_bytes / final_accuracy. - Lower is better. - - Args: - bytes_up: total uploaded bytes - bytes_down: total downloaded bytes - final_accuracy: final test accuracy in [0,1] - eps: small constant to avoid division by zero - - Returns: - float - """ - total_bytes = float(bytes_up) + float(bytes_down) - acc = float(test_acc_avg) +def _read_or_create_emissions_dataframe(emissions_path): + # Reuse the existing file when present, otherwise create the expected columns. + if os.path.exists(emissions_path): + return pd.read_csv(emissions_path) + + return pd.DataFrame( + columns=[ + "id", + "role", + "energy_grid", + "emissions", + "workload", + "CPU_model", + "GPU_model", + ] + ) + + +def _build_emissions_row(tracker, role, workload, sample_size, participant_idx): + # Convert CodeCarbon's final data object into the CSV row persisted by trustworthiness. + emissions_data = tracker.final_emissions_data + energy_grid = (emissions_data.emissions / emissions_data.energy_consumed) * 1000 + + return { + "id": participant_idx, + "role": role, + "energy_grid": [energy_grid], + "emissions": [emissions_data.emissions], + "workload": workload, + "CPU_model": emissions_data.cpu_model if emissions_data.cpu_model else "None", + "GPU_model": emissions_data.gpu_model if emissions_data.gpu_model else "None", + "CPU_used": bool(emissions_data.cpu_energy), + "GPU_used": bool(emissions_data.gpu_energy), + "energy_consumed": emissions_data.energy_consumed, + "sample_size": sample_size, + } - if acc < eps: - acc = eps - return total_bytes / acc +def comm_efficiency(bytes_up: int, bytes_down: int, test_acc_avg: float, eps: float = 1e-12) -> float: + # Communication efficiency is total transferred bytes divided by final accuracy. + total_bytes = float(bytes_up) + float(bytes_down) + accuracy = max(float(test_acc_avg), eps) + return total_bytes / accuracy diff --git a/nebula/addons/trustworthiness/helpers/scoring.py b/nebula/addons/trustworthiness/helpers/scoring.py index 5103626c8..955bf5421 100644 --- a/nebula/addons/trustworthiness/helpers/scoring.py +++ b/nebula/addons/trustworthiness/helpers/scoring.py @@ -4,40 +4,31 @@ logger = logging.getLogger(__name__) -def get_mapped_score(score_key, score_map): - """ - Finds the score by the score_key in the score_map. - Args: - score_key (string): The key to look up in the score_map. - score_map (dict): The score map defined in the eval_metrics.json file. +def _is_number(value): + # Score calculations expect real numeric values; booleans are handled explicitly. + return isinstance(value, (int, float, np.number)) and not isinstance(value, bool) + + +def _warn_not_number(value): + # Keep the warning format consistent across all numeric scoring functions. + logger.warning("Input value is not a number") + logger.warning(f"{value}") - Returns: - float: The normalized score of [0, 1]. - """ - score = 0 + +def get_mapped_score(score_key, score_map): + # Normalize the configured score map and return the normalized value for the input key. if score_map is None: logger.warning("Score map is missing") - else: - keys = [key for key, value in score_map.items()] - scores = [value for key, value in score_map.items()] - normalized_scores = get_normalized_scores(scores) - normalized_score_map = dict(zip(keys, normalized_scores, strict=False)) - score = normalized_score_map.get(score_key, np.nan) + return 0 - return score + normalized_scores = get_normalized_scores(list(score_map.values())) + normalized_score_map = dict(zip(score_map.keys(), normalized_scores, strict=False)) + return normalized_score_map.get(score_key, np.nan) def get_normalized_scores(scores): - """ - Calculates the normalized scores of a list. - - Args: - scores (list): The values that will be normalized. - - Returns: - list: The normalized list. - """ + # Convert a list of raw configured scores to the [0, 1] range. if scores is None or len(scores) == 0: return [] @@ -46,145 +37,89 @@ def get_normalized_scores(scores): if max_score == min_score: return [1.0 for _ in scores] - normalized = [(x - min_score) / (max_score - min_score) for x in scores] - return normalized + return [(score - min_score) / (max_score - min_score) for score in scores] def get_range_score(value, ranges, direction="asc"): - """ - Maps the value to a range and gets the score by the range and direction. - - Args: - value (int): The input score. - ranges (list): The ranges defined. - direction (string): Asc means the higher the range the higher the score, desc means otherwise. - - Returns: - float: The normalized score of [0, 1]. - """ - - if not (type(value) == int or type(value) == float): - logger.warning("Input value is not a number") - logger.warning(f"{value}") + # Place the value in one of the configured bins and normalize that bin index. + if not _is_number(value): + _warn_not_number(value) return 0 - else: - score = 0 - if ranges is None: - logger.warning("Score ranges are missing") - else: - total_bins = len(ranges) + 1 - bin = np.digitize(value, ranges, right=True) - score = 1 - (bin / total_bins) if direction == "desc" else bin / total_bins - return score + if ranges is None: + logger.warning("Score ranges are missing") + return 0 -def get_map_value_score(score_key, score_map): - """ - Finds the score by the score_key in the score_map and returns the value. + total_bins = len(ranges) + 1 + bin_index = np.digitize(value, ranges, right=True) + score = bin_index / total_bins + return 1 - score if direction == "desc" else score - Args: - score_key (string): The key to look up in the score_map. - score_map (dict): The score map defined in the eval_metrics.json file. - Returns: - float: The score obtained in the score_map. - """ - score = 0 +def get_map_value_score(score_key, score_map): + # Return the exact configured score for maps that already store normalized values. if score_map is None: logger.warning("Score map is missing") - else: - score = score_map[score_key] - return score - - -def get_true_score(value, direction): - """ - Returns the negative of the value if direction is 'desc', otherwise returns value. + return 0 - Args: - value (int): The input score. - direction (string): Asc means the higher the range the higher the score, desc means otherwise. + return score_map[score_key] - Returns: - float: The score obtained. - """ +def get_true_score(value, direction): + # Booleans are direct scores; numeric values can be inverted for descending metrics. if value is True: return 1 - elif value is False: + if value is False: return 0 - else: - if not (type(value) == int or type(value) == float): - logger.warning("Input value is not a number") - logger.warning(f"{value}.") - return 0 - else: - if direction == "desc": - return 1 - value - else: - return value - -def get_scaled_score(value, scale: list, direction: str): - """ - Maps a score of a specific scale into the scale between zero and one. + if not _is_number(value): + _warn_not_number(value) + return 0 - Args: - value (int or float): The raw value of the metric. - scale (list): List containing the minimum and maximum value the value can fall in between. + return 1 - value if direction == "desc" else value - Returns: - float: The normalized score of [0, 1]. - """ - score = 0 - try: - value_min, value_max = scale[0], scale[1] - except Exception: - logger.warning("Score minimum or score maximum is missing. The minimum has been set to 0 and the maximum to 1") - value_min, value_max = 0, 1 +def get_scaled_score(value, scale: list, direction: str): + # Clamp a metric from its configured scale into the [0, 1] score range. if value is None or value == "": logger.warning("Score value is missing. Set value to zero") - else: - low, high = 0, 1 - if value >= value_max: - score = 1 - elif value <= value_min: - score = 0 - else: - diff = value_max - value_min - diffScale = high - low - score = (float(value) - value_min) * (float(diffScale) / diff) + low - if direction == "desc": - score = high - score + return 0 - return score + if not _is_number(value): + _warn_not_number(value) + return 0 + value_min, value_max = _get_scale_bounds(scale) + if value_max == value_min: + score = 1 + elif value >= value_max: + score = 1 + elif value <= value_min: + score = 0 + else: + score = (float(value) - value_min) / (value_max - value_min) -def get_value(value): - """ - Get the value of a metric. + return 1 - score if direction == "desc" else score - Args: - value (float): The value of the metric. - Returns: - float: The value of the metric. - """ +def _get_scale_bounds(scale): + # Fall back to the default [0, 1] scale when the config is incomplete. + try: + return scale[0], scale[1] + except (TypeError, IndexError): + logger.warning("Score minimum or score maximum is missing. The minimum has been set to 0 and the maximum to 1") + return 0, 1 + +def get_value(value): + # Factsheet operations use this when a metric only needs the raw input value. return value def check_properties(*args): - """ - Check if all the arguments have values. - - Args: - args (list): All the arguments. - - Returns: - float: The mean of arguments that have values. - """ + # Return the fraction of required properties that are filled. + if not args: + return 0 - result = map(lambda x: x is not None and x != "", args) - return np.mean(list(result)) + filled = [value is not None and value != "" for value in args] + return np.mean(filled) diff --git a/nebula/addons/trustworthiness/helpers/trust_reports.py b/nebula/addons/trustworthiness/helpers/trust_reports.py index 08d1798ec..11e09208d 100644 --- a/nebula/addons/trustworthiness/helpers/trust_reports.py +++ b/nebula/addons/trustworthiness/helpers/trust_reports.py @@ -2,79 +2,86 @@ import json import os -def load_trust_report_json_dumped(scenario_name: str, participant_id: int) -> str: - """ - Read a participant trustworthiness JSON file and return it - serialized as a string with json.dumps(...). - - Args: - scenario_name (str): Scenario/experiment name. - participant_id (int): Participant ID. +SCORE_KEYS = {"trust_score", "score"} +NAMED_ENTRY_KEYS = {"score", "metrics", "notions", "pillars"} +NAMED_ENTRY_PATH_KEY = "__named_entry__" - Returns: - str: JSON content serialized as a string. - Raises: - FileNotFoundError: If the file does not exist. - ValueError: If the file content is not valid JSON. - """ +def _logs_dir() -> str: + # Return the configured logs directory required by trust report exchange. logs_dir = os.environ.get("NEBULA_LOGS_DIR") if not logs_dir: raise ValueError("The NEBULA_LOGS_DIR environment variable is not defined.") + return logs_dir - file_name = f"nebula_trust_results_{participant_id}.json" - file_path = os.path.join( - logs_dir, - scenario_name, - "trustworthiness", - file_name, - ) +def _trustworthiness_dir(scenario_name: str) -> str: + # Return the scenario trustworthiness directory used by report JSON files. + return os.path.join(_logs_dir(), scenario_name, "trustworthiness") + + +def _trust_report_path(scenario_name: str, participant_id: int | str) -> str: + # Return the local trust report path for one participant. + return os.path.join(_trustworthiness_dir(scenario_name), f"nebula_trust_results_{participant_id}.json") + + +def _read_json_file(file_path: str) -> dict: + # Load a JSON object and raise clear errors for missing or invalid files. if not os.path.exists(file_path): raise FileNotFoundError(f"The file does not exist: {file_path}") try: - with open(file_path, "r", encoding="utf-8") as f: - trust_report = json.load(f) - except json.JSONDecodeError as e: - raise ValueError(f"The file does not contain valid JSON: {file_path}") from e + with open(file_path, "r", encoding="utf-8") as file: + return json.load(file) + except json.JSONDecodeError as error: + raise ValueError(f"The file does not contain valid JSON: {file_path}") from error - return json.dumps(trust_report) +def _write_json_file(file_path: str, data: dict) -> str: + # Write a formatted JSON object, creating the parent directory if needed. + directory = os.path.dirname(file_path) + if directory: + os.makedirs(directory, exist_ok=True) -def load_trust_report_json(scenario_name: str, participant_id: int | str) -> dict: - trust_report_json = load_trust_report_json_dumped(scenario_name, participant_id) - return json.loads(trust_report_json) + with open(file_path, "w", encoding="utf-8") as file: + json.dump(data, file, indent=4) + return file_path -def create_local_trust_report_copy(scenario_name: str, participant_id: int | str, suffix: str = "global") -> tuple[dict, str]: - trust_report = load_trust_report_json(scenario_name, participant_id) - logs_dir = os.environ.get("NEBULA_LOGS_DIR") - if not logs_dir: - raise ValueError("The NEBULA_LOGS_DIR environment variable is not defined.") - trust_dir = os.path.join(logs_dir, scenario_name, "trustworthiness") - os.makedirs(trust_dir, exist_ok=True) +def _is_score_entry(key, value) -> bool: + # Trust report scores are numeric values stored under score-like keys. + return key in SCORE_KEYS and _is_numeric_score(value) - file_path = os.path.join(trust_dir, f"nebula_trust_results_{participant_id}_{suffix}.json") - with open(file_path, "w", encoding="utf-8") as f: - json.dump(trust_report, f, indent=4) - return trust_report, file_path +def load_trust_report_json_dumped(scenario_name: str, participant_id: int) -> str: + # Load one participant report and return it serialized for network messages. + return json.dumps(load_trust_report_json(scenario_name, participant_id)) -def save_trust_report_json(file_path: str, trust_report: dict) -> str: - directory = os.path.dirname(file_path) - if directory: - os.makedirs(directory, exist_ok=True) +def load_trust_report_json(scenario_name: str, participant_id: int | str) -> dict: + # Load one participant trustworthiness report as a dictionary. + return _read_json_file(_trust_report_path(scenario_name, participant_id)) - with open(file_path, "w", encoding="utf-8") as f: - json.dump(trust_report, f, indent=4) - return file_path +def create_local_trust_report_copy(scenario_name: str, participant_id: int | str, suffix: str = "global") -> tuple[dict, str]: + # Copy a participant report to a local aggregation output file. + trust_report = load_trust_report_json(scenario_name, participant_id) + file_path = os.path.join( + _trustworthiness_dir(scenario_name), + f"nebula_trust_results_{participant_id}_{suffix}.json", + ) + + return trust_report, _write_json_file(file_path, trust_report) + + +def save_trust_report_json(file_path: str, trust_report: dict) -> str: + # Save a trust report and return the written file path. + return _write_json_file(file_path, trust_report) def accumulate_weighted_trustscores(report: dict, weight: float, score_accumulator: dict, weight_accumulator: dict): + # Add all score values from a report into weighted accumulators. if weight <= 0: raise ValueError("The aggregation weight must be greater than 0.") @@ -88,6 +95,7 @@ def accumulate_weighted_trustscores(report: dict, weight: float, score_accumulat def build_weighted_trustscores_report(template_report: dict, score_accumulator: dict, weight_accumulator: dict) -> dict: + # Return a deep-copied report with every score replaced by its weighted mean. aggregated_report = copy.deepcopy(template_report) _apply_weighted_trustscores_recursive( obj=aggregated_report, @@ -99,22 +107,23 @@ def build_weighted_trustscores_report(template_report: dict, score_accumulator: def _accumulate_weighted_trustscores_recursive(obj, weight: float, path: tuple, score_accumulator: dict, weight_accumulator: dict): + # Walk a trust report and accumulate weighted sums for every score path. if isinstance(obj, dict): - structural_named_entry = _get_structural_named_entry(obj) - if structural_named_entry is not None: - _, nested_value = structural_named_entry + named_entry = _get_structural_named_entry(obj) + if named_entry is not None: + _, nested_value = named_entry _accumulate_weighted_trustscores_recursive( obj=nested_value, weight=weight, - path=path + ("__named_entry__",), + path=path + (NAMED_ENTRY_PATH_KEY,), score_accumulator=score_accumulator, weight_accumulator=weight_accumulator, ) return for key, value in obj.items(): - if key in {"trust_score", "score"} and _is_numeric_score(value): - score_path = path + (key,) + score_path = path + (key,) + if _is_score_entry(key, value): score_accumulator[score_path] = score_accumulator.get(score_path, 0.0) + (float(value) * weight) weight_accumulator[score_path] = weight_accumulator.get(score_path, 0.0) + weight continue @@ -122,7 +131,7 @@ def _accumulate_weighted_trustscores_recursive(obj, weight: float, path: tuple, _accumulate_weighted_trustscores_recursive( obj=value, weight=weight, - path=path + (key,), + path=score_path, score_accumulator=score_accumulator, weight_accumulator=weight_accumulator, ) @@ -140,21 +149,22 @@ def _accumulate_weighted_trustscores_recursive(obj, weight: float, path: tuple, def _apply_weighted_trustscores_recursive(obj, path: tuple, score_accumulator: dict, weight_accumulator: dict): + # Walk a report copy and replace score values with weighted averages. if isinstance(obj, dict): - structural_named_entry = _get_structural_named_entry(obj) - if structural_named_entry is not None: - entry_key, nested_value = structural_named_entry + named_entry = _get_structural_named_entry(obj) + if named_entry is not None: + entry_key, nested_value = named_entry obj[entry_key] = _apply_weighted_trustscores_recursive( obj=nested_value, - path=path + ("__named_entry__",), + path=path + (NAMED_ENTRY_PATH_KEY,), score_accumulator=score_accumulator, weight_accumulator=weight_accumulator, ) return obj for key, value in obj.items(): - if key in {"trust_score", "score"} and _is_numeric_score(value): - score_path = path + (key,) + score_path = path + (key,) + if _is_score_entry(key, value): total_weight = weight_accumulator.get(score_path) if total_weight: obj[key] = round(score_accumulator[score_path] / total_weight, 6) @@ -162,7 +172,7 @@ def _apply_weighted_trustscores_recursive(obj, path: tuple, score_accumulator: d obj[key] = _apply_weighted_trustscores_recursive( obj=value, - path=path + (key,), + path=score_path, score_accumulator=score_accumulator, weight_accumulator=weight_accumulator, ) @@ -176,10 +186,12 @@ def _apply_weighted_trustscores_recursive(obj, path: tuple, score_accumulator: d score_accumulator=score_accumulator, weight_accumulator=weight_accumulator, ) + return obj def _get_structural_named_entry(obj: dict): + # Detect wrappers like {"Privacy": {"score": ..., "metrics": ...}}. if len(obj) != 1: return None @@ -187,11 +199,12 @@ def _get_structural_named_entry(obj: dict): if not isinstance(nested_value, dict): return None - if any(key in nested_value for key in ("score", "metrics", "notions", "pillars")): + if any(key in nested_value for key in NAMED_ENTRY_KEYS): return entry_key, nested_value return None def _is_numeric_score(value): + # Booleans are ints in Python, but they are not trust score values here. return isinstance(value, (int, float)) and not isinstance(value, bool) diff --git a/nebula/addons/trustworthiness/trustworthiness.py b/nebula/addons/trustworthiness/trustworthiness.py index 17b9a4ef8..1996171ba 100644 --- a/nebula/addons/trustworthiness/trustworthiness.py +++ b/nebula/addons/trustworthiness/trustworthiness.py @@ -35,7 +35,7 @@ from codecarbon import EmissionsTracker from nebula.addons.trustworthiness.per_round_metrics import PerRoundTrustMetrics from datetime import datetime -from nebula.addons.trustworthiness.factsheet import CflFactsheet +from nebula.addons.trustworthiness.cfl_factsheet import CflFactsheet from nebula.addons.trustworthiness.metric import TrustMetricManager from nebula.addons.trustworthiness.dfl_factsheet import DflFactsheet from nebula.addons.trustworthiness.graphics import Graphics @@ -996,6 +996,7 @@ async def _process_experiment_finish_event(self, efe: ExperimentFinishEvent): bytes_sent = self._engine.reporter.acc_bytes_sent bytes_recv = self._engine.reporter.acc_bytes_recv + # Persist the trainer-reported DP budget so factsheets can score privacy. privacy_metrics = self._engine.trainer.get_privacy_metrics() dp_enabled=bool(privacy_metrics.get("dp_enabled", False)) dp_epsilon=privacy_metrics.get("dp_epsilon") diff --git a/nebula/core/aggregation/updatehandlers/sdflupdatehandler.py b/nebula/core/aggregation/updatehandlers/sdflupdatehandler.py index 4ebd15ba3..b91e82ef4 100644 --- a/nebula/core/aggregation/updatehandlers/sdflupdatehandler.py +++ b/nebula/core/aggregation/updatehandlers/sdflupdatehandler.py @@ -54,8 +54,10 @@ def __init__(self, aggregator, addr, buffersize=MAX_UPDATE_BUFFER_SIZE): self._addr = addr self._aggregator: Aggregator = aggregator self._buffersize = buffersize + # Store the last used update plus a short history per source to tolerate late/missing updates. self._updates_storage: dict[str, tuple[Update, deque[Update]]] = {} self._updates_storage_lock = Locker(name="updates_storage_lock", async_lock=True) + # SDFL aggregation waits for a dynamic set of trainer sources each round. self._sources_expected = set() self._sources_received = set() self._round_updates_lock = Locker(name="round_updates_lock", async_lock=True) @@ -91,6 +93,7 @@ async def round_expected_updates(self, federation_nodes: set): """ await self._update_federation_lock.acquire_async() await self._updates_storage_lock.acquire_async() + # Reset per-round reception state while preserving per-node history buffers. self._sources_expected = federation_nodes.copy() self._sources_received.clear() @@ -144,6 +147,7 @@ async def storage_update(self, updt_received_event: UpdateReceivedEvent): updt_received_event (UpdateReceivedEvent): Event with model update data. """ if updt_received_event.is_reputation_update(): + # Reputation model updates are consumed by the reputation addon, not by aggregation. logging.debug("Discard reputation-only update in SDFL aggregation storage") return @@ -168,6 +172,7 @@ async def storage_update(self, updt_received_event: UpdateReceivedEvent): f"Updates received ({len(self._sources_received)}/{len(self._sources_expected)}) | Missing nodes: {updates_left}" ) if self._round_updates_lock.locked() and not updates_left: + # Release aggregation as soon as the last expected trainer update arrives. all_rec = await self._all_updates_received() if all_rec: await self._notify() @@ -194,6 +199,7 @@ async def get_round_updates(self): self._nodes_using_historic.clear() updates = {} for sr in self._sources_received: + # Use the newest update unless it was already consumed in a previous aggregation. source_historic = self.us[sr][1] last_updt_received = self.us[sr][0] updt: Update = None @@ -217,6 +223,8 @@ async def before_aggregation(self, updates: dict[str, tuple[object, float]], fed if not hasattr(engine, "_reputation") or engine._reputation is None: return + # The aggregator may receive updates from non-neighbor trainers through forwarding. + # Their reputation is inferred from reputation tables shared by expected trainers. round_num = await engine.get_round() expected_table_nodes = engine.get_sdfl_expected_trainers() target_nodes = set(federation_nodes) | set(updates.keys()) @@ -285,6 +293,7 @@ async def notify_if_all_updates_received(self): Set a notification trigger and notify aggregator if all updates are already received. """ logging.info("Set notification when all expected updates received") + # Hold this lock while the caller is waiting; _notify releases it once ready. await self._round_updates_lock.acquire_async() await self._updates_storage_lock.acquire_async() all_received = await self._all_updates_received() @@ -306,6 +315,7 @@ async def _notify(self): """ await self._notification_sent_lock.acquire_async() if self._notification: + # Multiple updates can race to complete the round; notify the aggregator once. await self._notification_sent_lock.release_async() return self._notification = True diff --git a/nebula/core/engine.py b/nebula/core/engine.py index 7fde4164e..cb05c57db 100644 --- a/nebula/core/engine.py +++ b/nebula/core/engine.py @@ -559,6 +559,7 @@ async def _reputation_share_callback(self, source, message): async def _reputationtable_table_callback(self, source, message): try: + # Reputation tables are an SDFL-only control plane for indirect reputation. if self.config.participant["scenario_args"].get("federation") != "SDFL": return if self.rb.get_role_name(True) != "aggregator": @@ -580,6 +581,7 @@ async def _reputationtable_table_callback(self, source, message): reputation_table, received_from=source, ) + # Start or refresh the async collection window for this SDFL round. expected_nodes = self.get_sdfl_expected_trainers() timeout = float( self.config.participant["defense_args"] @@ -685,6 +687,7 @@ async def _sdflmodel_trainer_update_callback(self, source, message): ) return + # Valid trainer updates are converted into the normal aggregation event stream. decoded_model = self.trainer.deserialize_model(message.parameters) event = UpdateReceivedEvent( @@ -729,6 +732,7 @@ async def _sdflmodel_global_model_callback(self, source, message): ) return + # Trainers apply the aggregator's global model and unblock their SDFL round wait. decoded_model = self.trainer.deserialize_model(message.parameters) self.trainer.set_model_parameters(decoded_model) diff --git a/nebula/core/models/nebulamodel.py b/nebula/core/models/nebulamodel.py index b2d6065d8..3a270ae88 100755 --- a/nebula/core/models/nebulamodel.py +++ b/nebula/core/models/nebulamodel.py @@ -215,6 +215,7 @@ def __init__( self._latest_validation_metrics = {} self._train_extra_metrics = {} + # DP trainers update these fields after querying the Opacus accountant. self.dp_enabled = False self.dp_epsilon = None self.dp_delta = None diff --git a/nebula/core/network/forwarder.py b/nebula/core/network/forwarder.py index e6831eec7..9eccc15fe 100755 --- a/nebula/core/network/forwarder.py +++ b/nebula/core/network/forwarder.py @@ -143,8 +143,10 @@ def _allow_forward_after_learning_finished(self, msg: bytes) -> bool: if message_type == "trustscores_message": return True if message_type == "sdflmodel_message": + # Trainers may finish their local cycle before the forwarded global model arrives. return message_wrapper.sdflmodel_message.action == nebula_pb2.SdflmodelMessage.Action.GLOBAL_MODEL if message_type == "reputationtable_message": + # SDFL reputation tables can be forwarded while the aggregator is waiting. return True return False except Exception as e: diff --git a/nebula/core/network/messages.py b/nebula/core/network/messages.py index 0d8a036ed..9963ff6ea 100644 --- a/nebula/core/network/messages.py +++ b/nebula/core/network/messages.py @@ -87,6 +87,7 @@ def _define_message_templates(self): }, }, "sdflmodel": { + # SDFL uses a dedicated model channel for forwarded trainer/global updates. "parameters": ["action", "target", "parameters", "weight", "round", "node_id"], "defaults": { "weight": 1, @@ -100,6 +101,7 @@ def _define_message_templates(self): }, }, "reputationtable": { + # Reputation tables carry one-hop trust scores for SDFL indirect reputation. "parameters": ["action", "node_id", "round", "reputation_table_json"], "defaults": { "node_id": self.addr, @@ -262,11 +264,13 @@ def _should_forward_message(self, message_type, message_wrapper): return True if self.cm.config.participant["scenario_args"]["federation"] == "SDFL" and message_type == "sdflmodel_message": + # SDFL model messages must still flow after the generic learning-finished gate. return True if ( self.cm.config.participant["scenario_args"]["federation"] == "SDFL" and message_type == "reputationtable_message" ): + # Reputation tables can arrive late while aggregation is waiting for trust evidence. return True def create_message(self, message_type: str, action: str = "", *args, **kwargs): diff --git a/nebula/core/node.py b/nebula/core/node.py index a5fa22f1d..772c9d832 100755 --- a/nebula/core/node.py +++ b/nebula/core/node.py @@ -210,6 +210,7 @@ async def main(config: Config): trainer_str = config.participant["training_args"]["trainer"] dp_enabled = config.participant["training_args"]["dp"]["enabled"] if trainer_str == "lightning": + # DP is implemented as a Lightning-specific trainer wrapper around Opacus. if dp_enabled: trainer = LightningDP else: diff --git a/nebula/core/noderole.py b/nebula/core/noderole.py index ce02f1cd4..3d7f68456 100644 --- a/nebula/core/noderole.py +++ b/nebula/core/noderole.py @@ -327,6 +327,7 @@ async def resolve_missing_updates(self): class SDFLRoleMixin: async def _send_reputation_model_update(self): + # SDFL reputation evaluates direct neighbors from the latest local model update. model_params = self._engine.trainer.get_model_parameters() serialized_model = ( model_params @@ -346,6 +347,7 @@ async def _send_reputation_model_update(self): logging.info("SDFL reputation | No direct neighbors to send model/update") return + # Reputation model updates use the regular model channel and stay one-hop local. logging.info(f"SDFL reputation | Broadcasting model/update to direct neighbors: {neighbors}") await asyncio.gather( *[ @@ -357,9 +359,11 @@ async def _send_reputation_model_update(self): class SDFLAggregatorRoleBehavior(SDFLRoleMixin, AggregatorRoleBehavior): async def before_round_start(self): + # Leadership transfer must be acknowledged before the new aggregator starts a round. await self._engine.wait_pending_leadership_ack() async def extended_learning_cycle(self): + # SDFL aggregators collect trainer updates, publish the global model, then rotate leadership. await self._engine.trainer.test() await self._send_reputation_model_update() await self._engine._waiting_model_updates() @@ -370,12 +374,14 @@ async def _before_leadership_transfer(self, successor): await self._engine.mark_leadership_transfer_pending(successor) async def select_nodes_to_wait(self): + # The aggregator waits for all expected trainers, not just currently direct neighbors. nodes = self._engine.get_sdfl_expected_trainers() if nodes: return nodes return await super().select_nodes_to_wait() async def _send_global_model(self) -> None: + # Send the aggregated model through the SDFL forwarding channel. model_params = self._engine.trainer.get_model_parameters() serialized_model = ( model_params @@ -500,6 +506,7 @@ async def extended_learning_cycle(self): await self._engine.trainer.test() self._prepare_waiting_global_model() + # Trainers train locally, exchange reputation evidence, send their update, then wait for aggregation. await self._engine.trainning_in_progress_lock.acquire_async() try: await self._engine.trainer.train() @@ -507,6 +514,7 @@ async def extended_learning_cycle(self): await self._engine.trainning_in_progress_lock.release_async() if self._engine._reputation is not None: + # Process reputation model updates that arrived before the local table is computed. await self._engine._reputation.process_pending_sdfl_reputation_updates(self._engine.round) await self._send_reputation_model_update() @@ -515,10 +523,12 @@ async def extended_learning_cycle(self): await self._waiting_global_model() def _prepare_waiting_global_model(self): + # Reset the per-round event used by trainers to block until a GLOBAL_MODEL arrives. self._engine._global_model_source = None self._engine._global_model_received.clear() async def _calculate_and_send_reputation_table(self): + # Trainers publish direct-neighbor reputation tables for the aggregator to combine. if self._engine._reputation is None: return @@ -544,6 +554,7 @@ async def _calculate_and_send_reputation_table(self): await self._engine._reputation.calculate_and_send_sdfl_reputation_table() async def _send_trainer_update(self): + # Broadcast the local trainer update; forwarding delivers it to the current aggregator. model_params = self._engine.trainer.get_model_parameters() serialized_model = ( model_params @@ -585,6 +596,7 @@ async def _send_trainer_update(self): logging.warning("SDFL trainer | No neighbors available to send TRAINER_UPDATE") async def _waiting_global_model(self): + # A trainer continues only after the aggregator's GLOBAL_MODEL is received or times out. timeout = self._config.participant["aggregator_args"]["aggregation_timeout"] logging.info(f"💤 Waiting global SDFL model in round {self._engine.round}.") try: diff --git a/nebula/core/training/dp.py b/nebula/core/training/dp.py index 56a2508f8..15b094c88 100644 --- a/nebula/core/training/dp.py +++ b/nebula/core/training/dp.py @@ -1,4 +1,5 @@ class SimpleDPState: + # Minimal mutable state used to pass Opacus-wrapped objects between hooks. def __init__(self): self.extras = {} @@ -17,6 +18,8 @@ def __init__( poisson_sampling=True, clipping="flat", ): + # Fixed DP-SGD controls. Epsilon is not configured here; it is computed + # from the accountant as the consumed privacy budget after training. self.noise_multiplier = float(noise_multiplier) self.max_grad_norm = float(max_grad_norm) self.target_delta = target_delta @@ -27,11 +30,14 @@ def __init__( self._privacy_engine = None def on_train_start(self, model, optimizer, state): + # Import Opacus lazily so non-DP trainers do not need to load it. from opacus import PrivacyEngine dataloader = state.extras["dataloader"] model.train() + # Keep one PrivacyEngine per plugin instance so the accountant composes + # privacy loss across Nebula rounds instead of resetting every round. if self._privacy_engine is None: self._privacy_engine = PrivacyEngine( accountant=self.accountant, @@ -49,12 +55,14 @@ def on_train_start(self, model, optimizer, state): clipping=self.clipping, ) + # Replace the training components with DP-aware versions used by LightningDP. state.extras["privacy_engine"] = privacy_engine state.extras["model"] = private_model state.extras["optimizer"] = private_optimizer state.extras["dataloader"] = private_dataloader def on_train_end(self, state): + # Query the accumulated epsilon for the configured delta after this round. privacy_engine = state.extras.get("privacy_engine") private_model = state.extras.get("model") @@ -67,6 +75,8 @@ def on_train_end(self, state): pass if private_model is not None: + # Clean Opacus hook state so the same model can continue through later + # Nebula phases without stale per-sample gradient hooks. try: private_model.zero_grad(set_to_none=True) except Exception: diff --git a/nebula/core/training/lightning.py b/nebula/core/training/lightning.py index 0c869d1de..f5988ef00 100755 --- a/nebula/core/training/lightning.py +++ b/nebula/core/training/lightning.py @@ -390,6 +390,7 @@ def show_current_learning_rate(self): self.model.show_current_learning_rate() def get_privacy_metrics(self): + # Non-DP trainers expose the same metrics contract with neutral values. return { "dp_enabled": False, "dp_epsilon": 0, diff --git a/nebula/core/training/lightning_dp.py b/nebula/core/training/lightning_dp.py index c15a164b9..bed08a973 100644 --- a/nebula/core/training/lightning_dp.py +++ b/nebula/core/training/lightning_dp.py @@ -19,11 +19,13 @@ class LightningDP(Lightning): def __init__(self, model, datamodule, config=None): super().__init__(model, datamodule, config) + # The DP plugin owns the Opacus PrivacyEngine and its cumulative accountant. self._dp_plugin = self.create_dp_plugin() self.dp_epsilon = None self.dp_delta = None def create_dp_plugin(self): + # Translate Nebula participant config into the fixed DP-SGD controls used by Opacus. dp_config = self.config.participant["training_args"].get("dp") if dp_config is None or not dp_config.get("enabled", False): @@ -40,6 +42,7 @@ def create_dp_plugin(self): ) def _train_sync(self): + # Keep the public Lightning trainer contract: train once and return loss/accuracy. try: self._fit_with_dp() @@ -63,6 +66,7 @@ def _train_sync(self): raise def _get_training_device(self): + # Resolve the effective device for any manual DP path that needs it. if ( self.config.participant["device_args"]["accelerator"] == "gpu" and torch.cuda.is_available() @@ -73,6 +77,7 @@ def _get_training_device(self): return torch.device("cpu") def _log_manual_metrics(self, phase, metrics): + # Log manually computed metrics using the same naming scheme as Lightning. output = metrics.compute() output = { f"{phase}/{key.replace('Multiclass', '').split('/')[-1]}": value.detach() @@ -88,9 +93,11 @@ def _log_manual_metrics(self, phase, metrics): self._logger.log_data(output, step=self.model.global_number[phase]) def _fit_with_dp(self): + # Bridge Nebula's Lightning trainer with Opacus' private optimizer/dataloader. state = SimpleDPState() if hasattr(self.model, "clear_optimizer_override"): + # Start from a clean optimizer so a previous round cannot leak into this fit. self.model.clear_optimizer_override() try: @@ -102,6 +109,7 @@ def _fit_with_dp(self): optimizer = self.model.configure_optimizers() state.extras["dataloader"] = train_dataloader + # Opacus wraps the model, optimizer and dataloader, and updates the accountant. self._dp_plugin.on_train_start(self.model, optimizer, state) private_optimizer = state.extras["optimizer"] @@ -114,6 +122,8 @@ def _fit_with_dp(self): # the original LightningModule and a DPOptimizer through configure_optimizers. self.model.dp_enabled = True self.model.set_optimizer_override(private_optimizer) + # Lightning still drives the training loop; the injected optimizer/dataloader + # make the loop perform DP-SGD instead of standard SGD. self._trainer.fit( self.model, train_dataloaders=private_dataloader, @@ -123,6 +133,7 @@ def _fit_with_dp(self): self.model.train() finally: + # Always restore the model/trainer state, even if Lightning raises. self.model.dp_enabled = False if hasattr(self.model, "clear_optimizer_override"): self.model.clear_optimizer_override() @@ -132,6 +143,7 @@ def _fit_with_dp(self): dp_epsilon = state.extras.get("dp_epsilon") if dp_epsilon is not None: + # Store the accumulated privacy budget for logging and trustworthiness reports. dp_delta = state.extras["dp_delta"] self.dp_epsilon = float(dp_epsilon) @@ -153,6 +165,7 @@ def _fit_with_dp(self): ) def get_privacy_metrics(self): + # Trustworthiness consumes these values at experiment finish. return { "dp_enabled": True, "dp_epsilon": self.dp_epsilon, From 309e5f0718704503416a47744cbef7af92411b4b Mon Sep 17 00:00:00 2001 From: "Juan J." Date: Wed, 3 Jun 2026 10:54:01 +0200 Subject: [PATCH 57/66] Refactoring and metrics fixed: Privacy and explainability --- .../trustworthiness/helpers/explainability.py | 438 ++++++------------ .../addons/trustworthiness/helpers/privacy.py | 94 +--- 2 files changed, 155 insertions(+), 377 deletions(-) diff --git a/nebula/addons/trustworthiness/helpers/explainability.py b/nebula/addons/trustworthiness/helpers/explainability.py index ce9809c3e..96b066a1f 100644 --- a/nebula/addons/trustworthiness/helpers/explainability.py +++ b/nebula/addons/trustworthiness/helpers/explainability.py @@ -1,5 +1,3 @@ -import copy -import gc import logging import math @@ -11,298 +9,160 @@ logger = logging.getLogger(__name__) -def get_feature_importance_cv(model, test_sample): - """ - Calculates the coefficient of variation of the feature importance. - Args: - model (object): The model. - test_sample (object): One test sample to calculate the feature importance. +def _feature_importance_cv_from_values(vals): + # Higher CV means attributions differ more across features, i.e. a more selective explanation. + vals = np.asarray(vals, dtype=float).reshape(-1) + vals = np.nan_to_num(vals, nan=0.0, posinf=0.0, neginf=0.0) + vals = vals[vals > 0] - Returns: - float: The coefficient of variation of the feature importance. - """ - - try: - vals = np.asarray(_get_feature_importances(model, test_sample), dtype=float).reshape(-1) - vals = np.nan_to_num(vals, nan=0.0, posinf=0.0, neginf=0.0) - vals = vals[vals > 0] - - if len(vals) <= 1: - return 0.0 + if len(vals) <= 1: + return 0.0 - cv = float(variation(vals)) - if math.isnan(cv) or math.isinf(cv): - return 1.0 - return max(0.0, cv) - except Exception as exc: - logger.warning("Could not compute feature importance CV with shap") - logger.warning(exc) + cv = float(variation(vals)) + if math.isnan(cv) or math.isinf(cv): return 1.0 + return max(0.0, cv) def _get_feature_importances(model, test_sample): - """ - Computes global feature importances from SHAP values. - - Args: - model (object): The model. - test_sample (object): One test sample batch. - - Returns: - np.ndarray: Global importances per feature. - """ + # Computes global feature importances with a simple modality-aware policy: + # SHAP for tabular tensors and Integrated Gradients for image-like tensors. if not isinstance(model, torch.nn.Module): logger.warning("Model is not a torch.nn.Module") return np.array([]) - def _clone_model(model_ref, device): - optimizer_attrs = ("_optimizer", "_optimizer_override") - optimizer_state = {} - try: - for attr in optimizer_attrs: - if hasattr(model_ref, attr): - optimizer_state[attr] = getattr(model_ref, attr) - setattr(model_ref, attr, None) - - model_clone = copy.deepcopy(model_ref) - for attr in optimizer_attrs: - if hasattr(model_clone, attr): - setattr(model_clone, attr, None) - - model_clone.to(device) - model_clone.eval() - return model_clone - except Exception as exc: - logger.warning("Could not clone model for SHAP, using original model") - logger.warning(exc) - model_ref.eval() - return model_ref - finally: - for attr, value in optimizer_state.items(): - setattr(model_ref, attr, value) - - def _prepare_shap_inputs(sample): - if not (isinstance(sample, (tuple, list)) and len(sample) >= 1): - return None, None, None - - batched_data = sample[0] - if not torch.is_tensor(batched_data) or batched_data.ndim == 0 or batched_data.size(0) == 0: - return None, None, None - - if not torch.is_floating_point(batched_data): - batched_data = batched_data.float() - - batch_size = int(batched_data.size(0)) - input_shape = tuple(int(dim) for dim in batched_data.shape[1:]) - - if batch_size == 1: - return batched_data[:1], batched_data[:1], input_shape - - background_size = min(max(8, batch_size // 4), 32, batch_size - 1) - explainable = batch_size - background_size - explain_size = min(max(4, explainable), 32, explainable) - - background = batched_data[:background_size] - test_data = batched_data[background_size:background_size + explain_size] - - if test_data.size(0) == 0: - test_data = batched_data[: min(batch_size, 32)] - - return background, test_data, input_shape - - def _compute_shap_values(model_ref, background, test_data): - explainer_errors = [] - - for explainer_name in ("DeepExplainer", "GradientExplainer"): - explainer = None - try: - if explainer_name == "DeepExplainer": - explainer = shap.DeepExplainer(model_ref, background) - return explainer.shap_values(test_data, check_additivity=False) - - explainer = shap.GradientExplainer(model_ref, background) - return explainer.shap_values(test_data) - except Exception as exc: - explainer_errors.append(f"{explainer_name}: {exc}") - finally: - # SHAP explainers may register autograd hooks. If we explain on the - # original model, those hooks can leak into later ART metrics. - del explainer - gc.collect() - - raise RuntimeError("; ".join(explainer_errors)) - - def _compute_gradient_importances(model_ref, test_data): - was_training = bool(getattr(model_ref, "training", False)) - model_ref.eval() - - try: - inputs = test_data.detach().clone().requires_grad_(True) - model_ref.zero_grad(set_to_none=True) - - outputs = model_ref(inputs) - if isinstance(outputs, (tuple, list)): - outputs = outputs[0] - - if outputs.ndim == 1: - score = outputs.sum() - else: - score = outputs.reshape(outputs.shape[0], -1).max(dim=1).values.sum() - - score.backward() - if inputs.grad is None: - return np.array([]) - - importances = torch.abs(inputs.grad * inputs).mean(dim=0) - importances = importances.detach().cpu().numpy().reshape(-1) - importances = np.nan_to_num(importances, nan=0.0, posinf=0.0, neginf=0.0) - return np.maximum(importances, 0.0) - finally: - if was_training: - model_ref.train() - - def _feature_axes_from_shape(arr_shape, input_shape, n_samples): - input_shape = tuple(input_shape) - input_rank = len(input_shape) - - if input_rank == 0 or len(arr_shape) < input_rank: - return None - - if len(arr_shape) >= input_rank + 1 and tuple(arr_shape[1:1 + input_rank]) == input_shape: - return tuple(range(1, 1 + input_rank)) - - if len(arr_shape) >= input_rank + 2 and arr_shape[1] == n_samples and tuple(arr_shape[2:2 + input_rank]) == input_shape: - return tuple(range(2, 2 + input_rank)) - - candidates = [] - for start in range(len(arr_shape) - input_rank + 1): - if tuple(arr_shape[start:start + input_rank]) == input_shape: - candidates.append(start) + if not isinstance(test_sample, (tuple, list)) or len(test_sample) < 1: + return np.array([]) - if not candidates: - return None + inputs = test_sample[0] + if not torch.is_tensor(inputs) or inputs.ndim < 2 or inputs.size(0) == 0: + return np.array([]) - # Prefer matches that do not consume the leading sample/output axes. - non_leading = [start for start in candidates if start > 0] - if non_leading: - candidates = non_leading + try: + device = next(model.parameters()).device + except Exception: + device = torch.device("cpu") - if len(arr_shape) > 1 and arr_shape[1] == n_samples: - non_output_sample = [start for start in candidates if start > 1] - if non_output_sample: - candidates = non_output_sample + inputs = inputs.to(device) + if not torch.is_floating_point(inputs): + inputs = inputs.float() - start = candidates[0] - return tuple(range(start, start + input_rank)) + was_training = bool(getattr(model, "training", False)) + model.eval() try: - try: - device = next(model.parameters()).device - except Exception: - device = torch.device("cpu") + if inputs.ndim == 2: + logger.info("Computing tabular feature importances with SHAP, input_shape=%s", tuple(inputs.shape)) + importances = _get_shap_importances(model, inputs) + else: + logger.info("Computing image-like feature importances with Integrated Gradients, input_shape=%s", tuple(inputs.shape)) + importances = _get_integrated_gradients_importances(model, inputs) - background, test_data, input_shape = _prepare_shap_inputs(test_sample) - if background is None or test_data is None or input_shape is None: - return np.array([]) + logger.info("Computed feature importances, n_features=%s, total_importance=%s", len(importances), float(np.sum(importances))) + return importances + except Exception as exc: + logger.warning("Could not compute feature importances") + logger.warning(exc) + return np.array([]) + finally: + if was_training: + model.train() - background = background.to(device) - test_data = test_data.to(device) - - shap_model = _clone_model(model, device) - try: - shap_values = _compute_shap_values(shap_model, background, test_data) - except Exception as exc: - logger.debug("Could not compute feature importances with SHAP, using gradient fallback: %s", exc) - shap_model = None - gc.collect() - - gradient_model = _clone_model(model, device) - try: - return _compute_gradient_importances(gradient_model, test_data) - except Exception as fallback_exc: - logger.debug("Could not compute feature importances with gradient fallback: %s", fallback_exc) - return np.array([]) - finally: - del gradient_model - gc.collect() - finally: - if shap_model is not None: - del shap_model - gc.collect() - - if shap_values is None: - return np.array([]) - if isinstance(shap_values, (list, tuple)): - arrays = [np.asarray(val, dtype=float) for val in shap_values if val is not None] - if not arrays: - return np.array([]) - shap_arr = np.stack(arrays, axis=0) - else: - shap_arr = np.asarray(shap_values, dtype=float) +def _get_shap_importances(model, inputs): + # SHAP is a natural fit for tabular data: one attribution per input column. + if inputs.size(0) < 2: + return np.array([]) + + background_size = min(16, inputs.size(0) - 1) + background = inputs[:background_size] + explained = inputs[background_size:] - if shap_arr.size == 0: + logger.info("SHAP background_size=%s, explained_size=%s", int(background.size(0)), int(explained.size(0))) + explainer = shap.GradientExplainer(model, background) + shap_values = explainer.shap_values(explained) + + if isinstance(shap_values, (list, tuple)): + arrays = [np.asarray(values, dtype=float) for values in shap_values if values is not None] + if not arrays: return np.array([]) + shap_arr = np.stack(arrays, axis=0) + importances = np.mean(np.abs(shap_arr), axis=(0, 1)) + else: + shap_arr = np.asarray(shap_values, dtype=float) + if shap_arr.ndim == 3: + importances = np.mean(np.abs(shap_arr), axis=(0, 2)) + else: + importances = np.mean(np.abs(shap_arr), axis=0) + + return _clean_importances(importances) + + +def _get_integrated_gradients_importances(model, inputs, steps=16): + # Zero baseline is simple and works well for normalized image tensors. + logger.info("Integrated Gradients steps=%s", int(steps)) + baseline = torch.zeros_like(inputs) + total_gradients = torch.zeros_like(inputs) + + for alpha in torch.linspace(0.0, 1.0, steps, device=inputs.device): + scaled_inputs = (baseline + alpha * (inputs - baseline)).detach().requires_grad_(True) + model.zero_grad(set_to_none=True) - shap_arr = np.nan_to_num(shap_arr, nan=0.0, posinf=0.0, neginf=0.0) - feature_axes = _feature_axes_from_shape(tuple(shap_arr.shape), input_shape, int(test_data.size(0))) - - if feature_axes is None: - # Conservative fallback: treat the first axis as samples when possible and - # flatten the remaining dimensions into features. - if shap_arr.ndim == 1: - importances = np.abs(shap_arr) - else: - aggregate_axes = (0,) - importances = np.mean(np.abs(shap_arr), axis=aggregate_axes) + outputs = model(scaled_inputs) + if isinstance(outputs, (tuple, list)): + outputs = outputs[0] + + # Explain the model's predicted class for each sample. + if outputs.ndim == 1: + score = outputs.sum() else: - aggregate_axes = tuple(idx for idx in range(shap_arr.ndim) if idx not in feature_axes) - if aggregate_axes: - importances = np.mean(np.abs(shap_arr), axis=aggregate_axes) - else: - importances = np.abs(shap_arr) - - importances = np.asarray(importances, dtype=float).reshape(-1) - importances = np.nan_to_num(importances, nan=0.0, posinf=0.0, neginf=0.0) - return np.maximum(importances, 0.0) - except Exception as exc: - logger.debug("Could not compute feature importances") - logger.debug(exc) - return np.array([]) + score = outputs.reshape(outputs.shape[0], -1).max(dim=1).values.sum() + gradients = torch.autograd.grad(score, scaled_inputs)[0] + total_gradients += gradients.detach() -def get_alpha_score(model, test_sample, alpha=0.8): - """ - Computes alpha score from global feature importances. - """ - try: - vals = np.asarray(_get_feature_importances(model, test_sample), dtype=float).reshape(-1) - vals = np.nan_to_num(vals, nan=0.0, posinf=0.0, neginf=0.0) - vals = np.maximum(vals, 0.0) - total_features = len(vals) - if total_features == 0 or np.sum(vals) <= 1e-12: - return 1.0 - - try: - alpha = float(alpha) - except Exception: - alpha = 0.8 - alpha = min(max(alpha, 0.0), 1.0) - - vals_sorted = np.sort(vals)[::-1] - cum_sum = np.cumsum(vals_sorted) - threshold = float(alpha) * np.sum(vals_sorted) - idx = np.searchsorted(cum_sum, threshold) - return float(min(total_features, idx + 1) / total_features) - except Exception as exc: - logger.warning("Could not compute alpha score") - logger.warning(exc) + attributions = (inputs - baseline) * total_gradients / float(steps) + importances = torch.abs(attributions).mean(dim=0) + + if importances.ndim == 3: + # For RGB images, keep one importance value per spatial position. + importances = importances.mean(dim=0) + + return _clean_importances(importances.detach().cpu().numpy()) + + +def _clean_importances(importances): + importances = np.asarray(importances, dtype=float).reshape(-1) + importances = np.nan_to_num(importances, nan=0.0, posinf=0.0, neginf=0.0) + return np.maximum(importances, 0.0) + + +def _alpha_score_from_values(vals, alpha=0.8): + # Fraction of features needed to explain alpha of the attribution mass; lower is better. + vals = np.asarray(vals, dtype=float).reshape(-1) + vals = np.nan_to_num(vals, nan=0.0, posinf=0.0, neginf=0.0) + vals = np.maximum(vals, 0.0) + total_features = len(vals) + if total_features == 0 or np.sum(vals) <= 1e-12: return 1.0 + try: + alpha = float(alpha) + except Exception: + alpha = 0.8 + alpha = min(max(alpha, 0.0), 1.0) + + vals_sorted = np.sort(vals)[::-1] + cum_sum = np.cumsum(vals_sorted) + threshold = float(alpha) * np.sum(vals_sorted) + idx = np.searchsorted(cum_sum, threshold) + return float(min(total_features, idx + 1) / total_features) + -def _get_spread_base(model, test_sample, divergence=True): - vals = _get_feature_importances(model, test_sample) +def _spread_base_from_values(vals, divergence=True): + # Entropy ratio measures spread; JS divergence measures distance from uniform attribution. + vals = np.asarray(vals, dtype=float).reshape(-1) tol = 1e-8 if len(vals) == 0 or np.sum(vals) < tol: @@ -324,44 +184,8 @@ def _get_spread_base(model, test_sample, divergence=True): return float(np.clip(metric, 0.0, 1.0)) -def get_spread_ratio(model, test_sample): - """ - Computes spread ratio from global feature importances. - """ - try: - return _get_spread_base(model, test_sample, divergence=False) - except Exception as exc: - logger.warning("Could not compute spread ratio") - logger.warning(exc) - return 1.0 - - -def get_spread_divergence(model, test_sample): - """ - Computes spread divergence from global feature importances. - """ - try: - return _get_spread_base(model, test_sample, divergence=True) - except Exception as exc: - logger.warning("Could not compute spread divergence") - logger.warning(exc) - return 0.0 - - def get_explainability_metrics_summary(model, test_dataloader, max_batches=4): - """ - Computes explainability metrics over multiple test batches and returns - their mean values. - - Args: - model (object): The model. - test_dataloader: Test dataloader providing batches. - max_batches (int): Maximum number of batches to use. - - Returns: - dict: Mean values for feature_importance_cv, alpha_score, - spread_ratio and spread_divergence. - """ + # Computes explainability metrics over multiple test batches and returns their mean values. summary = { "feature_importance_cv": 1.0, "alpha_score": 1.0, @@ -387,10 +211,12 @@ def get_explainability_metrics_summary(model, test_dataloader, max_batches=4): if batch_idx >= max_batches: break - fi_values.append(float(get_feature_importance_cv(model, test_sample))) - alpha_values.append(float(get_alpha_score(model, test_sample))) - spread_ratio_values.append(float(get_spread_ratio(model, test_sample))) - spread_divergence_values.append(float(get_spread_divergence(model, test_sample))) + # Compute attributions once per batch and derive all explainability metrics from them. + importances = _get_feature_importances(model, test_sample) + fi_values.append(float(_feature_importance_cv_from_values(importances))) + alpha_values.append(float(_alpha_score_from_values(importances))) + spread_ratio_values.append(float(_spread_base_from_values(importances, divergence=False))) + spread_divergence_values.append(float(_spread_base_from_values(importances, divergence=True))) except Exception as exc: logger.warning("Could not compute explainability metrics summary") logger.warning(exc) diff --git a/nebula/addons/trustworthiness/helpers/privacy.py b/nebula/addons/trustworthiness/helpers/privacy.py index f6ed327c1..b33c062f2 100644 --- a/nebula/addons/trustworthiness/helpers/privacy.py +++ b/nebula/addons/trustworthiness/helpers/privacy.py @@ -11,17 +11,7 @@ logger = logging.getLogger(__name__) def get_global_privacy_risk(dp, epsilon, n): - """ - Calculates the global privacy risk by epsilon and the number of clients. - - Args: - dp (bool): Indicates if differential privacy is used or not. - epsilon (int): The epsilon value. - n (int): The number of clients in the scenario. - - Returns: - float: The global privacy risk. - """ + # Calculates the global privacy risk by epsilon and the number of clients. try: epsilon = float(epsilon) @@ -36,17 +26,7 @@ def get_global_privacy_risk(dp, epsilon, n): def get_global_privacy_risk_dfl(dp, epsilon, n): - """ - Calculates the global privacy risk by epsilon and the number of clients. - - Args: - dp (bool): Indicates if differential privacy is used or not. - epsilon (int): The epsilon value. - n (int): The number of neighbours. - - Returns: - float: The global privacy risk. - """ + # Calculates the global privacy risk by epsilon and the number of clients for DFL. try: epsilon = float(epsilon) @@ -61,17 +41,7 @@ def get_global_privacy_risk_dfl(dp, epsilon, n): def _collect_per_sample_losses(model, dataloader, max_samples=5000): - """ - Compute per-sample cross-entropy losses for a dataloader. - - Args: - model (torch.nn.Module): The model to evaluate. - dataloader: DataLoader providing (samples, labels). - max_samples (int): Maximum number of samples to process. - - Returns: - np.ndarray: Losses per sample. - """ + # Compute per-sample cross-entropy losses for a dataloader. if not isinstance(model, torch.nn.Module) or dataloader is None: return np.array([]) @@ -110,7 +80,12 @@ def _collect_per_sample_losses(model, dataloader, max_samples=5000): logits = outputs[0] if isinstance(outputs, (tuple, list)) else outputs batch_losses = criterion(logits, labels) - losses.append(batch_losses.detach().cpu().numpy()) + batch_losses_np = batch_losses.detach().cpu().numpy() + batch_losses_np = batch_losses_np[np.isfinite(batch_losses_np)] + if batch_losses_np.size == 0: + continue + + losses.append(batch_losses_np) collected += int(batch_losses.shape[0]) if not losses: @@ -119,22 +94,8 @@ def _collect_per_sample_losses(model, dataloader, max_samples=5000): return np.concatenate(losses, axis=0) -def get_epsilon_star(model, train_dataloader, test_dataloader, max_samples=5000): - """ - Compute empirical epsilon* from train/test loss distributions. - - This follows the same core structure as privacy_metrics_core.epsilon_star, - adapted to PyTorch models and DataLoaders used in Nebula. - - Args: - model (torch.nn.Module): Model to evaluate. - train_dataloader: Training DataLoader. - test_dataloader: Test DataLoader. - max_samples (int): Maximum samples to evaluate per split. - - Returns: - float: Empirical epsilon* value. Returns 0.0 on failure. - """ +def get_epsilon_star(model, train_dataloader, test_dataloader, max_samples=5000, percentile=95): + # Compute empirical epsilon* from train/test loss distributions. try: loss_train = _collect_per_sample_losses(model, train_dataloader, max_samples=max_samples) loss_test = _collect_per_sample_losses(model, test_dataloader, max_samples=max_samples) @@ -147,9 +108,11 @@ def get_epsilon_star(model, train_dataloader, test_dataloader, max_samples=5000) fpr, tpr, _ = roc_curve(y_true, scores) - fpr = np.clip(fpr, 1e-10, 1 - 1e-10) - tpr = np.clip(tpr, 1e-10, 1 - 1e-10) - fnr = 1 - tpr + fpr_floor = 1.0 / len(loss_test) + fnr_floor = 1.0 / len(loss_train) + + fpr = np.clip(fpr, fpr_floor, 1 - fpr_floor) + fnr = np.clip(1 - tpr, fnr_floor, 1 - fnr_floor) delta = 1.0 / len(loss_train) if len(loss_train) > 0 else 1e-5 @@ -158,9 +121,12 @@ def get_epsilon_star(model, train_dataloader, test_dataloader, max_samples=5000) m3 = (fnr - delta) / (1 - fpr) m4 = (fpr - delta) / (1 - fnr) - epsilon_star_val = np.log( - np.nanmax(np.maximum.reduce([m1, m2, m3, m4, np.ones_like(m1)])) - ) + ratios = np.maximum.reduce([m1, m2, m3, m4, np.ones_like(m1)]) + ratios = ratios[np.isfinite(ratios)] + if ratios.size == 0: + return 0.0 + + epsilon_star_val = np.log(np.nanpercentile(ratios, percentile)) if np.isnan(epsilon_star_val) or np.isinf(epsilon_star_val): return 0.0 @@ -173,21 +139,7 @@ def get_epsilon_star(model, train_dataloader, test_dataloader, max_samples=5000) def get_mia_auc(model, train_dataloader, test_dataloader, max_samples=5000): - """ - Compute membership inference attack AUC using per-sample loss as the attack score. - - Lower loss suggests a sample is more likely to be a training member, so the - attack score is defined as negative loss. - - Args: - model (torch.nn.Module): Model to evaluate. - train_dataloader: Training DataLoader. - test_dataloader: Test DataLoader. - max_samples (int): Maximum samples to evaluate per split. - - Returns: - float: ROC-AUC of the loss-threshold membership attack. Returns 0.5 on failure. - """ + # Compute membership inference attack AUC using per-sample loss as the attack score. try: loss_train = _collect_per_sample_losses(model, train_dataloader, max_samples=max_samples) loss_test = _collect_per_sample_losses(model, test_dataloader, max_samples=max_samples) From e91ffb0d637f317dc1788a4922b54a17bec97a7a Mon Sep 17 00:00:00 2001 From: "Juan J." Date: Wed, 3 Jun 2026 13:33:46 +0200 Subject: [PATCH 58/66] Quality_model and fairness refactored, train accuracy and macro f1 score are obtained from training --- .../addons/trustworthiness/cfl_factsheet.py | 6 +- .../addons/trustworthiness/dfl_factsheet.py | 4 +- .../trustworthiness/factsheet_populators.py | 5 +- .../addons/trustworthiness/helpers/csv_io.py | 18 ++ .../trustworthiness/helpers/model_quality.py | 232 +++--------------- .../helpers/scenario_metrics.py | 35 ++- .../addons/trustworthiness/trustworthiness.py | 52 ++-- nebula/core/engine.py | 4 +- nebula/core/models/cifar10/resnet.py | 2 +- nebula/core/models/nebulamodel.py | 30 ++- nebula/core/nebulaevents.py | 10 +- nebula/core/network/messages.py | 4 +- nebula/core/pb/nebula.proto | 2 + nebula/core/pb/nebula_pb2.py | 12 +- nebula/core/training/lightning.py | 34 ++- 15 files changed, 193 insertions(+), 257 deletions(-) diff --git a/nebula/addons/trustworthiness/cfl_factsheet.py b/nebula/addons/trustworthiness/cfl_factsheet.py index 88eedab28..b8cbe104b 100755 --- a/nebula/addons/trustworthiness/cfl_factsheet.py +++ b/nebula/addons/trustworthiness/cfl_factsheet.py @@ -20,7 +20,6 @@ get_dp_global, get_elapsed_time, get_entropy_list, - get_participant_loss_accuracy, get_underfitting_score, ) from nebula.addons.trustworthiness.factsheet_common import ( @@ -104,7 +103,8 @@ def populate_factsheet_cfl( factsheet["performance"]["test_acc_avg"] = result_avg_loss_accuracy[1] test_acc_cv = get_cv(std=result_avg_loss_accuracy[2], mean=result_avg_loss_accuracy[1]) factsheet["fairness"]["test_acc_cv"] = 1 if test_acc_cv > 1 else test_acc_cv - _, participant_test_acc = get_participant_loss_accuracy(scenario_name, participant_idx) + factsheet["performance"]["test_macro_f1"] = result_avg_loss_accuracy[3] + factsheet["performance"]["train_accuracy"] = result_avg_loss_accuracy[4] # Compute CFL privacy risk from aggregate DP settings and client count. dp_enabled, dp_epsilon = get_dp_global(scenario_name) @@ -142,7 +142,7 @@ def populate_factsheet_cfl( model, train_loader, test_loader, - participant_test_acc, + factsheet["performance"]["test_acc_avg"], ) # Enrich CodeCarbon emissions with CPU/GPU benchmark metadata. diff --git a/nebula/addons/trustworthiness/dfl_factsheet.py b/nebula/addons/trustworthiness/dfl_factsheet.py index 2fb2fd115..d8f4a6afd 100644 --- a/nebula/addons/trustworthiness/dfl_factsheet.py +++ b/nebula/addons/trustworthiness/dfl_factsheet.py @@ -99,7 +99,9 @@ def populate_factsheet_dfl( factsheet["performance"]["test_acc"] = float(final_acc) # Load local communication and privacy values reported by the participant. - bytes_sent, bytes_recv, *_ = load_data_results_participant(scenario_name, participant_idx) + bytes_sent, bytes_recv, _, _, _, macro_f1, train_accuracy, *_ = load_data_results_participant(scenario_name, participant_idx) + factsheet["performance"]["test_macro_f1"] = macro_f1 + factsheet["performance"]["train_accuracy"] = train_accuracy factsheet["system"]["model_size"] = get_bytes_model(model) diff --git a/nebula/addons/trustworthiness/factsheet_populators.py b/nebula/addons/trustworthiness/factsheet_populators.py index 5ace6b034..25f5180bd 100644 --- a/nebula/addons/trustworthiness/factsheet_populators.py +++ b/nebula/addons/trustworthiness/factsheet_populators.py @@ -8,8 +8,6 @@ from nebula.addons.trustworthiness.helpers.model_quality import ( get_coefficient_of_variation, get_generalized_entropy_index, - get_macro_f1_score, - get_overfitting_score, get_theil_index, get_well_calibration_error, ) @@ -113,7 +111,6 @@ def populate_common_model_quality_metrics( test_sample, ): # Populate model quality, privacy, and fairness metrics shared by all profiles. - factsheet["performance"]["test_macro_f1"] = get_macro_f1_score(model, test_loader) # Privacy metrics derived from train/test behavior. factsheet["privacy"]["epsilon_star"] = get_epsilon_star(model, train_loader, test_loader) @@ -122,7 +119,7 @@ def populate_common_model_quality_metrics( factsheet["privacy"]["mia_auc_score"] = 1 - 2 * abs(factsheet["privacy"]["mia_auc"] - 0.5) # Fairness and calibration metrics expressed as inverse scores. - overfitting_value = get_overfitting_score(model, train_loader, test_accuracy) + overfitting_value = max(0.0, float(factsheet["performance"]["train_accuracy"]) - float(test_accuracy)) factsheet["fairness"]["inverse_overfitting"] = inverse_score(overfitting_value) well_calibration_error_value = get_well_calibration_error(model, test_loader) diff --git a/nebula/addons/trustworthiness/helpers/csv_io.py b/nebula/addons/trustworthiness/helpers/csv_io.py index 40bd7fda0..c924fc887 100644 --- a/nebula/addons/trustworthiness/helpers/csv_io.py +++ b/nebula/addons/trustworthiness/helpers/csv_io.py @@ -16,6 +16,8 @@ "accuracy", "loss", "val_accuracy", + "macro_f1", + "train_accuracy", "dp_enabled", "dp_epsilon", ] @@ -30,6 +32,8 @@ "model_size", "local_entropy", "val_accuracy", + "macro_f1", + "train_accuracy", "dp_enabled", "dp_epsilon", ] @@ -135,6 +139,8 @@ def load_data_results_participant(experiment_name: str, participant_id: int | st row = _read_first_csv_row( _trustworthiness_path(experiment_name, f"data_results_{participant_id}.csv") ) + macro_f1 = row["macro_f1"] or 0.0 + train_accuracy = row["train_accuracy"] or 0.0 return ( int(float(row["bytes_sent"])), @@ -142,6 +148,8 @@ def load_data_results_participant(experiment_name: str, participant_id: int | st float(row["accuracy"]), float(row["loss"]), float(row["val_accuracy"]), + float(macro_f1), + float(train_accuracy), _to_bool(row["dp_enabled"]), float(row["dp_epsilon"]), ) @@ -185,6 +193,8 @@ def save_trustworthiness_reports_csv( "model_size": report["model_size"], "local_entropy": report["local_entropy"], "val_accuracy": report["val_accuracy"], + "macro_f1": report["macro_f1"], + "train_accuracy": report["train_accuracy"], "dp_enabled": report["dp_enabled"], "dp_epsilon": report["dp_epsilon"], } @@ -231,6 +241,8 @@ def save_results_csv_cfl( model_size: int, local_entropy: float, val_accuracy: float, + macro_f1: float, + train_accuracy: float, dp_enabled: bool, dp_epsilon: float, ): @@ -248,6 +260,8 @@ def save_results_csv_cfl( "model_size": model_size, "local_entropy": local_entropy, "val_accuracy": val_accuracy, + "macro_f1": macro_f1, + "train_accuracy": train_accuracy, "dp_enabled": dp_enabled, "dp_epsilon": dp_epsilon, }, @@ -296,6 +310,8 @@ def save_results_csv( accuracy: float, loss: float, val_accuracy: float, + macro_f1: float, + train_accuracy: float, dp_enabled: bool, dp_epsilon: float, ): @@ -310,6 +326,8 @@ def save_results_csv( "accuracy": accuracy, "loss": loss, "val_accuracy": val_accuracy, + "macro_f1": macro_f1, + "train_accuracy": train_accuracy, "dp_enabled": dp_enabled, "dp_epsilon": dp_epsilon, }, diff --git a/nebula/addons/trustworthiness/helpers/model_quality.py b/nebula/addons/trustworthiness/helpers/model_quality.py index 0b87937fe..4887f5719 100644 --- a/nebula/addons/trustworthiness/helpers/model_quality.py +++ b/nebula/addons/trustworthiness/helpers/model_quality.py @@ -3,105 +3,16 @@ import numpy as np import torch -from sklearn.metrics import f1_score logger = logging.getLogger(__name__) -def _get_model_accuracy(model, dataloader): - """ - Calculates model accuracy over a dataloader. - - Args: - model (torch.nn.Module): Model to evaluate. - dataloader (DataLoader): Dataloader with (x, y) batches. - - Returns: - float: Accuracy in [0, 1]. - """ - if not isinstance(model, torch.nn.Module): - logger.warning("Model is not a torch.nn.Module") - return 0.0 - - try: - device = next(model.parameters()).device - except Exception: - device = torch.device("cpu") - - model.eval() - correct = 0 - total = 0 - - with torch.no_grad(): - for x, y in dataloader: - x = x.to(device) - y = y.to(device) - - out = model(x) - logits = out[0] if isinstance(out, (tuple, list)) else out - preds = logits.argmax(dim=1) - - correct += (preds == y).sum().item() - total += y.size(0) - - return correct / total if total > 0 else 0.0 - - -def get_macro_f1_score(model, dataloader): - """ - Calculates macro F1 score over a dataloader. - - Args: - model (torch.nn.Module): Model to evaluate. - dataloader (DataLoader): Dataloader with (x, y) batches. - - Returns: - float: Macro F1 score in [0, 1]. - """ - if not isinstance(model, torch.nn.Module): - logger.warning("Model is not a torch.nn.Module") - return 0.0 - - try: - device = next(model.parameters()).device - except Exception: - device = torch.device("cpu") - - model.eval() - y_true = [] - y_pred = [] - - with torch.no_grad(): - for x, y in dataloader: - x = x.to(device) - y = y.to(device) - - out = model(x) - logits = out[0] if isinstance(out, (tuple, list)) else out - preds = logits.argmax(dim=1) - - y_true.extend(y.detach().cpu().numpy().tolist()) - y_pred.extend(preds.detach().cpu().numpy().tolist()) - - if not y_true: - return 0.0 - - return float(f1_score(y_true, y_pred, average="macro", zero_division=0)) - - def _extract_model_logits(model_output): - """ - Normalize the output returned by a model forward pass into a logits tensor. - - Some models may return tuples/lists; for trust metrics we always consume the - first element as the classification output. - """ + # Normalize the output returned by a model forward pass into a logits tensor. return model_output[0] if isinstance(model_output, (tuple, list)) else model_output def _prepare_class_targets(y): - """ - Convert different target representations into a flat class-index tensor. - """ + # Convert different target representations into a flat class-index tensor. if not torch.is_tensor(y): y = torch.as_tensor(y) @@ -115,14 +26,7 @@ def _prepare_class_targets(y): def _logits_to_probabilities(logits): - """ - Convert model outputs into a probability matrix of shape (N, C). - - Supports: - - multiclass logits/log-probabilities with shape (N, C) - - binary logits with shape (N,) or (N, 1) - - already-normalized probability matrices - """ + # Convert model outputs into a probability matrix of shape (N, C). if not torch.is_tensor(logits): logits = torch.as_tensor(logits) @@ -151,29 +55,20 @@ def _logits_to_probabilities(logits): def _collect_classification_statistics(model, dataloader): - """ - Collect prediction statistics required by calibration and inequality metrics. - - Returns: - tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]: - predicted labels, true labels, prediction confidences, correctness flags, - and probability assigned to the true class. - """ + # Collect prediction statistics required by calibration and inequality metrics. if not isinstance(model, torch.nn.Module): logger.warning("Model is not a torch.nn.Module") empty = np.array([], dtype=float) - return empty, empty, empty, empty, empty + return empty, empty, empty try: device = next(model.parameters()).device except Exception: device = torch.device("cpu") - preds_all = [] - targets_all = [] - confidences_all = [] - correct_all = [] - true_probs_all = [] + confidences = [] + correct = [] + true_probs = [] model.eval() with torch.no_grad(): @@ -188,19 +83,18 @@ def _collect_classification_statistics(model, dataloader): x = x.to(device) y = _prepare_class_targets(y).to(device) - out = model(x) - logits = _extract_model_logits(out) - probs = _logits_to_probabilities(logits) + # Metrics consume probabilities even when the model returns raw logits + # or wraps the classification output in a tuple/list. + probs = _logits_to_probabilities(_extract_model_logits(model(x))) if probs.ndim != 2 or probs.size(0) == 0: continue - if y.numel() != probs.size(0): - n = min(int(y.numel()), int(probs.size(0))) - if n == 0: - continue - y = y[:n] - probs = probs[:n] + n = min(int(y.numel()), int(probs.size(0))) + if n == 0: + continue + y = y[:n] + probs = probs[:n] valid_mask = (y >= 0) & (y < probs.size(1)) if not torch.any(valid_mask): @@ -209,74 +103,39 @@ def _collect_classification_statistics(model, dataloader): y = y[valid_mask] probs = probs[valid_mask] + # Confidence is the predicted-class probability. true_probs is the + # probability assigned to the actual class, used as a continuous benefit. conf, preds = probs.max(dim=1) - true_probs = probs.gather(1, y.view(-1, 1)).squeeze(1) - correct = preds.eq(y).float() + confidences.append(conf.cpu()) + correct.append(preds.eq(y).float().cpu()) + true_probs.append(probs.gather(1, y.view(-1, 1)).squeeze(1).cpu()) - preds_all.extend(preds.detach().cpu().numpy().tolist()) - targets_all.extend(y.detach().cpu().numpy().tolist()) - confidences_all.extend(conf.detach().cpu().numpy().tolist()) - correct_all.extend(correct.detach().cpu().numpy().tolist()) - true_probs_all.extend(true_probs.detach().cpu().numpy().tolist()) + if not confidences: + empty = np.array([], dtype=float) + return empty, empty, empty return ( - np.asarray(preds_all, dtype=int), - np.asarray(targets_all, dtype=int), - np.asarray(confidences_all, dtype=float), - np.asarray(correct_all, dtype=float), - np.asarray(true_probs_all, dtype=float), + torch.cat(confidences).numpy(), + torch.cat(correct).numpy(), + torch.cat(true_probs).numpy(), ) -def get_overfitting_score(model, train_dataloader, test_accuracy): - """ - Calculates overfitting as the positive train-test accuracy gap. - - Args: - model (torch.nn.Module): Model to evaluate on training data. - train_dataloader (DataLoader): Training dataloader. - test_accuracy (float): Test accuracy in [0, 1]. - - Returns: - float: Positive train-test accuracy gap. - """ - try: - train_accuracy = _get_model_accuracy(model, train_dataloader) - return max(0.0, float(train_accuracy) - float(test_accuracy)) - except Exception as exc: - logger.warning("Could not compute overfitting score") - logger.warning(exc) - return 0.0 - - def get_well_calibration_error(model, test_dataloader, n_bins=10): - """ - Calculates a well-calibration error style metric using prediction confidence. - - For multiclass models, confidence is taken as the max softmax probability and - the observed outcome is whether the prediction is correct. - - Args: - model (torch.nn.Module): Model to evaluate. - test_dataloader (DataLoader): Test dataloader. - n_bins (int): Number of quantile bins. - - Returns: - float: Calibration error in [0, 1] when computation succeeds. - """ + # Calculates a well-calibration error style metric using prediction confidence. if not isinstance(model, torch.nn.Module): logger.warning("Model is not a torch.nn.Module") - return 0.0 + return 1.0 try: n_bins = max(2, int(n_bins)) except Exception: n_bins = 10 - _, _, confidences, correct, _ = _collect_classification_statistics(model, test_dataloader) + confidences, correct, _ = _collect_classification_statistics(model, test_dataloader) if len(confidences) == 0 or len(correct) == 0: - return 0.0 + return 1.0 confidences = np.clip(np.asarray(confidences, dtype=float), 0.0, 1.0) correct = np.clip(np.asarray(correct, dtype=float), 0.0, 1.0) @@ -285,6 +144,7 @@ def get_well_calibration_error(model, test_dataloader, n_bins=10): ece = 0.0 total = float(len(confidences)) + # ECE compares empirical accuracy and average confidence within each bin. for idx in range(n_bins): left = bin_edges[idx] right = bin_edges[idx + 1] @@ -305,32 +165,20 @@ def get_well_calibration_error(model, test_dataloader, n_bins=10): def get_generalized_entropy_index(model, test_dataloader, alpha=2): - """ - Calculates generalized entropy index from model predictions. - - Args: - model (torch.nn.Module): Model to evaluate. - test_dataloader (DataLoader): Test dataloader. - alpha (float): GEI alpha parameter. - - Returns: - float: Generalized entropy index value. - """ + # Calculates generalized entropy index from model predictions. try: - _, _, _, _, true_class_probs = _collect_classification_statistics(model, test_dataloader) + _, _, true_class_probs = _collect_classification_statistics(model, test_dataloader) if len(true_class_probs) == 0: return 0.0 - # Use the probability assigned to the true class as a continuous, positive - # benefit. This works consistently for multiclass neural models on both - # images and tabular data, and avoids collapsing the metric to a coarse - # correct/incorrect indicator. eps = 1e-12 b = np.clip(np.asarray(true_class_probs, dtype=float), eps, 1.0) mu = float(np.mean(b)) if mu <= 0: return 0.0 + # GEI measures dispersion around the mean benefit. Lower values mean the + # model gives more even true-class confidence across samples. ratio = np.clip(b / mu, eps, None) if alpha == 0: @@ -352,16 +200,12 @@ def get_generalized_entropy_index(model, test_dataloader, alpha=2): def get_theil_index(model, test_dataloader): - """ - Convenience wrapper for generalized entropy index with alpha=1. - """ + # Convenience wrapper for generalized entropy index with alpha=1. return get_generalized_entropy_index(model, test_dataloader, alpha=1) def get_coefficient_of_variation(model, test_dataloader): - """ - Calculates coefficient of variation from GEI(alpha=2). - """ + # Calculates coefficient of variation from GEI(alpha=2). try: gei = get_generalized_entropy_index(model, test_dataloader, alpha=2) return float(np.sqrt(2 * gei)) diff --git a/nebula/addons/trustworthiness/helpers/scenario_metrics.py b/nebula/addons/trustworthiness/helpers/scenario_metrics.py index 1d1f35615..081c3db39 100644 --- a/nebula/addons/trustworthiness/helpers/scenario_metrics.py +++ b/nebula/addons/trustworthiness/helpers/scenario_metrics.py @@ -69,12 +69,18 @@ def _find_participant_row_by_int_id(data, participant_id): def _client_count(data): # Global CSVs include the server row, so client averages exclude one row. - return max(1, len(data) - 1) + return len(_client_rows(data)) + + +def _client_rows(data): + # CFL writes client reports first and appends the server row last. + return data.iloc[:-1] if len(data) > 1 else data def _mean_client_column(data, column_name): # Average a global metric across clients while keeping the historical server-row exclusion. - return data[column_name].sum() / _client_count(data) + clients = _client_rows(data) + return clients[column_name].sum() / max(1, len(clients)) def get_bytes_model(model): @@ -99,14 +105,17 @@ def get_bytes_sent_recv(scenario_name): def get_avg_loss_accuracy(scenario_name): - # Return client-average test loss, test accuracy and accuracy standard deviation. + # Return client-average test loss, accuracy, accuracy std, macro F1 and train accuracy. data = _read_global_results(scenario_name) + clients = _client_rows(data) avg_loss = _mean_client_column(data, "loss") avg_accuracy = _mean_client_column(data, "accuracy") - std_accuracy = statistics.stdev(data["accuracy"]) if len(data) > 1 else 0.0 + std_accuracy = statistics.stdev(clients["accuracy"]) if len(clients) > 1 else 0.0 + avg_macro_f1 = _mean_client_column(data, "macro_f1") + avg_train_accuracy = _mean_client_column(data, "train_accuracy") - return avg_loss, avg_accuracy, std_accuracy + return avg_loss, avg_accuracy, std_accuracy, avg_macro_f1, avg_train_accuracy def get_underfitting_score(scenario_name, participant_id): @@ -137,28 +146,30 @@ def get_dp_local(scenario_name, participant_id): def get_dp_global(scenario_name): # Return CFL DP settings, averaging epsilon across client rows when DP is enabled. data = _read_global_results(scenario_name) + clients = _client_rows(data) - if data["dp_enabled"].iloc[0] == False: + if clients["dp_enabled"].iloc[0] == False: return False, 0.0 return True, _mean_client_column(data, "dp_epsilon") def get_avg_class_imbalance_model_size(scenario_name): - # Return average class imbalance and model size across all global result rows. + # Return average class imbalance and model size across client rows. data = _read_global_results(scenario_name) - number_files = len(data) + clients = _client_rows(data) + number_files = max(1, len(clients)) - avg_class_imbalance = data["class_imbalance"].sum() / number_files - avg_model_size = data["model_size"].sum() / number_files + avg_class_imbalance = clients["class_imbalance"].sum() / number_files + avg_model_size = clients["model_size"].sum() / number_files return avg_class_imbalance, avg_model_size def get_entropy_list(scenario_name): - # Return local entropy values so callers can normalize the distribution. + # Return client entropy values so callers can normalize the distribution. data = _read_global_results(scenario_name) - return data["local_entropy"].tolist() + return _client_rows(data)["local_entropy"].tolist() def stop_emissions_tracking_and_save( diff --git a/nebula/addons/trustworthiness/trustworthiness.py b/nebula/addons/trustworthiness/trustworthiness.py index 1996171ba..39778c22c 100644 --- a/nebula/addons/trustworthiness/trustworthiness.py +++ b/nebula/addons/trustworthiness/trustworthiness.py @@ -69,8 +69,8 @@ def get_sample_size(self) -> float: raise NotImplementedError @abstractmethod - def get_metrics(self) -> tuple[float, float]: - # Return the latest test loss and accuracy. + def get_metrics(self) -> tuple[float, float, float]: + # Return the latest test loss, accuracy and macro F1. raise NotImplementedError @abstractmethod @@ -93,8 +93,10 @@ def __init__(self, engine: Engine, idx, trust_files_route, workload: str, role_l self._sample_size = sample_size self._current_loss = None self._current_accuracy = None + self._current_macro_f1 = None self._current_val_loss = None self._current_val_accuracy = None + self._current_train_accuracy = None self._experiment_name = "" self._per_round = None self._role_label = role_label @@ -134,11 +136,11 @@ def get_sample_size(self): def get_metrics(self): # Return the latest test metrics observed through events. - return (self._current_loss, self._current_accuracy) + return (self._current_loss, self._current_accuracy, self._current_macro_f1) def get_validation_metrics(self): - # Return the latest validation metrics observed through events. - return (self._current_val_loss, self._current_val_accuracy) + # Return the latest validation metrics and train accuracy observed through events. + return (self._current_val_loss, self._current_val_accuracy, self._current_train_accuracy) def _is_reputation_enabled(self) -> bool: # Read the reputation toggle from the participant defense config. @@ -230,18 +232,20 @@ async def _process_aggregation_event(self, age: AggregationEvent): async def _process_test_metrics_event(self, tme: TestMetricsEvent): # Cache final test metrics and forward them to per-round trust metrics. - cur_loss, cur_acc = await tme.get_event_data() + cur_loss, cur_acc, cur_macro_f1 = await tme.get_event_data() if cur_loss is not None and cur_acc is not None: self._current_loss, self._current_accuracy = cur_loss, cur_acc + self._current_macro_f1 = cur_macro_f1 if self._per_round is not None: await self._per_round.on_test_metrics(self._engine, float(cur_loss), float(cur_acc)) async def _process_validation_metrics_event(self, vme: ValidationMetricsEvent): # Cache final validation metrics for final trustworthiness outputs. - cur_loss, cur_acc = await vme.get_event_data() + cur_loss, cur_acc, train_acc = await vme.get_event_data() if cur_loss is not None and cur_acc is not None: self._current_val_loss, self._current_val_accuracy = cur_loss, cur_acc + self._current_train_accuracy = train_acc class TrustWorkloadTrainer(BaseTrustWorkload): @@ -312,7 +316,7 @@ async def _send_cfl_trustworthiness_report(self, experiment_name: str): def _build_cfl_trustworthiness_report(self, experiment_name: str) -> dict: # Load local metrics and shape them as a trustworthiness message payload. - bytes_sent, bytes_recv, accuracy, loss, val_accuracy, dp_enabled, dp_epsilon = load_data_results_participant( + bytes_sent, bytes_recv, accuracy, loss, val_accuracy, macro_f1, train_accuracy, dp_enabled, dp_epsilon = load_data_results_participant( experiment_name, self._idx, ) @@ -340,6 +344,8 @@ def _build_cfl_trustworthiness_report(self, experiment_name: str) -> dict: "model_size": get_bytes_model(self._engine.trainer.model), "local_entropy": get_local_entropy(self._idx, experiment_name), "val_accuracy": val_accuracy, + "macro_f1": macro_f1, + "train_accuracy": train_accuracy, "dp_enabled": dp_enabled, "dp_epsilon": dp_epsilon, } @@ -349,7 +355,7 @@ def _log_cfl_trustworthiness_report(self, server_addr: str, report: dict): logging.info( "[TW SEND] dest=%s node_id=%s bytes_sent=%s bytes_recv=%s " "accuracy=%s loss=%s role=%s energy_grid=%s emissions=%s workload=%s " - "cpu_model=%s gpu_model=%s cpu_used=%s gpu_used=%s energy_consumed=%s sample_size=%s class_imbalance=%s model_size=%s local_entropy=%s val_accuracy=%s dp_enabled=%s dp_epsilon=%s", + "cpu_model=%s gpu_model=%s cpu_used=%s gpu_used=%s energy_consumed=%s sample_size=%s class_imbalance=%s model_size=%s local_entropy=%s val_accuracy=%s dp_enabled=%s dp_epsilon=%s macro_f1=%s train_accuracy=%s", server_addr, str(self._idx), report["bytes_sent"], @@ -372,6 +378,8 @@ def _log_cfl_trustworthiness_report(self, server_addr: str, report: dict): report["val_accuracy"], report["dp_enabled"], report["dp_epsilon"], + report["macro_f1"], + report["train_accuracy"], ) async def _finish_trustscores_exchange(self, federation, trust_config, experiment_name): @@ -840,7 +848,7 @@ def _save_trustworthiness_reports_once(self): async def _save_local_server_report_and_generate_factsheet(self, trust_config, experiment_name): # Add the server's own local report and generate final trust artifacts. - bytes_sent, bytes_recv, _, _, val_accuracy, dp_enabled, dp_epsilon = load_data_results_participant( + bytes_sent, bytes_recv, _, _, val_accuracy, _, _, dp_enabled, dp_epsilon = load_data_results_participant( self._experiment_name, self._idx, ) @@ -859,7 +867,7 @@ async def _save_local_server_report_and_generate_factsheet(self, trust_config, e model_size = get_bytes_model(self._engine.trainer.model) local_entropy = get_local_entropy(self._idx, experiment_name) - save_results_csv_cfl(self._experiment_name, self._idx, bytes_sent, bytes_recv, 0, 0, class_imbalance, model_size, local_entropy, val_accuracy, dp_enabled, dp_epsilon) + save_results_csv_cfl(self._experiment_name, self._idx, bytes_sent, bytes_recv, 0, 0, class_imbalance, model_size, local_entropy, val_accuracy, 0, 0, dp_enabled, dp_epsilon) save_emissions_csv_cfl(self._experiment_name, self._idx, role, energy_grid, emissions, workload, cpu_model, gpu_model, cpu_used, gpu_used, energy_consumed, sample_size) await self._generate_factsheet(trust_config, experiment_name) @@ -887,7 +895,9 @@ async def register_trustworthiness_report(self, source, message): "local_entropy": message.local_entropy, "val_accuracy": message.val_accuracy, "dp_enabled": message.dp_enabled, - "dp_epsilon": message.dp_epsilon + "dp_epsilon": message.dp_epsilon, + "macro_f1": message.macro_f1, + "train_accuracy": message.train_accuracy, } logging.info( @@ -987,8 +997,8 @@ async def _process_experiment_finish_event(self, efe: ExperimentFinishEvent): await self.tw.finish_experiment_role_pre_actions() - last_loss, last_accuracy = self.tw.get_metrics() - _, last_val_accuracy = self.tw.get_validation_metrics() + last_loss, last_accuracy, last_macro_f1 = self.tw.get_metrics() + _, last_val_accuracy, last_train_accuracy = self.tw.get_validation_metrics() if last_val_accuracy is None: last_val_accuracy = 0.0 @@ -1008,7 +1018,19 @@ async def _process_experiment_finish_event(self, efe: ExperimentFinishEvent): sample_size = self.tw.get_sample_size() # Final operations - save_results_csv(self._experiment_name, self._idx, bytes_sent, bytes_recv, last_accuracy, last_loss, last_val_accuracy, dp_enabled, dp_epsilon) + save_results_csv( + self._experiment_name, + self._idx, + bytes_sent, + bytes_recv, + last_accuracy, + last_loss, + last_val_accuracy, + last_macro_f1, + last_train_accuracy, + dp_enabled, + dp_epsilon, + ) stop_emissions_tracking_and_save(self._tracker, self._trust_dir_files, f'emissions_{self._idx}.csv', self._role.value, workload, sample_size, self._idx) await self.tw.finish_experiment_role_post_actions(self._trust_config, self._experiment_name) diff --git a/nebula/core/engine.py b/nebula/core/engine.py index cb05c57db..831abf549 100644 --- a/nebula/core/engine.py +++ b/nebula/core/engine.py @@ -618,7 +618,9 @@ async def _trustworthiness_report_callback(self, source, message): "local_entropy": message.local_entropy, "val_accuracy": message.val_accuracy, "dp_enabled": message.dp_enabled, - "dp_epsilon": message.dp_epsilon + "dp_epsilon": message.dp_epsilon, + "macro_f1": message.macro_f1, + "train_accuracy": message.train_accuracy, } logging.info(f"handle_trustworthiness_message | Trigger | {report}") diff --git a/nebula/core/models/cifar10/resnet.py b/nebula/core/models/cifar10/resnet.py index a0b6d1f15..d2da13f3b 100755 --- a/nebula/core/models/cifar10/resnet.py +++ b/nebula/core/models/cifar10/resnet.py @@ -47,7 +47,7 @@ def __init__( MulticlassAccuracy(num_classes=num_classes), MulticlassPrecision(num_classes=num_classes), MulticlassRecall(num_classes=num_classes), - MulticlassF1Score(num_classes=num_classes), + MulticlassF1Score(num_classes=num_classes, average="macro"), ]) self.train_metrics = metrics.clone(prefix="Train/") self.val_metrics = metrics.clone(prefix="Validation/") diff --git a/nebula/core/models/nebulamodel.py b/nebula/core/models/nebulamodel.py index 3a270ae88..a6557fed6 100755 --- a/nebula/core/models/nebulamodel.py +++ b/nebula/core/models/nebulamodel.py @@ -83,10 +83,18 @@ def log_metrics_end(self, phase): f"{phase}/{key.replace('Multiclass', '').split('/')[-1]}": value.detach() for key, value in output.items() } + output_values = { + key: float(value.detach().cpu().item()) for key, value in output.items() + } + + if phase == "Train": + self._latest_train_metrics = output_values + if phase == "Validation": - self._latest_validation_metrics = { - key: float(value.detach().cpu().item()) for key, value in output.items() - } + self._latest_validation_metrics = output_values + + if phase in {"Test", "Test (Local)"}: + self._latest_test_metrics = output_values if phase == "Train" and self._train_extra_metrics: output.update({ @@ -180,7 +188,7 @@ def __init__( MulticlassAccuracy(num_classes=num_classes), MulticlassPrecision(num_classes=num_classes), MulticlassRecall(num_classes=num_classes), - MulticlassF1Score(num_classes=num_classes), + MulticlassF1Score(num_classes=num_classes, average="macro"), ]) self.train_metrics = metrics.clone(prefix="Train/") self.val_metrics = metrics.clone(prefix="Validation/") @@ -212,7 +220,9 @@ def __init__( self._current_loss = -1 self._optimizer = None self._optimizer_override = None + self._latest_train_metrics = {} self._latest_validation_metrics = {} + self._latest_test_metrics = {} self._train_extra_metrics = {} # DP trainers update these fields after querying the Opacus accountant. @@ -293,6 +303,18 @@ def get_loss(self): def get_latest_validation_metrics(self): return self._latest_validation_metrics + def get_latest_train_metrics(self): + return self._latest_train_metrics + + def get_latest_test_metrics(self): + return self._latest_test_metrics + + def get_latest_train_accuracy(self): + return self._latest_train_metrics.get("Train/Accuracy") + + def get_latest_test_macro_f1(self): + return self._latest_test_metrics.get("Test (Local)/F1Score") + def modify_learning_rate(self, new_lr): logging.info(f"Modifiying | learning rate, new value: {new_lr}") self.learning_rate = new_lr diff --git a/nebula/core/nebulaevents.py b/nebula/core/nebulaevents.py index ecdd482da..583f8facd 100644 --- a/nebula/core/nebulaevents.py +++ b/nebula/core/nebulaevents.py @@ -464,24 +464,26 @@ async def get_event_data(self): return (self.latitude, self.longitude) class TestMetricsEvent(AddonEvent): - def __init__(self, loss, accuracy): + def __init__(self, loss, accuracy, macro_f1=None): self._loss = loss self._accuracy = accuracy + self._macro_f1 = macro_f1 def __str__(self): return "TestMetricsEvent" async def get_event_data(self): - return (self._loss, self._accuracy) + return (self._loss, self._accuracy, self._macro_f1) class ValidationMetricsEvent(AddonEvent): - def __init__(self, loss, accuracy): + def __init__(self, loss, accuracy, train_accuracy=None): self._loss = loss self._accuracy = accuracy + self._train_accuracy = train_accuracy def __str__(self): return "ValidationMetricsEvent" async def get_event_data(self): - return (self._loss, self._accuracy) + return (self._loss, self._accuracy, self._train_accuracy) diff --git a/nebula/core/network/messages.py b/nebula/core/network/messages.py index 9963ff6ea..29e7088bc 100644 --- a/nebula/core/network/messages.py +++ b/nebula/core/network/messages.py @@ -134,7 +134,9 @@ def _define_message_templates(self): "local_entropy", "val_accuracy", "dp_enabled", - "dp_epsilon" + "dp_epsilon", + "macro_f1", + "train_accuracy" ], "defaults": {}, }, diff --git a/nebula/core/pb/nebula.proto b/nebula/core/pb/nebula.proto index c05d6131f..13b0fa74d 100755 --- a/nebula/core/pb/nebula.proto +++ b/nebula/core/pb/nebula.proto @@ -185,6 +185,8 @@ message TrustworthinessMessage { float val_accuracy = 20; bool dp_enabled = 21; float dp_epsilon = 22; + double macro_f1 = 23; + double train_accuracy = 24; } message TrustscoresMessage { diff --git a/nebula/core/pb/nebula_pb2.py b/nebula/core/pb/nebula_pb2.py index bb470f06b..bc06160a2 100644 --- a/nebula/core/pb/nebula_pb2.py +++ b/nebula/core/pb/nebula_pb2.py @@ -13,7 +13,7 @@ -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0cnebula.proto\x12\x06nebula\"\xa6\x06\n\x07Wrapper\x12\x0e\n\x06source\x18\x01 \x01(\t\x12\x35\n\x11\x64iscovery_message\x18\x02 \x01(\x0b\x32\x18.nebula.DiscoveryMessageH\x00\x12\x31\n\x0f\x63ontrol_message\x18\x03 \x01(\x0b\x32\x16.nebula.ControlMessageH\x00\x12\x37\n\x12\x66\x65\x64\x65ration_message\x18\x04 \x01(\x0b\x32\x19.nebula.FederationMessageH\x00\x12-\n\rmodel_message\x18\x05 \x01(\x0b\x32\x14.nebula.ModelMessageH\x00\x12\x37\n\x12\x63onnection_message\x18\x06 \x01(\x0b\x32\x19.nebula.ConnectionMessageH\x00\x12\x33\n\x10response_message\x18\x07 \x01(\x0b\x32\x17.nebula.ResponseMessageH\x00\x12\x37\n\x12reputation_message\x18\x08 \x01(\x0b\x32\x19.nebula.ReputationMessageH\x00\x12\x33\n\x10\x64iscover_message\x18\t \x01(\x0b\x32\x17.nebula.DiscoverMessageH\x00\x12-\n\roffer_message\x18\n \x01(\x0b\x32\x14.nebula.OfferMessageH\x00\x12+\n\x0clink_message\x18\x0b \x01(\x0b\x32\x13.nebula.LinkMessageH\x00\x12\x41\n\x17trustworthiness_message\x18\x0c \x01(\x0b\x32\x1e.nebula.TrustworthinessMessageH\x00\x12\x39\n\x13trustscores_message\x18\r \x01(\x0b\x32\x1a.nebula.TrustscoresMessageH\x00\x12\x35\n\x11sdflmodel_message\x18\x0e \x01(\x0b\x32\x18.nebula.SdflmodelMessageH\x00\x12\x41\n\x17reputationtable_message\x18\x0f \x01(\x0b\x32\x1e.nebula.ReputationtableMessageH\x00\x42\t\n\x07message\"\x9e\x01\n\x10\x44iscoveryMessage\x12/\n\x06\x61\x63tion\x18\x01 \x01(\x0e\x32\x1f.nebula.DiscoveryMessage.Action\x12\x10\n\x08latitude\x18\x02 \x01(\x02\x12\x11\n\tlongitude\x18\x03 \x01(\x02\"4\n\x06\x41\x63tion\x12\x0c\n\x08\x44ISCOVER\x10\x00\x12\x0c\n\x08REGISTER\x10\x01\x12\x0e\n\nDEREGISTER\x10\x02\"\xd1\x01\n\x0e\x43ontrolMessage\x12-\n\x06\x61\x63tion\x18\x01 \x01(\x0e\x32\x1d.nebula.ControlMessage.Action\x12\x0b\n\x03log\x18\x02 \x01(\t\"\x82\x01\n\x06\x41\x63tion\x12\t\n\x05\x41LIVE\x10\x00\x12\x0c\n\x08OVERHEAD\x10\x01\x12\x0c\n\x08MOBILITY\x10\x02\x12\x0c\n\x08RECOVERY\x10\x03\x12\r\n\tWEAK_LINK\x10\x04\x12\x17\n\x13LEADERSHIP_TRANSFER\x10\x05\x12\x1b\n\x17LEADERSHIP_TRANSFER_ACK\x10\x06\"\xcd\x01\n\x11\x46\x65\x64\x65rationMessage\x12\x30\n\x06\x61\x63tion\x18\x01 \x01(\x0e\x32 .nebula.FederationMessage.Action\x12\x11\n\targuments\x18\x02 \x03(\t\x12\r\n\x05round\x18\x03 \x01(\x05\"d\n\x06\x41\x63tion\x12\x14\n\x10\x46\x45\x44\x45RATION_START\x10\x00\x12\x0e\n\nREPUTATION\x10\x01\x12\x1e\n\x1a\x46\x45\x44\x45RATION_MODELS_INCLUDED\x10\x02\x12\x14\n\x10\x46\x45\x44\x45RATION_READY\x10\x03\"A\n\x0cModelMessage\x12\x12\n\nparameters\x18\x01 \x01(\x0c\x12\x0e\n\x06weight\x18\x02 \x01(\x03\x12\r\n\x05round\x18\x03 \x01(\x05\"\xc7\x01\n\x10SdflmodelMessage\x12/\n\x06\x61\x63tion\x18\x01 \x01(\x0e\x32\x1f.nebula.SdflmodelMessage.Action\x12\x0e\n\x06target\x18\x02 \x01(\t\x12\x12\n\nparameters\x18\x03 \x01(\x0c\x12\x0e\n\x06weight\x18\x04 \x01(\x03\x12\r\n\x05round\x18\x05 \x01(\x05\x12\x0f\n\x07node_id\x18\x06 \x01(\t\".\n\x06\x41\x63tion\x12\x12\n\x0eTRAINER_UPDATE\x10\x00\x12\x10\n\x0cGLOBAL_MODEL\x10\x01\"\x8f\x01\n\x11\x43onnectionMessage\x12\x30\n\x06\x61\x63tion\x18\x01 \x01(\x0e\x32 .nebula.ConnectionMessage.Action\"H\n\x06\x41\x63tion\x12\x0b\n\x07\x43ONNECT\x10\x00\x12\x0e\n\nDISCONNECT\x10\x01\x12\x10\n\x0cLATE_CONNECT\x10\x02\x12\x0f\n\x0bRESTRUCTURE\x10\x03\"\x95\x01\n\x0f\x44iscoverMessage\x12.\n\x06\x61\x63tion\x18\x01 \x01(\x0e\x32\x1e.nebula.DiscoverMessage.Action\"R\n\x06\x41\x63tion\x12\x11\n\rDISCOVER_JOIN\x10\x00\x12\x12\n\x0e\x44ISCOVER_NODES\x10\x01\x12\x10\n\x0cLATE_CONNECT\x10\x02\x12\x0f\n\x0bRESTRUCTURE\x10\x03\"\xce\x01\n\x0cOfferMessage\x12+\n\x06\x61\x63tion\x18\x01 \x01(\x0e\x32\x1b.nebula.OfferMessage.Action\x12\x13\n\x0bn_neighbors\x18\x02 \x01(\x02\x12\x0c\n\x04loss\x18\x03 \x01(\x02\x12\x12\n\nparameters\x18\x04 \x01(\x0c\x12\x0e\n\x06rounds\x18\x05 \x01(\x05\x12\r\n\x05round\x18\x06 \x01(\x05\x12\x0e\n\x06\x65pochs\x18\x07 \x01(\x05\"+\n\x06\x41\x63tion\x12\x0f\n\x0bOFFER_MODEL\x10\x00\x12\x10\n\x0cOFFER_METRIC\x10\x01\"w\n\x0bLinkMessage\x12*\n\x06\x61\x63tion\x18\x01 \x01(\x0e\x32\x1a.nebula.LinkMessage.Action\x12\r\n\x05\x61\x64\x64rs\x18\x02 \x01(\t\"-\n\x06\x41\x63tion\x12\x0e\n\nCONNECT_TO\x10\x00\x12\x13\n\x0f\x44ISCONNECT_FROM\x10\x01\"\x89\x01\n\x11ReputationMessage\x12\x0f\n\x07node_id\x18\x01 \x01(\t\x12\r\n\x05score\x18\x02 \x01(\x02\x12\r\n\x05round\x18\x03 \x01(\x05\x12\x30\n\x06\x61\x63tion\x18\x04 \x01(\x0e\x32 .nebula.ReputationMessage.Action\"\x13\n\x06\x41\x63tion\x12\t\n\x05SHARE\x10\x00\"\xa3\x01\n\x16ReputationtableMessage\x12\x0f\n\x07node_id\x18\x01 \x01(\t\x12\r\n\x05round\x18\x02 \x01(\x05\x12\x1d\n\x15reputation_table_json\x18\x03 \x01(\t\x12\x35\n\x06\x61\x63tion\x18\x04 \x01(\x0e\x32%.nebula.ReputationtableMessage.Action\"\x13\n\x06\x41\x63tion\x12\t\n\x05TABLE\x10\x00\"#\n\x0fResponseMessage\x12\x10\n\x08response\x18\x01 \x01(\t\"\x80\x04\n\x16TrustworthinessMessage\x12\x35\n\x06\x61\x63tion\x18\x01 \x01(\x0e\x32%.nebula.TrustworthinessMessage.Action\x12\x0f\n\x07node_id\x18\x02 \x01(\t\x12\x12\n\nbytes_sent\x18\x03 \x01(\x03\x12\x12\n\nbytes_recv\x18\x04 \x01(\x03\x12\x10\n\x08\x61\x63\x63uracy\x18\x05 \x01(\x01\x12\x0c\n\x04loss\x18\x06 \x01(\x01\x12\x0c\n\x04role\x18\x07 \x01(\t\x12\x13\n\x0b\x65nergy_grid\x18\x08 \x01(\x01\x12\x11\n\temissions\x18\t \x01(\x01\x12\x10\n\x08workload\x18\n \x01(\t\x12\x11\n\tcpu_model\x18\x0b \x01(\t\x12\x11\n\tgpu_model\x18\x0c \x01(\t\x12\x10\n\x08\x63pu_used\x18\r \x01(\x08\x12\x10\n\x08gpu_used\x18\x0e \x01(\x08\x12\x17\n\x0f\x65nergy_consumed\x18\x0f \x01(\x01\x12\x13\n\x0bsample_size\x18\x10 \x01(\x05\x12\x17\n\x0f\x63lass_imbalance\x18\x11 \x01(\x02\x12\x12\n\nmodel_size\x18\x12 \x01(\x03\x12\x15\n\rlocal_entropy\x18\x13 \x01(\x02\x12\x14\n\x0cval_accuracy\x18\x14 \x01(\x02\x12\x12\n\ndp_enabled\x18\x15 \x01(\x08\x12\x12\n\ndp_epsilon\x18\x16 \x01(\x02\"\x14\n\x06\x41\x63tion\x12\n\n\x06REPORT\x10\x00\"\x88\x01\n\x12TrustscoresMessage\x12\x31\n\x06\x61\x63tion\x18\x01 \x01(\x0e\x32!.nebula.TrustscoresMessage.Action\x12\x0f\n\x07node_id\x18\x02 \x01(\t\x12\x19\n\x11trust_report_json\x18\x03 \x01(\t\"\x13\n\x06\x41\x63tion\x12\t\n\x05SHARE\x10\x00\x62\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0cnebula.proto\x12\x06nebula\"\xa6\x06\n\x07Wrapper\x12\x0e\n\x06source\x18\x01 \x01(\t\x12\x35\n\x11\x64iscovery_message\x18\x02 \x01(\x0b\x32\x18.nebula.DiscoveryMessageH\x00\x12\x31\n\x0f\x63ontrol_message\x18\x03 \x01(\x0b\x32\x16.nebula.ControlMessageH\x00\x12\x37\n\x12\x66\x65\x64\x65ration_message\x18\x04 \x01(\x0b\x32\x19.nebula.FederationMessageH\x00\x12-\n\rmodel_message\x18\x05 \x01(\x0b\x32\x14.nebula.ModelMessageH\x00\x12\x37\n\x12\x63onnection_message\x18\x06 \x01(\x0b\x32\x19.nebula.ConnectionMessageH\x00\x12\x33\n\x10response_message\x18\x07 \x01(\x0b\x32\x17.nebula.ResponseMessageH\x00\x12\x37\n\x12reputation_message\x18\x08 \x01(\x0b\x32\x19.nebula.ReputationMessageH\x00\x12\x33\n\x10\x64iscover_message\x18\t \x01(\x0b\x32\x17.nebula.DiscoverMessageH\x00\x12-\n\roffer_message\x18\n \x01(\x0b\x32\x14.nebula.OfferMessageH\x00\x12+\n\x0clink_message\x18\x0b \x01(\x0b\x32\x13.nebula.LinkMessageH\x00\x12\x41\n\x17trustworthiness_message\x18\x0c \x01(\x0b\x32\x1e.nebula.TrustworthinessMessageH\x00\x12\x39\n\x13trustscores_message\x18\r \x01(\x0b\x32\x1a.nebula.TrustscoresMessageH\x00\x12\x35\n\x11sdflmodel_message\x18\x0e \x01(\x0b\x32\x18.nebula.SdflmodelMessageH\x00\x12\x41\n\x17reputationtable_message\x18\x0f \x01(\x0b\x32\x1e.nebula.ReputationtableMessageH\x00\x42\t\n\x07message\"\x9e\x01\n\x10\x44iscoveryMessage\x12/\n\x06\x61\x63tion\x18\x01 \x01(\x0e\x32\x1f.nebula.DiscoveryMessage.Action\x12\x10\n\x08latitude\x18\x02 \x01(\x02\x12\x11\n\tlongitude\x18\x03 \x01(\x02\"4\n\x06\x41\x63tion\x12\x0c\n\x08\x44ISCOVER\x10\x00\x12\x0c\n\x08REGISTER\x10\x01\x12\x0e\n\nDEREGISTER\x10\x02\"\xd1\x01\n\x0e\x43ontrolMessage\x12-\n\x06\x61\x63tion\x18\x01 \x01(\x0e\x32\x1d.nebula.ControlMessage.Action\x12\x0b\n\x03log\x18\x02 \x01(\t\"\x82\x01\n\x06\x41\x63tion\x12\t\n\x05\x41LIVE\x10\x00\x12\x0c\n\x08OVERHEAD\x10\x01\x12\x0c\n\x08MOBILITY\x10\x02\x12\x0c\n\x08RECOVERY\x10\x03\x12\r\n\tWEAK_LINK\x10\x04\x12\x17\n\x13LEADERSHIP_TRANSFER\x10\x05\x12\x1b\n\x17LEADERSHIP_TRANSFER_ACK\x10\x06\"\xcd\x01\n\x11\x46\x65\x64\x65rationMessage\x12\x30\n\x06\x61\x63tion\x18\x01 \x01(\x0e\x32 .nebula.FederationMessage.Action\x12\x11\n\targuments\x18\x02 \x03(\t\x12\r\n\x05round\x18\x03 \x01(\x05\"d\n\x06\x41\x63tion\x12\x14\n\x10\x46\x45\x44\x45RATION_START\x10\x00\x12\x0e\n\nREPUTATION\x10\x01\x12\x1e\n\x1a\x46\x45\x44\x45RATION_MODELS_INCLUDED\x10\x02\x12\x14\n\x10\x46\x45\x44\x45RATION_READY\x10\x03\"A\n\x0cModelMessage\x12\x12\n\nparameters\x18\x01 \x01(\x0c\x12\x0e\n\x06weight\x18\x02 \x01(\x03\x12\r\n\x05round\x18\x03 \x01(\x05\"\xc7\x01\n\x10SdflmodelMessage\x12/\n\x06\x61\x63tion\x18\x01 \x01(\x0e\x32\x1f.nebula.SdflmodelMessage.Action\x12\x0e\n\x06target\x18\x02 \x01(\t\x12\x12\n\nparameters\x18\x03 \x01(\x0c\x12\x0e\n\x06weight\x18\x04 \x01(\x03\x12\r\n\x05round\x18\x05 \x01(\x05\x12\x0f\n\x07node_id\x18\x06 \x01(\t\".\n\x06\x41\x63tion\x12\x12\n\x0eTRAINER_UPDATE\x10\x00\x12\x10\n\x0cGLOBAL_MODEL\x10\x01\"\x8f\x01\n\x11\x43onnectionMessage\x12\x30\n\x06\x61\x63tion\x18\x01 \x01(\x0e\x32 .nebula.ConnectionMessage.Action\"H\n\x06\x41\x63tion\x12\x0b\n\x07\x43ONNECT\x10\x00\x12\x0e\n\nDISCONNECT\x10\x01\x12\x10\n\x0cLATE_CONNECT\x10\x02\x12\x0f\n\x0bRESTRUCTURE\x10\x03\"\x95\x01\n\x0f\x44iscoverMessage\x12.\n\x06\x61\x63tion\x18\x01 \x01(\x0e\x32\x1e.nebula.DiscoverMessage.Action\"R\n\x06\x41\x63tion\x12\x11\n\rDISCOVER_JOIN\x10\x00\x12\x12\n\x0e\x44ISCOVER_NODES\x10\x01\x12\x10\n\x0cLATE_CONNECT\x10\x02\x12\x0f\n\x0bRESTRUCTURE\x10\x03\"\xce\x01\n\x0cOfferMessage\x12+\n\x06\x61\x63tion\x18\x01 \x01(\x0e\x32\x1b.nebula.OfferMessage.Action\x12\x13\n\x0bn_neighbors\x18\x02 \x01(\x02\x12\x0c\n\x04loss\x18\x03 \x01(\x02\x12\x12\n\nparameters\x18\x04 \x01(\x0c\x12\x0e\n\x06rounds\x18\x05 \x01(\x05\x12\r\n\x05round\x18\x06 \x01(\x05\x12\x0e\n\x06\x65pochs\x18\x07 \x01(\x05\"+\n\x06\x41\x63tion\x12\x0f\n\x0bOFFER_MODEL\x10\x00\x12\x10\n\x0cOFFER_METRIC\x10\x01\"w\n\x0bLinkMessage\x12*\n\x06\x61\x63tion\x18\x01 \x01(\x0e\x32\x1a.nebula.LinkMessage.Action\x12\r\n\x05\x61\x64\x64rs\x18\x02 \x01(\t\"-\n\x06\x41\x63tion\x12\x0e\n\nCONNECT_TO\x10\x00\x12\x13\n\x0f\x44ISCONNECT_FROM\x10\x01\"\x89\x01\n\x11ReputationMessage\x12\x0f\n\x07node_id\x18\x01 \x01(\t\x12\r\n\x05score\x18\x02 \x01(\x02\x12\r\n\x05round\x18\x03 \x01(\x05\x12\x30\n\x06\x61\x63tion\x18\x04 \x01(\x0e\x32 .nebula.ReputationMessage.Action\"\x13\n\x06\x41\x63tion\x12\t\n\x05SHARE\x10\x00\"\xa3\x01\n\x16ReputationtableMessage\x12\x0f\n\x07node_id\x18\x01 \x01(\t\x12\r\n\x05round\x18\x02 \x01(\x05\x12\x1d\n\x15reputation_table_json\x18\x03 \x01(\t\x12\x35\n\x06\x61\x63tion\x18\x04 \x01(\x0e\x32%.nebula.ReputationtableMessage.Action\"\x13\n\x06\x41\x63tion\x12\t\n\x05TABLE\x10\x00\"#\n\x0fResponseMessage\x12\x10\n\x08response\x18\x01 \x01(\t\"\xaa\x04\n\x16TrustworthinessMessage\x12\x35\n\x06\x61\x63tion\x18\x01 \x01(\x0e\x32%.nebula.TrustworthinessMessage.Action\x12\x0f\n\x07node_id\x18\x02 \x01(\t\x12\x12\n\nbytes_sent\x18\x03 \x01(\x03\x12\x12\n\nbytes_recv\x18\x04 \x01(\x03\x12\x10\n\x08\x61\x63\x63uracy\x18\x05 \x01(\x01\x12\x0c\n\x04loss\x18\x06 \x01(\x01\x12\x0c\n\x04role\x18\x07 \x01(\t\x12\x13\n\x0b\x65nergy_grid\x18\x08 \x01(\x01\x12\x11\n\temissions\x18\t \x01(\x01\x12\x10\n\x08workload\x18\n \x01(\t\x12\x11\n\tcpu_model\x18\x0b \x01(\t\x12\x11\n\tgpu_model\x18\x0c \x01(\t\x12\x10\n\x08\x63pu_used\x18\r \x01(\x08\x12\x10\n\x08gpu_used\x18\x0e \x01(\x08\x12\x17\n\x0f\x65nergy_consumed\x18\x0f \x01(\x01\x12\x13\n\x0bsample_size\x18\x10 \x01(\x05\x12\x17\n\x0f\x63lass_imbalance\x18\x11 \x01(\x02\x12\x12\n\nmodel_size\x18\x12 \x01(\x03\x12\x15\n\rlocal_entropy\x18\x13 \x01(\x02\x12\x14\n\x0cval_accuracy\x18\x14 \x01(\x02\x12\x12\n\ndp_enabled\x18\x15 \x01(\x08\x12\x12\n\ndp_epsilon\x18\x16 \x01(\x02\x12\x10\n\x08macro_f1\x18\x17 \x01(\x01\x12\x16\n\x0etrain_accuracy\x18\x18 \x01(\x01\"\x14\n\x06\x41\x63tion\x12\n\n\x06REPORT\x10\x00\"\x88\x01\n\x12TrustscoresMessage\x12\x31\n\x06\x61\x63tion\x18\x01 \x01(\x0e\x32!.nebula.TrustscoresMessage.Action\x12\x0f\n\x07node_id\x18\x02 \x01(\t\x12\x19\n\x11trust_report_json\x18\x03 \x01(\t\"\x13\n\x06\x41\x63tion\x12\t\n\x05SHARE\x10\x00\x62\x06proto3') _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'nebula_pb2', globals()) @@ -67,11 +67,11 @@ _RESPONSEMESSAGE._serialized_start=2617 _RESPONSEMESSAGE._serialized_end=2652 _TRUSTWORTHINESSMESSAGE._serialized_start=2655 - _TRUSTWORTHINESSMESSAGE._serialized_end=3167 - _TRUSTWORTHINESSMESSAGE_ACTION._serialized_start=3147 - _TRUSTWORTHINESSMESSAGE_ACTION._serialized_end=3167 - _TRUSTSCORESMESSAGE._serialized_start=3170 - _TRUSTSCORESMESSAGE._serialized_end=3306 + _TRUSTWORTHINESSMESSAGE._serialized_end=3209 + _TRUSTWORTHINESSMESSAGE_ACTION._serialized_start=3189 + _TRUSTWORTHINESSMESSAGE_ACTION._serialized_end=3209 + _TRUSTSCORESMESSAGE._serialized_start=3212 + _TRUSTSCORESMESSAGE._serialized_end=3348 _TRUSTSCORESMESSAGE_ACTION._serialized_start=2430 _TRUSTSCORESMESSAGE_ACTION._serialized_end=2449 # @@protoc_insertion_point(module_scope) diff --git a/nebula/core/training/lightning.py b/nebula/core/training/lightning.py index f5988ef00..b7551ea5d 100755 --- a/nebula/core/training/lightning.py +++ b/nebula/core/training/lightning.py @@ -295,9 +295,9 @@ async def train(self): try: self.create_trainer() logging.info(f"{'=' * 10} [Training] Started (check training logs for progress) {'=' * 10}") - val_loss, val_accuracy = await asyncio.to_thread(self._train_sync) + val_loss, val_accuracy, train_accuracy = await asyncio.to_thread(self._train_sync) logging.info(f"{'=' * 10} [Training] Finished (check training logs for progress) {'=' * 10}") - vme = ValidationMetricsEvent(val_loss, val_accuracy) + vme = ValidationMetricsEvent(val_loss, val_accuracy, train_accuracy) await EventManager.get_instance().publish_addonevent(vme) except Exception as e: logging_training.error(f"Error training model: {e}") @@ -317,39 +317,51 @@ def _train_sync(self): loss = raw_loss.item() if hasattr(raw_loss, "item") else raw_loss accuracy = validation_metrics.get("Validation/Accuracy") - return loss, accuracy + train_accuracy = None + get_train_accuracy = getattr(self.model, "get_latest_train_accuracy", None) + if callable(get_train_accuracy): + train_accuracy = get_train_accuracy() + + return loss, accuracy, train_accuracy except Exception as e: logging_training.error(f"Error in _train_sync: {e}") tb = traceback.format_exc() logging_training.error(f"Traceback: {tb}") # If "raise", the exception will be managed by the main thread - return None, None + return None, None, None async def test(self): try: self.create_trainer() logging.info(f"{'=' * 10} [Testing] Started (check training logs for progress) {'=' * 10}") - loss, accuracy = await asyncio.to_thread(self._test_sync) + loss, accuracy, macro_f1 = await asyncio.to_thread(self._test_sync) logging.info(f"{'=' * 10} [Testing] Finished (check training logs for progress) {'=' * 10}") - tme = TestMetricsEvent(loss, accuracy) + tme = TestMetricsEvent(loss, accuracy, macro_f1) await EventManager.get_instance().publish_addonevent(tme) except Exception as e: logging_training.error(f"Error testing model: {e}") logging_training.error(traceback.format_exc()) + def _metric_value(self, value): + return value.item() if hasattr(value, "item") else value + def _test_sync(self): try: self._trainer.test(self.model, self.datamodule, verbose=True) metrics = self._trainer.callback_metrics - loss = metrics.get('val_loss/dataloader_idx_0', None).item() - accuracy = metrics.get('val_accuracy/dataloader_idx_0', None).item() - return loss, accuracy + loss = self._metric_value(metrics.get('val_loss/dataloader_idx_0')) + accuracy = self._metric_value(metrics.get('val_accuracy/dataloader_idx_0')) + macro_f1 = None + get_macro_f1 = getattr(self.model, "get_latest_test_macro_f1", None) + if callable(get_macro_f1): + macro_f1 = get_macro_f1() + + return loss, accuracy, macro_f1 except Exception as e: logging_training.error(f"Error in _test_sync: {e}") tb = traceback.format_exc() logging_training.error(f"Traceback: {tb}") - # If "raise", the exception will be managed by the main thread - return None, None + return None, None, None def cleanup(self): if self._trainer is not None: From bc7ef3a099a59d8fbd51f065c9ba6b812a052510 Mon Sep 17 00:00:00 2001 From: "Juan J." Date: Thu, 4 Jun 2026 12:02:16 +0200 Subject: [PATCH 59/66] Adversarial Training CAPGD V1 --- .../defenses/adversarial_training/__init__.py | 59 ++++ .../base.py} | 0 .../config.py} | 9 +- .../defense.py} | 28 +- .../image.py} | 4 +- .../logging.py} | 2 +- .../defenses/adversarial_training/tabular.py | 199 +++++++++++ .../defenses/adversarial_training_tabular.py | 323 ------------------ nebula/core/datasets/tabular_metadata.py | 52 +++ 9 files changed, 332 insertions(+), 344 deletions(-) create mode 100644 nebula/addons/defenses/adversarial_training/__init__.py rename nebula/addons/defenses/{adversarial_training_base.py => adversarial_training/base.py} (100%) rename nebula/addons/defenses/{adversarial_training_config.py => adversarial_training/config.py} (90%) rename nebula/addons/defenses/{adversarial_training.py => adversarial_training/defense.py} (91%) rename nebula/addons/defenses/{adversarial_training_image.py => adversarial_training/image.py} (96%) rename nebula/addons/defenses/{adversarial_training_logging.py => adversarial_training/logging.py} (99%) create mode 100644 nebula/addons/defenses/adversarial_training/tabular.py delete mode 100644 nebula/addons/defenses/adversarial_training_tabular.py diff --git a/nebula/addons/defenses/adversarial_training/__init__.py b/nebula/addons/defenses/adversarial_training/__init__.py new file mode 100644 index 000000000..c417b1b93 --- /dev/null +++ b/nebula/addons/defenses/adversarial_training/__init__.py @@ -0,0 +1,59 @@ +from nebula.addons.defenses.adversarial_training.defense import ( + CAA_TABULAR_DATASETS, + ERR_ALPHA, + ERR_APPLY_PROBABILITY, + ERR_CLIP_BOUNDS, + ERR_EPSILON, + ERR_IMAGE_ATTACK, + ERR_LOSS_WEIGHTS, + ERR_MIXED_WEIGHTS, + ERR_MODE, + ERR_STEPS, + ERR_TABULAR_ATTACK, + ERR_TABULAR_METADATA, + ERR_UNSUPPORTED_ATTACK, + IMAGE_ADVERSARIAL_ATTACKS, + IMAGE_DATASET_NORMALIZATION, + TABULAR_ADVERSARIAL_ATTACKS, + AdversarialExampleGenerator, + AdversarialTrainingConfig, + AdversarialTrainingDefense, + ImageAdversarialExampleGenerator, + ImageFGSMGenerator, + ImagePGDGenerator, + TabularAdversarialExampleGenerator, + TabularCAAGenerator, + TabularCAPGDGenerator, + TabularConstraintSet, + apply_adversarial_training_if_enabled, +) + +__all__ = [ + "CAA_TABULAR_DATASETS", + "ERR_ALPHA", + "ERR_APPLY_PROBABILITY", + "ERR_CLIP_BOUNDS", + "ERR_EPSILON", + "ERR_IMAGE_ATTACK", + "ERR_LOSS_WEIGHTS", + "ERR_MIXED_WEIGHTS", + "ERR_MODE", + "ERR_STEPS", + "ERR_TABULAR_ATTACK", + "ERR_TABULAR_METADATA", + "ERR_UNSUPPORTED_ATTACK", + "IMAGE_ADVERSARIAL_ATTACKS", + "IMAGE_DATASET_NORMALIZATION", + "TABULAR_ADVERSARIAL_ATTACKS", + "AdversarialExampleGenerator", + "AdversarialTrainingConfig", + "AdversarialTrainingDefense", + "ImageAdversarialExampleGenerator", + "ImageFGSMGenerator", + "ImagePGDGenerator", + "TabularAdversarialExampleGenerator", + "TabularCAAGenerator", + "TabularCAPGDGenerator", + "TabularConstraintSet", + "apply_adversarial_training_if_enabled", +] diff --git a/nebula/addons/defenses/adversarial_training_base.py b/nebula/addons/defenses/adversarial_training/base.py similarity index 100% rename from nebula/addons/defenses/adversarial_training_base.py rename to nebula/addons/defenses/adversarial_training/base.py diff --git a/nebula/addons/defenses/adversarial_training_config.py b/nebula/addons/defenses/adversarial_training/config.py similarity index 90% rename from nebula/addons/defenses/adversarial_training_config.py rename to nebula/addons/defenses/adversarial_training/config.py index 930144b5d..fcb6c5aa2 100644 --- a/nebula/addons/defenses/adversarial_training_config.py +++ b/nebula/addons/defenses/adversarial_training/config.py @@ -2,9 +2,11 @@ from typing import Any IMAGE_ADVERSARIAL_ATTACKS = {"fgsm", "pgd"} +TABULAR_ADVERSARIAL_ATTACKS = {"capgd"} CAA_TABULAR_DATASETS = {"AdultCensus"} ERR_IMAGE_ATTACK = "image adversarial_training.attack must be one of: fgsm, pgd" +ERR_TABULAR_ATTACK = "tabular adversarial_training.attack must be one of: capgd" ERR_MODE = "adversarial_training.mode must be one of: clean, adversarial, mixed" ERR_EPSILON = "adversarial_training.epsilon must be >= 0" ERR_ALPHA = "adversarial_training.alpha must be >= 0" @@ -51,8 +53,9 @@ def config_from_participant(participant_config: dict[str, Any]) -> AdversarialTr dataset_name = participant_config.get("data_args", {}).get("dataset") domain = str(raw.get("domain", "image")).lower() - # Tabular adversarial training exposes a single attack: CAA. - attack = "caa" if domain == "tabular" else str(raw.get("attack", "fgsm")).lower() + attack = str(raw.get("attack", "capgd" if domain == "tabular" else "fgsm")).lower() + if domain == "tabular" and attack == "caa": + attack = "capgd" return AdversarialTrainingConfig( enabled=True, @@ -78,6 +81,8 @@ def validate_config(config: AdversarialTrainingConfig) -> None: raise ValueError(ERR_MODE) if config.domain == "image" and config.attack not in IMAGE_ADVERSARIAL_ATTACKS: raise ValueError(ERR_IMAGE_ATTACK) + if config.domain == "tabular" and config.attack not in TABULAR_ADVERSARIAL_ATTACKS: + raise ValueError(ERR_TABULAR_ATTACK) if config.epsilon < 0: raise ValueError(ERR_EPSILON) if config.alpha is not None and config.alpha < 0: diff --git a/nebula/addons/defenses/adversarial_training.py b/nebula/addons/defenses/adversarial_training/defense.py similarity index 91% rename from nebula/addons/defenses/adversarial_training.py rename to nebula/addons/defenses/adversarial_training/defense.py index 6dbc7aea0..d4e8da2fd 100644 --- a/nebula/addons/defenses/adversarial_training.py +++ b/nebula/addons/defenses/adversarial_training/defense.py @@ -3,8 +3,8 @@ import torch -from nebula.addons.defenses.adversarial_training_base import AdversarialExampleGenerator -from nebula.addons.defenses.adversarial_training_config import ( +from nebula.addons.defenses.adversarial_training.base import AdversarialExampleGenerator +from nebula.addons.defenses.adversarial_training.config import ( CAA_TABULAR_DATASETS, ERR_ALPHA, ERR_APPLY_PROBABILITY, @@ -15,23 +15,26 @@ ERR_MIXED_WEIGHTS, ERR_MODE, ERR_STEPS, + ERR_TABULAR_ATTACK, ERR_TABULAR_METADATA, ERR_UNSUPPORTED_ATTACK, IMAGE_ADVERSARIAL_ATTACKS, IMAGE_DATASET_NORMALIZATION, + TABULAR_ADVERSARIAL_ATTACKS, AdversarialTrainingConfig, config_from_participant, validate_config, ) -from nebula.addons.defenses.adversarial_training_image import ( +from nebula.addons.defenses.adversarial_training.image import ( ImageAdversarialExampleGenerator, ImageFGSMGenerator, ImagePGDGenerator, ) -from nebula.addons.defenses.adversarial_training_logging import AdversarialTrainingSampleLogger -from nebula.addons.defenses.adversarial_training_tabular import ( +from nebula.addons.defenses.adversarial_training.logging import AdversarialTrainingSampleLogger +from nebula.addons.defenses.adversarial_training.tabular import ( TabularAdversarialExampleGenerator, TabularCAAGenerator, + TabularCAPGDGenerator, TabularConstraintSet, ) from nebula.core.datasets.tabular_metadata import CATEGORICAL, CONTINUOUS, INTEGER, TabularAdversarialMetadata @@ -62,18 +65,8 @@ def from_participant_config( validate_config(config) if config.domain == "tabular": - # CAA needs dataset metadata. Keep the allow-list explicit while more tabular datasets are added. - if config.dataset_name not in CAA_TABULAR_DATASETS: - logging.warning( - "[AdversarialTrainingDefense] Skipping CAA tabular adversarial training: " - "dataset '%s' is not supported yet", - config.dataset_name, - ) - return None - metadata = cls._get_tabular_metadata(partition) - # For tabular data, the only valid adversarial-training generator is CAA. - return cls(config=config, generator=TabularCAAGenerator(config, metadata)) + return cls(config=config, generator=TabularCAPGDGenerator(config, metadata)) if config.domain == "image": # Image attacks run in normalized model space, so each dataset must provide mean/std. @@ -254,10 +247,12 @@ def apply_adversarial_training_if_enabled(model, participant_config: dict[str, A "ERR_MIXED_WEIGHTS", "ERR_MODE", "ERR_STEPS", + "ERR_TABULAR_ATTACK", "ERR_TABULAR_METADATA", "ERR_UNSUPPORTED_ATTACK", "IMAGE_ADVERSARIAL_ATTACKS", "IMAGE_DATASET_NORMALIZATION", + "TABULAR_ADVERSARIAL_ATTACKS", "AdversarialExampleGenerator", "AdversarialTrainingConfig", "AdversarialTrainingDefense", @@ -266,6 +261,7 @@ def apply_adversarial_training_if_enabled(model, participant_config: dict[str, A "ImagePGDGenerator", "TabularAdversarialExampleGenerator", "TabularCAAGenerator", + "TabularCAPGDGenerator", "TabularConstraintSet", "apply_adversarial_training_if_enabled", ] diff --git a/nebula/addons/defenses/adversarial_training_image.py b/nebula/addons/defenses/adversarial_training/image.py similarity index 96% rename from nebula/addons/defenses/adversarial_training_image.py rename to nebula/addons/defenses/adversarial_training/image.py index d9a84ae1c..cd6dbd129 100644 --- a/nebula/addons/defenses/adversarial_training_image.py +++ b/nebula/addons/defenses/adversarial_training/image.py @@ -1,7 +1,7 @@ import torch -from nebula.addons.defenses.adversarial_training_base import AdversarialExampleGenerator -from nebula.addons.defenses.adversarial_training_config import AdversarialTrainingConfig +from nebula.addons.defenses.adversarial_training.base import AdversarialExampleGenerator +from nebula.addons.defenses.adversarial_training.config import AdversarialTrainingConfig class ImageAdversarialExampleGenerator(AdversarialExampleGenerator): diff --git a/nebula/addons/defenses/adversarial_training_logging.py b/nebula/addons/defenses/adversarial_training/logging.py similarity index 99% rename from nebula/addons/defenses/adversarial_training_logging.py rename to nebula/addons/defenses/adversarial_training/logging.py index 2e0e489cb..013a398ac 100644 --- a/nebula/addons/defenses/adversarial_training_logging.py +++ b/nebula/addons/defenses/adversarial_training/logging.py @@ -2,7 +2,7 @@ import torch -from nebula.addons.defenses.adversarial_training_config import AdversarialTrainingConfig +from nebula.addons.defenses.adversarial_training.config import AdversarialTrainingConfig from nebula.config.config import TRAINING_LOGGER logging_training = logging.getLogger(TRAINING_LOGGER) diff --git a/nebula/addons/defenses/adversarial_training/tabular.py b/nebula/addons/defenses/adversarial_training/tabular.py new file mode 100644 index 000000000..2d1d954e8 --- /dev/null +++ b/nebula/addons/defenses/adversarial_training/tabular.py @@ -0,0 +1,199 @@ +import torch +import torch.nn.functional as F + +from nebula.addons.defenses.adversarial_training.base import AdversarialExampleGenerator +from nebula.addons.defenses.adversarial_training.config import AdversarialTrainingConfig +from nebula.core.datasets.tabular_metadata import CATEGORICAL, CONTINUOUS, INTEGER, TabularAdversarialMetadata + + +class TabularConstraintSet: + """Projects tabular attack candidates back to the valid feature domain.""" + + def __init__(self, metadata: TabularAdversarialMetadata): + # The metadata is dataset-level and immutable; derived tensors are cached per device/dtype. + self.metadata = metadata + self._tensor_cache: dict[tuple[torch.device, torch.dtype], dict[str, torch.Tensor]] = {} + + def tensors(self, x: torch.Tensor) -> dict[str, torch.Tensor]: + # Masks and bounds are reused in every CAPGD step, so build them once for each tensor placement. + key = (x.device, x.dtype) + cached = self._tensor_cache.get(key) + if cached is not None: + return cached + + # Masks have shape (1, n_features), which broadcasts over the batch dimension. + cached = { + "continuous": self._feature_type_mask(x, CONTINUOUS), + "integer": self._feature_type_mask(x, INTEGER), + "categorical": self._feature_type_mask(x, CATEGORICAL), + "min": torch.tensor(self.metadata.feature_min_norm, dtype=x.dtype, device=x.device).view(1, -1), + "max": torch.tensor(self.metadata.feature_max_norm, dtype=x.dtype, device=x.device).view(1, -1), + } + cached["numeric"] = cached["continuous"] | cached["integer"] + cached["perturbable"] = cached["numeric"] | cached["categorical"] + cached["integer_step"] = self._integer_steps(cached["min"]) + self._tensor_cache[key] = cached + return cached + + def perturbable_mask(self, x: torch.Tensor) -> torch.Tensor: + # Used by the attack step to avoid moving immutable features in the first place. + return self.tensors(x)["perturbable"] + + def project(self, x_candidate: torch.Tensor, x_clean: torch.Tensor, epsilon: float) -> torch.Tensor: + """Clamp numeric features, round integers, restore immutable features and fix one-hot groups.""" + tensors = self.tensors(x_clean) + lower, upper = self._bounds(x_clean, epsilon, tensors) + + # First force every value into its valid interval, then apply type-specific fixes. + x_projected = torch.max(torch.min(x_candidate, upper), lower) + x_projected = self._project_integer_features(x_projected, x_clean, lower, upper, tensors) + x_projected = self.project_categorical_groups(x_projected) + # Immutable features are copied back from the original clean sample as the final guardrail. + return torch.where(tensors["perturbable"], x_projected, x_clean) + + def categorical_gradient_step(self, x_candidate: torch.Tensor, grad: torch.Tensor) -> torch.Tensor: + if not self.metadata.categorical_groups: + return x_candidate + + # One-hot columns are discrete: instead of adding a fractional gradient, + # activate the category whose gradient most increases the adversarial loss. + x_stepped = x_candidate.clone() + for group in self.metadata.categorical_groups: + group_tensor = torch.tensor(group, dtype=torch.long, device=x_candidate.device) + selected = grad.index_select(1, group_tensor).argmax(dim=1) + x_stepped[:, group_tensor] = F.one_hot(selected, num_classes=len(group)).to(dtype=x_candidate.dtype) + return x_stepped + + def project_categorical_groups(self, x_candidate: torch.Tensor) -> torch.Tensor: + if not self.metadata.categorical_groups: + return x_candidate + + # Projection must always leave each one-hot group with exactly one active feature. + x_projected = x_candidate.clone() + for group in self.metadata.categorical_groups: + group_tensor = torch.tensor(group, dtype=torch.long, device=x_candidate.device) + selected = x_candidate.index_select(1, group_tensor).argmax(dim=1) + x_projected[:, group_tensor] = F.one_hot(selected, num_classes=len(group)).to(dtype=x_candidate.dtype) + return x_projected + + def _feature_type_mask(self, x: torch.Tensor, feature_type: str) -> torch.Tensor: + return torch.tensor( + [value == feature_type for value in self.metadata.feature_types], + dtype=torch.bool, + device=x.device, + ).view(1, -1) + + def _bounds( + self, + x_clean: torch.Tensor, + epsilon: float, + tensors: dict[str, torch.Tensor], + ) -> tuple[torch.Tensor, torch.Tensor]: + # Numeric features are restricted both by dataset bounds and by the epsilon ball around x_clean. + numeric_lower = torch.maximum(tensors["min"], x_clean - float(epsilon)) + numeric_upper = torch.minimum(tensors["max"], x_clean + float(epsilon)) + # Categorical features are handled by one-hot projection, not by an epsilon ball. + lower = torch.where(tensors["categorical"], tensors["min"], numeric_lower) + upper = torch.where(tensors["categorical"], tensors["max"], numeric_upper) + return lower, upper + + def _integer_steps(self, minimum: torch.Tensor) -> torch.Tensor: + # Default step=1 is harmless for non-integer columns because the integer mask gates usage later. + integer_steps = torch.ones_like(minimum) + for idx, step in (self.metadata.integer_step_norm or {}).items(): + integer_steps[0, int(idx)] = float(step) + return integer_steps + + def _project_integer_features( + self, + x_projected: torch.Tensor, + x_clean: torch.Tensor, + lower: torch.Tensor, + upper: torch.Tensor, + tensors: dict[str, torch.Tensor], + ) -> torch.Tensor: + integer_mask = tensors["integer"] + if not integer_mask.any(): + return x_projected + + # Integer features may be normalized, so the valid values form a shifted grid: + # min, min + step, min + 2*step, ... + step = torch.clamp(tensors["integer_step"], min=torch.finfo(x_projected.dtype).eps) + grid_lower = torch.ceil((lower - tensors["min"]) / step) * step + tensors["min"] + grid_upper = torch.floor((upper - tensors["min"]) / step) * step + tensors["min"] + rounded = torch.round((x_projected - tensors["min"]) / step) * step + tensors["min"] + rounded = torch.max(torch.min(rounded, grid_upper), grid_lower) + + # If epsilon is smaller than the normalized integer step, no valid integer move exists. + has_valid_grid = grid_lower <= grid_upper + rounded = torch.where(has_valid_grid, rounded, x_clean) + return torch.where(integer_mask, rounded, x_projected) + + +class TabularAdversarialExampleGenerator(AdversarialExampleGenerator): + """Base generator for constrained tabular adversarial examples.""" + + def __init__(self, config: AdversarialTrainingConfig, metadata: TabularAdversarialMetadata): + # Generators share the same constraint layer; only the search strategy should vary. + self.config = config + self.metadata = metadata + self.constraints = TabularConstraintSet(metadata) + + def _alpha(self, epsilon: float) -> float: + # By default, distribute the epsilon budget evenly across CAPGD steps. + if self.config.alpha is not None: + return float(self.config.alpha) + return float(epsilon) / max(int(self.config.steps), 1) + + def _margin(self, logits: torch.Tensor, y: torch.Tensor) -> torch.Tensor: + # Positive margin means some wrong class already beats the true class. + true_logits = logits.gather(1, y.view(-1, 1)).squeeze(1) + true_class_mask = F.one_hot(y, num_classes=logits.size(1)).bool() + other_logits = logits.masked_fill(true_class_mask, float("-inf")) + return other_logits.max(dim=1).values - true_logits + + +class TabularCAPGDGenerator(TabularAdversarialExampleGenerator): + """First-phase constrained tabular CAPGD generator.""" + + def generate(self, model, x, y, criterion): + # Sample one attack strength for this batch, matching the image generator behavior. + epsilon = self._sample_epsilon(x.device) + x_clean = x.detach() + if epsilon <= 0.0: + return x_clean + + steps = max(int(self.config.steps), 1) + step_size = self._alpha(epsilon) + perturbable_mask = self.constraints.perturbable_mask(x_clean).to(dtype=x_clean.dtype) + + x_adv = x_clean.clone() + best_adv = x_adv.clone() + best_score = torch.full((x_clean.size(0),), float("-inf"), dtype=x_clean.dtype, device=x_clean.device) + + for _ in range(steps): + # CAPGD step: move in the sign of the loss gradient, but only on perturbable features. + x_grad = x_adv.detach().requires_grad_(True) + logits = model(x_grad) + loss = criterion(logits, y) + grad = torch.autograd.grad(loss, x_grad, only_inputs=True)[0] + + candidate = x_adv.detach() + float(step_size) * grad.sign() * perturbable_mask + candidate = self.constraints.categorical_gradient_step(candidate, grad) + # This is the key tabular rule: never score or return an invalid candidate. + candidate = self.constraints.project(candidate, x_clean, epsilon) + + with torch.no_grad(): + # Keep the strongest candidate per sample, not just the last step. + candidate_score = self._margin(model(candidate), y) + better = candidate_score > best_score + best_adv = torch.where(better.view(-1, 1), candidate, best_adv) + best_score = torch.where(better, candidate_score, best_score) + + x_adv = candidate + + return best_adv.detach() + + +# Compatibility alias while old configs/UI still refer to the future CAA attack. +TabularCAAGenerator = TabularCAPGDGenerator diff --git a/nebula/addons/defenses/adversarial_training_tabular.py b/nebula/addons/defenses/adversarial_training_tabular.py deleted file mode 100644 index 0c62e1217..000000000 --- a/nebula/addons/defenses/adversarial_training_tabular.py +++ /dev/null @@ -1,323 +0,0 @@ -import torch -import torch.nn.functional as F - -from nebula.addons.defenses.adversarial_training_base import AdversarialExampleGenerator -from nebula.addons.defenses.adversarial_training_config import AdversarialTrainingConfig -from nebula.core.datasets.tabular_metadata import CATEGORICAL, CONTINUOUS, INTEGER, TabularAdversarialMetadata - - -class TabularConstraintSet: - """Projection and mutation rules derived from tabular metadata.""" - - def __init__(self, metadata: TabularAdversarialMetadata): - # Store metadata and cache derived tensors by device/dtype for speed. - self.metadata = metadata - self._tensor_cache: dict[tuple[torch.device, torch.dtype], dict[str, torch.Tensor]] = {} - - def tensors(self, x: torch.Tensor) -> dict[str, torch.Tensor]: - # Return reusable masks, bounds and integer steps for a batch tensor. - key = (x.device, x.dtype) - cached = self._tensor_cache.get(key) - if cached is not None: - return cached - - # Convert metadata lists to tensors once per device/dtype; CAA uses them in every step. - cached = { - "continuous": torch.tensor( - [feature_type == CONTINUOUS for feature_type in self.metadata.feature_types], - dtype=torch.bool, - device=x.device, - ).view(1, -1), - "integer": torch.tensor( - [feature_type == INTEGER for feature_type in self.metadata.feature_types], - dtype=torch.bool, - device=x.device, - ).view(1, -1), - "categorical": torch.tensor( - [feature_type == CATEGORICAL for feature_type in self.metadata.feature_types], - dtype=torch.bool, - device=x.device, - ).view(1, -1), - "min": torch.tensor(self.metadata.feature_min_norm, dtype=x.dtype, device=x.device).view(1, -1), - "max": torch.tensor(self.metadata.feature_max_norm, dtype=x.dtype, device=x.device).view(1, -1), - } - cached["numeric"] = cached["continuous"] | cached["integer"] - cached["perturbable"] = cached["numeric"] | cached["categorical"] - cached["integer_step"] = self._integer_steps(cached["min"]) - self._tensor_cache[key] = cached - return cached - - def perturbable_mask(self, x: torch.Tensor) -> torch.Tensor: - # Expose the final boolean mask used to block immutable features. - return self.tensors(x)["perturbable"] - - def project(self, x_adv: torch.Tensor, x_clean: torch.Tensor, epsilon: float) -> torch.Tensor: - # Project a candidate back to valid tabular values around the clean sample. - tensors = self.tensors(x_clean) - # Numeric features are bounded by epsilon; categorical one-hot features use dataset bounds. - numeric_lower = torch.maximum(tensors["min"], x_clean - float(epsilon)) - numeric_upper = torch.minimum(tensors["max"], x_clean + float(epsilon)) - lower = torch.where(tensors["categorical"], tensors["min"], numeric_lower) - upper = torch.where(tensors["categorical"], tensors["max"], numeric_upper) - x_adv = torch.max(torch.min(x_adv, upper), lower) - - x_adv = self._project_integer_features(x_adv, x_clean, lower, upper, tensors) - x_adv = self.project_categorical_groups(x_adv) - return torch.where(tensors["perturbable"], x_adv, x_clean) - - def project_categorical_groups(self, x_adv: torch.Tensor) -> torch.Tensor: - # Enforce one-hot validity after gradient or evolutionary changes. - if not self.metadata.categorical_groups: - return x_adv - - # Each one-hot group must end with exactly one active category. - x_projected = x_adv.clone() - for group in self.metadata.categorical_groups: - group_tensor = torch.tensor(group, dtype=torch.long, device=x_adv.device) - group_values = x_adv.index_select(1, group_tensor) - selected = group_values.argmax(dim=1) - one_hot = F.one_hot(selected, num_classes=len(group)).to(dtype=x_adv.dtype) - x_projected[:, group_tensor] = one_hot - return x_projected - - def categorical_gradient_step(self, x_candidate: torch.Tensor, grad: torch.Tensor) -> torch.Tensor: - # Apply a discrete gradient step to categorical one-hot groups. - if not self.metadata.categorical_groups: - return x_candidate - - # For one-hot features, choose the category with the largest adversarial gradient. - x_stepped = x_candidate.clone() - for group in self.metadata.categorical_groups: - group_tensor = torch.tensor(group, dtype=torch.long, device=x_candidate.device) - selected = grad.index_select(1, group_tensor).argmax(dim=1) - one_hot = F.one_hot(selected, num_classes=len(group)).to(dtype=x_candidate.dtype) - x_stepped[:, group_tensor] = one_hot - return x_stepped - - def randomize_categorical_groups( - self, - candidates: torch.Tensor, - mutation_probability: float, - ) -> torch.Tensor: - # Randomly switch categories for evolutionary exploration. - if not self.metadata.categorical_groups: - return candidates - - original_shape = candidates.shape - flat_candidates = candidates.reshape(-1, original_shape[-1]).clone() - for group in self.metadata.categorical_groups: - # Mutation explores alternative categories when the gradient phase is not enough. - group_tensor = torch.tensor(group, dtype=torch.long, device=candidates.device) - current = flat_candidates.index_select(1, group_tensor).argmax(dim=1) - random_choice = torch.randint(len(group), current.shape, device=candidates.device) - mutate = torch.rand(current.shape, device=candidates.device) < float(mutation_probability) - selected = torch.where(mutate, random_choice, current) - one_hot = F.one_hot(selected, num_classes=len(group)).to(dtype=candidates.dtype) - flat_candidates[:, group_tensor] = one_hot - return flat_candidates.reshape(original_shape) - - def _integer_steps(self, minimum: torch.Tensor) -> torch.Tensor: - # Build the normalized integer grid spacing tensor from metadata. - integer_steps = torch.ones_like(minimum) - for idx, step in (self.metadata.integer_step_norm or {}).items(): - integer_steps[0, int(idx)] = float(step) - return integer_steps - - def _project_integer_features( - self, - x_adv: torch.Tensor, - x_clean: torch.Tensor, - lower: torch.Tensor, - upper: torch.Tensor, - tensors: dict[str, torch.Tensor], - ) -> torch.Tensor: - # Round integer columns while keeping them inside the allowed epsilon interval. - integer_mask = tensors["integer"] - if not integer_mask.any(): - return x_adv - - # Integer features live on a normalized grid, so round to the closest valid grid value. - step = torch.clamp(tensors["integer_step"], min=torch.finfo(x_adv.dtype).eps) - projected_integer = torch.round((x_adv - tensors["min"]) / step) * step + tensors["min"] - grid_lower = torch.ceil((lower - tensors["min"]) / step) * step + tensors["min"] - grid_upper = torch.floor((upper - tensors["min"]) / step) * step + tensors["min"] - projected_integer = torch.max(torch.min(projected_integer, grid_upper), grid_lower) - has_valid_grid = grid_lower <= grid_upper - projected_integer = torch.where(has_valid_grid, projected_integer, x_clean) - return torch.where(integer_mask, projected_integer, x_adv) - - -class TabularAdversarialExampleGenerator(AdversarialExampleGenerator): - """Base generator for constrained tabular adversarial examples.""" - - def __init__(self, config: AdversarialTrainingConfig, metadata: TabularAdversarialMetadata): - # Share config, metadata and constraints across CAA phases. - self.config = config - self.metadata = metadata - self.constraints = TabularConstraintSet(metadata) - - def _alpha(self, epsilon: float) -> float: - # Use an explicit alpha when provided; otherwise distribute epsilon across steps. - if self.config.alpha is not None: - return float(self.config.alpha) - return float(epsilon) / max(int(self.config.steps), 1) - - def _margin(self, logits: torch.Tensor, y: torch.Tensor) -> torch.Tensor: - # Score how close each sample is to being misclassified. - # Positive margin means some wrong class beats the true class. - true_logits = logits.gather(1, y.view(-1, 1)).squeeze(1) - other_logits = logits.masked_fill(F.one_hot(y, num_classes=logits.size(1)).bool(), float("-inf")) - return other_logits.max(dim=1).values - true_logits - - def _success_mask(self, model, x_adv: torch.Tensor, y: torch.Tensor) -> torch.Tensor: - # Mark samples whose adversarial version changes the model prediction. - with torch.no_grad(): - return torch.argmax(model(x_adv), dim=1) != y - - def _better_mask( - self, - candidate_success: torch.Tensor, - candidate_score: torch.Tensor, - best_success: torch.Tensor, - best_score: torch.Tensor, - ) -> torch.Tensor: - # Prefer successful attacks, then candidates with a better adversarial margin. - return (candidate_success & ~best_success) | ( - (candidate_success == best_success) & (candidate_score > best_score) - ) - - -class TabularCAAGenerator(TabularAdversarialExampleGenerator): - """CAA-style generator for constrained tabular adversarial training.""" - - def generate(self, model, x, y, criterion): - # Generate a constrained tabular adversarial batch with CAA. - epsilon = self._sample_epsilon(x.device) - x_clean = x.detach() - if epsilon <= 0.0: - return x_clean - - # First try a gradient-guided CAA search; then mutate only samples that still resist. - x_adv = self._capgd_phase(model, x_clean, y, criterion, epsilon) - failed = ~self._success_mask(model, x_adv, y) - if failed.any(): - x_fallback = self._evolutionary_phase(model, x_clean[failed], y[failed], x_adv[failed], epsilon) - x_adv = x_adv.clone() - x_adv[failed] = x_fallback - return x_adv.detach() - - def _capgd_phase(self, model, x_clean: torch.Tensor, y: torch.Tensor, criterion, epsilon: float) -> torch.Tensor: - # Run the gradient-based part of CAA with projection after every candidate step. - steps = max(int(self.config.steps), 1) - step_size = self._alpha(epsilon) - perturbable_mask = self.constraints.perturbable_mask(x_clean) - x_adv = x_clean.clone() - best_adv = x_adv.clone() - best_score = torch.full((x_clean.size(0),), float("-inf"), dtype=x_clean.dtype, device=x_clean.device) - best_success = torch.zeros(x_clean.size(0), dtype=torch.bool, device=x_clean.device) - previous_loss = None - - for _ in range(steps): - x_grad = x_adv.detach().requires_grad_(True) - logits = model(x_grad) - loss = criterion(logits, y) - grad = torch.autograd.grad(loss, x_grad, only_inputs=True)[0] - - candidate = x_adv.detach() + float(step_size) * grad.sign() * perturbable_mask - candidate = self.constraints.categorical_gradient_step(candidate, grad) - candidate = self.constraints.project(candidate, x_clean, epsilon) - - with torch.no_grad(): - candidate_logits = model(candidate) - candidate_score = self._margin(candidate_logits, y) - candidate_success = torch.argmax(candidate_logits, dim=1) != y - # Keep successful adversarial samples first; otherwise keep the highest margin. - better = self._better_mask(candidate_success, candidate_score, best_success, best_score) - best_adv = torch.where(better.view(-1, 1), candidate, best_adv) - best_score = torch.where(better, candidate_score, best_score) - best_success = best_success | candidate_success - - candidate_loss = F.cross_entropy(candidate_logits, y) - if previous_loss is not None and candidate_loss <= previous_loss: - step_size *= 0.75 - previous_loss = candidate_loss - - x_adv = candidate - - return best_adv.detach() - - def _evolutionary_phase( - self, - model, - x_clean: torch.Tensor, - y: torch.Tensor, - x_seed: torch.Tensor, - epsilon: float, - ) -> torch.Tensor: - # Use random mutations as a fallback for samples not solved by the gradient phase. - if x_clean.numel() == 0: - return x_clean - - tensors = self.constraints.tensors(x_clean) - perturbable_mask = tensors["perturbable"].to(dtype=x_clean.dtype) - batch_size = x_clean.size(0) - population_size = min(max(int(self.config.steps) * 4, 8), 32) - generations = min(max(int(self.config.steps), 3), 20) - mutation_scale = max(float(epsilon) / 2.0, torch.finfo(x_clean.dtype).eps) - - best_adv = self.constraints.project(x_seed.detach(), x_clean, epsilon) - with torch.no_grad(): - best_logits = model(best_adv) - best_score = self._margin(best_logits, y) - best_success = torch.argmax(best_logits, dim=1) != y - - for _ in range(generations): - random_noise = torch.empty( - population_size, - *x_clean.shape, - dtype=x_clean.dtype, - device=x_clean.device, - ).uniform_(-float(epsilon), float(epsilon)) - mutations = torch.randn( - population_size, - *x_clean.shape, - dtype=x_clean.dtype, - device=x_clean.device, - ) * mutation_scale - candidates = x_clean.unsqueeze(0) + random_noise * perturbable_mask - candidates[0] = best_adv + mutations[0] * perturbable_mask - if population_size > 1: - candidates[1:] = candidates[1:] + mutations[1:] * perturbable_mask - candidates = self.constraints.randomize_categorical_groups(candidates, mutation_probability=0.35) - - flat_candidates = candidates.reshape(population_size * batch_size, -1) - flat_clean = x_clean.repeat(population_size, 1) - # Every random candidate is projected back to the valid tabular domain before scoring. - flat_candidates = self.constraints.project(flat_candidates, flat_clean, epsilon) - repeated_y = y.repeat(population_size) - - with torch.no_grad(): - logits = model(flat_candidates) - scores = self._margin(logits, repeated_y).view(population_size, batch_size) - successes = (torch.argmax(logits, dim=1) != repeated_y).view(population_size, batch_size) - candidate_rank = scores + successes.to(dtype=scores.dtype) * 1_000.0 - best_population_idx = candidate_rank.argmax(dim=0) - - selected = flat_candidates.view(population_size, batch_size, -1)[ - best_population_idx, - torch.arange(batch_size, device=x_clean.device), - ] - selected_score = scores[ - best_population_idx, - torch.arange(batch_size, device=x_clean.device), - ] - selected_success = successes[ - best_population_idx, - torch.arange(batch_size, device=x_clean.device), - ] - better = self._better_mask(selected_success, selected_score, best_success, best_score) - best_adv = torch.where(better.view(-1, 1), selected, best_adv) - best_score = torch.where(better, selected_score, best_score) - best_success = best_success | selected_success - - return best_adv.detach() diff --git a/nebula/core/datasets/tabular_metadata.py b/nebula/core/datasets/tabular_metadata.py index e7596fcad..e34397f6a 100644 --- a/nebula/core/datasets/tabular_metadata.py +++ b/nebula/core/datasets/tabular_metadata.py @@ -12,9 +12,14 @@ ERR_FEATURE_MIN_LENGTH = "feature_min_norm length must match feature_names length" ERR_FEATURE_MAX_LENGTH = "feature_max_norm length must match feature_names length" ERR_UNSUPPORTED_FEATURE_TYPES = "Unsupported tabular feature types: {feature_types}" +ERR_FEATURE_BOUNDS = "feature_min_norm must be <= feature_max_norm for every feature" +ERR_INTEGER_STEP_INDEX = "integer_step_norm contains invalid feature indices: {indices}" +ERR_INTEGER_STEP_VALUE = "integer_step_norm values must be > 0" +ERR_INTEGER_STEP_TYPE = "integer_step_norm contains non-integer feature indices: {indices}" ERR_CATEGORICAL_GROUP_SIZE = "categorical_groups entries must contain at least two feature indices" ERR_CATEGORICAL_GROUP_INDEX = "categorical_groups contains invalid feature indices: {indices}" ERR_CATEGORICAL_GROUP_TYPE = "categorical_groups contains non-categorical feature indices: {indices}" +ERR_CATEGORICAL_GROUP_OVERLAP = "categorical_groups contains duplicated feature indices: {indices}" ERR_CATEGORICAL_GROUP_COVERAGE = "categorical feature indices missing from categorical_groups: {indices}" @@ -22,6 +27,8 @@ class TabularAdversarialMetadata: """Minimal metadata for tabular adversarial training.""" + # These fields describe the exact vector received by the model after preprocessing. + # Bounds and steps must use the same normalized space as the training tensors. feature_names: list[str] feature_types: list[str] feature_min_norm: list[float] @@ -30,6 +37,8 @@ class TabularAdversarialMetadata: categorical_groups: list[list[int]] | None = None def __post_init__(self): + # Fail early if a dataset exposes incomplete metadata. The attack relies on + # these arrays lining up feature-by-feature. n_features = len(self.feature_names) if len(self.feature_types) != n_features: raise ValueError(ERR_FEATURE_TYPES_LENGTH) @@ -37,9 +46,43 @@ def __post_init__(self): raise ValueError(ERR_FEATURE_MIN_LENGTH) if len(self.feature_max_norm) != n_features: raise ValueError(ERR_FEATURE_MAX_LENGTH) + + # Every feature needs a valid normalized interval so projection can clamp safely. + invalid_bounds = [ + idx + for idx, (min_value, max_value) in enumerate( + zip(self.feature_min_norm, self.feature_max_norm, strict=True) + ) + if min_value > max_value + ] + if invalid_bounds: + raise ValueError(ERR_FEATURE_BOUNDS) invalid_types = set(self.feature_types) - {CONTINUOUS, INTEGER, CATEGORICAL, NON_PERTURBABLE} if invalid_types: raise ValueError(ERR_UNSUPPORTED_FEATURE_TYPES.format(feature_types=sorted(invalid_types))) + + # Integer steps represent the normalized distance between consecutive integer values. + # They only make sense for features marked as INTEGER. + invalid_step_indices = [ + idx + for idx in (self.integer_step_norm or {}) + if int(idx) < 0 or int(idx) >= n_features + ] + if invalid_step_indices: + raise ValueError(ERR_INTEGER_STEP_INDEX.format(indices=invalid_step_indices)) + non_integer_step_indices = [ + idx + for idx in (self.integer_step_norm or {}) + if self.feature_types[int(idx)] != INTEGER + ] + if non_integer_step_indices: + raise ValueError(ERR_INTEGER_STEP_TYPE.format(indices=non_integer_step_indices)) + if any(step <= 0 for step in (self.integer_step_norm or {}).values()): + raise ValueError(ERR_INTEGER_STEP_VALUE) + + # Categorical groups represent one original categorical column after one-hot encoding. + # Each group must be disjoint so projection can activate exactly one value per group. + grouped_counts: dict[int, int] = {} for group in self.categorical_groups or []: if len(group) < 2: raise ValueError(ERR_CATEGORICAL_GROUP_SIZE) @@ -49,7 +92,14 @@ def __post_init__(self): non_categorical_indices = [idx for idx in group if self.feature_types[idx] != CATEGORICAL] if non_categorical_indices: raise ValueError(ERR_CATEGORICAL_GROUP_TYPE.format(indices=non_categorical_indices)) + for idx in group: + grouped_counts[idx] = grouped_counts.get(idx, 0) + 1 + + duplicated_group_indices = sorted(idx for idx, count in grouped_counts.items() if count > 1) + if duplicated_group_indices: + raise ValueError(ERR_CATEGORICAL_GROUP_OVERLAP.format(indices=duplicated_group_indices)) + # A categorical feature without a group cannot be projected back to a valid one-hot state. grouped_categorical_indices = { idx for group in self.categorical_groups or [] @@ -65,10 +115,12 @@ def __post_init__(self): raise ValueError(ERR_CATEGORICAL_GROUP_COVERAGE.format(indices=missing_categorical_indices)) def to_dict(self) -> dict[str, Any]: + # Partitions persist metadata as JSON-like dictionaries in HDF5 attributes. return asdict(self) @classmethod def from_dict(cls, data: dict[str, Any]) -> TabularAdversarialMetadata: + # HDF5/JSON round-trips can turn integer keys into strings; normalize them here. return cls( feature_names=[str(value) for value in data["feature_names"]], feature_types=[str(value) for value in data["feature_types"]], From 3d232b8f030584bc931c20185ccd7659534be28d Mon Sep 17 00:00:00 2001 From: "Juan J." Date: Fri, 5 Jun 2026 11:14:35 +0200 Subject: [PATCH 60/66] Adversarial Training: Frontend updated and tabular finished, target loss and max loss implemented --- .../defenses/adversarial_training/__init__.py | 14 +-- .../defenses/adversarial_training/config.py | 63 ++++++++----- .../defenses/adversarial_training/defense.py | 40 ++++---- .../defenses/adversarial_training/image.py | 9 +- .../defenses/adversarial_training/tabular.py | 51 ++++++++-- nebula/controller/scenarios.py | 10 +- .../frontend/config/participant.json.example | 9 +- .../js/deployment/adversarial-training.js | 92 +++++++++++-------- nebula/frontend/templates/deployment.html | 54 ++++------- 9 files changed, 189 insertions(+), 153 deletions(-) diff --git a/nebula/addons/defenses/adversarial_training/__init__.py b/nebula/addons/defenses/adversarial_training/__init__.py index c417b1b93..844956335 100644 --- a/nebula/addons/defenses/adversarial_training/__init__.py +++ b/nebula/addons/defenses/adversarial_training/__init__.py @@ -1,12 +1,9 @@ from nebula.addons.defenses.adversarial_training.defense import ( - CAA_TABULAR_DATASETS, ERR_ALPHA, ERR_APPLY_PROBABILITY, - ERR_CLIP_BOUNDS, ERR_EPSILON, ERR_IMAGE_ATTACK, - ERR_LOSS_WEIGHTS, - ERR_MIXED_WEIGHTS, + ERR_LOSS_INCREASE, ERR_MODE, ERR_STEPS, ERR_TABULAR_ATTACK, @@ -15,6 +12,7 @@ IMAGE_ADVERSARIAL_ATTACKS, IMAGE_DATASET_NORMALIZATION, TABULAR_ADVERSARIAL_ATTACKS, + TABULAR_ADVERSARIAL_DATASETS, AdversarialExampleGenerator, AdversarialTrainingConfig, AdversarialTrainingDefense, @@ -22,21 +20,17 @@ ImageFGSMGenerator, ImagePGDGenerator, TabularAdversarialExampleGenerator, - TabularCAAGenerator, TabularCAPGDGenerator, TabularConstraintSet, apply_adversarial_training_if_enabled, ) __all__ = [ - "CAA_TABULAR_DATASETS", "ERR_ALPHA", "ERR_APPLY_PROBABILITY", - "ERR_CLIP_BOUNDS", "ERR_EPSILON", "ERR_IMAGE_ATTACK", - "ERR_LOSS_WEIGHTS", - "ERR_MIXED_WEIGHTS", + "ERR_LOSS_INCREASE", "ERR_MODE", "ERR_STEPS", "ERR_TABULAR_ATTACK", @@ -45,6 +39,7 @@ "IMAGE_ADVERSARIAL_ATTACKS", "IMAGE_DATASET_NORMALIZATION", "TABULAR_ADVERSARIAL_ATTACKS", + "TABULAR_ADVERSARIAL_DATASETS", "AdversarialExampleGenerator", "AdversarialTrainingConfig", "AdversarialTrainingDefense", @@ -52,7 +47,6 @@ "ImageFGSMGenerator", "ImagePGDGenerator", "TabularAdversarialExampleGenerator", - "TabularCAAGenerator", "TabularCAPGDGenerator", "TabularConstraintSet", "apply_adversarial_training_if_enabled", diff --git a/nebula/addons/defenses/adversarial_training/config.py b/nebula/addons/defenses/adversarial_training/config.py index fcb6c5aa2..ca003a87f 100644 --- a/nebula/addons/defenses/adversarial_training/config.py +++ b/nebula/addons/defenses/adversarial_training/config.py @@ -3,18 +3,16 @@ IMAGE_ADVERSARIAL_ATTACKS = {"fgsm", "pgd"} TABULAR_ADVERSARIAL_ATTACKS = {"capgd"} -CAA_TABULAR_DATASETS = {"AdultCensus"} +TABULAR_ADVERSARIAL_DATASETS = {"AdultCensus"} ERR_IMAGE_ATTACK = "image adversarial_training.attack must be one of: fgsm, pgd" ERR_TABULAR_ATTACK = "tabular adversarial_training.attack must be one of: capgd" -ERR_MODE = "adversarial_training.mode must be one of: clean, adversarial, mixed" +ERR_MODE = "adversarial_training.mode must be one of: adversarial, mixed" ERR_EPSILON = "adversarial_training.epsilon must be >= 0" ERR_ALPHA = "adversarial_training.alpha must be >= 0" ERR_STEPS = "adversarial_training.steps must be >= 1" ERR_APPLY_PROBABILITY = "adversarial_training.apply_probability must be in [0, 1]" -ERR_LOSS_WEIGHTS = "adversarial_training loss weights must be >= 0" -ERR_MIXED_WEIGHTS = "adversarial_training mixed mode requires at least one positive loss weight" -ERR_CLIP_BOUNDS = "adversarial_training.clip_min must be smaller than clip_max" +ERR_LOSS_INCREASE = "adversarial_training loss increase thresholds must be >= 0 and target <= max" ERR_TABULAR_METADATA = "Tabular adversarial training requires tabular_metadata" ERR_UNSUPPORTED_ATTACK = "Unsupported adversarial training attack: {attack}" @@ -36,13 +34,13 @@ class AdversarialTrainingConfig: epsilon: float = 8.0 / 255.0 alpha: float | None = None steps: int = 1 + mode: str = "mixed" clean_weight: float = 0.5 adversarial_weight: float = 0.5 - mode: str = "mixed" - apply_probability: float = 1.0 - clip_min: float = 0.0 - clip_max: float = 1.0 + apply_probability: float = 0.3 log_adversarial_metrics: bool = True + target_loss_increase: float | None = None + max_loss_increase: float | None = None def config_from_participant(participant_config: dict[str, Any]) -> AdversarialTrainingConfig | None: @@ -54,8 +52,9 @@ def config_from_participant(participant_config: dict[str, Any]) -> AdversarialTr dataset_name = participant_config.get("data_args", {}).get("dataset") domain = str(raw.get("domain", "image")).lower() attack = str(raw.get("attack", "capgd" if domain == "tabular" else "fgsm")).lower() - if domain == "tabular" and attack == "caa": - attack = "capgd" + + mode = str(raw.get("mode", "mixed")).lower() + clean_weight, adversarial_weight = _loss_weights_for_mode(mode) return AdversarialTrainingConfig( enabled=True, @@ -65,19 +64,29 @@ def config_from_participant(participant_config: dict[str, Any]) -> AdversarialTr epsilon=float(raw.get("epsilon", 8.0 / 255.0)), alpha=float(raw["alpha"]) if raw.get("alpha") is not None else None, steps=int(raw.get("steps", 1)), - clean_weight=float(raw.get("clean_weight", 0.5)), - adversarial_weight=float(raw.get("adversarial_weight", 0.5)), - mode=str(raw.get("mode", "mixed")).lower(), - apply_probability=float(raw.get("apply_probability", 1.0)), - clip_min=float(raw.get("clip_min", 0.0)), - clip_max=float(raw.get("clip_max", 1.0)), - log_adversarial_metrics=bool(raw.get("log_adversarial_metrics", True)), + mode=mode, + clean_weight=clean_weight, + adversarial_weight=adversarial_weight, + apply_probability=float(raw.get("apply_probability", 0.3)), + log_adversarial_metrics=True, + target_loss_increase=float(raw["target_loss_increase"]) + if raw.get("target_loss_increase") is not None + else None, + max_loss_increase=float(raw["max_loss_increase"]) + if raw.get("max_loss_increase") is not None + else None, ) +def _loss_weights_for_mode(mode: str) -> tuple[float, float]: + if mode == "adversarial": + return 0.0, 1.0 + return 0.5, 0.5 + + def validate_config(config: AdversarialTrainingConfig) -> None: # Fail early when a frontend/backend config value cannot produce a valid attack. - if config.mode not in {"clean", "adversarial", "mixed"}: + if config.mode not in {"adversarial", "mixed"}: raise ValueError(ERR_MODE) if config.domain == "image" and config.attack not in IMAGE_ADVERSARIAL_ATTACKS: raise ValueError(ERR_IMAGE_ATTACK) @@ -91,9 +100,13 @@ def validate_config(config: AdversarialTrainingConfig) -> None: raise ValueError(ERR_STEPS) if not 0.0 <= config.apply_probability <= 1.0: raise ValueError(ERR_APPLY_PROBABILITY) - if config.clean_weight < 0 or config.adversarial_weight < 0: - raise ValueError(ERR_LOSS_WEIGHTS) - if config.mode == "mixed" and config.clean_weight + config.adversarial_weight == 0: - raise ValueError(ERR_MIXED_WEIGHTS) - if config.clip_min >= config.clip_max: - raise ValueError(ERR_CLIP_BOUNDS) + if config.target_loss_increase is not None and config.target_loss_increase < 0: + raise ValueError(ERR_LOSS_INCREASE) + if config.max_loss_increase is not None and config.max_loss_increase < 0: + raise ValueError(ERR_LOSS_INCREASE) + if ( + config.target_loss_increase is not None + and config.max_loss_increase is not None + and config.target_loss_increase > config.max_loss_increase + ): + raise ValueError(ERR_LOSS_INCREASE) diff --git a/nebula/addons/defenses/adversarial_training/defense.py b/nebula/addons/defenses/adversarial_training/defense.py index d4e8da2fd..259784ab4 100644 --- a/nebula/addons/defenses/adversarial_training/defense.py +++ b/nebula/addons/defenses/adversarial_training/defense.py @@ -5,14 +5,11 @@ from nebula.addons.defenses.adversarial_training.base import AdversarialExampleGenerator from nebula.addons.defenses.adversarial_training.config import ( - CAA_TABULAR_DATASETS, ERR_ALPHA, ERR_APPLY_PROBABILITY, - ERR_CLIP_BOUNDS, ERR_EPSILON, ERR_IMAGE_ATTACK, - ERR_LOSS_WEIGHTS, - ERR_MIXED_WEIGHTS, + ERR_LOSS_INCREASE, ERR_MODE, ERR_STEPS, ERR_TABULAR_ATTACK, @@ -21,6 +18,7 @@ IMAGE_ADVERSARIAL_ATTACKS, IMAGE_DATASET_NORMALIZATION, TABULAR_ADVERSARIAL_ATTACKS, + TABULAR_ADVERSARIAL_DATASETS, AdversarialTrainingConfig, config_from_participant, validate_config, @@ -33,7 +31,6 @@ from nebula.addons.defenses.adversarial_training.logging import AdversarialTrainingSampleLogger from nebula.addons.defenses.adversarial_training.tabular import ( TabularAdversarialExampleGenerator, - TabularCAAGenerator, TabularCAPGDGenerator, TabularConstraintSet, ) @@ -126,12 +123,6 @@ def compute_training_step(self, model, x, y, criterion): loss = criterion(logits, y) return loss, logits, {} - # "clean" mode keeps the normal training step but still goes through the defense hook. - if self.config.mode == "clean": - logits = model(x) - loss = criterion(logits, y) - return loss, logits, {} - # Generate x_adv once and reuse it for logging, adversarial loss and metrics. x_adv = self.generator.generate(model, x, y, criterion) self._log_adversarial_samples(model, x, x_adv, y) @@ -147,11 +138,8 @@ def compute_training_step(self, model, x, y, criterion): clean_logits = model(x) clean_loss = criterion(clean_logits, y) - total_weight = self.config.clean_weight + self.config.adversarial_weight - # "mixed" combines clean and adversarial losses with user-provided weights. - loss = ( - self.config.clean_weight * clean_loss + self.config.adversarial_weight * adv_loss - ) / total_weight + # "mixed" uses a fixed 50/50 clean/adversarial objective. + loss = self.config.clean_weight * clean_loss + self.config.adversarial_weight * adv_loss return loss, clean_logits, self._extra_metrics({ "Clean Loss": clean_loss, @@ -176,7 +164,7 @@ def _extra_metrics(self, metrics): def _log_tabular_metadata(tabular_metadata: TabularAdversarialMetadata) -> None: - # Log a compact metadata summary to make CAA setup auditable. + # Log a compact metadata summary to make CAPGD setup auditable. integer_features = _feature_names_by_type(tabular_metadata, {INTEGER}) continuous_features = _feature_names_by_type(tabular_metadata, {CONTINUOUS}) categorical_features = _feature_names_by_type(tabular_metadata, {CATEGORICAL}) @@ -225,26 +213,32 @@ def apply_adversarial_training_if_enabled(model, participant_config: dict[str, A model.set_adversarial_training(defense) logging.info( "[AdversarialTrainingDefense] Enabled | dataset=%s | attack=%s | epsilon_max=%s | " - "epsilon_range=[%.6f, %.6f] | epsilon_step=%.6f | mode=%s", + "epsilon_range=[%.6f, %.6f] | epsilon_step=%.6f | steps=%s | mode=%s | " + "clean_weight=%.2f | adversarial_weight=%.2f | apply_probability=%.2f | " + "target_loss_increase=%s | max_loss_increase=%s | log_adversarial_metrics=%s", defense.config.dataset_name, defense.config.attack, defense.config.epsilon, defense.config.epsilon / 4.0, defense.config.epsilon, defense.config.epsilon / 8.0, + defense.config.steps, defense.config.mode, + defense.config.clean_weight, + defense.config.adversarial_weight, + defense.config.apply_probability, + defense.config.target_loss_increase, + defense.config.max_loss_increase, + defense.config.log_adversarial_metrics, ) __all__ = [ - "CAA_TABULAR_DATASETS", "ERR_ALPHA", "ERR_APPLY_PROBABILITY", - "ERR_CLIP_BOUNDS", "ERR_EPSILON", "ERR_IMAGE_ATTACK", - "ERR_LOSS_WEIGHTS", - "ERR_MIXED_WEIGHTS", + "ERR_LOSS_INCREASE", "ERR_MODE", "ERR_STEPS", "ERR_TABULAR_ATTACK", @@ -253,6 +247,7 @@ def apply_adversarial_training_if_enabled(model, participant_config: dict[str, A "IMAGE_ADVERSARIAL_ATTACKS", "IMAGE_DATASET_NORMALIZATION", "TABULAR_ADVERSARIAL_ATTACKS", + "TABULAR_ADVERSARIAL_DATASETS", "AdversarialExampleGenerator", "AdversarialTrainingConfig", "AdversarialTrainingDefense", @@ -260,7 +255,6 @@ def apply_adversarial_training_if_enabled(model, participant_config: dict[str, A "ImageFGSMGenerator", "ImagePGDGenerator", "TabularAdversarialExampleGenerator", - "TabularCAAGenerator", "TabularCAPGDGenerator", "TabularConstraintSet", "apply_adversarial_training_if_enabled", diff --git a/nebula/addons/defenses/adversarial_training/image.py b/nebula/addons/defenses/adversarial_training/image.py index cd6dbd129..585231f32 100644 --- a/nebula/addons/defenses/adversarial_training/image.py +++ b/nebula/addons/defenses/adversarial_training/image.py @@ -3,6 +3,9 @@ from nebula.addons.defenses.adversarial_training.base import AdversarialExampleGenerator from nebula.addons.defenses.adversarial_training.config import AdversarialTrainingConfig +IMAGE_CLIP_MIN = 0.0 +IMAGE_CLIP_MAX = 1.0 + class ImageAdversarialExampleGenerator(AdversarialExampleGenerator): def __init__(self, config: AdversarialTrainingConfig, mean: tuple[float, ...], std: tuple[float, ...]): @@ -33,15 +36,15 @@ def _bounds(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: # Convert valid pixel bounds to the normalized space where the model operates. mean = self._channel_tensor(self.mean, x) std = self._channel_tensor(self.std, x) - lower = (float(self.config.clip_min) - mean) / std - upper = (float(self.config.clip_max) - mean) / std + lower = (IMAGE_CLIP_MIN - mean) / std + upper = (IMAGE_CLIP_MAX - mean) / std return lower, upper def denormalize(self, x: torch.Tensor) -> torch.Tensor: # Convert normalized tensors back to pixel scale for logging. mean = self._channel_tensor(self.mean, x) std = self._channel_tensor(self.std, x) - return (x * std + mean).clamp(float(self.config.clip_min), float(self.config.clip_max)) + return (x * std + mean).clamp(IMAGE_CLIP_MIN, IMAGE_CLIP_MAX) def _project(self, x_adv: torch.Tensor, x_clean: torch.Tensor, epsilon: float) -> torch.Tensor: # Keep the adversarial image inside both the epsilon ball and valid pixel bounds. diff --git a/nebula/addons/defenses/adversarial_training/tabular.py b/nebula/addons/defenses/adversarial_training/tabular.py index 2d1d954e8..4786a6ecf 100644 --- a/nebula/addons/defenses/adversarial_training/tabular.py +++ b/nebula/addons/defenses/adversarial_training/tabular.py @@ -40,7 +40,7 @@ def perturbable_mask(self, x: torch.Tensor) -> torch.Tensor: return self.tensors(x)["perturbable"] def project(self, x_candidate: torch.Tensor, x_clean: torch.Tensor, epsilon: float) -> torch.Tensor: - """Clamp numeric features, round integers, restore immutable features and fix one-hot groups.""" + # Clamp numeric features, round integers, restore immutable features and fix one-hot groups. tensors = self.tensors(x_clean) lower, upper = self._bounds(x_clean, epsilon, tensors) @@ -152,6 +152,10 @@ def _margin(self, logits: torch.Tensor, y: torch.Tensor) -> torch.Tensor: other_logits = logits.masked_fill(true_class_mask, float("-inf")) return other_logits.max(dim=1).values - true_logits + def _per_sample_loss(self, logits: torch.Tensor, y: torch.Tensor) -> torch.Tensor: + # CAPGD needs per-sample scores so each row can stop once it is hard enough. + return F.cross_entropy(logits, y, reduction="none") + class TabularCAPGDGenerator(TabularAdversarialExampleGenerator): """First-phase constrained tabular CAPGD generator.""" @@ -170,6 +174,8 @@ def generate(self, model, x, y, criterion): x_adv = x_clean.clone() best_adv = x_adv.clone() best_score = torch.full((x_clean.size(0),), float("-inf"), dtype=x_clean.dtype, device=x_clean.device) + use_loss_window = self._use_loss_window() + clean_loss = self._clean_loss(model, x_clean, y) if use_loss_window else None for _ in range(steps): # CAPGD step: move in the sign of the loss gradient, but only on perturbable features. @@ -184,16 +190,49 @@ def generate(self, model, x, y, criterion): candidate = self.constraints.project(candidate, x_clean, epsilon) with torch.no_grad(): - # Keep the strongest candidate per sample, not just the last step. - candidate_score = self._margin(model(candidate), y) - better = candidate_score > best_score + # Keep the best candidate per sample, not just the last step. + candidate_logits = model(candidate) + if use_loss_window: + candidate_score = self._loss_increase(candidate_logits, y, clean_loss) + better = self._loss_window_better(candidate_score, best_score) + else: + candidate_score = self._margin(candidate_logits, y) + better = candidate_score > best_score best_adv = torch.where(better.view(-1, 1), candidate, best_adv) best_score = torch.where(better, candidate_score, best_score) + if self._target_reached(best_score): + break + x_adv = candidate return best_adv.detach() + def _use_loss_window(self) -> bool: + return self.config.target_loss_increase is not None or self.config.max_loss_increase is not None + + def _clean_loss(self, model, x_clean: torch.Tensor, y: torch.Tensor) -> torch.Tensor: + # Baseline difficulty. Candidate scores become loss(candidate) - loss(clean). + with torch.no_grad(): + return self._per_sample_loss(model(x_clean), y) -# Compatibility alias while old configs/UI still refer to the future CAA attack. -TabularCAAGenerator = TabularCAPGDGenerator + def _loss_increase( + self, + candidate_logits: torch.Tensor, + y: torch.Tensor, + clean_loss: torch.Tensor, + ) -> torch.Tensor: + return self._per_sample_loss(candidate_logits, y) - clean_loss + + def _loss_window_better(self, candidate_score: torch.Tensor, best_score: torch.Tensor) -> torch.Tensor: + # A candidate must make the sample harder. If max_loss_increase is set, reject overshoots. + valid = candidate_score > 0.0 + if self.config.max_loss_increase is not None: + valid = valid & (candidate_score <= float(self.config.max_loss_increase)) + return valid & (candidate_score > best_score) + + def _target_reached(self, best_score: torch.Tensor) -> bool: + # Once every sample has reached the requested hardness, stop taking stronger steps. + if self.config.target_loss_increase is None: + return False + return bool((best_score >= float(self.config.target_loss_increase)).all().item()) diff --git a/nebula/controller/scenarios.py b/nebula/controller/scenarios.py index f9fa18a17..7325e650a 100644 --- a/nebula/controller/scenarios.py +++ b/nebula/controller/scenarios.py @@ -748,11 +748,9 @@ def __init__(self, scenario, user=None): for key in ( "epsilon", "alpha", - "clean_weight", - "adversarial_weight", "apply_probability", - "clip_min", - "clip_max", + "target_loss_increase", + "max_loss_increase", ): if key in adversarial_training and adversarial_training[key] is not None: participant_config["defense_args"]["adversarial_training"][key] = float( @@ -766,10 +764,6 @@ def __init__(self, scenario, user=None): participant_config["defense_args"]["adversarial_training"]["mode"] = str( adversarial_training["mode"] ) - if "log_adversarial_metrics" in adversarial_training: - participant_config["defense_args"]["adversarial_training"]["log_adversarial_metrics"] = bool( - adversarial_training["log_adversarial_metrics"] - ) participant_config["device_args"]["accelerator"] = self.scenario.accelerator participant_config["device_args"]["gpu_id"] = self.scenario.gpu_id participant_config["device_args"]["logging"] = self.scenario.logginglevel diff --git a/nebula/frontend/config/participant.json.example b/nebula/frontend/config/participant.json.example index e3c65f409..88c017ba1 100755 --- a/nebula/frontend/config/participant.json.example +++ b/nebula/frontend/config/participant.json.example @@ -115,13 +115,10 @@ "attack": "fgsm", "epsilon": 0.03, "steps": 1, - "clean_weight": 0.5, - "adversarial_weight": 0.5, "mode": "mixed", - "apply_probability": 1.0, - "clip_min": 0.0, - "clip_max": 1.0, - "log_adversarial_metrics": true + "apply_probability": 0.3, + "target_loss_increase": null, + "max_loss_increase": null }, "reputation": { "enabled": false, diff --git a/nebula/frontend/static/js/deployment/adversarial-training.js b/nebula/frontend/static/js/deployment/adversarial-training.js index ceb965246..b02addf4e 100644 --- a/nebula/frontend/static/js/deployment/adversarial-training.js +++ b/nebula/frontend/static/js/deployment/adversarial-training.js @@ -8,22 +8,20 @@ const AdversarialTrainingManager = (function() { alpha: null, steps: 1, mode: "mixed", - clean_weight: 0.5, - adversarial_weight: 0.5, - apply_probability: 1.0, - clip_min: 0.0, - clip_max: 1.0, - log_adversarial_metrics: true + apply_probability: 0.3, + log_adversarial_metrics: true, + target_loss_increase: null, + max_loss_increase: null }; const IMAGE_DATASETS = new Set(["MNIST", "FashionMNIST", "EMNIST", "CIFAR10", "CIFAR100"]); - const CAA_TABULAR_DATASETS = new Set(["AdultCensus"]); + const TABULAR_ADVERSARIAL_DATASETS = new Set(["AdultCensus"]); const IMAGE_ATTACK_OPTIONS = [ {value: "fgsm", label: "FGSM"}, {value: "pgd", label: "PGD"} ]; const TABULAR_ATTACK_OPTIONS = [ - {value: "caa", label: "CAA"} + {value: "capgd", label: "CAPGD"} ]; function initializeAdversarialTraining() { @@ -73,11 +71,16 @@ const AdversarialTrainingManager = (function() { function toggleAttackSettings(attack) { const pgdSettings = document.getElementById("adversarial-training-pgd-settings"); const stepsTitle = document.getElementById("adversarialTrainingStepsTitle"); + const lossWindowSettings = document.getElementById("adversarial-training-loss-window-settings"); + const domain = document.getElementById("adversarialTrainingDomain")?.value || DEFAULT_ADVERSARIAL_TRAINING_CONFIG.domain; if (!pgdSettings) return; - pgdSettings.style.display = ["pgd", "caa"].includes(attack) ? "block" : "none"; + pgdSettings.style.display = ["pgd", "capgd"].includes(attack) ? "block" : "none"; + if (lossWindowSettings) { + lossWindowSettings.style.display = domain === "tabular" ? "block" : "none"; + } if (stepsTitle) { - stepsTitle.textContent = attack === "caa" ? "CAA search steps" : "PGD steps"; + stepsTitle.textContent = domain === "tabular" ? "CAPGD steps" : "PGD steps"; } } @@ -91,7 +94,7 @@ const AdversarialTrainingManager = (function() { if (datasetNote) { datasetNote.style.display = domain === "unsupported" ? "block" : "none"; - datasetNote.textContent = "Adversarial Training for tabular datasets currently supports AdultCensus with CAA."; + datasetNote.textContent = "Adversarial Training for tabular datasets currently supports AdultCensus with CAPGD."; } if (domainInput) { domainInput.value = domain === "unsupported" ? "tabular" : domain; @@ -116,7 +119,7 @@ const AdversarialTrainingManager = (function() { if (IMAGE_DATASETS.has(dataset)) { return "image"; } - if (CAA_TABULAR_DATASETS.has(dataset)) { + if (TABULAR_ADVERSARIAL_DATASETS.has(dataset)) { return "tabular"; } return "unsupported"; @@ -126,7 +129,7 @@ const AdversarialTrainingManager = (function() { const attackSelect = document.getElementById("adversarialTrainingAttack"); if (!attackSelect) return; - // Tabular datasets intentionally expose only CAA; image datasets expose FGSM/PGD. + // Tabular datasets intentionally expose only CAPGD; image datasets expose FGSM/PGD. const options = domain === "tabular" ? TABULAR_ATTACK_OPTIONS : IMAGE_ATTACK_OPTIONS; const currentAttack = preferredAttack || attackSelect.value; attackSelect.innerHTML = ""; @@ -167,7 +170,7 @@ const AdversarialTrainingManager = (function() { function getAdversarialTrainingConfig() { const domain = document.getElementById("adversarialTrainingDomain")?.value || DEFAULT_ADVERSARIAL_TRAINING_CONFIG.domain; const attack = domain === "tabular" - ? "caa" + ? "capgd" : (document.getElementById("adversarialTrainingAttack")?.value || DEFAULT_ADVERSARIAL_TRAINING_CONFIG.attack); const config = { enabled: Boolean(document.getElementById("adversarialTrainingSwitch")?.checked), @@ -177,17 +180,27 @@ const AdversarialTrainingManager = (function() { alpha: optionalNumberValue("adversarialTrainingAlpha", DEFAULT_ADVERSARIAL_TRAINING_CONFIG.alpha), steps: integerValue("adversarialTrainingSteps", DEFAULT_ADVERSARIAL_TRAINING_CONFIG.steps), mode: document.getElementById("adversarialTrainingMode")?.value || DEFAULT_ADVERSARIAL_TRAINING_CONFIG.mode, - clean_weight: numberValue("adversarialTrainingCleanWeight", DEFAULT_ADVERSARIAL_TRAINING_CONFIG.clean_weight), - adversarial_weight: numberValue("adversarialTrainingAdversarialWeight", DEFAULT_ADVERSARIAL_TRAINING_CONFIG.adversarial_weight), apply_probability: numberValue("adversarialTrainingApplyProbability", DEFAULT_ADVERSARIAL_TRAINING_CONFIG.apply_probability), - clip_min: numberValue("adversarialTrainingClipMin", DEFAULT_ADVERSARIAL_TRAINING_CONFIG.clip_min), - clip_max: numberValue("adversarialTrainingClipMax", DEFAULT_ADVERSARIAL_TRAINING_CONFIG.clip_max), - log_adversarial_metrics: Boolean(document.getElementById("adversarialTrainingLogMetrics")?.checked) + target_loss_increase: optionalNumberValue( + "adversarialTrainingTargetLossIncrease", + DEFAULT_ADVERSARIAL_TRAINING_CONFIG.target_loss_increase + ), + max_loss_increase: optionalNumberValue( + "adversarialTrainingMaxLossIncrease", + DEFAULT_ADVERSARIAL_TRAINING_CONFIG.max_loss_increase + ), + log_adversarial_metrics: true }; if (config.alpha === null || config.attack !== "pgd") { delete config.alpha; } + if (config.target_loss_increase === null) { + delete config.target_loss_increase; + } + if (config.max_loss_increase === null) { + delete config.max_loss_increase; + } return config; } @@ -204,17 +217,15 @@ const AdversarialTrainingManager = (function() { setValue("adversarialTrainingEpsilon", adversarialTrainingConfig.epsilon); setValue("adversarialTrainingAlpha", adversarialTrainingConfig.alpha ?? ""); setValue("adversarialTrainingSteps", adversarialTrainingConfig.steps); - setValue("adversarialTrainingMode", adversarialTrainingConfig.mode); - setValue("adversarialTrainingCleanWeight", adversarialTrainingConfig.clean_weight); - setValue("adversarialTrainingAdversarialWeight", adversarialTrainingConfig.adversarial_weight); + setValue( + "adversarialTrainingMode", + ["mixed", "adversarial"].includes(adversarialTrainingConfig.mode) + ? adversarialTrainingConfig.mode + : DEFAULT_ADVERSARIAL_TRAINING_CONFIG.mode + ); setValue("adversarialTrainingApplyProbability", adversarialTrainingConfig.apply_probability); - setValue("adversarialTrainingClipMin", adversarialTrainingConfig.clip_min); - setValue("adversarialTrainingClipMax", adversarialTrainingConfig.clip_max); - - const logMetricsInput = document.getElementById("adversarialTrainingLogMetrics"); - if (logMetricsInput) { - logMetricsInput.checked = Boolean(adversarialTrainingConfig.log_adversarial_metrics); - } + setValue("adversarialTrainingTargetLossIncrease", adversarialTrainingConfig.target_loss_increase ?? ""); + setValue("adversarialTrainingMaxLossIncrease", adversarialTrainingConfig.max_loss_increase ?? ""); updateDatasetAvailability(); const domain = document.getElementById("adversarialTrainingDomain")?.value || adversarialTrainingConfig.domain; @@ -241,20 +252,27 @@ const AdversarialTrainingManager = (function() { if (config.epsilon < 0) { return "[Adversarial Training] Epsilon must be greater than or equal to 0."; } - if (["pgd", "caa"].includes(config.attack) && config.steps < 1) { + if (["pgd", "capgd"].includes(config.attack) && config.steps < 1) { return "[Adversarial Training] Search steps must be at least 1."; } - if (config.clean_weight < 0 || config.adversarial_weight < 0) { - return "[Adversarial Training] Loss weights must be greater than or equal to 0."; - } - if (config.mode === "mixed" && config.clean_weight + config.adversarial_weight === 0) { - return "[Adversarial Training] Mixed mode needs at least one positive loss weight."; + if (!["mixed", "adversarial"].includes(config.mode)) { + return "[Adversarial Training] Training mode must be Clean + adversarial or Adversarial only."; } if (config.apply_probability < 0 || config.apply_probability > 1) { return "[Adversarial Training] Apply probability must be between 0 and 1."; } - if (config.clip_min >= config.clip_max) { - return "[Adversarial Training] Min bound must be smaller than max bound."; + if (config.target_loss_increase !== undefined && config.target_loss_increase < 0) { + return "[Adversarial Training] Target loss increase must be greater than or equal to 0."; + } + if (config.max_loss_increase !== undefined && config.max_loss_increase < 0) { + return "[Adversarial Training] Max loss increase must be greater than or equal to 0."; + } + if ( + config.target_loss_increase !== undefined + && config.max_loss_increase !== undefined + && config.target_loss_increase > config.max_loss_increase + ) { + return "[Adversarial Training] Target loss increase must be smaller than or equal to max loss increase."; } return null; } diff --git a/nebula/frontend/templates/deployment.html b/nebula/frontend/templates/deployment.html index 5ad22ac87..bfa249739 100755 --- a/nebula/frontend/templates/deployment.html +++ b/nebula/frontend/templates/deployment.html @@ -588,7 +588,7 @@
    Enable/Disable Adversarial Training
    style="display: inline; width: 80px; height: 30px;"> +
    Apply probability
    +
    + +
    + From 800b5931edca5371185bb5206ca3ba014e800878 Mon Sep 17 00:00:00 2001 From: "Juan J." Date: Fri, 5 Jun 2026 18:30:03 +0200 Subject: [PATCH 61/66] CAPGD name changed to constrained PGD. Adult Cencus and Breast Cancer finished. Other minor changes. --- .../defenses/adversarial_training/__init__.py | 4 +- .../defenses/adversarial_training/config.py | 8 +- .../defenses/adversarial_training/defense.py | 8 +- .../defenses/adversarial_training/tabular.py | 12 +- .../trustworthiness/factsheet_common.py | 42 +++- .../core/datasets/adultcensus/adultcensus.py | 187 ++++++++---------- .../datasets/breast_cancer/breast_cancer.py | 148 +++++++------- nebula/core/datasets/covtype/covtype.py | 133 +++++++------ nebula/core/datasets/tabular_metadata.py | 147 ++++++++++++++ .../js/deployment/adversarial-training.js | 16 +- nebula/frontend/templates/deployment.html | 4 +- 11 files changed, 445 insertions(+), 264 deletions(-) diff --git a/nebula/addons/defenses/adversarial_training/__init__.py b/nebula/addons/defenses/adversarial_training/__init__.py index 844956335..ddc977538 100644 --- a/nebula/addons/defenses/adversarial_training/__init__.py +++ b/nebula/addons/defenses/adversarial_training/__init__.py @@ -20,7 +20,7 @@ ImageFGSMGenerator, ImagePGDGenerator, TabularAdversarialExampleGenerator, - TabularCAPGDGenerator, + TabularConstrainedPGDGenerator, TabularConstraintSet, apply_adversarial_training_if_enabled, ) @@ -47,7 +47,7 @@ "ImageFGSMGenerator", "ImagePGDGenerator", "TabularAdversarialExampleGenerator", - "TabularCAPGDGenerator", + "TabularConstrainedPGDGenerator", "TabularConstraintSet", "apply_adversarial_training_if_enabled", ] diff --git a/nebula/addons/defenses/adversarial_training/config.py b/nebula/addons/defenses/adversarial_training/config.py index ca003a87f..a5ca04a15 100644 --- a/nebula/addons/defenses/adversarial_training/config.py +++ b/nebula/addons/defenses/adversarial_training/config.py @@ -2,11 +2,11 @@ from typing import Any IMAGE_ADVERSARIAL_ATTACKS = {"fgsm", "pgd"} -TABULAR_ADVERSARIAL_ATTACKS = {"capgd"} -TABULAR_ADVERSARIAL_DATASETS = {"AdultCensus"} +TABULAR_ADVERSARIAL_ATTACKS = {"constrained_pgd"} +TABULAR_ADVERSARIAL_DATASETS = {"AdultCensus", "BreastCancer", "Covtype"} ERR_IMAGE_ATTACK = "image adversarial_training.attack must be one of: fgsm, pgd" -ERR_TABULAR_ATTACK = "tabular adversarial_training.attack must be one of: capgd" +ERR_TABULAR_ATTACK = "tabular adversarial_training.attack must be one of: constrained_pgd" ERR_MODE = "adversarial_training.mode must be one of: adversarial, mixed" ERR_EPSILON = "adversarial_training.epsilon must be >= 0" ERR_ALPHA = "adversarial_training.alpha must be >= 0" @@ -51,7 +51,7 @@ def config_from_participant(participant_config: dict[str, Any]) -> AdversarialTr dataset_name = participant_config.get("data_args", {}).get("dataset") domain = str(raw.get("domain", "image")).lower() - attack = str(raw.get("attack", "capgd" if domain == "tabular" else "fgsm")).lower() + attack = str(raw.get("attack", "constrained_pgd" if domain == "tabular" else "fgsm")).lower() mode = str(raw.get("mode", "mixed")).lower() clean_weight, adversarial_weight = _loss_weights_for_mode(mode) diff --git a/nebula/addons/defenses/adversarial_training/defense.py b/nebula/addons/defenses/adversarial_training/defense.py index 259784ab4..4cd6b6923 100644 --- a/nebula/addons/defenses/adversarial_training/defense.py +++ b/nebula/addons/defenses/adversarial_training/defense.py @@ -31,7 +31,7 @@ from nebula.addons.defenses.adversarial_training.logging import AdversarialTrainingSampleLogger from nebula.addons.defenses.adversarial_training.tabular import ( TabularAdversarialExampleGenerator, - TabularCAPGDGenerator, + TabularConstrainedPGDGenerator, TabularConstraintSet, ) from nebula.core.datasets.tabular_metadata import CATEGORICAL, CONTINUOUS, INTEGER, TabularAdversarialMetadata @@ -63,7 +63,7 @@ def from_participant_config( if config.domain == "tabular": metadata = cls._get_tabular_metadata(partition) - return cls(config=config, generator=TabularCAPGDGenerator(config, metadata)) + return cls(config=config, generator=TabularConstrainedPGDGenerator(config, metadata)) if config.domain == "image": # Image attacks run in normalized model space, so each dataset must provide mean/std. @@ -164,7 +164,7 @@ def _extra_metrics(self, metrics): def _log_tabular_metadata(tabular_metadata: TabularAdversarialMetadata) -> None: - # Log a compact metadata summary to make CAPGD setup auditable. + # Log a compact metadata summary to make constrained PGD setup auditable. integer_features = _feature_names_by_type(tabular_metadata, {INTEGER}) continuous_features = _feature_names_by_type(tabular_metadata, {CONTINUOUS}) categorical_features = _feature_names_by_type(tabular_metadata, {CATEGORICAL}) @@ -255,7 +255,7 @@ def apply_adversarial_training_if_enabled(model, participant_config: dict[str, A "ImageFGSMGenerator", "ImagePGDGenerator", "TabularAdversarialExampleGenerator", - "TabularCAPGDGenerator", + "TabularConstrainedPGDGenerator", "TabularConstraintSet", "apply_adversarial_training_if_enabled", ] diff --git a/nebula/addons/defenses/adversarial_training/tabular.py b/nebula/addons/defenses/adversarial_training/tabular.py index 4786a6ecf..2ae280be0 100644 --- a/nebula/addons/defenses/adversarial_training/tabular.py +++ b/nebula/addons/defenses/adversarial_training/tabular.py @@ -15,7 +15,7 @@ def __init__(self, metadata: TabularAdversarialMetadata): self._tensor_cache: dict[tuple[torch.device, torch.dtype], dict[str, torch.Tensor]] = {} def tensors(self, x: torch.Tensor) -> dict[str, torch.Tensor]: - # Masks and bounds are reused in every CAPGD step, so build them once for each tensor placement. + # Masks and bounds are reused in every constrained PGD step, so build them once per placement. key = (x.device, x.dtype) cached = self._tensor_cache.get(key) if cached is not None: @@ -140,7 +140,7 @@ def __init__(self, config: AdversarialTrainingConfig, metadata: TabularAdversari self.constraints = TabularConstraintSet(metadata) def _alpha(self, epsilon: float) -> float: - # By default, distribute the epsilon budget evenly across CAPGD steps. + # By default, distribute the epsilon budget evenly across constrained PGD steps. if self.config.alpha is not None: return float(self.config.alpha) return float(epsilon) / max(int(self.config.steps), 1) @@ -153,12 +153,12 @@ def _margin(self, logits: torch.Tensor, y: torch.Tensor) -> torch.Tensor: return other_logits.max(dim=1).values - true_logits def _per_sample_loss(self, logits: torch.Tensor, y: torch.Tensor) -> torch.Tensor: - # CAPGD needs per-sample scores so each row can stop once it is hard enough. + # The attack needs per-sample scores so each row can stop once it is hard enough. return F.cross_entropy(logits, y, reduction="none") -class TabularCAPGDGenerator(TabularAdversarialExampleGenerator): - """First-phase constrained tabular CAPGD generator.""" +class TabularConstrainedPGDGenerator(TabularAdversarialExampleGenerator): + """Constrained PGD generator for tabular adversarial examples.""" def generate(self, model, x, y, criterion): # Sample one attack strength for this batch, matching the image generator behavior. @@ -178,7 +178,7 @@ def generate(self, model, x, y, criterion): clean_loss = self._clean_loss(model, x_clean, y) if use_loss_window else None for _ in range(steps): - # CAPGD step: move in the sign of the loss gradient, but only on perturbable features. + # PGD step: move in the sign of the loss gradient, but only on perturbable features. x_grad = x_adv.detach().requires_grad_(True) logits = model(x_grad) loss = criterion(logits, y) diff --git a/nebula/addons/trustworthiness/factsheet_common.py b/nebula/addons/trustworthiness/factsheet_common.py index 7cfbe11d9..9c08c62c2 100644 --- a/nebula/addons/trustworthiness/factsheet_common.py +++ b/nebula/addons/trustworthiness/factsheet_common.py @@ -90,6 +90,28 @@ def inverse_score(value): return 1 / (1 + value) +def get_enabled_defences(data): + # Return the active training-time defences declared in the scenario. + defences = [] + if data.get("reputation", {}).get("enabled", False): + defences.append("reputation-based defence") + if data.get("feature_squeezing", {}).get("enabled", False): + defences.append("feature squeezing") + if data.get("adversarial_training", {}).get("enabled", False): + defences.append(_format_adversarial_training_defence(data["adversarial_training"])) + return defences + + +def _format_adversarial_training_defence(adversarial_training): + attack = str(adversarial_training.get("attack", "")).upper() + domain = str(adversarial_training.get("domain", "")).lower() + if attack: + return f"adversarial training with {attack}" + if domain: + return f"adversarial training for {domain} data" + return "adversarial training" + + def build_project_background(data): # Build the natural-language scenario description used in factsheets. federation = data["federation"] @@ -99,7 +121,7 @@ def build_project_background(data): aggregation_algorithm = data["agg_algorithm"] n_rounds = int(data["rounds"]) attack = data["attack_params"]["attacks"] - with_reputation = data["reputation"]["enabled"] + enabled_defences = get_enabled_defences(data) base = ( "For the project setup, the most important aspects are the following: " @@ -113,8 +135,9 @@ def build_project_background(data): else: attack_text = "No attacks are used. " - if with_reputation: - defence_text = "A reputation-based defence is used, and the trustworthiness of the project is desired." + if enabled_defences: + defence_list = ", ".join(enabled_defences) + defence_text = f"The active defence mechanisms are: {defence_list}. The trustworthiness of the project is desired." else: defence_text = "No defence mechanism is used, and the trustworthiness of the project is desired." @@ -145,6 +168,19 @@ def populate_common_pre_train_sections(factsheet, data, model): factsheet["configuration"]["reputation_enabled"] = bool( data.get("reputation", {}).get("enabled", False) ) + adversarial_training = data.get("adversarial_training", {}) or {} + factsheet["configuration"]["adversarial_training"] = bool( + adversarial_training.get("enabled", False) + ) + factsheet["configuration"]["adversarial_training_domain"] = ( + adversarial_training.get("domain", "") if adversarial_training.get("enabled", False) else "" + ) + factsheet["configuration"]["adversarial_training_attack"] = ( + adversarial_training.get("attack", "") if adversarial_training.get("enabled", False) else "" + ) + factsheet["configuration"]["adversarial_training_mode"] = ( + adversarial_training.get("mode", "") if adversarial_training.get("enabled", False) else "" + ) factsheet["configuration"]["visualization"] = True factsheet["configuration"]["monitoring"] = True factsheet["configuration"]["total_round_num"] = int(data["rounds"]) diff --git a/nebula/core/datasets/adultcensus/adultcensus.py b/nebula/core/datasets/adultcensus/adultcensus.py index 51aa5e7a2..062ffc380 100644 --- a/nebula/core/datasets/adultcensus/adultcensus.py +++ b/nebula/core/datasets/adultcensus/adultcensus.py @@ -1,5 +1,6 @@ # nebula/core/datasets/adultcensus/adultcensus.py +import logging import os from typing import Any, ClassVar @@ -8,7 +9,11 @@ from torch.utils.data import Dataset from nebula.core.datasets.nebuladataset import NebulaDataset, NebulaPartitionHandler -from nebula.core.datasets.tabular_metadata import CATEGORICAL, CONTINUOUS, INTEGER, TabularAdversarialMetadata +from nebula.core.datasets.tabular_metadata import ( + build_tabular_adversarial_metadata, +) + +logger = logging.getLogger(__name__) class AdultCensusTorchDataset(Dataset): @@ -25,6 +30,7 @@ def __init__( continuous_features: list[int] | None = None, integer_features: list[int] | None = None, categorical_features: list[int] | None = None, + non_perturbable_features: list[int] | None = None, categorical_groups: list[list[int]] | None = None, tabular_metadata: dict | None = None, ): @@ -41,7 +47,7 @@ def __init__( self.x: np.ndarray = x.astype(np.float32, copy=False) self.y: np.ndarray = y_arr.astype(np.int64, copy=False) - # Nebula conventions + # Nebula dataset conventions used by partitioning, logging and model setup. self.data: np.ndarray = self.x self.targets: np.ndarray = self.y self.classes: list[str] = ["<=50K", ">50K"] @@ -49,6 +55,7 @@ def __init__( self.continuous_features = continuous_features or [] self.integer_features = integer_features or [] self.categorical_features = categorical_features or [] + self.non_perturbable_features = non_perturbable_features or [] self.categorical_groups = categorical_groups or [] self.tabular_metadata = tabular_metadata self.input_dim = int(self.x.shape[1]) @@ -120,6 +127,10 @@ class AdultCensusDataset(NebulaDataset): "sex", "native-country", ] + # Experimental wide attack surface for testing constrained PGD thoroughly. + # This intentionally allows broad changes, including categorical flips. + PERTURBABLE_INTEGER_COLUMNS: ClassVar[list[str]] = list(INTEGER_COLUMNS) + PERTURBABLE_CATEGORICAL_COLUMNS: ClassVar[list[str]] = list(CATEGORICAL_COLUMNS) def __init__( self, @@ -219,17 +230,18 @@ def load_adult_census_dataset(self) -> tuple[AdultCensusTorchDataset, AdultCensu "AdultCensusDataset requires pandas + scikit-learn. Install them (e.g., pip install pandas scikit-learn)." ) from e - # 1) Load from OpenML + # Raw Adult Census uses mixed pandas columns; the model receives the + # numeric matrix produced later by the ColumnTransformer. bunch = fetch_openml(data_id=1590, as_frame=True, data_home=data_dir) X_df = bunch.data.copy() y_raw = bunch.target - # 2) Target -> {0,1} - # Normalize spaces to avoid variants like ' >50K' + # Normalize target labels to {0, 1}; 1 means income >50K. y_str = y_raw.astype(str).str.strip() y: np.ndarray = (y_str == ">50K").astype(np.int64).to_numpy() - # 3) Replace '?' markers with np.nan and drop rows with missing configured features. + # Adult encodes missing values as '?'. Drop incomplete rows so the + # adversarial metadata is based on real observed feature ranges. X_df = X_df.replace(r"^\s*\?\s*$", np.nan, regex=True) self._validate_manual_schema(X_df.columns) @@ -243,12 +255,12 @@ def load_adult_census_dataset(self) -> tuple[AdultCensusTorchDataset, AdultCensu valid_rows = ~X_df[configured_columns].isna().any(axis=1) removed_rows = int((~valid_rows).sum()) if removed_rows: - import logging - logging.getLogger().info("[AdultCensus] Dropping %s rows with NA values", removed_rows) + logger.info("[AdultCensus] Dropping %s rows with NA values", removed_rows) X_df = X_df.loc[valid_rows].copy() y = y[valid_rows.to_numpy()] - # 4) Preprocess + # Numeric columns are standardized; categorical columns become one-hot + # columns. Constrained PGD metadata is built after this, in model input space. numeric_transformer = Pipeline( steps=[ ("impute", SimpleImputer(strategy="median")), @@ -273,7 +285,7 @@ def load_adult_census_dataset(self) -> tuple[AdultCensusTorchDataset, AdultCensu preprocessor = ColumnTransformer(transformers=transformers, remainder="drop") - # 5) Split then fit on train + # Fit preprocessing only on train to avoid leaking test statistics. X_train_df, X_test_df, y_train, y_test = train_test_split( X_df, y, @@ -285,10 +297,7 @@ def load_adult_census_dataset(self) -> tuple[AdultCensusTorchDataset, AdultCensu X_train = preprocessor.fit_transform(X_train_df) X_test = preprocessor.transform(X_test_df) - try: - feature_names = [str(name) for name in preprocessor.get_feature_names_out()] - except Exception: - feature_names = [f"feature_{i}" for i in range(X_train.shape[1])] + feature_names = self._feature_names(preprocessor, X_train.shape[1]) # In case some sklearn path returns sparse matrices, densify safely if hasattr(X_train, "toarray"): @@ -296,101 +305,77 @@ def load_adult_census_dataset(self) -> tuple[AdultCensusTorchDataset, AdultCensu if hasattr(X_test, "toarray"): X_test = X_test.toarray() - X_train_np: np.ndarray = np.asarray(X_train, dtype=np.float32) - import logging - logging.getLogger().info(f"[AdultCensus] X_train shape = {X_train_np.shape}") - logging.getLogger().info(f"[AdultCensus] INPUT_DIM (post-OHE) = {int(X_train_np.shape[1])}") + X_train_np = np.asarray(X_train, dtype=np.float32) X_test_np: np.ndarray = np.asarray(X_test, dtype=np.float32) - continuous_features = [ - idx for idx, name in enumerate(feature_names) - if name.startswith("continuous__") - ] - integer_features = [ - idx for idx, name in enumerate(feature_names) - if name.startswith("integer__") - ] - categorical_features = [ - idx for idx, name in enumerate(feature_names) - if name.startswith("categorical__") - ] - continuous_feature_set = set(continuous_features) - integer_feature_set = set(integer_features) - categorical_feature_set = set(categorical_features) - assigned_feature_set = continuous_feature_set | integer_feature_set | categorical_feature_set - unknown_features = [ - feature_names[idx] - for idx in range(len(feature_names)) - if idx not in assigned_feature_set - ] - if unknown_features: - raise ValueError(f"AdultCensusDataset generated untyped features: {unknown_features}") - feature_type_by_idx = { - **{idx: CONTINUOUS for idx in continuous_feature_set}, - **{idx: INTEGER for idx in integer_feature_set}, - **{idx: CATEGORICAL for idx in categorical_feature_set}, - } + metadata = self._build_adversarial_metadata(feature_names, X_train_np, preprocessor) + logger.info("[AdultCensus] X_train shape = %s", X_train_np.shape) + logger.info("[AdultCensus] INPUT_DIM (post-OHE) = %s", int(X_train_np.shape[1])) + self._log_adversarial_metadata(metadata, feature_names) - categorical_groups = self._build_categorical_groups(feature_names) - integer_step_norm = {} - if integer_features: - integer_scaler = preprocessor.named_transformers_["integer"].named_steps["scaler"] - integer_step_norm = { - idx: float(1.0 / scale) - for idx, scale in zip(integer_features, integer_scaler.scale_, strict=False) - } - tabular_metadata = TabularAdversarialMetadata( - feature_names=feature_names, - feature_types=[feature_type_by_idx[idx] for idx in range(len(feature_names))], - feature_min_norm=np.min(X_train_np, axis=0).astype(float).tolist(), - feature_max_norm=np.max(X_train_np, axis=0).astype(float).tolist(), - integer_step_norm=integer_step_norm, - categorical_groups=categorical_groups, - ).to_dict() - logging.getLogger().info( - "[AdultCensus] Tabular adversarial feature mask | continuous=%s | integer=%s | " - "categorical=%s | categorical_groups=%s | continuous_features=%s | integer_features=%s | " - "integer_step_norm=%s", - len(continuous_features), - len(integer_features), - len(categorical_features), - len(categorical_groups), - [feature_names[idx] for idx in continuous_features], - [feature_names[idx] for idx in integer_features], - integer_step_norm, - ) + train_ds = self._make_dataset(X_train_np, y_train, feature_names, metadata) + test_ds = self._make_dataset(X_test_np, y_test, feature_names, metadata) + + return train_ds, test_ds - train_ds = AdultCensusTorchDataset( - X_train_np, - np.asarray(y_train, dtype=np.int64), + @staticmethod + def _feature_names(preprocessor, n_features: int) -> list[str]: + try: + return [str(name) for name in preprocessor.get_feature_names_out()] + except Exception: + return [f"feature_{idx}" for idx in range(n_features)] + + @staticmethod + def _make_dataset(x, y, feature_names, metadata) -> AdultCensusTorchDataset: + return AdultCensusTorchDataset( + x, + np.asarray(y, dtype=np.int64), feature_names=feature_names, - continuous_features=continuous_features, - integer_features=integer_features, - categorical_features=categorical_features, - categorical_groups=categorical_groups, - tabular_metadata=tabular_metadata, + continuous_features=[], + integer_features=metadata["integer_features"], + categorical_features=metadata["categorical_features"], + non_perturbable_features=metadata["non_perturbable_features"], + categorical_groups=metadata["categorical_groups"], + tabular_metadata=metadata["tabular_metadata"], ) - test_ds = AdultCensusTorchDataset( - X_test_np, - np.asarray(y_test, dtype=np.int64), + + @classmethod + def _build_adversarial_metadata(cls, feature_names, x_train, preprocessor) -> dict[str, Any]: + # Dataset responsibility ends here: declare which raw columns are perturbable. + # The shared metadata builder maps those declarations to transformed model features. + integer_scaler = preprocessor.named_transformers_["integer"].named_steps["scaler"] + integer_step_by_column = { + column: float(1.0 / scale) + for column, scale in zip(cls.INTEGER_COLUMNS, integer_scaler.scale_, strict=False) + } + return build_tabular_adversarial_metadata( feature_names=feature_names, - continuous_features=continuous_features, - integer_features=integer_features, - categorical_features=categorical_features, - categorical_groups=categorical_groups, - tabular_metadata=tabular_metadata, + x_train=x_train, + continuous_columns=cls.CONTINUOUS_COLUMNS, + integer_columns=cls.INTEGER_COLUMNS, + categorical_columns=cls.CATEGORICAL_COLUMNS, + perturbable_integer_columns=cls.PERTURBABLE_INTEGER_COLUMNS, + perturbable_categorical_columns=cls.PERTURBABLE_CATEGORICAL_COLUMNS, + integer_step_by_column=integer_step_by_column, ) - return train_ds, test_ds - - @classmethod - def _build_categorical_groups(cls, feature_names: list[str]) -> list[list[int]]: - groups = [] - for column in cls.CATEGORICAL_COLUMNS: - prefix = f"categorical__{column}_" - group = [idx for idx, name in enumerate(feature_names) if name.startswith(prefix)] - if group: - groups.append(group) - return groups + @staticmethod + def _log_adversarial_metadata(metadata: dict[str, Any], feature_names: list[str]) -> None: + integer_features = metadata["integer_features"] + categorical_features = metadata["categorical_features"] + non_perturbable_features = metadata["non_perturbable_features"] + logger.info( + "[AdultCensus] Tabular adversarial feature mask | integer=%s | categorical=%s | " + "categorical_groups=%s | non_perturbable=%s | integer_features=%s | " + "categorical_preview=%s | non_perturbable_preview=%s | integer_step_norm=%s", + len(integer_features), + len(categorical_features), + len(metadata["categorical_groups"]), + len(non_perturbable_features), + [feature_names[idx] for idx in integer_features], + [feature_names[idx] for idx in categorical_features[:20]], + [feature_names[idx] for idx in non_perturbable_features[:20]], + metadata["integer_step_norm"], + ) def generate_non_iid_map(self, dataset, partition: str = "dirichlet", partition_parameter: float = 0.5): if partition == "dirichlet": diff --git a/nebula/core/datasets/breast_cancer/breast_cancer.py b/nebula/core/datasets/breast_cancer/breast_cancer.py index ac7446770..f5e53ed7e 100644 --- a/nebula/core/datasets/breast_cancer/breast_cancer.py +++ b/nebula/core/datasets/breast_cancer/breast_cancer.py @@ -1,12 +1,15 @@ +import logging import os -from typing import Tuple, Any +from typing import Any import numpy as np import torch from torch.utils.data import Dataset from nebula.core.datasets.nebuladataset import NebulaDataset, NebulaPartitionHandler -from nebula.core.datasets.tabular_metadata import CONTINUOUS, INTEGER, NON_PERTURBABLE, TabularAdversarialMetadata +from nebula.core.datasets.tabular_metadata import build_tabular_adversarial_metadata + +logger = logging.getLogger(__name__) class BreastCancerTorchDataset(Dataset): @@ -38,14 +41,14 @@ def __init__( self.x = x.astype(np.float32, copy=False) self.y = y.astype(np.int64, copy=False) - # Nebula conventions (some utilities expect these) + # Nebula dataset conventions used by partitioning, logging and model setup. self.data = self.x self.targets = self.y self.classes = ["0", "1"] self.feature_names = feature_names or [f"feature_{i}" for i in range(self.x.shape[1])] - self.continuous_features = continuous_features or list(range(self.x.shape[1])) - self.integer_features = integer_features or [] - self.non_perturbable_features = non_perturbable_features or [] + self.continuous_features = list(range(self.x.shape[1])) if continuous_features is None else continuous_features + self.integer_features = [] if integer_features is None else integer_features + self.non_perturbable_features = [] if non_perturbable_features is None else non_perturbable_features self.binary_features = [] self.tabular_metadata = tabular_metadata self.input_dim = int(self.x.shape[1]) @@ -53,7 +56,7 @@ def __init__( def __len__(self) -> int: return int(self.y.shape[0]) - def __getitem__(self, idx: int) -> Tuple[torch.Tensor, torch.Tensor]: + def __getitem__(self, idx: int) -> tuple[torch.Tensor, torch.Tensor]: x_i = torch.from_numpy(self.x[idx]) y_i = torch.tensor(self.y[idx], dtype=torch.long) return x_i, y_i @@ -97,7 +100,9 @@ class BreastCancerDataset(NebulaDataset): - tabular features (30) - deterministic stratified train/test split """ - PERTURBABLE_CONTINUOUS_COLUMNS = [ + # Raw sklearn feature names. These names are also the schema used to decide + # which variables adversarial training may perturb. + FEATURE_COLUMNS = [ "mean radius", "mean texture", "mean perimeter", @@ -129,8 +134,11 @@ class BreastCancerDataset(NebulaDataset): "worst symmetry", "worst fractal dimension", ] + # Breast Cancer has only continuous medical measurements. Keeping this as a + # list makes perturbability a dataset-level decision: remove a column here + # and the shared metadata builder will mark it as non-perturbable. + PERTURBABLE_CONTINUOUS_COLUMNS = list(FEATURE_COLUMNS) PERTURBABLE_INTEGER_COLUMNS = [] - NON_PERTURBABLE_COLUMNS = [] def __init__( self, @@ -164,30 +172,7 @@ def initialize_dataset(self): self.data_partitioning(plot=True) - @classmethod - def _validate_manual_schema(cls, columns) -> None: - continuous_columns = set(cls.PERTURBABLE_CONTINUOUS_COLUMNS) - integer_columns = set(cls.PERTURBABLE_INTEGER_COLUMNS) - non_perturbable_columns = set(cls.NON_PERTURBABLE_COLUMNS) - overlapping_columns = sorted( - (continuous_columns & integer_columns) - | (continuous_columns & non_perturbable_columns) - | (integer_columns & non_perturbable_columns) - ) - if overlapping_columns: - raise ValueError(f"BreastCancerDataset columns configured twice: {overlapping_columns}") - - configured_columns = continuous_columns | integer_columns | non_perturbable_columns - dataset_columns = set(columns) - missing_columns = sorted(configured_columns - dataset_columns) - if missing_columns: - raise ValueError(f"BreastCancerDataset is missing configured columns: {missing_columns}") - unconfigured_columns = sorted(dataset_columns - configured_columns) - if unconfigured_columns: - raise ValueError(f"BreastCancerDataset has unconfigured columns: {unconfigured_columns}") - def load_breast_cancer_dataset(self): - # Local cache directory (aunque load_breast_cancer no descarga, seguimos el patrón) data_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data") os.makedirs(data_dir, exist_ok=True) @@ -219,55 +204,70 @@ def load_breast_cancer_dataset(self): x_train = scaler.fit_transform(x_train) x_test = scaler.transform(x_test) + # Constrained PGD receives standardized tensors, so metadata bounds must also be + # computed in this transformed model-input space. x_train_np = np.asarray(x_train, dtype=np.float32) x_test_np = np.asarray(x_test, dtype=np.float32) - continuous_features = [ - idx for idx, name in enumerate(feature_names) - if name in self.PERTURBABLE_CONTINUOUS_COLUMNS - ] - integer_features = [ - idx for idx, name in enumerate(feature_names) - if name in self.PERTURBABLE_INTEGER_COLUMNS - ] - non_perturbable_features = [ - idx for idx, name in enumerate(feature_names) - if name in self.NON_PERTURBABLE_COLUMNS - ] - continuous_feature_set = set(continuous_features) - integer_feature_set = set(integer_features) - tabular_metadata = TabularAdversarialMetadata( - feature_names=feature_names, - feature_types=[ - CONTINUOUS if idx in continuous_feature_set - else INTEGER if idx in integer_feature_set - else NON_PERTURBABLE - for idx in range(len(feature_names)) - ], - feature_min_norm=np.min(x_train_np, axis=0).astype(float).tolist(), - feature_max_norm=np.max(x_train_np, axis=0).astype(float).tolist(), - integer_step_norm={}, - ).to_dict() - - train_ds = BreastCancerTorchDataset( - x_train_np, - y_train, + metadata = self._build_adversarial_metadata(feature_names, x_train_np) + self._log_adversarial_metadata(metadata, feature_names) + + return ( + self._make_dataset(x_train_np, y_train, feature_names, metadata), + self._make_dataset(x_test_np, y_test, feature_names, metadata), + ) + + @classmethod + def _validate_manual_schema(cls, columns) -> None: + dataset_columns = set(columns) + expected_columns = set(cls.FEATURE_COLUMNS) + missing_columns = sorted(expected_columns - dataset_columns) + extra_columns = sorted(dataset_columns - expected_columns) + if missing_columns or extra_columns: + raise ValueError( + "BreastCancerDataset schema mismatch: " + f"missing={missing_columns}, extra={extra_columns}" + ) + + @classmethod + def _build_adversarial_metadata(cls, feature_names, x_train): + # The dataset only declares perturbable columns. The shared builder + # turns that declaration into feature types, bounds and masks for constrained PGD. + return build_tabular_adversarial_metadata( feature_names=feature_names, - continuous_features=continuous_features, - integer_features=integer_features, - non_perturbable_features=non_perturbable_features, - tabular_metadata=tabular_metadata, + x_train=x_train, + continuous_columns=cls.FEATURE_COLUMNS, + integer_columns=[], + categorical_columns=[], + perturbable_continuous_columns=cls.PERTURBABLE_CONTINUOUS_COLUMNS, + perturbable_integer_columns=cls.PERTURBABLE_INTEGER_COLUMNS, ) - test_ds = BreastCancerTorchDataset( - x_test_np, - y_test, + + @staticmethod + def _make_dataset(x, y, feature_names, metadata) -> BreastCancerTorchDataset: + # Store the same metadata on train and test. Training uses it to create + # adversarial examples; evaluation can inspect it for robustness reports. + return BreastCancerTorchDataset( + x, + y, feature_names=feature_names, - continuous_features=continuous_features, - integer_features=integer_features, - non_perturbable_features=non_perturbable_features, - tabular_metadata=tabular_metadata, + continuous_features=metadata["continuous_features"], + integer_features=metadata["integer_features"], + non_perturbable_features=metadata["non_perturbable_features"], + tabular_metadata=metadata["tabular_metadata"], ) - return train_ds, test_ds + @staticmethod + def _log_adversarial_metadata(metadata: dict[str, Any], feature_names: list[str]) -> None: + continuous_features = metadata["continuous_features"] + non_perturbable_features = metadata["non_perturbable_features"] + logger.info( + "[BreastCancer] Tabular adversarial feature mask | continuous=%s | " + "non_perturbable=%s | continuous_features=%s | non_perturbable_preview=%s", + len(continuous_features), + len(non_perturbable_features), + [feature_names[idx] for idx in continuous_features], + [feature_names[idx] for idx in non_perturbable_features[:20]], + ) def generate_non_iid_map(self, dataset, partition: str = "dirichlet", partition_parameter: float = 0.5): if partition == "dirichlet": diff --git a/nebula/core/datasets/covtype/covtype.py b/nebula/core/datasets/covtype/covtype.py index 22a24c682..e3be64318 100644 --- a/nebula/core/datasets/covtype/covtype.py +++ b/nebula/core/datasets/covtype/covtype.py @@ -1,14 +1,17 @@ # nebula/core/datasets/covtype/covtype.py +import logging import os -from typing import Tuple, Any +from typing import Any import numpy as np import torch from torch.utils.data import Dataset from nebula.core.datasets.nebuladataset import NebulaDataset, NebulaPartitionHandler -from nebula.core.datasets.tabular_metadata import CONTINUOUS, INTEGER, NON_PERTURBABLE, TabularAdversarialMetadata +from nebula.core.datasets.tabular_metadata import build_tabular_adversarial_metadata + +logger = logging.getLogger(__name__) class CovtypeTorchDataset(Dataset): @@ -44,6 +47,7 @@ def __init__( self.x = x.astype(np.float32, copy=False) self.y = y.astype(np.int64, copy=False) + # Nebula dataset conventions used by partitioning, logging and model setup. self.data = self.x self.targets = self.y @@ -60,7 +64,7 @@ def __init__( def __len__(self) -> int: return int(self.y.shape[0]) - def __getitem__(self, idx: int) -> Tuple[torch.Tensor, torch.Tensor]: + def __getitem__(self, idx: int) -> tuple[torch.Tensor, torch.Tensor]: x_i = torch.from_numpy(self.x[idx]) y_i = torch.tensor(self.y[idx], dtype=torch.long) return x_i, y_i @@ -120,7 +124,7 @@ class CovtypeDataset(NebulaDataset): Requirements: - scikit-learn must be installed (for fetch_covtype + train_test_split). """ - PERTURBABLE_CONTINUOUS_COLUMNS = [ + CONTINUOUS_COLUMNS = [ "Elevation", "Aspect", "Slope", @@ -132,8 +136,7 @@ class CovtypeDataset(NebulaDataset): "Hillshade_3pm", "Horizontal_Distance_To_Fire_Points", ] - PERTURBABLE_INTEGER_COLUMNS = [] - NON_PERTURBABLE_COLUMNS = [ + BINARY_COLUMNS = [ "Wilderness_Area_0", "Wilderness_Area_1", "Wilderness_Area_2", @@ -179,6 +182,12 @@ class CovtypeDataset(NebulaDataset): "Soil_Type_38", "Soil_Type_39", ] + # Covtype has two kinds of inputs: + # - terrain measurements, which constrained PGD may perturb; + # - binary wilderness/soil indicators, which stay immutable to avoid broken + # one-hot-like combinations. + PERTURBABLE_CONTINUOUS_COLUMNS = list(CONTINUOUS_COLUMNS) + PERTURBABLE_INTEGER_COLUMNS = [] def __init__( self, @@ -218,16 +227,16 @@ def initialize_dataset(self): @classmethod def _default_feature_names(cls, n_features: int) -> list[str]: - configured_columns = cls.PERTURBABLE_CONTINUOUS_COLUMNS + cls.NON_PERTURBABLE_COLUMNS + configured_columns = cls.CONTINUOUS_COLUMNS + cls.BINARY_COLUMNS if n_features == len(configured_columns): return configured_columns return [f"feature_{i}" for i in range(n_features)] @classmethod def _validate_manual_schema(cls, columns) -> None: - continuous_columns = set(cls.PERTURBABLE_CONTINUOUS_COLUMNS) + continuous_columns = set(cls.CONTINUOUS_COLUMNS) integer_columns = set(cls.PERTURBABLE_INTEGER_COLUMNS) - non_perturbable_columns = set(cls.NON_PERTURBABLE_COLUMNS) + non_perturbable_columns = set(cls.BINARY_COLUMNS) overlapping_columns = sorted( (continuous_columns & integer_columns) | (continuous_columns & non_perturbable_columns) @@ -250,7 +259,6 @@ def load_covtype_dataset(self): Loads Covtype via sklearn, performs a deterministic train/test split, and wraps into torch Datasets. """ - # Local cache directory for sklearn dataset downloads data_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data") os.makedirs(data_dir, exist_ok=True) @@ -274,27 +282,13 @@ def load_covtype_dataset(self): try: self._validate_manual_schema(feature_names) except ValueError: - if x.shape[1] != len(self.PERTURBABLE_CONTINUOUS_COLUMNS) + len(self.NON_PERTURBABLE_COLUMNS): + if x.shape[1] != len(self.CONTINUOUS_COLUMNS) + len(self.BINARY_COLUMNS): raise - import logging - logging.getLogger().info( + logger.info( "[Covtype] Replacing sklearn feature names with canonical Covtype names for adversarial metadata" ) feature_names = self._default_feature_names(x.shape[1]) self._validate_manual_schema(feature_names) - continuous_features = [ - idx for idx, name in enumerate(feature_names) - if name in self.PERTURBABLE_CONTINUOUS_COLUMNS - ] - integer_features = [ - idx for idx, name in enumerate(feature_names) - if name in self.PERTURBABLE_INTEGER_COLUMNS - ] - non_perturbable_features = [ - idx for idx, name in enumerate(feature_names) - if name in self.NON_PERTURBABLE_COLUMNS - ] - binary_features = non_perturbable_features # Map labels to 0..6 (CrossEntropyLoss convention) # If already 0..6, this is harmless for 1..7 only if we detect min. @@ -330,50 +324,69 @@ def load_covtype_dataset(self): stratify=y_test, ) - # Covtype has continuous features followed by binary wilderness/soil indicators. - # Scale only the continuous block; keep binary indicators as 0/1. + # Scale only the perturbable terrain measurements. The binary columns + # must remain 0/1 because they encode wilderness and soil indicators. scaler = StandardScaler() x_train = np.asarray(x_train, dtype=np.float32).copy() x_test = np.asarray(x_test, dtype=np.float32).copy() + continuous_features = [ + idx for idx, name in enumerate(feature_names) + if name in self.CONTINUOUS_COLUMNS + ] x_train[:, continuous_features] = scaler.fit_transform(x_train[:, continuous_features]) x_test[:, continuous_features] = scaler.transform(x_test[:, continuous_features]) - continuous_feature_set = set(continuous_features) - integer_feature_set = set(integer_features) - tabular_metadata = TabularAdversarialMetadata( - feature_names=feature_names, - feature_types=[ - CONTINUOUS if idx in continuous_feature_set - else INTEGER if idx in integer_feature_set - else NON_PERTURBABLE - for idx in range(len(feature_names)) - ], - feature_min_norm=np.min(x_train, axis=0).astype(float).tolist(), - feature_max_norm=np.max(x_train, axis=0).astype(float).tolist(), - integer_step_norm={}, - ).to_dict() - - train_ds = CovtypeTorchDataset( - x_train, - y_train, + metadata = self._build_adversarial_metadata(feature_names, x_train) + self._log_adversarial_metadata(metadata, feature_names) + + return ( + self._make_dataset(x_train, y_train, feature_names, metadata), + self._make_dataset(x_test, y_test, feature_names, metadata), + ) + + @staticmethod + def _make_dataset( + x: np.ndarray, + y: np.ndarray, + feature_names: list[str], + metadata: dict[str, Any], + ) -> CovtypeTorchDataset: + return CovtypeTorchDataset( + x, + y, feature_names=feature_names, - continuous_features=continuous_features, - integer_features=integer_features, - non_perturbable_features=non_perturbable_features, - binary_features=binary_features, - tabular_metadata=tabular_metadata, + continuous_features=metadata["continuous_features"], + integer_features=metadata["integer_features"], + non_perturbable_features=metadata["non_perturbable_features"], + binary_features=metadata["non_perturbable_features"], + tabular_metadata=metadata["tabular_metadata"], ) - test_ds = CovtypeTorchDataset( - x_test, - y_test, + + @classmethod + def _build_adversarial_metadata(cls, feature_names, x_train): + # Dataset responsibility: declare which variables are perturbable. The + # shared builder maps those declarations to feature masks and bounds. + return build_tabular_adversarial_metadata( feature_names=feature_names, - continuous_features=continuous_features, - integer_features=integer_features, - non_perturbable_features=non_perturbable_features, - binary_features=binary_features, - tabular_metadata=tabular_metadata, + x_train=x_train, + continuous_columns=cls.CONTINUOUS_COLUMNS, + integer_columns=[], + categorical_columns=[], + perturbable_continuous_columns=cls.PERTURBABLE_CONTINUOUS_COLUMNS, + perturbable_integer_columns=cls.PERTURBABLE_INTEGER_COLUMNS, ) - return train_ds, test_ds + @staticmethod + def _log_adversarial_metadata(metadata: dict[str, Any], feature_names: list[str]) -> None: + continuous_features = metadata["continuous_features"] + non_perturbable_features = metadata["non_perturbable_features"] + logger.info( + "[Covtype] Tabular adversarial feature mask | continuous=%s | non_perturbable=%s | " + "continuous_features=%s | non_perturbable_preview=%s", + len(continuous_features), + len(non_perturbable_features), + [feature_names[idx] for idx in continuous_features], + [feature_names[idx] for idx in non_perturbable_features[:20]], + ) def generate_non_iid_map(self, dataset, partition: str = "dirichlet", partition_parameter: float = 0.5): if partition == "dirichlet": diff --git a/nebula/core/datasets/tabular_metadata.py b/nebula/core/datasets/tabular_metadata.py index e34397f6a..85d240099 100644 --- a/nebula/core/datasets/tabular_metadata.py +++ b/nebula/core/datasets/tabular_metadata.py @@ -132,3 +132,150 @@ def from_dict(cls, data: dict[str, Any]) -> TabularAdversarialMetadata: for group in data.get("categorical_groups") or [] ], ) + + +def build_tabular_adversarial_metadata( + *, + feature_names: list[str], + x_train, + continuous_columns: list[str] | tuple[str, ...] = (), + integer_columns: list[str] | tuple[str, ...] = (), + categorical_columns: list[str] | tuple[str, ...] = (), + perturbable_continuous_columns: list[str] | tuple[str, ...] = (), + perturbable_integer_columns: list[str] | tuple[str, ...] = (), + perturbable_categorical_columns: list[str] | tuple[str, ...] = (), + integer_step_by_column: dict[str, float] | None = None, +) -> dict[str, Any]: + """Build tabular adversarial metadata from dataset-level perturbability lists.""" + # Datasets should only decide which raw columns are perturbable. This helper + # maps that decision to the transformed feature vector consumed by the model. + _validate_perturbable_columns( + continuous_columns=continuous_columns, + integer_columns=integer_columns, + categorical_columns=categorical_columns, + perturbable_continuous_columns=perturbable_continuous_columns, + perturbable_integer_columns=perturbable_integer_columns, + perturbable_categorical_columns=perturbable_categorical_columns, + ) + + perturbable_continuous = set(perturbable_continuous_columns) + perturbable_integer = set(perturbable_integer_columns) + perturbable_categorical = set(perturbable_categorical_columns) + + # Continuous/integer transformed features usually keep their raw column name + # after an optional transformer prefix, for example "integer__age". + continuous_features = [ + idx + for idx, name in enumerate(feature_names) + if _raw_feature_name(name) in perturbable_continuous + ] + integer_features = [ + idx + for idx, name in enumerate(feature_names) + if _raw_feature_name(name) in perturbable_integer + ] + # One raw categorical column becomes several one-hot features, for example + # "categorical__sex_Female" and "categorical__sex_Male". + categorical_features = [ + idx + for idx, name in enumerate(feature_names) + if _categorical_column_name(name, categorical_columns) in perturbable_categorical + ] + + continuous_feature_set = set(continuous_features) + integer_feature_set = set(integer_features) + categorical_feature_set = set(categorical_features) + perturbable_feature_set = continuous_feature_set | integer_feature_set | categorical_feature_set + non_perturbable_features = [ + idx + for idx in range(len(feature_names)) + if idx not in perturbable_feature_set + ] + + categorical_groups = _categorical_groups(feature_names, perturbable_categorical) + integer_step_norm = _integer_step_norm(feature_names, integer_features, integer_step_by_column or {}) + # The attack consumes only TabularAdversarialMetadata. The extra lists are + # returned so dataset wrappers and logs can expose the same mask clearly. + tabular_metadata = TabularAdversarialMetadata( + feature_names=feature_names, + feature_types=[ + CONTINUOUS if idx in continuous_feature_set + else INTEGER if idx in integer_feature_set + else CATEGORICAL if idx in categorical_feature_set + else NON_PERTURBABLE + for idx in range(len(feature_names)) + ], + feature_min_norm=[float(value) for value in x_train.min(axis=0)], + feature_max_norm=[float(value) for value in x_train.max(axis=0)], + integer_step_norm=integer_step_norm, + categorical_groups=categorical_groups, + ).to_dict() + + return { + "continuous_features": continuous_features, + "integer_features": integer_features, + "categorical_features": categorical_features, + "non_perturbable_features": non_perturbable_features, + "categorical_groups": categorical_groups, + "integer_step_norm": integer_step_norm, + "tabular_metadata": tabular_metadata, + } + + +def _validate_perturbable_columns( + *, + continuous_columns, + integer_columns, + categorical_columns, + perturbable_continuous_columns, + perturbable_integer_columns, + perturbable_categorical_columns, +) -> None: + invalid_continuous = sorted(set(perturbable_continuous_columns) - set(continuous_columns)) + invalid_integer = sorted(set(perturbable_integer_columns) - set(integer_columns)) + invalid_categorical = sorted(set(perturbable_categorical_columns) - set(categorical_columns)) + if invalid_continuous or invalid_integer or invalid_categorical: + raise ValueError( + "Perturbable columns must exist in the dataset schema: " + f"continuous={invalid_continuous}, integer={invalid_integer}, categorical={invalid_categorical}" + ) + + +def _raw_feature_name(feature_name: str) -> str: + # Strip sklearn ColumnTransformer prefixes such as "integer__" or + # "categorical__" while leaving plain feature names untouched. + return feature_name.split("__", maxsplit=1)[1] if "__" in feature_name else feature_name + + +def _categorical_column_name(feature_name: str, categorical_columns) -> str | None: + # Recover the raw categorical column name from a one-hot feature name. + raw_name = _raw_feature_name(feature_name) + for column in categorical_columns: + if raw_name.startswith(f"{column}_"): + return column + return None + + +def _categorical_groups(feature_names: list[str], perturbable_categorical_columns: set[str]) -> list[list[int]]: + # Constrained PGD projects each group back to exactly one active one-hot value. + groups = [] + for column in perturbable_categorical_columns: + prefix = f"categorical__{column}_" + group = [idx for idx, name in enumerate(feature_names) if name.startswith(prefix)] + if group: + groups.append(group) + return groups + + +def _integer_step_norm( + feature_names: list[str], + integer_features: list[int], + integer_step_by_column: dict[str, float], +) -> dict[int, float]: + # Integer columns may be scaled. The step tells constrained PGD what "+1 raw unit" + # means in the normalized model-input space. + return { + idx: float(integer_step_by_column[_raw_feature_name(feature_names[idx])]) + for idx in integer_features + if _raw_feature_name(feature_names[idx]) in integer_step_by_column + } diff --git a/nebula/frontend/static/js/deployment/adversarial-training.js b/nebula/frontend/static/js/deployment/adversarial-training.js index b02addf4e..cb072cf9e 100644 --- a/nebula/frontend/static/js/deployment/adversarial-training.js +++ b/nebula/frontend/static/js/deployment/adversarial-training.js @@ -15,13 +15,13 @@ const AdversarialTrainingManager = (function() { }; const IMAGE_DATASETS = new Set(["MNIST", "FashionMNIST", "EMNIST", "CIFAR10", "CIFAR100"]); - const TABULAR_ADVERSARIAL_DATASETS = new Set(["AdultCensus"]); + const TABULAR_ADVERSARIAL_DATASETS = new Set(["AdultCensus", "BreastCancer", "Covtype"]); const IMAGE_ATTACK_OPTIONS = [ {value: "fgsm", label: "FGSM"}, {value: "pgd", label: "PGD"} ]; const TABULAR_ATTACK_OPTIONS = [ - {value: "capgd", label: "CAPGD"} + {value: "constrained_pgd", label: "Constrained PGD"} ]; function initializeAdversarialTraining() { @@ -75,12 +75,12 @@ const AdversarialTrainingManager = (function() { const domain = document.getElementById("adversarialTrainingDomain")?.value || DEFAULT_ADVERSARIAL_TRAINING_CONFIG.domain; if (!pgdSettings) return; - pgdSettings.style.display = ["pgd", "capgd"].includes(attack) ? "block" : "none"; + pgdSettings.style.display = ["pgd", "constrained_pgd"].includes(attack) ? "block" : "none"; if (lossWindowSettings) { lossWindowSettings.style.display = domain === "tabular" ? "block" : "none"; } if (stepsTitle) { - stepsTitle.textContent = domain === "tabular" ? "CAPGD steps" : "PGD steps"; + stepsTitle.textContent = domain === "tabular" ? "Constrained PGD steps" : "PGD steps"; } } @@ -94,7 +94,7 @@ const AdversarialTrainingManager = (function() { if (datasetNote) { datasetNote.style.display = domain === "unsupported" ? "block" : "none"; - datasetNote.textContent = "Adversarial Training for tabular datasets currently supports AdultCensus with CAPGD."; + datasetNote.textContent = "Adversarial Training for tabular datasets currently supports AdultCensus, BreastCancer, and Covtype with constrained PGD."; } if (domainInput) { domainInput.value = domain === "unsupported" ? "tabular" : domain; @@ -129,7 +129,7 @@ const AdversarialTrainingManager = (function() { const attackSelect = document.getElementById("adversarialTrainingAttack"); if (!attackSelect) return; - // Tabular datasets intentionally expose only CAPGD; image datasets expose FGSM/PGD. + // Tabular datasets intentionally expose only constrained PGD; image datasets expose FGSM/PGD. const options = domain === "tabular" ? TABULAR_ATTACK_OPTIONS : IMAGE_ATTACK_OPTIONS; const currentAttack = preferredAttack || attackSelect.value; attackSelect.innerHTML = ""; @@ -170,7 +170,7 @@ const AdversarialTrainingManager = (function() { function getAdversarialTrainingConfig() { const domain = document.getElementById("adversarialTrainingDomain")?.value || DEFAULT_ADVERSARIAL_TRAINING_CONFIG.domain; const attack = domain === "tabular" - ? "capgd" + ? "constrained_pgd" : (document.getElementById("adversarialTrainingAttack")?.value || DEFAULT_ADVERSARIAL_TRAINING_CONFIG.attack); const config = { enabled: Boolean(document.getElementById("adversarialTrainingSwitch")?.checked), @@ -252,7 +252,7 @@ const AdversarialTrainingManager = (function() { if (config.epsilon < 0) { return "[Adversarial Training] Epsilon must be greater than or equal to 0."; } - if (["pgd", "capgd"].includes(config.attack) && config.steps < 1) { + if (["pgd", "constrained_pgd"].includes(config.attack) && config.steps < 1) { return "[Adversarial Training] Search steps must be at least 1."; } if (!["mixed", "adversarial"].includes(config.mode)) { diff --git a/nebula/frontend/templates/deployment.html b/nebula/frontend/templates/deployment.html index bfa249739..01e9ed5a4 100755 --- a/nebula/frontend/templates/deployment.html +++ b/nebula/frontend/templates/deployment.html @@ -588,7 +588,7 @@
    Enable/Disable Adversarial Training
    style="display: inline; width: 80px; height: 30px;"> - Image datasets use FGSM/PGD. AdultCensus uses CAPGD for tabular adversarial training. + Image datasets use FGSM/PGD. AdultCensus, BreastCancer, and Covtype use constrained PGD for tabular adversarial training. From c774e9d2bd25cd168164579fdf6676c7e23bbb84 Mon Sep 17 00:00:00 2001 From: "Juan J." Date: Mon, 8 Jun 2026 14:37:20 +0200 Subject: [PATCH 62/66] Adversarial training for tabular data updated: Margin window implemented. Covtype and KDDCUP99 updated and fixed for adversarial training --- .../defenses/adversarial_training/__init__.py | 4 + .../defenses/adversarial_training/config.py | 26 ++- .../defenses/adversarial_training/defense.py | 10 +- .../defenses/adversarial_training/tabular.py | 51 ++++- nebula/core/datasets/covtype/covtype.py | 44 ++-- nebula/core/datasets/kddcup99/kddcup99.py | 201 ++++++++++-------- .../frontend/config/participant.json.example | 5 +- .../js/deployment/adversarial-training.js | 89 +++++++- nebula/frontend/templates/deployment.html | 28 ++- 9 files changed, 333 insertions(+), 125 deletions(-) diff --git a/nebula/addons/defenses/adversarial_training/__init__.py b/nebula/addons/defenses/adversarial_training/__init__.py index ddc977538..772ac6cda 100644 --- a/nebula/addons/defenses/adversarial_training/__init__.py +++ b/nebula/addons/defenses/adversarial_training/__init__.py @@ -1,9 +1,11 @@ from nebula.addons.defenses.adversarial_training.defense import ( ERR_ALPHA, ERR_APPLY_PROBABILITY, + ERR_CANDIDATE_SELECTION, ERR_EPSILON, ERR_IMAGE_ATTACK, ERR_LOSS_INCREASE, + ERR_MARGIN_WINDOW, ERR_MODE, ERR_STEPS, ERR_TABULAR_ATTACK, @@ -28,9 +30,11 @@ __all__ = [ "ERR_ALPHA", "ERR_APPLY_PROBABILITY", + "ERR_CANDIDATE_SELECTION", "ERR_EPSILON", "ERR_IMAGE_ATTACK", "ERR_LOSS_INCREASE", + "ERR_MARGIN_WINDOW", "ERR_MODE", "ERR_STEPS", "ERR_TABULAR_ATTACK", diff --git a/nebula/addons/defenses/adversarial_training/config.py b/nebula/addons/defenses/adversarial_training/config.py index a5ca04a15..8cbcdddcd 100644 --- a/nebula/addons/defenses/adversarial_training/config.py +++ b/nebula/addons/defenses/adversarial_training/config.py @@ -3,7 +3,7 @@ IMAGE_ADVERSARIAL_ATTACKS = {"fgsm", "pgd"} TABULAR_ADVERSARIAL_ATTACKS = {"constrained_pgd"} -TABULAR_ADVERSARIAL_DATASETS = {"AdultCensus", "BreastCancer", "Covtype"} +TABULAR_ADVERSARIAL_DATASETS = {"AdultCensus", "BreastCancer", "Covtype", "KDDCUP99"} ERR_IMAGE_ATTACK = "image adversarial_training.attack must be one of: fgsm, pgd" ERR_TABULAR_ATTACK = "tabular adversarial_training.attack must be one of: constrained_pgd" @@ -12,7 +12,11 @@ ERR_ALPHA = "adversarial_training.alpha must be >= 0" ERR_STEPS = "adversarial_training.steps must be >= 1" ERR_APPLY_PROBABILITY = "adversarial_training.apply_probability must be in [0, 1]" +ERR_CANDIDATE_SELECTION = ( + "tabular adversarial_training.candidate_selection must be one of: none, loss_window, margin_window" +) ERR_LOSS_INCREASE = "adversarial_training loss increase thresholds must be >= 0 and target <= max" +ERR_MARGIN_WINDOW = "adversarial_training margin thresholds must satisfy target_margin <= max_margin" ERR_TABULAR_METADATA = "Tabular adversarial training requires tabular_metadata" ERR_UNSUPPORTED_ATTACK = "Unsupported adversarial training attack: {attack}" @@ -39,8 +43,11 @@ class AdversarialTrainingConfig: adversarial_weight: float = 0.5 apply_probability: float = 0.3 log_adversarial_metrics: bool = True + candidate_selection: str = "none" target_loss_increase: float | None = None max_loss_increase: float | None = None + target_margin: float | None = 0.0 + max_margin: float | None = 0.5 def config_from_participant(participant_config: dict[str, Any]) -> AdversarialTrainingConfig | None: @@ -69,12 +76,19 @@ def config_from_participant(participant_config: dict[str, Any]) -> AdversarialTr adversarial_weight=adversarial_weight, apply_probability=float(raw.get("apply_probability", 0.3)), log_adversarial_metrics=True, + candidate_selection=str(raw.get("candidate_selection", "none")).lower(), target_loss_increase=float(raw["target_loss_increase"]) if raw.get("target_loss_increase") is not None else None, max_loss_increase=float(raw["max_loss_increase"]) if raw.get("max_loss_increase") is not None else None, + target_margin=float(raw["target_margin"]) + if raw.get("target_margin") is not None + else 0.0, + max_margin=float(raw["max_margin"]) + if raw.get("max_margin") is not None + else 0.5, ) @@ -92,6 +106,10 @@ def validate_config(config: AdversarialTrainingConfig) -> None: raise ValueError(ERR_IMAGE_ATTACK) if config.domain == "tabular" and config.attack not in TABULAR_ADVERSARIAL_ATTACKS: raise ValueError(ERR_TABULAR_ATTACK) + if config.domain == "tabular" and config.candidate_selection not in {"none", "loss_window", "margin_window"}: + raise ValueError(ERR_CANDIDATE_SELECTION) + if config.domain == "image" and config.candidate_selection != "none": + raise ValueError(ERR_CANDIDATE_SELECTION) if config.epsilon < 0: raise ValueError(ERR_EPSILON) if config.alpha is not None and config.alpha < 0: @@ -110,3 +128,9 @@ def validate_config(config: AdversarialTrainingConfig) -> None: and config.target_loss_increase > config.max_loss_increase ): raise ValueError(ERR_LOSS_INCREASE) + if ( + config.target_margin is not None + and config.max_margin is not None + and config.target_margin > config.max_margin + ): + raise ValueError(ERR_MARGIN_WINDOW) diff --git a/nebula/addons/defenses/adversarial_training/defense.py b/nebula/addons/defenses/adversarial_training/defense.py index 4cd6b6923..b79285009 100644 --- a/nebula/addons/defenses/adversarial_training/defense.py +++ b/nebula/addons/defenses/adversarial_training/defense.py @@ -7,9 +7,11 @@ from nebula.addons.defenses.adversarial_training.config import ( ERR_ALPHA, ERR_APPLY_PROBABILITY, + ERR_CANDIDATE_SELECTION, ERR_EPSILON, ERR_IMAGE_ATTACK, ERR_LOSS_INCREASE, + ERR_MARGIN_WINDOW, ERR_MODE, ERR_STEPS, ERR_TABULAR_ATTACK, @@ -215,7 +217,8 @@ def apply_adversarial_training_if_enabled(model, participant_config: dict[str, A "[AdversarialTrainingDefense] Enabled | dataset=%s | attack=%s | epsilon_max=%s | " "epsilon_range=[%.6f, %.6f] | epsilon_step=%.6f | steps=%s | mode=%s | " "clean_weight=%.2f | adversarial_weight=%.2f | apply_probability=%.2f | " - "target_loss_increase=%s | max_loss_increase=%s | log_adversarial_metrics=%s", + "candidate_selection=%s | target_loss_increase=%s | max_loss_increase=%s | " + "target_margin=%s | max_margin=%s | log_adversarial_metrics=%s", defense.config.dataset_name, defense.config.attack, defense.config.epsilon, @@ -227,8 +230,11 @@ def apply_adversarial_training_if_enabled(model, participant_config: dict[str, A defense.config.clean_weight, defense.config.adversarial_weight, defense.config.apply_probability, + defense.config.candidate_selection, defense.config.target_loss_increase, defense.config.max_loss_increase, + defense.config.target_margin, + defense.config.max_margin, defense.config.log_adversarial_metrics, ) @@ -236,9 +242,11 @@ def apply_adversarial_training_if_enabled(model, participant_config: dict[str, A __all__ = [ "ERR_ALPHA", "ERR_APPLY_PROBABILITY", + "ERR_CANDIDATE_SELECTION", "ERR_EPSILON", "ERR_IMAGE_ATTACK", "ERR_LOSS_INCREASE", + "ERR_MARGIN_WINDOW", "ERR_MODE", "ERR_STEPS", "ERR_TABULAR_ATTACK", diff --git a/nebula/addons/defenses/adversarial_training/tabular.py b/nebula/addons/defenses/adversarial_training/tabular.py index 2ae280be0..661556d9b 100644 --- a/nebula/addons/defenses/adversarial_training/tabular.py +++ b/nebula/addons/defenses/adversarial_training/tabular.py @@ -174,7 +174,9 @@ def generate(self, model, x, y, criterion): x_adv = x_clean.clone() best_adv = x_adv.clone() best_score = torch.full((x_clean.size(0),), float("-inf"), dtype=x_clean.dtype, device=x_clean.device) + best_distance = torch.full((x_clean.size(0),), float("inf"), dtype=x_clean.dtype, device=x_clean.device) use_loss_window = self._use_loss_window() + use_margin_window = self._use_margin_window() clean_loss = self._clean_loss(model, x_clean, y) if use_loss_window else None for _ in range(steps): @@ -195,13 +197,18 @@ def generate(self, model, x, y, criterion): if use_loss_window: candidate_score = self._loss_increase(candidate_logits, y, clean_loss) better = self._loss_window_better(candidate_score, best_score) + elif use_margin_window: + candidate_score = self._margin(candidate_logits, y) + candidate_distance = self._margin_window_distance(candidate_score) + better = self._margin_window_better(candidate_score, candidate_distance, best_score, best_distance) + best_distance = torch.where(better, candidate_distance, best_distance) else: candidate_score = self._margin(candidate_logits, y) better = candidate_score > best_score best_adv = torch.where(better.view(-1, 1), candidate, best_adv) best_score = torch.where(better, candidate_score, best_score) - if self._target_reached(best_score): + if self._target_reached(best_score, best_distance): break x_adv = candidate @@ -209,7 +216,10 @@ def generate(self, model, x, y, criterion): return best_adv.detach() def _use_loss_window(self) -> bool: - return self.config.target_loss_increase is not None or self.config.max_loss_increase is not None + return self.config.candidate_selection == "loss_window" + + def _use_margin_window(self) -> bool: + return self.config.candidate_selection == "margin_window" def _clean_loss(self, model, x_clean: torch.Tensor, y: torch.Tensor) -> torch.Tensor: # Baseline difficulty. Candidate scores become loss(candidate) - loss(clean). @@ -231,8 +241,35 @@ def _loss_window_better(self, candidate_score: torch.Tensor, best_score: torch.T valid = valid & (candidate_score <= float(self.config.max_loss_increase)) return valid & (candidate_score > best_score) - def _target_reached(self, best_score: torch.Tensor) -> bool: - # Once every sample has reached the requested hardness, stop taking stronger steps. - if self.config.target_loss_increase is None: - return False - return bool((best_score >= float(self.config.target_loss_increase)).all().item()) + def _margin_window_distance(self, margin: torch.Tensor) -> torch.Tensor: + # Distance is zero inside the window and positive outside. This gives a + # soft fallback when discrete tabular steps jump over the desired range. + distance = torch.zeros_like(margin) + if self.config.target_margin is not None: + target = torch.full_like(margin, float(self.config.target_margin)) + distance = torch.maximum(distance, target - margin) + if self.config.max_margin is not None: + maximum = torch.full_like(margin, float(self.config.max_margin)) + distance = torch.maximum(distance, margin - maximum) + return distance + + def _margin_window_better( + self, + candidate_score: torch.Tensor, + candidate_distance: torch.Tensor, + best_score: torch.Tensor, + best_distance: torch.Tensor, + ) -> torch.Tensor: + closer = candidate_distance < best_distance + same_distance = candidate_distance == best_distance + stronger = candidate_score > best_score + return closer | (same_distance & stronger) + + def _target_reached(self, best_score: torch.Tensor, best_distance: torch.Tensor) -> bool: + if self._use_loss_window(): + if self.config.target_loss_increase is None: + return False + return bool((best_score >= float(self.config.target_loss_increase)).all().item()) + if self._use_margin_window(): + return bool((best_distance <= torch.finfo(best_distance.dtype).eps).all().item()) + return False diff --git a/nebula/core/datasets/covtype/covtype.py b/nebula/core/datasets/covtype/covtype.py index e3be64318..ec3ef65d9 100644 --- a/nebula/core/datasets/covtype/covtype.py +++ b/nebula/core/datasets/covtype/covtype.py @@ -16,7 +16,7 @@ class CovtypeTorchDataset(Dataset): """ - Simple torch Dataset wrapper for tabular Covtype data. + Torch Dataset wrapper for tabular Covtype data. Returns: x: torch.float32 tensor of shape (n_features,) @@ -81,18 +81,16 @@ class CovtypePartitionHandler(NebulaPartitionHandler): def __init__(self, file_path: str, prefix: str, config: Any, empty: bool = False): super().__init__(file_path, prefix, config, empty) - # For tabular data we typically don't apply torchvision transforms. - # If you later want normalization here, do it explicitly and carefully - # (train stats vs test stats, per-partition stats, etc.). + # Tabular features are already preprocessed before partitioning, so no + # torchvision-style transform is applied here. self.transform = None def __getitem__(self, idx: int): data, target = super().__getitem__(idx) - # Defensive: depending on how NebulaPartitionHandler stores/returns, - # "data" might be list/tuple/np.ndarray. Ensure we end up with 1D float32 tensor. + # Partition storage can return lists, numpy arrays or tensors. The model + # expects a 1D float32 tensor for each tabular sample. if isinstance(data, tuple): - # Some vision datasets store (img, meta). For tabular we ignore extras. data = data[0] if isinstance(data, torch.Tensor): @@ -100,7 +98,7 @@ def __getitem__(self, idx: int): else: x = torch.tensor(np.asarray(data), dtype=torch.float32) - # Ensure target in [0..num_classes-1] and torch.long + # Targets are stored as class indices and consumed by CrossEntropyLoss. if isinstance(target, torch.Tensor): y = target.to(dtype=torch.long) else: @@ -119,7 +117,7 @@ class CovtypeDataset(NebulaDataset): Notes: - Covtype has 7 classes. - Features are tabular (54 features in the classic version). - - We provide a simple train/test split with fixed seed. + - Deterministic stratified train/test split. Requirements: - scikit-learn must be installed (for fetch_covtype + train_test_split). @@ -184,10 +182,14 @@ class CovtypeDataset(NebulaDataset): ] # Covtype has two kinds of inputs: # - terrain measurements, which constrained PGD may perturb; - # - binary wilderness/soil indicators, which stay immutable to avoid broken - # one-hot-like combinations. + # - binary wilderness/soil indicators, which are already one-hot-like. + # + # The binary groups are immutable in the current metadata. This avoids + # invalid wilderness/soil combinations while still exercising constrained + # PGD on the numeric part of the dataset. PERTURBABLE_CONTINUOUS_COLUMNS = list(CONTINUOUS_COLUMNS) PERTURBABLE_INTEGER_COLUMNS = [] + NON_PERTURBABLE_COLUMNS = list(BINARY_COLUMNS) def __init__( self, @@ -236,7 +238,7 @@ def _default_feature_names(cls, n_features: int) -> list[str]: def _validate_manual_schema(cls, columns) -> None: continuous_columns = set(cls.CONTINUOUS_COLUMNS) integer_columns = set(cls.PERTURBABLE_INTEGER_COLUMNS) - non_perturbable_columns = set(cls.BINARY_COLUMNS) + non_perturbable_columns = set(cls.NON_PERTURBABLE_COLUMNS) overlapping_columns = sorted( (continuous_columns & integer_columns) | (continuous_columns & non_perturbable_columns) @@ -290,13 +292,13 @@ def load_covtype_dataset(self): feature_names = self._default_feature_names(x.shape[1]) self._validate_manual_schema(feature_names) - # Map labels to 0..6 (CrossEntropyLoss convention) - # If already 0..6, this is harmless for 1..7 only if we detect min. + # sklearn usually returns labels in 1..7. CrossEntropyLoss expects + # zero-based class indices, so map them to 0..6 when needed. y = np.asarray(y).reshape(-1) if y.min() == 1: y = y - 1 - # Split "grande" + # Build a deterministic stratified train/test split. x_train, x_test, y_train, y_test = train_test_split( x, y, test_size=self.test_size, @@ -305,7 +307,8 @@ def load_covtype_dataset(self): stratify=y, ) - # Submuestreo estratificado (corto y determinista) + # Optional stratified limits keep experiments manageable without + # changing the class distribution unnecessarily. if self.train_limit is not None and len(y_train) > self.train_limit: x_train, _, y_train, _ = train_test_split( x_train, y_train, @@ -324,8 +327,8 @@ def load_covtype_dataset(self): stratify=y_test, ) - # Scale only the perturbable terrain measurements. The binary columns - # must remain 0/1 because they encode wilderness and soil indicators. + # Scale only the terrain measurements. The binary columns must remain + # exact 0/1 values because they encode wilderness and soil indicators. scaler = StandardScaler() x_train = np.asarray(x_train, dtype=np.float32).copy() x_test = np.asarray(x_test, dtype=np.float32).copy() @@ -364,7 +367,8 @@ def _make_dataset( @classmethod def _build_adversarial_metadata(cls, feature_names, x_train): # Dataset responsibility: declare which variables are perturbable. The - # shared builder maps those declarations to feature masks and bounds. + # shared builder marks every other feature, including binary indicators, + # as non-perturbable and creates the masks consumed by constrained PGD. return build_tabular_adversarial_metadata( feature_names=feature_names, x_train=x_train, @@ -380,7 +384,7 @@ def _log_adversarial_metadata(metadata: dict[str, Any], feature_names: list[str] continuous_features = metadata["continuous_features"] non_perturbable_features = metadata["non_perturbable_features"] logger.info( - "[Covtype] Tabular adversarial feature mask | continuous=%s | non_perturbable=%s | " + "[Covtype] Tabular adversarial feature mask | continuous=%s | binary_non_perturbable=%s | " "continuous_features=%s | non_perturbable_preview=%s", len(continuous_features), len(non_perturbable_features), diff --git a/nebula/core/datasets/kddcup99/kddcup99.py b/nebula/core/datasets/kddcup99/kddcup99.py index 644af9e3b..72c3db0f4 100644 --- a/nebula/core/datasets/kddcup99/kddcup99.py +++ b/nebula/core/datasets/kddcup99/kddcup99.py @@ -1,18 +1,20 @@ import logging import os -from typing import Tuple, Any +from typing import Any import numpy as np import torch from torch.utils.data import Dataset from nebula.core.datasets.nebuladataset import NebulaDataset, NebulaPartitionHandler -from nebula.core.datasets.tabular_metadata import CONTINUOUS, INTEGER, NON_PERTURBABLE, TabularAdversarialMetadata +from nebula.core.datasets.tabular_metadata import build_tabular_adversarial_metadata + +logger = logging.getLogger(__name__) class KDDCUP99TorchDataset(Dataset): """ - Simple torch Dataset wrapper for tabular KDDCUP99 data. + Torch Dataset wrapper for tabular KDDCUP99 data. Returns: x: torch.float32 tensor of shape (n_features,) @@ -59,7 +61,7 @@ def __init__( def __len__(self) -> int: return int(self.y.shape[0]) - def __getitem__(self, idx: int) -> Tuple[torch.Tensor, torch.Tensor]: + def __getitem__(self, idx: int) -> tuple[torch.Tensor, torch.Tensor]: x_i = torch.from_numpy(self.x[idx]) y_i = torch.tensor(self.y[idx], dtype=torch.long) return x_i, y_i @@ -76,14 +78,15 @@ class KDDCUP99PartitionHandler(NebulaPartitionHandler): def __init__(self, file_path: str, prefix: str, config: Any, empty: bool = False): super().__init__(file_path, prefix, config, empty) - # For tabular data we typically don't apply torchvision transforms. + # Tabular features are already preprocessed before partitioning, so no + # torchvision-style transform is applied here. self.transform = None def __getitem__(self, idx: int): data, target = super().__getitem__(idx) - # Defensive: depending on how NebulaPartitionHandler stores/returns, - # "data" might be list/tuple/np.ndarray. Ensure we end up with 1D float32 tensor. + # Partition storage can return lists, numpy arrays or tensors. The model + # expects a 1D float32 tensor for each tabular sample. if isinstance(data, tuple): data = data[0] @@ -92,7 +95,7 @@ def __getitem__(self, idx: int): else: x = torch.tensor(np.asarray(data), dtype=torch.float32) - # Ensure target in [0..num_classes-1] and torch.long + # Targets are stored as class indices and consumed by CrossEntropyLoss. if isinstance(target, torch.Tensor): y = target.to(dtype=torch.long) else: @@ -112,7 +115,7 @@ class KDDCUP99Dataset(NebulaDataset): - KDDCUP99 is a tabular intrusion-detection dataset. - sklearn fetch_kddcup99 exposes 41 features. - Targets are mapped to a binary task: normal vs attack. - - Some columns are categorical/string-like, so we one-hot encode them. + - Categorical string columns are one-hot encoded. - Targets may come as bytes/strings, so we decode before mapping labels. Requirements: @@ -162,7 +165,7 @@ class KDDCUP99Dataset(NebulaDataset): "dst_host_rerror_rate", "dst_host_srv_rerror_rate", ] - PERTURBABLE_CONTINUOUS_COLUMNS = [ + CONTINUOUS_COLUMNS = [ "serror_rate", "srv_serror_rate", "rerror_rate", @@ -179,7 +182,7 @@ class KDDCUP99Dataset(NebulaDataset): "dst_host_rerror_rate", "dst_host_srv_rerror_rate", ] - PERTURBABLE_INTEGER_COLUMNS = [ + INTEGER_COLUMNS = [ "duration", "src_bytes", "dst_bytes", @@ -198,10 +201,12 @@ class KDDCUP99Dataset(NebulaDataset): "dst_host_count", "dst_host_srv_count", ] - NON_PERTURBABLE_RAW_COLUMNS = [ + CATEGORICAL_COLUMNS = [ "protocol_type", "service", "flag", + ] + NON_PERTURBABLE_COLUMNS = [ "land", "logged_in", "root_shell", @@ -209,6 +214,12 @@ class KDDCUP99Dataset(NebulaDataset): "is_host_login", "is_guest_login", ] + # KDDCUP99 exposes mixed network-traffic features. For the first supported + # adversarial-training version, constrained PGD may perturb numeric traffic + # measurements and counters. Protocol/service/flag one-hot columns and + # binary login/status flags stay immutable to avoid invalid records. + PERTURBABLE_CONTINUOUS_COLUMNS = list(CONTINUOUS_COLUMNS) + PERTURBABLE_INTEGER_COLUMNS = list(INTEGER_COLUMNS) def __init__( self, @@ -259,18 +270,22 @@ def _ensure_raw_feature_names(cls, x): @classmethod def _validate_manual_schema(cls, columns) -> None: - continuous_columns = set(cls.PERTURBABLE_CONTINUOUS_COLUMNS) - integer_columns = set(cls.PERTURBABLE_INTEGER_COLUMNS) - non_perturbable_columns = set(cls.NON_PERTURBABLE_RAW_COLUMNS) + continuous_columns = set(cls.CONTINUOUS_COLUMNS) + integer_columns = set(cls.INTEGER_COLUMNS) + categorical_columns = set(cls.CATEGORICAL_COLUMNS) + non_perturbable_columns = set(cls.NON_PERTURBABLE_COLUMNS) overlapping_columns = sorted( (continuous_columns & integer_columns) + | (continuous_columns & categorical_columns) | (continuous_columns & non_perturbable_columns) + | (integer_columns & categorical_columns) | (integer_columns & non_perturbable_columns) + | (categorical_columns & non_perturbable_columns) ) if overlapping_columns: raise ValueError(f"KDDCUP99Dataset columns configured twice: {overlapping_columns}") - configured_columns = continuous_columns | integer_columns | non_perturbable_columns + configured_columns = continuous_columns | integer_columns | categorical_columns | non_perturbable_columns dataset_columns = set(columns) missing_columns = sorted(configured_columns - dataset_columns) if missing_columns: @@ -284,7 +299,6 @@ def load_kddcup99_dataset(self): Loads KDDCUP99 via sklearn, performs deterministic preprocessing and train/test split, and wraps into torch Datasets. """ - # Local cache directory for sklearn dataset downloads data_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "data") os.makedirs(data_dir, exist_ok=True) @@ -296,7 +310,6 @@ def load_kddcup99_dataset(self): except Exception as e: raise ImportError( "KDDCUP99Dataset requires scikit-learn and pandas. " - "Install them (e.g., pip install scikit-learn pandas)." ) from e kdd = fetch_kddcup99( @@ -312,7 +325,8 @@ def load_kddcup99_dataset(self): x = kdd.data y = kdd.target - # Defensive conversion to pandas objects + # fetch_kddcup99 can return numpy arrays depending on sklearn version. + # The preprocessing below expects pandas columns. if not hasattr(x, "columns"): x = pd.DataFrame(x) if not hasattr(y, "astype"): @@ -320,47 +334,30 @@ def load_kddcup99_dataset(self): x = self._ensure_raw_feature_names(x) self._validate_manual_schema(x.columns) - # Decode bytes -> str where needed def _decode_if_bytes(v): if isinstance(v, (bytes, bytearray)): return v.decode("utf-8", errors="ignore") return v - # Some KDDCUP99 columns are categorical (e.g. protocol/service/flag). - # We decode bytes and one-hot encode object/category columns. + # Decode bytes before one-hot encoding categorical columns and mapping labels. for col in x.columns: if x[col].dtype == object: x[col] = x[col].map(_decode_if_bytes) y = y.map(_decode_if_bytes) - # One-hot encode categorical columns, keep numeric ones as-is. + # One-hot encode protocol/service/flag and keep numeric columns as-is. x = pd.get_dummies(x, drop_first=False) feature_names = [str(col) for col in x.columns] - logging.getLogger().info("[KDDCUP99] Encoded feature dimension: %s", len(feature_names)) - continuous_features = [ - x.columns.get_loc(col) - for col in self.PERTURBABLE_CONTINUOUS_COLUMNS - if col in x.columns - ] - integer_features = [ - x.columns.get_loc(col) - for col in self.PERTURBABLE_INTEGER_COLUMNS - if col in x.columns - ] - perturbable_features = set(continuous_features) | set(integer_features) - non_perturbable_features = [i for i in range(len(feature_names)) if i not in perturbable_features] - binary_features = non_perturbable_features + logger.info("[KDDCUP99] Encoded feature dimension: %s", len(feature_names)) # Map labels to a binary task: 0 = normal, 1 = attack. y = pd.Series(y).astype(str) y = y.str.strip() y = (y != "normal.").astype(np.int64).to_numpy(copy=False) - - classes = ["normal", "attack"] self.num_classes = 2 - # Split "grande" + # Build a deterministic stratified train/test split. x_train, x_test, y_train, y_test = train_test_split( x, y, test_size=self.test_size, @@ -369,7 +366,8 @@ def _decode_if_bytes(v): stratify=y, ) - # Submuestreo estratificado (corto y determinista) + # Optional stratified limits keep experiments manageable without + # changing the class distribution unnecessarily. if self.train_limit is not None and len(y_train) > self.train_limit: x_train, _, y_train, _ = train_test_split( x_train, y_train, @@ -378,7 +376,7 @@ def _decode_if_bytes(v): shuffle=True, stratify=y_train, ) - logging.getLogger().info("[KDDCUP99] Limited train split to %s samples", len(y_train)) + logger.info("[KDDCUP99] Limited train split to %s samples", len(y_train)) if self.test_limit is not None and len(y_test) > self.test_limit: x_test, _, y_test, _ = train_test_split( @@ -388,64 +386,97 @@ def _decode_if_bytes(v): shuffle=True, stratify=y_test, ) - logging.getLogger().info("[KDDCUP99] Limited test split to %s samples", len(y_test)) + logger.info("[KDDCUP99] Limited test split to %s samples", len(y_test)) x_train_np = x_train.astype(np.float32).to_numpy(copy=True) x_test_np = x_test.astype(np.float32).to_numpy(copy=True) - # Scale perturbable numeric columns after splitting. One-hot and binary flags stay unchanged. + # Scale perturbable numeric columns after splitting. One-hot categorical + # columns and binary flags remain exact 0/1 values. + continuous_features = self._column_indices(x_train.columns, self.CONTINUOUS_COLUMNS) + integer_features = self._column_indices(x_train.columns, self.INTEGER_COLUMNS) scaled_features = continuous_features + integer_features + integer_step_by_column = {} if scaled_features: scaler = StandardScaler() x_train_np[:, scaled_features] = scaler.fit_transform(x_train_np[:, scaled_features]) x_test_np[:, scaled_features] = scaler.transform(x_test_np[:, scaled_features]) - integer_step_norm = {} - if integer_features: - integer_step_norm = { - idx: float(1.0 / scale) - for idx, scale in zip(integer_features, scaler.scale_[len(continuous_features):], strict=False) + integer_scales = scaler.scale_[len(continuous_features):] + integer_step_by_column = { + column: float(1.0 / scale) + for column, scale in zip(self.INTEGER_COLUMNS, integer_scales, strict=False) } - continuous_feature_set = set(continuous_features) - integer_feature_set = set(integer_features) - tabular_metadata = TabularAdversarialMetadata( - feature_names=feature_names, - feature_types=[ - CONTINUOUS if idx in continuous_feature_set - else INTEGER if idx in integer_feature_set - else NON_PERTURBABLE - for idx in range(len(feature_names)) - ], - feature_min_norm=np.min(x_train_np, axis=0).astype(float).tolist(), - feature_max_norm=np.max(x_train_np, axis=0).astype(float).tolist(), - integer_step_norm=integer_step_norm, - ).to_dict() - - train_ds = KDDCUP99TorchDataset( - x_train_np, - y_train, - feature_names=feature_names, - continuous_features=continuous_features, - integer_features=integer_features, - non_perturbable_features=non_perturbable_features, - binary_features=binary_features, - tabular_metadata=tabular_metadata, + + metadata = self._build_adversarial_metadata(feature_names, x_train_np, integer_step_by_column) + self._log_adversarial_metadata(metadata, feature_names) + + return ( + self._make_dataset(x_train_np, y_train, feature_names, metadata), + self._make_dataset(x_test_np, y_test, feature_names, metadata), ) - test_ds = KDDCUP99TorchDataset( - x_test_np, - y_test, + + @staticmethod + def _column_indices(columns, names: list[str]) -> list[int]: + return [columns.get_loc(name) for name in names if name in columns] + + @staticmethod + def _make_dataset( + x: np.ndarray, + y: np.ndarray, + feature_names: list[str], + metadata: dict[str, Any], + ) -> KDDCUP99TorchDataset: + dataset = KDDCUP99TorchDataset( + x, + y, feature_names=feature_names, - continuous_features=continuous_features, - integer_features=integer_features, - non_perturbable_features=non_perturbable_features, - binary_features=binary_features, - tabular_metadata=tabular_metadata, + continuous_features=metadata["continuous_features"], + integer_features=metadata["integer_features"], + non_perturbable_features=metadata["non_perturbable_features"], + binary_features=metadata["non_perturbable_features"], + tabular_metadata=metadata["tabular_metadata"], ) + dataset.classes = ["normal", "attack"] + return dataset - # Optional: preserve original class names for inspection/debugging - train_ds.classes = classes - test_ds.classes = classes + @classmethod + def _build_adversarial_metadata( + cls, + feature_names: list[str], + x_train: np.ndarray, + integer_step_by_column: dict[str, float], + ) -> dict[str, Any]: + # Dataset responsibility: declare which raw variables are perturbable. + # The shared builder maps that declaration to transformed feature masks, + # bounds and integer steps in model-input space. + return build_tabular_adversarial_metadata( + feature_names=feature_names, + x_train=x_train, + continuous_columns=cls.CONTINUOUS_COLUMNS, + integer_columns=cls.INTEGER_COLUMNS, + categorical_columns=cls.CATEGORICAL_COLUMNS, + perturbable_continuous_columns=cls.PERTURBABLE_CONTINUOUS_COLUMNS, + perturbable_integer_columns=cls.PERTURBABLE_INTEGER_COLUMNS, + integer_step_by_column=integer_step_by_column, + ) - return train_ds, test_ds + @staticmethod + def _log_adversarial_metadata(metadata: dict[str, Any], feature_names: list[str]) -> None: + continuous_features = metadata["continuous_features"] + integer_features = metadata["integer_features"] + non_perturbable_features = metadata["non_perturbable_features"] + logger.info( + "[KDDCUP99] Tabular adversarial feature mask | continuous=%s | integer=%s | " + "non_perturbable=%s | continuous_features=%s | integer_features=%s | " + "non_perturbable_preview=%s | integer_step_norm=%s", + len(continuous_features), + len(integer_features), + len(non_perturbable_features), + [feature_names[idx] for idx in continuous_features], + [feature_names[idx] for idx in integer_features], + [feature_names[idx] for idx in non_perturbable_features[:20]], + metadata["integer_step_norm"], + ) def generate_non_iid_map(self, dataset, partition: str = "dirichlet", partition_parameter: float = 0.5): if partition == "dirichlet": diff --git a/nebula/frontend/config/participant.json.example b/nebula/frontend/config/participant.json.example index 88c017ba1..9d9552fa3 100755 --- a/nebula/frontend/config/participant.json.example +++ b/nebula/frontend/config/participant.json.example @@ -117,8 +117,11 @@ "steps": 1, "mode": "mixed", "apply_probability": 0.3, + "candidate_selection": "none", "target_loss_increase": null, - "max_loss_increase": null + "max_loss_increase": null, + "target_margin": 0, + "max_margin": 0.5 }, "reputation": { "enabled": false, diff --git a/nebula/frontend/static/js/deployment/adversarial-training.js b/nebula/frontend/static/js/deployment/adversarial-training.js index cb072cf9e..be1f3aca5 100644 --- a/nebula/frontend/static/js/deployment/adversarial-training.js +++ b/nebula/frontend/static/js/deployment/adversarial-training.js @@ -10,12 +10,15 @@ const AdversarialTrainingManager = (function() { mode: "mixed", apply_probability: 0.3, log_adversarial_metrics: true, + candidate_selection: "none", target_loss_increase: null, - max_loss_increase: null + max_loss_increase: null, + target_margin: 0, + max_margin: 0.5 }; const IMAGE_DATASETS = new Set(["MNIST", "FashionMNIST", "EMNIST", "CIFAR10", "CIFAR100"]); - const TABULAR_ADVERSARIAL_DATASETS = new Set(["AdultCensus", "BreastCancer", "Covtype"]); + const TABULAR_ADVERSARIAL_DATASETS = new Set(["AdultCensus", "BreastCancer", "Covtype", "KDDCUP99"]); const IMAGE_ATTACK_OPTIONS = [ {value: "fgsm", label: "FGSM"}, {value: "pgd", label: "PGD"} @@ -27,6 +30,7 @@ const AdversarialTrainingManager = (function() { function initializeAdversarialTraining() { setupAdversarialTrainingSwitch(); setupAttackSelector(); + setupCandidateSelectionSelector(); setupDatasetAwareness(); setAdversarialTrainingConfig(DEFAULT_ADVERSARIAL_TRAINING_CONFIG); } @@ -52,6 +56,15 @@ const AdversarialTrainingManager = (function() { }); } + function setupCandidateSelectionSelector() { + const candidateSelectionSelect = document.getElementById("adversarialTrainingCandidateSelection"); + if (!candidateSelectionSelect) return; + + candidateSelectionSelect.addEventListener("change", function() { + toggleCandidateSelectionSettings(this.value); + }); + } + function setupDatasetAwareness() { const datasetSelect = document.getElementById("datasetSelect"); if (!datasetSelect) return; @@ -71,17 +84,39 @@ const AdversarialTrainingManager = (function() { function toggleAttackSettings(attack) { const pgdSettings = document.getElementById("adversarial-training-pgd-settings"); const stepsTitle = document.getElementById("adversarialTrainingStepsTitle"); + const candidateSelectionSettings = document.getElementById("adversarial-training-candidate-selection-settings"); const lossWindowSettings = document.getElementById("adversarial-training-loss-window-settings"); + const marginWindowSettings = document.getElementById("adversarial-training-margin-window-settings"); const domain = document.getElementById("adversarialTrainingDomain")?.value || DEFAULT_ADVERSARIAL_TRAINING_CONFIG.domain; if (!pgdSettings) return; pgdSettings.style.display = ["pgd", "constrained_pgd"].includes(attack) ? "block" : "none"; - if (lossWindowSettings) { - lossWindowSettings.style.display = domain === "tabular" ? "block" : "none"; + if (candidateSelectionSettings) { + candidateSelectionSettings.style.display = domain === "tabular" ? "block" : "none"; } if (stepsTitle) { stepsTitle.textContent = domain === "tabular" ? "Constrained PGD steps" : "PGD steps"; } + if (domain !== "tabular") { + if (lossWindowSettings) lossWindowSettings.style.display = "none"; + if (marginWindowSettings) marginWindowSettings.style.display = "none"; + return; + } + toggleCandidateSelectionSettings( + document.getElementById("adversarialTrainingCandidateSelection")?.value + || DEFAULT_ADVERSARIAL_TRAINING_CONFIG.candidate_selection + ); + } + + function toggleCandidateSelectionSettings(candidateSelection) { + const lossWindowSettings = document.getElementById("adversarial-training-loss-window-settings"); + const marginWindowSettings = document.getElementById("adversarial-training-margin-window-settings"); + if (lossWindowSettings) { + lossWindowSettings.style.display = candidateSelection === "loss_window" ? "block" : "none"; + } + if (marginWindowSettings) { + marginWindowSettings.style.display = candidateSelection === "margin_window" ? "block" : "none"; + } } function updateDatasetAvailability() { @@ -94,7 +129,7 @@ const AdversarialTrainingManager = (function() { if (datasetNote) { datasetNote.style.display = domain === "unsupported" ? "block" : "none"; - datasetNote.textContent = "Adversarial Training for tabular datasets currently supports AdultCensus, BreastCancer, and Covtype with constrained PGD."; + datasetNote.textContent = "Adversarial Training for tabular datasets currently supports AdultCensus, BreastCancer, Covtype, and KDDCUP99 with constrained PGD."; } if (domainInput) { domainInput.value = domain === "unsupported" ? "tabular" : domain; @@ -181,6 +216,8 @@ const AdversarialTrainingManager = (function() { steps: integerValue("adversarialTrainingSteps", DEFAULT_ADVERSARIAL_TRAINING_CONFIG.steps), mode: document.getElementById("adversarialTrainingMode")?.value || DEFAULT_ADVERSARIAL_TRAINING_CONFIG.mode, apply_probability: numberValue("adversarialTrainingApplyProbability", DEFAULT_ADVERSARIAL_TRAINING_CONFIG.apply_probability), + candidate_selection: document.getElementById("adversarialTrainingCandidateSelection")?.value + || DEFAULT_ADVERSARIAL_TRAINING_CONFIG.candidate_selection, target_loss_increase: optionalNumberValue( "adversarialTrainingTargetLossIncrease", DEFAULT_ADVERSARIAL_TRAINING_CONFIG.target_loss_increase @@ -189,18 +226,35 @@ const AdversarialTrainingManager = (function() { "adversarialTrainingMaxLossIncrease", DEFAULT_ADVERSARIAL_TRAINING_CONFIG.max_loss_increase ), + target_margin: optionalNumberValue( + "adversarialTrainingTargetMargin", + DEFAULT_ADVERSARIAL_TRAINING_CONFIG.target_margin + ), + max_margin: optionalNumberValue( + "adversarialTrainingMaxMargin", + DEFAULT_ADVERSARIAL_TRAINING_CONFIG.max_margin + ), log_adversarial_metrics: true }; if (config.alpha === null || config.attack !== "pgd") { delete config.alpha; } - if (config.target_loss_increase === null) { + if (config.domain !== "tabular") { + delete config.candidate_selection; + } + if (config.candidate_selection !== "loss_window" || config.target_loss_increase === null) { delete config.target_loss_increase; } - if (config.max_loss_increase === null) { + if (config.candidate_selection !== "loss_window" || config.max_loss_increase === null) { delete config.max_loss_increase; } + if (config.candidate_selection !== "margin_window" || config.target_margin === null) { + delete config.target_margin; + } + if (config.candidate_selection !== "margin_window" || config.max_margin === null) { + delete config.max_margin; + } return config; } @@ -224,8 +278,16 @@ const AdversarialTrainingManager = (function() { : DEFAULT_ADVERSARIAL_TRAINING_CONFIG.mode ); setValue("adversarialTrainingApplyProbability", adversarialTrainingConfig.apply_probability); + setValue( + "adversarialTrainingCandidateSelection", + ["none", "loss_window", "margin_window"].includes(adversarialTrainingConfig.candidate_selection) + ? adversarialTrainingConfig.candidate_selection + : DEFAULT_ADVERSARIAL_TRAINING_CONFIG.candidate_selection + ); setValue("adversarialTrainingTargetLossIncrease", adversarialTrainingConfig.target_loss_increase ?? ""); setValue("adversarialTrainingMaxLossIncrease", adversarialTrainingConfig.max_loss_increase ?? ""); + setValue("adversarialTrainingTargetMargin", adversarialTrainingConfig.target_margin ?? 0); + setValue("adversarialTrainingMaxMargin", adversarialTrainingConfig.max_margin ?? 0.5); updateDatasetAvailability(); const domain = document.getElementById("adversarialTrainingDomain")?.value || adversarialTrainingConfig.domain; @@ -261,6 +323,12 @@ const AdversarialTrainingManager = (function() { if (config.apply_probability < 0 || config.apply_probability > 1) { return "[Adversarial Training] Apply probability must be between 0 and 1."; } + if ( + config.candidate_selection !== undefined + && !["none", "loss_window", "margin_window"].includes(config.candidate_selection) + ) { + return "[Adversarial Training] Candidate selection must be None, Loss window, or Margin window."; + } if (config.target_loss_increase !== undefined && config.target_loss_increase < 0) { return "[Adversarial Training] Target loss increase must be greater than or equal to 0."; } @@ -274,6 +342,13 @@ const AdversarialTrainingManager = (function() { ) { return "[Adversarial Training] Target loss increase must be smaller than or equal to max loss increase."; } + if ( + config.target_margin !== undefined + && config.max_margin !== undefined + && config.target_margin > config.max_margin + ) { + return "[Adversarial Training] Target margin must be smaller than or equal to max margin."; + } return null; } diff --git a/nebula/frontend/templates/deployment.html b/nebula/frontend/templates/deployment.html index 01e9ed5a4..f5171f14e 100755 --- a/nebula/frontend/templates/deployment.html +++ b/nebula/frontend/templates/deployment.html @@ -588,7 +588,7 @@
    Enable/Disable Adversarial Training
    style="display: inline; width: 80px; height: 30px;"> + + From 48c124232a4a3e6938e41c95b62086a734ed6c34 Mon Sep 17 00:00:00 2001 From: "Juan J." Date: Wed, 10 Jun 2026 18:40:37 +0200 Subject: [PATCH 63/66] Robustness metrics revised and fixed, trustworthiness metrics revised, models updated: Data type, factsheets updated, tabular robustness metrics deleted --- .../defenses/adversarial_training/config.py | 11 +- .../defenses/adversarial_training/defense.py | 3 +- .../addons/trustworthiness/cfl_factsheet.py | 4 +- .../configs/eval_metrics_cfl.json | 28 +- .../configs/eval_metrics_cfl_images.json | 28 +- .../configs/eval_metrics_cfl_tabular.json | 70 +-- .../configs/eval_metrics_dfl.json | 28 +- .../configs/eval_metrics_dfl_images.json | 28 +- .../configs/eval_metrics_dfl_tabular.json | 70 +-- .../configs/factsheet_template_cfl.json | 8 +- .../factsheet_template_cfl_images.json | 8 +- .../factsheet_template_cfl_tabular.json | 7 +- .../configs/factsheet_template_dfl.json | 8 +- .../factsheet_template_dfl_images.json | 8 +- .../factsheet_template_dfl_tabular.json | 7 +- .../addons/trustworthiness/dfl_factsheet.py | 1 + .../trustworthiness/factsheet_common.py | 50 +- .../trustworthiness/factsheet_populators.py | 78 ++- .../trustworthiness/helpers/robustness.py | 473 +++++++++++++----- nebula/core/datasets/image_metadata.py | 14 + nebula/core/datasets/kddcup99/kddcup99.py | 4 +- nebula/core/models/adultcensus/mlp.py | 1 + nebula/core/models/breast_cancer/mlp.py | 1 + nebula/core/models/cifar10/cnn.py | 1 + nebula/core/models/cifar10/cnnV2.py | 1 + nebula/core/models/cifar10/cnnV3.py | 1 + nebula/core/models/cifar10/fastermobilenet.py | 1 + nebula/core/models/cifar10/resnet.py | 1 + nebula/core/models/cifar10/simplemobilenet.py | 1 + nebula/core/models/cifar100/cnn.py | 1 + nebula/core/models/covtype/mlp.py | 1 + nebula/core/models/emnist/cnn.py | 1 + nebula/core/models/emnist/mlp.py | 1 + nebula/core/models/fashionmnist/cnn.py | 1 + nebula/core/models/fashionmnist/mlp.py | 1 + nebula/core/models/kddcup99/mlp.py | 1 + nebula/core/models/mnist/cnn.py | 1 + nebula/core/models/mnist/mlp.py | 1 + nebula/core/models/sentiment140/cnn.py | 1 + nebula/core/models/sentiment140/rnn.py | 1 + 40 files changed, 571 insertions(+), 384 deletions(-) create mode 100644 nebula/core/datasets/image_metadata.py diff --git a/nebula/addons/defenses/adversarial_training/config.py b/nebula/addons/defenses/adversarial_training/config.py index 8cbcdddcd..48dd73e6e 100644 --- a/nebula/addons/defenses/adversarial_training/config.py +++ b/nebula/addons/defenses/adversarial_training/config.py @@ -1,6 +1,8 @@ from dataclasses import dataclass from typing import Any +from nebula.core.datasets.image_metadata import IMAGE_DATASET_NORMALIZATION + IMAGE_ADVERSARIAL_ATTACKS = {"fgsm", "pgd"} TABULAR_ADVERSARIAL_ATTACKS = {"constrained_pgd"} TABULAR_ADVERSARIAL_DATASETS = {"AdultCensus", "BreastCancer", "Covtype", "KDDCUP99"} @@ -20,15 +22,6 @@ ERR_TABULAR_METADATA = "Tabular adversarial training requires tabular_metadata" ERR_UNSUPPORTED_ATTACK = "Unsupported adversarial training attack: {attack}" -IMAGE_DATASET_NORMALIZATION = { - "MNIST": ((0.5,), (0.5,)), - "FashionMNIST": ((0.5,), (0.5,)), - "EMNIST": ((0.5,), (0.5,)), - "CIFAR10": ((0.4914, 0.4822, 0.4465), (0.2471, 0.2435, 0.2616)), - "CIFAR100": ((0.4914, 0.4822, 0.4465), (0.2471, 0.2435, 0.2616)), -} - - @dataclass(frozen=True) class AdversarialTrainingConfig: enabled: bool = False diff --git a/nebula/addons/defenses/adversarial_training/defense.py b/nebula/addons/defenses/adversarial_training/defense.py index b79285009..1e0cbc8de 100644 --- a/nebula/addons/defenses/adversarial_training/defense.py +++ b/nebula/addons/defenses/adversarial_training/defense.py @@ -25,6 +25,7 @@ config_from_participant, validate_config, ) +from nebula.core.datasets.image_metadata import get_image_normalization from nebula.addons.defenses.adversarial_training.image import ( ImageAdversarialExampleGenerator, ImageFGSMGenerator, @@ -69,7 +70,7 @@ def from_participant_config( if config.domain == "image": # Image attacks run in normalized model space, so each dataset must provide mean/std. - normalization = IMAGE_DATASET_NORMALIZATION.get(config.dataset_name) + normalization = get_image_normalization(config.dataset_name) if normalization is None: logging.warning( "[AdversarialTrainingDefense] Skipping adversarial training: dataset '%s' has no image bounds", diff --git a/nebula/addons/trustworthiness/cfl_factsheet.py b/nebula/addons/trustworthiness/cfl_factsheet.py index b8cbe104b..1144571db 100755 --- a/nebula/addons/trustworthiness/cfl_factsheet.py +++ b/nebula/addons/trustworthiness/cfl_factsheet.py @@ -23,7 +23,6 @@ get_underfitting_score, ) from nebula.addons.trustworthiness.factsheet_common import ( - cap_score, get_factsheet_path, get_factsheet_template_name, get_trustworthiness_dir, @@ -67,6 +66,7 @@ def populate_factsheet_cfl( data["federation"], model, self.factsheet_template_file_nm, + dataset_name=data["dataset"], ) try: @@ -129,7 +129,7 @@ def populate_factsheet_cfl( # Convert class imbalance and runtime summaries into factsheet fields. class_imbalance_score = get_class_imbalance_score(avg_class_imbalance) - factsheet["fairness"]["class_imbalance"] = cap_score(class_imbalance_score) + factsheet["fairness"]["class_imbalance"] = class_imbalance_score populate_reputation(factsheet, reputation_summary) underfitting_score = get_underfitting_score(scenario_name, participant_idx) diff --git a/nebula/addons/trustworthiness/configs/eval_metrics_cfl.json b/nebula/addons/trustworthiness/configs/eval_metrics_cfl.json index 635d2e9a0..520e32ed6 100755 --- a/nebula/addons/trustworthiness/configs/eval_metrics_cfl.json +++ b/nebula/addons/trustworthiness/configs/eval_metrics_cfl.json @@ -7,7 +7,7 @@ "inputs": [ { "source": "factsheet", - "field_path": "performance/clipped_test_clever" + "field_path": "performance/test_clever_score" } ], "operation": "get_value", @@ -27,40 +27,40 @@ "description": "Inverse loss sensitivity score; higher values indicate lower sensitivity of the loss to input perturbations.", "weight": 0.2 }, - "clipped_adversarial_accuracy": { + "adversarial_accuracy": { "inputs": [ { "source": "factsheet", - "field_path": "performance/clipped_test_adv_accuracy" + "field_path": "performance/test_adv_accuracy" } ], "operation": "get_value", "type": "true_score", - "description": "Adversarial accuracy clipped to the expected score range; higher values indicate better predictive performance under adversarial perturbations.", + "description": "Adversarial accuracy; higher values indicate better predictive performance under adversarial perturbations.", "weight": 0.2 }, - "clipped_empirical_robustness": { + "empirical_robustness_score": { "inputs": [ { "source": "factsheet", - "field_path": "performance/clipped_test_empirical_robustness" + "field_path": "performance/test_empirical_robustness_score" } ], "operation": "get_value", "type": "true_score", - "description": "Empirical robustness clipped to the expected score range; higher values indicate stronger resistance to adversarial perturbations.", + "description": "Empirical robustness score; higher values indicate stronger resistance to adversarial perturbations.", "weight": 0.15 }, - "clipped_confidence_score": { + "confidence_score": { "inputs": [ { "source": "factsheet", - "field_path": "performance/clipped_test_confidence_score" + "field_path": "performance/test_confidence_score" } ], "operation": "get_value", "type": "true_score", - "description": "Confidence score clipped to the expected score range; higher values indicate more stable predictive confidence.", + "description": "Confidence score; higher values indicate more stable predictive confidence.", "weight": 0.1 }, "inverse_attack_success_rate": { @@ -665,7 +665,7 @@ }, { "source": "factsheet", - "field_path": "performance/clipped_test_clever" + "field_path": "performance/test_clever_score" }, { "source": "factsheet", @@ -673,15 +673,15 @@ }, { "source": "factsheet", - "field_path": "performance/clipped_test_adv_accuracy" + "field_path": "performance/test_adv_accuracy" }, { "source": "factsheet", - "field_path": "performance/clipped_test_empirical_robustness" + "field_path": "performance/test_empirical_robustness_score" }, { "source": "factsheet", - "field_path": "performance/clipped_test_confidence_score" + "field_path": "performance/test_confidence_score" }, { "source": "factsheet", diff --git a/nebula/addons/trustworthiness/configs/eval_metrics_cfl_images.json b/nebula/addons/trustworthiness/configs/eval_metrics_cfl_images.json index 635d2e9a0..520e32ed6 100755 --- a/nebula/addons/trustworthiness/configs/eval_metrics_cfl_images.json +++ b/nebula/addons/trustworthiness/configs/eval_metrics_cfl_images.json @@ -7,7 +7,7 @@ "inputs": [ { "source": "factsheet", - "field_path": "performance/clipped_test_clever" + "field_path": "performance/test_clever_score" } ], "operation": "get_value", @@ -27,40 +27,40 @@ "description": "Inverse loss sensitivity score; higher values indicate lower sensitivity of the loss to input perturbations.", "weight": 0.2 }, - "clipped_adversarial_accuracy": { + "adversarial_accuracy": { "inputs": [ { "source": "factsheet", - "field_path": "performance/clipped_test_adv_accuracy" + "field_path": "performance/test_adv_accuracy" } ], "operation": "get_value", "type": "true_score", - "description": "Adversarial accuracy clipped to the expected score range; higher values indicate better predictive performance under adversarial perturbations.", + "description": "Adversarial accuracy; higher values indicate better predictive performance under adversarial perturbations.", "weight": 0.2 }, - "clipped_empirical_robustness": { + "empirical_robustness_score": { "inputs": [ { "source": "factsheet", - "field_path": "performance/clipped_test_empirical_robustness" + "field_path": "performance/test_empirical_robustness_score" } ], "operation": "get_value", "type": "true_score", - "description": "Empirical robustness clipped to the expected score range; higher values indicate stronger resistance to adversarial perturbations.", + "description": "Empirical robustness score; higher values indicate stronger resistance to adversarial perturbations.", "weight": 0.15 }, - "clipped_confidence_score": { + "confidence_score": { "inputs": [ { "source": "factsheet", - "field_path": "performance/clipped_test_confidence_score" + "field_path": "performance/test_confidence_score" } ], "operation": "get_value", "type": "true_score", - "description": "Confidence score clipped to the expected score range; higher values indicate more stable predictive confidence.", + "description": "Confidence score; higher values indicate more stable predictive confidence.", "weight": 0.1 }, "inverse_attack_success_rate": { @@ -665,7 +665,7 @@ }, { "source": "factsheet", - "field_path": "performance/clipped_test_clever" + "field_path": "performance/test_clever_score" }, { "source": "factsheet", @@ -673,15 +673,15 @@ }, { "source": "factsheet", - "field_path": "performance/clipped_test_adv_accuracy" + "field_path": "performance/test_adv_accuracy" }, { "source": "factsheet", - "field_path": "performance/clipped_test_empirical_robustness" + "field_path": "performance/test_empirical_robustness_score" }, { "source": "factsheet", - "field_path": "performance/clipped_test_confidence_score" + "field_path": "performance/test_confidence_score" }, { "source": "factsheet", diff --git a/nebula/addons/trustworthiness/configs/eval_metrics_cfl_tabular.json b/nebula/addons/trustworthiness/configs/eval_metrics_cfl_tabular.json index 635d2e9a0..a75400052 100755 --- a/nebula/addons/trustworthiness/configs/eval_metrics_cfl_tabular.json +++ b/nebula/addons/trustworthiness/configs/eval_metrics_cfl_tabular.json @@ -3,65 +3,29 @@ "resilience_to_attacks": { "weight": 0.4, "metrics": { - "certified_robustness": { + "adversarial_accuracy": { "inputs": [ { "source": "factsheet", - "field_path": "performance/clipped_test_clever" + "field_path": "performance/test_adv_accuracy" } ], "operation": "get_value", "type": "true_score", - "description": "Cross Lipschitz Extreme Value for network Robustness: attack-agnostic estimator of the lower bound βL", - "weight": 0.2 - }, - "inverse_loss_sensitivity": { - "inputs": [ - { - "source": "factsheet", - "field_path": "performance/inverse_test_loss_sensitivity" - } - ], - "operation": "get_value", - "type": "true_score", - "description": "Inverse loss sensitivity score; higher values indicate lower sensitivity of the loss to input perturbations.", - "weight": 0.2 + "description": "Adversarial accuracy; higher values indicate better predictive performance under adversarial perturbations.", + "weight": 0.4444444444 }, - "clipped_adversarial_accuracy": { + "confidence_score": { "inputs": [ { "source": "factsheet", - "field_path": "performance/clipped_test_adv_accuracy" + "field_path": "performance/test_confidence_score" } ], "operation": "get_value", "type": "true_score", - "description": "Adversarial accuracy clipped to the expected score range; higher values indicate better predictive performance under adversarial perturbations.", - "weight": 0.2 - }, - "clipped_empirical_robustness": { - "inputs": [ - { - "source": "factsheet", - "field_path": "performance/clipped_test_empirical_robustness" - } - ], - "operation": "get_value", - "type": "true_score", - "description": "Empirical robustness clipped to the expected score range; higher values indicate stronger resistance to adversarial perturbations.", - "weight": 0.15 - }, - "clipped_confidence_score": { - "inputs": [ - { - "source": "factsheet", - "field_path": "performance/clipped_test_confidence_score" - } - ], - "operation": "get_value", - "type": "true_score", - "description": "Confidence score clipped to the expected score range; higher values indicate more stable predictive confidence.", - "weight": 0.1 + "description": "Confidence score; higher values indicate more stable predictive confidence.", + "weight": 0.2222222222 }, "inverse_attack_success_rate": { "inputs": [ @@ -73,7 +37,7 @@ "operation": "get_value", "type": "true_score", "description": "Inverse attack success rate; higher values indicate a lower fraction of successful adversarial attacks.", - "weight": 0.15 + "weight": 0.3333333334 } } }, @@ -665,23 +629,11 @@ }, { "source": "factsheet", - "field_path": "performance/clipped_test_clever" - }, - { - "source": "factsheet", - "field_path": "performance/inverse_test_loss_sensitivity" - }, - { - "source": "factsheet", - "field_path": "performance/clipped_test_adv_accuracy" - }, - { - "source": "factsheet", - "field_path": "performance/clipped_test_empirical_robustness" + "field_path": "performance/test_adv_accuracy" }, { "source": "factsheet", - "field_path": "performance/clipped_test_confidence_score" + "field_path": "performance/test_confidence_score" }, { "source": "factsheet", diff --git a/nebula/addons/trustworthiness/configs/eval_metrics_dfl.json b/nebula/addons/trustworthiness/configs/eval_metrics_dfl.json index 80cb9486e..b43295c1d 100755 --- a/nebula/addons/trustworthiness/configs/eval_metrics_dfl.json +++ b/nebula/addons/trustworthiness/configs/eval_metrics_dfl.json @@ -7,7 +7,7 @@ "inputs": [ { "source": "factsheet", - "field_path": "performance/clipped_test_clever" + "field_path": "performance/test_clever_score" } ], "operation": "get_value", @@ -27,40 +27,40 @@ "description": "Inverse loss sensitivity score; higher values indicate lower sensitivity of the loss to input perturbations.", "weight": 0.2 }, - "clipped_adversarial_accuracy": { + "adversarial_accuracy": { "inputs": [ { "source": "factsheet", - "field_path": "performance/clipped_test_adv_accuracy" + "field_path": "performance/test_adv_accuracy" } ], "operation": "get_value", "type": "true_score", - "description": "Adversarial accuracy clipped to the expected score range; higher values indicate better predictive performance under adversarial perturbations.", + "description": "Adversarial accuracy; higher values indicate better predictive performance under adversarial perturbations.", "weight": 0.2 }, - "clipped_empirical_robustness": { + "empirical_robustness_score": { "inputs": [ { "source": "factsheet", - "field_path": "performance/clipped_test_empirical_robustness" + "field_path": "performance/test_empirical_robustness_score" } ], "operation": "get_value", "type": "true_score", - "description": "Empirical robustness clipped to the expected score range; higher values indicate stronger resistance to adversarial perturbations.", + "description": "Empirical robustness score; higher values indicate stronger resistance to adversarial perturbations.", "weight": 0.15 }, - "clipped_confidence_score": { + "confidence_score": { "inputs": [ { "source": "factsheet", - "field_path": "performance/clipped_test_confidence_score" + "field_path": "performance/test_confidence_score" } ], "operation": "get_value", "type": "true_score", - "description": "Confidence score clipped to the expected score range; higher values indicate more stable predictive confidence.", + "description": "Confidence score; higher values indicate more stable predictive confidence.", "weight": 0.1 }, "inverse_attack_success_rate": { @@ -645,7 +645,7 @@ }, { "source": "factsheet", - "field_path": "performance/clipped_test_clever" + "field_path": "performance/test_clever_score" }, { "source": "factsheet", @@ -653,15 +653,15 @@ }, { "source": "factsheet", - "field_path": "performance/clipped_test_adv_accuracy" + "field_path": "performance/test_adv_accuracy" }, { "source": "factsheet", - "field_path": "performance/clipped_test_empirical_robustness" + "field_path": "performance/test_empirical_robustness_score" }, { "source": "factsheet", - "field_path": "performance/clipped_test_confidence_score" + "field_path": "performance/test_confidence_score" }, { "source": "factsheet", diff --git a/nebula/addons/trustworthiness/configs/eval_metrics_dfl_images.json b/nebula/addons/trustworthiness/configs/eval_metrics_dfl_images.json index 80cb9486e..b43295c1d 100755 --- a/nebula/addons/trustworthiness/configs/eval_metrics_dfl_images.json +++ b/nebula/addons/trustworthiness/configs/eval_metrics_dfl_images.json @@ -7,7 +7,7 @@ "inputs": [ { "source": "factsheet", - "field_path": "performance/clipped_test_clever" + "field_path": "performance/test_clever_score" } ], "operation": "get_value", @@ -27,40 +27,40 @@ "description": "Inverse loss sensitivity score; higher values indicate lower sensitivity of the loss to input perturbations.", "weight": 0.2 }, - "clipped_adversarial_accuracy": { + "adversarial_accuracy": { "inputs": [ { "source": "factsheet", - "field_path": "performance/clipped_test_adv_accuracy" + "field_path": "performance/test_adv_accuracy" } ], "operation": "get_value", "type": "true_score", - "description": "Adversarial accuracy clipped to the expected score range; higher values indicate better predictive performance under adversarial perturbations.", + "description": "Adversarial accuracy; higher values indicate better predictive performance under adversarial perturbations.", "weight": 0.2 }, - "clipped_empirical_robustness": { + "empirical_robustness_score": { "inputs": [ { "source": "factsheet", - "field_path": "performance/clipped_test_empirical_robustness" + "field_path": "performance/test_empirical_robustness_score" } ], "operation": "get_value", "type": "true_score", - "description": "Empirical robustness clipped to the expected score range; higher values indicate stronger resistance to adversarial perturbations.", + "description": "Empirical robustness score; higher values indicate stronger resistance to adversarial perturbations.", "weight": 0.15 }, - "clipped_confidence_score": { + "confidence_score": { "inputs": [ { "source": "factsheet", - "field_path": "performance/clipped_test_confidence_score" + "field_path": "performance/test_confidence_score" } ], "operation": "get_value", "type": "true_score", - "description": "Confidence score clipped to the expected score range; higher values indicate more stable predictive confidence.", + "description": "Confidence score; higher values indicate more stable predictive confidence.", "weight": 0.1 }, "inverse_attack_success_rate": { @@ -645,7 +645,7 @@ }, { "source": "factsheet", - "field_path": "performance/clipped_test_clever" + "field_path": "performance/test_clever_score" }, { "source": "factsheet", @@ -653,15 +653,15 @@ }, { "source": "factsheet", - "field_path": "performance/clipped_test_adv_accuracy" + "field_path": "performance/test_adv_accuracy" }, { "source": "factsheet", - "field_path": "performance/clipped_test_empirical_robustness" + "field_path": "performance/test_empirical_robustness_score" }, { "source": "factsheet", - "field_path": "performance/clipped_test_confidence_score" + "field_path": "performance/test_confidence_score" }, { "source": "factsheet", diff --git a/nebula/addons/trustworthiness/configs/eval_metrics_dfl_tabular.json b/nebula/addons/trustworthiness/configs/eval_metrics_dfl_tabular.json index 80cb9486e..b7770033d 100755 --- a/nebula/addons/trustworthiness/configs/eval_metrics_dfl_tabular.json +++ b/nebula/addons/trustworthiness/configs/eval_metrics_dfl_tabular.json @@ -3,65 +3,29 @@ "resilience_to_attacks": { "weight": 0.4, "metrics": { - "certified_robustness": { + "adversarial_accuracy": { "inputs": [ { "source": "factsheet", - "field_path": "performance/clipped_test_clever" + "field_path": "performance/test_adv_accuracy" } ], "operation": "get_value", "type": "true_score", - "description": "Cross Lipschitz Extreme Value for network Robustness: attack-agnostic estimator of the lower bound βL", - "weight": 0.2 - }, - "inverse_loss_sensitivity": { - "inputs": [ - { - "source": "factsheet", - "field_path": "performance/inverse_test_loss_sensitivity" - } - ], - "operation": "get_value", - "type": "true_score", - "description": "Inverse loss sensitivity score; higher values indicate lower sensitivity of the loss to input perturbations.", - "weight": 0.2 + "description": "Adversarial accuracy; higher values indicate better predictive performance under adversarial perturbations.", + "weight": 0.4444444444 }, - "clipped_adversarial_accuracy": { + "confidence_score": { "inputs": [ { "source": "factsheet", - "field_path": "performance/clipped_test_adv_accuracy" + "field_path": "performance/test_confidence_score" } ], "operation": "get_value", "type": "true_score", - "description": "Adversarial accuracy clipped to the expected score range; higher values indicate better predictive performance under adversarial perturbations.", - "weight": 0.2 - }, - "clipped_empirical_robustness": { - "inputs": [ - { - "source": "factsheet", - "field_path": "performance/clipped_test_empirical_robustness" - } - ], - "operation": "get_value", - "type": "true_score", - "description": "Empirical robustness clipped to the expected score range; higher values indicate stronger resistance to adversarial perturbations.", - "weight": 0.15 - }, - "clipped_confidence_score": { - "inputs": [ - { - "source": "factsheet", - "field_path": "performance/clipped_test_confidence_score" - } - ], - "operation": "get_value", - "type": "true_score", - "description": "Confidence score clipped to the expected score range; higher values indicate more stable predictive confidence.", - "weight": 0.1 + "description": "Confidence score; higher values indicate more stable predictive confidence.", + "weight": 0.2222222222 }, "inverse_attack_success_rate": { "inputs": [ @@ -73,7 +37,7 @@ "operation": "get_value", "type": "true_score", "description": "Inverse attack success rate; higher values indicate a lower fraction of successful adversarial attacks.", - "weight": 0.15 + "weight": 0.3333333334 } } }, @@ -645,23 +609,11 @@ }, { "source": "factsheet", - "field_path": "performance/clipped_test_clever" - }, - { - "source": "factsheet", - "field_path": "performance/inverse_test_loss_sensitivity" - }, - { - "source": "factsheet", - "field_path": "performance/clipped_test_adv_accuracy" - }, - { - "source": "factsheet", - "field_path": "performance/clipped_test_empirical_robustness" + "field_path": "performance/test_adv_accuracy" }, { "source": "factsheet", - "field_path": "performance/clipped_test_confidence_score" + "field_path": "performance/test_confidence_score" }, { "source": "factsheet", diff --git a/nebula/addons/trustworthiness/configs/factsheet_template_cfl.json b/nebula/addons/trustworthiness/configs/factsheet_template_cfl.json index 0ba2db196..61a465fa0 100755 --- a/nebula/addons/trustworthiness/configs/factsheet_template_cfl.json +++ b/nebula/addons/trustworthiness/configs/factsheet_template_cfl.json @@ -47,11 +47,11 @@ "test_acc_avg": "", "test_macro_f1": "", "clipped_test_feature_importance_cv": "", - "clipped_test_clever": "", + "test_clever_score": "", "inverse_test_loss_sensitivity": "", - "clipped_test_adv_accuracy": "", - "clipped_test_empirical_robustness": "", - "clipped_test_confidence_score": "", + "test_adv_accuracy": "", + "test_empirical_robustness_score": "", + "test_confidence_score": "", "inverse_test_attack_success_rate": "" }, "fairness": { diff --git a/nebula/addons/trustworthiness/configs/factsheet_template_cfl_images.json b/nebula/addons/trustworthiness/configs/factsheet_template_cfl_images.json index 0ba2db196..61a465fa0 100755 --- a/nebula/addons/trustworthiness/configs/factsheet_template_cfl_images.json +++ b/nebula/addons/trustworthiness/configs/factsheet_template_cfl_images.json @@ -47,11 +47,11 @@ "test_acc_avg": "", "test_macro_f1": "", "clipped_test_feature_importance_cv": "", - "clipped_test_clever": "", + "test_clever_score": "", "inverse_test_loss_sensitivity": "", - "clipped_test_adv_accuracy": "", - "clipped_test_empirical_robustness": "", - "clipped_test_confidence_score": "", + "test_adv_accuracy": "", + "test_empirical_robustness_score": "", + "test_confidence_score": "", "inverse_test_attack_success_rate": "" }, "fairness": { diff --git a/nebula/addons/trustworthiness/configs/factsheet_template_cfl_tabular.json b/nebula/addons/trustworthiness/configs/factsheet_template_cfl_tabular.json index 0ba2db196..75b539f1b 100755 --- a/nebula/addons/trustworthiness/configs/factsheet_template_cfl_tabular.json +++ b/nebula/addons/trustworthiness/configs/factsheet_template_cfl_tabular.json @@ -47,11 +47,8 @@ "test_acc_avg": "", "test_macro_f1": "", "clipped_test_feature_importance_cv": "", - "clipped_test_clever": "", - "inverse_test_loss_sensitivity": "", - "clipped_test_adv_accuracy": "", - "clipped_test_empirical_robustness": "", - "clipped_test_confidence_score": "", + "test_adv_accuracy": "", + "test_confidence_score": "", "inverse_test_attack_success_rate": "" }, "fairness": { diff --git a/nebula/addons/trustworthiness/configs/factsheet_template_dfl.json b/nebula/addons/trustworthiness/configs/factsheet_template_dfl.json index 031be171e..c5ea5af60 100755 --- a/nebula/addons/trustworthiness/configs/factsheet_template_dfl.json +++ b/nebula/addons/trustworthiness/configs/factsheet_template_dfl.json @@ -48,11 +48,11 @@ "test_acc": "", "test_macro_f1": "", "clipped_test_feature_importance_cv": "", - "clipped_test_clever": "", + "test_clever_score": "", "inverse_test_loss_sensitivity": "", - "clipped_test_adv_accuracy": "", - "clipped_test_empirical_robustness": "", - "clipped_test_confidence_score": "", + "test_adv_accuracy": "", + "test_empirical_robustness_score": "", + "test_confidence_score": "", "inverse_test_attack_success_rate": "" }, "fairness": { diff --git a/nebula/addons/trustworthiness/configs/factsheet_template_dfl_images.json b/nebula/addons/trustworthiness/configs/factsheet_template_dfl_images.json index 031be171e..c5ea5af60 100755 --- a/nebula/addons/trustworthiness/configs/factsheet_template_dfl_images.json +++ b/nebula/addons/trustworthiness/configs/factsheet_template_dfl_images.json @@ -48,11 +48,11 @@ "test_acc": "", "test_macro_f1": "", "clipped_test_feature_importance_cv": "", - "clipped_test_clever": "", + "test_clever_score": "", "inverse_test_loss_sensitivity": "", - "clipped_test_adv_accuracy": "", - "clipped_test_empirical_robustness": "", - "clipped_test_confidence_score": "", + "test_adv_accuracy": "", + "test_empirical_robustness_score": "", + "test_confidence_score": "", "inverse_test_attack_success_rate": "" }, "fairness": { diff --git a/nebula/addons/trustworthiness/configs/factsheet_template_dfl_tabular.json b/nebula/addons/trustworthiness/configs/factsheet_template_dfl_tabular.json index 031be171e..5e2e841a8 100755 --- a/nebula/addons/trustworthiness/configs/factsheet_template_dfl_tabular.json +++ b/nebula/addons/trustworthiness/configs/factsheet_template_dfl_tabular.json @@ -48,11 +48,8 @@ "test_acc": "", "test_macro_f1": "", "clipped_test_feature_importance_cv": "", - "clipped_test_clever": "", - "inverse_test_loss_sensitivity": "", - "clipped_test_adv_accuracy": "", - "clipped_test_empirical_robustness": "", - "clipped_test_confidence_score": "", + "test_adv_accuracy": "", + "test_confidence_score": "", "inverse_test_attack_success_rate": "" }, "fairness": { diff --git a/nebula/addons/trustworthiness/dfl_factsheet.py b/nebula/addons/trustworthiness/dfl_factsheet.py index d8f4a6afd..8cb8b752c 100644 --- a/nebula/addons/trustworthiness/dfl_factsheet.py +++ b/nebula/addons/trustworthiness/dfl_factsheet.py @@ -62,6 +62,7 @@ def populate_factsheet_dfl( data["federation"], model, self.factsheet_template_file_nm, + dataset_name=data["dataset"], ) factsheet_file = get_factsheet_path(scenario_name, self.factsheet_file_nm) diff --git a/nebula/addons/trustworthiness/factsheet_common.py b/nebula/addons/trustworthiness/factsheet_common.py index 9c08c62c2..2b6051e4a 100644 --- a/nebula/addons/trustworthiness/factsheet_common.py +++ b/nebula/addons/trustworthiness/factsheet_common.py @@ -8,32 +8,55 @@ # Shared helpers for trustworthiness factsheet generation. DATA_TYPE_IMAGES = "images" DATA_TYPE_TABULAR = "tabular" +DATASET_DATA_TYPES = { + "mnist": DATA_TYPE_IMAGES, + "fashionmnist": DATA_TYPE_IMAGES, + "emnist": DATA_TYPE_IMAGES, + "cifar10": DATA_TYPE_IMAGES, + "cifar100": DATA_TYPE_IMAGES, + "kddcup99": DATA_TYPE_TABULAR, + "adultcensus": DATA_TYPE_TABULAR, + "breastcancer": DATA_TYPE_TABULAR, + "covtype": DATA_TYPE_TABULAR, + "sentiment140": DATA_TYPE_TABULAR, +} + + +def get_dataset_data_type(dataset_name): + # Infer the data type from Nebula's built-in dataset names. + if dataset_name is None: + return "" + + normalized_name = str(dataset_name).strip().lower().replace("_", "").replace("-", "") + return DATASET_DATA_TYPES.get(normalized_name, "") -def get_model_data_type(model): - # Return the data type declared by the model, when available. +def get_model_data_type(model, dataset_name=None): + # Return the model-declared data type, falling back to the dataset name. if not hasattr(model, "get_data_type"): - return "" + return get_dataset_data_type(dataset_name) try: data_type = model.get_data_type() except AttributeError: - return "" + return get_dataset_data_type(dataset_name) if data_type is None: - return "" - return str(data_type).strip() + return get_dataset_data_type(dataset_name) + + data_type = str(data_type).strip() + return data_type or get_dataset_data_type(dataset_name) -def get_normalized_model_data_type(model): +def get_normalized_model_data_type(model, dataset_name=None): # Normalize the model data type before matching templates or profiles. - return get_model_data_type(model).lower() + return get_model_data_type(model, dataset_name=dataset_name).lower() -def get_factsheet_template_name(federation, model, default_template_name): +def get_factsheet_template_name(federation, model, default_template_name, dataset_name=None): # Select a data-type-specific template when one exists for the federation. federation_prefix = "dfl" if str(federation).upper() in {"DFL", "SDFL"} else "cfl" - data_type = get_normalized_model_data_type(model) + data_type = get_normalized_model_data_type(model, dataset_name=dataset_name) if data_type not in {DATA_TYPE_IMAGES, DATA_TYPE_TABULAR}: return default_template_name @@ -90,6 +113,11 @@ def inverse_score(value): return 1 / (1 + value) +def inverse_bounded_score(value): + # Invert an error already bounded in [0, 1] while keeping the full score range. + return min(max(1 - float(value), 0.0), 1.0) + + def get_enabled_defences(data): # Return the active training-time defences declared in the scenario. defences = [] @@ -153,7 +181,7 @@ def populate_common_pre_train_sections(factsheet, data, model): factsheet["project"]["background"] = build_project_background(data) factsheet["data"]["provenance"] = data["dataset"] - factsheet["data"]["type"] = get_model_data_type(model) + factsheet["data"]["type"] = get_model_data_type(model, dataset_name=data["dataset"]) factsheet["data"]["preprocessing"] = data["topology"] factsheet["participants"]["client_num"] = data["n_nodes"] or "" diff --git a/nebula/addons/trustworthiness/factsheet_populators.py b/nebula/addons/trustworthiness/factsheet_populators.py index 25f5180bd..bee3b1e22 100644 --- a/nebula/addons/trustworthiness/factsheet_populators.py +++ b/nebula/addons/trustworthiness/factsheet_populators.py @@ -17,7 +17,7 @@ ) from nebula.addons.trustworthiness.helpers.robustness import ( attack_success_rate, - compute_adversarial_accuracy_art, + get_adversarial_accuracy, get_clever_score, get_confidence_score, get_empirical_robustness_score, @@ -30,6 +30,7 @@ DATA_TYPE_TABULAR, cap_score, get_normalized_model_data_type, + inverse_bounded_score, inverse_score, ) @@ -52,7 +53,9 @@ def populate_profile_metrics( ): # Select the profile-specific populator, falling back to the shared metric set. federation_profile = get_federation_profile(federation) - data_type = get_normalized_model_data_type(model) + data_type = str(factsheet.get("data", {}).get("type", "")).strip().lower() + if not data_type: + data_type = get_normalized_model_data_type(model) populator = PROFILE_POPULATORS.get((federation_profile, data_type), populate_common_profile_metrics) populator( @@ -65,23 +68,27 @@ def populate_profile_metrics( def populate_cfl_images_metrics(factsheet, model, train_loader, test_loader, test_accuracy): - # Populate the current shared metrics for CFL image factsheets. + # Image factsheets include all image-compatible robustness metrics. populate_common_profile_metrics(factsheet, model, train_loader, test_loader, test_accuracy) + populate_image_robustness_metrics(factsheet, model, test_loader) def populate_cfl_tabular_metrics(factsheet, model, train_loader, test_loader, test_accuracy): - # Populate the current shared metrics for CFL tabular factsheets. + # Tabular factsheets use only metrics shared by valid tabular and image workflows. populate_common_profile_metrics(factsheet, model, train_loader, test_loader, test_accuracy) + remove_image_only_robustness_metrics(factsheet) def populate_dfl_images_metrics(factsheet, model, train_loader, test_loader, test_accuracy): - # Populate the current shared metrics for DFL/SDFL image factsheets. + # Image factsheets include all image-compatible robustness metrics. populate_common_profile_metrics(factsheet, model, train_loader, test_loader, test_accuracy) + populate_image_robustness_metrics(factsheet, model, test_loader) def populate_dfl_tabular_metrics(factsheet, model, train_loader, test_loader, test_accuracy): - # Populate the current shared metrics for DFL/SDFL tabular factsheets. + # Tabular factsheets use only metrics shared by valid tabular and image workflows. populate_common_profile_metrics(factsheet, model, train_loader, test_loader, test_accuracy) + remove_image_only_robustness_metrics(factsheet) def populate_common_profile_metrics(factsheet, model, train_loader, test_loader, test_accuracy): @@ -99,7 +106,7 @@ def populate_common_profile_metrics(factsheet, model, train_loader, test_loader, test_sample, ) populate_common_explainability_metrics(factsheet, explainability_metrics) - populate_common_robustness_metrics(factsheet, model, test_loader, test_sample) + populate_common_robustness_metrics(factsheet, model, test_loader) def populate_common_model_quality_metrics( @@ -120,10 +127,10 @@ def populate_common_model_quality_metrics( # Fairness and calibration metrics expressed as inverse scores. overfitting_value = max(0.0, float(factsheet["performance"]["train_accuracy"]) - float(test_accuracy)) - factsheet["fairness"]["inverse_overfitting"] = inverse_score(overfitting_value) + factsheet["fairness"]["inverse_overfitting"] = inverse_bounded_score(overfitting_value) well_calibration_error_value = get_well_calibration_error(model, test_loader) - factsheet["fairness"]["inverse_well_calibration_error"] = inverse_score(well_calibration_error_value) + factsheet["fairness"]["inverse_well_calibration_error"] = inverse_bounded_score(well_calibration_error_value) generalized_entropy_index_value = get_generalized_entropy_index(model, test_loader) factsheet["fairness"]["inverse_generalized_entropy_index"] = inverse_score(generalized_entropy_index_value) @@ -134,9 +141,9 @@ def populate_common_model_quality_metrics( coefficient_of_variation_value = get_coefficient_of_variation(model, test_loader) factsheet["fairness"]["inverse_coefficient_of_variation"] = inverse_score(coefficient_of_variation_value) - # Confidence is capped so factsheet scores stay within the expected range. + # Confidence is already a probability-like score in [0, 1]. value_confidence_score = get_confidence_score(model, test_sample) - factsheet["performance"]["clipped_test_confidence_score"] = cap_score(value_confidence_score) + factsheet["performance"]["test_confidence_score"] = value_confidence_score def populate_common_explainability_metrics(factsheet, explainability_metrics): @@ -149,36 +156,53 @@ def populate_common_explainability_metrics(factsheet, explainability_metrics): factsheet["performance"]["clipped_test_feature_importance_cv"] = cap_score(feature_importance) -def populate_common_robustness_metrics(factsheet, model, test_loader, test_sample): - # Populate adversarial robustness metrics shared by the current factsheet profiles. +def populate_common_robustness_metrics(factsheet, model, test_loader): + # Populate robustness metrics valid for both image and tabular datasets. lr = factsheet["configuration"]["learning_rate"] num_classes = model.get_num_classes() - # Sample-based robustness scores. + # Loader-based adversarial accuracy. + value_adv_accuracy = get_adversarial_accuracy(model, test_loader, num_classes, lr) + factsheet["performance"]["test_adv_accuracy"] = value_adv_accuracy + + # Attack success is inverted so higher remains better in the factsheet. + value_attack_success_rate = attack_success_rate( + model, + test_loader, + ) + factsheet["performance"]["inverse_test_attack_success_rate"] = 1 - value_attack_success_rate + + +def populate_image_robustness_metrics(factsheet, model, test_loader): + # Populate image-only continuous-input robustness metrics. + lr = factsheet["configuration"]["learning_rate"] + num_classes = model.get_num_classes() + test_sample = next(iter(test_loader)) + value_clever = get_clever_score(model, test_sample, num_classes, lr) - factsheet["performance"]["clipped_test_clever"] = cap_score(value_clever) + factsheet["performance"]["test_clever_score"] = value_clever value_loss_sensitivity = get_loss_sensitivity_score(model, test_sample, num_classes, lr) factsheet["performance"]["inverse_test_loss_sensitivity"] = inverse_score(value_loss_sensitivity) - # Loader-based adversarial accuracy. - value_adv_accuracy = compute_adversarial_accuracy_art(model, test_loader, num_classes, lr) - factsheet["performance"]["clipped_test_adv_accuracy"] = cap_score(value_adv_accuracy) - value_empirical_robustness = get_empirical_robustness_score( model, test_sample, num_classes, lr, ) - factsheet["performance"]["clipped_test_empirical_robustness"] = cap_score(value_empirical_robustness) - - # Attack success is inverted so higher remains better in the factsheet. - value_attack_success_rate = attack_success_rate( - model, - test_sample, - ) - factsheet["performance"]["inverse_test_attack_success_rate"] = 1 - value_attack_success_rate + factsheet["performance"]["test_empirical_robustness_score"] = value_empirical_robustness + + +def remove_image_only_robustness_metrics(factsheet): + # Drop stale values when an existing factsheet was created before tabular metrics were split. + performance = factsheet.get("performance", {}) + for field in ( + "test_clever_score", + "inverse_test_loss_sensitivity", + "test_empirical_robustness_score", + ): + performance.pop(field, None) PROFILE_POPULATORS = { diff --git a/nebula/addons/trustworthiness/helpers/robustness.py b/nebula/addons/trustworthiness/helpers/robustness.py index 13611842b..a64fc5001 100644 --- a/nebula/addons/trustworthiness/helpers/robustness.py +++ b/nebula/addons/trustworthiness/helpers/robustness.py @@ -6,13 +6,23 @@ import torch.nn.functional as F from art.estimators.classification import PyTorchClassifier from art.metrics import clever_u, empirical_robustness, loss_sensitivity +from nebula.core.datasets.image_metadata import get_image_normalization from torch import nn, optim logger = logging.getLogger(__name__) R_L2 = 2 +ROBUSTNESS_EPSILON = 0.03 +# ART CLEVER is an L2 lower-bound estimate; the attack radius maps to a full trust score. +CLEVER_REFERENCE = R_L2 +# ART empirical robustness is a relative perturbation distance; this maps 0.2 to a full trust score. +EMPIRICAL_ROBUSTNESS_REFERENCE = 0.2 +TABULAR_ATTACK_STEPS = 3 +ADVERSARIAL_LOG_SAMPLES = 2 +ADVERSARIAL_LOG_FEATURES = 12 def _build_art_classifier(model, input_shape, nb_classes, learning_rate): + # Wrap the PyTorch model with the ART classifier interface used by ART metrics. criterion = nn.CrossEntropyLoss() optimizer = optim.Adam(model.parameters(), learning_rate) @@ -26,6 +36,7 @@ def _build_art_classifier(model, input_shape, nb_classes, learning_rate): def _validate_test_sample_tensors(test_sample): + # Shared guard for sample-based metrics that expect a non-empty (x, y) batch. if not (isinstance(test_sample, (tuple, list)) and len(test_sample) >= 2): raise ValueError("`test_sample` must contain samples and labels.") @@ -37,26 +48,207 @@ def _validate_test_sample_tensors(test_sample): def _coerce_max_samples(max_samples, default=8): + # Keep metric calls bounded even if configuration values are missing or invalid. try: return max(1, int(max_samples)) except Exception: return default +def _coerce_tabular_metadata(metadata): + # Accept both serialized dataset metadata and the typed metadata object. + if metadata is None: + return None + + # Keep tabular-only imports lazy so image workflows do not depend on them. + from nebula.core.datasets.tabular_metadata import TabularAdversarialMetadata + + if isinstance(metadata, TabularAdversarialMetadata): + return metadata + return TabularAdversarialMetadata.from_dict(metadata) + + +def _get_tabular_metadata_from_dataset(dataset): + # Dataloaders can wrap datasets; walk through wrappers until metadata is found. + if dataset is None: + return None + + metadata = getattr(dataset, "tabular_metadata", None) + if metadata is not None: + return _coerce_tabular_metadata(metadata) + + return _get_tabular_metadata_from_dataset(getattr(dataset, "dataset", None)) + + +def _get_tabular_metadata_from_loader(data_loader): + # Return None for image datasets, which keeps the adversarial path on FGSM. + return _get_tabular_metadata_from_dataset(getattr(data_loader, "dataset", None)) + + +def _get_dataset_name_from_dataset(dataset): + # Dataset wrappers keep the real dataset in `.dataset`; walk through them. + if dataset is None: + return None + + dataset_name = getattr(dataset, "dataset_name", None) + if dataset_name is not None: + return dataset_name + + config = getattr(dataset, "config", None) + participant = getattr(config, "participant", None) + if isinstance(config, dict): + participant = config.get("participant", participant) + if isinstance(participant, dict): + dataset_name = participant.get("data_args", {}).get("dataset") + if dataset_name is not None: + return dataset_name + + return _get_dataset_name_from_dataset(getattr(dataset, "dataset", None)) + + +def _get_image_normalization_from_loader(data_loader): + # Resolve image mean/std from shared dataset metadata instead of inferring by channels. + dataset_name = _get_dataset_name_from_dataset(getattr(data_loader, "dataset", None)) + normalization = get_image_normalization(dataset_name) + if normalization is not None: + logger.info("[Robustness] Image normalization loaded | dataset=%s | mean/std=%s", dataset_name, normalization) + return normalization + + +def _build_fixed_epsilon_tabular_generator(epsilon, tabular_metadata): + # Reuse the tabular adversarial-training generator, but make evaluation deterministic. + from nebula.addons.defenses.adversarial_training.config import AdversarialTrainingConfig + from nebula.addons.defenses.adversarial_training.tabular import TabularConstrainedPGDGenerator + + class FixedEpsilonTabularConstrainedPGDGenerator(TabularConstrainedPGDGenerator): + def _sample_epsilon(self, device): + # Training samples epsilon; factsheet metrics should use the requested epsilon exactly. + self.last_epsilon = float(self.config.epsilon) + return self.last_epsilon + + config = AdversarialTrainingConfig( + domain="tabular", + attack="constrained_pgd", + epsilon=float(epsilon), + steps=TABULAR_ATTACK_STEPS, + candidate_selection="none", + ) + return FixedEpsilonTabularConstrainedPGDGenerator(config, tabular_metadata) + + +def _build_tabular_generator(epsilon, tabular_metadata): + # A missing generator intentionally means "use the image/default FGSM path". + tabular_metadata = _coerce_tabular_metadata(tabular_metadata) + if tabular_metadata is None: + return None + + return _build_fixed_epsilon_tabular_generator(epsilon, tabular_metadata) + + +def _attack_name(tabular_generator): + # Keep log messages explicit about which adversarial path is active. + return "tabular_constrained_pgd" if tabular_generator is not None else "fgsm" + + +def _tensor_range(tensor): + # Compact numeric summary for batch-level logging. + if tensor.numel() == 0: + return "empty" + + tensor = tensor.detach().float().cpu() + return "min={:.6f}, max={:.6f}, mean={:.6f}".format( + tensor.min().item(), + tensor.max().item(), + tensor.mean().item(), + ) + + +def _format_preview_vector(vector, feature_names=None, max_features=ADVERSARIAL_LOG_FEATURES): + # Log only a small prefix of the flattened vector to keep factsheet logs readable. + values = vector.detach().flatten().float().cpu().tolist() + preview_values = values[:max_features] + + if feature_names: + names = list(feature_names)[:max_features] + items = [ + "{}={:.6f}".format(name, float(value)) + for name, value in zip(names, preview_values, strict=False) + ] + else: + items = ["{:.6f}".format(float(value)) for value in preview_values] + + suffix = ", ..." if len(values) > max_features else "" + return "[" + ", ".join(items) + suffix + "]" + + +def _log_adversarial_generation(metric_name, samples, labels, x_adv, epsilon, tabular_generator, batch_idx): + # Log one representative batch per metric invocation to inspect generated samples. + if batch_idx != 0: + return + + attack = _attack_name(tabular_generator) + clean = samples.detach().cpu() + adv = x_adv.detach().cpu() + delta = adv - clean + flat_delta = delta.reshape(delta.shape[0], -1).float() + feature_names = getattr(getattr(tabular_generator, "metadata", None), "feature_names", None) + + logger.info( + "[Robustness] %s adversarial generation | attack=%s | epsilon=%.6f | " + "clean_shape=%s | adv_shape=%s | clean=%s | adv=%s | " + "delta_linf=%.6f | delta_l2_mean=%.6f", + metric_name, + attack, + float(epsilon), + tuple(clean.shape), + tuple(adv.shape), + _tensor_range(clean), + _tensor_range(adv), + flat_delta.abs().max().item() if flat_delta.numel() else 0.0, + flat_delta.norm(p=2, dim=1).mean().item() if flat_delta.numel() else 0.0, + ) + + n_preview = min(int(clean.shape[0]), ADVERSARIAL_LOG_SAMPLES) + labels_cpu = labels.detach().cpu() if torch.is_tensor(labels) else labels + for sample_idx in range(n_preview): + label = labels_cpu[sample_idx].item() if torch.is_tensor(labels_cpu) else None + logger.info( + "[Robustness] %s adversarial sample %s | attack=%s | label=%s | " + "clean=%s | adversarial=%s | delta=%s", + metric_name, + sample_idx, + attack, + label, + _format_preview_vector(clean[sample_idx], feature_names), + _format_preview_vector(adv[sample_idx], feature_names), + _format_preview_vector(delta[sample_idx]), + ) + + +def _generate_adversarial_samples( + model, + samples, + labels, + epsilon=ROBUSTNESS_EPSILON, + tabular_generator=None, + image_normalization=None, +): + # Central switch: FGSM for images, constrained PGD for tabular datasets. + if tabular_generator is None: + return fgsm_attack( + model, + samples, + labels, + epsilon=epsilon, + image_normalization=image_normalization, + ) + + return tabular_generator.generate(model, samples, labels, nn.CrossEntropyLoss()) + + def get_clever_score(model, test_sample, nb_classes, learning_rate, max_samples=8): - """ - Calculates the CLEVER score as the mean score over multiple samples. - - Args: - model (object): The model. - test_sample (object): A batch from the test dataloader. - nb_classes (int): The nb_classes of the model. - learning_rate (float): The learning rate of the model. - max_samples (int): Maximum number of samples from the batch to evaluate. - - Returns: - float: Mean CLEVER score across the selected samples. - """ + # Calculates and scales ART CLEVER into a trust score. + samples, _ = _validate_test_sample_tensors(test_sample) input_shape = tuple(samples.shape[1:]) if samples.dim() >= 2 else tuple(samples.shape) @@ -69,6 +261,7 @@ def get_clever_score(model, test_sample, nb_classes, learning_rate, max_samples= clever_scores = [] for idx in range(n_samples): + # ART CLEVER evaluates one input at a time without the batch dimension. background = samples[idx].detach().cpu() sample_np = background.numpy() @@ -92,23 +285,19 @@ def get_clever_score(model, test_sample, nb_classes, learning_rate, max_samples= if not clever_scores: return 0.0 - return float(np.mean(clever_scores)) + raw_score = float(np.mean(clever_scores)) + score = min(max(raw_score / CLEVER_REFERENCE, 0.0), 1.0) + logger.info( + "[Robustness] CLEVER | raw_l2=%.6f | reference=%.6f | score=%.6f", + raw_score, + CLEVER_REFERENCE, + score, + ) + return score def get_loss_sensitivity_score(model, test_sample, nb_classes, learning_rate, max_samples=8): + # Calculates the loss sensitivity score as the mean score over multiple samples. - """ - Calculates the loss sensitivity score as the mean score over multiple samples. - - Args: - model (object): The model. - test_sample (object): A batch from the test dataloader. - nb_classes (int): The nb_classes of the model. - learning_rate (float): The learning rate of the model. - max_samples (int): Maximum number of samples from the batch to evaluate. - - Returns: - float: Mean loss sensitivity score across the selected samples. - """ samples, labels = _validate_test_sample_tensors(test_sample) max_samples = _coerce_max_samples(max_samples) @@ -119,6 +308,7 @@ def get_loss_sensitivity_score(model, test_sample, nb_classes, learning_rate, ma sensitivity_scores = [] for idx in range(n_samples): + # ART loss_sensitivity expects a batch and one-hot labels. sample = samples[idx].detach().cpu().unsqueeze(0) label = labels[idx].detach().cpu().unsqueeze(0) label = F.one_hot(label, num_classes=nb_classes).float() @@ -141,39 +331,54 @@ def get_loss_sensitivity_score(model, test_sample, nb_classes, learning_rate, ma return float(np.mean(sensitivity_scores)) -def compute_adversarial_accuracy_art( +def get_adversarial_accuracy( model, test_loader, nb_classes, learning_rate, - epsilon=0.03 + epsilon=ROBUSTNESS_EPSILON ): - """ - Computes adversarial accuracy using FGSM attack. - - Args: - model (object): The model. - test_loader (DataLoader): DataLoader providing test samples. - nb_classes (int): The nb_classes of the model. - learning_rate (float): The learning rate of the model. - epsilon (float): Maximum perturbation magnitude for the attacks. - - Returns: - float: The adversarial accuracy score. - """ + # Computes adversarial accuracy on generated adversarial samples. device = torch.device("cuda" if torch.cuda.is_available() else "cpu") model.eval() model.to(device) + # If metadata exists, adversarial examples preserve tabular feature constraints. + tabular_generator = _build_tabular_generator( + epsilon, + _get_tabular_metadata_from_loader(test_loader), + ) + image_normalization = None if tabular_generator is not None else _get_image_normalization_from_loader(test_loader) + logger.info( + "[Robustness] adversarial accuracy | attack=%s | epsilon=%.6f", + _attack_name(tabular_generator), + float(epsilon), + ) correct = 0 total = 0 - for samples, labels in test_loader: + for batch_idx, (samples, labels) in enumerate(test_loader): samples = samples.to(device) labels = labels.to(device) - x_adv = fgsm_attack(model, samples, labels, epsilon=epsilon) + x_adv = _generate_adversarial_samples( + model, + samples, + labels, + epsilon=epsilon, + tabular_generator=tabular_generator, + image_normalization=image_normalization, + ) + _log_adversarial_generation( + "adversarial_accuracy", + samples, + labels, + x_adv, + epsilon, + tabular_generator, + batch_idx, + ) with torch.no_grad(): outputs = model(x_adv) @@ -195,21 +400,8 @@ def get_empirical_robustness_score( attack_params = None, max_samples = 128, ): - """ - Calculates the Empirical Robustness score using Adversarial Robustness Toolbox (ART). - - Args: - model (object): The model. - test_sample (object): A batch from the test dataloader (samples, labels). - nb_classes (int): The nb_classes of the model. - learning_rate (float): The learning rate of the model. - attack_name (str): Attack key supported by ART empirical_robustness. - attack_params (dict | None): Optional attack parameters. - max_samples (int): Max number of samples from the batch to use. - - Returns: - float: Empirical robustness score (>= 0.0). If it cannot be computed, returns 0.0. - """ + # Calculates and scales ART empirical robustness into a trust score. + try: samples, _ = _validate_test_sample_tensors(test_sample) @@ -219,20 +411,27 @@ def get_empirical_robustness_score( classifier = _build_art_classifier(model, samples.shape[1:], nb_classes, learning_rate) - score = empirical_robustness( + raw_score = empirical_robustness( classifier=classifier, x=x, attack_name=attack_name, attack_params=attack_params, ) - if isinstance(score, np.ndarray): - score = float(np.mean(score)) + if isinstance(raw_score, np.ndarray): + raw_score = float(np.mean(raw_score)) - if score is None or (isinstance(score, float) and math.isnan(score)): + if raw_score is None or (isinstance(raw_score, float) and math.isnan(raw_score)): return 0.0 - return float(score) + score = min(max(float(raw_score) / EMPIRICAL_ROBUSTNESS_REFERENCE, 0.0), 1.0) + logger.info( + "[Robustness] empirical robustness | raw_distance=%.6f | reference=%.6f | score=%.6f", + float(raw_score), + EMPIRICAL_ROBUSTNESS_REFERENCE, + score, + ) + return score except Exception as exc: logger.warning("Could not compute empirical robustness (ART). Returning 0.0") @@ -240,25 +439,27 @@ def get_empirical_robustness_score( return 0.0 -def _get_image_normalization_for_samples(samples): - if not isinstance(samples, torch.Tensor) or samples.ndim < 4: - return None +def _get_image_normalization_for_samples(samples, image_normalization=None): + # Image normalization must come from dataset metadata; do not infer it by channel count. + if image_normalization is not None: + return image_normalization - channels = int(samples.shape[1]) - if channels == 1: - return (0.5,), (0.5,) - if channels == 3: - return (0.4914, 0.4822, 0.4465), (0.2471, 0.2435, 0.2616) + if isinstance(samples, torch.Tensor) and samples.ndim >= 4: + logger.warning( + "[Robustness] Image normalization missing; FGSM will perturb without normalized-space clamping." + ) return None def _channel_tensor(values, samples): + # Broadcast channel statistics over the batch and spatial dimensions. shape = [1, len(values)] + [1] * max(samples.dim() - 2, 0) return torch.tensor(values, dtype=samples.dtype, device=samples.device).view(*shape) -def _fgsm_step_and_clamp(samples, grad, epsilon): - normalization = _get_image_normalization_for_samples(samples) +def _fgsm_step_and_clamp(samples, grad, epsilon, image_normalization=None): + # Clamp image attacks in normalized space; leave non-image tensors unclamped here. + normalization = _get_image_normalization_for_samples(samples, image_normalization=image_normalization) if normalization is None: return samples + epsilon * grad.sign() @@ -275,19 +476,9 @@ def _fgsm_step_and_clamp(samples, grad, epsilon): return torch.max(torch.min(x_adv, upper), lower) -def fgsm_attack(model, samples, labels, epsilon=0.03): - """ - Performs an FGSM (Fast Gradient Sign Method) adversarial attack on a batch of samples. +def fgsm_attack(model, samples, labels, epsilon=ROBUSTNESS_EPSILON, image_normalization=None): + # Performs an FGSM (Fast Gradient Sign Method) adversarial attack on a batch of samples. - Args: - model (torch.nn.Module): The PyTorch model to attack. - samples (torch.Tensor): Input samples to perturb, shape (B, ...). - labels (torch.Tensor): True labels corresponding to the samples. - epsilon (float, optional): Maximum perturbation magnitude for the attack. Defaults to 0.03. - - Returns: - torch.Tensor: Adversarially perturbed samples with the same shape as `samples`. - """ try: device = next(model.parameters()).device except Exception: @@ -295,13 +486,21 @@ def fgsm_attack(model, samples, labels, epsilon=0.03): samples = samples.clone().detach().to(device) labels = labels.to(device) + # Gradients are needed only with respect to the input batch. samples.requires_grad = True outputs = model(samples) logits = outputs[0] if isinstance(outputs, (tuple, list)) else outputs loss = nn.CrossEntropyLoss()(logits, labels) grad = torch.autograd.grad(loss, samples, only_inputs=True)[0] - x_adv = _fgsm_step_and_clamp(samples, grad, epsilon) + x_adv = _fgsm_step_and_clamp(samples, grad, epsilon, image_normalization=image_normalization) + logger.debug( + "[Robustness] FGSM batch generated | epsilon=%.6f | samples_shape=%s | grad=%s | adv=%s", + float(epsilon), + tuple(samples.shape), + _tensor_range(grad), + _tensor_range(x_adv), + ) return x_adv.detach() @@ -312,18 +511,8 @@ def get_confidence_score( max_samples = 128, use_true_label = True, ): - """ - Calculates the confidence score. - - Args: - model (object): The model. - test_sample (object): A batch from the test dataloader (samples, labels). - max_samples (int): Max number of samples from the batch to use. - use_true_label (bool): Whether to compute confidence with respect to the true labels. Defaults to True. - - Returns: - float: Confidence score. - """ + # Calculates the confidence score. + try: if not isinstance(model, torch.nn.Module): logger.warning("Model is not a torch.nn.Module") @@ -350,6 +539,7 @@ def get_confidence_score( probs = torch.softmax(logits, dim=1) if use_true_label and isinstance(y, torch.Tensor): + # True-label confidence is used when labels are available. if y.ndim > 1: y_idx = torch.argmax(y, dim=1) else: @@ -368,46 +558,65 @@ def get_confidence_score( return 0.0 -def attack_success_rate(model, test_sample,epsilon=0.03): - """ - Calculates the ASR. - - Args: - model (object): The model. - test_sample (object): A batch from the test dataloader (samples, labels). - epsilon (float): Maximum perturbation magnitude for the attacks. - - Returns: - float: The ASR. - """ +def attack_success_rate(model, test_loader, epsilon=ROBUSTNESS_EPSILON): + # Computes ASR over originally correct predictions only. device = torch.device("cuda" if torch.cuda.is_available() else "cpu") model.eval() model.to(device) + # Tabular datasets use constrained PGD; image datasets fall back to FGSM. + tabular_generator = _build_tabular_generator( + epsilon, + _get_tabular_metadata_from_loader(test_loader), + ) + image_normalization = None if tabular_generator is not None else _get_image_normalization_from_loader(test_loader) + logger.info( + "[Robustness] attack success rate | attack=%s | epsilon=%.6f", + _attack_name(tabular_generator), + float(epsilon), + ) - images, labels = test_sample - images = images.to(device) - labels = labels.to(device) - - with torch.no_grad(): - outputs = model(images) - logits = outputs[0] if isinstance(outputs, (tuple, list)) else outputs - preds = logits.argmax(dim=1) - - correct_mask = preds.eq(labels) - num_correct = correct_mask.sum().item() + successful_attacks = 0 + num_correct = 0 - if num_correct == 0: - return 0.0 + for batch_idx, (samples, labels) in enumerate(test_loader): + samples = samples.to(device) + labels = labels.to(device) - x_adv = fgsm_attack(model, images, labels, epsilon=epsilon) + with torch.no_grad(): + outputs = model(samples) + logits = outputs[0] if isinstance(outputs, (tuple, list)) else outputs + preds = logits.argmax(dim=1) - with torch.no_grad(): - outputs_adv = model(x_adv) - logits_adv = outputs_adv[0] if isinstance(outputs_adv, (tuple, list)) else outputs_adv - preds_adv = logits_adv.argmax(dim=1) + correct_mask = preds.eq(labels) + batch_correct = correct_mask.sum().item() + if batch_correct == 0: + # ASR is defined over clean-correct samples, so this batch contributes nothing. + continue + + x_adv = _generate_adversarial_samples( + model, + samples, + labels, + epsilon=epsilon, + tabular_generator=tabular_generator, + image_normalization=image_normalization, + ) + _log_adversarial_generation( + "attack_success_rate", + samples, + labels, + x_adv, + epsilon, + tabular_generator, + batch_idx, + ) - successful_attacks = (correct_mask & preds_adv.ne(labels)).sum().item() + with torch.no_grad(): + outputs_adv = model(x_adv) + logits_adv = outputs_adv[0] if isinstance(outputs_adv, (tuple, list)) else outputs_adv + preds_adv = logits_adv.argmax(dim=1) - asr = successful_attacks / num_correct + successful_attacks += (correct_mask & preds_adv.ne(labels)).sum().item() + num_correct += batch_correct - return asr + return successful_attacks / num_correct if num_correct > 0 else 0.0 diff --git a/nebula/core/datasets/image_metadata.py b/nebula/core/datasets/image_metadata.py new file mode 100644 index 000000000..0b206fbf8 --- /dev/null +++ b/nebula/core/datasets/image_metadata.py @@ -0,0 +1,14 @@ +IMAGE_DATASET_NORMALIZATION = { + "MNIST": ((0.5,), (0.5,)), + "FashionMNIST": ((0.5,), (0.5,)), + "EMNIST": ((0.5,), (0.5,)), + "CIFAR10": ((0.4914, 0.4822, 0.4465), (0.2471, 0.2435, 0.2616)), + "CIFAR100": ((0.4914, 0.4822, 0.4465), (0.2471, 0.2435, 0.2616)), +} + + +def get_image_normalization(dataset_name): + # Shared source of image mean/std values used by attacks in normalized model space. + if dataset_name is None: + return None + return IMAGE_DATASET_NORMALIZATION.get(str(dataset_name)) diff --git a/nebula/core/datasets/kddcup99/kddcup99.py b/nebula/core/datasets/kddcup99/kddcup99.py index 72c3db0f4..db6ae8488 100644 --- a/nebula/core/datasets/kddcup99/kddcup99.py +++ b/nebula/core/datasets/kddcup99/kddcup99.py @@ -233,8 +233,8 @@ def __init__( seed: int = 42, config_dir: str | None = None, test_size: float = 0.2, - train_limit: int | None = 12000, - test_limit: int | None = 2000, + train_limit: int | None = 20000, + test_limit: int | None = 4000, subset: str | None = None, percent10: bool = True, ): diff --git a/nebula/core/models/adultcensus/mlp.py b/nebula/core/models/adultcensus/mlp.py index d3fff9e94..3a7c2595c 100644 --- a/nebula/core/models/adultcensus/mlp.py +++ b/nebula/core/models/adultcensus/mlp.py @@ -26,6 +26,7 @@ def __init__( ): # NebulaModel expects something like input_channels first; for tabular we pass input_dim there. super().__init__(input_dim, num_classes, learning_rate, metrics, confusion_matrix, seed) + self.data_type = data_type self.config = {"beta1": 0.9, "beta2": 0.999, "amsgrad": True} diff --git a/nebula/core/models/breast_cancer/mlp.py b/nebula/core/models/breast_cancer/mlp.py index 11a6ec833..577d16290 100644 --- a/nebula/core/models/breast_cancer/mlp.py +++ b/nebula/core/models/breast_cancer/mlp.py @@ -20,6 +20,7 @@ def __init__( # pero en la práctica se usa ese primer argumento como "input shape info". # Para tabular, pasamos input_dim en input_channels para mantener la firma. super().__init__(input_dim, num_classes, learning_rate, metrics, confusion_matrix, seed) + self.data_type = data_type # Mantengo el mismo patrón que tu MLP de FashionMNIST. self.config = {"beta1": 0.9, "beta2": 0.999, "amsgrad": True} diff --git a/nebula/core/models/cifar10/cnn.py b/nebula/core/models/cifar10/cnn.py index db9df51e6..cdd70ddcf 100755 --- a/nebula/core/models/cifar10/cnn.py +++ b/nebula/core/models/cifar10/cnn.py @@ -15,6 +15,7 @@ def __init__( data_type="Images", ): super().__init__(input_channels, num_classes, learning_rate, metrics, confusion_matrix, seed) + self.data_type = data_type self.config = {"beta1": 0.851436, "beta2": 0.999689, "amsgrad": True} diff --git a/nebula/core/models/cifar10/cnnV2.py b/nebula/core/models/cifar10/cnnV2.py index 0b23eef34..f5bcb5c6f 100755 --- a/nebula/core/models/cifar10/cnnV2.py +++ b/nebula/core/models/cifar10/cnnV2.py @@ -15,6 +15,7 @@ def __init__( data_type="Images", ): super().__init__(input_channels, num_classes, learning_rate, metrics, confusion_matrix, seed) + self.data_type = data_type self.config = {"beta1": 0.851436, "beta2": 0.999689, "amsgrad": True} diff --git a/nebula/core/models/cifar10/cnnV3.py b/nebula/core/models/cifar10/cnnV3.py index 8b4585208..2aff83dd0 100755 --- a/nebula/core/models/cifar10/cnnV3.py +++ b/nebula/core/models/cifar10/cnnV3.py @@ -15,6 +15,7 @@ def __init__( data_type="Images", ): super().__init__(input_channels, num_classes, learning_rate, metrics, confusion_matrix, seed) + self.data_type = data_type self.config = {"beta1": 0.851436, "beta2": 0.999689, "amsgrad": True} diff --git a/nebula/core/models/cifar10/fastermobilenet.py b/nebula/core/models/cifar10/fastermobilenet.py index 7be70c64d..20ec7704a 100755 --- a/nebula/core/models/cifar10/fastermobilenet.py +++ b/nebula/core/models/cifar10/fastermobilenet.py @@ -16,6 +16,7 @@ def __init__( data_type="Images", ): super().__init__(input_channels, num_classes, learning_rate, metrics, confusion_matrix, seed) + self.data_type = data_type self.config = {"beta1": 0.851436, "beta2": 0.999689, "amsgrad": True} diff --git a/nebula/core/models/cifar10/resnet.py b/nebula/core/models/cifar10/resnet.py index d2da13f3b..255191511 100755 --- a/nebula/core/models/cifar10/resnet.py +++ b/nebula/core/models/cifar10/resnet.py @@ -42,6 +42,7 @@ def __init__( data_type="Images", ): super().__init__() + self.data_type = data_type if metrics is None: metrics = MetricCollection([ MulticlassAccuracy(num_classes=num_classes), diff --git a/nebula/core/models/cifar10/simplemobilenet.py b/nebula/core/models/cifar10/simplemobilenet.py index 9791f5735..b394a101d 100755 --- a/nebula/core/models/cifar10/simplemobilenet.py +++ b/nebula/core/models/cifar10/simplemobilenet.py @@ -21,6 +21,7 @@ def __init__( data_type="Images", ): super().__init__(input_channels, num_classes, learning_rate, metrics, confusion_matrix, seed) + self.data_type = data_type self.config = {"beta1": 0.851436, "beta2": 0.999689, "amsgrad": True} diff --git a/nebula/core/models/cifar100/cnn.py b/nebula/core/models/cifar100/cnn.py index 0a005973f..6c2de6b41 100755 --- a/nebula/core/models/cifar100/cnn.py +++ b/nebula/core/models/cifar100/cnn.py @@ -15,6 +15,7 @@ def __init__( data_type="Images", ): super().__init__(input_channels, num_classes, learning_rate, metrics, confusion_matrix, seed) + self.data_type = data_type self.config = { "lr": 8.0505e-05, diff --git a/nebula/core/models/covtype/mlp.py b/nebula/core/models/covtype/mlp.py index 0f684dd06..6eb55a348 100644 --- a/nebula/core/models/covtype/mlp.py +++ b/nebula/core/models/covtype/mlp.py @@ -20,6 +20,7 @@ def __init__( # pero en la práctica se usa ese primer argumento como "input shape info". # Para tabular, pasamos input_dim en input_channels para mantener la firma. super().__init__(input_dim, num_classes, learning_rate, metrics, confusion_matrix, seed) + self.data_type = data_type # Mantengo el mismo patrón que tu MLP de FashionMNIST. self.config = {"beta1": 0.9, "beta2": 0.999, "amsgrad": True} diff --git a/nebula/core/models/emnist/cnn.py b/nebula/core/models/emnist/cnn.py index 17c7f6040..22bd80a2e 100755 --- a/nebula/core/models/emnist/cnn.py +++ b/nebula/core/models/emnist/cnn.py @@ -15,6 +15,7 @@ def __init__( data_type="Images", ): super().__init__(input_channels, num_classes, learning_rate, metrics, confusion_matrix, seed) + self.data_type = data_type self.config = {"beta1": 0.851436, "beta2": 0.999689, "amsgrad": True} diff --git a/nebula/core/models/emnist/mlp.py b/nebula/core/models/emnist/mlp.py index 6d5e420e6..4887165fc 100755 --- a/nebula/core/models/emnist/mlp.py +++ b/nebula/core/models/emnist/mlp.py @@ -15,6 +15,7 @@ def __init__( data_type="Images", ): super().__init__(input_channels, num_classes, learning_rate, metrics, confusion_matrix, seed) + self.data_type = data_type self.config = {"beta1": 0.851436, "beta2": 0.999689, "amsgrad": True} diff --git a/nebula/core/models/fashionmnist/cnn.py b/nebula/core/models/fashionmnist/cnn.py index 72837d204..bef3d1eca 100755 --- a/nebula/core/models/fashionmnist/cnn.py +++ b/nebula/core/models/fashionmnist/cnn.py @@ -15,6 +15,7 @@ def __init__( data_type="Images", ): super().__init__(input_channels, num_classes, learning_rate, metrics, confusion_matrix, seed) + self.data_type = data_type self.config = {"beta1": 0.851436, "beta2": 0.999689, "amsgrad": True} diff --git a/nebula/core/models/fashionmnist/mlp.py b/nebula/core/models/fashionmnist/mlp.py index 4704674e0..ac289c7d5 100755 --- a/nebula/core/models/fashionmnist/mlp.py +++ b/nebula/core/models/fashionmnist/mlp.py @@ -15,6 +15,7 @@ def __init__( data_type="Images", ): super().__init__(input_channels, num_classes, learning_rate, metrics, confusion_matrix, seed) + self.data_type = data_type self.config = {"beta1": 0.851436, "beta2": 0.999689, "amsgrad": True} diff --git a/nebula/core/models/kddcup99/mlp.py b/nebula/core/models/kddcup99/mlp.py index 539f3ad98..2de38af46 100644 --- a/nebula/core/models/kddcup99/mlp.py +++ b/nebula/core/models/kddcup99/mlp.py @@ -16,6 +16,7 @@ def __init__( data_type="Tabular", ): super().__init__(input_channels, num_classes, learning_rate, metrics, confusion_matrix, seed) + self.data_type = data_type self.input_size = input_size self.example_input_array = torch.zeros(1, self.input_size) diff --git a/nebula/core/models/mnist/cnn.py b/nebula/core/models/mnist/cnn.py index 78e520b4e..94bdcbdc5 100755 --- a/nebula/core/models/mnist/cnn.py +++ b/nebula/core/models/mnist/cnn.py @@ -15,6 +15,7 @@ def __init__( data_type="Images", ): super().__init__(input_channels, num_classes, learning_rate, metrics, confusion_matrix, seed) + self.data_type = data_type self.config = {"beta1": 0.851436, "beta2": 0.999689, "amsgrad": True} diff --git a/nebula/core/models/mnist/mlp.py b/nebula/core/models/mnist/mlp.py index 9fdc48bb7..f316dc110 100755 --- a/nebula/core/models/mnist/mlp.py +++ b/nebula/core/models/mnist/mlp.py @@ -15,6 +15,7 @@ def __init__( data_type="Images", ): super().__init__(input_channels, num_classes, learning_rate, metrics, confusion_matrix, seed) + self.data_type = data_type self.example_input_array = torch.zeros(1, 1, 28, 28) self.learning_rate = learning_rate diff --git a/nebula/core/models/sentiment140/cnn.py b/nebula/core/models/sentiment140/cnn.py index ab305fda7..f5c2d9d46 100755 --- a/nebula/core/models/sentiment140/cnn.py +++ b/nebula/core/models/sentiment140/cnn.py @@ -17,6 +17,7 @@ def __init__( data_type="Tabular", ): super().__init__(input_channels, num_classes, learning_rate, metrics, confusion_matrix, seed) + self.data_type = data_type self.config = {"beta1": 0.851436, "beta2": 0.999689, "amsgrad": True} self.example_input_array = torch.zeros(1, 1, 28, 28) diff --git a/nebula/core/models/sentiment140/rnn.py b/nebula/core/models/sentiment140/rnn.py index aa1915d53..d02b1e76e 100755 --- a/nebula/core/models/sentiment140/rnn.py +++ b/nebula/core/models/sentiment140/rnn.py @@ -15,6 +15,7 @@ def __init__( data_type="Tabular", ): super().__init__(input_channels, num_classes, learning_rate, metrics, confusion_matrix, seed) + self.data_type = data_type self.config = {"beta1": 0.851436, "beta2": 0.999689, "amsgrad": True} From 8e0a7a1657d1728339a7397801606aee4a8e56c3 Mon Sep 17 00:00:00 2001 From: "Juan J." Date: Thu, 11 Jun 2026 13:08:01 +0200 Subject: [PATCH 64/66] Citations --- README.md | 29 +++++++++++++++++++ .../trustworthiness/helpers/model_quality.py | 5 ++++ .../core/datasets/adultcensus/adultcensus.py | 2 ++ .../datasets/breast_cancer/breast_cancer.py | 3 ++ nebula/core/datasets/covtype/covtype.py | 2 ++ nebula/core/datasets/kddcup99/kddcup99.py | 3 ++ nebula/core/training/dp.py | 3 ++ 7 files changed, 47 insertions(+) diff --git a/README.md b/README.md index 5dfc69a53..725d78280 100755 --- a/README.md +++ b/README.md @@ -159,4 +159,33 @@ We would like to thank the following projects for their contributions which have - [FastAPI](https://github.com/tiangolo/fastapi) for the RESTful API - [Fedstellar](https://github.com/CyberDataLab/fedstellar) platform and [p2pfl](https://github.com/pguijas/p2pfl/) library - [Adversarial Robustness Toolbox (ART)](https://github.com/Trusted-AI/adversarial-robustness-toolbox) for the implementation of adversarial attacks +- [Opacus](https://github.com/meta-pytorch/opacus) for differential privacy training support +- [AI Fairness 360 (AIF360)](https://github.com/Trusted-AI/AIF360) for fairness metric definitions +- [HolisticAI](https://github.com/holistic-ai/holisticai) for trustworthiness and fairness metric definitions - [D3.js](https://github.com/d3/d3-force) for the network visualizations + +## Third-party Differential Privacy + +NEBULA uses Opacus for differential privacy training: + +- Yousefpour, A., Shilov, I., Sablayrolles, A., Testuggine, D., Prasad, K., Malek, M., Nguyen, J., Ghosh, S., Bharadwaj, A., Zhao, J., Cormode, G., & Mironov, I. (2021). Opacus: User-Friendly Differential Privacy Library in PyTorch. arXiv:2109.12298. Licensed under Apache License 2.0: https://github.com/meta-pytorch/opacus/blob/main/LICENSE + +## Third-party Trustworthiness Metrics + +NEBULA implements some trustworthiness and fairness metrics following definitions documented in external toolkits: + +- AI Fairness 360 (AIF360). AI Fairness 360 [Software]. https://github.com/Trusted-AI/AIF360. Licensed under Apache License 2.0: https://github.com/Trusted-AI/AIF360/blob/main/LICENSE + +- Holistic AI. HolisticAI [Software]. https://github.com/holistic-ai/holisticai. Licensed under Apache License 2.0: https://github.com/holistic-ai/holisticai/blob/main/LICENSE + +## Third-party Tabular Datasets + +NEBULA preprocesses these datasets for experiments, including splitting, scaling, encoding, label mapping, filtering, and/or sample limiting depending on the dataset. + +- Becker, B. & Kohavi, R. (1996). Adult [Dataset]. UCI Machine Learning Repository. https://doi.org/10.24432/C5XW20. Licensed under CC BY 4.0: https://creativecommons.org/licenses/by/4.0/ + +- Blackard, J. (1998). Covertype [Dataset]. UCI Machine Learning Repository. https://doi.org/10.24432/C50K5N. Licensed under CC BY 4.0: https://creativecommons.org/licenses/by/4.0/ + +- Wolberg, W., Mangasarian, O., Street, N., & Street, W. (1993). Breast Cancer Wisconsin (Diagnostic) [Dataset]. UCI Machine Learning Repository. https://doi.org/10.24432/C5DW2B. Licensed under CC BY 4.0: https://creativecommons.org/licenses/by/4.0/ + +- Stolfo, S., Fan, W., Lee, W., Prodromidis, A., & Chan, P. (1999). KDD Cup 1999 Data [Dataset]. UCI Machine Learning Repository. https://doi.org/10.24432/C51C7N. Licensed under CC BY 4.0: https://creativecommons.org/licenses/by/4.0/ diff --git a/nebula/addons/trustworthiness/helpers/model_quality.py b/nebula/addons/trustworthiness/helpers/model_quality.py index 4887f5719..979583607 100644 --- a/nebula/addons/trustworthiness/helpers/model_quality.py +++ b/nebula/addons/trustworthiness/helpers/model_quality.py @@ -4,6 +4,11 @@ import numpy as np import torch +# AIF360: AI Fairness 360 [Software]. https://github.com/Trusted-AI/AIF360 +# Licensed under Apache License 2.0: https://github.com/Trusted-AI/AIF360/blob/main/LICENSE +# HolisticAI: open-source library to assess and improve AI trustworthiness. +# Licensed under Apache License 2.0: https://github.com/holistic-ai/holisticai/blob/main/LICENSE + logger = logging.getLogger(__name__) def _extract_model_logits(model_output): diff --git a/nebula/core/datasets/adultcensus/adultcensus.py b/nebula/core/datasets/adultcensus/adultcensus.py index 062ffc380..6618ccad9 100644 --- a/nebula/core/datasets/adultcensus/adultcensus.py +++ b/nebula/core/datasets/adultcensus/adultcensus.py @@ -1,4 +1,6 @@ # nebula/core/datasets/adultcensus/adultcensus.py +# Becker, B. & Kohavi, R. (1996). Adult [Dataset]. UCI Machine Learning Repository. https://doi.org/10.24432/C5XW20. +# Licensed under CC BY 4.0: https://creativecommons.org/licenses/by/4.0/ import logging import os diff --git a/nebula/core/datasets/breast_cancer/breast_cancer.py b/nebula/core/datasets/breast_cancer/breast_cancer.py index f5e53ed7e..04fbcf9ae 100644 --- a/nebula/core/datasets/breast_cancer/breast_cancer.py +++ b/nebula/core/datasets/breast_cancer/breast_cancer.py @@ -1,3 +1,6 @@ +# Wolberg, W., Mangasarian, O., Street, N., & Street, W. (1993). Breast Cancer Wisconsin (Diagnostic) [Dataset]. UCI Machine Learning Repository. https://doi.org/10.24432/C5DW2B. +# Licensed under CC BY 4.0: https://creativecommons.org/licenses/by/4.0/ + import logging import os from typing import Any diff --git a/nebula/core/datasets/covtype/covtype.py b/nebula/core/datasets/covtype/covtype.py index ec3ef65d9..2ef0a360c 100644 --- a/nebula/core/datasets/covtype/covtype.py +++ b/nebula/core/datasets/covtype/covtype.py @@ -1,4 +1,6 @@ # nebula/core/datasets/covtype/covtype.py +# Blackard, J. (1998). Covertype [Dataset]. UCI Machine Learning Repository. https://doi.org/10.24432/C50K5N. +# Licensed under CC BY 4.0: https://creativecommons.org/licenses/by/4.0/ import logging import os diff --git a/nebula/core/datasets/kddcup99/kddcup99.py b/nebula/core/datasets/kddcup99/kddcup99.py index db6ae8488..494265bbe 100644 --- a/nebula/core/datasets/kddcup99/kddcup99.py +++ b/nebula/core/datasets/kddcup99/kddcup99.py @@ -1,3 +1,6 @@ +# Stolfo, S., Fan, W., Lee, W., Prodromidis, A., & Chan, P. (1999). KDD Cup 1999 Data [Dataset]. UCI Machine Learning Repository. https://doi.org/10.24432/C51C7N. +# Licensed under CC BY 4.0: https://creativecommons.org/licenses/by/4.0/ + import logging import os from typing import Any diff --git a/nebula/core/training/dp.py b/nebula/core/training/dp.py index 15b094c88..446511409 100644 --- a/nebula/core/training/dp.py +++ b/nebula/core/training/dp.py @@ -1,3 +1,6 @@ +# Opacus: User-Friendly Differential Privacy Library in PyTorch. Yousefpour et al. (2021). arXiv:2109.12298. +# Licensed under Apache License 2.0: https://github.com/meta-pytorch/opacus/blob/main/LICENSE + class SimpleDPState: # Minimal mutable state used to pass Opacus-wrapped objects between hooks. def __init__(self): From 51e6bc3833185f86e4c68b6e7f81dce930ad122a Mon Sep 17 00:00:00 2001 From: "Juan J." Date: Thu, 11 Jun 2026 16:26:42 +0200 Subject: [PATCH 65/66] DP error fixed --- nebula/core/models/breast_cancer/mlp.py | 6 ------ nebula/core/models/covtype/mlp.py | 6 ------ nebula/core/models/nebulamodel.py | 1 - nebula/core/training/lightning_dp.py | 9 +++++++-- 4 files changed, 7 insertions(+), 15 deletions(-) diff --git a/nebula/core/models/breast_cancer/mlp.py b/nebula/core/models/breast_cancer/mlp.py index 577d16290..27c6a51ba 100644 --- a/nebula/core/models/breast_cancer/mlp.py +++ b/nebula/core/models/breast_cancer/mlp.py @@ -16,13 +16,9 @@ def __init__( seed=None, data_type="Tabular", ): - # OJO: NebulaModel está pensado para imágenes (input_channels), - # pero en la práctica se usa ese primer argumento como "input shape info". - # Para tabular, pasamos input_dim en input_channels para mantener la firma. super().__init__(input_dim, num_classes, learning_rate, metrics, confusion_matrix, seed) self.data_type = data_type - # Mantengo el mismo patrón que tu MLP de FashionMNIST. self.config = {"beta1": 0.9, "beta2": 0.999, "amsgrad": True} self.example_input_array = torch.rand(1, input_dim) @@ -34,8 +30,6 @@ def __init__( self.l3 = torch.nn.Linear(128, num_classes) def forward(self, x): - # En tabular, x debe ser (batch, input_dim). - # A veces puede venir con dimensión extra (batch, 1, input_dim) por loaders. if x.dim() == 3 and x.size(1) == 1: x = x.squeeze(1) diff --git a/nebula/core/models/covtype/mlp.py b/nebula/core/models/covtype/mlp.py index 6eb55a348..bb93fbc97 100644 --- a/nebula/core/models/covtype/mlp.py +++ b/nebula/core/models/covtype/mlp.py @@ -16,13 +16,9 @@ def __init__( seed=None, data_type="Tabular", ): - # OJO: NebulaModel está pensado para imágenes (input_channels), - # pero en la práctica se usa ese primer argumento como "input shape info". - # Para tabular, pasamos input_dim en input_channels para mantener la firma. super().__init__(input_dim, num_classes, learning_rate, metrics, confusion_matrix, seed) self.data_type = data_type - # Mantengo el mismo patrón que tu MLP de FashionMNIST. self.config = {"beta1": 0.9, "beta2": 0.999, "amsgrad": True} self.example_input_array = torch.rand(1, input_dim) @@ -34,8 +30,6 @@ def __init__( self.l3 = torch.nn.Linear(128, num_classes) def forward(self, x): - # En tabular, x debe ser (batch, input_dim). - # A veces puede venir con dimensión extra (batch, 1, input_dim) por loaders. if x.dim() == 3 and x.size(1) == 1: x = x.squeeze(1) diff --git a/nebula/core/models/nebulamodel.py b/nebula/core/models/nebulamodel.py index a6557fed6..5973f1518 100755 --- a/nebula/core/models/nebulamodel.py +++ b/nebula/core/models/nebulamodel.py @@ -160,7 +160,6 @@ def generate_confusion_matrix(self, phase, print_cm=False, plot_cm=False): del cm_numpy, classes, fig, ax - # Restablecer la matriz de confusión if phase == "Test (Local)": self.cm.reset() else: diff --git a/nebula/core/training/lightning_dp.py b/nebula/core/training/lightning_dp.py index bed08a973..6fde329f4 100644 --- a/nebula/core/training/lightning_dp.py +++ b/nebula/core/training/lightning_dp.py @@ -42,7 +42,7 @@ def create_dp_plugin(self): ) def _train_sync(self): - # Keep the public Lightning trainer contract: train once and return loss/accuracy. + # Keep the public Lightning trainer contract: train once and return validation and training metrics. try: self._fit_with_dp() @@ -57,7 +57,12 @@ def _train_sync(self): loss = raw_loss.item() if hasattr(raw_loss, "item") else raw_loss accuracy = validation_metrics.get("Validation/Accuracy") - return loss, accuracy + train_accuracy = None + get_train_accuracy = getattr(self.model, "get_latest_train_accuracy", None) + if callable(get_train_accuracy): + train_accuracy = get_train_accuracy() + + return loss, accuracy, train_accuracy except Exception as e: logging_training.error(f"Error in _train_sync with Differential Privacy: {e}") From ce3f59680a01452da335ee2d85b62750b9f74065 Mon Sep 17 00:00:00 2001 From: enriquetomasmb Date: Fri, 12 Jun 2026 10:59:47 +0200 Subject: [PATCH 66/66] Refactor code for improved readability and consistency --- .github/PULL_REQUEST_TEMPLATE.md | 4 +- CLA.md | 4 +- COMMERCIAL_INFO.md | 4 +- CONTRIBUTING.md | 4 +- app/deployer.py | 30 ++++---- app/windows/install.ps1 | 2 +- docs/_prebuilt/commercial-faq.md | 10 +-- .../attacks/communications/floodingattack.py | 4 +- .../attacks/model/gllneuroninversion.py | 2 +- .../addons/attacks/model/swappingweights.py | 2 +- .../networksimulation/networksimulator.py | 2 +- nebula/config/config.py | 2 +- nebula/controller/controller.py | 72 +++++++++---------- nebula/controller/http_helpers.py | 18 ++--- nebula/core/datasets/datamodule.py | 2 +- nebula/core/role.py | 2 +- .../staticarbitrationpolicy.py | 8 +-- .../distanceneighborpolicy.py | 4 +- .../neighborpolicies/fcneighborpolicy.py | 6 +- .../neighborpolicies/idleneighborpolicy.py | 8 +-- .../neighborpolicies/ringneighborpolicy.py | 8 +-- .../awareness/satraining/satraining.py | 13 ++-- .../trainingpolicy/bpstrainingpolicy.py | 16 ++--- .../satraining/trainingpolicy/fastreboot.py | 4 +- .../trainingpolicy/htstrainingpolicy.py | 30 ++++---- .../trainingpolicy/qdstrainingpolicy.py | 60 ++++++++-------- .../trainingpolicy/trainingpolicy.py | 14 ++-- .../distcandidateselector.py | 10 +-- .../candidateselection/fccandidateselector.py | 2 +- .../ringcandidateselector.py | 2 +- .../stdcandidateselector.py | 6 +- .../modelhandlers/defaultmodelhandler.py | 6 +- .../modelhandlers/stdmodelhandler.py | 6 +- nebula/frontend/static/css/deployment.css | 2 +- .../frontend/static/js/deployment/topology.js | 26 +++---- .../static/js/deployment/ui-controls.js | 38 +++++----- nebula/physical/api.py | 6 +- nebula/physical/node.sh | 4 +- nebula/utils.py | 2 +- 39 files changed, 222 insertions(+), 223 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index f61055dbb..ace900221 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -17,5 +17,5 @@ Please fill out the following template to help us review your pull request. ## Signed-off-by -Signed-off-by: *Your Name (email)* -Date: *YYYY-MM-DD* \ No newline at end of file +Signed-off-by: *Your Name (email)* +Date: *YYYY-MM-DD* diff --git a/CLA.md b/CLA.md index 6a87eb8c4..d321de5ea 100644 --- a/CLA.md +++ b/CLA.md @@ -18,5 +18,5 @@ By submitting a pull request, patch or code snippet, you agree that: your contribution and that Authors may use, sell or license the software containing your contribution at its sole discretion. -Signed-off-by: *Enrique Tomás Martínez Beltrán (enriquetomas@um.es)* -Date: *2025-06-25* \ No newline at end of file +Signed-off-by: *Enrique Tomás Martínez Beltrán (enriquetomas@um.es)* +Date: *2025-06-25* diff --git a/COMMERCIAL_INFO.md b/COMMERCIAL_INFO.md index c22826a6e..81bcec4a7 100644 --- a/COMMERCIAL_INFO.md +++ b/COMMERCIAL_INFO.md @@ -1,6 +1,6 @@ # NEBULA Enterprise License -This repository is published under **GNU AGPL v3.0**. +This repository is published under **GNU AGPL v3.0**. If you wish to embed NEBULA in closed-source products, offer it as a hosted service, or obtain an SLA, please e-mail **enriquetomas@um.es** and **alberto.huertas@um.es**. -A bespoke commercial agreement (OEM / subscription / SaaS) will be provided on request. \ No newline at end of file +A bespoke commercial agreement (OEM / subscription / SaaS) will be provided on request. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a123f7cc5..6a2779a85 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,7 +6,7 @@ Follow conventional-commit style. ## 2 • Sign the CLA When you open your first Pull Request, **CLA-assistant** will block the merge until you tick the box confirming you accept the -[ICLA](CLA.md). +[ICLA](CLA.md). Add a Developer-Certificate-of-Origin line in every commit: ``` @@ -23,4 +23,4 @@ The pull request will be reviewed by the maintainers. The maintainers will provide feedback on the pull request. ## 6 • Merge the Pull Request -The pull request will be merged by the maintainers. \ No newline at end of file +The pull request will be merged by the maintainers. diff --git a/app/deployer.py b/app/deployer.py index 968ef62da..fc21f23ef 100644 --- a/app/deployer.py +++ b/app/deployer.py @@ -289,17 +289,17 @@ def run_script(self, script): def kill_script_processes(self, pids_file): """ Forcefully terminates processes listed in a given PID file, including their child processes. - + Args: pids_file (str): Path to the file containing PIDs, one per line. - + Behavior: - Reads the PIDs from the file. - For each PID, checks if the process exists. - If it exists, kills all child processes recursively before killing the main process. - Handles and logs exceptions such as missing processes or invalid PID entries. - Logs warnings and errors appropriately. - + Typical use case: Used to clean up running processes related to a scenario or script that has been deleted or stopped. """ @@ -344,7 +344,7 @@ def run_observer(): """ Starts a watchdog observer to monitor the configuration directory for changes. - This function is typically used to execute additional scripts or trigger events + This function is typically used to execute additional scripts or trigger events during the execution of a federated learning session by monitoring file system changes. Main functionalities: @@ -357,7 +357,7 @@ def run_observer(): - Trigger specific actions during a federation lifecycle. Note: - The observer runs in a blocking mode and will keep the process alive + The observer runs in a blocking mode and will keep the process alive until manually stopped or interrupted. """ # Watchdog for running additional scripts in the host machine (i.e. during the execution of a federation) @@ -373,7 +373,7 @@ class Deployer: """ Handles the configuration and initialization of deployment parameters for the NEBULA system. - This class reads and stores various deployment-related settings such as port assignments, + This class reads and stores various deployment-related settings such as port assignments, environment paths, logging configuration, and system mode (production, development, or simulation). Main functionalities: @@ -438,7 +438,7 @@ def configure_logger(self): """ Configures the logging system for the deployment controller. - This method sets up both console and file logging with a consistent format and appropriate log levels. + This method sets up both console and file logging with a consistent format and appropriate log levels. It also ensures that Uvicorn loggers are properly configured to avoid duplicate log outputs. Main functionalities: @@ -452,7 +452,7 @@ def configure_logger(self): - Ensures clean and consistent logging output during deployment. Note: - This method does not set up file logging directly, but prepares the base configuration + This method does not set up file logging directly, but prepares the base configuration and Uvicorn logger behavior for further logging use. """ log_console_format = "[%(asctime)s] [%(name)s] [%(levelname)s] %(message)s" @@ -475,7 +475,7 @@ def ensure_directory_access(self, directory_path: str) -> str: """ Ensures that the specified directory exists and is writable. - This method attempts to create the directory if it does not exist and verifies + This method attempts to create the directory if it does not exist and verifies write access by writing and deleting a temporary metadata file. Args: @@ -521,8 +521,8 @@ def start(self): """ Starts the NEBULA deployment process and all associated services. - This method initializes the NEBULA platform by setting up the environment, - checking port availability, starting key services (controller, frontend, WAF), + This method initializes the NEBULA platform by setting up the environment, + checking port availability, starting key services (controller, frontend, WAF), and launching a filesystem observer to react to configuration changes. Main functionalities: @@ -539,7 +539,7 @@ def start(self): - Central entry point for managing NEBULA components during deployment. Note: - The method blocks indefinitely until manually interrupted, + The method blocks indefinitely until manually interrupted, and ensures graceful shutdown upon receiving SIGINT or SIGTERM. """ banner = """ @@ -616,8 +616,8 @@ def signal_handler(self, sig, frame): """ Handles system termination signals to ensure a clean shutdown. - This method is triggered when the application receives SIGTERM or SIGINT signals - (e.g., via Ctrl+C or `kill`). It logs the event, performs cleanup actions, and + This method is triggered when the application receives SIGTERM or SIGINT signals + (e.g., via Ctrl+C or `kill`). It logs the event, performs cleanup actions, and terminates the process gracefully. Args: @@ -749,7 +749,7 @@ def run_controller(self): ) network_name = f"{os.environ['USER']}_nebula-net-base" - + try: subprocess.check_call(["nvidia-smi"]) self.gpu_available = True diff --git a/app/windows/install.ps1 b/app/windows/install.ps1 index 88aa7f1ac..6d9dc7004 100644 --- a/app/windows/install.ps1 +++ b/app/windows/install.ps1 @@ -1,2 +1,2 @@ # Run make install -make install \ No newline at end of file +make install diff --git a/docs/_prebuilt/commercial-faq.md b/docs/_prebuilt/commercial-faq.md index 197ef81a9..274c0fcbd 100644 --- a/docs/_prebuilt/commercial-faq.md +++ b/docs/_prebuilt/commercial-faq.md @@ -1,13 +1,13 @@ # Commercial FAQ — NEBULA Enterprise -**Q 1. What does the commercial license cover?** +**Q 1. What does the commercial license cover?** To be determined. -**Q 2. Does the commercial edition include extra features?** +**Q 2. Does the commercial edition include extra features?** To be determined. -**Q 3. Pricing model?** +**Q 3. Pricing model?** To be determined. -**Q 4. Can we contribute back fixes?** -Absolutely; your patches remain under AGPL in the community edition, and you can keep proprietary extensions private under the commercial agreement. \ No newline at end of file +**Q 4. Can we contribute back fixes?** +Absolutely; your patches remain under AGPL in the community edition, and you can keep proprietary extensions private under the commercial agreement. diff --git a/nebula/addons/attacks/communications/floodingattack.py b/nebula/addons/attacks/communications/floodingattack.py index 146854fa3..73dc0394c 100644 --- a/nebula/addons/attacks/communications/floodingattack.py +++ b/nebula/addons/attacks/communications/floodingattack.py @@ -69,9 +69,9 @@ async def wrapper(*args, **kwargs): ) _, *new_args = args # Exclude self argument await func(*new_args, **kwargs) - _, *new_args = args + _, *new_args = args return await func(*new_args) - + return wrapper return decorator diff --git a/nebula/addons/attacks/model/gllneuroninversion.py b/nebula/addons/attacks/model/gllneuroninversion.py index 64cb3d215..e52a5a930 100644 --- a/nebula/addons/attacks/model/gllneuroninversion.py +++ b/nebula/addons/attacks/model/gllneuroninversion.py @@ -66,4 +66,4 @@ def model_attack(self, received_weights): # Inject random noise of the same shape and type received_weights[target_key] = torch.empty_like(target_weights).uniform_(0, noise_scale) - return received_weights \ No newline at end of file + return received_weights diff --git a/nebula/addons/attacks/model/swappingweights.py b/nebula/addons/attacks/model/swappingweights.py index 95aa89208..36eb1e7a0 100644 --- a/nebula/addons/attacks/model/swappingweights.py +++ b/nebula/addons/attacks/model/swappingweights.py @@ -109,4 +109,4 @@ def model_attack(self, received_weights): if self.layer_idx + 2 < len(layer_keys): received_weights[layer_keys[self.layer_idx + 2]] = received_weights[layer_keys[self.layer_idx + 2]][:, perm] - return received_weights \ No newline at end of file + return received_weights diff --git a/nebula/addons/networksimulation/networksimulator.py b/nebula/addons/networksimulation/networksimulator.py index e296a1527..9dfd4853e 100644 --- a/nebula/addons/networksimulation/networksimulator.py +++ b/nebula/addons/networksimulation/networksimulator.py @@ -6,7 +6,7 @@ class NetworkSimulator(ABC): Abstract base class representing a network simulator interface. This interface defines the required methods for controlling and simulating network conditions between nodes. - A concrete implementation is expected to manage artificial delays, bandwidth restrictions, packet loss, + A concrete implementation is expected to manage artificial delays, bandwidth restrictions, packet loss, or other configurable conditions typically used in network emulation or testing. Required asynchronous methods: diff --git a/nebula/config/config.py b/nebula/config/config.py index 5ef336e3a..cae3cf7f8 100755 --- a/nebula/config/config.py +++ b/nebula/config/config.py @@ -55,7 +55,7 @@ def reset_logging_configuration(self): self.__set_default_logging(mode="a") self.__set_training_logging(mode="a") - + def shutdown_logging(self): """ Properly shuts down all loggers and their handlers in the system. diff --git a/nebula/controller/controller.py b/nebula/controller/controller.py index a00d142d1..0d7142dbe 100755 --- a/nebula/controller/controller.py +++ b/nebula/controller/controller.py @@ -264,24 +264,24 @@ async def get_available_gpu(): def validate_physical_fields(data: dict): if data.get("deployment") != "physical": - return - + return + ips = data.get("physical_ips") if not ips: raise HTTPException( status_code=400, detail="physical deployment requires 'physical_ips'" ) - + if len(ips) != data.get("n_nodes"): raise HTTPException( status_code=400, detail="'physical_ips' must have the same length as 'n_nodes'" ) - + try: for ip in ips: - ipaddress.ip_address(ip) + ipaddress.ip_address(ip) print(ip) except ValueError as e: raise HTTPException(status_code=400, detail=str(e)) @@ -347,21 +347,21 @@ async def stop_scenario( ): """ Stops the execution of a federated learning scenario and performs cleanup operations. - + This endpoint: - Stops all participant containers associated with the specified scenario. - Removes Docker containers and network resources tied to the scenario and user. - Sets the scenario's status to "finished" in the database. - Optionally finalizes all active scenarios if the 'all' flag is set. - + Args: scenario_name (str): Name of the scenario to stop. username (str): User who initiated the stop operation. all (bool): Whether to stop all running scenarios instead of just one (default: False). - + Raises: HTTPException: Returns a 500 status code if any step fails. - + Note: This function does not currently trigger statistics generation. """ @@ -847,27 +847,27 @@ async def discover_vpn(): stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, ) - + # 2) Wait for it to finish and capture stdout/stderr out, err = await proc.communicate() if proc.returncode != 0: # If the CLI returned an error, raise to be caught below raise RuntimeError(err.decode()) - + # 3) Parse the JSON output data = json.loads(out.decode()) - + # 4) Collect only the IPv4 addresses from each peer ips = [] for peer in data.get("Peer", {}).values(): for ip in peer.get("TailscaleIPs", []): - if ":" not in ip: + if ":" not in ip: # Skip IPv6 entries (they contain colons) ips.append(ip) - + # 5) Return the list of IPv4s return {"ips": ips} - + except Exception as e: # 6) Log any failure and respond with HTTP 500 logging.error(f"Error discovering VPN devices: {e}") @@ -877,14 +877,14 @@ async def discover_vpn(): @app.get("/physical/run/{ip}", tags=["physical"]) async def physical_run(ip: str): status, data = await remote_get(ip, "/run/") - + if status == 200: return data if status is None: raise HTTPException(status_code=502, detail=f"Node unreachable: {data}") raise HTTPException(status_code=status, detail=data) - - + + @app.get("/physical/stop/{ip}", tags=["physical"]) async def physical_stop(ip: str): status, data = await remote_get(ip, "/stop/") @@ -893,8 +893,8 @@ async def physical_stop(ip: str): if status is None: raise HTTPException(status_code=502, detail=f"Node unreachable: {data}") raise HTTPException(status_code=status, detail=data) - - + + @app.put("/physical/setup/{ip}", tags=["physical"], status_code=status.HTTP_201_CREATED) async def physical_setup( @@ -903,7 +903,7 @@ async def physical_setup( global_test: UploadFile = File(..., description="Global Dataset*.h5*"), train_set: UploadFile = File(..., description="Training dataset*.h5*"), ): - + form = aiohttp.FormData() await config.seek(0) form.add_field("config", config.file, @@ -914,17 +914,17 @@ async def physical_setup( await train_set.seek(0) form.add_field("train_set", train_set.file, filename=train_set.filename, content_type="application/octet-stream") - + status_code, data = await remote_post_form( ip, "/setup/", form, method="PUT" ) - + if status_code == 201: return data if status_code is None: raise HTTPException(status_code=502, detail=f"Node unreachable: {data}") raise HTTPException(status_code=status_code, detail=data) - + # ────────────────────────────────────────────────────────────── # Physical · single-node state # ────────────────────────────────────────────────────────────── @@ -932,22 +932,22 @@ async def physical_setup( async def get_physical_node_state(ip: str): """ Query a single Raspberry Pi (or other node) for its training state. - + Parameters ---------- ip : str IP address or hostname of the node. - + Returns ------- dict - • running (bool) – True if a training process is active. + • running (bool) – True if a training process is active. • error (str) – Optional error message when the node is unreachable or returns a non-200 HTTP status. """ # Short global timeout so a dead node doesn't block the whole request timeout = aiohttp.ClientTimeout(total=3) # seconds - + try: async with aiohttp.ClientSession(timeout=timeout) as session: async with session.get(f"http://{ip}/state/") as resp: @@ -960,8 +960,8 @@ async def get_physical_node_state(ip: str): except Exception as exc: # Network errors, timeouts, DNS failures, … return {"running": False, "error": str(exc)} - - + + # ────────────────────────────────────────────────────────────── # Physical · aggregate state for an entire scenario # ────────────────────────────────────────────────────────────── @@ -969,12 +969,12 @@ async def get_physical_node_state(ip: str): async def get_physical_scenario_state(scenario_name: str): """ Check the training state of *every* physical node assigned to a scenario. - + Parameters ---------- scenario_name : str Scenario identifier. - + Returns ------- dict @@ -989,16 +989,16 @@ async def get_physical_scenario_state(scenario_name: str): scenario = await get_scenario_by_name(scenario_name) if not scenario: raise HTTPException(status_code=404, detail="Scenario not found") - + nodes = await list_nodes_by_scenario_name(scenario_name) if not nodes: raise HTTPException(status_code=404, detail="No nodes found for scenario") - + # 2) Probe all nodes concurrently ips = [n["ip"] for n in nodes] tasks = [get_physical_node_state(ip) for ip in ips] states = await asyncio.gather(*tasks) # parallel HTTP calls - + # 3) Aggregate results nodes_state = dict(zip(ips, states)) any_running = any(s.get("running") for s in states) @@ -1007,7 +1007,7 @@ async def get_physical_scenario_state(scenario_name: str): all_available = all( (not s.get("running")) and (not s.get("error")) for s in states ) - + return { "running": any_running, "nodes_state": nodes_state, diff --git a/nebula/controller/http_helpers.py b/nebula/controller/http_helpers.py index ed60f44e5..886cc57e7 100644 --- a/nebula/controller/http_helpers.py +++ b/nebula/controller/http_helpers.py @@ -1,13 +1,13 @@ from __future__ import annotations - + import logging from typing import Optional, Union - + import aiohttp from aiohttp import FormData - + _TIMEOUT = aiohttp.ClientTimeout(total=15) - + async def _request_json( method: str, host: str, @@ -27,12 +27,12 @@ async def _request_json( except Exception as exc: logging.error("[%s] %s%s – %s", method.upper(), host, endpoint, exc) return None, str(exc) - - + + async def remote_get(host: str, endpoint: str): return await _request_json("GET", host, endpoint) - - + + async def remote_post_form( host: str, endpoint: str, @@ -40,4 +40,4 @@ async def remote_post_form( *, method: str = "POST", ): - return await _request_json(method, host, endpoint, data=form) \ No newline at end of file + return await _request_json(method, host, endpoint, data=form) diff --git a/nebula/core/datasets/datamodule.py b/nebula/core/datasets/datamodule.py index 04413f35a..aae9bf820 100755 --- a/nebula/core/datasets/datamodule.py +++ b/nebula/core/datasets/datamodule.py @@ -46,7 +46,7 @@ def __init__( self.data_val = None self.global_te_subset = None self.local_te_subset = None - + def get_samples_per_label(self): return self._samples_per_label diff --git a/nebula/core/role.py b/nebula/core/role.py index 6bc4343f8..dc5281983 100755 --- a/nebula/core/role.py +++ b/nebula/core/role.py @@ -10,7 +10,7 @@ class Role(Enum): PROXY = "proxy" IDLE = "idle" SERVER = "server" - + def factory_node_role(role: str) -> Role: if role == "trainer": return Role.TRAINER diff --git a/nebula/core/situationalawareness/awareness/arbitrationpolicies/staticarbitrationpolicy.py b/nebula/core/situationalawareness/awareness/arbitrationpolicies/staticarbitrationpolicy.py index e0dedab5c..dd17b5c35 100644 --- a/nebula/core/situationalawareness/awareness/arbitrationpolicies/staticarbitrationpolicy.py +++ b/nebula/core/situationalawareness/awareness/arbitrationpolicies/staticarbitrationpolicy.py @@ -8,11 +8,11 @@ class SAP(ArbitrationPolicy): # Static Arbitatrion Policy """ Static Arbitration Policy for the Reasoner module. - This class implements a fixed priority arbitration mechanism for - SA (Situational Awareness) components. Each SA component category + This class implements a fixed priority arbitration mechanism for + SA (Situational Awareness) components. Each SA component category is assigned a static weight representing its priority level. - In case of conflicting SA commands, the policy selects the command + In case of conflicting SA commands, the policy selects the command whose originating component has the highest priority weight. Attributes: @@ -21,7 +21,7 @@ class SAP(ArbitrationPolicy): # Static Arbitatrion Policy Methods: init(config): Placeholder for initialization with external configuration. - tie_break(sac1, sac2): Resolves conflicts between two SA commands by + tie_break(sac1, sac2): Resolves conflicts between two SA commands by comparing their category weights, returning True if sac1 wins. """ def __init__(self, verbose): diff --git a/nebula/core/situationalawareness/awareness/sanetwork/neighborpolicies/distanceneighborpolicy.py b/nebula/core/situationalawareness/awareness/sanetwork/neighborpolicies/distanceneighborpolicy.py index a1d29c675..421cb8e37 100644 --- a/nebula/core/situationalawareness/awareness/sanetwork/neighborpolicies/distanceneighborpolicy.py +++ b/nebula/core/situationalawareness/awareness/sanetwork/neighborpolicies/distanceneighborpolicy.py @@ -16,7 +16,7 @@ class DistanceNeighborPolicy(NeighborPolicy): - When to discard or replace existing neighbors. - Keeping track of current neighbors and known nodes with their distances. - The policy operates under the assumption that physical proximity + The policy operates under the assumption that physical proximity can be beneficial for performance and robustness in the network. Attributes: @@ -26,7 +26,7 @@ class DistanceNeighborPolicy(NeighborPolicy): addr (str | None): The address of this node (used for self-identification). neighbors_lock (Locker): Async lock for safe access to `neighbors`. nodes_known_lock (Locker): Async lock for safe access to `nodes_known`. - nodes_distances (dict[str, tuple[float, tuple[float, float]]] | None): + nodes_distances (dict[str, tuple[float, tuple[float, float]]] | None): Mapping from node IDs to a tuple containing (distance, (latitude, longitude)). nodes_distances_lock (Locker): Async lock for safe access to `nodes_distances`. _verbose (bool): Whether to enable verbose logging for debugging purposes. diff --git a/nebula/core/situationalawareness/awareness/sanetwork/neighborpolicies/fcneighborpolicy.py b/nebula/core/situationalawareness/awareness/sanetwork/neighborpolicies/fcneighborpolicy.py index e395a199a..443282f65 100644 --- a/nebula/core/situationalawareness/awareness/sanetwork/neighborpolicies/fcneighborpolicy.py +++ b/nebula/core/situationalawareness/awareness/sanetwork/neighborpolicies/fcneighborpolicy.py @@ -8,8 +8,8 @@ class FCNeighborPolicy(NeighborPolicy): """ Neighbor policy for fully-connected (FC) structured topologies. - This policy assumes a fully-connected topology where every node should attempt - to connect to all known nodes. It always accepts incoming neighbor connections + This policy assumes a fully-connected topology where every node should attempt + to connect to all known nodes. It always accepts incoming neighbor connections and considers the neighbor list incomplete if there are known nodes that are not yet connected. The goal is to maintain full connectivity across all known nodes in the federation. @@ -23,7 +23,7 @@ class FCNeighborPolicy(NeighborPolicy): nodes_known_lock (Locker): Async lock for safe access to `nodes_known`. _verbose (bool): Whether to enable verbose logging for debugging purposes. """ - + def __init__(self): self.max_neighbors = None self.nodes_known = set() diff --git a/nebula/core/situationalawareness/awareness/sanetwork/neighborpolicies/idleneighborpolicy.py b/nebula/core/situationalawareness/awareness/sanetwork/neighborpolicies/idleneighborpolicy.py index 648c8605e..d1d7d5025 100644 --- a/nebula/core/situationalawareness/awareness/sanetwork/neighborpolicies/idleneighborpolicy.py +++ b/nebula/core/situationalawareness/awareness/sanetwork/neighborpolicies/idleneighborpolicy.py @@ -8,11 +8,11 @@ class IDLENeighborPolicy(NeighborPolicy): """ Neighbor policy for minimal connectivity scenarios. - This policy only attempts to discover or establish new neighbor connections - if the node is currently isolated (i.e., has no neighbors). All incoming + This policy only attempts to discover or establish new neighbor connections + if the node is currently isolated (i.e., has no neighbors). All incoming connection requests are accepted regardless of the current neighbor state. - This policy is suitable for scenarios where minimal intervention is preferred, + This policy is suitable for scenarios where minimal intervention is preferred, and connections are formed opportunistically rather than proactively. Attributes: @@ -24,7 +24,7 @@ class IDLENeighborPolicy(NeighborPolicy): nodes_known_lock (Locker): Async lock for thread-safe access to `nodes_known`. _verbose (bool): Enables verbose logging for debugging and traceability. """ - + def __init__(self): self.max_neighbors = None self.nodes_known = set() diff --git a/nebula/core/situationalawareness/awareness/sanetwork/neighborpolicies/ringneighborpolicy.py b/nebula/core/situationalawareness/awareness/sanetwork/neighborpolicies/ringneighborpolicy.py index afd9b1d59..e6933b5b7 100644 --- a/nebula/core/situationalawareness/awareness/sanetwork/neighborpolicies/ringneighborpolicy.py +++ b/nebula/core/situationalawareness/awareness/sanetwork/neighborpolicies/ringneighborpolicy.py @@ -10,9 +10,9 @@ class RINGNeighborPolicy(NeighborPolicy): """ Neighbor policy for ring topologies. - This policy maintains a strict limit on the number of neighbors per node, - enforcing a ring-like structure. Each node connects to a fixed number of - neighbors (by default 2), and excess connections are detected and marked + This policy maintains a strict limit on the number of neighbors per node, + enforcing a ring-like structure. Each node connects to a fixed number of + neighbors (by default 2), and excess connections are detected and marked for removal. The policy ensures: @@ -34,7 +34,7 @@ class RINGNeighborPolicy(NeighborPolicy): _excess_neighbors_removed_lock (Locker): Lock for accessing the removal tracking set. _verbose (bool): Enables verbose logging. """ - + RECENTLY_REMOVED_BAN_TIME = 20 def __init__(self): diff --git a/nebula/core/situationalawareness/awareness/satraining/satraining.py b/nebula/core/situationalawareness/awareness/satraining/satraining.py index 94c18f40c..813cc11fe 100644 --- a/nebula/core/situationalawareness/awareness/satraining/satraining.py +++ b/nebula/core/situationalawareness/awareness/satraining/satraining.py @@ -6,9 +6,9 @@ from nebula.addons.functions import print_msg_box from nebula.core.situationalawareness.awareness.sareasoner import SAReasoner, SAMComponent from nebula.core.eventmanager import EventManager - -RESTRUCTURE_COOLDOWN = 5 - + +RESTRUCTURE_COOLDOWN = 5 + class SATraining(SAMComponent): """ SATraining is a Situational Awareness (SA) component responsible for enhancing @@ -24,7 +24,7 @@ class SATraining(SAMComponent): _sar (SAReasoner): Reference to the shared situational reasoner. _trainning_policy: Instantiated training policy strategy. """ - + def __init__(self, config): """ Initialize the SATraining component with a given configuration. @@ -61,7 +61,7 @@ def tp(self): """ Returns the currently active training policy instance. """ - return self._trainning_policy + return self._trainning_policy async def init(self): """ @@ -69,7 +69,7 @@ async def init(self): This setup enables the policy to make informed decisions based on local topology. """ config = {} - config["nodes"] = set(await self.sar.get_nodes_known(neighbors_only=True)) + config["nodes"] = set(await self.sar.get_nodes_known(neighbors_only=True)) await self.tp.init(config) async def sa_component_actions(self): @@ -79,4 +79,3 @@ async def sa_component_actions(self): """ logging.info("SA Trainng evaluating current scenario") asyncio.create_task(self.tp.get_evaluation_results()) - diff --git a/nebula/core/situationalawareness/awareness/satraining/trainingpolicy/bpstrainingpolicy.py b/nebula/core/situationalawareness/awareness/satraining/trainingpolicy/bpstrainingpolicy.py index 0353a8020..32874c6ae 100644 --- a/nebula/core/situationalawareness/awareness/satraining/trainingpolicy/bpstrainingpolicy.py +++ b/nebula/core/situationalawareness/awareness/satraining/trainingpolicy/bpstrainingpolicy.py @@ -4,18 +4,18 @@ from nebula.core.nebulaevents import RoundEndEvent class BPSTrainingPolicy(TrainingPolicy): - + def __init__(self, config=None): pass - + async def init(self, config): - await self.register_sa_agent() + await self.register_sa_agent() async def get_evaluation_results(self): sac = factory_sa_command( "connectivity", SACommandAction.MAINTAIN_CONNECTIONS, - self, + self, "", SACommandPRIO.LOW, False, @@ -24,15 +24,15 @@ async def get_evaluation_results(self): ) await self.suggest_action(sac) await self.notify_all_suggestions_done(RoundEndEvent) - + async def get_agent(self) -> str: return "SATraining_BPSTP" async def register_sa_agent(self): await SuggestionBuffer.get_instance().register_event_agents(RoundEndEvent, self) - + async def suggest_action(self, sac : SACommand): await SuggestionBuffer.get_instance().register_suggestion(RoundEndEvent, self, sac) - + async def notify_all_suggestions_done(self, event_type): - await SuggestionBuffer.get_instance().notify_all_suggestions_done_for_agent(self, event_type) \ No newline at end of file + await SuggestionBuffer.get_instance().notify_all_suggestions_done_for_agent(self, event_type) diff --git a/nebula/core/situationalawareness/awareness/satraining/trainingpolicy/fastreboot.py b/nebula/core/situationalawareness/awareness/satraining/trainingpolicy/fastreboot.py index dd8fc438d..39a0791c4 100644 --- a/nebula/core/situationalawareness/awareness/satraining/trainingpolicy/fastreboot.py +++ b/nebula/core/situationalawareness/awareness/satraining/trainingpolicy/fastreboot.py @@ -24,13 +24,13 @@ def __init__( self._upgrade_lr = FR_LEARNING_RATE # Increased value for learning rate self._current_lr = VANILLA_LEARNING_RATE self._verbose = config["verbose"] - + self._learning_rate_lock = Locker(name="learning_rate_lock", async_lock=True) self._weight_modifier = {} self._weight_modifier_lock = Locker(name="weight_modifier_lock", async_lock=True) self._fr_in_progress = False - + async def init(self, config): #await EventManager.get_instance().subscribe_node_event(UpdateNeighborEvent) #await EventManager.get_instance().subscribe_node_event(AggregationEvent) diff --git a/nebula/core/situationalawareness/awareness/satraining/trainingpolicy/htstrainingpolicy.py b/nebula/core/situationalawareness/awareness/satraining/trainingpolicy/htstrainingpolicy.py index e37209ece..29b0524ba 100644 --- a/nebula/core/situationalawareness/awareness/satraining/trainingpolicy/htstrainingpolicy.py +++ b/nebula/core/situationalawareness/awareness/satraining/trainingpolicy/htstrainingpolicy.py @@ -6,19 +6,19 @@ # "Hybrid Training Strategy" (HTS) class HTSTrainingPolicy(TrainingPolicy): """ - Implements a Hybrid Training Strategy (HTS) that combines multiple training policies - (e.g., QDS, FRTS) to collaboratively decide on the evaluation and potential pruning + Implements a Hybrid Training Strategy (HTS) that combines multiple training policies + (e.g., QDS, FRTS) to collaboratively decide on the evaluation and potential pruning of neighbors in a decentralized federated learning scenario. - + Attributes: TRAINING_POLICY (set): Names of training policy classes to instantiate and manage. """ - + TRAINING_POLICY = { "Quality-Driven Selection", "Fast Reboot Training Strategy", } - + def __init__(self, config): """ Initializes the HTS policy with the node's address and verbosity level. @@ -33,34 +33,34 @@ def __init__(self, config): self._verbose = config["verbose"] self._training_policies : set[TrainingPolicy] = set() self._training_policies.add([factory_training_policy(x, config) for x in self.TRAINING_POLICY]) - + def __str__(self): - return "HTS" - + return "HTS" + @property def tps(self): - return self._training_policies + return self._training_policies async def init(self, config): for tp in self.tps: - await tp.init(config) + await tp.init(config) async def update_neighbors(self, node, remove=False): pass - + async def get_evaluation_results(self): """ Asynchronously calls the `get_evaluation_results` of each policy, and logs the nodes each policy would remove. - + Returns: None (future version may merge all evaluations). """ nodes_to_remove = dict() for tp in self.tps: nodes_to_remove[tp] = await tp.get_evaluation_results() - + for tp, nodes in nodes_to_remove.items(): logging.info(f"Training Policy: {tp}, nodes to remove: {nodes}") - - return None \ No newline at end of file + + return None diff --git a/nebula/core/situationalawareness/awareness/satraining/trainingpolicy/qdstrainingpolicy.py b/nebula/core/situationalawareness/awareness/satraining/trainingpolicy/qdstrainingpolicy.py index f067f7e84..535097b45 100644 --- a/nebula/core/situationalawareness/awareness/satraining/trainingpolicy/qdstrainingpolicy.py +++ b/nebula/core/situationalawareness/awareness/satraining/trainingpolicy/qdstrainingpolicy.py @@ -15,13 +15,13 @@ class QDSTrainingPolicy(TrainingPolicy): """ Implements a Quality-Driven Selection (QDS) strategy for training in DFL. - + This policy tracks the cosine similarity of neighbor model updates over time, and detects nodes that are inactive or provide redundant updates. Based on these evaluations, the policy suggests disconnecting such nodes to promote better model convergence and network efficiency. """ - + MAX_HISTORIC_SIZE = 10 SIMILARITY_THRESHOLD = 0.73 INACTIVE_THRESHOLD = 3 @@ -31,7 +31,7 @@ class QDSTrainingPolicy(TrainingPolicy): def __init__(self, config : dict): """ Initializes the QDS training policy. - + Args: config (dict): Configuration dictionary with keys: - "addr": Local node address. @@ -46,7 +46,7 @@ def __init__(self, config : dict): self._last_check = 0 self._check_done = False self._evaluation_results = set() - + def __str__(self): return "QDS" @@ -94,21 +94,21 @@ async def _process_aggregation_event(self, agg_ev : AggregationEvent): for addr, updt in updates.items(): if addr == self._addr: continue if not addr in self._nodes.keys(): continue - + deque_history, missed_count = self._nodes[addr] if addr in missing_nodes: if self._verbose: logging.info(f"Node inactivity counter increased for: {addr}") self._nodes[addr] = (deque_history, missed_count + 1) # Inactive rounds counter +1 else: self._nodes[addr] = (deque_history, 0) # Reset inactive counter - - #TODO Do it for the ones not using the last update received cause they are missing this round + + #TODO Do it for the ones not using the last update received cause they are missing this round (model,_) = updt - (self_model, _) = self_updt + (self_model, _) = self_updt cos_sim = cosine_metric(self_model, model, similarity=True) self._nodes[addr][0].append(cos_sim) self._evaluation_results = await self.evaluate() - + async def _get_nodes(self): """ Safely returns a copy of the current node tracking dictionary. @@ -118,8 +118,8 @@ async def _get_nodes(self): """ async with self._nodes_lock: nodes = self._nodes.copy() - return nodes - + return nodes + async def evaluate(self): """ Evaluates the current neighbor set to determine inactive or redundant nodes. @@ -131,10 +131,10 @@ async def evaluate(self): self._grace_rounds -= 1 if self._verbose: logging.info("Grace time hasnt finished...") return None - + if self._verbose: logging.info("Evaluation in process") - - result = set() + + result = set() if self._last_check == 0: self._check_done = True nodes = await self._get_nodes() @@ -149,18 +149,18 @@ async def evaluate(self): if self._verbose: logging.info(f"Node: {node} hadn't participated in any of the last {self.INACTIVE_THRESHOLD} rounds") else: if self._verbose: logging.info(f"Node: {node} inactivity counter: {inactivity_counter}") - + if node not in self._round_missing_nodes: if last_sim < self.SIMILARITY_THRESHOLD: if self._verbose: logging.info(f"Node: {node} got a similarity value of: {last_sim} under threshold: {self.SIMILARITY_THRESHOLD}") else: if self._verbose: logging.info(f"Node: {node} got a redundant model, cossine simmilarity: {last_sim} over threshold: {self.SIMILARITY_THRESHOLD}") redundant_nodes.add((node, last_sim)) - + if self._verbose: logging.info(f"Inactive nodes on aggregations: {inactive_nodes}") if self._verbose: logging.info(f"Redundant nodes on aggregations: {redundant_nodes}") if inactive_nodes: - result = result.union(inactive_nodes) + result = result.union(inactive_nodes) if len(redundant_nodes): sorted_redundant_nodes = sorted(redundant_nodes, key=lambda x: x[1]) n_discarded = math.ceil((len(redundant_nodes)/2)) @@ -171,11 +171,11 @@ async def evaluate(self): else: if self._verbose: logging.info(f"Evaluation is on cooldown... | {self.CHECK_COOLDOWN - self._last_check} rounds remaining") self._check_done = False - + self._last_check = (self._last_check + 1) % self.CHECK_COOLDOWN - + return result - + async def get_evaluation_results(self): """ Triggers suggested actions based on last evaluation results. @@ -186,14 +186,14 @@ async def get_evaluation_results(self): for node_discarded in self._evaluation_results: args = (node_discarded, False, True) sac = factory_sa_command( - "connectivity", + "connectivity", SACommandAction.DISCONNECT, - self, - node_discarded, - SACommandPRIO.MEDIUM, - False, - CommunicationsManager.get_instance().disconnect, - *args + self, + node_discarded, + SACommandPRIO.MEDIUM, + False, + CommunicationsManager.get_instance().disconnect, + *args ) await self.suggest_action(sac) await self.notify_all_suggestions_done(RoundEndEvent) @@ -203,9 +203,9 @@ async def get_agent(self) -> str: async def register_sa_agent(self): await SuggestionBuffer.get_instance().register_event_agents(RoundEndEvent, self) - + async def suggest_action(self, sac : SACommand): await SuggestionBuffer.get_instance().register_suggestion(RoundEndEvent, self, sac) - + async def notify_all_suggestions_done(self, event_type): - await SuggestionBuffer.get_instance().notify_all_suggestions_done_for_agent(self, event_type) \ No newline at end of file + await SuggestionBuffer.get_instance().notify_all_suggestions_done_for_agent(self, event_type) diff --git a/nebula/core/situationalawareness/awareness/satraining/trainingpolicy/trainingpolicy.py b/nebula/core/situationalawareness/awareness/satraining/trainingpolicy/trainingpolicy.py index cd9dae7c1..74d1b426f 100644 --- a/nebula/core/situationalawareness/awareness/satraining/trainingpolicy/trainingpolicy.py +++ b/nebula/core/situationalawareness/awareness/satraining/trainingpolicy/trainingpolicy.py @@ -2,7 +2,7 @@ from nebula.core.situationalawareness.awareness.sautils.samoduleagent import SAModuleAgent class TrainingPolicy(SAModuleAgent): - + @abstractmethod async def init(self, config): pass @@ -10,20 +10,20 @@ async def init(self, config): @abstractmethod async def get_evaluation_results(self): pass - - + + def factory_training_policy(training_policy, config) -> TrainingPolicy: from nebula.core.situationalawareness.awareness.satraining.trainingpolicy.bpstrainingpolicy import BPSTrainingPolicy from nebula.core.situationalawareness.awareness.satraining.trainingpolicy.qdstrainingpolicy import QDSTrainingPolicy from nebula.core.situationalawareness.awareness.satraining.trainingpolicy.htstrainingpolicy import HTSTrainingPolicy from nebula.core.situationalawareness.awareness.satraining.trainingpolicy.fastreboot import FastReboot - + options = { "Broad-Propagation Strategy": BPSTrainingPolicy, # "Broad-Propagation Strategy" (BPS) -- default value "Quality-Driven Selection": QDSTrainingPolicy, # "Quality-Driven Selection" (QDS) "Hybrid Training Strategy": HTSTrainingPolicy, # "Hybrid Training Strategy" (HTS) "Fast Reboot Training Strategy": FastReboot, # "Fast Reboot Training Strategy" (FRTS) - } - + } + cs = options.get(training_policy, BPSTrainingPolicy) - return cs(config) \ No newline at end of file + return cs(config) diff --git a/nebula/core/situationalawareness/discovery/candidateselection/distcandidateselector.py b/nebula/core/situationalawareness/discovery/candidateselection/distcandidateselector.py index d389f8bbb..fec0c1b09 100644 --- a/nebula/core/situationalawareness/discovery/candidateselection/distcandidateselector.py +++ b/nebula/core/situationalawareness/discovery/candidateselection/distcandidateselector.py @@ -10,17 +10,17 @@ class DistanceCandidateSelector(CandidateSelector): """ Selects candidate nodes based on their physical proximity. - This selector uses geolocation data to filter candidates within a - maximum distance threshold. It listens for GPS updates and maintains + This selector uses geolocation data to filter candidates within a + maximum distance threshold. It listens for GPS updates and maintains a mapping of node identifiers to their distances and coordinates. Attributes: - MAX_DISTANCE_THRESHOLD (int): Maximum distance (in meters) allowed + MAX_DISTANCE_THRESHOLD (int): Maximum distance (in meters) allowed for a node to be considered a valid candidate. candidates (list): List of candidate nodes to be evaluated. - candidates_lock (Locker): Async lock for managing concurrent access + candidates_lock (Locker): Async lock for managing concurrent access to the candidate list. - nodes_distances (dict): Maps node IDs to a tuple containing the + nodes_distances (dict): Maps node IDs to a tuple containing the distance and GPS coordinates. nodes_distances_lock (Locker): Async lock for the distance mapping. _verbose (bool): Flag to enable verbose logging for debugging. diff --git a/nebula/core/situationalawareness/discovery/candidateselection/fccandidateselector.py b/nebula/core/situationalawareness/discovery/candidateselection/fccandidateselector.py index 5e82db6b8..b0840b804 100644 --- a/nebula/core/situationalawareness/discovery/candidateselection/fccandidateselector.py +++ b/nebula/core/situationalawareness/discovery/candidateselection/fccandidateselector.py @@ -24,7 +24,7 @@ class FCCandidateSelector(CandidateSelector): Inherits from: CandidateSelector: Base class interface for candidate selection logic. """ - + def __init__(self): self.candidates = [] self.candidates_lock = Locker(name="candidates_lock") diff --git a/nebula/core/situationalawareness/discovery/candidateselection/ringcandidateselector.py b/nebula/core/situationalawareness/discovery/candidateselection/ringcandidateselector.py index 5a90df6c5..d1b3bb33c 100644 --- a/nebula/core/situationalawareness/discovery/candidateselection/ringcandidateselector.py +++ b/nebula/core/situationalawareness/discovery/candidateselection/ringcandidateselector.py @@ -27,7 +27,7 @@ class RINGCandidateSelector(CandidateSelector): Inherits from: CandidateSelector: Base interface for candidate selection strategies. """ - + def __init__(self): self._candidates = [] self._rejected_candidates = [] diff --git a/nebula/core/situationalawareness/discovery/candidateselection/stdcandidateselector.py b/nebula/core/situationalawareness/discovery/candidateselection/stdcandidateselector.py index fb20b59a7..bbb5fd7db 100644 --- a/nebula/core/situationalawareness/discovery/candidateselection/stdcandidateselector.py +++ b/nebula/core/situationalawareness/discovery/candidateselection/stdcandidateselector.py @@ -9,8 +9,8 @@ class STDandidateSelector(CandidateSelector): Candidate selector for scenarios without a predefined structural topology. In cases where the federation topology is not explicitly structured, - this selector chooses candidates based on the average number of neighbors - indicated in their offers. It selects approximately as many candidates as the + this selector chooses candidates based on the average number of neighbors + indicated in their offers. It selects approximately as many candidates as the average neighbor count, aiming to balance connectivity dynamically. Attributes: @@ -27,7 +27,7 @@ class STDandidateSelector(CandidateSelector): Inherits from: CandidateSelector: Base interface for candidate selection strategies. """ - + def __init__(self): self.candidates = [] self.candidates_lock = Locker(name="candidates_lock") diff --git a/nebula/core/situationalawareness/discovery/modelhandlers/defaultmodelhandler.py b/nebula/core/situationalawareness/discovery/modelhandlers/defaultmodelhandler.py index fa8aec8d4..bd16bae8b 100644 --- a/nebula/core/situationalawareness/discovery/modelhandlers/defaultmodelhandler.py +++ b/nebula/core/situationalawareness/discovery/modelhandlers/defaultmodelhandler.py @@ -7,14 +7,14 @@ class DefaultModelHandler(ModelHandler): """ Provides the initial default model. - This handler returns the baseline model with default weights, - typically used at the start of the federation or when no suitable + This handler returns the baseline model with default weights, + typically used at the start of the federation or when no suitable model offers have been received from peers. Inherits from: ModelHandler: Provides the base interface for model operations. """ - + def __init__(self): self.model = None self.rounds = 0 diff --git a/nebula/core/situationalawareness/discovery/modelhandlers/stdmodelhandler.py b/nebula/core/situationalawareness/discovery/modelhandlers/stdmodelhandler.py index 15975dee1..83506249e 100644 --- a/nebula/core/situationalawareness/discovery/modelhandlers/stdmodelhandler.py +++ b/nebula/core/situationalawareness/discovery/modelhandlers/stdmodelhandler.py @@ -4,7 +4,7 @@ class STDModelHandler(ModelHandler): """ - Handles the selection and acquisition of the most up-to-date model + Handles the selection and acquisition of the most up-to-date model during the discovery phase of the federation process. This handler choose the first model received. @@ -13,10 +13,10 @@ class STDModelHandler(ModelHandler): ModelHandler: Provides the base interface for model operations. Intended Use: - Used during the initial, when a node discovers others and must + Used during the initial, when a node discovers others and must align itself with the most recent global model state. """ - + def __init__(self): self.model = None self.rounds = 0 diff --git a/nebula/frontend/static/css/deployment.css b/nebula/frontend/static/css/deployment.css index 03fa0ab30..2fecffe9a 100644 --- a/nebula/frontend/static/css/deployment.css +++ b/nebula/frontend/static/css/deployment.css @@ -234,4 +234,4 @@ button[title]:hover::after { #predefined-topology-nodes:disabled{ background:#e9ecef; cursor:not-allowed; -} \ No newline at end of file +} diff --git a/nebula/frontend/static/js/deployment/topology.js b/nebula/frontend/static/js/deployment/topology.js index 28d5ec0ad..5b6a99ce9 100644 --- a/nebula/frontend/static/js/deployment/topology.js +++ b/nebula/frontend/static/js/deployment/topology.js @@ -533,7 +533,7 @@ const TopologyManager = (function() { function updateIPsAndPorts() { const isPhysical = document.getElementById("physical-devices-radio").checked; - + /* ⬅︎ if physical deployment get default IPs */ if (isPhysical) { gData.nodes.forEach((node, idx) => { @@ -541,11 +541,11 @@ const TopologyManager = (function() { }); return; } - + /* Docker or Process → generate sintetic IPs */ const isProcess = document.getElementById("process-radio").checked; const baseIP = "192.168.50"; - + gData.nodes.forEach((node, idx) => { node.ip = isProcess ? "127.0.0.1" : `${baseIP}.${idx + 2}`; node.port = (45001 + idx).toString(); @@ -572,23 +572,23 @@ const TopologyManager = (function() { function setPhysicalIPs(ipList = []) { if (!ipList.length) return; - + /* 1. Update input for the user */ const nodesInput = document.getElementById('predefined-topology-nodes'); if (nodesInput) { nodesInput.value = ipList.length; - nodesInput.disabled = true; - nodesInput.classList.add('disabled'); + nodesInput.disabled = true; + nodesInput.classList.add('disabled'); } - + /* 2. Regenerate topology */ generatePredefinedTopology(); // ← create Nodes and Links - + /* 3. Assign IPs */ gData.nodes.forEach((n, idx) => { n.ip = ipList[idx] || n.ip; // if more nodes than IPs }); - + updateGraph(); // redraw } @@ -660,7 +660,7 @@ const TopologyManager = (function() { generatePredefinedTopology(); return; } - + // Ensure each node has the required properties data.nodes = data.nodes.map(node => ({ id: node.id, @@ -670,13 +670,13 @@ const TopologyManager = (function() { neighbors: node.neighbors || [], links: node.links || [] })); - + // Ensure each link has the required properties data.links = data.links.map(link => ({ source: link.source, target: link.target })); - + gData = data; updateGraph(); }, @@ -690,7 +690,7 @@ const TopologyManager = (function() { nodes: [], links: [] }; - // Update graph + // Update graph if (Graph) { Graph.graphData(gData); } diff --git a/nebula/frontend/static/js/deployment/ui-controls.js b/nebula/frontend/static/js/deployment/ui-controls.js index ed02efa76..97a76dce9 100644 --- a/nebula/frontend/static/js/deployment/ui-controls.js +++ b/nebula/frontend/static/js/deployment/ui-controls.js @@ -11,11 +11,11 @@ const UIControls = (function() { /* === control Physical + Predefined => block input === */ document.querySelectorAll('input[name="deploymentRadioOptions"]') .forEach(r => r.addEventListener('change', togglePredefinedNodesInput)); - + ['custom-topology-btn', 'predefined-topology-btn'] .forEach(id => document.getElementById(id) .addEventListener('change', togglePredefinedNodesInput)); - + togglePredefinedNodesInput(); setupVpnDiscover(); setupParticipantDisplay(); @@ -650,33 +650,33 @@ const UIControls = (function() { const radios = document.querySelectorAll('input[name="deploymentRadioOptions"]'); const discoverBtn = document.getElementById('discoverDevicesBtn'); if (!discoverBtn || !radios.length) return; - + const toggle = () => { const sel = document.querySelector('input[name="deploymentRadioOptions"]:checked'); discoverBtn.disabled = sel.value !== 'physical'; }; - + radios.forEach(r => r.addEventListener('change', toggle)); toggle(); } - + function setupVpnDiscover() { const discoverBtn = document.getElementById('discoverDevicesBtn'); if (!discoverBtn) return; - + discoverBtn.addEventListener('click', async () => { try { const res = await fetch('/platform/api/discover-vpn'); if (!res.ok) throw new Error(res.statusText); - + const { ips } = await res.json(); - + const form = document.getElementById('vpn-form'); form.innerHTML = ''; - + const currentScenario = window.ScenarioManager.getScenariosList()[window.ScenarioManager.getActualScenario()]; const selectedIPs = currentScenario?.physical_ips || []; - + ips.forEach(ip => { const wrapper = document.createElement('div'); wrapper.classList.add('form-check'); @@ -687,18 +687,18 @@ const UIControls = (function() { `; form.appendChild(wrapper); }); - + const modal = new bootstrap.Modal(document.getElementById('vpnModal')); modal.show(); - + document.getElementById('vpn-accept-btn').onclick = () => { const selected = Array.from(form.querySelectorAll('input:checked')) .map(i => i.value); - + window.ScenarioManager.setPhysicalIPs(selected); - + window.TopologyManager.setPhysicalIPs(selected); - + modal.hide(); }; } catch (err) { @@ -707,19 +707,19 @@ const UIControls = (function() { } }); } - + function togglePredefinedNodesInput() { const deployment = document.querySelector('input[name="deploymentRadioOptions"]:checked')?.value; const isPredefined = document.getElementById('predefined-topology-btn').checked; const nodesInput = document.getElementById('predefined-topology-nodes'); - + if (!nodesInput) return; - + const disable = deployment === 'physical' && isPredefined; nodesInput.disabled = disable; nodesInput.classList.toggle('disabled', disable); } - + function setupDeploymentRadios() { const radios = document.querySelectorAll('input[name="deploymentRadioOptions"]'); radios.forEach(radio => { diff --git a/nebula/physical/api.py b/nebula/physical/api.py index 6d44428e7..ac6b17ffe 100644 --- a/nebula/physical/api.py +++ b/nebula/physical/api.py @@ -379,8 +379,8 @@ def setup_new_run(): Expected multipart-form fields ------------------------------- - * **config** – JSON with scenario, network and security arguments - * **global_test** – shared evaluation dataset (`*.h5`) + * **config** – JSON with scenario, network and security arguments + * **global_test** – shared evaluation dataset (`*.h5`) * **train_set** – participant-specific training dataset (`*.h5`) The function rewrites paths inside *config*, validates neighbour IPs @@ -489,4 +489,4 @@ def setup_new_run(): # ----------------------------------------------------------------------------- if __name__ == "__main__": # Local testing: python main.py - app.run(host="0.0.0.0", port=8000, debug=False) \ No newline at end of file + app.run(host="0.0.0.0", port=8000, debug=False) diff --git a/nebula/physical/node.sh b/nebula/physical/node.sh index 6fe24f688..f7f2469ba 100644 --- a/nebula/physical/node.sh +++ b/nebula/physical/node.sh @@ -1,7 +1,7 @@ ############################################################################### # RUN NEBULA PHYSICAL NODE ───────────────────────────────────── ############################################################################### -VENV_DIR=".venv" +VENV_DIR=".venv" APP_PORT=8000 source "${VENV_DIR}/bin/activate" @@ -13,4 +13,4 @@ fi echo "· Launching Gunicorn (Flask) on port ${APP_PORT} …" export FLASK_APP=api.py -exec gunicorn -w 1 -b "0.0.0.0:${APP_PORT}" "api:app" \ No newline at end of file +exec gunicorn -w 1 -b "0.0.0.0:${APP_PORT}" "api:app" diff --git a/nebula/utils.py b/nebula/utils.py index cfc7a558b..9d7557968 100644 --- a/nebula/utils.py +++ b/nebula/utils.py @@ -174,7 +174,7 @@ def check_docker_by_prefix(cls, prefix): for container in containers: if container.name.startswith(prefix): return True - + return False except docker.errors.APIError: