spaghetti.yaml

# universal_tabular_route_conformal.yaml
# Goal: always-on prediction (reg + multiclass) with arbitrary NaNs up to 70%,
# route/reconstruction head for robustness + difficulty, split conformal calibration,
# and a case-based architecture selector.

experiment:
  name: "universal_route_conformal"
  seed: 42

data:
  # Generic schema; works for single-row samples or row-set/subtable samples.
  input_mode: "auto"            # auto | single_row | row_set
  target_types: ["regression", "multiclass"]  # enable either or both

  missingness:
    nan_value: "NaN"
    # This is an *inference/evaluation tolerance* target (how sparse a single row can be
    # while still expecting the model to function via mask-aware routing/reconstruction).
    # It is NOT a recommendation for how aggressively to corrupt data during training.
    max_missing_rate: 0.70
    fill_strategy: "zero"       # zero | mean | learned_token
    provide_mask: true          # always feed binary mask m to model

  splits:
    train_frac: 0.50
    calib_frac: 0.30            # <=10k points recommended for now
    test_frac: 0.20
    stratify_for_classification: true

training_augmentation:
  # Denoising-style training: additional random masking on top of naturally missing values.
  # Keep this moderate; masking away ~70% of observed values during training typically throws
  # away too much information and is not the intent of `data.missingness.max_missing_rate`.
  enable_random_masking: true
  masking_rate_schedule:
    type: "mixture_uniform"     # mixture_uniform | fixed | curriculum
    p_min: 0.0
    p_max: 0.50
  masking_pattern:
    type: "mixed"               # mixed | mcar | block | feature_biased
    mcar_prob: 0.60
    block_prob: 0.25
    feature_biased_prob: 0.15
  feature_biased:
    # Optional: if you have column missingness priors, plug them here; else ignored.
    use_empirical_column_missingness: true

model:
  # One trunk, three heads (reg + clf + route).
  use_column_id_embeddings: true
  column_id_embedding_dim: 16
  mask_embedding:
    type: "concat"              # concat | additive

  architecture_selector:
    type: "rule_based_auto"
    rules:
      # Case D: row-set/subtable input
      - if: { input_mode: "row_set" }
        encoder_case: "D_set_encoder"

      # Case C: very wide table
      - if: { feature_count_ge: 512 }
        encoder_case: "C_feature_token_transformer"

      # Case B: mixed types (if metadata exists)
      - if: { has_categorical_features: true }
        encoder_case: "B_mixed_embeddings"

      # Default Case A
      - else: {}
        encoder_case: "C_feature_token_transformer"

  encoders:
    A_mlp_baseline:
      type: "mlp"
      input_representation: ["x_filled", "mask", "col_id_emb_optional"]
      hidden_dims: [512, 256, 128]
      dropout: 0.10
      normalization: "layernorm"     # layernorm | batchnorm | none
      activation: "gelu"

    B_mixed_embeddings:
      type: "mixed_tabular"
      numeric:
        encoder: "linear"            # linear | small_mlp
        small_mlp_dims: [64, 64]
      categorical:
        embedding_dim_default: 32
        handle_oov: "bucket"
      fusion:
        type: "mlp"
        hidden_dims: [512, 256, 128]
        dropout: 0.10
        normalization: "layernorm"

    C_feature_token_transformer:
      type: "feature_token_transformer"
      token_components: ["value_emb", "col_id_emb", "mask_emb"]
      d_model: 256
      n_heads: 8
      n_layers: 4
      dropout: 0.10
      pooling: "cls"                 # cls | mean
      value_embedding:
        numeric: "linear"
        categorical: "embedding"

    D_set_encoder:
      # For inputs that are sets of rows/subtables. Each row is encoded with one of A/B/C.
      type: "set_encoder"
      row_encoder_case: "auto"       # auto | A_mlp_baseline | B_mixed_embeddings | C_feature_token_transformer
      set_aggregator:
        type: "deepsets"             # deepsets | set_transformer
        deepsets_pool: "mean"        # mean | sum | max
        set_transformer:
          d_model: 256
          n_heads: 8
          n_layers: 2
      output_dim: 256

  heads:
    regression:
      enabled: true
      type: "mlp_head"
      hidden_dims: [128, 64]
      out_dim: 1
      loss: "huber"                  # mse | huber
      huber_delta: 1.0

    multiclass:
      enabled: true
      type: "mlp_head"
      hidden_dims: [128, 64]
      num_classes: "auto"            # inferred from labels
      loss: "cross_entropy"
      label_smoothing: 0.0

    route:
      # Route head reconstructs features for robustness + difficulty score.
      enabled: true
      type: "mlp_reconstruction"
      hidden_dims: [256, 256]
      # route loss computed only on additionally-masked entries during training,
      # and difficulty computed only on observed entries at inference.
      loss:
        type: "nll"            # masked_l1 | masked_l2 | gaussian_nll
        weight: 1.0

difficulty:
  # Difficulty drives binning and optional adaptive conformal scaling.
  compute_from:
    - missing_rate
    - route_self_consistency_error
  route_error:
    norm: "l2"                       # l1 | l2
    on_entries: "observed_only"       # observed_only
  combine:
    type: "linear"
    weights:
      missing_rate: 1.0
      route_error: 0.5
  normalize:
    route_error: "robust_zscore"      # none | zscore | robust_zscore
  out_of_range_policy: "use_hardest_bin"  # use_hardest_bin

conformal:
  enabled: true
  method: "split"
  alpha: 0.1

  binning:
    type: "quantile_bins"
    num_bins: 1  # Disable bins on tiny Iris; use global threshold
    min_bin_size: 800                # if violated, merge bins until satisfied
    feature_for_bins: "difficulty"   # difficulty | missing_rate
  method: "split"
  alpha: 0.2

  binning:
    type: "quantile_bins"
    num_bins: 4
    min_bin_size: 800                # if violated, merge bins until satisfied
    feature_for_bins: "difficulty"   # difficulty | missing_rate

  regression:
    enabled: true
    score: "abs_residual"            # abs_residual
    adaptive_width:
      enabled: false                 # safety-first can be true, but start false for stability
      scale_from: "difficulty"
      g:
        type: "1_plus"               # 1_plus => g(d)=1+d (after normalization)
      epsilon: 1e-6
    output:
      always_return_point: true
      return_interval: true

  multiclass:
    enabled: true
    set_method: "RAPS"                # APS | RAPS
    aps:
      score: "cumprob_true_rank"     # cumulative probability up to true label rank
    raps:
      enabled: true
      lambda: 0.1
      k_reg: 5
    output:
      always_return_top1: true
      return_prediction_set: true

optimization:
  optimizer: "adamw"
  lr: 3e-4
  weight_decay: 1e-3
  batch_size: 256
  max_epochs: 50
  early_stopping:
    enabled: true
    metric: "calib_coverage_worst_bin"  # safety-first: track worst-bin behavior offline
    patience: 8
  pretraining:
    enabled: true
    epochs: 20

evaluation:
  offline_stress_tests:
    enabled: true
    # Evaluate coverage + set/interval size under synthetic missingness up to 70%.
    regimes:
      - { name: "clean", type: "none" }
      - { name: "mcar_10", type: "mcar", p: 0.10 }
      - { name: "mcar_30", type: "mcar", p: 0.30 }
      - { name: "mcar_50", type: "mcar", p: 0.50 }
      - { name: "mcar_70", type: "mcar", p: 0.70 }
      - { name: "block_50", type: "block", p: 0.50 }
      - { name: "feature_biased_50", type: "feature_biased", p: 0.50 }
      - { name: "nmar_high_50", type: "nmar", p: 0.50, quantile: 0.80, direction: "high" }
      - { name: "nmar_low_50", type: "nmar", p: 0.50, quantile: 0.20, direction: "low" }

  report:
    regression_metrics: ["coverage", "avg_interval_width", "worst_regime_coverage"]
    multiclass_metrics: ["coverage", "avg_set_size", "worst_regime_coverage"]
    by_bin: true