SBNSoftware · mvicenzi · May 22, 2025 · Feb 8, 2025 · Feb 8, 2025 · Apr 12, 2025
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -55,6 +55,7 @@ find_package(ifdhc REQUIRED )
 find_package(ifdh_art REQUIRED )
 find_package(Range-v3 REQUIRED )
 find_package(sbnobj REQUIRED )
+find_package(sbnalg REQUIRED )
 find_package(icarus_signal_processing REQUIRED )
 find_package(icarusalg REQUIRED )
 find_package(icarusutil REQUIRED )

diff --git a/fcl/SkipTesting.txt b/fcl/SkipTesting.txt
@@ -18,6 +18,8 @@ services_compat_no_overburden_icarus_v3.fcl
 services_compat_overburden_icarus_v3.fcl
 set_caf_calconst.fcl
 
+override_services_for_simulation.fcl
+
 
 ################################################################################
 # broken, won't fix

diff --git a/fcl/caf/cafmakerjob_icarus.fcl b/fcl/caf/cafmakerjob_icarus.fcl
@@ -22,6 +22,8 @@ services:
   SpaceChargeService: @local::icarus_spacecharge
 }
 
+#include "override_services_for_simulation.fcl"
+
 outputs:
 {
   outpid:

diff --git a/fcl/detsim/CMakeLists.txt b/fcl/detsim/CMakeLists.txt
@@ -1,5 +1,6 @@
 # add underlying subfolders
 add_subdirectory(SBNNoise)
+add_subdirectory(partial)
 add_subdirectory(commissioning)
 add_subdirectory(archive)
 

diff --git a/fcl/detsim/commissioning/run4642like_detsim_icarus.fcl b/fcl/detsim/commissioning/run4642like_detsim_icarus.fcl
@@ -25,7 +25,7 @@
 #
 
 #include "services_common_icarus.fcl"
-#include "opdetsim_pmt_icarus.fcl"
+#include "pmtsimulation_icarus.fcl"
 #include "rootoutput_icarus.fcl"
 
 process_name: DetSim4642
@@ -38,7 +38,7 @@ services: @local::icarus_common_services  # from services_common_icarus.fcl
 physics: {
   producers: {
     opdaq: {
-      @table::icarus_simpmt_noise  # from opdetsim_pmt_icarus.fcl
+      @table::icarus_pmtsimulationalg_202202_noise  # from pmtsimulation_icarus.fcl
 
       #
       # enabled readout window

diff --git a/fcl/detsim/commissioning/run4759like_detsim_icarus.fcl b/fcl/detsim/commissioning/run4759like_detsim_icarus.fcl
@@ -25,7 +25,7 @@
 #
 
 #include "services_common_icarus.fcl"
-#include "opdetsim_pmt_icarus.fcl"
+#include "pmtsimulation_icarus.fcl"
 #include "rootoutput_icarus.fcl"
 
 process_name: DetSim4759
@@ -38,7 +38,7 @@ services: @local::icarus_common_services  # from services_common_icarus.fcl
 physics: {
   producers: {
     opdaq: {
-      @table::icarus_simpmt_noise  # from opdetsim_pmt_icarus.fcl
+      @table::icarus_pmtsimulationalg_202202_noise  # from pmtsimulation_icarus.fcl
 
       #
       # enabled readout window

diff --git a/fcl/detsim/detsim_1d_icarus.fcl b/fcl/detsim/detsim_1d_icarus.fcl
@@ -99,3 +99,6 @@ services.Geometry.GDML: "icarus_complete_20220518_overburden.gdml"
 services.Geometry.ROOT: "icarus_complete_20220518_overburden.gdml"
 physics.producers.crtdaq.G4ModuleLabel: "largeant"
 physics.producers.opdaq.InputModule: "largeant"
+physics.producers.opdaq.ApplyTimingDelays:        false  # keep false for non-overlay MC
+physics.producers.opdaq.UseGainDatabase:          false  # keep false for non-overlay MC
+physics.producers.opdaq.UseChannelStatusDatabase: false  # keep false for non-overlay MC
diff --git a/fcl/detsim/detsim_2d_icarus.fcl b/fcl/detsim/detsim_2d_icarus.fcl
@@ -55,3 +55,6 @@ outputs: {
 services.Geometry.GDML: "icarus_complete_20220518_overburden.gdml"
 services.Geometry.ROOT: "icarus_complete_20220518_overburden.gdml"
 physics.producers.crtdaq.G4ModuleLabel: "shifted"
+physics.producers.opdaq.ApplyTimingDelays:        false  # keep false for non-overlay MC
+physics.producers.opdaq.UseGainDatabase:          false  # keep false for non-overlay MC
+physics.producers.opdaq.UseChannelStatusDatabase: false  # keep false for non-overlay MC
diff --git a/fcl/detsim/detsim_2d_icarus_refactored.fcl b/fcl/detsim/detsim_2d_icarus_refactored.fcl
@@ -53,5 +53,8 @@ outputs: {
 
 physics.producers.crtdaq.G4ModuleLabel: "shifted"
 physics.producers.opdaq.InputModule: "pdfastsim"
+physics.producers.opdaq.ApplyTimingDelays:        false  # keep false for non-overlay MC
+physics.producers.opdaq.UseGainDatabase:          false  # keep false for non-overlay MC
+physics.producers.opdaq.UseChannelStatusDatabase: false  # keep false for non-overlay MC
 physics.producers.shifted.InitAuxDetSimChannelLabel: "genericcrt"
 physics.producers.shifted.InitSimPhotonsLabel: "pdfastsim"
diff --git a/fcl/detsim/detsim_2d_icarus_refactored_Run4.fcl b/fcl/detsim/detsim_2d_icarus_refactored_Run4.fcl
@@ -0,0 +1,7 @@
+#include "detsim_2d_icarus_refactored.fcl"
+
+# Run3/4 optical tune
+physics.producers.opdaq: @local::icarus_simpmt_run4
+physics.producers.opdaq.ApplyTimingDelays:        false  # keep false for non-overlay MC
+physics.producers.opdaq.UseGainDatabase:          false  # keep false for non-overlay MC
+physics.producers.opdaq.UseChannelStatusDatabase: false  # keep false for non-overlay MC
diff --git a/fcl/detsim/detsim_2d_icarus_refactored_overlay.fcl b/fcl/detsim/detsim_2d_icarus_refactored_overlay.fcl
@@ -1,3 +1,10 @@
 #include "detsim_2d_icarus_refactored.fcl"
 
 physics.producers.daq: @local::icarus_simwire_wirecell_shifted_overlay
+
+# turn off mc noise on pmt waveforms 
+physics.producers.opdaq: @local::icarus_simpmt_nonoise
+physics.producers.opdaq.ApplyTimingDelays:        true
+physics.producers.opdaq.UseGainDatabase:          true
+physics.producers.opdaq.UseChannelStatusDatabase: true
+
diff --git a/fcl/detsim/detsim_2d_icarus_refactored_overlay_Run4.fcl b/fcl/detsim/detsim_2d_icarus_refactored_overlay_Run4.fcl
@@ -0,0 +1,7 @@
+#include "detsim_2d_icarus_refactored_overlay.fcl"
+
+# Run3/4 optical tune (with no noise)
+physics.producers.opdaq: @local::icarus_simpmt_run4_nonoise
+physics.producers.opdaq.ApplyTimingDelays:        true
+physics.producers.opdaq.UseGainDatabase:          true
+physics.producers.opdaq.UseChannelStatusDatabase: true
diff --git a/fcl/detsim/detsim_2d_icarus_refactored_yzsim.fcl b/fcl/detsim/detsim_2d_icarus_refactored_yzsim.fcl
@@ -62,6 +62,9 @@ outputs: {
 
 physics.producers.crtdaq.G4ModuleLabel: "shifted"
 physics.producers.opdaq.InputModule: "pdfastsim"
+physics.producers.opdaq.ApplyTimingDelays:        false  # keep false for non-overlay MC
+physics.producers.opdaq.UseGainDatabase:          false  # keep false for non-overlay MC
+physics.producers.opdaq.UseChannelStatusDatabase: false  # keep false for non-overlay MC
 physics.producers.shifted.InitAuxDetSimChannelLabel: "genericcrt"
 physics.producers.shifted.InitSimPhotonsLabel: "pdfastsim"
 physics.producers.filtersed.InitSimEnergyDepositLabel: "shifted"

diff --git a/fcl/detsim/detsim_2d_icarus_refactored_yzsim_Run4.fcl b/fcl/detsim/detsim_2d_icarus_refactored_yzsim_Run4.fcl
@@ -1,3 +1,9 @@
 #include "detsim_2d_icarus_refactored_yzsim.fcl"
 
 physics.producers.daq.wcls_main.params.YZScaleMapJson: "yzmap_gain_icarus_v4_run4.json"
+
+# Run3/4 optical tune
+physics.producers.opdaq: @local::icarus_simpmt_run4
+physics.producers.opdaq.ApplyTimingDelays:        false  # keep false for non-overlay MC
+physics.producers.opdaq.UseGainDatabase:          false  # keep false for non-overlay MC
+physics.producers.opdaq.UseChannelStatusDatabase: false  # keep false for non-overlay MC
diff --git a/fcl/detsim/detsim_2d_icarus_refactored_yzsim_notrigger.fcl b/fcl/detsim/detsim_2d_icarus_refactored_yzsim_notrigger.fcl
@@ -57,3 +57,6 @@ outputs: {
 }
 
 physics.producers.opdaq.InputModule: "pdfastsim"
+physics.producers.opdaq.ApplyTimingDelays:        false  # keep false for non-overlay MC
+physics.producers.opdaq.UseGainDatabase:          false  # keep false for non-overlay MC
+physics.producers.opdaq.UseChannelStatusDatabase: false  # keep false for non-overlay MC
diff --git a/fcl/detsim/detsim_2d_icarus_refactored_yzsim_notrigger_overlay.fcl b/fcl/detsim/detsim_2d_icarus_refactored_yzsim_notrigger_overlay.fcl
@@ -1,3 +1,9 @@
 #include "detsim_2d_icarus_refactored_yzsim_notrigger.fcl"
 
 physics.producers.daq: @local::icarus_simwire_wirecell_yz_overlay
+
+# turn off mc noise on pmt waveforms 
+physics.producers.opdaq: @local::icarus_simpmt_nonoise
+physics.producers.opdaq.ApplyTimingDelays:        true
+physics.producers.opdaq.UseGainDatabase:          true
+physics.producers.opdaq.UseChannelStatusDatabase: true
diff --git a/fcl/detsim/detsim_2d_icarus_refactored_yzsim_overlay.fcl b/fcl/detsim/detsim_2d_icarus_refactored_yzsim_overlay.fcl
@@ -1,3 +1,9 @@
 #include "detsim_2d_icarus_refactored_yzsim.fcl"
 
 physics.producers.daq: @local::icarus_simwire_wirecell_yz_overlay
+
+# turn off mc noise on pmt waveforms 
+physics.producers.opdaq: @local::icarus_simpmt_nonoise
+physics.producers.opdaq.ApplyTimingDelays:        true
+physics.producers.opdaq.UseGainDatabase:          true
+physics.producers.opdaq.UseChannelStatusDatabase: true
diff --git a/fcl/detsim/detsim_2d_icarus_refactored_yzsim_overlay_Run4.fcl b/fcl/detsim/detsim_2d_icarus_refactored_yzsim_overlay_Run4.fcl
@@ -1,3 +1,9 @@
 #include "detsim_2d_icarus_refactored_yzsim_overlay.fcl"
 
 physics.producers.daq.wcls_main.params.YZScaleMapJson: "yzmap_gain_icarus_v4_run4.json"
+
+# Run3/4 optical tune (with no noise)
+physics.producers.opdaq: @local::icarus_simpmt_run4_nonoise
+physics.producers.opdaq.ApplyTimingDelays:        true
+physics.producers.opdaq.UseGainDatabase:          true
+physics.producers.opdaq.UseChannelStatusDatabase: true
diff --git a/fcl/detsim/partial/CMakeLists.txt b/fcl/detsim/partial/CMakeLists.txt
@@ -0,0 +1,8 @@
+# Install fcl files in /job subdirectory.
+install_fhicl()
+
+# Also put a copy in the source tree.
+
+FILE(GLOB fcl_files *.fcl)
+install_source( EXTRAS ${fcl_files} )
+
diff --git a/fcl/detsim/partial/detsim_opdetonly_icarus.fcl b/fcl/detsim/partial/detsim_opdetonly_icarus.fcl
@@ -0,0 +1,40 @@
+#
+# File:    detsim_opdetonly_icarus.fcl
+# Purpose: Simulation of the optical detector response.
+# Date:    January 27, 2025
+# Author:  Gianluca Petrillo (petrillo@slac.stanford.edu)
+#
+# This job configuration runs the standard detector simulation, like
+# `detsim_1d_icarus.fcl` does, but only for the optical detector.
+#
+#
+# Output
+# -------
+#
+# * `std::vector<raw::OpDetWaveform>` (`opdaq`): raw PMT waveforms
+#
+#
+#
+#
+# Changes
+# --------
+#
+# 20250127 (petrillo@slac.stanford.edu) [v1.0]
+#   original version (from `detsim_1d_icarus.fcl`)
+#
+
+#include "detsim_1d_icarus.fcl"
+
+process_name: OpDetSim
+
+services.DetPedestalService:         @erase
+services.ChannelStatusService:       @erase
+services.IICARUSChannelMap:          @erase
+services.SignalShapingICARUSService: @erase
+
+# run only the `opdaq` module:
+physics.producers: {
+  opdaq: @local::physics.producers.opdaq  # simplify configuration removing all other producers
+}
+physics.simulate: [ opdaq ]
+
diff --git a/fcl/detsim/partial/detsim_opdetonly_icarus_overlay_reprocessing_run2.fcl b/fcl/detsim/partial/detsim_opdetonly_icarus_overlay_reprocessing_run2.fcl
@@ -0,0 +1,150 @@
+## File:    stage0_run2_opdetonly_icarus_data_reprocessing.fcl
+## Purpose: Reprocessing of the optical products for the 2025 SBN "spring" OVERLAY MC production
+
+#include "detsim_2d_icarus_refactored.fcl"
+
+## This job selectively re-runs only the PMT/CRT DetSim path on top of an existing file.
+## This is has been put together only for the post-processing of the 2025 SBN MC OVERLAY
+## productions, and assumes to be starting with its mcstage1 files.
+## It is valid only for RUN-2 productions because it selects the Run2 DetSim tune.
+
+process_name: OpDetSim
+
+services.DetPedestalService:         @erase
+services.ChannelStatusService:       @erase
+services.SignalShapingICARUSService: @erase
+
+source.inputCommands: [
+  "keep *",
+
+  # `stage0` comes from decoding data: should be kept!
+  # `MCstage0` comes running stage0 on the overlay products:
+  #  so pmt/trigger/crt stuff should be thrown out!
+  # note: I would really like to drop these things as well:
+  # "drop raw::OpDetWaveform*_opdaq_*_*",    # drop old MC waveforms (if any)
+  # "drop *_pmtfixedthrinit_*_*",        # drop triggersim product
+  # "drop *_pmtlvdsgatesinit_*_*",       # drop triggersim product
+  # "drop *_pmttriggerwindowsinit_*_*",  # drop triggersim product
+  # "drop *_triggersimgatesinit_*_*",    # drop triggersim product
+  # "drop *_emuTriggerUnshifted_*_*",    # drop triggersim product
+  # but I can't: it triggers art genocidal insticts thus removing all
+  # downstream products in the dependency tree that we still need.
+
+  "drop raw::OpDetWaveform*_shifted_*_*",  # drop old MC (shifted) waveforms (if any)
+  "drop *_overlayOpWaveforms_*_*",         # drop old Overlay waveforms (if any)
+  "drop *_ophit*_*_*",                     # drop MCstage0 ophits
+  "drop *_mcophit_*_*",                    # drop MCstageo mcophit
+  "drop *_opflash*_*_*",                   # drop MCsstage0 east/west opflashes
+
+  "drop *_pmtfixedthr_*_*",            # drop MCstage0 trigger product
+  "drop *_pmtlvdsgates_*_*",           # drop MCstage0 trigger product
+  "drop *_pmtlvdsgates_*_*",           # drop MCstage0 trigger product
+  "drop *_pmtbaselines_*_MCstage0",    # drop MCstage0 baselines, not stage0!
+  "drop *_pmttriggerwindows_*_*",      # drop MCstage0 trigger product
+  "drop *_emuTrigger_*_*",             # drop MCstage0 trigger product
+
+  "drop *_crtdaq_*_*",        # drop CRT detsim product
+  "drop *_mccrthit_*_*",      # drop CRT mc hits
+  "drop *_overlayCRTHit_*_*", # drop CRT overlay hits
+  "drop *_crttrack_*_*",      # drop CRT tracks (MCstage0)
+  "drop *_crtpmt_*_*"         # drop CRTPMT matches (MCstage0)
+]
+
+# Running only the optical/trigger/crt modules
+# Simplify configuration removing all other producers (tpc)
+
+physics.producers: {
+
+  # opdaq Run-1/2 tune for overlays (no noise)
+  opdaq: @local::icarus_simpmt_run2_nonoise 
+
+  # triggersim producers 
+  @table::icarus_shifting_triggersim.producers
+
+  # crt hits
+  crtdaq: @local::icarus_crtsim
+
+}
+
+# since some "old" products need to be kept in the tree
+# we have products with the same tag, and different process_name
+# make sure to select exactly what you need! 
+
+# build waveforms from simPhotons (already shifted)
+physics.producers.opdaq.InputModule: "shifted::DetSim"
+physics.producers.opdaq.ApplyTimingDelays:        true
+physics.producers.opdaq.UseGainDatabase:          true
+physics.producers.opdaq.UseChannelStatusDatabase: true
+
+# make sure the triggersim chain used the "new" products
+# instead of the "old" ones which I cannot drop on input
+physics.producers.pmtfixedthrinit.OpticalWaveforms: "opdaq::OpDetSim"                    # this was re-generated with same tag
+physics.producers.pmtlvdsgatesinit.TriggerGatesTag: "pmtfixedthrinit::OpDetSim"          # this was re-generated with same tag
+physics.producers.pmttriggerwindowsinit.TriggerGatesTag: "pmtlvdsgatesinit::OpDetSim"    # this was re-generated with same tag
+physics.producers.triggersimgatesinit.BeamGateTag: "beamgate"                            # keep using old Gen product
+physics.producers.emuTriggerUnshifted.BeamGates: "triggersimgatesinit::OpDetSim"         # this was re-generated with same tag
+physics.producers.emuTriggerUnshifted.TriggerGatesTag: "pmttriggerwindowsinit::OpDetSim" # this was re-generated with same tag
+
+# need to update the shifting module to pick up new labels
+# some were regenerated (tag still good), others were previously shifted
+# applying a second shift is okay: everything is consistent
+# (eg. waveforms made from `shifted` photons)
+# NOTE: still shifing energy deposits although tpc info not regenerated
+# however forced to use `filtersed` as `shifted` has been consumed
+
+physics.producers.shifted.InputTriggerLabel: "emuTriggerUnshifted::OpDetSim"       # this was re-generated with same tag
+physics.producers.shifted.InitAuxDetSimChannelLabel: "shifted::DetSim"             # need to start from previously-shifted, not `genericcrt`
+physics.producers.shifted.InitBeamGateInfoLabel: "triggersimgatesinit::OpDetSim"   # this was re-generated with same tag
+physics.producers.shifted.InitSimEnergyDepositLabel: "filtersed::DetSim"           # need to start from previously-shifted, but `shifted` no longer available
+physics.producers.shifted.InitSimEnergyDepositLiteLabel: "filtersed::DetSim"       # need to start from previously-shifted, but `shifted` no longer available
+physics.producers.shifted.InitSimPhotonsLabel: "shifted::DetSim"                   # need to start from previously-shifted, not `pdfastsim`
+physics.producers.shifted.InitWaveformLabel: "opdaq::OpDetSim"                     # this was re-generated with same tag
+
+# need to update shiting of priorSCE deposits:
+# this is important if future reprocessings will need to regenerate SimPhotons
+
+physics.producers.shiftedpriorSCE.InputTriggerLabel: "emuTriggerUnshifted::OpDetSim"      # this was re-generated with same tag
+physics.producers.shiftedpriorSCE.InitSimEnergyDepositLabel: "shiftedpriorSCE::DetSim"  # need to start from previously-shifted, not `ionization:priorSCE`
+
+# and finally make sure the crtsim also uses the newly shifted products
+physics.producers.crtsim.G4ModuleLabel: "shifted::OpDetSim"
+
+# The reprocessing flow is the following:
+# - build new pmt waveforms from photons
+# - simulate the trigger
+# - shift all products to the trigger:
+#   -- we are shifting photons, pmt waveforms, energy depositis, aux sim channels
+#   -- note: since tpc is not re-simulated, no longer fully coherent with trigger time
+# - build CRT data from aux sim channels
+
+physics.simulate: [ opdaq, @sequence::icarus_shifting_triggersim.path, crtdaq ]
+
+
+# drop duplicate produtcs in output
+# this doesn't trigger art genocidal instincts
+
+outputs.rootoutput.outputCommands: [
+  "keep *",
+
+  # drop `opdaq`, but keep `shifted` for overlaying
+  "drop *_opdaq_*_*",
+
+  # drop old trigger products now that new ones exist
+  # specify the old process name "DetSim"
+  "drop *_pmtfixedthrinit_*_DetSim",        
+  "drop *_pmtlvdsgatesinit_*_DetSim",
+  "drop *_pmttriggerwindowsinit_*_DetSim",  
+  "drop *_triggersimgatesinit_*_DetSim",    
+  "drop *_emuTriggerUnshifted_*_DetSim", 
+
+  # drop old "shifted" products since we shifted things again
+  "drop sim::SimEnergyDeposits_filtersed_*_DetSim",        # this has been regenerated into `shifted`
+  # mcreco looks for `filtersed` in stage1: potentially can be changed to look for `shifted`?
+  # however since tpc reco products are not shifted, better to keep old deposits for that matching
+  # "drop sim::SimEnergyDepositLites_filtersed_*_DetSim",    # this has been regenerated into `shifted`
+  "drop sim::SimEnergyDeposits_shiftedpriorSCE_*_DetSim",  # this has been regenerated into the same tag
+  "drop sim::AuxDetSimChannel_shifted_*_DetSim",           # this has been regenerated into the same tag
+  "drop sim::SimPhotons_shifted_*_DetSim",                 # this has been regenerated into the same tag
+  "drop sim::BeamGateInfo_shifted_*_DetSim"                # this has been regenerated into the same tag
+
+]
diff --git a/fcl/detsim/partial/detsim_opdetonly_icarus_overlay_reprocessing_run4.fcl b/fcl/detsim/partial/detsim_opdetonly_icarus_overlay_reprocessing_run4.fcl
@@ -0,0 +1,16 @@
+## File:    stage0_run4_opdetonly_icarus_data_reprocessing.fcl
+## Purpose: Reprocessing of the optical products for the 2025 SBN "spring" OVERLAY MC production
+
+#include "detsim_opdetonly_icarus_overlay_reprocessing_run2.fcl"
+
+## This job selectively re-runs only the PMT/CRT DetSim path on top of an existing file.
+## This is has been put together only for the post-processing of the 2025 SBN MC OVERLAY
+## productions, and assumes to be starting with its mcstage1 files.
+## It is valid only for RUN-3/RUN-4 productions because it selects their common DetSim tune.
+
+# switch to Run3/Run-4 tune
+physics.producers.opdaq: @local::icarus_simpmt_run4_nonoise
+physics.producers.opdaq.InputModule: "shifted::DetSim"
+physics.producers.opdaq.ApplyTimingDelays:        true
+physics.producers.opdaq.UseGainDatabase:          true
+physics.producers.opdaq.UseChannelStatusDatabase: true
-Original file line number
+Diff line change
@@ Expand Up / @@ -22,6 +22,8 @@ services: @@
       SpaceChargeService: @local::icarus_spacecharge
     }
+    #include "override_services_for_simulation.fcl"
     outputs:
     {
       outpid:
@@ Expand Down @@