From 8afb2634be004f9fa71506d88e230e769f389d88 Mon Sep 17 00:00:00 2001
From: klendathu2k <jwebb@bnl.gov>
Date: Thu, 23 Jan 2025 14:19:04 -0500
Subject: [PATCH 01/25] Start by committing the ana451 2024p009 production
 file.

---
 ...GGERED_EVENT_run2auau_ana451_2024p009.yaml | 350 ++++++++++++++++++
 1 file changed, 350 insertions(+)
 create mode 100644 run2auau/DST_TRIGGERED_EVENT_run2auau_ana451_2024p009.yaml

diff --git a/run2auau/DST_TRIGGERED_EVENT_run2auau_ana451_2024p009.yaml b/run2auau/DST_TRIGGERED_EVENT_run2auau_ana451_2024p009.yaml
new file mode 100644
index 0000000..d1623fb
--- /dev/null
+++ b/run2auau/DST_TRIGGERED_EVENT_run2auau_ana451_2024p009.yaml
@@ -0,0 +1,350 @@
+#
+# Testbed for file ranges
+#
+DST_TRIGGERED_EVENT_run2auau_ana451_2024p009:
+   params:
+     name:       DST_TRIGGERED_EVENT_run2auau
+     build:      ana.451
+     build_name: ana451
+     dbtag:      2024p009
+     logbase :   $(name)_$(build)_$(tag)-$INT(run,{RUNFMT})-$INT(seg,{SEGFMT})
+     outbase :   $(name)_$(build)_$(tag)
+     script  :   run.sh
+     payload :   ./ProdFlow/run2auau/TriggerProduction/
+     mem     :   2048M
+     neventsper: 5000
+     noverflow:  1000
+     rsync  : "ProdFlow/run2auau/TriggerProduction/*,cups.py,bachi.py,odbc.ini"
+
+
+
+   #
+   # input query:
+   #
+   # This builds a list of all runs known to the file catalog "datasets" table.
+   # The query should return:
+   # 1. The source of the information (formatted as database name/table name)
+   # 2. The run number
+   # 3. A sequence number (a placeholder fixed at zero for event builders)
+   # 4. And a space-separated list of logical filenames 
+   #
+   # The {*_condition} parameters (run, seg, limit) are substituted by kaedama
+   # based on (optional) command line options.
+   #
+   # For now, require GL1 files...
+   #
+   input:
+      db: daqdb
+      direct_path: /sphenix/lustre01/sphnxpro/physics/*/*/
+      query: |-
+         with config as (
+            select 
+               {neventsper} as nevents,
+               {noverflow} as noverflow
+         ),
+
+         fullevents as (
+              select runnumber,max(lastevent)    as lastevent
+              from filelist, config
+              where (filename similar to '/bbox%/GL1_(beam|physics)%.evt'        and lastevent>2 )
+
+                 {run_condition} and runnumber>54748
+
+              group by runnumber
+              having
+                     every(transferred_to_sdcc)   and
+                     max(lastevent)>1000          and
+                     sum( case when filename similar to '/bbox%/GL1_(beam|physics)%' then 1 else 0 end )>0
+
+              order by runnumber
+         ),
+
+         fulljobs as (
+         select 
+                'daqdb/filelist'                                                                                    as source      , 
+                runnumber                                                                                                          , 
+                0                                                                                                   as segment     , 
+                string_agg( split_part(filename,'/',-1), ' ' order by filename)                                             as files       ,   
+                string_agg( firstevent::text || ':' || lastevent::text || ':' || split_part(filename,'/',-1), ' ' order by filename )                    as fileranges  
+
+         from  
+                filelist,config
+         where 
+           ( 
+             (filename  similar to '/bbox%/%emcal%(beam|physics)%.prdf'    and lastevent>2 ) or 
+             (filename  similar to '/bbox%/%HCal%(beam|physics)%.prdf'     and lastevent>2 ) or
+             (filename  similar to '/bbox%/%LL1%(beam|physics)%.prdf'      and lastevent>2 ) or
+             (filename  similar to '/bbox%/GL1_(beam|physics)%.evt'       and lastevent>2 ) or
+             (filename  similar to '/bbox%/%mbd%(beam|physics)%.prdf'      and lastevent>2 ) or
+             (filename  similar to '/bbox%/%ZDC%(beam|physics)%.prdf'      and lastevent>2 ) 
+
+           )
+
+           {run_condition}
+
+                 and runnumber>53880
+
+         group by runnumber
+
+         having
+                every(transferred_to_sdcc)   and
+                max(lastevent)>1000          and
+                sum( case when filename similar to '/bbox%/GL1_(beam|physics)%' then 1 else 0 end )>0 and
+                (
+                   sum( case when filename similar to '/bbox%/%emcal%(beam|physics)%'  then 1 else 0 end )>0 or
+                   sum( case when filename similar to '/bbox%/%HCal%(beam|physics)%'   then 1 else 0 end )>0 
+                )
+
+         order by runnumber
+         ),
+
+         fullrun as (
+
+             select fulljobs.*,fullevents.lastevent from fulljobs join fullevents on fulljobs.runnumber=fullevents.runnumber
+
+         ),
+
+         unsegmented as ( 
+
+             select *,'full run' as runtype from fullrun where true 
+         ),
+
+         segmented as (
+
+            select source, 
+                   runnumber,
+                   generate_series(0,unsegmented.lastevent/config.nevents) as segment,
+                   files,
+                   fileranges,
+                   lastevent,
+                   nevents
+
+             from unsegmented,config
+
+         ),
+
+         segmentedA as ( 
+
+         select source,
+                runnumber,
+                segment,
+                files,
+                fileranges,
+                lastevent, 
+                segment*config.nevents as myfirstevent,
+                least( (segment+1)*config.nevents - 1, lastevent ) as mylastevent,
+                config.nevents 
+ 
+                from segmented,config
+
+          ),
+
+
+          segmentedB as (
+
+               select *,
+                      mylastevent-myfirstevent as nprocessing,
+                      lastevent-mylastevent-1  as nremaining from segmentedA,config  -- b/c we count from zero
+       
+          ),
+
+          segmentedC as (
+
+
+               select source, runnumber,
+                      segment,
+                      files,
+                      fileranges,
+                      myfirstevent,
+                      (
+                          select 
+                             case
+                                when nprocessing<config.noverflow  then -1
+                                when nremaining>=config.nevents    then mylastevent
+                                when nremaining>=config.noverflow  then mylastevent
+                                when nremaining<config.noverflow   then lastevent
+                                else                                    -2             -- unhandled, should be an error condition
+                             end
+                      ) as mylastevent,
+                      lastevent,
+                      nprocessing,
+                      nremaining
+
+                      from segmentedB,config
+
+          )
+
+
+          select 
+               source,
+                 runnumber,
+                 segment,
+                 files,
+                 fileranges,
+                 
+                 (select count(*) from regexp_matches( files,      ' ', 'g' ))::integer =
+                 (select count(*) from regexp_matches( fileranges, ' ', 'g' ))::integer as sanity,
+                 myfirstevent as firstevent,
+                 mylastevent  as lastevent,
+                 lastevent    as runs_last_event
+
+                 from segmentedC
+
+                 where runnumber>53880
+
+          order by runnumber desc, segment desc;
+                 
+
+   #
+   # Again I note the need to ensure that the arguments are properly specified given the
+   # definition of the payload script.
+   #
+   job:
+     executable            : "{payload}/run.sh"
+     arguments             : "$(nevents) {outbase} {logbase} $(run) $(seg) $(outdir) $(buildarg) $(tag) $(inputs) $(ranges) {neventsper} {logdir} {histdir} {PWD} {rsync} $(firstevent) $(lastevent) $(runs_last_event)"
+     output_destination    : '{logdir}'
+     log                   : '{condor}/{logbase}.condor'
+     accounting_group      : "group_sphenix.mdc2"
+     accounting_group_user : "sphnxpro"
+     priority : '3900'
+     request_xferslots: '1'
+
+
+
+
+
+# Downstream products
+DST_CALOFITTING_run2auau_ana451_2024p009:
+   params:
+     name:       DST_CALOFITTING_run2auau
+     nevents:    5000
+     build:      ana.451
+     build_name: ana451
+     dbtag:      2024p009
+     logbase :   $(name)_$(build)_$(tag)-$INT(run,{RUNFMT})-$INT(seg,{SEGFMT})
+     outbase :   $(name)_$(build)_$(tag)
+     script  :   runy2fitting.sh
+     payload :   ./ProdFlow/run2auau/CaloProduction/
+     mem     :   1024MB
+     neventsper: 50000
+     rsync  : "./ProdFlow/run2auau/CaloProduction/*,cups.py,bachi.py,odbc.ini"
+
+   input:
+     db: filecatalog
+     query: |-
+         select 
+                'filecatalog/datasets'                                  as source      , 
+                runnumber                                                              , 
+                segment                                                                ,
+                filename                                                as files       ,
+                filename || ':' || 0 || ':' || events                   as fileranges 
+         from  
+                datasets
+         where 
+                filename like 'DST_TRIGGERED_EVENT_run2auau_ana451_2024p009%'
+                {run_condition}
+
+                 and runnumber>53880
+           
+         {limit_condition}
+
+         ;
+   job:
+     executable            : "{payload}/runy2fitting.sh"
+     arguments             : "{nevents} {outbase} {logbase} $(run) $(seg) $(outdir) $(buildarg) $(tag) $(inputs) $(ranges) {neventsper} {logdir} {histdir} {PWD} {rsync}"
+     output_destination    : '{logdir}'
+     log                   : '{condor}/{logbase}.condor'
+     accounting_group      : "group_sphenix.mdc2"
+     accounting_group_user : "sphnxpro"
+     priority : '3900'
+
+
+
+# Downstream products
+DST_CALO_run2auau_new_2024p007:
+   params:
+     name:       DST_CALO_run2auau
+     build:      new
+     build_name: new
+     dbtag:      2024p007
+     logbase :   $(name)_$(build)_$(tag)-$INT(run,{RUNFMT})-$INT(seg,{SEGFMT})
+     outbase :   $(name)_$(build)_$(tag)
+     script  :   runy2calib.sh
+     payload :   ./ProdFlow/run2auau/CaloProduction/
+     mem     :   4096MB
+     neventsper: 50000
+     rsync  : "./ProdFlow/run2auau/CaloProduction/*,cups.py,bachi.py,odbc.ini"
+
+   input:
+     db: filecatalog
+     query: |-
+         select 
+                'filecatalog/datasets'                                  as source      , 
+                runnumber                                                              , 
+                segment                                                                ,
+                filename                                                as files       ,
+                filename || ':' || 0 || ':' || events                   as fileranges 
+         from  
+                datasets
+         where 
+                filename like 'DST_CALOFITTING_run2auau_new_2024p007%'
+                {run_condition}
+
+                 and runnumber>53880
+           
+         {limit_condition}
+
+         ;
+   job:
+     executable            : "{payload}/runy2calib.sh"
+     arguments             : "$(nevents) {outbase} {logbase} $(run) $(seg) $(outdir) $(buildarg) $(tag) $(inputs) $(ranges) {neventsper} {logdir} {histdir} {PWD} {rsync}"
+     output_destination    : '{logdir}'
+     log                   : '{condor}/{logbase}.condor'
+     accounting_group      : "group_sphenix.mdc2"
+     accounting_group_user : "sphnxpro"
+     priority : '3900'
+
+
+# Downstream products
+DST_JETS_run2auau_new_2024p007:
+   params:
+     name:       DST_JETS_run2auau
+     build:      new
+     build_name: new
+     dbtag:      2024p007
+     logbase :   $(name)_$(build)_$(tag)-$INT(run,{RUNFMT})-$INT(seg,{SEGFMT})
+     outbase :   $(name)_$(build)_$(tag)
+     script  :   runjets.sh
+     payload :   ./ProdFlow/run2auau/JetProduction/
+     mem     :   2096MB
+     neventsper: 50000
+     rsync  : "./ProdFlow/run2auau/JetProduction/*,cups.py,bachi.py,odbc.ini"
+
+   input:
+     db: filecatalog
+     query: |-
+         select 
+                'filecatalog/datasets'                                  as source      , 
+                runnumber                                                              , 
+                segment                                                                ,
+                filename                                                as files       ,
+                'X'                   as fileranges 
+         from  
+                datasets
+         where 
+                filename like 'DST_CALO_run2auau_new_2024p007%'
+                {run_condition}
+                and runnumber>53880
+           
+         {limit_condition}
+
+         ;
+   job:
+     executable            : "{payload}/runjets.sh"
+     arguments             : "$(nevents) {outbase} {logbase} $(run) $(seg) $(outdir) $(buildarg) $(tag) $(inputs) $(ranges) {neventsper} {logdir} {histdir} {PWD} {rsync}"
+     output_destination    : '{logdir}'
+     log                   : '{condor}/{logbase}.condor'
+     accounting_group      : "group_sphenix.mdc2"
+     accounting_group_user : "sphnxpro"
+     priority : '3900'
+

From eb38d3facf8a75608665831e06ac7844e9d1485c Mon Sep 17 00:00:00 2001
From: klendathu2k <jwebb@bnl.gov>
Date: Thu, 23 Jan 2025 16:09:41 -0500
Subject: [PATCH 02/25] Runs DST_CALO_FITTING off of the ana451 2024p010
 unversioned DST_TRIGGERED_EVENT production.  Fitting and downstream products
 will be run in ana.462 against cdbtag 2024p010.

---
 ...GGERED_EVENT_run2auau_ana462_2024p010.yaml | 352 ++++++++++++++++++
 1 file changed, 352 insertions(+)
 create mode 100644 run2auau/DST_TRIGGERED_EVENT_run2auau_ana462_2024p010.yaml

diff --git a/run2auau/DST_TRIGGERED_EVENT_run2auau_ana462_2024p010.yaml b/run2auau/DST_TRIGGERED_EVENT_run2auau_ana462_2024p010.yaml
new file mode 100644
index 0000000..8b1efea
--- /dev/null
+++ b/run2auau/DST_TRIGGERED_EVENT_run2auau_ana462_2024p010.yaml
@@ -0,0 +1,352 @@
+# Downstream products
+DST_CALOFITTING_run2auau_ana462_2024p010:
+   params:
+     name:       DST_CALOFITTING_run2auau
+     nevents:    5000
+     build:      ana.462
+     build_name: ana462
+     dbtag:      2024p010
+     version : 1
+     logbase :   $(name)_$(build)_$(tag)_$(version)-$INT(run,{RUNFMT})-$INT(seg,{SEGFMT})
+     outbase :   $(name)_$(build)_$(tag)_$(version)
+     script  :   runy2fitting.sh
+     payload :   ./ProdFlow/run2auau/CaloProduction/
+     mem     :   1024MB
+     neventsper: 50000
+     rsync  : "./ProdFlow/run2auau/CaloProduction/*,cups.py,bachi.py,odbc.ini"
+
+   input:
+     db: filecatalog
+     query: |-
+         select 
+                'filecatalog/datasets'                                  as source      , 
+                runnumber                                                              , 
+                segment                                                                ,
+                filename                                                as files       ,
+                filename || ':' || 0 || ':' || events                   as fileranges 
+         from  
+                datasets
+         where 
+                filename like 'DST_TRIGGERED_EVENT_run2auau_ana451_2024p009%'
+                {run_condition}
+
+                 and runnumber>53880
+           
+         {limit_condition}
+
+         ;
+   job:
+     arguments             : "{nevents} {outbase} {logbase} $(run) $(seg) $(outdir) $(buildarg) $(tag) $(inputs) $(ranges) {neventsper} {logdir} {histdir} {PWD} {rsync}"
+     output_destination    : '{logdir}'
+     log                   : '{condor}/{logbase}.condor'
+     accounting_group      : "group_sphenix.mdc2"
+     accounting_group_user : "sphnxpro"
+     priority : '3900'
+
+
+
+# Downstream products
+DST_CALO_run2auau_new_2024p010:
+   params:
+     name:       DST_CALO_run2auau
+     build:      ana.462
+     build_name: ana462
+     dbtag:      2024p010
+     version : 1
+     logbase :   $(name)_$(build)_$(tag)_$(version)-$INT(run,{RUNFMT})-$INT(seg,{SEGFMT})
+     outbase :   $(name)_$(build)_$(tag)_$(version)
+     script  :   runy2calib.sh
+     payload :   ./ProdFlow/run2auau/CaloProduction/
+     mem     :   4096MB
+     neventsper: 50000
+     rsync  : "./ProdFlow/run2auau/CaloProduction/*,cups.py,bachi.py,odbc.ini"
+
+   input:
+     db: filecatalog
+     query: |-
+         select 
+                'filecatalog/datasets'                                  as source      , 
+                runnumber                                                              , 
+                segment                                                                ,
+                filename                                                as files       ,
+                filename || ':' || 0 || ':' || events                   as fileranges 
+         from  
+                datasets
+         where 
+                filename like 'DST_CALOFITTING_run2auau_ana462_2024p010%'
+                {run_condition}
+
+                 and runnumber>53880
+           
+         {limit_condition}
+
+         ;
+   job:
+     arguments             : "$(nevents) {outbase} {logbase} $(run) $(seg) $(outdir) $(buildarg) $(tag) $(inputs) $(ranges) {neventsper} {logdir} {histdir} {PWD} {rsync}"
+     output_destination    : '{logdir}'
+     log                   : '{condor}/{logbase}.condor'
+     accounting_group      : "group_sphenix.mdc2"
+     accounting_group_user : "sphnxpro"
+     priority : '3900'
+
+
+# Downstream products
+DST_JETS_run2auau_new_2024p010:
+   params:
+     name:       DST_JETS_run2auau
+     build:      ana.462
+     build_name: ana462
+     dbtag:      2024p010
+     version : 1
+     logbase :   $(name)_$(build)_$(tag)_$(version)-$INT(run,{RUNFMT})-$INT(seg,{SEGFMT})
+     outbase :   $(name)_$(build)_$(tag)_$(version)
+     script  :   runjets.sh
+     payload :   ./ProdFlow/run2auau/JetProduction/
+     mem     :   2096MB
+     neventsper: 50000
+     rsync  : "./ProdFlow/run2auau/JetProduction/*,cups.py,bachi.py,odbc.ini"
+
+   input:
+     db: filecatalog
+     query: |-
+         select 
+                'filecatalog/datasets'                                  as source      , 
+                runnumber                                                              , 
+                segment                                                                ,
+                filename                                                as files       ,
+                'X'                   as fileranges 
+         from  
+                datasets
+         where 
+                filename like 'DST_CALO_run2auau_ana462_2024p010%'
+                {run_condition}
+                and runnumber>53880
+           
+         {limit_condition}
+
+         ;
+   job:
+     arguments             : "$(nevents) {outbase} {logbase} $(run) $(seg) $(outdir) $(buildarg) $(tag) $(inputs) $(ranges) {neventsper} {logdir} {histdir} {PWD} {rsync}"
+     output_destination    : '{logdir}'
+     log                   : '{condor}/{logbase}.condor'
+     accounting_group      : "group_sphenix.mdc2"
+     accounting_group_user : "sphnxpro"
+     priority : '3900'
+
+
+
+
+
+# ----------------------------------------------------------------------------------------------------------------------------------------------
+# Disable the DST_TRIGGERED_EVENT rule
+DST_TRIGGERED_EVENT_run2auau_ana462_2024p010:
+   params:
+     name:       DST_TRIGGERED_EVENT_run2auau
+     build:      -------
+     build_name: ------
+     dbtag:      --------
+     version :   -
+     logbase :   $(name)_$(build)_$(tag)_$(version)-$INT(run,{RUNFMT})-$INT(seg,{SEGFMT})
+     outbase :   $(name)_$(build)_$(tag)_$(version)
+     script  :   run.sh
+     payload :   ./ProdFlow/run2auau/TriggerProduction/
+     mem     :   2048M
+     neventsper: 5000
+     noverflow:  1000
+     rsync  : "-----"
+#    rsync  : "ProdFlow/run2auau/TriggerProduction/*,cups.py,bachi.py,odbc.ini"
+
+
+   #
+   # input query:
+   #
+   # This builds a list of all runs known to the file catalog "datasets" table.
+   # The query should return:
+   # 1. The source of the information (formatted as database name/table name)
+   # 2. The run number
+   # 3. A sequence number (a placeholder fixed at zero for event builders)
+   # 4. And a space-separated list of logical filenames 
+   #
+   # The {*_condition} parameters (run, seg, limit) are substituted by kaedama
+   # based on (optional) command line options.
+   #
+   # For now, require GL1 files...
+   #
+   input:
+      db: daqdb
+      direct_path: /sphenix/lustre01/sphnxpro/physics/*/*/
+      query: |-
+         with config as (
+            select 
+               {neventsper} as nevents,
+               {noverflow} as noverflow
+         ),
+
+         fullevents as (
+              select runnumber,max(lastevent)    as lastevent
+              from filelist, config
+              where (filename similar to '/bbox%/GL1_(beam|physics)%.evt'        and lastevent>2 )
+
+                 {run_condition} and runnumber>54748
+
+              group by runnumber
+              having
+                     every(transferred_to_sdcc)   and
+                     max(lastevent)>1000          and
+                     sum( case when filename similar to '/bbox%/GL1_(beam|physics)%' then 1 else 0 end )>0
+
+              order by runnumber
+         ),
+
+         fulljobs as (
+         select 
+                'daqdb/filelist'                                                                                    as source      , 
+                runnumber                                                                                                          , 
+                0                                                                                                   as segment     , 
+                string_agg( split_part(filename,'/',-1), ' ' order by filename)                                             as files       ,   
+                string_agg( firstevent::text || ':' || lastevent::text || ':' || split_part(filename,'/',-1), ' ' order by filename )                    as fileranges  
+
+         from  
+                filelist,config
+         where 
+           ( 
+             (filename  similar to '/bbox%/%emcal%(beam|physics)%.prdf'    and lastevent>2 ) or 
+             (filename  similar to '/bbox%/%HCal%(beam|physics)%.prdf'     and lastevent>2 ) or
+             (filename  similar to '/bbox%/%LL1%(beam|physics)%.prdf'      and lastevent>2 ) or
+             (filename  similar to '/bbox%/GL1_(beam|physics)%.evt'       and lastevent>2 ) or
+             (filename  similar to '/bbox%/%mbd%(beam|physics)%.prdf'      and lastevent>2 ) or
+             (filename  similar to '/bbox%/%ZDC%(beam|physics)%.prdf'      and lastevent>2 ) 
+
+           )
+
+           {run_condition}
+
+                 and runnumber>53880
+
+         group by runnumber
+
+         having
+                every(transferred_to_sdcc)   and
+                max(lastevent)>1000          and
+                sum( case when filename similar to '/bbox%/GL1_(beam|physics)%' then 1 else 0 end )>0 and
+                (
+                   sum( case when filename similar to '/bbox%/%emcal%(beam|physics)%'  then 1 else 0 end )>0 or
+                   sum( case when filename similar to '/bbox%/%HCal%(beam|physics)%'   then 1 else 0 end )>0 
+                )
+
+         order by runnumber
+         ),
+
+         fullrun as (
+
+             select fulljobs.*,fullevents.lastevent from fulljobs join fullevents on fulljobs.runnumber=fullevents.runnumber
+
+         ),
+
+         unsegmented as ( 
+
+             select *,'full run' as runtype from fullrun where true 
+         ),
+
+         segmented as (
+
+            select source, 
+                   runnumber,
+                   generate_series(0,unsegmented.lastevent/config.nevents) as segment,
+                   files,
+                   fileranges,
+                   lastevent,
+                   nevents
+
+             from unsegmented,config
+
+         ),
+
+         segmentedA as ( 
+
+         select source,
+                runnumber,
+                segment,
+                files,
+                fileranges,
+                lastevent, 
+                segment*config.nevents as myfirstevent,
+                least( (segment+1)*config.nevents - 1, lastevent ) as mylastevent,
+                config.nevents 
+ 
+                from segmented,config
+
+          ),
+
+
+          segmentedB as (
+
+               select *,
+                      mylastevent-myfirstevent as nprocessing,
+                      lastevent-mylastevent-1  as nremaining from segmentedA,config  -- b/c we count from zero
+       
+          ),
+
+          segmentedC as (
+
+
+               select source, runnumber,
+                      segment,
+                      files,
+                      fileranges,
+                      myfirstevent,
+                      (
+                          select 
+                             case
+                                when nprocessing<config.noverflow  then -1
+                                when nremaining>=config.nevents    then mylastevent
+                                when nremaining>=config.noverflow  then mylastevent
+                                when nremaining<config.noverflow   then lastevent
+                                else                                    -2             -- unhandled, should be an error condition
+                             end
+                      ) as mylastevent,
+                      lastevent,
+                      nprocessing,
+                      nremaining
+
+                      from segmentedB,config
+
+          )
+
+
+          select 
+               source,
+                 runnumber,
+                 segment,
+                 files,
+                 fileranges,
+                 
+                 (select count(*) from regexp_matches( files,      ' ', 'g' ))::integer =
+                 (select count(*) from regexp_matches( fileranges, ' ', 'g' ))::integer as sanity,
+                 myfirstevent as firstevent,
+                 mylastevent  as lastevent,
+                 lastevent    as runs_last_event
+
+                 from segmentedC
+
+                 where runnumber>53880
+
+          order by runnumber desc, segment desc;
+                 
+
+   #
+   # Again I note the need to ensure that the arguments are properly specified given the
+   # definition of the payload script.
+   #
+   job:
+     arguments             : "$(nevents) {outbase} {logbase} $(run) $(seg) $(outdir) $(buildarg) $(tag) $(inputs) $(ranges) {neventsper} {logdir} {histdir} {PWD} {rsync} $(firstevent) $(lastevent) $(runs_last_event) {PWD} {rsync}"
+     output_destination    : '{logdir}'
+     log                   : '{condor}/{logbase}.condor'
+     accounting_group      : "group_sphenix.mdc2"
+     accounting_group_user : "sphnxpro"
+     priority : '3900'
+     request_xferslots: '1'
+
+
+
+

From 364191c33302321b535d6d1a1d1c6c7d128990b5 Mon Sep 17 00:00:00 2001
From: klendathu2k <jwebb@bnl.gov>
Date: Fri, 24 Jan 2025 11:07:04 -0500
Subject: [PATCH 03/25] First draft of the run2pp reproduction ana462 2024p010

---
 ...RIGGERED_EVENT_run2pp_ana462_2024p010.yaml | 361 ++++++++++++++++++
 1 file changed, 361 insertions(+)
 create mode 100644 run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml

diff --git a/run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml b/run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml
new file mode 100644
index 0000000..cfc7cdf
--- /dev/null
+++ b/run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml
@@ -0,0 +1,361 @@
+#
+#FileCatalog=# select dataset,dsttype,count(filename),sum(events) from datasets where dsttype='DST_CALOFITTING_run2pp' group by dataset,dsttype;
+#     dataset     |        dsttype         | count  |     sum     
+#-----------------+------------------------+--------+-------------
+# ana451_2024p009 | DST_CALOFITTING_run2pp |  50343 |  4976087047
+# ana446_2024p007 | DST_CALOFITTING_run2pp | 148286 | 14632177735
+#(2 rows)
+#
+
+# Downstream products
+DST_CALO_run2pp_new_2024p010:
+   params:
+     name:       DST_CALO_run2pp
+     build:      ana.462
+     build_name: ana462
+     dbtag:      2024p010
+     version : 1
+     logbase :   $(name)_$(build)_$(tag)_$(version)-$INT(run,{RUNFMT})-$INT(seg,{SEGFMT})
+     outbase :   $(name)_$(build)_$(tag)_$(version)
+     script  :   runy2calib.sh
+     payload :   ./ProdFlow/run2pp/CaloProduction/
+     mem     :   4096MB
+     neventsper: 50000
+     rsync  : "./ProdFlow/run2pp/CaloProduction/*,cups.py,bachi.py,odbc.ini"
+
+   input:
+     db: filecatalog
+     query: |-
+         select 
+                'filecatalog/datasets'                                  as source      , 
+                runnumber                                                              , 
+                segment                                                                ,
+                filename                                                as files       ,
+                filename || ':' || 0 || ':' || events                   as fileranges 
+         from  
+                datasets
+         where 
+                dataset='ana446_2024p007' and dsttype='DST_CALOFITTING_run2pp'
+                {run_condition}
+
+                 and runnumber>53880
+           
+         {limit_condition}
+
+         ;
+   job:
+     arguments             : "$(nevents) {outbase} {logbase} $(run) $(seg) $(outdir) $(buildarg) $(tag) $(inputs) $(ranges) {neventsper} {logdir} {histdir} {PWD} {rsync}"
+     output_destination    : '{logdir}'
+     log                   : '{condor}/{logbase}.condor'
+     accounting_group      : "group_sphenix.mdc2"
+     accounting_group_user : "sphnxpro"
+     priority : '3900'
+
+
+# Downstream products
+DST_JETS_run2pp_new_2024p010:
+   params:
+     name:       DST_JETS_run2pp
+     build:      ana.462
+     build_name: ana462
+     dbtag:      2024p010
+     version : 1
+     logbase :   $(name)_$(build)_$(tag)_$(version)-$INT(run,{RUNFMT})-$INT(seg,{SEGFMT})
+     outbase :   $(name)_$(build)_$(tag)_$(version)
+     script  :   runjets.sh
+     payload :   ./ProdFlow/run2pp/JetProduction/
+     mem     :   2096MB
+     neventsper: 50000
+     rsync  : "./ProdFlow/run2pp/JetProduction/*,cups.py,bachi.py,odbc.ini"
+
+   input:
+     db: filecatalog
+     query: |-
+         select 
+                'filecatalog/datasets'                                  as source      , 
+                runnumber                                                              , 
+                segment                                                                ,
+                filename                                                as files       ,
+                'X'                   as fileranges 
+         from  
+                datasets
+         where 
+                filename like 'DST_CALO_run2pp_ana462_2024p010%'
+                {run_condition}
+                and runnumber>53880
+           
+         {limit_condition}
+
+         ;
+   job:
+     arguments             : "$(nevents) {outbase} {logbase} $(run) $(seg) $(outdir) $(buildarg) $(tag) $(inputs) $(ranges) {neventsper} {logdir} {histdir} {PWD} {rsync}"
+     output_destination    : '{logdir}'
+     log                   : '{condor}/{logbase}.condor'
+     accounting_group      : "group_sphenix.mdc2"
+     accounting_group_user : "sphnxpro"
+     priority : '3900'
+
+
+
+
+
+# ----------------------------------------------------------------------------------------------------------------------------------------------
+# Disable the DST_TRIGGERED_EVENT rule
+XXX_TRIGGERED_EVENT_run2pp_ana462_2024p010:
+   params:
+     name:       DST_TRIGGERED_EVENT_run2pp
+     build:      xxxxxxx
+     build_name: xxxxxx
+     dbtag:      xxxxxxxx
+     version :   x
+     logbase :   $(name)_$(build)_$(tag)_$(version)-$INT(run,{RUNFMT})-$INT(seg,{SEGFMT})
+     outbase :   $(name)_$(build)_$(tag)_$(version)
+     script  :   run.sh
+     payload :   ./ProdFlow/run2pp/TriggerProduction/
+     mem     :   2048M
+     neventsper: 5000
+     noverflow:  1000
+     rsync  : "-----"
+#    rsync  : "ProdFlow/run2pp/TriggerProduction/*,cups.py,bachi.py,odbc.ini"
+
+
+   #
+   # input query:
+   #
+   # This builds a list of all runs known to the file catalog "datasets" table.
+   # The query should return:
+   # 1. The source of the information (formatted as database name/table name)
+   # 2. The run number
+   # 3. A sequence number (a placeholder fixed at zero for event builders)
+   # 4. And a space-separated list of logical filenames 
+   #
+   # The {*_condition} parameters (run, seg, limit) are substituted by kaedama
+   # based on (optional) command line options.
+   #
+   # For now, require GL1 files...
+   #
+   input:
+      db: daqdb
+      direct_path: /sphenix/lustre01/sphnxpro/physics/*/*/
+      query: |-
+         with config as (
+            select 
+               {neventsper} as nevents,
+               {noverflow} as noverflow
+         ),
+
+         fullevents as (
+              select runnumber,max(lastevent)    as lastevent
+              from filelist, config
+              where (filename similar to '/bbox%/GL1_(beam|physics)%.evt'        and lastevent>2 )
+
+                 {run_condition} and runnumber>54748
+
+              group by runnumber
+              having
+                     every(transferred_to_sdcc)   and
+                     max(lastevent)>1000          and
+                     sum( case when filename similar to '/bbox%/GL1_(beam|physics)%' then 1 else 0 end )>0
+
+              order by runnumber
+         ),
+
+         fulljobs as (
+         select 
+                'daqdb/filelist'                                                                                    as source      , 
+                runnumber                                                                                                          , 
+                0                                                                                                   as segment     , 
+                string_agg( split_part(filename,'/',-1), ' ' order by filename)                                             as files       ,   
+                string_agg( firstevent::text || ':' || lastevent::text || ':' || split_part(filename,'/',-1), ' ' order by filename )                    as fileranges  
+
+         from  
+                filelist,config
+         where 
+           ( 
+             (filename  similar to '/bbox%/%emcal%(beam|physics)%.prdf'    and lastevent>2 ) or 
+             (filename  similar to '/bbox%/%HCal%(beam|physics)%.prdf'     and lastevent>2 ) or
+             (filename  similar to '/bbox%/%LL1%(beam|physics)%.prdf'      and lastevent>2 ) or
+             (filename  similar to '/bbox%/GL1_(beam|physics)%.evt'       and lastevent>2 ) or
+             (filename  similar to '/bbox%/%mbd%(beam|physics)%.prdf'      and lastevent>2 ) or
+             (filename  similar to '/bbox%/%ZDC%(beam|physics)%.prdf'      and lastevent>2 ) 
+
+           )
+
+           {run_condition}
+
+                 and runnumber>53880
+
+         group by runnumber
+
+         having
+                every(transferred_to_sdcc)   and
+                max(lastevent)>1000          and
+                sum( case when filename similar to '/bbox%/GL1_(beam|physics)%' then 1 else 0 end )>0 and
+                (
+                   sum( case when filename similar to '/bbox%/%emcal%(beam|physics)%'  then 1 else 0 end )>0 or
+                   sum( case when filename similar to '/bbox%/%HCal%(beam|physics)%'   then 1 else 0 end )>0 
+                )
+
+         order by runnumber
+         ),
+
+         fullrun as (
+
+             select fulljobs.*,fullevents.lastevent from fulljobs join fullevents on fulljobs.runnumber=fullevents.runnumber
+
+         ),
+
+         unsegmented as ( 
+
+             select *,'full run' as runtype from fullrun where true 
+         ),
+
+         segmented as (
+
+            select source, 
+                   runnumber,
+                   generate_series(0,unsegmented.lastevent/config.nevents) as segment,
+                   files,
+                   fileranges,
+                   lastevent,
+                   nevents
+
+             from unsegmented,config
+
+         ),
+
+         segmentedA as ( 
+
+         select source,
+                runnumber,
+                segment,
+                files,
+                fileranges,
+                lastevent, 
+                segment*config.nevents as myfirstevent,
+                least( (segment+1)*config.nevents - 1, lastevent ) as mylastevent,
+                config.nevents 
+ 
+                from segmented,config
+
+          ),
+
+
+          segmentedB as (
+
+               select *,
+                      mylastevent-myfirstevent as nprocessing,
+                      lastevent-mylastevent-1  as nremaining from segmentedA,config  -- b/c we count from zero
+       
+          ),
+
+          segmentedC as (
+
+
+               select source, runnumber,
+                      segment,
+                      files,
+                      fileranges,
+                      myfirstevent,
+                      (
+                          select 
+                             case
+                                when nprocessing<config.noverflow  then -1
+                                when nremaining>=config.nevents    then mylastevent
+                                when nremaining>=config.noverflow  then mylastevent
+                                when nremaining<config.noverflow   then lastevent
+                                else                                    -2             -- unhandled, should be an error condition
+                             end
+                      ) as mylastevent,
+                      lastevent,
+                      nprocessing,
+                      nremaining
+
+                      from segmentedB,config
+
+          )
+
+
+          select 
+               source,
+                 runnumber,
+                 segment,
+                 files,
+                 fileranges,
+                 
+                 (select count(*) from regexp_matches( files,      ' ', 'g' ))::integer =
+                 (select count(*) from regexp_matches( fileranges, ' ', 'g' ))::integer as sanity,
+                 myfirstevent as firstevent,
+                 mylastevent  as lastevent,
+                 lastevent    as runs_last_event
+
+                 from segmentedC
+
+                 where runnumber>53880
+
+          order by runnumber desc, segment desc;
+                 
+
+   #
+   # Again I note the need to ensure that the arguments are properly specified given the
+   # definition of the payload script.
+   #
+   job:
+     arguments             : "$(nevents) {outbase} {logbase} $(run) $(seg) $(outdir) $(buildarg) $(tag) $(inputs) $(ranges) {neventsper} {logdir} {histdir} {PWD} {rsync} $(firstevent) $(lastevent) $(runs_last_event) {PWD} {rsync}"
+     output_destination    : '{logdir}'
+     log                   : '{condor}/{logbase}.condor'
+     accounting_group      : "group_sphenix.mdc2"
+     accounting_group_user : "sphnxpro"
+     priority : '3900'
+     request_xferslots: '1'
+
+
+
+
+# Downstream products
+XXX_CALOFITTING_run2pp_ana462_2024p010:
+   params:
+     name:       DST_CALOFITTING_run2pp
+     nevents:    5000
+     build:      ana.462
+     build_name: ana462
+     dbtag:      2024p010
+     version : 1
+     logbase :   $(name)_$(build)_$(tag)_$(version)-$INT(run,{RUNFMT})-$INT(seg,{SEGFMT})
+     outbase :   $(name)_$(build)_$(tag)_$(version)
+     script  :   runy2fitting.sh
+     payload :   ./ProdFlow/run2pp/CaloProduction/
+     mem     :   1024MB
+     neventsper: 50000
+     rsync  : "./ProdFlow/run2pp/CaloProduction/*,cups.py,bachi.py,odbc.ini"
+
+   input:
+     db: filecatalog
+     query: |-
+         select 
+                'filecatalog/datasets'                                  as source      , 
+                runnumber                                                              , 
+                segment                                                                ,
+                filename                                                as files       ,
+                filename || ':' || 0 || ':' || events                   as fileranges 
+         from  
+                datasets
+         where 
+                filename like 'DST_TRIGGERED_EVENT_run2pp_ana451_2024p009%'
+                {run_condition}
+
+                 and runnumber>53880
+           
+         {limit_condition}
+
+         ;
+   job:
+     arguments             : "{nevents} {outbase} {logbase} $(run) $(seg) $(outdir) $(buildarg) $(tag) $(inputs) $(ranges) {neventsper} {logdir} {histdir} {PWD} {rsync}"
+     output_destination    : '{logdir}'
+     log                   : '{condor}/{logbase}.condor'
+     accounting_group      : "group_sphenix.mdc2"
+     accounting_group_user : "sphnxpro"
+     priority : '3900'
+
+
+

From b5bdf607e2eddf7ae0e01903d126139ae20e3bbd Mon Sep 17 00:00:00 2001
From: klendathu2k <jwebb@bnl.gov>
Date: Fri, 24 Jan 2025 11:07:56 -0500
Subject: [PATCH 04/25] ... b/c yaml doesn't like unquoted '-' here ...

---
 .../DST_TRIGGERED_EVENT_run2auau_ana462_2024p010.yaml     | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/run2auau/DST_TRIGGERED_EVENT_run2auau_ana462_2024p010.yaml b/run2auau/DST_TRIGGERED_EVENT_run2auau_ana462_2024p010.yaml
index 8b1efea..8ef74f9 100644
--- a/run2auau/DST_TRIGGERED_EVENT_run2auau_ana462_2024p010.yaml
+++ b/run2auau/DST_TRIGGERED_EVENT_run2auau_ana462_2024p010.yaml
@@ -142,10 +142,10 @@ DST_JETS_run2auau_new_2024p010:
 DST_TRIGGERED_EVENT_run2auau_ana462_2024p010:
    params:
      name:       DST_TRIGGERED_EVENT_run2auau
-     build:      -------
-     build_name: ------
-     dbtag:      --------
-     version :   -
+     build:      xxxxxxx
+     build_name: xxxxxx
+     dbtag:      xxxxxxxx
+     version :   x
      logbase :   $(name)_$(build)_$(tag)_$(version)-$INT(run,{RUNFMT})-$INT(seg,{SEGFMT})
      outbase :   $(name)_$(build)_$(tag)_$(version)
      script  :   run.sh

From 3b92ac6447b098d77853d1abae041aa5b6521722 Mon Sep 17 00:00:00 2001
From: klendathu2k <jwebb@bnl.gov>
Date: Fri, 24 Jan 2025 11:23:39 -0500
Subject: [PATCH 05/25] ... b/c this is the pp run limit ...

---
 run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml b/run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml
index cfc7cdf..bd7409c 100644
--- a/run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml
+++ b/run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml
@@ -38,7 +38,7 @@ DST_CALO_run2pp_new_2024p010:
                 dataset='ana446_2024p007' and dsttype='DST_CALOFITTING_run2pp'
                 {run_condition}
 
-                 and runnumber>53880
+                 and runnumber<=53880
            
          {limit_condition}
 

From 48b7a9da7f9ce913a65f78b64629a75c7134a0f1 Mon Sep 17 00:00:00 2001
From: klendathu2k <jwebb@bnl.gov>
Date: Fri, 24 Jan 2025 12:12:11 -0500
Subject: [PATCH 06/25] ... test

---
 run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml b/run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml
index bd7409c..1eee091 100644
--- a/run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml
+++ b/run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml
@@ -5,7 +5,6 @@
 # ana451_2024p009 | DST_CALOFITTING_run2pp |  50343 |  4976087047
 # ana446_2024p007 | DST_CALOFITTING_run2pp | 148286 | 14632177735
 #(2 rows)
-#
 
 # Downstream products
 DST_CALO_run2pp_new_2024p010:

From c99864e69c958875f940ec480064364fd1ed7b71 Mon Sep 17 00:00:00 2001
From: klendathu2k <jwebb@bnl.gov>
Date: Sat, 25 Jan 2025 10:02:11 -0500
Subject: [PATCH 07/25] Add the jet skim workflow.

---
 ...RIGGERED_EVENT_run2pp_ana462_2024p010.yaml | 48 +++++++++++++++++++
 1 file changed, 48 insertions(+)

diff --git a/run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml b/run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml
index 1eee091..2c883e0 100644
--- a/run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml
+++ b/run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml
@@ -97,6 +97,54 @@ DST_JETS_run2pp_new_2024p010:
 
 
 
+# Downstream products
+DST_JETSKIMMED_run2pp_ana462_2024p010:
+   params:
+     name:       DST_JETSKIMMED_run2pp
+     build:      ana.462
+     build_name: ana462
+     dbtag:      2024p010
+     version:    1
+     logbase :   $(name)_$(build)_$(tag)_$(version)-$INT(run,{RUNFMT})-$INT(seg,{SEGFMT})
+     outbase :   $(name)_$(build)_$(tag)_$(version)
+     script  :   runy2jetskim.sh
+     payload :   ./ProdFlow/run2pp/JetProduction/
+     mem     :   4096MB
+     neventsper: 50000
+     rsync  : "./ProdFlow/run2pp/JetProduction/*,cups.py,bachi.py"
+
+   input:
+     db: filecatalog
+     query: |-
+         select
+                'filecatalog/datasets'                                  as source      ,
+                runnumber                                                              ,
+                segment                                                                ,
+                filename                                                as files       ,
+                filename || ':' || 0 || ':' || events                   as fileranges
+         from
+                datasets
+         where
+                filename like 'DST_CALOFITTING_run2pp_ana462_2024p010%'
+                {run_condition}
+
+                 and runnumber<=53880
+
+         {limit_condition}
+
+         ;
+   job:
+     executable            : "{payload}/runy2jetskim.sh"
+     arguments             : "$(nevents) {outbase} {logbase} $(run) $(seg) $(outdir) $(buildarg) $(tag) $(inputs) $(ranges) {neventsper} {logdir} {histdir} {PWD} {rsync}"
+     output_destination    : '{logdir}'
+     log                   : '{condor}/{logbase}.condor'
+     accounting_group      : "group_sphenix.mdc2"
+     accounting_group_user : "sphnxpro"
+     priority : '3900'
+
+
+
+
 
 # ----------------------------------------------------------------------------------------------------------------------------------------------
 # Disable the DST_TRIGGERED_EVENT rule

From b75c9bc200b4d17b4008c139a4b6630f8cbcd04c Mon Sep 17 00:00:00 2001
From: klendathu2k <jwebb@bnl.gov>
Date: Sat, 25 Jan 2025 15:16:21 -0500
Subject: [PATCH 08/25] Further name fixup.  Use $logbase for output
 filenaming.  Histogram file should have the HIST prefix in place of DST.

---
 .../DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml   |  4 ++--
 run2pp/JetProduction/runy2jetskim.sh                  | 11 ++++++++---
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml b/run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml
index 2c883e0..dd57366 100644
--- a/run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml
+++ b/run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml
@@ -98,9 +98,9 @@ DST_JETS_run2pp_new_2024p010:
 
 
 # Downstream products
-DST_JETSKIMMED_run2pp_ana462_2024p010:
+DST_JETCALO_run2pp_ana462_2024p010:
    params:
-     name:       DST_JETSKIMMED_run2pp
+     name:       DST_JETCALO_run2pp
      build:      ana.462
      build_name: ana462
      dbtag:      2024p010
diff --git a/run2pp/JetProduction/runy2jetskim.sh b/run2pp/JetProduction/runy2jetskim.sh
index e268221..1cb3d97 100644
--- a/run2pp/JetProduction/runy2jetskim.sh
+++ b/run2pp/JetProduction/runy2jetskim.sh
@@ -97,12 +97,14 @@ out1=HIST_${logbase#DST_}.root
 nevents=-1
 status_f4a=0
 
+# there should be only one input file in this workflow
 for infile_ in ${inputs[@]}; do
     infile=$( basename ${infile_} )
     cp -v ${infile_} .
-    outfile1=${infile/CALOFITTING/JETCALO}
-    outfile2=${infile/CALOFITTING/JET}
-    outhist=${infile/DST_CALOFITTING/HIST_CALOQASKIMMED}
+#${infile/CALOFITTING/JETCALO}
+    outfile1=${logbase}.root
+    outfile2=${logbase/JETCALO/JET}.root
+    outhist=${logbase/DST_JETCALO/HIST_CALOQASKIMMED}.root
     root.exe -q -b Fun4All_JetSkimmedProductionYear2.C\(${nevents},\"${infile}\",\"${outfile1}\",\"${outfile2}\",\"${outhist}\",\"${dbtag}\"\);  status_f4a=$?
 
     # Stageout the (single) DST created in the macro run
@@ -120,6 +122,9 @@ for infile_ in ${inputs[@]}; do
 	echo Stageout ${hfile} to ${histdir}
         ./stageout.sh ${hfile} ${histdir}
     done
+
+    break # as noted... there should only be one input file in this workflow.   Force a break at this point.
+
 done
 
 ls -lah

From 0aaf6a805fb11808f414f8e01107fb37c9c08671 Mon Sep 17 00:00:00 2001
From: klendathu2k <jwebb@bnl.gov>
Date: Sat, 25 Jan 2025 15:33:29 -0500
Subject: [PATCH 09/25] And get the correct input dataset.

---
 run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml b/run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml
index dd57366..52cbb8d 100644
--- a/run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml
+++ b/run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml
@@ -125,7 +125,7 @@ DST_JETCALO_run2pp_ana462_2024p010:
          from
                 datasets
          where
-                filename like 'DST_CALOFITTING_run2pp_ana462_2024p010%'
+                dataset='ana446_2024p007' and dsttype='DST_CALOFITTING_run2pp'
                 {run_condition}
 
                  and runnumber<=53880

From a2b923b8701267eb42a0b5ab86ea989cde00b279 Mon Sep 17 00:00:00 2001
From: klendathu2k <jwebb@bnl.gov>
Date: Sat, 25 Jan 2025 18:17:36 -0500
Subject: [PATCH 10/25] We must remove the executable so that the commonn job
 wrapper is picked up.

---
 run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml b/run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml
index 52cbb8d..0928346 100644
--- a/run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml
+++ b/run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml
@@ -134,7 +134,6 @@ DST_JETCALO_run2pp_ana462_2024p010:
 
          ;
    job:
-     executable            : "{payload}/runy2jetskim.sh"
      arguments             : "$(nevents) {outbase} {logbase} $(run) $(seg) $(outdir) $(buildarg) $(tag) $(inputs) $(ranges) {neventsper} {logdir} {histdir} {PWD} {rsync}"
      output_destination    : '{logdir}'
      log                   : '{condor}/{logbase}.condor'

From 4a3abd4971a264f2f7766c12e95ddd113a1b99d2 Mon Sep 17 00:00:00 2001
From: klendathu2k <jwebb@bnl.gov>
Date: Mon, 27 Jan 2025 11:01:10 -0500
Subject: [PATCH 11/25] additional debug information... nothing to see here

---
 run2pp/JetProduction/runy2jetskim.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/run2pp/JetProduction/runy2jetskim.sh b/run2pp/JetProduction/runy2jetskim.sh
index 1cb3d97..5cc3896 100644
--- a/run2pp/JetProduction/runy2jetskim.sh
+++ b/run2pp/JetProduction/runy2jetskim.sh
@@ -36,6 +36,8 @@ export USER="$(id -u -n)"
 export LOGNAME=${USER}
 export HOME=/sphenix/u/${USER}
 hostname
+echo cupsid=${cupsid}
+
 
 source /opt/sphenix/core/bin/sphenix_setup.sh -n ${7}
 echo OFFLINE_MAIN: $OFFLINE_MAIN
@@ -55,6 +57,8 @@ fi
 
 if [[ "${inputs}" == *"dbinput"* ]]; then
     echo "Getting inputs via cups.  ranges is not set."
+    echo ./cups.py -r ${runnumber} -s ${segment} -d ${outbase} getinputs
+         ./cups.py -r ${runnumber} -s ${segment} -d ${outbase} getinputs
     inputs=( $(./cups.py -r ${runnumber} -s ${segment} -d ${outbase} getinputs) )
 fi
 

From 9fbca39635835302819cb7d216a310673edbc393 Mon Sep 17 00:00:00 2001
From: klendathu2k <jwebb@bnl.gov>
Date: Mon, 27 Jan 2025 11:01:54 -0500
Subject: [PATCH 12/25] Get the correct order.  This does not matter for the
 current production, since we do not rely on an odbc.ini file.

---
 run2pp/CaloProduction/runy2calib.sh | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/run2pp/CaloProduction/runy2calib.sh b/run2pp/CaloProduction/runy2calib.sh
index 2cb9730..42cbaaf 100644
--- a/run2pp/CaloProduction/runy2calib.sh
+++ b/run2pp/CaloProduction/runy2calib.sh
@@ -46,11 +46,6 @@ for i in ${payload[@]}; do
     cp --verbose ${subdir}/${i} .
 done
 
-if [[ "${inputs}" == *"dbinput"* ]]; then
-    echo "Getting inputs via cups.  ranges is not set."
-    inputs=( $(./cups.py -r ${runnumber} -s ${segment} -d ${outbase} getinputs) )
-fi
-
 if [ -e odbc.ini ]; then
 echo export ODBCINI=./odbc.ini
      export ODBCINI=./odbc.ini
@@ -58,6 +53,11 @@ else
      echo No odbc.ini file detected.  Using system odbc.ini
 fi
 
+if [[ "${inputs}" == *"dbinput"* ]]; then
+    echo "Getting inputs via cups.  ranges is not set."
+    inputs=( $(./cups.py -r ${runnumber} -s ${segment} -d ${outbase} getinputs) )
+fi
+
 # Debugging info
 ./cups.py -r ${runnumber} -s ${segment} -d ${outbase} info
 

From e377aa66c8d2bac14fb1d663dd103a0fc0d6e9cf Mon Sep 17 00:00:00 2001
From: klendathu2k <jwebb@bnl.gov>
Date: Mon, 27 Jan 2025 11:02:57 -0500
Subject: [PATCH 13/25] Current DST_CALO production used the input filename and
 substituted for the dsttype.  Consequently the output filename does not
 reflect the correct build and cdbtag.  A subsequent fixup will be run to
 rename.

---
 run2pp/CaloProduction/runy2calib.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/run2pp/CaloProduction/runy2calib.sh b/run2pp/CaloProduction/runy2calib.sh
index 42cbaaf..5c67859 100644
--- a/run2pp/CaloProduction/runy2calib.sh
+++ b/run2pp/CaloProduction/runy2calib.sh
@@ -94,7 +94,7 @@ status_f4a=0
 for infile_ in ${inputs[@]}; do
     infile=$( basename ${infile_} )
     cp -v ${infile_} .
-    outfile=${infile/CALOFITTING/CALO}
+    outfile=${logbase}.root
     outhist=${outfile/DST_CALO/HIST_CALOQA}
     root.exe -q -b Fun4All_Year2_Calib.C\(${nevents},\"${infile}\",\"${outfile}\",\"${outhist}\",\"${dbtag}\"\);  status_f4a=$?
 

From 36875e9f5c470ae9493b1f3b6d21e7c332d5d8c5 Mon Sep 17 00:00:00 2001
From: klendathu2k <jwebb@bnl.gov>
Date: Mon, 27 Jan 2025 11:03:37 -0500
Subject: [PATCH 14/25] Corresponding fix for auau, before submitting.

---
 run2auau/CaloProduction/runy2calib.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/run2auau/CaloProduction/runy2calib.sh b/run2auau/CaloProduction/runy2calib.sh
index a46dd74..1837d55 100644
--- a/run2auau/CaloProduction/runy2calib.sh
+++ b/run2auau/CaloProduction/runy2calib.sh
@@ -35,6 +35,7 @@ trap sighandler SIGTERM SIGINT SIGKILL
 export USER="$(id -u -n)"
 export LOGNAME=${USER}
 export HOME=/sphenix/u/${USER}
+echo "cupsid=${cupsid}"
 hostname
 
 source /opt/sphenix/core/bin/sphenix_setup.sh -n ${7}
@@ -96,7 +97,7 @@ status_f4a=0
 for infile_ in ${inputs[@]}; do
     infile=$( basename ${infile_} )
     cp -v ${infile_} .
-    outfile=${infile/CALOFITTING/CALO}
+    outfile=${logbase}.root
     outhist=${outfile/DST_CALO/HIST_CALOQA}
     root.exe -q -b Fun4All_Year2_Calib.C\(${nevents},\"${infile}\",\"${outfile}\",\"${outhist}\",\"${dbtag}\"\);  status_f4a=$?
 

From 9e9605a1cbf023ccf1c4872cd5a4fe6a95a89435 Mon Sep 17 00:00:00 2001
From: klendathu2k <jwebb@bnl.gov>
Date: Mon, 27 Jan 2025 11:04:50 -0500
Subject: [PATCH 15/25] Rework of the run2auau workflow for the ana462 2024p010
 production.

---
 ...GGERED_EVENT_run2auau_ana462_2024p010.yaml | 117 ++++++++++++------
 1 file changed, 82 insertions(+), 35 deletions(-)

diff --git a/run2auau/DST_TRIGGERED_EVENT_run2auau_ana462_2024p010.yaml b/run2auau/DST_TRIGGERED_EVENT_run2auau_ana462_2024p010.yaml
index 8ef74f9..05c97f6 100644
--- a/run2auau/DST_TRIGGERED_EVENT_run2auau_ana462_2024p010.yaml
+++ b/run2auau/DST_TRIGGERED_EVENT_run2auau_ana462_2024p010.yaml
@@ -1,17 +1,16 @@
 # Downstream products
-DST_CALOFITTING_run2auau_ana462_2024p010:
+DST_CALO_run2auau_new_2024p010:
    params:
-     name:       DST_CALOFITTING_run2auau
-     nevents:    5000
+     name:       DST_CALO_run2auau
      build:      ana.462
      build_name: ana462
      dbtag:      2024p010
      version : 1
      logbase :   $(name)_$(build)_$(tag)_$(version)-$INT(run,{RUNFMT})-$INT(seg,{SEGFMT})
      outbase :   $(name)_$(build)_$(tag)_$(version)
-     script  :   runy2fitting.sh
+     script  :   runy2calib.sh
      payload :   ./ProdFlow/run2auau/CaloProduction/
-     mem     :   1024MB
+     mem     :   4096MB
      neventsper: 50000
      rsync  : "./ProdFlow/run2auau/CaloProduction/*,cups.py,bachi.py,odbc.ini"
 
@@ -27,16 +26,16 @@ DST_CALOFITTING_run2auau_ana462_2024p010:
          from  
                 datasets
          where 
-                filename like 'DST_TRIGGERED_EVENT_run2auau_ana451_2024p009%'
+                dataset='ana450_2024p009' and dsttype='DST_CALOFITTING_run2auau'
                 {run_condition}
 
-                 and runnumber>53880
+                and runnumber in  ( 54909, 54911, 54912, 54913, 54914, 54915, 54916, 54918, 54920 )
            
          {limit_condition}
 
          ;
    job:
-     arguments             : "{nevents} {outbase} {logbase} $(run) $(seg) $(outdir) $(buildarg) $(tag) $(inputs) $(ranges) {neventsper} {logdir} {histdir} {PWD} {rsync}"
+     arguments             : "$(nevents) {outbase} {logbase} $(run) $(seg) $(outdir) $(buildarg) $(tag) $(inputs) $(ranges) {neventsper} {logdir} {histdir} {PWD} {rsync}"
      output_destination    : '{logdir}'
      log                   : '{condor}/{logbase}.condor'
      accounting_group      : "group_sphenix.mdc2"
@@ -44,22 +43,21 @@ DST_CALOFITTING_run2auau_ana462_2024p010:
      priority : '3900'
 
 
-
 # Downstream products
-DST_CALO_run2auau_new_2024p010:
+DST_JETS_run2auau_new_2024p010:
    params:
-     name:       DST_CALO_run2auau
+     name:       DST_JETS_run2auau
      build:      ana.462
      build_name: ana462
      dbtag:      2024p010
      version : 1
      logbase :   $(name)_$(build)_$(tag)_$(version)-$INT(run,{RUNFMT})-$INT(seg,{SEGFMT})
      outbase :   $(name)_$(build)_$(tag)_$(version)
-     script  :   runy2calib.sh
-     payload :   ./ProdFlow/run2auau/CaloProduction/
-     mem     :   4096MB
+     script  :   runjets.sh
+     payload :   ./ProdFlow/run2auau/JetProduction/
+     mem     :   2096MB
      neventsper: 50000
-     rsync  : "./ProdFlow/run2auau/CaloProduction/*,cups.py,bachi.py,odbc.ini"
+     rsync  : "./ProdFlow/run2auau/JetProduction/*,cups.py,bachi.py,odbc.ini"
 
    input:
      db: filecatalog
@@ -69,14 +67,13 @@ DST_CALO_run2auau_new_2024p010:
                 runnumber                                                              , 
                 segment                                                                ,
                 filename                                                as files       ,
-                filename || ':' || 0 || ':' || events                   as fileranges 
+                'X'                   as fileranges 
          from  
                 datasets
          where 
-                filename like 'DST_CALOFITTING_run2auau_ana462_2024p010%'
+                filename like 'DST_CALO_run2auau_ana462_2024p010%'
                 {run_condition}
-
-                 and runnumber>53880
+                and runnumber>53880
            
          {limit_condition}
 
@@ -90,38 +87,41 @@ DST_CALO_run2auau_new_2024p010:
      priority : '3900'
 
 
+
+
 # Downstream products
-DST_JETS_run2auau_new_2024p010:
+DST_JETCALO_run2auau_ana462_2024p010:
    params:
-     name:       DST_JETS_run2auau
+     name:       DST_JETCALO_run2auau
      build:      ana.462
      build_name: ana462
      dbtag:      2024p010
-     version : 1
+     version:    1
      logbase :   $(name)_$(build)_$(tag)_$(version)-$INT(run,{RUNFMT})-$INT(seg,{SEGFMT})
      outbase :   $(name)_$(build)_$(tag)_$(version)
-     script  :   runjets.sh
+     script  :   runy2jetskim.sh
      payload :   ./ProdFlow/run2auau/JetProduction/
-     mem     :   2096MB
+     mem     :   4096MB
      neventsper: 50000
-     rsync  : "./ProdFlow/run2auau/JetProduction/*,cups.py,bachi.py,odbc.ini"
+     rsync  : "./ProdFlow/run2auau/JetProduction/*,cups.py,bachi.py"
 
    input:
      db: filecatalog
      query: |-
-         select 
-                'filecatalog/datasets'                                  as source      , 
-                runnumber                                                              , 
+         select
+                'filecatalog/datasets'                                  as source      ,
+                runnumber                                                              ,
                 segment                                                                ,
                 filename                                                as files       ,
-                'X'                   as fileranges 
-         from  
+                filename || ':' || 0 || ':' || events                   as fileranges
+         from
                 datasets
-         where 
-                filename like 'DST_CALO_run2auau_ana462_2024p010%'
+         where
+                dataset='ana446_2024p007' and dsttype='DST_CALOFITTING_run2auau'
                 {run_condition}
-                and runnumber>53880
-           
+
+                 and runnumber<=53880
+
          {limit_condition}
 
          ;
@@ -139,7 +139,7 @@ DST_JETS_run2auau_new_2024p010:
 
 # ----------------------------------------------------------------------------------------------------------------------------------------------
 # Disable the DST_TRIGGERED_EVENT rule
-DST_TRIGGERED_EVENT_run2auau_ana462_2024p010:
+XXX_TRIGGERED_EVENT_run2auau_ana462_2024p010:
    params:
      name:       DST_TRIGGERED_EVENT_run2auau
      build:      xxxxxxx
@@ -350,3 +350,50 @@ DST_TRIGGERED_EVENT_run2auau_ana462_2024p010:
 
 
 
+# Downstream products
+XXX_CALOFITTING_run2auau_ana462_2024p010:
+   params:
+     name:       DST_CALOFITTING_run2auau
+     nevents:    5000
+     build:      ana.462
+     build_name: ana462
+     dbtag:      2024p010
+     version : 1
+     logbase :   $(name)_$(build)_$(tag)_$(version)-$INT(run,{RUNFMT})-$INT(seg,{SEGFMT})
+     outbase :   $(name)_$(build)_$(tag)_$(version)
+     script  :   runy2fitting.sh
+     payload :   ./ProdFlow/run2auau/CaloProduction/
+     mem     :   1024MB
+     neventsper: 50000
+     rsync  : "./ProdFlow/run2auau/CaloProduction/*,cups.py,bachi.py,odbc.ini"
+
+   input:
+     db: filecatalog
+     query: |-
+         select 
+                'filecatalog/datasets'                                  as source      , 
+                runnumber                                                              , 
+                segment                                                                ,
+                filename                                                as files       ,
+                filename || ':' || 0 || ':' || events                   as fileranges 
+         from  
+                datasets
+         where 
+                filename like 'DST_TRIGGERED_EVENT_run2auau_ana451_2024p009%'
+                {run_condition}
+
+                 and runnumber>53880
+           
+         {limit_condition}
+
+         ;
+   job:
+     arguments             : "{nevents} {outbase} {logbase} $(run) $(seg) $(outdir) $(buildarg) $(tag) $(inputs) $(ranges) {neventsper} {logdir} {histdir} {PWD} {rsync}"
+     output_destination    : '{logdir}'
+     log                   : '{condor}/{logbase}.condor'
+     accounting_group      : "group_sphenix.mdc2"
+     accounting_group_user : "sphnxpro"
+     priority : '3900'
+
+
+

From ed3a26325297dd9b9cc05a6be1ee9c83755d6c3d Mon Sep 17 00:00:00 2001
From: klendathu2k <jwebb@bnl.gov>
Date: Mon, 27 Jan 2025 15:46:18 -0500
Subject: [PATCH 16/25] Setup triggered event builder production.

---
 ...RIGGERED_EVENT_run2pp_ana462_2024p010.yaml | 32 ++++++++-----------
 1 file changed, 13 insertions(+), 19 deletions(-)

diff --git a/run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml b/run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml
index 0928346..3c1c2ad 100644
--- a/run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml
+++ b/run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml
@@ -147,13 +147,12 @@ DST_JETCALO_run2pp_ana462_2024p010:
 
 # ----------------------------------------------------------------------------------------------------------------------------------------------
 # Disable the DST_TRIGGERED_EVENT rule
-XXX_TRIGGERED_EVENT_run2pp_ana462_2024p010:
+DST_TRIGGERED_EVENT_run2pp_ana462_2024p010:
    params:
      name:       DST_TRIGGERED_EVENT_run2pp
-     build:      xxxxxxx
-     build_name: xxxxxx
-     dbtag:      xxxxxxxx
-     version :   x
+     build:      ana.462
+     build_name: ana462
+     dbtag:      nocdbtag
      logbase :   $(name)_$(build)_$(tag)_$(version)-$INT(run,{RUNFMT})-$INT(seg,{SEGFMT})
      outbase :   $(name)_$(build)_$(tag)_$(version)
      script  :   run.sh
@@ -161,8 +160,7 @@ XXX_TRIGGERED_EVENT_run2pp_ana462_2024p010:
      mem     :   2048M
      neventsper: 5000
      noverflow:  1000
-     rsync  : "-----"
-#    rsync  : "ProdFlow/run2pp/TriggerProduction/*,cups.py,bachi.py,odbc.ini"
+     rsync  : "ProdFlow/run2pp/TriggerProduction/*,cups.py"
 
 
    #
@@ -181,7 +179,7 @@ XXX_TRIGGERED_EVENT_run2pp_ana462_2024p010:
    # For now, require GL1 files...
    #
    input:
-      db: daqdb
+      db: raw
       direct_path: /sphenix/lustre01/sphnxpro/physics/*/*/
       query: |-
          with config as (
@@ -195,7 +193,7 @@ XXX_TRIGGERED_EVENT_run2pp_ana462_2024p010:
               from filelist, config
               where (filename similar to '/bbox%/GL1_(beam|physics)%.evt'        and lastevent>2 )
 
-                 {run_condition} and runnumber>54748
+                 {run_condition} and runnumber>46000
 
               group by runnumber
               having
@@ -229,7 +227,7 @@ XXX_TRIGGERED_EVENT_run2pp_ana462_2024p010:
 
            {run_condition}
 
-                 and runnumber>53880
+                 and runnumber<=53880
 
          group by runnumber
 
@@ -337,23 +335,19 @@ XXX_TRIGGERED_EVENT_run2pp_ana462_2024p010:
 
                  from segmentedC
 
-                 where runnumber>53880
+                 where runnumber<=53880
 
-          order by runnumber desc, segment desc;
+          order by runnumber desc, segment;
                  
-
-   #
-   # Again I note the need to ensure that the arguments are properly specified given the
-   # definition of the payload script.
-   #
    job:
      arguments             : "$(nevents) {outbase} {logbase} $(run) $(seg) $(outdir) $(buildarg) $(tag) $(inputs) $(ranges) {neventsper} {logdir} {histdir} {PWD} {rsync} $(firstevent) $(lastevent) $(runs_last_event) {PWD} {rsync}"
      output_destination    : '{logdir}'
      log                   : '{condor}/{logbase}.condor'
      accounting_group      : "group_sphenix.mdc2"
      accounting_group_user : "sphnxpro"
-     priority : '3900'
-     request_xferslots: '1'
+     priority              : '3900'
+     request_xferslots     : '1'
+     periodicremove:       : "(JobStatus == 2) && (time() - EnteredCurrentStatus) > (3 * 24 * 3600)"
 
 
 

From 43a33798920780d41a1ae7d4d736dc6ebc0be24a Mon Sep 17 00:00:00 2001
From: klendathu2k <jwebb@bnl.gov>
Date: Mon, 27 Jan 2025 16:06:31 -0500
Subject: [PATCH 17/25] Must have version>0.  Fix the double ":" in the
 periodic remove expresion.  Switch to single quotes as well...

---
 run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml b/run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml
index 3c1c2ad..d9c80a4 100644
--- a/run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml
+++ b/run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml
@@ -13,7 +13,7 @@ DST_CALO_run2pp_new_2024p010:
      build:      ana.462
      build_name: ana462
      dbtag:      2024p010
-     version : 1
+     version :   1
      logbase :   $(name)_$(build)_$(tag)_$(version)-$INT(run,{RUNFMT})-$INT(seg,{SEGFMT})
      outbase :   $(name)_$(build)_$(tag)_$(version)
      script  :   runy2calib.sh
@@ -153,6 +153,7 @@ DST_TRIGGERED_EVENT_run2pp_ana462_2024p010:
      build:      ana.462
      build_name: ana462
      dbtag:      nocdbtag
+     version :   1
      logbase :   $(name)_$(build)_$(tag)_$(version)-$INT(run,{RUNFMT})-$INT(seg,{SEGFMT})
      outbase :   $(name)_$(build)_$(tag)_$(version)
      script  :   run.sh
@@ -347,7 +348,7 @@ DST_TRIGGERED_EVENT_run2pp_ana462_2024p010:
      accounting_group_user : "sphnxpro"
      priority              : '3900'
      request_xferslots     : '1'
-     periodicremove:       : "(JobStatus == 2) && (time() - EnteredCurrentStatus) > (3 * 24 * 3600)"
+     periodicremove        : '(JobStatus==2)&&(time()-EnteredCurrentStatus)>(3*24*3600)'
 
 
 

From 2d88e644b3099f5e4535c4aa9c3f9ee9ab60cae8 Mon Sep 17 00:00:00 2001
From: klendathu2k <jwebb@bnl.gov>
Date: Mon, 27 Jan 2025 22:24:25 -0500
Subject: [PATCH 18/25] Past productions have used 100k events / output file...

---
 run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml b/run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml
index d9c80a4..3b0a332 100644
--- a/run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml
+++ b/run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml
@@ -159,8 +159,8 @@ DST_TRIGGERED_EVENT_run2pp_ana462_2024p010:
      script  :   run.sh
      payload :   ./ProdFlow/run2pp/TriggerProduction/
      mem     :   2048M
-     neventsper: 5000
-     noverflow:  1000
+     neventsper: 100000
+     noverflow:   20000
      rsync  : "ProdFlow/run2pp/TriggerProduction/*,cups.py"
 
 

From c07263778f0323ed79cbc8628ece3232c69c7ecc Mon Sep 17 00:00:00 2001
From: klendathu2k <jwebb@bnl.gov>
Date: Mon, 27 Jan 2025 22:27:38 -0500
Subject: [PATCH 19/25] ... dropping by x2 ...

---
 run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml b/run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml
index 3b0a332..18307f6 100644
--- a/run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml
+++ b/run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml
@@ -159,8 +159,8 @@ DST_TRIGGERED_EVENT_run2pp_ana462_2024p010:
      script  :   run.sh
      payload :   ./ProdFlow/run2pp/TriggerProduction/
      mem     :   2048M
-     neventsper: 100000
-     noverflow:   20000
+     neventsper: 50000
+     noverflow:  10000
      rsync  : "ProdFlow/run2pp/TriggerProduction/*,cups.py"
 
 

From 18ff8071f1a1c47f0fd5840b45a2a16c1318d406 Mon Sep 17 00:00:00 2001
From: klendathu2k <jwebb@bnl.gov>
Date: Tue, 28 Jan 2025 16:38:52 -0500
Subject: [PATCH 20/25] Workflow and script used to fix the filenames and
 catalog entries for the DST_CALO_run2pp_ana462_2024p010_v001 production. 
 Added to the repo posthoc.

---
 .../FIX_CALO_run2pp_ana462_2024p010.yaml      |  52 +++++++
 .../fixy2calib.sh                             | 129 ++++++++++++++++++
 2 files changed, 181 insertions(+)
 create mode 100644 run2pp/FixDstCalo_ana462_2024p010_v001/FIX_CALO_run2pp_ana462_2024p010.yaml
 create mode 100644 run2pp/FixDstCalo_ana462_2024p010_v001/fixy2calib.sh

diff --git a/run2pp/FixDstCalo_ana462_2024p010_v001/FIX_CALO_run2pp_ana462_2024p010.yaml b/run2pp/FixDstCalo_ana462_2024p010_v001/FIX_CALO_run2pp_ana462_2024p010.yaml
new file mode 100644
index 0000000..991485e
--- /dev/null
+++ b/run2pp/FixDstCalo_ana462_2024p010_v001/FIX_CALO_run2pp_ana462_2024p010.yaml
@@ -0,0 +1,52 @@
+# -------------------------------------------------------------------------------------------------
+# Fix the filenames.  The output was staged with the wrong filename, based on the input file rather
+# than the logbase name.
+RENAME_CALO_run2pp_ana462_2024p010:
+   params:
+     name:       DST_CALO_run2pp
+     build:      ana.462
+     build_name: ana462
+     dbtag:      2024p010
+     version :   1
+     logbase :   RENAME_CALO_run2pp_$(build)_$(tag)_$(version)-$INT(run,{RUNFMT})-$INT(seg,{SEGFMT})
+     outbase :   DST_CALO_run2pp_$(build)_$(tag)_$(version)
+     script  :   fixy2calib.sh
+#    payload :   ./ProdFlow/run2pp/CaloProduction/
+     payload :   ./ProdFlow/run2pp/FixDstCalo_ana462_2024p010_v001/
+     mem     :   4096MB
+     neventsper: 50000
+#     rsync  : "./ProdFlow/run2pp/CaloProduction/*,cups.py"
+#     rsync  : "./ProdFlow/run2pp/test/*,cups.py"
+     rsync  : "./ProdFlow/run2pp/FixDstCalo_ana462_2024p010_v001/*,cups.py"
+
+   input:
+     db: filecatalog
+     query: |-
+         select 
+                'filecatalog/datasets'                                   as source      , 
+                runnumber                                                               , 
+                0                                                        as segment     ,
+                string_agg( distinct split_part(filename,'/',-1), ' ' )  as files       ,                   
+                'na'                                                     as fileranges 
+         from  
+                datasets
+         where 
+                dataset='ana446_2024p007' and dsttype='DST_CALO_run2pp'
+                {run_condition}
+
+                 and runnumber<=53880
+
+         group by runnumber
+           
+         {limit_condition}
+
+         ;
+   job:
+     arguments             : "$(nevents) {outbase} {logbase} $(run) $(seg) $(outdir) $(buildarg) $(tag) $(inputs) $(ranges) {neventsper} {logdir} {histdir} {PWD} {rsync}"
+     output_destination    : '{logdir}'
+     log                   : '{condor}/{logbase}.condor'
+     accounting_group      : "group_sphenix.mdc2"
+     accounting_group_user : "sphnxpro"
+     priority : '3900'
+
+
diff --git a/run2pp/FixDstCalo_ana462_2024p010_v001/fixy2calib.sh b/run2pp/FixDstCalo_ana462_2024p010_v001/fixy2calib.sh
new file mode 100644
index 0000000..2b4615b
--- /dev/null
+++ b/run2pp/FixDstCalo_ana462_2024p010_v001/fixy2calib.sh
@@ -0,0 +1,129 @@
+#!/usr/bin/bash
+
+nevents=${1}
+outbase=${2}
+logbase=${3}
+runnumber=${4}
+segment=${5}
+outdir=${6}
+build=${7/./}
+dbtag=${8}
+inputs=(`echo ${9} | tr "," " "`)  # array of input files 
+ranges=(`echo ${10} | tr "," " "`)  # array of input files with ranges appended
+neventsper=${11:-1000}
+logdir=${12:-.}
+histdir=${13:-.}
+subdir=${14}
+payload=(`echo ${15} | tr ","  " "`) # array of files to be rsynced
+#-----
+export cupsid=${@: -1}
+
+sighandler()
+{
+echo "signal handler"
+mv ${logbase}.out ${logdir#file:/}
+mv ${logbase}.err ${logdir#file:/}
+echo ./cups.py -v -r ${runnumber} -s ${segment} -d ${outbase} finished -e ${status_f4a} --nevents 0 --inc 
+     ./cups.py -v -r ${runnumber} -s ${segment} -d ${outbase} finished -e 255 --nevents 0 
+}
+
+# On evict (term,stp) or hold (kill) branch to signal handler
+trap sighandler SIGTERM SIGINT SIGKILL  
+
+{
+
+export USER="$(id -u -n)"
+export LOGNAME=${USER}
+export HOME=/sphenix/u/${USER}
+hostname
+
+source /opt/sphenix/core/bin/sphenix_setup.sh -n ${7}
+echo OFFLINE_MAIN: $OFFLINE_MAIN
+#export ODBCINI=./odbc.ini
+
+# Stagein
+for i in ${payload[@]}; do
+    cp --verbose ${subdir}/${i} .
+done
+
+if [ -e odbc.ini ]; then
+echo export ODBCINI=./odbc.ini
+     export ODBCINI=./odbc.ini
+else
+     echo No odbc.ini file detected.  Using system odbc.ini
+fi
+
+if [[ "${inputs}" == *"dbinput"* ]]; then
+    echo "Getting inputs via cups.  ranges is not set."
+    inputs=( $(./cups.py -r ${runnumber} -s ${segment} -d ${outbase} getinputs) )
+fi
+
+# Debugging info
+./cups.py -r ${runnumber} -s ${segment} -d ${outbase} info
+
+echo ..............................................................................................
+echo $@
+echo .............................................................................................. 
+echo nevents: $nevents
+echo outbase: $outbase
+echo logbase: $logbase
+echo runnumb: $runnumber
+echo segment: $segment
+echo outdir:  $outdir
+echo build:   $build
+echo dbtag:   $dbtag
+echo inputs:  ${inputs[@]}
+echo nper:    $neventsper
+echo logdir:  $logdir
+echo histdir: $histdir
+echo .............................................................................................. 
+
+#______________________________________________________________________________________ running __
+#
+./cups.py -r ${runnumber} -s ${segment} -d ${outbase} running
+#_________________________________________________________________________________________________
+
+nevents=-1
+status_f4a=0
+
+echo "The DST_CALO_run2pp_ana462_2024p010_v001 production was created with the wrong output filename."
+echo "Renaming each file for run ${runnumber}.  Catalog to be updated one all files are moved."
+
+for infile_ in ${inputs[@]}; do
+
+    base=$( basename ${infile_} )
+
+    export dst0=${infile_}
+    export dstB=${base/DST_CALO_run2pp_ana446_2024p007/${outbase}}
+    export dstF=${outdir}/${dstB}
+    
+    export hist0=${histdir}/${base/DST_CALO/HIST_CALOQA}
+    export histF=${histdir}/${dstB/DST_CALO/HIST_CALOQA}
+
+#   echo mv --verbose ${dst0}  ${dstF}
+    mv --verbose ${dst0}  ${dstF}
+#   echo mv --verbose ${hist0} ${histF}
+    mv --verbose ${hist0} ${histF}
+
+#   random sleep for up to 30s between copies
+    sleep $(( RANDOM % 30 ))
+
+done
+
+ls -lah
+
+#______________________________________________________________________________________ finished __
+echo ./cups.py -v -r ${runnumber} -s ${segment} -d ${outbase} finished -e ${status_f4a} --nevents -1 
+     ./cups.py -v -r ${runnumber} -s ${segment} -d ${outbase} finished -e ${status_f4a} --nevents -1 
+#_________________________________________________________________________________________________
+
+
+
+echo "bdee bdee bdee, That's All Folks!"
+} >  ${logdir#file:/}/${logbase}.out 2> ${logdir#file:/}/${logbase}.err
+
+if [ -e cups.stat ]; then
+    cp cups.stat ${logdir#file:/}/${logbase}.dbstat
+fi
+
+exit $status_f4a

From f52a12377a2e8d505bbc566d945295102fafa1df Mon Sep 17 00:00:00 2001
From: klendathu2k <jwebb@bnl.gov>
Date: Thu, 30 Jan 2025 16:50:29 -0500
Subject: [PATCH 21/25] Restore seperate log and error files in fitting
 workflow.

---
 run2pp/CaloProduction/runy2fitting.sh | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/run2pp/CaloProduction/runy2fitting.sh b/run2pp/CaloProduction/runy2fitting.sh
index 9921d1b..d288de1 100644
--- a/run2pp/CaloProduction/runy2fitting.sh
+++ b/run2pp/CaloProduction/runy2fitting.sh
@@ -125,8 +125,7 @@ echo ./cups.py -v -r ${runnumber} -s ${segment} -d ${outbase} finished -e ${stat
 
 echo "bdee bdee bdee, That's All Folks!"
 
-} | tee ${logdir#file:/}/${logbase}.out 
-#> ${logdir#file:/}/${logbase}.out 2> ${logdir#file:/}/${logbase}.err
+} > ${logdir#file:/}/${logbase}.out 2> ${logdir#file:/}/${logbase}.err
 
 if [ -e cups.stat ]; then
     cp cups.stat ${logdir#file:/}/${logbase}.dbstat

From 65782c225bafc28c1b8ec61665454db3d9bea274 Mon Sep 17 00:00:00 2001
From: klendathu2k <jwebb@bnl.gov>
Date: Thu, 30 Jan 2025 20:51:30 -0500
Subject: [PATCH 22/25] Update payload, process all events in input, and use
 the new event production.

---
 run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml b/run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml
index 18307f6..15873b6 100644
--- a/run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml
+++ b/run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml
@@ -357,7 +357,7 @@ DST_TRIGGERED_EVENT_run2pp_ana462_2024p010:
 XXX_CALOFITTING_run2pp_ana462_2024p010:
    params:
      name:       DST_CALOFITTING_run2pp
-     nevents:    5000
+     nevents:    0
      build:      ana.462
      build_name: ana462
      dbtag:      2024p010
@@ -368,7 +368,7 @@ XXX_CALOFITTING_run2pp_ana462_2024p010:
      payload :   ./ProdFlow/run2pp/CaloProduction/
      mem     :   1024MB
      neventsper: 50000
-     rsync  : "./ProdFlow/run2pp/CaloProduction/*,cups.py,bachi.py,odbc.ini"
+     rsync  : "./ProdFlow/run2pp/CaloProduction/*,cups.py"
 
    input:
      db: filecatalog
@@ -382,7 +382,7 @@ XXX_CALOFITTING_run2pp_ana462_2024p010:
          from  
                 datasets
          where 
-                filename like 'DST_TRIGGERED_EVENT_run2pp_ana451_2024p009%'
+                filename like 'DST_TRIGGERED_EVENT_run2pp_ana462_nocdbtag%'
                 {run_condition}
 
                  and runnumber>53880

From 9e58f1550c3d290c20d0a9c33e592f3a269c1e59 Mon Sep 17 00:00:00 2001
From: klendathu2k <jwebb@bnl.gov>
Date: Thu, 30 Jan 2025 20:56:35 -0500
Subject: [PATCH 23/25] ... fix run range for pp.

---
 run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml b/run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml
index 15873b6..9ef5f71 100644
--- a/run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml
+++ b/run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml
@@ -354,7 +354,7 @@ DST_TRIGGERED_EVENT_run2pp_ana462_2024p010:
 
 
 # Downstream products
-XXX_CALOFITTING_run2pp_ana462_2024p010:
+DST_CALOFITTING_run2pp_ana462_2024p010:
    params:
      name:       DST_CALOFITTING_run2pp
      nevents:    0
@@ -385,7 +385,7 @@ XXX_CALOFITTING_run2pp_ana462_2024p010:
                 filename like 'DST_TRIGGERED_EVENT_run2pp_ana462_nocdbtag%'
                 {run_condition}
 
-                 and runnumber>53880
+                 and runnumber<=53880
            
          {limit_condition}
 

From 95d10d099c4cc8a0fbfadd0c25cd70fd55280aa8 Mon Sep 17 00:00:00 2001
From: klendathu2k <jwebb@bnl.gov>
Date: Fri, 21 Feb 2025 14:01:04 -0500
Subject: [PATCH 24/25] Updated inputs and versioning

---
 run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml b/run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml
index 9ef5f71..04efc04 100644
--- a/run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml
+++ b/run2pp/DST_TRIGGERED_EVENT_run2pp_ana462_2024p010.yaml
@@ -13,7 +13,7 @@ DST_CALO_run2pp_new_2024p010:
      build:      ana.462
      build_name: ana462
      dbtag:      2024p010
-     version :   1
+     version :   2
      logbase :   $(name)_$(build)_$(tag)_$(version)-$INT(run,{RUNFMT})-$INT(seg,{SEGFMT})
      outbase :   $(name)_$(build)_$(tag)_$(version)
      script  :   runy2calib.sh
@@ -34,7 +34,7 @@ DST_CALO_run2pp_new_2024p010:
          from  
                 datasets
          where 
-                dataset='ana446_2024p007' and dsttype='DST_CALOFITTING_run2pp'
+                dataset='ana472_2024p010_v001' and dsttype='DST_CALOFITTING_run2pp'
                 {run_condition}
 
                  and runnumber<=53880
@@ -104,14 +104,14 @@ DST_JETCALO_run2pp_ana462_2024p010:
      build:      ana.462
      build_name: ana462
      dbtag:      2024p010
-     version:    1
+     version:    2
      logbase :   $(name)_$(build)_$(tag)_$(version)-$INT(run,{RUNFMT})-$INT(seg,{SEGFMT})
      outbase :   $(name)_$(build)_$(tag)_$(version)
      script  :   runy2jetskim.sh
      payload :   ./ProdFlow/run2pp/JetProduction/
      mem     :   4096MB
      neventsper: 50000
-     rsync  : "./ProdFlow/run2pp/JetProduction/*,cups.py,bachi.py"
+     rsync  : "./ProdFlow/run2pp/JetProduction/*,cups.py"
 
    input:
      db: filecatalog
@@ -125,7 +125,7 @@ DST_JETCALO_run2pp_ana462_2024p010:
          from
                 datasets
          where
-                dataset='ana446_2024p007' and dsttype='DST_CALOFITTING_run2pp'
+                dataset='ana462_2024p010_v001' and dsttype='DST_CALOFITTING_run2pp'
                 {run_condition}
 
                  and runnumber<=53880

From 6bacd0db2305d767ce1c5c6adee223ba2c627d1a Mon Sep 17 00:00:00 2001
From: klendathu2k <jwebb@bnl.gov>
Date: Fri, 21 Feb 2025 14:02:20 -0500
Subject: [PATCH 25/25] Run calo fitting w/ ana468 2024p012

---
 ...ST_CALOFITTING_run2pp_ana468_2024p012.yaml | 194 ++++++++++++++++++
 1 file changed, 194 insertions(+)
 create mode 100644 run2pp/DST_CALOFITTING_run2pp_ana468_2024p012.yaml

diff --git a/run2pp/DST_CALOFITTING_run2pp_ana468_2024p012.yaml b/run2pp/DST_CALOFITTING_run2pp_ana468_2024p012.yaml
new file mode 100644
index 0000000..c219c08
--- /dev/null
+++ b/run2pp/DST_CALOFITTING_run2pp_ana468_2024p012.yaml
@@ -0,0 +1,194 @@
+#
+#FileCatalog=# select dataset,dsttype,count(filename),sum(events) from datasets where dsttype='DST_CALOFITTING_run2pp' group by dataset,dsttype;
+#     dataset     |        dsttype         | count  |     sum     
+#-----------------+------------------------+--------+-------------
+# ana451_2024p009 | DST_CALOFITTING_run2pp |  50343 |  4976087047
+# ana446_2024p007 | DST_CALOFITTING_run2pp | 148286 | 14632177735
+#(2 rows)
+
+# Downstream products
+DST_CALO_run2pp_new_2024p010:
+   params:
+     name:       DST_CALO_run2pp
+     build:      ana.462
+     build_name: ana462
+     dbtag:      2024p010
+     version :   2
+     logbase :   $(name)_$(build)_$(tag)_$(version)-$INT(run,{RUNFMT})-$INT(seg,{SEGFMT})
+     outbase :   $(name)_$(build)_$(tag)_$(version)
+     script  :   runy2calib.sh
+     payload :   ./ProdFlow/run2pp/CaloProduction/
+     mem     :   4096MB
+     neventsper: 50000
+     rsync  : "./ProdFlow/run2pp/CaloProduction/*,cups.py,bachi.py,odbc.ini"
+
+   input:
+     db: filecatalog
+     query: |-
+         select 
+                'filecatalog/datasets'                                  as source      , 
+                runnumber                                                              , 
+                segment                                                                ,
+                filename                                                as files       ,
+                filename || ':' || 0 || ':' || events                   as fileranges 
+         from  
+                datasets
+         where 
+                dataset='ana472_2024p010_v001' and dsttype='DST_CALOFITTING_run2pp'
+                {run_condition}
+
+                 and runnumber<=53880
+           
+         {limit_condition}
+
+         ;
+   job:
+     arguments             : "$(nevents) {outbase} {logbase} $(run) $(seg) $(outdir) $(buildarg) $(tag) $(inputs) $(ranges) {neventsper} {logdir} {histdir} {PWD} {rsync}"
+     output_destination    : '{logdir}'
+     log                   : '{condor}/{logbase}.condor'
+     accounting_group      : "group_sphenix.mdc2"
+     accounting_group_user : "sphnxpro"
+     priority : '3900'
+
+
+# Downstream products
+DST_JETS_run2pp_new_2024p010:
+   params:
+     name:       DST_JETS_run2pp
+     build:      ana.462
+     build_name: ana462
+     dbtag:      2024p010
+     version : 1
+     logbase :   $(name)_$(build)_$(tag)_$(version)-$INT(run,{RUNFMT})-$INT(seg,{SEGFMT})
+     outbase :   $(name)_$(build)_$(tag)_$(version)
+     script  :   runjets.sh
+     payload :   ./ProdFlow/run2pp/JetProduction/
+     mem     :   2096MB
+     neventsper: 50000
+     rsync  : "./ProdFlow/run2pp/JetProduction/*,cups.py,bachi.py,odbc.ini"
+
+   input:
+     db: filecatalog
+     query: |-
+         select 
+                'filecatalog/datasets'                                  as source      , 
+                runnumber                                                              , 
+                segment                                                                ,
+                filename                                                as files       ,
+                'X'                   as fileranges 
+         from  
+                datasets
+         where 
+                filename like 'DST_CALO_run2pp_ana462_2024p010%'
+                {run_condition}
+                and runnumber>53880
+           
+         {limit_condition}
+
+         ;
+   job:
+     arguments             : "$(nevents) {outbase} {logbase} $(run) $(seg) $(outdir) $(buildarg) $(tag) $(inputs) $(ranges) {neventsper} {logdir} {histdir} {PWD} {rsync}"
+     output_destination    : '{logdir}'
+     log                   : '{condor}/{logbase}.condor'
+     accounting_group      : "group_sphenix.mdc2"
+     accounting_group_user : "sphnxpro"
+     priority : '3900'
+
+
+
+
+# Downstream products
+DST_JETCALO_run2pp_ana462_2024p010:
+   params:
+     name:       DST_JETCALO_run2pp
+     build:      ana.462
+     build_name: ana462
+     dbtag:      2024p010
+     version:    2
+     logbase :   $(name)_$(build)_$(tag)_$(version)-$INT(run,{RUNFMT})-$INT(seg,{SEGFMT})
+     outbase :   $(name)_$(build)_$(tag)_$(version)
+     script  :   runy2jetskim.sh
+     payload :   ./ProdFlow/run2pp/JetProduction/
+     mem     :   4096MB
+     neventsper: 50000
+     rsync  : "./ProdFlow/run2pp/JetProduction/*,cups.py"
+
+   input:
+     db: filecatalog
+     query: |-
+         select
+                'filecatalog/datasets'                                  as source      ,
+                runnumber                                                              ,
+                segment                                                                ,
+                filename                                                as files       ,
+                filename || ':' || 0 || ':' || events                   as fileranges
+         from
+                datasets
+         where
+                dataset='ana462_2024p010_v001' and dsttype='DST_CALOFITTING_run2pp'
+                {run_condition}
+
+                 and runnumber<=53880
+
+         {limit_condition}
+
+         ;
+   job:
+     arguments             : "$(nevents) {outbase} {logbase} $(run) $(seg) $(outdir) $(buildarg) $(tag) $(inputs) $(ranges) {neventsper} {logdir} {histdir} {PWD} {rsync}"
+     output_destination    : '{logdir}'
+     log                   : '{condor}/{logbase}.condor'
+     accounting_group      : "group_sphenix.mdc2"
+     accounting_group_user : "sphnxpro"
+     priority : '3900'
+
+
+
+
+
+# Downstream products
+DST_CALOFITTING_run2pp_ana468_2024p012:
+   params:
+     name:       DST_CALOFITTING_run2pp
+     nevents:    0
+     build:      ana.468
+     build_name: ana468
+     dbtag:      2024p012
+     version : 1
+     logbase :   $(name)_$(build)_$(tag)_$(version)-$INT(run,{RUNFMT})-$INT(seg,{SEGFMT})
+     outbase :   $(name)_$(build)_$(tag)_$(version)
+     script  :   runy2fitting.sh
+     payload :   ./ProdFlow/run2pp/CaloProduction/
+     mem     :   1024MB
+     neventsper: 50000
+     rsync  : "./ProdFlow/run2pp/CaloProduction/*,cups.py"
+
+   input:
+     db: filecatalog
+     query: |-
+         select 
+                'filecatalog/datasets'                                  as source      , 
+                runnumber                                                              , 
+                segment                                                                ,
+                filename                                                as files       ,
+                filename || ':' || 0 || ':' || events                   as fileranges 
+         from  
+                datasets
+         where 
+                filename like 'DST_TRIGGERED_EVENT_run2pp_ana462_nocdbtag%'
+                {run_condition}
+
+                 and runnumber<=53880
+           
+         {limit_condition}
+
+         ;
+   job:
+     arguments             : "{nevents} {outbase} {logbase} $(run) $(seg) $(outdir) $(buildarg) $(tag) $(inputs) $(ranges) {neventsper} {logdir} {histdir} {PWD} {rsync}"
+     output_destination    : '{logdir}'
+     log                   : '{condor}/{logbase}.condor'
+     accounting_group      : "group_sphenix.mdc2"
+     accounting_group_user : "sphnxpro"
+     priority : '3900'
+
+
+