Prod trigger run2 ana462 2024p010 v001

run2auau/CaloProduction/runy2calib.sh

@@ -35,6 +35,7 @@ trap sighandler SIGTERM SIGINT SIGKILL
 export USER="$(id -u -n)"
 export LOGNAME=${USER}
 export HOME=/sphenix/u/${USER}
+echo "cupsid=${cupsid}"
 hostname
 
 source /opt/sphenix/core/bin/sphenix_setup.sh -n ${7}
@@ -96,7 +97,7 @@ status_f4a=0
 for infile_ in ${inputs[@]}; do
     infile=$( basename ${infile_} )
     cp -v ${infile_} .
-    outfile=${infile/CALOFITTING/CALO}
+    outfile=${logbase}.root
     outhist=${outfile/DST_CALO/HIST_CALOQA}
     root.exe -q -b Fun4All_Year2_Calib.C\(${nevents},\"${infile}\",\"${outfile}\",\"${outhist}\",\"${dbtag}\"\);  status_f4a=$?
 

run2auau/DST_TRIGGERED_EVENT_run2auau_ana451_2024p009.yaml

@@ -0,0 +1,350 @@
+#
+# Testbed for file ranges
+#
+DST_TRIGGERED_EVENT_run2auau_ana451_2024p009:
+   params:
+     name:       DST_TRIGGERED_EVENT_run2auau
+     build:      ana.451
+     build_name: ana451
+     dbtag:      2024p009
+     logbase :   $(name)_$(build)_$(tag)-$INT(run,{RUNFMT})-$INT(seg,{SEGFMT})
+     outbase :   $(name)_$(build)_$(tag)
+     script  :   run.sh
+     payload :   ./ProdFlow/run2auau/TriggerProduction/
+     mem     :   2048M
+     neventsper: 5000
+     noverflow:  1000
+     rsync  : "ProdFlow/run2auau/TriggerProduction/*,cups.py,bachi.py,odbc.ini"
+
+
+
+   #
+   # input query:
+   #
+   # This builds a list of all runs known to the file catalog "datasets" table.
+   # The query should return:
+   # 1. The source of the information (formatted as database name/table name)
+   # 2. The run number
+   # 3. A sequence number (a placeholder fixed at zero for event builders)
+   # 4. And a space-separated list of logical filenames 
+   #
+   # The {*_condition} parameters (run, seg, limit) are substituted by kaedama
+   # based on (optional) command line options.
+   #
+   # For now, require GL1 files...
+   #
+   input:
+      db: daqdb
+      direct_path: /sphenix/lustre01/sphnxpro/physics/*/*/
+      query: |-
+         with config as (
+            select 
+               {neventsper} as nevents,
+               {noverflow} as noverflow
+         ),
+
+         fullevents as (
+              select runnumber,max(lastevent)    as lastevent
+              from filelist, config
+              where (filename similar to '/bbox%/GL1_(beam|physics)%.evt'        and lastevent>2 )
+
+                 {run_condition} and runnumber>54748
+
+              group by runnumber
+              having
+                     every(transferred_to_sdcc)   and
+                     max(lastevent)>1000          and
+                     sum( case when filename similar to '/bbox%/GL1_(beam|physics)%' then 1 else 0 end )>0
+
+              order by runnumber
+         ),
+
+         fulljobs as (
+         select 
+                'daqdb/filelist'                                                                                    as source      , 
+                runnumber                                                                                                          , 
+                0                                                                                                   as segment     , 
+                string_agg( split_part(filename,'/',-1), ' ' order by filename)                                             as files       ,   
+                string_agg( firstevent::text || ':' || lastevent::text || ':' || split_part(filename,'/',-1), ' ' order by filename )                    as fileranges  
+
+         from  
+                filelist,config
+         where 
+           ( 
+             (filename  similar to '/bbox%/%emcal%(beam|physics)%.prdf'    and lastevent>2 ) or 
+             (filename  similar to '/bbox%/%HCal%(beam|physics)%.prdf'     and lastevent>2 ) or
+             (filename  similar to '/bbox%/%LL1%(beam|physics)%.prdf'      and lastevent>2 ) or
+             (filename  similar to '/bbox%/GL1_(beam|physics)%.evt'       and lastevent>2 ) or
+             (filename  similar to '/bbox%/%mbd%(beam|physics)%.prdf'      and lastevent>2 ) or
+             (filename  similar to '/bbox%/%ZDC%(beam|physics)%.prdf'      and lastevent>2 ) 
+
+           )
+
+           {run_condition}
+
+                 and runnumber>53880
+
+         group by runnumber
+
+         having
+                every(transferred_to_sdcc)   and
+                max(lastevent)>1000          and
+                sum( case when filename similar to '/bbox%/GL1_(beam|physics)%' then 1 else 0 end )>0 and
+                (
+                   sum( case when filename similar to '/bbox%/%emcal%(beam|physics)%'  then 1 else 0 end )>0 or
+                   sum( case when filename similar to '/bbox%/%HCal%(beam|physics)%'   then 1 else 0 end )>0 
+                )
+
+         order by runnumber
+         ),
+
+         fullrun as (
+
+             select fulljobs.*,fullevents.lastevent from fulljobs join fullevents on fulljobs.runnumber=fullevents.runnumber
+
+         ),
+
+         unsegmented as ( 
+
+             select *,'full run' as runtype from fullrun where true 
+         ),
+
+         segmented as (
+
+            select source, 
+                   runnumber,
+                   generate_series(0,unsegmented.lastevent/config.nevents) as segment,
+                   files,
+                   fileranges,
+                   lastevent,
+                   nevents
+
+             from unsegmented,config
+
+         ),
+
+         segmentedA as ( 
+
+         select source,
+                runnumber,
+                segment,
+                files,
+                fileranges,
+                lastevent, 
+                segment*config.nevents as myfirstevent,
+                least( (segment+1)*config.nevents - 1, lastevent ) as mylastevent,
+                config.nevents 
+
+                from segmented,config
+
+          ),
+
+
+          segmentedB as (
+
+               select *,
+                      mylastevent-myfirstevent as nprocessing,
+                      lastevent-mylastevent-1  as nremaining from segmentedA,config  -- b/c we count from zero
+
+          ),
+
+          segmentedC as (
+
+
+               select source, runnumber,
+                      segment,
+                      files,
+                      fileranges,
+                      myfirstevent,
+                      (
+                          select 
+                             case
+                                when nprocessing<config.noverflow  then -1
+                                when nremaining>=config.nevents    then mylastevent
+                                when nremaining>=config.noverflow  then mylastevent
+                                when nremaining<config.noverflow   then lastevent
+                                else                                    -2             -- unhandled, should be an error condition
+                             end
+                      ) as mylastevent,
+                      lastevent,
+                      nprocessing,
+                      nremaining
+
+                      from segmentedB,config
+
+          )
+
+
+          select 
+               source,
+                 runnumber,
+                 segment,
+                 files,
+                 fileranges,
+
+                 (select count(*) from regexp_matches( files,      ' ', 'g' ))::integer =
+                 (select count(*) from regexp_matches( fileranges, ' ', 'g' ))::integer as sanity,
+                 myfirstevent as firstevent,
+                 mylastevent  as lastevent,
+                 lastevent    as runs_last_event
+
+                 from segmentedC
+
+                 where runnumber>53880
+
+          order by runnumber desc, segment desc;
+
+
+   #
+   # Again I note the need to ensure that the arguments are properly specified given the
+   # definition of the payload script.
+   #
+   job:
+     executable            : "{payload}/run.sh"
+     arguments             : "$(nevents) {outbase} {logbase} $(run) $(seg) $(outdir) $(buildarg) $(tag) $(inputs) $(ranges) {neventsper} {logdir} {histdir} {PWD} {rsync} $(firstevent) $(lastevent) $(runs_last_event)"
+     output_destination    : '{logdir}'
+     log                   : '{condor}/{logbase}.condor'
+     accounting_group      : "group_sphenix.mdc2"
+     accounting_group_user : "sphnxpro"
+     priority : '3900'
+     request_xferslots: '1'
+
+
+
+
+
+# Downstream products
+DST_CALOFITTING_run2auau_ana451_2024p009:
+   params:
+     name:       DST_CALOFITTING_run2auau
+     nevents:    5000
+     build:      ana.451
+     build_name: ana451
+     dbtag:      2024p009
+     logbase :   $(name)_$(build)_$(tag)-$INT(run,{RUNFMT})-$INT(seg,{SEGFMT})
+     outbase :   $(name)_$(build)_$(tag)
+     script  :   runy2fitting.sh
+     payload :   ./ProdFlow/run2auau/CaloProduction/
+     mem     :   1024MB
+     neventsper: 50000
+     rsync  : "./ProdFlow/run2auau/CaloProduction/*,cups.py,bachi.py,odbc.ini"
+
+   input:
+     db: filecatalog
+     query: |-
+         select 
+                'filecatalog/datasets'                                  as source      , 
+                runnumber                                                              , 
+                segment                                                                ,
+                filename                                                as files       ,
+                filename || ':' || 0 || ':' || events                   as fileranges 
+         from  
+                datasets
+         where 
+                filename like 'DST_TRIGGERED_EVENT_run2auau_ana451_2024p009%'
+                {run_condition}
+
+                 and runnumber>53880
+
+         {limit_condition}
+
+         ;
+   job:
+     executable            : "{payload}/runy2fitting.sh"
+     arguments             : "{nevents} {outbase} {logbase} $(run) $(seg) $(outdir) $(buildarg) $(tag) $(inputs) $(ranges) {neventsper} {logdir} {histdir} {PWD} {rsync}"
+     output_destination    : '{logdir}'
+     log                   : '{condor}/{logbase}.condor'
+     accounting_group      : "group_sphenix.mdc2"
+     accounting_group_user : "sphnxpro"
+     priority : '3900'
+
+
+
+# Downstream products
+DST_CALO_run2auau_new_2024p007:
+   params:
+     name:       DST_CALO_run2auau
+     build:      new
+     build_name: new
+     dbtag:      2024p007
+     logbase :   $(name)_$(build)_$(tag)-$INT(run,{RUNFMT})-$INT(seg,{SEGFMT})
+     outbase :   $(name)_$(build)_$(tag)
+     script  :   runy2calib.sh
+     payload :   ./ProdFlow/run2auau/CaloProduction/
+     mem     :   4096MB
+     neventsper: 50000
+     rsync  : "./ProdFlow/run2auau/CaloProduction/*,cups.py,bachi.py,odbc.ini"
+
+   input:
+     db: filecatalog
+     query: |-
+         select 
+                'filecatalog/datasets'                                  as source      , 
+                runnumber                                                              , 
+                segment                                                                ,
+                filename                                                as files       ,
+                filename || ':' || 0 || ':' || events                   as fileranges 
+         from  
+                datasets
+         where 
+                filename like 'DST_CALOFITTING_run2auau_new_2024p007%'
+                {run_condition}
+
+                 and runnumber>53880
+
+         {limit_condition}
+
+         ;
+   job:
+     executable            : "{payload}/runy2calib.sh"
+     arguments             : "$(nevents) {outbase} {logbase} $(run) $(seg) $(outdir) $(buildarg) $(tag) $(inputs) $(ranges) {neventsper} {logdir} {histdir} {PWD} {rsync}"
+     output_destination    : '{logdir}'
+     log                   : '{condor}/{logbase}.condor'
+     accounting_group      : "group_sphenix.mdc2"
+     accounting_group_user : "sphnxpro"
+     priority : '3900'
+
+
+# Downstream products
+DST_JETS_run2auau_new_2024p007:
+   params:
+     name:       DST_JETS_run2auau
+     build:      new
+     build_name: new
+     dbtag:      2024p007
+     logbase :   $(name)_$(build)_$(tag)-$INT(run,{RUNFMT})-$INT(seg,{SEGFMT})
+     outbase :   $(name)_$(build)_$(tag)
+     script  :   runjets.sh
+     payload :   ./ProdFlow/run2auau/JetProduction/
+     mem     :   2096MB
+     neventsper: 50000
+     rsync  : "./ProdFlow/run2auau/JetProduction/*,cups.py,bachi.py,odbc.ini"
+
+   input:
+     db: filecatalog
+     query: |-
+         select 
+                'filecatalog/datasets'                                  as source      , 
+                runnumber                                                              , 
+                segment                                                                ,
+                filename                                                as files       ,
+                'X'                   as fileranges 
+         from  
+                datasets
+         where 
+                filename like 'DST_CALO_run2auau_new_2024p007%'
+                {run_condition}
+                and runnumber>53880
+
+         {limit_condition}
+
+         ;
+   job:
+     executable            : "{payload}/runjets.sh"
+     arguments             : "$(nevents) {outbase} {logbase} $(run) $(seg) $(outdir) $(buildarg) $(tag) $(inputs) $(ranges) {neventsper} {logdir} {histdir} {PWD} {rsync}"
+     output_destination    : '{logdir}'
+     log                   : '{condor}/{logbase}.condor'
+     accounting_group      : "group_sphenix.mdc2"
+     accounting_group_user : "sphnxpro"
+     priority : '3900'
+