add job_status.log

Author: s-poll

.gitignore

@@ -1,5 +1,6 @@
 # logs
 ctl/logs
+ctl/job_status.log
 # run directories
 run/
 # data directories

ctl/control_tsmp2.sh

@@ -62,6 +62,9 @@ pfl_node=2
 # DebugMode: No job submission. Just config
 debugmode=false
 
+# log job status
+joblog=true
+
 ###########################################
 
 ###
@@ -182,8 +185,6 @@ else
   source ${ctl_dir}/pre_ctl/pre.job
 fi
 
-echo $submit_pre" for preprocessing"
-
 # get jobid
 pre_id=$(echo $submit_pre | awk 'END{print $(NF)}')
 

ctl/pos_ctl/pos.job

@@ -18,6 +18,7 @@ IFS=' ' read -r -a lvis <<< "$lvisstr"
 ###
 # source environment
 ###
+timestart=$(date +%s)
 source ${tsmp2_env} # should be changed to post-processing env.
 
 ###
@@ -57,4 +58,10 @@ pos_cleanup
 
 fi # cleanup
 
+###
+# Log job
+###
+timeend=$(date +%s)
+logging_job_status "pos"
+
 exit 0

ctl/pre_ctl/pre.job

@@ -18,6 +18,7 @@ IFS=' ' read -r -a lvis <<< "$lvisstr"
 ###
 # source environment
 ###
+timestart=$(date +%s)
 #source ${tsmp2_env} # chang pre-processing env. according to comp. model
 
 ###
@@ -57,4 +58,10 @@ pre_cleanup
 
 fi # cleanup
 
+###
+# Log job
+###
+timeend=$(date +%s)
+logging_job_status "pre"
+
 exit 0

ctl/sim_ctl/sim.job

@@ -18,6 +18,7 @@ IFS=' ' read -r -a lvis <<< "$lvisstr"
 ###
 # source environment
 ###
+timestart=$(date +%s)
 source ${tsmp2_env}
 
 ###
@@ -57,4 +58,10 @@ sim_cleanup
 
 fi # cleanup
 
+###
+# Log simulation
+###
+timeend=$(date +%s)
+logging_job_status "sim"
+
 exit 0

ctl/sim_ctl/sim_cleanup.sh

@@ -96,7 +96,8 @@ cp -v ${sim_dir}/slm_multiprog_mapping.conf ${simout_dir}/log/
 # sim logs
 mv ${log_dir}/${SLURM_JOB_NAME}_${SLURM_JOB_ID}.{err,out} ${simout_dir}/log/.
 
-#echo "SLURM node ID LIST:"${SLURM_JOB_NODELIST}
+# job info
+echo $(scontrol show job ${SLURM_JOB_ID}) > ${simout_dir}/log/job_info.log
 
 # remove run directory
 rm -rf ${sim_dir:?}

ctl/utils_tsmp2.sh

@@ -83,3 +83,14 @@ check_var_def() {
     fi
   fi
 } # check_var_def
+
+logging_job_status(){
+  local step="$1"
+
+  if [ "$joblog" = true ]; then
+    job_state=$(scontrol show job $SLURM_JOB_ID | grep "JobState=" | cut -d= -f2 | cut -d' ' -f1)
+    printf "%10s %3s %15s %14s %10s %14s %9s\n" "${expid}" "${step}" "${modelid}" \
+        "${dateshort}" "${job_state}" "$(date '+%Y%m%d%H%M%S')" $(date -u -d "0 $timeend sec - $timestart sec" +"%H:%M:%S") \
+        >> ${ctl_dir}/job_status.log
+  fi
+} # logging_job_status

ctl/vis_ctl/vis.job

@@ -18,6 +18,7 @@ IFS=' ' read -r -a lvis <<< "$lvisstr"
 ###
 # source environment
 ###
+timestart=$(date +%s)
 source ${tsmp2_env} # should be changed to visualisation env.
 
 ###
@@ -57,4 +58,10 @@ vis_cleanup
 
 fi # cleanup
 
+###
+# Log job
+###
+timeend=$(date +%s)
+logging_job_status "vis"
+
 exit 0