Source code for lst_auto_rta.Auto_Check

#!/usr/bin/env python

import datetime
import logging
import os
import time

import pymongo


[docs] def now(): """ Returns the current timestamp in seconds, relative to the Unix epoch. :return: float """ return datetime.datetime.timestamp(datetime.datetime.now())
[docs] def get_current_run(): """ Attempts to retrieve the current run number from the lst1_obs_summary database. :return: int """ try: client = pymongo.MongoClient("lst101") database = client["lst1_obs_summary"] camera_collection = database["camera"] summaries = camera_collection.find({}) run_table = [] summaries = camera_collection.find({}) for summary in summaries: if summary["tstart"] > datetime.datetime.timestamp(datetime.datetime.now()) - 3600 * 24 * 100: run_table.append(summary["run_number"]) if run_table[-2] != run_table[-1]: return run_table[-1] else: return run_table[-1] + 1 except Exception as e: logging.error("Error DB !!!!!") return -1
[docs] def get_night_timestamp(today): """ Get current timestamp (YYYYMMDD) with respect to the night. It will consider a same night everything run between 08h00 until 07h59 of the next day. Ex: An observation launched at 04h00 (local time) will refer to the previous day. :param today: datetime object datetime object with current local time :return: timestamp: str """ # TODO check that today is a datetime object ? if today.hour < 8: # script launched between yesterday = today - datetime.timedelta(days=1) timestamp = f"{yesterday.year:04d}{yesterday.month:02d}{yesterday.day:02d}" else: timestamp = f"{today.year:04d}{today.month:02d}{today.day:02d}" return timestamp
[docs] def today_to_directory(today): return today[0:4]+"/"+today[4:6]+"/"+today[6:8]
[docs] def main(): """ Initialize current_run_is and quit variables and get the current timestamp with Start_time = now() Initialize the logging system by setting the log file name and logging level Change the current working directory to /fefs/onsite/pipeline/rta/data Find the current reservations for nodes using the scontrol command and parse the output to find the individual node names Modify a configuration file with sed commands, replacing placeholders with the node names found in step 4 Check if the ib0 network interface is in connected mode for each of the nodes and log the results """ current_run_is = -1 quit = False Start_time = now() # today = str(datetime.datetime.now().year) + str(datetime.datetime.now().month) + str(datetime.datetime.now().day) today = get_night_timestamp(datetime.datetime.now()) logging.basicConfig(filename="log_AutoCheck_" + today + ".txt", level=logging.INFO) logging.info("Start RTA Auto Check for the day " + today) RTA_ready = True loop_id = 0 data_directory = "'/fefs/onsite/pipeline/rta/data/" os.system("rm -f /fefs/onsite/pipeline/rta/data/plots/*") while quit == False and RTA_ready: time.sleep(10) # print(current_run_is+1) if current_run_is == -1: current_run_is = get_current_run() if current_run_is == -1: logging.info("None") current_run_is = -1 continue if current_run_is != get_current_run(): time.sleep(1) current_run_is = get_current_run() if current_run_is != -1: logging.info("Start RTA check for run " + str(current_run_is)) time.sleep(2) logging.info(datetime.datetime.now()) logging.info( "srun --reservation=rta_one_node ./Auto_Check_DL1.py -da " + today_to_directory(today) + " -r " + str(current_run_is + 1) ) os.system( "srun --reservation=rta_one_node ./Auto_Check_DL1.py -da " + today_to_directory(today) + " -r " + str(current_run_is + 1) ) os.system( "convert -density 300 /fefs/onsite/pipeline/rta/data/" + today_to_directory(today) + "/" + str(current_run_is + 1) + "/plots/output_DL1.pdf /fefs/onsite/pipeline/rta/data/" + today_to_directory(today) + "/" + str(current_run_is + 1) + "/plots/Check_DL1.png" ) logging.info(datetime.datetime.now()) logging.info( "srun --reservation=rta_one_node ./Auto_Check_DL2.py -da " + today_to_directory(today) + " -r " + str(current_run_is + 1) ) os.system( "srun --reservation=rta_one_node ./Auto_Check_DL2.py -da " + today_to_directory(today) + " -r " + str(current_run_is + 1) ) os.system( "convert -density 300 /fefs/onsite/pipeline/rta/data/" + today_to_directory(today) + "/" + str(current_run_is + 1) + "/plots/output_DL2.pdf /fefs/onsite/pipeline/rta/data/" + today_to_directory(today) + "/" + str(current_run_is + 1) + "/plots/Check_DL2.png" ) logging.info(datetime.datetime.now()) logging.info( "srun --reservation=rta_one_node ./merge_DL3.py --input-filter 'dl3_v06*' -d '/fefs/onsite/pipeline/rta/data/" + today_to_directory(today) + "/" + str(current_run_is + 1) + "/DL3/' -o '/fefs/onsite/pipeline/rta/data/" + today_to_directory(today) + "/" + str(current_run_is + 1) + "/DL3/' -r " + str(current_run_is + 1) ) os.system( "srun --reservation=rta_one_node ./merge_DL3.py --input-filter 'dl3_v06*' -d '/fefs/onsite/pipeline/rta/data/" + today_to_directory(today) + "/" + str(current_run_is + 1) + "/DL3/' -o '/fefs/onsite/pipeline/rta/data/" + today + "/" + str(current_run_is + 1) + "/DL3/' -r " + str(current_run_is + 1) ) time.sleep(10) os.system( "srun --reservation=rta_one_node ./merge_DL3.py --input-filter 'dl3_v06*' -d '/fefs/onsite/pipeline/rta/data/" + today_to_directory(today) + "/" + str(current_run_is) + "/DL3/' -o '/fefs/onsite/pipeline/rta/data/" + today_to_directory(today) + "/" + str(current_run_is) + "/DL3/' -r " + str(current_run_is) ) logging.info( "srun --reservation=rta_one_node ./merge_DL3.py --input-filter 'dl3_v06*' -d '/fefs/onsite/pipeline/rta/data/" + today_to_directory(today) + "/" + str(current_run_is + 1) + "/DL3/' -o '/fefs/onsite/pipeline/rta/data/" + today_to_directory(today) + "/" + str(current_run_is + 1) + "/DL3/' -r " + str(current_run_is + 1) ) if loop_id % 1 == 0: logging.info("Start RTA check for run " + str(current_run_is)) time.sleep(2) logging.info(datetime.datetime.now()) logging.info( "srun --reservation=rta_one_node ./Auto_Check_DL1.py -da " + today_to_directory(today) + " -r " + str(current_run_is + 1) ) os.system( "srun --reservation=rta_one_node ./Auto_Check_DL1.py -da " + today_to_directory(today) + " -r " + str(current_run_is + 1) ) os.system( "convert -density 300 /fefs/onsite/pipeline/rta/data/" + today_to_directory(today) + "/" + str(current_run_is + 1) + "/plots/output_DL1.pdf /fefs/onsite/pipeline/rta/data/" + today_to_directory(today) + "/" + str(current_run_is + 1) + "/Check_DL1.png" ) logging.info(datetime.datetime.now()) logging.info( "srun --reservation=rta_one_node ./Auto_Check_DL2.py -da " + today_to_directory(today) + " -r " + str(current_run_is + 1) ) os.system( "srun --reservation=rta_one_node ./Auto_Check_DL2.py -da " + today_to_directory(today) + " -r " + str(current_run_is + 1) ) os.system( "convert -density 300 /fefs/onsite/pipeline/rta/data/" + today_to_directory(today) + "/" + str(current_run_is + 1) + "/plots/output_DL2.pdf /fefs/onsite/pipeline/rta/data/" + today_to_directory(today) + "/" + str(current_run_is + 1) + "/plots/Check_DL2.png" ) logging.info(datetime.datetime.now()) logging.info( "srun --reservation=rta_one_node ./merge_DL3.py --input-filter 'dl3_v06*' -d '/fefs/onsite/pipeline/rta/data/" + today_to_directory(today) + "/" + str(current_run_is + 1) + "/DL3/' -o '/fefs/onsite/pipeline/rta/data/" + today_to_directory(today) + "/" + str(current_run_is + 1) + "/DL3/' -r " + str(current_run_is + 1) ) os.system( "srun --reservation=rta_one_node ./merge_DL3.py --input-filter 'dl3_v06*' -d '/fefs/onsite/pipeline/rta/data/" + today_to_directory(today) + "/" + str(current_run_is + 1) + "/DL3/' -o '/fefs/onsite/pipeline/rta/data/" + today_to_directory(today) + "/" + str(current_run_is + 1) + "/DL3/' -r " + str(current_run_is + 1) ) os.system( "cp -rf /fefs/onsite/pipeline/rta/data/" + today_to_directory(today) + "/" + str(current_run_is + 1) + "/plots /fefs/onsite/pipeline/rta/data/." ) logging.info( "cp -rf /fefs/onsite/pipeline/rta/data/" + today_to_directory(today) + "/" + str(current_run_is + 1) + "/plots /fefs/onsite/pipeline/rta/data/." ) os.system("rm -f /fefs/onsite/pipeline/rta/data/plot_shifters/*") logging.info("rm -f /fefs/onsite/pipeline/rta/data/plot_shifters/*") os.system( "cp -f /fefs/onsite/pipeline/rta/data/" + today_to_directory(today) + "/" + str(current_run_is + 1) + "/plots/*_standard_theta2.png /fefs/onsite/pipeline/rta/data/plot_shifters/" ) logging.info( "cp -f /fefs/onsite/pipeline/rta/data/" + today_to_directory(today) + "/" + str(current_run_is + 1) + "/plots/*_standard_theta2.png /fefs/onsite/pipeline/rta/data/plot_shifters/" ) os.system( "cp -f /fefs/onsite/pipeline/rta/data/" + today_to_directory(today) + "/" + str(current_run_is + 1) + "/plots/*_standard__excess_significance_over_time.png /fefs/onsite/pipeline/rta/data/plot_shifters/" ) logging.info( "cp -f /fefs/onsite/pipeline/rta/data/" + today_to_directory(today) + "/" + str(current_run_is + 1) + "/plots/*_standard__excess_significance_over_time.png /fefs/onsite/pipeline/rta/data/plot_shifters/" ) os.system( "cp -f /fefs/onsite/pipeline/rta/data/" + today_to_directory(today) + "/" + str(current_run_is + 1) + "/plots/*_standard__spectra.png /fefs/onsite/pipeline/rta/data/plot_shifters/" ) logging.info( "cp -f /fefs/onsite/pipeline/rta/data/" + today_to_directory(today) + "/" + str(current_run_is + 1) + "/plots/*_standard__spectra.png /fefs/onsite/pipeline/rta/data/plot_shifters/" ) os.system( "cp -f /fefs/onsite/pipeline/rta/data/" + today_to_directory(today) + "/" + str(current_run_is + 1) + "/plots/*_standard__sky_map.png /fefs/onsite/pipeline/rta/data/plot_shifters/" ) logging.info( "cp -f /fefs/onsite/pipeline/rta/data/" + today_to_directory(today) + "/" + str(current_run_is + 1) + "/plots/*_standard__sky_map.png /fefs/onsite/pipeline/rta/data/plot_shifters/" ) loop_id = loop_id + 1 if (datetime.datetime.timestamp(datetime.datetime.now()) - Start_time) > 14 * 3600: quit = True logging.info("End of the night, Stop the RTA check")
if __name__ == "__main__": main()