#!/usr/bin/env python
import datetime
import logging
import os
import time
import pymongo
[docs]
def wait_for_directory(directory, interval=5):
"""
Wait for the directory to exist
:param directory: directory path
:param interval: Time interval between verification
"""
while not os.path.exists(directory):
print(f"{directory} does not exist yet. let's wait {interval} seconds...")
time.sleep(interval)
print(f"The directory {directory} finally exist !!!")
[docs]
def now():
"""
Returns the current timestamp in seconds, relative to the Unix epoch.
:return: float
"""
return datetime.datetime.timestamp(datetime.datetime.now())
[docs]
def get_current_run(
db_hostname: str = "lst101",
obs_target_query_timeout_s: int = 10,
) -> int:
"""
Standalone retrieval of the current run number from the lst1_obs_summary DB.
Robust to duplicated run_number entries.
"""
try:
with pymongo.MongoClient(db_hostname) as client:
collection = client.get_database("lst1_obs_summary").get_collection("telescope")
camera_data_index = [("data.camera.tstart", pymongo.DESCENDING)]
keep_querying = True
query_idx = 0
visited_ids: Set[object] = set()
obs_id: Optional[int] = None
nb_duplicated_run_numbers = 0
while keep_querying:
cursor = collection.find(
{
"data.camera.run_number": {"$exists": True},
"data.camera.tstart": {"$exists": True},
"data.structure.tstart": {"$exists": True},
},
{
"_id": True,
"data.camera.run_number": True,
"data.camera.tstart": True,
"data.structure.tstart": True,
},
sort=camera_data_index,
max_time_ms=obs_target_query_timeout_s * 1000,
skip=query_idx,
limit=1,
)
try:
query_data = next(cursor)
finally:
# Close cursor explicitly for older pymongo versions
cursor.close()
# If the DB changes while iterating, we may get the same _id again: ignore it
if query_data["_id"] in visited_ids:
query_idx += 1
continue
visited_ids.add(query_data["_id"])
camera_list = sorted(query_data["data"]["camera"], key=lambda x: x["tstart"], reverse=True)
structure_list = sorted(query_data["data"]["structure"], key=lambda x: x["tstart"], reverse=True)
# If the DB is incoherent for the latest run: raise error.
if (len(structure_list) != len(camera_list)) and (query_idx == 0):
raise pymongo.errors.PyMongoError(
"TCU DB query returned inconsistent data.structure (length {}) "
"and data.camera (length {})".format(len(structure_list), len(camera_list))
)
if query_idx == 0:
obs_id = camera_list[0]["run_number"]
comparison_start_idx = 1 if query_idx == 0 else 0
for field in camera_list[comparison_start_idx:]:
if field["run_number"] != obs_id:
keep_querying = False
break
nb_duplicated_run_numbers += 1
query_idx += 1
if obs_id is None:
logging.error("DB query succeeded but no obs_id was extracted")
return -1
return int(obs_id + nb_duplicated_run_numbers)
except pymongo.errors.ExecutionTimeout:
logging.error("DB query timeout (max_time_ms exceeded) while retrieving current run number")
return -1
except (StopIteration, KeyError, TypeError) as e:
logging.error(f"DB returned unexpected/empty data while retrieving current run number: {e}")
return -1
except pymongo.errors.PyMongoError as e:
logging.error(f"MongoDB error while retrieving current run number: {e}")
return -1
except Exception as e:
logging.error(f"Unexpected error while retrieving current run number: {e}")
return -1
[docs]
def get_night_timestamp(today):
"""
Get current timestamp (YYYYMMDD) with respect to the night.
It will consider a same night everything run between 08h00 until 07h59 of the next day.
Ex: An observation launched at 04h00 (local time) will refer to the previous day.
:param today: datetime object
datetime object with current local time
:return:
timestamp: str
"""
# TODO check that today is a datetime object ?
if today.hour < 8: # script launched between
yesterday = today - datetime.timedelta(days=1)
timestamp = f"{yesterday.year:04d}{yesterday.month:02d}{yesterday.day:02d}"
else:
timestamp = f"{today.year:04d}{today.month:02d}{today.day:02d}"
return timestamp
[docs]
def Obtain_RA_DEC(runid):
try:
client = pymongo.MongoClient("lst101")
database = client["bridgesmonitoring"]
print(client)
client = pymongo.MongoClient("lst101")
database = client["lst1_obs_summary"]
camera_collection = database["telescope"]
summaries = camera_collection.find({"data.camera.run_number": runid})
for summary in summaries:
camera = summary["data"]["camera"]
count = 0
structure = summary["data"]["structure"]
for c in camera:
if c["run_number"] == runid:
run_count = count
count = count + 1
run_parameters = structure[run_count]
return (
run_parameters["target"]["ra"],
run_parameters["target"]["dec"],
run_parameters["target"]["source_ra"],
run_parameters["target"]["source_dec"],
)
except Exception as e:
# client = pymongo.MongoClient('lst101')
# database = client['lst1_obs_summary']
# camera_collection = database['camera']
# summaries = camera_collection.find({})
# for summary in summaries:
# if (summary['tstart'] > datetime.datetime.timestamp(datetime.datetime.now())-3600*24*100):
# if (summary['run_number']==runid and summary["kind"] != "data_taking"):
# return [0,0,0,0]
logging.error("Error DB !!!!!")
return [0, 0, 0, 0]
[docs]
def today_to_directory(today):
return today[0:4]+"/"+today[4:6]+"/"+today[6:8]
[docs]
def main():
"""
Initialize current_run_is and quit variables and get the current timestamp with Start_time = now()
Initialize the logging system by setting the log file name and logging level
Change the current working directory to /fefs/onsite/pipeline/rta/data
Find the current reservations for nodes using the scontrol command and parse the output to find the individual node names
Modify a configuration file with sed commands, replacing placeholders with the node names found in step 4
Check if the ib0 network interface is in connected mode for each of the nodes and log the results
"""
current_run_is = -1
quit = False
Start_time = now()
# today = str(datetime.datetime.now().year) + str(datetime.datetime.now().month) + str(datetime.datetime.now().day)
today = get_night_timestamp(datetime.datetime.now())
wait_for_directory("/fefs/onsite/pipeline/rta/data/"+today_to_directory(today))
logging.basicConfig(filename="/fefs/onsite/pipeline/rta/data/"+today_to_directory(today)+"/log_High_Level_" + today + ".txt", level=logging.INFO)
logging.info("Start RTA High Level for the day " + today)
RTA_ready = True
loop_id = 0
ra_dec = [0, 0, 0, 0]
data_directory = "/fefs/onsite/pipeline/rta/data/"
while quit == False and RTA_ready:
time.sleep(10)
# print(current_run_is+1)
if current_run_is == -1:
current_run_is = get_current_run()
if current_run_is == -1:
logging.info("None")
current_run_is = -1
continue
if current_run_is != get_current_run():
time.sleep(1)
current_run_is = get_current_run()
if current_run_is != -1:
logging.info("Start RTA check for run " + str(current_run_is))
time.sleep(2)
ra_dec = Obtain_RA_DEC(current_run_is + 1)
logging.info(
# ~ "srun --reservation=rta_one_node ./Theta_square.py -da "
"./Theta_square.py -da "
+ today_to_directory(today)
+ " -r "
+ str(current_run_is + 1)
+ " -RA "
+ str(ra_dec[2])
+ " -DEC "
+ str(ra_dec[3])
)
os.system(
# ~ "srun --reservation=rta_one_node ./Theta_square.py -da "
"./Theta_square.py -da "
+ today_to_directory(today)
+ " -r "
+ str(current_run_is + 1)
+ " -RA "
+ str(ra_dec[2])
+ " -DEC "
+ str(ra_dec[3])
)
logging.info(
# ~ "srun --reservation=rta_one_node ./Spectra.py -da "
"./Spectra.py -da "
+ today_to_directory(today)
+ " -r "
+ str(current_run_is + 1)
+ " -RA "
+ str(ra_dec[2])
+ " -DEC "
+ str(ra_dec[3])
)
os.system(
# ~ "srun --reservation=rta_one_node ./Spectra.py -da "
"./Spectra.py -da "
+ today_to_directory(today)
+ " -r "
+ str(current_run_is + 1)
+ " -RA "
+ str(ra_dec[2])
+ " -DEC "
+ str(ra_dec[3])
)
logging.info(
# ~ "srun --reservation=rta_one_node ./rta_var.py --dl3_folder "
"./rta_var.py --dl3_folder "
+ data_directory
+ today_to_directory(today)
+ "/"
+ str(current_run_is + 1)
+ "/DL3 -ra "
+ str(ra_dec[2])
+ " -dec "
+ str(ra_dec[3])
+ " --output_dir "+ data_directory
+ today_to_directory(today)
+ "/"
+ str(current_run_is + 1)
+ "/plots --date "
+ today
)
os.system(
# ~ "srun --reservation=rta_one_node ./rta_var.py --dl3_folder "
"./rta_var.py --dl3_folder "
+ data_directory
+ today_to_directory(today)
+ "/"
+ str(current_run_is + 1)
+ "/DL3 -ra "
+ str(ra_dec[2])
+ " -dec "
+ str(ra_dec[3])
+ " --output_dir " + data_directory
+ today_to_directory(today)
+ "/"
+ str(current_run_is + 1)
+ "/plots --date "
+ today
)
if loop_id % 1 == 0:
logging.info("Start RTA check for run " + str(current_run_is))
time.sleep(2)
logging.info(datetime.datetime.now())
ra_dec = Obtain_RA_DEC(current_run_is + 1)
logging.info(
"./Theta_square.py -da "
+ today_to_directory(today)
+ " -r "
+ str(current_run_is + 1)
+ " -RA "
+ str(ra_dec[2])
+ " -DEC "
+ str(ra_dec[3])
)
os.system(
"./Theta_square.py -da "
+ today_to_directory(today)
+ " -r "
+ str(current_run_is + 1)
+ " -RA "
+ str(ra_dec[2])
+ " -DEC "
+ str(ra_dec[3])
)
logging.info(
"./Spectra.py -da "
+ today_to_directory(today)
+ " -r "
+ str(current_run_is + 1)
+ " -RA "
+ str(ra_dec[2])
+ " -DEC "
+ str(ra_dec[3])
)
os.system(
"./Spectra.py -da "
+ today_to_directory(today)
+ " -r "
+ str(current_run_is + 1)
+ " -RA "
+ str(ra_dec[2])
+ " -DEC "
+ str(ra_dec[3])
)
logging.info(
"./rta_var.py --dl3_folder "+ data_directory
+ today_to_directory(today)
+ "/"
+ str(current_run_is + 1)
+ "/DL3 -ra "
+ str(ra_dec[2])
+ " -dec "
+ str(ra_dec[3])
+ " --output_dir "+ data_directory
+ today_to_directory(today)
+ "/"
+ str(current_run_is + 1)
+ "/plots --date "
+ today
)
os.system(
"./rta_var.py --dl3_folder "+ data_directory
+ today_to_directory(today)
+ "/"
+ str(current_run_is + 1)
+ "/DL3 -ra "
+ str(ra_dec[2])
+ " -dec "
+ str(ra_dec[3])
+ " --output_dir "+ data_directory
+ today_to_directory(today)
+ "/"
+ str(current_run_is + 1)
+ "/plots --date "
+ today
)
loop_id = loop_id + 1
if (
datetime.datetime.timestamp(datetime.datetime.now()) - Start_time
) > 14 * 3600:
quit = True
logging.info("End of the night, Stop the RTA check")
if __name__ == "__main__":
main()