Coverage for lst_auto_rta / Auto_Check.py: 0%
125 statements
« prev ^ index » next coverage.py v7.13.4, created at 2026-03-08 14:48 +0000
1#!/usr/bin/env python
3import datetime
4import logging
5import os
6import time
8import pymongo
10def wait_for_directory(directory, interval=5):
11 """
12 Wait for the directory to exist
14 :param directory: directory path
15 :param interval: Time interval between verification
16 """
17 while not os.path.exists(directory):
18 print(f"{directory} does not exist yet. let's wait {interval} seconds...")
19 time.sleep(interval)
20 print(f"The directory {directory} finally exist !!!")
22def now():
23 """
24 Returns the current timestamp in seconds, relative to the Unix epoch.
26 :return: float
27 """
28 return datetime.datetime.timestamp(datetime.datetime.now())
31#def get_current_run():
32# """
33# Attempts to retrieve the current run number from the lst1_obs_summary database.
34#
35# :return: int
36# """
37# try:
38# client = pymongo.MongoClient("lst101")
39# database = client["lst1_obs_summary"]
40# camera_collection = database["camera"]
41# summaries = camera_collection.find({})
42# run_table = []
43# summaries = camera_collection.find({})
44# for summary in summaries:
45# if summary["tstart"] > datetime.datetime.timestamp(datetime.datetime.now()) - 3600 * 24 * 100:
46# run_table.append(summary["run_number"])
47# if run_table[-2] != run_table[-1]:
48# return run_table[-1]
49# else:
50# return run_table[-1] + 1
51# except Exception as e:
52# logging.error("Error DB !!!!!")
53# return -1
55def get_current_run(
56 db_hostname: str = "lst101",
57 obs_target_query_timeout_s: int = 10,
58) -> int:
59 """
60 Standalone retrieval of the current run number from the lst1_obs_summary DB.
61 Robust to duplicated run_number entries.
62 """
63 try:
64 with pymongo.MongoClient(db_hostname) as client:
65 collection = client.get_database("lst1_obs_summary").get_collection("telescope")
66 camera_data_index = [("data.camera.tstart", pymongo.DESCENDING)]
68 keep_querying = True
69 query_idx = 0
70 visited_ids: Set[object] = set()
72 obs_id: Optional[int] = None
73 nb_duplicated_run_numbers = 0
75 while keep_querying:
76 cursor = collection.find(
77 {
78 "data.camera.run_number": {"$exists": True},
79 "data.camera.tstart": {"$exists": True},
80 "data.structure.tstart": {"$exists": True},
81 },
82 {
83 "_id": True,
84 "data.camera.run_number": True,
85 "data.camera.tstart": True,
86 "data.structure.tstart": True,
87 },
88 sort=camera_data_index,
89 max_time_ms=obs_target_query_timeout_s * 1000,
90 skip=query_idx,
91 limit=1,
92 )
94 try:
95 query_data = next(cursor)
96 finally:
97 # Close cursor explicitly for older pymongo versions
98 cursor.close()
100 # If the DB changes while iterating, we may get the same _id again: ignore it
101 if query_data["_id"] in visited_ids:
102 query_idx += 1
103 continue
104 visited_ids.add(query_data["_id"])
106 camera_list = sorted(query_data["data"]["camera"], key=lambda x: x["tstart"], reverse=True)
107 structure_list = sorted(query_data["data"]["structure"], key=lambda x: x["tstart"], reverse=True)
109 # If the DB is incoherent for the latest run: raise error.
110 if (len(structure_list) != len(camera_list)) and (query_idx == 0):
111 raise pymongo.errors.PyMongoError(
112 "TCU DB query returned inconsistent data.structure (length {}) "
113 "and data.camera (length {})".format(len(structure_list), len(camera_list))
114 )
116 if query_idx == 0:
117 obs_id = camera_list[0]["run_number"]
119 comparison_start_idx = 1 if query_idx == 0 else 0
120 for field in camera_list[comparison_start_idx:]:
121 if field["run_number"] != obs_id:
122 keep_querying = False
123 break
124 nb_duplicated_run_numbers += 1
126 query_idx += 1
128 if obs_id is None:
129 logging.error("DB query succeeded but no obs_id was extracted")
130 return -1
132 return int(obs_id + nb_duplicated_run_numbers)
134 except pymongo.errors.ExecutionTimeout:
135 logging.error("DB query timeout (max_time_ms exceeded) while retrieving current run number")
136 return -1
137 except (StopIteration, KeyError, TypeError) as e:
138 logging.error(f"DB returned unexpected/empty data while retrieving current run number: {e}")
139 return -1
140 except pymongo.errors.PyMongoError as e:
141 logging.error(f"MongoDB error while retrieving current run number: {e}")
142 return -1
143 except Exception as e:
144 logging.error(f"Unexpected error while retrieving current run number: {e}")
145 return -1
147def get_night_timestamp(today):
148 """
149 Get current timestamp (YYYYMMDD) with respect to the night.
150 It will consider a same night everything run between 08h00 until 07h59 of the next day.
152 Ex: An observation launched at 04h00 (local time) will refer to the previous day.
154 :param today: datetime object
155 datetime object with current local time
157 :return:
158 timestamp: str
159 """
160 # TODO check that today is a datetime object ?
161 if today.hour < 8: # script launched between
162 yesterday = today - datetime.timedelta(days=1)
163 timestamp = f"{yesterday.year:04d}{yesterday.month:02d}{yesterday.day:02d}"
164 else:
165 timestamp = f"{today.year:04d}{today.month:02d}{today.day:02d}"
167 return timestamp
169def today_to_directory(today):
170 return today[0:4]+"/"+today[4:6]+"/"+today[6:8]
173def main():
174 """
175 Initialize current_run_is and quit variables and get the current timestamp with Start_time = now()
176 Initialize the logging system by setting the log file name and logging level
177 Change the current working directory to /fefs/onsite/pipeline/rta/data
178 Find the current reservations for nodes using the scontrol command and parse the output to find the individual node names
179 Modify a configuration file with sed commands, replacing placeholders with the node names found in step 4
180 Check if the ib0 network interface is in connected mode for each of the nodes and log the results
181 """
182 current_run_is = -1
183 quit = False
185 Start_time = now()
187 # today = str(datetime.datetime.now().year) + str(datetime.datetime.now().month) + str(datetime.datetime.now().day)
188 today = get_night_timestamp(datetime.datetime.now())
189 wait_for_directory("/fefs/onsite/pipeline/rta/data/"+today_to_directory(today))
190 logging.basicConfig(filename="/fefs/onsite/pipeline/rta/data/"+today_to_directory(today)+"/log_AutoCheck_" + today + ".txt", level=logging.INFO)
191 logging.info("Start RTA Auto Check for the day " + today)
193 RTA_ready = True
195 loop_id = 0
197 data_directory = "'/fefs/onsite/pipeline/rta/data/"
198 os.system("rm -f /fefs/onsite/pipeline/rta/data/plots/*")
199 while quit == False and RTA_ready:
200 time.sleep(10)
201 # print(current_run_is+1)
202 if current_run_is == -1:
203 current_run_is = get_current_run()
204 if current_run_is == -1:
205 logging.info("None")
206 current_run_is = -1
207 continue
208 if current_run_is != get_current_run():
209 time.sleep(1)
210 current_run_is = get_current_run()
211 if current_run_is != -1:
212 logging.info("Start RTA check for run " + str(current_run_is))
213 time.sleep(2)
214 #logging.info(datetime.datetime.now())
215 #logging.info(
216 # "srun --reservation=rta_one_node ./Auto_Check_DL1.py -da "
217 # + today_to_directory(today)
218 # + " -r "
219 # + str(current_run_is + 1)
220 #)
221 #os.system(
222 # "srun --reservation=rta_one_node ./Auto_Check_DL1.py -da "
223 # + today_to_directory(today)
224 # + " -r "
225 # + str(current_run_is + 1)
226 #)
227 #os.system(
228 # "convert -density 300 /fefs/onsite/pipeline/rta/data/"
229 # + today_to_directory(today)
230 # + "/"
231 # + str(current_run_is + 1)
232 # + "/plots/output_DL1.pdf /fefs/onsite/pipeline/rta/data/"
233 # + today_to_directory(today)
234 # + "/"
235 # + str(current_run_is + 1)
236 # + "/plots/Check_DL1.png"
237 #)
239 #logging.info(datetime.datetime.now())
240 #logging.info(
241 # "srun --reservation=rta_one_node ./Auto_Check_DL2.py -da "
242 # + today_to_directory(today)
243 # + " -r "
244 # + str(current_run_is + 1)
245 #)
246 #os.system(
247 # "srun --reservation=rta_one_node ./Auto_Check_DL2.py -da "
248 # + today_to_directory(today)
249 # + " -r "
250 # + str(current_run_is + 1)
251 #)
252 #os.system(
253 # "convert -density 300 /fefs/onsite/pipeline/rta/data/"
254 # + today_to_directory(today)
255 # + "/"
256 # + str(current_run_is + 1)
257 # + "/plots/output_DL2.pdf /fefs/onsite/pipeline/rta/data/"
258 # + today_to_directory(today)
259 # + "/"
260 # + str(current_run_is + 1)
261 # + "/plots/Check_DL2.png"
262 #)
264 logging.info(datetime.datetime.now())
265 logging.info(
266 "./merge_DL3.py --input-filter 'dl3_v06*' -d '/fefs/onsite/pipeline/rta/data/"
267 + today_to_directory(today)
268 + "/"
269 + str(current_run_is + 1)
270 + "/DL3/' -o '/fefs/onsite/pipeline/rta/data/"
271 + today_to_directory(today)
272 + "/"
273 + str(current_run_is + 1)
274 + "/DL3/' -r "
275 + str(current_run_is + 1)
276 )
277 os.system(
278 "./merge_DL3.py --input-filter 'dl3_v06*' -d '/fefs/onsite/pipeline/rta/data/"
279 + today_to_directory(today)
280 + "/"
281 + str(current_run_is + 1)
282 + "/DL3/' -o '/fefs/onsite/pipeline/rta/data/"
283 + today
284 + "/"
285 + str(current_run_is + 1)
286 + "/DL3/' -r "
287 + str(current_run_is + 1)
288 )
290 time.sleep(10)
291 os.system(
292 "./merge_DL3.py --input-filter 'dl3_v06*' -d '/fefs/onsite/pipeline/rta/data/"
293 + today_to_directory(today)
294 + "/"
295 + str(current_run_is)
296 + "/DL3/' -o '/fefs/onsite/pipeline/rta/data/"
297 + today_to_directory(today)
298 + "/"
299 + str(current_run_is)
300 + "/DL3/' -r "
301 + str(current_run_is)
302 )
304 logging.info(
305 "./merge_DL3.py --input-filter 'dl3_v06*' -d '/fefs/onsite/pipeline/rta/data/"
306 + today_to_directory(today)
307 + "/"
308 + str(current_run_is + 1)
309 + "/DL3/' -o '/fefs/onsite/pipeline/rta/data/"
310 + today_to_directory(today)
311 + "/"
312 + str(current_run_is + 1)
313 + "/DL3/' -r "
314 + str(current_run_is + 1)
315 )
316 if loop_id % 1 == 0:
317 logging.info("Start RTA check for run " + str(current_run_is))
318 time.sleep(2)
319 logging.info(datetime.datetime.now())
320 #logging.info(
321 # "srun --reservation=rta_one_node ./Auto_Check_DL1.py -da " + today_to_directory(today) + " -r " + str(current_run_is + 1)
322 #)
323 #os.system(
324 # "srun --reservation=rta_one_node ./Auto_Check_DL1.py -da " + today_to_directory(today) + " -r " + str(current_run_is + 1)
325 #)
326 #os.system(
327 # "convert -density 300 /fefs/onsite/pipeline/rta/data/"
328 # + today_to_directory(today)
329 # + "/"
330 # + str(current_run_is + 1)
331 # + "/plots/output_DL1.pdf /fefs/onsite/pipeline/rta/data/"
332 # + today_to_directory(today)
333 # + "/"
334 # + str(current_run_is + 1)
335 # + "/Check_DL1.png"
336 #)
338 #logging.info(datetime.datetime.now())
339 #logging.info(
340 # "srun --reservation=rta_one_node ./Auto_Check_DL2.py -da " + today_to_directory(today) + " -r " + str(current_run_is + 1)
341 #)
342 #os.system(
343 # "srun --reservation=rta_one_node ./Auto_Check_DL2.py -da " + today_to_directory(today) + " -r " + str(current_run_is + 1)
344 #)
345 #os.system(
346 # "convert -density 300 /fefs/onsite/pipeline/rta/data/"
347 # + today_to_directory(today)
348 # + "/"
349 # + str(current_run_is + 1)
350 # + "/plots/output_DL2.pdf /fefs/onsite/pipeline/rta/data/"
351 # + today_to_directory(today)
352 # + "/"
353 # + str(current_run_is + 1)
354 # + "/plots/Check_DL2.png"
355 #)
357 logging.info(datetime.datetime.now())
358 logging.info(
359 "./merge_DL3.py --input-filter 'dl3_v06*' -d '/fefs/onsite/pipeline/rta/data/"
360 + today_to_directory(today)
361 + "/"
362 + str(current_run_is + 1)
363 + "/DL3/' -o '/fefs/onsite/pipeline/rta/data/"
364 + today_to_directory(today)
365 + "/"
366 + str(current_run_is + 1)
367 + "/DL3/' -r "
368 + str(current_run_is + 1)
369 )
370 os.system(
371 "./merge_DL3.py --input-filter 'dl3_v06*' -d '/fefs/onsite/pipeline/rta/data/"
372 + today_to_directory(today)
373 + "/"
374 + str(current_run_is + 1)
375 + "/DL3/' -o '/fefs/onsite/pipeline/rta/data/"
376 + today_to_directory(today)
377 + "/"
378 + str(current_run_is + 1)
379 + "/DL3/' -r "
380 + str(current_run_is + 1)
381 )
383 os.system(
384 "cp -rf /fefs/onsite/pipeline/rta/data/"
385 + today_to_directory(today)
386 + "/"
387 + str(current_run_is + 1)
388 + "/plots /fefs/onsite/pipeline/rta/data/."
389 )
390 logging.info(
391 "cp -rf /fefs/onsite/pipeline/rta/data/"
392 + today_to_directory(today)
393 + "/"
394 + str(current_run_is + 1)
395 + "/plots /fefs/onsite/pipeline/rta/data/."
396 )
397 os.system("rm -f /fefs/onsite/pipeline/rta/data/plot_shifters/*")
398 logging.info("rm -f /fefs/onsite/pipeline/rta/data/plot_shifters/*")
399 os.system(
400 "cp -f /fefs/onsite/pipeline/rta/data/"
401 + today_to_directory(today)
402 + "/"
403 + str(current_run_is + 1)
404 + "/plots/*_standard_theta2.png /fefs/onsite/pipeline/rta/data/plot_shifters/"
405 )
406 logging.info(
407 "cp -f /fefs/onsite/pipeline/rta/data/"
408 + today_to_directory(today)
409 + "/"
410 + str(current_run_is + 1)
411 + "/plots/*_standard_theta2.png /fefs/onsite/pipeline/rta/data/plot_shifters/"
412 )
413 os.system(
414 "cp -f /fefs/onsite/pipeline/rta/data/"
415 + today_to_directory(today)
416 + "/"
417 + str(current_run_is + 1)
418 + "/plots/*_standard__excess_significance_over_time.png /fefs/onsite/pipeline/rta/data/plot_shifters/"
419 )
420 logging.info(
421 "cp -f /fefs/onsite/pipeline/rta/data/"
422 + today_to_directory(today)
423 + "/"
424 + str(current_run_is + 1)
425 + "/plots/*_standard__excess_significance_over_time.png /fefs/onsite/pipeline/rta/data/plot_shifters/"
426 )
427 os.system(
428 "cp -f /fefs/onsite/pipeline/rta/data/"
429 + today_to_directory(today)
430 + "/"
431 + str(current_run_is + 1)
432 + "/plots/*_standard__spectra.png /fefs/onsite/pipeline/rta/data/plot_shifters/"
433 )
434 logging.info(
435 "cp -f /fefs/onsite/pipeline/rta/data/"
436 + today_to_directory(today)
437 + "/"
438 + str(current_run_is + 1)
439 + "/plots/*_standard__spectra.png /fefs/onsite/pipeline/rta/data/plot_shifters/"
440 )
441 os.system(
442 "cp -f /fefs/onsite/pipeline/rta/data/"
443 + today_to_directory(today)
444 + "/"
445 + str(current_run_is + 1)
446 + "/plots/*_standard__sky_map.png /fefs/onsite/pipeline/rta/data/plot_shifters/"
447 )
448 logging.info(
449 "cp -f /fefs/onsite/pipeline/rta/data/"
450 + today_to_directory(today)
451 + "/"
452 + str(current_run_is + 1)
453 + "/plots/*_standard__sky_map.png /fefs/onsite/pipeline/rta/data/plot_shifters/"
454 )
455 loop_id = loop_id + 1
456 if (datetime.datetime.timestamp(datetime.datetime.now()) - Start_time) > 14 * 3600:
457 quit = True
458 logging.info("End of the night, Stop the RTA check")
461if __name__ == "__main__":
462 main()