Coverage for src / lstautorta / Auto_Check.py: 0%
88 statements
« prev ^ index » next coverage.py v7.13.5, created at 2026-08-10 11:56 +0000
1#!/usr/bin/env python
3import datetime
4import logging
5import os
6import time
8from lstautorta.shared_observation import DEFAULT_SHARED_OBS_PATH, wait_for_shared_obs
11def wait_for_directory(directory, interval=5):
12 """
13 Wait for the directory to exist
15 :param directory: directory path
16 :param interval: Time interval between verification
17 """
18 while not os.path.exists(directory):
19 print(f"{directory} does not exist yet. let's wait {interval} seconds...")
20 time.sleep(interval)
21 print(f"The directory {directory} finally exist !!!")
24def now():
25 """
26 Returns the current timestamp in seconds, relative to the Unix epoch.
28 :return: float
29 """
30 return datetime.datetime.timestamp(datetime.datetime.now())
33# def get_current_run():
34# """
35# Attempts to retrieve the current run number from the lst1_obs_summary database.
36#
37# :return: int
38# """
39# try:
40# client = pymongo.MongoClient("lst101")
41# database = client["lst1_obs_summary"]
42# camera_collection = database["camera"]
43# summaries = camera_collection.find({})
44# run_table = []
45# summaries = camera_collection.find({})
46# for summary in summaries:
47# if summary["tstart"] > datetime.datetime.timestamp(datetime.datetime.now()) - 3600 * 24 * 100:
48# run_table.append(summary["run_number"])
49# if run_table[-2] != run_table[-1]:
50# return run_table[-1]
51# else:
52# return run_table[-1] + 1
53# except Exception as e:
54# logging.error("Error DB !!!!!")
55# return -1
58def get_current_run(
59 db_hostname: str = "lst101",
60 obs_target_query_timeout_s: int = 10,
61) -> int:
62 """
63 Retrieve the current run number from the shared observation file.
64 Waits until the file is available.
65 """
66 logging.info("Waiting for shared observation file at %s", DEFAULT_SHARED_OBS_PATH)
67 obs_info = wait_for_shared_obs(DEFAULT_SHARED_OBS_PATH)
68 return int(obs_info.obs_id)
71def get_night_timestamp(today):
72 """
73 Get current timestamp (YYYYMMDD) with respect to the night.
74 It will consider a same night everything run between 08h00 until 07h59 of the next day.
76 Ex: An observation launched at 04h00 (local time) will refer to the previous day.
78 :param today: datetime object
79 datetime object with current local time
81 :return:
82 timestamp: str
83 """
84 # TODO check that today is a datetime object ?
85 if today.hour < 8: # script launched between
86 yesterday = today - datetime.timedelta(days=1)
87 timestamp = f"{yesterday.year:04d}{yesterday.month:02d}{yesterday.day:02d}"
88 else:
89 timestamp = f"{today.year:04d}{today.month:02d}{today.day:02d}"
91 return timestamp
94def today_to_directory(today):
95 return today[0:4] + "/" + today[4:6] + "/" + today[6:8]
98def main():
99 """
100 Initialize current_run_is and quit variables and get the current timestamp with Start_time = now()
101 Initialize the logging system by setting the log file name and logging level
102 Change the current working directory to /fefs/onsite/pipeline/rta/data
103 Find the current reservations for nodes using the scontrol command and parse the output to find the individual node names
104 Modify a configuration file with sed commands, replacing placeholders with the node names found in step 4
105 Check if the ib0 network interface is in connected mode for each of the nodes and log the results
106 """
107 current_run_is = -1
108 quit = False
110 Start_time = now()
112 # today = str(datetime.datetime.now().year) + str(datetime.datetime.now().month) + str(datetime.datetime.now().day)
113 today = get_night_timestamp(datetime.datetime.now())
114 wait_for_directory("/fefs/onsite/pipeline/rta/data/" + today_to_directory(today))
115 logging.basicConfig(
116 filename="/fefs/onsite/pipeline/rta/data/" + today_to_directory(today) + "/log_AutoCheck_" + today + ".txt",
117 level=logging.INFO,
118 )
119 logging.info("Start RTA Auto Check for the day " + today)
121 RTA_ready = True
123 loop_id = 0
125 data_directory = "'/fefs/onsite/pipeline/rta/data/"
126 os.system("rm -f /fefs/onsite/pipeline/rta/data/plots/*")
127 while quit == False and RTA_ready:
128 time.sleep(10)
129 # print(current_run_is+1)
130 if current_run_is == -1:
131 current_run_is = get_current_run()
132 if current_run_is == -1:
133 logging.info("None")
134 current_run_is = -1
135 continue
136 if current_run_is != get_current_run():
137 time.sleep(1)
138 current_run_is = get_current_run()
139 if current_run_is != -1:
140 logging.info("Start RTA check for run " + str(current_run_is))
141 time.sleep(2)
142 logging.info(datetime.datetime.now())
143 logging.info(
144 "srun --reservation=rta_one_node ./Auto_Check_DL1.py -da "
145 + today_to_directory(today)
146 + " -r "
147 + str(current_run_is + 1)
148 )
149 # os.system(
150 # "srun --reservation=rta_one_node ./Auto_Check_DL1.py -da "
151 # + today_to_directory(today)
152 # + " -r "
153 # + str(current_run_is + 1)
154 # )
155 # os.system(
156 # "convert -density 300 /fefs/onsite/pipeline/rta/data/"
157 # + today_to_directory(today)
158 # + "/"
159 # + str(current_run_is + 1)
160 # + "/plots/output_DL1.pdf /fefs/onsite/pipeline/rta/data/"
161 # + today_to_directory(today)
162 # + "/"
163 # + str(current_run_is + 1)
164 # + "/plots/Check_DL1.png"
165 # )
167 logging.info(datetime.datetime.now())
168 logging.info(
169 "srun --reservation=rta_one_node ./Auto_Check_DL2.py -da "
170 + today_to_directory(today)
171 + " -r "
172 + str(current_run_is + 1)
173 )
174 # os.system(
175 # "srun --reservation=rta_one_node ./Auto_Check_DL2.py -da "
176 # + today_to_directory(today)
177 # + " -r "
178 # + str(current_run_is + 1)
179 # )
180 # os.system(
181 # "convert -density 300 /fefs/onsite/pipeline/rta/data/"
182 # + today_to_directory(today)
183 # + "/"
184 # + str(current_run_is + 1)
185 # + "/plots/output_DL2.pdf /fefs/onsite/pipeline/rta/data/"
186 # + today_to_directory(today)
187 # + "/"
188 # + str(current_run_is + 1)
189 # + "/plots/Check_DL2.png"
190 # )
192 logging.info(datetime.datetime.now())
193 logging.info(
194 "./merge_DL3.py --input-filter 'dl3_v06*' -d '/fefs/onsite/pipeline/rta/data/"
195 + today_to_directory(today)
196 + "/"
197 + str(current_run_is)
198 + "/DL3/' -o '/fefs/onsite/pipeline/rta/data/"
199 + today_to_directory(today)
200 + "/"
201 + str(current_run_is)
202 + "/DL3/' -r "
203 + str(current_run_is)
204 )
205 os.system(
206 "./merge_DL3.py --input-filter 'dl3_v06*' -d '/fefs/onsite/pipeline/rta/data/"
207 + today_to_directory(today)
208 + "/"
209 + str(current_run_is)
210 + "/DL3/' -o '/fefs/onsite/pipeline/rta/data/"
211 + today
212 + "/"
213 + str(current_run_is)
214 + "/DL3/' -r "
215 + str(current_run_is)
216 )
218 time.sleep(10)
219 os.system(
220 "./merge_DL3.py --input-filter 'dl3_v06*' -d '/fefs/onsite/pipeline/rta/data/"
221 + today_to_directory(today)
222 + "/"
223 + str(current_run_is - 1)
224 + "/DL3/' -o '/fefs/onsite/pipeline/rta/data/"
225 + today_to_directory(today)
226 + "/"
227 + str(current_run_is - 1)
228 + "/DL3/' -r "
229 + str(current_run_is - 1)
230 )
232 logging.info(
233 "./merge_DL3.py --input-filter 'dl3_v06*' -d '/fefs/onsite/pipeline/rta/data/"
234 + today_to_directory(today)
235 + "/"
236 + str(current_run_is - 1)
237 + "/DL3/' -o '/fefs/onsite/pipeline/rta/data/"
238 + today_to_directory(today)
239 + "/"
240 + str(current_run_is - 1)
241 + "/DL3/' -r "
242 + str(current_run_is - 1)
243 )
244 if loop_id % 1 == 0:
245 logging.info("Start RTA check for run " + str(current_run_is))
246 time.sleep(2)
247 logging.info(datetime.datetime.now())
248 logging.info(
249 "srun --reservation=rta_one_node ./Auto_Check_DL1.py -da "
250 + today_to_directory(today)
251 + " -r "
252 + str(current_run_is + 1)
253 )
254 # os.system(
255 # "srun --reservation=rta_one_node ./Auto_Check_DL1.py -da " + today_to_directory(today) + " -r " + str(current_run_is + 1)
256 # )
257 # os.system(
258 # "convert -density 300 /fefs/onsite/pipeline/rta/data/"
259 # + today_to_directory(today)
260 # + "/"
261 # + str(current_run_is + 1)
262 # + "/plots/output_DL1.pdf /fefs/onsite/pipeline/rta/data/"
263 # + today_to_directory(today)
264 # + "/"
265 # + str(current_run_is + 1)
266 # + "/Check_DL1.png"
267 # )
269 logging.info(datetime.datetime.now())
270 logging.info(
271 "srun --reservation=rta_one_node ./Auto_Check_DL2.py -da "
272 + today_to_directory(today)
273 + " -r "
274 + str(current_run_is + 1)
275 )
276 # os.system(
277 # "srun --reservation=rta_one_node ./Auto_Check_DL2.py -da " + today_to_directory(today) + " -r " + str(current_run_is + 1)
278 # )
279 # os.system(
280 # "convert -density 300 /fefs/onsite/pipeline/rta/data/"
281 # + today_to_directory(today)
282 # + "/"
283 # + str(current_run_is + 1)
284 # + "/plots/output_DL2.pdf /fefs/onsite/pipeline/rta/data/"
285 # + today_to_directory(today)
286 # + "/"
287 # + str(current_run_is + 1)
288 # + "/plots/Check_DL2.png"
289 # )
291 logging.info(datetime.datetime.now())
292 logging.info(
293 "./merge_DL3.py --input-filter 'dl3_v06*' -d '/fefs/onsite/pipeline/rta/data/"
294 + today_to_directory(today)
295 + "/"
296 + str(current_run_is)
297 + "/DL3/' -o '/fefs/onsite/pipeline/rta/data/"
298 + today_to_directory(today)
299 + "/"
300 + str(current_run_is)
301 + "/DL3/' -r "
302 + str(current_run_is)
303 )
304 os.system(
305 "./merge_DL3.py --input-filter 'dl3_v06*' -d '/fefs/onsite/pipeline/rta/data/"
306 + today_to_directory(today)
307 + "/"
308 + str(current_run_is)
309 + "/DL3/' -o '/fefs/onsite/pipeline/rta/data/"
310 + today_to_directory(today)
311 + "/"
312 + str(current_run_is)
313 + "/DL3/' -r "
314 + str(current_run_is)
315 )
317 os.system(
318 "cp -rf /fefs/onsite/pipeline/rta/data/"
319 + today_to_directory(today)
320 + "/"
321 + str(current_run_is)
322 + "/plots /fefs/onsite/pipeline/rta/data/."
323 )
324 logging.info(
325 "cp -rf /fefs/onsite/pipeline/rta/data/"
326 + today_to_directory(today)
327 + "/"
328 + str(current_run_is)
329 + "/plots /fefs/onsite/pipeline/rta/data/."
330 )
331 os.system("rm -f /fefs/onsite/pipeline/rta/data/plot_shifters/*")
332 logging.info("rm -f /fefs/onsite/pipeline/rta/data/plot_shifters/*")
333 os.system(
334 "cp -f /fefs/onsite/pipeline/rta/data/"
335 + today_to_directory(today)
336 + "/"
337 + str(current_run_is)
338 + "/plots/*_standard_theta2.png /fefs/onsite/pipeline/rta/data/plot_shifters/"
339 )
340 logging.info(
341 "cp -f /fefs/onsite/pipeline/rta/data/"
342 + today_to_directory(today)
343 + "/"
344 + str(current_run_is)
345 + "/plots/*_standard_theta2.png /fefs/onsite/pipeline/rta/data/plot_shifters/"
346 )
347 os.system(
348 "cp -f /fefs/onsite/pipeline/rta/data/"
349 + today_to_directory(today)
350 + "/"
351 + str(current_run_is)
352 + "/plots/*_standard__excess_significance_over_time.png /fefs/onsite/pipeline/rta/data/plot_shifters/"
353 )
354 logging.info(
355 "cp -f /fefs/onsite/pipeline/rta/data/"
356 + today_to_directory(today)
357 + "/"
358 + str(current_run_is)
359 + "/plots/*_standard__excess_significance_over_time.png /fefs/onsite/pipeline/rta/data/plot_shifters/"
360 )
361 os.system(
362 "cp -f /fefs/onsite/pipeline/rta/data/"
363 + today_to_directory(today)
364 + "/"
365 + str(current_run_is)
366 + "/plots/*_standard__spectra.png /fefs/onsite/pipeline/rta/data/plot_shifters/"
367 )
368 logging.info(
369 "cp -f /fefs/onsite/pipeline/rta/data/"
370 + today_to_directory(today)
371 + "/"
372 + str(current_run_is)
373 + "/plots/*_standard__spectra.png /fefs/onsite/pipeline/rta/data/plot_shifters/"
374 )
375 os.system(
376 "cp -f /fefs/onsite/pipeline/rta/data/"
377 + today_to_directory(today)
378 + "/"
379 + str(current_run_is)
380 + "/plots/*_standard__sky_map.png /fefs/onsite/pipeline/rta/data/plot_shifters/"
381 )
382 logging.info(
383 "cp -f /fefs/onsite/pipeline/rta/data/"
384 + today_to_directory(today)
385 + "/"
386 + str(current_run_is)
387 + "/plots/*_standard__sky_map.png /fefs/onsite/pipeline/rta/data/plot_shifters/"
388 )
389 loop_id = loop_id + 1
390 if (datetime.datetime.timestamp(datetime.datetime.now()) - Start_time) > 14 * 3600:
391 quit = True
392 logging.info("End of the night, Stop the RTA check")
395if __name__ == "__main__":
396 main()