Coverage for lst_auto_rta/Auto_Check.py: 0%
105 statements
« prev ^ index » next coverage.py v7.6.10, created at 2025-01-26 14:48 +0000
1#!/usr/bin/env python
3import datetime
4import logging
5import os
6import time
8import pymongo
10def wait_for_directory(directory, interval=5):
11 """
12 Wait for the directory to exist
14 :param directory: directory path
15 :param interval: Time interval between verification
16 """
17 while not os.path.exists(directory):
18 print(f"{directory} does not exist yet. let's wait {interval} seconds...")
19 time.sleep(interval)
20 print(f"The directory {directory} finally exist !!!")
22def now():
23 """
24 Returns the current timestamp in seconds, relative to the Unix epoch.
26 :return: float
27 """
28 return datetime.datetime.timestamp(datetime.datetime.now())
31def get_current_run():
32 """
33 Attempts to retrieve the current run number from the lst1_obs_summary database.
35 :return: int
36 """
37 try:
38 client = pymongo.MongoClient("lst101")
39 database = client["lst1_obs_summary"]
40 camera_collection = database["camera"]
41 summaries = camera_collection.find({})
42 run_table = []
43 summaries = camera_collection.find({})
44 for summary in summaries:
45 if summary["tstart"] > datetime.datetime.timestamp(datetime.datetime.now()) - 3600 * 24 * 100:
46 run_table.append(summary["run_number"])
47 if run_table[-2] != run_table[-1]:
48 return run_table[-1]
49 else:
50 return run_table[-1] + 1
51 except Exception as e:
52 logging.error("Error DB !!!!!")
53 return -1
56def get_night_timestamp(today):
57 """
58 Get current timestamp (YYYYMMDD) with respect to the night.
59 It will consider a same night everything run between 08h00 until 07h59 of the next day.
61 Ex: An observation launched at 04h00 (local time) will refer to the previous day.
63 :param today: datetime object
64 datetime object with current local time
66 :return:
67 timestamp: str
68 """
69 # TODO check that today is a datetime object ?
70 if today.hour < 8: # script launched between
71 yesterday = today - datetime.timedelta(days=1)
72 timestamp = f"{yesterday.year:04d}{yesterday.month:02d}{yesterday.day:02d}"
73 else:
74 timestamp = f"{today.year:04d}{today.month:02d}{today.day:02d}"
76 return timestamp
78def today_to_directory(today):
79 return today[0:4]+"/"+today[4:6]+"/"+today[6:8]
82def main():
83 """
84 Initialize current_run_is and quit variables and get the current timestamp with Start_time = now()
85 Initialize the logging system by setting the log file name and logging level
86 Change the current working directory to /fefs/onsite/pipeline/rta/data
87 Find the current reservations for nodes using the scontrol command and parse the output to find the individual node names
88 Modify a configuration file with sed commands, replacing placeholders with the node names found in step 4
89 Check if the ib0 network interface is in connected mode for each of the nodes and log the results
90 """
91 current_run_is = -1
92 quit = False
94 Start_time = now()
96 # today = str(datetime.datetime.now().year) + str(datetime.datetime.now().month) + str(datetime.datetime.now().day)
97 today = get_night_timestamp(datetime.datetime.now())
98 wait_for_directory("/fefs/onsite/pipeline/rta/data/"+today_to_directory(today))
99 logging.basicConfig(filename="/fefs/onsite/pipeline/rta/data/"+today_to_directory(today)+"/log_AutoCheck_" + today + ".txt", level=logging.INFO)
100 logging.info("Start RTA Auto Check for the day " + today)
102 RTA_ready = True
104 loop_id = 0
106 data_directory = "'/fefs/onsite/pipeline/rta/data/"
107 os.system("rm -f /fefs/onsite/pipeline/rta/data/plots/*")
108 while quit == False and RTA_ready:
109 time.sleep(10)
110 # print(current_run_is+1)
111 if current_run_is == -1:
112 current_run_is = get_current_run()
113 if current_run_is == -1:
114 logging.info("None")
115 current_run_is = -1
116 continue
117 if current_run_is != get_current_run():
118 time.sleep(1)
119 current_run_is = get_current_run()
120 if current_run_is != -1:
121 logging.info("Start RTA check for run " + str(current_run_is))
122 time.sleep(2)
123 logging.info(datetime.datetime.now())
124 logging.info(
125 "srun --reservation=rta_one_node ./Auto_Check_DL1.py -da "
126 + today_to_directory(today)
127 + " -r "
128 + str(current_run_is + 1)
129 )
130 os.system(
131 "srun --reservation=rta_one_node ./Auto_Check_DL1.py -da "
132 + today_to_directory(today)
133 + " -r "
134 + str(current_run_is + 1)
135 )
136 #os.system(
137 # "convert -density 300 /fefs/onsite/pipeline/rta/data/"
138 # + today_to_directory(today)
139 # + "/"
140 # + str(current_run_is + 1)
141 # + "/plots/output_DL1.pdf /fefs/onsite/pipeline/rta/data/"
142 # + today_to_directory(today)
143 # + "/"
144 # + str(current_run_is + 1)
145 # + "/plots/Check_DL1.png"
146 #)
148 logging.info(datetime.datetime.now())
149 logging.info(
150 "srun --reservation=rta_one_node ./Auto_Check_DL2.py -da "
151 + today_to_directory(today)
152 + " -r "
153 + str(current_run_is + 1)
154 )
155 os.system(
156 "srun --reservation=rta_one_node ./Auto_Check_DL2.py -da "
157 + today_to_directory(today)
158 + " -r "
159 + str(current_run_is + 1)
160 )
161 #os.system(
162 # "convert -density 300 /fefs/onsite/pipeline/rta/data/"
163 # + today_to_directory(today)
164 # + "/"
165 # + str(current_run_is + 1)
166 # + "/plots/output_DL2.pdf /fefs/onsite/pipeline/rta/data/"
167 # + today_to_directory(today)
168 # + "/"
169 # + str(current_run_is + 1)
170 # + "/plots/Check_DL2.png"
171 #)
173 logging.info(datetime.datetime.now())
174 logging.info(
175 "srun --reservation=rta_one_node ./merge_DL3.py --input-filter 'dl3_v06*' -d '/fefs/onsite/pipeline/rta/data/"
176 + today_to_directory(today)
177 + "/"
178 + str(current_run_is + 1)
179 + "/DL3/' -o '/fefs/onsite/pipeline/rta/data/"
180 + today_to_directory(today)
181 + "/"
182 + str(current_run_is + 1)
183 + "/DL3/' -r "
184 + str(current_run_is + 1)
185 )
186 os.system(
187 "srun --reservation=rta_one_node ./merge_DL3.py --input-filter 'dl3_v06*' -d '/fefs/onsite/pipeline/rta/data/"
188 + today_to_directory(today)
189 + "/"
190 + str(current_run_is + 1)
191 + "/DL3/' -o '/fefs/onsite/pipeline/rta/data/"
192 + today
193 + "/"
194 + str(current_run_is + 1)
195 + "/DL3/' -r "
196 + str(current_run_is + 1)
197 )
199 time.sleep(10)
200 os.system(
201 "srun --reservation=rta_one_node ./merge_DL3.py --input-filter 'dl3_v06*' -d '/fefs/onsite/pipeline/rta/data/"
202 + today_to_directory(today)
203 + "/"
204 + str(current_run_is)
205 + "/DL3/' -o '/fefs/onsite/pipeline/rta/data/"
206 + today_to_directory(today)
207 + "/"
208 + str(current_run_is)
209 + "/DL3/' -r "
210 + str(current_run_is)
211 )
213 logging.info(
214 "srun --reservation=rta_one_node ./merge_DL3.py --input-filter 'dl3_v06*' -d '/fefs/onsite/pipeline/rta/data/"
215 + today_to_directory(today)
216 + "/"
217 + str(current_run_is + 1)
218 + "/DL3/' -o '/fefs/onsite/pipeline/rta/data/"
219 + today_to_directory(today)
220 + "/"
221 + str(current_run_is + 1)
222 + "/DL3/' -r "
223 + str(current_run_is + 1)
224 )
225 if loop_id % 1 == 0:
226 logging.info("Start RTA check for run " + str(current_run_is))
227 time.sleep(2)
228 logging.info(datetime.datetime.now())
229 logging.info(
230 "srun --reservation=rta_one_node ./Auto_Check_DL1.py -da " + today_to_directory(today) + " -r " + str(current_run_is + 1)
231 )
232 os.system(
233 "srun --reservation=rta_one_node ./Auto_Check_DL1.py -da " + today_to_directory(today) + " -r " + str(current_run_is + 1)
234 )
235 #os.system(
236 # "convert -density 300 /fefs/onsite/pipeline/rta/data/"
237 # + today_to_directory(today)
238 # + "/"
239 # + str(current_run_is + 1)
240 # + "/plots/output_DL1.pdf /fefs/onsite/pipeline/rta/data/"
241 # + today_to_directory(today)
242 # + "/"
243 # + str(current_run_is + 1)
244 # + "/Check_DL1.png"
245 #)
247 logging.info(datetime.datetime.now())
248 logging.info(
249 "srun --reservation=rta_one_node ./Auto_Check_DL2.py -da " + today_to_directory(today) + " -r " + str(current_run_is + 1)
250 )
251 os.system(
252 "srun --reservation=rta_one_node ./Auto_Check_DL2.py -da " + today_to_directory(today) + " -r " + str(current_run_is + 1)
253 )
254 #os.system(
255 # "convert -density 300 /fefs/onsite/pipeline/rta/data/"
256 # + today_to_directory(today)
257 # + "/"
258 # + str(current_run_is + 1)
259 # + "/plots/output_DL2.pdf /fefs/onsite/pipeline/rta/data/"
260 # + today_to_directory(today)
261 # + "/"
262 # + str(current_run_is + 1)
263 # + "/plots/Check_DL2.png"
264 #)
266 logging.info(datetime.datetime.now())
267 logging.info(
268 "srun --reservation=rta_one_node ./merge_DL3.py --input-filter 'dl3_v06*' -d '/fefs/onsite/pipeline/rta/data/"
269 + today_to_directory(today)
270 + "/"
271 + str(current_run_is + 1)
272 + "/DL3/' -o '/fefs/onsite/pipeline/rta/data/"
273 + today_to_directory(today)
274 + "/"
275 + str(current_run_is + 1)
276 + "/DL3/' -r "
277 + str(current_run_is + 1)
278 )
279 os.system(
280 "srun --reservation=rta_one_node ./merge_DL3.py --input-filter 'dl3_v06*' -d '/fefs/onsite/pipeline/rta/data/"
281 + today_to_directory(today)
282 + "/"
283 + str(current_run_is + 1)
284 + "/DL3/' -o '/fefs/onsite/pipeline/rta/data/"
285 + today_to_directory(today)
286 + "/"
287 + str(current_run_is + 1)
288 + "/DL3/' -r "
289 + str(current_run_is + 1)
290 )
292 os.system(
293 "cp -rf /fefs/onsite/pipeline/rta/data/"
294 + today_to_directory(today)
295 + "/"
296 + str(current_run_is + 1)
297 + "/plots /fefs/onsite/pipeline/rta/data/."
298 )
299 logging.info(
300 "cp -rf /fefs/onsite/pipeline/rta/data/"
301 + today_to_directory(today)
302 + "/"
303 + str(current_run_is + 1)
304 + "/plots /fefs/onsite/pipeline/rta/data/."
305 )
306 os.system("rm -f /fefs/onsite/pipeline/rta/data/plot_shifters/*")
307 logging.info("rm -f /fefs/onsite/pipeline/rta/data/plot_shifters/*")
308 os.system(
309 "cp -f /fefs/onsite/pipeline/rta/data/"
310 + today_to_directory(today)
311 + "/"
312 + str(current_run_is + 1)
313 + "/plots/*_standard_theta2.png /fefs/onsite/pipeline/rta/data/plot_shifters/"
314 )
315 logging.info(
316 "cp -f /fefs/onsite/pipeline/rta/data/"
317 + today_to_directory(today)
318 + "/"
319 + str(current_run_is + 1)
320 + "/plots/*_standard_theta2.png /fefs/onsite/pipeline/rta/data/plot_shifters/"
321 )
322 os.system(
323 "cp -f /fefs/onsite/pipeline/rta/data/"
324 + today_to_directory(today)
325 + "/"
326 + str(current_run_is + 1)
327 + "/plots/*_standard__excess_significance_over_time.png /fefs/onsite/pipeline/rta/data/plot_shifters/"
328 )
329 logging.info(
330 "cp -f /fefs/onsite/pipeline/rta/data/"
331 + today_to_directory(today)
332 + "/"
333 + str(current_run_is + 1)
334 + "/plots/*_standard__excess_significance_over_time.png /fefs/onsite/pipeline/rta/data/plot_shifters/"
335 )
336 os.system(
337 "cp -f /fefs/onsite/pipeline/rta/data/"
338 + today_to_directory(today)
339 + "/"
340 + str(current_run_is + 1)
341 + "/plots/*_standard__spectra.png /fefs/onsite/pipeline/rta/data/plot_shifters/"
342 )
343 logging.info(
344 "cp -f /fefs/onsite/pipeline/rta/data/"
345 + today_to_directory(today)
346 + "/"
347 + str(current_run_is + 1)
348 + "/plots/*_standard__spectra.png /fefs/onsite/pipeline/rta/data/plot_shifters/"
349 )
350 os.system(
351 "cp -f /fefs/onsite/pipeline/rta/data/"
352 + today_to_directory(today)
353 + "/"
354 + str(current_run_is + 1)
355 + "/plots/*_standard__sky_map.png /fefs/onsite/pipeline/rta/data/plot_shifters/"
356 )
357 logging.info(
358 "cp -f /fefs/onsite/pipeline/rta/data/"
359 + today_to_directory(today)
360 + "/"
361 + str(current_run_is + 1)
362 + "/plots/*_standard__sky_map.png /fefs/onsite/pipeline/rta/data/plot_shifters/"
363 )
364 loop_id = loop_id + 1
365 if (datetime.datetime.timestamp(datetime.datetime.now()) - Start_time) > 14 * 3600:
366 quit = True
367 logging.info("End of the night, Stop the RTA check")
370if __name__ == "__main__":
371 main()