Coverage for lst_auto_rta/Auto_Check.py: 0%
103 statements
« prev ^ index » next coverage.py v7.6.4, created at 2024-11-03 14:47 +0000
1#!/usr/bin/env python
3import datetime
4import logging
5import os
6import time
8import pymongo
11def now():
12 """
13 Returns the current timestamp in seconds, relative to the Unix epoch.
15 :return: float
16 """
17 return datetime.datetime.timestamp(datetime.datetime.now())
20def get_current_run():
21 """
22 Attempts to retrieve the current run number from the lst1_obs_summary database.
24 :return: int
25 """
26 try:
27 client = pymongo.MongoClient("lst101")
28 database = client["lst1_obs_summary"]
29 camera_collection = database["camera"]
30 summaries = camera_collection.find({})
31 run_table = []
32 summaries = camera_collection.find({})
33 for summary in summaries:
34 if summary["tstart"] > datetime.datetime.timestamp(datetime.datetime.now()) - 3600 * 24 * 100:
35 run_table.append(summary["run_number"])
36 if run_table[-2] != run_table[-1]:
37 return run_table[-1]
38 else:
39 return run_table[-1] + 1
40 except Exception as e:
41 logging.error("Error DB !!!!!")
42 return -1
45def get_night_timestamp(today):
46 """
47 Get current timestamp (YYYYMMDD) with respect to the night.
48 It will consider a same night everything run between 08h00 until 07h59 of the next day.
50 Ex: An observation launched at 04h00 (local time) will refer to the previous day.
52 :param today: datetime object
53 datetime object with current local time
55 :return:
56 timestamp: str
57 """
58 # TODO check that today is a datetime object ?
59 if today.hour < 8: # script launched between
60 yesterday = today - datetime.timedelta(days=1)
61 timestamp = f"{yesterday.year:04d}{yesterday.month:02d}{yesterday.day:02d}"
62 else:
63 timestamp = f"{today.year:04d}{today.month:02d}{today.day:02d}"
65 return timestamp
67def today_to_directory(today):
68 return today[0:4]+"/"+today[4:6]+"/"+today[6:8]
71def main():
72 """
73 Initialize current_run_is and quit variables and get the current timestamp with Start_time = now()
74 Initialize the logging system by setting the log file name and logging level
75 Change the current working directory to /fefs/onsite/pipeline/rta/data
76 Find the current reservations for nodes using the scontrol command and parse the output to find the individual node names
77 Modify a configuration file with sed commands, replacing placeholders with the node names found in step 4
78 Check if the ib0 network interface is in connected mode for each of the nodes and log the results
79 """
80 current_run_is = -1
81 quit = False
83 Start_time = now()
85 # today = str(datetime.datetime.now().year) + str(datetime.datetime.now().month) + str(datetime.datetime.now().day)
86 today = get_night_timestamp(datetime.datetime.now())
87 logging.basicConfig(filename="log_AutoCheck_" + today + ".txt", level=logging.INFO)
88 logging.info("Start RTA Auto Check for the day " + today)
90 RTA_ready = True
92 loop_id = 0
94 data_directory = "'/fefs/onsite/pipeline/rta/data/"
95 os.system("rm -f /fefs/onsite/pipeline/rta/data/plots/*")
96 while quit == False and RTA_ready:
97 time.sleep(10)
98 # print(current_run_is+1)
99 if current_run_is == -1:
100 current_run_is = get_current_run()
101 if current_run_is == -1:
102 logging.info("None")
103 current_run_is = -1
104 continue
105 if current_run_is != get_current_run():
106 time.sleep(1)
107 current_run_is = get_current_run()
108 if current_run_is != -1:
109 logging.info("Start RTA check for run " + str(current_run_is))
110 time.sleep(2)
111 logging.info(datetime.datetime.now())
112 logging.info(
113 "srun --reservation=rta_one_node ./Auto_Check_DL1.py -da "
114 + today_to_directory(today)
115 + " -r "
116 + str(current_run_is + 1)
117 )
118 os.system(
119 "srun --reservation=rta_one_node ./Auto_Check_DL1.py -da "
120 + today_to_directory(today)
121 + " -r "
122 + str(current_run_is + 1)
123 )
124 os.system(
125 "convert -density 300 /fefs/onsite/pipeline/rta/data/"
126 + today_to_directory(today)
127 + "/"
128 + str(current_run_is + 1)
129 + "/plots/output_DL1.pdf /fefs/onsite/pipeline/rta/data/"
130 + today_to_directory(today)
131 + "/"
132 + str(current_run_is + 1)
133 + "/plots/Check_DL1.png"
134 )
136 logging.info(datetime.datetime.now())
137 logging.info(
138 "srun --reservation=rta_one_node ./Auto_Check_DL2.py -da "
139 + today_to_directory(today)
140 + " -r "
141 + str(current_run_is + 1)
142 )
143 os.system(
144 "srun --reservation=rta_one_node ./Auto_Check_DL2.py -da "
145 + today_to_directory(today)
146 + " -r "
147 + str(current_run_is + 1)
148 )
149 os.system(
150 "convert -density 300 /fefs/onsite/pipeline/rta/data/"
151 + today_to_directory(today)
152 + "/"
153 + str(current_run_is + 1)
154 + "/plots/output_DL2.pdf /fefs/onsite/pipeline/rta/data/"
155 + today_to_directory(today)
156 + "/"
157 + str(current_run_is + 1)
158 + "/plots/Check_DL2.png"
159 )
161 logging.info(datetime.datetime.now())
162 logging.info(
163 "srun --reservation=rta_one_node ./merge_DL3.py --input-filter 'dl3_v06*' -d '/fefs/onsite/pipeline/rta/data/"
164 + today_to_directory(today)
165 + "/"
166 + str(current_run_is + 1)
167 + "/DL3/' -o '/fefs/onsite/pipeline/rta/data/"
168 + today_to_directory(today)
169 + "/"
170 + str(current_run_is + 1)
171 + "/DL3/' -r "
172 + str(current_run_is + 1)
173 )
174 os.system(
175 "srun --reservation=rta_one_node ./merge_DL3.py --input-filter 'dl3_v06*' -d '/fefs/onsite/pipeline/rta/data/"
176 + today_to_directory(today)
177 + "/"
178 + str(current_run_is + 1)
179 + "/DL3/' -o '/fefs/onsite/pipeline/rta/data/"
180 + today
181 + "/"
182 + str(current_run_is + 1)
183 + "/DL3/' -r "
184 + str(current_run_is + 1)
185 )
187 time.sleep(10)
188 os.system(
189 "srun --reservation=rta_one_node ./merge_DL3.py --input-filter 'dl3_v06*' -d '/fefs/onsite/pipeline/rta/data/"
190 + today_to_directory(today)
191 + "/"
192 + str(current_run_is)
193 + "/DL3/' -o '/fefs/onsite/pipeline/rta/data/"
194 + today_to_directory(today)
195 + "/"
196 + str(current_run_is)
197 + "/DL3/' -r "
198 + str(current_run_is)
199 )
201 logging.info(
202 "srun --reservation=rta_one_node ./merge_DL3.py --input-filter 'dl3_v06*' -d '/fefs/onsite/pipeline/rta/data/"
203 + today_to_directory(today)
204 + "/"
205 + str(current_run_is + 1)
206 + "/DL3/' -o '/fefs/onsite/pipeline/rta/data/"
207 + today_to_directory(today)
208 + "/"
209 + str(current_run_is + 1)
210 + "/DL3/' -r "
211 + str(current_run_is + 1)
212 )
213 if loop_id % 1 == 0:
214 logging.info("Start RTA check for run " + str(current_run_is))
215 time.sleep(2)
216 logging.info(datetime.datetime.now())
217 logging.info(
218 "srun --reservation=rta_one_node ./Auto_Check_DL1.py -da " + today_to_directory(today) + " -r " + str(current_run_is + 1)
219 )
220 os.system(
221 "srun --reservation=rta_one_node ./Auto_Check_DL1.py -da " + today_to_directory(today) + " -r " + str(current_run_is + 1)
222 )
223 os.system(
224 "convert -density 300 /fefs/onsite/pipeline/rta/data/"
225 + today_to_directory(today)
226 + "/"
227 + str(current_run_is + 1)
228 + "/plots/output_DL1.pdf /fefs/onsite/pipeline/rta/data/"
229 + today_to_directory(today)
230 + "/"
231 + str(current_run_is + 1)
232 + "/Check_DL1.png"
233 )
235 logging.info(datetime.datetime.now())
236 logging.info(
237 "srun --reservation=rta_one_node ./Auto_Check_DL2.py -da " + today_to_directory(today) + " -r " + str(current_run_is + 1)
238 )
239 os.system(
240 "srun --reservation=rta_one_node ./Auto_Check_DL2.py -da " + today_to_directory(today) + " -r " + str(current_run_is + 1)
241 )
242 os.system(
243 "convert -density 300 /fefs/onsite/pipeline/rta/data/"
244 + today_to_directory(today)
245 + "/"
246 + str(current_run_is + 1)
247 + "/plots/output_DL2.pdf /fefs/onsite/pipeline/rta/data/"
248 + today_to_directory(today)
249 + "/"
250 + str(current_run_is + 1)
251 + "/plots/Check_DL2.png"
252 )
254 logging.info(datetime.datetime.now())
255 logging.info(
256 "srun --reservation=rta_one_node ./merge_DL3.py --input-filter 'dl3_v06*' -d '/fefs/onsite/pipeline/rta/data/"
257 + today_to_directory(today)
258 + "/"
259 + str(current_run_is + 1)
260 + "/DL3/' -o '/fefs/onsite/pipeline/rta/data/"
261 + today_to_directory(today)
262 + "/"
263 + str(current_run_is + 1)
264 + "/DL3/' -r "
265 + str(current_run_is + 1)
266 )
267 os.system(
268 "srun --reservation=rta_one_node ./merge_DL3.py --input-filter 'dl3_v06*' -d '/fefs/onsite/pipeline/rta/data/"
269 + today_to_directory(today)
270 + "/"
271 + str(current_run_is + 1)
272 + "/DL3/' -o '/fefs/onsite/pipeline/rta/data/"
273 + today_to_directory(today)
274 + "/"
275 + str(current_run_is + 1)
276 + "/DL3/' -r "
277 + str(current_run_is + 1)
278 )
280 os.system(
281 "cp -rf /fefs/onsite/pipeline/rta/data/"
282 + today_to_directory(today)
283 + "/"
284 + str(current_run_is + 1)
285 + "/plots /fefs/onsite/pipeline/rta/data/."
286 )
287 logging.info(
288 "cp -rf /fefs/onsite/pipeline/rta/data/"
289 + today_to_directory(today)
290 + "/"
291 + str(current_run_is + 1)
292 + "/plots /fefs/onsite/pipeline/rta/data/."
293 )
294 os.system("rm -f /fefs/onsite/pipeline/rta/data/plot_shifters/*")
295 logging.info("rm -f /fefs/onsite/pipeline/rta/data/plot_shifters/*")
296 os.system(
297 "cp -f /fefs/onsite/pipeline/rta/data/"
298 + today_to_directory(today)
299 + "/"
300 + str(current_run_is + 1)
301 + "/plots/*_standard_theta2.png /fefs/onsite/pipeline/rta/data/plot_shifters/"
302 )
303 logging.info(
304 "cp -f /fefs/onsite/pipeline/rta/data/"
305 + today_to_directory(today)
306 + "/"
307 + str(current_run_is + 1)
308 + "/plots/*_standard_theta2.png /fefs/onsite/pipeline/rta/data/plot_shifters/"
309 )
310 os.system(
311 "cp -f /fefs/onsite/pipeline/rta/data/"
312 + today_to_directory(today)
313 + "/"
314 + str(current_run_is + 1)
315 + "/plots/*_standard__excess_significance_over_time.png /fefs/onsite/pipeline/rta/data/plot_shifters/"
316 )
317 logging.info(
318 "cp -f /fefs/onsite/pipeline/rta/data/"
319 + today_to_directory(today)
320 + "/"
321 + str(current_run_is + 1)
322 + "/plots/*_standard__excess_significance_over_time.png /fefs/onsite/pipeline/rta/data/plot_shifters/"
323 )
324 os.system(
325 "cp -f /fefs/onsite/pipeline/rta/data/"
326 + today_to_directory(today)
327 + "/"
328 + str(current_run_is + 1)
329 + "/plots/*_standard__spectra.png /fefs/onsite/pipeline/rta/data/plot_shifters/"
330 )
331 logging.info(
332 "cp -f /fefs/onsite/pipeline/rta/data/"
333 + today_to_directory(today)
334 + "/"
335 + str(current_run_is + 1)
336 + "/plots/*_standard__spectra.png /fefs/onsite/pipeline/rta/data/plot_shifters/"
337 )
338 os.system(
339 "cp -f /fefs/onsite/pipeline/rta/data/"
340 + today_to_directory(today)
341 + "/"
342 + str(current_run_is + 1)
343 + "/plots/*_standard__sky_map.png /fefs/onsite/pipeline/rta/data/plot_shifters/"
344 )
345 logging.info(
346 "cp -f /fefs/onsite/pipeline/rta/data/"
347 + today_to_directory(today)
348 + "/"
349 + str(current_run_is + 1)
350 + "/plots/*_standard__sky_map.png /fefs/onsite/pipeline/rta/data/plot_shifters/"
351 )
352 loop_id = loop_id + 1
353 if (datetime.datetime.timestamp(datetime.datetime.now()) - Start_time) > 14 * 3600:
354 quit = True
355 logging.info("End of the night, Stop the RTA check")
358if __name__ == "__main__":
359 main()