Coverage for lst_auto_rta/Auto_Check.py: 0%

103 statements  

« prev     ^ index     » next       coverage.py v7.6.7, created at 2024-11-17 14:47 +0000

1#!/usr/bin/env python 

2 

3import datetime 

4import logging 

5import os 

6import time 

7 

8import pymongo 

9 

10 

11def now(): 

12 """ 

13 Returns the current timestamp in seconds, relative to the Unix epoch. 

14 

15 :return: float 

16 """ 

17 return datetime.datetime.timestamp(datetime.datetime.now()) 

18 

19 

20def get_current_run(): 

21 """ 

22 Attempts to retrieve the current run number from the lst1_obs_summary database. 

23 

24 :return: int 

25 """ 

26 try: 

27 client = pymongo.MongoClient("lst101") 

28 database = client["lst1_obs_summary"] 

29 camera_collection = database["camera"] 

30 summaries = camera_collection.find({}) 

31 run_table = [] 

32 summaries = camera_collection.find({}) 

33 for summary in summaries: 

34 if summary["tstart"] > datetime.datetime.timestamp(datetime.datetime.now()) - 3600 * 24 * 100: 

35 run_table.append(summary["run_number"]) 

36 if run_table[-2] != run_table[-1]: 

37 return run_table[-1] 

38 else: 

39 return run_table[-1] + 1 

40 except Exception as e: 

41 logging.error("Error DB !!!!!") 

42 return -1 

43 

44 

45def get_night_timestamp(today): 

46 """ 

47 Get current timestamp (YYYYMMDD) with respect to the night. 

48 It will consider a same night everything run between 08h00 until 07h59 of the next day. 

49 

50 Ex: An observation launched at 04h00 (local time) will refer to the previous day. 

51 

52 :param today: datetime object 

53 datetime object with current local time 

54 

55 :return: 

56 timestamp: str 

57 """ 

58 # TODO check that today is a datetime object ? 

59 if today.hour < 8: # script launched between 

60 yesterday = today - datetime.timedelta(days=1) 

61 timestamp = f"{yesterday.year:04d}{yesterday.month:02d}{yesterday.day:02d}" 

62 else: 

63 timestamp = f"{today.year:04d}{today.month:02d}{today.day:02d}" 

64 

65 return timestamp 

66 

67def today_to_directory(today): 

68 return today[0:4]+"/"+today[4:6]+"/"+today[6:8] 

69 

70 

71def main(): 

72 """ 

73 Initialize current_run_is and quit variables and get the current timestamp with Start_time = now() 

74 Initialize the logging system by setting the log file name and logging level 

75 Change the current working directory to /fefs/onsite/pipeline/rta/data 

76 Find the current reservations for nodes using the scontrol command and parse the output to find the individual node names 

77 Modify a configuration file with sed commands, replacing placeholders with the node names found in step 4 

78 Check if the ib0 network interface is in connected mode for each of the nodes and log the results 

79 """ 

80 current_run_is = -1 

81 quit = False 

82 

83 Start_time = now() 

84 

85 # today = str(datetime.datetime.now().year) + str(datetime.datetime.now().month) + str(datetime.datetime.now().day) 

86 today = get_night_timestamp(datetime.datetime.now()) 

87 logging.basicConfig(filename="log_AutoCheck_" + today + ".txt", level=logging.INFO) 

88 logging.info("Start RTA Auto Check for the day " + today) 

89 

90 RTA_ready = True 

91 

92 loop_id = 0 

93 

94 data_directory = "'/fefs/onsite/pipeline/rta/data/" 

95 os.system("rm -f /fefs/onsite/pipeline/rta/data/plots/*") 

96 while quit == False and RTA_ready: 

97 time.sleep(10) 

98 # print(current_run_is+1) 

99 if current_run_is == -1: 

100 current_run_is = get_current_run() 

101 if current_run_is == -1: 

102 logging.info("None") 

103 current_run_is = -1 

104 continue 

105 if current_run_is != get_current_run(): 

106 time.sleep(1) 

107 current_run_is = get_current_run() 

108 if current_run_is != -1: 

109 logging.info("Start RTA check for run " + str(current_run_is)) 

110 time.sleep(2) 

111 logging.info(datetime.datetime.now()) 

112 logging.info( 

113 "srun --reservation=rta_one_node ./Auto_Check_DL1.py -da " 

114 + today_to_directory(today) 

115 + " -r " 

116 + str(current_run_is + 1) 

117 ) 

118 os.system( 

119 "srun --reservation=rta_one_node ./Auto_Check_DL1.py -da " 

120 + today_to_directory(today) 

121 + " -r " 

122 + str(current_run_is + 1) 

123 ) 

124 os.system( 

125 "convert -density 300 /fefs/onsite/pipeline/rta/data/" 

126 + today_to_directory(today) 

127 + "/" 

128 + str(current_run_is + 1) 

129 + "/plots/output_DL1.pdf /fefs/onsite/pipeline/rta/data/" 

130 + today_to_directory(today) 

131 + "/" 

132 + str(current_run_is + 1) 

133 + "/plots/Check_DL1.png" 

134 ) 

135 

136 logging.info(datetime.datetime.now()) 

137 logging.info( 

138 "srun --reservation=rta_one_node ./Auto_Check_DL2.py -da " 

139 + today_to_directory(today) 

140 + " -r " 

141 + str(current_run_is + 1) 

142 ) 

143 os.system( 

144 "srun --reservation=rta_one_node ./Auto_Check_DL2.py -da " 

145 + today_to_directory(today) 

146 + " -r " 

147 + str(current_run_is + 1) 

148 ) 

149 os.system( 

150 "convert -density 300 /fefs/onsite/pipeline/rta/data/" 

151 + today_to_directory(today) 

152 + "/" 

153 + str(current_run_is + 1) 

154 + "/plots/output_DL2.pdf /fefs/onsite/pipeline/rta/data/" 

155 + today_to_directory(today) 

156 + "/" 

157 + str(current_run_is + 1) 

158 + "/plots/Check_DL2.png" 

159 ) 

160 

161 logging.info(datetime.datetime.now()) 

162 logging.info( 

163 "srun --reservation=rta_one_node ./merge_DL3.py --input-filter 'dl3_v06*' -d '/fefs/onsite/pipeline/rta/data/" 

164 + today_to_directory(today) 

165 + "/" 

166 + str(current_run_is + 1) 

167 + "/DL3/' -o '/fefs/onsite/pipeline/rta/data/" 

168 + today_to_directory(today) 

169 + "/" 

170 + str(current_run_is + 1) 

171 + "/DL3/' -r " 

172 + str(current_run_is + 1) 

173 ) 

174 os.system( 

175 "srun --reservation=rta_one_node ./merge_DL3.py --input-filter 'dl3_v06*' -d '/fefs/onsite/pipeline/rta/data/" 

176 + today_to_directory(today) 

177 + "/" 

178 + str(current_run_is + 1) 

179 + "/DL3/' -o '/fefs/onsite/pipeline/rta/data/" 

180 + today 

181 + "/" 

182 + str(current_run_is + 1) 

183 + "/DL3/' -r " 

184 + str(current_run_is + 1) 

185 ) 

186 

187 time.sleep(10) 

188 os.system( 

189 "srun --reservation=rta_one_node ./merge_DL3.py --input-filter 'dl3_v06*' -d '/fefs/onsite/pipeline/rta/data/" 

190 + today_to_directory(today) 

191 + "/" 

192 + str(current_run_is) 

193 + "/DL3/' -o '/fefs/onsite/pipeline/rta/data/" 

194 + today_to_directory(today) 

195 + "/" 

196 + str(current_run_is) 

197 + "/DL3/' -r " 

198 + str(current_run_is) 

199 ) 

200 

201 logging.info( 

202 "srun --reservation=rta_one_node ./merge_DL3.py --input-filter 'dl3_v06*' -d '/fefs/onsite/pipeline/rta/data/" 

203 + today_to_directory(today) 

204 + "/" 

205 + str(current_run_is + 1) 

206 + "/DL3/' -o '/fefs/onsite/pipeline/rta/data/" 

207 + today_to_directory(today) 

208 + "/" 

209 + str(current_run_is + 1) 

210 + "/DL3/' -r " 

211 + str(current_run_is + 1) 

212 ) 

213 if loop_id % 1 == 0: 

214 logging.info("Start RTA check for run " + str(current_run_is)) 

215 time.sleep(2) 

216 logging.info(datetime.datetime.now()) 

217 logging.info( 

218 "srun --reservation=rta_one_node ./Auto_Check_DL1.py -da " + today_to_directory(today) + " -r " + str(current_run_is + 1) 

219 ) 

220 os.system( 

221 "srun --reservation=rta_one_node ./Auto_Check_DL1.py -da " + today_to_directory(today) + " -r " + str(current_run_is + 1) 

222 ) 

223 os.system( 

224 "convert -density 300 /fefs/onsite/pipeline/rta/data/" 

225 + today_to_directory(today) 

226 + "/" 

227 + str(current_run_is + 1) 

228 + "/plots/output_DL1.pdf /fefs/onsite/pipeline/rta/data/" 

229 + today_to_directory(today) 

230 + "/" 

231 + str(current_run_is + 1) 

232 + "/Check_DL1.png" 

233 ) 

234 

235 logging.info(datetime.datetime.now()) 

236 logging.info( 

237 "srun --reservation=rta_one_node ./Auto_Check_DL2.py -da " + today_to_directory(today) + " -r " + str(current_run_is + 1) 

238 ) 

239 os.system( 

240 "srun --reservation=rta_one_node ./Auto_Check_DL2.py -da " + today_to_directory(today) + " -r " + str(current_run_is + 1) 

241 ) 

242 os.system( 

243 "convert -density 300 /fefs/onsite/pipeline/rta/data/" 

244 + today_to_directory(today) 

245 + "/" 

246 + str(current_run_is + 1) 

247 + "/plots/output_DL2.pdf /fefs/onsite/pipeline/rta/data/" 

248 + today_to_directory(today) 

249 + "/" 

250 + str(current_run_is + 1) 

251 + "/plots/Check_DL2.png" 

252 ) 

253 

254 logging.info(datetime.datetime.now()) 

255 logging.info( 

256 "srun --reservation=rta_one_node ./merge_DL3.py --input-filter 'dl3_v06*' -d '/fefs/onsite/pipeline/rta/data/" 

257 + today_to_directory(today) 

258 + "/" 

259 + str(current_run_is + 1) 

260 + "/DL3/' -o '/fefs/onsite/pipeline/rta/data/" 

261 + today_to_directory(today) 

262 + "/" 

263 + str(current_run_is + 1) 

264 + "/DL3/' -r " 

265 + str(current_run_is + 1) 

266 ) 

267 os.system( 

268 "srun --reservation=rta_one_node ./merge_DL3.py --input-filter 'dl3_v06*' -d '/fefs/onsite/pipeline/rta/data/" 

269 + today_to_directory(today) 

270 + "/" 

271 + str(current_run_is + 1) 

272 + "/DL3/' -o '/fefs/onsite/pipeline/rta/data/" 

273 + today_to_directory(today) 

274 + "/" 

275 + str(current_run_is + 1) 

276 + "/DL3/' -r " 

277 + str(current_run_is + 1) 

278 ) 

279 

280 os.system( 

281 "cp -rf /fefs/onsite/pipeline/rta/data/" 

282 + today_to_directory(today) 

283 + "/" 

284 + str(current_run_is + 1) 

285 + "/plots /fefs/onsite/pipeline/rta/data/." 

286 ) 

287 logging.info( 

288 "cp -rf /fefs/onsite/pipeline/rta/data/" 

289 + today_to_directory(today) 

290 + "/" 

291 + str(current_run_is + 1) 

292 + "/plots /fefs/onsite/pipeline/rta/data/." 

293 ) 

294 os.system("rm -f /fefs/onsite/pipeline/rta/data/plot_shifters/*") 

295 logging.info("rm -f /fefs/onsite/pipeline/rta/data/plot_shifters/*") 

296 os.system( 

297 "cp -f /fefs/onsite/pipeline/rta/data/" 

298 + today_to_directory(today) 

299 + "/" 

300 + str(current_run_is + 1) 

301 + "/plots/*_standard_theta2.png /fefs/onsite/pipeline/rta/data/plot_shifters/" 

302 ) 

303 logging.info( 

304 "cp -f /fefs/onsite/pipeline/rta/data/" 

305 + today_to_directory(today) 

306 + "/" 

307 + str(current_run_is + 1) 

308 + "/plots/*_standard_theta2.png /fefs/onsite/pipeline/rta/data/plot_shifters/" 

309 ) 

310 os.system( 

311 "cp -f /fefs/onsite/pipeline/rta/data/" 

312 + today_to_directory(today) 

313 + "/" 

314 + str(current_run_is + 1) 

315 + "/plots/*_standard__excess_significance_over_time.png /fefs/onsite/pipeline/rta/data/plot_shifters/" 

316 ) 

317 logging.info( 

318 "cp -f /fefs/onsite/pipeline/rta/data/" 

319 + today_to_directory(today) 

320 + "/" 

321 + str(current_run_is + 1) 

322 + "/plots/*_standard__excess_significance_over_time.png /fefs/onsite/pipeline/rta/data/plot_shifters/" 

323 ) 

324 os.system( 

325 "cp -f /fefs/onsite/pipeline/rta/data/" 

326 + today_to_directory(today) 

327 + "/" 

328 + str(current_run_is + 1) 

329 + "/plots/*_standard__spectra.png /fefs/onsite/pipeline/rta/data/plot_shifters/" 

330 ) 

331 logging.info( 

332 "cp -f /fefs/onsite/pipeline/rta/data/" 

333 + today_to_directory(today) 

334 + "/" 

335 + str(current_run_is + 1) 

336 + "/plots/*_standard__spectra.png /fefs/onsite/pipeline/rta/data/plot_shifters/" 

337 ) 

338 os.system( 

339 "cp -f /fefs/onsite/pipeline/rta/data/" 

340 + today_to_directory(today) 

341 + "/" 

342 + str(current_run_is + 1) 

343 + "/plots/*_standard__sky_map.png /fefs/onsite/pipeline/rta/data/plot_shifters/" 

344 ) 

345 logging.info( 

346 "cp -f /fefs/onsite/pipeline/rta/data/" 

347 + today_to_directory(today) 

348 + "/" 

349 + str(current_run_is + 1) 

350 + "/plots/*_standard__sky_map.png /fefs/onsite/pipeline/rta/data/plot_shifters/" 

351 ) 

352 loop_id = loop_id + 1 

353 if (datetime.datetime.timestamp(datetime.datetime.now()) - Start_time) > 14 * 3600: 

354 quit = True 

355 logging.info("End of the night, Stop the RTA check") 

356 

357 

358if __name__ == "__main__": 

359 main()