Coverage for lib/manaudit/lib_datou_audit.py: 4%

1import datetime 

2import types 

3 

4import pandas.core.dtypes.generic 

5 

6 

7# on pourrait extraire le datou des audit_info : il est juste au debut Ah ha ah ! 

8# Mais on a pas de fonction pour construire le datou à partir des json 

9# on pourrait aussi extraire les man_audit_info du datou plutot que du audit, enfin bon ! 

10# il faut nécessairement chercher dans les datou, ou au moins le dict qui le représente le param json utile 

11 

12def parse_audit_info(audit_info, list_action_for_df_to_correct = [], map_modif_df_manual = {}): 

13 try: 

14 # En fait il suffit de trouver load_tab 

15 if "io_exec" not in audit_info: 

16 return {}, {} 

17 

18 panda_table_content_as_markdown = "" 

19 panda_table_content_as_json = {} 

20 data_to_parse_for_json = "" 

21 suffix = "" 

22 nb_manual_action_df_for_col_audit = 0 

23 

24 map_col_id_to_id_df = {} 

25 

26 for steps in audit_info["io_exec"]: 

27 if not steps.isdigit(): 

28 continue 

29 type_step = audit_info["io_exec"][steps]["datou_step"] 

30 param_json = audit_info["io_exec"][steps]["param_json"] 

31 input = audit_info["io_exec"][steps]["input"] 

32 output = audit_info["io_exec"][steps]["output"] 

33 if type_step == "load_tab": 

34 if "assoc" in param_json: 

35 print("Will maybe faile, we need to find the right assoc") 

36 if panda_table_content_as_markdown != "": 

37 print("Will maybe fail, since we have two load tab step") 

38 panda_table_content_as_markdown = input["result"] if "result" in input else "" 

39 if type_step == "image_to_text": 

40 suffix = "google_ocr" 

41 # param_json = 

42 if type_step == "map_reduce": 

43 print("We want the second map_reduce, like id_step 7, ou peut-etre n'importe laquelle le champt result de input ?? ") 

44 if "text" in output: 

45 data_to_parse_for_json = output["text"] 

46 if type_step == "format": 

47 if "df_complet_as_markdown" in output: 

48 panda_table_content_as_markdown = output["df_complet_as_markdown"] 

49 if "df_complet_as_json" in output: 

50 panda_table_content_as_json = output["df_complet_as_json"] 

51 pass 

52 

53 from lib.batch.lib_batch import create_pandas_table_from_text 

54 

55 df = create_pandas_table_from_text(panda_table_content_as_markdown) 

56 import pandas as pd 

57 if panda_table_content_as_json != {} and type(panda_table_content_as_json) != types.NoneType and panda_table_content_as_json != "": 

58 df = pd.read_json(panda_table_content_as_json, convert_dates=["datet", "date_entree_hospitalisationt", "date_sortie_hospitalisationt", "date_fin_arret_travailt", "date_debut_arret_travailt"]) 

59 if type(df) == type(None): 

60 print("pd none from : " + str(panda_table_content_as_markdown)) 

61 df = pd.DataFrame(columns=["Null Column"]) 

62 else: 

63 df["intro_correct"] = "Le ..." 

64 df["cr_correct"] = "Le ..." 

65 df["cr_correct_typo"] = "" 

66 df["intro_correct_typo"] = "" 

67 df["cr_front_init"] = "init" 

68 df["intro_front_init"] = "init" 

69 df["auto_state"] = """<input type="checkbox" value="" name="bordered-checkbox" class="w-4 h-4 text-blue-600 bg-gray-100 border-gray-300 rounded focus:ring-blue-500 dark:focus:ring-blue-600 dark:ring-offset-gray-800 focus:ring-2 dark:bg-gray-700 dark:border-gray-600">Automation impossible""" 

70 df["auto_state_val"] = "false" 

71 df["class_vert_rouge_valider"] = "false" 

72 

73 from lib.lib_util import parse_json_from_prompt_result, \ 

74 complete_date_and_order_json_to_mettre_en_forme, append_id_by_order 

75 list_json_to_mettre_en_forme = parse_json_from_prompt_result(data_to_parse_for_json) 

76 list_json_to_mettre_en_forme = append_id_by_order(list_json_to_mettre_en_forme) 

77 

78 # if order_by_date: 

79 # list_json_to_mettre_en_forme = complete_date_and_order_json_to_mettre_en_forme(list_json_to_mettre_en_forme) 

80 

81 from lib.lib_util import add_parsing_meta_info_to_table 

82 # VR 29-3-24 : a mon avis cela a deja été réalisé 

83# df = add_parsing_meta_info_to_table(df, list_json_to_mettre_en_forme) 

84 

85 if len(list_action_for_df_to_correct) > 0: 

86 for action in list_action_for_df_to_correct: 

87 type_action = action["type_action"] if "type_action" in action else "default" 

88 if type_action == "df_meta_info_correct": 

89 key_action = create_key_action(action) 

90 count = (map_modif_df_manual[key_action]["count"] + 1) if key_action in map_modif_df_manual and "count" in map_modif_df_manual[key_action] else 1 

91 map_modif_df_manual[key_action] = action 

92 map_modif_df_manual[key_action]["count"] = count 

93 map_modif_df_manual[key_action]["audit_or_datou"] = "audit" 

94 

95 id_line = action["id"] 

96 if id_line == 'NaN': 

97 continue 

98 manual_value = action["manual_value"] 

99 col_name = action["col_name"] 

100 # refacto renommer front_value et rajouter un 

101 manual = bool(action["manual"]) if "manual" in action else False 

102 if manual : 

103 nb_manual_action_df_for_col_audit += 1 

104 #df.loc[int(id_line)][col_name] = manual_value 

105 try : 

106 # maanging incomplete parsing of document => not clear why it creates this but there are some link in audit bug 

107 id_line_int = int(str(id_line).replace(".0", "")) 

108 except Exception as e: 

109 print(" Pb with action id_line : " + str(id_line) + " " + str(e) + " parsing as float then int !") 

110 if id_line != 'None': 

111 print("ERROR treated as warning need to be audited") 

112 id_line_int = int(float(id_line)) 

113 else : 

114 continue 

115 

116 if map_col_id_to_id_df == {}: 

117 for i in range(len(df)): 

118 idx_df = df.index[i] 

119 idx_from_data = df.loc[idx_df]["id"] 

120 if str(idx_from_data).replace(".0", "").isdigit(): 

121 map_col_id_to_id_df[str(idx_from_data).replace(".0", "")] = idx_df 

122 else : 

123 print("ERROR should fail, wrong data will be saved ! " + str(idx_from_data)) 

124 if str(idx_df).isdigit(): 

125 map_col_id_to_id_df[str(idx_df)] = idx_df 

126 if str(id_line).replace(".0", "") in map_col_id_to_id_df: 

127 id_line_df = map_col_id_to_id_df[str(id_line).replace(".0", "")] 

128 # df.iloc[id_line_df][col_name] = manual_value 

129 df.loc[id_line_df, col_name] = manual_value 

130 # df.loc[:,('one','second')] 

131 else: 

132 print(" Missing id line in df : " + str(id_line) + " in " + str(map_col_id_to_id_df.keys())) 

133 

134# df.loc[id_line_int, col_name] = manual_value 

135 else: 

136 print("type action not supported : " + str(type_action)) 

137 

138 df_to_html = df.to_html(classes="table pdt-table table-striped", escape=False) 

139 

140 # TODO VR : a mettre dans la html clairement ! contrairement à ce que dit chat gpt 

141 return {"dataframe_html" : df_to_html, 

142 "dataframe_text" : panda_table_content_as_markdown, 

143 "suffix" : suffix, 

144 "df" : df, 

145 "paragraphs" : []}, {"nb_manual_action_df_for_col_audit" : nb_manual_action_df_for_col_audit, "map_modif_df_manual" : map_modif_df_manual} 

146 except Exception as e: 

147 print(str(e)) 

148 return {"dataframe_html": None, 

149 "dataframe_text": "", 

150 "suffix": ""}, {"nb_manual_action_df_for_col_audit" : nb_manual_action_df_for_col_audit, "map_modif_df_manual" : map_modif_df_manual} 

151 

152# dataframe_html = df.to_html() 

153 

154def load_sub_json(main_json, key_list_slash): 

155 try: 

156 if key_list_slash == "": 

157 return main_json 

158 separator = "/" 

159 # if "/" not in key_list_slash and "." in key_list_slash: 

160 # separator = "." 

161 list_keys = key_list_slash.lstrip(separator).split(separator) 

162 sub_json = main_json 

163 for k in list_keys: 

164 if type(sub_json) == list: 

165 if not k.isdigit(): 

166 print("Unexpected index for array in load_sub_json : " + str(k)) 

167 return None 

168 k_int = int(k) 

169 if k_int >= len(sub_json): 

170 print("Not enough value in array in load_sub_json") 

171 return None 

172 sub_json = sub_json[k_int] 

173 elif k in sub_json: # expected type dict 

174 sub_json = sub_json[k] 

175 # if type(sub_json) not in (dict, list): 

176 # return sub_json 

177 else: 

178 # Dict with key as integer, so we need to convert k to int used in io_exec 

179 if k.isdigit(): 

180 if int(k) in sub_json: 

181 sub_json = sub_json[int(k)] 

182 else: 

183 print(" missing key " + k + " in " + str(sub_json.keys())) 

184 return None 

185 return sub_json 

186 except Exception as e: 

187 print("Exception in load_sub_json") 

188 print(str(e)) 

189 return None 

190 

191def set_sub_json(main_json, key_list_slash, value): 

192 separator = "/" 

193# if "/" not in key_list_slash and "." in key_list_slash: 

194# separator = "." 

195 list_keys = key_list_slash.lstrip(separator).split(separator) 

196 if key_list_slash.lstrip(separator) == "": 

197 return value 

198 sub_json = main_json 

199 for k in list_keys[:-1]: 

200 if type(sub_json) == list: 

201 if not k.isdigit(): 

202 print("Unexpected index for array in load_sub_json : " + str(k)) 

203 return None 

204 k_int = int(k) 

205 if k_int >= len(sub_json): 

206 print("Not enough value in array in load_sub_json") 

207 return None 

208 sub_json = sub_json[k_int] 

209 elif k in sub_json: 

210 sub_json = sub_json[k] 

211 else: 

212 print(" missing key " + k + " in " + str(sub_json.keys())) 

213 sub_json[list_keys[-1]] = value 

214 return main_json 

215 

216def create_key_action(action): 

217 if "type_action" not in action or action["type_action"] != "df_meta_info_correct" or "col_name" not in action or "id" not in action: 

218 return None 

219 return str(action["id"]) + "_" + str(action["col_name"]) 

220 

221def is_numericf(input): 

222 if input == "NaN": 

223 return False 

224 try: 

225 float(input) 

226 return True 

227 except ValueError: 

228 return False 

229 except Exception as e: 

230 print("Unexpected error treated as warning") 

231 print(str(e)) 

232 return False 

233 

234 

235 

236def count_time_lab_by_user(all_result_manual_correct, delta_min_between_save_minutes = 10): 

237 map_user_id_time_modif = {} 

238 try: 

239 for result in all_result_manual_correct: 

240 user_id = result["user_id"] if "user_id" in result else 0 

241 if user_id not in map_user_id_time_modif: 

242 map_user_id_time_modif[user_id] = [] 

243 if "created_at" in result and type(result["created_at"]) == datetime.datetime: 

244 map_user_id_time_modif[user_id].append(result["created_at"]) 

245 except Exception as e: 

246 print(str(e)) 

247 

248 map_result_time_by_user = {} 

249 

250 for user_id in map_user_id_time_modif: 

251 map_user_id_time_modif[user_id] = sorted(list(set(map_user_id_time_modif[user_id]))) 

252 count_minutes = 0 

253 list_intervals = [] 

254 previous_time = None 

255 for time_modif in map_user_id_time_modif[user_id]: 

256 if previous_time == None: 

257 previous_time = time_modif 

258 list_intervals.append([time_modif, time_modif]) 

259 else: 

260 delta = time_modif - previous_time 

261 delta_in_minutes = delta.total_seconds() / 60.0 

262 if delta_in_minutes <= delta_min_between_save_minutes: 

263 # same interval 

264 list_intervals[-1][1] = time_modif 

265 else: 

266 list_intervals.append([time_modif, time_modif]) 

267 previous_time = time_modif 

268 

269 for interval in list_intervals: 

270 delta = interval[1] - interval[0] 

271 delta_in_minutes = delta.total_seconds() / 60.0 

272 count_minutes += delta_in_minutes + delta_min_between_save_minutes 

273 map_result_time_by_user[user_id] = {"intervals" : list_intervals, 

274 "total_minutes" : count_minutes} 

275 

276 return map_result_time_by_user 

277 

278 

279 

280def list_action_by_user(all_result_manual_correct, df_complete): 

281 map_user_id_list_pages = {} 

282 map_user_id_list_pages_for_split = {} 

283 map_user_id_time_modif = {} 

284 try: 

285 for result in all_result_manual_correct: 

286 if "manual_input_info" not in result or "list_actions" not in result["manual_input_info"]: 

287 print("Warning missing actions in result : " + str(result)) 

288 continue 

289 user_id = result["user_id"] if "user_id" in result else 0 

290 if user_id not in map_user_id_list_pages: 

291 map_user_id_list_pages[user_id] = [] 

292 map_user_id_time_modif[user_id] = [] 

293 map_user_id_list_pages_for_split[user_id] = [] 

294 if "created_at" in result and type(result["created_at"]) == datetime.datetime: 

295 map_user_id_time_modif[user_id].append(result["created_at"]) 

296 for action in result["manual_input_info"]["list_actions"]: 

297 if "id_page" in action: 

298 if not str(action["id_page"]).isdigit(): 

299 print("How come this id_page isn't an int : " + str(action["id_page"])) 

300 continue 

301 id_page = int(action["id_page"]) 

302 if id_page not in map_user_id_list_pages[user_id]: 

303 map_user_id_list_pages[user_id].append(id_page) 

304 if "col_name" in action and action["col_name"] == "Liste des pages": 

305 if id_page not in map_user_id_list_pages_for_split[user_id]: 

306 map_user_id_list_pages_for_split[user_id].append(id_page) 

307 elif "id" not in action or not is_numericf(str(action["id"])): 

308 print("Warning missing id in action : " + str(action)) 

309 continue 

310 else: 

311 id = int(float(action["id"])) 

312 list_pages = df_complete.loc[df_complete["id"] == id, "Liste des pages"].values[0] if "Liste des pages" in df_complete.columns else "" 

313 if list_pages == "" or not list_pages.replace(",", "").isdigit(): 

314 print("Warning missing list_pages in df_complete for id : " + str(id)) 

315 continue 

316 list_pages_as_list_int = [int(x) for x in list_pages.split(",") if x.strip().isdigit()] 

317 map_user_id_list_pages[user_id].extend(list_pages_as_list_int) 

318 if "col_name" in action and action["col_name"] == "Liste des pages": 

319 map_user_id_list_pages_for_split[user_id].extend(list_pages_as_list_int) 

320 for user_id in map_user_id_list_pages: 

321 map_user_id_list_pages[user_id] = sorted(list(set(map_user_id_list_pages[user_id]))) 

322 for user_id in map_user_id_list_pages_for_split: 

323 map_user_id_list_pages_for_split[user_id] = sorted(list(set(map_user_id_list_pages_for_split[user_id]))) 

324 except Exception as e: 

325 print(str(e)) 

326 

327 return map_user_id_list_pages, map_user_id_time_modif, map_user_id_list_pages_for_split 

328 

329def modify_audit_info_from_manual_correction(audit_info, all_result_manual_correct, 

330 manual_action_to_audit_data = {}, 

331 project_id = None, 

332 lib_user_data_internal = None): 

333 if manual_action_to_audit_data == {}: 

334 if lib_user_data_internal == None: 

335 print (" We need access to database to get the conf_project and saxia param") 

336 else : 

337 print(" We will load the configuration saxia from the project_id") 

338 if project_id == None: 

339 print("Error project_id is None and no manual_action_to_audit_data") 

340 

341 # We can still try to get the project_id from the hash_id_token, but won't do it right now 

342 hash_id_treatment = audit_info["config"]["complete_param_json"]["hash_id_treatment"] if "config" in audit_info and "complete_param_json" in audit_info["config"] and "hash_id_treatment" in audit_info["config"]["complete_param_json"] else None 

343 all_result = lib_user_data_internal.load_data_audit(hash_id_treatment=hash_id_treatment) 

344 # print(" all_result : " + str(all_result.keys())) 

345 project_id = all_result["project_id"] if "project_id" in all_result else None 

346 

347 if project_id == None: 

348 print(" We can't do anything") 

349 else : 

350 conf_project = lib_user_data_internal.load_conf_project(project_id) 

351 saxia_conf = conf_project["saxia"] if "saxia" in conf_project else {} 

352 assoc_conf = saxia_conf["assoc"] if "assoc" in saxia_conf else {} 

353 manual_action_to_audit_data = assoc_conf["manual_action_to_audit_data"] if "manual_action_to_audit_data" in assoc_conf else {} 

354 

355 nb_modif_class_manual = 0 

356 nb_manual_action_df = 0 

357 map_modif_df_manual = {} # permettrait d'ailleurs une projection 

358 paragraphs = [] 

359 df_auto = None 

360 try: 

361 # key_as_list_slash = "io_exec/0/output/paragraphs" 

362 key_as_list_slash = "io_exec/3/input/paragraphs" 

363 key_as_list_slash = manual_action_to_audit_data["class_paragraph"]["audit_data"] if "class_paragraph" in manual_action_to_audit_data and "audit_data" in manual_action_to_audit_data["class_paragraph"] else key_as_list_slash 

364 audit_info_corrected = audit_info 

365 

366 map_col_id_to_id_df = {} 

367 

368 

369 list_action_for_df_to_correct = [] 

370 paragraphs = load_sub_json(audit_info_corrected, key_as_list_slash) 

371 # order modification by increasing date 

372 all_result_manual_correct_ordered = sorted(all_result_manual_correct, key=lambda x: x["created_at"]) 

373 map_count_modif_per_doc = {} 

374 map_modif_type_document = {} 

375 for result in all_result_manual_correct_ordered: 

376 created_at_as_string = result["created_at"].strftime("%y%m%d_%H:%m:%S") 

377 print(" Modification from created_at_as_string :" + created_at_as_string + " result id : " + str(result["id"])) 

378 manual_input_info = result["manual_input_info"] if "manual_input_info" in result else {} 

379 list_actions = manual_input_info["list_actions"] if "list_actions" in manual_input_info else [] 

380 for action in list_actions: 

381 type_action = action["type_action"] if "type_action" in action else "default" 

382 if type_action == "class_paragraph": 

383 nb_modif_class_manual += 1 

384 # class_action stocke un id_page au lieu d'un id_file => impact important pour corriger, donc on va sans doute garder comme cela 8/4/24 

385 id_page = action["id_page"] 

386 if id_page not in map_count_modif_per_doc: 

387 map_count_modif_per_doc[id_page] = 0 

388 map_count_modif_per_doc[id_page] += 1 

389 id_paragraph = action["id_paragraph"] if "id_paragraph" in action else None 

390 manual_class = action["manual_class"] + "_class" if "manual_class" in action else None 

391 manual_class.replace("_class_class", "_class") 

392 id_page_int = int(id_page) 

393 if id_paragraph == None: 

394 print("ERROR due certainly to modified crops modified again by switch 2025-05 ") 

395 continue 

396 id_paragraph_int = int(id_paragraph) 

397 if id_page_int < len(paragraphs): 

398 if id_paragraph_int < len(paragraphs[id_page_int]): 

399 paragraphs[id_page_int][id_paragraph_int]["class"] = manual_class 

400 else: 

401 print("ERROR id_paragraph_int greater than len(paragraphs[id_page_int])" + str(id_paragraph_int) + " nb : " + str(len(paragraphs[id_page_int])) + " for " + str(id_page_int)) 

402 continue 

403 else : 

404 print("ERROR id_page_int greater than len(paragraphs)" + str(id_page_int) + " nb : " + str(len(paragraphs))) 

405 continue 

406 

407# audit_info_corrected["io_exec"]["3"]["input"]["paragraphs"] = paragraphs 

408 

409 elif type_action == "df_meta_info_correct": 

410 key_action = create_key_action(action) 

411 count = (map_modif_df_manual[key_action]["count"] + 1) if key_action in map_modif_df_manual and "count" in map_modif_df_manual[key_action] else 1 

412 map_modif_df_manual[key_action] = action 

413 map_modif_df_manual[key_action]["count"] = count 

414 map_modif_df_manual[key_action]["audit_or_datou"] = "datou" 

415 

416 # list_action_for_df_to_correct.append(action) 

417 if True: 

418 key_for_df_as_md = "io_exec/6/output" # df_complet_as_markdown 

419 key_for_df_as_md = "io_exec/9/output/df_complet_as_markdown" # df_complet_as_markdown # C'est 9 en prod grrr => ca va pas du tout grrr 

420 key_for_df_as_md = manual_action_to_audit_data["df_meta_info_correct"]["audit_data"] if "df_meta_info_correct" in manual_action_to_audit_data and "audit_data" in \ 

421 manual_action_to_audit_data["df_meta_info_correct"] else key_for_df_as_md 

422 df_as_md = load_sub_json(audit_info_corrected, key_for_df_as_md) 

423 key_for_df_as_json = "io_exec/9/output/df_complet_as_json" 

424 key_for_df_as_json = manual_action_to_audit_data["df_meta_info_correct_json"]["audit_data"] if "df_meta_info_correct_json" in manual_action_to_audit_data and "audit_data" in \ 

425 manual_action_to_audit_data["df_meta_info_correct_json"] else key_for_df_as_json 

426 df_as_json = load_sub_json(audit_info_corrected, key_for_df_as_json) 

427 

428 

429 if df_as_json != None: 

430 import pandas as pd 

431 df = pd.read_json(df_as_json, convert_dates=["datet", "date_entree_hospitalisationt", "date_sortie_hospitalisationt", "date_fin_arret_travailt", "date_debut_arret_travailt"]) 

432 

433 #from copy import deepcopy 

434 if type(df_auto) == types.NoneType: 

435 df_auto = df.copy() # deep=True 

436 #df_auto = deepcopy(df_as_json) 

437 

438 elif df_as_md != None: 

439 print("ERROR SINCE 06-2024 : WE NEVER GO THROUGH THIS !?! ") 

440 # from lib.batch.lib_batch import create_pandas_table_from_text 

441 # df = create_pandas_table_from_text(df_as_md, verbose = False) 

442 print("ON 08-2024 : WE COMMENT THE TWO PREVIOUS LINE AND RETURN => LOAD AUDIT WILL CRASH ") 

443 return None 

444 else: 

445 print("ERROR FROM THE START ") 

446 from lib.batch.lib_batch import init_df_synchronize 

447 df = init_df_synchronize() #pandas.core.dtypes.generic.create_pandas_abc_type("DataFrame", []) 

448 if action["col_name"] in df.columns: 

449 id_line = action["id"] 

450 from lib.manaudit.lib_datou_audit import is_numericf 

451 if not is_numericf(id_line): 

452# if id_line == "" or (id_line != "0" and not str(id_line).rstrip(".0").isdigit()): 

453 print(" action not treated since input data is wrong : " + str(id_line) + " " + str(action)) 

454 continue 

455 manual_value = action["manual_value"] 

456 col_name = action["col_name"] 

457 if col_name == "datet": 

458 #print("Will maybe fail : " + str(action)) 

459 if type(df.loc[0,"datet"]) != pandas._libs.tslibs.timestamps.Timestamp and type(df.loc[0,"datet"]) != datetime.datetime and type(df.loc[0,"datet"]) != pandas._libs.tslibs.nattype.NaTType: 

460 print("Will fail now : " + str(action) + " result id : " + str(result["id"])) 

461 print(str(type(df.loc[0,"datet"]))) 

462 if col_name == "date_entree_hospitalisationt" or col_name == "date_sortie_hospitalisationt" or col_name == "date_fin_arret_travailt" or col_name == "date_debut_arret_travailt" or col_name == "datet": 

463 import dateparser 

464 manual_value_parsed = dateparser.parse(manual_value) 

465 print(manual_value_parsed) 

466 if manual_value_parsed == None: 

467 continue 

468 if col_name == "document_type": 

469 map_modif_type_document[id_line] = manual_value 

470 if map_col_id_to_id_df == {}: 

471 for i in range(len(df)): 

472 idx_df = df.index[i] 

473 idx_from_data = df.loc[idx_df]["id"] 

474 if str(idx_from_data).replace(".0", "").isdigit(): 

475 map_col_id_to_id_df[str(idx_from_data).replace(".0", "")] = idx_df 

476 else: 

477 print("ERROR should fail, wrong data will be saved ! : " + str(idx_from_data)) 

478# if str(idx_df).isdigit(): 

479# map_col_id_to_id_df[str(idx_df)] = idx_df 

480 if str(id_line).replace(".0", "") in map_col_id_to_id_df: 

481 id_line_df = map_col_id_to_id_df[str(id_line).replace(".0", "")] 

482 #df.iloc[id_line_df][col_name] = manual_value 

483 if col_name == "datet" and ("20" not in str(manual_value) and "19" not in str(manual_value)): 

484 print("Protect wrong date : " + str(manual_value)) 

485 manual_value = None 

486 df.loc[id_line_df, col_name] = manual_value 

487 else: 

488 print(" Missing id line in df : " + str(id_line) + " in " + str(map_col_id_to_id_df.keys())) 

489 

490# df.iloc[int(float(id_line))][col_name] = manual_value 

491 print("TO TEST") 

492 df_as_md = df.to_markdown() 

493 df_as_json = df.to_json() 

494 

495 # refacto renommer front_value et rajouter un 

496 manual = bool(action["manual"]) if "manual" in action else False 

497 if manual : 

498 nb_manual_action_df += 1 

499 

500 set_sub_json(audit_info_corrected, key_for_df_as_md, df_as_md) 

501 set_sub_json(audit_info_corrected, key_for_df_as_json, df_as_json) 

502 # audit_info_corrected["io_exec"]["9"]["output"]["df_complet_as_markdown"] = df.to_markdown() 

503 else: 

504# print("col_name not in df : " + str(action["col_name"]) + " in " + str(df.columns)) 

505 list_action_for_df_to_correct.append(action) 

506 elif type_action == "crops": 

507 id_page = action["id_page"] if "id_page" in action else None 

508 if id_page == None: 

509 print("ERROR treated as warning id_page not in action") 

510 continue 

511 paragraphs[id_page] #["crops"] = action["crops"] if "crops" in action else None 

512 

513 if "modify" in action: 

514 for modification in action["modify"]: 

515 if "id" not in modification: 

516 print("ERROR id of paragraph not in modification") 

517 continue 

518 if "x" in modification: 

519 x = modification["x"] 

520 paragraphs[id_page][int(id)]["x"] = x 

521 if "y" in modification: 

522 y = modification["y"] 

523 paragraphs[id_page][int(id)]["y"] = y 

524 if "w" in modification: 

525 w = modification["w"] 

526 paragraphs[id_page][int(id)]["w"] = w 

527 if "h" in modification: 

528 h = modification["h"] 

529 paragraphs[id_page][int(id)]["h"] = h 

530 if "text" in modification: 

531 text = modification["text"] 

532 paragraphs[id_page][int(id)]["text"] = text 

533 

534 if "delete" in action: 

535 for id in action["delete"]: 

536 if not str(id).isdigit(): 

537 print("Eror due to deleted crop that have no id") 

538 continue 

539 if int(id) < len(paragraphs[id_page]): 

540 paragraphs[id_page][int(id)] = {} 

541# del paragraphs[id_page][int(id)] 

542 else: 

543 print("ERROR id of paragraph not in modification") 

544 if "add" in action: 

545 for one_new_par in action["add"]: 

546 new_id = len(paragraphs[id_page]) 

547 one_new_par["id"] = new_id 

548 paragraphs[id_page].append(one_new_par) 

549 #if id not in paragraphs[id_page]: 

550 # paragraphs[id_page][id] = {} 

551 #else: 

552 # print("ERROR id of paragraph not in modification") 

553 else: 

554 print("type action not supported : " + str(type_action)) 

555 except Exception as e: 

556 print(str(e)) 

557 print("Error treated as warning (to be audited) in modify_audit_info_from_manual_correction") 

558 audit_info_corrected = audit_info 

559# list_action_for_df_to_correct = [] 

560 

561 

562 

563 results, audit_info_from_datou = parse_audit_info(audit_info_corrected, list_action_for_df_to_correct, map_modif_df_manual) 

564 nb_manual_action_df_for_col_audit = audit_info_from_datou["nb_manual_action_df_for_col_audit"] if "nb_manual_action_df_for_col_audit" in audit_info_from_datou else -4 

565 map_modif_df_manual = audit_info_from_datou["map_modif_df_manual"] if "map_modif_df_manual" in audit_info_from_datou else {} 

566 audit_info_count = {"nb_modif_class_manual" : nb_modif_class_manual, "nb_manual_action_df" : nb_manual_action_df, 

567 "nb_manual_action_df_for_col_audit" : nb_manual_action_df_for_col_audit, 

568 "map_modif_df_manual" : map_modif_df_manual, 

569 "map_count_modif_per_doc" : map_count_modif_per_doc, 

570 "map_modif_type_document" : map_modif_type_document} 

571 results["paragraphs"] = paragraphs 

572 return df_auto, results, audit_info_count 

573 

574 

575 

576def load_audit_info_and_apply_manual_correction(hash_id_treatment_auto = None, 

577 hash_id_treatment_manual = None, 

578 lpgss = None, 

579 limit = None, 

580 project_id = None, 

581 ids_manual = None, 

582 manual_action_to_audit_data = {}): 

583 import json 

584 df_auto = None 

585 all_result = lpgss.load_data_audit(hash_id_treatment=hash_id_treatment_manual, col_csv="*") 

586 # print(" all result : " + str(all_result)) 

587 if type(all_result) == type(None): 

588 print(" all_result seems none from hash_id_treatment : " + str(hash_id_treatment_manual)) 

589 all_result = {} 

590 print(" all_result : " + str(all_result.keys())) 

591 info_date = all_result["info_date"] if "info_date" in all_result else {} 

592 info_date["test_var_info_date"] = "We want to find in which hit this data is recorded, either hit_main or cons_hit" 

593 audit_json_file_content = all_result["audit_info"] if "audit_info" in all_result else {} 

594 id_file = all_result["id_file"] if "id_file" in all_result else None 

595 try: 

596 audit_json_file_content_as_json = json.loads(audit_json_file_content) 

597 except Exception as e: 

598 audit_json_file_content_as_json = audit_json_file_content 

599 print( 

600 str("A présent c'est sur le chemin critique, je pense qu'un ajax règlerait le pb, mais je ne comprends pas")) 

601 print(str(e)) 

602 

603 try: 

604 from lib.manaudit.lib_datou_audit import parse_audit_info, modify_audit_info_from_manual_correction 

605 if limit != None and int(limit) == 0: 

606 all_results = [] 

607 else: 

608 all_results = lpgss.load_data_manual(ids_manual=ids_manual, 

609 hash_id_treatment=hash_id_treatment_manual, 

610 limit=limit) 

611 if len(all_results) == 0: 

612 results, audit_info_count = parse_audit_info(audit_json_file_content) 

613 nb_manual_action_df_for_col_audit = audit_info_count[ 

614 "nb_manual_action_df_for_col_audit"] if "nb_manual_action_df_for_col_audit" in audit_info_count else -3 

615# audit_info_count = {} 

616 df_auto_as_json = None 

617 print("There are no correction, to be tested") 

618 df_auto = results["df"].copy() # deep = True 

619 else: 

620 df_auto, results, audit_info_count = modify_audit_info_from_manual_correction( 

621 audit_json_file_content_as_json, 

622 all_results, 

623 manual_action_to_audit_data=manual_action_to_audit_data, 

624 project_id=project_id) 

625 except Exception as e: 

626 results = {"error": str(e)} 

627 audit_info_count = {} 

628 print(str(e)) 

629 print("LOG_TO_PARSE : Error parsing audit result json manual : " + str(hash_id_treatment_manual) + " auto " + str(hash_id_treatment_auto)) 

630 

631 hash_id_treatment_rerun = all_result["info_date"]["consolidate_hash_id_treatment"] if "info_date" in all_result and "consolidate_hash_id_treatment" in all_result["info_date"] else None 

632 df_cons = all_result["info_consolidate"]["df_cons"] if "info_consolidate" in all_result and "df_cons" in all_result["info_consolidate"] else None 

633 

634 return df_auto, df_cons, hash_id_treatment_rerun, results, audit_info_count, id_file, audit_json_file_content_as_json, all_results, info_date 

635 

636def get_list_backup(histo_folder, mtd_id = None): 

637 # Dans le dossier histo_folder on veut parser des fichiers json avec des noms du type : datou_anon_42_0213_22.json ou 42 est le datou_id et celui la c'est celui de 22h le 31 février, il faudrait avoir l'année de manière optionnel 2024 par defaut 

638 # On veut aussi pouvoir filtrer par mtd_id mais c'est moins important 

639 # On veut aussi pouvoir trier par date pour afficher dans le front 

640 import os 

641 import json 

642 map_datou_id_date_backup = {} 

643 map_proj_id_date_backup = {} 

644 if histo_folder == None: 

645 return map_datou_id_date_backup, map_proj_id_date_backup 

646 for filename in os.listdir(histo_folder): 

647 if filename.endswith(".json"): 

648 print("Open " + str(filename)) 

649 with open(os.path.join(histo_folder, filename)) as f: 

650 try: 

651 data = json.load(f) 

652 except Exception as e: 

653 print(" Error reading " + filename + " : " + str(e)) 

654 continue 

655 

656 if "project_id" in data: 

657 map_proj_id_date_backup[data["project_id"]] = data 

658 continue 

659 

660 if type(data) == list : 

661 if len(data) == 1: 

662 data = data[0] 

663 else : 

664 print("Maybe warnind size of data in " + filename + " : " + str(len(data)) + " avoiding this file ") 

665 

666 if "id" not in data: # to manage the case of the intricate export 

667 if "datou" not in data or len(data["datou"]) == 0: 

668 print("Unexpected data in " + filename + " : " + str(data) + " avoiding this file ") 

669 continue 

670 

671 data = data["datou"][0] 

672 

673 if mtd_id != None: 

674 if "id" in data and int(data["id"]) != int(mtd_id): 

675 continue 

676 if "id" in data and data["id"] not in map_datou_id_date_backup: 

677 map_datou_id_date_backup[data["id"]] = {} 

678 

679 print(" filename : " + str(filename)) 

680 

681 import re 

682 # exemple de nom de fichier pour parser la date : datou_anon_42_0213_22.json (et vérifier l'id du datou) 

683 

684 suffix = "" 

685 try: 

686 date_parsed = datetime.datetime.now() 

687 m = re.search(r"_(\d+)_([\s\w\W-]*_)?(\d{4})?(\d{2})(\d{2})_(\d{2})", filename) 

688 

689 if m: 

690 if len(m.groups()) == 5 and m.group(2) != None and m.group(2) != "": 

691 print(" PArsed 5 groups ! " + str(m.groups()))# a tester avec le truc ligne 92 

692 date_parsed = datetime.datetime(int(m.group(2)), int(m.group(3)), int(m.group(4)), 

693 int(m.group(5))) 

694 elif len(m.groups()) == 5: 

695 print(" PArsed 4 real groups ! " + str(m.groups())) 

696 default_year = 2024 

697 month = int(m.group(3)) 

698 day = int(m.group(4)) 

699 hour = int(m.group(5)) 

700 

701 date_parsed = datetime.datetime(default_year, month, day, hour) 

702 elif len(m.groups()) == 6: 

703 print(" Parsed 6 groups ! " + str(m.groups())) 

704 default_year = 2024 if (m.group(3) == None or m.group(3) != "") else int(m.group(3)) 

705 month = int(m.group(4))