Coverage for auth/lib_conf

1#!/usr/bin/env python

2# -*- coding: utf-8 -*-

3__author__ = 'moilerat'

5__license__ = "LICENSETOBEDEFINED : on premise treatment custom"

7import json

8import socket

10"""lib_conf_system.py Access to configuration of the whole system"""

11__credits__ = [""]

12try:

13 from lib.lib_version import version_name

14except Exception as e:

15 print(str(e))

16 version_name = "0.0.0"

17__version__ = version_name

18__maintainer__ = "moilerat and Safia Team"

19__email__ = "safia.dev@rubbia.fr"

20__status__ = "Alpha Dev See version_name"

21# TODO VR 8-7-23 HEader to be deploy on all files on 14-7-23 or only in lib_version

23HC_DEFAULT_CONF_PROJECT_ID = 70

25import os

27class LibConfSystem():

28 def __init__(self):

29 self.conf = {}

31 import credentials

32 self.conf_json_raw = credentials.conf

33 print(" _cr_cnf_ begin 10000 : " + str(self.conf_json_raw)[:10000])

34 print(" _cr_cnf_ : " + os.path.abspath(credentials.__file__))

35 print(" _cr_cnf_ : " + str(credentials.__file__))

36 #print(" _cr_cnf_ : " + str(_cr_cnf_.__le__))

38 api_key_at = self.get_custom_conf('AIRTABLE.api_key')

39 base_id_pi = self.get_custom_conf('AIRTABLE.base_id_pi')

40 base_id_apia = self.get_custom_conf('AIRTABLE.base_id_apia')

42 self.at_pe_conf = (api_key_at, base_id_pi) # fot otp connect as of 7-7-23

43 self.at_apia_conf = (api_key_at, base_id_apia) # fot external data like github_parameter as of 7-7-23

45 PG_HOST = self.get_custom_conf('PG.PG_HOST')

46 PG_USER = self.get_custom_conf('PG.PG_USER')

47 PG_PASSWORD = self.get_custom_conf('PG.PG_PASSWORD')

48 PG_DB = self.get_custom_conf('PG.PG_DB')

49 PG_PORT = self.get_custom_conf('PG.PG_PORT')

51 self.pg_conf = (PG_HOST, PG_USER, PG_PASSWORD, PG_DB, PG_PORT)

53 OPENAI_API_KEY = self.get_custom_conf('SAFIA.OPENAI_API_KEY')

54 self.openai_api_key = OPENAI_API_KEY

56 LIGHTON_API_KEY = self.get_custom_conf('SAFIA.API_LIGHTON_KEY')

57 self.lighton_api_key = LIGHTON_API_KEY

59 STRIPE_PUBLIC_KEY = self.get_custom_conf("STRIPE.PUBLIC_KEY")

60 self.stripe_public_key = STRIPE_PUBLIC_KEY

61 STRIPE_SECRET_KEY = self.get_custom_conf("STRIPE.SECRET_KEY")

62 self.stripe_secret_key = STRIPE_SECRET_KEY

64 FVS_PATH = self.get_custom_conf("SAFIA.FVS_PATH")

65 self.fvs_path = FVS_PATH

67 CERT_SSL = self.get_custom_conf("SAFIA.CERT_SSL", "/home/safia/.ssl/safia.app.crt")

68 self.cert_ssl = CERT_SSL

70 KEY_SSL = self.get_custom_conf("SAFIA.KEY_SSL", "/home/safia/.ssl/safia.app.key")

71 self.key_ssl = KEY_SSL

73 CONF_SAFIA_HTTP_APACHE2_SSL = self.get_custom_conf("SAFIA.CONF_SAFIA_HTTP_APACHE2_SSL", False)

74 self.conf_safia_http_apache2_ssl = CONF_SAFIA_HTTP_APACHE2_SSL

76 GOOGLE_API_KEY = self.get_custom_conf('SAFIA.GOOGLE_API_KEY')

77 self.google_api_key = GOOGLE_API_KEY

79 # VR 2-10 temporary for migration of parametrization

80 default_value_google_credential_cracra = os.path.join(os.environ["GITSAFIA"], "prompt/python", "google_credential_cracra.json") if "GITSAFIA" in os.environ else os.path.join(os.environ["HOME"], "workarea/git/Safia/prompt/python", "google_credential_cracra.json")

81 GOOGLE_APPLICATION_CREDENTIALS = self.get_custom_conf("SAFIA.GOOGLE_APPLICATION_CREDENTIALS", default_value_google_credential_cracra)

82 self.google_api_key = GOOGLE_APPLICATION_CREDENTIALS

84 ADMIN_GROUP_ID = self.get_custom_conf("SAFIA.ADMIN_GROUP_ID")

85 self.admin_group_id = ADMIN_GROUP_ID

87 default_conf_project_ID = self.get_custom_conf("SAFIA.DEFAULT_CONF_PROJECT_ID", HC_DEFAULT_CONF_PROJECT_ID)

88 self.default_conf_project_id = default_conf_project_ID

90 from lib.lib_util import load_json

91 from pkg_resources import resource_filename

92# config_file = resource_filename("server", "config.json")

93# config_file_from_devops = resource_filename("", "config_devops_file.json")

94 try:

95 devops_complete_config = load_json("../config_devops_file.json")

96 from lib.manaudit.lib_datou_audit import load_sub_json

97 config_dict = load_sub_json(devops_complete_config, "VersionConfig/app_exec")

98 except Exception as e:

99 print("Error loading config file ../config_devops_file.json " + str(e))

100 print("Default HC conf choosen ")

101 config_dict = {}

102 self.non_secure_conf = config_dict

103

104 def get_pg_conf(self):

105 return self.pg_conf

106

107 def get_at_pe_conf(self):

108 return self.at_pe_conf

109

110 def get_at_apia_conf(self):

111 return self.at_apia_conf

112

113 def get_openai_api_key(self):

114 return self.openai_api_key

115

116 def get_google_api_key(self):

117 return self.google_api_key

118

119 def set_conf_google_environ(self):

120 import os

121 os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = self.google_api_key

122

123 def get_lighton_api_key(self):

124 return self.lighton_api_key

125

126 def get_stripe_public_key(self):

127 return self.stripe_public_key

128

129 def get_stripe_secret_key(self):

130 return self.stripe_secret_key

131

132 def get_fvs_path(self):

133 return self.fvs_path

134

135 def get_cert_ssl(self):

136 return self.cert_ssl

137

138 def get_key_ssl(self):

139 return self.key_ssl

140

141 def get_conf_safia_http_apache2_ssl(self):

142 return self.conf_safia_http_apache2_ssl

143

144 def get_admin_group_id(self):

145 return self.admin_group_id

146

147 def get_custom_conf(self, path, default_value = None):

148 if not hasattr(self, "conf_json_raw") or self.conf_json_raw == None or self.conf_json_raw == {}:

149 print("Missing conf_json_raw, internal error, returning None !")

150 return default_value

151

152 # - [ ] TODO lib_conf_system.py check path is . and alphabet de what

153 sub_path = path.split(".")

154 data = self.conf_json_raw

155 for s in sub_path:

156 if s not in data:

157 print("Missing info for conf variable " + path + " returning None")

158 return default_value

159 data = data[s]

160

161 return data

162

163 def get_non_secure_conf(self, key = None, default_value = None):

164 if key == None:

165 return self.non_secure_conf

166 elif self.non_secure_conf == None:

167 print("Warning : non_secure_conf is None, returning default_value (maybe None) for key " + str(key))

168 return default_value

169 elif "/" in key:

170 from lib.manaudit.lib_datou_audit import load_sub_json

171 val = load_sub_json(self.non_secure_conf, key)

172 if val == None:

173 return default_value

174 return val

175 elif key not in self.non_secure_conf:

176 return default_value

177 else:

178 return self.non_secure_conf[key] if key in self.non_secure_conf else default_value

179

180 def get_app_type(self):

181 app_type = self.get_non_secure_conf("app_type")

182 if app_type == None:

183 return "default" # ou safia ?

184 else:

185 return app_type

186

187 def get_config_app(self):

188 from auth.lib_conf_app import all_conf

189 app_type = self.get_app_type()

190 if app_type not in all_conf:

191 print("App type " + app_type + " not found in all_conf")

192 return {}

193 else:

194 return all_conf[app_type]

195

196 def get_default_conf_project_id(self):

197 app_type = self.get_default_conf_project_id("app_type")

198 if app_type == None:

199 return "default" # ou safia ?

200 else:

201 return app_type

202

203 # VR TODO refacto 31-7-24 : Pourrait etre déplacer dans une librairie gérant les différents types de configuration et en laissant ici ceux lié à l'application

204 def aux_parse_json_load_and_instantiate_config(self, sub_json_config_with_reference, sub_keys_of_current_conf):

205 if type(sub_json_config_with_reference) == dict:

206 return self.aux_parse_json_load_and_instantiate_config_dict(sub_json_config_with_reference, sub_keys_of_current_conf)

207 elif type(sub_json_config_with_reference) == list:

208 return self.aux_parse_json_load_and_instantiate_config_array(sub_json_config_with_reference, sub_keys_of_current_conf)

209 elif type(sub_json_config_with_reference) == str:

210 return sub_json_config_with_reference

211 elif type(sub_json_config_with_reference) == int:

212 return sub_json_config_with_reference

213 elif type(sub_json_config_with_reference) == float:

214 return sub_json_config_with_reference

215 elif type(sub_json_config_with_reference) == bool:

216 return sub_json_config_with_reference

217 elif type(sub_json_config_with_reference) == "<class 'datetime.datetime'>":

218 return sub_json_config_with_reference

219 else:

220 print("ERROR (we should never be here and we could then simplify the code, we could also simply set a warning a return the value) in aux_parse_json_load_and_instantiate_config, type not recognized " + str(type(sub_json_config_with_reference)))

221

222 def aux_parse_json_load_and_instantiate_config_array(self, sub_json_config, sub_keys):

223 instance_json_arr = []

224 for i, sub_json in enumerate(sub_json_config):

225 instance_json_arr.append(self.aux_parse_json_load_and_instantiate_config(sub_json, sub_keys + "/" + str(i)))

226 return instance_json_arr

227

228 # VR 28-10-24 : je ne comprends pas comment ca marche, mais c'est utilisé !

229 # peut-etre que "saxia.config.default" est deprecated au 25/10/24, clairement c'est une recherche de key word !

230 def aux_parse_json_load_and_instantiate_config_dict(self, sub_json_config, sub_keys = ""):

231 has_default_conf = "saxia.config.default" in sub_json_config.keys()

232 has_project_id_conf = "saxia.config.project" in sub_json_config.keys()

233 if not has_default_conf and not has_project_id_conf:

234 instance_sub_json = dict()

235 for k in sub_json_config:

236 instance_sub_json[k] = self.aux_parse_json_load_and_instantiate_config(sub_json_config[k], sub_keys + "/" + k)

237 return instance_sub_json

238

239 # VR TODO pour l'instant on ne fait pas d'instantiation imbriquée, c'est à dire que les projets "template" doivent avoir les configurations correctes

240 # Ce comportement est géré (sans les droits dans la nouvelles librairies LFC à déployer, lib_formal_conf)

241 if has_default_conf:

242 print("DEPRECATED 28/10/24 TO DELETE 11/11/24")

243 default_conf = sub_json_config["saxia.config.default"]

244 else :

245 default_conf = {}

246

247 project_configuration_referenced = {}

248 if has_project_id_conf:

249 if "project_id" in sub_json_config["saxia.config.project"]:

250 project_id = sub_json_config["saxia.config.project"]["project_id"]

251 else:

252 project_id = self.get_default_conf_project_id()

253

254 if project_id != None:

255 from server.safia import lpgss_singleton

256 project_configuration_referenced = lpgss_singleton.load_conf_project(project_id)

257 from lib.manaudit.lib_datou_audit import load_sub_json

258 project_configuration_referenced = load_sub_json(project_configuration_referenced, sub_keys.lstrip("/"))

259 else:

260 print("Warning : internal error treated as warning in 7-24 implementation and configuration thanks HC_DEFAULT_CONF_PROJECT_ID")

261 project_configuration_referenced = {}

262

263 from pydantic.utils import deep_update

264

265 # ne sert strictement à rien car default_conf est toujours vide !

266 instantite_conf = deep_update(project_configuration_referenced, default_conf)

267 return instantite_conf

268

269 def load_and_instantiate_config(self, raw_json_config):

270 parsed_and_load_json_config = self.aux_parse_json_load_and_instantiate_config(raw_json_config, "")

271 return parsed_and_load_json_config

272

273# --id_version=lamed --job=saxia.save_config -v

274# --job=saxia.load_config -v --in_file=../devops/histo_config/lamed_alpha_air-victor.home_config_version_20250530_22

275

276 def save_all_useful_config(self, root_output = "temp", id_version = "tav"):

277 all_to_save = {}

278

279 import datetime

280 now = datetime.datetime.now()

281 date_str = now.strftime("%Y%m%d_%H")

282

283 dir_version = id_version + "_alpha_" + socket.gethostname() + "_" + "config_version_" + date_str

284 print(" dir_version : " + str(dir_version))

285 print(" complte_die : " + str(os.path.join(root_output, dir_version)))

286

287 # VR TODO 31-7-24 : en fait on est plus de 86400 secondes avantce jour

288 # Liste des configs utiles

289 list_datou_useful = [25, 40, 44, 45, 46, 51, 52, 55] #, 23 # ?? 23 pour edit_document_voice ? => et non pas 23 !

290 # A trouver dans saxia.config ? anon_gpt, extract, consolidate, anon_bert, enchaine_anon_extract

291 # J'ai oublié deux datous qui sont dans les map_reduce des premiers

292 list_intricate_datou = [39, 23]

293 from server.safia import lpgss_singleton

294 for did in list_datou_useful + list_intricate_datou:

295 from lib.lib_safia_system import LibSafiaSystem

296 lss_hack = LibSafiaSystem(lpgss_singleton)

297 conf_datou = lss_hack.get_datou(did, dont_instantiate_config_prepare = True, list_datou_ids = [])

298# if (len(conf_datou) > 0 and "steps" in conf_datou[0] and len(conf_datou[0]["steps"]) > 0 and "param_json" in conf_datou[0]["steps"][0] and "version" in conf_datou[0]["steps"][0]["param_json"]) :

299 if (len(conf_datou) > 0 and "steps" in conf_datou[0] and len(conf_datou[0]["steps"]) > 0 and "param_json" in conf_datou[0]["steps"][0]) :

300 conf_datou[0]["steps"][0]["param_json"]["version"] = id_version

301 # TODO VR change history datou and add year plop plop plop in history

302 datou_file_name = "datou_" + str(did) + "_" + dir_version + ".json"

303 print("datou_file_name : " + str(datou_file_name))

304 all_to_save[datou_file_name] = conf_datou

305 datou_file_name_wo_year = "datou_" + str(did) + "_" + date_str[4:] + ".json"

306 print("scp " + datou_file_name + " safia@marlene.fotonower.com:/home/safia/workarea/git/Safia/prompt/python/server/static/onedrive/workarea_anon/list_datou_backup/" + datou_file_name_wo_year)

307

308 config_app = self.get_config_app()

309 all_to_save["config_app"] = config_app

310 # Project ID

311 list_useful_project_id = [70, 91, 121, 94, 96] # et puis toutes les configurations users, mais c'est deux là sont sans doute suffisante !

312

313 # 91

314 list_useful_project_id.append(114)

315 list_useful_project_id.append(118)

316 list_useful_project_id.append(122)

317 list_useful_project_id.append(123)

318

319 conf_proj_default_project = {}

320 for pid in list_useful_project_id:

321 conf_proj = lpgss_singleton.load_conf_project(pid)

322 conf_proj["version"] = id_version

323 conf_proj["project_id"] = pid

324 import copy

325 del_tav_version_in_sub_config(conf_proj, copy.deepcopy(conf_proj))

326 all_to_save["proj_" + str(pid)] = conf_proj

327 # reminder HC_DEFAULT_CONF_PROJECT_ID = 70

328 if pid != HC_DEFAULT_CONF_PROJECT_ID:

329 from jsondiff import diff

330 diff_is = diff(conf_proj_default_project, conf_proj)

331 print("pid : " + str(pid))

332 if "saxia" in diff_is and "format" in diff_is["saxia"]:

333 print("diff_is : " + str(diff_is["saxia"]["format"]))

334

335 else :

336 conf_proj_default_project = conf_proj

337

338 all_to_save["safia_system"] = {"version": __version__}

339 output = os.path.join(root_output, dir_version)

340

341 if not os.path.exists(output):

342 os.makedirs(output)

343

344 for k in all_to_save:

345 with open(os.path.join(output, k + ".json"), "w") as f:

346 # TypeError: Object of type datetime is not JSON serializable

347 json.dump(all_to_save[k], f, default=str)

348

349 print(" ALL SAVED IN THIS FOLDER : " + str(output))

350

351 # Lié mais snas doute dupliqué et ne va pas du tout ici :

352 # VR TODO 31-7-24 :

353 # - chargement de la configuration d'un projet en indiquant si on veut garder la configuration provenant d'une reference ou si on veut la copier

354

355# --job=saxia.load_config --in_file="../devops/histo_config/tav_test/tav_alpha_Air-victor.home_config_version_20241106_00/datou_40_tav_alpha_Air-victor.home_config_version_20241106_00.json" --root_output="../devops/histo_config/tav_test/tav_alpha_Air-victor.home_config_version_20241106_00" --id_version="tav"

356

357 def load_config(self,

358 root_output = "temp", # "../devops/histo_config/",

359 id_version = "tav",

360 verbose = False,

361 admin_user_id = -1):

362 print("First we save the current configuration as tav (developement mode)")

363 self.save_all_useful_config(root_output, "tav")

364

365 from lib.manaudit.lib_datou_audit import get_list_backup

366 try:

367 histo_folder = os.path.join(root_output, "")

368 list_backup, map_proj_id_date_backup = get_list_backup(histo_folder)

369 except Exception as e:

370 print(str(e))

371

372 print("Just go on http://localhost:4999/datou?id=40&histo=True&histo_folder=../" + root_output)

373

374 for p in map_proj_id_date_backup:

375 print("Project " + str(p) + " : " + str(map_proj_id_date_backup[p]))

376 if "project_id" not in map_proj_id_date_backup[p]:

377 print("Missing project id")

378 continue

379

380 project_id = map_proj_id_date_backup[p]["project_id"]

381

382 from server.safia import lpgss_singleton

383 from lib.lib_safia_system import LibSafiaSystem

384 lss_hack = LibSafiaSystem(lpgss_singleton)

385 lss_hack.update_project_info(project_id, {"configuration" : map_proj_id_date_backup[p]})

386

387 print("Ca ca ne marche pas à cause des user => ca pourrait en fait remplacer le user quoi !, donc on va utiliser l'interface pour l'instant")

388 print("TODO build the link !")

389 for d in list_backup:

390 expected_one_datou_in_map = list_backup[d]

391 if len(expected_one_datou_in_map) != 1:

392 print(" Uncoherence")

393 continue

394 one_datou = expected_one_datou_in_map[list(expected_one_datou_in_map.keys())[0]]

395

396 from server.safia import lpgss_singleton

397 from lib.lib_safia_system import LibSafiaSystem

398 lss_hack = LibSafiaSystem(lpgss_singleton)

399 lss_hack.user_id = admin_user_id

400 if admin_user_id == -1:

401 print("Warning : admin_user_id is -1, you may have issues if you are not admin")

402 continue

403 lss_hack.update_datou(one_datou)

404

405 print(" Very dirty help : ")

406 print("You need to copy the file per datou in the correct folder and load them from the interface per datou with histo=True")

407 print("""

408 It should not be in $GITSAFIA/prompt/python/server/static/onedrive/workarea_anon/list_datou_backup/

409 It should be in $GITSAFIA/prompt/devops/histo_config/ and aleph / bet and others version

410 """)

411

412 cmd_list = ["ls", "-l", "$GITSAFIA/prompt/devops/histo_config/"]

413 print(" ".join(cmd_list))

414 try :

415 os.system("ls $GITSAFIA/prompt/devops/histo_config/")

416

417 import subprocess

418

419 subprocess.run(cmd_list)

420 except Exception as e:

421 print(str(e))

422

423 if "GITSAFIA" in os.environ:

424 print("GITSAFIA is defined to " + str(os.environ["GITSAFIA"]))

425 print("Just go on http://localhost:4999/datou?id=40&histo=True&histo_folder=../../devops/histo_config/")

426

427 print("http://localhost:4999/datou?id=40&histo=True&histo_folder=/Users/moilerat/Documents/Fotonower/Safia/prompt/devops/histo_config/bet_marlene_config_version_20240802_08")

428

429 print(" Maybe the best : http://localhost:4999/datou?id=40&histo=True&histo_folder=../../devops/histo_config/tav_test/tav_alpha_Air-victor.home_config_version_20241106_00 ")

430

431 print(" Very dirty help because we need to parse the list of stuff as well as the server used ! ")

432

433 print(" Maybe the best : http://localhost:4999/datou?histo=True&histo_folder=../../devops/histo_config/tav_test/tav_alpha_Air-victor.home_config_version_20241106_00 ")

434

435def del_tav_version_in_sub_config(config, ref_config):

436 if type(config) == dict:

437 for k in ref_config:

438 if k == "version" and ref_config[k] == "tav":

439 if k in config:

440 del config[k]

441 else:

442 print("Warning : trying to delete version tav but not found in config")

443 else:

444 del_tav_version_in_sub_config(config[k], ref_config[k])

445 elif type(config) == list:

446 for item in config:

447 import copy

448 del_tav_version_in_sub_config(item, copy.deepcopy(item))

449 return False

450

451

452def find_tav_version_in_sub_config(config):

453 if type(config) == dict:

454 for k in config:

455 if k == "version" and config[k] == "tav":

456 return True

457 else:

458 found_in_sub = find_tav_version_in_sub_config(config[k])

459 if found_in_sub:

460 return True

461 elif type(config) == list:

462 for item in config:

463 found_in_sub = find_tav_version_in_sub_config(item)

464 if found_in_sub:

465 return True

466 return False

467

468def collect_version_from_datou_and_proj_and_app_recursively(list_datous = [], list_proj_config = [],

469 host_url = None):

470 from lib.lib_version import version_name, version_date

471

472 map_entity_version = {}

473 for datou in list_datous:

474 datou_id = None

475 if "id" in datou:

476 datou_id = datou["id"]

477 else:

478 continue

479 if "steps" in datou and len(datou["steps"]) > 0 and "param_json" in datou["steps"][0] and "version" in datou["steps"][0]["param_json"]:

480 version_id = datou["steps"][0]["param_json"]["version"]

481 if version_id not in map_entity_version:

482 map_entity_version[version_id] = []

483 map_entity_version[version_id].append("D" + str(datou_id))

484

485 for proj in list_proj_config:

486 project_id = None

487 if "project_id" in proj:

488 project_id = proj["project_id"]

489 else :

490 continue

491 if "version" in proj:

492 version_id = proj["version"]

493 found_tav = find_tav_version_in_sub_config(proj)

494 if found_tav:

495 version_id = "tav"

496 else:

497 continue

498 if version_id not in map_entity_version:

499 map_entity_version[version_id] = []

500 map_entity_version[version_id].append("P" + str(project_id))

501

502 all_version_obj = "-".join(list(map(lambda x: x + ":" + str("&".join(map_entity_version[x])), map_entity_version.keys())))

503

504 version_string = version_name + " from " + version_date + " conf " + all_version_obj

505 if host_url != None:

506 version_string += " from " + host_url

507 hostname = os.environ["HOSTNAME"] if "HOSTNAME" in os.environ else os.uname().nodename

508 if hostname != None:

509 version_string += " on " + hostname

510

511 import datetime

512 now = datetime.datetime.now()

513 date_str = now.strftime("%Y-%m-%d %H:%M:%S")

514 version_string += " at " + date_str

515

516 return version_string

517

518lcs_global_singleton = LibConfSystem()

519

520

Coverage for auth/lib_conf_system.py: 42%

370 statements