python /home/admin/mtr/script_for_cron.py -j default -m 20 -a 'python3 /home/admin/workarea/git/Velours/python/prod/memo/memo.py --dashboard_places=SyctomXV_Mask_23Q4 -j sla_mensuel -v --date=2025-03 --rootfolder_sla=/data_2/data_log/job/memo/SyctomXV ' -s sla_mensuel -M 0 -S 0 -U 90,80,95 import MySQLdb succeeded Import error (python version) Need env var : LOGRASPI Raspi : , Date : 2025-03 _cr_cnf_ : /home/admin/mtr/.credentials/credentials.py _cr_cnf_ : /home/admin/mtr/.credentials/credentials.py warning , we can't find thcl infos in json_data warning , we can't find pdt infos in json_data nb_day : (5, 31) TODO : Insert select and so on SELECT id, name, misc_info, datou_id_for_right FROM MTRPhoto.dashboard_places WHERE name="SyctomXV_Mask_23Q4"; SELECT name, created_at,limit_max FROM MTRDatou.mtr_datou WHERE id=4932 SELECT mtd.id, mtdt.`type`, mtd.`param`, mtd.param_json, mtdt.nb_input, mtdt.nb_output, mtdt.prod, mtdt.is_local, mtdt.is_datou_depend, mtdt.is_photo_id_local FROM MTRDatou.mtr_datou_step mtd, MTRDatou.mtr_datou_step_types mtdt WHERE mtdt.`id`=mtd.`type` AND mtd.mtd_id=4932 SELECT mtd.id, mtd.mtd_id, mdsdt.id, mdsdt.name, mdsdt.description, msid.output_or_input, msid.data_order_id, mdsdt.type FROM MTRDatou.mtr_datou_step mtd, MTRDatou.mtr_datou_steptype_io_datatypes msid, MTRDatou.mtr_datou_step_data_types mdsdt WHERE mtd.`type`=msid.`mtr_datou_step_type` AND mtd.mtd_id= 4932 AND msid.data_type=mdsdt.id SELECT mts_id_output, id_output, mts_id_input, id_input FROM MTRDatou.mtr_datou_step_by_step WHERE mtd_id=4932 # VR 17-11-17 : to create in DB ! Here we check the datou graph and we reorder steps ! Tree builded and cycle checked, now we need to re-order the steps ! We have currenlty an error because there is no dependence between the last step for the case tile - detect - glue We can either keep the depence of, it is better to keep an order compatible with the id of steps if we do not have sons, so a lexical order : (number_son, step_id) DONE and to test : checkNoCycle ! We are managing only one step so we do not consider checkConsistencyNbInputNbOutput ! We are managing only one step so we do not consider checkConsistencyTypeOutputInput ! {"input_values":[]} We expected an STS datou with a camera_place_name, so there is a PB, datou_id: 4932 ! {'id': 4932, 'name': 'split_time_score_cod_upm_SyctomXV_Mask_23Q1', 'limit_max': 150, 'nb_steps': 1, 'estimated_time': 8640, 'list_steps': [{'id': 14066, 'type': 'split_time_score_with_photo', 'param': '0', 'json_param_as_string': "{'cod': 1, 'camera_place_name': 'SyctomXV_Mask_23Q4', 'ETA': 8640, 'param_moyenne_mobile': 5, 'thcl': 3379, 'list_task_complete': [''], 'list_tasks': 'LOAD,ORDER,CONVENTION,SPLIT_BY_DARK,SPLIT_TIME, CREATE_PORT_LAUNCH_CARAC_FILL_DASHBOARD,READ_PLATE', 'forced_hashtag': 'entrant', 'list_hashtag_class_create': 'entrant', 'ponderation': '0.001,0.001,0.001,0.001,0.001', 'host': 'www.fotonower.com', 'interval': 10, 'datou_proportion_papier': {'test': 370, 'entrant': 4742}, 'check_plate_format': 1, 'input_default_values': []}", 'json_param': {'cod': 1, 'camera_place_name': 'SyctomXV_Mask_23Q4', 'ETA': 8640, 'param_moyenne_mobile': 5, 'thcl': 3379, 'list_task_complete': [''], 'list_tasks': 'LOAD,ORDER,CONVENTION,SPLIT_BY_DARK,SPLIT_TIME, CREATE_PORT_LAUNCH_CARAC_FILL_DASHBOARD,READ_PLATE', 'forced_hashtag': 'entrant', 'list_hashtag_class_create': 'entrant', 'ponderation': '0.001,0.001,0.001,0.001,0.001', 'host': 'www.fotonower.com', 'interval': 10, 'datou_proportion_papier': {'test': 370, 'entrant': 4742}, 'check_plate_format': 1, 'input_default_values': []}, 'nb_input': 2, 'nb_output': 1, 'prod': False, 'is_local': False, 'is_datou_depend': False, 'is_photo_id_local': False, 'list_data': []}], 'depends_step': []} check if it is the case where we have multiple sts in one datou yes, we found a datou with multiple sts, we will choose the last one by default SELECT name, created_at,limit_max FROM MTRDatou.mtr_datou WHERE id=370 SELECT mtd.id, mtdt.`type`, mtd.`param`, mtd.param_json, mtdt.nb_input, mtdt.nb_output, mtdt.prod, mtdt.is_local, mtdt.is_datou_depend, mtdt.is_photo_id_local FROM MTRDatou.mtr_datou_step mtd, MTRDatou.mtr_datou_step_types mtdt WHERE mtdt.`id`=mtd.`type` AND mtd.mtd_id=370 SELECT mtd.id, mtd.mtd_id, mdsdt.id, mdsdt.name, mdsdt.description, msid.output_or_input, msid.data_order_id, mdsdt.type FROM MTRDatou.mtr_datou_step mtd, MTRDatou.mtr_datou_steptype_io_datatypes msid, MTRDatou.mtr_datou_step_data_types mdsdt WHERE mtd.`type`=msid.`mtr_datou_step_type` AND mtd.mtd_id= 370 AND msid.data_type=mdsdt.id SELECT mts_id_output, id_output, mts_id_input, id_input FROM MTRDatou.mtr_datou_step_by_step WHERE mtd_id=370 # VR 17-11-17 : to create in DB ! Here we check the datou graph and we reorder steps ! Tree builded and cycle checked, now we need to re-order the steps ! We have currenlty an error because there is no dependence between the last step for the case tile - detect - glue We can either keep the depence of, it is better to keep an order compatible with the id of steps if we do not have sons, so a lexical order : (number_son, step_id) All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! DONE and to test : checkNoCycle ! Here we check the consistency of inputs/outputs number between the given ones and the db ! eke 1-6-18 : checkConsistencyNbInputNbOutput should be processed after step reordering ! Number of inputs / outputs for each step checked ! Here we check the consistency of outputs/inputs types during steps connections eke 1-6-18 : checkConsistencyTypeOutputInput should be processed after checkConsistencyNbInputNbOutput ! We ignore checkConsistencyTypeOutputInput for datou_step final ! We ignore checkConsistencyTypeOutputInput for datou_step final ! We ignore checkConsistencyTypeOutputInput for datou_step final ! DataTypes for each output/input checked ! {"input_values":[]} Unexpected error treated as WARNING for now expected given param in final step {'min_score': 0.7, 'name_pipeline': 'proportion_papier_tapis', 'hashtag_type': 705, 'hashtag_proportion': 492668766, 'hashtag_parmi': 538914404, 'hashtag_to_consider': 'papier_only', 'hashtag_background': 493012381} Found hashtag_parmi in final step, will be used and useful if different from matiere_majoritaire in classifier ventilate_hashtags_in_portfolio step not found ! SELECT name, created_at,limit_max FROM MTRDatou.mtr_datou WHERE id=4742 SELECT mtd.id, mtdt.`type`, mtd.`param`, mtd.param_json, mtdt.nb_input, mtdt.nb_output, mtdt.prod, mtdt.is_local, mtdt.is_datou_depend, mtdt.is_photo_id_local FROM MTRDatou.mtr_datou_step mtd, MTRDatou.mtr_datou_step_types mtdt WHERE mtdt.`id`=mtd.`type` AND mtd.mtd_id=4742 SELECT mtd.id, mtd.mtd_id, mdsdt.id, mdsdt.name, mdsdt.description, msid.output_or_input, msid.data_order_id, mdsdt.type FROM MTRDatou.mtr_datou_step mtd, MTRDatou.mtr_datou_steptype_io_datatypes msid, MTRDatou.mtr_datou_step_data_types mdsdt WHERE mtd.`type`=msid.`mtr_datou_step_type` AND mtd.mtd_id= 4742 AND msid.data_type=mdsdt.id SELECT mts_id_output, id_output, mts_id_input, id_input FROM MTRDatou.mtr_datou_step_by_step WHERE mtd_id=4742 # VR 17-11-17 : to create in DB ! Here we check the datou graph and we reorder steps ! Tree builded and cycle checked, now we need to re-order the steps ! We have currenlty an error because there is no dependence between the last step for the case tile - detect - glue We can either keep the depence of, it is better to keep an order compatible with the id of steps if we do not have sons, so a lexical order : (number_son, step_id) All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! DONE and to test : checkNoCycle ! Here we check the consistency of inputs/outputs number between the given ones and the db ! eke 1-6-18 : checkConsistencyNbInputNbOutput should be processed after step reordering ! WARNING : number of outputs for step 13169 copy_chis is not consistent : 2 used against 1 in the step definition ! WARNING : number of outputs for step 13170 consolidate_hashtags_from_manual_portfolio is not consistent : 2 used against 1 in the step definition ! WARNING : number of inputs for step 13167 rle_unique_nms_with_priority is not consistent : 3 used against 1 in the step definition ! WARNING : number of outputs for step 13167 rle_unique_nms_with_priority is not consistent : 2 used against 1 in the step definition ! WARNING : number of outputs for step 13174 ventilate_hashtags_in_portfolio is not consistent : 2 used against 1 in the step definition ! Step 13168 final have less outputs used (1) than in the step definition (2) : some outputs may be not used ! Step 13171 blur_detection have less inputs used (0) than in the step definition (1) : maybe we manage optionnal inputs ! Step 13172 brightness have less inputs used (0) than in the step definition (1) : maybe we manage optionnal inputs ! Step 13173 send_mail_cod have less inputs used (4) than in the step definition (5) : maybe we manage optionnal inputs ! Step 13175 split_time_score have less inputs used (1) than in the step definition (2) : maybe we manage optionnal inputs ! Number of inputs / outputs for each step checked ! Here we check the consistency of outputs/inputs types during steps connections eke 1-6-18 : checkConsistencyTypeOutputInput should be processed after checkConsistencyNbInputNbOutput ! We ignore checkConsistencyTypeOutputInput for datou_step final ! WARNING : output 0 of step 13169 have datatype=11 whereas input 0 of step 13167 have datatype=2 WARNING : type of output 1 of step 13170 doesn't seem to be define in the database( WARNING : type of input 3 of step 13168 doesn't seem to be define in the database( We ignore checkConsistencyTypeOutputInput for datou_step final ! We ignore checkConsistencyTypeOutputInput for datou_step final ! WARNING : type of output 1 of step 13170 doesn't seem to be define in the database( WARNING : type of input 1 of step 13167 doesn't seem to be define in the database( WARNING : output 0 of step 13174 have datatype=10 whereas input 3 of step 13173 have datatype=6 WARNING : type of input 1 of step 13174 doesn't seem to be define in the database( WARNING : output 1 of step 13167 have datatype=7 whereas input 1 of step 13174 have datatype=None WARNING : type of output 1 of step 13174 doesn't seem to be define in the database( WARNING : type of input 4 of step 13168 doesn't seem to be define in the database( We ignore checkConsistencyTypeOutputInput for datou_step final ! WARNING : type of output 1 of step 13169 doesn't seem to be define in the database( WARNING : type of input 1 of step 13170 doesn't seem to be define in the database( DataTypes for each output/input checked ! {"input_values":[]} Unexpected error treated as WARNING for now expected given param in final step {'name_pipeline': 'aggregate_carac_ratio', 'hashtag_type': 4857, 'hashtag_proportion': 'sac_om_plein,textiles,verre,organique,dasri,masque,encombrant,autre_non_emballage', 'hashtag_parmi': 'papier,carton,metal,pet_clair,pehd,flux_dev,film_dev_souple,ela', 'hashtag_weights': {'papier': 3, 'carton': 2.8, 'metal': 6, 'pet_clair': 1.4, 'pehd': 2.8, 'flux_dev': 2.8, 'pet_opaque': 3, 'barquette_opaque': 2.8, 'film_dev_souple': 0.8, 'ela': 3.0, 'sac_om_plein': 6.5, 'textiles': 3, 'verre': 10.0, 'organique': 4.0, 'dasri': 2.0, 'masque': 1, 'encombrant': 7, 'autre_emballage': 6.0, 'autre_non_emballage': 6.0}, 'check_plaque': 1} Found hashtag_parmi in final step, will be used and useful if different from matiere_majoritaire in classifier Fonction SLA_MENSUEL nb_day : (5, 31) TODO : Insert select and so on SELECT id, name, misc_info, datou_id_for_right FROM MTRPhoto.dashboard_places WHERE name="SyctomXV_Mask_23Q4"; SELECT name, created_at,limit_max FROM MTRDatou.mtr_datou WHERE id=4932 SELECT mtd.id, mtdt.`type`, mtd.`param`, mtd.param_json, mtdt.nb_input, mtdt.nb_output, mtdt.prod, mtdt.is_local, mtdt.is_datou_depend, mtdt.is_photo_id_local FROM MTRDatou.mtr_datou_step mtd, MTRDatou.mtr_datou_step_types mtdt WHERE mtdt.`id`=mtd.`type` AND mtd.mtd_id=4932 SELECT mtd.id, mtd.mtd_id, mdsdt.id, mdsdt.name, mdsdt.description, msid.output_or_input, msid.data_order_id, mdsdt.type FROM MTRDatou.mtr_datou_step mtd, MTRDatou.mtr_datou_steptype_io_datatypes msid, MTRDatou.mtr_datou_step_data_types mdsdt WHERE mtd.`type`=msid.`mtr_datou_step_type` AND mtd.mtd_id= 4932 AND msid.data_type=mdsdt.id SELECT mts_id_output, id_output, mts_id_input, id_input FROM MTRDatou.mtr_datou_step_by_step WHERE mtd_id=4932 # VR 17-11-17 : to create in DB ! Here we check the datou graph and we reorder steps ! Tree builded and cycle checked, now we need to re-order the steps ! We have currenlty an error because there is no dependence between the last step for the case tile - detect - glue We can either keep the depence of, it is better to keep an order compatible with the id of steps if we do not have sons, so a lexical order : (number_son, step_id) DONE and to test : checkNoCycle ! We are managing only one step so we do not consider checkConsistencyNbInputNbOutput ! We are managing only one step so we do not consider checkConsistencyTypeOutputInput ! {"input_values":[]} We expected an STS datou with a camera_place_name, so there is a PB, datou_id: 4932 ! {'id': 4932, 'name': 'split_time_score_cod_upm_SyctomXV_Mask_23Q1', 'limit_max': 150, 'nb_steps': 1, 'estimated_time': 8640, 'list_steps': [{'id': 14066, 'type': 'split_time_score_with_photo', 'param': '0', 'json_param_as_string': "{'cod': 1, 'camera_place_name': 'SyctomXV_Mask_23Q4', 'ETA': 8640, 'param_moyenne_mobile': 5, 'thcl': 3379, 'list_task_complete': [''], 'list_tasks': 'LOAD,ORDER,CONVENTION,SPLIT_BY_DARK,SPLIT_TIME, CREATE_PORT_LAUNCH_CARAC_FILL_DASHBOARD,READ_PLATE', 'forced_hashtag': 'entrant', 'list_hashtag_class_create': 'entrant', 'ponderation': '0.001,0.001,0.001,0.001,0.001', 'host': 'www.fotonower.com', 'interval': 10, 'datou_proportion_papier': {'test': 370, 'entrant': 4742}, 'check_plate_format': 1, 'input_default_values': []}", 'json_param': {'cod': 1, 'camera_place_name': 'SyctomXV_Mask_23Q4', 'ETA': 8640, 'param_moyenne_mobile': 5, 'thcl': 3379, 'list_task_complete': [''], 'list_tasks': 'LOAD,ORDER,CONVENTION,SPLIT_BY_DARK,SPLIT_TIME, CREATE_PORT_LAUNCH_CARAC_FILL_DASHBOARD,READ_PLATE', 'forced_hashtag': 'entrant', 'list_hashtag_class_create': 'entrant', 'ponderation': '0.001,0.001,0.001,0.001,0.001', 'host': 'www.fotonower.com', 'interval': 10, 'datou_proportion_papier': {'test': 370, 'entrant': 4742}, 'check_plate_format': 1, 'input_default_values': []}, 'nb_input': 2, 'nb_output': 1, 'prod': False, 'is_local': False, 'is_datou_depend': False, 'is_photo_id_local': False, 'list_data': []}], 'depends_step': []} check if it is the case where we have multiple sts in one datou yes, we found a datou with multiple sts, we will choose the last one by default SELECT name, created_at,limit_max FROM MTRDatou.mtr_datou WHERE id=370 SELECT mtd.id, mtdt.`type`, mtd.`param`, mtd.param_json, mtdt.nb_input, mtdt.nb_output, mtdt.prod, mtdt.is_local, mtdt.is_datou_depend, mtdt.is_photo_id_local FROM MTRDatou.mtr_datou_step mtd, MTRDatou.mtr_datou_step_types mtdt WHERE mtdt.`id`=mtd.`type` AND mtd.mtd_id=370 SELECT mtd.id, mtd.mtd_id, mdsdt.id, mdsdt.name, mdsdt.description, msid.output_or_input, msid.data_order_id, mdsdt.type FROM MTRDatou.mtr_datou_step mtd, MTRDatou.mtr_datou_steptype_io_datatypes msid, MTRDatou.mtr_datou_step_data_types mdsdt WHERE mtd.`type`=msid.`mtr_datou_step_type` AND mtd.mtd_id= 370 AND msid.data_type=mdsdt.id SELECT mts_id_output, id_output, mts_id_input, id_input FROM MTRDatou.mtr_datou_step_by_step WHERE mtd_id=370 # VR 17-11-17 : to create in DB ! Here we check the datou graph and we reorder steps ! Tree builded and cycle checked, now we need to re-order the steps ! We have currenlty an error because there is no dependence between the last step for the case tile - detect - glue We can either keep the depence of, it is better to keep an order compatible with the id of steps if we do not have sons, so a lexical order : (number_son, step_id) All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! DONE and to test : checkNoCycle ! Here we check the consistency of inputs/outputs number between the given ones and the db ! eke 1-6-18 : checkConsistencyNbInputNbOutput should be processed after step reordering ! Number of inputs / outputs for each step checked ! Here we check the consistency of outputs/inputs types during steps connections eke 1-6-18 : checkConsistencyTypeOutputInput should be processed after checkConsistencyNbInputNbOutput ! We ignore checkConsistencyTypeOutputInput for datou_step final ! We ignore checkConsistencyTypeOutputInput for datou_step final ! We ignore checkConsistencyTypeOutputInput for datou_step final ! DataTypes for each output/input checked ! {"input_values":[]} Unexpected error treated as WARNING for now expected given param in final step {'min_score': 0.7, 'name_pipeline': 'proportion_papier_tapis', 'hashtag_type': 705, 'hashtag_proportion': 492668766, 'hashtag_parmi': 538914404, 'hashtag_to_consider': 'papier_only', 'hashtag_background': 493012381} Found hashtag_parmi in final step, will be used and useful if different from matiere_majoritaire in classifier ventilate_hashtags_in_portfolio step not found ! SELECT name, created_at,limit_max FROM MTRDatou.mtr_datou WHERE id=4742 SELECT mtd.id, mtdt.`type`, mtd.`param`, mtd.param_json, mtdt.nb_input, mtdt.nb_output, mtdt.prod, mtdt.is_local, mtdt.is_datou_depend, mtdt.is_photo_id_local FROM MTRDatou.mtr_datou_step mtd, MTRDatou.mtr_datou_step_types mtdt WHERE mtdt.`id`=mtd.`type` AND mtd.mtd_id=4742 SELECT mtd.id, mtd.mtd_id, mdsdt.id, mdsdt.name, mdsdt.description, msid.output_or_input, msid.data_order_id, mdsdt.type FROM MTRDatou.mtr_datou_step mtd, MTRDatou.mtr_datou_steptype_io_datatypes msid, MTRDatou.mtr_datou_step_data_types mdsdt WHERE mtd.`type`=msid.`mtr_datou_step_type` AND mtd.mtd_id= 4742 AND msid.data_type=mdsdt.id SELECT mts_id_output, id_output, mts_id_input, id_input FROM MTRDatou.mtr_datou_step_by_step WHERE mtd_id=4742 # VR 17-11-17 : to create in DB ! Here we check the datou graph and we reorder steps ! Tree builded and cycle checked, now we need to re-order the steps ! We have currenlty an error because there is no dependence between the last step for the case tile - detect - glue We can either keep the depence of, it is better to keep an order compatible with the id of steps if we do not have sons, so a lexical order : (number_son, step_id) All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! DONE and to test : checkNoCycle ! Here we check the consistency of inputs/outputs number between the given ones and the db ! eke 1-6-18 : checkConsistencyNbInputNbOutput should be processed after step reordering ! WARNING : number of outputs for step 13169 copy_chis is not consistent : 2 used against 1 in the step definition ! WARNING : number of outputs for step 13170 consolidate_hashtags_from_manual_portfolio is not consistent : 2 used against 1 in the step definition ! WARNING : number of inputs for step 13167 rle_unique_nms_with_priority is not consistent : 3 used against 1 in the step definition ! WARNING : number of outputs for step 13167 rle_unique_nms_with_priority is not consistent : 2 used against 1 in the step definition ! WARNING : number of outputs for step 13174 ventilate_hashtags_in_portfolio is not consistent : 2 used against 1 in the step definition ! Step 13168 final have less outputs used (1) than in the step definition (2) : some outputs may be not used ! Step 13171 blur_detection have less inputs used (0) than in the step definition (1) : maybe we manage optionnal inputs ! Step 13172 brightness have less inputs used (0) than in the step definition (1) : maybe we manage optionnal inputs ! Step 13173 send_mail_cod have less inputs used (4) than in the step definition (5) : maybe we manage optionnal inputs ! Step 13175 split_time_score have less inputs used (1) than in the step definition (2) : maybe we manage optionnal inputs ! Number of inputs / outputs for each step checked ! Here we check the consistency of outputs/inputs types during steps connections eke 1-6-18 : checkConsistencyTypeOutputInput should be processed after checkConsistencyNbInputNbOutput ! We ignore checkConsistencyTypeOutputInput for datou_step final ! WARNING : output 0 of step 13169 have datatype=11 whereas input 0 of step 13167 have datatype=2 WARNING : type of output 1 of step 13170 doesn't seem to be define in the database( WARNING : type of input 3 of step 13168 doesn't seem to be define in the database( We ignore checkConsistencyTypeOutputInput for datou_step final ! We ignore checkConsistencyTypeOutputInput for datou_step final ! WARNING : type of output 1 of step 13170 doesn't seem to be define in the database( WARNING : type of input 1 of step 13167 doesn't seem to be define in the database( WARNING : output 0 of step 13174 have datatype=10 whereas input 3 of step 13173 have datatype=6 WARNING : type of input 1 of step 13174 doesn't seem to be define in the database( WARNING : output 1 of step 13167 have datatype=7 whereas input 1 of step 13174 have datatype=None WARNING : type of output 1 of step 13174 doesn't seem to be define in the database( WARNING : type of input 4 of step 13168 doesn't seem to be define in the database( We ignore checkConsistencyTypeOutputInput for datou_step final ! WARNING : type of output 1 of step 13169 doesn't seem to be define in the database( WARNING : type of input 1 of step 13170 doesn't seem to be define in the database( DataTypes for each output/input checked ! {"input_values":[]} Unexpected error treated as WARNING for now expected given param in final step {'name_pipeline': 'aggregate_carac_ratio', 'hashtag_type': 4857, 'hashtag_proportion': 'sac_om_plein,textiles,verre,organique,dasri,masque,encombrant,autre_non_emballage', 'hashtag_parmi': 'papier,carton,metal,pet_clair,pehd,flux_dev,film_dev_souple,ela', 'hashtag_weights': {'papier': 3, 'carton': 2.8, 'metal': 6, 'pet_clair': 1.4, 'pehd': 2.8, 'flux_dev': 2.8, 'pet_opaque': 3, 'barquette_opaque': 2.8, 'film_dev_souple': 0.8, 'ela': 3.0, 'sac_om_plein': 6.5, 'textiles': 3, 'verre': 10.0, 'organique': 4.0, 'dasri': 2.0, 'masque': 1, 'encombrant': 7, 'autre_emballage': 6.0, 'autre_non_emballage': 6.0}, 'check_plaque': 1} Found hashtag_parmi in final step, will be used and useful if different from matiere_majoritaire in classifier place_id : 188 month : 03 mtr_id : dashboard_places : SyctomXV_Mask_23Q4 date : 2025-03 year : 2025 raspi : 0 select count(distinct mtr_photo_id) from MTRUser.mtr_portfolio_photos where mtr_portfolio_id in (select id from MTRUser.mtr_portfolios where id in (select mtr_portfolio_id from MTRPhoto.dashboard_results where dashboard_run_id in(select last_run_id from MTRPhoto.dashboard_entry_day where dashboard_place_id in (select id from MTRPhoto.dashboard_places where name = 'SyctomXV_Mask_23Q4' and date like '%2025-03%') and created_at like '%2025-03%'))); nb_day : (5, 31) after unwanted_material_data We return tuple() but get_coverage_data_from_list_of_date return unexpectedly data from stuff in error with raspi=0 so we force here ! after coverage_data SELECT DATE(date_debut) as date, sum(qualite*nombre_balle)/sum(nombre_balle), count(distinct mtr_photo_id) FROM MTRPhoto.dashboard_results dr, MTRUser.mtr_portfolio_photos mpp, MTRPhoto.dashboard_entry_day ded where mpp.mtr_portfolio_id=dr.mtr_portfolio_id and ded.dashboard_place_id = 188 and month(ded.date) = 03 and year(ded.date) = 2025 and dr.dashboard_run_id=ded.last_run_id and dr.qualite>=0 GROUP BY ded.date order by dr.date_debut; SELECT DATE(date_debut) as date, sum(qualite*nombre_balle)/sum(nombre_balle), count(distinct mtr_photo_id) FROM MTRPhoto.dashboard_results dr, MTRUser.mtr_portfolio_photos mpp, MTRPhoto.dashboard_entry_day ded where mpp.mtr_portfolio_id=dr.mtr_portfolio_id and dr.hashtag = '_______entrant' and ded.dashboard_place_id = 188 and month(ded.date) = 03 and year(ded.date) = 2025 and dr.dashboard_run_id=ded.last_run_id and dr.qualite>=0 GROUP BY ded.date order by dr.date_debut; select avg(cps.score) as score, SUBSTRING_INDEX(SUBSTRING_INDEX(mp.name,'__', -1), '_', 1) as sbstr, SUBSTRING_INDEX(SUBSTRING_INDEX(mp.name,'_', -2), '_', 1) as sbstr2, SUBSTRING_INDEX(mp.name,'_', -1) as sbstr3 FROM MTRUser.mtr_portfolio_photos mpp, MTRUser.mtr_portfolios mp,MTRPhoto.class_photo_score cps,MTRBack.hashtags h,MTRPhoto.dashboard_results dr,MTRPhoto.dashboard_entry_day ded, MTRPhoto.dashboard_places dp WHERE mp.id=mpp.mtr_portfolio_id and cps.photo_id=mpp.mtr_photo_id and h.hashtag_id=cps.hashtag_id and dr.mtr_portfolio_id = mpp.mtr_portfolio_id and dr.dashboard_run_id = ded.last_run_id and ded.dashboard_place_id = dp.id and h.hashtag in ('blur', 'mean_blur', 'clear') and dp.id = 188 and mp.name like '%032025%' and cps.thcl = 1533 GROUP BY sbstr ; after number_of_batch date_start : 2025-03-01 : dt_date_just_month_year : 2025-03-01 00:00:00 : VR TODO TOCHECK : due to this bug it shouldn't have being able to work, or maybe it was due to a change directory not done ! VR TODO TOCHECK : due to this bug it shouldn't have being able to work, or maybe it was due to a change directory not done ! VR TODO TOCHECK : due to this bug it shouldn't have being able to work, or maybe it was due to a change directory not done ! VR TODO TOCHECK : due to this bug it shouldn't have being able to work, or maybe it was due to a change directory not done ! VR TODO TOCHECK : due to this bug it shouldn't have being able to work, or maybe it was due to a change directory not done ! VR TODO TOCHECK : due to this bug it shouldn't have being able to work, or maybe it was due to a change directory not done ! VR TODO TOCHECK : due to this bug it shouldn't have being able to work, or maybe it was due to a change directory not done ! VR TODO TOCHECK : due to this bug it shouldn't have being able to work, or maybe it was due to a change directory not done ! VR TODO TOCHECK : due to this bug it shouldn't have being able to work, or maybe it was due to a change directory not done ! VR TODO TOCHECK : due to this bug it shouldn't have being able to work, or maybe it was due to a change directory not done ! VR TODO TOCHECK : due to this bug it shouldn't have being able to work, or maybe it was due to a change directory not done ! VR TODO TOCHECK : due to this bug it shouldn't have being able to work, or maybe it was due to a change directory not done ! VR TODO TOCHECK : due to this bug it shouldn't have being able to work, or maybe it was due to a change directory not done ! VR TODO TOCHECK : due to this bug it shouldn't have being able to work, or maybe it was due to a change directory not done ! VR TODO TOCHECK : due to this bug it shouldn't have being able to work, or maybe it was due to a change directory not done ! VR TODO TOCHECK : due to this bug it shouldn't have being able to work, or maybe it was due to a change directory not done ! VR TODO TOCHECK : due to this bug it shouldn't have being able to work, or maybe it was due to a change directory not done ! VR TODO TOCHECK : due to this bug it shouldn't have being able to work, or maybe it was due to a change directory not done ! VR TODO TOCHECK : due to this bug it shouldn't have being able to work, or maybe it was due to a change directory not done ! VR TODO TOCHECK : due to this bug it shouldn't have being able to work, or maybe it was due to a change directory not done ! VR TODO TOCHECK : due to this bug it shouldn't have being able to work, or maybe it was due to a change directory not done ! VR TODO TOCHECK : due to this bug it shouldn't have being able to work, or maybe it was due to a change directory not done ! VR TODO TOCHECK : due to this bug it shouldn't have being able to work, or maybe it was due to a change directory not done ! VR TODO TOCHECK : due to this bug it shouldn't have being able to work, or maybe it was due to a change directory not done ! VR TODO TOCHECK : due to this bug it shouldn't have being able to work, or maybe it was due to a change directory not done ! VR TODO TOCHECK : due to this bug it shouldn't have being able to work, or maybe it was due to a change directory not done ! VR TODO TOCHECK : due to this bug it shouldn't have being able to work, or maybe it was due to a change directory not done ! VR TODO TOCHECK : due to this bug it shouldn't have being able to work, or maybe it was due to a change directory not done ! VR TODO TOCHECK : due to this bug it shouldn't have being able to work, or maybe it was due to a change directory not done ! Error parsing crontab ! [Errno 2] No such file or directory: '' You better check your LOGRASPI env !!! after pl.get_datou_sts_from_crontab : verbose : True no sts found, try to find from database SELECT dri.id FROM MTRPhoto.dashboard_run_ids dri, MTRPhoto.dashboard_entry_day ded, MTRPhoto.dashboard_places dp WHERE dp.name= "SyctomXV_Mask_23Q4" AND ded.dashboard_place_id=dp.id AND dri.dashboard_entry_day=ded.id AND dri.id=ded.last_run_id AND ded.date >= "2025-03-01" AND ded.date <= "2025-03-31" apple3 {'test': {'mat': 'test', 'pht': 0, 'datou_carac_id': 370, 'unwanted_material': [], 'hashtag_majoritaire_from_carac': 538914404}, 'entrant': {'mat': 'entrant', 'pht': 4857, 'datou_carac_id': 4742, 'unwanted_material': [], 'hashtag_majoritaire_from_carac': 'papier,carton,metal,pet_clair,pehd,flux_dev,film_dev_souple,ela'}} SELECT h.hashtag as unwanted_material, substr(dr.hashtag,8) as main_material, ptp.type as pht_type, sum(pcr.value*dr.nombre_balle)/sum(dr.nombre_balle) as ratio, count(distinct mpp.mtr_photo_id) as nb_photo, group_concat(distinct ptp.mtr_portfolio_id_2) as list_port_cont, group_concat(distinct concat(cast(ptp.mtr_portfolio_id_1 as char), ":", cast(ptp.mtr_portfolio_id_2 as char))) as assoc_port, group_concat(distinct concat(cast(ptp.mtr_portfolio_id_1 as char), ":", h.hashtag, ":", cast(ptp.type as char), ":", cast(ptp.mtr_portfolio_id_2 as char))) as assoc_mat FROM MTRPhoto.dashboard_results dr, MTRPhoto.mtr_port_to_port_ids ptp, MTRUser.mtr_portfolio_photos mpp, MTRUser.portfolio_carac_ratio pcr, MTRBack.hashtags h WHERE dr.dashboard_run_id IN (1712112,1712045,1711948,1711912) AND dr.mtr_portfolio_id=ptp.mtr_portfolio_id_1 AND dr.qualite >= 0 AND mpp.mtr_portfolio_id=ptp.mtr_portfolio_id_2 AND pcr.portfolio_id=ptp.mtr_portfolio_id_1 AND h.hashtag_id = pcr.hashtag_id AND ptp.type = pcr.hashtag_type AND mpp.hide_status = 0 AND ptp.hashtag_id=h.hashtag_id AND ptp.type IN (0,4857) group by h.hashtag, dr.hashtag, ptp.type; VR TODO TO BETTER PARSE ! ({'unwanted_material': 'autre_non_emballage', 'main_material': 'entrant', 'pht_type': 4857, 'ratio': 0.01541413348688883, 'nb_photo': 11, 'list_port_cont': '21947850,21947869,21950398,21950821,21950841,21956772,21956792', 'assoc_port': '21917053:21950821,21917053:21950841,21917067:21956772,21917067:21956792,21917162:21947850,21917162:21947869,21917360:21950398', 'assoc_mat': '21917053:autre_non_emballage:4857:21950821,21917053:autre_non_emballage:4857:21950841,21917067:autre_non_emballage:4857:21956772,21917067:autre_non_emballage:4857:21956792,21917162:autre_non_emballage:4857:21947850,21917162:autre_non_emballage:4857:21947869,21917360:autre_non_emballage:4857:21950398'}, {'unwanted_material': 'carton', 'main_material': 'entrant', 'pht_type': 4857, 'ratio': 0.6498830700745826, 'nb_photo': 1039, 'list_port_cont': '21947836,21947855,21950384,21950430,21950807,21950827,21956758,21956778,21984388,21984407,22001990,22002009,22015820,22015840,22017290,22017326,22054963,22054988', 'assoc_port': '21917053:21950807,21917053:21950827,21917067:21956758,21917067:21956778,21917161:21984388,21917161:21984407,21917162:21947836,21917162:21947855,21917357:22015820,21917357:22015840,21917360:21950384,21917360:21950430,21917361:22001990,21917361:22002009,21917362:22017290,21917362:22017326,21918917:22054963,21918917:22054988', 'assoc_mat': '21917053:carton:4857:21950807,21917053:carton:4857:21950827,21917067:carton:4857:21956758,21917067:carton:4857:21956778,21917161:carton:4857:21984388,21917161:carton:4857:21984407,21917162:carton:4857:21947836,21917162:carton:4857:21947855,21917357:carton:4857:22015820,21917357:carton:4857:22015840,21917360:carton:4857:21950384,21917360:carton:4857:21950430,21917361:carton:4857:22001990,21917361:carton:4857:22002009,21917362:carton:4857:22017290,21917362:carton:4857:22017326,21918917:carton:4857:22054963,21918917:carton:4857:22054988'}, {'unwanted_material': 'dasri', 'main_material': 'entrant', 'pht_type': 4857, 'ratio': 0.0, 'nb_photo': 1, 'list_port_cont': '21950395', 'assoc_port': '21917360:21950395', 'assoc_mat': '21917360:dasri:4857:21950395'}, {'unwanted_material': 'ela', 'main_material': 'entrant', 'pht_type': 4857, 'ratio': 0.002121527027143947, 'nb_photo': 5, 'list_port_cont': '21950813,21950833,21956764,21956784,21984394,21984413,22015826,22015846,22017296,22017332', 'assoc_port': '21917053:21950813,21917053:21950833,21917067:21956764,21917067:21956784,21917161:21984394,21917161:21984413,21917357:22015826,21917357:22015846,21917362:22017296,21917362:22017332', 'assoc_mat': '21917053:ela:4857:21950813,21917053:ela:4857:21950833,21917067:ela:4857:21956764,21917067:ela:4857:21956784,21917161:ela:4857:21984394,21917161:ela:4857:21984413,21917357:ela:4857:22015826,21917357:ela:4857:22015846,21917362:ela:4857:22017296,21917362:ela:4857:22017332'}, {'unwanted_material': 'film_dev_souple', 'main_material': 'entrant', 'pht_type': 4857, 'ratio': 0.02799164863432263, 'nb_photo': 164, 'list_port_cont': '21947841,21947860,21950389,21950435,21950812,21950832,21956763,21956783,21984393,21984412,22001995,22002014,22015825,22015845,22017295,22017331,22054968,22054993', 'assoc_port': '21917053:21950812,21917053:21950832,21917067:21956763,21917067:21956783,21917161:21984393,21917161:21984412,21917162:21947841,21917162:21947860,21917357:22015825,21917357:22015845,21917360:21950389,21917360:21950435,21917361:22001995,21917361:22002014,21917362:22017295,21917362:22017331,21918917:22054968,21918917:22054993', 'assoc_mat': '21917053:film_dev_souple:4857:21950812,21917053:film_dev_souple:4857:21950832,21917067:film_dev_souple:4857:21956763,21917067:film_dev_souple:4857:21956783,21917161:film_dev_souple:4857:21984393,21917161:film_dev_souple:4857:21984412,21917162:film_dev_souple:4857:21947841,21917162:film_dev_souple:4857:21947860,21917357:film_dev_souple:4857:22015825,21917357:film_dev_souple:4857:22015845,21917360:film_dev_souple:4857:21950389,21917360:film_dev_souple:4857:21950435,21917361:film_dev_souple:4857:22001995,21917361:film_dev_souple:4857:22002014,21917362:film_dev_souple:4857:22017295,21917362:film_dev_souple:4857:22017331,21918917:film_dev_souple:4857:22054968,21918917:film_dev_souple:4857:22054993'}, {'unwanted_material': 'flux_dev', 'main_material': 'entrant', 'pht_type': 4857, 'ratio': 0.026120294248382987, 'nb_photo': 66, 'list_port_cont': '21947840,21947859,21950388,21950434,21950811,21950831,21956762,21956782,21984392,21984411,22001994,22002013,22015824,22015844,22017294,22017330,22054967,22054992', 'assoc_port': '21917053:21950811,21917053:21950831,21917067:21956762,21917067:21956782,21917161:21984392,21917161:21984411,21917162:21947840,21917162:21947859,21917357:22015824,21917357:22015844,21917360:21950388,21917360:21950434,21917361:22001994,21917361:22002013,21917362:22017294,21917362:22017330,21918917:22054967,21918917:22054992', 'assoc_mat': '21917053:flux_dev:4857:21950811,21917053:flux_dev:4857:21950831,21917067:flux_dev:4857:21956762,21917067:flux_dev:4857:21956782,21917161:flux_dev:4857:21984392,21917161:flux_dev:4857:21984411,21917162:flux_dev:4857:21947840,21917162:flux_dev:4857:21947859,21917357:flux_dev:4857:22015824,21917357:flux_dev:4857:22015844,21917360:flux_dev:4857:21950388,21917360:flux_dev:4857:21950434,21917361:flux_dev:4857:22001994,21917361:flux_dev:4857:22002013,21917362:flux_dev:4857:22017294,21917362:flux_dev:4857:22017330,21918917:flux_dev:4857:22054967,21918917:flux_dev:4857:22054992'}, {'unwanted_material': 'metal', 'main_material': 'entrant', 'pht_type': 4857, 'ratio': 0.024127419912394713, 'nb_photo': 44, 'list_port_cont': '21947837,21950385,21950431,21950808,21950828,21956759,21956779,22001991,22002010,22015821,22015841,22017291,22017327', 'assoc_port': '21917053:21950808,21917053:21950828,21917067:21956759,21917067:21956779,21917162:21947837,21917357:22015821,21917357:22015841,21917360:21950385,21917360:21950431,21917361:22001991,21917361:22002010,21917362:22017291,21917362:22017327', 'assoc_mat': '21917053:metal:4857:21950808,21917053:metal:4857:21950828,21917067:metal:4857:21956759,21917067:metal:4857:21956779,21917162:metal:4857:21947837,21917357:metal:4857:22015821,21917357:metal:4857:22015841,21917360:metal:4857:21950385,21917360:metal:4857:21950431,21917361:metal:4857:22001991,21917361:metal:4857:22002010,21917362:metal:4857:22017291,21917362:metal:4857:22017327'}, {'unwanted_material': 'organique', 'main_material': 'entrant', 'pht_type': 4857, 'ratio': 0.004472507743608766, 'nb_photo': 1, 'list_port_cont': '21950440', 'assoc_port': '21917360:21950440', 'assoc_mat': '21917360:organique:4857:21950440'}, {'unwanted_material': 'papier', 'main_material': 'entrant', 'pht_type': 4857, 'ratio': 0.1400314294980711, 'nb_photo': 235, 'list_port_cont': '21947835,21947854,21950383,21950429,21950806,21950826,21956757,21956777,21984387,21984406,22001989,22002008,22015819,22015839,22017289,22017325,22054962,22054987', 'assoc_port': '21917053:21950806,21917053:21950826,21917067:21956757,21917067:21956777,21917161:21984387,21917161:21984406,21917162:21947835,21917162:21947854,21917357:22015819,21917357:22015839,21917360:21950383,21917360:21950429,21917361:22001989,21917361:22002008,21917362:22017289,21917362:22017325,21918917:22054962,21918917:22054987', 'assoc_mat': '21917053:papier:4857:21950806,21917053:papier:4857:21950826,21917067:papier:4857:21956757,21917067:papier:4857:21956777,21917161:papier:4857:21984387,21917161:papier:4857:21984406,21917162:papier:4857:21947835,21917162:papier:4857:21947854,21917357:papier:4857:22015819,21917357:papier:4857:22015839,21917360:papier:4857:21950383,21917360:papier:4857:21950429,21917361:papier:4857:22001989,21917361:papier:4857:22002008,21917362:papier:4857:22017289,21917362:papier:4857:22017325,21918917:papier:4857:22054962,21918917:papier:4857:22054987'}, {'unwanted_material': 'pehd', 'main_material': 'entrant', 'pht_type': 4857, 'ratio': 0.008176693463765301, 'nb_photo': 12, 'list_port_cont': '21947839,21947858,21950387,21950433,21950810,21950830,21984391,21984410,22015823,22015843,22017293,22017329', 'assoc_port': '21917053:21950810,21917053:21950830,21917161:21984391,21917161:21984410,21917162:21947839,21917162:21947858,21917357:22015823,21917357:22015843,21917360:21950387,21917360:21950433,21917362:22017293,21917362:22017329', 'assoc_mat': '21917053:pehd:4857:21950810,21917053:pehd:4857:21950830,21917161:pehd:4857:21984391,21917161:pehd:4857:21984410,21917162:pehd:4857:21947839,21917162:pehd:4857:21947858,21917357:pehd:4857:22015823,21917357:pehd:4857:22015843,21917360:pehd:4857:21950387,21917360:pehd:4857:21950433,21917362:pehd:4857:22017293,21917362:pehd:4857:22017329'}, {'unwanted_material': 'pet_clair', 'main_material': 'entrant', 'pht_type': 4857, 'ratio': 0.029335934831367275, 'nb_photo': 163, 'list_port_cont': '21947838,21947857,21950386,21950432,21950809,21950829,21956760,21956780,21984390,21984409,22001992,22002011,22015822,22015842,22017292,22017328,22054965,22054990', 'assoc_port': '21917053:21950809,21917053:21950829,21917067:21956760,21917067:21956780,21917161:21984390,21917161:21984409,21917162:21947838,21917162:21947857,21917357:22015822,21917357:22015842,21917360:21950386,21917360:21950432,21917361:22001992,21917361:22002011,21917362:22017292,21917362:22017328,21918917:22054965,21918917:22054990', 'assoc_mat': '21917053:pet_clair:4857:21950809,21917053:pet_clair:4857:21950829,21917067:pet_clair:4857:21956760,21917067:pet_clair:4857:21956780,21917161:pet_clair:4857:21984390,21917161:pet_clair:4857:21984409,21917162:pet_clair:4857:21947838,21917162:pet_clair:4857:21947857,21917357:pet_clair:4857:22015822,21917357:pet_clair:4857:22015842,21917360:pet_clair:4857:21950386,21917360:pet_clair:4857:21950432,21917361:pet_clair:4857:22001992,21917361:pet_clair:4857:22002011,21917362:pet_clair:4857:22017292,21917362:pet_clair:4857:22017328,21918917:pet_clair:4857:22054965,21918917:pet_clair:4857:22054990'}, {'unwanted_material': 'sac_om_plein', 'main_material': 'entrant', 'pht_type': 4857, 'ratio': 0.16562486153652237, 'nb_photo': 65, 'list_port_cont': '21947843,21947862,21950391,21950437,21950814,21950834,21956765,21956785,21984395,21984414,22001997,22002016,22015827,22015847,22017297,22017333,22054971,22054995', 'assoc_port': '21917053:21950814,21917053:21950834,21917067:21956765,21917067:21956785,21917161:21984395,21917161:21984414,21917162:21947843,21917162:21947862,21917357:22015827,21917357:22015847,21917360:21950391,21917360:21950437,21917361:22001997,21917361:22002016,21917362:22017297,21917362:22017333,21918917:22054971,21918917:22054995', 'assoc_mat': '21917053:sac_om_plein:4857:21950814,21917053:sac_om_plein:4857:21950834,21917067:sac_om_plein:4857:21956765,21917067:sac_om_plein:4857:21956785,21917161:sac_om_plein:4857:21984395,21917161:sac_om_plein:4857:21984414,21917162:sac_om_plein:4857:21947843,21917162:sac_om_plein:4857:21947862,21917357:sac_om_plein:4857:22015827,21917357:sac_om_plein:4857:22015847,21917360:sac_om_plein:4857:21950391,21917360:sac_om_plein:4857:21950437,21917361:sac_om_plein:4857:22001997,21917361:sac_om_plein:4857:22002016,21917362:sac_om_plein:4857:22017297,21917362:sac_om_plein:4857:22017333,21918917:sac_om_plein:4857:22054971,21918917:sac_om_plein:4857:22054995'}, {'unwanted_material': 'textiles', 'main_material': 'entrant', 'pht_type': 4857, 'ratio': 0.007983810261584477, 'nb_photo': 7, 'list_port_cont': '21950438,21950815,21950835,21984396,21984415,22001998,22002017', 'assoc_port': '21917053:21950815,21917053:21950835,21917161:21984396,21917161:21984415,21917360:21950438,21917361:22001998,21917361:22002017', 'assoc_mat': '21917053:textiles:4857:21950815,21917053:textiles:4857:21950835,21917161:textiles:4857:21984396,21917161:textiles:4857:21984415,21917360:textiles:4857:21950438,21917361:textiles:4857:22001998,21917361:textiles:4857:22002017'}, {'unwanted_material': 'verre', 'main_material': 'entrant', 'pht_type': 4857, 'ratio': 0.002323954391622627, 'nb_photo': 1, 'list_port_cont': '21947864', 'assoc_port': '21917162:21947864', 'assoc_mat': '21917162:verre:4857:21947864'}) select count(distinct mtr_photo_id) from MTRUser.mtr_portfolio_photos where mtr_portfolio_id in (select mtr_portfolio_id from MTRPhoto.dashboard_results where dashboard_run_id in(1712112,1712045,1711948,1711912)); after get_hostname_from_raspi hasthag : entrant hasthag that could be used but not yet : _______papier,carton,metal,pet_clair,pehd,flux_dev,film_dev_souple,ela SELECT substr(hashtag,8), substr(date_debut,1,7) as month, (sum(qualite*nombre_balle)/sum(nombre_balle)) FROM MTRPhoto.dashboard_results WHERE hashtag = '_______entrant' and qualite >=0 and dashboard_run_id in (select last_run_id from MTRPhoto.dashboard_entry_day where dashboard_place_id=188 and date >='2025-03-01' and date <= '2025-03-31') GROUP BY month order by month; after impurety_average_per_hashtag {'number_of_analysed_photos': ((93,),), 'unwanted_material_monthly_percentage': ((0.22246814709383067,),), 'material_repartition': (('entrant', 93),), 'material_dict': {'entrant': 93}, 'coverage_data': (), 'impurety_average_data': ((datetime.date(2025, 3, 1), 0.30656812781950904, 21), (datetime.date(2025, 3, 4), 0.1853987599174162, 36), (datetime.date(2025, 3, 5), 0.12077183578984403, 10), (datetime.date(2025, 3, 25), 0.276169821316118, 26)), 'impurety_average_data_per_hashtag': {'entrant': ((datetime.date(2025, 3, 1), 0.30656812781950904, 21), (datetime.date(2025, 3, 4), 0.1853987599174162, 36), (datetime.date(2025, 3, 5), 0.12077183578984403, 10), (datetime.date(2025, 3, 25), 0.276169821316118, 26))}, 'quality_array': [], 'number_of_batch': ((12,),), 'datou_id_sts': 4932, 'map_hashtag_number_mat': {'hash_to_mat_nb': ({'unwanted_material': 'autre_non_emballage', 'main_material': 'entrant', 'pht_type': 4857, 'ratio': 0.01541413348688883, 'nb_photo': 11, 'list_port_cont': '21947850,21947869,21950398,21950821,21950841,21956772,21956792', 'assoc_port': '21917053:21950821,21917053:21950841,21917067:21956772,21917067:21956792,21917162:21947850,21917162:21947869,21917360:21950398', 'assoc_mat': '21917053:autre_non_emballage:4857:21950821,21917053:autre_non_emballage:4857:21950841,21917067:autre_non_emballage:4857:21956772,21917067:autre_non_emballage:4857:21956792,21917162:autre_non_emballage:4857:21947850,21917162:autre_non_emballage:4857:21947869,21917360:autre_non_emballage:4857:21950398'}, {'unwanted_material': 'carton', 'main_material': 'entrant', 'pht_type': 4857, 'ratio': 0.6498830700745826, 'nb_photo': 1039, 'list_port_cont': '21947836,21947855,21950384,21950430,21950807,21950827,21956758,21956778,21984388,21984407,22001990,22002009,22015820,22015840,22017290,22017326,22054963,22054988', 'assoc_port': '21917053:21950807,21917053:21950827,21917067:21956758,21917067:21956778,21917161:21984388,21917161:21984407,21917162:21947836,21917162:21947855,21917357:22015820,21917357:22015840,21917360:21950384,21917360:21950430,21917361:22001990,21917361:22002009,21917362:22017290,21917362:22017326,21918917:22054963,21918917:22054988', 'assoc_mat': '21917053:carton:4857:21950807,21917053:carton:4857:21950827,21917067:carton:4857:21956758,21917067:carton:4857:21956778,21917161:carton:4857:21984388,21917161:carton:4857:21984407,21917162:carton:4857:21947836,21917162:carton:4857:21947855,21917357:carton:4857:22015820,21917357:carton:4857:22015840,21917360:carton:4857:21950384,21917360:carton:4857:21950430,21917361:carton:4857:22001990,21917361:carton:4857:22002009,21917362:carton:4857:22017290,21917362:carton:4857:22017326,21918917:carton:4857:22054963,21918917:carton:4857:22054988'}, {'unwanted_material': 'dasri', 'main_material': 'entrant', 'pht_type': 4857, 'ratio': 0.0, 'nb_photo': 1, 'list_port_cont': '21950395', 'assoc_port': '21917360:21950395', 'assoc_mat': '21917360:dasri:4857:21950395'}, {'unwanted_material': 'ela', 'main_material': 'entrant', 'pht_type': 4857, 'ratio': 0.002121527027143947, 'nb_photo': 5, 'list_port_cont': '21950813,21950833,21956764,21956784,21984394,21984413,22015826,22015846,22017296,22017332', 'assoc_port': '21917053:21950813,21917053:21950833,21917067:21956764,21917067:21956784,21917161:21984394,21917161:21984413,21917357:22015826,21917357:22015846,21917362:22017296,21917362:22017332', 'assoc_mat': '21917053:ela:4857:21950813,21917053:ela:4857:21950833,21917067:ela:4857:21956764,21917067:ela:4857:21956784,21917161:ela:4857:21984394,21917161:ela:4857:21984413,21917357:ela:4857:22015826,21917357:ela:4857:22015846,21917362:ela:4857:22017296,21917362:ela:4857:22017332'}, {'unwanted_material': 'film_dev_souple', 'main_material': 'entrant', 'pht_type': 4857, 'ratio': 0.02799164863432263, 'nb_photo': 164, 'list_port_cont': '21947841,21947860,21950389,21950435,21950812,21950832,21956763,21956783,21984393,21984412,22001995,22002014,22015825,22015845,22017295,22017331,22054968,22054993', 'assoc_port': '21917053:21950812,21917053:21950832,21917067:21956763,21917067:21956783,21917161:21984393,21917161:21984412,21917162:21947841,21917162:21947860,21917357:22015825,21917357:22015845,21917360:21950389,21917360:21950435,21917361:22001995,21917361:22002014,21917362:22017295,21917362:22017331,21918917:22054968,21918917:22054993', 'assoc_mat': '21917053:film_dev_souple:4857:21950812,21917053:film_dev_souple:4857:21950832,21917067:film_dev_souple:4857:21956763,21917067:film_dev_souple:4857:21956783,21917161:film_dev_souple:4857:21984393,21917161:film_dev_souple:4857:21984412,21917162:film_dev_souple:4857:21947841,21917162:film_dev_souple:4857:21947860,21917357:film_dev_souple:4857:22015825,21917357:film_dev_souple:4857:22015845,21917360:film_dev_souple:4857:21950389,21917360:film_dev_souple:4857:21950435,21917361:film_dev_souple:4857:22001995,21917361:film_dev_souple:4857:22002014,21917362:film_dev_souple:4857:22017295,21917362:film_dev_souple:4857:22017331,21918917:film_dev_souple:4857:22054968,21918917:film_dev_souple:4857:22054993'}, {'unwanted_material': 'flux_dev', 'main_material': 'entrant', 'pht_type': 4857, 'ratio': 0.026120294248382987, 'nb_photo': 66, 'list_port_cont': '21947840,21947859,21950388,21950434,21950811,21950831,21956762,21956782,21984392,21984411,22001994,22002013,22015824,22015844,22017294,22017330,22054967,22054992', 'assoc_port': '21917053:21950811,21917053:21950831,21917067:21956762,21917067:21956782,21917161:21984392,21917161:21984411,21917162:21947840,21917162:21947859,21917357:22015824,21917357:22015844,21917360:21950388,21917360:21950434,21917361:22001994,21917361:22002013,21917362:22017294,21917362:22017330,21918917:22054967,21918917:22054992', 'assoc_mat': '21917053:flux_dev:4857:21950811,21917053:flux_dev:4857:21950831,21917067:flux_dev:4857:21956762,21917067:flux_dev:4857:21956782,21917161:flux_dev:4857:21984392,21917161:flux_dev:4857:21984411,21917162:flux_dev:4857:21947840,21917162:flux_dev:4857:21947859,21917357:flux_dev:4857:22015824,21917357:flux_dev:4857:22015844,21917360:flux_dev:4857:21950388,21917360:flux_dev:4857:21950434,21917361:flux_dev:4857:22001994,21917361:flux_dev:4857:22002013,21917362:flux_dev:4857:22017294,21917362:flux_dev:4857:22017330,21918917:flux_dev:4857:22054967,21918917:flux_dev:4857:22054992'}, {'unwanted_material': 'metal', 'main_material': 'entrant', 'pht_type': 4857, 'ratio': 0.024127419912394713, 'nb_photo': 44, 'list_port_cont': '21947837,21950385,21950431,21950808,21950828,21956759,21956779,22001991,22002010,22015821,22015841,22017291,22017327', 'assoc_port': '21917053:21950808,21917053:21950828,21917067:21956759,21917067:21956779,21917162:21947837,21917357:22015821,21917357:22015841,21917360:21950385,21917360:21950431,21917361:22001991,21917361:22002010,21917362:22017291,21917362:22017327', 'assoc_mat': '21917053:metal:4857:21950808,21917053:metal:4857:21950828,21917067:metal:4857:21956759,21917067:metal:4857:21956779,21917162:metal:4857:21947837,21917357:metal:4857:22015821,21917357:metal:4857:22015841,21917360:metal:4857:21950385,21917360:metal:4857:21950431,21917361:metal:4857:22001991,21917361:metal:4857:22002010,21917362:metal:4857:22017291,21917362:metal:4857:22017327'}, {'unwanted_material': 'organique', 'main_material': 'entrant', 'pht_type': 4857, 'ratio': 0.004472507743608766, 'nb_photo': 1, 'list_port_cont': '21950440', 'assoc_port': '21917360:21950440', 'assoc_mat': '21917360:organique:4857:21950440'}, {'unwanted_material': 'papier', 'main_material': 'entrant', 'pht_type': 4857, 'ratio': 0.1400314294980711, 'nb_photo': 235, 'list_port_cont': '21947835,21947854,21950383,21950429,21950806,21950826,21956757,21956777,21984387,21984406,22001989,22002008,22015819,22015839,22017289,22017325,22054962,22054987', 'assoc_port': '21917053:21950806,21917053:21950826,21917067:21956757,21917067:21956777,21917161:21984387,21917161:21984406,21917162:21947835,21917162:21947854,21917357:22015819,21917357:22015839,21917360:21950383,21917360:21950429,21917361:22001989,21917361:22002008,21917362:22017289,21917362:22017325,21918917:22054962,21918917:22054987', 'assoc_mat': '21917053:papier:4857:21950806,21917053:papier:4857:21950826,21917067:papier:4857:21956757,21917067:papier:4857:21956777,21917161:papier:4857:21984387,21917161:papier:4857:21984406,21917162:papier:4857:21947835,21917162:papier:4857:21947854,21917357:papier:4857:22015819,21917357:papier:4857:22015839,21917360:papier:4857:21950383,21917360:papier:4857:21950429,21917361:papier:4857:22001989,21917361:papier:4857:22002008,21917362:papier:4857:22017289,21917362:papier:4857:22017325,21918917:papier:4857:22054962,21918917:papier:4857:22054987'}, {'unwanted_material': 'pehd', 'main_material': 'entrant', 'pht_type': 4857, 'ratio': 0.008176693463765301, 'nb_photo': 12, 'list_port_cont': '21947839,21947858,21950387,21950433,21950810,21950830,21984391,21984410,22015823,22015843,22017293,22017329', 'assoc_port': '21917053:21950810,21917053:21950830,21917161:21984391,21917161:21984410,21917162:21947839,21917162:21947858,21917357:22015823,21917357:22015843,21917360:21950387,21917360:21950433,21917362:22017293,21917362:22017329', 'assoc_mat': '21917053:pehd:4857:21950810,21917053:pehd:4857:21950830,21917161:pehd:4857:21984391,21917161:pehd:4857:21984410,21917162:pehd:4857:21947839,21917162:pehd:4857:21947858,21917357:pehd:4857:22015823,21917357:pehd:4857:22015843,21917360:pehd:4857:21950387,21917360:pehd:4857:21950433,21917362:pehd:4857:22017293,21917362:pehd:4857:22017329'}, {'unwanted_material': 'pet_clair', 'main_material': 'entrant', 'pht_type': 4857, 'ratio': 0.029335934831367275, 'nb_photo': 163, 'list_port_cont': '21947838,21947857,21950386,21950432,21950809,21950829,21956760,21956780,21984390,21984409,22001992,22002011,22015822,22015842,22017292,22017328,22054965,22054990', 'assoc_port': '21917053:21950809,21917053:21950829,21917067:21956760,21917067:21956780,21917161:21984390,21917161:21984409,21917162:21947838,21917162:21947857,21917357:22015822,21917357:22015842,21917360:21950386,21917360:21950432,21917361:22001992,21917361:22002011,21917362:22017292,21917362:22017328,21918917:22054965,21918917:22054990', 'assoc_mat': '21917053:pet_clair:4857:21950809,21917053:pet_clair:4857:21950829,21917067:pet_clair:4857:21956760,21917067:pet_clair:4857:21956780,21917161:pet_clair:4857:21984390,21917161:pet_clair:4857:21984409,21917162:pet_clair:4857:21947838,21917162:pet_clair:4857:21947857,21917357:pet_clair:4857:22015822,21917357:pet_clair:4857:22015842,21917360:pet_clair:4857:21950386,21917360:pet_clair:4857:21950432,21917361:pet_clair:4857:22001992,21917361:pet_clair:4857:22002011,21917362:pet_clair:4857:22017292,21917362:pet_clair:4857:22017328,21918917:pet_clair:4857:22054965,21918917:pet_clair:4857:22054990'}, {'unwanted_material': 'sac_om_plein', 'main_material': 'entrant', 'pht_type': 4857, 'ratio': 0.16562486153652237, 'nb_photo': 65, 'list_port_cont': '21947843,21947862,21950391,21950437,21950814,21950834,21956765,21956785,21984395,21984414,22001997,22002016,22015827,22015847,22017297,22017333,22054971,22054995', 'assoc_port': '21917053:21950814,21917053:21950834,21917067:21956765,21917067:21956785,21917161:21984395,21917161:21984414,21917162:21947843,21917162:21947862,21917357:22015827,21917357:22015847,21917360:21950391,21917360:21950437,21917361:22001997,21917361:22002016,21917362:22017297,21917362:22017333,21918917:22054971,21918917:22054995', 'assoc_mat': '21917053:sac_om_plein:4857:21950814,21917053:sac_om_plein:4857:21950834,21917067:sac_om_plein:4857:21956765,21917067:sac_om_plein:4857:21956785,21917161:sac_om_plein:4857:21984395,21917161:sac_om_plein:4857:21984414,21917162:sac_om_plein:4857:21947843,21917162:sac_om_plein:4857:21947862,21917357:sac_om_plein:4857:22015827,21917357:sac_om_plein:4857:22015847,21917360:sac_om_plein:4857:21950391,21917360:sac_om_plein:4857:21950437,21917361:sac_om_plein:4857:22001997,21917361:sac_om_plein:4857:22002016,21917362:sac_om_plein:4857:22017297,21917362:sac_om_plein:4857:22017333,21918917:sac_om_plein:4857:22054971,21918917:sac_om_plein:4857:22054995'}, {'unwanted_material': 'textiles', 'main_material': 'entrant', 'pht_type': 4857, 'ratio': 0.007983810261584477, 'nb_photo': 7, 'list_port_cont': '21950438,21950815,21950835,21984396,21984415,22001998,22002017', 'assoc_port': '21917053:21950815,21917053:21950835,21917161:21984396,21917161:21984415,21917360:21950438,21917361:22001998,21917361:22002017', 'assoc_mat': '21917053:textiles:4857:21950815,21917053:textiles:4857:21950835,21917161:textiles:4857:21984396,21917161:textiles:4857:21984415,21917360:textiles:4857:21950438,21917361:textiles:4857:22001998,21917361:textiles:4857:22002017'}, {'unwanted_material': 'verre', 'main_material': 'entrant', 'pht_type': 4857, 'ratio': 0.002323954391622627, 'nb_photo': 1, 'list_port_cont': '21947864', 'assoc_port': '21917162:21947864', 'assoc_mat': '21917162:verre:4857:21947864'}), 'main_stat': {'nb_photos': 93}}, 'hostname': (), 'impurety_average_per_hashtag': [(('entrant', '2025-03', 0.22246814709383067),)]} TODO : Insert select and so on SELECT id, name, misc_info, datou_id_for_right FROM MTRPhoto.dashboard_places WHERE name="SyctomXV_Mask_23Q4"; SELECT name, created_at,limit_max FROM MTRDatou.mtr_datou WHERE id=4932 SELECT mtd.id, mtdt.`type`, mtd.`param`, mtd.param_json, mtdt.nb_input, mtdt.nb_output, mtdt.prod, mtdt.is_local, mtdt.is_datou_depend, mtdt.is_photo_id_local FROM MTRDatou.mtr_datou_step mtd, MTRDatou.mtr_datou_step_types mtdt WHERE mtdt.`id`=mtd.`type` AND mtd.mtd_id=4932 SELECT mtd.id, mtd.mtd_id, mdsdt.id, mdsdt.name, mdsdt.description, msid.output_or_input, msid.data_order_id, mdsdt.type FROM MTRDatou.mtr_datou_step mtd, MTRDatou.mtr_datou_steptype_io_datatypes msid, MTRDatou.mtr_datou_step_data_types mdsdt WHERE mtd.`type`=msid.`mtr_datou_step_type` AND mtd.mtd_id= 4932 AND msid.data_type=mdsdt.id SELECT mts_id_output, id_output, mts_id_input, id_input FROM MTRDatou.mtr_datou_step_by_step WHERE mtd_id=4932 # VR 17-11-17 : to create in DB ! Here we check the datou graph and we reorder steps ! Tree builded and cycle checked, now we need to re-order the steps ! We have currenlty an error because there is no dependence between the last step for the case tile - detect - glue We can either keep the depence of, it is better to keep an order compatible with the id of steps if we do not have sons, so a lexical order : (number_son, step_id) DONE and to test : checkNoCycle ! We are managing only one step so we do not consider checkConsistencyNbInputNbOutput ! We are managing only one step so we do not consider checkConsistencyTypeOutputInput ! {"input_values":[]} We expected an STS datou with a camera_place_name, so there is a PB, datou_id: 4932 ! {'id': 4932, 'name': 'split_time_score_cod_upm_SyctomXV_Mask_23Q1', 'limit_max': 150, 'nb_steps': 1, 'estimated_time': 8640, 'list_steps': [{'id': 14066, 'type': 'split_time_score_with_photo', 'param': '0', 'json_param_as_string': "{'cod': 1, 'camera_place_name': 'SyctomXV_Mask_23Q4', 'ETA': 8640, 'param_moyenne_mobile': 5, 'thcl': 3379, 'list_task_complete': [''], 'list_tasks': 'LOAD,ORDER,CONVENTION,SPLIT_BY_DARK,SPLIT_TIME, CREATE_PORT_LAUNCH_CARAC_FILL_DASHBOARD,READ_PLATE', 'forced_hashtag': 'entrant', 'list_hashtag_class_create': 'entrant', 'ponderation': '0.001,0.001,0.001,0.001,0.001', 'host': 'www.fotonower.com', 'interval': 10, 'datou_proportion_papier': {'test': 370, 'entrant': 4742}, 'check_plate_format': 1, 'input_default_values': []}", 'json_param': {'cod': 1, 'camera_place_name': 'SyctomXV_Mask_23Q4', 'ETA': 8640, 'param_moyenne_mobile': 5, 'thcl': 3379, 'list_task_complete': [''], 'list_tasks': 'LOAD,ORDER,CONVENTION,SPLIT_BY_DARK,SPLIT_TIME, CREATE_PORT_LAUNCH_CARAC_FILL_DASHBOARD,READ_PLATE', 'forced_hashtag': 'entrant', 'list_hashtag_class_create': 'entrant', 'ponderation': '0.001,0.001,0.001,0.001,0.001', 'host': 'www.fotonower.com', 'interval': 10, 'datou_proportion_papier': {'test': 370, 'entrant': 4742}, 'check_plate_format': 1, 'input_default_values': []}, 'nb_input': 2, 'nb_output': 1, 'prod': False, 'is_local': False, 'is_datou_depend': False, 'is_photo_id_local': False, 'list_data': []}], 'depends_step': []} check if it is the case where we have multiple sts in one datou yes, we found a datou with multiple sts, we will choose the last one by default SELECT name, created_at,limit_max FROM MTRDatou.mtr_datou WHERE id=370 SELECT mtd.id, mtdt.`type`, mtd.`param`, mtd.param_json, mtdt.nb_input, mtdt.nb_output, mtdt.prod, mtdt.is_local, mtdt.is_datou_depend, mtdt.is_photo_id_local FROM MTRDatou.mtr_datou_step mtd, MTRDatou.mtr_datou_step_types mtdt WHERE mtdt.`id`=mtd.`type` AND mtd.mtd_id=370 SELECT mtd.id, mtd.mtd_id, mdsdt.id, mdsdt.name, mdsdt.description, msid.output_or_input, msid.data_order_id, mdsdt.type FROM MTRDatou.mtr_datou_step mtd, MTRDatou.mtr_datou_steptype_io_datatypes msid, MTRDatou.mtr_datou_step_data_types mdsdt WHERE mtd.`type`=msid.`mtr_datou_step_type` AND mtd.mtd_id= 370 AND msid.data_type=mdsdt.id SELECT mts_id_output, id_output, mts_id_input, id_input FROM MTRDatou.mtr_datou_step_by_step WHERE mtd_id=370 # VR 17-11-17 : to create in DB ! Here we check the datou graph and we reorder steps ! Tree builded and cycle checked, now we need to re-order the steps ! We have currenlty an error because there is no dependence between the last step for the case tile - detect - glue We can either keep the depence of, it is better to keep an order compatible with the id of steps if we do not have sons, so a lexical order : (number_son, step_id) All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! DONE and to test : checkNoCycle ! Here we check the consistency of inputs/outputs number between the given ones and the db ! eke 1-6-18 : checkConsistencyNbInputNbOutput should be processed after step reordering ! Number of inputs / outputs for each step checked ! Here we check the consistency of outputs/inputs types during steps connections eke 1-6-18 : checkConsistencyTypeOutputInput should be processed after checkConsistencyNbInputNbOutput ! We ignore checkConsistencyTypeOutputInput for datou_step final ! We ignore checkConsistencyTypeOutputInput for datou_step final ! We ignore checkConsistencyTypeOutputInput for datou_step final ! DataTypes for each output/input checked ! {"input_values":[]} Unexpected error treated as WARNING for now expected given param in final step {'min_score': 0.7, 'name_pipeline': 'proportion_papier_tapis', 'hashtag_type': 705, 'hashtag_proportion': 492668766, 'hashtag_parmi': 538914404, 'hashtag_to_consider': 'papier_only', 'hashtag_background': 493012381} Found hashtag_parmi in final step, will be used and useful if different from matiere_majoritaire in classifier ventilate_hashtags_in_portfolio step not found ! SELECT name, created_at,limit_max FROM MTRDatou.mtr_datou WHERE id=4742 SELECT mtd.id, mtdt.`type`, mtd.`param`, mtd.param_json, mtdt.nb_input, mtdt.nb_output, mtdt.prod, mtdt.is_local, mtdt.is_datou_depend, mtdt.is_photo_id_local FROM MTRDatou.mtr_datou_step mtd, MTRDatou.mtr_datou_step_types mtdt WHERE mtdt.`id`=mtd.`type` AND mtd.mtd_id=4742 SELECT mtd.id, mtd.mtd_id, mdsdt.id, mdsdt.name, mdsdt.description, msid.output_or_input, msid.data_order_id, mdsdt.type FROM MTRDatou.mtr_datou_step mtd, MTRDatou.mtr_datou_steptype_io_datatypes msid, MTRDatou.mtr_datou_step_data_types mdsdt WHERE mtd.`type`=msid.`mtr_datou_step_type` AND mtd.mtd_id= 4742 AND msid.data_type=mdsdt.id SELECT mts_id_output, id_output, mts_id_input, id_input FROM MTRDatou.mtr_datou_step_by_step WHERE mtd_id=4742 # VR 17-11-17 : to create in DB ! Here we check the datou graph and we reorder steps ! Tree builded and cycle checked, now we need to re-order the steps ! We have currenlty an error because there is no dependence between the last step for the case tile - detect - glue We can either keep the depence of, it is better to keep an order compatible with the id of steps if we do not have sons, so a lexical order : (number_son, step_id) All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! All sons are already in current list ! DONE and to test : checkNoCycle ! Here we check the consistency of inputs/outputs number between the given ones and the db ! eke 1-6-18 : checkConsistencyNbInputNbOutput should be processed after step reordering ! WARNING : number of outputs for step 13169 copy_chis is not consistent : 2 used against 1 in the step definition ! WARNING : number of outputs for step 13170 consolidate_hashtags_from_manual_portfolio is not consistent : 2 used against 1 in the step definition ! WARNING : number of inputs for step 13167 rle_unique_nms_with_priority is not consistent : 3 used against 1 in the step definition ! WARNING : number of outputs for step 13167 rle_unique_nms_with_priority is not consistent : 2 used against 1 in the step definition ! WARNING : number of outputs for step 13174 ventilate_hashtags_in_portfolio is not consistent : 2 used against 1 in the step definition ! Step 13168 final have less outputs used (1) than in the step definition (2) : some outputs may be not used ! Step 13171 blur_detection have less inputs used (0) than in the step definition (1) : maybe we manage optionnal inputs ! Step 13172 brightness have less inputs used (0) than in the step definition (1) : maybe we manage optionnal inputs ! Step 13173 send_mail_cod have less inputs used (4) than in the step definition (5) : maybe we manage optionnal inputs ! Step 13175 split_time_score have less inputs used (1) than in the step definition (2) : maybe we manage optionnal inputs ! Number of inputs / outputs for each step checked ! Here we check the consistency of outputs/inputs types during steps connections eke 1-6-18 : checkConsistencyTypeOutputInput should be processed after checkConsistencyNbInputNbOutput ! We ignore checkConsistencyTypeOutputInput for datou_step final ! WARNING : output 0 of step 13169 have datatype=11 whereas input 0 of step 13167 have datatype=2 WARNING : type of output 1 of step 13170 doesn't seem to be define in the database( WARNING : type of input 3 of step 13168 doesn't seem to be define in the database( We ignore checkConsistencyTypeOutputInput for datou_step final ! We ignore checkConsistencyTypeOutputInput for datou_step final ! WARNING : type of output 1 of step 13170 doesn't seem to be define in the database( WARNING : type of input 1 of step 13167 doesn't seem to be define in the database( WARNING : output 0 of step 13174 have datatype=10 whereas input 3 of step 13173 have datatype=6 WARNING : type of input 1 of step 13174 doesn't seem to be define in the database( WARNING : output 1 of step 13167 have datatype=7 whereas input 1 of step 13174 have datatype=None WARNING : type of output 1 of step 13174 doesn't seem to be define in the database( WARNING : type of input 4 of step 13168 doesn't seem to be define in the database( We ignore checkConsistencyTypeOutputInput for datou_step final ! WARNING : type of output 1 of step 13169 doesn't seem to be define in the database( WARNING : type of input 1 of step 13170 doesn't seem to be define in the database( DataTypes for each output/input checked ! {"input_values":[]} Unexpected error treated as WARNING for now expected given param in final step {'name_pipeline': 'aggregate_carac_ratio', 'hashtag_type': 4857, 'hashtag_proportion': 'sac_om_plein,textiles,verre,organique,dasri,masque,encombrant,autre_non_emballage', 'hashtag_parmi': 'papier,carton,metal,pet_clair,pehd,flux_dev,film_dev_souple,ela', 'hashtag_weights': {'papier': 3, 'carton': 2.8, 'metal': 6, 'pet_clair': 1.4, 'pehd': 2.8, 'flux_dev': 2.8, 'pet_opaque': 3, 'barquette_opaque': 2.8, 'film_dev_souple': 0.8, 'ela': 3.0, 'sac_om_plein': 6.5, 'textiles': 3, 'verre': 10.0, 'organique': 4.0, 'dasri': 2.0, 'masque': 1, 'encombrant': 7, 'autre_emballage': 6.0, 'autre_non_emballage': 6.0}, 'check_plaque': 1} Found hashtag_parmi in final step, will be used and useful if different from matiere_majoritaire in classifier {'number_of_analysed_photos': ((93,),), 'unwanted_material_monthly_percentage': ((0.22246814709383067,),), 'material_repartition': (('entrant', 93),), 'material_dict': {'entrant': 93}, 'coverage_data': (), 'impurety_average_data': ((datetime.date(2025, 3, 1), 0.30656812781950904, 21), (datetime.date(2025, 3, 4), 0.1853987599174162, 36), (datetime.date(2025, 3, 5), 0.12077183578984403, 10), (datetime.date(2025, 3, 25), 0.276169821316118, 26)), 'impurety_average_data_per_hashtag': {'entrant': ((datetime.date(2025, 3, 1), 0.30656812781950904, 21), (datetime.date(2025, 3, 4), 0.1853987599174162, 36), (datetime.date(2025, 3, 5), 0.12077183578984403, 10), (datetime.date(2025, 3, 25), 0.276169821316118, 26))}, 'quality_array': [], 'number_of_batch': ((12,),), 'datou_id_sts': 4932, 'map_hashtag_number_mat': {'hash_to_mat_nb': ({'unwanted_material': 'autre_non_emballage', 'main_material': 'entrant', 'pht_type': 4857, 'ratio': 0.01541413348688883, 'nb_photo': 11, 'list_port_cont': '21947850,21947869,21950398,21950821,21950841,21956772,21956792', 'assoc_port': '21917053:21950821,21917053:21950841,21917067:21956772,21917067:21956792,21917162:21947850,21917162:21947869,21917360:21950398', 'assoc_mat': '21917053:autre_non_emballage:4857:21950821,21917053:autre_non_emballage:4857:21950841,21917067:autre_non_emballage:4857:21956772,21917067:autre_non_emballage:4857:21956792,21917162:autre_non_emballage:4857:21947850,21917162:autre_non_emballage:4857:21947869,21917360:autre_non_emballage:4857:21950398'}, {'unwanted_material': 'carton', 'main_material': 'entrant', 'pht_type': 4857, 'ratio': 0.6498830700745826, 'nb_photo': 1039, 'list_port_cont': '21947836,21947855,21950384,21950430,21950807,21950827,21956758,21956778,21984388,21984407,22001990,22002009,22015820,22015840,22017290,22017326,22054963,22054988', 'assoc_port': '21917053:21950807,21917053:21950827,21917067:21956758,21917067:21956778,21917161:21984388,21917161:21984407,21917162:21947836,21917162:21947855,21917357:22015820,21917357:22015840,21917360:21950384,21917360:21950430,21917361:22001990,21917361:22002009,21917362:22017290,21917362:22017326,21918917:22054963,21918917:22054988', 'assoc_mat': '21917053:carton:4857:21950807,21917053:carton:4857:21950827,21917067:carton:4857:21956758,21917067:carton:4857:21956778,21917161:carton:4857:21984388,21917161:carton:4857:21984407,21917162:carton:4857:21947836,21917162:carton:4857:21947855,21917357:carton:4857:22015820,21917357:carton:4857:22015840,21917360:carton:4857:21950384,21917360:carton:4857:21950430,21917361:carton:4857:22001990,21917361:carton:4857:22002009,21917362:carton:4857:22017290,21917362:carton:4857:22017326,21918917:carton:4857:22054963,21918917:carton:4857:22054988'}, {'unwanted_material': 'dasri', 'main_material': 'entrant', 'pht_type': 4857, 'ratio': 0.0, 'nb_photo': 1, 'list_port_cont': '21950395', 'assoc_port': '21917360:21950395', 'assoc_mat': '21917360:dasri:4857:21950395'}, {'unwanted_material': 'ela', 'main_material': 'entrant', 'pht_type': 4857, 'ratio': 0.002121527027143947, 'nb_photo': 5, 'list_port_cont': '21950813,21950833,21956764,21956784,21984394,21984413,22015826,22015846,22017296,22017332', 'assoc_port': '21917053:21950813,21917053:21950833,21917067:21956764,21917067:21956784,21917161:21984394,21917161:21984413,21917357:22015826,21917357:22015846,21917362:22017296,21917362:22017332', 'assoc_mat': '21917053:ela:4857:21950813,21917053:ela:4857:21950833,21917067:ela:4857:21956764,21917067:ela:4857:21956784,21917161:ela:4857:21984394,21917161:ela:4857:21984413,21917357:ela:4857:22015826,21917357:ela:4857:22015846,21917362:ela:4857:22017296,21917362:ela:4857:22017332'}, {'unwanted_material': 'film_dev_souple', 'main_material': 'entrant', 'pht_type': 4857, 'ratio': 0.02799164863432263, 'nb_photo': 164, 'list_port_cont': '21947841,21947860,21950389,21950435,21950812,21950832,21956763,21956783,21984393,21984412,22001995,22002014,22015825,22015845,22017295,22017331,22054968,22054993', 'assoc_port': '21917053:21950812,21917053:21950832,21917067:21956763,21917067:21956783,21917161:21984393,21917161:21984412,21917162:21947841,21917162:21947860,21917357:22015825,21917357:22015845,21917360:21950389,21917360:21950435,21917361:22001995,21917361:22002014,21917362:22017295,21917362:22017331,21918917:22054968,21918917:22054993', 'assoc_mat': '21917053:film_dev_souple:4857:21950812,21917053:film_dev_souple:4857:21950832,21917067:film_dev_souple:4857:21956763,21917067:film_dev_souple:4857:21956783,21917161:film_dev_souple:4857:21984393,21917161:film_dev_souple:4857:21984412,21917162:film_dev_souple:4857:21947841,21917162:film_dev_souple:4857:21947860,21917357:film_dev_souple:4857:22015825,21917357:film_dev_souple:4857:22015845,21917360:film_dev_souple:4857:21950389,21917360:film_dev_souple:4857:21950435,21917361:film_dev_souple:4857:22001995,21917361:film_dev_souple:4857:22002014,21917362:film_dev_souple:4857:22017295,21917362:film_dev_souple:4857:22017331,21918917:film_dev_souple:4857:22054968,21918917:film_dev_souple:4857:22054993'}, {'unwanted_material': 'flux_dev', 'main_material': 'entrant', 'pht_type': 4857, 'ratio': 0.026120294248382987, 'nb_photo': 66, 'list_port_cont': '21947840,21947859,21950388,21950434,21950811,21950831,21956762,21956782,21984392,21984411,22001994,22002013,22015824,22015844,22017294,22017330,22054967,22054992', 'assoc_port': '21917053:21950811,21917053:21950831,21917067:21956762,21917067:21956782,21917161:21984392,21917161:21984411,21917162:21947840,21917162:21947859,21917357:22015824,21917357:22015844,21917360:21950388,21917360:21950434,21917361:22001994,21917361:22002013,21917362:22017294,21917362:22017330,21918917:22054967,21918917:22054992', 'assoc_mat': '21917053:flux_dev:4857:21950811,21917053:flux_dev:4857:21950831,21917067:flux_dev:4857:21956762,21917067:flux_dev:4857:21956782,21917161:flux_dev:4857:21984392,21917161:flux_dev:4857:21984411,21917162:flux_dev:4857:21947840,21917162:flux_dev:4857:21947859,21917357:flux_dev:4857:22015824,21917357:flux_dev:4857:22015844,21917360:flux_dev:4857:21950388,21917360:flux_dev:4857:21950434,21917361:flux_dev:4857:22001994,21917361:flux_dev:4857:22002013,21917362:flux_dev:4857:22017294,21917362:flux_dev:4857:22017330,21918917:flux_dev:4857:22054967,21918917:flux_dev:4857:22054992'}, {'unwanted_material': 'metal', 'main_material': 'entrant', 'pht_type': 4857, 'ratio': 0.024127419912394713, 'nb_photo': 44, 'list_port_cont': '21947837,21950385,21950431,21950808,21950828,21956759,21956779,22001991,22002010,22015821,22015841,22017291,22017327', 'assoc_port': '21917053:21950808,21917053:21950828,21917067:21956759,21917067:21956779,21917162:21947837,21917357:22015821,21917357:22015841,21917360:21950385,21917360:21950431,21917361:22001991,21917361:22002010,21917362:22017291,21917362:22017327', 'assoc_mat': '21917053:metal:4857:21950808,21917053:metal:4857:21950828,21917067:metal:4857:21956759,21917067:metal:4857:21956779,21917162:metal:4857:21947837,21917357:metal:4857:22015821,21917357:metal:4857:22015841,21917360:metal:4857:21950385,21917360:metal:4857:21950431,21917361:metal:4857:22001991,21917361:metal:4857:22002010,21917362:metal:4857:22017291,21917362:metal:4857:22017327'}, {'unwanted_material': 'organique', 'main_material': 'entrant', 'pht_type': 4857, 'ratio': 0.004472507743608766, 'nb_photo': 1, 'list_port_cont': '21950440', 'assoc_port': '21917360:21950440', 'assoc_mat': '21917360:organique:4857:21950440'}, {'unwanted_material': 'papier', 'main_material': 'entrant', 'pht_type': 4857, 'ratio': 0.1400314294980711, 'nb_photo': 235, 'list_port_cont': '21947835,21947854,21950383,21950429,21950806,21950826,21956757,21956777,21984387,21984406,22001989,22002008,22015819,22015839,22017289,22017325,22054962,22054987', 'assoc_port': '21917053:21950806,21917053:21950826,21917067:21956757,21917067:21956777,21917161:21984387,21917161:21984406,21917162:21947835,21917162:21947854,21917357:22015819,21917357:22015839,21917360:21950383,21917360:21950429,21917361:22001989,21917361:22002008,21917362:22017289,21917362:22017325,21918917:22054962,21918917:22054987', 'assoc_mat': '21917053:papier:4857:21950806,21917053:papier:4857:21950826,21917067:papier:4857:21956757,21917067:papier:4857:21956777,21917161:papier:4857:21984387,21917161:papier:4857:21984406,21917162:papier:4857:21947835,21917162:papier:4857:21947854,21917357:papier:4857:22015819,21917357:papier:4857:22015839,21917360:papier:4857:21950383,21917360:papier:4857:21950429,21917361:papier:4857:22001989,21917361:papier:4857:22002008,21917362:papier:4857:22017289,21917362:papier:4857:22017325,21918917:papier:4857:22054962,21918917:papier:4857:22054987'}, {'unwanted_material': 'pehd', 'main_material': 'entrant', 'pht_type': 4857, 'ratio': 0.008176693463765301, 'nb_photo': 12, 'list_port_cont': '21947839,21947858,21950387,21950433,21950810,21950830,21984391,21984410,22015823,22015843,22017293,22017329', 'assoc_port': '21917053:21950810,21917053:21950830,21917161:21984391,21917161:21984410,21917162:21947839,21917162:21947858,21917357:22015823,21917357:22015843,21917360:21950387,21917360:21950433,21917362:22017293,21917362:22017329', 'assoc_mat': '21917053:pehd:4857:21950810,21917053:pehd:4857:21950830,21917161:pehd:4857:21984391,21917161:pehd:4857:21984410,21917162:pehd:4857:21947839,21917162:pehd:4857:21947858,21917357:pehd:4857:22015823,21917357:pehd:4857:22015843,21917360:pehd:4857:21950387,21917360:pehd:4857:21950433,21917362:pehd:4857:22017293,21917362:pehd:4857:22017329'}, {'unwanted_material': 'pet_clair', 'main_material': 'entrant', 'pht_type': 4857, 'ratio': 0.029335934831367275, 'nb_photo': 163, 'list_port_cont': '21947838,21947857,21950386,21950432,21950809,21950829,21956760,21956780,21984390,21984409,22001992,22002011,22015822,22015842,22017292,22017328,22054965,22054990', 'assoc_port': '21917053:21950809,21917053:21950829,21917067:21956760,21917067:21956780,21917161:21984390,21917161:21984409,21917162:21947838,21917162:21947857,21917357:22015822,21917357:22015842,21917360:21950386,21917360:21950432,21917361:22001992,21917361:22002011,21917362:22017292,21917362:22017328,21918917:22054965,21918917:22054990', 'assoc_mat': '21917053:pet_clair:4857:21950809,21917053:pet_clair:4857:21950829,21917067:pet_clair:4857:21956760,21917067:pet_clair:4857:21956780,21917161:pet_clair:4857:21984390,21917161:pet_clair:4857:21984409,21917162:pet_clair:4857:21947838,21917162:pet_clair:4857:21947857,21917357:pet_clair:4857:22015822,21917357:pet_clair:4857:22015842,21917360:pet_clair:4857:21950386,21917360:pet_clair:4857:21950432,21917361:pet_clair:4857:22001992,21917361:pet_clair:4857:22002011,21917362:pet_clair:4857:22017292,21917362:pet_clair:4857:22017328,21918917:pet_clair:4857:22054965,21918917:pet_clair:4857:22054990'}, {'unwanted_material': 'sac_om_plein', 'main_material': 'entrant', 'pht_type': 4857, 'ratio': 0.16562486153652237, 'nb_photo': 65, 'list_port_cont': '21947843,21947862,21950391,21950437,21950814,21950834,21956765,21956785,21984395,21984414,22001997,22002016,22015827,22015847,22017297,22017333,22054971,22054995', 'assoc_port': '21917053:21950814,21917053:21950834,21917067:21956765,21917067:21956785,21917161:21984395,21917161:21984414,21917162:21947843,21917162:21947862,21917357:22015827,21917357:22015847,21917360:21950391,21917360:21950437,21917361:22001997,21917361:22002016,21917362:22017297,21917362:22017333,21918917:22054971,21918917:22054995', 'assoc_mat': '21917053:sac_om_plein:4857:21950814,21917053:sac_om_plein:4857:21950834,21917067:sac_om_plein:4857:21956765,21917067:sac_om_plein:4857:21956785,21917161:sac_om_plein:4857:21984395,21917161:sac_om_plein:4857:21984414,21917162:sac_om_plein:4857:21947843,21917162:sac_om_plein:4857:21947862,21917357:sac_om_plein:4857:22015827,21917357:sac_om_plein:4857:22015847,21917360:sac_om_plein:4857:21950391,21917360:sac_om_plein:4857:21950437,21917361:sac_om_plein:4857:22001997,21917361:sac_om_plein:4857:22002016,21917362:sac_om_plein:4857:22017297,21917362:sac_om_plein:4857:22017333,21918917:sac_om_plein:4857:22054971,21918917:sac_om_plein:4857:22054995'}, {'unwanted_material': 'textiles', 'main_material': 'entrant', 'pht_type': 4857, 'ratio': 0.007983810261584477, 'nb_photo': 7, 'list_port_cont': '21950438,21950815,21950835,21984396,21984415,22001998,22002017', 'assoc_port': '21917053:21950815,21917053:21950835,21917161:21984396,21917161:21984415,21917360:21950438,21917361:22001998,21917361:22002017', 'assoc_mat': '21917053:textiles:4857:21950815,21917053:textiles:4857:21950835,21917161:textiles:4857:21984396,21917161:textiles:4857:21984415,21917360:textiles:4857:21950438,21917361:textiles:4857:22001998,21917361:textiles:4857:22002017'}, {'unwanted_material': 'verre', 'main_material': 'entrant', 'pht_type': 4857, 'ratio': 0.002323954391622627, 'nb_photo': 1, 'list_port_cont': '21947864', 'assoc_port': '21917162:21947864', 'assoc_mat': '21917162:verre:4857:21947864'}), 'main_stat': {'nb_photos': 93}}, 'hostname': (), 'impurety_average_per_hashtag': [(('entrant', '2025-03', 0.22246814709383067),)]} filepath : /data_2/data_log/job/memo/SyctomXV/sla_mensuel_SyctomXV_Mask_23Q4_mois_03_annee_2025.pdf Before plot_photos_and_unwanted_material_summary # VR 17-11-17 : to create in DB ! Here we check the datou graph and we reorder steps ! Tree builded and cycle checked, now we need to re-order the steps ! We have currenlty an error because there is no dependence between the last step for the case tile - detect - glue We can either keep the depence of, it is better to keep an order compatible with the id of steps if we do not have sons, so a lexical order : (number_son, step_id) DONE and to test : checkNoCycle ! We are managing only one step so we do not consider checkConsistencyNbInputNbOutput ! We are managing only one step so we do not consider checkConsistencyTypeOutputInput ! Before organize_and_plot_unwanted_material_stats {'entrant': {'autre_non_emballage': ['autre_non_emballage', '1.54%', 11], 'carton': ['carton', '64.99%', 1039], 'dasri': ['dasri', '0.0%', 1], 'ela': ['ela', '0.21%', 5], 'film_dev_souple': ['film_dev_souple', '2.8%', 164], 'flux_dev': ['flux_dev', '2.61%', 66], 'metal': ['metal', '2.41%', 44], 'organique': ['organique', '0.45%', 1], 'papier': ['papier', '14.0%', 235], 'pehd': ['pehd', '0.82%', 12], 'pet_clair': ['pet_clair', '2.93%', 163], 'sac_om_plein': ['sac_om_plein', '16.56%', 65], 'textiles': ['textiles', '0.8%', 7], 'verre': ['verre', '0.23%', 1]}} before organize_and_plot_impurety_average_data before organize_and_plot_quality_photo_data finish generating filepath : /data_2/data_log/job/memo/SyctomXV/sla_mensuel_SyctomXV_Mask_23Q4_mois_03_annee_2025.pdf le fichier se trouve ici : /data_2/data_log/job/memo/SyctomXV 2.69user 2.71system 0:03.05elapsed 176%CPU (0avgtext+0avgdata 191196maxresident)k 31664inputs+216outputs (159major+65922minor)pagefaults 0swaps