Coverage for lib/lib_github.py: 6%
1471 statements
« prev ^ index » next coverage.py v7.9.1, created at 2026-02-18 02:40 +0100
1__author__ = 'moilerat'
3# --job=git.user -v --with_comment --check_issue_re --list_repos=fotonower/projects
6import sys
8def print_mem():
9 import logging
10 logger = logging.getLogger()
11 local_vars = list(locals().items())
12 for var, obj in local_vars:
13 print(var, sys.getsizeof(obj))
14 logger.warning(" var : " + str(sys.getsizeof(obj)))
17import datetime
18import sys
20# TODO move mainly to dev_test ou alpha
22# pip3 install PyGithub d'après https://github.com/FlorianMarckmann/Test_stage_ftn/
23from github import Github
24import json
26## Usage
28## Append git. to these job for use from prompt
30# VR 25-12-21 : excluding issues with label
31# --job=count --excluded_labels="project:rubbia,bug,enhancement,business,client,documentation,api,project:cod,raspi,it_maintenance,reporting work,model:supervised:classification,ready_for_qa,project:qualipapia,refacto,rh,project:broca,compte_rendu,project:datou,model:gan,model:unsupervised:vse,project:datou_loco,model:statistics,recherche,duplicate"
32# --nb_days_tolerance=0 --excluded_labels="project:rubbia,bug,enhancement,business,client,documentation,api,project:cod,raspi,it_maintenance,reporting work,model:supervised:classification,ready_for_qa,project:qualipapia,refacto,rh,project:broca,compte_rendu,project:datou,model:gan,model:unsupervised:vse,project:datou_loco,model:statistics,recherche,duplicate"
35# --repos=fotonower/API,fotonower/raspi-fotonower-x,fotonower/Metolour,fotonower/Velours,fotonower/projects,fotonower/Admin
37# Just for one repo
38# -v --repos=fotonower/Metolour
40# Check reporting automatically
41# --job=report
43# Job normal
44# --nb_days_tolerance=20
46# List of all projects
47# project:cod,project:adenes,project:broca,project:circpack,project:qualipapia,project:mpa,project:datou,documentation,bug,it_maintenance,reporting work,business,commercial,recherche,rh,refacto
50# <!DOCTYPE NETSCAPE-Bookmark-file-1>
51# <!-- This is an automatically generated file.
52# It will be read and overwritten.
53# DO NOT EDIT! -->
54# <META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=UTF-8">
55# <TITLE>Bookmarks</TITLE>
56# <H1>Bookmarks</H1>
57# <DL><p>
58# <DT><H3 ADD_DATE="1368544705" LAST_MODIFIED="1644685392" PERSONAL_TOOLBAR_FOLDER="true">Barre de favoris</H3>
59# <DL><p>
60# <DT><H3 ADD_DATE="1509912547" LAST_MODIFIED="1644103407">perso</H3>
61# <DL><p>
62# <DT><H3 ADD_DATE="1580984692" LAST_MODIFIED="1644685336">louve</H3>
63# <DL><p>
64# <DT><A HREF="https://wiki.supermarches-cooperatifs.fr/doku.php?id=thematiques:informatique" ADD_DATE="1580984683" ICON="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAACKElEQVQ4jW2SO48dRRCFv1PdM3P37rXXFjL7HwiRkACRkRAS8ovgJ5CSgwh4SIBIIOBN4MAPJJCQsAUBIJ7W4pmqQzD33sUSFXWru07Vd6p0enp6frLbPdUAeodlgdbE/0RmurWmrKrm6A+Su32z2z3dW38dW4Fc0SVAEpLwIdtGIQREtATGbfBK74AgDC0k97GzLIuWeSZzARsb+jAwbTYGmOdZ0lqnL60xCrvsiObMJDM5e+wGZ4+fIwWS+PXH+/z12y8exwkJS3JI7j0TWkcS8zKDxHxxwbMvvsQTzzzHvW/vMk4nfP7Om9z54ieiDdgAVtl0tybbkiTbK7etabvlqw/e5f3XXmV3dp2qYhgnqorVIhGSYn/jkCxWU+wil5l8+JBl/gc798au/w/RD4eDCHvmWhaefP4Frt04p48jX773Fj98c4thnI7FHhG4DINBEfx8/x7f3fyaabvlwR+/E9H2/JdxFLhEWDuJaHx/6yafvf0Gp9eu04eR1jsY+A9CHIoeEMwqVDZ9mtheucr2ytV9srHWYscOlGm3tmau+thFH0daNKoKl49w8n63bMp2X1rTKKmqGIaBqlIfRm5/8hEXf/9JHyfKxR5erTcqE9ZFUldm0YcF1CQ8DB2M7nz6MYpg2OxnLxjb4AiRuRRGZWdX1Yg9rQ9JZoHMsJmOhq7LYbIWsgRob7hPembejl4vA4SCqiQiVlAenVlVERGUq1yaKD78FyIYKx9nIeZ/AAAAAElFTkSuQmCC">TEST thematiques:informatique [Supermarchés Coopératifs : le wiki]</A>
65# <DT><A HREF="https://forum.supermarches-cooperatifs.fr/t/le-choix-de-loutil-de-gestion-des-cooperatives/1287/8" ADD_DATE="1580984695" ICON="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAACKElEQVQ4jW2SO48dRRCFv1PdM3P37rXXFjL7HwiRkACRkRAS8ovgJ5CSgwh4SIBIIOBN4MAPJJCQsAUBIJ7W4pmqQzD33sUSFXWru07Vd6p0enp6frLbPdUAeodlgdbE/0RmurWmrKrm6A+Su32z2z3dW38dW4Fc0SVAEpLwIdtGIQREtATGbfBK74AgDC0k97GzLIuWeSZzARsb+jAwbTYGmOdZ0lqnL60xCrvsiObMJDM5e+wGZ4+fIwWS+PXH+/z12y8exwkJS3JI7j0TWkcS8zKDxHxxwbMvvsQTzzzHvW/vMk4nfP7Om9z54ieiDdgAVtl0tybbkiTbK7etabvlqw/e5f3XXmV3dp2qYhgnqorVIhGSYn/jkCxWU+wil5l8+JBl/gc798au/w/RD4eDCHvmWhaefP4Frt04p48jX773Fj98c4thnI7FHhG4DINBEfx8/x7f3fyaabvlwR+/E9H2/JdxFLhEWDuJaHx/6yafvf0Gp9eu04eR1jsY+A9CHIoeEMwqVDZ9mtheucr2ytV9srHWYscOlGm3tmau+thFH0daNKoKl49w8n63bMp2X1rTKKmqGIaBqlIfRm5/8hEXf/9JHyfKxR5erTcqE9ZFUldm0YcF1CQ8DB2M7nz6MYpg2OxnLxjb4AiRuRRGZWdX1Yg9rQ9JZoHMsJmOhq7LYbIWsgRob7hPembejl4vA4SCqiQiVlAenVlVERGUq1yaKD78FyIYKx9nIeZ/AAAAAElFTkSuQmCC">TEST Le choix de l'outil de gestion des coopératives - Informatique / ERP / ODOO - Supermarchés Coopératifs : le forum</A>
66# <DL><p>
67# <DL><p>
68# <DT><A HREF="https://github.com/issues?q=is%3Aopen+is%3Aissue+org%3Afotonower+archived%3Afalse+label%3Aready_for_qa" ADD_DATE="1644685336" ICON="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAACKElEQVQ4jW2SO48dRRCFv1PdM3P37rXXFjL7HwiRkACRkRAS8ovgJ5CSgwh4SIBIIOBN4MAPJJCQsAUBIJ7W4pmqQzD33sUSFXWru07Vd6p0enp6frLbPdUAeodlgdbE/0RmurWmrKrm6A+Su32z2z3dW38dW4Fc0SVAEpLwIdtGIQREtATGbfBK74AgDC0k97GzLIuWeSZzARsb+jAwbTYGmOdZ0lqnL60xCrvsiObMJDM5e+wGZ4+fIwWS+PXH+/z12y8exwkJS3JI7j0TWkcS8zKDxHxxwbMvvsQTzzzHvW/vMk4nfP7Om9z54ieiDdgAVtl0tybbkiTbK7etabvlqw/e5f3XXmV3dp2qYhgnqorVIhGSYn/jkCxWU+wil5l8+JBl/gc798au/w/RD4eDCHvmWhaefP4Frt04p48jX773Fj98c4thnI7FHhG4DINBEfx8/x7f3fyaabvlwR+/E9H2/JdxFLhEWDuJaHx/6yafvf0Gp9eu04eR1jsY+A9CHIoeEMwqVDZ9mtheucr2ytV9srHWYscOlGm3tmau+thFH0daNKoKl49w8n63bMp2X1rTKKmqGIaBqlIfRm5/8hEXf/9JHyfKxR5erTcqE9ZFUldm0YcF1CQ8DB2M7nz6MYpg2OxnLxjb4AiRuRRGZWdX1Yg9rQ9JZoHMsJmOhq7LYbIWsgRob7hPembejl4vA4SCqiQiVlAenVlVERGUq1yaKD78FyIYKx9nIeZ/AAAAAElFTkSuQmCC">TEST Issues label:ready_for_qa</A>
69# </DL><p>
70# </DL><p>
72head_favoris = """<!DOCTYPE NETSCAPE-Bookmark-file-1>
73<!-- This is an automatically generated file.
74 It will be read and overwritten.
75 DO NOT EDIT! -->
76<META HTTP-EQUIV="Content-Type" CONTENT="text/html; charset=UTF-8">
77<TITLE>Bookmarks</TITLE>
78<H1>Bookmarks</H1>
79"""
80image_ftn = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAYAAAAf8/9hAAACKElEQVQ4jW2SO48dRRCFv1PdM3P37rXXFjL7HwiRkACRkRAS8ovgJ5CSgwh4SIBIIOBN4MAPJJCQsAUBIJ7W4pmqQzD33sUSFXWru07Vd6p0enp6frLbPdUAeodlgdbE/0RmurWmrKrm6A+Su32z2z3dW38dW4Fc0SVAEpLwIdtGIQREtATGbfBK74AgDC0k97GzLIuWeSZzARsb+jAwbTYGmOdZ0lqnL60xCrvsiObMJDM5e+wGZ4+fIwWS+PXH+/z12y8exwkJS3JI7j0TWkcS8zKDxHxxwbMvvsQTzzzHvW/vMk4nfP7Om9z54ieiDdgAVtl0tybbkiTbK7etabvlqw/e5f3XXmV3dp2qYhgnqorVIhGSYn/jkCxWU+wil5l8+JBl/gc798au/w/RD4eDCHvmWhaefP4Frt04p48jX773Fj98c4thnI7FHhG4DINBEfx8/x7f3fyaabvlwR+/E9H2/JdxFLhEWDuJaHx/6yafvf0Gp9eu04eR1jsY+A9CHIoeEMwqVDZ9mtheucr2ytV9srHWYscOlGm3tmau+thFH0daNKoKl49w8n63bMp2X1rTKKmqGIaBqlIfRm5/8hEXf/9JHyfKxR5erTcqE9ZFUldm0YcF1CQ8DB2M7nz6MYpg2OxnLxjb4AiRuRRGZWdX1Yg9rQ9JZoHMsJmOhq7LYbIWsgRob7hPembejl4vA4SCqiQiVlAenVlVERGUq1yaKD78FyIYKx9nIeZ/AAAAAElFTkSuQmCC"
84# - [ ] add date in name
85# - [ ] add date in link
87def create_favoris(map_user_issues, file="favoris.html"):
88 with open(file, "w") as f:
89 f.write(head_favoris)
90 f.write("<DL><p>\n")
91 f.write(" <DT><H3>Listes de tickets ouvert depuis trop longtemps</H3>\n")
92 f.write(" <DL><p>\n")
94 for user in map_user_issues:
95 f.write(" <DT><H3>" + str(user) + "</H3>\n")
96 f.write(" <DL><p>\n")
97 # print("<a name='" + str(datestr) + "_" + str(user) + "' ></a>")
98 if len(map_user_issues[user]) > 0:
99 # print ("For user : " + str(user))
100 if "list_inactiv" in map_user_issues[user]:
101 for issue in map_user_issues[user]["list_inactiv"]:
102 # print(issue["link"] + " for " + str(issue["nb_days_inactiv"]) + " days")
103 f.write(
104 " <DT><A HREF=\"" + issue["link"] + "\" ICON=\"" + image_ftn + "\">" + issue[
105 "link"] + " for " + str(issue["nb_days_inactiv"]) + "</A>\n")
106 f.write(" </DL><p>\n")
108 f.write(" </DL><p>\n")
109 f.write("</DL><p>\n")
111 return 0
115def issue_atmo_has_label(issue, list_labels):
116 found = False
117 if "labels" not in issue:
118 return found
119 if issue["labels"] == None:
120 return found
121 for label in issue["labels"]:
122 if label != None and "name" in label and label["name"] in list_labels:
123 found = True
125 return found
129def issue_has_label(issue, list_labels):
130 found = False
131 for label in issue.labels:
132 if label.name.lower() in list_labels:
133 found = True
135 return found
138def check_issue_body(issue, list_comments, parse_link_wiki, link_issue):
139 if issue["concat_body"] != None:
140 has_up_anchor = issue["concat_body"].lower().find("<a name=up></a>")
141 if has_up_anchor < 0:
142 print(
143 "Missing anchor up (now using top) in body of issue " + str(issue["name"]) + " " + str(issue["title"]))
145 map_wiki_issue_link = {}
147 import re
148 # Check if the issue body has a link to a wiki page starting with https://github.com/fotonower/Fotonower/wiki
149 wiki_links = re.findall(r'(https\:\/\/github\.com\/fotonower\/Fotonower\/wiki[\/]?[^\s]*)',
150 issue["concat_body"])
151 # m = re.search(r'\[\[([^\]]+)\]\]', issue.body)
152 if len(wiki_links) > 0:
153 print("Found a link to a wiki page in the body of issue " + str(issue)[:50] + " wiki_link : " + str(wiki_links))
154 map_wiki_issue_link = {link_issue: wiki_links}
156 nbi = issue["number"]
157 link_issue
159 # internal_comment_links = re.findall(r'(https\:\/\/www\.github\.com\/fotonower\/saxia\/issues\/11\/[^\s]*)', issue.body)
160 link_issue_for_regexp = link_issue.replace("/", "\/").replace(":", "\:").replace("www.", "[www.]?").replace(".",
161 "\.")
162 search_regex = r"({0}\#issuecomment\-[\d]*)".format(link_issue_for_regexp)
163 print(" search_regex : " + str(search_regex))
164 internal_comment_links = re.findall(search_regex, issue["concat_body"])
165 # VR TODO on pourrait aussi chercher les lien vers des anchor contenu dans les commentaires, mais c'est beaucoup plus compliqué !
166 # VR TODO on pourrait aussi récupérer l'id du comment pour lister plus facilement tous les commentaires qui ne sont pas lister
168 present_comment_links = []
169 for m in internal_comment_links:
170 link = None
171 if type(m) == str:
172 link = m
173 if link != None and link not in present_comment_links:
174 present_comment_links.append(link)
176 missing_comment_links = []
177 nb_total_comments = 0
178 map_id_comment_missing_link_body_to_comment = {}
179 for c in list_comments:
180 nb_total_comments = nb_total_comments + 1
181 c_id = str(c["git_io"]["id"] if "git_io" in c and "id" in c["git_io"] else c["name"].split("#")[-1])
182 link_comment = link_issue + "#issuecomment-" + c_id
183 label_missing_link_body_to_comment = False
184 if link_comment.replace("www.",
185 "") not in present_comment_links and link_comment not in missing_comment_links and "issue-" not in link_comment and "DEFAULTFISTR" not in link_comment:
186 print("Missing link to comment " + str(c["name"]) + " in issue " + str(issue["name"]) + " " + str(
187 issue["title"]) + " link : " + str(link_comment))
188 missing_comment_links.append(link_comment.replace(".www", ""))
189 label_missing_link_body_to_comment = True
190 map_id_comment_missing_link_body_to_comment[c_id] = label_missing_link_body_to_comment
191 if c["body"] != None:
192 wiki_links_comments = re.findall(r'(https\:\/\/github\.com\/fotonower\/Fotonower\/wiki[\/]?[^\s]*)',
193 c["body"])
194 if len(wiki_links_comments) > 0:
195 print("Found a link to a wiki page in the comment of issue " + str(
196 issue["name"]) + " wiki_link : " + str(wiki_links_comments))
197 map_wiki_issue_link[link_comment] = wiki_links_comments
199 ret = {"has_up_anchor": has_up_anchor >= 0, "wiki_links": map_wiki_issue_link}
201 ret["internal_comment_links"] = internal_comment_links
202 ret["missing_comment_links"] = missing_comment_links
203 ret["draft_reason"] = (str(len(missing_comment_links)) + "/" + str(nb_total_comments - 1) + " MISSING\n") if len(
204 missing_comment_links) > 0 else "OK\n"
206 ret["map_id_comment_missing_link_body_to_comment"] = map_id_comment_missing_link_body_to_comment
208 return ret
210 map_id_comment_missing_link_body_to_comment = {}
211 for c in list_comments:
212 c_id = str(c["git_io"]["id"] if "git_io" in c and "id" in c["git_io"] else c["name"].split("#")[-1])
213 map_id_comment_missing_link_body_to_comment[c_id] = True
215 return {"map_id_comment_missing_link_body_to_comment" : map_id_comment_missing_link_body_to_comment}
219def from_label_to_map_color_name(labels):
220 list_map_labels = []
221 for label in labels:
222 map_label = {}
223 map_label["name"] = label.name
224 map_label["color"] = label.color
225 list_map_labels.append(map_label)
226 return list_map_labels
229def check_rate(headers):
230 # print("TODO")
232 xratelimitremaining = headers['x-ratelimit-remaining'] if "x-ratelimit-remaining" in headers else 0
233 xratelimitlimit = headers['x-ratelimit-limit'] if "x-ratelimit-limit" in headers else 0
234 if int(xratelimitlimit) > 0 and int(xratelimitremaining) * 5 < int(xratelimitlimit):
235 print(" x-ratelimit-remaining : " + str(xratelimitremaining))
236 print(" x-ratelimit-limit : " + str(xratelimitlimit))
237 print("Now we aren't blind anymore, to avoid rate limit issues, we wait for 1 minute before continuing !")
238 import time
239 time.sleep(60)
242# 'x-ratelimit-limit' (140650522216608) '5000'
243# 'x-ratelimit-remaining' (140650522216528) '4993'
244# 'x-ratelimit-reset' (140650522216368) '1684285562'
245# 'x-ratelimit-resource' (140650522216288) 'core'
246# 'x-ratelimit-used' (140650522216448) '7'
248def util_is_naive(dated_at):
249 is_not_naive = dated_at.tzinfo is not None and dated_at.tzinfo.utcoffset(dated_at) is not None
250 return is_not_naive
252# f1 gag 3-8-25
253def load_github_to_json_with_type(list_repos=["fotonower/Velours", "fotonower/projects", "fotonower/Admin"],
254 # override by default argument_parser
255 list_excluded_labels=["project:rubbia", "project:cod", "project:adenes", "project:broca",
256 "project:circpack", "project:qualipapia", "project:mpa", "project:datou",
257 "documentation", "bug", "it_maintenance", "reporting work", "business",
258 "commercial", "recherche", "rh", "refacto"],
259 token="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",
260 verbose=False, only_count=False,
261 with_comment=False,
262 check_issue_re=False,
263 load_all=False,
264 list_required_labels=[],
265 last_number_days = None,
266 previous_number_days = None,
267 limit=0,
268 offset=0,
269 list_issues=[],
270 parse_link_wiki=True,
271 cache_data_file=None,
272 find_detailed_update_at = False,
273 list_labels_to_parse = [],
274 config_atmo_for_name = "default"):
275 from datetime import datetime
276 start = datetime.now()
278 g = Github(token)
280 reponame = list_repos[0].replace("/", "_") if len(list_repos) == 1 else "all_repos_" + str(len(list_repos))
282 from tzlocal import get_localzone
283 print(" start cached_at : " + str(start) + " local timezone : " + str(get_localzone()))
284 data_load = {"issues": [], "type": "repo",
285 "name" : reponame,
286 "title" : reponame, "cached_at" : start, "cache_local_timezome" : get_localzone()}
288 count_issue = 0
290 for repo_name in list_repos:
292 repo = g.get_repo(repo_name)
293 state_to_load = 'all' if (load_all) else 'open' # or closed
295 if list_required_labels == []:
296 open_issues = repo.get_issues(state=state_to_load)
297 else:
298 if "audit" in list_required_labels:
299 with_comment = True
300 find_detailed_update_at = True
301 open_issues = repo.get_issues(state=state_to_load, labels=list_required_labels)
303 for issue in open_issues:
304 import sys
305 sys.stdout.write(";")
306 sys.stdout.flush()
307 if len(list_issues) > 0 and (repo_name.replace("fotonower/", "") + "/" + str(issue.number)) not in list_issues:
308 continue
309 if count_issue < offset:
310 count_issue += 1
311 continue
312 if count_issue == (limit + offset) and limit > 0:
313 print("Limit number issue to parse")
314 break
315 check_rate(issue._headers)
317 # Exclusion is more powerful
318 if not issue_has_label(issue, list_excluded_labels) and (
319 len(list_required_labels) == 0 or issue_has_label(issue, list_required_labels) or "issue" in list_required_labels):
320 if issue.body == None:
321 print("issue.body is None, but I want this issue !")
322 # continue
324 if util_is_naive(issue.updated_at):
325 print(" Issue NOTNAIVE " + str(issue.number) + " has a timezone, we convert it to UTC : " + str(
326 issue.updated_at) + " ! ")
327 else:
328 print(" Issue NAIVE " + str(issue.number) + " has no timezone, we assume it is in UTC : " + str(
329 issue.updated_at) + " ! ")
331 if last_number_days or previous_number_days:
332 from datetime import date, datetime
333 aujourdhui = datetime.now()
334 delta = aujourdhui - issue.updated_at
336 if last_number_days and delta.days > last_number_days:
337 print("Issue has not been modified recently, we don't use it ! ")
338 continue
339 if previous_number_days and delta.days < previous_number_days:
340 print("Issue has not been modified recently, we don't use it ! ")
341 continue
343 concat_body = issue.body if issue.body is not None else ""
344 size_body = len(issue.body.lower()) if issue.body is not None else 0
345 size_total = 0
346 all_comments = []
347 if with_comment:
348 all_comments_as_git = issue.get_comments()
349 for comment_git in all_comments_as_git:
350 comment = {"body": comment_git.body,
351 "updated_at": comment_git.updated_at,
352 "name": issue.repository.name + "/" + str(issue.number) + "#issuecomment-" + str(
353 comment_git.id),
354 "type": "comment",
355 "git_io": {"id": comment_git.id}}
356 all_comments.append(comment)
358 data_one_issue = {"id": issue.id,
359 "title": issue.title,
360 "number": issue.number,
361 "repo": issue.repository.name,
362 "assignees": issue.assignees,
363 "size_body": size_body,
364 "updated_at": issue.updated_at,
365 "created_at": issue.created_at,
366 "concat_body": concat_body,
367 "labels": from_label_to_map_color_name(issue.labels) + [{"name": "issue", "color": "9b0f3d"}],
368 "comments_data": all_comments,
369 "name": issue.repository.name + "/" + str(issue.number),
370 "type": "issue"}
371 data_load["issues"].append(data_one_issue)
372 count_issue = count_issue + 1
373 else:
374 print("WARNING This issue has some of the excluded label, we don't treat it ! " + str(issue.number) + " " + str(
375 issue.title) + " labels : " + str(issue.labels))
377 print("Time to load issues from github : " + str(datetime.now() - start))
379 if len(list_issues) == 1:
380 # .repository.name
381 print(" WE EXPECT type list[str] : " + str(type(list_issues)) + " type(list_issues[0]) " + str(type(list_issues[0])))
382 montcucq_manque_pas_par = list_issues[0].replace("/", "_")
383 reponame = f"{montcucq_manque_pas_par}_single_issue"
384 elif config_atmo_for_name != "default":
385 reponame = f"{reponame}_{config_atmo_for_name}_atmo_config"
386 elif limit > 0:
387 reponame = f"{reponame}_limit_{limit}_offset_{offset}"
389 if cache_data_file is None:
390 cache_data_file = f"data_{reponame}.json"
391 print("CACHEDATAFILE We save the data in cache file : " + str(cache_data_file))
392 print("CACHEDATAFILE TIME cached_at : " + str(data_load["cached_at"] if "cached_at" in data_load else "unknown") + " local timezone : " + str(data_load["cache_local_timezone"] if "cache_local_timezone" in data_load else "unknown"))
393 with open(cache_data_file, "w") as outfile:
394 json.dump(data_load, outfile, indent=4, default=str)
396 ret_atmo_data = parse_atmo_from_json_cache(data_load=data_load,
397 list_repos=list_repos, # override by default argument_parser
398 list_excluded_labels=list_excluded_labels,
399 verbose=verbose, only_count=only_count,
400 with_comment=with_comment,
401 check_issue_re=check_issue_re,
402 list_required_labels=list_required_labels,
403 last_number_days=last_number_days,
404 limit=limit,
405 parse_link_wiki=parse_link_wiki,
406 list_labels_to_parse = list_labels_to_parse,
407 list_issues=list_issues)
408 return ret_atmo_data, reponame # as repo_name_with_cond
410def compute_info_edit_atmo(content_this_comment, content, checkbox_name = "", title = ""):
411 list_find_begin_end = content_this_comment.split(content)
412 if len(list_find_begin_end) > 2:
413 print("WARNING : content_this_comment.split(content) has more than 2 elements, we take the first one ! we don't accept edition from ATMO then : " + str(checkbox_name) + " " + str(title) + " for this content : " + content)
414 return {}
415 else :
416 if "\n" not in list_find_begin_end[0]:
417 begin_char = 0
418 else :
419 len_until_before_last = 0
420 begin_char = len(list_find_begin_end[0]) - len(list_find_begin_end[0].split("\n")[-1])
422 # begin_char = len(list_find_begin_end[0])
423 end_char = len(list_find_begin_end[0]) + len(content)
424 len_content = end_char - begin_char
425 info_edit_atmo = {"begin_char": begin_char, "end_char": end_char, "len_content": len_content}
426 return info_edit_atmo
429def parse_checkbox_as_atmo_doc_func(comment_name, content_this_comment, c_updated_at,
430 c_label, list_excluded_labels = [],
431 list_required_labels = [],
432 list_labels_to_parse = []):
433 import re
434 checkbox_data = []
435 search_regex = r"\[([x| ])\](.+)\n"
436 content_this_comment = content_this_comment.replace("NBSP", " ")
437 content_this_comment = content_this_comment.replace("\xa0", " ")
438 content_this_comment = content_this_comment.replace("\u00a0", " ")
440 all_checkboxs = re.findall(search_regex, content_this_comment)
441 count_checkbox = 0
442 for checkbox in all_checkboxs:
443 if len(checkbox) != 2:
444 print("Unexpected format parsed checkboxes ")
446 done_or_not, content = checkbox
447 title = content
449 # or DDB328
450 if done_or_not == " ":
451 label_todone = "todo"
452 color_todone = "f46aff"
453 else :
454 label_todone = "done"
455 color_todone = "20fe32"
456 labels = [{"name" : label_todone, "color" : color_todone}]
457 from copy import deepcopy
458 labels = deepcopy(c_label) if c_label is not None else []
459 labels.insert(0, {"name" : label_todone, "color" : color_todone})
461 checkbox_name = comment_name + "/CB" + str(count_checkbox)
462 one_checkbox_as_atmo_data = {"name" : checkbox_name,
463 "title" : title,
464 "content" : content,
465 "labels" : labels,
466 "links" : [{"label":"hierarchical", "target":comment_name}],
467 "type" : "checkbox",
468 "updated_at" : c_updated_at}
470 info_edit_atmo = compute_info_edit_atmo(content_this_comment, content)
472 one_checkbox_as_atmo_data["info_edit_atmo"] = info_edit_atmo
474 if len(list_excluded_labels) > 0 and issue_atmo_has_label(one_checkbox_as_atmo_data, list_excluded_labels):
475 print("WARNING This checkbox has some of the excluded label, we don't treat it ! " + str(one_checkbox_as_atmo_data["name"]) + " " + str(
476 one_checkbox_as_atmo_data["title"]) + " labels : " + str(one_checkbox_as_atmo_data["labels"]))
477 continue
479 checkbox_data.append(one_checkbox_as_atmo_data)
480 count_checkbox += 1
482 return checkbox_data
484# f4 gag 3-8-25
485def parse_atmo_from_json_cache(data_load={},
486 list_repos=["fotonower/Velours", "fotonower/projects", "fotonower/Admin"],
487 # override by default argument_parser
488 list_excluded_labels=[], # "project:rubbia", "project:cod", "project:adenes",
489 #"project:broca", "project:circpack", "project:qualipapia",
490 # "project:mpa", "project:datou", "documentation", "bug",
491 # "it_maintenance", "reporting work", "business",
492 # "commercial", "recherche", "rh", "refacto"
493 verbose=False, only_count=False,
494 with_comment=False,
495 check_issue_re=False,
496 list_required_labels=[],
497 last_number_days=None,
498 previous_number_days=None,
499 limit=0,
500 parse_link_wiki=True,
501 parse_checkbox_as_atmo_doc = True,
502 config_parse_wip = {},
503 list_labels_to_parse = [],
504 list_issues = []):
505 import datetime
506 from datetime import timezone
507 start = datetime.datetime.now()
508 todaynn = datetime.datetime.now(timezone.utc)
509 today = datetime.datetime.now()
511 data = {"issues_data": []}
512 if "name" in data_load:
513 data["name"] = data_load["name"]
514 if "title" in data_load:
515 data["title"] = data_load["title"]
516 if "type" in data_load:
517 data["type"] = data_load["type"]
519 content_recently_updated = ""
520 map_wiki_issue_link = {} # For check_issue_re
522 count_issue = 0
523 for data_one_issue in (data_load["issues"] if "issues" in data_load else []):
525 if list_required_labels != [] and not issue_atmo_has_label(data_one_issue, list_required_labels) and not "issue" in list_required_labels:
526 print("WARNING This issue has not the required label, we don't treat it ! " + str(data_one_issue["number"]) + " " + str(
527 data_one_issue["title"]) + " labels : " + str(data_one_issue["labels"]))
528 continue
530 if list_excluded_labels != [] and issue_atmo_has_label(data_one_issue, list_excluded_labels):
531 print("WARNING This issue has some of the excluded label, we don't treat it ! " + str(data_one_issue["number"]) + " " + str(
532 data_one_issue["title"]) + " labels : " + str(data_one_issue["labels"]))
533 continue
535 updated_at_str = data_one_issue["updated_at"] if type(data_one_issue["updated_at"]) == str else data_one_issue[
536 "updated_at"].strftime("%Y-%m-%d %H:%M:%S")
538 content_recently_updated += "Repo : " + data_one_issue["repo"] + " issue : " + data_one_issue[
539 "title"] + " updated at " + " id : " + str(data_one_issue["id"]) + " Last Update : " + updated_at_str + "\n"
540 content_recently_updated += "\nbody :\n" + str(data_one_issue["concat_body"])
542 concat_body = data_one_issue["concat_body"] if "concat_body" in data_one_issue else ""
543 size_body = len(concat_body)
545 size_total = 0
546 if with_comment:
548 count_to_do_this_comment = len(
549 concat_body.split("[ ]")) - 1 if concat_body is not None else 0
550 count_done_this_comment = len(
551 concat_body.lower().split("[x]")) - 1 if concat_body is not None else 0
553 title = (data_one_issue["title"] if "title" in data_one_issue else "") + " " + \
554 ("TODO : " + str(count_to_do_this_comment) + "/" + str(
555 count_to_do_this_comment + count_done_this_comment) + " \n size : " + str(
556 len(concat_body))) if parse_checkbox_as_atmo_doc else data_one_issue["title"]
558 fake_first_comment = {"body": concat_body, "title" : title,
559 "name" : data_one_issue["name"] + "#DEFAULTFISTR",
560 "git_io": {"nb": data_one_issue["id"]},
561 "type" : "comment",
562 "labels" : [{"color" : "a2eeef", "name" : "comment"}],
563 "links" : [{"label":"hierarchical", "target":data_one_issue["name"]}]}
564 info_from_first_comment = parse_comment_with_config_parse(concat_body, context_node_parent=fake_first_comment,
565 list_labels_to_parse=list_labels_to_parse,
566 verbose=verbose)
567 fake_first_comment["checkboxs_data"] = info_from_first_comment
569 all_comments = data_one_issue["comments_data"] if "comments_data" in data_one_issue else []
571 for c in all_comments:
572 c["links"] = [{"label":"hierarchical", "target":data_one_issue["name"]}]
573 # Conversion git to json
574 comment_name = "comment_default_name_todo" if "name" not in c else c["name"]
575 c_body = c["body"]
577 title = c_body.strip()
579 if title.startswith("[up](#"):
580 title = title[6:] # remove [up](# and before
581 if "#" in title:
582 title = title.replace("##", "#").replace("##", "#").replace("##", "#") # remove double #
583 title = title.split("#")[1]
584 if "\n" in title:
585 # 100 c'est déjà trop long => je ne comprends pas pourquoi ce superbug arrive parfois et parfois non !
586 title = title.split("\n")[0].strip()[:400]
587 else :
588 title = title[:400] #c["name"]
589 else:
590 print("Missing info title !")
591 title = c["name"]
593 c["title"] = title # comment_name
594 c_updated_at = c["updated_at"] if "updated_at" in c else datetime.datetime.now()
595 # print(str(c.body))
596 c_updated_at_str = c_updated_at if type(c_updated_at) == str else c_updated_at.strftime("%Y-%m-%d %H:%M:%S")
597# c_updated_at = datetime.datetime.strptime(c_updated_at_str, "%Y-%m-%d %H:%M:%S")
599 if "+" in c_updated_at_str: # or "-" in updated_at_str.split(" ")[-1]:
600 c_updated_at = datetime.datetime.strptime(c_updated_at_str, "%Y-%m-%d %H:%M:%S%z")
601 else:
602 c_updated_at = datetime.datetime.strptime(c_updated_at_str, "%Y-%m-%d %H:%M:%S")
604# content_this_comment = "\nComment last modified at " + c_updated_at_str + "\n"
605 content_this_comment = ""
606 content_this_comment += c_body if c_body is not None else ""
607 if last_number_days:
608 if util_is_naive(c_updated_at):
609 nb_day_aux = todaynn - c_updated_at
610 else:
611 nb_day_aux = today - c_updated_at
613 if nb_day_aux.days <= last_number_days:
614 print(" This comment has been modified after aujourdhui minus last_number_days : " + str(
615 last_number_days))
616 content_recently_updated += content_this_comment
617 if parse_checkbox_as_atmo_doc:
618 print("TODO parse checkbox as atmo doc => première version done")
620 c["labels"] = [ d for d in data_one_issue["labels"] if d["name"] != "issue"] if "labels" in data_one_issue else []
621 c["labels"].insert(0, {"name": "comment", "color": "0075ca"})
623 # VR SANDBOX use marko to parse markdown
624 test = parse_comment_with_config_parse(content_this_comment, context_node_parent=c,
625 list_labels_to_parse = list_labels_to_parse,
626 verbose = verbose)
628 checkboxs_data = []
629# checkboxs_data = parse_checkbox_as_atmo_doc_func(comment_name, content_this_comment,
630# c_updated_at, c["labels"],
631# list_excluded_labels=list_excluded_labels)
632 checkboxs_data.extend(test)
634 count_to_do_this_comment = len(content_this_comment.split("[ ]")) - 1 if content_this_comment is not None else 0
635 count_done_this_comment = len(content_this_comment.lower().split("[x]")) - 1 if content_this_comment is not None else 0
636 c["title"] += " TODO : " + str(count_to_do_this_comment) + "/" + str(
637 count_to_do_this_comment + count_done_this_comment) + " \n size : " + str(
638 len(content_this_comment)) # + " \n updated at : " + c_updated_at_str
640 c["checkboxs_data"] = checkboxs_data
641 concat_body += content_this_comment
642 # print(c_body)
643 size_total += len(c_body) if c_body is not None else 0
645 all_comments.append(fake_first_comment)
647 else:
648 all_comments = []
650 print(" l502 updated_at_str : " + str(updated_at_str))
651 print("+" in updated_at_str)
652 if "+" in updated_at_str: # or "-" in updated_at_str.split(" ")[-1]:
653 updated_at_as_date = datetime.datetime.strptime(updated_at_str, "%Y-%m-%d %H:%M:%S%z")
654 else :
655 updated_at_as_date = datetime.datetime.strptime(updated_at_str, "%Y-%m-%d %H:%M:%S")
656 if util_is_naive(updated_at_as_date):
657 nb_day_since_activ = todaynn - updated_at_as_date
658 else:
659 nb_day_since_activ = today - updated_at_as_date
661 if last_number_days != None and nb_day_since_activ.days > last_number_days:
662 print("WARNING This issue has not been modified after aujourdhui minus last_number_days : " + str(
663 last_number_days) + ", we don't use it ! ")
664 continue
665 if previous_number_days != None and nb_day_since_activ.days < previous_number_days:
666 print("WARNING This issue has not been modified before aujourdhui minus previous_number_days : " + str(
667 previous_number_days) + ", we don't use it ! ")
668 continue
670 count_to_do = len(concat_body.split("[ ]")) - 1 if concat_body is not None else 0
671 count_done = len(concat_body.lower().split("[x]")) - 1 if concat_body is not None else 0
673 # if verbose:
674 # print(" count_to_do : " + str(count_to_do) + " count_done : " + str(count_done) + " size_body : " + str(size_body) + " size_total : " + str(size_total))
676 link_issue = "https://www.github.com/fotonower/" + data_one_issue["repo"] + "/issues/" + str(
677 data_one_issue["number"])
679 links = find_link_issue(data_one_issue["concat_body"])
681# VR TODO 5-8-25 : this makes stuff not very visible !
682 if "name" in data:
683 links.append({"target" : data["name"], "label" : "hierarchical"})
684 else:
685 print("WARNING data has no name ! : " + str(data.keys()))
687 res_check_issue = {}
688 if check_issue_re:
689 res_check_issue = check_issue_body(data_one_issue, all_comments, parse_link_wiki, link_issue)
691 if "wiki_links" in res_check_issue:
692 for issue_link in res_check_issue["wiki_links"]:
693 for wiki_link in res_check_issue["wiki_links"][issue_link]:
694 if wiki_link not in map_wiki_issue_link:
695 map_wiki_issue_link[wiki_link] = []
696 if issue_link not in map_wiki_issue_link[wiki_link]:
697 map_wiki_issue_link[wiki_link].append(issue_link)
698 if "missing_comment_links" in res_check_issue:
699 for missing_link in res_check_issue["missing_comment_links"]:
700 print("Missing link to comment : " + missing_link)
702 if "map_id_comment_missing_link_body_to_comment" in res_check_issue:
703 map_id_comment_missing_link_body_to_comment = res_check_issue["map_id_comment_missing_link_body_to_comment"]
704 for comment in all_comments:
705 c_id = str(comment["git_io"]["id"] if "git_io" in comment and "id" in comment["git_io"] else comment["name"].split("#")[-1])
706 if c_id in map_id_comment_missing_link_body_to_comment:
707 comment["label_missing_link_body_to_comment"] = map_id_comment_missing_link_body_to_comment[c_id]
708 if map_id_comment_missing_link_body_to_comment[c_id] == False:
709 comment["labels"].append({"name": "missing_link_body_to_comment", "color": "ff0000"})
710 else:
711 comment["label_missing_link_body_to_comment"] = "unknown"
713 draft_reason = res_check_issue[
714 "draft_reason"] if check_issue_re and "draft_reason" in res_check_issue else "OK\n"
715 complete_info = (draft_reason) + " nb_day_since_activ : " + str(
716 nb_day_since_activ.days)
717 title = data_one_issue["title"] + " " + str(count_to_do) + "/" + str(
718 count_to_do + count_done) + " \n size_total : " + str(
719 size_body + size_total) + " \n size_body : " + str(size_body) + " " + complete_info
721 if not only_count:
722 print("ISSUE:" + str(data_one_issue["title"]) + " TODO " + str(count_to_do) + " DONE " + str(
723 count_done) + " " \
724 + str(data_one_issue["number"]) + " " + ",".join(
725 map(lambda x: x["name"], data_one_issue["labels"])) \
726 + " " + link_issue + " body : " + str(size_body) + " and total comments : " + str(size_total))
728 # if (size_body + size_total < 16000):
729 # print("TO PARSE")
730 # print(concat_body)
731 # print("TO CONTINUE")
733 # VR TODO REFACTO 30-7-25 : on ne veut pas de git_issue mais plutot faire un extend ?
734 data_issue_form_comment_and_parsing = {
735 "count_done": count_done,
736 "count_to_do": count_to_do,
737 "size_total": size_total,
738 "link": link_issue,
739 "links" : links,
740 "title" : title}
741# "comments_data" : all_comments} # a priori dans le data_one_issue
742 data_one_issue.update(data_issue_form_comment_and_parsing)
743 data_one_issue["title"] = title
744 data["issues_data"].append(data_one_issue)
745 count_issue += 1
747 data["recently_updated"] = content_recently_updated
749 print("Number issues : " + str(count_issue))
751 if parse_link_wiki:
752 data["wiki_links"] = map_wiki_issue_link
753 if len(map_wiki_issue_link) > 0:
754 print("Found " + str(len(map_wiki_issue_link)) + " links to wiki pages in issues and comments")
755 for wiki_link in map_wiki_issue_link:
756 print(
757 "Wiki link : " + wiki_link.rstrip(")") + "/_edit\n# Lien Retour :\n<details>\n\n - " + "\n - ".join(
758 map_wiki_issue_link[wiki_link]) + "\n</details>")
760 ## Broken, maybe due to date added or something else
761 # with open("data.json", "r+") as outfile:
762 # outfile.seek(0)
763 # json.dump(data, outfile)
765 print("Time to parse issues : " + str(datetime.datetime.now() - start))
767 return data
769def append_node_and_edge(issue, map_node_to_node, list_nb_days_since_activ_for_darken,
770 map_node_to_father_hierarchical,
771 config_gag = {"list_link_display" : ["hierarchical", "allow"], "criteria_size" : "nb_days_activity"},
772 config_filter = {"required_labels" : [],
773 "excluded_labels" : [],
774 "list_issues" : []}):
776 max_size_title = 256 if "max_size_title" not in config_gag else config_gag["max_size_title"]
778 import datetime
779 from datetime import timezone
780 todaynn = datetime.datetime.now(timezone.utc)
781 today = datetime.datetime.now()
783 try :
784 expected_fields = set(ATMODataFTN.model_fields.keys())
785 submitted_fields = set(issue.keys())
786 unknown_fields = submitted_fields - expected_fields
788 from copy import deepcopy
789 issue_copy = deepcopy(issue)
791 # VR 5-8-25 : on a pas besoin de faire de deepcopy car la recursion portent en dernier sur les objets externes qui contiennet le reste dans des champs inattendu en attendant le refacto qui changera les data_issues en data
792 # mais on en a besoin pour updated_at
793 for d in unknown_fields: # and d != "updated_at":
794 del issue_copy[d]
796 one_atmo_data = ATMODataFTN(**issue_copy)
797 if len(one_atmo_data.title) > max_size_title:
798 one_atmo_data.title = one_atmo_data.title[:max_size_title]
799 print(f"WARNING : title is {max_size_title} chars, maybe truncated : " + one_atmo_data.title)
800 one_atmo_data.labels.append({"name" : "OBJECTION", "color" : "ff0000"})
801 #if ATMODataFTN.parse_raw(json.dumps(issue, default=str)): # and not validate_json
802# if ATMODataFTN.parse_raw(**issue): => ca ca marche pas
803 # print("Issue is valid")
804# one_atmo_data = ATMODataFTN(**json.loads(json.dumps(issue, default=str)))
806 # if unknown_fields:
807 # print(f"Log these unexpected fields: {unknown_fields}")
810 except Exception as e:
811 print(str(e))
812 print("Issue is NOT valid")
813 print(json.dumps(issue, indent=2, default=str)[:1000])
814 print(" Expected : ")
815 print(json.dumps(ATMODataFTN.model_json_schema(), indent=2))
816 print("ERROR DATA NOT TREATED")
817 print(str(e))
818 return
820 index = one_atmo_data.name
822 # print(issue["concat_body"])
824 links = one_atmo_data.links
826# if len(links) > 0:
827# print(str(links))
829 criteria_size = config_gag.get("criteria_size", "nb_days_activity")
830 if criteria_size not in ["none", "size_total", "nb_days_activity", "nb_todo", "nb_checkboxs", "nb_done"]:
831 print("Invalid criteria_size: " + criteria_size)
832 return
834 value_criteria_for_size = None
836 if criteria_size == "nb_days_activity": # and "updated_at" not in issue:
837 if "updated_at" in issue and type(issue["updated_at"]) == datetime.datetime:
838 updated_at_as_date = issue["updated_at"]
839 elif "updated_at" in issue and type(issue["updated_at"]) == str:
840 if "+" in issue["updated_at"]: # or "-" in updated_at_str.split(" ")[-1]:
841 updated_at_as_date = datetime.datetime.strptime(issue["updated_at"], "%Y-%m-%d %H:%M:%S%z")
842 else :
843 updated_at_as_date = datetime.datetime.strptime(issue["updated_at"], "%Y-%m-%d %H:%M:%S")
844 else:
845 if "updated_at" in issue:
846 print("Type not expected " + str(type(issue["updated_at"])) + " for updated_at : ")
847 else:
848 print("updated_at missing in issue ")
849 updated_at_as_date = None
851 if updated_at_as_date != None:
852 # print(" updated_at_as_date is not naive " + str(util_is_naive(updated_at_as_date)))
853 # print(" todaynn is not naive " + str(util_is_naive(todaynn)))
854 # print(" today is not naive " + str(util_is_naive(today)))
855 if util_is_naive(updated_at_as_date):
856 nb_day_since_activ = todaynn - updated_at_as_date
857 else:
858 nb_day_since_activ = today - updated_at_as_date
859# nb_day_since_activ_days = nb_day_since_activ.days
860 value_criteria_for_size = nb_day_since_activ.days
861 else:
862# nb_day_since_activ_days = -1
863 value_criteria_for_size = -1
864 elif criteria_size == "size_total":
865 if "size_total" in issue and type(issue["size_total"]) == int:
866 value_criteria_for_size = -issue["size_total"]
867 elif "body" in issue and type(issue["body"]) == str:
868 issue["size_total"] = len(issue["body"])
869 value_criteria_for_size = -len(issue["body"])
870 else:
871 issue["size_total"] = 1
872 value_criteria_for_size = -1
873 else:
874 print("Invalid criteria_size: " + criteria_size + " for now")
875 value_criteria_for_size = -1
877 title = one_atmo_data.title
879 # labels = one_atmo_data.labels
880 labels = issue["labels"] if "labels" in issue else []
881 # VR TODO 5-8-25 replace by one_atmo_data.labels
882 issue_copy["labels"] = one_atmo_data.labels
884 list_excluded_labels = config_filter["excluded_labels"] if "excluded_labels" in config_filter else []
885 list_required_labels = config_filter["required_labels"] if "required_labels" in config_filter else []
886 if len(list_excluded_labels) > 0 and issue_atmo_has_label(issue_copy, list_excluded_labels):
887 return None
888 if len(list_required_labels) > 0 and not issue_atmo_has_label(issue_copy, list_required_labels):
889 return None
891 if value_criteria_for_size != None:
892 list_nb_days_since_activ_for_darken.append(value_criteria_for_size)
893 else:
894 print("TO DEBUG")
896 # Search label hierarchical
897 for link in links:
898 if link == None:
899 continue
900 if "label" in link and link["label"].lower() == "hierarchical":
901 if "target" in link and link["target"] != "":
902 target = link["target"]
903 if index not in map_node_to_father_hierarchical:
904 map_node_to_father_hierarchical[index] = target
905 else :
906 if map_node_to_father_hierarchical[index].lower() != target.lower():
907 # print("OBJECTION WARNING : " + str(index) + " already has a target which is " + map_node_to_father_hierarchical[index] + " in hierarchical link, we don't update it !")
908 # print("We add an OBJECTION label to the node " + str(index))
909 one_atmo_data.labels.append({"name" : "OBJECTION", "color" : "ff0000"})
910 print("OBJECTION WARNING : " + str(index) + " already has a target which is " + map_node_to_father_hierarchical[index] + " and not " + target + " in hierarchical link, we don't update it !")
911 else:
912 print("dummy duplicate code")
914 if index in map_node_to_node:
915 if map_node_to_node[index] != {"links": links,
916 "title": title,
917 "labels": one_atmo_data.labels,
918 "assigned": len(one_atmo_data.assignee),
919 criteria_size: value_criteria_for_size,
920 "type": one_atmo_data.type}:
921 print("OBJECTION WARNING : index " + str(index) + " already in map_node_to_node, we don't overwrite it !")
922 else:
923 print("dummy duplicate code")
924# map_node_to_node[index]["labels"].append({"name" : "OBJECTION", "color" : "ff0000"})
925 else:
926 map_node_to_node[index] = {"links": links,
927 "title": title,
928 "labels": one_atmo_data.labels,
929 "assigned": len(one_atmo_data.assignee),
930 criteria_size: value_criteria_for_size,
931 "type": one_atmo_data.type} # one_atmo_data.type
932 return issue
934def build_gag_data_from_dict_rec(data, map_node_to_node,
935 list_nb_days_since_activ_for_darken,
936 map_node_to_father_hierarchical,
937 config_gag = {"list_link_display" : [], "criteria_size" : "nb_days_activity"}, # possible value for criteria_size : "none", "size_total", "nb_days_activity", "nb_todo", "nb_checkboxs", "nb_done"
938 config_filter = {"required_labels" : [],
939 "excluded_labels" : [],
940 "list_issues" : []}):
941 data_loop = []
942 used_data = [] # va contenir des dupliquer
943 if data == None:
944 print("Due to add generic parsing of md ")
945 return map_node_to_node, []
946 if "issues_data" in data:
947 data_loop = data["issues_data"]
948 elif "comments_data" in data :
949 data_loop = data["comments_data"]
950 elif "checkboxs_data" in data:
951 data_loop = data["checkboxs_data"]
952 else :
953 print("Pas utile celui-ci mais l'autre et on enleve celui la")
954# ret = append_node_and_edge(data, map_node_to_node, list_nb_days_since_activ_for_darken, map_node_to_father_hierarchical, config_gag=config_gag,
955# config_filter=config_filter)
956 # will avoid useless append to the tree, which by the way made warning for the unique hierarchical tree building
957 # Since this is an internal node, we manage the stat in this special usecase doomly
958 # VR TO TEST
959# if ret != None:
960# used_data.append(ret)
961# return map_node_to_node, [0]
962 print("Looping")
964 for d in data_loop:
965 ret = build_gag_data_from_dict_rec(d, map_node_to_node, list_nb_days_since_activ_for_darken,
966 map_node_to_father_hierarchical, config_gag=config_gag,
967 config_filter=config_filter)
968 if ret != None:
969 used_data.extend(ret)
971 ret = append_node_and_edge(data, map_node_to_node, list_nb_days_since_activ_for_darken, map_node_to_father_hierarchical, config_gag=config_gag,
972 config_filter=config_filter)
973 if ret != None:
974 # En fait ici il faudrait enlever l'internal loop si on veut applaitir la donner, mais je crois qu'il faut faire un deepcopy, ca ne me plait pas, faut mieux faire dans l'autre sens et tout copier sauf le data_loop
975 used_data.append(ret)
977 return used_data
981def util_check_data_inside_parent_node_tree(map_node_to_father_hierarchical, current_node_name, potential_parent_node_name):
982 count_height = 0
983 while current_node_name in map_node_to_father_hierarchical:
984 if count_height > 100:
985 print("There seems to be a cycle in the supposed hierarchical tree, so there could be an infinite loop too bad " + str(current_node_name))
986 break
987 count_height += 1
988 if current_node_name.lower() == potential_parent_node_name.lower():
989 return True
990 current_node_name = map_node_to_father_hierarchical[current_node_name]
992 # Ca c'est juste pour gérer le cas où on met tous les repo, mais en meme temps on s'en fout car on devrait juste pas mettre de condition (je crois)
993 if current_node_name.lower() == potential_parent_node_name.lower():
994 return True
995 return False
999# f16 gag 3-8-25
1000def build_gag_data_from_dict(data,
1001 config_gag = {"node": {"type" : ["issue", "comment"],
1002 "color": "blue"},
1003 "edge": {"type": ["https"]}},
1004 config_filter = {"required_labels" : [],
1005 "excluded_labels" : [],
1006 "list_issues" : []}):
1007 # works without concat_body
1009 # J'ai du remplacer
1010 # - \" par "
1011 # - les débuts de concat_body \\' par "
1012 # works with concat_body
1016 # if len(data["issues"]) > 0 and util_is_naive(data["issues"][0]["updated_at"]):
1017 # print("FORCE NOT NAIVE")
1018 # today = datetime.datetime.now(timezone.utc)
1020 list_nb_days_since_activ_for_darken = []
1022 map_node_to_node = {} # - [x] VR 3-8-25 renamed from map_issue_link_to_issues
1023 map_node_to_father_hierarchical = {}
1025 used_data = build_gag_data_from_dict_rec(data, map_node_to_node,
1026 list_nb_days_since_activ_for_darken,
1027 map_node_to_father_hierarchical,
1028 config_gag=config_gag,
1029 config_filter=config_filter)
1031# for issue in data["issues_data"]:
1032# append_node_and_edge(issue, map_node_to_node, list_nb_days_since_activ_for_darken)
1036 if "list_issues" in config_filter and len(config_filter["list_issues"]) > 0:
1037 list_to_delete = []
1038 for current_node_name in map_node_to_node:
1039 found_issue = False
1040 for potential_parent_node in config_filter["list_issues"]:
1041 found_issue = util_check_data_inside_parent_node_tree(map_node_to_father_hierarchical, current_node_name, potential_parent_node)
1042 if found_issue:
1043 break
1044 if not found_issue:
1045 list_to_delete.append(current_node_name)
1047 for current_node_name in list_to_delete:
1048 del map_node_to_node[current_node_name]
1049 if current_node_name in map_node_to_father_hierarchical:
1050 del map_node_to_father_hierarchical[current_node_name]
1052 # Je veux aussi supprimer dans l'autre sens
1053 list_link_hierarchy_to_delete = []
1054 for current_node_name in map_node_to_father_hierarchical:
1055 if map_node_to_father_hierarchical[current_node_name] in list_to_delete or map_node_to_father_hierarchical[current_node_name] not in map_node_to_father_hierarchical:
1056 list_link_hierarchy_to_delete.append(current_node_name)
1057 for current_node_name in list_link_hierarchy_to_delete:
1058 del map_node_to_father_hierarchical[current_node_name]
1060 list_distrib_nb_day_activ = stat_nb_day_activ(list_nb_days_since_activ_for_darken)
1062 return map_node_to_node, list_distrib_nb_day_activ, map_node_to_father_hierarchical, used_data
1066def init_user_count_activ():
1067 return {"nb_activ": 0, "nb_inactiv": 0, "list_inactiv": [], "size_body": 0, "size_total": 0, "count_to_do": 0,
1068 "count_done": 0}
1071def append_inactiv_issue(issue):
1072 # VR 14-02-2022 : if one wants more legend in issue, it should be here !
1073 return {"link": issue["link"], "nb_days_inactiv": issue["nb_days_inactiv"]}
1076def compute_display_user_activ(map_user_issues):
1077 map_user_issues_resume = {}
1078 for user in map_user_issues:
1079 nb_issue_activ = map_user_issues[user]["nb_activ"]
1080 nb_issue_inactiv = map_user_issues[user]["nb_inactiv"]
1081 prop = int(100 * nb_issue_activ / (nb_issue_activ + nb_issue_inactiv)) if (
1082 nb_issue_activ + nb_issue_inactiv > 0) else 0
1083 map_user_issues_resume[user] = {"user": user, "nb_issue_activ": nb_issue_activ,
1084 "nb_issue_inactiv": nb_issue_inactiv, "prop": prop,
1085 "count_to_do": map_user_issues[user]["count_to_do"],
1086 "count_done": map_user_issues[user]["count_done"],
1087 "size_body": map_user_issues[user]["size_body"],
1088 "size_total": map_user_issues[user]["size_total"]}
1090 return map_user_issues_resume
1093def display_user_activ_inactiv(list_resume_to_display_sorted):
1094 for data in list_resume_to_display_sorted:
1095 nb_issue_activ = data["nb_issue_activ"]
1096 nb_issue_inactiv = data["nb_issue_inactiv"]
1097 prop = data["prop"]
1098 user = data["user"]
1099 begin_message = "For user " + str(user) + " activ : " + str(nb_issue_activ) + "/" + str(
1100 nb_issue_activ + nb_issue_inactiv)
1101 complete_to_tab = max(0, 45 - len(begin_message))
1102 percent_achieve = str(round(100. * data["count_done"] / float(data["count_to_do"] + data["count_done"]))) if (
1103 data[
1104 "count_to_do"] +
1105 data[
1106 "count_done"]) > 0 else "0"
1107 print(begin_message + (" " * complete_to_tab) + str(prop) + "% " + " size_body : " + str(
1108 data["size_body"]) + " size_total : " + str(data["size_total"]) +
1109 " count_to_do : " + str(data["count_to_do"]) + " count_done : " + str(
1110 data["count_done"]) + " prop : " + str(percent_achieve) + "%")
1112 # Marseille
1113 from datetime import datetime, timedelta
1114 original_date = datetime(2023, 8, 26, 13, 0)
1115 nb_hour = int(float(percent_achieve) * 10.08)
1116 # print(" nb hour : " + str(nb_hour))
1117 # new_date = original_date + timedelta(hours = nb_hour) # Affichez la
1118 # print("La nouvelle date et heure est: ", new_date) # Utilisez la fonction add_hours(10)
1120 # ORga 10-23
1121 date1 = datetime(2024, 1, 21)
1122 date2 = datetime(2023, 6, 8)
1123 # Compter les jours entre les deux dates
1124 diff = date1 - date2
1125 total_hours = diff.days * 24.
1126 nb_hour = int(float(percent_achieve) * total_hours / 100)
1127 # Calculer la date et l'heure correspondant à la proportion p de temps passé
1128 new_date = date2 + timedelta(hours=nb_hour)
1130 print("La date et l'heure correspondant à la proportion", percent_achieve, "de temps passé est:", new_date)
1133def list_inactiv_issue_per_user(list_issues):
1134 for issue in list_issues:
1135 print(issue["link"] + " for " + str(issue["nb_days_inactiv"]) + " days")
1138def sum_for_user(map_user, map_issue, list_to_sum=["count_to_do", "count_done", "size_body", "size_total"]):
1139 for s in list_to_sum:
1140 if s in map_user and s in map_issue and str(map_issue[s]).isdigit() and type(map_user[s]) == type(1):
1141 map_user[s] += int(map_issue[s])
1143def group_by_user_from_versatile_data(data, nb_days_tolerance = 30):
1144 # Add data and list_users
1145 # list_users = ["unassigned"]
1147 list_issues = []
1149 import datetime
1150 from datetime import timezone
1151 todaynn = datetime.datetime.now(timezone.utc)
1152 today = datetime.datetime.now()
1154 datestr = today.strftime("%d%m%Y")
1156 map_user_issues = {"unassigned": init_user_count_activ()}
1158 nb_total_activ = 0
1159 nb_total_inactiv = 0
1161 map_user_issues["ALL"] = init_user_count_activ()
1162 map_user_issues["ALLu"] = init_user_count_activ()
1163 for issue in data["issues_data"]:
1164 if list_issues == [] or (issue["repo"] + "/" + str(issue["number"])) in list_issues:
1165 updated_at = issue["updated_at"]
1166 if util_is_naive(updated_at):
1167 inactiv_period = todaynn - updated_at
1168 else:
1169 inactiv_period = today - updated_at
1171 nb_days_inactiv = inactiv_period.days
1173 # c = a - b
1174 # seconds = c.total_seconds()
1176 activ = nb_days_inactiv < nb_days_tolerance
1178 issue["nb_days_inactiv"] = nb_days_inactiv
1180 if activ:
1181 map_user_issues["ALL"]["nb_activ"] += 1
1182 else:
1183 map_user_issues["ALL"]["nb_inactiv"] += 1
1184 # map_user_issues["ALL"]["list_inactiv"].append(append_inactiv_issue(issue))
1186 if len(issue["assignees"]) == 0:
1187 if activ:
1188 map_user_issues["unassigned"]["nb_activ"] += 1
1189 # nb_total_activ += 1
1190 else:
1191 map_user_issues["unassigned"]["nb_inactiv"] += 1
1192 # nb_total_inactiv += 1
1193 map_user_issues["unassigned"]["list_inactiv"].append(append_inactiv_issue(issue))
1195 map_user_issues["unassigned"]["size_body"] += issue["size_body"]
1196 sum_for_user(map_user_issues["unassigned"], issue)
1198 for user in issue["assignees"]:
1199 if user.login not in map_user_issues:
1200 map_user_issues[user.login] = init_user_count_activ()
1202 for user in issue["assignees"]:
1203 username = user.login
1204 sum_for_user(map_user_issues[username], issue)
1205 if activ:
1206 nb_total_activ += 1
1207 map_user_issues[username]["nb_activ"] += 1
1208 else:
1209 nb_total_inactiv += 1
1210 map_user_issues[username]["nb_inactiv"] += 1
1211 inactiv_link = append_inactiv_issue(issue)
1212 map_user_issues[username]["list_inactiv"].append(inactiv_link)
1214 sum_for_user(map_user_issues["ALLu"], issue)
1215 sum_for_user(map_user_issues["ALL"], issue)
1217 map_user_issues["ALLu"]["nb_activ"] = nb_total_activ
1218 map_user_issues["ALLu"]["nb_inactiv"] = nb_total_inactiv
1220 print("For tolerance nb of days of : " + str(nb_days_tolerance))
1221 resume_to_display = compute_display_user_activ(map_user_issues)
1222 list_resume_to_display_sorted = sorted(resume_to_display.values(), key=lambda x: -x["prop"])
1223 display_user_activ_inactiv(list_resume_to_display_sorted)
1225 for user in map_user_issues:
1226 print("<a name='" + str(datestr) + "_" + str(user) + "' ></a>")
1227 if len(map_user_issues[user]) > 0:
1228 print("For user : " + str(user))
1229 if "list_inactiv" in map_user_issues[user]:
1230 list_inactiv_issue_per_user(map_user_issues[user]["list_inactiv"])
1232 create_favoris(map_user_issues)
1234 return map_user_issues["ALL"]
1236def load_github_to_json_prepare_gag_and_group_by_user(list_repos, list_excluded_labels,
1237 list_issues,
1238 token, nb_days_tolerance=30, verbose=False,
1239 with_comment=False, check_issue_re=False,
1240 limit=0,
1241 offset=0,
1242 load_all=False,
1243 list_labels_to_parse=[],
1244 config_atmo_for_name = "default"):
1245 data, repo_name_with_cond = \
1246 load_github_to_json_with_type(list_repos, list_excluded_labels, token,
1247 verbose, with_comment=with_comment,
1248 check_issue_re=check_issue_re,
1249 limit=limit, offset=offset,
1250 list_issues=list_issues,
1251 load_all=load_all,
1252 list_labels_to_parse=list_labels_to_parse,
1253 config_atmo_for_name = config_atmo_for_name)
1255 data_by_user = group_by_user_from_versatile_data(data, nb_days_tolerance = nb_days_tolerance)
1257 return data_by_user, data, repo_name_with_cond
1260def find_link_issue(text):
1261 import re
1262 # Je pense qu'il n'y a jamais de www.github.com
1263 pattern = r'https://github\.com/fotonower/([\-\w]+)/issues/(\d+)'
1264 matches = re.findall(pattern, text)
1266 list_found = []
1268 for match in matches:
1269 repo, issue_number = match
1270 index = repo + "/" + str(issue_number)
1271 if index not in list_found:
1272 list_found.append(index)
1274 list_with_label = list(map(lambda x : {"label" : "http", "target" : x}, list_found))
1275 return list_with_label
1278def stat_nb_day_activ(list_nb_days_since_activ_for_darken):
1279 if len(list_nb_days_since_activ_for_darken) == 0:
1280 return [0]
1281 list_decil_nb_day_activ = []
1283 nb_decile = 8
1284 list_nb_days_since_activ_for_darken.append(-30)
1285 list_nb_days_since_activ_for_darken.sort(reverse=True)
1286 # for i in range(nb_decile - 1, 0, -1):
1287 for i in range(nb_decile):
1288 subscrit = int(i * float(len(list_nb_days_since_activ_for_darken)) / float(nb_decile))
1289 if subscrit < len(list_nb_days_since_activ_for_darken):
1290 list_decil_nb_day_activ.append(list_nb_days_since_activ_for_darken[subscrit])
1291 list_decil_nb_day_activ.append(min(list_nb_days_since_activ_for_darken)) # => ca pourrait tout faire foirer pour le type decil !
1293 return list_decil_nb_day_activ
1296def convert_activ_to_size(nb_day_activ, list_decil_nb_day_activ,
1297 type_scale = "decil", # decil,linear,log()not done)
1298 verbose = False):
1300 if type_scale == "decil":
1301 i = 0
1302 while i < len(list_decil_nb_day_activ) and nb_day_activ < list_decil_nb_day_activ[i]:
1303 i = i + 1
1305 i = i + 1 # pas sur enfin bon
1306 elif type_scale == "linear":
1307 min_l = min(list_decil_nb_day_activ) if len(list_decil_nb_day_activ) > 0 else 0
1308 max_l = max(list_decil_nb_day_activ) if len(list_decil_nb_day_activ) > 0 else 0
1310 surface_linear_square = len(list_decil_nb_day_activ) * len(list_decil_nb_day_activ) * (nb_day_activ * nb_day_activ - max_l * max_l) / (max_l * max_l - min_l * min_l) if max_l != min_l else 25
1312 import math
1313 if surface_linear_square > 0:
1314 val = int((math.sqrt(surface_linear_square)))
1315 else:
1316 val = int((math.sqrt(-surface_linear_square)))
1317 if val < 0:
1318 i = -val
1319 else:
1320 i = val
1322 else :
1323 print ("Unexpected type_scale " + str(type_scale) + " using decil")
1324 i = 2
1326 if verbose:
1327 print(" value_criteria_size : " + str(nb_day_activ) + " height : " + str(i))
1329 return i
1332# en fait non je court-ciruite, il faut juste rajouter un #
1333def hex_to_rgb(value):
1334 value = value.lstrip('#')
1335 lv = len(value)
1336 color_as_tuple = tuple(int(value[i:i + lv // 3], 16) for i in range(0, lv, lv // 3))
1337 color_as_csv = "".join(map(str, color_as_tuple))
1338 return "#" + value
1342def mise_en_forme_text_replace_some_voila_par_voila(input_text):
1343 max_length_line = 32
1345 list_part = input_text.split(" ")
1346# max_length_no_space = max(list_part, key=len)
1347 new_list_part = []
1348 for part in list_part:
1349 if len(part) > max_length_line:
1350 new_list_part.append(part[:max_length_line])
1351 new_list_part.append(part[max_length_line:])
1352 else:
1353 new_list_part.append(part)
1355 if len(input_text)/4 < max_length_line:
1356 max_length_line = int(len(input_text)/4)
1358 new_text_with_new_line = ""
1359 nb_char_from_start_line = 0
1360 for part in new_list_part:
1361 if nb_char_from_start_line + len(part) > max_length_line:
1362 new_text_with_new_line += "\n"
1363 nb_char_from_start_line = 0
1364 else :
1365 new_text_with_new_line += " "
1366 new_text_with_new_line += part
1367 nb_char_from_start_line += len(part)
1369# while max_length_no_space > 64:
1370# input_text = input_text[:64] + " " + input_text[64:]
1372 return new_text_with_new_line
1376def mise_en_forme_text(input_text,
1377 format_gag_mise_en_forme_text_tooltip = "default"): # VR 20-8-25 TO DOC en CDC : one_short_line
1378 return mise_en_forme_text_replace_some_voila_par_voila(input_text[:256])
1380 if len(input_text.split("\n")) > 3:
1381 print("VR 20-8-25 : ununderstood case !")
1382 if format_gag_mise_en_forme_text_tooltip == "one_short_line":
1383 return input_text.split("\n")[0]
1384 return input_text
1385 input_text_bak = input_text
1386 # Une ligne de deux caractère fais la largeur egale à la hauteur d'une ligne : on egalise les deux
1387 # nb_line * size_split = nb_char
1388 # Egaliser les hauteur : nb_line = size_split / 2
1389 displayed_text_with_endline = ""
1390 import math
1391 size_split = int(math.sqrt(2 * len(input_text))) + 1
1392 while len(input_text) > 0:
1393 displayed_text_with_endline += input_text[:size_split] + "\n"
1394 input_text = input_text[size_split:]
1396 # VR 20-8-25 : for debugging purposse to avoid wrong counting of line number in dot .gv
1397 if format_gag_mise_en_forme_text_tooltip == "one_short_line":
1398 displayed_text_with_endline = input_text_bak[:size_split].split("\n")[0]
1400 return displayed_text_with_endline
1402def highest(issue_link, map_issue_link_to_issues):
1403 if issue_link.startswith("http"):
1404 return False
1405 if issue_link not in map_issue_link_to_issues:
1406 print("WARNING not really expected : issue link " + str(issue_link) + " not in map_issue_link_to_issues in highest function")
1407 return True
1408 if "links" not in map_issue_link_to_issues[issue_link]:
1409 return True
1410 for link in map_issue_link_to_issues[issue_link]["links"]:
1411 if link["label"].lower() == "hierarchical" and link["target"] in map_issue_link_to_issues:
1412 return False
1413 return True
1415def make_gag(map_issue_link_to_issues,
1416 list_distrib_nb_day_activ,
1417 out_path='doctest-output/Fotonower_Issues',
1418 with_title_in_node=True,
1419 with_linked_issue=False,
1420 format_gag = {"map_label_to_format_for_edges" :
1421 {"MOYEN" : {"shape" : "oval", "fillcolor" : "peachpuff"},
1422 "OBJECTIF" : {"shape" : "hexagon", "fillcolor" : "yellow"},
1423 "TACHE" : {"shape" : "rectangle", "fillcolor" : "lightblue"}}
1424 ,"format": "svg", "layout": "fdp", "overlap": "prism",
1425 "filter_links" : {"required_labels": [],
1426 "excluded_labels": [],
1427 "list_issues": []},
1428 "criteria_size": "nb_days_activity"},
1429 parsed_data_used_for_gag = None,
1430 title_di_graph = "Fotonower Issues Graph",
1431 link_title = "http://atmo.fotonower.com:8091",
1432 generate_dynamic_css = False,
1433 dynamic_css_config = None):
1434 format_gag_mise_en_forme_text_tooltip = format_gag["format_text_tooltip"] if "format_text_tooltip" in format_gag else "default"
1435 import os
1436 out_folder = os.path.dirname(out_path)
1437 out_file = os.path.basename(out_path)
1439 print("Now build the dot graph and call dot")
1440 import graphviz
1442 import os
1443 os.system("pwd")
1445 # , "URL": link_title} => disturb a lot the rendering and action in the interface
1447 map_label_to_format_for_edges = format_gag["map_label_to_format_for_edges"] if "map_label_to_format_for_edges" in format_gag else {}
1448 edge_format = format_gag["edge_format"] if "edge_format" in format_gag else {}
1450 map_cluster_to_title = {}
1452 list_clusters = []
1453 list_title_to_dels = []
1454 for issue in map_issue_link_to_issues:
1455 if "labels" in map_issue_link_to_issues[issue] and len(map_issue_link_to_issues[issue]["labels"]) > 0:
1456 for label in map_issue_link_to_issues[issue]["labels"]:
1457 if "cluster" == label["name"].lower() and issue not in list_clusters:
1458 list_clusters.append(issue)
1459 break
1460 # VR TODO peut-etre rajouter une option dans config gag pour activer ou non cette fonctionnalité
1461 if "use_title_node" in format_gag and format_gag["use_title_node"]:
1462 if "title" == label["name"].lower():
1463 found_cluster = False
1464 if "links" in map_issue_link_to_issues[issue]:
1465 for link in map_issue_link_to_issues[issue]["links"]:
1466 if link["label"].lower() == "cluster":
1467 found_cluster = True
1468 map_cluster_to_title[link["target"]] = map_issue_link_to_issues[issue]["title"].strip().strip("TITLE").strip()
1469 list_title_to_dels.append(issue)
1470 break
1471 if not found_cluster:
1472 # VR TODO il aurait fallu chercher uniquement le cluster hierarchical, par ailleurs ils sont un peu doublonner par le fait qu'il y a un lien aussi, ca se mord
1473 #if highest(link["target"], map_issue_link_to_issues):
1474 title_di_graph = map_issue_link_to_issues[issue]["title"].strip().strip("TITLE").strip()
1475 list_title_to_dels.append(issue)
1476 #else:
1477 # print("WARNING : title label found without cluster link for issue " + str(issue))
1478# map_cluster_to_title[issue] = map_issue_link_to_issues[issue]["title"]
1480 if "delete_title_node" in format_gag and format_gag["delete_title_node"]:
1481 for t in list_title_to_dels:
1482 if t in map_issue_link_to_issues:
1483 del map_issue_link_to_issues[t]
1485 # VR TODO DOC indicate that it works only for one layer clsuter
1486 map_issue_to_cluster = {}
1487 for node in list_clusters:
1488 del map_issue_link_to_issues[node]
1489 for issue in map_issue_link_to_issues:
1490 for link in map_issue_link_to_issues[issue]["links"]:
1491 if link["target"] in list_clusters:
1492 map_issue_to_cluster[issue] = link["target"]
1494 map_cluster_to_subgraph_object = {}
1496 label_color_cluster = "peachpuff"
1497 if "cluster" in map_label_to_format_for_edges :
1498 label_color_cluster = map_label_to_format_for_edges["cluster"]["fillcolor"] if "fillcolor" in \
1499 map_label_to_format_for_edges["cluster"] else label_color_cluster
1501 dot = graphviz.Digraph(out_path,
1502 graph_attr = {"label": title_di_graph,
1503 "fontsize": "100px",
1504 "labelloc": "t",
1505 "fontname": "helvetica",
1506 "style": "bold"},
1507 comment='Some Data on ' + os.path.basename(out_path)) #, title="Some Data on " + os.path.basename(out_path))
1509 # dot.subgraph(name='child', node_attr={'shape': 'box'})
1510 for cluster in list_clusters:
1511 label_cluster_display = cluster
1512 if "cluster" in label_cluster_display:
1513 label_cluster_display = label_cluster_display.split("cluster")[-1].strip("_")
1514 title_cluster = cluster
1515 graph_attr={"fillcolor": label_color_cluster, "style":"filled,rounded,dotted",
1516 "label" : label_cluster_display, "fontsize":"39px"}
1518 if cluster in map_cluster_to_title:
1519 title_cluster = map_cluster_to_title[cluster]
1520 graph_attr["label"] = title_cluster
1521 map_cluster_to_subgraph_object[cluster] = graphviz.Digraph("cluster_" + title_cluster,
1522 comment=title_cluster,
1523 # title=cluster,
1524 graph_attr=graph_attr)
1525# dot.subgraph(map_cluster_to_subgraph_object[cluster])
1526# map_cluster_to_subgraph_object[cluster] = dot.subgraph(name=cluster)
1527 # dot.subgraph(name=cluster, comment=cluster, _attributes={"class": "cluster"})
1528# map_issue_link_to_issues
1530 map_node_gaglink = {}
1532 for i in map_issue_link_to_issues:
1533 pair_repo_number = i.split("/")
1534 issue = map_issue_link_to_issues[i]
1535 repo = pair_repo_number[0]
1536 nb = pair_repo_number[1] if len(pair_repo_number) > 1 else "-1"
1537 label_color = hex_to_rgb(issue["labels"][0]["color"]) if len(issue["labels"]) > 0 else "blue"
1539 # new_node.attr(fillcolor="blue", style="filled", color=label_color)
1540 # , fillcolor="blue", style="filled" OU color='blue',style='filled')
1542 shape = 'square' if issue["assigned"] == 0 else 'oval'
1543 image_path = None
1544 style_node = None
1545 fontcolor = "black"
1546 for label in issue["labels"]:
1547 if label["name"] in map_label_to_format_for_edges:
1548 shape = map_label_to_format_for_edges[label["name"]]["shape"] if "shape" in map_label_to_format_for_edges[label["name"]] else shape
1549 label_color = map_label_to_format_for_edges[label["name"]]["fillcolor"] if "fillcolor" in map_label_to_format_for_edges[label["name"]] else label_color
1550 image_path = map_label_to_format_for_edges[label["name"]]["image"] if "image" in map_label_to_format_for_edges[label["name"]] else None
1551 style_node = map_label_to_format_for_edges[label["name"]]["style"] if "style" in map_label_to_format_for_edges[label["name"]] else None
1552 fontcolor = map_label_to_format_for_edges[label["name"]]["fontcolor"] if "fontcolor" in map_label_to_format_for_edges[label["name"]] else fontcolor
1554 criteria_size = format_gag["criteria_size"] if "criteria_size" in format_gag else "nb_days_activity"
1555 if criteria_size not in ["none", "size_total", "nb_days_activity"]:
1556 print("Invalid criteria_size: " + criteria_size + " using nb_days_activity")
1557 continue
1558# list_nb_days_since_activ_for_darken.append(criteria_size)
1560 if criteria_size == "nb_days_activity":
1561 value_criteria_for_size = issue["nb_days_activity"] if "nb_days_activity" in issue else 30
1562 type_scale = "decil"
1563 elif criteria_size == "size_total":
1564 value_criteria_for_size = issue["size_total"]
1565 type_scale = "linear"
1566 else:
1567 continue
1569 if "type_scale" in format_gag:
1570 type_scale = format_gag["type_scale"]
1572 fontsize = None
1573 height = None
1574# fontsize = convert_activ_to_size(issue[criteria_size], list_distrib_nb_day_activ)
1575 height = convert_activ_to_size(issue[criteria_size] if criteria_size in issue else 30,
1576 list_distrib_nb_day_activ, type_scale=type_scale)
1578 # new_node.attr(shape='square')
1579 # shape='oval'
1581 # dot.node(i, i
1582 # TODO document nice color : https://graphviz.org/doc/info/colors.html#svg
1584 # for i in map_issue_link_to_issues:
1585 displayed_text = i if not with_title_in_node else map_issue_link_to_issues[i]["title"]
1587 URL = map_issue_link_to_issues[i]["link"] if "link" in map_issue_link_to_issues[i] else "https://www.github.com/fotonower/" + repo + "/issues/" + nb
1588 for link in map_issue_link_to_issues[i]["links"]:
1589 if link["label"].upper() == "GAGLINK":
1590 map_node_gaglink[i] = link["target"]
1591 if "http" in link["target"]:
1592 URL = link["target"]
1594 type_val = map_issue_link_to_issues[i]["type"]
1595 labels_name = " ".join(map(lambda x: x["name"], map_issue_link_to_issues[i]["labels"]))
1596 # print(" labels_name : " + str(labels_name))
1598 displayed_text = mise_en_forme_text(displayed_text, format_gag_mise_en_forme_text_tooltip)
1599 tooltip_text = map_issue_link_to_issues[i]["title"]
1600 if "\\" in tooltip_text:
1601 print("REPLACE \\ by \ in tooltip_text")
1602 tooltip_text = tooltip_text.replace("\\", "__")
1603 if i in map_issue_to_cluster:
1604 cluster = map_issue_to_cluster[i]
1605 if cluster in map_cluster_to_subgraph_object:
1606 used_cluster_or_dot = map_cluster_to_subgraph_object[cluster]
1607 else:
1608 print("Internal error treated as warning ")
1609 used_cluster_or_dot = dot
1611# Est-ce du code utile ? VR 15-8-25
1612 # map_cluster_to_subgraph_object[cluster].node(i, displayed_text, fontsize=str(fontsize) + "pt",
1613 # tooltip=map_issue_link_to_issues[i]["title"],
1614 # href=URL, fillcolor=label_color,
1615 # style="filled", shape=shape,
1616 # _attributes={"class": type_val + " " + labels_name}))
1617 else :
1618 used_cluster_or_dot = dot
1620 import json
1621 list_args = {"tooltip":tooltip_text,
1622 "href":URL, "fillcolor":label_color,
1623 "style":"filled", "shape":shape,
1624 "_attributes":{"class": type_val + " " + labels_name,
1625 "data-json": json.dumps(map_issue_link_to_issues[i])}}
1626 if image_path != None:
1627 list_args["image"] = image_path
1628# list_args["image"] = "http://localhost:4993/static/" + image_path
1629# list_args["width"] = "20px"
1630# list_args["height"] = "20px"
1631# list_args["fixedsize"] = "true"
1632 print("We should verify this, on dirait que le shape is none quand meme !")
1633# list_args["shape"] = "none"
1634# del list_args["shape"]
1635 if fontcolor != None:
1636 list_args["fontcolor"] = fontcolor
1637 if style_node != None:
1638 list_args["style"] = style_node
1639 if fontsize != None:
1640 list_args["fontsize"] = str(fontsize) + "pt"
1641 if height != None:
1642 list_args["height"] = str(height) + "px"
1643 used_cluster_or_dot.node(i, displayed_text, **list_args)
1644 # , class=type_val) #
1645# used_cluster_or_dot.node(i, displayed_text, fontsize=str(fontsize) + "pt", tooltip=tooltip_text,
1646# href=URL, fillcolor=label_color,
1647# style="filled", shape=shape, _attributes={"class": type_val + " " + labels_name}) # , class=type_val) #
1649 filter_links = format_gag["filter_links"] if "filter_links" in format_gag else {"required_labels": [], "excluded_labels": [], "list_issues": []}
1650 for i in map_issue_link_to_issues:
1651 for link in map_issue_link_to_issues[i]["links"]:
1652 def filter_link(link, filter_links):
1653 if "label" in link and link["label"].lower() in filter_links["excluded_labels"]:
1654 return False
1655 if len(filter_links["required_labels"]) > 0 and "label" in link and link["label"].lower() in filter_links["required_labels"]:
1656 return True
1657# if "target" in link and link["target"] in filter_links["list_issues"]:
1658# return True
1659 return True
1660 if not filter_link(link, filter_links):
1661 continue
1662 if link == None:
1663 print("Unexpected format link : " + str(link) + " ignoring this link (none) due to parsing generic btw !")
1664 continue
1665 if "target" in link and "label" in link:
1666 label = link["label"]
1667 j = link["target"]
1668 else :
1669 print("Unexpected format link : " + str(link) + " ignoring this link !")
1670 continue
1671 # dot.edges([ij])
1672 # if j in map_issue_link_to_issues: # Keep the open or extracted graph
1673 if j not in map_issue_link_to_issues:
1674 if with_linked_issue:
1675 dot.node(j, "X:" + j, shape='circle',
1676 fontsize="6pt") # on pourrait le rajouter dans map_issue_link_to_issues
1677 if with_linked_issue or j in map_issue_link_to_issues:
1678 attributes_edge_init = {"class": label}
1679 attributes_edge = attributes_edge_init.copy()
1680 if label in edge_format:
1681 for attr_edge in edge_format[label]:
1682 if type(edge_format[label][attr_edge]) == str:
1683 attributes_edge[attr_edge] = edge_format[label][attr_edge]
1684 else:
1685 attributes_edge[attr_edge] = str(edge_format[label][attr_edge])
1686 # VR TODO : potentiellement il faudrait mieux que ce soit le js qui affiche ou non, mais cela me semble délicat !
1687 dot.edge(i, j, _attributes=attributes_edge) #, _attributes={"class": type_val}) # , constraint='false')
1689 # for node in dot.nodes:
1690 # node.attr('onmouseover', f'this.style.fill = "yellow"; this.style.stroke = "orange";')
1692 # print(dot.source)
1694 for cluster in list_clusters:
1695 dot.subgraph(map_cluster_to_subgraph_object[cluster])
1697 # doctest_mark_exe()
1699 # dot.attr(rankdir='LR', size='8,5')
1700 dot.attr('node', shape='oval', fontname='Helvetica')
1701 dot.attr('edge', fontsize='12')
1702 dot.attr('graph', splines='true', overlap='false')
1704 # graph [splines=true overlap=false];
1706 # neato, fdp (needs overlap=prism ?) , sfdp
1707 dot.attr(layout='fdp')
1708 dot.attr(overlap='prism')
1710 if parsed_data_used_for_gag != None:
1711 import json
1712 with open(out_path + ".json", 'w') as f:
1713 json.dump(parsed_data_used_for_gag, f)
1715 try:
1716 # VR TODO 31-8-25 REFACTO remove out_folder in dot.render since already in out_path, check impact on job suividev
1717 dot.render(directory=out_folder, format='svg') # .replace('\\', '/')
1718 # 'doctest-output/round-table.gv.pdf'
1719 dot.render(directory=out_folder, format='pdf')
1720 except Exception as e:
1721 print(" Exception in dot render : " + str(e))
1723 print("Doc rendered ! Exiting !")
1725 # Generate dynamic CSS if requested
1726 if generate_dynamic_css:
1727 svg_file_path = os.path.join(out_folder, out_file + '.svg')
1728 if os.path.exists(svg_file_path):
1729 if dynamic_css_config is None:
1730 dynamic_css_config = {
1731 'initial_delay': 1.0,
1732 'animation_duration': 10.0,
1733 'randomize_order': True
1734 }
1735 generate_dynamic_css_for_gag(
1736 svg_file_path,
1737 css_path=svg_file_path + '.dynamic.css',
1738 initial_delay=dynamic_css_config.get('initial_delay', 1.0),
1739 animation_duration=dynamic_css_config.get('animation_duration', 10.0),
1740 randomize_order=dynamic_css_config.get('randomize_order', True)
1741 )
1743 return map_node_gaglink
1745def generate_dynamic_css_for_gag(svg_path, css_path=None,
1746 initial_delay=1.0,
1747 animation_duration=10.0,
1748 randomize_order=True):
1749 """
1750 Generate CSS file for progressive SVG animation and inject it into the SVG
1752 Args:
1753 svg_path: Path to the SVG file
1754 css_path: Path for output CSS (default: svg_path + '.dynamic.css')
1755 initial_delay: Delay before animation starts (seconds)
1756 animation_duration: Total duration of animation (seconds)
1757 randomize_order: Whether to randomize node/cluster display order
1758 """
1759 import random
1760 import os
1761 import re
1763 if css_path is None:
1764 css_path = svg_path + '.dynamic.css'
1766 # Parse SVG to extract nodes, edges, clusters
1767 with open(svg_path, 'r', encoding='utf-8') as f:
1768 svg_content = f.read()
1770 # Extract all node IDs
1771 node_pattern = r'<g id="(node\d+)" class="node'
1772 edge_pattern = r'<g id="(edge\d+)" class="edge'
1773 cluster_pattern = r'<g id="(clust\d+)" class="cluster'
1775 nodes = re.findall(node_pattern, svg_content)
1776 edges = re.findall(edge_pattern, svg_content)
1777 clusters = re.findall(cluster_pattern, svg_content)
1779 # Build edge dependency map (edge -> [source_node, target_node])
1780 edge_dependencies = {}
1781 edge_full_pattern = r'<g id="(edge\d+)"[^>]*>[\s\S]*?<title>([^<]+)</title>'
1782 for edge_id, edge_title in re.findall(edge_full_pattern, svg_content):
1783 if '-->' in edge_title or '->' in edge_title:
1784 # Parse edge title to get source and target
1785 edge_title = edge_title.replace('-', '-').replace('>', '>')
1786 if '-->' in edge_title:
1787 parts = edge_title.split('-->')
1788 if len(parts) == 2:
1789 source = parts[0].strip()
1790 target = parts[1].strip()
1792 # Find corresponding node IDs
1793 source_node_pattern = r'<g id="(node\d+)"[^>]*>[\s\S]*?<title>' + re.escape(source) + r'</title>'
1794 target_node_pattern = r'<g id="(node\d+)"[^>]*>[\s\S]*?<title>' + re.escape(target) + r'</title>'
1796 source_matches = re.findall(source_node_pattern, svg_content)
1797 target_matches = re.findall(target_node_pattern, svg_content)
1799 if source_matches and target_matches:
1800 edge_dependencies[edge_id] = {
1801 'source': source_matches[0],
1802 'target': target_matches[0]
1803 }
1805 # Create randomized order for nodes and clusters
1806 all_elements = list(clusters) + list(nodes)
1807 if randomize_order:
1808 random.shuffle(all_elements)
1810 # Calculate delays for each element
1811 element_count = len(all_elements)
1812 if element_count > 0:
1813 delay_increment = animation_duration / element_count
1814 else:
1815 delay_increment = 0
1817 element_delays = {}
1818 for idx, elem_id in enumerate(all_elements):
1819 element_delays[elem_id] = initial_delay + (idx * delay_increment)
1821 # Generate CSS
1822 css_lines = []
1823 css_lines.append("/* CSS dynamique pour animation progressive du graphique */")
1824 css_lines.append("/* Généré automatiquement par generate_dynamic_css_for_gag */")
1825 css_lines.append("")
1826 css_lines.append("/* Tous les éléments commencent invisibles */")
1827 css_lines.append("svg g.node, svg g.cluster, svg g.edge {")
1828 css_lines.append(" opacity: 0;")
1829 css_lines.append("}")
1830 css_lines.append("")
1832 # Animation for each cluster
1833 for cluster_id in clusters:
1834 if cluster_id in element_delays:
1835 delay = element_delays[cluster_id]
1836 css_lines.append(f"/* Cluster {cluster_id} */")
1837 css_lines.append(f"svg g#{cluster_id} {{")
1838 css_lines.append(f" animation: fadeIn 0.5s ease-in {delay:.2f}s forwards;")
1839 css_lines.append("}")
1840 css_lines.append("")
1842 # Animation for each node
1843 for node_id in nodes:
1844 if node_id in element_delays:
1845 delay = element_delays[node_id]
1846 css_lines.append(f"/* Node {node_id} */")
1847 css_lines.append(f"svg g#{node_id} {{")
1848 css_lines.append(f" animation: fadeIn 0.5s ease-in {delay:.2f}s forwards;")
1849 css_lines.append("}")
1851 # Text within node appears at same time
1852 css_lines.append(f"svg g#{node_id} text {{")
1853 css_lines.append(f" animation: fadeIn 0.5s ease-in {delay:.2f}s forwards;")
1854 css_lines.append("}")
1855 css_lines.append("")
1857 # Animation for edges (appear after both connected nodes)
1858 for edge_id in edges:
1859 if edge_id in edge_dependencies:
1860 dep = edge_dependencies[edge_id]
1861 source_delay = element_delays.get(dep['source'], 0)
1862 target_delay = element_delays.get(dep['target'], 0)
1863 edge_delay = max(source_delay, target_delay) + 0.5 # 0.5s after last node
1864 else:
1865 # If we can't determine dependencies, show at end
1866 edge_delay = initial_delay + animation_duration
1868 css_lines.append(f"/* Edge {edge_id} */")
1869 css_lines.append(f"svg g#{edge_id} {{")
1870 css_lines.append(f" animation: fadeIn 0.3s ease-in {edge_delay:.2f}s forwards;")
1871 css_lines.append("}")
1872 css_lines.append("")
1874 # Add keyframe animation
1875 css_lines.append("/* Keyframe pour l'animation de fade-in */")
1876 css_lines.append("@keyframes fadeIn {")
1877 css_lines.append(" from {")
1878 css_lines.append(" opacity: 0;")
1879 css_lines.append(" }")
1880 css_lines.append(" to {")
1881 css_lines.append(" opacity: 1;")
1882 css_lines.append(" }")
1883 css_lines.append("}")
1885 css_content = '\n'.join(css_lines)
1887 # Write CSS file (for reference/debugging)
1888 with open(css_path, 'w', encoding='utf-8') as f:
1889 f.write(css_content)
1891 # NOUVEAU : Injecter le CSS directement dans le SVG
1892 # Find the first <svg> tag and inject <style> after it
1893 svg_tag_match = re.search(r'(<svg[^>]*>)', svg_content)
1894 if svg_tag_match:
1895 svg_tag_end = svg_tag_match.end()
1896 style_tag = f'\n<style type="text/css">\n<![CDATA[\n{css_content}\n]]>\n</style>\n'
1898 # Insert the style tag after the opening <svg> tag
1899 svg_content_with_css = svg_content[:svg_tag_end] + style_tag + svg_content[svg_tag_end:]
1901 # Write the modified SVG back to file
1902 with open(svg_path, 'w', encoding='utf-8') as f:
1903 f.write(svg_content_with_css)
1905 print(f"CSS injecté dans le SVG: {svg_path}")
1906 else:
1907 print("ATTENTION: Impossible de trouver la balise <svg> pour injecter le CSS")
1909 print(f"Dynamic CSS generated: {css_path}")
1910 print(f" - {len(clusters)} clusters")
1911 print(f" - {len(nodes)} nodes")
1912 print(f" - {len(edges)} edges")
1913 print(f" - Animation: {initial_delay}s delay + {animation_duration}s duration")
1915 return css_path
1917# f8 gag 3-8-25 graph_git_just_draw_from_data_not_load_DEPRECATED
1920def git_issue_to_retrieval_json(list_repos, list_excluded_labels, github_token,
1921 list_required_labels=[],
1922 verbose=False,
1923 with_comment=False,
1924 load_cache=False,
1925 with_closed_issue=False,
1926 out_path='doctest-output/Fotonower_Issues.json',
1927 load_all=False,
1928 last_number_days=None,
1929 limit=0,
1930 check_issue_re=False,
1931 only_recent_update=False): # pragma no cover safia git import => donc sans doute à renommer
1932 print("TO TEST")
1934 import os
1935 out_folder = os.path.dirname(out_path)
1936 out_file = os.path.basename(out_path)
1938 recently_updated_info = ""
1940 import json
1941 if not load_cache:
1942 data, repo_name_with_cond = load_github_to_json_with_type(list_repos, list_excluded_labels, github_token,
1943 verbose, with_comment=with_comment, check_issue_re=check_issue_re,
1944 load_all=load_all,
1945 list_required_labels=list_required_labels,
1946 last_number_days=last_number_days,
1947 limit=limit)
1949 data_for_retrieval = []
1951 if "recently_updated" in data:
1952 print("Recent update : " + str(data["recently_updated"]))
1954 aujourdhui = datetime.datetime.now()
1955 recently_updated_info = "recently_updated_days_" + str(last_number_days) + "_before_" + str(
1956 aujourdhui.strftime("%Y_%m_%d_%H_%M")) + "_" + "_".join(list_repos).replace("/", "_") + "_voila"
1957 text = data["recently_updated"]
1959 data_for_retrieval.append(
1960 {"id": recently_updated_info, "text": text, "url": "useless"}) # , "created_at":created_at})
1962 if not only_recent_update:
1963 for issue in data["issues"]:
1964 print(".")
1965 id = issue["repo"] + "/" + str(issue["number"])
1966 text = issue["title"] + issue["concat_body"]
1967 url = issue["link"]
1968 # created_at = issue["updated_at"]
1970 data_for_retrieval.append({"id": id, "text": text, "url": url}) # , "created_at":created_at})
1972 # source = item.get("source", None)
1973 # source_id = item.get("source_id", None)
1974 # created_at = item.get("created_at", None)
1975 # author = item.get("author", None)
1977 # "title":issue.title,
1978 # "labels":from_label_to_map(issue.labels),
1979 # "assignees":issue.assignees,
1980 # "count_done":count_done,
1981 # "count_to_do":count_to_do,
1982 # "size_body":size_body,
1983 # "size_total":size_total,
1984 # "updated_at":issue.updated_at,
1985 # "link":link,
1986 # "concat_body":concat_body})
1988 import ast
1989 json_data = ast.literal_eval(json.dumps(data_for_retrieval))
1990 if verbose:
1991 print(json_data)
1992 from lib.import_util.lib_path_to_vec import manual_dump_id_path_url
1993 if not os.path.exists(os.path.dirname(out_path)):
1994 os.makedirs(os.path.dirname(out_path))
1995 manual_dump_id_path_url(out_path + ".json", json_data)
1996 # with open(out_path + '.json', 'w') as f:
1997 # json.dump(str(json_data), f)
1999 # f.write(json_data)
2001 # data = "".join(f.readlines())
2002 # json_data = json.load(data)
2003 # import ast
2004 # json_data = ast.literal_eval(data)
2006 # Maybe to try => but makes TypeError: Object of type datetime is not JSON serializable
2007 # f.write(json.dumps(data))
2009 return out_path + ".json", recently_updated_info
2010 else:
2011 print("Not yet managing cache for issues to gpt retrieval")
2012 return "", ""
2015def append_comment(github_token, verbose=False,
2016 message_comment="Just test",
2017 OwnRepo="fotonower/Doc",
2018 issue_number=188): # pragma no cover because for later v1
2019 from github import Github
2021 g = Github(github_token)
2023 # Then get your repository
2024 repo = g.get_repo(OwnRepo)
2026 # Get the issue that you want to comment on
2027 issue = repo.get_issue(number=issue_number)
2029 # Add a comment to the issue
2030 return issue.create_comment(message_comment)
2033# Should we do it by issue with context ?
2034class GitRulesIssues(): # pragma no cover
2035 def __init__(self, list_rules, gpt_service, do_generate_title=False, check_comment_link=True):
2036 self.sub_rules = []
2037 self.list_rules = list_rules
2038 self.methodo_rules = "contains" # contains, regexp, nlp
2039 self.gpt_service = gpt_service
2040 self._uplink_new_line = "[up](#up)"
2041 self._do_generate_title = do_generate_title
2042 self._check_comment_link = True
2044 def enforce_comment(self, comment, title_issue, body_issue=""):
2046 print("Need to test the list of rules !")
2048 has_title, has_uplink = self._has_title_and_uplink(comment)
2050 new_body = comment.body
2051 if not has_uplink:
2052 print("MISSING : Missing UPLINK ")
2053 new_body = self._uplink_new_line + "\n" + new_body
2054 new_title = False
2055 if not has_title:
2056 print("MISSING : Missing TITLE ")
2058 if not has_title and self._do_generate_title:
2059 generated_title = self._generate_title(comment, title_issue)
2060 new_body = self._uplink_new_line + new_body
2061 new_body = "# " + generated_title + "\n" + new_body
2062 new_title = True
2063 if not has_uplink or new_title:
2064 comment.edit(new_body)
2065 print("Now need to insert !")
2067 id_comment = comment.id
2068 if self._check_comment_link:
2069 if str(id_comment) not in body_issue:
2070 print("MISSING : Missing link from body to comment : " + str(id_comment))
2071 return id_comment
2073 return None
2075 # - [ ] VR TODO 5-8-25 : je refais en faisant un split sur # puis sur \n => donc il faut le refacto tout cela
2076 def _has_title_and_uplink(self, comment):
2077 has_uplink = False
2078 has_title = False
2079 if self.methodo_rules == "contains":
2080 body_parse = comment.body
2081 # if body_parse.startswith(self._uplink_new_line):
2082 if self._uplink_new_line + "\r\n" in body_parse or self._uplink_new_line + "\n" in body_parse:
2083 has_uplink = True
2084 if body_parse.startswith(self._uplink_new_line):
2085 body_parse = body_parse.lstrip(self._uplink_new_line).lstrip("\r\n").lstrip("\n")
2086 # body_parse[len(self._uplink_new_line):]
2087 if body_parse.startswith("#"):
2088 has_title = True
2089 else:
2090 print("Other methodo_rules not yet implemented !")
2092 return has_title, has_uplink
2094 def _generate_title(self, comment, title_issue):
2095 request = "PLEASE PROPOSE A TITLE FOR THE FOLLOWING COMMENT OF THE ISSUE THAT NAMES IS " + title_issue + "\n" + comment.body
2096 result, nb_token, model = self.gpt_service.completion(request)
2098 print("result : " + result)
2099 return result
2102def git_smart_edit(list_repos, list_excluded_labels, github_token,
2103 openai_token,
2104 list_rules,
2105 list_required_labels=[],
2106 verbose=False,
2107 with_comment=False,
2108 load_cache=False,
2109 with_closed_issue=False,
2110 out_path='doctest-output/Fotonower_Issues.json',
2111 load_all=False): # pragma no cover v1
2112 print("TO TEST")
2114 do_generate_title = False
2116 print("List Rules to check or enforce : " + str(list_rules))
2118 from lib.lib_openai import OpenAIAPI # request_gpt
2119 gpt_service = OpenAIAPI(openai_token)
2121 git_rules = GitRulesIssues(list_rules, gpt_service, do_generate_title=do_generate_title)
2123 import os
2124 out_folder = os.path.dirname(out_path)
2125 out_file = os.path.basename(out_path)
2127 import json
2128 if not load_cache:
2129 check_issue_re = False
2130 data, repo_name_with_cond = load_github_to_json_with_type(list_repos, list_excluded_labels, github_token,
2131 verbose, with_comment=with_comment, check_issue_re=check_issue_re,
2132 load_all=load_all,
2133 list_required_labels=list_required_labels)
2135 data_for_retrieval = []
2137 for issue in data["issues_data"]:
2138 print(".")
2139 id = issue["repo"] + "/" + str(issue["number"])
2140 text = issue["title"] + issue["concat_body"]
2141 url = issue["link"]
2143 list_missing_comment = []
2144 for c in issue["comments_data_or_git"]:
2145 print("c : " + str(c))
2146 one_missing_comment = git_rules.enforce_comment(c, issue["title"], issue["concat_body"])
2147 if one_missing_comment != None:
2148 list_missing_comment.append(one_missing_comment)
2149 # new_body = "[up](#up)\n" + c.body
2150 # c.edit(new_body)
2152 # https://github.com/fotonower/projects/issues/374#issuecomment-1561219418
2153 #
2154 print("TO CHECK")
2156 if len(list_missing_comment) > 0:
2157 print(" MISSING : " + str(list_missing_comment))
2159 else:
2160 print("Not yet managing cache for issues to smart_edit")
2162 print("TO FINISH")
2165import subprocess
2168def subprocessCommand(command, timeout=10, verbose=False): # pragma no cover util
2169 """ permet de faire appel a des commandes shell """
2170 proc = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True)
2171 try:
2172 outs, errs = proc.communicate(timeout=timeout)
2173 if verbose:
2174 print(str(command) + " Error : " + str(errs))
2175 print(" Output : " + str(outs))
2176 except subprocess.TimeoutExpired:
2177 proc.kill()
2178 outs, errs = proc.communicate()
2179 if verbose:
2180 print(str(outs) + " : " + str(errs))
2181 return outs
2184def get_remote_repo_from_local_repo(timeout=40, verbose=False): # pragma no cover interface simplifé
2185 command = "git remote -v"
2186 # Expected
2187 # origin git@github.com:fotonower/Safia.git(fetch)
2188 # origin git@github.com:fotonower/Safia.git(push)
2189 remote_repo_raw = subprocessCommand(command, timeout, verbose)
2191 remote_repo = remote_repo_raw.decode()
2192 remote_repo_lines = remote_repo.split("\n")
2193 if len(remote_repo_lines) == 0:
2194 return None
2196 if verbose:
2197 print(remote_repo_lines)
2199 first_line_repo = remote_repo_lines[0]
2200 first_line_repo = first_line_repo.replace("origin\tgit@github.com:", "")
2201 first_line_repo = first_line_repo.replace(".git", "")
2202 first_line_repo = first_line_repo.replace("(fetch)", "")
2203 first_line_repo = first_line_repo.replace(" ", "")
2205 if "/" not in first_line_repo:
2206 print("Error !")
2208 return first_line_repo
2212import json
2213from pydantic import BaseModel, ConfigDict, field_validator
2215from typing import List
2218class ATMODataFTN(BaseModel):
2219 name: str
2220 links: List = []
2221# links : List[dict[str, str]] = []
2222 content : str = ""
2223 labels : List = [] # VR 5-8-25 : on peut mettre des labels vides
2224# labels : List[dict[str, str]] = []
2225 type : str = "issue"
2226 link : str = ""
2227 title : str # Doit etre plus petit que 128 caractères (et c'est déjà trop)
2228 assignee : List = []
2229# assignee : List[str] = []
2230 type : str = ""
2231# updated_at : str = ""
2232 # datetime) = datetime.datetime.now()
2233# data : list[ATMODataFTN]
2235 @field_validator("title")
2236 @classmethod
2237 def check_title_short(cls, title: str) -> str:
2238 if len(title) > 96:
2239 #raise ValueError("Employees must be at least 18 years old.")
2240 print("WARNING title too long, we don't reduce it to avoid objection char : " + str(title))
2241 # title = title[:960]
2242 if len(title) > 1048576: # VR TODO data just for test duplicate ca va pas le faire
2243 print("WARNING title is too long 1048576 , we put a default value")
2244 title = "We should add label OBJECTION"
2245 # title = title[:256]
2247 return title
2251# parse_labels (and what about bolors ?)
2252def parse_label_on_content_or_link(content_or_link_display, list_labels):
2253 list_labels_found = []
2254 default_color = "123456"
2255 for l in list_labels:
2256 if l.upper() in content_or_link_display:
2257 new_label = {"name" : l, "color" : default_color}
2258 list_labels_found.append(new_label)
2260 return list_labels_found
2264# TO use or deprecate
2265def parse_object_as_atmo_doc_func(content,
2266 context_node_parent_name,
2267 context_node_parent_updated_at,
2268 regexp_search,
2269 fields,
2270 forced_name_object = None,
2271 id_new_object = 0):
2272 print(" Here copy voila du parse des chekxbo et qu'on va mettre à jour ovilà !")
2273 return None
2276def util_get_count_iterate(context_parent_node_and_count_subobj):
2277 if context_parent_node_and_count_subobj == None or "count_childs" not in context_parent_node_and_count_subobj:
2278 print("ERROR unexpected behavior, we could quit even if it is quite harsh !")
2279 return "-1"
2280 count = context_parent_node_and_count_subobj["count_childs"]
2281 context_parent_node_and_count_subobj["count_childs"] += 1
2282 return str(count)
2286def convert_json_md_a_to_link(attrs_data, current_node_context = None, context_parent_node_and_count_subobj = {},
2287 list_labels_to_parse = [],
2288 text_link = None):
2289 import sys
2290 sys.stdout.flush()
2291 attrs = ''
2292 href = ''
2293 # vr we expect to loose the text
2294 # text = data['text'] if 'text' in data else ""
2295 print("Text from link lost, but need at least to parse for label on link ")
2296 if True:
2297 href = None
2298 name = None
2299 content = text_link if text_link != None else ""
2301 if "href" in attrs_data:
2302 href = attrs_data["href"]
2303 complete_target = ""
2304 if href.startswith("#"): # we consider a local link VR TODO CDC 15-8-25 : how can we decide the context of relative linking ? This depends on github issue indeed ! Do we want to have these name or our name with comment flat ?
2305 print("STRAT NAMING TO BE CORRECLTY DEFINED ! ")
2306 grand_parent_node_name = context_parent_node_and_count_subobj["grand_parent_node_name"] if context_parent_node_and_count_subobj != None and "grand_parent_node_name" in context_parent_node_and_count_subobj else "DEFAULT_GRAND_PARENT_NODE_NAME_BECAUSE_MISSING"
2307 complete_target = grand_parent_node_name + "/" + href.lstrip("#")
2308 if current_node_context != None and "name" in current_node_context and "/" in current_node_context["name"]:
2309 complete_target = "/".join((current_node_context["name"].split("/")[:-1])) + "/" + href.lstrip("#")
2310 if current_node_context != None and "name" in current_node_context and "#" in current_node_context["name"]:
2311 complete_target = "#".join((current_node_context["name"].split("#")[:-1])) + "#" + href.lstrip("#")
2312 else :
2313 complete_target = href
2314 type_or_label = "link"
2315 print("TODO use text maybe 15-8-25")
2316 list_label_found = parse_label_on_content_or_link(content, list_labels_to_parse)
2317 content_without_label = content
2318 for l in list_label_found:
2319 content_without_label = content_without_label.replace(l["name"], "").strip()
2320 if content_without_label != "":
2321 print("OBJECTION WARNING we loose stuff ! We could use the content without label : " + str(content_without_label))
2322 # content = content_without_label
2323 if current_node_context != None:
2324 print("OBJECTION ERROR DEV WARNING WE loose data and don't report (but we could) it unexpected case we cannot add OBJECTION label ! " + str(content) + " " + str(content_without_label))
2325# current_node_context["labels"].append({"name": "OBJECTION", "color": "FF0000"})
2326 else:
2327 print("OBJECTION ERROR WE loose data and don't report it unexpected case we cannot add OBJECTION label ! " + str(content) + " " + str(content_without_label))
2329 if len(list_label_found) > 0:
2330 type_or_label = list_label_found[0]["name"]
2331 default_link_label_hierarchical = "info" # VR TODO REFACTO DEV parse_label of link at this point !
2332 data_atmo = {"type": type_or_label, # useless
2333 "label": default_link_label_hierarchical,
2334 "target": complete_target,
2335 "content": content} # useless as well
2336 new_link = {"label" : type_or_label, "target" : complete_target}
2337 if current_node_context != None:
2338 if "links" not in current_node_context:
2339 current_node_context["links"] = []
2340 current_node_context["links"].append(new_link)
2341 return None
2342 elif "name" in attrs_data:
2343 name = attrs_data["name"]
2344 type_or_label = "a_name"
2345 content = attrs_data["text"] if "text" in attrs_data else content
2346 data_atmo = {"type": type_or_label, # useless
2347 "name": name,
2348 "content": content} # useless as well
2349 print("We will need to rename")
2350 if current_node_context != None and "name" in current_node_context:
2351 if "#" not in current_node_context["name"]:
2352 print("# Not sure what to do")
2353 if "/" not in current_node_context["name"]:
2354 print("Not sur what to do why we are in this state")
2355 else : # STRAT NAMING relative to comment
2356 new_name_thanks_ref = "/".join(current_node_context["name"].split("/")[:-1]) + "/" + name
2357 print("a_name RENAMING " + (current_node_context["name"].split("/")[-1]) + " TO " + name)
2358 current_node_context["name"] = new_name_thanks_ref
2359 else: # STRAT NAMING relative to issue
2360 if current_node_context["type"] == "comment" and "issuecomment" not in current_node_context["name"]:
2361 print("We do not rename because we are in a comment and it has been already renamed")
2362 else:
2363 new_name_thanks_ref = "#".join(current_node_context["name"].split("#")[:-1]) + "#" + name
2364 print("a_name RENAMING " + (current_node_context["name"].split("#")[-1]) + " TO " + name)
2365 print("VR TODO verify, je ne suis pas vraiment sur mais il me semble qu'il faut aussi changer le parent_node_name car les string ne sont pas des references contraitement aux dict dans python")
2366 if "parent_node_name" in context_parent_node_and_count_subobj:
2367 context_parent_node_and_count_subobj["parent_node_name"] = new_name_thanks_ref
2368 if "luca_name" in context_parent_node_and_count_subobj and current_node_context["name"] == context_parent_node_and_count_subobj["luca_name"]:
2369 context_parent_node_and_count_subobj["luca_name"] = new_name_thanks_ref
2370 current_node_context["name"] = new_name_thanks_ref
2372 # Test 14-8-25
2373# grand_parent_node_name = context_parent_node_and_count_subobj["grand_parent_node_name"] if context_parent_node_and_count_subobj != None and "grand_parent_node_name" in context_parent_node_and_count_subobj else "DEFAULT_GRAND_PARENT_NODE_NAME_BECAUSE_MISSING"
2374# current_node_context["name"] = grand_parent_node_name + "/" + name
2375# print("- [ ] VR TODO 15-8-25 : RENAMING, we need to have convention in a lot of case, for example we need the grand-parent ! ")
2377 # we don't return the ref_name_a if it was used
2378 return None
2379 else :
2380 print(" ERROR unexpected case NOT SURE what to do ! ")
2382 else :
2383 print("WARNING DATA LOST unexpected a not href or name : incorrectly parsed, format link unexpected : " + str(attrs_data))
2384 return None
2387 return data_atmo
2389# VR 15-8-25 : starting this in order to get back to the correct behavior of the checkbox and remove the former code of parsing
2390# VR 15-8-25 : maybe we could start with a refacto !
2391def get_todone_or_tic_thanks_re(complete_content):
2392 type_and_label = ""
2393 content = ""
2394 return content, type_and_label
2398# VR 15-8-25 : CDC how to manage color of label (same pb as cache label to list ? grrr grrr)
2399def append_label_to_checkbox(complete_content, list_labels):
2400 labels_found = []
2401 type_tic_or_todone = ""
2402 content = ""
2403 return content, type_tic_or_todone
2407def convert_json_md_a_to_cb_tic(data,
2408 config_assoc_list_to_atmo ={},
2409 option_flat_or_tree = "flat",
2410 context_parent_node_and_count_subobj = {},
2411 list_labels_to_parse = [],
2412 verbose=False):
2414 list_data_atmo = []
2415 if type(data) == list and len(data) > 0:
2416 # if type(data[0]) == str:
2417 for d in data:
2418 if type(d) == str:
2419 if d == None:
2420 print("warn remove one None data")
2421 continue
2422 labels = []
2424 content = d
2425 type_and_label = "tic"
2426 import re
2427 try:
2428 matches = re.match(r"(\[[x|X| ]\])?(.+)", d)
2429 except Exception as e:
2430 print("WRONG PARSING CONTINUE TIC OR TOE CHECKBOX " + str(e) + " " + str(d))
2431 type_and_label = "todone"
2432 continue
2433 group_match = matches.groups()
2434 if type(group_match) == tuple and len(group_match) == 2:
2435 if group_match[0] == None:
2436 type_and_label = "tic"
2437 content = d
2438 else:
2439 content = group_match[1]
2441 if group_match[0].lower() == "[x]":
2442 type_and_label = "done"
2443 elif group_match[0].lower() == "[ ]":
2444 type_and_label = "todo"
2445 else :
2446 type_and_label = "todone"
2447 print("Warning incorrect parsing of CB : " + str(d))
2449 else :
2450 type_and_label = "tic"
2451 print("WARNING checkbox with many checkbox or WTF ??!?!")
2453 if type_and_label == 'tic':
2454 type_tic_or_todone = "tic"
2455 label_todone = "tic"
2456 color_todone = "ffffff"
2457 elif type_and_label == "todo":
2458 type_tic_or_todone = "checkbox"
2459 label_todone = "todo"
2460 color_todone = "f46aff"
2461 elif type_and_label == "done":
2462 type_tic_or_todone = "checkbox"
2463 label_todone = "done"
2464 color_todone = "20fe32"
2465 elif type_and_label == "todone":
2466 print("ERROR treated as warning in detecting type_and_label from todone_or_tic Not really expected")
2467 type_tic_or_todone = "todone"
2468 label_todone = "todone"
2469 color_todone = "ffffff"
2470 else :
2471 type_tic_or_todone = "ticb" # or todone ?
2472 color_todone = "7d5c96"
2473 label = "undefined"
2474 print("ERROR treated as WARNING, unexpected type_and_label : " + str(type_and_label))
2476 labels = [{"name" : label_todone, "color" : color_todone}]
2477 list_labels_found = parse_label_on_content_or_link(content, list_labels_to_parse)
2478 labels.extend(list_labels_found)
2479 data_atmo = {"name" : "DEFAULT_NAME_WHILE_ADDING_COUNT","type" : type_tic_or_todone, "content" : content, "labels" : labels}
2480 parent_node_name = context_parent_node_and_count_subobj["parent_node_name"] if context_parent_node_and_count_subobj != None and "parent_node_name" in context_parent_node_and_count_subobj else "DEFAULT_PARENT_NODE_NAME_MISSING"
2481 data_atmo["name"] = parent_node_name + "/" + type_tic_or_todone.upper() + util_get_count_iterate(context_parent_node_and_count_subobj)
2482 data_atmo["title"] = content
2483 data_atmo["links"] = [{"label":"hierarchical", "target" : context_parent_node_and_count_subobj["luca_name"]}]
2485 if "luca_content" in context_parent_node_and_count_subobj:
2486 content_this_comment = context_parent_node_and_count_subobj["luca_content"]
2487 info_edit_atmo = compute_info_edit_atmo(content_this_comment, content)
2488 data_atmo["info_edit_atmo"] = info_edit_atmo
2490 list_data_atmo.append(data_atmo)
2491 elif d == None:
2492 continue
2493 else:
2494 print("Iteration over the rest ")
2495 # Despite the flat option we pass at the next level
2496# context_parent_node_and_count_subobj["grand_parent_node_name"] = context_parent_node_and_count_subobj["parent_node_name"]
2497# current_obj_atmo_used_for_rename = None
2498# if len(list_data_atmo) > 0:
2499# context_parent_node_and_count_subobj["parent_node_name"] = list_data_atmo[-1]["name"]
2500# current_obj_atmo_used_for_rename = list_data_atmo[-1]
2501# if len(list_data_atmo) > 1:
2502# # print("IF we have a name inside this, this is an ERROR UNEXPECTED TREATED AS WARNING ! We have trouble here, we don't know what to do with this !")
2503# current_obj_atmo_used_for_rename = None
2504 collected_object_output = decrit_rec_md_bs_json_tree(d,
2505 list_data_atmo, # current_obj_atmo_used_for_rename VR TODO REFACTO rename collected_object_output_local or at this level
2506 current_obj_atmo=None, # J'ai bien envie de mettre le data_atmo ici, et les cas ou il y en a plusieurs ou verra, de toute facon on aurait pas du etre là car cela aurait du etre géré avant, vraiment ?
2507 config_assoc_list_to_atmo=config_assoc_list_to_atmo,
2508 option_flat_or_tree=option_flat_or_tree,
2509 context_parent_node_and_count_subobj = context_parent_node_and_count_subobj,
2510 list_labels_to_parse=list_labels_to_parse,
2511 verbose = verbose)
2512 list_data_atmo.extend(collected_object_output)
2513 list_data_atmo = make_unique(list_data_atmo)
2514# del context_parent_node_and_count_subobj["grand_parent_node_name"]
2515 else :
2516# print("It seems we are at the end of the for loop : ")
2517# print("ALWAYS TRUE OR ERROR : " + str(d == data[-1]))
2518 if (d != data[-1]):
2519 print("ERROR exiting !")
2520 exit(1666)
2521 # TO remove on 18-8-25 print("VR 15-8-25 : how can a else be after a for ??? We observed two new caes, don't know if it is this end, and the first case is not yet debug anyway !")
2522 else:
2523 print("Unexpected type data")
2524 print(" We observed different cases, don't know if it is the last one, and code in test on 08-25 !")
2525 return []
2527 # data_atmo = {"type" : "tic", "content" : str(data)}
2528 return list_data_atmo
2532def convert_json_md_a_to_par(data, context_parent_node_and_count_subobj):
2534 name = context_parent_node_and_count_subobj["parent_node_name"] + "/PAR" + str(util_get_count_iterate(context_parent_node_and_count_subobj))
2535 labels = [{"name":"paragraph", "color" : "ff1122"}]
2536 data_atmo = {"type" : "paragraph", "content" : str(data), "name" : name, "title" : str(data), "labels" : labels}
2538 luca_name = context_parent_node_and_count_subobj["luca_name"] if context_parent_node_and_count_subobj != None and "luca_name" in context_parent_node_and_count_subobj else "DEFAULT_LUCA_NAME_MISSING"
2540 data_atmo["links"] = [{"label":"hierarchical", "target" : luca_name}]
2541 return data_atmo
2543def make_unique(collected_object_output):
2544 map_collected_object_output = {d["name"] : d for d in collected_object_output}
2546 collected_object_output = list(map_collected_object_output.values())
2547 return collected_object_output
2549def decrit_rec_md_bs_json_tree(input_data,
2550collected_object_output,
2551 current_obj_atmo = None,
2552 config_assoc_list_to_atmo ={},
2553 option_flat_or_tree = "flat",
2554 context_parent_node_and_count_subobj = {}, # VR TODO duplicate with current_obj_atmo but containing also the count of sub ojbect of a comment (or of something els if we activate option_flat_or_tree to tree)
2555 list_labels_to_parse = [],
2556 verbose = False):
2559 print_mem()
2560 local_vars = list(locals().items())
2561 if verbose:
2562 for var, obj in local_vars:
2563 print(var, sys.getsizeof(obj))
2566 if len(collected_object_output) > 1000000:
2567 print(len(collected_object_output))
2569 import bs4
2570 if type(input_data) == list:
2571 for d in input_data :
2572 if type(d) == dict and "attrs" in d:
2573 if len(set(d.keys()) - set(["attrs", "text"])) > 0:
2574 print("Unexpected case we loose data")
2575 text_link = d["text"] if "text" in d else None
2576 attrs_link = d["attrs"]
2577 res_link = convert_json_md_a_to_link(attrs_link, current_obj_atmo,
2578 context_parent_node_and_count_subobj,
2579 list_labels_to_parse=list_labels_to_parse,
2580 text_link = text_link)
2581 if res_link != None:
2582 print("ERROR refacto convert_json_md_a_to_link is not complete !")
2583 else:
2584 collected_object_output = decrit_rec_md_bs_json_tree(d, collected_object_output,
2585 current_obj_atmo, config_assoc_list_to_atmo,
2586 option_flat_or_tree,
2587 context_parent_node_and_count_subobj = context_parent_node_and_count_subobj,
2588 list_labels_to_parse = list_labels_to_parse,
2589 verbose = verbose)
2590 if len(collected_object_output) > 0 :
2591 current_obj_atmo = collected_object_output[-1]
2592 elif type(input_data) == dict : # or type(input_data) == bs4.element.XMLAttributeDict:
2593 for k in input_data:
2594 if k in config_assoc_list_to_atmo:
2595 if k == 'a' and type(input_data[k]) == dict and 'attrs' in input_data[k]:
2596 text_link = input_data[k]["text"] if "text" in input_data[k] else None
2597 attrs_link = input_data[k]["attrs"]
2598 context_parent_node_and_count_subobj["current_obj_atmo"] = current_obj_atmo
2599 res_link = convert_json_md_a_to_link(attrs_link, current_obj_atmo, context_parent_node_and_count_subobj,
2600 list_labels_to_parse = list_labels_to_parse,
2601 text_link = text_link)
2602 if current_obj_atmo == None or res_link == None:
2603 print(" Link was already treated !")
2604 else:
2605 print("WARNING : we shouldn't be there since link is not already treated")
2606 if "links" not in current_obj_atmo:
2607 current_obj_atmo["links"] = []
2608 if "type" in res_link and "href" == res_link["type"]:
2609 current_obj_atmo["links"].append(res_link)
2610 elif k == 'a' and type(input_data[k]) == dict and 'attrs' not in input_data[k]:
2611 print("ERROR UNEXPECTED PARSING GENERIC CASE")
2612 elif type(input_data[k]) == dict and "attrs" in input_data[k]:
2613 print("ERROR On n paasse jamais ici, c'est un cas ou on est dans depuis une lise")
2614# if len(set(list(input_data[k].keys())) - set(["attrs", "text"])) > 0:
2615 if len(set(input_data[k].keys()) - set(["attrs", "text"])) > 0:
2616 print("WARNING OBJECTION Unexpected case we loose data")
2617 if len(collected_object_output) > 0:
2618 if current_obj_atmo != None:
2619 print("Maybe we should do something in this ! voila !")
2620 collected_object_output[-1]["labels"].append({"name" : "OBJECTION", "color" : "ff0000"})
2621 else:
2622 print("ERROR OBJECTION unexpected case no object to inform !")
2624 text_link = input_data[k]["text"] if "text" in input_data[k] else None
2625 attrs_link = input_data[k]["attrs"]
2626 res_link = convert_json_md_a_to_link(attrs_link, current_obj_atmo,
2627 context_parent_node_and_count_subobj,
2628 list_labels_to_parse=list_labels_to_parse,
2629 text_link = text_link)
2630 elif k == 'li' and type(input_data[k]) == list and len(input_data[k]) > 0 and type(input_data[k][0]) == str:
2631 list_data_atmo = convert_json_md_a_to_cb_tic(input_data[k], config_assoc_list_to_atmo = config_assoc_list_to_atmo,
2632 option_flat_or_tree=option_flat_or_tree,
2633 context_parent_node_and_count_subobj = context_parent_node_and_count_subobj,
2634 list_labels_to_parse = list_labels_to_parse,
2635 verbose=verbose)
2636 collected_object_output.extend(list_data_atmo)
2637 collected_object_output = make_unique(collected_object_output)
2638 # current_obj_atmo = res_cb_tic
2639 current_obj_atmo = collected_object_output[-1]
2640# elif k == 'h1':
2641# print("VR pour l'instant je crois qu'on ne passe pas souvent là, mais de fait on voit les paragraph détaché !")
2642# res_par = convert_json_md_a_to_par(input_data[k], context_parent_node_and_count_subobj)
2643# collected_object_output.append(res_par)
2644# current_obj_atmo = res_par
2645 elif k == 'p' and type(input_data[k]) == str:
2646 list_data_atmo = convert_json_md_a_to_cb_tic([input_data[k]],
2647 config_assoc_list_to_atmo = config_assoc_list_to_atmo,
2648 option_flat_or_tree=option_flat_or_tree,
2649 context_parent_node_and_count_subobj = context_parent_node_and_count_subobj,
2650 list_labels_to_parse = list_labels_to_parse)
2651 collected_object_output.extend(list_data_atmo)
2652 collected_object_output = make_unique(collected_object_output)
2653 # current_obj_atmo = res_cb_tic
2654 current_obj_atmo = collected_object_output[-1] # Sans doute useless dans ce cat
2655 context_parent_node_and_count_subobj["current_obj_atmo"] = current_obj_atmo
2656 else:
2657 collected_object_output = decrit_rec_md_bs_json_tree(input_data[k], collected_object_output,
2658 current_obj_atmo, config_assoc_list_to_atmo,
2659 option_flat_or_tree,
2660 context_parent_node_and_count_subobj = context_parent_node_and_count_subobj,
2661 list_labels_to_parse=list_labels_to_parse,
2662 verbose = verbose)
2664 else :
2665 print(" We may loose a bit of data and should add the label OBJECTION ! :-) k " + str(k))
2667 if k not in ["body", "html", "ul", "text"]:
2668 print(" WARNING unexpected key in dict that is not parsed : " + str(k) + " type : " + str(type(input_data[k])))
2669 if len(collected_object_output) > 0:
2670 if current_obj_atmo != None:
2671 print("Maybe we should do something in this ! voila !")
2672 collected_object_output[-1]["labels"].append({"name" : "OBJECTION", "color" : "ff0000"})
2673 else:
2674 print("ERROR unexpected case no object to inform !")
2676 collected_object_output = decrit_rec_md_bs_json_tree(input_data[k],
2677 collected_object_output,
2678 current_obj_atmo, config_assoc_list_to_atmo,
2679 option_flat_or_tree,
2680 context_parent_node_and_count_subobj = context_parent_node_and_count_subobj,
2681 list_labels_to_parse=list_labels_to_parse,
2682 verbose = verbose)
2683 if len(collected_object_output) > 0:
2684 current_obj_atmo = collected_object_output[-1]
2685 elif type(input_data) == str:
2686 list_data_atmo = convert_json_md_a_to_cb_tic([input_data],
2687 config_assoc_list_to_atmo=config_assoc_list_to_atmo,
2688 option_flat_or_tree=option_flat_or_tree,
2689 context_parent_node_and_count_subobj=context_parent_node_and_count_subobj,
2690 list_labels_to_parse=list_labels_to_parse)
2691 collected_object_output.extend(list_data_atmo)
2692 collected_object_output = make_unique(collected_object_output)
2693 # current_obj_atmo = res_cb_tic
2694 current_obj_atmo = collected_object_output[-1] # Sans doute useless dans ce cat
2695 context_parent_node_and_count_subobj["current_obj_atmo"] = current_obj_atmo
2696 else :
2697 print(" WARNING OBJECTION MISSING DATA unexpected type input_data : " + str(type(input_data)))
2698 if len(collected_object_output) > 0:
2699 collected_object_output[-1]["labels"].append({"name" : "OBJECTION", "color" : "ff0000"})
2700 else:
2701 print("ERROR OBJECTION WARNING unexpected type input_data and no object to inform ! : " + str(type(input_data)))
2702 print("MISSING DATA : " + str(input_data))
2704 return collected_object_output
2707# This function
2708# There is some doc in the comment in and around it
2709# - [ ] VR TODO 15-8-25 : consolidate the doc in the expected document, currently in https://github.com/fotonower/BETATMO/issues/2#issuecomment-3137936894
2710# - [ ] VR TODO 15-8-25 : generalize parse_checkbox_as_atmo_doc_func to parse_object_as_atmo_doc_func with different type of object that couldn't be the same by the way !
2711# - [ ] VR OUTPUT unparsed content
2712# - [ ] CDC parse_object_as_atmo_doc_func and parse_checkbox_as_atmo_doc_func : should we parse line by line (this must be more efficient, argh or simpler than keeping the unparsed line, and furthermore there are some stuff like details or code_example that are multi (line) but we don't want to parse it for now)
2713# - [ ] inheritance of labels from parents will be done later (not like in checkbox parsing)
2714# - [ ] Veut-on différencier les internal des externals links ?
2715# - [ ] VR TODO 15-8-25 : refacto links parsing
2716# - [ ] doc devops pip3.10 install bs2json pip3.10 install marko pip3.10 install bs4
2717# [ ] VR CDC faut il passer le contexte dans decrit_rec_md_bs_json_tree notament pour les liens ?
2718def parse_comment_with_config_parse(comment,
2719 context_node_parent = None,
2720 config_parse = {"objects" : ["checkbox", "tic", "md_paragraph", "a_name",
2721 "implicit_hierarchical_link"],
2722 "options": ["link", "label_on_object", "label_on_link"]},
2723 list_labels_to_parse = ["DEV", "DOC", "BUG", "PLAN", "DONE"],
2724 verbose = False) : # PLAN or CDC
2726 import marko
2727 res_parse_marko = marko.convert(comment)
2730 if verbose:
2731 print(res_parse_marko)
2732 from bs4 import BeautifulSoup
2733 res_parse_bs = BeautifulSoup(res_parse_marko, 'xml')
2734 res_parse_bs = BeautifulSoup(res_parse_marko, 'lxml')
2736 if verbose:
2737 print(res_parse_bs)
2739 from bs2json import BS2Json
2741# html = '<html><head><title>Page Title</title></head><body><h1>My First Heading</h1><p>My first paragraph.</p></body></html>'
2742 bs2json = BS2Json(res_parse_bs)
2744 # Convert soup to JSON
2745 try :
2746 res_parse_bs_to_json = bs2json.convert()
2747 except Exception as e:
2748 print("TROP GROS SANS DOUTE")
2749 print(str(e))
2750 res_parse_bs_to_json = {}
2752 # Save JSON to file
2753# bs2json.save()
2755 # Print prettified output
2756 if verbose:
2757 bs2json.prettify()
2759 very_chiant_two_new_cases_li = {'html': {'body': {'p': {'a': {'attrs': {'href': '#top'}, 'text': 'up'}},
2760 'h1': ['Amélioration du rangement', 'Rangement récurrent'], 'ul': [{'li': {'text': '[ ]', 'a': {
2761 'attrs': {
2762 'href': 'https://docs.google.com/spreadsheets/d/1xZ06L4OkmIVjgg-vvq-XiJSPmYOwIZNsvdb-miq3GFE/edit?gid=327753039#gid=327753039'},
2763 'text': 'Typologie_inventaire_matos FTN'}}}, {'li': [{'p': {
2764 'a': {'attrs': {'href': 'https://github.com/fotonower/BETATMO/issues/5'},
2765 'text': 'Lié au taches domestiques'}}}, {
2766 'p': "[ ] Rangemeent recurrent table cuisine, comment l'organiser dans rangement ou dans cuisine ??"}]}]}}}
2768 if verbose:
2769 print(res_parse_bs_to_json)
2771 collected_object_output = []
2772 context_parent_node_and_count_subobj = {"parent_node_name" : context_node_parent["name"] if context_node_parent != None and "name" in context_node_parent else "DEFAULT_PARENT_NODE_NAME_BEECAUSE_MISSING",
2773 "luca_name": context_node_parent[
2774 "name"] if context_node_parent != None and "name" in context_node_parent else "DEFAULT_ONLY_ANCESTOR_NODE_FOR_FLAT_CASE",
2775 "count_childs" : 0}
2776 if context_node_parent != None and "body" in context_node_parent:
2777 context_parent_node_and_count_subobj["luca_content"] = context_node_parent["body"]
2779 collected_object_output = decrit_rec_md_bs_json_tree(res_parse_bs_to_json,
2780 collected_object_output,
2781 context_node_parent,
2782 config_assoc_list_to_atmo={"a": "link",
2783 "li": "cb",
2784 "h1": "paragraph",
2785 "h2": "sparagraph",
2786 "h3": "ssparagraph",
2787 "h4": "sssparagraph",
2788 "p" : "only_with_string",
2789 "attrs" : "link"},
2790 option_flat_or_tree="flat",
2791 context_parent_node_and_count_subobj = context_parent_node_and_count_subobj,
2792 list_labels_to_parse = list_labels_to_parse,
2793 verbose = verbose)
2795 if verbose:
2796 print(collected_object_output)
2799 # - [ ] VR tODO after debug Y'a pu qu'à rajouter les labels sur les liens, les labels sur les objets
2800 # - [ ] Et puis gérer les a name
2802 # expected info from context node
2803 list_keys_expected_context = ["father_node_name", "source"]
2804 # father_node_name will be commend node name for example
2805 # source can be git or atmo (not used on 15-8-25)
2807 list_config_parse_option_expected = ["checkbox", "tic", "md_paragraph", "a_name",
2808 "implicit_hierarchical_link",
2809 "link", "label_on_object", "label_on_link",
2810 "details", "code_quote", "source_code_reference"] # as of 15-8-25 we don't want to parse this
2813 regexp_search_diff_tic_cb = r"\[([x| ]?)\](.+)\n"
2817 # tic_and_cb
2818 # "label_on_object", "label_on_link"
2821 # "link", "label_on_object", "label_on_link"
2825 # - [ ] 15-8-25 vr todo lister les conversions de md_bs_json vers atmo ! eh he eh ! genre
2826 # a => link
2827 # p => CB or
2828 # h1.... => PAR
2829 # confusion entre a et link par ce process
2830 # comment fais t'on pour associer les a aux bons objets ? Et les link interne, sont ils gérés ?
2831 # => on refais un 'parsing simple', ou est-ce qu'on prend plutot des a name sur la meme ligne ? Oh ho ! => on dirait
2833 # optino flat et optino tree et peut-etre d'autres à venir !
2835 return collected_object_output
2837map_config_bijection_root_level = {
2838 "default" : {"issue" : "global_git",
2839 "comment" : "local"},
2840 "atmo" : {"issue" : "loca_atmol",
2841 "comment" : "local"}
2842}
2844map_todo_voila = {
2845 "a": "link",
2846 "p": "cb",
2847 "h1": "paragraph",
2848 "h2": "sparagraph",
2849 "h3": "ssparagraph",
2850 "h4": "sssparagraph"
2851}
2853def check_name_is_default(name, repo = "BETATMO", parent_comment = ""):
2855 if len(name.split("/")) == 3:
2856 return True
2857 else:
2858 return False
2860 import re
2861 f = re.match(r"{repo}\/(\s+)\#(\s*)\/{tic|cb|par}(\n+)", name.lower())
2862 if (f):
2863 print("WARNING default name detected, we should rename it : " + str(name) + " parent comment : " + str(parent_comment))
2864 return True
2866def get_type_from_label(labels):
2867 map_managed_type_prefix = {"tic" : "- ",
2868 "checkbox" : "- [ ]",
2869 "done" : "- [x]",
2870 "todo" : "- [ ]",
2871 "paragraph" : "# "}
2872 for l in labels:
2873 if "name" in l:
2874 label = l["name"]
2875 if label in map_managed_type_prefix:
2876 return map_managed_type_prefix[label]
2877 return ""
2880def build_comment_content_from_atmo_data(list_data_atmo_one_comment,
2881 context_node_parent_name = None,
2882grand_parent_node_name = None,
2883 config_export = {"root_level" : "default",
2884 "with_up_link" : True}):
2885 print("VR TODO 15-8-25 : build_comment_content_from_atmo_data not yet implemented")
2887 with_up_link = config_export["with_up_link"] if "with_up_link" in config_export else False
2889 count_objection = 0
2891 content_comment_mld = ""
2892 if with_up_link:
2893 content_comment_mld += "[up](#top)\n\n"
2895 if context_node_parent_name != None:
2896 # match {grand_parent_node_name}#issuecomment-commentid
2897 if grand_parent_node_name != None:
2898 if context_node_parent_name.startswith(grand_parent_node_name + "#issuecomment-"):
2899 context_node_parent_name = context_node_parent_name.replace(grand_parent_node_name + "#issuecomment-", "")
2900 if not context_node_parent_name.isdecimal():
2901 print("ERROR unexpected case for context_node_parent_name : " + str(context_node_parent_name))
2902 exit(1667)
2903 elif context_node_parent_name.startswith(grand_parent_node_name + "#issue"):
2904 print("On ne s'en occupe pas for now de cette verif")
2905 elif context_node_parent_name.startswith(grand_parent_node_name + "#"):
2906 print("We had relative naming")
2907 content_comment_mld += "<a name=" + context_node_parent_name.replace(grand_parent_node_name + "#", "") + " ></a>\n\n"
2908 else :
2909 print("This is really unexpected !")
2912 begin_char = len(content_comment_mld)
2914 # Loop over the list of data atmo for one comment
2915 for d in list_data_atmo_one_comment:
2916 if issue_atmo_has_label(d, "OBJECTION"):
2917 print("SKIP OBJECTION AND Can'T " + str(d))
2918 count_objection += 1
2919# continue
2920 type_data_prefix = get_type_from_label(d["labels"]) if "labels" in d and type(d["labels"]) == list else None
2921 if type_data_prefix == None:
2922 print("WARNING unhandled type in build_comment_content_from_atmo_data : " + str(d))
2923 continue
2924 content_node = d["title"] if "title" in d else ""
2926 links_str = ""
2927 if "links" in d and type(d["links"]) == list and len(d["links"]) > 0:
2928 print("OBJECTION TO TREAT " + str(d))
2929 for l in d["links"]:
2930 if l["label"].lower() != "hierarchical":
2931 # if links_str != "":
2932 links_str += " "
2933 target_href = l["target"]
2934 if grand_parent_node_name != None and not target_href.startswith("http") and target_href.startswith(grand_parent_node_name):
2935 target_href = target_href.replace(grand_parent_node_name, "")
2936 links_str += "[" + l["label"].upper() + "](" + target_href + ")"
2937 # count_objection += 1
2938 potential_a_name = ""
2939 if "name" in d:
2940 if not check_name_is_default(d["name"]):
2941 print("OBJECTION TO TREAT MAIS CA VA ETRE FACILE " + str(d))
2942# count_objection += 1
2943 d_name = d["name"]
2944 if grand_parent_node_name != None and d_name.startswith(grand_parent_node_name):
2945 d_name = d_name.replace(grand_parent_node_name + "#", "")
2946 potential_a_name = f" <a name={d_name}></a>"
2947 else:
2948 print("Verify it is default !?!")
2950 this_content = type_data_prefix + content_node + links_str + potential_a_name + "\n"
2952# info_edit_atmo = {"begin_char": begin_char, "end_char": end_char, "len_content": len_content}
2953 end_char = begin_char + len(this_content)
2955 d["info_edit_atmo"] = {"begin_char": begin_char, "end_char": end_char, "len_content": len(this_content)}
2956 begin_char = end_char
2958 content_comment_mld += this_content
2960 # Les passages à la ligne, c'est pas top !
2962 # PB : longueur, certains carachètre spécial (durant le premier test =>)
2963 # IDée : rajouter un label : NOBIJ ou OBJECTION => ca c'est bien !
2964 # Il y a une priorité de labels
2965 # - [ ] Gestion des name => comment
2966 # - [v] Gestion des links
2967 # - [ ] Rajout des labels sur les liens
2968 # - [ ] Gestion des (sub) paragraphes => en objection
2969 # - [ ] Gestion des détails => non
2970 # - [ ] Gestion des codes quotes => non
2971 # - [x] Clairement le link up n'est pas géré => et si mon gars par une option config_export
2973 print(content_comment_mld)
2975 return content_comment_mld, count_objection
2979def rebuild_git_data_from_atmo_data(map_data_atmo, map_hierarchy,
2980 context_node_parent_name = None,
2981grand_parent_node_name = None,
2982 config_export = {"root_level" : "default",
2983 "with_up_link" : True}):
2984 print("VR TODO 15-8-25 : rebuild_git_data_from_atmo_data not yet implemented")
2985 # VR TODO TACHE group by repo and issue and comment
2986 # Loop over the list of comment
2988 # Test on one comment
2989 if context_node_parent_name == None:
2990 context_node_parent_name = "BETATMO/14#issuecomment-3210372351"
2991 context_node_parent_name = "BETATMO/14#issuecomment-3289898907"
2992 grand_parent_node_name = "BETATMO/14"
2994 list_data_atmo_this_comment = []
2995 for k in map_data_atmo:
2996 d = map_data_atmo[k]
2997 if "links" in d and type(d["links"]) == list:
2998 for l in d["links"]: # en fait faut que les link hierarchy, on applique un tel filtre dans build_gag
2999 # VR TODO 15-8-25 REFACTO on pourrait aussi faire un truc plus propre en utilisant map_hierarchy
3000 if "target" in l and l["target"] == context_node_parent_name:
3001 d["name"] = k
3002 list_data_atmo_this_comment.append(d)
3003 break
3005 one_comment_content, count_objection = build_comment_content_from_atmo_data(list_data_atmo_this_comment,
3006 context_node_parent_name = context_node_parent_name,
3007 grand_parent_node_name= grand_parent_node_name,
3008 config_export = config_export)
3010# VR TODO do_we_miss_one_data
3012 if count_objection > 0:
3013 print("WARNING there were " + str(count_objection) + " OBJECTION labels")
3014 return None, count_objection
3016 return one_comment_content, 0
3018# --job=github.gitactions --min_date=">2026-02-08"
3019def github_gitactions(github_token, list_repos = [],
3020 verbose = False, min_date = None,
3021 conditions = {}):
3022 print("VR TODO 15-8-25 : github_gitactions not yet implemented")
3023 # List repos
3025 print("min_date : " + str(min_date))
3027 g = Github(github_token)
3029 g_ftn = g.get_organization("fotonower")
3030 list_runners_self = g_ftn.get_self_hosted_runners()
3032 print(str(list_runners_self))
3034 for r in list_runners_self:
3035 print("YEN A PAS, c'était pas ca")
3037 if list_repos == []:
3038 list_repos_sans_doute_paginated = g.get_repos()
3039 for r in list_repos_sans_doute_paginated:
3040 list_repos.append(r.full_name)
3042 for r in list_repos:
3043 try:
3044 repo = g.get_repo(r)
3045 except Exception as e:
3046 print(str(e))
3047 print("Repo " + str(r) + " has been deleted !")
3048 continue
3049 # repo.get_git_actions()
3050 # conditions actors, event created
3051 actor = "claude[bot]" if "actor" not in conditions else conditions["actor"]
3052 list_wf_runs = repo.get_workflow_runs(actor = actor, created=min_date)
3054 print(" list_wf_runs for repo : " + r + " " + str(list_wf_runs.totalCount))
3055 for w in list_wf_runs:
3056 print(w.actor.login + " " + w.event + " " + str(w.created_at))