release_iseus_status_auto_reply

scripts/release_issue_status/main.py

@@ -5,12 +5,15 @@
 from datetime import date, datetime
 import subprocess as sp
 from azure.storage.blob import BlobClient
+import reply_generator as rg
+from update_issue_body import update_issue_body, find_readme_link
+import traceback
 
 _NULL = ' '
 _FILE_OUT = 'release_issue_status.csv'
+_FILE_OUT_PYTHON = 'release_python_status.md
 _PYTHON_SDK_ADMINISTRATORS = {'msyyc', 'RAY-316', 'BigCat20196'}
 
-
 def my_print(cmd):
     print('==' + cmd + ' ==\n')
 
@@ -20,6 +23,20 @@ def print_check(cmd):
     sp.check_call(cmd, shell=True)
 
 
+def output_python_md(issue_status_python):
+    with open(_FILE_OUT_PYTHON, 'w') as file_out:
+        file_out.write('| issue | author | package | assignee | bot advice | created date of issue | delay from created date |\n')
+        file_out.write('| ------ | ------ | ------ | ------ | ------ | ------ | :-----: |\n')
+        file_out.writelines([item.output_python() for item in sorted(issue_status_python, key=_key_select)])
+
+
+def output_csv(issue_status):
+    with open(_FILE_OUT, 'w') as file_out:
+        file_out.write('language,issue,author,package,created date,delay from created date,latest update time,'
+                       'delay from latest update,status,bot advice\n')
+        file_out.writelines([item.output() for item in sorted(issue_status, key=_key_select)])
+
+
 class IssueStatus:
     link = _NULL
     author = _NULL
@@ -36,6 +53,7 @@ class IssueStatus:
     whether_author_comment = True
     issue_object = _NULL
     labels = _NULL
+    assignee = _NULL
 
     def output(self):
         return '{},{},{},{},{},{},{},{},{},{}\n'.format(self.language, self.link, self.author,
@@ -46,6 +64,13 @@ def output(self):
                                                         self.delay_from_latest_update,
                                                         self.status, self.bot_advice)
 
+
+    def output_python(self):
+        return '| [#{}]({}) | {} | {} | {} | {} | {} | {} |\n'.format(self.link.split('/')[-1], self.link, self.author, 
+                                                                      self.package, self.assignee, self.bot_advice, 
+                                                                      str(date.fromtimestamp(self.create_date)), 
+                                                                      self.delay_from_create_date)
+
 
 def _extract(str_list, key_word):
     for item in str_list:
@@ -86,8 +111,7 @@ def _extract_author_latest_comment(comments):
 def _whether_author_comment(comments):
     q = set(comment.user.login for comment in comments)
     diff = q.difference(_PYTHON_SDK_ADMINISTRATORS)
-
-    return  len(diff) > 0
+    return len(diff) > 0
 
 def _latest_comment_time(comments, delay_from_create_date):
     q = [(comment.updated_at.timestamp(), comment.user.login)
@@ -96,13 +120,16 @@ def _latest_comment_time(comments, delay_from_create_date):
 
     return delay_from_create_date if not q else int((time.time() - q[-1][0]) / 3600 / 24)
 
+
 def main():
     # get latest issue status
     g = Github(os.getenv('TOKEN'))  # please fill user_token
-    repo = g.get_repo('Azure/sdk-release-request')
-    label1 = repo.get_label('ManagementPlane')
-    open_issues = repo.get_issues(state='open', labels=[label1])
+    sdk_repo = g.get_repo('Azure/sdk-release-request')
+    rest_repo = g.get_repo('Azure/azure-rest-api-specs')
+    label1 = sdk_repo.get_label('ManagementPlane')
+    open_issues = sdk_repo.get_issues(state='open', labels=[label1])
     issue_status = []
+    issue_status_python = []
     duplicated_issue = dict()
     start_time = time.time()
     for item in open_issues:
@@ -124,7 +151,9 @@ def main():
         issue.issue_object = item
         issue.labels = [label.name for label in item.labels]
         issue.days_from_latest_commit = _latest_comment_time(item.get_comments(), issue.delay_from_create_date)
-
+        if item.assignee:
+            issue.assignee = item.assignee.login
+
         issue_status.append(issue)
         key = (issue.language, issue.package)
         duplicated_issue[key] = duplicated_issue.get(key, 0) + 1
@@ -135,18 +164,43 @@ def main():
     # rule2: if latest comment is from author, need response asap
     # rule3: if comment num is 0, it is new issue, better to deal with it asap
     # rule4: if delay from latest update is over 7 days, better to deal with it soon.
-    # rule5: if delay from created date is over 30 days and owner never reply, close it.
-    # rule6: if delay from created date is over 15 days and owner never reply, remind owner to handle it.
+    # rule5: if delay from created date is over 30 days, better to close.
+    # rule6: if delay from created date is over 30 days and owner never reply, close it.
+    # rule7: if delay from created date is over 15 days and owner never reply, remind owner to handle it.
     for item in issue_status:
         if item.status == 'release':
             item.bot_advice = 'better to release asap.'
         elif item.author == item.author_latest_comment:
             item.bot_advice = 'new comment for author.'
-        elif item.comment_num == 0:
+        elif item.comment_num == 0 and 'Python' in item.labels:
             item.bot_advice = 'new issue and better to confirm quickly.'
+            if 'auto-link' not in item.labels:
+                try:
+                    readme_link = update_issue_body(sdk_repo, rest_repo, item.link)
+                except Exception as e:
+                    item.bot_advice = 'failed to modify the body of the new issue. Please modify manually'
+                    item.labels.append('attention')
+                item.labels.append('auto-link')
+                item.issue_object.set_labels(*item.labels)
+            else:
+                try:
+                    readme_link = find_readme_link(sdk_repo, item.issue_object.number)
+                except Exception as e:
+                    print('Issue: {}  updates body failed'.format(item.issue_object.number))
+                    item.bot_advice = 'failed to find Readme link, Please check !!'
+                    item.labels.append('attention')
+                    continue
+            try:
+                reply = rg.begin_reply_generate(issue_object=item.issue_object, rest_repo=rest_repo, readme_link=readme_link)
+            except Exception as e:
+                item.bot_advice = 'auto reply failed, Please intervene manually !!'
+                print('Error from auto reply ========================')
+                print('Issue:{}'.format(item.issue_object.number))
+                print(traceback.format_exc())
+                print('==============================================')
         elif item.delay_from_latest_update >= 7:
             item.bot_advice = 'delay for a long time and better to handle now.'
-  
+
         if item.days_from_latest_commit >= 30 and item.language == 'Python' and '30days attention' not in item.labels:
             item.labels.append('30days attention')
             item.issue_object.set_labels(*item.labels)
@@ -157,17 +211,17 @@ def main():
                                              ' please deal with it ASAP. We will close the issue if there is still no response after 15 days!')
             item.labels.append('15days attention')
             item.issue_object.set_labels(*item.labels)
-
-
+
         # judge whether there is duplicated issue for same package
         if item.package != _NULL and duplicated_issue.get((item.language, item.package)) > 1:
             item.bot_advice = f'Warning:There is duplicated issue for {item.package}. ' + item.bot_advice
+
+        if item.language == 'Python':
+            issue_status_python.append(item)
 
     # output result
-    with open(_FILE_OUT, 'w') as file_out:
-        file_out.write('language,issue,author,package,created date,delay from created date,latest update time,'
-                       'delay from latest update,status,bot advice\n')
-        file_out.writelines([item.output() for item in sorted(issue_status, key=_key_select)])
+    output_python_md(issue_status_python)
+    output_csv(issue_status)
 
     # commit to github
     print_check('git add .')
@@ -179,7 +233,7 @@ def main():
                                              blob_name=_FILE_OUT)
     with open(_FILE_OUT, 'rb') as data:
         blob.upload_blob(data, overwrite=True)
-
+        
 
 if __name__ == '__main__':
     main()

scripts/release_issue_status/reply_generator.py

@@ -0,0 +1,102 @@
+import re
+
+link_dict = {}
+
+
+def begin_reply_generate(issue_object, rest_repo, readme_link):
+    def weather_change_readme():
+        # to see whether need change readme
+        contents = str(rest_repo.get_contents(link_dict['readme_path']).decoded_content)
+        pattern_tag = re.compile(r'tag: package-[\w+-.]+')
+        package_tag = pattern_tag.search(contents).group()
+        package_tag = package_tag.split(':')[1].strip()
+        readme_python_contents = str(rest_repo.get_contents(link_dict['readme_python_path']).decoded_content)
+        whether_multi_api = 'multi-api' in readme_python_contents
+        whether_same_tag = package_tag == link_dict['readme_tag']
+        whether_change_readme = not whether_same_tag or whether_multi_api and not 'MultiAPI' in labels
+        return whether_change_readme
+
+    # parse owner's comment and get links
+    def get_links():
+        comment_body = issue_object.body
+        pattern_readme = re.compile(r'/specification/([\w-]+/)+readme.md')
+        pattern_resource_manager = re.compile(r'/specification/([\w-]+/)+resource-manager')
+        pattern_tag = re.compile(r'package-[\w+-.]+')
+        readme_path = pattern_readme.search(readme_link).group()
+        readme_tag = pattern_tag.search(comment_body).group()
+        resource_manager = pattern_resource_manager.search(readme_link).group()
+        link_dict['readme_path'] = readme_path
+        link_dict['readme_python_path'] = readme_path[:readme_path.rfind('/')] + '/readme.python.md'
+        link_dict['readme_tag'] = readme_tag
+        link_dict['resource_manager'] = resource_manager
+
+    def get_latest_pr_from_readme():
+        commits = rest_repo.get_commits(path=link_dict['resource_manager'])
+        latest_commit = [commit for commit in commits][0]
+        latest_pr_brief = latest_commit.commit.message
+        latest_pr_number = re.findall('\(\#[0-9]+\)', latest_pr_brief)
+        latest_pr_number_int = []
+        for number in latest_pr_number:
+            number = int(re.search('\d+', number).group())
+            latest_pr_number_int.append(number)
+        latest_pr_number_int.sort()
+
+        return latest_pr_number_int[-1]
+
+    def latest_pr_parse():
+        latest_pr = rest_repo.get_issue(latest_pr_number)
+        latest_pr_comments = latest_pr.get_comments()
+        b = [i for i in latest_pr_comments]
+        for comment in latest_pr_comments:
+            if '<h3>Swagger Generation Artifacts</h3>' in comment.body:
+                return swagger_generator_parse(comment.body, issue_object, latest_pr_number)
+
+    def swagger_generator_parse(context, issue_object, latest_pr_number):
+        track1_info_model = ''
+        try:
+            if '<b> azure-sdk-for-python</b>' in context:
+                pattern_python_t1 = re.compile('<b> azure-sdk-for-python</b>.+?</details>', re.DOTALL)
+                python_t1 = re.search(pattern_python_t1, context).group()
+                prttern_python_track1 = re.compile('<ul>\s+?<li>\s+?<a.+</ul>', re.DOTALL)
+                python_track1_info = re.search(prttern_python_track1, python_t1).group()
+                track1_info_model = '<details open><summary><b> python-track1</b></summary>{} </details>'.format(
+                    python_track1_info)
+        except Exception as e:
+            print('track1 generate error')
+        pattern_python = re.compile('<b> azure-sdk-for-python-track2</b>.+?</details>', re.DOTALL)
+        python = re.search(pattern_python, context).group()
+        # the way that reply not contains [Release SDK Changes]
+        # pattern_python_track2 = re.compile('<ul>\s*?<li>\s*?<a.*</ul>', re.DOTALL)
+        pattern_python_track2 = re.compile('<b>track2_.*</ul>', re.DOTALL)
+        python_track2_info = re.search(pattern_python_track2, python).group()
+        track2_info_model = '<details open><summary><b> python-track2</b></summary>{} </details>'.format(
+            python_track2_info)
+        info_model = 'hi @{} Please check the package whether works well and the changelog info is as below:\n' \
+                     '{}\n{}\n' \
+                     '\n* (The version of the package is only a temporary version for testing)\n' \
+                     '\nhttps://github.com/Azure/azure-rest-api-specs/pull/{}\n' \
+            .format(issue_object.user.login, track1_info_model, track2_info_model, str(latest_pr_number))
+
+        return info_model
+
+    def reply_owner():
+        issue_object.create_comment(reply_content)
+
+    def add_label(label_name):
+        if label_name not in labels:
+            labels.append(label_name)
+        issue_object.set_labels(*labels)
+
+    get_links()
+    whether_change_readme = weather_change_readme()
+    # get issue labels
+    labels = [label.name for label in issue_object.labels]
+
+    if not whether_change_readme:
+        # get the latest pr about service
+        latest_pr_number = get_latest_pr_from_readme()
+        reply_content = latest_pr_parse()
+        reply_owner()
+        add_label('P1')
+    else:
+        print('issue {} need config readme***********'.format(issue_object.number))

scripts/release_issue_status/update_issue_body.py

@@ -0,0 +1,88 @@
+import re
+
+
+def update_issue_body(sdk_repo, rest_repo, issue_number):
+    # Get Issue Number
+    issue_info = sdk_repo.get_issue(number=issue_number)
+    issue_body = issue_info.body
+
+    issue_body_list = [i for i in issue_body.split("\n") if i]
+    # Get the link in issue body
+    for row in issue_body_list:
+        if 'link' in row.lower():
+            link = row.split(":", 1)[-1].strip()
+            break
+
+    if link.count('https') > 1:
+        link = link.split(']')[0]
+        link = link.replace('[', "").replace(']', "").replace('(', "").replace(')', "")
+
+    package_name, readme_link = get_pkname_and_readme_link(rest_repo, link)
+
+    issue_body_list.insert(0, f'\n{readme_link.replace("/readme.md", "")}')
+    issue_body_list.insert(1, package_name)
+    issue_body_up = ''
+    for raw in issue_body_list:
+        if raw == '---\r' or raw == '---':
+            issue_body_up += '\n'
+        issue_body_up += raw + '\n'
+
+    issue_info.edit(body=issue_body_up)
+    return readme_link
+
+
+def get_pkname_and_readme_link(rest_repo, link):
+    # change commit link to pull json link(i.e. https://github.com/Azure/azure-rest-api-specs/commit/77f5d3b5d2fbae17621ea124485788f496786758#diff-708c2fb843b022cac4af8c6f996a527440c1e0d328abb81f54670747bf14ab1a)
+    pk_name = ''
+    if 'commit' in link:
+        commit_sha = link.split('commit/')[-1]
+        commit = rest_repo.get_commit(commit_sha)
+        link = commit.files[0].blob_url
+
+    # if link is a pr, it can get both pakeage name and readme link.
+    if 'pull' in link:
+        pr_number = int(link.replace("https://github.com/Azure/azure-rest-api-specs/pull/", "").strip('/'))
+
+        # Get Readme link
+        pr_info = rest_repo.get_pull(number=pr_number)
+        pk_url_name = ''
+        for pr_changed_file in pr_info.get_files():
+            contents_url = pr_changed_file.contents_url
+            if '/resource-manager' in contents_url:
+                pk_url_name1 = re.findall(r'/specification/(.*?)/resource-manager/', contents_url)[0]
+                if not pk_url_name:
+                    pk_url_name = pk_url_name1
+                elif pk_url_name != pk_url_name1:
+                    print("\nexists multiple package names: {}, {} \n".format(pk_url_name, pk_url_name1))
+                    raise Exception('Not find readme link, because it exists multiple package names')
+
+        readme_link = 'https://github.com/Azure/azure-rest-api-specs/blob/main/specification/{}/' \
+                      'resource-manager/readme.python.md'.format(pk_url_name)
+
+
+    # if link is a rest url(i.e. https://github.com/Azure/azure-rest-api-specs/blob/main/specification/xxx/resource-manager/readme.python.md)
+    elif '/resource-manager' not in link:
+        # (i.e. https://github.com/Azure/azure-rest-api-specs/tree/main/specification/xxxx)
+        readme_link = link + '/resource-manager/readme.python.md'
+    else:
+        readme_link = link.split('/resource-manager')[0] + '/resource-manager/readme.python.md'
+    # get the package name by readme link
+    readme_link_part = '/specification' + readme_link.split('/specification')[-1]
+    readme_contents = str(rest_repo.get_contents(readme_link_part).decoded_content)
+    pk_name = re.findall(r'package-name: (.*?)\\n', readme_contents)[0]
+    readme_link = readme_link.replace('python.', '')
+
+    return pk_name, readme_link
+
+
+def find_readme_link(sdk_repo, issue_number):
+    # Get Issue Number
+    issue_info = sdk_repo.get_issue(number=issue_number)
+    issue_body = issue_info.body
+    issue_body_list = issue_body.split("\n")
+    for row in issue_body_list:
+        if 'resource-manager' in row:
+            readme_link = row + '/readme.md'
+            return readme_link
+    else:
+        raise Exception('Not find readme link,please check')