From 79169b1c9023177a6d6aaeb444168df82d5677fe Mon Sep 17 00:00:00 2001 From: ytshen Date: Fri, 12 Apr 2024 20:44:01 +0800 Subject: [PATCH 1/7] update actions deploy --- .github/workflows/main.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c810ee6..c0e5eb6 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -65,9 +65,10 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Publish in Repository - uses: peaceiris/actions-gh-pages@v3 + uses: peaceiris/actions-gh-pages@v4 with: - deploy_key: ${{ secrets.ACTIONS_DEPLOY_KEY }} + github_token: ${{ secrets.GITHUB_TOKEN }} +# deploy_key: ${{ secrets.ACTIONS_DEPLOY_KEY }} publish_branch: publish user_name: 'github-actions[bot]' user_email: 'github-actions[bot]@users.noreply.github.com' From b57fe304795aeb1ff8cec261fd8a56b8910f2952 Mon Sep 17 00:00:00 2001 From: ytshen Date: Fri, 12 Apr 2024 21:11:38 +0800 Subject: [PATCH 2/7] improve codes --- .github/workflows/main.yml | 2 +- file_cache.py | 27 ++++++++ github_operations.py | 111 ++++++++++++++++++++++++++++++ main.py | 137 +------------------------------------ 4 files changed, 141 insertions(+), 136 deletions(-) create mode 100644 file_cache.py create mode 100644 github_operations.py diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c0e5eb6..580c87e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -3,7 +3,7 @@ name: Zotero Addons Scraping on: workflow_dispatch: # manual schedule: - - cron: '25 19 * * *' # 3:25(beijing) + - cron: '25 4,10,19 * * *' # +8(beijing) push: branches: [master] diff --git a/file_cache.py b/file_cache.py new file mode 100644 index 0000000..16d750b --- /dev/null +++ b/file_cache.py @@ -0,0 +1,27 @@ +import os +import shutil +import hashlib + + +def folder_filename_hash(dir): + filenames = sorted(os.listdir(dir)) + folder_hash = hashlib.new('sha256') + for filename in filenames: + hash_obj = hashlib.new('sha256') + hash_obj.update(filename.encode('utf-8')) + folder_hash.update(hash_obj.hexdigest().encode('utf-8')) + return folder_hash.hexdigest() + + +def update_cache(cache_directory, runtime_xpi_directory, cache_hash_filename): + if not cache_directory or not runtime_xpi_directory or cache_directory == runtime_xpi_directory: + return + try: + shutil.rmtree(cache_directory) + shutil.move(runtime_xpi_directory, cache_directory) + folder_hash = folder_filename_hash(cache_directory) + with open(os.path.join(cache_directory, cache_hash_filename), 'w') as file: + file.write(folder_hash) + print(folder_hash) + except Exception as e: + print(f'update cache failed: {e}') diff --git a/github_operations.py b/github_operations.py new file mode 100644 index 0000000..5013760 --- /dev/null +++ b/github_operations.py @@ -0,0 +1,111 @@ +import json +import time +import requests + + +def github_api_headers(**kwargs): + result = { + 'Accept': 'application/vnd.github+json', + 'X-GitHub-Api-Version': '2022-11-28', + } + if github_token := kwargs.get('github_token'): + result['Authorization'] = f'token {github_token}' + return result + + +def report_issue(github_repository: str, title: str, body: str, **kwargs): + if not github_repository: + print('report issue repository not found') + return + try: + response = requests.post(f'https://api.github.com/repos/{github_repository}/issues', + headers=github_api_headers(github_token=kwargs.get('github_token')), + json={ + 'title': title, + 'body': body + }) + + if response.status_code == 201: + print('Issue created successfully.') + print('Issue URL:', response.json()['html_url']) + else: + print('Failed to create issue.') + print('Response:', response.content) + except Exception as e: + print(f'report issue failed: {e}') + + +def upload_json_to_release(github_repository, release_id, upload_file_name, upload_file, **kwargs): + headers = github_api_headers(github_token=kwargs.get('github_token')) + headers["Content-Type"] = "application/octet-stream" + upload_url = (f'https://uploads.github.com/repos/{github_repository}' + f'/releases/{release_id}/assets?name={upload_file_name}') + try: + with open(upload_file, "rb") as file: + upload_resp = requests.post(upload_url, data=file, headers=headers) + if upload_resp.status_code != 201: + print(f'upload release assets code: {upload_resp.status_code}') + else: + print('upload release assets succeed') + except Exception as e: + print(f'upload release assets failed: {e}') + + +def create_release(github_repository, **kwargs): + create_release_url = f'https://api.github.com/repos/{github_repository}/releases' + cur_time = int(time.time()) + param = { + 'tag_name': f'{cur_time}', + 'target_commitish': 'master', + 'name': f'{cur_time}', + 'body': f'![](https://img.shields.io/github/downloads/{github_repository}/{cur_time}/total?label=downloads)\n' + f'publish addon_infos.json', + 'draft': False, + 'prerelease': False, + 'generate_release_notes': False, + } + try: + create_resp = requests.post(create_release_url, json=param, + headers=github_api_headers(github_token=kwargs.get('github_token'))) + if create_resp.status_code == 201: + create_release_info = json.loads(create_resp.content) + release_id = create_release_info['id'] + return release_id + else: + print(f'create release code: {create_resp.status_code}') + except Exception as e: + print(f'create release failed: {e}') + + +def delete_cache(github_repository, github_token, remain_count=2): + headers = github_api_headers(github_token=github_token) + get_caches_url = (f'https://api.github.com/repos/{github_repository}' + f'/actions/caches?per_page=100&sort=last_accessed_at&direction=desc') + try: + caches_resp = requests.get(get_caches_url, headers=headers) + caches = json.loads(caches_resp.content) + if caches.get('total_count', 0) < remain_count: + return + delete_cache_url = f'https://api.github.com/repos/{github_repository}/actions/caches/' + for cache in caches.get('actions_caches', [])[remain_count:]: + cache_key = cache.get('key') + if cache_id := cache.get('id'): + try: + delete_cache_resp = requests.delete(f'{delete_cache_url}{cache_id}', headers=headers) + if delete_cache_resp.status_code == 204: + print(f'delete {cache_key} succeed') + else: + print(f'delete {cache_key} failed: {delete_cache_resp.text}') + except Exception as e: + print(f'delete cache for {cache_key} failed: {e}') + except Exception as e: + print(f'get caches failed: {e}') + + +def rate_limit(github_token): + try: + resp = requests.get('https://api.github.com/rate_limit', headers=github_api_headers(github_token=github_token)) + rate = json.loads(resp.content) + print(f'token rate {rate.get("rate")}') + except Exception as e: + print(f'get rate limit failed: {e}') diff --git a/main.py b/main.py index 4aab696..660cdb6 100644 --- a/main.py +++ b/main.py @@ -1,48 +1,11 @@ import concurrent.futures -import json -import os -import time import urllib.parse -import requests import argparse -import shutil -import hashlib from addon_info import * from fallback_infos import fallback_if_need from moz_addons import addon_details - - -def github_api_headers(**kwargs): - result = { - 'Accept': 'application/vnd.github+json', - 'X-GitHub-Api-Version': '2022-11-28', - } - if github_token := kwargs.get('github_token'): - result['Authorization'] = f'token {github_token}' - return result - - -def report_issue(repo: str, title: str, body: str, **kwargs): - if not repo: - print('report issue repo not found') - return - try: - response = requests.post(f'https://api.github.com/repos/{repo}/issues', - headers=github_api_headers(github_token=kwargs.get('github_token')), - json={ - 'title': title, - 'body': body - }) - - # 检查响应 - if response.status_code == 201: - print('Issue created successfully.') - print('Issue URL:', response.json()['html_url']) - else: - print('Failed to create issue.') - print('Response:', response.content) - except Exception as e: - print(f'report issue failed: {e}') +from github_operations import * +from file_cache import * def download_xpi(xpi_url: str, download_dir: str, unique_name: str, force_download: bool, **kwargs): @@ -247,102 +210,6 @@ def parse_addon_infos(input_dir, output_filepath, **kwargs): return addon_infos -def upload_json_to_release(github_repository, release_id, upload_file_name, upload_file, **kwargs): - headers = github_api_headers(github_token=kwargs.get('github_token')) - headers["Content-Type"] = "application/octet-stream" - upload_url = f'https://uploads.github.com/repos/{github_repository}/releases/{release_id}/assets?name={upload_file_name}' - try: - with open(upload_file, "rb") as file: - upload_resp = requests.post(upload_url, data=file, headers=headers) - if upload_resp.status_code != 201: - print(f'upload release assets code: {upload_resp.status_code}') - else: - print('upload release assets succeed') - except Exception as e: - print(f'upload release assets failed: {e}') - - -def create_release(github_repository, **kwargs): - create_release_url = f'https://api.github.com/repos/{github_repository}/releases' - cur_time = int(time.time()) - param = { - 'tag_name': f'{cur_time}', - 'target_commitish': 'master', - 'name': f'{cur_time}', - 'body': f'![](https://img.shields.io/github/downloads/{github_repository}/{cur_time}/total?label=downloads)\npublish addon_infos.json', - 'draft': False, - 'prerelease': False, - 'generate_release_notes': False, - } - try: - create_resp = requests.post(create_release_url, json=param, headers=github_api_headers(github_token=kwargs.get('github_token'))) - if create_resp.status_code == 201: - create_release_info = json.loads(create_resp.content) - release_id = create_release_info['id'] - return release_id - else: - print(f'create release code: {create_resp.status_code}') - except Exception as e: - print(f'create release failed: {e}') - - -def folder_filename_hash(dir): - filenames = sorted(os.listdir(dir)) - folder_hash = hashlib.new('sha256') - for filename in filenames: - hash_obj = hashlib.new('sha256') - hash_obj.update(filename.encode('utf-8')) - folder_hash.update(hash_obj.hexdigest().encode('utf-8')) - return folder_hash.hexdigest() - - -def update_cache(cache_directory, runtime_xpi_directory, cache_hash_filename): - if not cache_directory or not runtime_xpi_directory or cache_directory == runtime_xpi_directory: - return - try: - shutil.rmtree(cache_directory) - shutil.move(runtime_xpi_directory, cache_directory) - folder_hash = folder_filename_hash(cache_directory) - with open(os.path.join(cache_directory, cache_hash_filename), 'w') as file: - file.write(folder_hash) - print(folder_hash) - except Exception as e: - print(f'update cache failed: {e}') - - -def delete_cache(github_repository, github_token, remain_count=2): - headers = github_api_headers(github_token=github_token) - get_caches_url = f'https://api.github.com/repos/{github_repository}/actions/caches?per_page=100&sort=last_accessed_at&direction=desc' - try: - caches_resp = requests.get(get_caches_url, headers=headers) - caches = json.loads(caches_resp.content) - if caches.get('total_count', 0) < remain_count: - return - delete_cache_url = f'https://api.github.com/repos/{github_repository}/actions/caches/' - for cache in caches.get('actions_caches', [])[remain_count:]: - cache_key = cache.get('key') - if cache_id := cache.get('id'): - try: - delete_cache_resp = requests.delete(f'{delete_cache_url}{cache_id}', headers=headers) - if delete_cache_resp.status_code == 204: - print(f'delete {cache_key} succeed') - else: - print(f'delete {cache_key} failed: {delete_cache_resp.text}') - except Exception as e: - print(f'delete cache for {cache_key} failed: {e}') - except Exception as e: - print(f'get caches failed: {e}') - - -def rate_limit(github_token): - try: - resp = requests.get('https://api.github.com/rate_limit', headers=github_api_headers(github_token=github_token)) - rate = json.loads(resp.content) - print(f'token rate {rate.get("rate")}') - except Exception as e: - print(f'get rate limit failed: {e}') - - if __name__ == '__main__': parser = argparse.ArgumentParser(description='params') parser.add_argument('--github_repository', nargs='?', type=str, required=True, help='github repository') From a85ebdb43977910a45ac4e060a345d4f4e9def05 Mon Sep 17 00:00:00 2001 From: ytshen Date: Fri, 12 Apr 2024 21:14:09 +0800 Subject: [PATCH 3/7] update config --- .github/workflows/main.yml | 1 + main.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 580c87e..ffa3380 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -58,6 +58,7 @@ jobs: --cache_directory caches \ --runtime_xpi_directory xpis \ --cache_lockfile caches_lockfile \ + --create_release True \ # --previous_info_urls \ # https://raw.githubusercontent.com/${GITHUB_REPOSITORY}/publish/addon_infos.json \ # https://github.com/${GITHUB_REPOSITORY}/releases/latest/download/addon_infos.json diff --git a/main.py b/main.py index 660cdb6..3774961 100644 --- a/main.py +++ b/main.py @@ -221,6 +221,7 @@ if __name__ == '__main__': parser.add_argument('--cache_lockfile', nargs='?', default="caches_lockfile", type=str, help='hashfile for caches') parser.add_argument('--runtime_xpi_directory', nargs='?', default="xpis", type=str, help='folder for download xpi') parser.add_argument('--previous_info_urls', nargs='+', default=[], help='previous published info json to fallback') + parser.add_argument('--create_release', nargs='?', default=True, type=bool, help='create release in github') args = parser.parse_args() @@ -246,7 +247,7 @@ if __name__ == '__main__': runtime_xpi_directory=args.runtime_xpi_directory, previous_info_urls=args.previous_info_urls) - if release_id := create_release(args.github_repository, github_token=args.github_token): + if args.create_release and (release_id := create_release(args.github_repository, github_token=args.github_token)): upload_json_to_release(args.github_repository, release_id, upload_file_name='addon_infos.json', From 9eb736c410ab5c4e29da2e67b4cbbeacf40f2e1a Mon Sep 17 00:00:00 2001 From: ytshen Date: Fri, 12 Apr 2024 21:21:09 +0800 Subject: [PATCH 4/7] rm deploy key in readme --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index b974b50..7b54cf3 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,6 @@ e.g., ## Usage 1. `fork` this repository 2. Enable actions and scheduled workflow in `Actions` page of forked repository -3. Add a `Deploy key` in forked repository and named it with `ACTIONS_DEPLOY_KEY` ## Generated Format The output format for add-on information follows the format specified in [zotero-chinese/zotero-plugins](https://github.com/zotero-chinese/zotero-plugins). From 756c1fd17931588cd1140e7d4158979b1dc9b9a2 Mon Sep 17 00:00:00 2001 From: ytshen Date: Fri, 12 Apr 2024 22:50:36 +0800 Subject: [PATCH 5/7] auto rm release --- github_operations.py | 25 +++++++++++++++++++++++++ main.py | 1 + 2 files changed, 26 insertions(+) diff --git a/github_operations.py b/github_operations.py index 5013760..ec44819 100644 --- a/github_operations.py +++ b/github_operations.py @@ -102,6 +102,31 @@ def delete_cache(github_repository, github_token, remain_count=2): print(f'get caches failed: {e}') +def delete_release(github_repository, github_token, remain_count=2): + headers = github_api_headers(github_token=github_token) + get_release_url = (f'https://api.github.com/repos/{github_repository}' + f'/releases?per_page=100&page=1') + try: + releases_resp = requests.get(get_release_url, headers=headers) + releases = json.loads(releases_resp.content) + if len(releases) < remain_count: + return + delete_release_url = f'https://api.github.com/repos/{github_repository}/releases/' + for release in releases[remain_count:]: + release_tag = release.get('tag_name') + if release_id := release.get('id'): + try: + delete_release_resp = requests.delete(f'{delete_release_url}{release_id}', headers=headers) + if delete_release_resp.status_code == 204: + print(f'delete {release_tag} succeed') + else: + print(f'delete {release_tag} failed: {delete_release_resp.text}') + except Exception as e: + print(f'delete cache for {release_tag} failed: {e}') + except Exception as e: + print(f'get releases failed: {e}') + + def rate_limit(github_token): try: resp = requests.get('https://api.github.com/rate_limit', headers=github_api_headers(github_token=github_token)) diff --git a/main.py b/main.py index 3774961..6626ac5 100644 --- a/main.py +++ b/main.py @@ -253,6 +253,7 @@ if __name__ == '__main__': upload_file_name='addon_infos.json', upload_file=args.output, github_token=args.github_token) + delete_release(args.github_repository, github_token=args.github_token) update_cache(args.cache_directory, args.runtime_xpi_directory, args.cache_lockfile) From 29c768280293a77b8f3d074b903fd47ca96184b4 Mon Sep 17 00:00:00 2001 From: ytshen Date: Fri, 12 Apr 2024 23:16:14 +0800 Subject: [PATCH 6/7] automatically delete old tags --- github_operations.py | 36 +++++++++++++++++++++++++++++++++--- main.py | 1 + 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/github_operations.py b/github_operations.py index ec44819..d452a18 100644 --- a/github_operations.py +++ b/github_operations.py @@ -118,15 +118,45 @@ def delete_release(github_repository, github_token, remain_count=2): try: delete_release_resp = requests.delete(f'{delete_release_url}{release_id}', headers=headers) if delete_release_resp.status_code == 204: - print(f'delete {release_tag} succeed') + print(f'delete release {release_tag} succeed') else: - print(f'delete {release_tag} failed: {delete_release_resp.text}') + print(f'delete release {release_tag} failed: {delete_release_resp.text}') except Exception as e: - print(f'delete cache for {release_tag} failed: {e}') + print(f'delete release for {release_tag} failed: {e}') except Exception as e: print(f'get releases failed: {e}') +def delete_tag(github_repository, github_token, remain_count=2): + headers = github_api_headers(github_token=github_token) + get_tags_url = (f'https://api.github.com/repos/{github_repository}' + f'/git/refs/tags') + try: + tags_response = requests.get(get_tags_url, headers=headers) + tags = json.loads(tags_response.content) + if len(tags) < remain_count: + return + delete_tag_url = f'https://api.github.com/repos/{github_repository}/git/refs/tags/' + for tag in tags[remain_count:]: + if ref := tag.get('ref').replace('refs/tags/', ''): + if len(ref) < 10: + continue + try: # automatically tag name is timestamp + int(ref) + except ValueError: + continue + try: + delete_tag_resp = requests.delete(f'{delete_tag_url}{ref}', headers=headers) + if delete_tag_resp.status_code == 204: + print(f'delete tag {ref} succeed') + else: + print(f'delete tag {ref} failed: {delete_tag_resp.text}') + except Exception as e: + print(f'delete tag for {ref} failed: {e}') + except Exception as e: + print(f'get tags failed: {e}') + + def rate_limit(github_token): try: resp = requests.get('https://api.github.com/rate_limit', headers=github_api_headers(github_token=github_token)) diff --git a/main.py b/main.py index 6626ac5..8b7632c 100644 --- a/main.py +++ b/main.py @@ -254,6 +254,7 @@ if __name__ == '__main__': upload_file=args.output, github_token=args.github_token) delete_release(args.github_repository, github_token=args.github_token) + delete_tag(args.github_repository, github_token=args.github_token) update_cache(args.cache_directory, args.runtime_xpi_directory, args.cache_lockfile) From 326d17ab99a79a9abb5a82163b1338efdfdeccd4 Mon Sep 17 00:00:00 2001 From: ytshen Date: Fri, 12 Apr 2024 23:22:16 +0800 Subject: [PATCH 7/7] fix1 --- main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main.py b/main.py index 8b7632c..b33014a 100644 --- a/main.py +++ b/main.py @@ -248,13 +248,13 @@ if __name__ == '__main__': previous_info_urls=args.previous_info_urls) if args.create_release and (release_id := create_release(args.github_repository, github_token=args.github_token)): + delete_release(args.github_repository, github_token=args.github_token) + delete_tag(args.github_repository, github_token=args.github_token) upload_json_to_release(args.github_repository, release_id, upload_file_name='addon_infos.json', upload_file=args.output, github_token=args.github_token) - delete_release(args.github_repository, github_token=args.github_token) - delete_tag(args.github_repository, github_token=args.github_token) update_cache(args.cache_directory, args.runtime_xpi_directory, args.cache_lockfile)