From bd7f69e14d480acbc71f08de658e1de7bdbd5dce Mon Sep 17 00:00:00 2001 From: ytshen <> Date: Thu, 26 Dec 2024 21:09:14 +0800 Subject: [PATCH 1/5] fetch xpi from xpi's update json --- main.py | 89 ++++++++++++++++++++++++++++++++++----------------- moz_addons.py | 30 ++++++++++++++--- 2 files changed, 86 insertions(+), 33 deletions(-) diff --git a/main.py b/main.py index 5c3e0e6..297075e 100644 --- a/main.py +++ b/main.py @@ -3,7 +3,7 @@ import urllib.parse import argparse from addon_info import * from fallback_infos import fallback_if_need -from moz_addons import addon_details +from moz_addons import addon_details, compare_versions from github_operations import * from file_cache import * @@ -102,34 +102,10 @@ def parse(plugin: AddonInfo, **kwargs): release_asset = release_assets[0] if 'browser_download_url' not in release_asset: continue - xpi_url = release_asset['browser_download_url'] - release.releaseDate = release_asset['updated_at'] - release.xpiDownloadUrl = { - 'github': xpi_url, - 'ghProxy': 'https://ghproxy.com/?q=' + urllib.parse.quote(xpi_url), - 'kgithub': xpi_url.replace('github.com', 'kkgithub.com'), - } - xpi_filename = f'{plugin.owner}#{plugin.repository}+{release.tagName}@{release_asset["id"]}.xpi' - details = None - priority_sources = ['rdf', 'json'] - if release.targetZoteroVersion == '6': - priority_sources = ['json', 'rdf'] - try: - xpi_filepath = download_xpi(xpi_url=xpi_url, - download_dir=kwargs.get('runtime_xpi_directory'), - unique_name=xpi_filename, - force_download=False, - cache_dir=kwargs.get('cache_directory')) - details = addon_details(xpi_filepath, priority_sources=priority_sources) - except: - try: - xpi_filepath = download_xpi(xpi_url=xpi_url, - download_dir=kwargs.get('runtime_xpi_directory'), - unique_name=xpi_filename, - force_download=True) - details = addon_details(xpi_filepath, priority_sources=priority_sources) - except Exception as e: - print(f'fetch addon detail of {plugin.repo} with {xpi_url} failed: {e}') + details = parse_xpi_detail( + plugin=plugin, release=release, release_asset=release_asset, + **kwargs, + ) if details: if detail_id := details.id: @@ -180,6 +156,61 @@ def parse(plugin: AddonInfo, **kwargs): return plugin +def parse_xpi_detail(plugin: AddonInfo, release: AddonInfoRelease, release_asset, **kwargs): + priority_sources = ['rdf', 'json'] + if release.targetZoteroVersion == '6': + priority_sources = ['json', 'rdf'] + + def xpi_detail(xpi_url, xpi_filename): + try: + xpi_filepath = download_xpi(xpi_url=xpi_url, + download_dir=kwargs.get('runtime_xpi_directory'), + unique_name=xpi_filename, + force_download=False, + cache_dir=kwargs.get('cache_directory')) + details = addon_details(xpi_filepath, priority_sources=priority_sources) + return details + except: + try: + xpi_filepath = download_xpi(xpi_url=xpi_url, + download_dir=kwargs.get('runtime_xpi_directory'), + unique_name=xpi_filename, + force_download=True) + details = addon_details(xpi_filepath, priority_sources=priority_sources) + return details + except Exception as e: + print(f'fetch addon detail of {plugin.repo} with {xpi_url} failed: {e}') + + github_xpi_url = release_asset['browser_download_url'] + release.releaseDate = release_asset['updated_at'] + release.xpiDownloadUrl = { + 'github': github_xpi_url, + 'ghProxy': 'https://ghproxy.com/?q=' + urllib.parse.quote(github_xpi_url), + 'kgithub': github_xpi_url.replace('github.com', 'kkgithub.com'), + } + details = xpi_detail(xpi_url=github_xpi_url, + xpi_filename=f'{plugin.owner}#{plugin.repository}+{release.tagName}@{release_asset["id"]}.xpi') + + if details and details.update_url and details.id and details.version: + update_json_resp = requests.get(details.update_url, + headers=github_api_headers(github_token=kwargs.get('github_token'))) + update_json_info = json.loads(update_json_resp.content) + + updates = update_json_info.get('addons', {}).get(details.id, {}).get('updates', []) + xpi_urls = [e.get('update_link') for e in updates if compare_versions(e.get('version'), details.version) > 0] + + for xpi_url in xpi_urls: + update_details = xpi_detail(xpi_url=xpi_url, + xpi_filename=f'{plugin.owner}#{plugin.repository}+update{details.version}.xpi') + if update_details and update_details.check_compatible_for_zotero_version(release.zotero_check_version): + details = update_details + release.xpiDownloadUrl = { + 'github': xpi_url, + } + break + return details + + def fallback(addon_infos, previous_info_url, **kwargs): try: repos_resp = requests.get(previous_info_url, diff --git a/moz_addons.py b/moz_addons.py index 749ec49..ec49d63 100644 --- a/moz_addons.py +++ b/moz_addons.py @@ -43,6 +43,7 @@ class XpiDetail: self.name = None self.version = None self.description = None + self.update_url = None self.min_version = "*" self.max_version = "*" @@ -58,6 +59,10 @@ class XpiDetail: self.version = version if description := details.get('description'): self.description = description + if update_url := details.get('updateURL'): + self.update_url = update_url + if update_url := details.get('update_url'): + self.update_url = update_url if (min_version := details.get('min_version')) and (max_version := details.get('max_version')): if compare_versions(min_version.replace('*', '0'), self.min_version.replace('*', '999')) <= 0: self.min_version = min_version @@ -68,7 +73,7 @@ class XpiDetail: if isinstance(version, int): version = str(version) + '.*' if (min_version := self.min_version.replace('*', '0')) and ( - max_version := self.max_version.replace('*', '999')): + max_version := self.max_version.replace('*', '999')): return (compare_versions(min_version, version.replace('*', '999')) <= 0 <= compare_versions(max_version, version.replace('*', '0'))) @@ -125,8 +130,13 @@ def manifest_from_rdf(addon_path): def detail_from_manifest_json(addon_path, manifest): details = { - "name": manifest.get("name"), "version": manifest.get("version"), "description": manifest.get("description"), - "id": None, "min_version": None, "max_version": None + "name": manifest.get("name"), + "version": manifest.get("version"), + "description": manifest.get("description"), + "id": None, + "min_version": None, + "max_version": None, + "update_url": None, } for location in ("applications", "browser_specific_settings"): if details["id"]: @@ -136,6 +146,7 @@ def detail_from_manifest_json(addon_path, manifest): details["id"] = manifest[location][app].get("id") details["min_version"] = manifest[location][app].get("strict_min_version") details["max_version"] = manifest[location][app].get("strict_max_version") + details["update_url"] = manifest[location][app].get("update_url") break except KeyError: continue @@ -175,7 +186,15 @@ def detail_from_manifest_json(addon_path, manifest): def detail_from_manifest_rdf(manifest): - details = {"id": None, "name": None, "version": None, "description": None, "min_version": None, "max_version": None} + details = { + "id": None, + "name": None, + "version": None, + "description": None, + "min_version": None, + "max_version": None, + "updateURL": None, + } try: doc = minidom.parseString(manifest) @@ -238,6 +257,9 @@ def detail_from_manifest_rdf(manifest): extract_info(node, version_info) update_details(version_info) + if update_url := details.get('updateURL'): + details['update_url'] = update_url + return details except Exception as e: raise e From 830233993231b51195d4ce335d5883f2ff6a6ea0 Mon Sep 17 00:00:00 2001 From: ytshen <> Date: Thu, 26 Dec 2024 21:29:42 +0800 Subject: [PATCH 2/5] update --- main.py | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/main.py b/main.py index 297075e..acb5d0b 100644 --- a/main.py +++ b/main.py @@ -192,22 +192,24 @@ def parse_xpi_detail(plugin: AddonInfo, release: AddonInfoRelease, release_asset xpi_filename=f'{plugin.owner}#{plugin.repository}+{release.tagName}@{release_asset["id"]}.xpi') if details and details.update_url and details.id and details.version: - update_json_resp = requests.get(details.update_url, - headers=github_api_headers(github_token=kwargs.get('github_token'))) - update_json_info = json.loads(update_json_resp.content) + try: + update_json_resp = requests.get(details.update_url, + headers=github_api_headers(github_token=kwargs.get('github_token'))) + update_json_info = json.loads(update_json_resp.content) - updates = update_json_info.get('addons', {}).get(details.id, {}).get('updates', []) - xpi_urls = [e.get('update_link') for e in updates if compare_versions(e.get('version'), details.version) > 0] + updates = update_json_info.get('addons', {}).get(details.id, {}).get('updates', []) + xpi_urls = [e.get('update_link') for e in updates if compare_versions(e.get('version'), details.version) > 0] - for xpi_url in xpi_urls: - update_details = xpi_detail(xpi_url=xpi_url, - xpi_filename=f'{plugin.owner}#{plugin.repository}+update{details.version}.xpi') - if update_details and update_details.check_compatible_for_zotero_version(release.zotero_check_version): - details = update_details - release.xpiDownloadUrl = { - 'github': xpi_url, - } - break + for xpi_url in xpi_urls: + update_details = xpi_detail(xpi_url=xpi_url, + xpi_filename=f'{plugin.owner}#{plugin.repository}+update{details.version}.xpi') + if update_details and update_details.check_compatible_for_zotero_version(release.zotero_check_version): + details = update_details + release.xpiDownloadUrl = { + 'github': xpi_url, + } + break + except: pass return details From fe63c823e36d89e6e33da3b298ec1ddee81aa0f8 Mon Sep 17 00:00:00 2001 From: ytshen Date: Thu, 26 Dec 2024 23:06:08 +0800 Subject: [PATCH 3/5] ignore file download if 404 not found --- main.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/main.py b/main.py index acb5d0b..e60831b 100644 --- a/main.py +++ b/main.py @@ -25,11 +25,12 @@ def download_xpi(xpi_url: str, download_dir: str, unique_name: str, force_downlo return download_filepath response = requests.get(xpi_url, stream=True) - print(f'download {unique_name} from {xpi_url}') - with open(download_filepath, "wb") as file: - for chunk in response.iter_content(chunk_size=1024): - file.write(chunk) - return download_filepath + if response.status_code == 200: + print(f'download {unique_name} from {xpi_url}') + with open(download_filepath, "wb") as file: + for chunk in response.iter_content(chunk_size=1024): + file.write(chunk) + return download_filepath except Exception as e: print(f'download {unique_name} from {xpi_url} failed: {e}') From 66752b9ee0b1efaa1466a3b97aff68bc2934b401 Mon Sep 17 00:00:00 2001 From: ytshen Date: Thu, 26 Dec 2024 23:14:12 +0800 Subject: [PATCH 4/5] ignore xpi detail if id not parsed --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index e60831b..514b427 100644 --- a/main.py +++ b/main.py @@ -204,7 +204,7 @@ def parse_xpi_detail(plugin: AddonInfo, release: AddonInfoRelease, release_asset for xpi_url in xpi_urls: update_details = xpi_detail(xpi_url=xpi_url, xpi_filename=f'{plugin.owner}#{plugin.repository}+update{details.version}.xpi') - if update_details and update_details.check_compatible_for_zotero_version(release.zotero_check_version): + if update_details and update_details.id and update_details.check_compatible_for_zotero_version(release.zotero_check_version): details = update_details release.xpiDownloadUrl = { 'github': xpi_url, From a4be5dee12b148b5291599c9b04b7930956b3704 Mon Sep 17 00:00:00 2001 From: ytshen Date: Thu, 26 Dec 2024 23:17:27 +0800 Subject: [PATCH 5/5] delete some duplicate plugins --- ...t2#Zotero-Addons-Scrape-To-GitHub#zotero-gpt-pro.json | 9 --------- ...ero-Addons-Scrape-To-GitHub#zotero-reference-pro.json | 9 --------- ...#Zotero-Addons-Scrape-To-GitHub#zotero-style-pro.json | 9 --------- 3 files changed, 27 deletions(-) delete mode 100644 addons/syt2#Zotero-Addons-Scrape-To-GitHub#zotero-gpt-pro.json delete mode 100644 addons/syt2#Zotero-Addons-Scrape-To-GitHub#zotero-reference-pro.json delete mode 100644 addons/syt2#Zotero-Addons-Scrape-To-GitHub#zotero-style-pro.json diff --git a/addons/syt2#Zotero-Addons-Scrape-To-GitHub#zotero-gpt-pro.json b/addons/syt2#Zotero-Addons-Scrape-To-GitHub#zotero-gpt-pro.json deleted file mode 100644 index 9cc74a6..0000000 --- a/addons/syt2#Zotero-Addons-Scrape-To-GitHub#zotero-gpt-pro.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "repo": "syt2/Zotero-Addons-Scrape-To-GitHub", - "releases": [ - { - "targetZoteroVersion": "7", - "tagName": "zotero-gpt" - } - ] -} \ No newline at end of file diff --git a/addons/syt2#Zotero-Addons-Scrape-To-GitHub#zotero-reference-pro.json b/addons/syt2#Zotero-Addons-Scrape-To-GitHub#zotero-reference-pro.json deleted file mode 100644 index ca1a0ed..0000000 --- a/addons/syt2#Zotero-Addons-Scrape-To-GitHub#zotero-reference-pro.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "repo": "syt2/Zotero-Addons-Scrape-To-GitHub", - "releases": [ - { - "targetZoteroVersion": "7", - "tagName": "zotero-reference" - } - ] -} \ No newline at end of file diff --git a/addons/syt2#Zotero-Addons-Scrape-To-GitHub#zotero-style-pro.json b/addons/syt2#Zotero-Addons-Scrape-To-GitHub#zotero-style-pro.json deleted file mode 100644 index cbedb5d..0000000 --- a/addons/syt2#Zotero-Addons-Scrape-To-GitHub#zotero-style-pro.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "repo": "syt2/Zotero-Addons-Scrape-To-GitHub", - "releases": [ - { - "targetZoteroVersion": "7", - "tagName": "zotero-style" - } - ] -} \ No newline at end of file