From 234215f7851761e5b8909308ffea67de6e4ac6a3 Mon Sep 17 00:00:00 2001 From: ytshen Date: Sat, 13 Apr 2024 12:34:28 +0800 Subject: [PATCH 1/3] parse min and max version from xpi --- addon_info.py | 7 +++ main.py | 43 ++++++++++----- moz_addons.py | 147 +++++++++++++++++++++++++++++--------------------- 3 files changed, 124 insertions(+), 73 deletions(-) diff --git a/addon_info.py b/addon_info.py index 9bbf9a3..1262bae 100644 --- a/addon_info.py +++ b/addon_info.py @@ -22,6 +22,7 @@ class AddonInfoRelease: def __init__(self, targetZoteroVersion: str, tagName: str, xpiDownloadUrl: dict = None, releaseDate: str = None, id: str = None, xpiVersion: str = None, name: str = None, description: str = None, + minZoteroVersion: str = None, maxZoteroVersion: str = None, **kwargs): self.targetZoteroVersion = targetZoteroVersion self.tagName = tagName @@ -31,6 +32,8 @@ class AddonInfoRelease: self.xpiVersion = xpiVersion self.name = name self.description = description + self.minZoteroVersion = minZoteroVersion + self.maxZoteroVersion = maxZoteroVersion [setattr(self, key, value) for (key, value) in kwargs.items()] @property @@ -60,6 +63,10 @@ class AddonInfoRelease: result["name"] = self.name if self.description: result["description"] = self.description + if self.minZoteroVersion: + result["minZoteroVersion"] = self.minZoteroVersion + if self.maxZoteroVersion: + result["maxZoteroVersion"] = self.maxZoteroVersion return result diff --git a/main.py b/main.py index 4951dd2..9781287 100644 --- a/main.py +++ b/main.py @@ -47,7 +47,8 @@ def parse(plugin: AddonInfo, **kwargs): author_resp = requests.get(author_url, headers=headers) author_resp_info = json.loads(author_resp.content) - plugin.author.name = author_resp_info['name'] if 'name' in author_resp_info and author_resp_info['name'] else plugin.owner + plugin.author.name = author_resp_info['name'] if 'name' in author_resp_info and author_resp_info[ + 'name'] else plugin.owner if 'html_url' in author_resp_info: plugin.author.url = author_resp_info['html_url'] if 'avatar_url' in author_resp_info: @@ -109,21 +110,21 @@ def parse(plugin: AddonInfo, **kwargs): 'kgithub': xpi_url.replace('github.com', 'kkgithub.com'), } xpi_filename = f'{plugin.owner}#{plugin.repository}+{release.tagName}@{release_asset["id"]}.xpi' - details = {} + details = None try: xpi_filepath = download_xpi(xpi_url=xpi_url, download_dir=kwargs.get('runtime_xpi_directory'), unique_name=xpi_filename, force_download=False, cache_dir=kwargs.get('cache_directory')) - details = addon_details(xpi_filepath, zotero_versions=[release.zotero_check_version]) + details = addon_details(xpi_filepath) except: try: xpi_filepath = download_xpi(xpi_url=xpi_url, download_dir=kwargs.get('runtime_xpi_directory'), unique_name=xpi_filename, force_download=True) - details = addon_details(xpi_filepath, zotero_versions=[release.zotero_check_version]) + details = addon_details(xpi_filepath) except Exception as e: report_issue(kwargs.get('github_repository'), title=f'Parse xpi detail failed', @@ -134,16 +135,29 @@ def parse(plugin: AddonInfo, **kwargs): print(f'fetch addon detail of {plugin.repo} with {xpi_url} failed: {e}') if details: - if detail_id := details.get('id'): + if detail_id := details.id: release.id = detail_id - if detail_name := details.get('name'): + if detail_name := details.name: release.name = detail_name - if detail_version := details.get('version'): + if detail_version := details.version: release.xpiVersion = detail_version - if detail_desc := details.get('description'): + if detail_desc := details.description: release.description = detail_desc - if release.zotero_check_version not in details.get('zotero_versions', []): - invalid_releases.append(release) + if zotero_min_version := details.min_version: + release.minZoteroVersion = zotero_min_version + if zotero_max_version := details.max_version: + release.maxZoteroVersion = zotero_max_version + + if not details.check_compatible_for_zotero_version(release.zotero_check_version): + invalid_releases.append(release) + + if ((not release.minZoteroVersion or release.minZoteroVersion == '*') and + (not release.maxZoteroVersion or release.maxZoteroVersion == '*')): + report_issue(kwargs.get('github_repository'), + title=f'Parse {plugin.repo} of zotero version failed', + body=f'xpi:{plugin.repo}@{release.tagName} on {release.targetZoteroVersion}\n', + github_token=kwargs.get('github_token')) + print(plugin.repo, release.targetZoteroVersion, xpi_filename, release.minZoteroVersion, release.maxZoteroVersion) except Exception as e: print(f'handle {plugin.repo} request {release_url} failed: {e}') @@ -151,9 +165,11 @@ def parse(plugin: AddonInfo, **kwargs): for invalid_release in invalid_releases: print(plugin.repo, 'invalid', invalid_release.zotero_check_version) report_issue(kwargs.get('github_repository'), - title=f'Invalid xpi with zotero version {invalid_release.zotero_check_version}', + title=f'Invalid {plugin.repo} xpi with zotero version {invalid_release.zotero_check_version}', body=f'xpi:{plugin.repo}@{invalid_release.tagName}\n' - f'expect zotero version:{invalid_release.zotero_check_version}\n', + f'min zotero Version:{invalid_release.minZoteroVersion}\n' + f'max Zotero version:{invalid_release.maxZoteroVersion}' + f'expect Zotero version:{invalid_release.zotero_check_version}\n', github_token=kwargs.get('github_token')) plugin.releases.remove(invalid_release) return plugin @@ -161,7 +177,8 @@ def parse(plugin: AddonInfo, **kwargs): def fallback(addon_infos, previous_info_url, **kwargs): try: - repos_resp = requests.get(previous_info_url, headers=github_api_headers(github_token=kwargs.get('github_token'))) + repos_resp = requests.get(previous_info_url, + headers=github_api_headers(github_token=kwargs.get('github_token'))) repos_info = json.loads(repos_resp.content) addon_infos = fallback_if_need(addon_infos, repos_info) except Exception as e: diff --git a/moz_addons.py b/moz_addons.py index 16c515d..a285197 100644 --- a/moz_addons.py +++ b/moz_addons.py @@ -37,6 +37,41 @@ def compare_versions(version1, version2): return 0 +class XpiDetail: + def __init__(self): + self.id = None + self.name = None + self.version = None + self.description = None + self.min_version = "*" + self.max_version = "*" + + def _append_info(self, details: dict): + if id := details.get('id'): + if self.id and self.id != id: + print(f'Xpi ID not match? {self.id} <==> {id}') + return + self.id = id + if name := details.get('name'): + self.name = name + if version := details.get('version'): + self.version = version + if description := details.get('description'): + self.description = description + if (min_version := details.get('min_version')) and (max_version := details.get('max_version')): + if compare_versions(min_version.replace('*', '0'), self.min_version.replace('*', '999')) <= 0: + self.min_version = min_version + if compare_versions(max_version.replace('*', '999'), self.max_version.replace('*', '0')) >= 0: + self.max_version = max_version + + def check_compatible_for_zotero_version(self, version: str | int): + if isinstance(version, int): + version = str(version) + '.*' + if (min_version := self.min_version.replace('*', '0')) and (max_version := self.max_version.replace('*', '999')): + return (compare_versions(min_version, version.replace('*', '999')) <= 0 + <= compare_versions(max_version, version.replace('*', '0'))) + + def get_namespace_id(doc, url): attributes = doc.documentElement.attributes for i in range(attributes.length): @@ -87,23 +122,22 @@ def manifest_from_rdf(addon_path): print(f'Invalid Addon Path {addon_path}: {e}') -def detail_from_manifest_json(addon_path, manifest, **kwargs): - details = {"id": None, "name": manifest.get("name"), "version": manifest.get("version"), - "description": manifest.get("description")} +def detail_from_manifest_json(addon_path, manifest): + details = { + "name": manifest.get("name"), "version": manifest.get("version"), "description": manifest.get("description"), + "id": None, "min_version": None, "max_version": None + } for location in ("applications", "browser_specific_settings"): if details["id"]: break for app in ("zotero", "gecko"): - details["id"] = manifest[location][app].get("id") - - for version in kwargs.get('zotero_versions', []): - min_version = manifest[location][app].get("strict_min_version").replace('*', '0') - max_version = manifest[location][app].get("strict_max_version").replace('*', '999') - if (min_version and max_version and - compare_versions(min_version, version.replace('*', '999')) <= 0 and - compare_versions(version.replace('*', '0'), max_version) <= 0): - details.setdefault('zotero_versions', []).append(version) - break + try: + details["id"] = manifest[location][app].get("id") + details["min_version"] = manifest[location][app].get("strict_min_version") + details["max_version"] = manifest[location][app].get("strict_max_version") + break + except KeyError: + continue # handler for __MSG_{}__ items def extract_msg_placeholder(text): @@ -139,16 +173,14 @@ def detail_from_manifest_json(addon_path, manifest, **kwargs): return details -def detail_from_manifest_rdf(manifest, **kwargs): - details = {"id": None, "name": None, "version": None, "description": None} +def detail_from_manifest_rdf(manifest): + details = {"id": None, "name": None, "version": None, "description": None, "min_version": None, "max_version": None} try: doc = minidom.parseString(manifest) # Get the namespaces abbreviations em = get_namespace_id(doc, "http://www.mozilla.org/2004/em-rdf#") - rdf = get_namespace_id( - doc, "http://www.w3.org/1999/02/22-rdf-syntax-ns#" - ) + rdf = get_namespace_id(doc, "http://www.w3.org/1999/02/22-rdf-syntax-ns#") description = doc.getElementsByTagName(rdf + "Description").item(0) try: @@ -159,65 +191,60 @@ def detail_from_manifest_rdf(manifest, **kwargs): except Exception as e: pass - def extract_info(root, result): + def extract_info(node, result): try: - for entry, value in root.attributes.items(): + for entry, value in node.attributes.items(): entry = entry.replace(em, "") if entry in result.keys(): result.update({entry: value}) - for node in root.childNodes: - entry = node.nodeName.replace(em, "") + for child_node in node.childNodes: + entry = child_node.nodeName.replace(em, "") if entry in result.keys(): - result.update({entry: get_text(node)}) + result.update({entry: get_text(child_node)}) except: return - extract_info(description, details) - def check_version(version_info, version): - if (version_info['id'] == 'zotero@chnm.gmu.edu' and - (min_version := version_info['minVersion']) and (max_version := version_info['maxVersion'])): - min_version = min_version.replace('*', '0') - max_version = max_version.replace('*', '999') - return (compare_versions(min_version, version.replace('*', '999')) <= 0 and - compare_versions(version.replace('*', '0'), max_version) <= 0) - return False + def update_details(version_info): + if version_info['id'] != 'zotero@chnm.gmu.edu': + return + if (min_version := version_info['minVersion']) and (max_version := version_info['maxVersion']): + if exist_min_version := details['min_version']: + if compare_versions(min_version.replace('*', '0'), exist_min_version.replace('*', '999')) <= 0: + details['min_version'] = min_version + else: + details['min_version'] = min_version - for version in kwargs.get('zotero_versions', []): + if exist_max_version := details['max_version']: + if compare_versions(max_version.replace('*', '999'), exist_max_version.replace('*', '0')) >= 0: + details['max_version'] = max_version + else: + details['max_version'] = max_version + + + for targetApplication in description.getElementsByTagName(em + "targetApplication"): version_info = {'id': None, 'minVersion': None, 'maxVersion': None} - check_flag = True - for targetApplication in description.getElementsByTagName(em + "targetApplication"): - if not check_flag: - break - extract_info(targetApplication, version_info) - if check_version(version_info, version): - details.setdefault('zotero_versions', []).append(version) - break - for node in targetApplication.childNodes: - extract_info(node, version_info) - if check_version(version_info, version): - details.setdefault('zotero_versions', []).append(version) - check_flag = False - break + extract_info(targetApplication, version_info) + update_details(version_info) + + for node in targetApplication.childNodes: + version_info = {'id': None, 'minVersion': None, 'maxVersion': None} + extract_info(node, version_info) + update_details(version_info) return details except Exception as e: raise e -def addon_details(addon_path, **kwargs): +def addon_details(addon_path) -> XpiDetail: if not os.path.exists(addon_path): - raise IOError("Add-on path does not exist: %s" % addon_path) - result = {} + raise IOError(f"Add-on path does not exist: {addon_path}") + xpi_detail = XpiDetail() if ((manifest := manifest_from_json(addon_path)) and - (details := detail_from_manifest_json(addon_path, manifest, zotero_versions=kwargs.get('zotero_versions')))): - result = details + (details := detail_from_manifest_json(addon_path, manifest))): + xpi_detail._append_info(details) if ((manifest := manifest_from_rdf(addon_path)) and - (details := detail_from_manifest_rdf(manifest, zotero_versions=kwargs.get('zotero_versions')))): - for key in details: - if key not in result: - result[key] = details[key] - result['zotero_versions'] = list(set(result.get('zotero_versions', []) + details.get('zotero_versions', []))) - - return result - + (details := detail_from_manifest_rdf(manifest))): + xpi_detail._append_info(details) + return xpi_detail From d340f4793b46b2d60f5a30f4975abe4bf7504dbf Mon Sep 17 00:00:00 2001 From: ytshen Date: Sat, 13 Apr 2024 13:56:23 +0800 Subject: [PATCH 2/3] fix install.rdf not support z7 --- main.py | 1 - moz_addons.py | 3 +++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/main.py b/main.py index 9781287..e348067 100644 --- a/main.py +++ b/main.py @@ -157,7 +157,6 @@ def parse(plugin: AddonInfo, **kwargs): title=f'Parse {plugin.repo} of zotero version failed', body=f'xpi:{plugin.repo}@{release.tagName} on {release.targetZoteroVersion}\n', github_token=kwargs.get('github_token')) - print(plugin.repo, release.targetZoteroVersion, xpi_filename, release.minZoteroVersion, release.maxZoteroVersion) except Exception as e: print(f'handle {plugin.repo} request {release_url} failed: {e}') diff --git a/moz_addons.py b/moz_addons.py index a285197..ca53587 100644 --- a/moz_addons.py +++ b/moz_addons.py @@ -215,6 +215,9 @@ def detail_from_manifest_rdf(manifest): else: details['min_version'] = min_version + if compare_versions(max_version.replace('*', '999'), '6.*') > 0: + # rdf not support z7 + max_version = '6.*' if exist_max_version := details['max_version']: if compare_versions(max_version.replace('*', '999'), exist_max_version.replace('*', '0')) >= 0: details['max_version'] = max_version From 5d536a88b06d33b0df9e05c5149cabdf16a1e40a Mon Sep 17 00:00:00 2001 From: ytshen Date: Sat, 13 Apr 2024 14:32:36 +0800 Subject: [PATCH 3/3] rm unexpect rdf content --- main.py | 6 ++++++ moz_addons.py | 2 ++ 2 files changed, 8 insertions(+) diff --git a/main.py b/main.py index e348067..fccf3fa 100644 --- a/main.py +++ b/main.py @@ -157,6 +157,12 @@ def parse(plugin: AddonInfo, **kwargs): title=f'Parse {plugin.repo} of zotero version failed', body=f'xpi:{plugin.repo}@{release.tagName} on {release.targetZoteroVersion}\n', github_token=kwargs.get('github_token')) + else: + report_issue(kwargs.get('github_repository'), + title=f'Parse {plugin.repo} addon details failed', + body=f'xpi:{plugin.repo}@{release.tagName} on {release.targetZoteroVersion}\n', + github_token=kwargs.get('github_token')) + except Exception as e: print(f'handle {plugin.repo} request {release_url} failed: {e}') diff --git a/moz_addons.py b/moz_addons.py index ca53587..4339d64 100644 --- a/moz_addons.py +++ b/moz_addons.py @@ -204,6 +204,8 @@ def detail_from_manifest_rdf(manifest): except: return extract_info(description, details) + if not details.get('id') or (details.get('id').startswith("__") and details.get('id').endswith("__")): + return def update_details(version_info): if version_info['id'] != 'zotero@chnm.gmu.edu':