diff --git a/addon_info.py b/addon_info.py index b35d62f..9bbf9a3 100644 --- a/addon_info.py +++ b/addon_info.py @@ -33,6 +33,14 @@ class AddonInfoRelease: self.description = description [setattr(self, key, value) for (key, value) in kwargs.items()] + @property + def zotero_check_version(self) -> str: + if self.targetZoteroVersion == '6': + return '6.*' + elif self.targetZoteroVersion == '7': + return '7.*' + raise Exception(f'Invalid targetZoteroVersion({self.targetZoteroVersion})') + @property def __dict__(self): result = {} diff --git a/main.py b/main.py index b897697..4aab696 100644 --- a/main.py +++ b/main.py @@ -22,6 +22,29 @@ def github_api_headers(**kwargs): return result +def report_issue(repo: str, title: str, body: str, **kwargs): + if not repo: + print('report issue repo not found') + return + try: + response = requests.post(f'https://api.github.com/repos/{repo}/issues', + headers=github_api_headers(github_token=kwargs.get('github_token')), + json={ + 'title': title, + 'body': body + }) + + # 检查响应 + if response.status_code == 201: + print('Issue created successfully.') + print('Issue URL:', response.json()['html_url']) + else: + print('Failed to create issue.') + print('Response:', response.content) + except Exception as e: + print(f'report issue failed: {e}') + + def download_xpi(xpi_url: str, download_dir: str, unique_name: str, force_download: bool, **kwargs): try: download_filepath = os.path.join(download_dir, unique_name) @@ -82,6 +105,7 @@ def parse(plugin: AddonInfo, **kwargs): print(f'request {repo_url} failed: {e}') # fetch release info + invalid_releases = [] for release in plugin.releases: if not release.tagName: continue @@ -129,15 +153,21 @@ def parse(plugin: AddonInfo, **kwargs): unique_name=xpi_filename, force_download=False, cache_dir=kwargs.get('cache_directory')) - details = addon_details(xpi_filepath) + details = addon_details(xpi_filepath, zotero_versions=[release.zotero_check_version]) except: try: xpi_filepath = download_xpi(xpi_url=xpi_url, download_dir=kwargs.get('runtime_xpi_directory'), unique_name=xpi_filename, force_download=True) - details = addon_details(xpi_filepath) + details = addon_details(xpi_filepath, zotero_versions=[release.zotero_check_version]) except Exception as e: + report_issue(kwargs.get('github_repository'), + title=f'Parse xpi detail failed', + body=f'url:{xpi_url}\n' + f'repo:{plugin.repo}\n' + f'reason:{e}', + github_token=kwargs.get('github_token')) print(f'fetch addon detail of {plugin.repo} with {xpi_url} failed: {e}') if details: @@ -149,9 +179,20 @@ def parse(plugin: AddonInfo, **kwargs): release.xpiVersion = detail_version if detail_desc := details.get('description'): release.description = detail_desc + if release.zotero_check_version not in details.get('zotero_versions', []): + invalid_releases.append(release) except Exception as e: print(f'handle {plugin.repo} request {release_url} failed: {e}') + + for invalid_release in invalid_releases: + print(plugin.repo, 'invalid', invalid_release.zotero_check_version) + report_issue(kwargs.get('github_repository'), + title=f'Invalid xpi with zotero version {invalid_release.zotero_check_version}', + body=f'xpi:{plugin.repo}@{invalid_release.tagName}\n' + f'expect zotero version:{invalid_release.zotero_check_version}\n', + github_token=kwargs.get('github_token')) + plugin.releases.remove(invalid_release) return plugin @@ -183,7 +224,8 @@ def parse_addon_infos(input_dir, output_filepath, **kwargs): plugin, github_token=kwargs.get('github_token'), cache_directory=kwargs.get('cache_directory'), - runtime_xpi_directory=kwargs.get('runtime_xpi_directory')) + runtime_xpi_directory=kwargs.get('runtime_xpi_directory'), + github_repository=kwargs.get('github_repository')) for plugin in plugins] for future in concurrent.futures.as_completed(futures): @@ -331,6 +373,7 @@ if __name__ == '__main__': parse_addon_infos(args.input, args.output, + github_repository=args.github_repository, github_token=args.github_token, cache_directory=args.cache_directory, runtime_xpi_directory=args.runtime_xpi_directory, diff --git a/moz_addons.py b/moz_addons.py index efd5223..16c515d 100644 --- a/moz_addons.py +++ b/moz_addons.py @@ -1,3 +1,6 @@ +# modify from `https://github.com/ +# mozilla/gecko-dev/blob/ac19e2c0d7c09a2deaedbe6afc4cdcf1a4561456/testing/mozbase/mozprofile/mozprofile/addons.py#L213` + import zipfile import commentjson as json import os @@ -5,181 +8,216 @@ from xml.dom import minidom import re -# modify from `https://github.com/mozilla/gecko-dev/blob/ac19e2c0d7c09a2deaedbe6afc4cdcf1a4561456/testing/mozbase/mozprofile/mozprofile/addons.py#L213` -def addon_details(addon_path): - """ - Returns a dictionary of details about the addon. +def compare_versions(version1, version2): + parts1 = version1.replace('-', '.').split('.') + parts2 = version2.replace('-', '.').split('.') - :param addon_path: path to the add-on directory or XPI + for i in range(max(len(parts1), len(parts2))): + v1 = parts1[i] if i < len(parts1) else '0' + v2 = parts2[i] if i < len(parts2) else '0' + try: + num1 = int(v1) + except ValueError: + num1 = v1 + try: + num2 = int(v2) + except ValueError: + num2 = v2 - Returns:: + if isinstance(num1, int) and isinstance(num2, int): + if num1 < num2: + return -1 + if num1 > num2: + return 1 + else: + if v1 < v2: + return -1 + if v1 > v2: + return 1 + return 0 - {'id': u'rainbow@colors.org', # id of the addon - 'version': u'1.4', # version of the addon - 'name': u'Rainbow', # name of the addon - 'unpack': False } # whether to unpack the addon - """ - details = {"id": None, "unpack": False, "name": None, "version": None, "description": None} +def get_namespace_id(doc, url): + attributes = doc.documentElement.attributes + for i in range(attributes.length): + if attributes.item(i).value == url and ":" in attributes.item(i).name: + return attributes.item(i).name.split(":")[1] + ":" + return "" - def get_namespace_id(doc, url): - attributes = doc.documentElement.attributes - namespace = "" - for i in range(attributes.length): - if attributes.item(i).value == url: - if ":" in attributes.item(i).name: - # If the namespace is not the default one remove 'xlmns:' - namespace = attributes.item(i).name.split(":")[1] + ":" - break - return namespace - def get_text(element): - """Retrieve the text value of a given node""" - rc = [] - for node in element.childNodes: - if node.nodeType == node.TEXT_NODE: - rc.append(node.data) - return "".join(rc).strip() +def get_text(element): + """Retrieve the text value of a given node""" + rc = [] + for node in element.childNodes: + if node.nodeType == node.TEXT_NODE: + rc.append(node.data) + return "".join(rc).strip() - if not os.path.exists(addon_path): - raise IOError("Add-on path does not exist: %s" % addon_path) - is_webext = False +def manifest_from_json(addon_path): try: if zipfile.is_zipfile(addon_path): - # Bug 944361 - We cannot use 'with' together with zipFile because - # it will cause an exception thrown in Python 2.6. - try: - compressed_file = zipfile.ZipFile(addon_path, "r") - filenames = [f.filename for f in (compressed_file).filelist] + with zipfile.ZipFile(addon_path, "r") as compressed_file: + filenames = [f.filename for f in compressed_file.filelist] if "manifest.json" in filenames: - is_webext = True manifest = compressed_file.read("manifest.json").decode() manifest = json.loads(manifest) - elif "install.rdf" in filenames: - manifest = compressed_file.read("install.rdf") - else: - raise KeyError("No manifest") - finally: - compressed_file.close() + return manifest elif os.path.isdir(addon_path): - try: - with open(os.path.join(addon_path, "manifest.json")) as f: - manifest = json.loads(f.read()) - is_webext = True - except IOError: - with open(os.path.join(addon_path, "install.rdf")) as f: - manifest = f.read() - else: - raise IOError( - "Add-on path is neither an XPI nor a directory: %s" % addon_path - ) - except (IOError, KeyError) as e: - # reraise(AddonFormatError, AddonFormatError(str(e)), sys.exc_info()[2]) - raise e + with open(os.path.join(addon_path, "manifest.json")) as f: + manifest = json.loads(f.read()) + return manifest + except Exception as e: + print(f'Invalid Addon Path {addon_path}: {e}') - if is_webext: - details["version"] = manifest.get("version") - details["name"] = manifest.get("name") - details["description"] = manifest.get("description") - # Bug 1572404 - we support two locations for gecko-specific - # metadata. - for location in ("applications", "browser_specific_settings"): - for app in ("zotero", "gecko"): - try: - details["id"] = manifest[location][app]["id"] - break - except KeyError: - pass - # if details["id"] is None: - # details["id"] = cls._gen_iid(addon_path) - details["unpack"] = False - # handler for __MSG_{}__ items - def extract_MSG_placeholder(text): - if match := re.search(r'__MSG_(.*?)__', text): - return match.group(1) +def manifest_from_rdf(addon_path): + try: + if zipfile.is_zipfile(addon_path): + with zipfile.ZipFile(addon_path, "r") as compressed_file: + filenames = [f.filename for f in compressed_file.filelist] + if "install.rdf" in filenames: + manifest = compressed_file.read("install.rdf") + return manifest + elif os.path.isdir(addon_path): + with open(os.path.join(addon_path, "install.rdf")) as f: + manifest = json.loads(f.read()) + return manifest + except Exception as e: + print(f'Invalid Addon Path {addon_path}: {e}') - def load_locale_for_MSG(): - default_locale = manifest.get('default_locale') - locale_filename = f"_locales/{default_locale}/messages.json" - try: - if zipfile.is_zipfile(addon_path): - # Bug 944361 - We cannot use 'with' together with zipFile because - # it will cause an exception thrown in Python 2.6. - try: - compressed_file = zipfile.ZipFile(addon_path, "r") - filenames = [f.filename for f in (compressed_file).filelist] - if locale_filename in filenames: - locale = compressed_file.read(locale_filename).decode() - return json.loads(locale) - else: - raise KeyError(f"No {locale_filename}") - finally: - compressed_file.close() - elif os.path.isdir(addon_path): - try: - with open(os.path.join(addon_path, locale_filename)) as f: - return json.loads(f.read()) - except IOError: - raise KeyError(f"No {locale_filename}") - else: - raise IOError( - "Cannot found: %s" % locale_filename - ) - except (IOError, KeyError) as e: - # reraise(AddonFormatError, AddonFormatError(str(e)), sys.exc_info()[2]) - raise e - locale_for_MSG = None - for key in details: - if isinstance(details[key], str) and (placeholder := extract_MSG_placeholder(details[key])): - if not locale_for_MSG: - locale_for_MSG = load_locale_for_MSG() - if not locale_for_MSG: - break - if value := locale_for_MSG.get(placeholder, {}).get('message'): - details[key] = value +def detail_from_manifest_json(addon_path, manifest, **kwargs): + details = {"id": None, "name": manifest.get("name"), "version": manifest.get("version"), + "description": manifest.get("description")} + for location in ("applications", "browser_specific_settings"): + if details["id"]: + break + for app in ("zotero", "gecko"): + details["id"] = manifest[location][app].get("id") - else: + for version in kwargs.get('zotero_versions', []): + min_version = manifest[location][app].get("strict_min_version").replace('*', '0') + max_version = manifest[location][app].get("strict_max_version").replace('*', '999') + if (min_version and max_version and + compare_versions(min_version, version.replace('*', '999')) <= 0 and + compare_versions(version.replace('*', '0'), max_version) <= 0): + details.setdefault('zotero_versions', []).append(version) + break + + # handler for __MSG_{}__ items + def extract_msg_placeholder(text): + if match := re.search(r'__MSG_(.*?)__', text): + return match.group(1) + + def load_locale_for_msg(): + default_locale = manifest.get('default_locale') + locale_filename = f"_locales/{default_locale}/messages.json" try: - doc = minidom.parseString(manifest) - - # Get the namespaces abbreviations - em = get_namespace_id(doc, "http://www.mozilla.org/2004/em-rdf#") - rdf = get_namespace_id( - doc, "http://www.w3.org/1999/02/22-rdf-syntax-ns#" - ) - - description = doc.getElementsByTagName(rdf + "Description").item(0) - try: - descriptions = [e for e in doc.getElementsByTagName(rdf + "Description") - if len(e.getElementsByTagName(em + "targetApplication")) > 0] - if descriptions: - description = descriptions[0] - except Exception as e: - pass - - for entry, value in description.attributes.items(): - # Remove the namespace prefix from the tag for comparison - entry = entry.replace(em, "") - if entry in details.keys(): - details.update({entry: value}) - for node in description.childNodes: - # Remove the namespace prefix from the tag for comparison - entry = node.nodeName.replace(em, "") - if entry in details.keys(): - details.update({entry: get_text(node)}) + if zipfile.is_zipfile(addon_path): + with zipfile.ZipFile(addon_path, "r") as compressed_file: + filenames = [f.filename for f in compressed_file.filelist] + if locale_filename in filenames: + locale = compressed_file.read(locale_filename).decode() + return json.loads(locale) + elif os.path.isdir(addon_path): + with open(os.path.join(addon_path, locale_filename)) as f: + return json.loads(f.read()) except Exception as e: raise e - # turn unpack into a true/false value - if isinstance(details["unpack"], str): - details["unpack"] = details["unpack"].lower() == "true" - - # If no ID is set, the add-on is invalid - # if details.get("id") is None and not is_webext: - # raise AddonFormatError("Add-on id could not be found.") + locale_for_msg = None + for key in details: + if isinstance(details[key], str) and (placeholder := extract_msg_placeholder(details[key])): + if not locale_for_msg: + locale_for_msg = load_locale_for_msg() + if not locale_for_msg: + break + if value := locale_for_msg.get(placeholder, {}).get('message'): + details[key] = value return details + +def detail_from_manifest_rdf(manifest, **kwargs): + details = {"id": None, "name": None, "version": None, "description": None} + try: + doc = minidom.parseString(manifest) + + # Get the namespaces abbreviations + em = get_namespace_id(doc, "http://www.mozilla.org/2004/em-rdf#") + rdf = get_namespace_id( + doc, "http://www.w3.org/1999/02/22-rdf-syntax-ns#" + ) + + description = doc.getElementsByTagName(rdf + "Description").item(0) + try: + descriptions = [e for e in doc.getElementsByTagName(rdf + "Description") + if len(e.getElementsByTagName(em + "targetApplication")) > 0] + if descriptions: + description = descriptions[0] + except Exception as e: + pass + + def extract_info(root, result): + try: + for entry, value in root.attributes.items(): + entry = entry.replace(em, "") + if entry in result.keys(): + result.update({entry: value}) + for node in root.childNodes: + entry = node.nodeName.replace(em, "") + if entry in result.keys(): + result.update({entry: get_text(node)}) + except: + return + + extract_info(description, details) + + def check_version(version_info, version): + if (version_info['id'] == 'zotero@chnm.gmu.edu' and + (min_version := version_info['minVersion']) and (max_version := version_info['maxVersion'])): + min_version = min_version.replace('*', '0') + max_version = max_version.replace('*', '999') + return (compare_versions(min_version, version.replace('*', '999')) <= 0 and + compare_versions(version.replace('*', '0'), max_version) <= 0) + return False + + for version in kwargs.get('zotero_versions', []): + version_info = {'id': None, 'minVersion': None, 'maxVersion': None} + check_flag = True + for targetApplication in description.getElementsByTagName(em + "targetApplication"): + if not check_flag: + break + extract_info(targetApplication, version_info) + if check_version(version_info, version): + details.setdefault('zotero_versions', []).append(version) + break + for node in targetApplication.childNodes: + extract_info(node, version_info) + if check_version(version_info, version): + details.setdefault('zotero_versions', []).append(version) + check_flag = False + break + + return details + except Exception as e: + raise e + + +def addon_details(addon_path, **kwargs): + if not os.path.exists(addon_path): + raise IOError("Add-on path does not exist: %s" % addon_path) + result = {} + if ((manifest := manifest_from_json(addon_path)) and + (details := detail_from_manifest_json(addon_path, manifest, zotero_versions=kwargs.get('zotero_versions')))): + result = details + if ((manifest := manifest_from_rdf(addon_path)) and + (details := detail_from_manifest_rdf(manifest, zotero_versions=kwargs.get('zotero_versions')))): + for key in details: + if key not in result: + result[key] = details[key] + result['zotero_versions'] = list(set(result.get('zotero_versions', []) + details.get('zotero_versions', []))) + + return result +