parvej/zotero-addons-scraper
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Merge pull request #5 from syt2/dev/target_zotero_version_validation

Browse Source 
validation for target zotero version
This commit is contained in:
syt 2024-04-12 14:14:13 +08:00 committed by GitHub
commit 24eac29d1d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 244 additions and 155 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
addon_info.py
View File

@ -33,6 +33,14 @@ class AddonInfoRelease:
self.description = description
[setattr(self, key, value) for (key, value) in kwargs.items()]
@property
def zotero_check_version(self) -> str:
if self.targetZoteroVersion == '6':
return '6.*'
elif self.targetZoteroVersion == '7':
return '7.*'
raise Exception(f'Invalid targetZoteroVersion({self.targetZoteroVersion})')
@property
def __dict__(self):
result = {}

49
main.py
View File

@ -22,6 +22,29 @@ def github_api_headers(**kwargs):
return result
def report_issue(repo: str, title: str, body: str, **kwargs):
if not repo:
print('report issue repo not found')
return
try:
response = requests.post(f'https://api.github.com/repos/{repo}/issues',
headers=github_api_headers(github_token=kwargs.get('github_token')),
json={
'title': title,
'body': body
})
# 检查响应
if response.status_code == 201:
print('Issue created successfully.')
print('Issue URL:', response.json()['html_url'])
else:
print('Failed to create issue.')
print('Response:', response.content)
except Exception as e:
print(f'report issue failed: {e}')
def download_xpi(xpi_url: str, download_dir: str, unique_name: str, force_download: bool, **kwargs):
try:
download_filepath = os.path.join(download_dir, unique_name)
@ -82,6 +105,7 @@ def parse(plugin: AddonInfo, **kwargs):
print(f'request {repo_url} failed: {e}')
# fetch release info
invalid_releases = []
for release in plugin.releases:
if not release.tagName:
continue
@ -129,15 +153,21 @@ def parse(plugin: AddonInfo, **kwargs):
unique_name=xpi_filename,
force_download=False,
cache_dir=kwargs.get('cache_directory'))
details = addon_details(xpi_filepath)
details = addon_details(xpi_filepath, zotero_versions=[release.zotero_check_version])
except:
try:
xpi_filepath = download_xpi(xpi_url=xpi_url,
download_dir=kwargs.get('runtime_xpi_directory'),
unique_name=xpi_filename,
force_download=True)
details = addon_details(xpi_filepath)
details = addon_details(xpi_filepath, zotero_versions=[release.zotero_check_version])
except Exception as e:
report_issue(kwargs.get('github_repository'),
title=f'Parse xpi detail failed',
body=f'url:{xpi_url}\n'
f'repo:{plugin.repo}\n'
f'reason:{e}',
github_token=kwargs.get('github_token'))
print(f'fetch addon detail of {plugin.repo} with {xpi_url} failed: {e}')
if details:
@ -149,9 +179,20 @@ def parse(plugin: AddonInfo, **kwargs):
release.xpiVersion = detail_version
if detail_desc := details.get('description'):
release.description = detail_desc
if release.zotero_check_version not in details.get('zotero_versions', []):
invalid_releases.append(release)
except Exception as e:
print(f'handle {plugin.repo} request {release_url} failed: {e}')
for invalid_release in invalid_releases:
print(plugin.repo, 'invalid', invalid_release.zotero_check_version)
report_issue(kwargs.get('github_repository'),
title=f'Invalid xpi with zotero version {invalid_release.zotero_check_version}',
body=f'xpi:{plugin.repo}@{invalid_release.tagName}\n'
f'expect zotero version:{invalid_release.zotero_check_version}\n',
github_token=kwargs.get('github_token'))
plugin.releases.remove(invalid_release)
return plugin
@ -183,7 +224,8 @@ def parse_addon_infos(input_dir, output_filepath, **kwargs):
plugin,
github_token=kwargs.get('github_token'),
cache_directory=kwargs.get('cache_directory'),
runtime_xpi_directory=kwargs.get('runtime_xpi_directory'))
runtime_xpi_directory=kwargs.get('runtime_xpi_directory'),
github_repository=kwargs.get('github_repository'))
for plugin in plugins]
for future in concurrent.futures.as_completed(futures):
@ -331,6 +373,7 @@ if __name__ == '__main__':
parse_addon_infos(args.input,
args.output,
github_repository=args.github_repository,
github_token=args.github_token,
cache_directory=args.cache_directory,
runtime_xpi_directory=args.runtime_xpi_directory,

342
moz_addons.py
View File

@ -1,3 +1,6 @@
# modify from `https://github.com/
# mozilla/gecko-dev/blob/ac19e2c0d7c09a2deaedbe6afc4cdcf1a4561456/testing/mozbase/mozprofile/mozprofile/addons.py#L213`
import zipfile
import commentjson as json
import os
@ -5,181 +8,216 @@ from xml.dom import minidom
import re
# modify from `https://github.com/mozilla/gecko-dev/blob/ac19e2c0d7c09a2deaedbe6afc4cdcf1a4561456/testing/mozbase/mozprofile/mozprofile/addons.py#L213`
def addon_details(addon_path):
"""
Returns a dictionary of details about the addon.
def compare_versions(version1, version2):
parts1 = version1.replace('-', '.').split('.')
parts2 = version2.replace('-', '.').split('.')
:param addon_path: path to the add-on directory or XPI
for i in range(max(len(parts1), len(parts2))):
v1 = parts1[i] if i < len(parts1) else '0'
v2 = parts2[i] if i < len(parts2) else '0'
try:
num1 = int(v1)
except ValueError:
num1 = v1
try:
num2 = int(v2)
except ValueError:
num2 = v2
Returns::
if isinstance(num1, int) and isinstance(num2, int):
if num1 < num2:
return -1
if num1 > num2:
return 1
else:
if v1 < v2:
return -1
if v1 > v2:
return 1
return 0
{'id': u'rainbow@colors.org', # id of the addon
'version': u'1.4', # version of the addon
'name': u'Rainbow', # name of the addon
'unpack': False } # whether to unpack the addon
"""
details = {"id": None, "unpack": False, "name": None, "version": None, "description": None}
def get_namespace_id(doc, url):
attributes = doc.documentElement.attributes
for i in range(attributes.length):
if attributes.item(i).value == url and ":" in attributes.item(i).name:
return attributes.item(i).name.split(":")[1] + ":"
return ""
def get_namespace_id(doc, url):
attributes = doc.documentElement.attributes
namespace = ""
for i in range(attributes.length):
if attributes.item(i).value == url:
if ":" in attributes.item(i).name:
# If the namespace is not the default one remove 'xlmns:'
namespace = attributes.item(i).name.split(":")[1] + ":"
break
return namespace
def get_text(element):
"""Retrieve the text value of a given node"""
rc = []
for node in element.childNodes:
if node.nodeType == node.TEXT_NODE:
rc.append(node.data)
return "".join(rc).strip()
def get_text(element):
"""Retrieve the text value of a given node"""
rc = []
for node in element.childNodes:
if node.nodeType == node.TEXT_NODE:
rc.append(node.data)
return "".join(rc).strip()
if not os.path.exists(addon_path):
raise IOError("Add-on path does not exist: %s" % addon_path)
is_webext = False
def manifest_from_json(addon_path):
try:
if zipfile.is_zipfile(addon_path):
# Bug 944361 - We cannot use 'with' together with zipFile because
# it will cause an exception thrown in Python 2.6.
try:
compressed_file = zipfile.ZipFile(addon_path, "r")
filenames = [f.filename for f in (compressed_file).filelist]
with zipfile.ZipFile(addon_path, "r") as compressed_file:
filenames = [f.filename for f in compressed_file.filelist]
if "manifest.json" in filenames:
is_webext = True
manifest = compressed_file.read("manifest.json").decode()
manifest = json.loads(manifest)
elif "install.rdf" in filenames:
manifest = compressed_file.read("install.rdf")
else:
raise KeyError("No manifest")
finally:
compressed_file.close()
return manifest
elif os.path.isdir(addon_path):
try:
with open(os.path.join(addon_path, "manifest.json")) as f:
manifest = json.loads(f.read())
is_webext = True
except IOError:
with open(os.path.join(addon_path, "install.rdf")) as f:
manifest = f.read()
else:
raise IOError(
"Add-on path is neither an XPI nor a directory: %s" % addon_path
)
except (IOError, KeyError) as e:
# reraise(AddonFormatError, AddonFormatError(str(e)), sys.exc_info()[2])
raise e
with open(os.path.join(addon_path, "manifest.json")) as f:
manifest = json.loads(f.read())
return manifest
except Exception as e:
print(f'Invalid Addon Path {addon_path}: {e}')
if is_webext:
details["version"] = manifest.get("version")
details["name"] = manifest.get("name")
details["description"] = manifest.get("description")
# Bug 1572404 - we support two locations for gecko-specific
# metadata.
for location in ("applications", "browser_specific_settings"):
for app in ("zotero", "gecko"):
try:
details["id"] = manifest[location][app]["id"]
break
except KeyError:
pass
# if details["id"] is None:
# details["id"] = cls._gen_iid(addon_path)
details["unpack"] = False
# handler for __MSG_{}__ items
def extract_MSG_placeholder(text):
if match := re.search(r'__MSG_(.*?)__', text):
return match.group(1)
def manifest_from_rdf(addon_path):
try:
if zipfile.is_zipfile(addon_path):
with zipfile.ZipFile(addon_path, "r") as compressed_file:
filenames = [f.filename for f in compressed_file.filelist]
if "install.rdf" in filenames:
manifest = compressed_file.read("install.rdf")
return manifest
elif os.path.isdir(addon_path):
with open(os.path.join(addon_path, "install.rdf")) as f:
manifest = json.loads(f.read())
return manifest
except Exception as e:
print(f'Invalid Addon Path {addon_path}: {e}')
def load_locale_for_MSG():
default_locale = manifest.get('default_locale')
locale_filename = f"_locales/{default_locale}/messages.json"
try:
if zipfile.is_zipfile(addon_path):
# Bug 944361 - We cannot use 'with' together with zipFile because
# it will cause an exception thrown in Python 2.6.
try:
compressed_file = zipfile.ZipFile(addon_path, "r")
filenames = [f.filename for f in (compressed_file).filelist]
if locale_filename in filenames:
locale = compressed_file.read(locale_filename).decode()
return json.loads(locale)
else:
raise KeyError(f"No {locale_filename}")
finally:
compressed_file.close()
elif os.path.isdir(addon_path):
try:
with open(os.path.join(addon_path, locale_filename)) as f:
return json.loads(f.read())
except IOError:
raise KeyError(f"No {locale_filename}")
else:
raise IOError(
"Cannot found: %s" % locale_filename
)
except (IOError, KeyError) as e:
# reraise(AddonFormatError, AddonFormatError(str(e)), sys.exc_info()[2])
raise e
locale_for_MSG = None
for key in details:
if isinstance(details[key], str) and (placeholder := extract_MSG_placeholder(details[key])):
if not locale_for_MSG:
locale_for_MSG = load_locale_for_MSG()
if not locale_for_MSG:
break
if value := locale_for_MSG.get(placeholder, {}).get('message'):
details[key] = value
def detail_from_manifest_json(addon_path, manifest, **kwargs):
details = {"id": None, "name": manifest.get("name"), "version": manifest.get("version"),
"description": manifest.get("description")}
for location in ("applications", "browser_specific_settings"):
if details["id"]:
break
for app in ("zotero", "gecko"):
details["id"] = manifest[location][app].get("id")
else:
for version in kwargs.get('zotero_versions', []):
min_version = manifest[location][app].get("strict_min_version").replace('*', '0')
max_version = manifest[location][app].get("strict_max_version").replace('*', '999')
if (min_version and max_version and
compare_versions(min_version, version.replace('*', '999')) <= 0 and
compare_versions(version.replace('*', '0'), max_version) <= 0):
details.setdefault('zotero_versions', []).append(version)
break
# handler for __MSG_{}__ items
def extract_msg_placeholder(text):
if match := re.search(r'__MSG_(.*?)__', text):
return match.group(1)
def load_locale_for_msg():
default_locale = manifest.get('default_locale')
locale_filename = f"_locales/{default_locale}/messages.json"
try:
doc = minidom.parseString(manifest)
# Get the namespaces abbreviations
em = get_namespace_id(doc, "http://www.mozilla.org/2004/em-rdf#")
rdf = get_namespace_id(
doc, "http://www.w3.org/1999/02/22-rdf-syntax-ns#"
)
description = doc.getElementsByTagName(rdf + "Description").item(0)
try:
descriptions = [e for e in doc.getElementsByTagName(rdf + "Description")
if len(e.getElementsByTagName(em + "targetApplication")) > 0]
if descriptions:
description = descriptions[0]
except Exception as e:
pass
for entry, value in description.attributes.items():
# Remove the namespace prefix from the tag for comparison
entry = entry.replace(em, "")
if entry in details.keys():
details.update({entry: value})
for node in description.childNodes:
# Remove the namespace prefix from the tag for comparison
entry = node.nodeName.replace(em, "")
if entry in details.keys():
details.update({entry: get_text(node)})
if zipfile.is_zipfile(addon_path):
with zipfile.ZipFile(addon_path, "r") as compressed_file:
filenames = [f.filename for f in compressed_file.filelist]
if locale_filename in filenames:
locale = compressed_file.read(locale_filename).decode()
return json.loads(locale)
elif os.path.isdir(addon_path):
with open(os.path.join(addon_path, locale_filename)) as f:
return json.loads(f.read())
except Exception as e:
raise e
# turn unpack into a true/false value
if isinstance(details["unpack"], str):
details["unpack"] = details["unpack"].lower() == "true"
# If no ID is set, the add-on is invalid
# if details.get("id") is None and not is_webext:
# raise AddonFormatError("Add-on id could not be found.")
locale_for_msg = None
for key in details:
if isinstance(details[key], str) and (placeholder := extract_msg_placeholder(details[key])):
if not locale_for_msg:
locale_for_msg = load_locale_for_msg()
if not locale_for_msg:
break
if value := locale_for_msg.get(placeholder, {}).get('message'):
details[key] = value
return details
def detail_from_manifest_rdf(manifest, **kwargs):
details = {"id": None, "name": None, "version": None, "description": None}
try:
doc = minidom.parseString(manifest)
# Get the namespaces abbreviations
em = get_namespace_id(doc, "http://www.mozilla.org/2004/em-rdf#")
rdf = get_namespace_id(
doc, "http://www.w3.org/1999/02/22-rdf-syntax-ns#"
)
description = doc.getElementsByTagName(rdf + "Description").item(0)
try:
descriptions = [e for e in doc.getElementsByTagName(rdf + "Description")
if len(e.getElementsByTagName(em + "targetApplication")) > 0]
if descriptions:
description = descriptions[0]
except Exception as e:
pass
def extract_info(root, result):
try:
for entry, value in root.attributes.items():
entry = entry.replace(em, "")
if entry in result.keys():
result.update({entry: value})
for node in root.childNodes:
entry = node.nodeName.replace(em, "")
if entry in result.keys():
result.update({entry: get_text(node)})
except:
return
extract_info(description, details)
def check_version(version_info, version):
if (version_info['id'] == 'zotero@chnm.gmu.edu' and
(min_version := version_info['minVersion']) and (max_version := version_info['maxVersion'])):
min_version = min_version.replace('*', '0')
max_version = max_version.replace('*', '999')
return (compare_versions(min_version, version.replace('*', '999')) <= 0 and
compare_versions(version.replace('*', '0'), max_version) <= 0)
return False
for version in kwargs.get('zotero_versions', []):
version_info = {'id': None, 'minVersion': None, 'maxVersion': None}
check_flag = True
for targetApplication in description.getElementsByTagName(em + "targetApplication"):
if not check_flag:
break
extract_info(targetApplication, version_info)
if check_version(version_info, version):
details.setdefault('zotero_versions', []).append(version)
break
for node in targetApplication.childNodes:
extract_info(node, version_info)
if check_version(version_info, version):
details.setdefault('zotero_versions', []).append(version)
check_flag = False
break
return details
except Exception as e:
raise e
def addon_details(addon_path, **kwargs):
if not os.path.exists(addon_path):
raise IOError("Add-on path does not exist: %s" % addon_path)
result = {}
if ((manifest := manifest_from_json(addon_path)) and
(details := detail_from_manifest_json(addon_path, manifest, zotero_versions=kwargs.get('zotero_versions')))):
result = details
if ((manifest := manifest_from_rdf(addon_path)) and
(details := detail_from_manifest_rdf(manifest, zotero_versions=kwargs.get('zotero_versions')))):
for key in details:
if key not in result:
result[key] = details[key]
result['zotero_versions'] = list(set(result.get('zotero_versions', []) + details.get('zotero_versions', [])))
return result