parvej/zotero-addons-scraper
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

parse min and max version from xpi

Browse Source
This commit is contained in:
ytshen 2024-04-13 12:34:28 +08:00
parent 0d3a500588
commit 234215f785
3 changed files with 124 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
addon_info.py
View File

@ -22,6 +22,7 @@ class AddonInfoRelease:
def __init__(self, targetZoteroVersion: str, tagName: str,
xpiDownloadUrl: dict = None, releaseDate: str = None, id: str = None, xpiVersion: str = None,
name: str = None, description: str = None,
minZoteroVersion: str = None, maxZoteroVersion: str = None,
**kwargs):
self.targetZoteroVersion = targetZoteroVersion
self.tagName = tagName
@ -31,6 +32,8 @@ class AddonInfoRelease:
self.xpiVersion = xpiVersion
self.name = name
self.description = description
self.minZoteroVersion = minZoteroVersion
self.maxZoteroVersion = maxZoteroVersion
[setattr(self, key, value) for (key, value) in kwargs.items()]
@property
@ -60,6 +63,10 @@ class AddonInfoRelease:
result["name"] = self.name
if self.description:
result["description"] = self.description
if self.minZoteroVersion:
result["minZoteroVersion"] = self.minZoteroVersion
if self.maxZoteroVersion:
result["maxZoteroVersion"] = self.maxZoteroVersion
return result

41
main.py
View File

@ -47,7 +47,8 @@ def parse(plugin: AddonInfo, **kwargs):
author_resp = requests.get(author_url, headers=headers)
author_resp_info = json.loads(author_resp.content)
plugin.author.name = author_resp_info['name'] if 'name' in author_resp_info and author_resp_info['name'] else plugin.owner
plugin.author.name = author_resp_info['name'] if 'name' in author_resp_info and author_resp_info[
'name'] else plugin.owner
if 'html_url' in author_resp_info:
plugin.author.url = author_resp_info['html_url']
if 'avatar_url' in author_resp_info:
@ -109,21 +110,21 @@ def parse(plugin: AddonInfo, **kwargs):
'kgithub': xpi_url.replace('github.com', 'kkgithub.com'),
}
xpi_filename = f'{plugin.owner}#{plugin.repository}+{release.tagName}@{release_asset["id"]}.xpi'
details = {}
details = None
try:
xpi_filepath = download_xpi(xpi_url=xpi_url,
download_dir=kwargs.get('runtime_xpi_directory'),
unique_name=xpi_filename,
force_download=False,
cache_dir=kwargs.get('cache_directory'))
details = addon_details(xpi_filepath, zotero_versions=[release.zotero_check_version])
details = addon_details(xpi_filepath)
except:
try:
xpi_filepath = download_xpi(xpi_url=xpi_url,
download_dir=kwargs.get('runtime_xpi_directory'),
unique_name=xpi_filename,
force_download=True)
details = addon_details(xpi_filepath, zotero_versions=[release.zotero_check_version])
details = addon_details(xpi_filepath)
except Exception as e:
report_issue(kwargs.get('github_repository'),
title=f'Parse xpi detail failed',
@ -134,26 +135,41 @@ def parse(plugin: AddonInfo, **kwargs):
print(f'fetch addon detail of {plugin.repo} with {xpi_url} failed: {e}')
if details:
if detail_id := details.get('id'):
if detail_id := details.id:
release.id = detail_id
if detail_name := details.get('name'):
if detail_name := details.name:
release.name = detail_name
if detail_version := details.get('version'):
if detail_version := details.version:
release.xpiVersion = detail_version
if detail_desc := details.get('description'):
if detail_desc := details.description:
release.description = detail_desc
if release.zotero_check_version not in details.get('zotero_versions', []):
if zotero_min_version := details.min_version:
release.minZoteroVersion = zotero_min_version
if zotero_max_version := details.max_version:
release.maxZoteroVersion = zotero_max_version
if not details.check_compatible_for_zotero_version(release.zotero_check_version):
invalid_releases.append(release)
if ((not release.minZoteroVersion or release.minZoteroVersion == '*') and
(not release.maxZoteroVersion or release.maxZoteroVersion == '*')):
report_issue(kwargs.get('github_repository'),
title=f'Parse {plugin.repo} of zotero version failed',
body=f'xpi:{plugin.repo}@{release.tagName} on {release.targetZoteroVersion}\n',
github_token=kwargs.get('github_token'))
print(plugin.repo, release.targetZoteroVersion, xpi_filename, release.minZoteroVersion, release.maxZoteroVersion)
except Exception as e:
print(f'handle {plugin.repo} request {release_url} failed: {e}')
for invalid_release in invalid_releases:
print(plugin.repo, 'invalid', invalid_release.zotero_check_version)
report_issue(kwargs.get('github_repository'),
title=f'Invalid xpi with zotero version {invalid_release.zotero_check_version}',
title=f'Invalid {plugin.repo} xpi with zotero version {invalid_release.zotero_check_version}',
body=f'xpi:{plugin.repo}@{invalid_release.tagName}\n'
f'expect zotero version:{invalid_release.zotero_check_version}\n',
f'min zotero Version:{invalid_release.minZoteroVersion}\n'
f'max Zotero version:{invalid_release.maxZoteroVersion}'
f'expect Zotero version:{invalid_release.zotero_check_version}\n',
github_token=kwargs.get('github_token'))
plugin.releases.remove(invalid_release)
return plugin
@ -161,7 +177,8 @@ def parse(plugin: AddonInfo, **kwargs):
def fallback(addon_infos, previous_info_url, **kwargs):
try:
repos_resp = requests.get(previous_info_url, headers=github_api_headers(github_token=kwargs.get('github_token')))
repos_resp = requests.get(previous_info_url,
headers=github_api_headers(github_token=kwargs.get('github_token')))
repos_info = json.loads(repos_resp.content)
addon_infos = fallback_if_need(addon_infos, repos_info)
except Exception as e:

137
moz_addons.py
View File

@ -37,6 +37,41 @@ def compare_versions(version1, version2):
return 0
class XpiDetail:
def __init__(self):
self.id = None
self.name = None
self.version = None
self.description = None
self.min_version = "*"
self.max_version = "*"
def _append_info(self, details: dict):
if id := details.get('id'):
if self.id and self.id != id:
print(f'Xpi ID not match? {self.id} <==> {id}')
return
self.id = id
if name := details.get('name'):
self.name = name
if version := details.get('version'):
self.version = version
if description := details.get('description'):
self.description = description
if (min_version := details.get('min_version')) and (max_version := details.get('max_version')):
if compare_versions(min_version.replace('*', '0'), self.min_version.replace('*', '999')) <= 0:
self.min_version = min_version
if compare_versions(max_version.replace('*', '999'), self.max_version.replace('*', '0')) >= 0:
self.max_version = max_version
def check_compatible_for_zotero_version(self, version: str | int):
if isinstance(version, int):
version = str(version) + '.*'
if (min_version := self.min_version.replace('*', '0')) and (max_version := self.max_version.replace('*', '999')):
return (compare_versions(min_version, version.replace('*', '999')) <= 0
<= compare_versions(max_version, version.replace('*', '0')))
def get_namespace_id(doc, url):
attributes = doc.documentElement.attributes
for i in range(attributes.length):
@ -87,23 +122,22 @@ def manifest_from_rdf(addon_path):
print(f'Invalid Addon Path {addon_path}: {e}')
def detail_from_manifest_json(addon_path, manifest, **kwargs):
details = {"id": None, "name": manifest.get("name"), "version": manifest.get("version"),
"description": manifest.get("description")}
def detail_from_manifest_json(addon_path, manifest):
details = {
"name": manifest.get("name"), "version": manifest.get("version"), "description": manifest.get("description"),
"id": None, "min_version": None, "max_version": None
}
for location in ("applications", "browser_specific_settings"):
if details["id"]:
break
for app in ("zotero", "gecko"):
try:
details["id"] = manifest[location][app].get("id")
for version in kwargs.get('zotero_versions', []):
min_version = manifest[location][app].get("strict_min_version").replace('*', '0')
max_version = manifest[location][app].get("strict_max_version").replace('*', '999')
if (min_version and max_version and
compare_versions(min_version, version.replace('*', '999')) <= 0 and
compare_versions(version.replace('*', '0'), max_version) <= 0):
details.setdefault('zotero_versions', []).append(version)
details["min_version"] = manifest[location][app].get("strict_min_version")
details["max_version"] = manifest[location][app].get("strict_max_version")
break
except KeyError:
continue
# handler for __MSG_{}__ items
def extract_msg_placeholder(text):
@ -139,16 +173,14 @@ def detail_from_manifest_json(addon_path, manifest, **kwargs):
return details
def detail_from_manifest_rdf(manifest, **kwargs):
details = {"id": None, "name": None, "version": None, "description": None}
def detail_from_manifest_rdf(manifest):
details = {"id": None, "name": None, "version": None, "description": None, "min_version": None, "max_version": None}
try:
doc = minidom.parseString(manifest)
# Get the namespaces abbreviations
em = get_namespace_id(doc, "http://www.mozilla.org/2004/em-rdf#")
rdf = get_namespace_id(
doc, "http://www.w3.org/1999/02/22-rdf-syntax-ns#"
)
rdf = get_namespace_id(doc, "http://www.w3.org/1999/02/22-rdf-syntax-ns#")
description = doc.getElementsByTagName(rdf + "Description").item(0)
try:
@ -159,65 +191,60 @@ def detail_from_manifest_rdf(manifest, **kwargs):
except Exception as e:
pass
def extract_info(root, result):
def extract_info(node, result):
try:
for entry, value in root.attributes.items():
for entry, value in node.attributes.items():
entry = entry.replace(em, "")
if entry in result.keys():
result.update({entry: value})
for node in root.childNodes:
entry = node.nodeName.replace(em, "")
for child_node in node.childNodes:
entry = child_node.nodeName.replace(em, "")
if entry in result.keys():
result.update({entry: get_text(node)})
result.update({entry: get_text(child_node)})
except:
return
extract_info(description, details)
def check_version(version_info, version):
if (version_info['id'] == 'zotero@chnm.gmu.edu' and
(min_version := version_info['minVersion']) and (max_version := version_info['maxVersion'])):
min_version = min_version.replace('*', '0')
max_version = max_version.replace('*', '999')
return (compare_versions(min_version, version.replace('*', '999')) <= 0 and
compare_versions(version.replace('*', '0'), max_version) <= 0)
return False
def update_details(version_info):
if version_info['id'] != 'zotero@chnm.gmu.edu':
return
if (min_version := version_info['minVersion']) and (max_version := version_info['maxVersion']):
if exist_min_version := details['min_version']:
if compare_versions(min_version.replace('*', '0'), exist_min_version.replace('*', '999')) <= 0:
details['min_version'] = min_version
else:
details['min_version'] = min_version
if exist_max_version := details['max_version']:
if compare_versions(max_version.replace('*', '999'), exist_max_version.replace('*', '0')) >= 0:
details['max_version'] = max_version
else:
details['max_version'] = max_version
for version in kwargs.get('zotero_versions', []):
version_info = {'id': None, 'minVersion': None, 'maxVersion': None}
check_flag = True
for targetApplication in description.getElementsByTagName(em + "targetApplication"):
if not check_flag:
break
version_info = {'id': None, 'minVersion': None, 'maxVersion': None}
extract_info(targetApplication, version_info)
if check_version(version_info, version):
details.setdefault('zotero_versions', []).append(version)
break
update_details(version_info)
for node in targetApplication.childNodes:
version_info = {'id': None, 'minVersion': None, 'maxVersion': None}
extract_info(node, version_info)
if check_version(version_info, version):
details.setdefault('zotero_versions', []).append(version)
check_flag = False
break
update_details(version_info)
return details
except Exception as e:
raise e
def addon_details(addon_path, **kwargs):
def addon_details(addon_path) -> XpiDetail:
if not os.path.exists(addon_path):
raise IOError("Add-on path does not exist: %s" % addon_path)
result = {}
raise IOError(f"Add-on path does not exist: {addon_path}")
xpi_detail = XpiDetail()
if ((manifest := manifest_from_json(addon_path)) and
(details := detail_from_manifest_json(addon_path, manifest, zotero_versions=kwargs.get('zotero_versions')))):
result = details
(details := detail_from_manifest_json(addon_path, manifest))):
xpi_detail._append_info(details)
if ((manifest := manifest_from_rdf(addon_path)) and
(details := detail_from_manifest_rdf(manifest, zotero_versions=kwargs.get('zotero_versions')))):
for key in details:
if key not in result:
result[key] = details[key]
result['zotero_versions'] = list(set(result.get('zotero_versions', []) + details.get('zotero_versions', [])))
return result
(details := detail_from_manifest_rdf(manifest))):
xpi_detail._append_info(details)
return xpi_detail