parvej/zotero-addons-scraper
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Merge pull request #34 from syt2/dev/check_xpi_from_xpi_update_json

Browse Source 
Dev/check xpi from xpi update json
This commit is contained in:
syt 2024-12-26 23:18:17 +08:00 committed by GitHub
commit 9fa26c3826
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 94 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
addons/syt2#Zotero-Addons-Scrape-To-GitHub#zotero-gpt-pro.json
View File

@ -1,9 +0,0 @@
{
"repo": "syt2/Zotero-Addons-Scrape-To-GitHub",
"releases": [
{
"targetZoteroVersion": "7",
"tagName": "zotero-gpt"
}
]
}

9
addons/syt2#Zotero-Addons-Scrape-To-GitHub#zotero-reference-pro.json
View File

@ -1,9 +0,0 @@
{
"repo": "syt2/Zotero-Addons-Scrape-To-GitHub",
"releases": [
{
"targetZoteroVersion": "7",
"tagName": "zotero-reference"
}
]
}

9
addons/syt2#Zotero-Addons-Scrape-To-GitHub#zotero-style-pro.json
View File

@ -1,9 +0,0 @@
{
"repo": "syt2/Zotero-Addons-Scrape-To-GitHub",
"releases": [
{
"targetZoteroVersion": "7",
"tagName": "zotero-style"
}
]
}

92
main.py
View File

@ -3,7 +3,7 @@ import urllib.parse
import argparse
from addon_info import *
from fallback_infos import fallback_if_need
from moz_addons import addon_details
from moz_addons import addon_details, compare_versions
from github_operations import *
from file_cache import *
@ -25,6 +25,7 @@ def download_xpi(xpi_url: str, download_dir: str, unique_name: str, force_downlo
return download_filepath
response = requests.get(xpi_url, stream=True)
if response.status_code == 200:
print(f'download {unique_name} from {xpi_url}')
with open(download_filepath, "wb") as file:
for chunk in response.iter_content(chunk_size=1024):
@ -102,34 +103,10 @@ def parse(plugin: AddonInfo, **kwargs):
release_asset = release_assets[0]
if 'browser_download_url' not in release_asset:
continue
xpi_url = release_asset['browser_download_url']
release.releaseDate = release_asset['updated_at']
release.xpiDownloadUrl = {
'github': xpi_url,
'ghProxy': 'https://ghproxy.com/?q=' + urllib.parse.quote(xpi_url),
'kgithub': xpi_url.replace('github.com', 'kkgithub.com'),
}
xpi_filename = f'{plugin.owner}#{plugin.repository}+{release.tagName}@{release_asset["id"]}.xpi'
details = None
priority_sources = ['rdf', 'json']
if release.targetZoteroVersion == '6':
priority_sources = ['json', 'rdf']
try:
xpi_filepath = download_xpi(xpi_url=xpi_url,
download_dir=kwargs.get('runtime_xpi_directory'),
unique_name=xpi_filename,
force_download=False,
cache_dir=kwargs.get('cache_directory'))
details = addon_details(xpi_filepath, priority_sources=priority_sources)
except:
try:
xpi_filepath = download_xpi(xpi_url=xpi_url,
download_dir=kwargs.get('runtime_xpi_directory'),
unique_name=xpi_filename,
force_download=True)
details = addon_details(xpi_filepath, priority_sources=priority_sources)
except Exception as e:
print(f'fetch addon detail of {plugin.repo} with {xpi_url} failed: {e}')
details = parse_xpi_detail(
plugin=plugin, release=release, release_asset=release_asset,
**kwargs,
)
if details:
if detail_id := details.id:
@ -180,6 +157,63 @@ def parse(plugin: AddonInfo, **kwargs):
return plugin
def parse_xpi_detail(plugin: AddonInfo, release: AddonInfoRelease, release_asset, **kwargs):
priority_sources = ['rdf', 'json']
if release.targetZoteroVersion == '6':
priority_sources = ['json', 'rdf']
def xpi_detail(xpi_url, xpi_filename):
try:
xpi_filepath = download_xpi(xpi_url=xpi_url,
download_dir=kwargs.get('runtime_xpi_directory'),
unique_name=xpi_filename,
force_download=False,
cache_dir=kwargs.get('cache_directory'))
details = addon_details(xpi_filepath, priority_sources=priority_sources)
return details
except:
try:
xpi_filepath = download_xpi(xpi_url=xpi_url,
download_dir=kwargs.get('runtime_xpi_directory'),
unique_name=xpi_filename,
force_download=True)
details = addon_details(xpi_filepath, priority_sources=priority_sources)
return details
except Exception as e:
print(f'fetch addon detail of {plugin.repo} with {xpi_url} failed: {e}')
github_xpi_url = release_asset['browser_download_url']
release.releaseDate = release_asset['updated_at']
release.xpiDownloadUrl = {
'github': github_xpi_url,
'ghProxy': 'https://ghproxy.com/?q=' + urllib.parse.quote(github_xpi_url),
'kgithub': github_xpi_url.replace('github.com', 'kkgithub.com'),
}
details = xpi_detail(xpi_url=github_xpi_url,
xpi_filename=f'{plugin.owner}#{plugin.repository}+{release.tagName}@{release_asset["id"]}.xpi')
if details and details.update_url and details.id and details.version:
try:
update_json_resp = requests.get(details.update_url,
headers=github_api_headers(github_token=kwargs.get('github_token')))
update_json_info = json.loads(update_json_resp.content)
updates = update_json_info.get('addons', {}).get(details.id, {}).get('updates', [])
xpi_urls = [e.get('update_link') for e in updates if compare_versions(e.get('version'), details.version) > 0]
for xpi_url in xpi_urls:
update_details = xpi_detail(xpi_url=xpi_url,
xpi_filename=f'{plugin.owner}#{plugin.repository}+update{details.version}.xpi')
if update_details and update_details.id and update_details.check_compatible_for_zotero_version(release.zotero_check_version):
details = update_details
release.xpiDownloadUrl = {
'github': xpi_url,
}
break
except: pass
return details
def fallback(addon_infos, previous_info_url, **kwargs):
try:
repos_resp = requests.get(previous_info_url,

28
moz_addons.py
View File

@ -43,6 +43,7 @@ class XpiDetail:
self.name = None
self.version = None
self.description = None
self.update_url = None
self.min_version = "*"
self.max_version = "*"
@ -58,6 +59,10 @@ class XpiDetail:
self.version = version
if description := details.get('description'):
self.description = description
if update_url := details.get('updateURL'):
self.update_url = update_url
if update_url := details.get('update_url'):
self.update_url = update_url
if (min_version := details.get('min_version')) and (max_version := details.get('max_version')):
if compare_versions(min_version.replace('*', '0'), self.min_version.replace('*', '999')) <= 0:
self.min_version = min_version
@ -125,8 +130,13 @@ def manifest_from_rdf(addon_path):
def detail_from_manifest_json(addon_path, manifest):
details = {
"name": manifest.get("name"), "version": manifest.get("version"), "description": manifest.get("description"),
"id": None, "min_version": None, "max_version": None
"name": manifest.get("name"),
"version": manifest.get("version"),
"description": manifest.get("description"),
"id": None,
"min_version": None,
"max_version": None,
"update_url": None,
}
for location in ("applications", "browser_specific_settings"):
if details["id"]:
@ -136,6 +146,7 @@ def detail_from_manifest_json(addon_path, manifest):
details["id"] = manifest[location][app].get("id")
details["min_version"] = manifest[location][app].get("strict_min_version")
details["max_version"] = manifest[location][app].get("strict_max_version")
details["update_url"] = manifest[location][app].get("update_url")
break
except KeyError:
continue
@ -175,7 +186,15 @@ def detail_from_manifest_json(addon_path, manifest):
def detail_from_manifest_rdf(manifest):
details = {"id": None, "name": None, "version": None, "description": None, "min_version": None, "max_version": None}
details = {
"id": None,
"name": None,
"version": None,
"description": None,
"min_version": None,
"max_version": None,
"updateURL": None,
}
try:
doc = minidom.parseString(manifest)
@ -238,6 +257,9 @@ def detail_from_manifest_rdf(manifest):
extract_info(node, version_info)
update_details(version_info)
if update_url := details.get('updateURL'):
details['update_url'] = update_url
return details
except Exception as e:
raise e