parvej/zotero-addons-scraper
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

update json infos

Browse Source
This commit is contained in:
shenyutao 2024-03-07 20:37:51 +08:00
parent 6d48e275a8
commit a05282b91d
6 changed files with 203 additions and 166 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
README.md
View File

@ -19,7 +19,6 @@ Adding a new add-on entry `{owner}#{repo}.json` in the [addons](addons) folder.
e.g.,
``` json
{
"name": "Zotero Addons",
"repo": "syt2/zotero-addons",
"releases": [
{
@ -33,10 +32,7 @@ e.g.,
]
}
```
- `name`(optional):
Name of the add-on.
If not provided, `name` will be extracted automatically from the XPI provided in release.
- `repo`(required):
GitHub repository of the plugin.
- `releases`(required):
@ -48,7 +44,8 @@ e.g.,
- `tagName`(required):
The release tag name in the GitHub repository.
Supports `"latest"`, `"pre"` or `a specified tag name`.
*If `"latest"` or `"pre"` is used, script will automatically retrieve the latest release tag.*
*If `"latest"` is used, script will automatically retrieve the latest release tag.*
*If `"pre"` is used, script will automatically retrieve the latest pre-release tag.*
## Usage

103
addon_info.py Normal file
View File

@ -0,0 +1,103 @@
class AddonInfoAuthor:
def __init__(self, name: str = None, url: str = None, avatar: str = None,
**kwargs):
self.name = name
self.url = url
self.avatar = avatar
[setattr(self, key, value) for (key, value) in kwargs.items()]
@property
def __dict__(self):
result = {}
if self.name:
result["name"] = self.name
if self.url:
result["url"] = self.url
if self.avatar:
result["avatar"] = self.avatar
return result
class AddonInfoRelease:
def __init__(self, targetZoteroVersion: str, tagName: str,
xpiDownloadUrl: dict = None, releaseDate: str = None, id: str = None, xpiVersion: str = None,
name: str = None, description: str = None,
**kwargs):
self.targetZoteroVersion = targetZoteroVersion
self.tagName = tagName
self.xpiDownloadUrl = xpiDownloadUrl
self.releaseDate = releaseDate
self.id = id
self.xpiVersion = xpiVersion
self.name = name
self.description = description
[setattr(self, key, value) for (key, value) in kwargs.items()]
@property
def __dict__(self):
result = {}
if self.targetZoteroVersion:
result["targetZoteroVersion"] = self.targetZoteroVersion
if self.tagName:
result["tagName"] = self.tagName
if self.xpiDownloadUrl:
result["xpiDownloadUrl"] = self.xpiDownloadUrl
if self.releaseDate:
result["releaseDate"] = self.releaseDate
if self.id:
result["id"] = self.id
if self.xpiVersion:
result["xpiVersion"] = self.xpiVersion
if self.name:
result["name"] = self.name
if self.description:
result["description"] = self.description
return result
class AddonInfo:
def __init__(self, repo: str,
releases: list[dict] | list[AddonInfoRelease],
name: str = None,
description: str = None,
stars: int = None,
author: dict | AddonInfoAuthor = None,
**kwargs):
self.repo = repo
self.releases = [e if isinstance(e, AddonInfoRelease) else AddonInfoRelease(**(e if e else {})) for e in releases]
self.name = name
self.description = description
self.stars = stars
self.author = author if isinstance(author, AddonInfoAuthor) else AddonInfoAuthor(**(author if author else {}))
[setattr(self, key, value) for (key, value) in kwargs.items()]
@property
def owner(self):
if len(self.repo.split('/')) != 2:
return None
return self.repo.split('/')[0]
@property
def repository(self):
if len(self.repo.split('/')) != 2:
return None
return self.repo.split('/')[1]
@property
def __dict__(self):
result = {}
if self.repo:
result["repo"] = self.repo
if self.releases:
result["releases"] = [release.__dict__ for release in self.releases]
if self.name:
result["name"] = self.name
if self.description:
result["description"] = self.description
if self.stars:
result["stars"] = self.stars
# todo: Deprecation, used for z7 < 1.5.3 and z6 < 0.6.7
result["star"] = self.stars
if self.author:
result["author"] = self.author.__dict__
return result

77
addon_keys.py
View File

@ -1,77 +0,0 @@
name = "name"
repo = "repo"
releases = "releases"
targetZoteroVersion = "targetZoteroVersion"
tagName = "tagName"
currentVersion = "currentVersion"
xpiDownloadUrl = "xpiDownloadUrl"
github = "github"
gitee = "gitee"
ghProxy = "ghProxy"
jsdeliver = "jsdeliver"
kgithub = "kgithub"
releaseData = "releaseData"
releaseDate = "releaseDate"
downloadCount = "downloadCount"
assetId = "assetId"
id = "id"
xpiVersion = "xpiVersion"
description = "description"
star = "star"
stars = "stars"
author = "author"
url = "url"
avatar = "avatar"
# from https://github.com/zotero-chinese/zotero-plugins/blob/main/src/plugins.ts
# export interface PluginInfo {
# // plugin id
# // @deprecated This property is deprecated. Use `id` in releases instead.
# id?: string;
#
# // plugin name
# name?: string;
#
# // plugin repository address on GitHub
# // e.g., syt2/zotero-addons
# repo: string;
#
# // plugin releases information
# releases: Array<{
# // Zotero version corresponding to current release
# targetZoteroVersion: string;
#
# // Download channel corresponding to the current release version
# // `latest`: Latest stable release
# // `pre`: Latest pre-release
# // `string`: Corresponding to the `git.tag_name` of the release;
# tagName: "latest" | "pre" | string;
# // @deprecated This property is deprecated. Use `tagName` instead.
# currentVersion?: string;
# xpiDownloadUrl?: {
# github: string;
# gitee: string;
# ghProxy: string;
# jsdeliver: string;
# kgithub: string;
# };
# releaseDate?: string;
# // @deprecated This property is deprecated. Use `releaseDate` instead.
# releaseData?: string;
# downloadCount?: number;
# assetId?: number;
# // plugin xpi id
# id?: string;
# // plugin xpi version
# xpiVersion?: string;
# }>;
#
# description?: string;
# stars?: number;
# author?: {
# name: string;
# url: string;
# avatar: string;
# };
# }

14
fallback_infos.py
View File

@ -1,15 +1,13 @@
from addon_keys import *
fallback_associated_unique_id = 'p_unique_associated_id##80502300-9DE6-4510-8768-EC42B0EF14E6'
fallback_associated_exclude_ids = 'p_exclude_associated_ids##073F5543-C70A-46AA-8529-E05168852D8F'
fallback_associated_map = {
fallback_associated_unique_id: repo,
fallback_associated_exclude_ids: [id],
repo: {
releases: {
fallback_associated_unique_id: tagName,
fallback_associated_exclude_ids: [currentVersion],
fallback_associated_unique_id: 'repo',
fallback_associated_exclude_ids: ['id'],
'repo': {
'releases': {
fallback_associated_unique_id: 'tagName',
fallback_associated_exclude_ids: ['currentVersion'],
}
}
}

110
main.py
View File

@ -7,7 +7,7 @@ import requests
import argparse
import shutil
import hashlib
from addon_keys import *
from addon_info import *
from fallback_infos import fallback_if_need
from moz_addons import addon_details
@ -49,70 +49,60 @@ def download_xpi(xpi_url: str, download_dir: str, unique_name: str, force_downlo
# 输出格式参考 [zotero-chinese/zotero-plugins](https://github.com/zotero-chinese/zotero-plugins)
def parse(plugin, **kwargs):
if repo not in plugin or len(plugin[repo].split('/')) != 2 or releases not in plugin or len(plugin[releases]) <= 0:
def parse(plugin: AddonInfo, **kwargs):
if not plugin.owner or not plugin.releases or len(plugin.releases) <= 0:
return
plugin.name = plugin.repository
headers = github_api_headers(github_token=kwargs.get('github_token'))
# fetch author info
author_url = f"https://api.github.com/users/{plugin[repo].split('/')[0]}"
author_url = f"https://api.github.com/users/{plugin.owner}"
try:
author_resp = requests.get(author_url, headers=headers)
author_resp_info = json.loads(author_resp.content)
plugin_author = {}
plugin_author[name] = author_resp_info['name'] if 'name' in author_resp_info and author_resp_info['name'] \
else plugin[repo].split('/')[0]
plugin.author.name = author_resp_info['name'] if 'name' in author_resp_info and author_resp_info['name'] else plugin.owner
if 'html_url' in author_resp_info:
plugin_author[url] = author_resp_info['html_url']
plugin.author.url = author_resp_info['html_url']
if 'avatar_url' in author_resp_info:
plugin_author[avatar] = author_resp_info['avatar_url']
if plugin_author.keys():
plugin[author] = plugin_author
plugin.author.avatar = author_resp_info['avatar_url']
except Exception as e:
print(f'request {author_url} failed: {e}')
# fetch repo info
repo_url = f'https://api.github.com/repos/{plugin[repo]}'
repo_url = f'https://api.github.com/repos/{plugin.repo}'
try:
repos_resp = requests.get(repo_url, headers=headers)
repos_info = json.loads(repos_resp.content)
if repos_info['description'] and description not in plugin:
plugin[description] = repos_info['description']
if repos_info['stargazers_count'] is not None:
plugin[stars] = repos_info['stargazers_count']
# todo: remove
plugin[star] = repos_info['stargazers_count']
if 'description' in repos_info and repos_info['description'] and not plugin.description:
plugin.description = repos_info['description']
if 'stargazers_count' in repos_info and repos_info['stargazers_count'] is not None:
plugin.stars = repos_info['stargazers_count']
except Exception as e:
print(f'request {repo_url} failed: {e}')
# fetch release info
release_infos = []
for release in plugin[releases]:
if tagName not in release:
for release in plugin.releases:
if not release.tagName:
continue
release_url = f'https://api.github.com/repos/{plugin[repo]}/releases'
if release[tagName] == 'latest':
release_url = f'https://api.github.com/repos/{plugin.repo}/releases'
if release.tagName == 'latest':
release_url += '/latest'
elif release[tagName] != 'pre':
release_url += f'/tags/{release[tagName]}'
elif release.tagName != 'pre':
release_url += f'/tags/{release.tagName}'
try:
release_resp = requests.get(release_url, headers=headers)
release_info = json.loads(release_resp.content)
if release[tagName] == 'pre':
if release.tagName == 'pre':
release_info = [info for info in release_info if info['prerelease']]
if release_info:
release_info = release_info[0]
else:
continue
if 'tag_name' in release_info:
release[tagName] = release_info['tag_name']
# todo: 删除currentVersion
# zotero-addons插件相关：
# 等待影响降低后再删除 (影响2023-12-14日[1.4.0]之前的版本)
release[currentVersion] = release_info['tag_name']
release.tagName = release_info['tag_name']
if 'assets' not in release_info:
continue
@ -125,24 +115,13 @@ def parse(plugin, **kwargs):
if 'browser_download_url' not in release_asset:
continue
xpi_url = release_asset['browser_download_url']
if 'id' in release_asset:
release[assetId] = release_asset['id']
if 'download_count' in release_asset:
release[downloadCount] = release_asset['download_count']
release[releaseDate] = release_asset['updated_at']
# todo: releaseData
# zotero-addons插件相关：
# 等待影响降低后再删除 (影响2023-12-18日[1.4.1]之前的版本)
release[releaseData] = release_asset['updated_at']
release[xpiDownloadUrl] = {
release.releaseDate = release_asset['updated_at']
release.xpiDownloadUrl = {
'github': xpi_url,
'ghProxy': 'https://ghproxy.com/?q=' + urllib.parse.quote(xpi_url),
'kgithub': xpi_url.replace('github.com', 'kkgithub.com'),
}
xpi_filename = f'{plugin[repo].replace("/", "#")}+' \
f'{release.get(currentVersion, "0")}@' \
f'{release.get(assetId, "0")}.xpi'
xpi_filename = f'{plugin.owner}#{plugin.repository}+{release.tagName}@{release_asset["id"]}.xpi'
details = {}
try:
xpi_filepath = download_xpi(xpi_url=xpi_url,
@ -159,35 +138,20 @@ def parse(plugin, **kwargs):
force_download=True)
details = addon_details(xpi_filepath)
except Exception as e:
print(f'fetch addon detail of {plugin[repo]} failed: {e}')
print(f'fetch addon detail of {plugin.repo} with {xpi_url} failed: {e}')
if details:
if detail_id := details.get('id'):
release[id] = detail_id
# todo: 删除plugin外层的id
# zotero-addons插件相关：
# 等待影响降低后再删除 (影响2023-12-14日[1.4.0]之前的版本)
plugin[id] = detail_id
if not plugin.get(name) and (detail_name := details.get('name')):
plugin[name] = detail_name
release.id = detail_id
if detail_name := details.get('name'):
release.name = detail_name
if detail_version := details.get('version'):
release[xpiVersion] = detail_version
if not plugin.get(description) and (detail_desc := details.get('description')):
if detail_desc != '__MSG_description__':
plugin[description] = detail_desc
else:
# todo: read from i18n
pass
release_infos.append(release)
release.xpiVersion = detail_version
if detail_desc := details.get('description'):
release.description = detail_desc
except Exception as e:
print(f'handle {plugin[repo]} request {release_url} failed: {e}')
plugin[releases] = release_infos
if name not in plugin:
plugin[name] = plugin[repo].split('/')[-1]
print(f'handle {plugin.repo} request {release_url} failed: {e}')
return plugin
@ -208,7 +172,10 @@ def parse_addon_infos(input_dir, output_filepath, **kwargs):
continue
addon_json_filepath = os.path.join(input_dir, addon_json_filename)
with open(addon_json_filepath, 'r') as file:
plugins.append(json.load(file))
try:
plugins.append(AddonInfo(**json.load(file)))
except Exception as e:
print(f'parse initial addon info from {addon_json_filepath} failed: {e}')
addon_infos = []
with concurrent.futures.ThreadPoolExecutor() as executor:
@ -223,10 +190,11 @@ def parse_addon_infos(input_dir, output_filepath, **kwargs):
if addon_info := future.result():
addon_infos.append(addon_info)
addon_infos = [info.__dict__ for info in addon_infos]
for previous_info_url in kwargs.get('previous_info_urls', []):
addon_infos = fallback(addon_infos, previous_info_url, github_token=kwargs.get('github_token'))
addon_infos.sort(key=lambda item: item[stars] if stars in item else 0, reverse=True)
addon_infos.sort(key=lambda item: item.get('stars') if item.get('stars') else 0, reverse=True)
dir = os.path.dirname(output_filepath)
if dir and not os.path.exists(dir):

56
moz_addons.py
View File

@ -2,6 +2,7 @@ import zipfile
import commentjson as json
import os
from xml.dom import minidom
import re
# modify from `https://github.com/mozilla/gecko-dev/blob/ac19e2c0d7c09a2deaedbe6afc4cdcf1a4561456/testing/mozbase/mozprofile/mozprofile/addons.py#L213`
@ -78,13 +79,13 @@ def addon_details(addon_path):
raise e
if is_webext:
details["version"] = manifest["version"]
details["name"] = manifest["name"]
details["description"] = manifest["description"]
details["version"] = manifest.get("version")
details["name"] = manifest.get("name")
details["description"] = manifest.get("description")
# Bug 1572404 - we support two locations for gecko-specific
# metadata.
for location in ("applications", "browser_specific_settings"):
for app in ("gecko", "zotero"):
for app in ("zotero", "gecko"):
try:
details["id"] = manifest[location][app]["id"]
break
@ -93,6 +94,53 @@ def addon_details(addon_path):
# if details["id"] is None:
# details["id"] = cls._gen_iid(addon_path)
details["unpack"] = False
# handler for __MSG_{}__ items
def extract_MSG_placeholder(text):
if match := re.search(r'__MSG_(.*?)__', text):
return match.group(1)
def load_locale_for_MSG():
default_locale = manifest.get('default_locale')
locale_filename = f"_locales/{default_locale}/messages.json"
try:
if zipfile.is_zipfile(addon_path):
# Bug 944361 - We cannot use 'with' together with zipFile because
# it will cause an exception thrown in Python 2.6.
try:
compressed_file = zipfile.ZipFile(addon_path, "r")
filenames = [f.filename for f in (compressed_file).filelist]
if locale_filename in filenames:
locale = compressed_file.read(locale_filename).decode()
return json.loads(locale)
else:
raise KeyError(f"No {locale_filename}")
finally:
compressed_file.close()
elif os.path.isdir(addon_path):
try:
with open(os.path.join(addon_path, locale_filename)) as f:
return json.loads(f.read())
except IOError:
raise KeyError(f"No {locale_filename}")
else:
raise IOError(
"Cannot found: %s" % locale_filename
)
except (IOError, KeyError) as e:
# reraise(AddonFormatError, AddonFormatError(str(e)), sys.exc_info()[2])
raise e
locale_for_MSG = None
for key in details:
if isinstance(details[key], str) and (placeholder := extract_MSG_placeholder(details[key])):
if not locale_for_MSG:
locale_for_MSG = load_locale_for_MSG()
if not locale_for_MSG:
break
if value := locale_for_MSG.get(placeholder, {}).get('message'):
details[key] = value
else:
try:
doc = minidom.parseString(manifest)