improve codes
This commit is contained in:
ytshen
parent
79169b1c90
commit
b57fe30479
2
.github/workflows/main.yml
vendored
2
.github/workflows/main.yml
vendored
@ -3,7 +3,7 @@ name: Zotero Addons Scraping
|
||||
on:
|
||||
workflow_dispatch: # manual
|
||||
schedule:
|
||||
- cron: '25 19 * * *' # 3:25(beijing)
|
||||
- cron: '25 4,10,19 * * *' # +8(beijing)
|
||||
push:
|
||||
branches: [master]
|
||||
|
||||
|
||||
27
file_cache.py
Normal file
27
file_cache.py
Normal file
@ -0,0 +1,27 @@
|
||||
import os
|
||||
import shutil
|
||||
import hashlib
|
||||
|
||||
|
||||
def folder_filename_hash(dir):
|
||||
filenames = sorted(os.listdir(dir))
|
||||
folder_hash = hashlib.new('sha256')
|
||||
for filename in filenames:
|
||||
hash_obj = hashlib.new('sha256')
|
||||
hash_obj.update(filename.encode('utf-8'))
|
||||
folder_hash.update(hash_obj.hexdigest().encode('utf-8'))
|
||||
return folder_hash.hexdigest()
|
||||
|
||||
|
||||
def update_cache(cache_directory, runtime_xpi_directory, cache_hash_filename):
|
||||
if not cache_directory or not runtime_xpi_directory or cache_directory == runtime_xpi_directory:
|
||||
return
|
||||
try:
|
||||
shutil.rmtree(cache_directory)
|
||||
shutil.move(runtime_xpi_directory, cache_directory)
|
||||
folder_hash = folder_filename_hash(cache_directory)
|
||||
with open(os.path.join(cache_directory, cache_hash_filename), 'w') as file:
|
||||
file.write(folder_hash)
|
||||
print(folder_hash)
|
||||
except Exception as e:
|
||||
print(f'update cache failed: {e}')
|
||||
111
github_operations.py
Normal file
111
github_operations.py
Normal file
@ -0,0 +1,111 @@
|
||||
import json
|
||||
import time
|
||||
import requests
|
||||
|
||||
|
||||
def github_api_headers(**kwargs):
|
||||
result = {
|
||||
'Accept': 'application/vnd.github+json',
|
||||
'X-GitHub-Api-Version': '2022-11-28',
|
||||
}
|
||||
if github_token := kwargs.get('github_token'):
|
||||
result['Authorization'] = f'token {github_token}'
|
||||
return result
|
||||
|
||||
|
||||
def report_issue(github_repository: str, title: str, body: str, **kwargs):
|
||||
if not github_repository:
|
||||
print('report issue repository not found')
|
||||
return
|
||||
try:
|
||||
response = requests.post(f'https://api.github.com/repos/{github_repository}/issues',
|
||||
headers=github_api_headers(github_token=kwargs.get('github_token')),
|
||||
json={
|
||||
'title': title,
|
||||
'body': body
|
||||
})
|
||||
|
||||
if response.status_code == 201:
|
||||
print('Issue created successfully.')
|
||||
print('Issue URL:', response.json()['html_url'])
|
||||
else:
|
||||
print('Failed to create issue.')
|
||||
print('Response:', response.content)
|
||||
except Exception as e:
|
||||
print(f'report issue failed: {e}')
|
||||
|
||||
|
||||
def upload_json_to_release(github_repository, release_id, upload_file_name, upload_file, **kwargs):
|
||||
headers = github_api_headers(github_token=kwargs.get('github_token'))
|
||||
headers["Content-Type"] = "application/octet-stream"
|
||||
upload_url = (f'https://uploads.github.com/repos/{github_repository}'
|
||||
f'/releases/{release_id}/assets?name={upload_file_name}')
|
||||
try:
|
||||
with open(upload_file, "rb") as file:
|
||||
upload_resp = requests.post(upload_url, data=file, headers=headers)
|
||||
if upload_resp.status_code != 201:
|
||||
print(f'upload release assets code: {upload_resp.status_code}')
|
||||
else:
|
||||
print('upload release assets succeed')
|
||||
except Exception as e:
|
||||
print(f'upload release assets failed: {e}')
|
||||
|
||||
|
||||
def create_release(github_repository, **kwargs):
|
||||
create_release_url = f'https://api.github.com/repos/{github_repository}/releases'
|
||||
cur_time = int(time.time())
|
||||
param = {
|
||||
'tag_name': f'{cur_time}',
|
||||
'target_commitish': 'master',
|
||||
'name': f'{cur_time}',
|
||||
'body': f'\n'
|
||||
f'publish addon_infos.json',
|
||||
'draft': False,
|
||||
'prerelease': False,
|
||||
'generate_release_notes': False,
|
||||
}
|
||||
try:
|
||||
create_resp = requests.post(create_release_url, json=param,
|
||||
headers=github_api_headers(github_token=kwargs.get('github_token')))
|
||||
if create_resp.status_code == 201:
|
||||
create_release_info = json.loads(create_resp.content)
|
||||
release_id = create_release_info['id']
|
||||
return release_id
|
||||
else:
|
||||
print(f'create release code: {create_resp.status_code}')
|
||||
except Exception as e:
|
||||
print(f'create release failed: {e}')
|
||||
|
||||
|
||||
def delete_cache(github_repository, github_token, remain_count=2):
|
||||
headers = github_api_headers(github_token=github_token)
|
||||
get_caches_url = (f'https://api.github.com/repos/{github_repository}'
|
||||
f'/actions/caches?per_page=100&sort=last_accessed_at&direction=desc')
|
||||
try:
|
||||
caches_resp = requests.get(get_caches_url, headers=headers)
|
||||
caches = json.loads(caches_resp.content)
|
||||
if caches.get('total_count', 0) < remain_count:
|
||||
return
|
||||
delete_cache_url = f'https://api.github.com/repos/{github_repository}/actions/caches/'
|
||||
for cache in caches.get('actions_caches', [])[remain_count:]:
|
||||
cache_key = cache.get('key')
|
||||
if cache_id := cache.get('id'):
|
||||
try:
|
||||
delete_cache_resp = requests.delete(f'{delete_cache_url}{cache_id}', headers=headers)
|
||||
if delete_cache_resp.status_code == 204:
|
||||
print(f'delete {cache_key} succeed')
|
||||
else:
|
||||
print(f'delete {cache_key} failed: {delete_cache_resp.text}')
|
||||
except Exception as e:
|
||||
print(f'delete cache for {cache_key} failed: {e}')
|
||||
except Exception as e:
|
||||
print(f'get caches failed: {e}')
|
||||
|
||||
|
||||
def rate_limit(github_token):
|
||||
try:
|
||||
resp = requests.get('https://api.github.com/rate_limit', headers=github_api_headers(github_token=github_token))
|
||||
rate = json.loads(resp.content)
|
||||
print(f'token rate {rate.get("rate")}')
|
||||
except Exception as e:
|
||||
print(f'get rate limit failed: {e}')
|
||||
137
main.py
137
main.py
@ -1,48 +1,11 @@
|
||||
import concurrent.futures
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import urllib.parse
|
||||
import requests
|
||||
import argparse
|
||||
import shutil
|
||||
import hashlib
|
||||
from addon_info import *
|
||||
from fallback_infos import fallback_if_need
|
||||
from moz_addons import addon_details
|
||||
|
||||
|
||||
def github_api_headers(**kwargs):
|
||||
result = {
|
||||
'Accept': 'application/vnd.github+json',
|
||||
'X-GitHub-Api-Version': '2022-11-28',
|
||||
}
|
||||
if github_token := kwargs.get('github_token'):
|
||||
result['Authorization'] = f'token {github_token}'
|
||||
return result
|
||||
|
||||
|
||||
def report_issue(repo: str, title: str, body: str, **kwargs):
|
||||
if not repo:
|
||||
print('report issue repo not found')
|
||||
return
|
||||
try:
|
||||
response = requests.post(f'https://api.github.com/repos/{repo}/issues',
|
||||
headers=github_api_headers(github_token=kwargs.get('github_token')),
|
||||
json={
|
||||
'title': title,
|
||||
'body': body
|
||||
})
|
||||
|
||||
# 检查响应
|
||||
if response.status_code == 201:
|
||||
print('Issue created successfully.')
|
||||
print('Issue URL:', response.json()['html_url'])
|
||||
else:
|
||||
print('Failed to create issue.')
|
||||
print('Response:', response.content)
|
||||
except Exception as e:
|
||||
print(f'report issue failed: {e}')
|
||||
from github_operations import *
|
||||
from file_cache import *
|
||||
|
||||
|
||||
def download_xpi(xpi_url: str, download_dir: str, unique_name: str, force_download: bool, **kwargs):
|
||||
@ -247,102 +210,6 @@ def parse_addon_infos(input_dir, output_filepath, **kwargs):
|
||||
return addon_infos
|
||||
|
||||
|
||||
def upload_json_to_release(github_repository, release_id, upload_file_name, upload_file, **kwargs):
|
||||
headers = github_api_headers(github_token=kwargs.get('github_token'))
|
||||
headers["Content-Type"] = "application/octet-stream"
|
||||
upload_url = f'https://uploads.github.com/repos/{github_repository}/releases/{release_id}/assets?name={upload_file_name}'
|
||||
try:
|
||||
with open(upload_file, "rb") as file:
|
||||
upload_resp = requests.post(upload_url, data=file, headers=headers)
|
||||
if upload_resp.status_code != 201:
|
||||
print(f'upload release assets code: {upload_resp.status_code}')
|
||||
else:
|
||||
print('upload release assets succeed')
|
||||
except Exception as e:
|
||||
print(f'upload release assets failed: {e}')
|
||||
|
||||
|
||||
def create_release(github_repository, **kwargs):
|
||||
create_release_url = f'https://api.github.com/repos/{github_repository}/releases'
|
||||
cur_time = int(time.time())
|
||||
param = {
|
||||
'tag_name': f'{cur_time}',
|
||||
'target_commitish': 'master',
|
||||
'name': f'{cur_time}',
|
||||
'body': f'\npublish addon_infos.json',
|
||||
'draft': False,
|
||||
'prerelease': False,
|
||||
'generate_release_notes': False,
|
||||
}
|
||||
try:
|
||||
create_resp = requests.post(create_release_url, json=param, headers=github_api_headers(github_token=kwargs.get('github_token')))
|
||||
if create_resp.status_code == 201:
|
||||
create_release_info = json.loads(create_resp.content)
|
||||
release_id = create_release_info['id']
|
||||
return release_id
|
||||
else:
|
||||
print(f'create release code: {create_resp.status_code}')
|
||||
except Exception as e:
|
||||
print(f'create release failed: {e}')
|
||||
|
||||
|
||||
def folder_filename_hash(dir):
|
||||
filenames = sorted(os.listdir(dir))
|
||||
folder_hash = hashlib.new('sha256')
|
||||
for filename in filenames:
|
||||
hash_obj = hashlib.new('sha256')
|
||||
hash_obj.update(filename.encode('utf-8'))
|
||||
folder_hash.update(hash_obj.hexdigest().encode('utf-8'))
|
||||
return folder_hash.hexdigest()
|
||||
|
||||
|
||||
def update_cache(cache_directory, runtime_xpi_directory, cache_hash_filename):
|
||||
if not cache_directory or not runtime_xpi_directory or cache_directory == runtime_xpi_directory:
|
||||
return
|
||||
try:
|
||||
shutil.rmtree(cache_directory)
|
||||
shutil.move(runtime_xpi_directory, cache_directory)
|
||||
folder_hash = folder_filename_hash(cache_directory)
|
||||
with open(os.path.join(cache_directory, cache_hash_filename), 'w') as file:
|
||||
file.write(folder_hash)
|
||||
print(folder_hash)
|
||||
except Exception as e:
|
||||
print(f'update cache failed: {e}')
|
||||
|
||||
|
||||
def delete_cache(github_repository, github_token, remain_count=2):
|
||||
headers = github_api_headers(github_token=github_token)
|
||||
get_caches_url = f'https://api.github.com/repos/{github_repository}/actions/caches?per_page=100&sort=last_accessed_at&direction=desc'
|
||||
try:
|
||||
caches_resp = requests.get(get_caches_url, headers=headers)
|
||||
caches = json.loads(caches_resp.content)
|
||||
if caches.get('total_count', 0) < remain_count:
|
||||
return
|
||||
delete_cache_url = f'https://api.github.com/repos/{github_repository}/actions/caches/'
|
||||
for cache in caches.get('actions_caches', [])[remain_count:]:
|
||||
cache_key = cache.get('key')
|
||||
if cache_id := cache.get('id'):
|
||||
try:
|
||||
delete_cache_resp = requests.delete(f'{delete_cache_url}{cache_id}', headers=headers)
|
||||
if delete_cache_resp.status_code == 204:
|
||||
print(f'delete {cache_key} succeed')
|
||||
else:
|
||||
print(f'delete {cache_key} failed: {delete_cache_resp.text}')
|
||||
except Exception as e:
|
||||
print(f'delete cache for {cache_key} failed: {e}')
|
||||
except Exception as e:
|
||||
print(f'get caches failed: {e}')
|
||||
|
||||
|
||||
def rate_limit(github_token):
|
||||
try:
|
||||
resp = requests.get('https://api.github.com/rate_limit', headers=github_api_headers(github_token=github_token))
|
||||
rate = json.loads(resp.content)
|
||||
print(f'token rate {rate.get("rate")}')
|
||||
except Exception as e:
|
||||
print(f'get rate limit failed: {e}')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(description='params')
|
||||
parser.add_argument('--github_repository', nargs='?', type=str, required=True, help='github repository')
|
||||
|
||||
Loading…
Reference in New Issue
Block a user