diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0f343cb..1c0e1d4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -23,6 +23,23 @@ jobs: - name: Checkout Repository uses: actions/checkout@v4 + - name: Checkout publish branch (for release_cache) + uses: actions/checkout@v4 + with: + ref: publish + path: publish_branch + continue-on-error: true + + - name: Restore release_cache from publish branch + run: | + if [ -d "publish_branch/release_cache" ]; then + cp -r publish_branch/release_cache ./release_cache + echo "Restored release_cache from publish branch" + ls -la release_cache | head -20 + else + echo "No release_cache found in publish branch, starting fresh" + fi + - name: Set up Python uses: actions/setup-python@v5 with: @@ -32,68 +49,30 @@ jobs: run: | pip install -r requirements.txt - - name: Store Cache Folder - uses: actions/cache/restore@v4 - id: restore-cache - with: - path: | - caches - key: ${{ runner.os }}-files-${{ hashFiles('caches/caches_lockfile') }} - restore-keys: | - ${{ runner.os }}-files- - - - name: Get Cache Hash - id: get-cache-hash - run: | - echo "CACHE_HASH=$CACHE_HASH" >> "$GITHUB_OUTPUT" - env: - CACHE_HASH: ${{ hashFiles('caches/caches_lockfile') }} - - - name: Run Script + - name: Run Scraper run: | python main.py \ -i addons \ -o published/addon_infos.json \ - --github_repository ${GITHUB_REPOSITORY} \ - --github_token ${GITHUB_TOKEN} \ - --cache_directory caches \ - --runtime_xpi_directory xpis \ - --cache_lockfile caches_lockfile \ - --create_release True \ -# --previous_info_urls \ -# https://raw.githubusercontent.com/${GITHUB_REPOSITORY}/publish/addon_infos.json \ -# https://github.com/${GITHUB_REPOSITORY}/releases/latest/download/addon_infos.json + --release-cache-dir release_cache \ + --github_repository "${GITHUB_REPOSITORY}" \ + --github_token "${GITHUB_TOKEN}" \ + --create_release True env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Publish in Repository + - name: Copy release_cache to published folder + run: | + cp -r release_cache published/release_cache + echo "Contents of published folder:" + ls -la published + + - name: Publish to publish branch uses: peaceiris/actions-gh-pages@v4 with: github_token: ${{ secrets.GITHUB_TOKEN }} -# deploy_key: ${{ secrets.ACTIONS_DEPLOY_KEY }} publish_branch: publish user_name: 'github-actions[bot]' user_email: 'github-actions[bot]@users.noreply.github.com' publish_dir: ./published - - name: Check Cache Need Update - id: check-cache-update - run: | - if [[ "$PRE_CACHE_HASH" != "$CUR_CACHE_HASH" ]]; then - echo "SAVE_CACHE=true" >> "$GITHUB_OUTPUT" - else - echo "SAVE_CACHE=false" >> "$GITHUB_OUTPUT" - fi - env: - PRE_CACHE_HASH: ${{ steps.get-cache-hash.outputs.CACHE_HASH }} - CUR_CACHE_HASH: ${{ hashFiles('caches/caches_lockfile') }} - - - name: Save Cache Folder - if: steps.check-cache-update.outputs.SAVE_CACHE == 'true' - uses: actions/cache/save@v4 - id: save-cache - with: - path: | - caches - key: ${{ runner.os }}-files-${{ hashFiles('caches/caches_lockfile') }} - diff --git a/scripts/migrate_addons.py b/scripts/migrate_addons.py new file mode 100644 index 0000000..a422a27 --- /dev/null +++ b/scripts/migrate_addons.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 +""" +Migrate addons/*.json from old format to simplified format. + +Old format: +{ + "repo": "owner/name", + "releases": [ + {"targetZoteroVersion": "7", "tagName": "latest"}, + {"targetZoteroVersion": "6", "tagName": "v1.0.0"} + ] +} + +New format: +{ + "repo": "owner/name" +} + +The release information is now managed by release_cache.json instead. + +Usage: + python scripts/migrate_addons.py --input addons --output addons_new + python scripts/migrate_addons.py --input addons --in-place +""" + +import argparse +import json +import shutil +from pathlib import Path + + +def migrate_addon_file(input_path: Path, output_path: Path) -> bool: + """Migrate a single addon config file. + + Args: + input_path: Path to input file. + output_path: Path to output file. + + Returns: + True if migration was successful. + """ + try: + with open(input_path, "r", encoding="utf-8") as f: + data = json.load(f) + + # Extract only the repo field + repo = data.get("repo") + if not repo: + print(f" Warning: {input_path.name} has no 'repo' field, skipping") + return False + + new_data = {"repo": repo} + + # Write output + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, "w", encoding="utf-8") as f: + json.dump(new_data, f, ensure_ascii=False, indent=2) + f.write("\n") + + return True + except Exception as e: + print(f" Error processing {input_path.name}: {e}") + return False + + +def migrate_all(input_dir: Path, output_dir: Path, in_place: bool = False) -> dict: + """Migrate all addon config files. + + Args: + input_dir: Input directory containing addon configs. + output_dir: Output directory for migrated configs. + in_place: If True, overwrite original files. + + Returns: + Statistics dict. + """ + stats = {"total": 0, "migrated": 0, "skipped": 0, "errors": 0} + + if not input_dir.exists(): + print(f"Error: Input directory not found: {input_dir}") + return stats + + json_files = list(input_dir.glob("*.json")) + stats["total"] = len(json_files) + + print(f"Found {len(json_files)} addon config files") + + for json_file in sorted(json_files): + if in_place: + output_path = json_file + else: + output_path = output_dir / json_file.name + + print(f"Processing {json_file.name}...") + + # Read and check if already migrated + with open(json_file, "r", encoding="utf-8") as f: + data = json.load(f) + + if "releases" not in data: + print(f" Already in new format, skipping") + stats["skipped"] += 1 + if not in_place: + shutil.copy(json_file, output_path) + continue + + if migrate_addon_file(json_file, output_path): + stats["migrated"] += 1 + print(f" Migrated successfully") + else: + stats["errors"] += 1 + + return stats + + +def main(): + parser = argparse.ArgumentParser( + description="Migrate addons/*.json to simplified format" + ) + parser.add_argument( + "--input", + type=str, + default="addons", + help="Input directory (default: addons)", + ) + parser.add_argument( + "--output", + type=str, + default="addons_migrated", + help="Output directory (default: addons_migrated)", + ) + parser.add_argument( + "--in-place", + action="store_true", + help="Modify files in place (overwrites original files)", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Show what would be done without making changes", + ) + + args = parser.parse_args() + + input_dir = Path(args.input) + output_dir = Path(args.output) + + if args.dry_run: + print("Dry run mode - no changes will be made") + print(f"Input directory: {input_dir}") + print(f"Output directory: {output_dir}") + + json_files = list(input_dir.glob("*.json")) + print(f"\nWould process {len(json_files)} files:") + + for json_file in sorted(json_files): + with open(json_file, "r", encoding="utf-8") as f: + data = json.load(f) + if "releases" in data: + releases = data.get("releases", []) + print(f" {json_file.name}: would remove {len(releases)} releases") + else: + print(f" {json_file.name}: already migrated") + return + + if args.in_place: + confirm = input( + f"This will modify files in {input_dir} in place. Continue? [y/N] " + ) + if confirm.lower() != "y": + print("Aborted") + return + + stats = migrate_all(input_dir, output_dir, in_place=args.in_place) + + print(f"\nMigration complete:") + print(f" Total files: {stats['total']}") + print(f" Migrated: {stats['migrated']}") + print(f" Skipped (already new format): {stats['skipped']}") + print(f" Errors: {stats['errors']}") + + if not args.in_place: + print(f"\nMigrated files are in: {output_dir}") + print("To use them, run:") + print(f" rm -rf {input_dir} && mv {output_dir} {input_dir}") + + +if __name__ == "__main__": + main() diff --git a/src/zotero_scraper/__main__.py b/src/zotero_scraper/__main__.py index 2f3dc2c..733118b 100644 --- a/src/zotero_scraper/__main__.py +++ b/src/zotero_scraper/__main__.py @@ -4,9 +4,10 @@ import argparse import sys from pathlib import Path -from .cache.file_cache import FileCache +from .cache.release_cache import ReleaseCache from .config.settings import ScraperConfig -from .services.scraper import AddonScraper +from .services.cache_builder import ReleaseCacheBuilder +from .services.cache_scraper import CacheScraper from .utils.logging import setup_logging @@ -16,21 +17,57 @@ def parse_args() -> argparse.Namespace: description="Scrape Zotero addon information from GitHub" ) - # Required arguments + # Mode options + parser.add_argument( + "--build-cache-only", + action="store_true", + help="Only build/update release cache, don't generate output", + ) + parser.add_argument( + "--skip-build-cache", + action="store_true", + help="Skip building cache, only generate output from existing cache", + ) + + # Release cache options + parser.add_argument( + "--release-cache-dir", + type=str, + default="release_cache", + help="Release cache directory (default: release_cache)", + ) + parser.add_argument( + "--full-rebuild", + action="store_true", + help="Rebuild cache from scratch, ignoring existing cache", + ) + parser.add_argument( + "--parallel", + action="store_true", + help="Build cache in parallel (faster but uses more resources)", + ) + parser.add_argument( + "--max-workers", + type=int, + default=4, + help="Max parallel workers for cache building (default: 4)", + ) + + # GitHub options parser.add_argument( "--github_repository", type=str, - required=True, - help="GitHub repository (owner/repo)", + default=None, + help="GitHub repository (owner/repo) for publishing", ) - - # Optional arguments parser.add_argument( "--github_token", type=str, default=None, help="GitHub API token", ) + + # Input/Output options parser.add_argument( "-i", "--input", @@ -45,17 +82,13 @@ def parse_args() -> argparse.Namespace: default="addon_infos.json", help="Output JSON file path (default: addon_infos.json)", ) + + # Cache options parser.add_argument( "--cache_directory", type=str, default="caches", - help="Cache directory (default: caches)", - ) - parser.add_argument( - "--cache_lockfile", - type=str, - default="caches_lockfile", - help="Cache lockfile name (default: caches_lockfile)", + help="XPI download cache directory (default: caches)", ) parser.add_argument( "--runtime_xpi_directory", @@ -63,18 +96,24 @@ def parse_args() -> argparse.Namespace: default="xpis", help="Runtime XPI directory (default: xpis)", ) + + # Fallback options parser.add_argument( "--previous_info_urls", nargs="+", default=[], help="URLs to previous addon_infos.json for fallback", ) + + # Publish options parser.add_argument( "--create_release", type=lambda x: x.lower() in ("true", "1", "yes"), default=True, help="Create GitHub release (default: True)", ) + + # Logging parser.add_argument( "--log_level", type=str, @@ -87,7 +126,13 @@ def parse_args() -> argparse.Namespace: def main() -> int: - """Main entry point.""" + """Main entry point. + + Default flow: + 1. Build/update release cache (parse new releases) + 2. Generate addon_infos.json from cache + 3. Publish to GitHub release (if --create_release) + """ args = parse_args() # Setup logging @@ -99,8 +144,8 @@ def main() -> int: logger = logging.getLogger("zotero_scraper") # Validate arguments - if not args.github_repository: - logger.error("--github_repository is required") + if args.create_release and not args.github_repository: + logger.error("--github_repository is required when --create_release is True") return 1 # Create configuration @@ -110,35 +155,61 @@ def main() -> int: config.cache.cache_dir.mkdir(parents=True, exist_ok=True) config.cache.runtime_xpi_dir.mkdir(parents=True, exist_ok=True) - # Initialize scraper - scraper = AddonScraper(config) + # Initialize release cache + release_cache_dir = Path(args.release_cache_dir) + release_cache = ReleaseCache(release_cache_dir) + + # Step 1: Build/update cache (unless skipped) + if not args.skip_build_cache: + logger.info("Step 1: Building/updating release cache...") + + builder = ReleaseCacheBuilder(config, release_cache) + + if config.github.token: + builder.github.get_rate_limit() + + if args.parallel: + stats = builder.build_cache_parallel( + full_rebuild=args.full_rebuild, + max_workers=args.max_workers, + ) + else: + stats = builder.build_cache( + full_rebuild=args.full_rebuild, + ) + + logger.info(f"Cache build stats: {stats}") + + if args.build_cache_only: + logger.info("Done! (build-cache-only mode)") + return 0 + else: + if not release_cache_dir.exists(): + logger.error( + f"Release cache directory not found: {release_cache_dir}. " + "Cannot use --skip-build-cache without existing cache." + ) + return 1 + logger.info("Step 1: Skipped (using existing cache)") + + # Step 2: Generate addon_infos.json from cache + logger.info("Step 2: Generating addon_infos.json from cache...") + + scraper = CacheScraper(config, release_cache) - # Check rate limit if config.github.token: scraper.github.get_rate_limit() - # Run scraper - logger.info("Starting addon scraper...") addon_infos = scraper.scrape_all() - logger.info(f"Scraped {len(addon_infos)} addons") + logger.info(f"Generated info for {len(addon_infos)} addons") - # Publish if requested - if args.create_release: - logger.info("Publishing to GitHub release...") + # Step 3: Publish to GitHub release (if requested) + if args.create_release and args.github_repository: + logger.info("Step 3: Publishing to GitHub release...") scraper.publisher.publish(config.output_file) - - # Update cache - logger.info("Updating cache...") - cache = FileCache( - cache_dir=config.cache.cache_dir, - runtime_dir=config.cache.runtime_xpi_dir, - lockfile_name=config.cache.lockfile_name, - ) - cache.update_cache() - - # Clean up old caches - if config.github.token: scraper.publisher.cleanup_caches(keep_count=1) + else: + logger.info("Step 3: Skipped (--create_release is False or no repository)") logger.info("Done!") return 0 diff --git a/src/zotero_scraper/cache/__init__.py b/src/zotero_scraper/cache/__init__.py index 5222cc5..1831225 100644 --- a/src/zotero_scraper/cache/__init__.py +++ b/src/zotero_scraper/cache/__init__.py @@ -1,5 +1,11 @@ """Cache module.""" from .file_cache import FileCache +from .release_cache import CachedRelease, ReleaseCache, RepoCache -__all__ = ["FileCache"] +__all__ = [ + "CachedRelease", + "FileCache", + "ReleaseCache", + "RepoCache", +] diff --git a/src/zotero_scraper/cache/release_cache.py b/src/zotero_scraper/cache/release_cache.py new file mode 100644 index 0000000..f9aa202 --- /dev/null +++ b/src/zotero_scraper/cache/release_cache.py @@ -0,0 +1,346 @@ +"""Release cache service for storing parsed release information.""" + +import json +from dataclasses import asdict, dataclass, field +from datetime import UTC, datetime +from pathlib import Path +from typing import Any, Optional + +from ..utils.logging import get_logger + +logger = get_logger("cache.release_cache") + + +@dataclass +class CachedRelease: + """Cached release information parsed from XPI.""" + + tag: str + published_at: str + xpi_asset_id: int + xpi_name: str + xpi_download_url: str + # Parsed from XPI manifest + addon_id: Optional[str] = None + addon_name: Optional[str] = None + addon_version: Optional[str] = None + addon_description: Optional[str] = None + min_zotero_version: Optional[str] = None + max_zotero_version: Optional[str] = None + # Parse status + parse_success: bool = True + parse_error: Optional[str] = None + + def to_dict(self) -> dict[str, Any]: + """Convert to dictionary.""" + return {k: v for k, v in asdict(self).items() if v is not None} + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> "CachedRelease": + """Create from dictionary.""" + return cls( + tag=data.get("tag", ""), + published_at=data.get("published_at", ""), + xpi_asset_id=data.get("xpi_asset_id", 0), + xpi_name=data.get("xpi_name", ""), + xpi_download_url=data.get("xpi_download_url", ""), + addon_id=data.get("addon_id"), + addon_name=data.get("addon_name"), + addon_version=data.get("addon_version"), + addon_description=data.get("addon_description"), + min_zotero_version=data.get("min_zotero_version"), + max_zotero_version=data.get("max_zotero_version"), + parse_success=data.get("parse_success", True), + parse_error=data.get("parse_error"), + ) + + def is_compatible_with(self, zotero_version: str) -> bool: + """Check if this release is compatible with a Zotero version. + + Args: + zotero_version: Target version like "6" or "7". + + Returns: + True if compatible. + """ + if not self.parse_success or not self.min_zotero_version: + return False + + # Normalize versions + min_ver = self.min_zotero_version.replace("*", "0") + max_ver = (self.max_zotero_version or "999").replace("*", "999") + + # Check version range + target_min = float(zotero_version) + target_max = target_min + 0.999 + + try: + min_float = float(min_ver.split(".")[0]) + max_float = float(max_ver.split(".")[0]) if max_ver != "999" else 999 + except ValueError: + return False + + return min_float <= target_max and max_float >= target_min + + +@dataclass +class RepoCache: + """Cache for a single repository.""" + + last_checked: str = "" + checked_releases: list[CachedRelease] = field(default_factory=list) + + def to_dict(self) -> dict[str, Any]: + """Convert to dictionary.""" + return { + "last_checked": self.last_checked, + "checked_releases": [r.to_dict() for r in self.checked_releases], + } + + @classmethod + def from_dict(cls, data: dict[str, Any]) -> "RepoCache": + """Create from dictionary.""" + releases = [ + CachedRelease.from_dict(r) + for r in data.get("checked_releases", []) + ] + return cls( + last_checked=data.get("last_checked", ""), + checked_releases=releases, + ) + + def get_checked_tags(self) -> set[str]: + """Get set of already checked tags.""" + return {r.tag for r in self.checked_releases} + + def get_release_by_tag(self, tag: str) -> Optional[CachedRelease]: + """Get cached release by tag.""" + for release in self.checked_releases: + if release.tag == tag: + return release + return None + + def add_release(self, release: CachedRelease) -> None: + """Add or update a release in cache.""" + # Remove existing release with same tag + self.checked_releases = [ + r for r in self.checked_releases if r.tag != release.tag + ] + self.checked_releases.append(release) + + def remove_deleted_releases(self, current_tags: set[str]) -> list[str]: + """Remove releases that no longer exist on GitHub. + + Args: + current_tags: Set of tags that currently exist on GitHub. + + Returns: + List of removed tags. + """ + removed = [] + new_releases = [] + for release in self.checked_releases: + if release.tag in current_tags: + new_releases.append(release) + else: + removed.append(release.tag) + self.checked_releases = new_releases + return removed + + def get_best_release_for_zotero( + self, zotero_version: str + ) -> Optional[CachedRelease]: + """Get the best (latest compatible) release for a Zotero version. + + Args: + zotero_version: Target version like "6" or "7". + + Returns: + Best compatible release or None. + """ + compatible = [ + r for r in self.checked_releases + if r.is_compatible_with(zotero_version) + ] + + if not compatible: + return None + + # Sort by published_at descending to get latest + compatible.sort(key=lambda r: r.published_at, reverse=True) + return compatible[0] + + +class ReleaseCache: + """Service for managing release cache. + + Uses a directory structure where each repo has its own JSON file: + release_cache/ + ├── windingwind#zotero-pdf-translate.json + ├── MuiseDestiny#zotero-gpt.json + └── ... + """ + + def __init__(self, cache_dir: Path): + """Initialize release cache. + + Args: + cache_dir: Directory to store cache files. + """ + self.cache_dir = cache_dir + self._cache: dict[str, RepoCache] = {} + self._dirty: set[str] = set() # Track which repos need saving + + def _repo_to_filename(self, repo: str) -> str: + """Convert repo name to filename.""" + # owner/name -> owner#name.json + return repo.replace("/", "#") + ".json" + + def _filename_to_repo(self, filename: str) -> str: + """Convert filename to repo name.""" + # owner#name.json -> owner/name + return filename.replace(".json", "").replace("#", "/") + + def _get_repo_file(self, repo: str) -> Path: + """Get the cache file path for a repo.""" + return self.cache_dir / self._repo_to_filename(repo) + + def _load_repo(self, repo: str) -> RepoCache: + """Load cache for a single repo.""" + cache_file = self._get_repo_file(repo) + + if not cache_file.exists(): + return RepoCache() + + try: + with open(cache_file, "r", encoding="utf-8") as f: + data = json.load(f) + return RepoCache.from_dict(data) + except Exception as e: + logger.warning(f"Failed to load cache for {repo}: {e}") + return RepoCache() + + def _save_repo(self, repo: str) -> None: + """Save cache for a single repo.""" + if repo not in self._cache: + return + + try: + self.cache_dir.mkdir(parents=True, exist_ok=True) + cache_file = self._get_repo_file(repo) + + with open(cache_file, "w", encoding="utf-8") as f: + json.dump(self._cache[repo].to_dict(), f, ensure_ascii=False, indent=2) + + self._dirty.discard(repo) + except Exception as e: + logger.error(f"Failed to save cache for {repo}: {e}") + + def save(self) -> None: + """Save all dirty (modified) repos.""" + saved_count = 0 + for repo in list(self._dirty): + self._save_repo(repo) + saved_count += 1 + + if saved_count > 0: + logger.info(f"Saved cache for {saved_count} repos") + + def save_all(self) -> None: + """Save all repos in memory (regardless of dirty state).""" + for repo in self._cache: + self._save_repo(repo) + logger.info(f"Saved cache for {len(self._cache)} repos") + + def get_repo_cache(self, repo: str) -> RepoCache: + """Get cache for a repository. + + Args: + repo: Repository in "owner/name" format. + + Returns: + RepoCache instance (loads from file or creates empty one). + """ + if repo not in self._cache: + self._cache[repo] = self._load_repo(repo) + return self._cache[repo] + + def update_repo_checked_time(self, repo: str) -> None: + """Update the last checked time for a repository. + + Args: + repo: Repository in "owner/name" format. + """ + cache = self.get_repo_cache(repo) + cache.last_checked = datetime.now(UTC).isoformat() + self._dirty.add(repo) + + def get_unchecked_tags( + self, repo: str, all_tags: list[str] + ) -> list[str]: + """Get tags that haven't been checked yet. + + Args: + repo: Repository in "owner/name" format. + all_tags: List of all tags from GitHub. + + Returns: + List of unchecked tags. + """ + cache = self.get_repo_cache(repo) + checked = cache.get_checked_tags() + return [tag for tag in all_tags if tag not in checked] + + def add_release(self, repo: str, release: CachedRelease) -> None: + """Add a release to cache. + + Args: + repo: Repository in "owner/name" format. + release: CachedRelease instance. + """ + cache = self.get_repo_cache(repo) + cache.add_release(release) + self._dirty.add(repo) + + def sync_with_github( + self, repo: str, current_tags: set[str] + ) -> list[str]: + """Sync cache with GitHub, removing deleted releases. + + Args: + repo: Repository in "owner/name" format. + current_tags: Set of tags that currently exist on GitHub. + + Returns: + List of removed tags. + """ + cache = self.get_repo_cache(repo) + removed = cache.remove_deleted_releases(current_tags) + if removed: + self._dirty.add(repo) + return removed + + def get_repos(self) -> list[str]: + """Get all cached repository names from directory.""" + if not self.cache_dir.exists(): + return [] + + repos = [] + for cache_file in self.cache_dir.glob("*.json"): + repo = self._filename_to_repo(cache_file.name) + repos.append(repo) + return repos + + def get_stats(self) -> dict[str, Any]: + """Get cache statistics.""" + repos = self.get_repos() + total_releases = 0 + + for repo in repos: + cache = self.get_repo_cache(repo) + total_releases += len(cache.checked_releases) + + return { + "repos": len(repos), + "total_releases": total_releases, + } diff --git a/src/zotero_scraper/config/settings.py b/src/zotero_scraper/config/settings.py index b2439bb..5ab3948 100644 --- a/src/zotero_scraper/config/settings.py +++ b/src/zotero_scraper/config/settings.py @@ -35,7 +35,6 @@ class CacheConfig: cache_dir: Path = field(default_factory=lambda: Path("caches")) runtime_xpi_dir: Path = field(default_factory=lambda: Path("xpis")) - lockfile_name: str = "caches_lockfile" @dataclass(frozen=True) @@ -70,7 +69,6 @@ class ScraperConfig: cache=CacheConfig( cache_dir=Path(args.cache_directory), runtime_xpi_dir=Path(args.runtime_xpi_directory), - lockfile_name=args.cache_lockfile, ), previous_info_urls=args.previous_info_urls or [], create_release=args.create_release, diff --git a/src/zotero_scraper/services/__init__.py b/src/zotero_scraper/services/__init__.py index e7fbe10..3a3ecef 100644 --- a/src/zotero_scraper/services/__init__.py +++ b/src/zotero_scraper/services/__init__.py @@ -1,11 +1,15 @@ """Business services module.""" +from .cache_builder import ReleaseCacheBuilder +from .cache_scraper import CacheScraper from .fallback import FallbackService from .publisher import PublisherService from .scraper import AddonScraper __all__ = [ "AddonScraper", + "CacheScraper", "FallbackService", "PublisherService", + "ReleaseCacheBuilder", ] diff --git a/src/zotero_scraper/services/cache_builder.py b/src/zotero_scraper/services/cache_builder.py new file mode 100644 index 0000000..7564881 --- /dev/null +++ b/src/zotero_scraper/services/cache_builder.py @@ -0,0 +1,359 @@ +"""Service for building and updating release cache.""" + +import json +from concurrent.futures import ThreadPoolExecutor, as_completed +from pathlib import Path +from typing import Any, Optional + +from ..cache.release_cache import CachedRelease, ReleaseCache +from ..clients.downloader import XPIDownloader +from ..clients.github import GitHubClient, ReleaseAsset +from ..config.constants import ContentTypes +from ..config.settings import ScraperConfig +from ..parsers.xpi_parser import XPIParser +from ..utils.logging import get_logger + +logger = get_logger("services.cache_builder") + + +class ReleaseCacheBuilder: + """Service for building release cache by parsing all releases.""" + + def __init__( + self, + config: ScraperConfig, + release_cache: ReleaseCache, + ): + """Initialize cache builder. + + Args: + config: Scraper configuration. + release_cache: Release cache instance. + """ + self.config = config + self.cache = release_cache + self.github = GitHubClient(config.github) + self.downloader = XPIDownloader(config.cache) + self.xpi_parser = XPIParser() + + def build_cache( + self, + repos: Optional[list[str]] = None, + full_rebuild: bool = False, + max_releases_per_repo: Optional[int] = None, + ) -> dict[str, Any]: + """Build or update release cache for all repositories. + + Args: + repos: Optional list of repos to process. If None, loads from input dir. + full_rebuild: If True, ignore existing cache and rebuild everything. + max_releases_per_repo: Limit releases to process per repo (for testing). + + Returns: + Statistics about the build process. + """ + if repos is None: + repos = self._load_repos_from_input() + + logger.info(f"Building cache for {len(repos)} repositories") + + stats = { + "repos_processed": 0, + "repos_failed": 0, + "new_releases_parsed": 0, + "releases_deleted": 0, + "errors": [], + } + + for i, repo in enumerate(repos, 1): + logger.info(f"[{i}/{len(repos)}] Processing {repo}") + + try: + result = self._process_repo( + repo, + full_rebuild=full_rebuild, + max_releases=max_releases_per_repo, + ) + stats["repos_processed"] += 1 + stats["new_releases_parsed"] += result.get("new_releases", 0) + stats["releases_deleted"] += result.get("deleted_releases", 0) + except Exception as e: + logger.error(f"Failed to process {repo}: {e}") + stats["repos_failed"] += 1 + stats["errors"].append({"repo": repo, "error": str(e)}) + + # Save cache after each repo (immediate persistence) + self.cache.save() + + # Final save + self.cache.save() + + logger.info( + f"Cache build complete: {stats['repos_processed']} repos, " + f"{stats['new_releases_parsed']} new releases, " + f"{stats['releases_deleted']} deleted releases" + ) + + return stats + + def build_cache_parallel( + self, + repos: Optional[list[str]] = None, + full_rebuild: bool = False, + max_releases_per_repo: Optional[int] = None, + max_workers: int = 4, + ) -> dict[str, Any]: + """Build cache in parallel (faster but uses more resources). + + Args: + repos: Optional list of repos to process. + full_rebuild: If True, ignore existing cache. + max_releases_per_repo: Limit releases per repo. + max_workers: Number of parallel workers. + + Returns: + Statistics about the build process. + """ + if repos is None: + repos = self._load_repos_from_input() + + logger.info(f"Building cache for {len(repos)} repos with {max_workers} workers") + + stats = { + "repos_processed": 0, + "repos_failed": 0, + "new_releases_parsed": 0, + "releases_deleted": 0, + "errors": [], + } + + with ThreadPoolExecutor(max_workers=max_workers) as executor: + futures = { + executor.submit( + self._process_repo, + repo, + full_rebuild=full_rebuild, + max_releases=max_releases_per_repo, + ): repo + for repo in repos + } + + for future in as_completed(futures): + repo = futures[future] + try: + result = future.result() + stats["repos_processed"] += 1 + stats["new_releases_parsed"] += result.get("new_releases", 0) + stats["releases_deleted"] += result.get("deleted_releases", 0) + logger.info( + f"[{stats['repos_processed']}/{len(repos)}] " + f"Completed {repo}: {result.get('new_releases', 0)} new releases" + ) + except Exception as e: + logger.error(f"Failed to process {repo}: {e}") + stats["repos_failed"] += 1 + stats["errors"].append({"repo": repo, "error": str(e)}) + + self.cache.save() + + logger.info( + f"Cache build complete: {stats['repos_processed']} repos, " + f"{stats['new_releases_parsed']} new releases" + ) + + return stats + + def _load_repos_from_input(self) -> list[str]: + """Load repository list from input directory.""" + repos = [] + + if not self.config.input_dir.exists(): + logger.error(f"Input directory not found: {self.config.input_dir}") + return repos + + for config_file in self.config.input_dir.glob("*.json"): + try: + with open(config_file, "r", encoding="utf-8") as f: + data = json.load(f) + if repo := data.get("repo"): + repos.append(repo) + except Exception as e: + logger.error(f"Failed to load {config_file}: {e}") + + return repos + + def _process_repo( + self, + repo: str, + full_rebuild: bool = False, + max_releases: Optional[int] = None, + ) -> dict[str, int]: + """Process a single repository. + + Args: + repo: Repository in "owner/name" format. + full_rebuild: If True, reprocess all releases. + max_releases: Limit number of releases to process. + + Returns: + Statistics dict with new_releases and deleted_releases counts. + """ + parts = repo.split("/") + if len(parts) != 2: + raise ValueError(f"Invalid repo format: {repo}") + + owner, name = parts + + # Get all releases from GitHub + releases = self.github.get_releases(owner, name) + if not releases: + logger.warning(f"No releases found for {repo}") + return {"new_releases": 0, "deleted_releases": 0} + + # Get current tags from GitHub + current_tags = {r.get("tag_name") for r in releases if r.get("tag_name")} + + # Sync cache - remove deleted releases + deleted_tags = self.cache.sync_with_github(repo, current_tags) + if deleted_tags: + logger.info(f"{repo}: Removed {len(deleted_tags)} deleted releases: {deleted_tags}") + + # Get tags to process + if full_rebuild: + tags_to_process = list(current_tags) + else: + tags_to_process = self.cache.get_unchecked_tags(repo, list(current_tags)) + + if max_releases: + tags_to_process = tags_to_process[:max_releases] + + if not tags_to_process: + logger.debug(f"{repo}: No new releases to process") + self.cache.update_repo_checked_time(repo) + return {"new_releases": 0, "deleted_releases": len(deleted_tags)} + + logger.info(f"{repo}: Processing {len(tags_to_process)} new releases") + + # Process each new release + new_count = 0 + for release_data in releases: + tag = release_data.get("tag_name") + if tag not in tags_to_process: + continue + + cached = self._parse_release(repo, release_data) + if cached: + self.cache.add_release(repo, cached) + new_count += 1 + logger.debug(f"{repo}@{tag}: Parsed successfully") + else: + # Still add to cache with parse_success=False to avoid re-parsing + cached = CachedRelease( + tag=tag, + published_at=release_data.get("published_at", ""), + xpi_asset_id=0, + xpi_name="", + xpi_download_url="", + parse_success=False, + parse_error="No XPI found or parse failed", + ) + self.cache.add_release(repo, cached) + logger.debug(f"{repo}@{tag}: Parse failed, cached as failed") + + self.cache.update_repo_checked_time(repo) + + return {"new_releases": new_count, "deleted_releases": len(deleted_tags)} + + def _parse_release( + self, repo: str, release_data: dict[str, Any] + ) -> Optional[CachedRelease]: + """Parse a single release and return CachedRelease. + + Args: + repo: Repository in "owner/name" format. + release_data: Release data from GitHub API. + + Returns: + CachedRelease or None if no XPI found. + """ + tag = release_data.get("tag_name", "") + published_at = release_data.get("published_at", "") + assets = release_data.get("assets", []) + + # Find XPI asset + xpi_asset = self._find_xpi_asset(assets) + if not xpi_asset: + return None + + xpi_url = xpi_asset.get("browser_download_url", "") + xpi_name = xpi_asset.get("name", "") + xpi_id = xpi_asset.get("id", 0) + + # Download XPI + owner, name = repo.split("/") + filename = f"{owner}#{name}+{tag}@{xpi_id}.xpi" + xpi_path = self.downloader.download(xpi_url, filename) + + if not xpi_path: + return CachedRelease( + tag=tag, + published_at=published_at, + xpi_asset_id=xpi_id, + xpi_name=xpi_name, + xpi_download_url=xpi_url, + parse_success=False, + parse_error="Download failed", + ) + + # Parse XPI + details = self.xpi_parser.parse(xpi_path) + + if not details or not details.id: + return CachedRelease( + tag=tag, + published_at=published_at, + xpi_asset_id=xpi_id, + xpi_name=xpi_name, + xpi_download_url=xpi_url, + parse_success=False, + parse_error="Parse failed or no addon ID", + ) + + return CachedRelease( + tag=tag, + published_at=published_at, + xpi_asset_id=xpi_id, + xpi_name=xpi_name, + xpi_download_url=xpi_url, + addon_id=details.id, + addon_name=details.name, + addon_version=details.version, + addon_description=details.description, + min_zotero_version=details.min_version, + max_zotero_version=details.max_version, + parse_success=True, + ) + + def _find_xpi_asset(self, assets: list[dict]) -> Optional[dict]: + """Find XPI asset from release assets.""" + # Sort by updated_at descending + sorted_assets = sorted( + assets, key=lambda a: a.get("updated_at", ""), reverse=True + ) + + # First try XPI content type + for asset in sorted_assets: + if asset.get("content_type") == ContentTypes.XPI: + return asset + + # Fallback to ZIP content type + for asset in sorted_assets: + if asset.get("content_type") == ContentTypes.ZIP: + return asset + + # Fallback to .xpi extension + for asset in sorted_assets: + if asset.get("name", "").endswith(".xpi"): + return asset + + return None diff --git a/src/zotero_scraper/services/cache_scraper.py b/src/zotero_scraper/services/cache_scraper.py new file mode 100644 index 0000000..eb117ea --- /dev/null +++ b/src/zotero_scraper/services/cache_scraper.py @@ -0,0 +1,213 @@ +"""Scraper service that uses release cache to generate output.""" + +import json +from concurrent.futures import ThreadPoolExecutor, as_completed +from pathlib import Path +from typing import Any, Optional + +from ..cache.release_cache import CachedRelease, ReleaseCache +from ..clients.github import GitHubClient +from ..config.constants import XPIProxy +from ..config.settings import ScraperConfig +from ..models.addon import AddonInfo, AddonRelease, Author, XpiDownloadUrls +from ..utils.logging import get_logger +from .fallback import FallbackService +from .publisher import PublisherService + +logger = get_logger("services.cache_scraper") + +# Target Zotero versions to generate releases for +TARGET_ZOTERO_VERSIONS = ["7", "6"] + + +class CacheScraper: + """Scraper that uses release cache to generate addon information.""" + + def __init__( + self, + config: ScraperConfig, + release_cache: ReleaseCache, + ): + """Initialize cache-based scraper. + + Args: + config: Scraper configuration. + release_cache: Release cache instance. + """ + self.config = config + self.cache = release_cache + self.github = GitHubClient(config.github) + self.fallback = FallbackService() + self.publisher = PublisherService(self.github, config.github) + + def scrape_all(self) -> list[dict[str, Any]]: + """Generate addon information from cache. + + Returns: + List of addon info dictionaries. + """ + repos = self._load_repos_from_input() + logger.info(f"Generating addon info for {len(repos)} repositories") + + addon_infos: list[AddonInfo] = [] + + with ThreadPoolExecutor(max_workers=self.config.max_workers) as executor: + futures = { + executor.submit(self._process_repo, repo): repo + for repo in repos + } + + for future in as_completed(futures): + repo = futures[future] + try: + if result := future.result(): + addon_infos.append(result) + except Exception as e: + logger.error(f"Failed to process {repo}: {e}") + + # Convert to dict + result = [info.to_dict() for info in addon_infos] + + # Apply fallback from previous versions + for url in self.config.previous_info_urls: + result = self.fallback.apply_fallback(result, url) + + # Sort by stars (descending) + result.sort(key=lambda x: x.get("stars") or 0, reverse=True) + + # Save output + self._save_output(result) + + return result + + def _load_repos_from_input(self) -> list[str]: + """Load repository list from input directory.""" + repos = [] + + if not self.config.input_dir.exists(): + logger.error(f"Input directory not found: {self.config.input_dir}") + return repos + + for config_file in self.config.input_dir.glob("*.json"): + try: + with open(config_file, "r", encoding="utf-8") as f: + data = json.load(f) + if repo := data.get("repo"): + repos.append(repo) + except Exception as e: + logger.error(f"Failed to load {config_file}: {e}") + + return repos + + def _process_repo(self, repo: str) -> Optional[AddonInfo]: + """Process a single repository using cache. + + Args: + repo: Repository in "owner/name" format. + + Returns: + AddonInfo or None if no valid releases. + """ + parts = repo.split("/") + if len(parts) != 2: + logger.warning(f"Invalid repo format: {repo}") + return None + + owner, name = parts + repo_cache = self.cache.get_repo_cache(repo) + + # Find best release for each Zotero version + releases: list[AddonRelease] = [] + for zotero_version in TARGET_ZOTERO_VERSIONS: + cached_release = repo_cache.get_best_release_for_zotero(zotero_version) + if cached_release: + release = self._cached_to_addon_release(cached_release, zotero_version) + if release: + releases.append(release) + + if not releases: + logger.debug(f"{repo}: No compatible releases found") + return None + + # Create AddonInfo + addon_info = AddonInfo( + repo=repo, + releases=releases, + name=name, + ) + + # Fetch author info + self._fetch_author_info(addon_info, owner) + + # Fetch repository info + self._fetch_repo_info(addon_info, owner, name) + + # Use addon info from first release + first_release = releases[0] + if first_release.name: + addon_info.name = first_release.name + if first_release.description and not addon_info.description: + addon_info.description = first_release.description + + return addon_info + + def _cached_to_addon_release( + self, cached: CachedRelease, zotero_version: str + ) -> Optional[AddonRelease]: + """Convert CachedRelease to AddonRelease. + + Args: + cached: Cached release data. + zotero_version: Target Zotero version. + + Returns: + AddonRelease or None. + """ + if not cached.parse_success: + return None + + xpi_download_url = XpiDownloadUrls( + github=cached.xpi_download_url, + ghProxy=XPIProxy.ghproxy_url(cached.xpi_download_url), + kgithub=XPIProxy.kkgithub_url(cached.xpi_download_url), + ) + + return AddonRelease( + targetZoteroVersion=zotero_version, + tagName=cached.tag, + xpiDownloadUrl=xpi_download_url, + releaseDate=cached.published_at, + id=cached.addon_id, + xpiVersion=cached.addon_version, + name=cached.addon_name, + description=cached.addon_description, + minZoteroVersion=cached.min_zotero_version, + maxZoteroVersion=cached.max_zotero_version, + ) + + def _fetch_author_info(self, addon_info: AddonInfo, owner: str) -> None: + """Fetch author info from GitHub.""" + if user_info := self.github.get_user(owner): + addon_info.author = Author( + name=user_info.name, + url=user_info.html_url, + avatar=user_info.avatar_url, + ) + + def _fetch_repo_info( + self, addon_info: AddonInfo, owner: str, name: str + ) -> None: + """Fetch repository info from GitHub.""" + if repo_info := self.github.get_repo(owner, name): + if not addon_info.description and repo_info.description: + addon_info.description = repo_info.description + addon_info.stars = repo_info.stargazers_count + + def _save_output(self, data: list[dict[str, Any]]) -> None: + """Save output to file.""" + self.config.output_file.parent.mkdir(parents=True, exist_ok=True) + + with open(self.config.output_file, "w", encoding="utf-8") as f: + json.dump(data, f, ensure_ascii=False) + + logger.info(f"Saved {len(data)} addons to {self.config.output_file}")