From 5cb3cd97947ddd3c63d9da8d8bcdaa69beb4f22f Mon Sep 17 00:00:00 2001 From: ytshen Date: Sun, 12 Apr 2026 11:27:39 +0800 Subject: [PATCH] feat: validate addon tags in ci --- .github/workflows/addon-tag-review.yml | 66 ++++++++ .github/workflows/main.yml | 38 ++++- README.md | 7 +- pyproject.toml | 1 + src/zotero_scraper/config/tags.py | 33 ++++ src/zotero_scraper/tag_review.py | 219 +++++++++++++++++++++++++ tests/unit/test_tag_review.py | 71 ++++++++ 7 files changed, 433 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/addon-tag-review.yml create mode 100644 src/zotero_scraper/config/tags.py create mode 100644 src/zotero_scraper/tag_review.py create mode 100644 tests/unit/test_tag_review.py diff --git a/.github/workflows/addon-tag-review.yml b/.github/workflows/addon-tag-review.yml new file mode 100644 index 0000000..714732d --- /dev/null +++ b/.github/workflows/addon-tag-review.yml @@ -0,0 +1,66 @@ +name: Addon Tag Review + +on: + pull_request: + paths: + - 'addons/**' + workflow_dispatch: + +permissions: + contents: read + +jobs: + review: + runs-on: ubuntu-latest + + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.x + + - name: Install Dependencies + run: | + pip install -r requirements.txt + + - name: Determine addon files to review + id: changed_addons + shell: bash + run: | + if [ "${{ github.event_name }}" = "pull_request" ]; then + git fetch origin "${{ github.base_ref }}" --depth=1 + files=$(git diff --name-only --diff-filter=AM "origin/${{ github.base_ref }}" HEAD -- addons || true) + else + files=$(git ls-files addons) + fi + + { + echo "changed_addons<> "$GITHUB_OUTPUT" + + - name: Review addon tags + if: steps.changed_addons.outputs.changed_addons != '' + shell: bash + env: + PYTHONPATH: src + run: | + mapfile -t files < <(printf '%s\n' "${{ steps.changed_addons.outputs.changed_addons }}" | sed '/^$/d') + if [ "${#files[@]}" -eq 0 ]; then + echo "No addon files to review" + exit 0 + fi + + python3 -m zotero_scraper.tag_review \ + --files "${files[@]}" \ + --summary-file "$GITHUB_STEP_SUMMARY" + + - name: No addon changes + if: steps.changed_addons.outputs.changed_addons == '' + run: echo "No addon files require tag review" diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 05377dc..84c1681 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -23,6 +23,8 @@ jobs: steps: - name: Checkout Repository uses: actions/checkout@v4 + with: + fetch-depth: 0 - name: Checkout publish branch (for release_cache) uses: actions/checkout@v4 @@ -50,6 +52,40 @@ jobs: run: | pip install -r requirements.txt + - name: Determine changed addon files + id: changed_addons + if: github.event_name == 'push' + shell: bash + run: | + before_sha="${{ github.event.before }}" + if [ -z "$before_sha" ] || [ "$before_sha" = "0000000000000000000000000000000000000000" ]; then + files=$(git diff-tree --no-commit-id --name-only -r "${{ github.sha }}" -- addons || true) + else + files=$(git diff --name-only --diff-filter=AM "$before_sha" "${{ github.sha }}" -- addons || true) + fi + + { + echo "changed_addons<> "$GITHUB_OUTPUT" + + - name: Validate changed addon tags + if: github.event_name == 'push' && steps.changed_addons.outputs.changed_addons != '' + shell: bash + env: + PYTHONPATH: src + run: | + mapfile -t files < <(printf '%s\n' "${{ steps.changed_addons.outputs.changed_addons }}" | sed '/^$/d') + if [ "${#files[@]}" -eq 0 ]; then + echo "No addon files changed" + exit 0 + fi + + python3 -m zotero_scraper.tag_review \ + --files "${files[@]}" \ + --summary-file "$GITHUB_STEP_SUMMARY" + - name: Run Scraper run: | python main.py \ @@ -61,6 +97,7 @@ jobs: --create_release True env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PYTHONPATH: src - name: Copy release_cache to published folder run: | @@ -77,4 +114,3 @@ jobs: user_email: 'github-actions[bot]@users.noreply.github.com' publish_dir: ./published force_orphan: true - diff --git a/README.md b/README.md index 0bbbcef..78d2ae0 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ including the following steps: - Retrieve [Zotero](https://www.zotero.org) add-on information specified in [addons](addons) folder - Parse add-on information into JSON data - Publish the JSON data to the [`publish`](https://github.com/syt2/zotero-addons-scraper/blob/publish/addon_infos.json) branch and the [latest release](https://github.com/syt2/zotero-addons-scraper/releases/latest) +- Validate changed addon tag metadata in CI ## Contributing Adding a new add-on entry `{owner}@{repo}` in the [addons](addons) folder. @@ -35,7 +36,11 @@ Available tags: | `integration` | Integration with external services (Notion, Obsidian, Word, etc.) | | `utility` | Zotero system tools (automation, deduplication, plugin management) | +### Tag validation in CI +- `pull_request` on `addons/**`: validates changed addon files against the allowed tag taxonomy +- `push` to `master`: validates changed addon files before the scraper publishes data +- Each addon config must declare at least one tag, use only allowed tag values, avoid duplicates, and keep tags in canonical order + ## Usage 1. `fork` this repository 2. Enable actions and scheduled workflow in `Actions` page of forked repository - diff --git a/pyproject.toml b/pyproject.toml index 21c905a..6304088 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,7 @@ dev = [ [project.scripts] zotero-scraper = "zotero_scraper.__main__:main" +zotero-tag-review = "zotero_scraper.tag_review:main" [tool.setuptools.packages.find] where = ["src"] diff --git a/src/zotero_scraper/config/tags.py b/src/zotero_scraper/config/tags.py new file mode 100644 index 0000000..aea6ed0 --- /dev/null +++ b/src/zotero_scraper/config/tags.py @@ -0,0 +1,33 @@ +"""Addon tag taxonomy shared by docs, validation, and AI review.""" + +from typing import Final + +ADDON_TAGS: Final[dict[str, str]] = { + "ai": "AI-powered features such as summarization, chat, or translation via LLMs.", + "metadata": ( + "Metadata retrieval, citation counts, impact factor, formatting, " + "or library enrichment." + ), + "reader": "PDF reading experience, annotation, and highlighting workflows.", + "notes": "Note-taking, markdown export, and knowledge management workflows.", + "attachment": "File management, attachment organization, OCR, and exports.", + "interface": "UI enhancements, themes, columns, and layout customizations.", + "integration": "Integration with external services such as Notion or Obsidian.", + "utility": "General Zotero tools such as automation and plugin management.", +} + +VALID_ADDON_TAGS: Final[tuple[str, ...]] = tuple(ADDON_TAGS.keys()) +MAX_ADDON_TAGS: Final[int] = 3 + + +def normalize_tags(tags: list[str]) -> list[str]: + """Return tags in canonical taxonomy order without duplicates.""" + tag_set = set(tags) + return [tag for tag in VALID_ADDON_TAGS if tag in tag_set] + + +def format_tag_catalog() -> str: + """Render the tag taxonomy as prompt-friendly text.""" + return "\n".join( + f"- {tag}: {description}" for tag, description in ADDON_TAGS.items() + ) diff --git a/src/zotero_scraper/tag_review.py b/src/zotero_scraper/tag_review.py new file mode 100644 index 0000000..58ec4f1 --- /dev/null +++ b/src/zotero_scraper/tag_review.py @@ -0,0 +1,219 @@ +"""CLI helpers for validating addon tag metadata.""" + +from __future__ import annotations + +import argparse +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from .config.tags import ADDON_TAGS, MAX_ADDON_TAGS, VALID_ADDON_TAGS, normalize_tags +from .utils.logging import get_logger, setup_logging + +logger = get_logger("tag_review") + + +@dataclass +class AddonConfigEntry: + """Addon config file loaded from the addons directory.""" + + path: Path + repo: str + data: dict[str, Any] + tags: list[str] + + @classmethod + def load(cls, path: Path) -> AddonConfigEntry: + """Load an addon config file.""" + text = path.read_text(encoding="utf-8").strip() + data: dict[str, Any] + if not text: + data = {} + else: + parsed = json.loads(text) + if not isinstance(parsed, dict): + raise ValueError("Addon config must be a JSON object") + data = parsed + + repo = path.name.replace("@", "/", 1) + tags = data.get("tags", []) + if tags is None: + tags = [] + if not isinstance(tags, list): + raise ValueError("'tags' must be a JSON array when present") + + return cls(path=path, repo=repo, data=data, tags=list(tags)) + + +@dataclass +class FileReviewResult: + """Validation outcome for one addon config file.""" + + path: Path + repo: str + status: str + current_tags: list[str] + messages: list[str] + + @property + def ok(self) -> bool: + return self.status == "ok" + + +class AddonTagReviewer: + """Validate addon tags locally.""" + + def review_file(self, path: Path) -> FileReviewResult: + """Review a single addon config file.""" + try: + entry = AddonConfigEntry.load(path) + messages = self._validate(entry) + except Exception as exc: + return FileReviewResult( + path=path, + repo=path.name.replace("@", "/", 1), + status="failed", + current_tags=[], + messages=[str(exc)], + ) + + if messages: + return FileReviewResult( + path=path, + repo=entry.repo, + status="failed", + current_tags=entry.tags, + messages=messages, + ) + + return FileReviewResult( + path=path, + repo=entry.repo, + status="ok", + current_tags=normalize_tags(entry.tags), + messages=[], + ) + + def _validate(self, entry: AddonConfigEntry) -> list[str]: + """Validate file structure and tag taxonomy.""" + messages: list[str] = [] + + unknown_keys = sorted(set(entry.data) - {"tags", "recommended"}) + if unknown_keys: + messages.append( + f"Unsupported addon config keys: {', '.join(unknown_keys)}" + ) + + if "recommended" in entry.data and not isinstance( + entry.data["recommended"], bool + ): + messages.append("'recommended' must be a boolean when present") + + if not entry.tags: + return messages + + non_string_tags = [tag for tag in entry.tags if not isinstance(tag, str)] + if non_string_tags: + messages.append("All tags must be strings") + return messages + + if len(entry.tags) != len(set(entry.tags)): + messages.append("Duplicate tags are not allowed") + + invalid_tags = [tag for tag in entry.tags if tag not in VALID_ADDON_TAGS] + if invalid_tags: + messages.append( + "Unknown tags: " + f"{', '.join(invalid_tags)}. Allowed tags: " + f"{', '.join(VALID_ADDON_TAGS)}" + ) + + if len(entry.tags) > MAX_ADDON_TAGS: + messages.append(f"At most {MAX_ADDON_TAGS} tags are allowed per addon") + + normalized = normalize_tags(entry.tags) + if entry.tags != normalized: + messages.append(f"Tags should use canonical order: {normalized}") + + return messages + + +def parse_args() -> argparse.Namespace: + """Parse command-line arguments for tag review.""" + parser = argparse.ArgumentParser( + description="Validate addon tags for addon config files." + ) + parser.add_argument( + "--files", + nargs="+", + help="Addon config files to review", + ) + parser.add_argument( + "--summary-file", + default="", + help="Optional markdown file to append the review summary to", + ) + parser.add_argument( + "--log-level", + choices=["DEBUG", "INFO", "WARNING", "ERROR"], + default="INFO", + help="Logging level", + ) + return parser.parse_args() + + +def render_summary(results: list[FileReviewResult]) -> str: + """Render a GitHub-friendly markdown summary.""" + if not results: + return "## Addon tag review\n\nNo addon files were provided for review.\n" + + lines = ["## Addon tag review", ""] + lines.append("| File | Status | Tags |") + lines.append("| --- | --- | --- |") + for result in results: + tags = ", ".join(result.current_tags) or "-" + lines.append(f"| `{result.path}` | {result.status} | {tags} |") + for message in result.messages: + lines.append(f"- `{result.path}`: {message}") + lines.append("") + lines.append("Allowed tags: " + ", ".join(f"`{tag}`" for tag in ADDON_TAGS)) + lines.append("") + return "\n".join(lines) + + +def main() -> int: + """CLI entry point used by GitHub Actions workflows.""" + args = parse_args() + + import logging + + setup_logging(level=getattr(logging, args.log_level)) + + file_paths = [Path(path) for path in args.files or [] if path] + if not file_paths: + logger.info("No addon files to review") + if args.summary_file: + Path(args.summary_file).write_text(render_summary([]), encoding="utf-8") + return 0 + + reviewer = AddonTagReviewer() + results: list[FileReviewResult] = [] + for path in file_paths: + if not path.exists(): + logger.info(f"Skipping deleted or missing addon file: {path}") + continue + logger.info(f"Reviewing addon tags: {path}") + results.append(reviewer.review_file(path)) + + summary = render_summary(results) + print(summary) + if args.summary_file: + Path(args.summary_file).write_text(summary, encoding="utf-8") + + failures = [result for result in results if not result.ok] + return 1 if failures else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/unit/test_tag_review.py b/tests/unit/test_tag_review.py new file mode 100644 index 0000000..5b76cc9 --- /dev/null +++ b/tests/unit/test_tag_review.py @@ -0,0 +1,71 @@ +"""Unit tests for addon tag review.""" + +from pathlib import Path + +from zotero_scraper.tag_review import AddonConfigEntry, AddonTagReviewer + + +def write_addon_file(path: Path, content: str) -> Path: + """Write addon config content to a test file.""" + path.write_text(content, encoding="utf-8") + return path + + +def test_addon_config_entry_loads_empty_file(temp_dir): + """Empty addon entries should load as an empty config object.""" + path = write_addon_file(temp_dir / "owner@repo", "") + + entry = AddonConfigEntry.load(path) + + assert entry.repo == "owner/repo" + assert entry.tags == [] + + +def test_review_allows_missing_tags(temp_dir): + """Empty addon files should pass when tags are omitted.""" + path = write_addon_file(temp_dir / "owner@repo", "") + reviewer = AddonTagReviewer() + + result = reviewer.review_file(path) + + assert result.status == "ok" + assert result.current_tags == [] + + +def test_review_rejects_unknown_tags(temp_dir): + """Unknown taxonomy values should fail validation.""" + path = write_addon_file(temp_dir / "owner@repo", '{"tags": ["unknown"]}\n') + reviewer = AddonTagReviewer() + + result = reviewer.review_file(path) + + assert result.status == "failed" + assert "Unknown tags" in result.messages[0] + + +def test_review_rejects_non_canonical_order(temp_dir): + """Tags should follow the configured canonical order.""" + path = write_addon_file( + temp_dir / "owner@repo", + '{"tags": ["reader", "ai"]}\n', + ) + reviewer = AddonTagReviewer() + + result = reviewer.review_file(path) + + assert result.status == "failed" + assert "canonical order" in result.messages[0] + + +def test_review_accepts_valid_tags(temp_dir): + """A valid addon config should pass validation.""" + path = write_addon_file( + temp_dir / "owner@repo", + '{"tags": ["ai", "notes"], "recommended": true}\n', + ) + reviewer = AddonTagReviewer() + + result = reviewer.review_file(path) + + assert result.status == "ok" + assert result.current_tags == ["ai", "notes"]