diff --git a/Code File/app.py b/Code File/app.py index d85a4f3..e9e0af9 100644 --- a/Code File/app.py +++ b/Code File/app.py @@ -75,11 +75,15 @@ def start_scrape(): if publishers: filters['publishers'] = publishers - # ── Generate output filename ────────────────────────────────────────────── + # ── Generate output filename and directory ───────────────────────────────── clean_query = re.sub(r'[^a-zA-Z0-9]', '_', query)[:20] timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + folder_name = f"scrape_{clean_query}_{timestamp}" + run_dir = os.path.join(BASE_DIR, "Data Folder", folder_name) + os.makedirs(run_dir, exist_ok=True) + filename = f"scrape_{clean_query}_{timestamp}.csv" - OUTPUT_CSV = os.path.join(BASE_DIR, "Data Folder", filename) + OUTPUT_CSV = os.path.join(run_dir, filename) def run_scraper(): try: @@ -128,13 +132,32 @@ def download(): return send_file(OUTPUT_CSV, as_attachment=True) return "File not found", 404 +@app.route('/pdf//') +def serve_pdf(folder, filename): + pdf_path = os.path.join(BASE_DIR, "Data Folder", folder, filename) + if os.path.exists(pdf_path): + return send_file(pdf_path, mimetype='application/pdf') + return "PDF file not found", 404 + @app.route('/data') def get_data(): if os.path.exists(OUTPUT_CSV): rows = [] + run_dir = os.path.dirname(OUTPUT_CSV) + folder_name = os.path.basename(run_dir) with open(OUTPUT_CSV, 'r', encoding='utf-8') as f: reader = csv.DictReader(f) for row in reader: + # Check if PDF exists locally + title = row.get("Title", "") + cleaned = re.sub(r'[^a-zA-Z0-9\s_]', ' ', title) + cleaned = re.sub(r'\s+', ' ', cleaned).strip() + cleaned = cleaned.replace(' ', '_') + pdf_filename = cleaned[:100].strip('_') + ".pdf" + + pdf_exists = os.path.exists(os.path.join(run_dir, pdf_filename)) + row["pdf_exists"] = pdf_exists + row["pdf_url"] = f"/pdf/{folder_name}/{pdf_filename}" if pdf_exists else "" rows.append(row) return jsonify(rows) return jsonify([]) diff --git a/Code File/scholar_scraper.py b/Code File/scholar_scraper.py index 40c0c85..17c7f79 100644 --- a/Code File/scholar_scraper.py +++ b/Code File/scholar_scraper.py @@ -1,5 +1,6 @@ from scholarly import scholarly, ProxyGenerator from scholarly._navigator import Navigator +from bs4 import BeautifulSoup import time import csv import stem.control @@ -280,6 +281,79 @@ def passes_filters(row_data, filters): return True, None + +# ── PDF Downloader Helpers ───────────────────────────────────────────────────── +def sanitize_filename(title): + """ + Cleans the title so that it only contains alphanumeric characters and underscores, + preventing any file system exceptions (dot, hyphens, slashes, or other special signs). + """ + cleaned = re.sub(r'[^a-zA-Z0-9\s_]', ' ', title) + cleaned = re.sub(r'\s+', ' ', cleaned).strip() + cleaned = cleaned.replace(' ', '_') + # Limit length to 100 characters to prevent path limit errors on Windows + cleaned = cleaned[:100].strip('_') + return cleaned + ".pdf" + + +def download_file(url, dest_path, headers=None, verify=True, timeout=20): + """Downloads a file and checks if the output is a valid PDF.""" + try: + resp = requests.get(url, headers=headers, timeout=timeout, verify=verify) + if resp.status_code == 200 and resp.content.startswith(b'%PDF'): + with open(dest_path, "wb") as f: + f.write(resp.content) + return True + except Exception: + pass + return False + + +def get_sidesgame_pdf_url(doi): + """Queries the sidesgame Sci-Hub mirror to get the direct PDF link for a DOI.""" + if not doi or doi == "N/A": + return None + url = f"https://sci-hub.sidesgame.com/{doi}" + headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + 'Accept': 'text/html,application/xhtml+xml,application/xml', + 'Accept-Language': 'en-US,en;q=0.5' + } + try: + resp = requests.get(url, headers=headers, timeout=15) + if resp.status_code == 200: + soup = BeautifulSoup(resp.text, 'html.parser') + # 1. Search for embed + embed = soup.find('embed', type='application/pdf') + if embed and embed.get('src'): + pdf_url = embed['src'] + if pdf_url.startswith('//'): + pdf_url = "https:" + pdf_url + return pdf_url + + # 2. Search for iframe + iframe = soup.find('iframe', id='pdf') + if iframe and iframe.get('src'): + pdf_url = iframe['src'] + if pdf_url.startswith('//'): + pdf_url = "https:" + pdf_url + return pdf_url + + # 3. Search for button locations + for btn in soup.find_all('button', onclick=re.compile(r"location\.href\s*=\s*['\"]([^'\"]+)['\"]")): + onclick = btn['onclick'] + onclick = onclick.replace('\\/', '/').replace('\\', '') + match = re.search(r"location\.href\s*=\s*['\"]([^'\"]+)['\"]", onclick) + if match: + pdf_url = match.group(1) + if pdf_url.startswith('//'): + pdf_url = "https:" + pdf_url + return pdf_url + except Exception: + pass + return None + + # ── Main scraper ─────────────────────────────────────────────────────────────── def search_scholar(query, output_file=None, target_count=20, year_low=None, year_high=None, @@ -455,10 +529,42 @@ def search_scholar(query, output_file=None, target_count=20, if full_abstract and len(full_abstract) > 20: row["Abstract/Snippet"] = full_abstract + # ── PDF Downloader Pipeline ────────────────────────────────────────── + output_dir = os.path.dirname(output_file) + pdf_filename = sanitize_filename(title) + pdf_filepath = os.path.join(output_dir, pdf_filename) + + downloaded = False + dl_headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36' + } + + # Step 1: Direct Google Scholar PDF link + if pdf_url and pdf_url != 'N/A' and pdf_url.startswith('http'): + my_print(f" -> [PDF] Downloading via direct Scholar link...") + if download_file(pdf_url, pdf_filepath, headers=dl_headers): + downloaded = True + my_print(f" ✅ PDF downloaded successfully: {pdf_filename}") + + # Step 2: Sci-Hub Fallback using DOI + if not downloaded and row.get('DOI') and row.get('DOI') != 'N/A': + my_print(f" -> [PDF] Attempting Sci-Hub mirror download...") + sh_pdf_url = get_sidesgame_pdf_url(row['DOI']) + if sh_pdf_url: + sh_headers = dl_headers.copy() + sh_headers['Referer'] = 'https://sci-hub.sidesgame.com/' + if download_file(sh_pdf_url, pdf_filepath, headers=sh_headers): + downloaded = True + my_print(f" ✅ PDF downloaded successfully via Sci-Hub: {pdf_filename}") + + if not downloaded: + my_print(" ❌ No free PDF downloaded for this paper.") + qualified += 1 my_print(f"[{qualified}/{target_count}] Title: {title}") my_print(f" Year: {year} | DOI: {crmeta['DOI']} | Quartile: {sjr_q} | Publisher: {crmeta['Publisher']}") + writer.writerow(row) file.flush() time.sleep(2) diff --git a/Code File/templates/data_view.html b/Code File/templates/data_view.html index 5840fb2..46a1677 100644 --- a/Code File/templates/data_view.html +++ b/Code File/templates/data_view.html @@ -45,6 +45,61 @@ } #tooltip strong { color: #000000; font-weight: bold; font-size: 0.8rem; } #tooltip.visible { opacity: 1; } + + /* Smart Action Badges */ + .action-links { + display: flex; + gap: 6px; + align-items: center; + } + .action-badge { + display: inline-block; + font-family: 'Space Mono', monospace; + font-size: 0.65rem; + padding: 3px 6px; + border-radius: 4px; + text-decoration: none; + font-weight: bold; + transition: all 0.15s; + } + /* Publisher Link */ + .badge-pub { + color: var(--accent); + border: 1px solid var(--accent); + background: transparent; + } + .badge-pub:hover { + background: var(--accent); + color: #ffffff; + } + /* Direct PDF Link */ + .badge-pdf { + color: #ffffff; + background: var(--green); + border: 1px solid var(--green); + } + .badge-pdf:hover { + background: var(--green-hov); + border-color: var(--green-hov); + } + /* Sci-Hub Link */ + .badge-scihub { + color: #ffffff; + background: var(--accent2); + border: 1px solid var(--accent2); + } + .badge-scihub:hover { + opacity: 0.9; + } + /* ResearchGate Link */ + .badge-rg { + color: var(--text-muted); + border: 1px solid var(--input-bdr); + background: transparent; + } + .badge-rg:hover { + background: rgba(42,45,74,0.06); + } @@ -82,7 +137,7 @@ PUBLISHER OPEN ACCESS COUNTRY - LINK + LINKS & DOWNLOADS @@ -119,8 +174,23 @@ function renderTable() { const titleEsc = (row.Title||'N/A').replace(/"/g, '"'); const abstractEsc = (row['Abstract/Snippet']||'No abstract').replace(/"/g, '"'); - const link = row['Paper Link'] && row['Paper Link'] !== 'N/A' - ? '↗' : '—'; + let actionsHtml = '
'; + + // 1. Publisher Link + if (row['Paper Link'] && row['Paper Link'] !== 'N/A') { + actionsHtml += 'PAGE'; + } + + // 2. Either Local PDF or ResearchGate Search + if (row['pdf_exists']) { + actionsHtml += 'PDF'; + } else { + let searchVal = row['DOI'] && row['DOI'] !== 'N/A' ? row['DOI'] : row.Title; + let searchUrl = 'https://www.google.com/search?q=' + encodeURIComponent(searchVal + ' site:researchgate.net'); + actionsHtml += 'RG SEARCH'; + } + + actionsHtml += '
'; tbody.innerHTML += '' + '' + (row.originalIndex) + '' @@ -135,7 +205,7 @@ function renderTable() { + '' + (row.Publisher||'N/A').substring(0,22) + '' + '' + (row['Open Access']||'N/A') + '' + '' + (row['Country (SCImago)']||'N/A') + '' - + '' + link + '' + + '' + actionsHtml + '' + ''; }); diff --git a/Data Folder/.gitkeep b/Data Folder/.gitkeep deleted file mode 100644 index 301ab1d..0000000 --- a/Data Folder/.gitkeep +++ /dev/null @@ -1 +0,0 @@ -# Keep this folder in the repository structure. Scraped CSV files are excluded via .gitignore. diff --git a/README.md b/README.md index 0cede8c..8165a77 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,16 @@ -# Google Scholar Scraper & Quality Filtering Dashboard (v1.0.1) +# Google Scholar Scraper & Quality Filtering Dashboard (v1.0.2) An enterprise-grade, web-based dashboard and automation engine to search, scrape, and filter Google Scholar articles. It features robust integration with the **Tor Network** for self-healing SOCKS5 proxy rotation, automatically bypassing Google's anti-bot blocks and pagination limits. ## 🚀 Key Features +* **Automated PDF Downloader Pipeline**: Downloads PDFs in 2 stages: + * **Direct Google Scholar Links**: Sourced directly from `eprint_url` if indexed. + * **Sci-Hub Fallback**: Automatically queries the `sci-hub.sidesgame.com` mirror using DOIs to scrape and download the PDF. +* **Query-Specific Run Directories**: Automatically structures output runs into `Data Folder/scrape_{query}_{timestamp}/` containing the CSV file and all downloaded PDF files. +* **Safe Filename Sanitization**: Automatically renames downloaded PDFs using cleaned paper titles. All special symbols (`.`, `-`, `:`, `/`, etc.) are stripped and replaced with underscores (`_`) to ensure perfect OS compatibility. +* **Hybrid Links Layout**: The preview page links column renders only the `PAGE` button and a toggle: either the green `PDF` button (if downloaded successfully) or a grey `RG SEARCH` button (pointing to a Google search on ResearchGate for manual human verification and 1-click download). * **Advanced Query Builder**: Build complex boolean queries (supporting `MUST`, `AND`, `OR` operators) directly from the visual interface. -* **Tor Circuit Rotation**: Automatically rotates Tor exit nodes using the Tor control port (`9151`) on detecting soft-blocks or captcha pages. -* **Resilient Pagination**: Syncs search offset states (`start_index`) to resume scraping seamlessly exactly where it was interrupted after an IP rotation. -* **Duplicate Elimination**: Automatically filters overlapping search results in real-time, ensuring unique outputs in final CSV records. -* **Academic Quality Filters**: - * Filter by **SCImago Journal Rank (SJR)**, Quartile rankings (`Q1` to `Q4`), and H-index via local database matching. - * Filter by minimum citation counts. - * Filter by Open Access status. - * Whitelist specific journals or publishers. -* **Live Output Terminal**: Simulates a live CLI output stream directly in the web browser using Server-Sent Events (SSE). -* **Live Data Table**: Sortable tabular view of scraped papers with interactive abstract hover tooltips. ---