Working Version 1.0.2: Integrated automated PDF downloader, safe filename sanitization, query-specific subdirectories, and local PDF viewer toggle

Code File/app.py

@@ -75,11 +75,15 @@ def start_scrape():
     if publishers:
         filters['publishers'] = publishers
 
-    # ── Generate output filename ──────────────────────────────────────────────
+    # ── Generate output filename and directory ─────────────────────────────────
     clean_query = re.sub(r'[^a-zA-Z0-9]', '_', query)[:20]
     timestamp   = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
+    folder_name = f"scrape_{clean_query}_{timestamp}"
+    run_dir     = os.path.join(BASE_DIR, "Data Folder", folder_name)
+    os.makedirs(run_dir, exist_ok=True)
+
     filename    = f"scrape_{clean_query}_{timestamp}.csv"
-    OUTPUT_CSV  = os.path.join(BASE_DIR, "Data Folder", filename)
+    OUTPUT_CSV  = os.path.join(run_dir, filename)
 
     def run_scraper():
         try:
@@ -128,13 +132,32 @@ def download():
         return send_file(OUTPUT_CSV, as_attachment=True)
     return "File not found", 404
 
+@app.route('/pdf/<folder>/<filename>')
+def serve_pdf(folder, filename):
+    pdf_path = os.path.join(BASE_DIR, "Data Folder", folder, filename)
+    if os.path.exists(pdf_path):
+        return send_file(pdf_path, mimetype='application/pdf')
+    return "PDF file not found", 404
+
 @app.route('/data')
 def get_data():
     if os.path.exists(OUTPUT_CSV):
         rows = []
+        run_dir = os.path.dirname(OUTPUT_CSV)
+        folder_name = os.path.basename(run_dir)
         with open(OUTPUT_CSV, 'r', encoding='utf-8') as f:
             reader = csv.DictReader(f)
             for row in reader:
+                # Check if PDF exists locally
+                title = row.get("Title", "")
+                cleaned = re.sub(r'[^a-zA-Z0-9\s_]', ' ', title)
+                cleaned = re.sub(r'\s+', ' ', cleaned).strip()
+                cleaned = cleaned.replace(' ', '_')
+                pdf_filename = cleaned[:100].strip('_') + ".pdf"
+                
+                pdf_exists = os.path.exists(os.path.join(run_dir, pdf_filename))
+                row["pdf_exists"] = pdf_exists
+                row["pdf_url"] = f"/pdf/{folder_name}/{pdf_filename}" if pdf_exists else ""
                 rows.append(row)
         return jsonify(rows)
     return jsonify([])

Code File/scholar_scraper.py

@@ -1,5 +1,6 @@
 from scholarly import scholarly, ProxyGenerator
 from scholarly._navigator import Navigator
+from bs4 import BeautifulSoup
 import time
 import csv
 import stem.control
@@ -280,6 +281,79 @@ def passes_filters(row_data, filters):
 
     return True, None
 
+
+# ── PDF Downloader Helpers ─────────────────────────────────────────────────────
+def sanitize_filename(title):
+    """
+    Cleans the title so that it only contains alphanumeric characters and underscores,
+    preventing any file system exceptions (dot, hyphens, slashes, or other special signs).
+    """
+    cleaned = re.sub(r'[^a-zA-Z0-9\s_]', ' ', title)
+    cleaned = re.sub(r'\s+', ' ', cleaned).strip()
+    cleaned = cleaned.replace(' ', '_')
+    # Limit length to 100 characters to prevent path limit errors on Windows
+    cleaned = cleaned[:100].strip('_')
+    return cleaned + ".pdf"
+
+
+def download_file(url, dest_path, headers=None, verify=True, timeout=20):
+    """Downloads a file and checks if the output is a valid PDF."""
+    try:
+        resp = requests.get(url, headers=headers, timeout=timeout, verify=verify)
+        if resp.status_code == 200 and resp.content.startswith(b'%PDF'):
+            with open(dest_path, "wb") as f:
+                f.write(resp.content)
+            return True
+    except Exception:
+        pass
+    return False
+
+
+def get_sidesgame_pdf_url(doi):
+    """Queries the sidesgame Sci-Hub mirror to get the direct PDF link for a DOI."""
+    if not doi or doi == "N/A":
+        return None
+    url = f"https://sci-hub.sidesgame.com/{doi}"
+    headers = {
+        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+        'Accept': 'text/html,application/xhtml+xml,application/xml',
+        'Accept-Language': 'en-US,en;q=0.5'
+    }
+    try:
+        resp = requests.get(url, headers=headers, timeout=15)
+        if resp.status_code == 200:
+            soup = BeautifulSoup(resp.text, 'html.parser')
+            # 1. Search for embed
+            embed = soup.find('embed', type='application/pdf')
+            if embed and embed.get('src'):
+                pdf_url = embed['src']
+                if pdf_url.startswith('//'):
+                    pdf_url = "https:" + pdf_url
+                return pdf_url
+            
+            # 2. Search for iframe
+            iframe = soup.find('iframe', id='pdf')
+            if iframe and iframe.get('src'):
+                pdf_url = iframe['src']
+                if pdf_url.startswith('//'):
+                    pdf_url = "https:" + pdf_url
+                return pdf_url
+            
+            # 3. Search for button locations
+            for btn in soup.find_all('button', onclick=re.compile(r"location\.href\s*=\s*['\"]([^'\"]+)['\"]")):
+                onclick = btn['onclick']
+                onclick = onclick.replace('\\/', '/').replace('\\', '')
+                match = re.search(r"location\.href\s*=\s*['\"]([^'\"]+)['\"]", onclick)
+                if match:
+                    pdf_url = match.group(1)
+                    if pdf_url.startswith('//'):
+                        pdf_url = "https:" + pdf_url
+                    return pdf_url
+    except Exception:
+        pass
+    return None
+
+
 # ── Main scraper ───────────────────────────────────────────────────────────────
 def search_scholar(query, output_file=None, target_count=20,
                    year_low=None, year_high=None,
@@ -455,10 +529,42 @@ def search_scholar(query, output_file=None, target_count=20,
                     if full_abstract and len(full_abstract) > 20:
                         row["Abstract/Snippet"] = full_abstract
                 
+                # ── PDF Downloader Pipeline ──────────────────────────────────────────
+                output_dir = os.path.dirname(output_file)
+                pdf_filename = sanitize_filename(title)
+                pdf_filepath = os.path.join(output_dir, pdf_filename)
+                
+                downloaded = False
+                dl_headers = {
+                    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
+                }
+                
+                # Step 1: Direct Google Scholar PDF link
+                if pdf_url and pdf_url != 'N/A' and pdf_url.startswith('http'):
+                    my_print(f"    -> [PDF] Downloading via direct Scholar link...")
+                    if download_file(pdf_url, pdf_filepath, headers=dl_headers):
+                        downloaded = True
+                        my_print(f"    ✅ PDF downloaded successfully: {pdf_filename}")
+                
+                # Step 2: Sci-Hub Fallback using DOI
+                if not downloaded and row.get('DOI') and row.get('DOI') != 'N/A':
+                    my_print(f"    -> [PDF] Attempting Sci-Hub mirror download...")
+                    sh_pdf_url = get_sidesgame_pdf_url(row['DOI'])
+                    if sh_pdf_url:
+                        sh_headers = dl_headers.copy()
+                        sh_headers['Referer'] = 'https://sci-hub.sidesgame.com/'
+                        if download_file(sh_pdf_url, pdf_filepath, headers=sh_headers):
+                            downloaded = True
+                            my_print(f"    ✅ PDF downloaded successfully via Sci-Hub: {pdf_filename}")
+                
+                if not downloaded:
+                    my_print("    ❌ No free PDF downloaded for this paper.")
+
                 qualified += 1
                 my_print(f"[{qualified}/{target_count}] Title: {title}")
                 my_print(f"    Year: {year} | DOI: {crmeta['DOI']} | Quartile: {sjr_q} | Publisher: {crmeta['Publisher']}")
 
+
                 writer.writerow(row)
                 file.flush()
                 time.sleep(2)

Code File/templates/data_view.html

@@ -45,6 +45,61 @@
     }
     #tooltip strong { color: #000000; font-weight: bold; font-size: 0.8rem; }
     #tooltip.visible { opacity: 1; }
+
+    /* Smart Action Badges */
+    .action-links {
+      display: flex;
+      gap: 6px;
+      align-items: center;
+    }
+    .action-badge {
+      display: inline-block;
+      font-family: 'Space Mono', monospace;
+      font-size: 0.65rem;
+      padding: 3px 6px;
+      border-radius: 4px;
+      text-decoration: none;
+      font-weight: bold;
+      transition: all 0.15s;
+    }
+    /* Publisher Link */
+    .badge-pub {
+      color: var(--accent);
+      border: 1px solid var(--accent);
+      background: transparent;
+    }
+    .badge-pub:hover {
+      background: var(--accent);
+      color: #ffffff;
+    }
+    /* Direct PDF Link */
+    .badge-pdf {
+      color: #ffffff;
+      background: var(--green);
+      border: 1px solid var(--green);
+    }
+    .badge-pdf:hover {
+      background: var(--green-hov);
+      border-color: var(--green-hov);
+    }
+    /* Sci-Hub Link */
+    .badge-scihub {
+      color: #ffffff;
+      background: var(--accent2);
+      border: 1px solid var(--accent2);
+    }
+    .badge-scihub:hover {
+      opacity: 0.9;
+    }
+    /* ResearchGate Link */
+    .badge-rg {
+      color: var(--text-muted);
+      border: 1px solid var(--input-bdr);
+      background: transparent;
+    }
+    .badge-rg:hover {
+      background: rgba(42,45,74,0.06);
+    }
   </style>
 </head>
 <body>
@@ -82,7 +137,7 @@
             <th onclick="sortTable(8, 'str')">PUBLISHER</th>
             <th onclick="sortTable(9, 'str')">OPEN ACCESS</th>
             <th onclick="sortTable(10, 'str')">COUNTRY</th>
-            <th>LINK</th>
+            <th>LINKS &amp; DOWNLOADS</th>
           </tr>
         </thead>
         <tbody id="tbody"></tbody>
@@ -119,8 +174,23 @@ function renderTable() {
     const titleEsc = (row.Title||'N/A').replace(/"/g, '&quot;');
     const abstractEsc = (row['Abstract/Snippet']||'No abstract').replace(/"/g, '&quot;');
     
-    const link = row['Paper Link'] && row['Paper Link'] !== 'N/A'
+    let actionsHtml = '<div class="action-links">';
-      ? '<a href="' + row['Paper Link'] + '" target="_blank" style="color:#3d5a99;">&#8599;</a>' : '—';
+    
+    // 1. Publisher Link
+    if (row['Paper Link'] && row['Paper Link'] !== 'N/A') {
+      actionsHtml += '<a href="' + row['Paper Link'] + '" target="_blank" class="action-badge badge-pub" title="Publisher Website">PAGE</a>';
+    }
+    
+    // 2. Either Local PDF or ResearchGate Search
+    if (row['pdf_exists']) {
+      actionsHtml += '<a href="' + row['pdf_url'] + '" target="_blank" class="action-badge badge-pdf" title="Open Downloaded PDF in Browser">PDF</a>';
+    } else {
+      let searchVal = row['DOI'] && row['DOI'] !== 'N/A' ? row['DOI'] : row.Title;
+      let searchUrl = 'https://www.google.com/search?q=' + encodeURIComponent(searchVal + ' site:researchgate.net');
+      actionsHtml += '<a href="' + searchUrl + '" target="_blank" class="action-badge badge-rg" title="Search ResearchGate on Google">RG SEARCH</a>';
+    }
+    
+    actionsHtml += '</div>';
       
     tbody.innerHTML += '<tr>'
       + '<td>' + (row.originalIndex) + '</td>'
@@ -135,7 +205,7 @@ function renderTable() {
       + '<td title="' + (row.Publisher||'').replace(/"/g, '&quot;') + '">' + (row.Publisher||'N/A').substring(0,22) + '</td>'
       + '<td>' + (row['Open Access']||'N/A') + '</td>'
       + '<td>' + (row['Country (SCImago)']||'N/A') + '</td>'
-      + '<td>' + link + '</td>'
+      + '<td>' + actionsHtml + '</td>'
       + '</tr>';
   });
   

Data Folder/.gitkeep

@@ -1 +0,0 @@
-# Keep this folder in the repository structure. Scraped CSV files are excluded via .gitignore.

README.md

@@ -1,20 +1,16 @@
-# Google Scholar Scraper & Quality Filtering Dashboard (v1.0.1)
+# Google Scholar Scraper & Quality Filtering Dashboard (v1.0.2)
 
 An enterprise-grade, web-based dashboard and automation engine to search, scrape, and filter Google Scholar articles. It features robust integration with the **Tor Network** for self-healing SOCKS5 proxy rotation, automatically bypassing Google's anti-bot blocks and pagination limits.
 
 ## 🚀 Key Features
 
+*   **Automated PDF Downloader Pipeline**: Downloads PDFs in 2 stages:
+    *   **Direct Google Scholar Links**: Sourced directly from `eprint_url` if indexed.
+    *   **Sci-Hub Fallback**: Automatically queries the `sci-hub.sidesgame.com` mirror using DOIs to scrape and download the PDF.
+*   **Query-Specific Run Directories**: Automatically structures output runs into `Data Folder/scrape_{query}_{timestamp}/` containing the CSV file and all downloaded PDF files.
+*   **Safe Filename Sanitization**: Automatically renames downloaded PDFs using cleaned paper titles. All special symbols (`.`, `-`, `:`, `/`, etc.) are stripped and replaced with underscores (`_`) to ensure perfect OS compatibility.
+*   **Hybrid Links Layout**: The preview page links column renders only the `PAGE` button and a toggle: either the green `PDF` button (if downloaded successfully) or a grey `RG SEARCH` button (pointing to a Google search on ResearchGate for manual human verification and 1-click download).
 *   **Advanced Query Builder**: Build complex boolean queries (supporting `MUST`, `AND`, `OR` operators) directly from the visual interface.
-*   **Tor Circuit Rotation**: Automatically rotates Tor exit nodes using the Tor control port (`9151`) on detecting soft-blocks or captcha pages.
-*   **Resilient Pagination**: Syncs search offset states (`start_index`) to resume scraping seamlessly exactly where it was interrupted after an IP rotation.
-*   **Duplicate Elimination**: Automatically filters overlapping search results in real-time, ensuring unique outputs in final CSV records.
-*   **Academic Quality Filters**:
-    *   Filter by **SCImago Journal Rank (SJR)**, Quartile rankings (`Q1` to `Q4`), and H-index via local database matching.
-    *   Filter by minimum citation counts.
-    *   Filter by Open Access status.
-    *   Whitelist specific journals or publishers.
-*   **Live Output Terminal**: Simulates a live CLI output stream directly in the web browser using Server-Sent Events (SSE).
-*   **Live Data Table**: Sortable tabular view of scraped papers with interactive abstract hover tooltips.
 
 ---