Initial commit: Set up Scholar Scraper dashboard with Tor integration and robust duplicate/resumption filters

.gitignore

@@ -0,0 +1,22 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.pyc
+*.pyo
+*.pyd
+
+# Virtual Environment
+.venv/
+venv/
+ENV/
+
+# Data Folder (Exclude all scraped results, but keep the directory template)
+Data Folder/*
+!Data Folder/.gitkeep
+river_erosion_papers.csv
+page2_response.html
+
+# IDE / System files
+.vscode/
+.idea/
+.DS_Store
+Thumbs.db

Code File/app.py

@@ -0,0 +1,145 @@
+import os
+import threading
+import queue
+import datetime
+import re
+from flask import Flask, render_template, request, Response, jsonify, send_file
+import csv
+import scholar_scraper
+
+app = Flask(__name__)
+log_queue = queue.Queue()
+
+BASE_DIR   = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+OUTPUT_CSV = os.path.join(BASE_DIR, "Data Folder", "scholar_results.csv")
+
+_stop_event    = threading.Event()
+_scrape_thread = None
+
+def enqueue_log(msg):
+    log_queue.put(msg)
+
+scholar_scraper.LOG_CALLBACK = enqueue_log
+
+@app.route('/')
+def index():
+    return render_template('index.html')
+
+@app.route('/preview')
+def preview():
+    return render_template('data_view.html')
+
+@app.route('/start-scrape', methods=['POST'])
+def start_scrape():
+    global OUTPUT_CSV, _stop_event, _scrape_thread
+    _stop_event.clear()
+
+    data         = request.json
+    query        = data.get('query')
+    raw_max      = data.get('max_results')
+    target_count = int(raw_max) if raw_max and str(raw_max).strip() else 1000000
+    year_low     = int(data['year_low'])  if data.get('year_low')  else None
+    year_high    = int(data['year_high']) if data.get('year_high') else None
+
+    # Drain any stale messages from a previous run
+    while not log_queue.empty():
+        try:
+            log_queue.get_nowait()
+        except Exception:
+            break
+
+    # ── Build filters dict ────────────────────────────────────────────────────
+    filters = {}
+
+    if data.get('open_access'):
+        filters['open_access'] = True
+
+    quartiles = [q for q in data.get('quartiles', []) if q]
+    if quartiles:
+        filters['quartiles'] = quartiles
+
+    if data.get('min_h_index') not in (None, '', 0, '0'):
+        filters['min_h_index'] = float(data['min_h_index'])
+
+    if data.get('min_sjr') not in (None, '', 0, '0'):
+        filters['min_sjr'] = float(data['min_sjr'])
+
+    if data.get('min_citations') not in (None, '', 0, '0'):
+        filters['min_citations'] = int(data['min_citations'])
+
+    journals = [j for j in data.get('journals', []) if j and j.strip()]
+    if journals:
+        filters['journals'] = journals
+
+    publishers = [p for p in data.get('publishers', []) if p and p.strip()]
+    if publishers:
+        filters['publishers'] = publishers
+
+    # ── Generate output filename ──────────────────────────────────────────────
+    clean_query = re.sub(r'[^a-zA-Z0-9]', '_', query)[:20]
+    timestamp   = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
+    filename    = f"scrape_{clean_query}_{timestamp}.csv"
+    OUTPUT_CSV  = os.path.join(BASE_DIR, "Data Folder", filename)
+
+    def run_scraper():
+        try:
+            scholar_scraper.setup_custom_tor()
+            scholar_scraper.search_scholar(
+                query,
+                output_file  = OUTPUT_CSV,
+                target_count = target_count,
+                year_low     = year_low,
+                year_high    = year_high,
+                stop_event   = _stop_event,
+                filters      = filters if filters else None
+            )
+        except Exception as e:
+            enqueue_log(f"ERROR: {str(e)}")
+        finally:
+            enqueue_log("STOPPED" if _stop_event.is_set() else "DONE")
+
+    _scrape_thread = threading.Thread(target=run_scraper, daemon=True)
+    _scrape_thread.start()
+    return jsonify({"status": "started", "file": filename})
+
+@app.route('/stop', methods=['POST'])
+def stop_scrape():
+    global _stop_event
+    _stop_event.set()
+    enqueue_log("[STOPPED] Stop signal sent. Finishing current paper...")
+    return jsonify({"status": "stopping"})
+
+@app.route('/stream')
+def stream():
+    def event_stream():
+        while True:
+            try:
+                msg = log_queue.get(timeout=30)
+                yield f"data: {msg}\n\n"
+            except Exception:
+                # Send a keepalive heartbeat comment so the browser
+                # doesn't time out the SSE connection during slow scrapes
+                yield ": heartbeat\n\n"
+    return Response(event_stream(), mimetype="text/event-stream")
+
+@app.route('/download')
+def download():
+    if os.path.exists(OUTPUT_CSV):
+        return send_file(OUTPUT_CSV, as_attachment=True)
+    return "File not found", 404
+
+@app.route('/data')
+def get_data():
+    if os.path.exists(OUTPUT_CSV):
+        rows = []
+        with open(OUTPUT_CSV, 'r', encoding='utf-8') as f:
+            reader = csv.DictReader(f)
+            for row in reader:
+                rows.append(row)
+        return jsonify(rows)
+    return jsonify([])
+
+if __name__ == '__main__':
+    while not log_queue.empty():
+        log_queue.get()
+    app.run(port=5000, debug=False)

Code File/check_content.py

@@ -0,0 +1,5 @@
+with open('blocked_page.html', 'r', encoding='utf-8') as f:
+    text = f.read()
+print("Length of file:", len(text))
+for keyword in ['Systemet', 'foretage', 'indlæser', 'cannot', 'perform', 'try again', 'robot', 'captcha', 'recaptcha', 'gs_alrt', 'gs_top', 'gs_res_ccl']:
+    print(f"'{keyword}' in file:", keyword.lower() in text.lower())

Code File/convert_scimago.py

@@ -0,0 +1,29 @@
+import csv
+import json
+
+csv_file = r'd:\Website\Life OS\Oppurtunity\Google Scholar Data\Assets\scimagojr 2025.csv'
+json_file = r'd:\Website\Life OS\Oppurtunity\Google Scholar Data\Assets\scimagojr_2025.json'
+
+scimago_dict = {}
+
+with open(csv_file, 'r', encoding='utf-8-sig') as f:
+    reader = csv.DictReader(f, delimiter=';')
+    for row in reader:
+        # Save exact title, but use lowercase stripped title as key
+        # Handle cases where multiple journals might have similar names by keeping the first/highest ranked one
+        key = row['Title'].strip().lower()
+        if key not in scimago_dict:
+            scimago_dict[key] = {
+                'Title': row['Title'].strip(),
+                'Quartile': row.get('SJR Best Quartile', ''),
+                'H-index': row.get('H index', ''),
+                'SJR': row.get('SJR', ''),
+                'Categories': row.get('Categories', ''),
+                'Areas': row.get('Areas', ''),
+                'Country': row.get('Country', '')
+            }
+
+with open(json_file, 'w', encoding='utf-8') as f:
+    json.dump(scimago_dict, f, indent=2)
+
+print(f"Successfully converted {len(scimago_dict)} journals to JSON at {json_file}")

Code File/dump_threads.py

@@ -0,0 +1,18 @@
+import threading
+import sys
+import traceback
+
+def dump_threads():
+    for th in threading.enumerate():
+        print(th, th.is_alive())
+    print('\n*** STACKTRACE - START ***\n')
+    for threadId, stack in sys._current_frames().items():
+        print(f"\n# ThreadID: {threadId}")
+        for filename, lineno, name, line in traceback.extract_stack(stack):
+            print(f'File: "{filename}", line {lineno}, in {name}')
+            if line:
+                print(f'  {line.strip()}')
+    print('\n*** STACKTRACE - END ***\n')
+
+import time
+time.sleep(2) # Give the background task a moment

Code File/scholar_scraper.py

@@ -0,0 +1,489 @@
+from scholarly import scholarly, ProxyGenerator
+from scholarly._navigator import Navigator
+import time
+import csv
+import stem.control
+import stem
+import re
+import requests
+import json
+import os
+
+LOG_CALLBACK = None
+STOP_EVENT   = None
+
+def my_print(*args, **kwargs):
+    msg = ' '.join(map(str, args))
+    if LOG_CALLBACK:
+        LOG_CALLBACK(msg)
+    print(*args, **kwargs)
+
+# ── Paths ──────────────────────────────────────────────────────────────────────
+BASE_DIR     = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+scimago_path = os.path.join(BASE_DIR, "Assets", "scimagojr_2025.json")
+
+SCIMAGO_DATA = {}
+if os.path.exists(scimago_path):
+    with open(scimago_path, "r", encoding="utf-8") as f:
+        SCIMAGO_DATA = json.load(f)
+
+# ── Tor helpers ────────────────────────────────────────────────────────────────
+def setup_custom_tor():
+    """
+    Directly inject a Tor SOCKS5 session into scholarly's Navigator singleton,
+    bypassing scholarly.use_proxy() which spawns an unreliable FreeProxy secondary.
+    Verifies the connection can actually reach Google Scholar before proceeding.
+    """
+    my_print("Configuring scholarly to use Tor SOCKS proxy...")
+
+    tor_proxies = {
+        'http://':  'socks5://127.0.0.1:9150',
+        'https://': 'socks5://127.0.0.1:9150'
+    }
+
+    # ── Build a working proxy session ──────────────────────────────────────────
+    pg = ProxyGenerator()
+    pg._proxies     = tor_proxies
+    pg._proxy_works = True
+    pg._can_refresh_tor = True
+    pg._tor_control_port = 9151
+    pg._tor_password = None
+    pg._new_session(proxies=tor_proxies)
+
+    # ── Inject directly into Navigator (skip FreeProxy secondary spawn) ─────────
+    nav = Navigator()
+    nav.pm1      = pg
+    nav.pm2      = pg          # use same Tor session as secondary too
+    nav._session1 = pg.get_session()
+    nav._session2 = pg.get_session()
+
+    # ── Verify Tor is actually reachable ───────────────────────────────────────
+    MAX_VERIFY_ATTEMPTS = 3
+    for attempt in range(1, MAX_VERIFY_ATTEMPTS + 1):
+        try:
+            resp = pg.get_session().get(
+                "https://scholar.google.com",
+                timeout=15,
+                follow_redirects=True
+            )
+            if resp.status_code == 200:
+                my_print(f"Proxy verified successfully! (attempt {attempt})")
+                return pg
+            elif resp.status_code == 429 or resp.status_code == 503:
+                my_print(f"Google Scholar blocked this Tor exit node (HTTP {resp.status_code}). Rotating IP...")
+                if rotate_tor_ip():
+                    pg._new_session(proxies=tor_proxies)
+                    nav._session1 = pg.get_session()
+                    nav._session2 = pg.get_session()
+            else:
+                my_print(f"Unexpected status {resp.status_code} from Scholar. Continuing anyway...")
+                return pg
+        except Exception as e:
+            my_print(f"Tor connectivity check failed (attempt {attempt}/{MAX_VERIFY_ATTEMPTS}): {e}")
+            if attempt < MAX_VERIFY_ATTEMPTS:
+                my_print("Retrying after IP rotation...")
+                rotate_tor_ip()
+                pg._new_session(proxies=tor_proxies)
+                nav._session1 = pg.get_session()
+                nav._session2 = pg.get_session()
+
+    my_print("WARNING: Could not verify Tor connection to Google Scholar. Proceeding anyway — results may be empty.")
+    my_print("         Make sure Tor Browser is open and running on port 9150.")
+    return pg
+
+
+def rotate_tor_ip():
+    """Rotate the Tor circuit via the control port. Returns True on success."""
+    my_print("Requesting new Tor IP...")
+    try:
+        with stem.control.Controller.from_port(port=9151) as controller:
+            controller.authenticate()
+            controller.signal(stem.Signal.NEWNYM)
+            time.sleep(5)
+            my_print("Switched to new Tor circuit (New IP assigned).")
+            return True
+    except Exception as e:
+        my_print(f"Failed to rotate IP: {e}")
+        my_print("Please ensure Tor Browser is OPEN in the background.")
+        return False
+
+# ── Crossref metadata ──────────────────────────────────────────────────────────
+def get_extended_metadata(title, url):
+    meta = {
+        "DOI": "N/A", "Publisher": "N/A", "Open Access": "N/A",
+        "Authors (Crossref)": "N/A", "Exact Journal": "N/A"
+    }
+    doi = None
+    doi_pattern = re.compile(r'(10\.\d{4,9}/[-._;()/:a-zA-Z0-9]+)')
+    if url:
+        match = doi_pattern.search(url)
+        if match:
+            doi = match.group(1).rstrip('.')
+    item = None
+    try:
+        if doi:
+            resp = requests.get(f"https://api.crossref.org/works/{doi}", timeout=5)
+            if resp.status_code == 200:
+                item = resp.json().get("message", {})
+        if not item:
+            resp = requests.get("https://api.crossref.org/works",
+                                params={"query.bibliographic": title, "rows": 1}, timeout=5)
+            if resp.status_code == 200:
+                items = resp.json().get("message", {}).get("items", [])
+                if items and title[:15].lower() in items[0].get("title", [""])[0].lower():
+                    item = items[0]
+        if item:
+            meta["DOI"]       = item.get("DOI", doi if doi else "N/A")
+            meta["Publisher"] = item.get("publisher", "N/A")
+            container         = item.get("container-title", [])
+            meta["Exact Journal"] = container[0] if container else "N/A"
+            licenses          = item.get("license", [])
+            is_oa             = any("creativecommons" in l.get("URL","").lower() for l in licenses)
+            meta["Open Access"] = "Yes (CC)" if is_oa else "Unknown"
+            authors = item.get("author", [])
+            a_list  = []
+            for a in authors:
+                name      = f"{a.get('given','')} {a.get('family','')}".strip()
+                orcid     = a.get("ORCID","").replace("http://orcid.org/","")
+                affils    = [aff.get("name","") for aff in a.get("affiliation",[])]
+                affil_str = f" [{', '.join(affils)}]" if affils else ""
+                orcid_str = f" (ORCID: {orcid})" if orcid else ""
+                a_list.append(f"{name}{orcid_str}{affil_str}")
+            meta["Authors (Crossref)"] = "; ".join(a_list) if a_list else "N/A"
+            
+            # Crossref sometimes has abstracts, though they often contain XML JATS tags
+            raw_abstract = item.get("abstract")
+            if raw_abstract:
+                clean_abs = re.sub(r'<[^>]+>', '', raw_abstract).strip()
+                if len(clean_abs) > 20:
+                    meta["Abstract (Crossref)"] = clean_abs
+                    
+    except Exception:
+        pass
+    return meta
+
+def get_full_abstract(doi, title=None):
+    if doi and doi != "N/A": 
+        try:
+            url = f"https://api.semanticscholar.org/graph/v1/paper/DOI:{doi}?fields=abstract"
+            resp = requests.get(url, timeout=5)
+            if resp.status_code == 200:
+                abs_text = resp.json().get('abstract')
+                if abs_text and len(abs_text) > 20:
+                    return abs_text
+        except Exception:
+            pass
+            
+    if title:
+        try:
+            url = f"https://api.semanticscholar.org/graph/v1/paper/search"
+            resp = requests.get(url, params={"query": title, "fields": "abstract,title", "limit": 1}, timeout=5)
+            if resp.status_code == 200:
+                items = resp.json().get('data', [])
+                if items:
+                    found_title = items[0].get('title', '').lower()
+                    if title[:20].lower() in found_title:
+                        abs_text = items[0].get('abstract')
+                        if abs_text and len(abs_text) > 20:
+                            return abs_text
+        except Exception:
+            pass
+
+        # Tertiary fallback: OpenAlex API
+        try:
+            url = f"https://api.openalex.org/works?search={title}"
+            resp = requests.get(url, timeout=5)
+            if resp.status_code == 200:
+                data = resp.json()
+                if data.get('results'):
+                    item = data['results'][0]
+                    inv = item.get('abstract_inverted_index')
+                    if inv:
+                        max_idx = max([max(pos) for pos in inv.values()])
+                        words = [""] * (max_idx + 1)
+                        for word, positions in inv.items():
+                            for pos in positions:
+                                words[pos] = word
+                        abs_text = " ".join(words)
+                        if len(abs_text) > 20:
+                            return abs_text
+        except Exception:
+            pass
+            
+    return None
+
+# ── Filter engine ──────────────────────────────────────────────────────────────
+def passes_filters(row_data, filters):
+    """
+    Returns (passed: bool, reason: str|None).
+    All active filters must pass for the paper to be accepted.
+    """
+    if not filters:
+        return True, None
+
+    # Open Access
+    if filters.get('open_access'):
+        oa = str(row_data.get('Open Access', '')).lower()
+        if 'yes' not in oa:
+            return False, 'Not open access'
+
+    # Quartile (list of accepted values, e.g. ['Q1','Q2'])
+    quartiles = [q for q in filters.get('quartiles', []) if q]
+    if quartiles:
+        paper_q = str(row_data.get('Quartile (SCImago)', 'N/A'))
+        if paper_q not in quartiles:
+            return False, f'Quartile {paper_q} (need {"/".join(quartiles)})'
+
+    # H-Index >=
+    min_h = filters.get('min_h_index')
+    if min_h is not None:
+        try:
+            h = float(str(row_data.get('H-index (SCImago)', 0) or 0).replace(',', '.'))
+            if h < float(min_h):
+                return False, f'H-Index {h} < {min_h}'
+        except Exception:
+            return False, 'H-Index N/A'
+
+    # SJR >=
+    min_sjr = filters.get('min_sjr')
+    if min_sjr is not None:
+        try:
+            sjr = float(str(row_data.get('SJR Score (SCImago)', 0) or 0).replace(',', '.'))
+            if sjr < float(min_sjr):
+                return False, f'SJR {sjr} < {min_sjr}'
+        except Exception:
+            return False, 'SJR N/A'
+
+    # Citations >=
+    min_cit = filters.get('min_citations')
+    if min_cit is not None:
+        try:
+            cit = int(str(row_data.get('Citations Count', 0) or 0))
+            if cit < int(min_cit):
+                return False, f'Citations {cit} < {min_cit}'
+        except Exception:
+            return False, 'Citations N/A'
+
+    # Journal (any match)
+    journals = [j.strip().lower() for j in filters.get('journals', []) if j and j.strip()]
+    if journals:
+        paper_j = str(row_data.get('Journal/Venue (Scholar)', '') or '').lower()
+        if not any(j in paper_j or paper_j in j for j in journals):
+            return False, 'Journal not matched'
+
+    # Publisher (any match)
+    publishers = [p.strip().lower() for p in filters.get('publishers', []) if p and p.strip()]
+    if publishers:
+        paper_p = str(row_data.get('Publisher', '') or '').lower()
+        if not any(p in paper_p or paper_p in p for p in publishers):
+            return False, 'Publisher not matched'
+
+    return True, None
+
+# ── Main scraper ───────────────────────────────────────────────────────────────
+def search_scholar(query, output_file=None, target_count=20,
+                   year_low=None, year_high=None,
+                   stop_event=None, filters=None):
+    if output_file is None:
+        output_file = os.path.join(BASE_DIR, "Data Folder", "scholar_results.csv")
+
+    my_print(f"Searching for: {query}")
+    if filters:
+        my_print(f"Active filters: {filters}")
+
+    # ── Init Scholar search ──
+    search_query = None
+    max_retries  = 5
+    for attempt in range(max_retries):
+        if stop_event and stop_event.is_set():
+            my_print("[STOPPED] Task cancelled by user.")
+            return
+        try:
+            search_query = scholarly.search_pubs(query, year_low=year_low, year_high=year_high)
+            break
+        except Exception as e:
+            my_print(f"Blocked on initial search (Attempt {attempt+1}/{max_retries}): {e}")
+            rotate_tor_ip()
+
+    if not search_query:
+        my_print("Failed to initialize search after multiple retries. Exiting.")
+        return
+
+    headers = [
+        "Title", "Year", "Journal/Venue (Scholar)",
+        "Quartile (SCImago)", "H-index (SCImago)", "SJR Score (SCImago)",
+        "Citations Count", "Abstract/Snippet", "Authors (Scholar)",
+        "Publisher", "Open Access", "Categories & Areas (SCImago)",
+        "Country (SCImago)", "Paper Link", "PDF Link",
+        "DOI", "Authors (Crossref)"
+    ]
+
+    # Safeguard: don't crawl more than 50x the target (prevents infinite loop)
+    MAX_CRAWL     = max(target_count * 50, 500)
+    qualified     = 0
+    crawled       = 0
+    seen_titles   = set()
+
+    with open(output_file, mode="w", newline="", encoding="utf-8") as file:
+        writer = csv.DictWriter(file, fieldnames=headers, extrasaction='ignore')
+        writer.writeheader()
+
+        # ── Pre-flight: detect immediate 0-result (blocked Tor exit node) ────────
+        ZERO_RESULT_RETRIES = 3
+        for zero_attempt in range(ZERO_RESULT_RETRIES):
+            try:
+                first_paper = next(search_query)
+                # Got at least one result — break and process it below
+                break
+            except StopIteration:
+                if zero_attempt < ZERO_RESULT_RETRIES - 1:
+                    my_print(f"[WARNING] Query returned 0 results immediately — Tor exit node may be blocked. "
+                             f"Rotating IP and retrying (attempt {zero_attempt + 1}/{ZERO_RESULT_RETRIES})...")
+                    rotated = rotate_tor_ip()
+                    if not rotated:
+                        my_print("IP rotation failed — Tor Browser may not be running. Cannot retry.")
+                        break
+                    # Re-initialise the search iterator with the new exit node
+                    try:
+                        search_query = scholarly.search_pubs(query, year_low=year_low, year_high=year_high)
+                        continue
+                    except Exception as e:
+                        my_print(f"Re-search after rotation failed: {e}")
+                        break
+                else:
+                    my_print("No results found after all retry attempts. The query may have no matching papers, "
+                             "or all Tor exit nodes are blocked by Google Scholar.")
+                    my_print("Try opening Tor Browser and running again, or use a VPN/different network.")
+                    first_paper = None
+                    break
+            except Exception:
+                first_paper = None
+                break
+        else:
+            first_paper = None
+
+        if first_paper is None:
+            # Genuinely no results or could not recover
+            pass
+        else:
+            # We have the first paper — inject it back into the processing loop
+            # by using an inline generator that yields first_paper then continues search_query
+            def _chain(first, rest):
+                yield first
+                yield from rest
+            search_query = _chain(first_paper, search_query)
+
+        while qualified < target_count and crawled < MAX_CRAWL:
+            if stop_event and stop_event.is_set():
+                my_print("[STOPPED] Task cancelled by user.")
+                break
+            try:
+                paper  = next(search_query)
+                crawled += 1
+                bib    = paper.get('bib', {})
+
+                title    = bib.get('title', 'N/A')
+                title_norm = title.strip().lower()
+                if title_norm in seen_titles:
+                    my_print(f"    [SKIP #{crawled}] Duplicate title detected (overlapping pages) — \"{title[:55]}\"")
+                    continue
+                seen_titles.add(title_norm)
+
+                year     = bib.get('pub_year', 'N/A')
+                venue    = bib.get('venue', 'N/A')
+                abstract = bib.get('abstract', 'N/A')
+                authors  = (", ".join(bib.get('author', []))
+                            if isinstance(bib.get('author'), list)
+                            else bib.get('author', 'N/A'))
+                citations = paper.get('num_citations', 0)
+                pub_url   = paper.get('pub_url', 'N/A')
+                pdf_url   = paper.get('eprint_url', 'N/A')
+
+                # Crossref
+                crmeta       = get_extended_metadata(title, pub_url)
+                exact_journal = crmeta.pop("Exact Journal", "N/A")
+                search_name  = (exact_journal if exact_journal != "N/A" else venue
+                                ).strip('… .').lower()
+
+                # SCImago lookup
+                scimago_match = SCIMAGO_DATA.get(search_name)
+                if not scimago_match:
+                    for sjr_title, sjr_info in SCIMAGO_DATA.items():
+                        if search_name == sjr_title or (len(search_name) > 8 and search_name in sjr_title):
+                            scimago_match = sjr_info
+                            break
+
+                sjr_q = scimago_match.get("Quartile", "N/A") if scimago_match else "N/A"
+                sjr_h = scimago_match.get("H-index", "N/A") if scimago_match else "N/A"
+                sjr_s = scimago_match.get("SJR",     "N/A") if scimago_match else "N/A"
+                sjr_cat = "N/A"
+                if scimago_match:
+                    cat_val  = scimago_match.get("Categories", "")
+                    area_val = scimago_match.get("Areas", "")
+                    if cat_val and area_val:
+                        sjr_cat = f"{cat_val} | {area_val}"
+                    elif cat_val or area_val:
+                        sjr_cat = cat_val or area_val
+                sjr_c = scimago_match.get("Country", "N/A") if scimago_match else "N/A"
+
+                row = {
+                    "Title": title, "Year": year, "Journal/Venue (Scholar)": venue,
+                    "Abstract/Snippet": abstract, "Authors (Scholar)": authors,
+                    "Citations Count": citations, "Paper Link": pub_url, "PDF Link": pdf_url,
+                    "Quartile (SCImago)": sjr_q, "H-index (SCImago)": sjr_h,
+                    "SJR Score (SCImago)": sjr_s,
+                    "Categories & Areas (SCImago)": sjr_cat, "Country (SCImago)": sjr_c
+                }
+                row.update(crmeta)
+
+                # ── Apply filters ──────────────────────────────────────────────
+                passed, reason = passes_filters(row, filters)
+                if not passed:
+                    short_title = title[:55] + ('...' if len(title) > 55 else '')
+                    my_print(f"    [SKIP #{crawled}] {reason} — \"{short_title}\"")
+                    time.sleep(1)
+                    continue
+
+                # ── Qualified paper ─────────────────────────────────────────────
+                
+                # Abstract Fetching Pipeline: Crossref -> Semantic Scholar -> OpenAlex
+                cr_abs = crmeta.get("Abstract (Crossref)")
+                if cr_abs:
+                    row["Abstract/Snippet"] = cr_abs
+                else:
+                    full_abstract = get_full_abstract(row.get('DOI'), row.get('Title'))
+                    if full_abstract and len(full_abstract) > 20:
+                        row["Abstract/Snippet"] = full_abstract
+                
+                qualified += 1
+                my_print(f"[{qualified}/{target_count}] Title: {title}")
+                my_print(f"    Year: {year} | DOI: {crmeta['DOI']} | Quartile: {sjr_q} | Publisher: {crmeta['Publisher']}")
+
+                writer.writerow(row)
+                file.flush()
+                time.sleep(2)
+
+            except StopIteration:
+                my_print("No more results found for this query.")
+                break
+            except Exception as e:
+                if stop_event and stop_event.is_set():
+                    my_print("[STOPPED] Task cancelled by user.")
+                    break
+                my_print(f"Blocked or error encountered: {e}")
+                rotate_tor_ip()
+                try:
+                    my_print(f"Resuming search from index {crawled}...")
+                    search_query = scholarly.search_pubs(query, year_low=year_low, year_high=year_high, start_index=crawled)
+                except Exception as re_err:
+                    my_print(f"Failed to re-initialize search iterator: {re_err}")
+
+    my_print(f"Extraction complete! Saved {qualified} results to '{output_file}'")
+    my_print(f"(Crawled {crawled} papers total to find {qualified} qualifying papers)")
+
+if __name__ == "__main__":
+    setup_custom_tor()
+    search_term = '"river bank erosion" AND ("GIS" OR "Remote Sensing" OR "DSAS" OR "morphological change")'
+    out_path    = os.path.join(BASE_DIR, "Data Folder", "river_erosion_papers.csv")
+    search_scholar(search_term, output_file=out_path, target_count=50)
+

Code File/static/script.js

@@ -0,0 +1,295 @@
+/* ═══════════════════════════════════════════════════════════════
+   QUERY BUILDER
+═══════════════════════════════════════════════════════════════ */
+let conditionCounter = 0;
+
+function quoteIfNeeded(term) {
+  term = term.trim();
+  return term.includes(' ') ? '"' + term + '"' : term;
+}
+
+function buildQuery() {
+  const mainVal = document.getElementById('qbMain').value.trim();
+  if (!mainVal) return '';
+  const mainParts = mainVal.split(',').map(p => p.trim()).filter(Boolean);
+  let query = mainParts.length === 1
+    ? mainParts[0]
+    : '(' + mainParts.map(quoteIfNeeded).join(' OR ') + ')';
+
+  document.querySelectorAll('.qb-cond-row').forEach(row => {
+    const type  = row.dataset.type;
+    const val   = row.querySelector('.qb-input').value.trim();
+    if (!val) return;
+    const terms = val.split(',').map(t => t.trim()).filter(Boolean);
+    if (!terms.length) return;
+    const group = terms.length === 1
+      ? quoteIfNeeded(terms[0])
+      : '(' + terms.map(quoteIfNeeded).join(' OR ') + ')';
+    query += ' ' + type + ' ' + group;
+  });
+  return query;
+}
+
+function updatePreview() {
+  const q  = buildQuery();
+  const el = document.getElementById('qbPreview');
+  el.innerHTML = q
+    ? '<strong>Query:</strong> ' + escapeHtml(q)
+    : 'Query will appear here…';
+}
+
+function addCondition(type) {
+  conditionCounter++;
+  const id  = 'cond_' + conditionCounter;
+  const ph  = type === 'AND'
+    ? 'e.g. GIS, remote sensing, DSAS'
+    : 'e.g. bankline migration';
+  const div = document.createElement('div');
+  div.className    = 'qb-row qb-cond-row';
+  div.dataset.type = type;
+  div.id = id;
+  div.innerHTML =
+    '<span class="qb-badge ' + type.toLowerCase() + '" onclick="toggleType(this)" title="Click to toggle">' + type + '</span>' +
+    '<input class="qb-input" type="text" placeholder="' + ph + '" oninput="updatePreview()">' +
+    '<button class="qb-del" onclick="removeCondition(\'' + id + '\')">&#x2715;</button>';
+  document.getElementById('qbConditions').appendChild(div);
+  updatePreview();
+}
+
+function toggleType(badge) {
+  const row  = badge.closest('.qb-cond-row');
+  const next = row.dataset.type === 'AND' ? 'OR' : 'AND';
+  row.dataset.type   = next;
+  badge.textContent  = next;
+  badge.className    = 'qb-badge ' + next.toLowerCase();
+  updatePreview();
+}
+
+function removeCondition(id) {
+  const el = document.getElementById(id);
+  if (el) el.remove();
+  updatePreview();
+}
+
+/* ═══════════════════════════════════════════════════════════════
+   QUARTILE CHIPS
+═══════════════════════════════════════════════════════════════ */
+function toggleChip(btn) {
+  btn.classList.toggle('active');
+}
+
+function getSelectedQuartiles() {
+  return [...document.querySelectorAll('#quartileChips .chip.active')]
+    .map(c => c.dataset.value);
+}
+
+/* ═══════════════════════════════════════════════════════════════
+   JOURNAL / PUBLISHER DYNAMIC FIELDS
+═══════════════════════════════════════════════════════════════ */
+let fieldCounters = { journal: 0, publisher: 0 };
+
+function addFilterField(type) {
+  fieldCounters[type]++;
+  const id = type + '_' + fieldCounters[type];
+  const ph = type === 'journal'
+    ? 'e.g. Natural Hazards'
+    : 'e.g. Elsevier BV';
+  const container = document.getElementById(type + 'Fields');
+  const div = document.createElement('div');
+  div.className = 'filter-input-row';
+  div.id = id;
+  div.innerHTML =
+    '<input type="text" placeholder="' + ph + '">' +
+    '<button class="qb-del" onclick="removeFilterField(\'' + id + '\')">&#x2715;</button>';
+  container.appendChild(div);
+}
+
+function removeFilterField(id) {
+  const el = document.getElementById(id);
+  if (el) el.remove();
+}
+
+function getFilterValues(type) {
+  return [...document.querySelectorAll('#' + type + 'Fields input')]
+    .map(i => i.value.trim()).filter(Boolean);
+}
+
+/* ═══════════════════════════════════════════════════════════════
+   SCRAPER CONTROL
+═══════════════════════════════════════════════════════════════ */
+let paperCount   = 0;
+let activeSource = null;
+let isRunning    = false;
+
+function setIPStatus(state, text) {
+  document.getElementById('liveDot').className = 'live-dot ' + state;
+  document.getElementById('liveText').textContent = text;
+}
+function setBottomStatus(active, text) {
+  const dot = document.getElementById('statusDot');
+  if (active) dot.classList.add('active'); else dot.classList.remove('active');
+  document.getElementById('statusText').textContent = text;
+}
+
+async function handleMainButton() {
+  isRunning ? await stopScrape() : await startScrape();
+}
+
+async function stopScrape() {
+  const btn = document.getElementById('startBtn');
+  btn.disabled = true;
+  btn.innerText = 'STOPPING...';
+  try { await fetch('/stop', { method: 'POST' }); } catch(e) {}
+}
+
+async function startScrape() {
+  const query = buildQuery();
+  if (!query) {
+    alert('Please enter at least one keyword in the MUST field.');
+    return;
+  }
+
+  const btn    = document.getElementById('startBtn');
+  const termSt = document.getElementById('termStatus');
+  isRunning    = true;
+  paperCount   = 0;
+  document.getElementById('paperCount').textContent = '';
+
+  btn.innerText = '■ STOP';
+  btn.className = 'btn btn-stop';
+  termSt.textContent = 'ACTIVE';
+  setBottomStatus(true, 'Scraping in progress...');
+  setIPStatus('ok', 'IP: STABLE — Connected via Tor');
+
+  document.getElementById('terminal').innerHTML = '';
+  appendLine('$ Query: ' + query, false, 't-muted');
+  appendLine('$ Initializing Tor proxy...', false, 't-muted');
+
+  // ── Collect all filter values ──────────────────────────────────────────────
+  const payload = {
+    query:        query,
+    max_results:  document.getElementById('max_results').value,
+    year_low:     document.getElementById('year_low').value   || null,
+    year_high:    document.getElementById('year_high').value  || null,
+    open_access:  document.getElementById('open_access').checked,
+    quartiles:    getSelectedQuartiles(),
+    min_h_index:  document.getElementById('min_h_index').value  || null,
+    min_sjr:      document.getElementById('min_sjr').value      || null,
+    min_citations:document.getElementById('min_citations').value || null,
+    journals:     getFilterValues('journal'),
+    publishers:   getFilterValues('publisher')
+  };
+
+  try {
+    await fetch('/start-scrape', {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify(payload)
+    });
+  } catch(e) {
+    appendLine('ERROR: Could not reach backend.', false, 't-error');
+    resetUI('idle');
+    return;
+  }
+
+  if (activeSource) { activeSource.close(); activeSource = null; }
+  const source = new EventSource('/stream');
+  activeSource = source;
+
+  source.onmessage = function(event) {
+    const raw = event.data;
+
+    if (raw === 'DONE') {
+      source.close(); activeSource = null;
+      appendLine('');
+      appendLine('EXTRACTION COMPLETE — ' + paperCount + ' papers saved.', false, 't-done');
+      setIPStatus('idle', 'IP: IDLE — Task complete');
+      resetUI('done');
+      return;
+    }
+    if (raw === 'STOPPED') {
+      source.close(); activeSource = null;
+      appendLine('');
+      appendLine('[STOPPED] Cancelled — ' + paperCount + ' papers saved.', false, 't-error');
+      setIPStatus('idle', 'IP: IDLE — Task stopped');
+      resetUI('idle');
+      return;
+    }
+
+    // IP noise → top bar only
+    if (raw.includes('Blocked') || raw.includes("codec can't encode") || raw.includes('charmap') || raw.includes('character maps to')) {
+      setIPStatus('blocked', 'IP: BLOCKED — Rotating circuit...');
+      return;
+    }
+    if (raw.includes('Requesting new Tor IP')) {
+      setIPStatus('rotating', 'IP: ROTATING — Requesting new circuit...');
+      return;
+    }
+    if ((raw.includes('Switched') && raw.includes('circuit')) || raw.includes('New IP assigned')) {
+      setIPStatus('ok', 'IP: ROTATED — New circuit active');
+      return;
+    }
+
+    // Normal output
+    const isIndented = raw.startsWith('    ') || raw.startsWith('\t') || raw.startsWith('[SKIP');
+    let cssClass = '';
+    if (raw.includes('Title:')) {
+      paperCount++;
+      document.getElementById('paperCount').textContent = paperCount + ' papers';
+      cssClass = 't-title';
+    } else if (raw.includes('[SKIP')) {
+      cssClass = 't-error';
+    } else if (raw.includes('[STOPPED]')) {
+      cssClass = 't-error';
+    } else if (raw.includes('Configuring') || raw.includes('Proxy') || raw.includes('Searching') || raw.startsWith('$')) {
+      cssClass = 't-muted';
+    }
+    appendLine(escapeHtml(raw), isIndented, cssClass);
+  };
+
+  source.onerror = function() {
+    source.close(); activeSource = null;
+    appendLine('Stream disconnected.', false, 't-error');
+    resetUI('idle');
+  };
+}
+
+function appendLine(text, isIndented, cssClass) {
+  const term = document.getElementById('terminal');
+  const div  = document.createElement('div');
+  div.className = 'term-line' + (isIndented ? '' : ' no-indent');
+  if (cssClass && text) {
+    const span = document.createElement('span');
+    span.className = cssClass;
+    span.textContent = text;
+    div.appendChild(span);
+  } else {
+    div.textContent = text;
+  }
+  term.appendChild(div);
+  term.scrollTop = term.scrollHeight;
+}
+
+function escapeHtml(str) {
+  return str.replace(/&/g,'&amp;').replace(/</g,'&lt;').replace(/>/g,'&gt;');
+}
+
+function resetUI(state) {
+  isRunning = false;
+  const btn    = document.getElementById('startBtn');
+  const termSt = document.getElementById('termStatus');
+  btn.disabled = false;
+  btn.innerText = '▶ RUN';
+  btn.className = 'btn btn-primary';
+  termSt.textContent = 'IDLE';
+  setBottomStatus(false, state === 'done'
+    ? 'Done — ' + paperCount + ' qualifying papers saved'
+    : 'Idle — ready for next run');
+}
+
+document.addEventListener('DOMContentLoaded', function() {
+  appendLine('$ Waiting for execution command...', false, 't-muted');
+  setIPStatus('idle', 'IP: IDLE — Not connected');
+  updatePreview();
+  // Start with one default journal and publisher slot hidden (add on demand)
+});

Code File/static/style.css

@@ -0,0 +1,583 @@
+@import url('https://fonts.googleapis.com/css2?family=Space+Mono:wght@400;700&family=Inter:wght@400;500;600&display=swap');
+
+:root {
+  --bg:          #e8e4dc;
+  --win-bg:      #f4f0e8;
+  --win-border:  #2a2d4a;
+  --titlebar-bg: #2a2d4a;
+  --titlebar-tx: #f4f0e8;
+  --accent:      #3d5a99;
+  --accent-hover:#2e4480;
+  --accent2:     #c4693a;
+  --green:       #3a7d5c;
+  --green-hov:   #2d6148;
+  --text:        #1a1c2e;
+  --text-muted:  #5a5d72;
+  --input-bg:    #fffef9;
+  --input-bdr:   #b8b4a8;
+  --term-bg:     #0f1117;
+  --term-green:  #39d98a;
+  --term-yellow: #f5c842;
+  --term-red:    #f25f5c;
+  --term-blue:   #60a5fa;
+  --term-dim:    #4a5568;
+  --shadow:      0 4px 20px rgba(42,45,74,0.15);
+  --radius:      12px;
+  --radius-sm:   6px;
+}
+
+/* No page scroll — everything fits in viewport */
+*, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
+html, body {
+  height: 100%;
+  overflow: hidden;
+}
+body {
+  font-family: 'Inter', sans-serif;
+  background: var(--bg);
+  background-image:
+    radial-gradient(circle at 20% 20%, rgba(61,90,153,0.08) 0%, transparent 50%),
+    radial-gradient(circle at 80% 80%, rgba(196,105,58,0.06) 0%, transparent 50%);
+  display: flex;
+  flex-direction: column;
+  align-items: center;
+  justify-content: center;
+  padding: 16px;
+}
+
+/* App shell fills viewport height */
+.app-shell {
+  width: 100%;
+  max-width: 1280px;
+  display: flex;
+  flex-direction: column;
+  gap: 12px;
+  height: 100%;
+}
+
+/* ── Top Bar ── */
+.app-topbar {
+  background: var(--titlebar-bg);
+  border-radius: var(--radius);
+  padding: 10px 20px;
+  display: flex;
+  align-items: center;
+  justify-content: space-between;
+  box-shadow: var(--shadow);
+  flex-shrink: 0;
+}
+.logo {
+  display: flex;
+  align-items: center;
+  gap: 12px;
+  color: var(--titlebar-tx);
+}
+.dot-row { display: flex; gap: 6px; }
+.dot { width: 12px; height: 12px; border-radius: 50%; }
+.dot-red   { background: #f25f5c; }
+.dot-amber { background: #f5c842; }
+.dot-green { background: #39d98a; }
+.logo h1 {
+  font-family: 'Space Mono', monospace;
+  font-size: 0.9rem;
+  letter-spacing: 0.08em;
+  color: var(--titlebar-tx);
+}
+
+/* ── Live Status Indicator (replaces badge) ── */
+.live-status {
+  display: flex;
+  align-items: center;
+  gap: 8px;
+  font-family: 'Space Mono', monospace;
+  font-size: 0.72rem;
+  color: rgba(244,240,232,0.6);
+  letter-spacing: 0.06em;
+}
+.live-dot {
+  width: 10px;
+  height: 10px;
+  border-radius: 50%;
+  background: #4a5568; /* idle = grey */
+  flex-shrink: 0;
+  transition: background 0.3s;
+}
+.live-dot.running  { background: #60a5fa; box-shadow: 0 0 8px rgba(96,165,250,0.6); animation: pulse 1.2s infinite; }
+.live-dot.blocked  { background: #f25f5c; box-shadow: 0 0 8px rgba(242,95,92,0.6); animation: pulse 0.6s infinite; }
+.live-dot.done     { background: #39d98a; box-shadow: 0 0 6px rgba(57,217,138,0.5); }
+@keyframes pulse { 0%,100%{opacity:1} 50%{opacity:0.35} }
+
+/* ── Two-Column Layout ── */
+.app-body {
+  display: grid;
+  grid-template-columns: 350px 1fr;
+  gap: 12px;
+  flex: 1;
+  min-height: 0; /* allow children to shrink */
+}
+
+/* ── Window Panel ── */
+.win-panel {
+  background: var(--win-bg);
+  border: 2px solid var(--win-border);
+  border-radius: var(--radius);
+  box-shadow: var(--shadow);
+  overflow: hidden;
+  display: flex;
+  flex-direction: column;
+}
+.win-title {
+  background: var(--titlebar-bg);
+  color: var(--titlebar-tx);
+  padding: 8px 14px;
+  font-family: 'Space Mono', monospace;
+  font-size: 0.75rem;
+  letter-spacing: 0.06em;
+  display: flex;
+  align-items: center;
+  gap: 8px;
+  flex-shrink: 0;
+}
+.win-body {
+  padding: 16px;
+  flex: 1;
+  overflow: hidden;
+  display: flex;
+  flex-direction: column;
+}
+
+/* ── Form ── */
+.form-section { margin-bottom: 14px; }
+.form-section label {
+  display: block;
+  font-family: 'Space Mono', monospace;
+  font-size: 0.68rem;
+  color: var(--text-muted);
+  letter-spacing: 0.08em;
+  text-transform: uppercase;
+  margin-bottom: 5px;
+}
+textarea, input[type="number"], input[type="text"] {
+  width: 100%;
+  background: var(--input-bg);
+  border: 1.5px solid var(--input-bdr);
+  border-radius: var(--radius-sm);
+  padding: 8px 10px;
+  font-family: 'Space Mono', monospace;
+  font-size: 0.75rem;
+  color: var(--text);
+  resize: none;
+  transition: border-color 0.2s, box-shadow 0.2s;
+  line-height: 1.4;
+}
+textarea:focus, input:focus {
+  outline: none;
+  border-color: var(--accent);
+  box-shadow: 0 0 0 3px rgba(61,90,153,0.15);
+}
+.row-2 { display: grid; grid-template-columns: 1fr 1fr; gap: 10px; }
+.divider { height: 1px; background: var(--input-bdr); margin: 12px 0; opacity: 0.5; }
+
+/* ── Buttons ── */
+.btn {
+  display: block;
+  width: 100%;
+  padding: 9px;
+  border: 2px solid transparent;
+  border-radius: var(--radius-sm);
+  font-family: 'Space Mono', monospace;
+  font-size: 0.75rem;
+  letter-spacing: 0.06em;
+  cursor: pointer;
+  text-align: center;
+  text-decoration: none;
+  transition: all 0.15s ease;
+  margin-bottom: 8px;
+}
+.btn-primary { background: var(--accent); color: #fff; border-color: var(--accent); }
+.btn-primary:hover { background: var(--accent-hover); border-color: var(--accent-hover); transform: translateY(-1px); }
+.btn-primary:disabled { background: var(--text-muted); border-color: var(--text-muted); cursor: not-allowed; transform: none; }
+.btn-secondary { background: transparent; color: var(--green); border-color: var(--green); }
+.btn-secondary:hover { background: var(--green); color: #fff; transform: translateY(-1px); }
+.btn-ghost { background: transparent; color: var(--text-muted); border-color: var(--input-bdr); }
+.btn-ghost:hover { background: var(--text-muted); color: #fff; transform: translateY(-1px); }
+
+/* ── Status Bar ── */
+.status-bar {
+  background: rgba(42,45,74,0.06);
+  border-radius: var(--radius-sm);
+  padding: 7px 10px;
+  font-family: 'Space Mono', monospace;
+  font-size: 0.68rem;
+  color: var(--text-muted);
+  margin-top: auto;
+  display: flex;
+  align-items: center;
+  gap: 8px;
+}
+.status-dot { width: 8px; height: 8px; border-radius: 50%; background: var(--text-muted); flex-shrink: 0; }
+.status-dot.active { background: var(--green); box-shadow: 0 0 6px rgba(57,217,138,0.5); animation: pulse 1.2s infinite; }
+
+/* ── Terminal Panel fills remaining height ── */
+.terminal-panel {
+  display: flex;
+  flex-direction: column;
+  min-height: 0;
+}
+.terminal-panel .win-panel {
+  height: 100%;
+}
+.terminal-wrap {
+  display: flex;
+  flex-direction: column;
+  flex: 1;
+  min-height: 0;
+}
+.terminal-path {
+  font-family: 'Space Mono', monospace;
+  font-size: 0.7rem;
+  color: var(--text-muted);
+  padding: 6px 14px;
+  background: rgba(42,45,74,0.04);
+  border-bottom: 1px solid var(--input-bdr);
+  letter-spacing: 0.04em;
+  flex-shrink: 0;
+}
+
+/* KEY TERMINAL FIX */
+.terminal {
+  background: var(--term-bg);
+  color: var(--term-green);
+  font-family: 'Space Mono', monospace;
+  font-size: 0.8rem;
+  line-height: 1.55;
+  /* Grows to fill remaining space, never overflows the layout */
+  flex: 1;
+  min-height: 0;
+  overflow-y: auto;
+  overflow-x: hidden;
+  padding: 12px 14px;
+}
+.terminal::-webkit-scrollbar { width: 5px; }
+.terminal::-webkit-scrollbar-track { background: #1a1f2e; }
+.terminal::-webkit-scrollbar-thumb { background: #3d5a99; border-radius: 3px; }
+
+/* Each terminal line — hanging indent so wrapped lines align */
+.term-line {
+  display: block;
+  white-space: pre-wrap;
+  word-break: break-word;
+  /* Hanging indent: shifts continuation lines right by 4ch */
+  padding-left: 4ch;
+  text-indent: -4ch;
+  margin-bottom: 1px;
+}
+/* Override indent for lines that start without leading spaces */
+.term-line.no-indent { padding-left: 0; text-indent: 0; }
+
+.term-line .t-title  { color: var(--term-yellow); }
+.term-line .t-error  { color: var(--term-red); }
+.term-line .t-muted  { color: var(--term-dim); }
+.term-line .t-done   { color: var(--term-green); font-weight: bold; }
+
+.terminal-footer {
+  background: #1a1f2e;
+  padding: 6px 14px;
+  display: flex;
+  align-items: center;
+  gap: 8px;
+  font-family: 'Space Mono', monospace;
+  font-size: 0.68rem;
+  color: var(--term-dim);
+  flex-shrink: 0;
+}
+.term-prompt { color: var(--term-green); }
+
+/* ── Data Table Page ── */
+html.data-page, html.data-page body { overflow: auto; height: auto; }
+.tbl-container { overflow-x: auto; max-height: 65vh; overflow-y: auto; }
+table { width: 100%; border-collapse: collapse; font-size: 0.8rem; }
+thead th {
+  position: sticky; top: 0;
+  background: var(--titlebar-bg);
+  color: var(--titlebar-tx);
+  font-family: 'Space Mono', monospace;
+  font-size: 0.68rem;
+  letter-spacing: 0.05em;
+  padding: 9px 12px;
+  text-align: left;
+  white-space: nowrap;
+}
+tbody td {
+  padding: 9px 12px;
+  border-bottom: 1px solid rgba(42,45,74,0.08);
+  color: var(--text);
+  max-width: 200px;
+  overflow: hidden;
+  text-overflow: ellipsis;
+  white-space: nowrap;
+}
+tbody tr:hover td { background: rgba(61,90,153,0.06); }
+.badge-q1 { color:#fff; background:#3a7d5c; padding:2px 7px; border-radius:4px; font-size:0.68rem; font-family:'Space Mono',monospace; }
+.badge-q2 { color:#fff; background:#3d5a99; padding:2px 7px; border-radius:4px; font-size:0.68rem; font-family:'Space Mono',monospace; }
+.badge-q3 { color:#fff; background:#c4693a; padding:2px 7px; border-radius:4px; font-size:0.68rem; font-family:'Space Mono',monospace; }
+.badge-q4 { color:#fff; background:#8b5a8b; padding:2px 7px; border-radius:4px; font-size:0.68rem; font-family:'Space Mono',monospace; }
+.badge-na { color:#888; background:rgba(0,0,0,0.06); padding:2px 7px; border-radius:4px; font-size:0.68rem; font-family:'Space Mono',monospace; }
+
+/* Stop button style */
+.btn-stop {
+  background: #8b1c1c;
+  color: #fff;
+  border-color: #8b1c1c;
+}
+.btn-stop:hover { background: #6e1515; border-color: #6e1515; transform: translateY(-1px); }
+
+/* IP rotating state */
+.live-dot.rotating { background: #f5c842; box-shadow: 0 0 8px rgba(245,200,66,0.6); animation: pulse 0.5s infinite; }
+
+/* ── One-row action buttons ── */
+.btn-row {
+  display: grid;
+  grid-template-columns: 1fr auto auto;
+  gap: 8px;
+  margin-bottom: 10px;
+}
+.btn-row .btn { margin-bottom: 0; padding: 9px 10px; }
+.btn-icon { min-width: 42px; }
+
+/* ── Query Builder ── */
+.qb-wrap {
+  display: flex;
+  flex-direction: column;
+  gap: 6px;
+}
+.qb-row {
+  display: flex;
+  align-items: center;
+  gap: 6px;
+}
+.qb-badge {
+  font-family: 'Space Mono', monospace;
+  font-size: 0.65rem;
+  font-weight: 700;
+  letter-spacing: 0.06em;
+  padding: 4px 8px;
+  border-radius: 4px;
+  cursor: default;
+  white-space: nowrap;
+  flex-shrink: 0;
+}
+.qb-badge.must  { background: #2a2d4a; color: #f4f0e8; }
+.qb-badge.and   { background: #3d5a99; color: #fff; cursor: pointer; }
+.qb-badge.or    { background: #c4693a; color: #fff; cursor: pointer; }
+.qb-badge.and:hover, .qb-badge.or:hover { opacity: 0.8; }
+
+.qb-input {
+  flex: 1;
+  padding: 6px 9px;
+  font-size: 0.75rem;
+}
+.qb-del {
+  background: transparent;
+  border: 1.5px solid var(--input-bdr);
+  border-radius: 4px;
+  width: 28px;
+  height: 28px;
+  display: flex;
+  align-items: center;
+  justify-content: center;
+  cursor: pointer;
+  color: var(--text-muted);
+  font-size: 1rem;
+  padding: 0;
+  flex-shrink: 0;
+  transition: all 0.15s;
+}
+.qb-del:hover { background: #f25f5c; border-color: #f25f5c; color: #fff; }
+
+.qb-add-row {
+  display: flex;
+  gap: 6px;
+  margin-top: 2px;
+}
+.qb-add-btn {
+  font-family: 'Space Mono', monospace;
+  font-size: 0.68rem;
+  padding: 5px 10px;
+  border-radius: 5px;
+  cursor: pointer;
+  border: 1.5px solid;
+  background: transparent;
+  transition: all 0.15s;
+  letter-spacing: 0.04em;
+  margin: 0;
+  width: auto;
+}
+.qb-add-btn.and { color: #3d5a99; border-color: #3d5a99; }
+.qb-add-btn.and:hover { background: #3d5a99; color: #fff; }
+.qb-add-btn.or  { color: #c4693a; border-color: #c4693a; }
+.qb-add-btn.or:hover  { background: #c4693a; color: #fff; }
+
+.qb-preview {
+  background: rgba(42,45,74,0.07);
+  border-radius: 5px;
+  padding: 6px 9px;
+  font-family: 'Space Mono', monospace;
+  font-size: 0.65rem;
+  color: var(--text-muted);
+  word-break: break-all;
+  margin-top: 4px;
+  min-height: 28px;
+  line-height: 1.5;
+}
+.qb-preview strong { color: var(--accent); }
+
+/* ── Left panel scroll ── */
+.win-body.scrollable {
+  overflow-y: auto;
+  overflow-x: hidden;
+  padding-right: 10px;
+}
+.win-body.scrollable::-webkit-scrollbar { width: 4px; }
+.win-body.scrollable::-webkit-scrollbar-track { background: transparent; }
+.win-body.scrollable::-webkit-scrollbar-thumb { background: var(--input-bdr); border-radius: 3px; }
+
+/* ── Config section labels ── */
+.cfg-label {
+  font-family: 'Space Mono', monospace;
+  font-size: 0.65rem;
+  color: var(--text-muted);
+  letter-spacing: 0.1em;
+  text-transform: uppercase;
+  margin-bottom: 7px;
+  margin-top: 2px;
+}
+.cfg-hint {
+  font-size: 0.6rem;
+  opacity: 0.6;
+  text-transform: none;
+  letter-spacing: 0;
+}
+
+/* ── Limits row ── */
+.limits-row {
+  display: grid;
+  grid-template-columns: 1fr 1fr 1fr auto;
+  gap: 7px;
+  align-items: end;
+  margin-bottom: 4px;
+}
+.lim-group label {
+  display: block;
+  font-family: 'Space Mono', monospace;
+  font-size: 0.62rem;
+  color: var(--text-muted);
+  letter-spacing: 0.08em;
+  margin-bottom: 4px;
+}
+.lim-group input {
+  padding: 6px 8px;
+  font-size: 0.73rem;
+}
+.oa-group { display: flex; flex-direction: column; align-items: center; gap: 4px; }
+.oa-group label:first-child { align-self: flex-start; }
+
+/* ── Toggle switch (Open Access) ── */
+.toggle-wrap {
+  position: relative;
+  display: inline-block;
+  width: 38px;
+  height: 20px;
+  cursor: pointer;
+}
+.toggle-wrap input { opacity: 0; width: 0; height: 0; }
+.toggle-slider {
+  position: absolute;
+  inset: 0;
+  background: var(--input-bdr);
+  border-radius: 20px;
+  transition: background 0.2s;
+}
+.toggle-slider::before {
+  content: '';
+  position: absolute;
+  width: 14px; height: 14px;
+  left: 3px; top: 3px;
+  background: white;
+  border-radius: 50%;
+  transition: transform 0.2s;
+}
+.toggle-wrap input:checked + .toggle-slider { background: var(--green); }
+.toggle-wrap input:checked + .toggle-slider::before { transform: translateX(18px); }
+
+/* ── Filter row label ── */
+.filter-row-label {
+  font-family: 'Space Mono', monospace;
+  font-size: 0.62rem;
+  color: var(--text-muted);
+  margin-bottom: 5px;
+}
+
+/* ── Quartile chips ── */
+.chip-row {
+  display: flex;
+  gap: 5px;
+  flex-wrap: wrap;
+  margin-bottom: 9px;
+}
+.chip {
+  font-family: 'Space Mono', monospace;
+  font-size: 0.7rem;
+  padding: 4px 10px;
+  border-radius: 5px;
+  border: 1.5px solid var(--input-bdr);
+  background: transparent;
+  color: var(--text-muted);
+  cursor: pointer;
+  transition: all 0.15s;
+}
+.chip:hover { border-color: var(--accent); color: var(--accent); }
+.chip.active { background: var(--accent); border-color: var(--accent); color: #fff; }
+
+/* ── Numeric threshold row ── */
+.numeric-row {
+  display: grid;
+  grid-template-columns: 1fr 1fr 1fr;
+  gap: 7px;
+}
+.num-group label {
+  display: block;
+  font-family: 'Space Mono', monospace;
+  font-size: 0.6rem;
+  color: var(--text-muted);
+  letter-spacing: 0.06em;
+  margin-bottom: 4px;
+}
+.num-group input { padding: 6px 8px; font-size: 0.72rem; }
+
+/* ── Journal / Publisher filter rows ── */
+.filter-input-row {
+  display: flex;
+  gap: 5px;
+  margin-bottom: 5px;
+  align-items: center;
+}
+.filter-input-row input {
+  flex: 1;
+  padding: 6px 9px;
+  font-size: 0.74rem;
+}
+.filter-add-btn {
+  font-family: 'Space Mono', monospace;
+  font-size: 0.68rem;
+  padding: 4px 10px;
+  border-radius: 5px;
+  cursor: pointer;
+  border: 1.5px solid var(--accent);
+  background: transparent;
+  color: var(--accent);
+  transition: all 0.15s;
+  margin-bottom: 2px;
+  width: auto;
+}
+.filter-add-btn:hover { background: var(--accent); color: #fff; }

Code File/templates/data_view.html

@@ -0,0 +1,248 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <title>Scrapy by PARVEJ - Live Data</title>
+  <link rel="stylesheet" href="{{ url_for('static', filename='style.css') }}">
+  <style>
+    .page-data { max-width: 100%; padding: 24px; height: 100vh; display: flex; flex-direction: column; }
+    body { justify-content: flex-start; margin: 0; height: 100vh; overflow: hidden; }
+    
+    /* Make top buttons and text white */
+    .top-text-white { color: #ffffff !important; }
+    .btn-ghost-white { background: transparent; color: #ffffff !important; border-color: rgba(255,255,255,0.4); }
+    .btn-ghost-white:hover { background: rgba(255,255,255,0.1); border-color: #ffffff; }
+
+    /* Increase table container height */
+    .win-panel-flex { flex: 1; display: flex; flex-direction: column; min-height: 0; margin-top: 0; }
+    .tbl-container { flex: 1; overflow-y: auto; max-height: none; }
+
+    /* Sortable headers */
+    th { cursor: pointer; user-select: none; position: relative; }
+    th:hover { background: rgba(0,0,0,0.1); }
+    th::after { content: '\2195'; margin-left: 5px; opacity: 0.3; font-size: 0.8em; }
+    th.sort-asc::after { content: '\2191'; opacity: 1; }
+    th.sort-desc::after { content: '\2193'; opacity: 1; }
+
+    /* Tooltip */
+    #tooltip {
+      position: fixed;
+      background: #E7E2D9;
+      color: #000000;
+      padding: 16px;
+      border-radius: 6px;
+      border: 1px solid rgba(0,0,0,0.1);
+      font-family: 'Space Mono', monospace;
+      font-size: 0.75rem;
+      width: max-content;
+      max-width: 800px;
+      pointer-events: none;
+      opacity: 0;
+      transition: opacity 0.2s;
+      z-index: 9999;
+      box-shadow: 0 4px 12px rgba(0,0,0,0.15);
+      line-height: 1.5;
+    }
+    #tooltip strong { color: #000000; font-weight: bold; font-size: 0.8rem; }
+    #tooltip.visible { opacity: 1; }
+  </style>
+</head>
+<body>
+<div class="app-shell page-data">
+
+  <div class="app-topbar">
+    <div class="logo">
+      <div class="dot-row">
+        <span class="dot dot-red"></span>
+        <span class="dot dot-amber"></span>
+        <span class="dot dot-green"></span>
+      </div>
+      <h1>SCHOLAR SCRAPER // LIVE DATA VIEW</h1>
+    </div>
+    <div style="display:flex;gap:10px;align-items:center;">
+      <span class="badge top-text-white" id="rowCount" style="background:transparent; border:none; font-size:0.8rem;">0 rows</span>
+      <a href="/" class="btn btn-ghost-white" style="width:auto;margin:0;padding:6px 14px;">&#8592; BACK</a>
+      <a href="/download" class="btn btn-ghost-white" style="width:auto;margin:0;padding:6px 14px;">&#11015; CSV</a>
+    </div>
+  </div>
+
+  <div class="win-panel win-panel-flex" style="padding-top: 0; overflow: hidden;">
+    <div class="tbl-container">
+      <table id="dataTable">
+        <thead>
+          <tr>
+            <th onclick="sortTable(0, 'num')">#</th>
+            <th onclick="sortTable(1, 'str')">TITLE</th>
+            <th onclick="sortTable(2, 'num')">YEAR</th>
+            <th onclick="sortTable(3, 'str')">JOURNAL</th>
+            <th onclick="sortTable(4, 'str')">QUARTILE</th>
+            <th onclick="sortTable(5, 'num')">H-INDEX</th>
+            <th onclick="sortTable(6, 'num')">SJR</th>
+            <th onclick="sortTable(7, 'num')">CITATIONS</th>
+            <th onclick="sortTable(8, 'str')">PUBLISHER</th>
+            <th onclick="sortTable(9, 'str')">OPEN ACCESS</th>
+            <th onclick="sortTable(10, 'str')">COUNTRY</th>
+            <th>LINK</th>
+          </tr>
+        </thead>
+        <tbody id="tbody"></tbody>
+      </table>
+    </div>
+  </div>
+
+</div>
+
+<!-- Tooltip Element -->
+<div id="tooltip"></div>
+
+<script>
+let currentData = [];
+let sortCol = -1;
+let sortAsc = true;
+
+function getBadge(q) {
+  if (!q || q === 'N/A') return '<span class="badge-na">N/A</span>';
+  if (q === 'Q1') return '<span class="badge-q1">Q1</span>';
+  if (q === 'Q2') return '<span class="badge-q2">Q2</span>';
+  if (q === 'Q3') return '<span class="badge-q3">Q3</span>';
+  if (q === 'Q4') return '<span class="badge-q4">Q4</span>';
+  return '<span class="badge-na">' + q + '</span>';
+}
+
+function renderTable() {
+  const tbody = document.getElementById('tbody');
+  tbody.innerHTML = '';
+  document.getElementById('rowCount').textContent = currentData.length + ' rows';
+  
+  currentData.forEach((row, idx) => {
+    // Escape quotes for HTML attributes safely
+    const titleEsc = (row.Title||'N/A').replace(/"/g, '&quot;');
+    const abstractEsc = (row['Abstract/Snippet']||'No abstract').replace(/"/g, '&quot;');
+    
+    const link = row['Paper Link'] && row['Paper Link'] !== 'N/A'
+      ? '<a href="' + row['Paper Link'] + '" target="_blank" style="color:#3d5a99;">&#8599;</a>' : '—';
+      
+    tbody.innerHTML += '<tr>'
+      + '<td>' + (row.originalIndex) + '</td>'
+      + '<td class="title-cell" data-fulltitle="' + titleEsc + '" data-abstract="' + abstractEsc + '">' 
+      + (row.Title||'N/A').substring(0,60) + '…</td>'
+      + '<td>' + (row.Year||'N/A') + '</td>'
+      + '<td title="' + (row['Journal/Venue (Scholar)']||'').replace(/"/g, '&quot;') + '">' + (row['Journal/Venue (Scholar)']||'N/A').substring(0,28) + '</td>'
+      + '<td>' + getBadge(row['Quartile (SCImago)']) + '</td>'
+      + '<td>' + (row['H-index (SCImago)']||'N/A') + '</td>'
+      + '<td>' + (row['SJR Score (SCImago)']||'N/A') + '</td>'
+      + '<td>' + (row['Citations Count']||'0') + '</td>'
+      + '<td title="' + (row.Publisher||'').replace(/"/g, '&quot;') + '">' + (row.Publisher||'N/A').substring(0,22) + '</td>'
+      + '<td>' + (row['Open Access']||'N/A') + '</td>'
+      + '<td>' + (row['Country (SCImago)']||'N/A') + '</td>'
+      + '<td>' + link + '</td>'
+      + '</tr>';
+  });
+  
+  attachTooltipEvents();
+}
+
+async function fetchData() {
+  const res = await fetch('/data');
+  let data = await res.json();
+  
+  // Assign original indices so sorting doesn't lose the '#' number
+  data.forEach((row, i) => { row.originalIndex = i + 1; });
+  
+  currentData = data;
+  if (sortCol !== -1) {
+    applySort();
+  } else {
+    renderTable();
+  }
+}
+
+// ── Sorting ────────────────────────────────────────────────────────────────
+const sortKeys = [
+  'originalIndex', 'Title', 'Year', 'Journal/Venue (Scholar)', 'Quartile (SCImago)', 
+  'H-index (SCImago)', 'SJR Score (SCImago)', 'Citations Count', 'Publisher', 'Open Access', 'Country (SCImago)'
+];
+
+function sortTable(colIndex, type) {
+  if (sortCol === colIndex) {
+    sortAsc = !sortAsc;
+  } else {
+    sortCol = colIndex;
+    sortAsc = true;
+  }
+  
+  // Update header arrows
+  document.querySelectorAll('th').forEach(th => th.classList.remove('sort-asc', 'sort-desc'));
+  const activeTh = document.querySelectorAll('th')[colIndex];
+  activeTh.classList.add(sortAsc ? 'sort-asc' : 'sort-desc');
+  
+  applySort();
+}
+
+function applySort() {
+  const key = sortKeys[sortCol];
+  currentData.sort((a, b) => {
+    let valA = a[key] || '';
+    let valB = b[key] || '';
+    
+    // Convert to numbers if possible for numeric columns
+    if (sortCol === 0 || sortCol === 2 || sortCol === 5 || sortCol === 6 || sortCol === 7) {
+      valA = parseFloat(String(valA).replace(',', '.')) || 0;
+      valB = parseFloat(String(valB).replace(',', '.')) || 0;
+      return sortAsc ? valA - valB : valB - valA;
+    }
+    
+    // String sort
+    valA = String(valA).toLowerCase();
+    valB = String(valB).toLowerCase();
+    if (valA < valB) return sortAsc ? -1 : 1;
+    if (valA > valB) return sortAsc ? 1 : -1;
+    return 0;
+  });
+  renderTable();
+}
+
+// ── Tooltip ────────────────────────────────────────────────────────────────
+function attachTooltipEvents() {
+  const tooltip = document.getElementById('tooltip');
+  
+  document.querySelectorAll('.title-cell').forEach(cell => {
+    cell.addEventListener('mouseenter', (e) => {
+      const t = e.target.dataset.fulltitle;
+      const a = e.target.dataset.abstract;
+      tooltip.innerHTML = '<strong>' + t + '</strong><br><br>' + a;
+      tooltip.classList.add('visible');
+    });
+    
+    cell.addEventListener('mousemove', (e) => {
+      const box = tooltip.getBoundingClientRect();
+      let left = e.clientX + 15;
+      let top = e.clientY + 15;
+      
+      // Prevent tooltip from overflowing the right of the screen
+      if (left + box.width > window.innerWidth) {
+        left = e.clientX - box.width - 15;
+        if (left < 15) left = 15;
+      }
+      
+      // Prevent tooltip from overflowing the bottom of the screen
+      if (top + box.height > window.innerHeight) {
+        top = e.clientY - box.height - 15;
+        if (top < 15) top = 15;
+      }
+      
+      tooltip.style.left = left + 'px';
+      tooltip.style.top = top + 'px';
+    });
+    
+    cell.addEventListener('mouseleave', () => {
+      tooltip.classList.remove('visible');
+    });
+  });
+}
+
+fetchData();
+setInterval(fetchData, 5000);
+</script>
+</body>
+</html>

Code File/templates/index.html

@@ -0,0 +1,156 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <title>Scrapy by PARVEJ</title>
+  <link rel="stylesheet" href="{{ url_for('static', filename='style.css') }}">
+</head>
+<body>
+<div class="app-shell">
+
+  <!-- Top Bar -->
+  <div class="app-topbar">
+    <div class="logo">
+      <div class="dot-row">
+        <span class="dot dot-red"></span>
+        <span class="dot dot-amber"></span>
+        <span class="dot dot-green"></span>
+      </div>
+      <h1>SCHOLAR SCRAPER // DASHBOARD</h1>
+    </div>
+    <div class="live-status">
+      <span class="live-dot" id="liveDot"></span>
+      <span id="liveText">IP: IDLE — Not connected</span>
+    </div>
+  </div>
+
+  <!-- Two-column body -->
+  <div class="app-body">
+
+    <!-- ══ LEFT: Configuration Panel ══════════════════════════════════════ -->
+    <div class="win-panel">
+      <div class="win-title"><span>&#9881;</span> CONFIGURATION</div>
+      <div class="win-body scrollable">
+
+        <!-- ── 1. Query Builder ── -->
+        <div class="cfg-label">SEARCH QUERY BUILDER</div>
+        <div class="qb-wrap" id="queryBuilder">
+          <div class="qb-row" id="qbMainRow">
+            <span class="qb-badge must">MUST</span>
+            <input class="qb-input" id="qbMain" type="text"
+              placeholder="e.g. riverbank erosion  (comma = OR)" oninput="updatePreview()">
+          </div>
+          <div id="qbConditions"></div>
+          <div class="qb-add-row">
+            <button class="qb-add-btn and" onclick="addCondition('AND')">+ AND</button>
+            <button class="qb-add-btn or"  onclick="addCondition('OR')">+ OR</button>
+          </div>
+          <div class="qb-preview" id="qbPreview">Query will appear here…</div>
+        </div>
+
+        <div class="divider"></div>
+
+        <!-- ── 2. Limits row (Max · Year From · Year To · Open Access) ── -->
+        <div class="limits-row">
+          <div class="lim-group">
+            <label>MAX</label>
+            <input type="number" id="max_results" placeholder="unlimit">
+          </div>
+          <div class="lim-group">
+            <label>FROM</label>
+            <input type="number" id="year_low" placeholder="Year">
+          </div>
+          <div class="lim-group">
+            <label>TO</label>
+            <input type="number" id="year_high" placeholder="Year">
+          </div>
+          <div class="lim-group oa-group">
+            <label>OA</label>
+            <label class="toggle-wrap">
+              <input type="checkbox" id="open_access">
+              <span class="toggle-slider"></span>
+            </label>
+          </div>
+        </div>
+
+        <div class="divider"></div>
+
+        <!-- ── 3. Quality Filters ── -->
+
+        <!-- Quartile chips -->
+        <div class="filter-row-label">Quartile (select any)</div>
+        <div class="chip-row" id="quartileChips">
+          <button class="chip" data-value="Q1" onclick="toggleChip(this)">Q1</button>
+          <button class="chip" data-value="Q2" onclick="toggleChip(this)">Q2</button>
+          <button class="chip" data-value="Q3" onclick="toggleChip(this)">Q3</button>
+          <button class="chip" data-value="Q4" onclick="toggleChip(this)">Q4</button>
+          <button class="chip" data-value="N/A" onclick="toggleChip(this)">N/A</button>
+        </div>
+
+        <!-- Numeric threshold row -->
+        <div class="numeric-row">
+          <div class="num-group">
+            <label>H-INDEX &#x2265;</label>
+            <input type="number" id="min_h_index" placeholder="e.g. 50" min="0">
+          </div>
+          <div class="num-group">
+            <label>SJR &#x2265;</label>
+            <input type="number" id="min_sjr" placeholder="e.g. 0.5" min="0" step="0.01">
+          </div>
+          <div class="num-group">
+            <label>CITATIONS &#x2265;</label>
+            <input type="number" id="min_citations" placeholder="e.g. 10" min="0">
+          </div>
+        </div>
+
+        <div class="divider"></div>
+
+        <!-- ── 4. Journal / Publisher Filter ── -->
+        <div id="journalFields"></div>
+        <div id="publisherFields"></div>
+        <div style="display: flex; gap: 8px; margin-bottom: 8px;">
+          <button class="filter-add-btn" onclick="addFilterField('journal')">+ Add Journal</button>
+          <button class="filter-add-btn" onclick="addFilterField('publisher')">+ Add Publisher</button>
+        </div>
+
+        <div class="divider"></div>
+
+        <!-- ── 6. Action buttons (one row) ── -->
+        <div class="btn-row">
+          <button id="startBtn" class="btn btn-primary" onclick="handleMainButton()">&#9654; RUN</button>
+          <a href="/preview" target="_blank" class="btn btn-secondary btn-icon" title="View Live Data Table">&#9776;</a>
+          <a href="/download" class="btn btn-ghost btn-icon" title="Download CSV">&#11015;</a>
+        </div>
+
+        <div class="status-bar">
+          <span class="status-dot" id="statusDot"></span>
+          <span id="statusText">Idle — ready to run</span>
+        </div>
+
+      </div><!-- /win-body -->
+    </div><!-- /left panel -->
+
+    <!-- ══ RIGHT: Terminal Panel ══════════════════════════════════════════ -->
+    <div class="win-panel terminal-panel">
+      <div class="win-title"><span>&#9654;</span> LIVE OUTPUT TERMINAL</div>
+      <div class="terminal-wrap">
+        <div class="terminal-path" style="display:none;"></div>
+        <div class="terminal" id="terminal"></div>
+        <div class="terminal-footer">
+          <span class="term-prompt">scholar@scraper:~$</span>
+          <span id="termStatus">IDLE</span>
+          <span style="margin-left:auto;" id="paperCount"></span>
+        </div>
+      </div>
+    </div>
+
+  </div><!-- /app-body -->
+</div><!-- /app-shell -->
+
+<script src="{{ url_for('static', filename='script.js') }}"></script>
+</body>
+</html>
+
+
+

Code File/test_crossref.py

@@ -0,0 +1,15 @@
+import requests
+import json
+import re
+
+doi = "10.1080/24749508.2025.2451450"
+resp = requests.get(f"https://api.crossref.org/works/{doi}")
+if resp.status_code == 200:
+    item = resp.json().get("message", {})
+    abstract = item.get("abstract", "")
+    print("RAW ABSTRACT:")
+    print(abstract)
+    print("\nCLEAN ABSTRACT:")
+    print(re.sub(r'<[^>]+>', '', abstract).strip())
+else:
+    print("Failed")

Code File/test_data.py

@@ -0,0 +1,6 @@
+import urllib.request
+try:
+    with urllib.request.urlopen('http://127.0.0.1:5000/data') as response:
+        print(len(response.read()))
+except Exception as e:
+    print(e)

Code File/test_newnym.py

@@ -0,0 +1,12 @@
+import socket
+socket.setdefaulttimeout(5)
+import stem.control
+try:
+    print("Connecting...")
+    with stem.control.Controller.from_port(port=9151) as controller:
+        controller.authenticate()
+        print("Connected! Sending NEWNYM...")
+        controller.signal(stem.Signal.NEWNYM)
+        print("Signal sent!")
+except Exception as e:
+    print("Error:", e)

Code File/test_openalex.py

@@ -0,0 +1,25 @@
+import requests
+import json
+
+title = "Implications of climate variability and changing seasonal hydrology for subarctic riverbank erosion"
+url = f"https://api.openalex.org/works?search={title}"
+resp = requests.get(url)
+if resp.status_code == 200:
+    data = resp.json()
+    if data.get('results'):
+        item = data['results'][0]
+        inv = item.get('abstract_inverted_index')
+        if inv:
+            # Reconstruct abstract
+            max_idx = max([max(pos) for pos in inv.values()])
+            words = [""] * (max_idx + 1)
+            for word, positions in inv.items():
+                for pos in positions:
+                    words[pos] = word
+            print(" ".join(words))
+        else:
+            print("No abstract in OpenAlex")
+    else:
+        print("No results in OpenAlex")
+else:
+    print("Failed")

Code File/test_path.py

@@ -0,0 +1,4 @@
+import os
+BASE_DIR = os.path.dirname(os.path.abspath('app.py'))
+# Note app.py is in 'Code File', BASE_DIR is one level up
+print("Current folder:", os.getcwd())

Code File/test_req.py

@@ -0,0 +1,11 @@
+import json
+import urllib.request
+
+url = 'http://127.0.0.1:5000/start-scrape'
+data = {'query': 'test erosion', 'max_results': '3'}
+req = urllib.request.Request(url, data=json.dumps(data).encode('utf-8'), headers={'Content-Type': 'application/json'})
+try:
+    with urllib.request.urlopen(req) as response:
+        print(response.read().decode())
+except Exception as e:
+    print(e)

Code File/test_s2.py

@@ -0,0 +1,10 @@
+import requests
+import json
+
+doi = "10.1080/24749508.2025.2451450"
+url = f"https://api.semanticscholar.org/graph/v1/paper/DOI:{doi}?fields=abstract"
+resp = requests.get(url)
+if resp.status_code == 200:
+    print(resp.json())
+else:
+    print(f"Failed: {resp.status_code} {resp.text}")

Code File/test_s2_func.py

@@ -0,0 +1,14 @@
+import requests
+
+def get_full_abstract(doi):
+    if doi == "N/A": return None
+    try:
+        url = f"https://api.semanticscholar.org/graph/v1/paper/DOI:{doi}?fields=abstract"
+        resp = requests.get(url, timeout=5)
+        if resp.status_code == 200:
+            return resp.json().get('abstract')
+    except Exception:
+        pass
+    return None
+
+print(get_full_abstract("10.1080/24749508.2025.2451450"))

Code File/test_scimago.py

@@ -0,0 +1,43 @@
+import csv
+import re
+import sys
+sys.stdout.reconfigure(encoding='utf-8')
+
+scimago_data = {}
+with open(r'd:\Website\Life OS\Oppurtunity\Google Scholar Data\Assets\scimagojr 2025.csv', 'r', encoding='utf-8-sig') as f:
+    reader = csv.DictReader(f, delimiter=';')
+    for row in reader:
+        title = row['Title'].strip().lower()
+        scimago_data[title] = {
+            'Title': row['Title'],
+            'Quartile': row.get('SJR Best Quartile', ''),
+            'H-index': row.get('H index', ''),
+            'SJR': row.get('SJR', ''),
+            'Categories': row.get('Categories', ''),
+            'Areas': row.get('Areas', ''),
+            'Country': row.get('Country', '')
+        }
+
+with open(r'd:\Website\Life OS\Oppurtunity\Google Scholar Data\river_erosion_papers.csv', 'r', encoding='utf-8') as f:
+    reader = csv.DictReader(f)
+    for row in reader:
+        journal_scholar = row.get('Journal/Venue (Scholar)', '').strip()
+        search_name = journal_scholar.strip('… ').lower()
+        
+        match = scimago_data.get(search_name)
+        
+        if not match:
+            for sjr_title, sjr_info in scimago_data.items():
+                if search_name == sjr_title or (len(search_name) > 8 and search_name in sjr_title):
+                    match = sjr_info
+                    break
+                    
+        print(f"Paper: {row.get('Title', '')[:40]}...")
+        print(f"Journal from Scholar: {journal_scholar}")
+        if match:
+            print(f"  [MATCHED AS]: {match['Title']}")
+            print(f"  -> Quartile: {match['Quartile']} | H-index: {match['H-index']} | SJR: {match['SJR']} | Country: {match['Country']}")
+            print(f"  -> Areas: {match['Areas']}")
+        else:
+            print("  [NOT FOUND] in SCImago")
+        print('-'*60)

Code File/test_stem.py

@@ -0,0 +1,9 @@
+import socket
+socket.setdefaulttimeout(5)
+import stem.control
+try:
+    with stem.control.Controller.from_port(port=9151) as controller:
+        controller.authenticate()
+        print("Connected!")
+except Exception as e:
+    print("Error:", e)

Code File/test_tags.py

@@ -0,0 +1,15 @@
+import requests
+proxies = {'http': 'socks5://127.0.0.1:9150', 'https': 'socks5://127.0.0.1:9150'}
+headers = {
+    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.0.0 Safari/537.36',
+    'accept-language': 'en-US,en',
+    'accept': 'text/html,application/xhtml+xml,application/xml'
+}
+try:
+    r = requests.get('https://scholar.google.com/scholar?q=urban+AND+%22green+space%22', proxies=proxies, headers=headers, timeout=10)
+    print("gs_res_ccl in text:", "gs_res_ccl" in r.text)
+    print("gs_top in text:", "gs_top" in r.text)
+    print("gs_ab_rt in text:", "gs_ab_rt" in r.text)
+    print("gs_nori in text:", "gs_nori" in r.text) # "Did not match any articles" container
+except Exception as e:
+    print("Failed:", e)

Data Folder/.gitkeep

@@ -0,0 +1 @@
+# Keep this folder in the repository structure. Scraped CSV files are excluded via .gitignore.

README.md

@@ -0,0 +1,95 @@
+# Google Scholar Scraper & Quality Filtering Dashboard
+
+An enterprise-grade, web-based dashboard and automation engine to search, scrape, and filter Google Scholar articles. It features robust integration with the **Tor Network** for self-healing SOCKS5 proxy rotation, automatically bypassing Google's anti-bot blocks and pagination limits.
+
+## 🚀 Key Features
+
+*   **Advanced Query Builder**: Build complex boolean queries (supporting `MUST`, `AND`, `OR` operators) directly from the visual interface.
+*   **Tor Circuit Rotation**: Automatically rotates Tor exit nodes using the Tor control port (`9151`) on detecting soft-blocks or captcha pages.
+*   **Resilient Pagination**: Syncs search offset states (`start_index`) to resume scraping seamlessly exactly where it was interrupted after an IP rotation.
+*   **Duplicate Elimination**: Automatically filters overlapping search results in real-time, ensuring unique outputs in final CSV records.
+*   **Academic Quality Filters**:
+    *   Filter by **SCImago Journal Rank (SJR)**, Quartile rankings (`Q1` to `Q4`), and H-index via local database matching.
+    *   Filter by minimum citation counts.
+    *   Filter by Open Access status.
+    *   Whitelist specific journals or publishers.
+*   **Live Output Terminal**: Simulates a live CLI output stream directly in the web browser using Server-Sent Events (SSE).
+*   **Live Data Table**: Sortable tabular view of scraped papers with interactive abstract hover tooltips.
+
+---
+
+## 🛠️ System Architecture
+
+```mermaid
+graph TD
+    A[Web UI Dashboard] -->|REST API & SSE /stream| B[Flask Backend]
+    B -->|Threaded Runner| C[Scraper Engine]
+    C -->|Construct Query| D[scholarly API]
+    D -->|SOCKS5 Proxy 9150| E[Tor Browser Network]
+    E -->|Fetch Page| F[Google Scholar]
+    F -->|Soft-Block / Captcha| C
+    C -->|NEWNYM Control Signal 9151| E
+    C -->|Lookup metadata| G[SCImago Local Database]
+    C -->|Fetch DOIs & Abstracts| H[Crossref / Semantic Scholar APIs]
+    C -->|Save clean results| I[CSV Export]
+```
+
+---
+
+## 📋 Installation & Setup
+
+### Prerequisites
+1.  **Python 3.10+** (Ensure it is added to your environment `PATH`).
+2.  **Tor Browser**: Must be installed and running in the background to handle proxy routing and circuit rotation.
+
+### Installation Steps
+
+1.  **Clone the repository** (or navigate to the project directory):
+    ```bash
+    cd "Google Scholar Data"
+    ```
+
+2.  **Verify or set up the Virtual Environment**:
+    If not already done, create a virtual environment:
+    ```bash
+    python -m venv .venv
+    ```
+
+3.  **Install Locked Dependencies**:
+    Install all required packages precisely using the pinned `requirements.txt`:
+    ```bash
+    .\.venv\Scripts\pip install -r requirements.txt
+    ```
+
+---
+
+## 🚦 How to Run
+
+1.  **Launch Tor Browser**:
+    *   Open the Tor Browser on your system.
+    *   Keep it running in the background. (This opens the SOCKS5 proxy on port `9150` and the control port on `9151`).
+
+2.  **Start the Scraper Dashboard**:
+    Double-click the **`Start Scraper.bat`** file in the root directory. This script will:
+    *   Kill any ghost servers on port `5000`.
+    *   Start the Flask backend in a minimized background window.
+    *   Automatically launch `http://127.0.0.1:5000` in your default browser.
+
+3.  **Configure and Run**:
+    *   Enter your search keywords in the configuration panel.
+    *   Set results limits, year ranges, or quality criteria.
+    *   Click **▶ RUN**. Watch the live terminal extract and verify papers in real-time.
+
+---
+
+## 📁 Repository Structure
+
+*   `Code File/` - Contains the Flask backend and core scraping scripts:
+    *   [app.py](file:///d:/Website/Life%20OS/Oppurtunity/Google%20Scholar%20Data/Code%20File/app.py) - Web server routes, SSE streaming handler, and UI controls.
+    *   [scholar_scraper.py](file:///d:/Website/Life%20OS/Oppurtunity/Google%20Scholar%20Data/Code%20File/scholar_scraper.py) - Core scraping logic, SCImago database matching, Tor IP rotation, and Crossref API metadata pipeline.
+    *   `templates/` - Frontend views ([index.html](file:///d:/Website/Life%20OS/Oppurtunity/Google%20Scholar%20Data/Code%20File/templates/index.html) and [data_view.html](file:///d:/Website/Life%20OS/Oppurtunity/Google%20Scholar%20Data/Code%20File/templates/data_view.html)).
+    *   `static/` - UI CSS styles and client-side JavaScript controllers.
+*   `Data Folder/` - Stores all downloaded and scraped CSV output results.
+*   `Assets/` - Local JSON metadata lists (e.g. `scimagojr_2025.json`).
+*   `requirements.txt` - Python package pin definitions ensuring system compatibility.
+*   `Start Scraper.bat` - Windows launcher script.

Start Scraper.bat

@@ -0,0 +1,23 @@
+@echo off
+echo ========================================================
+echo         SCHOLAR SCRAPER - DASHBOARD LAUNCHER
+echo ========================================================
+echo.
+
+echo [1/3] Checking for and closing any old ghost servers...
+for /f "tokens=5" %%a in ('netstat -aon ^| findstr :5000') do taskkill /F /PID %%a >nul 2>&1
+
+echo [2/3] Starting the Flask server...
+cd "Code File"
+:: Start the server in a new minimized command prompt window so it doesn't block
+start "Scholar Scraper Server" /MIN "..\.venv\Scripts\python.exe" app.py
+
+echo [3/3] Waiting for server to initialize...
+timeout /t 3 /nobreak >nul
+
+echo Opening dashboard in your default browser...
+start http://127.0.0.1:5000
+
+echo.
+echo Done! You can close this black window. The server will keep running in the minimized window.
+timeout /t 3 >nul

requirements.txt

@@ -0,0 +1,15 @@
+# ============================================================
+# Scholar Scraper — Pinned Dependencies
+# ============================================================
+# IMPORTANT: free-proxy MUST stay at 1.0.6
+# free-proxy >= 1.1.0 changed get_proxy_list() to require a
+# mandatory `repeat` argument, breaking scholarly 1.7.11.
+# DO NOT upgrade free-proxy without testing first.
+# ============================================================
+
+scholarly==1.7.11
+free_proxy==1.0.6      # PINNED — do not upgrade
+stem==1.8.2
+httpx==0.27.2
+requests==2.34.2
+flask>=2.0