490 lines
22 KiB
Python
490 lines
22 KiB
Python
from scholarly import scholarly, ProxyGenerator
|
|
from scholarly._navigator import Navigator
|
|
import time
|
|
import csv
|
|
import stem.control
|
|
import stem
|
|
import re
|
|
import requests
|
|
import json
|
|
import os
|
|
|
|
LOG_CALLBACK = None
|
|
STOP_EVENT = None
|
|
|
|
def my_print(*args, **kwargs):
|
|
msg = ' '.join(map(str, args))
|
|
if LOG_CALLBACK:
|
|
LOG_CALLBACK(msg)
|
|
print(*args, **kwargs)
|
|
|
|
# ── Paths ──────────────────────────────────────────────────────────────────────
|
|
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
scimago_path = os.path.join(BASE_DIR, "Assets", "scimagojr_2025.json")
|
|
|
|
SCIMAGO_DATA = {}
|
|
if os.path.exists(scimago_path):
|
|
with open(scimago_path, "r", encoding="utf-8") as f:
|
|
SCIMAGO_DATA = json.load(f)
|
|
|
|
# ── Tor helpers ────────────────────────────────────────────────────────────────
|
|
def setup_custom_tor():
|
|
"""
|
|
Directly inject a Tor SOCKS5 session into scholarly's Navigator singleton,
|
|
bypassing scholarly.use_proxy() which spawns an unreliable FreeProxy secondary.
|
|
Verifies the connection can actually reach Google Scholar before proceeding.
|
|
"""
|
|
my_print("Configuring scholarly to use Tor SOCKS proxy...")
|
|
|
|
tor_proxies = {
|
|
'http://': 'socks5://127.0.0.1:9150',
|
|
'https://': 'socks5://127.0.0.1:9150'
|
|
}
|
|
|
|
# ── Build a working proxy session ──────────────────────────────────────────
|
|
pg = ProxyGenerator()
|
|
pg._proxies = tor_proxies
|
|
pg._proxy_works = True
|
|
pg._can_refresh_tor = True
|
|
pg._tor_control_port = 9151
|
|
pg._tor_password = None
|
|
pg._new_session(proxies=tor_proxies)
|
|
|
|
# ── Inject directly into Navigator (skip FreeProxy secondary spawn) ─────────
|
|
nav = Navigator()
|
|
nav.pm1 = pg
|
|
nav.pm2 = pg # use same Tor session as secondary too
|
|
nav._session1 = pg.get_session()
|
|
nav._session2 = pg.get_session()
|
|
|
|
# ── Verify Tor is actually reachable ───────────────────────────────────────
|
|
MAX_VERIFY_ATTEMPTS = 3
|
|
for attempt in range(1, MAX_VERIFY_ATTEMPTS + 1):
|
|
try:
|
|
resp = pg.get_session().get(
|
|
"https://scholar.google.com",
|
|
timeout=15,
|
|
follow_redirects=True
|
|
)
|
|
if resp.status_code == 200:
|
|
my_print(f"Proxy verified successfully! (attempt {attempt})")
|
|
return pg
|
|
elif resp.status_code == 429 or resp.status_code == 503:
|
|
my_print(f"Google Scholar blocked this Tor exit node (HTTP {resp.status_code}). Rotating IP...")
|
|
if rotate_tor_ip():
|
|
pg._new_session(proxies=tor_proxies)
|
|
nav._session1 = pg.get_session()
|
|
nav._session2 = pg.get_session()
|
|
else:
|
|
my_print(f"Unexpected status {resp.status_code} from Scholar. Continuing anyway...")
|
|
return pg
|
|
except Exception as e:
|
|
my_print(f"Tor connectivity check failed (attempt {attempt}/{MAX_VERIFY_ATTEMPTS}): {e}")
|
|
if attempt < MAX_VERIFY_ATTEMPTS:
|
|
my_print("Retrying after IP rotation...")
|
|
rotate_tor_ip()
|
|
pg._new_session(proxies=tor_proxies)
|
|
nav._session1 = pg.get_session()
|
|
nav._session2 = pg.get_session()
|
|
|
|
my_print("WARNING: Could not verify Tor connection to Google Scholar. Proceeding anyway — results may be empty.")
|
|
my_print(" Make sure Tor Browser is open and running on port 9150.")
|
|
return pg
|
|
|
|
|
|
def rotate_tor_ip():
|
|
"""Rotate the Tor circuit via the control port. Returns True on success."""
|
|
my_print("Requesting new Tor IP...")
|
|
try:
|
|
with stem.control.Controller.from_port(port=9151) as controller:
|
|
controller.authenticate()
|
|
controller.signal(stem.Signal.NEWNYM)
|
|
time.sleep(5)
|
|
my_print("Switched to new Tor circuit (New IP assigned).")
|
|
return True
|
|
except Exception as e:
|
|
my_print(f"Failed to rotate IP: {e}")
|
|
my_print("Please ensure Tor Browser is OPEN in the background.")
|
|
return False
|
|
|
|
# ── Crossref metadata ──────────────────────────────────────────────────────────
|
|
def get_extended_metadata(title, url):
|
|
meta = {
|
|
"DOI": "N/A", "Publisher": "N/A", "Open Access": "N/A",
|
|
"Authors (Crossref)": "N/A", "Exact Journal": "N/A"
|
|
}
|
|
doi = None
|
|
doi_pattern = re.compile(r'(10\.\d{4,9}/[-._;()/:a-zA-Z0-9]+)')
|
|
if url:
|
|
match = doi_pattern.search(url)
|
|
if match:
|
|
doi = match.group(1).rstrip('.')
|
|
item = None
|
|
try:
|
|
if doi:
|
|
resp = requests.get(f"https://api.crossref.org/works/{doi}", timeout=5)
|
|
if resp.status_code == 200:
|
|
item = resp.json().get("message", {})
|
|
if not item:
|
|
resp = requests.get("https://api.crossref.org/works",
|
|
params={"query.bibliographic": title, "rows": 1}, timeout=5)
|
|
if resp.status_code == 200:
|
|
items = resp.json().get("message", {}).get("items", [])
|
|
if items and title[:15].lower() in items[0].get("title", [""])[0].lower():
|
|
item = items[0]
|
|
if item:
|
|
meta["DOI"] = item.get("DOI", doi if doi else "N/A")
|
|
meta["Publisher"] = item.get("publisher", "N/A")
|
|
container = item.get("container-title", [])
|
|
meta["Exact Journal"] = container[0] if container else "N/A"
|
|
licenses = item.get("license", [])
|
|
is_oa = any("creativecommons" in l.get("URL","").lower() for l in licenses)
|
|
meta["Open Access"] = "Yes (CC)" if is_oa else "Unknown"
|
|
authors = item.get("author", [])
|
|
a_list = []
|
|
for a in authors:
|
|
name = f"{a.get('given','')} {a.get('family','')}".strip()
|
|
orcid = a.get("ORCID","").replace("http://orcid.org/","")
|
|
affils = [aff.get("name","") for aff in a.get("affiliation",[])]
|
|
affil_str = f" [{', '.join(affils)}]" if affils else ""
|
|
orcid_str = f" (ORCID: {orcid})" if orcid else ""
|
|
a_list.append(f"{name}{orcid_str}{affil_str}")
|
|
meta["Authors (Crossref)"] = "; ".join(a_list) if a_list else "N/A"
|
|
|
|
# Crossref sometimes has abstracts, though they often contain XML JATS tags
|
|
raw_abstract = item.get("abstract")
|
|
if raw_abstract:
|
|
clean_abs = re.sub(r'<[^>]+>', '', raw_abstract).strip()
|
|
if len(clean_abs) > 20:
|
|
meta["Abstract (Crossref)"] = clean_abs
|
|
|
|
except Exception:
|
|
pass
|
|
return meta
|
|
|
|
def get_full_abstract(doi, title=None):
|
|
if doi and doi != "N/A":
|
|
try:
|
|
url = f"https://api.semanticscholar.org/graph/v1/paper/DOI:{doi}?fields=abstract"
|
|
resp = requests.get(url, timeout=5)
|
|
if resp.status_code == 200:
|
|
abs_text = resp.json().get('abstract')
|
|
if abs_text and len(abs_text) > 20:
|
|
return abs_text
|
|
except Exception:
|
|
pass
|
|
|
|
if title:
|
|
try:
|
|
url = f"https://api.semanticscholar.org/graph/v1/paper/search"
|
|
resp = requests.get(url, params={"query": title, "fields": "abstract,title", "limit": 1}, timeout=5)
|
|
if resp.status_code == 200:
|
|
items = resp.json().get('data', [])
|
|
if items:
|
|
found_title = items[0].get('title', '').lower()
|
|
if title[:20].lower() in found_title:
|
|
abs_text = items[0].get('abstract')
|
|
if abs_text and len(abs_text) > 20:
|
|
return abs_text
|
|
except Exception:
|
|
pass
|
|
|
|
# Tertiary fallback: OpenAlex API
|
|
try:
|
|
url = f"https://api.openalex.org/works?search={title}"
|
|
resp = requests.get(url, timeout=5)
|
|
if resp.status_code == 200:
|
|
data = resp.json()
|
|
if data.get('results'):
|
|
item = data['results'][0]
|
|
inv = item.get('abstract_inverted_index')
|
|
if inv:
|
|
max_idx = max([max(pos) for pos in inv.values()])
|
|
words = [""] * (max_idx + 1)
|
|
for word, positions in inv.items():
|
|
for pos in positions:
|
|
words[pos] = word
|
|
abs_text = " ".join(words)
|
|
if len(abs_text) > 20:
|
|
return abs_text
|
|
except Exception:
|
|
pass
|
|
|
|
return None
|
|
|
|
# ── Filter engine ──────────────────────────────────────────────────────────────
|
|
def passes_filters(row_data, filters):
|
|
"""
|
|
Returns (passed: bool, reason: str|None).
|
|
All active filters must pass for the paper to be accepted.
|
|
"""
|
|
if not filters:
|
|
return True, None
|
|
|
|
# Open Access
|
|
if filters.get('open_access'):
|
|
oa = str(row_data.get('Open Access', '')).lower()
|
|
if 'yes' not in oa:
|
|
return False, 'Not open access'
|
|
|
|
# Quartile (list of accepted values, e.g. ['Q1','Q2'])
|
|
quartiles = [q for q in filters.get('quartiles', []) if q]
|
|
if quartiles:
|
|
paper_q = str(row_data.get('Quartile (SCImago)', 'N/A'))
|
|
if paper_q not in quartiles:
|
|
return False, f'Quartile {paper_q} (need {"/".join(quartiles)})'
|
|
|
|
# H-Index >=
|
|
min_h = filters.get('min_h_index')
|
|
if min_h is not None:
|
|
try:
|
|
h = float(str(row_data.get('H-index (SCImago)', 0) or 0).replace(',', '.'))
|
|
if h < float(min_h):
|
|
return False, f'H-Index {h} < {min_h}'
|
|
except Exception:
|
|
return False, 'H-Index N/A'
|
|
|
|
# SJR >=
|
|
min_sjr = filters.get('min_sjr')
|
|
if min_sjr is not None:
|
|
try:
|
|
sjr = float(str(row_data.get('SJR Score (SCImago)', 0) or 0).replace(',', '.'))
|
|
if sjr < float(min_sjr):
|
|
return False, f'SJR {sjr} < {min_sjr}'
|
|
except Exception:
|
|
return False, 'SJR N/A'
|
|
|
|
# Citations >=
|
|
min_cit = filters.get('min_citations')
|
|
if min_cit is not None:
|
|
try:
|
|
cit = int(str(row_data.get('Citations Count', 0) or 0))
|
|
if cit < int(min_cit):
|
|
return False, f'Citations {cit} < {min_cit}'
|
|
except Exception:
|
|
return False, 'Citations N/A'
|
|
|
|
# Journal (any match)
|
|
journals = [j.strip().lower() for j in filters.get('journals', []) if j and j.strip()]
|
|
if journals:
|
|
paper_j = str(row_data.get('Journal/Venue (Scholar)', '') or '').lower()
|
|
if not any(j in paper_j or paper_j in j for j in journals):
|
|
return False, 'Journal not matched'
|
|
|
|
# Publisher (any match)
|
|
publishers = [p.strip().lower() for p in filters.get('publishers', []) if p and p.strip()]
|
|
if publishers:
|
|
paper_p = str(row_data.get('Publisher', '') or '').lower()
|
|
if not any(p in paper_p or paper_p in p for p in publishers):
|
|
return False, 'Publisher not matched'
|
|
|
|
return True, None
|
|
|
|
# ── Main scraper ───────────────────────────────────────────────────────────────
|
|
def search_scholar(query, output_file=None, target_count=20,
|
|
year_low=None, year_high=None,
|
|
stop_event=None, filters=None):
|
|
if output_file is None:
|
|
output_file = os.path.join(BASE_DIR, "Data Folder", "scholar_results.csv")
|
|
|
|
my_print(f"Searching for: {query}")
|
|
if filters:
|
|
my_print(f"Active filters: {filters}")
|
|
|
|
# ── Init Scholar search ──
|
|
search_query = None
|
|
max_retries = 5
|
|
for attempt in range(max_retries):
|
|
if stop_event and stop_event.is_set():
|
|
my_print("[STOPPED] Task cancelled by user.")
|
|
return
|
|
try:
|
|
search_query = scholarly.search_pubs(query, year_low=year_low, year_high=year_high)
|
|
break
|
|
except Exception as e:
|
|
my_print(f"Blocked on initial search (Attempt {attempt+1}/{max_retries}): {e}")
|
|
rotate_tor_ip()
|
|
|
|
if not search_query:
|
|
my_print("Failed to initialize search after multiple retries. Exiting.")
|
|
return
|
|
|
|
headers = [
|
|
"Title", "Year", "Journal/Venue (Scholar)",
|
|
"Quartile (SCImago)", "H-index (SCImago)", "SJR Score (SCImago)",
|
|
"Citations Count", "Abstract/Snippet", "Authors (Scholar)",
|
|
"Publisher", "Open Access", "Categories & Areas (SCImago)",
|
|
"Country (SCImago)", "Paper Link", "PDF Link",
|
|
"DOI", "Authors (Crossref)"
|
|
]
|
|
|
|
# Safeguard: don't crawl more than 50x the target (prevents infinite loop)
|
|
MAX_CRAWL = max(target_count * 50, 500)
|
|
qualified = 0
|
|
crawled = 0
|
|
seen_titles = set()
|
|
|
|
with open(output_file, mode="w", newline="", encoding="utf-8") as file:
|
|
writer = csv.DictWriter(file, fieldnames=headers, extrasaction='ignore')
|
|
writer.writeheader()
|
|
|
|
# ── Pre-flight: detect immediate 0-result (blocked Tor exit node) ────────
|
|
ZERO_RESULT_RETRIES = 3
|
|
for zero_attempt in range(ZERO_RESULT_RETRIES):
|
|
try:
|
|
first_paper = next(search_query)
|
|
# Got at least one result — break and process it below
|
|
break
|
|
except StopIteration:
|
|
if zero_attempt < ZERO_RESULT_RETRIES - 1:
|
|
my_print(f"[WARNING] Query returned 0 results immediately — Tor exit node may be blocked. "
|
|
f"Rotating IP and retrying (attempt {zero_attempt + 1}/{ZERO_RESULT_RETRIES})...")
|
|
rotated = rotate_tor_ip()
|
|
if not rotated:
|
|
my_print("IP rotation failed — Tor Browser may not be running. Cannot retry.")
|
|
break
|
|
# Re-initialise the search iterator with the new exit node
|
|
try:
|
|
search_query = scholarly.search_pubs(query, year_low=year_low, year_high=year_high)
|
|
continue
|
|
except Exception as e:
|
|
my_print(f"Re-search after rotation failed: {e}")
|
|
break
|
|
else:
|
|
my_print("No results found after all retry attempts. The query may have no matching papers, "
|
|
"or all Tor exit nodes are blocked by Google Scholar.")
|
|
my_print("Try opening Tor Browser and running again, or use a VPN/different network.")
|
|
first_paper = None
|
|
break
|
|
except Exception:
|
|
first_paper = None
|
|
break
|
|
else:
|
|
first_paper = None
|
|
|
|
if first_paper is None:
|
|
# Genuinely no results or could not recover
|
|
pass
|
|
else:
|
|
# We have the first paper — inject it back into the processing loop
|
|
# by using an inline generator that yields first_paper then continues search_query
|
|
def _chain(first, rest):
|
|
yield first
|
|
yield from rest
|
|
search_query = _chain(first_paper, search_query)
|
|
|
|
while qualified < target_count and crawled < MAX_CRAWL:
|
|
if stop_event and stop_event.is_set():
|
|
my_print("[STOPPED] Task cancelled by user.")
|
|
break
|
|
try:
|
|
paper = next(search_query)
|
|
crawled += 1
|
|
bib = paper.get('bib', {})
|
|
|
|
title = bib.get('title', 'N/A')
|
|
title_norm = title.strip().lower()
|
|
if title_norm in seen_titles:
|
|
my_print(f" [SKIP #{crawled}] Duplicate title detected (overlapping pages) — \"{title[:55]}\"")
|
|
continue
|
|
seen_titles.add(title_norm)
|
|
|
|
year = bib.get('pub_year', 'N/A')
|
|
venue = bib.get('venue', 'N/A')
|
|
abstract = bib.get('abstract', 'N/A')
|
|
authors = (", ".join(bib.get('author', []))
|
|
if isinstance(bib.get('author'), list)
|
|
else bib.get('author', 'N/A'))
|
|
citations = paper.get('num_citations', 0)
|
|
pub_url = paper.get('pub_url', 'N/A')
|
|
pdf_url = paper.get('eprint_url', 'N/A')
|
|
|
|
# Crossref
|
|
crmeta = get_extended_metadata(title, pub_url)
|
|
exact_journal = crmeta.pop("Exact Journal", "N/A")
|
|
search_name = (exact_journal if exact_journal != "N/A" else venue
|
|
).strip('… .').lower()
|
|
|
|
# SCImago lookup
|
|
scimago_match = SCIMAGO_DATA.get(search_name)
|
|
if not scimago_match:
|
|
for sjr_title, sjr_info in SCIMAGO_DATA.items():
|
|
if search_name == sjr_title or (len(search_name) > 8 and search_name in sjr_title):
|
|
scimago_match = sjr_info
|
|
break
|
|
|
|
sjr_q = scimago_match.get("Quartile", "N/A") if scimago_match else "N/A"
|
|
sjr_h = scimago_match.get("H-index", "N/A") if scimago_match else "N/A"
|
|
sjr_s = scimago_match.get("SJR", "N/A") if scimago_match else "N/A"
|
|
sjr_cat = "N/A"
|
|
if scimago_match:
|
|
cat_val = scimago_match.get("Categories", "")
|
|
area_val = scimago_match.get("Areas", "")
|
|
if cat_val and area_val:
|
|
sjr_cat = f"{cat_val} | {area_val}"
|
|
elif cat_val or area_val:
|
|
sjr_cat = cat_val or area_val
|
|
sjr_c = scimago_match.get("Country", "N/A") if scimago_match else "N/A"
|
|
|
|
row = {
|
|
"Title": title, "Year": year, "Journal/Venue (Scholar)": venue,
|
|
"Abstract/Snippet": abstract, "Authors (Scholar)": authors,
|
|
"Citations Count": citations, "Paper Link": pub_url, "PDF Link": pdf_url,
|
|
"Quartile (SCImago)": sjr_q, "H-index (SCImago)": sjr_h,
|
|
"SJR Score (SCImago)": sjr_s,
|
|
"Categories & Areas (SCImago)": sjr_cat, "Country (SCImago)": sjr_c
|
|
}
|
|
row.update(crmeta)
|
|
|
|
# ── Apply filters ──────────────────────────────────────────────
|
|
passed, reason = passes_filters(row, filters)
|
|
if not passed:
|
|
short_title = title[:55] + ('...' if len(title) > 55 else '')
|
|
my_print(f" [SKIP #{crawled}] {reason} — \"{short_title}\"")
|
|
time.sleep(1)
|
|
continue
|
|
|
|
# ── Qualified paper ─────────────────────────────────────────────
|
|
|
|
# Abstract Fetching Pipeline: Crossref -> Semantic Scholar -> OpenAlex
|
|
cr_abs = crmeta.get("Abstract (Crossref)")
|
|
if cr_abs:
|
|
row["Abstract/Snippet"] = cr_abs
|
|
else:
|
|
full_abstract = get_full_abstract(row.get('DOI'), row.get('Title'))
|
|
if full_abstract and len(full_abstract) > 20:
|
|
row["Abstract/Snippet"] = full_abstract
|
|
|
|
qualified += 1
|
|
my_print(f"[{qualified}/{target_count}] Title: {title}")
|
|
my_print(f" Year: {year} | DOI: {crmeta['DOI']} | Quartile: {sjr_q} | Publisher: {crmeta['Publisher']}")
|
|
|
|
writer.writerow(row)
|
|
file.flush()
|
|
time.sleep(2)
|
|
|
|
except StopIteration:
|
|
my_print("No more results found for this query.")
|
|
break
|
|
except Exception as e:
|
|
if stop_event and stop_event.is_set():
|
|
my_print("[STOPPED] Task cancelled by user.")
|
|
break
|
|
my_print(f"Blocked or error encountered: {e}")
|
|
rotate_tor_ip()
|
|
try:
|
|
my_print(f"Resuming search from index {crawled}...")
|
|
search_query = scholarly.search_pubs(query, year_low=year_low, year_high=year_high, start_index=crawled)
|
|
except Exception as re_err:
|
|
my_print(f"Failed to re-initialize search iterator: {re_err}")
|
|
|
|
my_print(f"Extraction complete! Saved {qualified} results to '{output_file}'")
|
|
my_print(f"(Crawled {crawled} papers total to find {qualified} qualifying papers)")
|
|
|
|
if __name__ == "__main__":
|
|
setup_custom_tor()
|
|
search_term = '"river bank erosion" AND ("GIS" OR "Remote Sensing" OR "DSAS" OR "morphological change")'
|
|
out_path = os.path.join(BASE_DIR, "Data Folder", "river_erosion_papers.csv")
|
|
search_scholar(search_term, output_file=out_path, target_count=50)
|
|
|