2022-06-02 19:14:49
[Script] Scraping ResearchGate all Publications
```python
from parsel import Selector
from playwright.sync_api import sync_playwright
import json
def scrape_researchgate_publications(query: str):
with sync_playwright() as p:
browser = p.chromium.launch(headless=True, slow_mo=50)
page = browser.new_page(user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.64 Safari/537.36")
publications = []
page_num = 1
while True:
page.goto(f"https://www.researchgate.net/search/publication?q={query}&page={page_num}")
selector = Selector(text=page.content())
for publication in selector.css(".nova-legacy-c-card__body--spacing-inherit"):
title = publication.css(".nova-legacy-v-publication-item__title .nova-legacy-e-link--theme-bare::text").get().title()
title_link = f'https://www.researchgate.net{publication.css(".nova-legacy-v-publication-item__title .nova-legacy-e-link--theme-bare::attr(href)").get()}'
publication_type = publication.css(".nova-legacy-v-publication-item__badge::text").get()
publication_date = publication.css(".nova-legacy-v-publication-item__meta-data-item:nth-child(1) span::text").get()
publication_doi = publication.css(".nova-legacy-v-publication-item__meta-data-item:nth-child(2) span").xpath("normalize-space()").get()
publication_isbn = publication.css(".nova-legacy-v-publication-item__meta-data-item:nth-child(3) span").xpath("normalize-space()").get()
authors = publication.css(".nova-legacy-v-person-inline-item__fullname::text").getall()
source_link = f'https://www.researchgate.net{publication.css(".nova-legacy-v-publication-item__preview-source .nova-legacy-e-link--theme-bare::attr(href)").get()}'
publications.append({
"title": title,
"link": title_link,
"source_link": source_link,
"publication_type": publication_type,
"publication_date": publication_date,
"publication_doi": publication_doi,
"publication_isbn": publication_isbn,
"authors": authors
})
print(f"page number: {page_num}")
# checks if next page arrow key is greyed out `attr(rel)` (inactive) and breaks out of the loop
if selector.css(".nova-legacy-c-button-group__item:nth-child(9) a::attr(rel)").get():
break
else:
page_num += 1
print(json.dumps(publications, indent=2, ensure_ascii=False))
browser.close()
scrape_researchgate_publications(query="coffee")
```
Outputs:
```json
[
{
"title":"The Social Life Of Coffee Turkey’S Local Coffees",
"link":"https://www.researchgate.netpublication/360540595_The_Social_Life_of_Coffee_Turkey%27s_Local_Coffees?_sg=kzuAi6HlFbSbnLEwtGr3BA_eiFtDIe1VEA4uvJlkBHOcbSjh5XlSQe6GpYvrbi12M0Z2MQ6grwnq9fI",
"source_link":"https://www.researchgate.netpublication/360540595_The_Social_Life_of_Coffee_Turkey%27s_Local_Coffees?_sg=kzuAi6HlFbSbnLEwtGr3BA_eiFtDIe1VEA4uvJlkBHOcbSjh5XlSQe6GpYvrbi12M0Z2MQ6grwnq9fI",
"publication_type":"Conference Paper",
"publication_date":"Apr 2022",
"publication_doi":null,
"publication_isbn":null,
"authors":[
"Gülşen Berat Torusdağ",
"Merve Uçkan Çakır",
"Cinucen Okat"
]
},
{
"title":"Coffee With The Algorithm",
"link":"https://www.researchgate.netpublication/359599064_Coffee_with_the_Algorithm?_sg=3KHP4SXHm_BSCowhgsa4a2B0xmiOUMyuHX2nfqVwRilnvd1grx55EWuJqO0VzbtuG-16TpsDTUywp0o",
"source_link":"https://www.researchgate.netNone",
"publication_type":"Chapter",
"publication_date":"Mar 2022",
"publication_doi":"DOI: 10.4324/9781003170884-10",
"publication_isbn":"ISBN: 9781003170884",
45 views16:14
D
