preprint_usage.Rmd

---
title: "R Notebook"
---

# Libraries

```{r}

library(tidyverse)
library(lubridate)
library(rvest)
library(janitor)

```

# Load data

```{r}

preprints <- bind_rows(
  read_csv("data/preprints_basic_20190101_20191231.csv"),
  read_csv("data/preprints_basic_20200101_20201031.csv"))

```

# Retrieve usage data for preprints

```{r}

# Scrape the bioRxiv and medRxiv websites for usage stats
getUsageData <- function(source, doi) {
  
  if(source == "biorxiv") {
    base_url = "https://www.biorxiv.org/content/"
  } else {
    base_url = "https://www.medrxiv.org/content/"
  }
  
  url <- paste0(base_url, doi, "v1.article-metrics")
  
  html <- read_html(url)
  
  data <- html %>%
    html_nodes(".highwire-stats") %>% 
    html_table(fill = TRUE) %>% .[[1]] %>%
    rename(date = 1) %>%
    mutate(source = source,
           doi = doi) %>%
    janitor::clean_names()
  
  # update progress bar
  pb$tick()$print()
  
  return(data)
  
}

# set counter for progress bar
pb <- progress_estimated(length(preprints$doi))

# Retrieve usage data. Sometimes the bioRxix/medRxiv websites time out and
# return an invalid response. So we conduct the iteration with purrr::safely
# to prevent errors interrupting the process
getUsageDataSafely <- safely(getUsageData)

usage_data <- map2(preprints$source, preprints$doi, 
                   function(x, y) getUsageDataSafely(x, y))

```

# Create final dataset

```{r}

# Parse the response returned by the 'safely' function
parseUsageData <- function(item) {
  if(item["error"] == "") {
    return()
  } else if(!length(item$result)) {
    return()
  } else {
    return(
      tibble(
        source = item$result$source,
        doi = as.character(item$result$doi),
        collection_date = as.character(item$result$date),
        abstract_views = as.numeric(item$result$abstract),
        full_text_views = if(length(item$result$full)) as.numeric(item$result$full) else rep(NA_integer_, length(item$result$doi)),
        pdf_downloads = as.numeric(item$result$pdf)
      )
    )
  }
}

map_dfr(usage_data, parseUsageData) %>%
  write_csv("data/preprint_usage_20190101_20201031.csv")

```