merging.Rmd

---
title: "R Notebook"
output: html_notebook
---
# packages

```{r}
pacman::p_load(tidyverse, here)
```


# All

```{r}
datasets <- dir(here("data", "yt"))[-1:-2]

newdata <- list()
for (jj in datasets) {
  cat(jj, ": \n")
  einzel <- get(load(here("data", "yt", jj)))
  newdata[[jj]] <- suppressWarnings(bind_rows(einzel))
}

yt_all_comments_fn <- suppressWarnings(bind_rows(newdata))

save(yt_all_comments_fn, file = here("data", "yt", "yt_all_comments_fn.Rdata"))
```

```{r}
load("C:/Users/Fabio/Documents/git_proj/altright_data/cernovich_comments.Rdata")
library(magrittr)
comments %>% 
  mutate(time = lubridate::as_date(created_time)) %>% 
  ggplot(aes(time, likes_count)) +
  geom_line()
range(comments$time)


```

# New Scrape merging

```{r}
datasets <- dir(here::here("data", "fb"))[c(-1,-28:-30)]

load(here::here("data", "fb", "fb_alexjones.Rdata"))

colnames(user_data$posts)
colnames(user_data$comments)

user_data$posts$page <- user_data$comments$page[1]

init_list <- get(load(here::here("data", "fb", "fb_alexjones.Rdata")))
init_posts <- init_list$posts
init_comments <- init_list$comments
for (jj in datasets) {
   cat(jj, "\n")
   init_posts <- plyr::rbind.fill(init_posts, get(load(here("data", "fb", jj)))$posts)
   init_comments <- plyr::rbind.fill(init_comments, get(load(here("data", "fb", jj)))$comments)
}
fb_all1 <- plyr::rbind.fill(init_posts, init_comments)

save(fb_all1, file = here("data", "fb", "fb_all1.Rdata"))

```

# Initital Scrape merging

```{r}
dat_files <- dir(here::here("data", "fb", "initial_scrape"))[-1:-2]
post_files <- dat_files[!grepl("comments", dat_files)]
comment_files <- dat_files[grepl("comments", dat_files)]


load(here("data", "fb", "initial_scrape", "abc.Rdata"))
load(here("data", "fb", "initial_scrape", "abc_comments.Rdata"))


colnames(page)
colnames(comments)

page$page <-  page$from_name[1]
comments$page <-  page$from_name[1]


init_posts <- page
init_comments <- comments

for (jj in seq_along(post_files)) {
  cat(post_files[jj], jj, "\n")
   init_posts <- plyr::rbind.fill(init_posts, get(load(here("data", "fb", "initial_scrape", post_files[jj]))))
  cat(comment_files[jj], jj, "\n")
   init_comments <- plyr::rbind.fill(init_comments, get(load(here("data", "fb", "initial_scrape", comment_files[jj]))))
}

fb_all2 <- plyr::rbind.fill(init_posts, init_comments)

save(fb_all2, file = here("data", "fb", "fb_all2.Rdata"))

```
# Merge it all

```{r}
fb_all <- plyr::rbind.fill(fb_all1, fb_all2)

fb_all %<>% 
  mutate(page = ifelse(is.na(page), from_name, page))

save(fb_all, file = here("data", "fb", "fb_all.Rdata"))
```

```{r}
load(here("data", "fb", "fb_all.Rdata"))

sjw_keywords <- c("blm", "black lives matter", "feminis", "race", "racis", "sexis", "postmodern", "marx", "antifa", "pc", "political correctness", "regressive", "safe space", "social justice", "warrior", "snowflake", "identity politics", "classical liberal", "white knight", "mra", "cultural, marxism", "white, genocide", "sjw", "cuck", "dindu", "dindu, nuffin", "george, soros", "soros", "We, must, secure, the, existence, of, our, people, and, a, future, for, white, children", "\\(\\(\\(.*?\\)\\)\\)", "anti, white", "beta", "cuckservative", "cultural, enrichment", "hatefact", "lügenpresse", "mainstream, media", "mass, immigration", "postmodernism", "race, realism", "red, pill", "remove, kebab", "kebab", "cockroach", "safe, space", "shekel", "shoah", "oh, vey", "soy, boy", "snowflake", "white, guilt", "triggered", "virtue, signal", "white, guilt", "feminazi", "professional, victim", "mangina", "third, wave, feminism", "joo", "white, nation", "kek", "deus vult", "ethnostate")

keywords1 <- paste(sjw_keywords, collapse = "|")

fb_all %>% 
  filter(!duplicated(message)) %>% 
  filter(stringr::str_detect(message, "anti-male")) #%>% 
  # group_by(page) %>% 
  # summarise(n = n())  %>%
  # left_join(fb_all %>% 
  # group_by(page) %>% 
  # summarise(total = n())) %>% 
  # mutate(perc = round(n / total * 100,3)) %>% 
  # arrange(perc)

table(fb_all$page)
```


## YouTube Merger
# packages

```{r}
pacman::p_load(tidyverse, here, svMisc, stringr, beepr)
# dir(here::here("data", "rebelmedia"))
# here::here()
#  yt_rebelmedia_comments_1_2148_fn <- plyr::rbind.fill(
#  get(load(here::here("data", "rebelmedia", "yt_all_rebelmedia_comments_1_767.Rdata"))),
#  get(load(here::here("data", "rebelmedia", "yt_all_rebelmedia_comments_768_1417_fn.Rdata"))),
#  get(load(here::here("data", "rebelmedia", "yt_all_rebelmedia_comments_1418_2148_fn.Rdata")))
# )
#  
# save(yt_rebelmedia_comments_1_2148_fn, file = "data/yt/yt_rebelmedia_comments_1_2148_fn.Rdata")
```


# All

```{r}
datasets <- dir(here::here("data", "yt"))[-1:-2]

names <- str_replace_all(datasets, "yt_", "") %>%
  str_replace_all("_comments.*?$", "")

names[19] <- "nyt"
cbind(datasets, names)

bind_it_all <- function(datasets, names){ 
newdata <- list()
for (jj in seq_along(datasets)) {
  cat(datasets[jj], ":", jj, "\n")
  einzel <- get(load(here::here("data", "yt", datasets[jj])))
  if(is.data.frame(einzel)) einzel$channel <- paste0(names[jj])
  newdata[[jj]] <- suppressWarnings(bind_rows(einzel))
}

yt_all_comments_fn <- suppressWarnings(bind_rows(newdata))

save(yt_all_comments_fn, file = here::here("data", "yt", "yt_all_comments_fn.Rdata"))
}

bind_it_all(datasets, names)

```

# Twitter Combine

```{r}
tw_sets <- dir(here::here("data"))[-c(1:2, 33:35)]

dat_list <- list()
for (jj in seq_along(tw_sets)) {
  dat_list[[jj]] <- do.call("rbind",(get(load(here("data", tw_sets[jj])))))
}
tw_partial3 <- do.call("rbind", dat_list)
save(tw_partial3, file = here("data", "tw", "tw_partial3.Rdata"))

table(tw_partial3$handle)

# tw_partial2 %>% 
#   filter(stringr::str_detect(text, "ethnostate"))
```

```{r}
load(here("data", "tw", "tw_partial1.Rdata"))
load(here("data", "tw", "tw_partial2.Rdata"))
load(here("data", "tw", "tw_partial3.Rdata"))

tw_partial123 <- rbind(tw_partial1, tw_partial2, tw_partial3)

save(tw_partial123, file = here("data", "tw", "tw_partial123.Rdata"))

table(tw_partial123$handle)

# tw_partial1_2 %>% 
#   filter(stringr::str_detect(text, "jew.|Jew.")) %>% 
#   group_by(handle) %>% 
#   summarise(n = n()) %>% 
#   left_join(tw_partial1_2 %>% 
#   group_by(handle) %>% 
#   summarise(total = n())) %>% 
#   mutate(perc = round(n / total * 100,3)) %>% 
#   arrange(perc)


```


# Keeping Track of Scrapers

```{r}
final_lists <- dir("C:/Users/Fabio/Desktop/altrighters")

options(scipen = 999)

finished_ones <- final_lists %>% 
  str_replace_all("final_list_|.Rdata", "") %>% 
  str_split("_") %>% 
  do.call("rbind", .) %>% 
  as.data.frame() %>% 
  rename(start = V1, end = V2) %>% 
  mutate_all(as.character) %>% 
  mutate_all(as.numeric) %>% 
  arrange(start, end) %>% 
  mutate(index = 1:n()) 

finished_ones_gathered <- finished_ones %>% 
  gather("position", "value", -index)

finished_ones_gathered %>% 
  ggplot(aes(1, value, group = index)) +
  geom_line() +
  coord_flip()

plotly::ggplotly()

all <- c()
for (jj in seq_along(finished_ones$start)) {
  temp <- finished_ones$start[jj]:finished_ones$end[jj]
  all <- append(all, temp)
}

every_id <- 1:390000

remainers  <- setdiff(every_id, all)

lastone <- c()
for (jj in seq_along(remainers)) {
  if ((remainers[jj] + 1) == remainers[jj + 1]) {
    NULL
  } else {
  lastone  <- append(lastone, remainers[jj + 1])
  }
}

finished_ones$start:finished_ones$end
```