eda_rvest.Rmd

# 网络爬虫 {#eda-rvest}

```{r, include=FALSE}
knitr::opts_chunk$set(
   echo         = TRUE, 
   eval         = FALSE,
   warning      = FALSE, 
   message      = FALSE,
   fig.showtext = TRUE
)
```

大神说[rvest](https://rvest.tidyverse.org/) 马上推出1.0版本了。

```{r rvest-1}
library(tidyverse)
library(rvest)
library(sf)
```

## 链家网

```{r rvest-2}
urls <-	paste0("https://sh.lianjia.com/ershoufang/pg", seq_along(1:2))


scrape_house_info  <- function(url) {

	web <- read_html(url)
	
	title <- web %>% 
	  html_nodes('.clear .title a') %>% 
	  html_text()

	houseinfo <- web %>% 
	  html_nodes('.houseInfo') %>%
	  html_text()

	
	price <- web %>% 
	  html_nodes('.totalPrice span') %>% 
	  html_text()

	price_per <- web %>% 
	  html_nodes('.unitPrice span') %>% 
	  html_text()

	df <- data.frame(title, houseinfo, price, price_per) 
	
	return(df)
}
```


```{r rvest-3}
tb <- urls %>% map_df(scrape_house_info)

tb %>% 
  head()
```




## 猪肉价格

```{r rvest-4}
df_price <-
  read_html("https://hangqing.zhuwang.cc/shengzhu/20190905/407978.html") %>%
  html_node(".tabzj") %>%
  html_table(header = T) %>% 
  set_names(
    c("region", "name", "price_today", "price_yestoday", "diff_last_day", "diff_last_week")
    ) %>% 
  mutate_at(vars(name), ~str_remove_all(., " ") ) %>% 
  mutate_at(vars(name), ~if_else( name == "黑龙江", "黑龙江省", .))

df_price %>% 
  head()
```




```{r rvest-5}
china <- st_read("./demo_data/chinamap_data/bou2_4p.shp") %>% 
  st_set_crs(4326) %>% 
  group_by(NAME) %>%
  summarize()
```


```{r rvest-6}
china_uni <- china %>% 
  mutate( NAME = iconv(NAME, "GBK", "UTF-8") ) %>% 
  mutate_at(vars(NAME), ~str_remove_all(., "自治区|回族|维吾尔|壮族") ) %>%
  mutate_at(vars(NAME), ~str_trim(.))
```


```{r rvest-7}
df <- left_join(china_uni, df_price, by = c("NAME" = "name"))
```




```{r rvest-8,fig.width= 8, fig.height= 8, eval=FALSE}
ggplot(data = df) + 
  geom_sf( aes(fill = price_today < 28), show.legend = FALSE) + 
  geom_sf_text(aes(label = NAME),
               size = 3
               ) +
  geom_sf_text(aes(label = price_today), 
               size = 3,
               #nudge_x = c(-0.4, 0.5, 0.7),
               nudge_y = c(-1, -1, -1)
               ) +
  coord_sf(crs = 4326) +
  ggtitle("全国猪肉价格地图")

```




```{r rvest-9, echo = F}
# remove the objects
# rm(list=ls())
rm(df, china_uni, china, df_price, tb, scrape_house_info, urls)
```


```{r rvest-10, echo = F, message = F, warning = F, results = "hide"}
pacman::p_unload(pacman::p_loaded(), character.only = TRUE)
```