# 第六章 R语言爬虫初介

### 6.1 HTML基础与R语言解析

#### 6.1.2 R语言中HTML的解析

    # 清空工作目录
    rm(list = ls())

    # install.packages(XML)
    library(XML)
    # install.packages(bitops)
    library(bitops)
    # install.packages(RCurl)
    library(RCurl)
    temp = getURL('http://movie.douban.com/subject/26862829/?from=subject-page')
    fanghua = htmlParse(temp)
    fanghua
    ##  <!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN" "http://www.w3.org/TR/REC-html40/loose.dtd">
    ##  <html>
    ##  <head><title>301 Moved Permanently</title></head>
    ##  <body bgcolor="white">
    ##  <center><h1>301 Moved Permanently</h1></center>
    ##  <hr>
    ##  <center>nginx</center>
    ##  </body>
    ##  </html>
    ##

### 6.2 XML与XPath表达式以及R爬虫应用

#### 6.2.2 如何在R语言中解析XML

    library(XML)
    nbadata = xmlParse(file = "./nbaplayer.xml")
    nbadata
    ##  <?xml version="1.0" encoding="ISO-8859-1"?>
    ##  <nbaplayer>
    ##    <team>
    ##      <city name="houston"> rockets</city>
    ##      <player> james harden</player>
    ##    </team>
    ##    <team>
    ##      <city name="boston"> celtics</city>
    ##      <player> kyrie irving</player>
    ##    </team>
    ##    <team>
    ##      <city name="goldenstates"> worriors</city>
    ##      <player> stephen curry</player>
    ##    </team>
    ##  </nbaplayer>
    ##

#### 6.2.3 XPath表达式

    # install.packages(rvest)
    library(rvest)
    # install.packages(xml2)
    library(xml2)
    # install.packages(dplyr)
    library(dplyr)
    read_html('http://movie.douban.com/subject/26862829/') %>% html_nodes(xpath = "//h1//span")
    ##  {xml_nodeset (2)}
    ##  [1] <span property="v:itemreviewed">芳华</span>
    ##  [2] <span class="year">(2017)</span>

### 6.3 HTTP协议

#### 6.3.2 URL语法

    # URL字符串的编码及解码
    char = 'Golden states worries is the NBA champion in 2017'
    URLencode(char, reserve = TRUE)
    ##  [1] "Golden%20states%20worries%20is%20the%20NBA%20champion%20in%202017"
    URLdecode(char)
    ##  [1] "Golden states worries is the NBA champion in 2017"

### 6.4 AJAX与网页动态加载

### 6.5 正则表达式与字符串处理函数

#### 6.5.2 R中正则表达式的使用方法

    # 基础字符处理函数的正则表达式应用
    example_text1 = c("23333#RRR#PP", "35555#CCCC", "bearclub#2017")
    # 以#进行字符串切分
    unlist(strsplit(example_text1, "#"))
    ##  [1] "23333"    "RRR"      "PP"       "35555"    "CCCC"     "bearclub"
    ##  [7] "2017"
    # 以空字符集进行字符串切分
    unlist(strsplit(example_text1, "\\s"))
    ##  [1] "23333#RRR#PP"  "35555#CCCC"    "bearclub#2017"
    # 以空字符替换字符串第一个#匹配
    sub("#", "", example_text1)
    ##  [1] "23333RRR#PP"  "35555CCCC"    "bearclub2017"
    # 以空字符集替换字符串全部#匹配
    gsub("#", "", example_text1)
    ##  [1] "23333RRRPP"   "35555CCCC"    "bearclub2017"
    # 查询字符串中是否存在3333或5555的特征并返回所在位置
    grep("[35]{4}", example_text1)
    ##  [1] 1 2
    # 查询字符串中是否存在3333或5555的特征并返回逻辑值
    grepl("[35]{4}", example_text1)
    ##  [1]  TRUE  TRUE FALSE

    # stringr包函数的正则表达式应用
    example_text2 = "1. A small sentence. -2. Another tiny sentence."
    # install.packages(stringr)
    library(stringr)
    # 提取small特征字符
    str_extract(example_text2, "small")
    ##  [1] "small"
    # 提取包含sentence特征的全部字符串
    unlist(str_extract_all(example_text2, "sentence"))
    ##  [1] "sentence" "sentence"
    # 提取以1开始的字符串
    str_extract(example_text2, "^1")
    ##  [1] "1"
    # 提取以句号结尾的字符
    unlist(str_extract_all(example_text2, ".$"))
    ##  [1] "."
    # 提取包含tiny或者sentence特征的字符串
    unlist(str_extract_all(example_text2, "tiny|sentence"))
    ##  [1] "sentence" "tiny"     "sentence"
    # 点号进行模糊匹配
    str_extract(example_text2, "sm.ll")
    ##  [1] "small"
    # 中括号表示可选字符串
    str_extract(example_text2, "sm[abc]ll")
    ##  [1] "small"
    str_extract(example_text2, "sm[a-p]ll")
    ##  [1] "small"
    # 提取全部单词字符
    unlist(str_extract_all(example_text2, "\\w+"))
    ##  [1] "1"        "A"        "small"    "sentence" "2"        "Another"
    ##  [7] "tiny"     "sentence"

### 6.6 R语言爬虫实战

#### 6.6.1 静态网页数据抓取利器rvest

    ## 抓取某网页二手房数据 ##
    # 加载所需的包
    library("xml2")
    library("rvest")
    library("dplyr")
    library("stringr")

    # 对爬取页数进行设定并创建数据框
    i = 1:100
    house_inf = data.frame()

    # 利用for循环封装爬虫代码，进行批量抓取
    for (i in 1:100) {
    # 发现url规律，利用字符串函数进行url拼接并规定编码
    web = read_html(str_c("http://hz.lianjia.com/ershoufang/pg", i), encoding = "UTF-8")
    # 提取房名信息
    house_name = web %>% html_nodes(".houseInfo a") %>% html_text()
    # 提取房名基本信息并消除空格
    house_basic_inf = web %>% html_nodes(".houseInfo") %>% html_text()
    house_basic_inf = str_replace_all(house_basic_inf, " ", "")
    # 提取二手房地址信息
    house_address = web %>% html_nodes(".positionInfo a") %>% html_text()
    # 提取二手房总价信息
    house_totalprice = web %>% html_nodes(".totalPrice") %>% html_text()
    # 提取二手房单价信息
    house_unitprice = web %>% html_nodes(".unitPrice span") %>% html_text()
    # 创建数据框存储以上信息
    house = data.frame(house_name, house_basic_inf, house_address, house_totalprice, house_unitprice)
    house_inf = rbind(house_inf, house)
    }

    # 将数据写入csv文档
    write.csv(house_inf, file = "./house_inf.csv")

#### 6.6.2 httr包实现对网页动态加载数据的抓取

    # install.packages(httr)
    library(httr)
    cookie = ""
    headers = c('Accept' = 'application/json',
                 'User-Agent' = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.117 Safari/537.36',
                 'Referer' = 'http://study.163.com/courses',
                 'edu-script-token' = '1c1f84a1b85a48aba8a4d440552f5f69',
                 'Connection' = 'keep-alive',
                 'Cookie' = cookie)
    # 构造参数信息
    payload = list('pageIndex' = 1, 'pageSize' = 50, 'relativeOffset' = 0,'frontCatgoryId' = '-1')

    # 二次请求的实际url
    url = "http://study.163.com/p/search/studycourse.json"
    # POST方法执行单词请求
    result = POST(url, add_headers(.headers = headers), body = payload, encode = "json")
    result
    ##  Response [http://study.163.com/p/search/studycourse.json]
    ##    Date: 2018-08-01 06:52
    ##    Status: 200
    ##    Content-Type: application/json;charset=UTF-8
    ##    Size: 84.6 kB