将信息分布在多个 View 中进行检索

Question

我想抓取this page左边的排名，它分布在 34 个 View 中，我相信(完全是抓取新手)是 Java 生成的。所有 View 都有相同的 url，所以我无法循环这些 View 。

据我所知，每个 View 似乎都有节点 #elferspielerhistorie_subcont_j td，以 j=0 开头。

我可以抓取第一个条目

library(rvest)
library(tidyverse)

elfer_url <- "http://www.kicker.de/news/fussball/bundesliga/spieltag/1-bundesliga/elfmeter-schuetzen-geschichte.html"

# first page
elfmeter <- read_html(elfer_url)
Schuetzen <- elfmeter %>% html_nodes("#elferspielerhistorie_subcont_0 td") %>% html_text()

我的“策略”是使用 RSelenium 单击下一页的链接，粘贴下一个节点并重新开始。然而，循环为接下来的 33 个 View 返回空条目(为了完整性，整个代码):

library(rvest)
library(tidyverse)
library(RSelenium)

elfer_url <- "http://www.kicker.de/news/fussball/bundesliga/spieltag/1-bundesliga/elfmeter-schuetzen-geschichte.html"

rD <- rsDriver(port = 4444L, browser = "firefox")
remDr <- rD$client
remDr$navigate(elfer_url)

# first page
elfmeter <- read_html(elfer_url)
Schuetzen <- elfmeter %>% html_nodes("#elferspielerhistorie_subcont_0 td") %>% html_text() %>% matrix(ncol=10, byrow=T) %>% data.frame()

clicknext <- remDr$findElements("xpath","//*[@id='ctl00_PlaceHolderContent_elfer_blaettern_elferhistorie_PagerForward']")

j <- 1
while (j<=34){
  clicknext[[1]]$clickElement()     # sends me to the right view
  #elfmeter <- read_html(elfer_url) # switching this on or off does not change things
  current.node <- paste0("#elferspielerhistorie_subcont_",j," td") # should be the node
  weitere_Schuetzen <- elfmeter %>% html_node(current.node) %>% html_text() %>% matrix(ncol=10, byrow=T) %>% data.frame() # returns empty result
  Schuetzen <- rbind(Schuetzen,weitere_Schuetzen)

  j <- j+1
}

Answer 1

由于 View 是动态生成的，因此您必须每次都获取页面源。下一个按钮的 ID 可能会发生变化，因此保存它以便在每次迭代时也能找到该按钮。

下面的代码应该可以工作。请注意，我还读出了循环完成时删除的那些空行:

library(rvest)
library(tidyverse)
library(RSelenium)

elfer_url <- "http://www.kicker.de/news/fussball/bundesliga/spieltag/1-bundesliga/elfmeter-schuetzen-geschichte.html"

rD <- rsDriver(port = 4447L, browser = "firefox")
remDr <- rD$client
remDr$navigate(elfer_url)

getTable <- function(x) {
  remDr$getPageSource()[[1]] %>% 
    read_html %>% 
    html_nodes(paste0("#elferspielerhistorie_subcont_", x, " table")) %>% 
    html_table(fill = T)  %>% 
    .[[1]] %>% 
    data.frame
}

# first page
data <- getTable(0)

for(j in 1:33) {
  next_button <- remDr$findElements("css","a[id=\"ctl00_PlaceHolderContent_elfer_blaettern_elferhistorie_PagerForward\"]") %>% .[[1]]
  remDr$executeScript(script = "arguments[0].scrollIntoView(true);", args = list(next_button))
  next_button$clickElement()
  # sometimes the loop is too fast and it cannot fetch the table. so pause here
  Sys.sleep(1)
  data <- rbind(data, getTable(j))

  j <- j+1

}
rD$server$stop()

data <- data[-which(data$Spieler == ""),]
dim(data)

> [1] 935  10