python - 使用 selenium 和 python 抓取 Instagram 关注者页面

Question

我有一个关于抓取 Instagram 关注者页面的问题。我有一个代码，但它只显示 9 个关注者。请帮助我。

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC


def login(driver):
    username = "xxxx@yahoo.com"  # <username here>
    password = "xxxx"  # <password here>

    # Load page
    driver.get("https://www.instagram.com/accounts/login/")

    # Login
    driver.find_element_by_xpath("//div/input[@name='username']").send_keys(username)
    driver.find_element_by_xpath("//div/input[@name='password']").send_keys(password)
    driver.find_element_by_xpath("//span/button").click()

    # Wait for the login page to load
    WebDriverWait(driver, 15).until(
        EC.presence_of_element_located((By.LINK_TEXT, "See All")))


def scrape_followers(driver, account):
    # Load account page
    driver.get("https://www.instagram.com/{0}/".format(account))

    # Click the 'Follower(s)' link
    driver.find_element_by_partial_link_text("follower").click()

    # Wait for the followers modal to load
    xpath = "//div[@style='position: relative; z-index: 1;']/div/div[2]/div/div[1]"
    WebDriverWait(driver, 10).until(
        EC.presence_of_element_located((By.XPATH, xpath)))

    # You'll need to figure out some scrolling magic here. Something that can
    # scroll to the bottom of the followers modal, and know when its reached
    # the bottom. This is pretty impractical for people with a lot of followers

    # Finally, scrape the followers
    xpath = "//div[@style='position: relative; z-index: 1;']//ul/li/div/div/div/div/a"
    followers_elems = driver.find_elements_by_xpath(xpath)

    return [e.text for e in followers_elems]


if __name__ == "__main__":
    driver = webdriver.Firefox()
    try:
        login(driver)
        followers = scrape_followers(driver, "instagram")
        print(followers)
    finally:
        driver.quit()

此代码取自另一个页面。我不明白如何向下滚动关注者页面。

Answer 1

您必须添加一个 for 循环，以便您可以向下滚动页面以查找关注者。这个 for 循环可以是这样的:

#Find the followers page
dialog = driver.find_element_by_xpath('/html/body/div[2]/div/div[2]/div/div[2]')
#find number of followers
allfoll=int(driver.find_element_by_xpath("//li[2]/a/span").text) 
#scroll down the page
for i in range(int(allfoll/2)):
    driver.execute_script("arguments[0].scrollTop = arguments[0].scrollHeight", dialog)
    time.sleep(random.randint(500,1000)/1000)
    print("Extract friends %",round((i/(allfoll/2)*100),2),"from","%100")