Instagram Selenium 网页抓取 - #Followers

Question

我正在抓取一个需要获取用户信息的 instagramm 页面：

1. 帖子数
2. 关注者数量

我设法在 instagram 上登录，然后搜索用户（在本例中为“leonardodicaprio”），然后转到他的页面。我无法选择文本。

有人可以帮忙吗？ 谢谢！

# -*- coding: utf-8 -*-
import scrapy
from scrapy_splash import SplashRequest
from scrapy.selector import Selector
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from shutil import which
import logging
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.support.wait import WebDriverWait
import time


class InstatestSpider(scrapy.Spider):
    name = 'instatest'
    allowed_domains = ['www.instagram.com']
    start_urls = ['https://www.instagram.com/accounts/login']


def __init__(self):
    chrome_option = Options()
    #chrome_option.add_argument("--headless")
    chrome_path = which("chromedriver")
    driver = webdriver.Chrome(executable_path=chrome_path, options = chrome_option)
    driver.set_window_size(1920, 1080)
    driver.get("https://www.instagram.com/accounts/login")
    logging.info('Website opened...')
    # username = driver.find_element_by_name("username")
    # username = driver.find_element(By.XPATH, '//input[@name="username"]')   

    username = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, '//input[@name="username"]')))
    password = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, '//input[@name="password"]')))

    username.clear()
    username.send_keys("username")
    logging.info('Typing Username...')
    password.clear()
    password.send_keys("password")
    logging.info('Typing Password...')
    Login_button = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, '//button[@type="submit"]'))).click()
    alert_1 = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, '//button[contains(text(), "Not Now")]'))).click()
    logging.info('Do NOT save password...')
    alert_2 = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, '//button[contains(text(), "Not Now")]'))).click()  #search for a text="Not Now"
    logging.info('Do NOT turn notifications on...')
    logging.info('Logging Successful...')

    influencer = "leonardodicaprio"
    driver.get("https://www.instagram.com/" + influencer + "/")
    time.sleep(5)

    driver.save_screenshot('Influencer_Home_Page.png')

P.S：对于追随者的数量，我想得到最接近数字的确切数字，如选择器中的标题属性中找到的。请看下图：

insta

运行时出现此错误：

error in jupyterlab

    wait = WebDriverWait(driver, 20)
    number_of_post = wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, "a[href$='profile_posts'] span"))).text
    print(number_of_post)
    number_of_follower = wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, "a[href$='followed_by_list'] span"))).get_attribute('title')
    print(number_of_follower)

error

更新代码：

# -*- coding: utf-8 -*-
import scrapy
from scrapy_splash import SplashRequest
from scrapy.selector import Selector
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from shutil import which
import logging
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.support.wait import WebDriverWait
import time


class InstatestSpider(scrapy.Spider):
    name = 'instatest'
    allowed_domains = ['www.instagram.com']
    start_urls = ['https://www.instagram.com/accounts/login']
   
def __init__(self):
    chrome_option = Options()
    #chrome_option.add_argument("--headless")
    chrome_path = which("chromedriver")
    driver = webdriver.Chrome(executable_path=chrome_path, options = chrome_option)
    driver.set_window_size(1920, 1080)
    driver.get("https://www.instagram.com/accounts/login")
    logging.info('Website opened...')
    # username = driver.find_element_by_name("username")
    # username = driver.find_element(By.XPATH, '//input[@name="username"]')   

    username = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, '//input[@name="username"]')))
    password = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, '//input[@name="password"]')))

    username.clear()
    username.send_keys("username")
    logging.info('Typing Username...')
    password.clear()
    password.send_keys("password")
    logging.info('Typing Password...')
    Login_button = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, '//button[@type="submit"]'))).click()
    alert_1 = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, '//button[contains(text(), "Not Now")]'))).click()
    logging.info('Do NOT save password...')
    alert_2 = WebDriverWait(driver, 10).until(EC.element_to_be_clickable((By.XPATH, '//button[contains(text(), "Not Now")]'))).click()  #search for a text="Not Now"
    logging.info('Do NOT turn notifications on...')
    logging.info('Logging Successful...')

    influencer = "leonardodicaprio"
    driver.get("https://www.instagram.com/" + influencer + "/")
    time.sleep(5)


    wait = WebDriverWait(driver, 20)
    number_of_post = wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, "a[href$='profile_posts'] span"))).text
    print(number_of_post)
    number_of_follower = wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, "a[href$='followed_by_list'] span"))).get_attribute('title')
    print(number_of_follower)

    driver.save_screenshot('Influencer_Home_Page.png')

Answer 1

您可以使用下面的CSS_SELECTOR，获取帖子数和关注者数。要获取标题，您可以使用.get_attribute()

wait = WebDriverWait(driver, 20)
number_of_post = wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, "a[href$='profile_posts'] span"))).text
print(number_of_post)
number_of_follower = wait.until(EC.visibility_of_element_located((By.CSS_SELECTOR, "a[href$='followed_by_list'] span"))).get_attribute('title')
print(number_of_follower)

进口：

from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC