BeautifulSoup.findAll 不打印任何内容

Question

我检查了登录是 200 作为响应，但结果什么也没打印。 这是代码：

import requests
from bs4 import BeautifulSoup

file_in = 'D:\OneDrive\Documents\GPIP\Files\scraping\idlinkedin.csv'
dataset = open(file_in, "r")

def login(iemail,ipassword):
    client = requests.Session()

    HOMEPAGE_URL = 'https://www.linkedin.com'
    LOGIN_URL = 'https://www.linkedin.com/uas/login-submit'

    html = client.get(HOMEPAGE_URL).content
    soup = BeautifulSoup(html, "html.parser")
    csrf = soup.find(id="loginCsrfParam-login")['value']

    login_information = {
        'session_key': iemail,
        'session_password': ipassword,
        'loginCsrfParam': csrf,
    }

    client.post(LOGIN_URL, data=login_information)

    for username in dataset:
        item_url = 'https://www.linkedin.com/in/' + username.strip()
        source_code = client.get(item_url)
        plain_text = source_code.text
        soup = BeautifulSoup(plain_text, features='html.parser')
        for item_name in soup.findAll('h1', {'class': 'pv-top-card-section__name inline t-24 t-black t-normal'}):
            print(item_name)

# MAIN
login('theusername','thepassword')

这行应该是打印姓名的账号，可惜结果什么都没有。

for item_name in soup.findAll('h1', {'class': 'pv-top-card-section__name inline t-24 t-black t-normal'}):
                print(item_name)

Answer 1

这里的问题是您正在根据您在浏览器元素中查看的内容（显示在 chrome 浏览器中的 F12 上）编写代码，而不是基于您在 requests.get 函数中获得的响应。我在使用 instagram.com 进行抓取时遇到了同样的问题-->（请参阅 git hub https://github.com/simplyshravan/python_learning/blob/master/Using_beautifulsoup.py 上的此链接）。

始终寻找你收到的东西，而不是它的样子。 所以在下面花了几个小时后，是从linkedin中提取用户信息的代码。

import requests
from bs4 import BeautifulSoup
import json

file_in = r'D:\OneDrive\Documents\GPIP\Files\scraping\idlinkedin.csv'
dataset = open(file_in, "r")

def login(iemail,ipassword):
    client = requests.Session()

    HOMEPAGE_URL = 'https://www.linkedin.com'
    LOGIN_URL = 'https://www.linkedin.com/uas/login-submit'

    html = client.get(HOMEPAGE_URL).content
    soup = BeautifulSoup(html, "html.parser")
    csrf = soup.find(id="loginCsrfParam-login")['value']

    login_information = {
        'session_key': iemail,
        'session_password': ipassword,
        'loginCsrfParam': csrf,
    }

    client.post(LOGIN_URL, data=login_information)

    for username in dataset:
        item_url = 'https://www.linkedin.com/in/' + username.strip()
        print(item_url)
        source_code = client.get(item_url)
        plain_text = source_code.text
        soup = BeautifulSoup(plain_text, 'html.parser')
        for item_name in soup.find_all('code'):
            if str(item_name).find('firstName') > -1:
                for i in json.loads(item_name.text)['included']:
                    #print(i)
                    if len(i['$deletedFields']) > 0:
                        if i['$type']=='com.linkedin.voyager.identity.shared.MiniProfile':
                            if i["publicIdentifier"]==username.strip():
                                    print(i['firstName']+' '+i['lastName'])
                                    print(i['lastName'])
                                    print(i['occupation'])
                                    break

# MAIN
login('username','password')