Python - 网络爬行/来自相同代码的不同结果？ /请求，bs4 / M1

Question

我正在学习 python 进行网络爬虫，但我完全卡住了。

每次我运行这段代码，结果都会改变。

很少，它可以工作，但几乎返回空列表。

为什么会这样？请告诉我

from indeed import extract_indeed_pages, extract_indeed_jobs


last_indeed_page = extract_indeed_pages()

print(last_indeed_page)

indeed_jobs = extract_indeed_jobs(last_indeed_page)

print(indeed_jobs)

import requests
from bs4 import BeautifulSoup

LIMIT = 50
URL = f"https://kr.indeed.com/jobs?q=React&l=%EC%84%9C%EC%9A%B8&radius=100&jt=fulltime&limit={LIMIT}"


def extract_indeed_pages():
    result = requests.get(URL)
    soup = BeautifulSoup(result.text, "html.parser")
    pagination = soup.find("div", {"class": "pagination"})

    links = pagination.find_all('a')
    pages = []
    for link in links[:-1]:
        pages.append(int(link.string))

    max_page = pages[-1]
    return max_page


def extract_indeed_jobs(last_page):

    jobs = []
    
    result = requests.get(f"{URL}&start={0*LIMIT}")
    soup = BeautifulSoup(result.text, "html.parser")
    results = soup.find_all("h2", {"class": "jobTitle"})
    jobs.append(results)

    return jobs

Answer 1

这是因为源代码中的 javascript。您可以按电脑上的ctrl + u 按钮查看网页。