创建特定的 Web Scraper

Question

我正在努力学习在 Python 中抓取，在这种情况下，我的想法是制作一个从网页获取数据的工具。我在建议“for”浏览页面并按原样收集每个框（项目）的数据时遇到问题：

• IDoffer
• 列表
• 标题
• 位置
• 内容
• 电话

这不是一项任务，这是我自己的主动性，但我没有向前迈进，感谢您的帮助。

这是我的代码：

from bs4 import BeautifulSoup
import requests

URL_BASE = "https://www.milanuncios.com/ofertas-de-empleo-en-madrid/?dias=3&demanda=n&pagina="
MAX_PAGES = 2
counter = 0

for i in range(0, MAX_PAGES):

    #Building the URL
    if i > 0:
        url = "%s%d" % (URL_BASE, i)
    else:
        url = URL_BASE

    #We make the request to the web
    req = requests.get(url)
    
    #We check that the request returns a Status Code = 200
    statusCode = req.status_code
    if statusCode == 200:

        #We pass the HTML content of the web to a BeautifulSoup () object
        html = BeautifulSoup(req.text, "html.parser")

        #We get all the divs where the inputs are
        entradas_IDoffer = html.find_all('div', {'class': 'aditem-header'})
        
        #We go through all the inputs and extract info
        for entrada1 in entradas_IDoffer:
            
            #THIS ARE SOME ATTEMPS
            #Title = entrada.find('div', {'class': 'aditem-detail-title'}).getText()
            #location = entrada.find('div', {'class': 'list-location-region'}).getText()
            #content = entrada.find('div', {'class': 'tx'}).getText()
            #phone = entrada.find('div', {'class': 'telefonos'}).getText()
        
            #Offer Title
            entradas_Title = html.find_all('div', {'class': 'aditem-detail'})
            for entrada2 in entradas_Title:
                counter += 1
                Title = entrada2.find('a', {'class': 'aditem-detail-title'}).getText()
                
            counter += 1
            IDoffer = entrada1.find('div', {'class': 'x5'}).getText()
                    
                    

        #Location
        #entradas_location = html.find_all('div', {'class': 'aditem-detail'})
        #for entrada4 in entradas_location:
        #    counter += 1
        #    location = entrada4.find('div', {'class': 'list-location-region'}).getText()

                    #Offer content
                    #entradas_content = html.find_all('div', {'class': 'aditem-detail'})
                    #for entrada3 in entradas_content:
                     #   counter += 1
                      #  content = entrada3.find('div', {'class': 'tx'}).getText()

            print("%d - %s  \n%s\n%s" % (counter, IDoffer.strip(),url,Title))

    else:
        try:
            r = requests.head(req)
            print(r.status_code)

        except requests.ConnectionError:
            print("failed to connect")
        break
        #If the page no longer exists and it gives me a 400

Answer 1

正确的entradas_IDoffer，

entradas_IDoffer = html.find_all("div", class_="aditem CardTestABClass")

标题位于“a”标签而非“div”下

title = entrada.find("a", class_="aditem-detail-title").text.strip()
location = entrada.find("div", class_="list-location-region").text.strip()
content = entrada.find("div", class_="tx").text.strip()

对其他数据这样做

他们可能正在使用 javascript 加载电话号码，因此您可能无法使用 bs4 获取，您可以使用 selenium 获取。

您编写了非常长的代码来遍历多个页面，只需使用范围遍历页面 1 和 2。将 url 放入格式化字符串中。

for page in range(1, 3):
    url =  f'https://www.milanuncios.com/ofertas-de-empleo-en-madrid/?dias=3&demanda=n&pagina={page}'

完整代码：

import requests
from bs4 import BeautifulSoup

for page in range(1, 5):
    url =  f'https://www.milanuncios.com/ofertas-de-empleo-en-madrid/?dias=3&demanda=n&pagina={page}'
    response = requests.get(url)
    soup = BeautifulSoup(response.text, 'html.parser')
    entradas_IDoffer = soup.find_all("div", class_="aditem CardTestABClass")

    for entrada in entradas_IDoffer:
        title = entrada.find("a", class_="aditem-detail-title").text.strip()
        ID = entrada.find("div", class_="x5").text.strip()
        location = entrada.find("div", class_="list-location-region").text.strip()
        content = entrada.find("div", class_="tx").text.strip()
        
        print(title, ID, location, content)