页面不抓取

Question

我正在尝试抓取此页面

https://www.vivareal.com.br/venda/pernambuco/recife/#onde=BR-Pernambuco-NULL-Recife

我刮了这个网站的第一页，然后用selenium点击到下一页，但我只能得到第一页的内容，当我刮第二页时，它与第一页的内容相同。我不知道如何解决这个问题，或者网页是否对抓取有一些保护。

有人可以帮我吗？

from bs4 import BeautifulSoup
import pandas as pd 
from time import sleep
from time import time
from random import randint
from IPython.core.display import clear_output
from warnings import warn
from selenium import webdriver

def scrape():
    cont = [True,True,True,True,False]

    for times in cont:

        if times != True:
            driver = webdriver.Firefox(executable_path = 'geckodriver')
            
            page = driver.get('https://www.vivareal.com.br/venda/pernambuco/recife/?#onde=BR-Pernambuco-NULL-Recife')
            
            sleep(15)

            titles = []
            addresses = []
            areas = []
            rooms = []
            bathes = []
            values = []

            start_time = time()
            request = 0
            
            soup = BeautifulSoup(page,'html.parser')
            imov = soup.find_all('div', class_='property-card__main-content')

            sleep(randint(8,15))

            # Monitor
            request += 1
            elapsed_time = time() - start_time
            print('Request: {}; Frequency: {} requests/s'.format(request, request/elapsed_time))
            clear_output(wait = True)

            # Throw a warning for non-200 status codes
            if page.status_code != 200:
                warn('Request: {}; Status code: {}'.format(requests, page.status_code))

            # Break the loop if the number of requests is greater than expected
            if request > 72:
                warn('Number of requests was greater than expected.')
                break
                
            for container in imov:
                # Título
                title = container.h2.a.get_text()
                t2 = title.strip()
                titles.append(t2)
                # Título

                # Endereço
                address = container.h2.span.get_text()
                a2 = address.strip()
                addresses.append(a2)
                # Endereço

                # Área
                area = container.li.span.get_text()
                ar2 = area.strip()
                areas.append(ar2)
                # Área

                # Quartos
                room = container.find(class_= "property-card__detail-item property-card__detail-room js-property-detail-rooms")
                room2 = room.find('span', class_="property-card__detail-value js-property-card-value").get_text()
                r2 = room2.strip()
                rooms.append(r2)
                # Quartos

                # Banheiros
                bath = container.find(class_= "property-card__detail-item property-card__detail-bathroom js-property-detail-bathroom")
                bath2 = bath.find('span', class_="property-card__detail-value js-property-card-value").get_text()
                b2 = bath2.strip()
                bathes.append(b2)
                # Banheiros

                # Valor
                value = container.section.div.get_text()
                v2 = value.strip()
                values.append(v2)
                # Valor

                # Dataframe e salvar
                vivareal = pd.DataFrame({
                    "title": titles, 
                    "address": addresses, 
                    "area": areas, 
                    "rooms":rooms,
                    "baths":bathes,
                    "value":values
                    })
                                
                vivareal.to_csv(r'output.csv')

            prox = driver.find_element_by_xpath('//*[@title="Próxima página"]')
            prox.click()
        else:
            print('Done!')
            
scrape()```

Answer 1

虽然你把click命令放在了最后，但是当它进入下一个循环时，第一个命令是创建一个新的驱动程序，然后调用该命令将Viva Real的主页带到Pernambuco。这是不希望的。你可以这样做：

def scrape():
    cont = [True,True,True,True,False]

    # You create the driver and access the main page only once
    driver = webdriver.Firefox(executable_path = 'geckodriver')
    page = driver.get('https://www.vivareal.com.br/venda/pernambuco/recife/?#onde=BR-Pernambuco-NULL-Recife')

    for times in cont:

        if times != True:
            # Wait to load every page
            sleep(15)

Answer 2

即使使用@MarceloBaliu 提供的修复程序，您的代码也无法正常工作。这是我的代码（终于！）为我工作。我之所以分享，是因为它可以帮助别人，就像我被这个网站所帮助一样。

from selenium import webdriver
from selenium.common.exceptions import WebDriverException, ElementClickInterceptedException, NoSuchElementException
from selenium.webdriver.common.by import By 
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from bs4 import BeautifulSoup
import re
import time
import pandas as pd

class ScraperVivaReal:
    wait_time = 5
    
    def __init__(self, url):
        # Initializing the webdriver
        options = webdriver.FirefoxOptions()
        options.add_argument('-headless')
        self.driver = webdriver.Firefox(options=options)
        self.driver.maximize_window()
        self.driver.get(url)
        time.sleep(self.wait_time)
        # Handling cookies acception
        WebDriverWait(self.driver, self.wait_time).until(EC.element_to_be_clickable((By.XPATH,'//*[@id="cookie-notifier-cta"]'))).click()
        time.sleep(self.wait_time/2)
        
    def __scrape_page__(self):
        result = []
        
        # Extracting data from the page
        try:
            soup = BeautifulSoup(self.driver.page_source, 'html.parser')
        except WebDriverException:
            print('Webdriver was manually quit by the user!') # I configure this exception before adding the option -headless to webdriver
            return result
        
        # Finding property cards containing search results
        div_list = soup.find_all('div', {'class':'property-card__content'})
        
        # Iterating each card
        for d in div_list:

            # Extracting info from card
            title = d.find('span', {'class': 'property-card__title js-cardLink js-card-title'}).get_text().strip()
            complete_address = d.find('span', {'class': 'property-card__address'}).get_text().strip()
            area = d.find('span', {'class': 'property-card__detail-value js-property-card-value property-card__detail-area js-property-card-detail-area'}).get_text().strip()
            rooms = d.find('li', {'class': 'property-card__detail-item property-card__detail-room js-property-detail-rooms'}).find('span', {'class': 'property-card__detail-value js-property-card-value'}).get_text().strip()
            baths = d.find('li', {'class': 'property-card__detail-item property-card__detail-bathroom js-property-detail-bathroom'}).find('span', {'class': 'property-card__detail-value js-property-card-value'}).get_text().strip()
            garage = d.find('li', {'class': 'property-card__detail-item property-card__detail-garage js-property-detail-garages'}).find('span', {'class': 'property-card__detail-value js-property-card-value'}).get_text().strip()
            
            # Extracting the price
            try:
                price = d.find('div', {'class':'property-card__price js-property-card-prices js-property-card__price-small'}).find('p').get_text().strip()
            except AttributeError:
                price = "N/I"
            
            # Splitting the address
            add_list = re.split(',|-', complete_address)
            add_list = [ item.strip() for item in add_list ]
            if len(add_list) == 2:
                city, st = add_list
                neibhood = 'N/I'
                address = 'N/I'
                number = 'N/I'
            if len(add_list) == 3:
                neibhood, city, st = add_list
                address = 'N/I'
                number = 'N/I'
            if len(add_list) == 4:
                address, neibhood, city, st = add_list
                number = 'N/I'
            elif len(add_list) == 5:
                address, number, neibhood, city, st = add_list
                
            # Adding the result into a dicionary and appending the dict to a result list
            row = { 'Título': title, 'Endereço': address, 'Número': number, 'Bairro': neibhood, 'Cidade': city, 'Estado': st, 'Área': area, 'Quartos': rooms, 'Banheiros': baths, 'Vagas': garage, 'Preço': price }
            result.append(row)
        return result
        
    def __next_page__(self):
        # Finding the "Next Page" button element
        next_element = self.driver.find_element_by_xpath('//*[@title="Próxima página"]')
        try:
            # Trying to click it
            next_element.click()
            time.sleep(self.wait_time)
            return True
        # Treating some exceptions (element not found and element not clickable)
        except ElementClickInterceptedException:
            print('"Próxima Página" element is not clickable!')
        except NoSuchElementException:
            print('"Próxima Página" element not found!')
        return False
        
    def run(self, output):
        has_next = True
        final_result = []
        # Getting the information!
        while has_next:
            results = self.__scrape_page__()
            final_result.extend(results)
            print('Got {} results! Total Found: {}'.format(len(results), len(final_result)))
            if len(results) == 0:
                break
            has_next = self.__next_page__()
        # Quitting Firefox
        self.driver.quit()
        # Exporting results to CSV
        df = pd.DataFrame(final_result)
        df.to_csv(output, sep=',')

S = ScraperVivaReal('https://www.vivareal.com.br/venda/sp/paulinia/')
S.run('output.csv')