滚动以使用 selenium 和 beautifulsoup 抓取更多细节

【问题标题】：scroll to scrape additional details using selenium and beautifulsoup滚动以使用 selenium 和 beautifulsoup 抓取更多细节
【发布时间】：2021-09-30 16:22:01
【问题描述】：

我有下面的 python 代码。我正在使用它从 zillow 中抓取房地产销售数据。它使用 selenium chrome 驱动程序打开 chrome 浏览器。然后导航到邮政编码的房产销售网址。然后我使用 beautifulsoup 来抓取属性的详细信息。我遇到的问题是，在下面的示例中，它仅抓取前 9 个属性数据。我认为它需要向下滚动并抓取更多属性。谁能建议如何抓取其余的属性信息？

代码：

import pandas as pd
import numpy as np

import os

from selenium import webdriver
from selenium.webdriver.chrome.options import Options


Options

options = Options()

chrome_options = Options() 



#re allows for matching text with regular expressions (including through BeautifulSoup)
#dateutil.parser provies .parse() to convert plain text dates in a variety of formats into datetime objects
import re, dateutil.parser
#BeautifulSoup provide a model for the source HTML
from bs4 import BeautifulSoup

#Webdriver is interface to the selected browser (PhantomJS)
from selenium import webdriver
#Ability to select values in HTML <select> tags
from selenium.webdriver.support import select
import time


from selenium.webdriver.common import action_chains, keys

from re import sub

import datetime




def property_details(zip_code):
    
    driver = webdriver.Chrome(executable_path=os.path.abspath("chromedriver"), chrome_options=chrome_options)

    
    zip_url = 'https://www.zillow.com/homes/for_sale/'+str(zip_code)+'_rb/'

    # waits between pulling data
    time.sleep(np.random.randint(low=3, high=7, size=1)[0])
    
    # opens website
    driver.get(zip_url)
    
    # getting page source data
    tstsoup = BeautifulSoup(driver.page_source)
    
    
    # getting list of prices

    price_parse=tstsoup.find_all('div',{'class':'list-card-price'})

    price_list=[]

    for x in range(len(price_parse)):

        price_list.append(int(sub(r'[^\d.]', '',price_parse[x].get_text())))
        
    # get address
    
    address_parse=tstsoup.find_all('address',{'class':'list-card-addr'})

    # address_parse[0].get_text()

    address_list=[]

    for i in range(len(address_parse)):

        address_list.append(address_parse[i].get_text())
        
    # get home details

    main_data=tstsoup.find_all('ul',{'class':'list-card-details'})
    detail_list=[]
    for data in main_data:
        d={  i.text.split(" ")[1] : i.text.split(" ")[0] for i in data.find_all("li",class_="")   }
        detail_list.append(d)
        
    # creating dict of property details    
        
    ts=str(datetime.datetime.now().isoformat())

    home_dict={}
    key_list=['bds','ba','sqft','acres']

    col_list=['acres', 'address', 'ba', 'bds', 'last_update', 'price', 'sqft',
           'zip_code']

    for x in range(len(detail_list)):

        for k in key_list:

            if k not in list(detail_list[x].keys()):

                detail_list[x][k]=-1

        # adding price data

        detail_list[x]['price']=price_list[x]

        # adding address

        detail_list[x]['address']=address_list[x]

        # add zip code

        detail_list[x]['zip_code']=zip_code

        # add state

    #     detail_list[x]['state']=state


        # adding last update date

        detail_list[x]['last_update']=ts

        # removing commas from numeric fields
#         detail_list[x]['sqft']=int(str(detail_list[x]['sqft']).replace(",", ""))

#         detail_list[x]['acres']=int(str(detail_list[x]['acres']).replace(",", ""))
        
        detail_list[x]['sqft']=detail_list[x]['sqft']

        detail_list[x]['acres']=detail_list[x]['acres']

        # cast strings to int

#         detail_list[x]['ba']=int(str(detail_list[x]['ba']).replace(",", ""))

#         detail_list[x]['bds']=int(str(detail_list[x]['bds']).replace(",", ""))
        
        detail_list[x]['ba']=detail_list[x]['ba']

        detail_list[x]['bds']=detail_list[x]['bds']

        # filtering for list of columns
        detail_list[x] = {k: v for k, v in detail_list[x].items() if k in col_list}


        # sorting by key

        detail_list[x]=dict(sorted(detail_list[x].items()))
        
        # turn into dataframe

    property_df=pd.DataFrame(detail_list)
    
    driver.quit()
    
    return property_df



# only returns first 9 entries

property_details(zip_code=94014)

尝试实施建议

更新：

from selenium.webdriver.common.action_chains import ActionChains

zip_code=94016



driver = webdriver.Chrome(executable_path=os.path.abspath("chromedriver"), chrome_options=chrome_options)

    
zip_url = 'https://www.zillow.com/homes/for_sale/'+str(zip_code)+'_rb/'

# waits between pulling data
time.sleep(np.random.randint(low=3, high=7, size=1)[0])

# opens website
driver.get(zip_url)

# getting page source data
tstsoup = BeautifulSoup(driver.page_source)


# getting list of prices

price_parse=tstsoup.find_all('div',{'class':'list-card-price'})

price_list=[]

for x in range(len(price_parse)):

    price_list.append(int(sub(r'[^\d.]', '',price_parse[x].get_text())))

# get address

address_parse=tstsoup.find_all('address',{'class':'list-card-addr'})

# address_parse[0].get_text()

address_list=[]

for i in range(len(address_parse)):

    address_list.append(address_parse[i].get_text())

# get home details

main_data=tstsoup.find_all('ul',{'class':'list-card-details'})
detail_list=[]
for data in main_data:
    d={  i.text.split(" ")[1] : i.text.split(" ")[0] for i in data.find_all("li",class_="")   }
    detail_list.append(d)

# creating dict of property details    

ts=str(datetime.datetime.now().isoformat())

home_dict={}
key_list=['bds','ba','sqft','acres']

col_list=['acres', 'address', 'ba', 'bds', 'last_update', 'price', 'sqft',
       'zip_code']

for x in range(len(detail_list)):

    for k in key_list:

        if k not in list(detail_list[x].keys()):

            detail_list[x][k]=-1

    # adding price data

    detail_list[x]['price']=price_list[x]

    # adding address

    detail_list[x]['address']=address_list[x]

    # add zip code

    detail_list[x]['zip_code']=zip_code

    # add state

#     detail_list[x]['state']=state


    # adding last update date

    detail_list[x]['last_update']=ts

    # removing commas from numeric fields
#         detail_list[x]['sqft']=int(str(detail_list[x]['sqft']).replace(",", ""))

#         detail_list[x]['acres']=int(str(detail_list[x]['acres']).replace(",", ""))

    detail_list[x]['sqft']=detail_list[x]['sqft']

    detail_list[x]['acres']=detail_list[x]['acres']

    # cast strings to int

#         detail_list[x]['ba']=int(str(detail_list[x]['ba']).replace(",", ""))

#         detail_list[x]['bds']=int(str(detail_list[x]['bds']).replace(",", ""))

    detail_list[x]['ba']=detail_list[x]['ba']

    detail_list[x]['bds']=detail_list[x]['bds']

    # filtering for list of columns
    detail_list[x] = {k: v for k, v in detail_list[x].items() if k in col_list}


    # sorting by key

    detail_list[x]=dict(sorted(detail_list[x].items()))
    
    print('made it')
    print('')

    # add scrolling

    actions = ActionChains(driver)

    footer = driver.find_elements_by_css_selector('div#region-info-footer')
    #if element is existing on the page the list will be non-empty and interpreted as True by python`
    if(footer):
        if not footer[0].is_displayed():
            #scroll the last currently available price_parse element into view. 
            #this will load a bulk of new elements
            actions.move_to_element(price_parse[-1]).perform()
            time.sleep(0.5)
            
            print('footer')
        else:
            #you can also use this case to exit the loop since you already scraped all the data on this page
            
            print('break')
            break
    else:        
        #if footer element is even not existing we 100% still not reached the bottom
        actions.move_to_element(price_parse[-1]).perform()
        time.sleep(0.5)
        print('scroll')

    # turn into dataframe
    
    
    print(len(detail_list))

在 break 语句处退出，不滚动

更新：

代码：

from selenium.webdriver.common.action_chains import ActionChains

zip_code=94014



driver = webdriver.Chrome(executable_path=os.path.abspath("chromedriver"), chrome_options=chrome_options)

    
zip_url = 'https://www.zillow.com/homes/for_sale/'+str(zip_code)+'_rb/'

# waits between pulling data
time.sleep(np.random.randint(low=3, high=7, size=1)[0])

# opens website
driver.get(zip_url)

# getting page source data
tstsoup = BeautifulSoup(driver.page_source)


# getting list of prices

price_parse=tstsoup.find_all('div',{'class':'list-card-price'})

price_list=[]

for x in range(len(price_parse)):

    price_list.append(int(sub(r'[^\d.]', '',price_parse[x].get_text())))

# get address

address_parse=tstsoup.find_all('address',{'class':'list-card-addr'})

# address_parse[0].get_text()

address_list=[]

for i in range(len(address_parse)):

    address_list.append(address_parse[i].get_text())

# get home details

main_data=tstsoup.find_all('ul',{'class':'list-card-details'})
detail_list=[]
for data in main_data:
    d={  i.text.split(" ")[1] : i.text.split(" ")[0] for i in data.find_all("li",class_="")   }
    detail_list.append(d)

# creating dict of property details    

ts=str(datetime.datetime.now().isoformat())

home_dict={}
key_list=['bds','ba','sqft','acres']

col_list=['acres', 'address', 'ba', 'bds', 'last_update', 'price', 'sqft',
       'zip_code']

for x in range(len(detail_list)):

    for k in key_list:

        if k not in list(detail_list[x].keys()):

            detail_list[x][k]=-1

    # adding price data

    detail_list[x]['price']=price_list[x]

    # adding address

    detail_list[x]['address']=address_list[x]

    # add zip code

    detail_list[x]['zip_code']=zip_code

    # add state

#     detail_list[x]['state']=state


    # adding last update date

    detail_list[x]['last_update']=ts

    # removing commas from numeric fields
#         detail_list[x]['sqft']=int(str(detail_list[x]['sqft']).replace(",", ""))

#         detail_list[x]['acres']=int(str(detail_list[x]['acres']).replace(",", ""))

    detail_list[x]['sqft']=detail_list[x]['sqft']

    detail_list[x]['acres']=detail_list[x]['acres']

    # cast strings to int

#         detail_list[x]['ba']=int(str(detail_list[x]['ba']).replace(",", ""))

#         detail_list[x]['bds']=int(str(detail_list[x]['bds']).replace(",", ""))

    detail_list[x]['ba']=detail_list[x]['ba']

    detail_list[x]['bds']=detail_list[x]['bds']

    # filtering for list of columns
    detail_list[x] = {k: v for k, v in detail_list[x].items() if k in col_list}


    # sorting by key

    detail_list[x]=dict(sorted(detail_list[x].items()))
    
    print('made it')
    print('')

    # add scrolling

    actions = ActionChains(driver)

    footer = driver.find_elements_by_css_selector('div#region-info-footer')
    #if element is existing on the page the list will be non-empty and interpreted as True by python`
    if(footer):
        if not footer[0].is_displayed():
            #scroll the last currently available price_parse element into view. 
            #this will load a bulk of new elements
            actions.move_to_element(price_parse[-1]).perform()
            
            # waits between pulling data
            time.sleep(np.random.randint(low=3, high=7, size=1)[0])
            
            print('footer')
        else:
            #you can also use this case to exit the loop since you already scraped all the data on this page
            
            print('break')
#             break
    else:        
        #if footer element is even not existing we 100% still not reached the bottom
        actions.move_to_element(price_parse[-1]).perform()
        
        # waits between pulling data
        time.sleep(np.random.randint(low=3, high=7, size=1)[0])
        
        print('scroll')

    # turn into dataframe
    
    
    print(len(detail_list))
    
    print(detail_list)
    print('')

输出：

made it

break
9
[{'acres': -1, 'address': '543 Green Ridge Dr APT 4, Daly City, CA 94014', 'ba': '2', 'bds': '2', 'last_update': '2021-07-26T21:32:21.259987', 'price': 898000, 'sqft': '1,443', 'zip_code': 94014}, {'bds': '2', 'ba': '2', 'sqft': '940'}, {'bds': '2', 'ba': '2', 'sqft': '1,394'}, {'bds': '2', 'ba': '2', 'sqft': '1,158'}, {'bds': '3', 'ba': '2', 'sqft': '1,640'}, {'bds': '3', 'ba': '3', 'sqft': '1,472'}, {'bds': '4', 'ba': '2', 'sqft': '1,770'}, {'bds': '2', 'ba': '1', 'sqft': '560'}, {'bds': '4', 'ba': '2', 'sqft': '1,770'}]

made it

break
9
[{'acres': -1, 'address': '543 Green Ridge Dr APT 4, Daly City, CA 94014', 'ba': '2', 'bds': '2', 'last_update': '2021-07-26T21:32:21.259987', 'price': 898000, 'sqft': '1,443', 'zip_code': 94014}, {'acres': -1, 'address': '340 San Diego Ave, Daly City, CA 94014', 'ba': '2', 'bds': '2', 'last_update': '2021-07-26T21:32:21.259987', 'price': 870000, 'sqft': '940', 'zip_code': 94014}, {'bds': '2', 'ba': '2', 'sqft': '1,394'}, {'bds': '2', 'ba': '2', 'sqft': '1,158'}, {'bds': '3', 'ba': '2', 'sqft': '1,640'}, {'bds': '3', 'ba': '3', 'sqft': '1,472'}, {'bds': '4', 'ba': '2', 'sqft': '1,770'}, {'bds': '2', 'ba': '1', 'sqft': '560'}, {'bds': '4', 'ba': '2', 'sqft': '1,770'}]

made it

break
9
[{'acres': -1, 'address': '543 Green Ridge Dr APT 4, Daly City, CA 94014', 'ba': '2', 'bds': '2', 'last_update': '2021-07-26T21:32:21.259987', 'price': 898000, 'sqft': '1,443', 'zip_code': 94014}, {'acres': -1, 'address': '340 San Diego Ave, Daly City, CA 94014', 'ba': '2', 'bds': '2', 'last_update': '2021-07-26T21:32:21.259987', 'price': 870000, 'sqft': '940', 'zip_code': 94014}, {'acres': -1, 'address': '691 Pointe Pacific Dr #4701, Daly City, CA 94014', 'ba': '2', 'bds': '2', 'last_update': '2021-07-26T21:32:21.259987', 'price': 854000, 'sqft': '1,394', 'zip_code': 94014}, {'bds': '2', 'ba': '2', 'sqft': '1,158'}, {'bds': '3', 'ba': '2', 'sqft': '1,640'}, {'bds': '3', 'ba': '3', 'sqft': '1,472'}, {'bds': '4', 'ba': '2', 'sqft': '1,770'}, {'bds': '2', 'ba': '1', 'sqft': '560'}, {'bds': '4', 'ba': '2', 'sqft': '1,770'}]

made it

break
9
[{'acres': -1, 'address': '543 Green Ridge Dr APT 4, Daly City, CA 94014', 'ba': '2', 'bds': '2', 'last_update': '2021-07-26T21:32:21.259987', 'price': 898000, 'sqft': '1,443', 'zip_code': 94014}, {'acres': -1, 'address': '340 San Diego Ave, Daly City, CA 94014', 'ba': '2', 'bds': '2', 'last_update': '2021-07-26T21:32:21.259987', 'price': 870000, 'sqft': '940', 'zip_code': 94014}, {'acres': -1, 'address': '691 Pointe Pacific Dr #4701, Daly City, CA 94014', 'ba': '2', 'bds': '2', 'last_update': '2021-07-26T21:32:21.259987', 'price': 854000, 'sqft': '1,394', 'zip_code': 94014}, {'acres': -1, 'address': '535 Mountain View Dr APT 8, Daly City, CA 94014', 'ba': '2', 'bds': '2', 'last_update': '2021-07-26T21:32:21.259987', 'price': 749000, 'sqft': '1,158', 'zip_code': 94014}, {'bds': '3', 'ba': '2', 'sqft': '1,640'}, {'bds': '3', 'ba': '3', 'sqft': '1,472'}, {'bds': '4', 'ba': '2', 'sqft': '1,770'}, {'bds': '2', 'ba': '1', 'sqft': '560'}, {'bds': '4', 'ba': '2', 'sqft': '1,770'}]

made it

break
9
[{'acres': -1, 'address': '543 Green Ridge Dr APT 4, Daly City, CA 94014', 'ba': '2', 'bds': '2', 'last_update': '2021-07-26T21:32:21.259987', 'price': 898000, 'sqft': '1,443', 'zip_code': 94014}, {'acres': -1, 'address': '340 San Diego Ave, Daly City, CA 94014', 'ba': '2', 'bds': '2', 'last_update': '2021-07-26T21:32:21.259987', 'price': 870000, 'sqft': '940', 'zip_code': 94014}, {'acres': -1, 'address': '691 Pointe Pacific Dr #4701, Daly City, CA 94014', 'ba': '2', 'bds': '2', 'last_update': '2021-07-26T21:32:21.259987', 'price': 854000, 'sqft': '1,394', 'zip_code': 94014}, {'acres': -1, 'address': '535 Mountain View Dr APT 8, Daly City, CA 94014', 'ba': '2', 'bds': '2', 'last_update': '2021-07-26T21:32:21.259987', 'price': 749000, 'sqft': '1,158', 'zip_code': 94014}, {'acres': -1, 'address': '157 Westlake Ave, Daly City, CA 94014', 'ba': '2', 'bds': '3', 'last_update': '2021-07-26T21:32:21.259987', 'price': 998888, 'sqft': '1,640', 'zip_code': 94014}, {'bds': '3', 'ba': '3', 'sqft': '1,472'}, {'bds': '4', 'ba': '2', 'sqft': '1,770'}, {'bds': '2', 'ba': '1', 'sqft': '560'}, {'bds': '4', 'ba': '2', 'sqft': '1,770'}]

made it

break
9
[{'acres': -1, 'address': '543 Green Ridge Dr APT 4, Daly City, CA 94014', 'ba': '2', 'bds': '2', 'last_update': '2021-07-26T21:32:21.259987', 'price': 898000, 'sqft': '1,443', 'zip_code': 94014}, {'acres': -1, 'address': '340 San Diego Ave, Daly City, CA 94014', 'ba': '2', 'bds': '2', 'last_update': '2021-07-26T21:32:21.259987', 'price': 870000, 'sqft': '940', 'zip_code': 94014}, {'acres': -1, 'address': '691 Pointe Pacific Dr #4701, Daly City, CA 94014', 'ba': '2', 'bds': '2', 'last_update': '2021-07-26T21:32:21.259987', 'price': 854000, 'sqft': '1,394', 'zip_code': 94014}, {'acres': -1, 'address': '535 Mountain View Dr APT 8, Daly City, CA 94014', 'ba': '2', 'bds': '2', 'last_update': '2021-07-26T21:32:21.259987', 'price': 749000, 'sqft': '1,158', 'zip_code': 94014}, {'acres': -1, 'address': '157 Westlake Ave, Daly City, CA 94014', 'ba': '2', 'bds': '3', 'last_update': '2021-07-26T21:32:21.259987', 'price': 998888, 'sqft': '1,640', 'zip_code': 94014}, {'acres': -1, 'address': '17 Honeysuckle Ct, Daly City, CA 94014', 'ba': '3', 'bds': '3', 'last_update': '2021-07-26T21:32:21.259987', 'price': 1049888, 'sqft': '1,472', 'zip_code': 94014}, {'bds': '4', 'ba': '2', 'sqft': '1,770'}, {'bds': '2', 'ba': '1', 'sqft': '560'}, {'bds': '4', 'ba': '2', 'sqft': '1,770'}]

made it

break
9
[{'acres': -1, 'address': '543 Green Ridge Dr APT 4, Daly City, CA 94014', 'ba': '2', 'bds': '2', 'last_update': '2021-07-26T21:32:21.259987', 'price': 898000, 'sqft': '1,443', 'zip_code': 94014}, {'acres': -1, 'address': '340 San Diego Ave, Daly City, CA 94014', 'ba': '2', 'bds': '2', 'last_update': '2021-07-26T21:32:21.259987', 'price': 870000, 'sqft': '940', 'zip_code': 94014}, {'acres': -1, 'address': '691 Pointe Pacific Dr #4701, Daly City, CA 94014', 'ba': '2', 'bds': '2', 'last_update': '2021-07-26T21:32:21.259987', 'price': 854000, 'sqft': '1,394', 'zip_code': 94014}, {'acres': -1, 'address': '535 Mountain View Dr APT 8, Daly City, CA 94014', 'ba': '2', 'bds': '2', 'last_update': '2021-07-26T21:32:21.259987', 'price': 749000, 'sqft': '1,158', 'zip_code': 94014}, {'acres': -1, 'address': '157 Westlake Ave, Daly City, CA 94014', 'ba': '2', 'bds': '3', 'last_update': '2021-07-26T21:32:21.259987', 'price': 998888, 'sqft': '1,640', 'zip_code': 94014}, {'acres': -1, 'address': '17 Honeysuckle Ct, Daly City, CA 94014', 'ba': '3', 'bds': '3', 'last_update': '2021-07-26T21:32:21.259987', 'price': 1049888, 'sqft': '1,472', 'zip_code': 94014}, {'acres': -1, 'address': '314 E Moltke St, Daly City, CA 94014', 'ba': '2', 'bds': '4', 'last_update': '2021-07-26T21:32:21.259987', 'price': 1250000, 'sqft': '1,770', 'zip_code': 94014}, {'bds': '2', 'ba': '1', 'sqft': '560'}, {'bds': '4', 'ba': '2', 'sqft': '1,770'}]

made it

break
9
[{'acres': -1, 'address': '543 Green Ridge Dr APT 4, Daly City, CA 94014', 'ba': '2', 'bds': '2', 'last_update': '2021-07-26T21:32:21.259987', 'price': 898000, 'sqft': '1,443', 'zip_code': 94014}, {'acres': -1, 'address': '340 San Diego Ave, Daly City, CA 94014', 'ba': '2', 'bds': '2', 'last_update': '2021-07-26T21:32:21.259987', 'price': 870000, 'sqft': '940', 'zip_code': 94014}, {'acres': -1, 'address': '691 Pointe Pacific Dr #4701, Daly City, CA 94014', 'ba': '2', 'bds': '2', 'last_update': '2021-07-26T21:32:21.259987', 'price': 854000, 'sqft': '1,394', 'zip_code': 94014}, {'acres': -1, 'address': '535 Mountain View Dr APT 8, Daly City, CA 94014', 'ba': '2', 'bds': '2', 'last_update': '2021-07-26T21:32:21.259987', 'price': 749000, 'sqft': '1,158', 'zip_code': 94014}, {'acres': -1, 'address': '157 Westlake Ave, Daly City, CA 94014', 'ba': '2', 'bds': '3', 'last_update': '2021-07-26T21:32:21.259987', 'price': 998888, 'sqft': '1,640', 'zip_code': 94014}, {'acres': -1, 'address': '17 Honeysuckle Ct, Daly City, CA 94014', 'ba': '3', 'bds': '3', 'last_update': '2021-07-26T21:32:21.259987', 'price': 1049888, 'sqft': '1,472', 'zip_code': 94014}, {'acres': -1, 'address': '314 E Moltke St, Daly City, CA 94014', 'ba': '2', 'bds': '4', 'last_update': '2021-07-26T21:32:21.259987', 'price': 1250000, 'sqft': '1,770', 'zip_code': 94014}, {'acres': -1, 'address': '172 Hillcrest Dr, Daly City, CA 94014', 'ba': '1', 'bds': '2', 'last_update': '2021-07-26T21:32:21.259987', 'price': 755000, 'sqft': '560', 'zip_code': 94014}, {'bds': '4', 'ba': '2', 'sqft': '1,770'}]

made it

break
9
[{'acres': -1, 'address': '543 Green Ridge Dr APT 4, Daly City, CA 94014', 'ba': '2', 'bds': '2', 'last_update': '2021-07-26T21:32:21.259987', 'price': 898000, 'sqft': '1,443', 'zip_code': 94014}, {'acres': -1, 'address': '340 San Diego Ave, Daly City, CA 94014', 'ba': '2', 'bds': '2', 'last_update': '2021-07-26T21:32:21.259987', 'price': 870000, 'sqft': '940', 'zip_code': 94014}, {'acres': -1, 'address': '691 Pointe Pacific Dr #4701, Daly City, CA 94014', 'ba': '2', 'bds': '2', 'last_update': '2021-07-26T21:32:21.259987', 'price': 854000, 'sqft': '1,394', 'zip_code': 94014}, {'acres': -1, 'address': '535 Mountain View Dr APT 8, Daly City, CA 94014', 'ba': '2', 'bds': '2', 'last_update': '2021-07-26T21:32:21.259987', 'price': 749000, 'sqft': '1,158', 'zip_code': 94014}, {'acres': -1, 'address': '157 Westlake Ave, Daly City, CA 94014', 'ba': '2', 'bds': '3', 'last_update': '2021-07-26T21:32:21.259987', 'price': 998888, 'sqft': '1,640', 'zip_code': 94014}, {'acres': -1, 'address': '17 Honeysuckle Ct, Daly City, CA 94014', 'ba': '3', 'bds': '3', 'last_update': '2021-07-26T21:32:21.259987', 'price': 1049888, 'sqft': '1,472', 'zip_code': 94014}, {'acres': -1, 'address': '314 E Moltke St, Daly City, CA 94014', 'ba': '2', 'bds': '4', 'last_update': '2021-07-26T21:32:21.259987', 'price': 1250000, 'sqft': '1,770', 'zip_code': 94014}, {'acres': -1, 'address': '172 Hillcrest Dr, Daly City, CA 94014', 'ba': '1', 'bds': '2', 'last_update': '2021-07-26T21:32:21.259987', 'price': 755000, 'sqft': '560', 'zip_code': 94014}, {'acres': -1, 'address': '309 E Moltke St, Daly City, CA 94014', 'ba': '2', 'bds': '4', 'last_update': '2021-07-26T21:32:21.259987', 'price': 1199000, 'sqft': '1,770', 'zip_code': 94014}]

【问题讨论】：

  • zillow 得到了很多回答！你甚至不需要硒！检查stackoverflow.com/a/68312200/7658985
  • @αԋɱҽԃαмєяιcαη 谢谢，您的链接看起来就像我正在寻找的一样。如何修改 params 参数以根据邮政编码拉取？
  • @user3476463 你试过我的答案了吗？

标签： python-3.x selenium selenium-webdriver beautifulsoup selenium-chromedriver


【解决方案1】：

我建议你使用 Selenium 方法。滚动直到页脚元素可见。
像这样的：

from selenium.webdriver.common.action_chains import ActionChains

actions = ActionChains(driver)

def property_details(zip_code):

#here comes all your code
------------------
-----------------
#after it just add this:
footer = driver.find_elements_by_css_selector('div#region-info-footer')
#if element is existing on the page the list will be non-empty and interpreted as True by python`
if(footer):
    if not footer[0].is_displayed():
        #scroll the last currently available price_parse element into view. 
        #this will load a bulk of new elements
        actions.move_to_element(price_parse[-1]).perform()
        time.sleep(0.5)
    else:
        #you can also use this case to exit the loop since you already scraped all the data on this page
        break
else:        
    #if footer element is even not existing we 100% still not reached the bottom
    actions.move_to_element(price_parse[-1]).perform()
    time.sleep(0.5)

【讨论】：

  • 感谢您的建议。我尝试实施它。我在我尝试过您的建议的原始帖子中添加了更新。它似乎在没有滚动的情况下退出了 break 语句。有什么我错过的吗，我是否正确实施了它？
  • 如果不使用那个中断怎么办？您还有其他情况可以退出循环吗？尝试使用 break 删除 else ，看看它是否有效
  • 我尝试注释掉中断。它运行了几次，但在我更改代码以检查每个循环的输出之前，zillow 现在在 google chrome 尝试启动浏览器后立即用验证码打击我。关于如何处理的任何提示？
  • 不，我不知道如何克服这个问题。这绝对是一个新的、额外的问题。所以我想我尽可能地回答了这个问题。所以请接受这个答案。
  • 看来，如果我等待一段时间，我可以再次运行代码而不会被验证码命中。所以每小时必须有某种限制。我已经在我的原始帖子中发布了另一个更新，我尝试将您的代码与我的代码结合起来。它似乎没有滚动侧边栏来拉入新属性。相反，它似乎混合了一种或两种不同的
猜你喜欢
  • 1970-01-01
  • 1970-01-01
  • 1970-01-01
  • 1970-01-01
  • 2019-09-29
  • 1970-01-01
  • 2019-05-06
  • 2018-11-08
  • 2021-12-28
相关资源
最近更新 更多
热门标签
Java Python linux javascript Mysql C# Docker 算法 前端 SpringBoot Redis Vue spring 设计模式 .net core .net kubernetes c++ 数据库 数据结构 大数据 js 机器学习 微服务 Android Go 程序员 面试 JVM ASP.net core 云原生 人工智能 后端 PHP git CSS golang k8s Nginx Django mybatis 深度学习 多线程 React 架构 devops 爬虫 云计算 Spring Boot LeetCode