如何在 20 个请求硒刮后等待 30 秒

Question

您好，我有一个 csv 文件，包含 300 个数据。

10 个请求后，网站停止给我结果。

如何在 10 个请求后暂停我的脚本 3 分钟

谢谢你

我的代码：

societelist =[]


import csv



with open('1.csv') as csvfile:
  reader = csv.reader(csvfile) 
  for row in reader:
    browser = webdriver.Firefox(options=options)
    browser.get("myurl".format(row[0]))
    time.sleep(20) 

    try:
        societe = browser.find_element_by_xpath('/html/body/div[3]/div[2]/div/div[1]/div[2]/div[1]/span[2]').text
    except NoSuchElementException:
        societe = 'Element not found'

    societelist.append(societe)
    print (row[0])
    browser.quit()

df = pd.DataFrame(list(zip(societelist)), columns=['societe'])

data = df.to_csv('X7878.csv', index=False)

Answer 1

用途：

import csv
societelist =[]

with open('1.csv') as csvfile:
    reader = csv.reader(csvfile) 
    for i, row in enumerate(reader):       # i gives the index of the row.
        browser = webdriver.Firefox(options=options)
        browser.get("myurl".format(row[0]))
        time.sleep(20) 

        try:
            societe = browser.find_element_by_xpath('/html/body/div[3]/div[2]/div/div[1]/div[2]/div[1]/span[2]').text
        except NoSuchElementException:
            societe = 'Element not found'

        societelist.append(societe)
        print(row[0])
        browser.quit()
        if not ((i+1) % 10):    
            time.sleep(180)

df = pd.DataFrame(list(zip(societelist)), columns=['societe'])
df.to_csv('X7878.csv', index=False)

在抓取后将每一行文本写入 Excel 的替代解决方案，而不是一次写入所有文本。

import csv
import win32com.client as win32

# Launch excel
excel = win32.Dispatch('Excel.Application')
excel.Visible = 1
wb = excel.Workbooks.Add()
ws = wb.Sheets(1)

# Read csv and scrape webpage
with open('1.csv') as csvfile:
    reader = csv.reader(csvfile) 
    for i, row in enumerate(reader):       # i gives the index of the row.
        browser = webdriver.Firefox(options=options)
        browser.get("myurl".format(row[0]))
        time.sleep(20) 

        try:
            societe = browser.find_element_by_xpath('/html/body/div[3]/div[2]/div/div[1]/div[2]/div[1]/span[2]').text
        except NoSuchElementException:
            societe = 'Element not found'

        # it may make sense to write the input text and the scraped value side by side.
        ws.Cells(i+1, 1).Value = row[0]
        ws.Cells(i+1, 2).Value = societe

        print(row[0], societe)
        browser.quit()
        if not ((i+1) % 10):    
            time.sleep(180)

# If you want to save the file programmatically and close excel.
path = r'C:\Users\jarodfrance\Documents\X7878.xlsx'
wb.SaveAs(path)
wb.Close()
excel.Quit()