豆瓣图书排行250抓取练习

第二天，利用BeautifulSoup解析库

import requests
import re
import time
from bs4 import BeautifulSoup

def get_one_page(url):
    proxies = {"http":"http://61.145.69.27","http":"http://121.61.0.208"}
    headers = {'User-Agent' : 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36'}
    response = requests.get(url,headers=headers,proxies=proxies)
    if response.status_code == 200:
        return response.text
    return None
def parse_one_page(html):
    soup = BeautifulSoup(html, 'lxml')
    for td in soup.find_all(attrs={'valign':'top'}):
        for a in td.find(name='a'):
            print(a.string)
        for p in td.find_all(name='p'):
            print(p.string)
        for span in td.find_all(name='span'):
            print(span.string)
def main(offset):
    url = 'https://book.douban.com/top250?start='+str(offset)
    html = get_one_page(url)
    parse_one_page(html)
if __name__ == '__main__':
    for i in range(10):
        main(i * 25)
        time.sleep(1)

运行结果：
不知道如何选择抓取节点，结果中有许多[None]