我的第一个爬虫，爬取北京地区短租房信息

# 导入程序所需要的库。
import requests
from bs4 import BeautifulSoup
import time

# 加入请求头伪装成浏览器
headers = {
    #通过Chrome浏览器复制User-Agent
    \'User-Agent\': \'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36\'
}


# 定义判断用户性别的函数
def judgment_sex(class_name):
    if class_name == [\'member_ico1\']:
        return \'女\'
    else:
        return \'男\'


# 获取详细页URL函数
def get_links(url):
    try:
        wb_date = requests.get(url, headers)
    except ConnectionAbortedError:
        print(\'拒绝连接\')
    soup = BeautifulSoup(wb_date.text, \'lxml\')
    links = soup.select(\'#page_list > ul > li > a\')
    for link in links:
        herf = link.get("href")
        get_info(herf)


# 获取网页信息函数
def get_info(url):
    wb_date = requests.get(url, headers)
    soup = BeautifulSoup(wb_date.text, \'lxml\')
    #通过浏览器copy selector
    tittles = soup.select(\'div.pho_info > h4\')
    addresses = soup.select(\'span.pr5\')
    prises = soup.select(\'#pricePart > div.day_l > span\')
    images = soup.select(\'#floatRightBox > div.js_box.clearfix > div.member_pic > a > img\')
    names = soup.select(\'#floatRightBox > div.js_box.clearfix > div.w_240 > h6 > a\')
    sexs = soup.select(\'#floatRightBox > div.js_box.clearfix > div.member_pic > div\')
    for tittle, address, prise, image, name, sex in zip(tittles, addresses, prises, images, names, sexs):
        date = {
            \'tittle\': tittle.get_text().strip(),
            \'address\': address.get_text().strip(),
            \'price\': prise.get_text(),
            \'image\': image.get("src"),
            \'name\': name.get_text(),
            \'sex\': judgment_sex(sex.get("class"))
        }
        print(date)


if __name__ == \'__main__\':

    urls = [\'http://bj.xiaozhu.com/search-duanzufang-p{}-0/\'.format(number) for number in range(1, 14)]
    for single_url in urls:
        get_links(single_url)
        #休眠十秒，防止被封IP
        time.sleep(10)

        #缺点：缺少IP管理，采用休眠方法，效率低