爬去某家房价的数据

想看看老家的房价，所以写了脚本，供大家分享下

import requests
import time
from bs4 import BeautifulSoup
# 写入数据库
from mysql import close_db


@close_db
def write_db(con, param):
    try:
        sql = "insert into house(adress, total_price, avg_price, area, title, url) " \
              "VALUES (%s,%s,%s,%s,%s,%s)"
        con.execute(sql, param)
    except Exception as e:
        print(e)


# 主方法
def main():
    # 给请求指定一个请求头来模拟chrome浏览器
    headers = {\'User-Agent\': \'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 \'
                             \'(KHTML, like Gecko) Chrome/54.0.2840.99 Safari/537.36\'}
    page_max = 24
    # 爬取地址
    for i in range(1, int(page_max) + 1):
        print("第几页：" + str(i))
        if i == 1:
            house = \'https://ty.lianjia.com/loupan/rs/\'
        else:
            house = \'https://ty.lianjia.com/loupan/pg\' + str(i)
        res = requests.get(house, headers=headers)
        soup = BeautifulSoup(res.text, \'html.parser\')
        li_max = soup.find(\'ul\', class_=\'resblock-list-wrapper\').find_all(\'li\')
        for li in li_max:
            try:
                house_param = {}
                param = []
                # 所在小区
                title = li.find(\'div\', class_=\'resblock-name\').find(\'a\').text
                house_param[\'title\'] = title
                # 小区地址
                address = li.find(\'div\', class_=\'resblock-location\').text
                address = address.replace(\'\n\', \'\')
                house_param[\'address\'] = address
                # 访问链接
                title_url = li.find(\'a\').attrs[\'href\']
                title_url = \'https://ty.fang.lianjia.com\' + title_url
                # 均价
                avg_price = li.find(\'div\', class_=\'resblock-price\').find(\'div\', class_=\'main-price\').find(\'span\',
                                                                                                          class_=\'number\').text
                house_param[\'avg_price\'] = avg_price
                # 建筑面积
                square_metre = li.find(\'div\', class_=\'resblock-area\').text
                square_metre = square_metre.replace(\'\n\', \'\')
                house_param[\'area\'] = square_metre
                # 总价
                total_price = li.find(\'div\', class_=\'resblock-price\').find(\'div\', class_=\'second\').text
                house_param[\'total_price\'] = total_price
                param.append(address)
                param.append(total_price)
                param.append(avg_price)
                param.append(square_metre)
                param.append(title)
                param.append(title_url)
                write_db(param)
                time.sleep(5)
            except Exception as e:
                print(e)


if __name__ == \'__main__\':
    main()

迭代器自己封装的，还不太成熟没有关闭数据库链接

import functools

import pymysql

# 打开数据库连接
config = {
    \'host\': \'localhost\',
    \'port\': 3306,
    \'user\': \'root\',
    \'passwd\': \'root\',
    \'db\': \'test\'
}
db = pymysql.connect(**config)
con = db.cursor()


def close_db(func):
    @functools.wraps(func)
    def wrapper(*args, **kw):
        r = func(con, *args, **kw)
        db.commit()
        return r

    return wrapper