#导入需要用到的模块
import requests
import pymysql
import time
from bs4 import BeautifulSoup
import tkinter as tk
import xlwt
import importlib,sys
importlib.reload(sys)
from PIL import Image,ImageTk
#背景图片
def resize( w_box, h_box, pil_image):
"""调整图片大小,适应窗体大小"""
"""arg:: w_box:new width h_box:new height pil_image:img"""
w, h = pil_image.size #获取图像的原始大小
f1 = 1.0*w_box/w
f2 = 1.0*h_box/h
factor = min([f1, f2])
width = int(w*factor)
height = int(h*factor)
return pil_image.resize((width, height), Image.ANTIALIAS)
#获取url下的页面内容，返回soup对象
def get_page(url):
responce = requests.get(url)
soup = BeautifulSoup(responce.text,\'html.parser\')
return soup
#封装成函数，作用是获取列表页下面的所有租房页面的链接，返回一个连接列表
def get_links(link_url):
soup = get_page(link_url)
links_div = soup.find_all(\'div\',class_="pic-panel")
links = [div.a.get(\'href\') for div in links_div]
return links
def get_house_info(house_url):
soup = get_page(house_url)
price = soup.find(\'span\',class_=\'total\').text #价格
unit = soup.find(\'span\',class_=\'unit\').text.strip() #单位 strip()函数去空格
house_info = soup.find_all(\'p\')
area = house_info[0].text[3:] #面积
layout = house_info[1].text[5:] #户型
floor = house_info[2].text[3:] #楼层
towards = house_info[3].text[5:] #朝向
subway = house_info[4].text[3:] #地铁
uptown = house_info[5].text[3:-8].strip() #小区
location = house_info[6].text[3:] #位置
info ={
\'价格\':price,
\'单位\':unit,
\'面积\':area,
\'户型\':layout,
\'楼层\':floor,
\'朝向\':towards,
\'地铁\':subway,
\'小区\':uptown,
\'位置\':location
}
return info
DATABASE = {
\'host\':\'localhost\',#如果是远程数据库，此处为远程服务器的ip地址
\'database\':\'examination\',
\'user\':\'root\',
\'password\':\'123456\',
\'charset\':\'utf8mb4\'
}
def get_db(setting):
return pymysql.connect(**setting)
def insert(db,house):
table_name=cityEntry.get()+\'_\'+localEntry.get()
values = "\'{}\',"* 8 +"\'{}\'"
sql_values = values.format(house[\'价格\'],house[\'单位\'],house[\'面积\'],house[\'户型\'],
house[\'楼层\'],house[\'朝向\'],house[\'地铁\'],house[\'小区\'],
house[\'位置\'])
sql = """
insert into {0}(price,unit,area,layout,floor,towards,subway,uptown,location)
values({1})
""".format(table_name,sql_values)
cursor = db.cursor()
cursor.execute(sql)
db.commit()
def creatTable(db):
table_name=cityEntry.get()+\'_\'+localEntry.get()
sql = """
CREATE TABLE `{}` (
`price` varchar(80) DEFAULT NULL,
`unit` varchar(80) DEFAULT NULL,
`area` varchar(80) DEFAULT NULL,
`layout` varchar(80) DEFAULT NULL,
`floor` varchar(80) DEFAULT NULL,
`towards` varchar(80) DEFAULT NULL,
`subway` varchar(80) DEFAULT NULL,
`uptown` varchar(80) DEFAULT NULL,
`location` varchar(80) DEFAULT NULL
);""".format(table_name)
cursor = db.cursor()
cursor.execute(sql)
db.commit()
def main():
db = get_db(DATABASE)
try:
creatTable(db)
except:
print("数据库已存在")
pass
num = int(numberEntry.get())
for i in range(num):
links = get_links("https://"+dict_loc[\'{}\'.format(cityEntry.get())]+".lianjia.com/zufang/"+dict_loc[\'{}\'.format(localEntry.get())]+"/pg{}/".format(i))
for link in links:
time.sleep(0.1)
house = get_house_info(link)
insert(db,house)
lableInit.config(text="{}市{}区数据获取成功".format(cityEntry.get(),localEntry.get()))
print(\'DONE\')
def quitw():
top.destroy()
def export():
db = get_db(DATABASE)
cursor = db.cursor()
table_name=cityEntry.get()+\'_\'+localEntry.get()
count = cursor.execute(\'select * from \'+table_name)
# 重置游标的位置
cursor.scroll(0,mode=\'absolute\')
# 搜取所有结果
results = cursor.fetchall()
# 获取MYSQL里面的数据字段名称
fields = cursor.description
workbook = xlwt.Workbook()
sheet = workbook.add_sheet(\'table_\'+table_name,cell_overwrite_ok=True)
# 写上字段信息
for field in range(0,len(fields)):
sheet.write(0,field,fields[field][0])
# 获取并写入数据段信息
row = 1
col = 0
for row in range(1,len(results)+1):
for col in range(0,len(fields)):
sheet.write(row,col,u\'%s\'%results[row-1][col])
workbook.save(r\'C:\Users\Lenovo\Desktop\{}.xls\'.format(table_name))
lableInit.config(text="共{}条数据导出成功!".format(count))
#构造字典
dict_loc = {
\'北京\':\'bj\',
\'东城\':\'dongcheng\',
\'西城\':\'xicheng\',
\'朝阳\':\'chaoyang\',
\'海淀\':\'haidian\',
\'丰台\':\'fengtai\',
\'上海\':\'sh\',
\'浦东\':\'pudong\',
\'宝山\':\'baoshan\',
\'杭州\':\'hz\',
\'西湖\':\'xihu\',
\'下城\':\'xiacheng\',
\'余杭\':\'yuhang\',
\'富阳\':\'fuyang\',
\'郑州\':\'zz\',
\'金水\':\'jinshui\',
\'中原\':\'zhongyuan\',
\'二七\':\'erqi\',
\'高新\':\'gaoxin\',
\'新郑市\':\'xinzhengshi\',
\'洛阳\':\'luoyang\',
\'嵩县\':\'songxian\',
\'新乡\':\'xinxiang\',
\'牧野\':\'muye\'
}
if __name__ == "__main__":
top = tk.Tk()
top.title("链家")
top.resizable(width=False,height=False)#设置不可拉伸
top.geometry("410x510") #设置窗口大小
#添加背景图片
canvas = tk.Canvas(top) #设置canvas
pil_image = Image.open(\'lianjiabg2.png\') #打开背景图片
pil_image_resize = resize(410,510,pil_image) #将它放大保存
im = ImageTk.PhotoImage(pil_image_resize)
canvas.create_image(205,255,image = im) #将图片加载到canvas来
canvas.place(x=0,y=0,height=510,width=410,anchor=\'nw\')#放到屏幕当中
#图片
photo = tk.PhotoImage(file="F:\CodeWorkspace\lianjia.png")
imgLabel = tk.Label(top,image=photo,bg=\'#fbfbfb\',width=410)
imgLabel.grid(row=0,column=0,columnspan=2)
#lable
Label = tk.Label(top,fg=\'#589e6e\',bg=\'#f9f7ba\',font = \'隶书 -20 \', text = "请输入您要查询的地区")
Label.grid(row=1,column=0,columnspan=2,pady=5)
#市
cityEntry = tk.Entry(top,width=12)
cityEntry.grid(row=2,column=0,padx=5,pady=10,sticky="E")
cityLabel = tk.Label(top,fg=\'#589e6e\',font = \'隶书 -20 \',bg=\'#d9f3e1\', text = "市")
cityLabel.grid(row=2,column=1,sticky="W")
#区
localEntry = tk.Entry(top,width=12)
localEntry.grid(row=3,column=0,padx=5,pady=10,sticky="E")
localLabel = tk.Label(top,fg=\'#589e6e\',font = \'隶书 -20 \',bg=\'#d9f3e1\', text = "区")
localLabel.grid(row=3,column=1,sticky="W")
#lable2
Label2 = tk.Label(top,fg=\'#589e6e\',bg=\'#f9f7ba\',font = \'隶书 -20 \', text = "请输入您要查询的页数\n(每页30条数据)")
Label2.grid(row=4,column=0,columnspan=2)
#信息数
numberEntry = tk.Entry(top,width=12)
numberEntry.grid(row=5,column=0,padx=5,pady=10,sticky="E")
numberLabel = tk.Label(top,fg=\'#589e6e\',font = \'隶书 -20 \',bg=\'#abe1c1\', text = "页")
numberLabel.grid(row=5,column=1,sticky="W")
#提交
submit = tk.Button(top,bg=\'#589e6e\',fg=\'white\',width=12,height=1,font = \'Helvetica -15 bold\', text="数据获取",command=main)
submit.grid(row=6,column=0,columnspan=2,padx=3,pady=5)
#lable3
Label3 = tk.Label(top,fg=\'#589e6e\',bg=\'#f9f7ba\',font = \'隶书 -20 \', text = "将数据导出为Excel格式")
Label3.grid(row=7,column=0,columnspan=2)
#导出excel
export = tk.Button(top,bg=\'#f9a33f\',fg=\'white\',width=12,height=1,font = \'Helvetica -15 bold\', text="导出数据",command=export)
export.grid(row=8,column=0,columnspan=2,padx=3,pady=5)
#退出
quitB = tk.Button(top,bg=\'#ff5757\',fg=\'white\',width=12,height=1,font = \'Helvetica -15 bold\', text="退出",command=quitw)
quitB.grid(row=9,column=0,columnspan=2,padx=3,pady=3)
#反馈
lableInitTitle = tk.Label(top,font = \'正楷 -12\',text="* * * 提 示 信 息 * * *",width=40,fg="#f9a33f")
lableInitTitle.grid(row=10,column=0,columnspan=2,ipady=5)
lableInit = tk.Label(top,bg=\'#d9f3e1\',font = \'正楷 -12 \',text="请在上方输入您要查询的信息",width=40,fg="red")
lableInit.grid(row=11,column=0,columnspan=2,ipady=5)