花瓣网爬虫

notes：

1、 参考https://www.cnblogs.com/nan86150/p/4272452.html 提供的思路

2、 思路：

　　使用requests库得到起始地址https://huaban.com/favorite/beauty/

　　第一次匹配网页中的   "pin_id":(\d*?), "user_id":\d*? 得到当前界面的所有pin_id

　　通过id构造地址，进入网页下载高清图片

　　进入网页后解析网页对信息筛选

 　　第一次筛选：　　app\["page"\](.*?), "text_meta" 内容 #因为网页中id有重复，所以第一次筛选缩小范围

　　第二次：提取第一次的内容，得到id     

　　第二次提取其中的key

　　通过key构造图片地址进行下载

　　翻页：

　　为满足异步加载，构造地址

　　https://huaban.com/favorite/beauty/?ivhdm0s5&max=914688397&limit=20&wfl=1
　　max后的值是页面最后一个id，所以每次页面下载完后就返回最后一个id，重新构造地址继续下载
3、创建文件函数是通过当前时间来创建文件夹
4、代码格式较混乱，欢迎交流
5、下载500张左右会封锁ip，需要更换ip，暂时未设置更换

6、下载文件

import requests
imgurl = \'http://img.ivsky.com/img/tupian/t/201411/01/xiaohuangren_tupian-007.jpg\'

imgdata = requests.get(imgurl).content

with open(\'s.jpg\', \'wb\') as f:

    f.write(imgdata)

 

#!/usr/bin/env python
# encoding: utf-8
import requests
import re
import os
import time
import urllib.request

def createFile():
    global path
    filetime = time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime(time.time()))
    path = \'./flower_beauty/\' + filetime
    if not os.path.exists(\'./flower_beauty\'):
        os.mkdir(\'./flower_beauty\')

    if not os.path.exists(path):
        os.mkdir(path)
    return path

def downloadimg(key,path):

    imgurl=\'https://hbimg.b0.upaiyun.com/\'+str(key)[2:-2]+\'_fw658\'
    urllib.request.urlretrieve(imgurl,path+\'/\'+str(key)[2:-2]+\'.jpg\')
    # print(imgurl)
def validname(title):
    rstr=\'[\/\\\:\*\?\"\<\>\|]\'
    valid_title=re.sub(rstr,\'\',title)
    return valid_title
def get_img_key(id):
    #得到图片id
    home_url=\'https://huaban.com/pins/\'
    url=home_url+str(id)+\'/\'
    webdata=requests.get(url)
    data=webdata.text
    #数据筛选找到key
    firstRE = re.compile(r\'app\["page"\](.*?), "text_meta"\')
    firstdata = firstRE.findall(data)
    # print(firstdata)
    key=re.findall(\'"key":"(.*?)"\',str(firstdata))
    # print(key)
    return key

    #对key构造地址
#749a992fb659e939ebc4f5690da60a81fed54c405068e-Illc20_fw658
    # with open(\'./tempdata.txt\',\'w\',encoding=\'utf-8\') as tempw:
    #     tempw.write(data)
    #     tempw.close()
    # with open(\'./tempdata.txt\',\'r+\',encoding=\'utf-8\') as tempr:
    #     data1=tempr.read()
    #     picurl = picRE.findall(data1,re.S)
    #     print(picurl)
    #     tempr.close()
# def get_img_id():
    # web_data=requests.get("https://huaban.com/favorite/beauty/")



def get_id(beauty_url):
    web_data = requests.get(beauty_url)
    pinRE = re.compile(\'"pin_id":(\d*?), "user_id":\d*?,\')
    pinid = pinRE.findall(web_data.text)
    # a=0
    # for id in pinid:
    #     get_img(id)
    #     a+=1
    #     print("第{}个".format(a))
    return pinid

def unique_id(id_list):
    id_list=list(set(id_list))
    return id_list


def get_next(beauty_url=\'https://huaban.com/favorite/beauty/\'):
    id_list = get_id(beauty_url)[:]  # 解析页面当前的id
    id_set = list(set(id_list))
    id_list_uniqune = unique_id(id_list)  # 对id列表去重
    # print(id_list_uniqune)
    #循环下载函数
    count=0
    for id in id_list_uniqune:
        count+=1
        key = get_img_key(id)  # 通过id得到key
        downloadimg(key, path)  # 通过key和path下载文件
        print("download {}".format(count))
        # if count>5:
        #     break
    return id_list[-1]
\'\'\'
198368485\', \'918789198\', \'198369677\', \'198363750\', \'350010205\',
\'670115744\', \'172796713\', \'898076056\', \'917512201\', \'313298032\',
\'918586402\', \'918920381\', \'525798891\', \'313304628\', \'918079727\',
\'564055088\', \'898132640\', \'918784856\', \'917512519\', \'212946705\',
\'918894995\', \'303222393\', \'904194691\', \'918944339\', \'918955317\',
\'217881447\', \'350010146\', \'898133035\', \'198375016\', \'344794002\',
\'898133172\', \'855305051\', \'918917448\', \'918978911\', \'918780514\',
\'917512027\', \'918957316\', \'898132699\', \'918079809\', \'296216360\', \'
394382196\', \'218761604\', \'670115745\', \'918991904\', \'670115742\',
\'918944392\', \'917393726\', \'388327469\', \'346550851\', \'918973592\',
\'918901743\', \'917511868\', \'313303957\', \'898133006\', \'313303569\',
\'299703741\', \'204765314\', \'917395347\', \'918956902\', \'918788348\',
\'206411589\', \'918081274\', \'802643792\', \'918896254\', \'670115743\',
\'670115740\', \'918081360\', \'296218662\', \'918994440\', \'918585230\',
\'913841090\', \'823423111\', \'918915500\', \'346550562\', \'383143365\',
\'898133053\', \'918979535\', \'670115748\', \'802683220\', \'670115749\',
\'898132715\', \'898133143\', \'346550402\', \'670115751\', \'802643684\',
\'313295155\', \'297719115\', \'917511136\']
https://huaban.com/favorite/beauty/?ivhdm0s5&max=914688397&limit=20&wfl=1
\'\'\'
if __name__ == \'__main__\':
    # list1=get_next(beauty_url=\'https://huaban.com/favorite/beauty/\')
    # list2=get_next(beauty_url=\'https://huaban.com/favorite/beauty/?ivhdm0s5&max=914688397&limit=20&wfl=1\')
    # for li1 in list1:
    #     if li1 in list2:
    #         print("double {}".format(li1))
    #     else:
    #         print("none")
    path=createFile()#创建文件夹，返回路径供下载函数使用
    pages=100
    last_id=get_next()
    for page in range(1,pages):
        beauty_url = \'https://huaban.com/favorite/beauty/?ivhdm0s5&max={}&limit=20&wfl=1\'.format(last_id)
        last_id=get_next(beauty_url)