python爬虫——批量下载百度图片

程序简介
程序/数据集下载
代码分析

程序简介

百度图片爬虫的封装接口2018年实现的，现在还能用...不错，谢谢百度的不封之恩，先将其贡献给所有热爱技术的开发者
输入：关键词、下载数量、重定尺寸(可省)
输出：自动创建文件夹下载对应数量的百度图片，图片由md5命令

程序/数据集下载

代码分析

导入模块

import numpy as np
import hashlib
import requests
import json
import cv2
import os

evalMd5函数用来计算图片md5，好进行命名和过滤相同图片

def evalMd5(sentence,charset=\'utf8\'):
    \'\'\'
    计算一段字符串的md5
    :param sentence: 字符串
    :param charset: 字符集
    :return: md5值
    \'\'\'
    #将字符串编码成bytes
    if type(sentence) != bytes:
        sentence = sentence.encode(charset)
    md5 = hashlib.md5(sentence).hexdigest()
    return md5

resizeImg函数用来重定图片尺寸

def resizeImg(oldPath,size,newPath):
    \'\'\'
    重定图片尺寸
    :param oldPath: 图片路径
    :param size: 重定大小
    :param newPath: 图片保存路径
    :return: None      
    \'\'\'
    oldPath = oldPath.replace(\'\\\',\'/\')
    newPath = newPath.replace(\'\\\',\'/\')
    oldImg = cv2.imdecode(np.fromfile(oldPath,dtype=np.uint8),-1)
    try:
        newImg = cv2.resize(oldImg,size,) #为图片重新指定尺寸
        cv2.imwrite(newPath,newImg)
        cv2.imencode(\'.\'+newPath.split(\'.\')[-1],newImg)[1].tofile(newPath)
    except:
        #图片格式不对发生错误，删除
        os.remove(oldPath)

核心函数download会调用上面的函数进行批量图片下载

def download(keyWord,imgNumber,imgSize=None):
    \'\'\'
    下载图片到关键词文件夹
    :param keyWord: 关键词
    :param imgNumber: 图片数量
    :param imgSize: 图片重定大小
    :return: None   
    \'\'\'
    #创建关键词文件夹
    dirname = keyWord
    if not os.path.exists(dirname):
        os.mkdir(dirname)
    #开始爬图片
    url = \'https://image.baidu.com/search/acjson\'#图片网址
    same = 0#重复下载数
    error = 0#错误数
    passNum = 0#无链接数
    for i in range(30,30*10000+30,30):
        param = {
            \'tn\': \'resultjson_com\',\'ipn\': \'rj\',
            \'ct\': 201326592,
            \'is\': \'\',
            \'fp\': \'result\',
            \'queryWord\': keyWord,
            \'cl\': 2,
            \'lm\': -1,
            \'ie\': \'utf-8\',
            \'oe\': \'utf-8\',
            \'adpicid\': \'\',
            \'st\': -1,
            \'z\': \'\',
            \'ic\': 0,
            \'word\': keyWord,
            \'s\': \'\',
            \'se\': \'\',
            \'tab\': \'\',
            \'width\': \'\',
            \'height\': \'\',
            \'face\': 0,
            \'istype\': 2,
            \'qc\': \'\',
            \'nc\': 1,
            \'fr\': \'\',
            \'pn\': i,
            \'rn\': 30,
            \'gsm\': \'1e\',
            \'1488942260214\': \'\'
            }
        #所有图片地址列表
        data = requests.get(url,params=param).text.replace(\'\\\',\'\\\\\')
        try:
            data = json.loads(data)[\'data\']
        except:
            #json数据可能不合法,直接跳过
            error += 1
            if error >=20:
                return None
            continue
        
        for item in data:
            imgUrl = item.get("middleURL")#图片地址
            if passNum>=20:
                return None
            if imgUrl is None:
                passNum+=1
                continue
            suffix = imgUrl.split(\'.\')[-1]#图片后缀
            imgContent = requests.get(imgUrl).content#图片内容
            imgMd5 = evalMd5(imgContent)#图片md5
            imgPath = os.path.join(dirname,\'%s.%s\'%(imgMd5,suffix))#图片路径
            oldFinish = len(os.listdir(dirname))
            open(imgPath, \'wb\').write(imgContent)#写入
            #重定尺寸
            if imgSize:
                resizeImg(imgPath,imgSize,imgPath)
            newFinish = len(os.listdir(dirname))
            print(\'key:%s goal:%d finish:%d\'%(keyWord,imgNumber,newFinish))
            #图片数达标,退出
            if newFinish >= imgNumber:
                return None
            #重复下载图片达到100次，说明已经下载完所有图片，退出
            if newFinish == oldFinish:
                same+=1
            if same >= 20:
                return

来测试一下看看效果吧~

imgNumber = 10
keys = [\'电子琴\',\'苹果\']
imgSize = None
for keyWord in keys:
    download(keyWord,imgNumber,imgSize)

key:电子琴 goal:10 finish:1
key:电子琴 goal:10 finish:2
key:电子琴 goal:10 finish:3
key:电子琴 goal:10 finish:4
key:电子琴 goal:10 finish:5
key:电子琴 goal:10 finish:6
key:电子琴 goal:10 finish:7
key:电子琴 goal:10 finish:8
key:电子琴 goal:10 finish:9
key:电子琴 goal:10 finish:10
key:苹果 goal:10 finish:1
key:苹果 goal:10 finish:2
key:苹果 goal:10 finish:3
key:苹果 goal:10 finish:4
key:苹果 goal:10 finish:5
key:苹果 goal:10 finish:6
key:苹果 goal:10 finish:7
key:苹果 goal:10 finish:8
key:苹果 goal:10 finish:9
key:苹果 goal:10 finish:10