google image - 守护式等待

google图片抓取

google图片是base64加密的,而且base64后的信息放在script信息里面

import pymysql
from lxml import etree
import logging
import requests
import time
import threading
from threading import RLock
import re
import os

lock = RLock()
import base64
import ssl

ssl._create_default_https_context = ssl._create_unverified_context

# 添加日志

logging.basicConfig(
    level=logging.INFO,  # 定义输出到文件的log级别，大于此级别的都被输出
    format=\'%(asctime)s  %(filename)s  %(levelname)s : %(message)s\',  # 定义输出log的格式
    datefmt=\'%Y-%m-%d %H:%M:%S\',  # 时间
    filename=\'drugimagesError.log\',  # log文件名
    filemode=\'a\')  # 写入模式“w”或“a”


class google_images(object):

    def __init__(self):
        self.strat_record = 1
        self.end_record = 10000001
        self.db = pymysql.connect(host=\'localhost\', port=3306, database=\'yao_zhi\', user=\'root\', password=\'root\',
                                  charset=\'utf8\')
        self.cursor = self.db.cursor()
        self.headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36"}

        while True:
            self.parse_page()

    def parse_page(self):
        lock.acquire()
        num = self.cursor.execute(
            "select id, me_pizhunwenhao, me_name, me_jixing, me_key from guo_cai_jin_kou_yao_pin where id > {} limit 1000".format(
                self.strat_record))
        lock.release()
        if str(num) == str(0):
            exit()

        data_tuple = self.cursor.fetchall()
        threading_list = []
        for data_one in data_tuple:
            id = data_one[0]
            approvalNumber = data_one[1]
            drugName = data_one[2]
            dosageForm = data_one[3]
            try:
                specifications = re.findall(r".+?,", data_one[4])[0]
            except:
                specifications = data_one[4]
            self.strat_record = id
            logging.info("id:%s  approvalNumber:%s   drugName:%s   dosageForm:%s   specifications:%s" % (
                id, approvalNumber, drugName, dosageForm, specifications))
            print("id:%s  approvalNumber:%s   drugName:%s   dosageForm:%s   specifications:%s" % (
                id, approvalNumber, drugName, dosageForm, specifications))

            if str(id) == str(self.end_record):
                exit()
            lock.acquire()
            num = self.cursor.execute("select id from drugimages where approvalNumber = \'{}\' ".format(approvalNumber))
            lock.release()
            if not num:
                t = threading.Thread(target=self.parse_page_data,
                                     args=(id, approvalNumber, drugName, dosageForm, specifications,))
                t.start()
                threading_list.append(t)
                time.sleep(3)

        for t_one in threading_list:
            t_one.join()

    def parse_page_data(self, id, approvalNumber, drugName, dosageForm, specifications):
        print("id:%s  approvalNumber:%s   drugName:%s   specifications:%s" % (
            id, approvalNumber, drugName, specifications))
        keyword = drugName + \' \' + dosageForm + \' \' + specifications
        url = \'https://www.google.com/search?biw=1920&bih=900&tbm=isch&q=%s\' % keyword
        print(url)
        data_particular = etree.HTML(requests.get(url=url, headers=self.headers).content)
        images_list = data_particular.xpath(\'//span[@id="xjs"]/script/text()\')[0]
        images_list_link = re.findall(r\'data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD.*"\]?\', images_list)

        num = 0
        for link in images_list_link:
            num += 1
            url_link = link.replace(\'"]\', \'\')
            image = url_link.encode(\'utf-8\').decode(\'unicode_escape\')
            image_data = image.replace(\'data:image/jpeg;base64,\', \'\')
            drugsql = "insert into drugimages(approvalNumber, drugName, specifications, image, num, durgid) values(\'{}\', \'{}\', \'{}\', \'{}\', {}, {})"
            drugsql_data = drugsql.format(approvalNumber, drugName, specifications, image, int(num), int(id))
            print(\'sql_data:%s\' % drugsql_data)
            logging.info("id:%s  approvalNumber:%s   drugName:%s   specifications:%s" % (
                id, approvalNumber, drugName, specifications))
            lock.acquire()
            self.cursor.execute(drugsql_data)
            self.db.commit()
            lock.release()
            pic_content = base64.b64decode(image_data)
            page_id = int(id / 1000)
            file = \'./images/\' + \'page\' + str(page_id) + \'/\'
            if not os.path.exists(file):
                os.makedirs(file)
            files = file + \'id\' + str(id) + \'/\'
            if not os.path.exists(files):
                os.makedirs(files)
            file = open(files + str(approvalNumber) + \'-\' + str(num) + \'.jpg\', \'wb\')
            file.write(pic_content)
            file.close()
            if str(num) == str(30):
                break


if __name__ == \'__main__\':
    google_images()