从谷歌图片中获取第 1 5 张图片

Question

我正在处理代码以从谷歌图像搜索中下载第一个 5 图像结果。但是，我遇到了以下代码的两个主要问题：

from bs4 import BeautifulSoup
import urllib.request
import os
import json

def get_soup(url,header):
    return BeautifulSoup(urllib.request.urlopen(urllib.request.Request(url,headers=header)),'html.parser')

query = input('>>> What image do you want? ') 
image_type=query
query= query.split()
query='+'.join(query)
url="https://www.google.co.in/search?q="+query+"&source=lnms&tbm=isch"
print ('>>> Base searching page from Google image:', url)

DIR="C:/Users/alex/Desktop/try"
header={'User-Agent':"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36"}
soup = get_soup(url,header)


ActualImages=[]# contains the link for Large original images, type of  image
for a in soup.find_all("div",{"class":"rg_meta"}):
    link , Type =json.loads(a.text)["ou"]  ,json.loads(a.text)["ity"]
    ActualImages.append((link,Type))

print('>>> Base page has', len(ActualImages),'images in total')

if not os.path.exists(DIR):
    os.mkdir(DIR)
DIR = os.path.join(DIR, query.split()[0])
if not os.path.exists(DIR):
    os.mkdir(DIR)

###print images
for i,(img,Type) in enumerate(ActualImages[:5]):
    try:
        req = urllib.request.Request(img, headers={'User-Agent' : header})
        raw_img = urllib.request.urlopen(req).read()

        cntr = len([i for i in os.listdir(DIR) if image_type in i]) + 1
        print(cntr)

        if len(Type)==0:
            f = open(os.path.join(DIR,image_type + "_"+ str(cntr)+".jpg"),'wb')
        else:
            f = open(os.path.join(DIR,image_type + "_"+ str(cntr)+"."+Type),'wb')
        f.write(raw_img)
        f.close()
    except Exception as e:
        print('>>> Could not load: '+img)
        print(e)

print ('>>> Finished!')

Q1：在行

req = urllib.request.Request(img, headers={'User-Agent' : header})

Python 将向我显示一个错误，说明预期的字符串或类似字节的对象，但如果我删除 headers={'User-Agent' : header} ，代码就可以正常工作。我知道标头充当许可，但是让它禁止代码运行是很奇怪的。有人可以帮忙解决这个问题吗？

Q2：根据几次测试，我有时会得到HTTP Error 403: Forbidden。我应该更改哪一部分以让 Python 继续尝试，直到我成功下载 5 次图像，而不是向我展示它尝试了 5 次但下载失败 1 次？

Answer 1

问题出在请求的标头中

只需换行

req = urllib.request.Request(img, headers={'User-Agent' : header})

到

req = urllib.request.Request(img, headers=header)

修改后的代码

from bs4 import BeautifulSoup
import urllib.request
import os
import json

def get_soup(url,header):
    return BeautifulSoup(urllib.request.urlopen(urllib.request.Request(url,headers=header)),'html.parser')

query = input('>>> What image do you want? ') 
image_type=query
query= query.split()
query='+'.join(query)
url="https://www.google.co.in/search?q="+query+"&source=lnms&tbm=isch"
print ('>>> Base searching page from Google image:', url)

DIR="/home/fly/Documents/py/"
header={'User-Agent':"Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36"}
soup = get_soup(url,header)


ActualImages=[]# contains the link for Large original images, type of  image
for a in soup.find_all("div",{"class":"rg_meta"}):
    link , Type =json.loads(a.text)["ou"]  ,json.loads(a.text)["ity"]
    ActualImages.append((link,Type))

print('>>> Base page has', len(ActualImages),'images in total')

if not os.path.exists(DIR):
    os.mkdir(DIR)
DIR = os.path.join(DIR, query.split()[0])
if not os.path.exists(DIR):
    os.mkdir(DIR)

###print images
for i,(img,Type) in enumerate(ActualImages[:5]):
    try:
        req = urllib.request.Request(img, headers=header)
        raw_img = urllib.request.urlopen(req).read()

        cntr = len([i for i in os.listdir(DIR) if image_type in i]) + 1
        print(cntr)

        if len(Type)==0:
            f = open(os.path.join(DIR,image_type + "_"+ str(cntr)+".jpg"),'wb')
        else:
            f = open(os.path.join(DIR,image_type + "_"+ str(cntr)+"."+Type),'wb')
        f.write(raw_img)
        f.close()
    except Exception as e:
        print('>>> Could not load: '+img)
        print(e)

print ('>>> Finished!')

输出

>>> What image do you want? cat
>>> Base searching page from Google image: https://www.google.co.in/search?q=cat&source=lnms&tbm=isch
>>> Base page has 100 images in total
1
2
3
4
5
>>> Finished!