无法加载图片，图片 url 自动更改 url

Question

我在学校有一个电子商务网站项目，我需要大量图片才能完成。所以我查阅了从 Youtube 下载图片的代码：John Watson Rooney 但是我在下载图像的一半时遇到问题 Url 更改为 'data:image/gif;base64,R0lGODdhFQAXAPAAANba3wAAACwAAAAAFQAXAAACFISPqcvtD6OctNqLs968+w+GolUAADs=' 所以我无法继续下载。

import requests
from bs4 import BeautifulSoup
import os
import base64

def imagedown(url, folder):
    try:
        os.mkdir(os.path.join(os.getcwd(), folder))
    except:
        pass
    os.chdir(os.path.join(os.getcwd(), folder))
    r = requests.get(url)
    soup = BeautifulSoup(r.text, 'html.parser')
    images = soup.find_all('img', class_='styles__productImage--3ZNPD')
    for image in images:
        name = image['alt']
        link = image['src']
        with open(name.replace('/', '').replace('?', '').replace('=', '').replace('|', '') + '.jpg', 'wb') as f:
            im = requests.get(link)
            f.write(im.content)
            print('Writing: ', name)

imagedown('https://www.redbubble.com/shop/?gender=gender-men&iaCode=u-tees&page=2&query=dog&sortOrder=relevant&style=u-tee-regular-crew', 'Images')

不知道哪里出错了，请帮帮我，谢谢

Answer 1

问题在于，当您向下滚动页面时，您尝试获取的图像会动态加载，因此您可以尝试使用 Selenium 之类的网络抓取工具，而不是使用 BeautifulSoup：

首先，使用pip install selenium安装它。

从以下URL下载谷歌浏览器驱动并保存，最新版本的谷歌浏览器也必须安装在你的电脑上。

这是下载图像的脚本，如您所见，我实现了一个循环以缓慢向下滚动以确保在下载图像之前加载图像：

import os
import urllib.request
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
import time

# set path to your chromedriver.exe
driver = webdriver.Chrome(executable_path="C:\\chromedriver.exe")
driver.maximize_window()
# launch URL
driver.get("https://www.redbubble.com/shop/?gender=gender-men&iaCode=u-tees&page=2&query=dog&sortOrder=relevant&style=u-tee-regular-crew")
driver.implicitly_wait(3)

body = driver.find_element_by_tag_name('body')

# Scroll the page slowly by sending down arrow keys and pausing between each stroke
scroll_pause_time = 0.05
scroll_strokes = 400

for i in range(scroll_strokes):
    # Send arrow down
    body.send_keys(Keys.ARROW_DOWN)
    # Wait to load page
    time.sleep(scroll_pause_time)

# find images
images = driver.find_elements_by_class_name('styles__productImage--3ZNPD')
save_folder = 'c:/temp/images/'

for image in images:
    name = image.get_attribute('alt')
    link = image.get_attribute('src')
    ext =  os.path.splitext(link)[1]
    urllib.request.urlretrieve(link, save_folder + name.replace('/', '').replace('?', '').replace('=', '').replace('|', '').replace('"', '') + ext)

# close browser
driver.quit()

Answer 2

这些图像被编码为base64字符串，所以你不需要下载它们，你可以简单地保存它们如下：

import requests
from bs4 import BeautifulSoup
from urllib.request import urlopen
import os
import re

def imagedown(url, folder):
    try:
        os.mkdir(os.path.join(os.getcwd(), folder))
    except:
        pass
    os.chdir(os.path.join(os.getcwd(), folder))
    r = requests.get(url)
    soup = BeautifulSoup(r.text, 'html.parser')
    images = soup.find_all('img', class_='styles__productImage--3ZNPD')
    for image in images:
        name = image['alt']
        link = image['src']
        ext = None
        data = None

        if link.startswith('data'):
            with urlopen(link) as response:
                if link.startswith('data:image/gif'):
                    ext = '.gif'
                data = response.read()
        else:
            ext = os.path.splitext(link)[1]
            data = requests.get(link).content
               
        with open(name.replace('/', '').replace('?', '').replace('=', '').replace('|', '') + ext, 'wb') as f:
            f.write(data)
            print('Writing: ', name)

imagedown('https://www.redbubble.com/shop/?gender=gender-men&iaCode=u-tees&page=2&query=dog&sortOrder=relevant&style=u-tee-regular-crew', 'Images')