如何从网站上抓取所有图像并将它们链接到我在 csv 中的产品？

Question

我正在尝试抓取网站上有关产品的所有信息。我已经处理了所有的标题、描述、skus、价格并将它们放入 csv 中，但是我需要有一个列 images 来存储我下载图像的名称（最后的任务是导入它们全部使用magento2)。

问题在于，我认为并非所有图像都已下载，并且如果您要运行代码，则需要事先创建文件夹 images_test。

p>

我将在此处附上我的代码。我真的需要在一天结束之前完成这个。

import requests
from bs4 import BeautifulSoup
import csv
import pandas as pd
import os

baseurl = 'https://www.k-beauty.ro/magazin/'

# an array for all the product links
productLinks = [] 

# going through all the pages of the shop
for x in range(1,8):
    r = requests.get(f'https://www.k-beauty.ro/magazin/page/{x}')
    soup = BeautifulSoup(r.content, 'lxml')

    productlist = soup.find_all('div', class_= 'product-element-top')
    # taking all the links to each product page
    for item in productlist:
        for link in item.find_all('a', href=True, class_='product-image-link'):
            productLinks.append(link['href'])
            # appending the links previously taken to the array

ProductItemsList = []



i = 0
j = 0
d={}
imagesList = []
folder = 'images_test'
rootFolder = os.getcwd()


os.chdir(folder)

# going through all the product pages and getting the info we need
for linkTest in productLinks:
    r = requests.get(linkTest)
    soup = BeautifulSoup(r.content, 'lxml')
    title = soup.find('h1', class_='product_title').text.strip()
    price = soup.find('p', class_ = 'price').text.strip()
    header = soup.find('div', class_ = 'woocommerce-product-details__short-description').text.strip()
    sku  = soup.find('span', class_ = 'sku').text.strip()
    categories = soup.find('span' , class_ = 'posted_in').text.strip()
    description = soup.find('div', class_ = 'wc-tab-inner').text.strip()
    brand = soup.find('div', id = 'tab-pwb_tab-content').text.strip()
    
    

# getting all the images on the website
    images = soup.find_all('img', class_ = 'attachment-woocommerce_thumbnail size-woocommerce_thumbnail')
    for image in images:
        if image not in d:
            d[image]=1
        else:
            continue
        i = i + 1
        name = str(i) +'img'
        i = int(i)
        print(name)
        link = image['src']
        print(link)
        # here I am adding the .jpg and saving the images
        with open(name + '.jpg', 'wb') as f:
            im = requests.get(link)
            print("URMEAZA DEBUG: {}".format(im))
            f.write(im.content)
        imagesList.append(name)
        # storing all the info about this product 
        
    for name in imagesList:
        product = {
            'sku': sku,
            'categories': categories,
            'name' : title,
            'description': description,
            'short_description': header,
            'price' : price[0:5],
            'images': name,
        }
    ProductItemsList.append(product)


os.chdir(rootFolder)

df =pd.DataFrame(ProductItemsList)
print(df)
df.to_csv('./K_Beauty_test.csv')

我将添加 csv 的图像

Answer 1

这是我的版本。

我不知道它是否能解决所有问题，但我现在没有时间 - 我必须关闭我的电脑。

我仍然看到一个问题：它从 Produse Similare 获取图像 - 所以它可能需要不同的 soup.find_all 来获取产品图像

EDIT soup.select('.product-images-inner img') 将只选择当前产品的图像，而不是滑块）

import os
import requests
from bs4 import BeautifulSoup
import pandas as pd

folder = 'images_test'
os.makedirs(folder, exist_ok=True)

root_folder = os.getcwd()

baseurl = 'https://www.k-beauty.ro/magazin/'

# an array for all the product links
product_links = [] 

# going through all the pages of the shop
for x in range(1, 8):
    url = f'https://www.k-beauty.ro/magazin/page/{x}'
    print(url)
    
    r = requests.get(url)
    soup = BeautifulSoup(r.content, 'lxml')

    product_list = soup.find_all('div', class_= 'product-element-top')
    # taking all the links to each product page
    for item in product_list:
        for link in item.find_all('a', href=True, class_='product-image-link'):
            product_links.append(link['href'])
            # appending the links previously taken to the array

product_items_list = []

i = 0
d = {}  # use as set() 

os.chdir(folder)

for link_test in product_links:

    r = requests.get(link_test)
    soup = BeautifulSoup(r.content, 'lxml')
    
    title = soup.find('h1', class_='product_title').text.strip()
    price = soup.find('p', class_ = 'price').text.strip()
    header = soup.find('div', class_ = 'woocommerce-product-details__short-description').text.strip()
    sku  = soup.find('span', class_ = 'sku').text.strip()
    categories = soup.find('span' , class_ = 'posted_in').text.strip()
    description = soup.find('div', class_ = 'wc-tab-inner').text.strip()
    brand = soup.find('div', id = 'tab-pwb_tab-content').text.strip()

    images = soup.select('.product-images-inner img')
    
    # --- before `for`-loop ---
    
    downloaded = []
    
    # --- `for`-loop ---
    
    for image in images:
        link = image['src']
        if link in d:
            name = d[link]
            downloaded.append(name)
        else:
            i += 1
            name = str(i) +'img.jpg'
            d[link] = name
            print('link:', link)
            print('name:', name)
            print('---')
            # here i am adding the .jpg and saving the images
            with open(name, 'wb') as f:
                im = requests.get(link)
                #print("URMEAZA DEBUG: {}".format(im))
                f.write(im.content)
            downloaded.append(name)
            
    # --- after `for`-loop ---
    
    # storing all the infos about this product 
    product = {
        'sku': sku,
        'categories': categories,
        'name' : title,
        'description': description,
        'short_description': header,
        'price' : price[0:5],
        'images': downloaded,
    }
    product_items_list.append(product)


os.chdir(root_folder)

df = pd.DataFrame(product_items_list)
print(df)
df.to_csv('K_Beauty_test.csv')