如何使用 puppeteer 下载页面上的图像？

Question

我是网络抓取的新手，想使用 puppeteer 下载网页上的所有图像：

const puppeteer = require('puppeteer');

let scrape = async () => {
  // Actual Scraping goes Here...

  const browser = await puppeteer.launch({headless: false});
  const page = await browser.newPage();
  await page.goto('https://memeculture69.tumblr.com/');

  //   Right click and save images

};

scrape().then((value) => {
    console.log(value); // Success!
});

我查看了API‌ docs，但不知道如何实现这一点。所以感谢你的帮助。

Answer 1

如果您想跳过手动 dom 遍历，您可以直接从页面响应将图像写入磁盘。

例子：

const puppeteer = require('puppeteer');
const fs = require('fs');
const path = require('path');

(async () => {
    const browser = await puppeteer.launch();
    const page = await browser.newPage();
    page.on('response', async response => {
        const url = response.url();
        if (response.request().resourceType() === 'image') {
            response.buffer().then(file => {
                const fileName = url.split('/').pop();
                const filePath = path.resolve(__dirname, fileName);
                const writeStream = fs.createWriteStream(filePath);
                writeStream.write(file);
            });
        }
    });
    await page.goto('https://memeculture69.tumblr.com/');
    await browser.close();
})();

Answer 2

这是另一个例子。它进入谷歌的通用搜索并下载左上角的谷歌图片。

const puppeteer = require('puppeteer');
const fs = require('fs');

async function run() {
    const browser = await puppeteer.launch({
        headless: false
    });
    const page = await browser.newPage();
    await page.setViewport({ width: 1200, height: 1200 });
    await page.goto('https://www.google.com/search?q=.net+core&rlz=1C1GGRV_enUS785US785&oq=.net+core&aqs=chrome..69i57j69i60l3j69i65j69i60.999j0j7&sourceid=chrome&ie=UTF-8');

    const IMAGE_SELECTOR = '#tsf > div:nth-child(2) > div > div.logo > a > img';
    let imageHref = await page.evaluate((sel) => {
        return document.querySelector(sel).getAttribute('src').replace('/', '');
    }, IMAGE_SELECTOR);

    console.log("https://www.google.com/" + imageHref);
    var viewSource = await page.goto("https://www.google.com/" + imageHref);
    fs.writeFile(".googles-20th-birthday-us-5142672481189888-s.png", await viewSource.buffer(), function (err) {
    if (err) {
        return console.log(err);
    }

    console.log("The file was saved!");
});

    browser.close();
}

run();

如果您有要下载的图像列表，则可以将选择器更改为根据需要以编程方式更改，然后从图像列表中向下一次下载一个。

Answer 3

您可以使用以下内容抓取页面上所有图像的所有src 属性的数组：

const images = await page.evaluate(() => Array.from(document.images, e => e.src));

然后您可以使用Node File System Module 和HTTP 或HTTPS Module 下载每个图像。

完整示例：

'use strict';

const fs = require('fs');
const https = require('https');
const puppeteer = require('puppeteer');

/* ============================================================
  Promise-Based Download Function
============================================================ */

const download = (url, destination) => new Promise((resolve, reject) => {
  const file = fs.createWriteStream(destination);

  https.get(url, response => {
    response.pipe(file);

    file.on('finish', () => {
      file.close(resolve(true));
    });
  }).on('error', error => {
    fs.unlink(destination);

    reject(error.message);
  });
});

/* ============================================================
  Download All Images
============================================================ */

(async () => {
  const browser = await puppeteer.launch();
  const page = await browser.newPage();

  let result;

  await page.goto('https://www.example.com/');

  const images = await page.evaluate(() => Array.from(document.images, e => e.src));

  for (let i = 0; i < images.length; i++) {
    result = await download(images[i], `image-${i}.png`);

    if (result === true) {
      console.log('Success:', images[i], 'has been downloaded successfully.');
    } else {
      console.log('Error:', images[i], 'was not downloaded.');
      console.error(result);
    }
  }

  await browser.close();
})();

Answer 4

我认为逻辑很简单。您只需要创建一个函数，该函数将获取图像的 url 并将其保存到您的目录中。 puppeteer 只会抓取图像 url 并将其传递给下载器功能。这是一个例子：

const puppeteer = require('puppeteer');
const fs = require('fs');
const request = require('request');

//  This is main download function which takes the url of your image
function download(uri, filename) {
  return new Promise((resolve, reject) => {
    request.head(uri, function (err, res, body) {
      request(uri).pipe(fs.createWriteStream(filename)).on('close', resolve);
    });
  });
}

let main = async () => {
  const browser = await puppeteer.launch({ headless: false });
  const page = await browser.newPage();
  await page.goto('https://memeculture69.tumblr.com/');
  await page.waitFor(1000);
  const imageUrl = await page.evaluate(
    // here we got the image url from the selector.
    () => document.querySelector('img.image')
  );
  // Now just simply pass the image url
  // to the downloader function to download  the image.
  await download(imageUrl, 'image.png');
};

main();

Answer 5

此代码将页面上找到的所有图像保存到图像文件夹中

page.on('response', async (response) => {
  const matches = /.*\.(jpg|png|svg|gif)$/.exec(response.url());
  if (matches && (matches.length === 2)) {
    const extension = matches[1];
    const buffer = await response.buffer();
    fs.writeFileSync(`images/${matches[0]}.${extension}`, buffer, 'base64');
  }
});

Answer 6

为了通过选择器下载图片，我做了以下操作：

1. 使用选择器为图像获取 uri

2. 将 uri 传递给下载函数

   const puppeteer = require('puppeteer');
   const fs = require('fs');
   var request = require('request');
   
   //download function
   var download = function (uri, filename, callback) {
       request.head(uri, function (err, res, body) {
           console.log('content-type:', res.headers['content-type']);
           console.log('content-length:', res.headers['content-length']);
           request(uri).pipe(fs.createWriteStream(filename)).on('close', callback);
       });
   };
   
   (async () => {
        const browser = await puppeteer.launch({
         headless: true,
         args: ['--no-sandbox', '--disable-setuid-sandbox'], //for no sandbox
       });
       const page = await browser.newPage();
       await page.goto('http://example.com');// your url here
   
       let imageLink = await page.evaluate(() => {
           const image = document.querySelector('#imageId');
           return image.src;
       })
   
       await download(imageLink, 'myImage.png', function () {
           console.log('done');
       });
   
       ...
   })();
   

资源：Downloading images with node.js

Answer 7

可以在不单独访问每个 url 的情况下获取所有图像。你需要监听所有对服务器的请求：

await page.setRequestInterception(true)
await page.on('request', function (request) {
   request.continue()
})
await page.on('response', async function (response) {
   // Filter those responses that are interesting
   const data = await response.buffer()
   // data contains the img information
})

Answer 8

您还可以根据请求类型进行过滤。

const blocked_resources = [
  'stylesheet',
  /*'image',*/
  'media',
  'font'
];

const _handleRequest = request => {
  const type = request.resourceType();
  if (blocked_resources.some(r => type === r)) {
    request.abort();
    return;
  }
  request.continue();
  return;
}