在节点js中的for循环中下载图像答案

【问题标题】：Download images in for loop in node js在节点js中的for循环中下载图像
【发布时间】：2020-02-27 07:22:07
【问题描述】：

我正在尝试使用 puppeteer 和节点 js 从维基百科页面 scrape 图像，并且我成功获得了链接。我还可以从页面下载 1 张图片。但是，每当我尝试循环调用我的下载函数时，它都会下载无效图像。当我尝试 scrape 一个图像但此功能无法循环工作时，下载功能起作用。如果有人有任何建议，请告诉我。提前致谢

这是我的代码

const puppeteer = require('puppeteer');
const fs = require('fs');
const request = require('request');


// download function

function download(uri, filename, callback) {
    request.head(uri, function(err, res, body) {
      request(uri)
      .pipe(fs.createWriteStream(filename))
      .on("close", callback);
   });
  }



let scrape = async ()=> {

    const browser = await puppeteer.launch({
        "headless": false
    });

    const page =await browser.newPage();        //opening new page

    await page.goto("https://www.wikipedia.org/");  // go to url

    const xpathselector = `//span[contains(text(), "Commons")]`;    //click on the commons buttons

    const commonlinks = await page.waitForXPath(xpathselector);     

    await page.waitFor(3000);

    await commonlinks.click();

    await page.waitFor(2000)

    const xpath = '//*[@id="mainpage-potd"]/div[1]/a/img';

    const imageXpath = await page.waitForXPath(xpath);
    const src = await imageXpath.evaluate(el => el.src)

    //downloading the 1st image from the page

    download(src, "image.jpg", function() {
        console.log("Image downloaded");
    });

    await page.waitFor(2000)

    //here we are going to another page

    const xpathselector1 ='//*[@id="mf-picture-picture"]/div[2]/ul/li[4]/a'     
    const previousPictture = await page.waitForXPath(xpathselector1);
    await page.waitFor(2000);
    previousPictture.click();

    await page.waitFor(1500);

    //getting urls of images

    const link_start = 0;
    const cue_card_links = await page.evaluate((selector) => {
        const anchors_node_list = document.querySelectorAll(selector);
        const anchors = [...anchors_node_list];
        return anchors.map(link => link.href);
      }, '#mw-content-text > div > table > tbody > tr > td > div > div > a');

      console.log("[#] Done getting links\n");

    for (let i = link_start; i < cue_card_links.length; i++) {
        let link = cue_card_links[i];

        //downloading all images from second page

        download(link, `image${i}.jpg`, function() {
            console.log("Image downloaded");
        });
    }



}

scrape();

【问题讨论】：

仅供参考，它是 scrape（和 scraping、scraper、scraped）不是废品

标签： javascript node.js linux web-scraping puppeteer

【解决方案1】：

下载应该是这样的

// download function

function download(files, callback) {
    let index = 0;
    var data = setInterval(async () => {
        let i = index++
        if (i === files.length)
            clearInterval(data)
        else {
            request.head(files[i % files.length], function (err, res, body) {
                request(files[i % files.length])
                    .pipe(fs.createWriteStream(`image${i}.jpg`))
                    .on("close", callback);
            });
        }
    }, 4000);
}

和去掉for循环调用的函数，for是sync fn，这样调用fn

 download(cue_card_links, function () { console.log("Image downloaded"); });

【讨论】：