Promise.all 是否第二次无法正常工作？为什么不？

Question

我刚刚完成了一个 T 恤网站的基本网络爬虫项目。

它通过一个硬编码的 url 进入，即主页。它将搜索任何产品页面，并将它们添加到 url。如果它找到另一个链接 (remainder)，它将再次抓取该链接并找到更多产品页面。它将产品页面添加到urlSet，然后再次抓取这些页面，获取 T 恤数据（价格、图片、标题），然后进行转换，然后将它们写入 CSV 文件。

由于某种原因，这不适用于使用“剩余”进行刮擦的第二次运行。

如果我删除了 url 的第二个刮擦，一切正常并且文件被正确写入。但是，如果我想获得其他产品页面，它似乎在某个地方失败了。

这是我的代码，我很抱歉发布了这么多，但如果没有正确的上下文，我不知道如何正确理解它，希望它已被评论好：

//TASK: Create a command line application that goes to an ecommerce site to get the latest prices.
    //Save the scraped data in a spreadsheet (CSV format).

'use strict';

//Modules being used:
var cheerio = require('cheerio');
var json2csv = require('json2csv');
var request = require('request');
var moment = require('moment');
var fs = require('fs');

//harcoded url
var url = 'http://shirts4mike.com/';

//url for tshirt pages
var urlSet = new Set();

var remainder;
var tshirtArray = [];


const requestPromise = function(url) {
    return new Promise(function(resolve, reject) {
        request(url, function(error, response, html) {

            if(error)return reject(error);

            if(!error && response.statusCode == 200){
                return resolve(html);   
            }       
        });
    });
}


// Go into webpage via url, load html and grab links shirt in url
function scrape (url) {
    console.log("Currently scraping " + url)
    return requestPromise(url)
        .then(function(html) {
            var $ = cheerio.load(html);

            var links = [];

            //get all the links
            $('a[href*=shirt]').each(function(){
                var a = $(this).attr('href');
                //add into link array
                links.push(url + a);
            });
            // return array of links
            return links;
        });
}


function nextStep (arrayOfLinks) { 
    var promiseArray = [];
    console.log(arrayOfLinks);
    for(var i = 0; i < arrayOfLinks.length; i++){
        promiseArray.push(requestPromise(arrayOfLinks[i]));
    }
    //return both the html of pages and their urls
    return Promise.all(promiseArray)
        .then(function(arrayOfHtml){
        return {arrayOfHtml: arrayOfHtml , arrayOfUrls: arrayOfLinks};
    });                 
}


//go through the html of each url and add to urlSet if there is a checkout button
//add to remainder otherwise to rescrape
function lastStep (obj){ 
    for(var i = 0;  i < obj.arrayOfHtml.length; i++){
        var $ = cheerio.load(obj.arrayOfHtml[i]);

        //if page has a submit it must be a product page
        if($('[type=submit]').length !== 0){

            //add page to set
            urlSet.add(obj.arrayOfUrls[i]);
            console.log(obj.arrayOfUrls[i]);

        } else if(remainder == undefined) {
            //if not a product page, add it to remainder so it another scrape can be performed.
            remainder = obj.arrayOfUrls[i];
            console.log("The remainder is " + remainder)                                     
        }
    }
    //return remainder for second run-through of scrape 
    return remainder;
}


//iterate through urlSet (product pages and grab html)
function lastScraperPt1(){
    //call lastScraper so we can grab data from the set (product pages)
        //scrape set, product pages
        var promiseArray = [];

        for(var item of urlSet){
            var url = item;

            promiseArray.push(requestPromise(url));
        }
        return Promise.all(promiseArray)
            .then(function(arrayOfHtml){
                return arrayOfHtml;
            });    
}


//iterate over the html of the product pages and store data as objects
function lastScraperPt2(html){
    for(var i = 0; i < html.length; i++){
        var $ = cheerio.load(html[i]);

        //grab data and store as variables
        var price = $('.price').text();
        var imgURL = $('.shirt-picture').find('img').attr('src');
        var title = $('body').find('.shirt-details > h1').text().slice(4);

        var tshirtObject = {};
        //add values into tshirt object
        tshirtObject.Title = title;
        tshirtObject.Price = price;
        tshirtObject.ImageURL = imgURL;
        tshirtObject.URL = url;
        tshirtObject.Date = moment().format('MMMM Do YYYY, h:mm:ss a');

        //add the object into the array of tshirts
        tshirtArray.push(tshirtObject);
    }
    convertJson2Csv();
}


//convert tshirt objects and save as CSV file
function convertJson2Csv(){
        //The scraper should generate a folder called `data` if it doesn’t exist.
        var dir ='./data';
        if(!fs.existsSync(dir)){
            fs.mkdirSync(dir);
        }

        var fields = ['Title', 'Price', 'ImageURL', 'URL', 'Date'];

        //convert tshirt data into CSV and pass in fields
        var csv = json2csv({ data: tshirtArray, fields: fields });

        //Name of file will be the date
        var fileDate = moment().format('MM-DD-YY');
        var fileName = dir + '/' + fileDate + '.csv';

        //Write file
        fs.writeFile(fileName, csv, {overwrite: true}, function(err) {
            console.log('file saved');
            if (err) throw err;
        });
}

scrape(url) //scrape from original entry point
    .then(nextStep) 
    .then(lastStep)
    .then(scrape) //scrape again but with remainder url
    .then(nextStep)
    .then(lastStep)
    .then(lastScraperPt1)
    .then(lastScraperPt2)
    .catch(function(err) {
        // handle any error from any request here
        console.log(err);
     });

我在控制台中将arrayOfLinks 记录在nextStep 中，所以我可以看到它们被正确抓取，我只是无法弄清楚为什么它们没有被正确传递到“lastStep”。

Currently scraping http://shirts4mike.com/
[ 'http://shirts4mike.com/shirts.php',
  'http://shirts4mike.com/shirts.php',
  'http://shirts4mike.com/shirt.php?id=108',
  'http://shirts4mike.com/shirt.php?id=107',
  'http://shirts4mike.com/shirt.php?id=106',
  'http://shirts4mike.com/shirt.php?id=105' ]
The remainder is http://shirts4mike.com/shirts.php
http://shirts4mike.com/shirt.php?id=108
http://shirts4mike.com/shirt.php?id=107
http://shirts4mike.com/shirt.php?id=106
http://shirts4mike.com/shirt.php?id=105
Currently scraping http://shirts4mike.com/shirts.php
[ 'http://shirts4mike.com/shirts.phpshirts.php',
  'http://shirts4mike.com/shirts.phpshirt.php?id=101',
  'http://shirts4mike.com/shirts.phpshirt.php?id=102',
  'http://shirts4mike.com/shirts.phpshirt.php?id=103',
  'http://shirts4mike.com/shirts.phpshirt.php?id=104',
  'http://shirts4mike.com/shirts.phpshirt.php?id=105',
  'http://shirts4mike.com/shirts.phpshirt.php?id=106',
  'http://shirts4mike.com/shirts.phpshirt.php?id=107',
  'http://shirts4mike.com/shirts.phpshirt.php?id=108' ]

但是如果我选择只调用第一个抓取而不调用第二个，像这样：

scrape(url) //scrape from original entry point
    .then(nextStep) 
    .then(lastStep)
    .then(lastScraperPt1)
    .then(lastScraperPt2)
    .catch(function(err) {
        // handle any error from any request here
        console.log(err);
     });

...然后一切正常。我只是没有访问所有的网址。

这里发生了什么，我该如何解决？谢谢各位

Answer 1

问题是tshirtArray 未在convertJson2Csv() 中定义。在lastlastScraperPt2 将tshirtArray 传递给convertJsonCsv()

convertJson2Csv(tshirtArray)

convertJson2Csv

function convertJson2Csv(tshirtArray) {
  // do stuff
}

Answer 2

您的lastStep 似乎有一个问题。看起来您的意思是 remainder 是另一个 url 数组。如果我错了，请纠正我。然而，发生的事情是if($('[type=submit]').length !== 0) 条件第一次失败时，您将自动进入下一个块，因为remainder 开始未定义。无论当前 url 是什么，您都可以将其分配给remainder。对于 for 循环的其余迭代，您将永远不会再次遇到remainder == undefined 的条件。因此，如果您最终只能获得一个分配给remainder 的网址，而您希望获得的更多网址将被忽略。

您可能希望将remainder 定义为remainder = [];。然后不要说else if (remainder == undefined)，而是说

} else {
    remainder.push(obj.arrayOfUrls[i]);
}

但是，当 scrape 只需要一个 url 时，您将一组 url 传递给 scrape。如果这是您想要的并且我假设您的意思是 remainder 是一个 url 数组是正确的，您可以定义一个新函数，如下所示：

function scrapeRemainders(remainders) {
  var promises = [];

  remainder.forEach(function (url) {
    promises.push(requestPromise(url));
  });

  return Promise.all(promises).then(function (results) {
    _.flattenDeep(results);
  })
}

然后，您可以将其替换为scrapeRemainders，而不是您的承诺链中的第二个scrape。此外，对于上一个函数中的_，您需要npm install lodash，然后是var _ = require('lodash')。顺便说一句，lodash 与 Promise 无关，但它是一个很好的数据操作工具。当你有机会的时候，你应该研究一下。

另外，在lastScraperPt1，你可以更改

return Promise.all(promiseArray)
    .then(function(arrayOfHtml){
        return arrayOfHtml;
    });

到

return Promise.all(promiseArray);

它做同样的事情。

希望这会有所帮助。如果这不能回答您的问题，请对我发表评论，我可以相应地更改我的答案。

Answer 3

全部修复，它在scrape() 中抓取了错误的网址。虽然我只是在将 statusCodes 记录到控制台后才知道这一点：

//TASK: Create a command line application that goes to an ecommerce site to get the latest prices.
    //Save the scraped data in a spreadsheet (CSV format).

'use strict';

//Modules being used:
var cheerio = require('cheerio');
var json2csv = require('json2csv');
var request = require('request');
var moment = require('moment');
var fs = require('fs');

//harcoded url
var urlHome = 'http://shirts4mike.com/';

//url for tshirt pages
var urlSet = [];

var tshirtArray = [];


const requestPromise = function(url) {
    return new Promise(function(resolve, reject) {
        request(url, function(error, response, html) {

            if(error) { 
                errorHandler(error);
                return reject(error);
            }

            if(!error && response.statusCode == 200){
                return resolve(html);   
            }

            if(response.statusCode !== 200){
                console.log("response code is " + response.statusCode);
            }

            return resolve("");      
        });
    });
}


// Go into webpage via url, load html and grab links shirt in url
function scrape (url) {
    console.log("Currently scraping " + url)
    return requestPromise(url)
        .then(function(html) {
            var $ = cheerio.load(html);

            var links = [];
            var URL = 'http://shirts4mike.com/';
            //get all the links
            $('a[href*=shirt]').each(function(){
                var a = $(this).attr('href');
                //add into link array
                links.push(URL + a);
            });
            // return array of links
            return links;
        });
}




function nextStep (arrayOfLinks) { 
    var promiseArray = [];
    console.log(arrayOfLinks);
    for(var i = 0; i < arrayOfLinks.length; i++){
        promiseArray.push(requestPromise(arrayOfLinks[i]));
    }
    //return both the html of pages and their urls
    return Promise.all(promiseArray)
        .then(function(arrayOfHtml){
        return {arrayOfHtml: arrayOfHtml , arrayOfUrls: arrayOfLinks};
    });                 
}


//go through the html of each url and add to urlSet if there is a checkout button
//add to remainder otherwise to rescrape
function lastStep (obj){ 
    for(var i = 0;  i < obj.arrayOfHtml.length; i++){
        var $ = cheerio.load(obj.arrayOfHtml[i]);

        //if page has a submit it must be a product page
        if($('[type=submit]').length !== 0){

            //add page to set
            urlSet.push(obj.arrayOfUrls[i]);
            console.log(obj.arrayOfUrls[i]);

        } else if(remainder == undefined) {
            //if not a product page, add it to remainder so it another scrape can be performed.
            var remainder = obj.arrayOfUrls[i];
            console.log("The remainder is " + remainder)                                     
        }
    }
    //return remainder for second run-through of scrape 
    return remainder;
}


//iterate through urlSet (product pages and grab html)
function lastScraperPt1(){
    //call lastScraper so we can grab data from the set (product pages)
        //scrape set, product pages
        var promiseArray = [];

        for(var item of urlSet){
            var url = item;

            promiseArray.push(requestPromise(url));
        }
        return Promise.all(promiseArray)
            .then(function(arrayOfHtml){
                return arrayOfHtml;
            });    
}


//iterate over the html of the product pages and store data as objects
function lastScraperPt2(html){
    for(var i = 0; i < html.length; i++){
        var $ = cheerio.load(html[i]);

        //grab data and store as variables
        var price = $('.price').text();
        var imgURL = $('.shirt-picture').find('img').attr('src');
        var title = $('body').find('.shirt-details > h1').text().slice(4);

        var tshirtObject = {};
        //add values into tshirt object
        tshirtObject.Title = title;
        tshirtObject.Price = price;
        tshirtObject.ImageURL = urlHome + imgURL;
        tshirtObject.URL = urlSet[i];
        tshirtObject.Date = moment().format('MMMM Do YYYY, h:mm:ss a');

        //add the object into the array of tshirts
        tshirtArray.push(tshirtObject);
    }
    return tshirtArray;
}


//conver tshirt objects and save as CSV file
function convertJson2Csv(tshirtArray){
        //The scraper should generate a folder called `data` if it doesn’t exist.
        var dir ='./data';
        if(!fs.existsSync(dir)){
            fs.mkdirSync(dir);
        }

        var fields = ['Title', 'Price', 'ImageURL', 'URL', 'Date'];

        //convert tshirt data into CSV and pass in fields
        var csv = json2csv({ data: tshirtArray, fields: fields });

        //Name of file will be the date
        var fileDate = moment().format('MM-DD-YY');
        var fileName = dir + '/' + fileDate + '.csv';

        //Write file
        fs.writeFile(fileName, csv, {overwrite: true}, function(err) {
            console.log('file saved');
            if (err) errorHandler(err);
        });
}


scrape(urlHome) //scrape from original entry point
    .then(nextStep) 
    .then(lastStep)
    .then(scrape)
    .then(nextStep)
    .then(lastStep)
    .then(lastScraperPt1)
    .then(lastScraperPt2)
    .then(convertJson2Csv)
    .catch(function(err) {
        // handle any error from any request here
        console.log(err);
     });


//If the site is down, an error message describing the issue should appear in the console. 
    //This is to be tested by disabling wifi on your device.
    //When an error occurs log it to a file scraper-error.log . It should append to the bottom of the file with a time stamp and error

var errorHandler = function (error) {
    console.log(error.message);
    console.log('The scraper could not not scrape data from ' + url + ' there is either a problem with your internet connection or the site may be down');
    /**
    * create new date for log file
     */
    var loggerDate = new Date();
    /**
     * create message as a variable
    */
    var errLog = '[' + loggerDate + '] ' + error.message + '\n';
    /**
    *when the error occurs, log that to the error logger file
    */
    fs.appendFile('scraper-error.log', errLog, function (err) {
        if (err) throw err;
        console.log('There was an error. The error was logged to scraper-error.log');
    });
};