节点：循环文件的行，异步处理每行，等待每个结果？

Question

我有一个包含姓名和出生日期信息的文件。对于文件中的每一行，我需要将数据提交到 Web 表单并查看我得到的结果。我正在使用 Node 和 Puppeteer（无头）以及 readline 来读取文件。

该代码适用于小文件，但是当我在完整的 5000 个名称甚至几百个名称上运行它时，我最终会得到数百个无头的 Chromium 实例，使我的机器陷入瘫痪，并可能造成令人困惑的超时错误。

我宁愿等待每个表单提交完成，或者以其他方式限制处理，以便一次处理的名称不超过 x 个。我尝试了几种方法，但没有一个能达到我想要的效果。我根本不是 JS 高手，所以可能存在有问题的设计。

有什么想法吗？

const puppeteer = require('puppeteer');
const fs = require('fs');
const readline = require('readline');
const BALLOT_TRACK_URL = 'https://www.example.com/ballottracking.aspx';
const VOTER_FILE = 'MailBallotsTT.tab';
const VOTER_FILE_SMALL = 'MailBallotsTTSmall.tab';
const COUNTY = 'Example County';


checkBallot = (async ( fName, lName, dob, county ) => {
  /* Initiate the Puppeteer browser */
  const browser = await puppeteer.launch({headless:true });
  const page = await browser.newPage();

  await page.goto( BALLOT_TRACK_URL, { waitUntil: 'networkidle0' });


  // fill out the form
  await page.type('#ctl00_ContentPlaceHolder1_FirstNameText', fName );
  await page.type('#ctl00_ContentPlaceHolder1_LastNameText', lName );
  await page.type('#ctl00_ContentPlaceHolder1_DateOfBirthText', dob );
  await page.type('#ctl00_ContentPlaceHolder1_CountyDropDown', county );

  let pageData = await page.content();

  // Extract the results from the page
  try {
  submitSelector = 'input[name="ctl00$ContentPlaceHolder1$RetrieveButton"]';
  tableSelector  = '#ctl00_ContentPlaceHolder1_ResultPanel > div > div > div > table > tbody > tr:nth-child(3) > td:nth-child(7) > div';

  foundSubmitSelector = await page.waitForSelector(submitSelector, { timeout: 5000 } );
  clickResult = await page.click( submitSelector );
  foundTable = await page.waitForSelector(tableSelector, { timeout: 5000 } )

  let data = await page.evaluate( ( theSelector ) => {
    let text = document.querySelector( theSelector ).innerHTML.replaceAll('<br>', '').trim();
    /* Returning an object filled with the scraped data */
    return {
      text
    }
  }, tableSelector );
  return data;
} catch (error) {
  return {
    text: error.message
  }
} finally {
  browser.close();
}
});

const mainFunction =  () => {

  const readInterface = readline.createInterface({
    input: fs.createReadStream( VOTER_FILE_SMALL ),
    output: null,
    console: false
  });
  
  readInterface.on('line', async(line) => {
    split = line.split( '\t' );
    fName = split[0];
    lName = split[1];
    dob = split[2];
    checkResult = await checkBallot( fName, lName, dob, COUNTY );
    console.log( line + '\t' + checkResult.text );
    to = await new Promise(resolve => setTimeout(resolve, 5000));
  });

};

mainFunction();

Answer 1

这里有一些代码实现了我在评论中的建议。我使用 setTimeout 来表示您拥有的异步代码，但原则上该方法应该很容易适应：

// Source file testReadFileSync.js

const fs = require( "fs" );

const data = fs.readFileSync( "./inputfile.txt", { encoding: "utf-8" } );
const lines = data.split( "\n" );
console.log( `There are ${lines.length} lines to process`);

var currentLine = 0;

function main(){
    // Check if we have processed all the lines
    if (currentLine == lines.length ) return;

    // get the current line number, then increment it;
    let lineNum = currentLine++;
    
    // Process the line
    asyncProcess( lineNum,  lines[ lineNum ] )

}


function asyncProcess( lineNum, line ){
    
    console.log( `Start processing line[${lineNum}]` );

    let delayMS = getRandomInt( 100 ) * 10
    setTimeout( function(){
        

        // ------------------------------------------------
        // this function represents all the async processing 
        // for one line.
        // After async has finished, we call main again
        // ------------------------------------------------

        console.log( `Finished processing line[${lineNum}]` );
        main();

    }, delayMS )

}

function getRandomInt(max) {
    return Math.floor(Math.random() * Math.floor(max));
}

// Start four parallel async processes, each of which will process one line at a time
main();
main();
main();
main();