如何通过 Express 发送抓取的网站数据？

Question

我在 Axios 中编写了一个爬虫并尝试通过 Express 发送文件，我在 Express 中有大约 10 个爬虫和 10 个 html 表单方法。

但是当我按下按钮时，它会下载空白文件，然后爬虫开始运行。

爬虫完成后应该下载文件并保存数据。但是该怎么做呢？

爬虫代码：

"use strict"

const axios = require("axios").default;
const cheerio = require("cheerio");
const fs = require("fs");
var excel = require('excel4node');

let writeStream = fs.createWriteStream('mpRera_Agents.csv');
writeStream.write(`AgentName,AgentPhone,AgentEmail\n`)

// let writeStream = fs.createWriteStream('mpRera_Promoters.csv');
// writeStream.write(`AgentName,Agentaddress,Agenttype,Agentrera,Agentlink\n`)

const Agenturl = "http://www.rera.mp.gov.in/agentsrcg-loop.php? 
   show=20&pagenum=1&search_txt=&search_state=&search_dist=&search_tehs=&_=1597665284486";


var workbook = new excel.Workbook();
var worksheet = workbook.addWorksheet('MP Agents');
var worksheet2 = workbook.addWorksheet('MP Promoters');

// var arr = []

class mprera{

makeRequest(urls){
    return new Promise((resolve, reject)=>{
        let url = axios.get(urls);
        let data = url.then((res)=>{
            if (res.status==200){
                resolve(res.data);
            }reject("response not 200");
        })
        
    })
}
getlink(url){
    return new Promise((resolve, reject)=>{

    
    var arr = []
    let soup = this.makeRequest(url);
    soup.then((res)=>{
        let $  = cheerio.load(res);
        let table = $("#example");
        let tbody = table.find("tbody");
        let tr = tbody.find("tr");
        for (var i = 0;i<tr.length;i++){
            let td = $(tr[i]).find("td");
            let link= $(td[6]).find("a").attr("href");
            arr.push(link);      
        }    
        resolve (arr);
    });
     
    })
};

getAgents(url){
    var link = this.getlink(url);
    link.then((data)=>{
        for (var i = 0;i<data.length;i++){

            let soup = this.makeRequest(data[i]);
            soup.then((res)=>{
                let $  = cheerio.load(res);
                let getDetails = $(".col-md-9").toArray();
                // let name = $(getDetails[1]).text().trim();
                // console.log(name);
                let phone = $(getDetails[5]).text().trim();
                let email = $(getDetails[6]).text().trim();
                // console.log(phone, email);
                writeStream.write(`${phone},${email} \n`);

            })
        }

    })

    };

}


module.exports = mprera;

我的快递代码：

const express = require("express");
const hbs = require("hbs");
const bodyParser = require("body-parser");
const path = require("path");
var json2csv = require('json2csv');

const mprera = require("./mprera");

let crawleer = new mprera();
const Agenturl = "http://www.rera.mp.gov.in/agentsrcg-loop.php?show=20&pagenum=1&search_txt=&search_state=&search_dist=&search_tehs=&_=1597665284486";


app = express();

//set path for views
app.set("views",path.join(__dirname,"views"))
app.set("view engine","hbs");

// for handling post requests
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ extended: false }));


// for public folders
app.use('/assets',express.static(__dirname + '/public'));

app.get("/", (req, res)=>{
    res.render("table");
})

app.get("/mprera", async (req, res)=>{
    console.log("starting");
    data = await crawleer.getAgents(Agenturl);
    res.attachment('filename.csv');
    res.send (data);

})


app.listen(8000, ()=>{
    console.log("server started");
})

Answer 1

crawler.getAgents 需要返回一个直到文件写入后才会解析的承诺。

比如：

crawler.getAgents = url => {
  return new Promise((resolve) => {
    stuff().then(() => {
      // write file here
      resolve()
    })
  })
}