在使用 pupeeteer 和 nodejs 抓取网页期间选择子元素

Question

我正在尝试使用 nodejs 和 puppeeter 抓取页面，这是我要抓取的页面内容的示例：

<article>
   <h3 class="item-title">Item 1 </h3>
   <img src="" alt="" alt="picture for this item" class="item-image">
   <a href="link-to-the-page" class="view-more">View more</a>
</article>

<article>
  <h3 class="item-title">Item 2 </h3>
  <img src="path-to-the-image" alt="" alt="picture for this item" class="item-image">
  <a href="link-to-the-page" class="view-more">View more</a>
</article>

<article>
  <h3 class="item-title">Item 2 </h3>
  <img src="path-to-the-image" alt="" alt="picture for this item" class="item-image">
  <a href="link-to-the-page" class="view-more">View more</a>
</article>

如您所见，article 标签中包含许多项目，我想获取文章列表及其对应的标题、图片链接和整页链接。这是我正在使用的代码：

var puppeeteer = require('puppeeteer');
var browser = await puppeeteer.launch();
var page = await browser.newPage();
await page.goto('link-to-the=page');
var articles = await page.evaluate(()=>{
     var articles = new Array();
     document.querySelectorAll("article").forEach(elt=> { articles.push(elt); });
     return articles;
});

//Here, for each elements in articles array, i try to get : the image link, the title, and the fullpage link.

var result = new Array();
articles.forEach(elt=>{
    var article = {};
    var item_title= elt.getElementsByClassName("item-title")[0];
    var fullpage_link = elt.getElementsByClassName("view-more")[0];
    var image_path = elt.getElementsByClassName("item-image")[0];

    article.title = item_title;
    article.link = fullpage_link;
    article.image = image_path;
    result.push(article);
});

return result;

但是当我运行这段代码时，这是我得到的错误：

(node:3960) UnhandledPromiseRejectionWarning: TypeError: elt.getElementsByClassName is not a function

另一个错误是文章数组包含这个：

[ {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {} ]

代码执行后。

有人可以帮忙吗？提前致谢。

Answer 1

只需在您的问题中使用 html（并修复其中的几个错误），这样的事情（使用 xpath）应该可以工作：

var DOMParser = require('xmldom').DOMParser;

var doc = new DOMParser().parseFromString(`<article>
   <h3 class="item-title">Item 1 </h3>
   <img src="path-to-the-image_1"  alt="picture for this item" class="item-image">
   <a href="link-to-the-page_1" class="view-more">View more</a>
</article>

<article>
  <h3 class="item-title">Item 2 </h3>
  <img src="path-to-the-image_2"  alt="picture for this item" class="item-image">
  <a href="link-to-the-page_2" class="view-more">View more</a>
</article>

<article>
  <h3 class="item-title">Item 3 </h3>
  <img src="path-to-the-image_3"  alt="picture for this item" class="item-image">
  <a href="link-to-the-page_3" class="view-more">View more</a>
</article>`
    ,'text/xml');

var xpath = require('xpath');
const articles = xpath.select("//article", doc);
var result = new Array();
articles.forEach(function(item){
  var article = {};
  title = xpath.select('./h3/text()',item);
  fpl = xpath.select('./a/@href',item);
  isrc = xpath.select('./img/@src',item);

  article.title = title[0].nodeValue;
  article.link = fpl[0].nodeValue;
  article.image = isrc[0].nodeValue;

  result.push(article);
});
console.log(result);

输出：

[ { title: 'Item 1 ',
    link: 'link-to-the-page_1',
    image: 'path-to-the-image_1' },

  { title: 'Item 2 ',
    link: 'link-to-the-page_2',
    image: 'path-to-the-image_2' },

  { title: 'Item 3 ',
    link: 'link-to-the-page_3',
    image: 'path-to-the-image_3' } ]