如何在不分页的情况下抓取下一页

Question

此链接“https://www.muscleandstrength.com/store/promos.html”会将您重定向到一个网站，该网站将向您显示他们所有 600 多种打折商品的列表。我的目标是把它们全部刮掉。但是我遇到的主要问题是它没有带有数字的常规分页（例如：prev 1, 2, 3, 4, 5 ... next）或我可以用来检索下一个的 GET url页。我必须单击“查看下一个 20 多个产品”才能显示下一批。我可以毫无问题地抓取一个带有分页的网站，但是这个让我很头疼。

下面的链接是点击下一个按钮时触发的主要 JS

https://cdn.muscleandstrength.com/store/media/js/ec00de517e571209f780e1b62a3967fd-1534086933.js

在里面你会发现这样一行：

e.post("www.muscleandstrength.com/store/promos/index/filter",{filter:JSON.stringify(t)}

如果您访问此链接“www.muscleandstrength.com/store/promos/index/filter”，并一直向下滚动到底部，您会发现类似的内容。

filters":{"classfilter":{"lowprice":125,"b1g1":123,"twopacks":119,"couponcode":118,"b1g50":79,"bxgy":36,"b2g1":16},"brandfilter":{"78":77,"84":45,"190":37,"383":26,"120":24,"82":23,"108":22,"42":21,"30":18,"150":18,"133":16,"151":14,"489":14,"490":13,"69":12,"65":11,"369":10,"193":9,"232":9,"53":8,"335":8,"81":7,"93":7,"426":7,"471":7,"67":6,"106":6,"423":6,"43":5,"98":5,"241":5,"267":5,"274":5,"432":5,"481":5,"493":5,"59":4,"64":4,"144":4,"279":4,"292":4,"389":4,"109":3,"183":3,"223":3,"330":3,"421":3,"444":3,"523":3,"76":2,"97":2,"111":2,"132":2,"147":2,"191":2,"196":2,"244":2,"331":2,"344":2,"346":2,"403":2,"406":2,"428":2,"435":2,"38":1,"46":1,"48":1,"51":1,"68":1,"92":1,"116":1,"139":1,"189":1,"194":1,"252":1,"302":1,"303":1,"322":1,"326":1,"380":1,"424":1,"439":1,"450":1,"472":1},"categoryfilter":{"21":119,"9":112,"13":97,"582":76,"57":75,"408":64,"29":62,"130":49,"26":45,"605":45,"58":42,"12":35,"10":34,"140":34,"131":29,"469":29,"44":28,"177":25,"670":25,"55":24,"580":23,"597":23,"22":22,"17":21,"27":21,"40":21,"464":21,"667":21,"28":19,"87":19,"219":19,"452":18,"441":17,"635":16,"684":16,"25":14,"42":14,"544":14,"694":14,"88":13,"132":13,"695":13,"220":12,"562":12,"147":11,"205":11,"53":10,"154":10,"389":10,"423":10,"425":10,"617":10,"51":9,"530":9,"623":9,"686":9,"16":8,"19":8,"86":8,"304":8,"492":8,"554":8,"592":8,"638":8,"52":7,"122":7,"378":7,"410":7,"509":7,"572":7,"594":7,"637":7,"83":6,"171":6,"180":6,"303":6,"409":6,"496":6,"559":6,"666":6,"59":5,"84":5,"142":5,"172":5,"187":5,"302":5,"390":5,"391":5,"400":5,"431":5,"466":5,"512":5,"583":5,"593":5,"632":5,"653":5,"690":5,"696":5,"20":4,"56":4}},"num_results":616}

我认为是负责显示下一批项目。

我的主要问题是，抓取所有这些项目的最简单方法是什么，不管它没有常规分页或我可以用来转到下一页的 GET url 的事实吗？

Answer 1

在浏览器中打开网页https://www.muscleandstrength.com/store/promos.html（我使用的是 Chrome）。按 F12 打开开发者工具。转到网络选项卡。在这里您可以看到所有记录的请求。如果您在网页上单击“查看下 20 个产品”，则会记录新的一个请求。过滤 XHR 请求。检索您可以在 Headers 中找到的所有产品的必要数据，如下所示：

要检索所有产品，您只需在过滤器中将总数量设置为"limit"，并将0 到"offset" 参数，重放相同的XHR 并解析响应。在 Preview 中可以发现响应实际上是 JSON，而要解析的 HTML 内容位于content 属性中：

下面的例子展示了如何做到这一点。转到控制台选项卡。复制以下代码，将其粘贴到控制台并按 Enter:

(function () {
    // retrieve all products
    var count = document.querySelector("span.search-result-available-count").innerText;
    var x = new XMLHttpRequest;
    x.open("POST","https://www.muscleandstrength.com/store/promos/index/filter",false);
    x.setRequestHeader("Content-Type","application/x-www-form-urlencoded; charset=UTF-8");
    x.send("filter=%7B%22order%22%3A%22sort_order+desc%2Csales_ranking+asc%22%2C%22brandfilter%22%3A%5B%5D%2C%22categoryfilter%22%3A%5B%5D%2C%22classfilter%22%3A%5B%5D%2C%22limit%22%3A"+count+"%2C%22offset%22%3A0%7D");
    // replace body content with received data
    document.body.innerHTML = JSON.parse(x.responseText).content;
    // create table for output
    var addCell = function(htmlContent) {
        var cell = row.insertCell(-1);
        cell.innerHTML = htmlContent;
    };
    var table = document.createElement("table");
    document.body.appendChild(table);
    table.style = 'margin:10px;'
    // add table header and body
    var tHead = table.createTHead();
    var row = tHead.insertRow(-1);
    ["#","Product","Price","Rating","Reviews"].forEach(addCell);
    var tBody = document.createElement("tbody");
    table.appendChild(tBody);
    // parse each product
    var products = document.querySelectorAll("li.grid-product");
    for (var i = 0; i < products.length; i++) {
        var product = products[i];
        // add row
        row = tBody.insertRow(-1);
        addCell(i+1);
        // parse name
        var m = product.querySelector("a.product-name").innerText.trim();
        addCell(m);
        // parse price
        var m = product.querySelector("div.product-price").innerText.trim();
        addCell(m);
        // parse rating
        var m = product.querySelector("a.star-rating span").style.width;
        addCell(m);
        // parse reviews
        var m = product.querySelector("span.review-count")?.innerText.replace(/\D/g,"") || '0';
        addCell(m);
    }
    // remove unnecessary content
    document.querySelector("div.promo-products").remove();
})();

请求是同步的，所以需要等待一段时间才能完成。最后将所有产品解析成表格，我的输出如下：

Answer 2

这是加载 DOM 的简单解决方案。 此循环功能将在循环中为您单击该查看更多按钮，直到完成。您将需要一些条件来检查是否完成，然后运行您的刮板功能。

取产品总数，去掉前60个（已经显示） 将其除以 20（单击查看更多按钮时加载的产品数量） 创建一个滚动到页面底部的函数，每次滚动超时 2 秒。

var prodElm = document.getElementsByClassName("search-result-available-count")[0].innerText;

var products = parseInt(prodElm.replace(/\D+/g, ''));

var numScrollEventsNeeded = Math.round((products - 60) / 20);

function timedScroller(num) {

  setTimeout(() => {

    window.scrollTo(0, document.body.scrollHeight);

    document.getElementsByClassName("view-more btn btn-blue")[0].click();

  }, num);

}

for (i = 0; i < numScrollEventsNeeded; i++) {

  timedScroller(i * 2000);

}