如何从charts.youtube.com 抓取音乐排行榜和见解页面？

Question

我使用 got 和 request-promise 模块进行scraping，但我得到了一个旋转圆圈作为响应。

基本上，我想要的是视频项目而不是旋转圈。

/* This example use request-promise */
    const rp = require('request-promise');
    const url = 'https://charts.youtube.com/charts/TrendingVideos/gb';

    rp(url)
        .then(function (html) {
            //success!
            fs.writeFileSync('./index.html', html);
        })
        .catch(function (err) {
            //handle error
        });


    /* This example use got */
    got('https://charts.youtube.com/').then(response => {
        fs.writeFileSync('./index.html', response.body);
        const dom = new JSDOM(response.body);
        dom.window.document.querySelectorAll('a').forEach(link => {
            console.log(link.href);
        });
    }).catch(err => {
        console.log(err);
    });

有人可以帮我吗？

Answer 1

数据来自以下 API：

POST https://charts.youtube.com/youtubei/v1/browse?alt=json&key=${apiKey}

您需要使用正则表达式从 html 中获取 api 密钥，然后使用所需的输入数据执行调用：

const got = require("got");
const util = require("util");

(async () => {
  let response = await got(
    "https://charts.youtube.com/charts/TrendingVideos/gb"
  );
  let keyRegex = /\"INNERTUBE_API_KEY\"\s*:\s*\"(.*?)\"/gm;
  let apiKey = keyRegex.exec(response.body)[1];

  response = await got.post(`https://charts.youtube.com/youtubei/v1/browse?alt=json&key=${apiKey}`,
    {
      json: {
        context: {
          client: {
            clientName: "WEB_MUSIC_ANALYTICS",
            clientVersion: "0.2",
            hl: "en",
            gl: "en",
            experimentIds: [],
            experimentsToken: "",
            theme: "MUSIC",
          },
          capabilities: {},
          request: {
            internalExperimentFlags: [],
          },
        },
        browseId: "FEmusic_analytics_charts_home",
        query:
          "chart_params_type=WEEK&perspective=CHART&flags=viral_video_chart&selected_chart=TRACKS&chart_params_id=weekly%3A0%3A0%3Agb",
      },
      responseType: "json",
      headers: {
        referer: "https://charts.youtube.com/charts/TrendingVideos/gb",
      },
    }
  );

  videoTrend = response.body.contents.sectionListRenderer.contents[0].musicAnalyticsSectionRenderer.content.videos;

  console.log(util.inspect(videoTrend, { showHidden: false, depth: null }));
})();

Try this on repl.it