发布 API 问题后尝试从 api 获取

Question

各位开发者，我又带着一个问题回来了，

我正在尝试从该网站https://www.iamsterdam.com/nl/uit-in-amsterdam/uit/agenda 获取一些数据。我首先抓取了网站，但这样做时我想到他们有一个 api，而且速度会快得多。所以我试图从我试过的api中获取数据：

get-website.js:

var webPage = require('webpage');
var page = webPage.create();
var settings = {
  operation: "POST",
  encoding: "utf8",
  headers: {
    "Content-Type": "application/json"
  },
  data: JSON.stringify({
    DateFilter: 03112016,
    PageId: "3418a37d-b907-4c80-9d67-9fec68d96568",
    Take: 2,
    Skip: 12,
    ViewMode: 1
  })
};

page.open('https://www.iamsterdam.com/api/AgendaApi/', settings, function(status) {
  console.log(page.content);
  phantom.exit();
});

get-website.php

$phantom_script= 'get-website.js'; 

$response =  exec ('phantomjs ' . $phantom_script);

echo  $response;

但是我回来的东西看起来不太好：

Message":"An error has occurred.","ExceptionMessage":"Page could not be found","ExceptionType":"System.ApplicationException","StackTrace":" at Axendo.SC.AM.Iamsterdam.Controllers.Api.AgendaApiController.GetResultsInternal(RequestModel requestModel)\r\n at lambda_method(Closure , Object , Object[] )\r\n
etc.

这是萤火虫的图片：

我希望有人可以帮助我。

Answer 1

有趣的问题。我有点惊讶该网站会在浏览器中甚至在 cURL 中支持 AJAX 请求，但在 PhantomJS 中不会。在这种情况下，您必须非常仔细地研究和复制请求，因为其中一个小细节可能会极大地影响服务器的响应。

原来，这是一个 cookie 和表单内容类型，必须进行相应设置。

var webPage = require('webpage');
var page = webPage.create();

// courtesy of http://stackoverflow.com/a/1714899/2715393
var serialize = function(obj) {
  var str = [];
  for(var p in obj)
    if (obj.hasOwnProperty(p)) {
      str.push(encodeURIComponent(p) + "=" + encodeURIComponent(obj[p]));
    }
  return str.join("&");
}

var settings = {
    operation: "POST",
    encoding: "utf8",
    headers: {
        "accept-encoding" : "identity", // https://github.com/ariya/phantomjs/issues/10930#issuecomment-81541618
        "x-requested-with" : "XMLHttpRequest",
        "accept-language" : "en;q=0.8,en-US;q=0.6",
        "authority" : "www.iamsterdam.com",
        "accept":"application/json, text/javascript, */*; q=0.01",
        "content-type" : "application/x-www-form-urlencoded; charset=UTF-8",
        "cookie" : "website#lang=nl"        
    },
    data: serialize({
        Genre: '',
        DateFilter: '03112016',
        DayPart: '',
        SearchTerm: '', 
        Neighbourhoud: '',
        DayRange: '',
        ViewMode: 1, 
        LastMinuteTickets : '',
        PageId: '3418a37d-b907-4c80-9d67-9fec68d96568',
        Skip: 0,
        Take: 12
    }) 
};

page.open('https://www.iamsterdam.com/api/AgendaApi/', settings, function(status) {
    console.log(page.content);
    phantom.exit();
});