尝试循环通过 Hubspot Deals API JSON 时出现错误 429

Question

我正在尝试将 Hubspot API 中的所有交易都放入 Google 表格中。当交易计数低于 250 时，以下 Google App 脚本运行良好。现在我的管道中有超过 250 笔交易，我收到 429 个错误，“一天内调用服务太多次：urlfetch。”和其他错误.

function getDeals() {
   // Prepare authentication to Hubspot
   
   var service = getService();
   var headers = {headers: {'Authorization': 'Bearer '+ service.getAccessToken()}};
   
   // Prepare pagination
   // Hubspot lets you take max 250 deals per request.
   // We need to make multiple request until we get all the deals.
   
   var keep_going = true;
   var offset = 0;
   var deals = Array();
   while(keep_going) {
      
      // We’ll take three properties from the deals: the source, the stage, the amount of the deal
      
      var url = API_URL + "/deals/v1/deal/paged?&includeAssociations=true&properties=dealstage&properties=source&properties=amount&properties=dealname&properties=num_associated_contacts&limit=250&offset&properties=hubspot_owner_id&limit=250&offset="+offset;
      var response = UrlFetchApp.fetch(url, headers);
      var result = JSON.parse(response.getContentText());
      Logger.log(result.deal)
      
      // Are there any more results, should we stop the pagination
      
      keep_going = result.hasMore;
      offset = result.offset;
      
      // For each deal, we take the stageId, source, amount, dealname, num_associated_contacts & hubspot_owner_id
      
      result.deals.forEach(function(deal) {
         var stageId = (deal.properties.hasOwnProperty("dealstage")) ? deal.properties.dealstage.value : "unknown";
         var source = (deal.properties.hasOwnProperty("source")) ? deal.properties.source.value : "unknown";
         var amount = (deal.properties.hasOwnProperty("amount")) ? deal.properties.amount.value : 0;
         var dealname = (deal.properties.hasOwnProperty("dealname")) ? deal.properties.dealname.value : "unknown";
         var hubspot_owner_id = (deal.properties.hasOwnProperty("hubspot_owner_id")) ? deal.properties.hubspot_owner_id.value : "unknown";
         var num_associated_contacts = (deal.properties.hasOwnProperty("num_associated_contacts")) ? deal.properties.num_associated_contacts.value : "unknown";
         deals.push([stageId,source,amount,dealname,num_associated_contacts,hubspot_owner_id]);
      });
   }
   return deals;
}

Answer 1

我认为您的问题来自您的offset 和limit URL 参数在url 中的多重规范：

"...&limit=250&offset&...&limit=250&offset=" + offset;

HubSpot 的 API 可能只期望某些关键字（例如 limit 和 offset）有一个值，这意味着您始终只访问结果的第一页 - 如果存在多个页面，您将永远不会停止调用此函数直到您用完 UrlFetchApp 配额并且脚本通过未处理的异常退出，因为 result.hasMore 始终为真。

我会重写您的脚本以使用 do-while 循环（同时也简化您的属性提取）。

function getDeals() {
  // Prepare authentication to Hubspot
  const service = getService();
  const fetchParams = {
    headers: {'Authorization': 'Bearer '+ service.getAccessToken()}
  };

  // Properties to collect from each deal:
  const desiredProps = [
    "dealstage",
    "source",
    "amount",
    "dealname",
    "num_associated_contacts",
    "hubspot_owner_id"
  ];
  const deals = [];

  // Hubspot lets you take max 250 deals per request.
  // Make multiple requests until we get all the deals.
  var offset = 0;
  var remainingPages = 100; // just in case.
  const url = API_URL + "/deals/v1/deal/paged?&includeAssociations=true&properties=dealstage&properties=source&properties=amount&properties=dealname&properties=num_associated_contacts&properties=hubspot_owner_id"
      + "&limit=250&offset=";

  do {
    var resp = UrlFetchApp.fetch(url + offset, fetchParams);
    var result = JSON.parse(response.getContentText());
    offset = result.offset;

    var pageDealInfo = result.deals.map(function (deal) {
      var dealInfo = desiredProperties.map(function (propName) {
        var val = deal.properties[propName];
        return (val === undefined ? "Unknown" : val;
      });
      /** add other things to dealInfo that aren't members of deal.properties
      dealInfo.push(deal.<something>);
       */
      return dealInfo;
    });

    // Add all the info for all the deals from this page of results.
    if (pageDealInfo.length)
      Array.prototype.push.apply(deals, pageDealInfo);
  } while (result.hasMore && --remainingPages);

  if (!remainingPages)
    console.warn({message: "Stopped deal queries due to own page limit - more deals exist!", currentOffset: offset, gatheredDealCount: deals.length});
  else
    console.log({message: "Finished deal queries", totalDeals: deals.length});

  return deals;
}

Answer 2

根据 HubSpots API 使用指南：

HubSpot 对 API 请求有以下限制：

• 每秒 10 个请求。

• 每天 40,000 个请求。此每日限制会根据 HubSpot 帐户的时区设置在午夜重置。

---

来源：https://developers.hubspot.com/apps/api_guidelines

交易 API 中最多可获取 250 条记录。

你应该这样做

除了交易列表，每个请求还会返回两个 值、偏移量和 hasMore。如果 hasMore 是真的，你需要做 另一个请求，使用偏移量获取交易的下一页 记录。

下一个请求中的offset 参数应该是上一个请求响应中返回的值。