【问题标题】:Javascript_api_promise problem / need faster solutionJavascript_api_promise 问题/需要更快的解决方案
【发布时间】:2020-08-31 00:42:50
【问题描述】:

agsi web 给单个毒气室运营商提供api,在一些国家运营商很少。我必须得到所有运营商的总和。例如在德国有 25 个 SSO(存储系统运营商)。我为他们每个人都有一个 api。

我设法得到了我想要的数组,但德国需要 20 秒(最大)

在图表中获取和使用这些数据的最佳方式是什么,我在本地存储数据。 (也许每天使用 D3 保存为 CSV 一次,或者使用 python 将数据作为 js 文件中的变量输入? 我将它用于学习目的,现在我还不够熟练地使用服务器端。

let apiKey = "709a7d013e4fda8f3e21166c33a1a691";
let urls = [
'https://cors-anywhere.herokuapp.com/https://agsi.gie.eu/api/data/21X000000001160J/DE',
'https://cors-anywhere.herokuapp.com/https://agsi.gie.eu/api/data/37X0000000000151/DE',
'https://cors-anywhere.herokuapp.com/https://agsi.gie.eu/api/data/37X0000000000224/DE',
'https://cors-anywhere.herokuapp.com/https://agsi.gie.eu/api/data/21X000000001125L/DE',
'https://cors-anywhere.herokuapp.com/https://agsi.gie.eu/api/data/21X0000000010849/DE',
'https://cors-anywhere.herokuapp.com/https://agsi.gie.eu/api/data/21X000000001368W/DE',
'https://cors-anywhere.herokuapp.com/https://agsi.gie.eu/api/data/21X0000000011756/DE',
'https://cors-anywhere.herokuapp.com/https://agsi.gie.eu/api/data/21X000000001090E/DE',
'https://cors-anywhere.herokuapp.com/https://agsi.gie.eu/api/data/21X0000000013805/DE',
'https://cors-anywhere.herokuapp.com/https://agsi.gie.eu/api/data/21X000000001262B/DE',
'https://cors-anywhere.herokuapp.com/https://agsi.gie.eu/api/data/21X000000001140P/DE',
'https://cors-anywhere.herokuapp.com/https://agsi.gie.eu/api/data/37X000000000042Z/DE',
'https://cors-anywhere.herokuapp.com/https://agsi.gie.eu/api/data/21X0000000011748/DE',
'https://cors-anywhere.herokuapp.com/https://agsi.gie.eu/api/data/11XNERGIE------1/DE',
'https://cors-anywhere.herokuapp.com/https://agsi.gie.eu/api/data/37X0000000000119/DE',
'https://cors-anywhere.herokuapp.com/https://agsi.gie.eu/api/data/25X-OMVGASSTORA5/DE',
'https://cors-anywhere.herokuapp.com/https://agsi.gie.eu/api/data/11XSWHANNOVERAG3/DE',
'https://cors-anywhere.herokuapp.com/https://agsi.gie.eu/api/data/21X000000001072G/DE',
'https://cors-anywhere.herokuapp.com/https://agsi.gie.eu/api/data/11XSWB-BREMEN--I/DE',
'https://cors-anywhere.herokuapp.com/https://agsi.gie.eu/api/data/37X000000000051Y/DE',
'https://cors-anywhere.herokuapp.com/https://agsi.gie.eu/api/data/21X000000001307F/DE',
'https://cors-anywhere.herokuapp.com/https://agsi.gie.eu/api/data/21X000000001310Q/DE',
'https://cors-anywhere.herokuapp.com/https://agsi.gie.eu/api/data/21X000000001127H/DE',
'https://cors-anywhere.herokuapp.com/https://agsi.gie.eu/api/data/21X000000001138C/DE',
];

let requests = urls.map((url) =>
  fetch(url, {
    method: "GET",
    headers: {
      "x-key": apiKey
    }
  }).then((res) => res.json())
);

Promise.all(requests).then((response) => arr_api_agsi(response));

function arr_api_agsi(response) {
  var arr_api=[];
  var l=[];
  var i;
  var j;
  var jj;
  var gas_date;
	var response_length=response.length
  for (i = 0; i < response_length; i++) {
         arr_api.push(response[i]);
         l.push(response[i].length);
  }
  var gas_vol=0;
  var arr_max_index=0;
  l.map((u)=>u.length>l[arr_max_index].length ? arr_max_index=l.indexOf(u.length):x=1);

  var gas_storage=[];
  var gas_storage_temp=[];
  var len_single_arr;
	var len_max_arr=arr_api[arr_max_index].length
  for(i=0;i<len_max_arr;i++){  //for all dates in max length array
  	len_max_arr_j=arr_api.length
    for(j=0;j<len_max_arr_j;j++){ //for all arrays
    	len_single_arr=arr_api[j].length
      for(jj=0;jj< len_single_arr;jj++){ //for all dates in single array
        if (arr_api[arr_max_index][i]["gasDayStartedOn"]==arr_api[j][jj]["gasDayStartedOn"] &&  isNaN(parseFloat(arr_api[j][jj]["gasInStorage"]))==false) {
          gas_vol+=parseFloat(arr_api[j][jj]["gasInStorage"]);
          gas_date=arr_api[j][jj]["gasDayStartedOn"];
        }
      }
    }
    gas_storage_temp.push([gas_date,gas_vol]);
    gas_storage=gas_storage.concat(gas_storage_temp);
    gas_storage_temp=[];
    gas_vol=0;
  }
console.log(gas_storage)
}

【问题讨论】:

    标签: javascript api promise


    【解决方案1】:

    您可以在此处使用转换器来解决您的性能问题。

    首先,安装deno

    这是一个您可以手动运行的脚本,如果您想要自动化,可以将其安排为 cron 作业。

    你可以这样运行它:deno run --allow-net gas.js

    gas.js

    import {
      pipe, fork, tap, map, filter, reduce, transform, get, not,
    } from 'https://deno.land/x/rubico/rubico.js'
    
    const apiKey = "709a7d013e4fda8f3e21166c33a1a691"
    
    const urls = [
      'https://cors-anywhere.herokuapp.com/https://agsi.gie.eu/api/data/21X000000001160J/DE',
      'https://cors-anywhere.herokuapp.com/https://agsi.gie.eu/api/data/37X0000000000151/DE',
      'https://cors-anywhere.herokuapp.com/https://agsi.gie.eu/api/data/37X0000000000224/DE',
      'https://cors-anywhere.herokuapp.com/https://agsi.gie.eu/api/data/21X000000001125L/DE',
      'https://cors-anywhere.herokuapp.com/https://agsi.gie.eu/api/data/21X0000000010849/DE',
      'https://cors-anywhere.herokuapp.com/https://agsi.gie.eu/api/data/21X000000001368W/DE',
      'https://cors-anywhere.herokuapp.com/https://agsi.gie.eu/api/data/21X0000000011756/DE',
      'https://cors-anywhere.herokuapp.com/https://agsi.gie.eu/api/data/21X000000001090E/DE',
      'https://cors-anywhere.herokuapp.com/https://agsi.gie.eu/api/data/21X0000000013805/DE',
      'https://cors-anywhere.herokuapp.com/https://agsi.gie.eu/api/data/21X000000001262B/DE',
      'https://cors-anywhere.herokuapp.com/https://agsi.gie.eu/api/data/21X000000001140P/DE',
      'https://cors-anywhere.herokuapp.com/https://agsi.gie.eu/api/data/37X000000000042Z/DE',
      'https://cors-anywhere.herokuapp.com/https://agsi.gie.eu/api/data/21X0000000011748/DE',
      'https://cors-anywhere.herokuapp.com/https://agsi.gie.eu/api/data/11XNERGIE------1/DE',
      'https://cors-anywhere.herokuapp.com/https://agsi.gie.eu/api/data/37X0000000000119/DE',
      'https://cors-anywhere.herokuapp.com/https://agsi.gie.eu/api/data/25X-OMVGASSTORA5/DE',
      'https://cors-anywhere.herokuapp.com/https://agsi.gie.eu/api/data/11XSWHANNOVERAG3/DE',
      'https://cors-anywhere.herokuapp.com/https://agsi.gie.eu/api/data/21X000000001072G/DE',
      'https://cors-anywhere.herokuapp.com/https://agsi.gie.eu/api/data/11XSWB-BREMEN--I/DE',
      'https://cors-anywhere.herokuapp.com/https://agsi.gie.eu/api/data/37X000000000051Y/DE',
      'https://cors-anywhere.herokuapp.com/https://agsi.gie.eu/api/data/21X000000001307F/DE',
      'https://cors-anywhere.herokuapp.com/https://agsi.gie.eu/api/data/21X000000001310Q/DE',
      'https://cors-anywhere.herokuapp.com/https://agsi.gie.eu/api/data/21X000000001127H/DE',
      'https://cors-anywhere.herokuapp.com/https://agsi.gie.eu/api/data/21X000000001138C/DE',
    ]
    
    const getData = url => fetch(url, {
      method: 'GET',
      headers: {
        'x-key': apiKey,
        'Origin': null,
      },
    }).then(res => res.json())
    
    const incrementMap = (y, xi) => {
      const [gasDate, gasVol] = xi
      if (y.has(gasDate)) {
        y.set(gasDate, y.get(gasDate) + gasVol)
      } else {
        y.set(gasDate, gasVol)
      }
      return y
    }
    
    // url => [gas_record]
    const toGasDateVolRecordsMap = url => pipe([
      getData,
      reduce(pipe([
        map(fork([
          get('gasDayStartedOn'),
          pipe([get('gasInStorage'), parseFloat]),
        ])),
        filter(pipe([get(1), not(isNaN)])),
      ])(incrementMap), new Map()),
      tap(() => console.log('done with', url)),
    ])(url)
    
    // mapA, mapB -> mapC
    const combineMaps = (y, xi) => {
      let newMap = y
      for (const record of xi) {
        newMap = incrementMap(newMap, record)
      }
      return y
    }
    
    console.log('beginning requests')
    const now = Date.now()
    const finalMap = await pipe([
      map(toGasDateVolRecordsMap),
      reduce(combineMaps, new Map()),
    ])(urls)
    
    console.log(finalMap, 'in', Date.now() - now, 'ms')
    

    这是我上次运行的结果

    Map {
      "2020-05-21" => 185.76999999999998,
      "2020-05-20" => 185.1071,
      "2020-05-19" => 184.6843,
      "2020-05-18" => 184.32969999999997,
      "2020-05-17" => 183.7246,
      "2020-05-16" => 182.92209999999997,
      "2020-05-15" => 182.33570000000003,
      "2020-05-14" => 182.1236,
      "2020-05-13" => 182.0282,
      "2020-05-12" => 181.98539999999997,
      "2020-05-11" => 182.32399999999998,
      "2020-05-10" => 181.7332,
      "2020-05-09" => 180.5206,
      "2020-05-08" => 179.3828,
      "2020-05-07" => 178.50409999999997,
      "2020-05-06" => 178.0811,
      "2020-05-05" => 177.8532,
      "2020-05-04" => 177.62040000000002,
      "2020-05-03" => 177.2416,
      "2020-05-02" => 176.63550000000004,
      "2020-05-01" => 175.9856,
      "2020-04-30" => 175.1376,
      "2020-04-29" => 174.47029999999998,
      "2020-04-28" => 173.93019999999999,
      "2020-04-27" => 173.61330000000004,
      "2020-04-26" => 173.28529999999998,
      "2020-04-25" => 172.62169999999998,
      "2020-04-24" => 171.8595,
      "2020-04-23" => 171.2766,
      "2020-04-22" => 170.8136,
      "2020-04-21" => 170.1925,
      "2020-04-20" => 169.55120000000002,
      "2020-04-19" => 169.0327,
      "2020-04-18" => 168.12290000000002,
      "2020-04-17" => 167.17200000000003,
      "2020-04-16" => 166.79960000000003,
      "2020-04-15" => 166.4389,
      "2020-04-14" => 166.3951,
      "2020-04-13" => 166.4094,
      "2020-04-12" => 165.82659999999998,
      "2020-04-11" => 164.98540000000003,
      "2020-04-10" => 164.2925,
      "2020-04-09" => 163.71460000000002,
      "2020-04-08" => 163.22520000000003,
      "2020-04-07" => 162.8278,
      "2020-04-06" => 162.46410000000003,
      "2020-04-05" => 162.05689999999998,
      "2020-04-04" => 161.66649999999998,
      "2020-04-03" => 161.48640000000003,
      "2020-04-02" => 161.7566,
      "2020-04-01" => 162.1767,
      "2020-03-31" => 160.53390000000005,
      "2020-03-30" => 162.0816,
      "2020-03-29" => 163.01540000000003,
      "2020-03-28" => 163.20020000000002,
      "2020-03-27" => 163.1253,
      "2020-03-26" => 163.52140000000003,
      "2020-03-25" => 164.3272,
      "2020-03-24" => 165.20749999999998,
      "2020-03-23" => 165.9939,
      "2020-03-22" => 166.9542,
      "2020-03-21" => 167.37539999999998,
      "2020-03-20" => 167.7758,
      "2020-03-19" => 168.15980000000002,
      "2020-03-18" => 168.52270000000001,
      "2020-03-17" => 168.68449999999999,
      "2020-03-16" => 168.9426,
      "2020-03-15" => 169.2491,
      "2020-03-14" => 169.4138,
      "2020-03-13" => 169.8962,
      "2020-03-12" => 170.3027,
      "2020-03-11" => 170.4511,
      "2020-03-10" => 170.6003,
      "2020-03-09" => 170.9641,
      "2020-03-08" => 171.48220000000003,
      "2020-03-07" => 171.61350000000002,
      "2020-03-06" => 172.0182,
      "2020-03-05" => 172.5045,
      "2020-03-04" => 173.2632,
      "2020-03-03" => 173.9052,
      "2020-03-02" => 174.4052,
      "2020-03-01" => 175.70919999999998,
      "2020-02-29" => 176.10870000000003,
      "2020-02-28" => 176.14049999999997,
      "2020-02-27" => 177.47449999999998,
      "2020-02-26" => 178.7926,
      "2020-02-25" => 179.75709999999998,
      "2020-02-24" => 180.1693,
      "2020-02-23" => 180.71120000000002,
      "2020-02-22" => 180.8336,
      "2020-02-21" => 181.0428,
      "2020-02-20" => 181.5118,
      "2020-02-19" => 182.02380000000002,
      "2020-02-18" => 182.5467,
      "2020-02-17" => 183.13459999999998,
      "2020-02-16" => 183.5387,
      "2020-02-15" => 183.539,
      "2020-02-14" => 183.67700000000005,
      "2020-02-13" => 184.24939999999998,
      "2020-02-12" => 185.1237,
      ... 3329 more items
    } in 7902 ms
    

    如果您不喜欢结尾处的Map,可以通过Array.from(finalMap) 获得Array

    【讨论】:

      猜你喜欢
      • 2020-09-26
      • 1970-01-01
      • 2022-11-29
      • 2011-02-07
      • 1970-01-01
      • 1970-01-01
      • 1970-01-01
      • 2011-07-01
      • 2011-05-05
      相关资源
      最近更新 更多