使用 Github GraphQL API 从一个 Promise.All 中获取不同数组的结果

Question

我正在为 Gatsby 制作一个自定义源插件，它将从 GitHub 存储库获取降价文件。存储库有单独的文件（blob）和文件夹（树），它们又包含文件。我需要在一个Promise.all 中获取所有文件（包括文件夹中的文件），但我不知道该怎么做。我已经设法从存储库中获取单个文件，并且我有一个函数可以从树中返回一组文件。但我不知道如何组合它们。

这是我的代码。 GraphQL 查询以获取存储库、树和文件信息：

const repositoryQuery = `
{
  viewer {
    repository(name: "repository-name") {
      object(expression: "master:") {
        ... on Tree {
          entries {
            name
            oid
            type
          }
        }
      }
    }
  }
}
`

const treeQuery = `
  query getTree($id: GitObjectID!) {
    viewer {
      repository(name: "repository-name") {
        object(oid: $id) {
          ... on Tree {
            entries {
              name
              oid
              type
            }
          }
        }
      }
    }
  }
`

const fileQuery = `
  query getFile($id: GitObjectID!) {
    viewer {
      repository(name: "repository-name") {
        object(oid: $id) {
          ... on Blob {
            text
          }
        }
      }
    }
  }
` 

还有函数本身：

const data = await client.request(repositoryQuery)

const getTree = async entry => {
  const data = await client.request(treeQuery, { id: entry.oid })
  const array = await data.viewer.repository.object.entries
  return array
}

const getFile = async entry => {
  const data = await client.request(fileQuery, { id: entry.oid })
  const result = await data.viewer.repository.object
  return result
}

const files = await Promise.all(
  data.viewer.repository.object.entries
    .filter(entry => entry.type !== "tree")
    .map(entry => {
      return (
        getFile(entry)
        .then(file => {
          return {
            data: file.text
          }
        })
      )
    }
  )
)

files.forEach(file =>
  createNode({...})
)

如何更新const files 以便它：

1. 运行getFile()，如果entry.type !== "tree"
2. 如果entry.type 是tree，则使用getTree() 获取树中的文件数组，然后为每个文件运行getFile()。
3. 将所有结果合并到一个数组中，以便我可以应用它们createNode。

非常感谢您的帮助。

Answer 1

您可以从用于递归遍历目录的 walk 函数中获得一些灵感。来自there。它看起来像这样：

async function walk(entry, isRoot) {
  if (isRoot){
    return await processEntry(entry);
  }
  let files = await getTreeEntryFromTree(repository, entry.oid);
  files = await Promise.all(files.data.viewer.repository.object.entries.map(async file => {
    return await processEntry(file);
  }));
  return files.reduce((all, folderContents) => all.concat(folderContents), []);
}

async function processEntry(entry){
  if (entry.type === "tree") {
    return walk(entry, false); 
  } else {
    let res = await getBlob(repository, entry.oid);
    return [{
      name: entry.name,
      oid: entry.oid,
      data:res.data.viewer.repository.object.text
    }];
  }
}

所以，它只是用树替换目录，并在您返回文件时请求每个文件的数据内容。

源插件的以下gatsby-node.js 代码（没有createSchemaCustomization）：

const { ApolloClient } = require("apollo-client")
const { InMemoryCache } = require("apollo-cache-inmemory")
const { HttpLink } = require("apollo-link-http")
const fetch = require("node-fetch")
const gql = require("graphql-tag")
const { setContext } = require('apollo-link-context');

const token = "YOUR_TOKEN";
const repository = "YOUR_REPO";

const authLink = setContext((_, { headers }) => {
  return {
    headers: {
      ...headers,
      authorization: token ? `Bearer ${token}` : null,
    }
  }
});

const defaultOptions = {
  watchQuery: {
    fetchPolicy: 'no-cache',
    errorPolicy: 'ignore',
  },
  query: {
    fetchPolicy: 'no-cache',
    errorPolicy: 'all',
  },
}

const client = new ApolloClient({
  link: authLink.concat(new HttpLink({ uri: 'https://api.github.com/graphql', fetch: fetch  })),
  cache: new InMemoryCache(),
  defaultOptions: defaultOptions,
});

exports.sourceNodes = async function sourceNodes(
  {
    actions,
    cache,
    createContentDigest,
    createNodeId,
    getNodesByType,
    getNode,
  },
  pluginOptions
) {
  const { createNode, touchNode, deleteNode } = actions
  const { data } = await getTreeFromRepo(repository)

  let sourceData = data;

  fileArr = []
  sourceData.viewer.repository.object.entries.map(it => {
    fileArr.push(walk(it, true))
  });
  let res = await Promise.all(fileArr)
  let result = res.flat();
  console.log(result);
  console.log(`got ${result.length} results`);
  return
}

async function walk(entry, isRoot) {
  if (isRoot){
    return await processEntry(entry);
  }
  let files = await getTreeEntryFromTree(repository, entry.oid);
  files = await Promise.all(files.data.viewer.repository.object.entries.map(async file => {
    return await processEntry(file);
  }));
  return files.reduce((all, folderContents) => all.concat(folderContents), []);
}

async function processEntry(entry){
  if (entry.type === "tree") {
    return walk(entry, false); 
  } else {
    let res = await getBlob(repository, entry.oid);
    return [{
      name: entry.name,
      oid: entry.oid,
      data:res.data.viewer.repository.object.text
    }];
  }
}

async function getTreeFromRepo(repo) {
    return await client.query({
      query: gql`
        query {
          viewer {
            repository(name: "${repo}") {
              object(expression: "master:") {
                ... on Tree {
                  entries {
                    name
                    oid
                    type
                  }
                }
              }
            }
          }
        }
      `,
    })
}

async function getTreeEntryFromTree(repo, oid) {
  return await client.query({
    query: gql`
      query getTree($id: GitObjectID!) {
        viewer {
          repository(name: "${repo}") {
            object(oid: $id) {
              ... on Tree {
                entries {
                  name
                  oid
                  type
                }
              }
            }
          }
        }
      }
    `,
    variables: {
      id: oid
    }
  })
}

async function getBlob(repo, oid){
  return await client.query({
    query: gql`
      query getFile($id: GitObjectID!) {
        viewer {
          repository(name: "${repo}") {
            object(oid: $id) {
              ... on Blob {
                text
              }
            }
          }
        }
      }
    `,
    variables: {
      id: oid
    }
  })
}

您需要替换上面代码中的 Github 令牌和 repo 名称。

它返回一个包含文件内容、名称和 oid 的对象数组

请注意，使用 ... on Blob { text } 返回 null for binary file ：

text (String) UTF8 文本数据，如果 Blob 是二进制则为 null

---

此外，还可以使用 Github API v3 在一次调用中递归遍历树，这大大减少了请求的数量。你会有这样的东西：

async function getAllEntries(repo, owner){
  return fetch(`https://api.github.com/repos/${owner}/${repo}/git/trees/master?recursive=1`,{
    headers: {
      'Authorization': `Bearer ${token}`,
    }
  })
  .then(response => response.json());
}

完整示例（用于 Gatsby 源插件）：

const { ApolloClient } = require("apollo-client")
const { InMemoryCache } = require("apollo-cache-inmemory")
const { HttpLink } = require("apollo-link-http")
const fetch = require("node-fetch")
const gql = require("graphql-tag")
const { setContext } = require('apollo-link-context');

const token = "YOUR_TOKEN";
const repository = "YOUR_REPO";
const owner = "YOUR_LOGIN";

const authLink = setContext((_, { headers }) => {
  return {
    headers: {
      ...headers,
      authorization: token ? `Bearer ${token}` : null,
    }
  }
});

const defaultOptions = {
  watchQuery: {
    fetchPolicy: 'no-cache',
    errorPolicy: 'ignore',
  },
  query: {
    fetchPolicy: 'no-cache',
    errorPolicy: 'all',
  },
}

const client = new ApolloClient({
  link: authLink.concat(new HttpLink({ uri: 'https://api.github.com/graphql', fetch: fetch  })),
  cache: new InMemoryCache(),
  defaultOptions: defaultOptions,
});

exports.sourceNodes = async function sourceNodes(
  {
    actions,
    cache,
    createContentDigest,
    createNodeId,
    getNodesByType,
    getNode,
  },
  pluginOptions
) {
  const { createNode, touchNode, deleteNode } = actions
  const { tree } = await getAllEntries(repository, owner)
  fileArr = []
  tree.map(it => {
    fileArr.push(walk(it, true))
  });
  let res = await Promise.all(fileArr)
  let result = res.filter(value => Object.keys(value).length !== 0);
  console.log(result);
  console.log(`got ${result.length} results`);
  return
}

async function walk(entry){
  if (entry.type === "blob") {
    let res = await getBlob(repository, entry.sha);
    return {
      name: entry.path,
      oid: entry.sha,
      data: res.data.viewer.repository.object.text
    };
  }
  return {};
}

async function getAllEntries(repo, owner){
  return fetch(`https://api.github.com/repos/${owner}/${repo}/git/trees/master?recursive=1`,{
    headers: {
      'Authorization': `Bearer ${token}`,
    }
  })
  .then(response => response.json());
}

async function getBlob(repo, oid){
  return await client.query({
    query: gql`
      query getFile($id: GitObjectID!) {
        viewer {
          repository(name: "${repo}") {
            object(oid: $id) {
              ... on Blob {
                text
              }
            }
          }
        }
      }
    `,
    variables: {
      id: oid
    }
  })
}

---

如果您需要不惜一切代价获取二进制内容，则需要使用 Github API v3，它直接在获取树结果中提供内容 url。内容URL返回base64编码的内容，见this file。

因此，如果您想要 base64 编码的内容（二进制 + 文本），您将拥有以下 gatsby-node.js（用于源插件）：

const fetch = require("node-fetch")

const token = "YOUR_TOKEN";
const repository = "YOUR_REPO";
const owner = "YOUR_LOGIN";

exports.sourceNodes = async function sourceNodes(
  {
    actions,
    cache,
    createContentDigest,
    createNodeId,
    getNodesByType,
    getNode,
  },
  pluginOptions
) {
  const { createNode, touchNode, deleteNode } = actions
  const { tree } = await getAllEntries(repository, owner)
  fileArr = []
  tree.map(it => {
    fileArr.push(walk(it, true))
  });
  let res = await Promise.all(fileArr)
  console.log(res);
  console.log(`got ${res.length} results`);
  return
}

async function walk(entry){
  if (entry.type === "blob") {
    let res = await getBlob(entry.url);
    return {
      name: entry.path,
      oid: entry.sha,
      data: res.content
    };
  }
  return {};
}

async function getAllEntries(repo, owner){
  return fetch(`https://api.github.com/repos/${owner}/${repo}/git/trees/master?recursive=1`, {
    headers: {
      'Authorization': `Bearer ${token}`,
    }
  })
  .then(response => response.json());
}

async function getBlob(url){
  return fetch(url, {
    headers: {
      'Authorization': `Bearer ${token}`,
    }
  })
  .then(response => response.json());
}

Answer 2

首先您可以遍历每棵树，然后为每棵树获取一个文件数组，这将为您提供一个二维数组：

.map(async entry => {
  const files = await getTree(entry);
  return Promise.all(
    files.map(file => getFile(file).then(fileRes => ({ data: fileRes.text })))
  );
)

然后你需要将结果展平，使其成为一维数组：

const files = allFiles.flat();

希望我正确理解了您的问题； getTree() 的结果是文件的一维数组（即[file1, file2, file3]），而不是多维数组（即[[file1, file2], [[file1, file2], [file1]], file1, file2]）。