按名称列出存储桶内容 GCS go

Question

我正在尝试使用 GO 读取 Google Cloud Storage 上存储桶的内容。 我可以这样做，但速度很慢。

bucket的内容是这样的：

bucket name 
-> folders with alphanumeric characters
----> 5 files into each of the folder
--------> each file has a json array inside

我想要做的是检查存储桶中所有文件夹的 jsons 文件的内容并查找特定值。以下代码可以工作，但速度很慢：

package backend

import (
    "encoding/json"
    "fmt"
    "golang.org/x/net/context"
    "golang.org/x/oauth2"
    "golang.org/x/oauth2/google"
    "google.golang.org/appengine"
    "google.golang.org/appengine/file"
    "google.golang.org/appengine/urlfetch"
    "google.golang.org/cloud"
    "google.golang.org/cloud/storage"
    "io"
    "io/ioutil"
    "net/http"
)

var bucket = "bucket_Name"

type jsonStruct struct {
    Gender string `json:"gender"`
    Age    string `json:"age"`
    ID     string `json:"id"`
    Done   int    `json:"done"`
}

type saveData struct {
    c       context.Context
    r       *http.Request       //http response
    w       http.ResponseWriter //http writer
    ctx     context.Context
    cleanUp []string // cleanUp is a list of filenames that need cleaning up at the end of the saving.
    failed  bool     // failed indicates that one or more of the saving steps failed.
}

func init() {
    http.HandleFunc("/", handleStatic)
    http.HandleFunc("/listBuckets", listBuckets)
}

func handleStatic(w http.ResponseWriter, r *http.Request) {
    w.Header().Set("Cache-Control", "no-cache")
    http.ServeFile(w, r, "static/"+r.URL.Path)
}

func listBuckets(w http.ResponseWriter, r *http.Request) {
    c := appengine.NewContext(r)
    if bucket == "" {
        var err error
        if bucket, err = file.DefaultBucketName(c); err != nil {
            // log.Errorf(c, "failed to get default GCS bucket name: %v", err)
            return
        }
    }
    hc := &http.Client{
        Transport: &oauth2.Transport{
            Source: google.AppEngineTokenSource(c, storage.ScopeFullControl),
            Base:   &urlfetch.Transport{Context: c},
        },
    }
    ctx := cloud.NewContext(appengine.AppID(c), hc)

    // structure to holds information needed to run the various saving functions
    d := &saveData{
        c:   c,
        r:   r,
        w:   w,
        ctx: ctx,
    }

    d.listBucket(bucket)
}

func (d *saveData) errorf(format string, args ...interface{}) {
    d.failed = true
    // log.Errorf(d.c, format, args...)
}

func (d *saveData) listBucket(bucket string) {
    io.WriteString(d.w, "\nListbucket result:\n")

    query := &storage.Query{}
    for query != nil {
        objs, err := storage.ListObjects(d.ctx, bucket, query)
        if err != nil {
            d.errorf("listBucket: unable to list bucket %q: %v", bucket, err)
            return
        }
        query = objs.Next
        for _, obj := range objs.Results {
            d.readFile(obj.Name)
        }

    }
}

func (d *saveData) readFile(fileName string) {

    rc, err := storage.NewReader(d.ctx, bucket, fileName)
    if err != nil {
        d.errorf("readFile: unable to open file from bucket %q, file %q: %v", bucket, fileName, err)
        return
    }
    defer rc.Close()
    slurp, err := ioutil.ReadAll(rc)
    if err != nil {
        d.errorf("readFile: unable to read data from bucket %q, file %q: %v", bucket, fileName, err)
        return
    }
    var userDetails jsonStruct
    err1 := json.Unmarshal(slurp, &userDetails)
    if err1 != nil {
        d.errorf("readFile: %v", err1)
        return
    }
    fmt.Fprintf(d.w, "done is: %v\n", userDetails.Done)
}

现在我基本上是从存储桶中读取文件夹名称，然后使用文件夹名称读取内容。是否可以将所有存储桶内容缓存在一个 go 变量中，然后处理该变量而不是读取每个文件夹的存储桶？

我真的需要更快，因为我需要实时呈现结果。

非常感谢

Answer 1

请参阅下面的简单 Go 示例代码，以列出 Google Cloud Storage 上的存储桶内容：

package main

import (
    "context"
    "fmt"
    "log"
    "os"

    "cloud.google.com/go/storage"
    "google.golang.org/api/iterator"
)

func main() {
    os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", 
              "C:\\Users\\Shubham Snehi\\Downloads\\awacs-dev-160bf0e57dc1.json")

    ctx := context.Background()

    client, err := storage.NewClient(ctx)
    if err != nil {
        log.Fatalf("Failed to create client: %v", err)
    }
    defer client.Close()

    // Sets the name for the new bucket.
    bucketName := "balatestawacs"

    // Creates a Bucket instance.
    bucket := client.Bucket(bucketName)

    it := bucket.Objects(ctx, nil)
    for {
        attrs, err := it.Next()
        if err == iterator.Done {
            break
        }
        if err != nil {
            panic(err)
        }
        fmt.Println(attrs.Owner)
    }
}