【发布时间】：2015-07-20 12:17:19
【问题描述】：
公共类 DemoServlet 扩展 HttpServlet { /* 用于从 REST api 获取数据并将其放入的代码 HashMap 集合，然后从该集合创建管道并将管道写入 bq 表 */
String[] 凭据 = {"--project=" + projectId, "--stagingLocation=" + stagingLocation}; PipelineOptionsFactory.Builder builder = PipelineOptionsFactory.fromArgs(credentials); PipelineOptions options = builder.create();
//Setting to run the pipeline in google cloud.By Changing this parameter we can run even in local mode
options.setRunner(DataflowPipelineRunner.class);
//Creating pipeline with above options
Pipeline youTubeChannelPipeline = Pipeline.create(options);
PCollection<String> lines = null;
lines = youTubeChannelPipeline.apply(Create.of(allValues)).setCoder(StringUtf8Coder.of());
//Transforming PCollection<String> to PCollection<TableRow> and then writing to bq
bqRows = lines.apply(ParDo.named("将字符串转换为 TableRow").of(new DoFn() { @Override public void processElement(ProcessContext pc) 抛出异常 { 表行 tmpRow = 新表行（）； String[] columns = pc.element().split(YoutubeConstants.COMA_DELIMITER); for (int i = 0; i
//Writing the rows in bq table
bqRows.apply(BigQueryIO.Write
.named("Write To BQ")
.to(destination_path)
.withSchema(schema)
.withWriteDisposition(BigQueryIO.Write.WriteDisposition.WRITE_APPEND)
.withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED));
youTubeChannelPipeline.run();
W 12:36:16.124 /demo
java.lang.RuntimeException: Failed to construct instance from factory method com.google.cloud.dataflow.sdk.runners.DataflowPipelineRunner#fromOptions
at com.google.cloud.dataflow.sdk.util.InstanceBuilder.buildFromMethod(InstanceBuilder.java:228)
at com.google.cloud.dataflow.sdk.util.InstanceBuilder.build(InstanceBuilder.java:161)
at com.google.cloud.dataflow.sdk.runners.PipelineRunner.fromOptions(PipelineRunner.java:52)
at com.google.cloud.dataflow.sdk.Pipeline.create(Pipeline.java:117)
at myapp.DemoServlet.doGet(DemoServlet.java:91)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:617)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:717)
at org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511)
at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1166)
at com.google.apphosting.utils.servlet.ParseBlobUploadFilter.doFilter(ParseBlobUploadFilter.java:125)
at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1157)
at com.google.apphosting.runtime.jetty.SaveSessionFilter.doFilter(SaveSessionFilter.java:35)
at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1157)
at com.google.apphosting.utils.servlet.JdbcMySqlConnectionCleanupFilter.doFilter(JdbcMySqlConnectionCleanupFilter.java:60)
at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1157)
at com.google.apphosting.utils.servlet.TransactionCleanupFilter.doFilter(TransactionCleanupFilter.java:43)
at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1157)
at org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:388)
at org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216)
at org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182)
at org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:765)
at org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:418)
at com.google.apphosting.runtime.jetty.AppVersionHandlerMap.handle(AppVersionHandlerMap.java:257)
at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
at org.mortbay.jetty.Server.handle(Server.java:326)
at org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542)
at org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:923)
at com.google.apphosting.runtime.jetty.RpcRequestParser.parseAvailable(RpcRequestParser.java:76)
at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404)
at com.google.apphosting.runtime.jetty.JettyServletEngineAdapter.serviceRequest(JettyServletEngineAdapter.java:146)
at com.google.apphosting.runtime.JavaRuntime$RequestRunnable.run(JavaRuntime.java:482)
at com.google.tracing.TraceContext$TraceContextRunnable.runInContext(TraceContext.java:437)
at com.google.tracing.TraceContext$TraceContextRunnable$1.run(TraceContext.java:444)
at com.google.tracing.CurrentContext.runInContext(CurrentContext.java:230)
at com.google.tracing.TraceContext$AbstractTraceContextCallback.runInInheritedContextNoUnref(TraceContext.java:308)
at com.google.tracing.TraceContext$AbstractTraceContextCallback.runInInheritedContext(TraceContext.java:300)
at com.google.tracing.TraceContext$TraceContextRunnable.run(TraceContext.java:441)
at com.google.apphosting.runtime.ThreadGroupPool$PoolEntry.run(ThreadGroupPool.java:234)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.reflect.InvocationTargetException
at com.google.appengine.runtime.Request.process-8f9b045d763b7321(Request.java)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:45)
at com.google.cloud.dataflow.sdk.util.InstanceBuilder.buildFromMethod(InstanceBuilder.java:221)
at com.google.cloud.dataflow.sdk.util.InstanceBuilder.build(InstanceBuilder.java:161)
at com.google.cloud.dataflow.sdk.runners.PipelineRunner.fromOptions(PipelineRunner.java:52)
at com.google.cloud.dataflow.sdk.Pipeline.create(Pipeline.java:117)
at myapp.DemoServlet.doGet(DemoServlet.java:91)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:617)
at javax.servlet.http.HttpServlet.service(HttpServlet.java:717)
at org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511)
at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1166)
at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1157)
at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1157)
at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1157)
at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1157)
at org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:388)
at org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216)
at org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182)
at org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:765)
at org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:418)
at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152)
at org.mortbay.jetty.Server.handle(Server.java:326)
at org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542)
at org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:923)
at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404)
at com.google.tracing.TraceContext$TraceContextRunnable.runInContext(TraceContext.java:437)
at com.google.tracing.TraceContext$TraceContextRunnable$1.run(TraceContext.java:444)
at com.google.tracing.CurrentContext.runInContext(CurrentContext.java:230)
at com.google.tracing.TraceContext$AbstractTraceContextCallback.runInInheritedContextNoUnref(TraceContext.java:308)
at com.google.tracing.TraceContext$AbstractTraceContextCallback.runInInheritedContext(TraceContext.java:300)
at com.google.tracing.TraceContext$TraceContextRunnable.run(TraceContext.java:441)
... 1 more
Caused by: java.lang.RuntimeException: Unable to get application default credentials. Please see https://developers.google.com/accounts/docs/application-default-credentials for details on how to specify credentials. This version of the SDK is dependent on the gcloud core component version 2015.02.05 or newer to be able to get credentials from the currently authorized user via gcloud auth.
at com.google.cloud.dataflow.sdk.util.Credentials.getCredential(Credentials.java:122)
at com.google.cloud.dataflow.sdk.util.GcpCredentialFactory.getCredential(GcpCredentialFactory.java:43)
at com.google.cloud.dataflow.sdk.options.GcpOptions$GcpUserCredentialsFactory.create(GcpOptions.java:185)
at com.google.cloud.dataflow.sdk.options.GcpOptions$GcpUserCredentialsFactory.create(GcpOptions.java:175)
at com.google.cloud.dataflow.sdk.options.ProxyInvocationHandler.getDefault(ProxyInvocationHandler.java:288)
at com.google.cloud.dataflow.sdk.options.ProxyInvocationHandler.invoke(ProxyInvocationHandler.java:127)
at com.sun.proxy.$Proxy20.getGcpCredential(Unknown Source)
at com.google.cloud.dataflow.sdk.util.Transport.newDataflowClient(Transport.java:120)
at com.google.cloud.dataflow.sdk.options.DataflowPipelineDebugOptions$DataflowClientFactory.create(DataflowPipelineDebugOptions.java:157)
at com.google.cloud.dataflow.sdk.options.DataflowPipelineDebugOptions$DataflowClientFactory.create(DataflowPipelineDebugOptions.java:154)
at com.google.cloud.dataflow.sdk.options.ProxyInvocationHandler.getDefault(ProxyInvocationHandler.java:288)
at com.google.cloud.dataflow.sdk.options.ProxyInvocationHandler.invoke(ProxyInvocationHandler.java:127)
at com.sun.proxy.$Proxy20.getDataflowClient(Unknown Source)
at com.google.cloud.dataflow.sdk.runners.DataflowPipelineRunner.<init>(DataflowPipelineRunner.java:160)
<continued in next message>
W 12:36:16.124 <continued from previous message>
at com.google.cloud.dataflow.sdk.runners.DataflowPipelineRunner.fromOptions(DataflowPipelineRunner.java:155)
... 34 more
Caused by: java.io.IOException: Application Default Credentials failed to create the Google App Engine service account credentials class com.google.api.client.googleapis.extensions.appengine.auth.oauth2.AppIdentityCredential$AppEngineCredentialWrapper. Check that the component 'google-api-client-appengine' is deployed.
at com.google.api.client.googleapis.auth.oauth2.DefaultCredentialProvider.tryGetAppEngineCredential(DefaultCredentialProvider.java:274)
at com.google.api.client.googleapis.auth.oauth2.DefaultCredentialProvider.getDefaultCredentialUnsynchronized(DefaultCredentialProvider.java:159)
at com.google.api.client.googleapis.auth.oauth2.DefaultCredentialProvider.getDefaultCredential(DefaultCredentialProvider.java:86)
at com.google.api.client.googleapis.auth.oauth2.GoogleCredential.getApplicationDefault(GoogleCredential.java:213)
at com.google.api.client.googleapis.auth.oauth2.GoogleCredential.getApplicationDefault(GoogleCredential.java:191)
at com.google.cloud.dataflow.sdk.util.Credentials.getCredential(Credentials.java:120)
... 48 more
Caused by: java.lang.ClassNotFoundException: com.google.api.client.googleapis.extensions.appengine.auth.oauth2.AppIdentityCredential$AppEngineCredentialWrapper
at com.google.appengine.runtime.Request.process-8f9b045d763b7321(Request.java)
at java.lang.Class.forName(Class.java:136)
at com.google.api.client.googleapis.auth.oauth2.DefaultCredentialProvider.forName(DefaultCredentialProvider.java:208)
at com.google.api.client.googleapis.auth.oauth2.DefaultCredentialProvider.tryGetAppEngineCredential(DefaultCredentialProvider.java:259)
... 53 more
【问题讨论】：
标签： google-app-engine google-cloud-dataflow