【问题标题】:Why I am getting ScalaTest-dispatcher NPE error with Intellij, maven and scala testing?为什么我在 Intellij、maven 和 scala 测试中遇到 ScalaTest-dispatcher NPE 错误?
【发布时间】:2021-01-17 01:06:00
【问题描述】:

当我尝试在本地运行 spark 测试时出现此错误:

Exception in thread "ScalaTest-dispatcher" java.lang.NullPointerException
    at org.apache.spark.sql.internal.SQLConf$$anonfun$14.apply(SQLConf.scala:133)
    at org.apache.spark.sql.internal.SQLConf$$anonfun$14.apply(SQLConf.scala:133)
    at scala.Option.map(Option.scala:146)
    at org.apache.spark.sql.internal.SQLConf$.get(SQLConf.scala:133)
    at org.apache.spark.sql.types.DataType.sameType(DataType.scala:88)
    at org.apache.spark.sql.catalyst.analysis.TypeCoercion$$anonfun$haveSameType$1.apply(TypeCoercion.scala:288)
    at org.apache.spark.sql.catalyst.analysis.TypeCoercion$$anonfun$haveSameType$1.apply(TypeCoercion.scala:288)
    at scala.collection.LinearSeqOptimized$class.forall(LinearSeqOptimized.scala:83)
    at scala.collection.immutable.List.forall(List.scala:84)
    at org.apache.spark.sql.catalyst.analysis.TypeCoercion$.haveSameType(TypeCoercion.scala:288)
    at org.apache.spark.sql.catalyst.expressions.ComplexTypeMergingExpression$class.dataTypeCheck(Expression.scala:717)
    at org.apache.spark.sql.catalyst.expressions.CaseWhen.dataTypeCheck(conditionalExpressions.scala:121)
    at org.apache.spark.sql.catalyst.expressions.ComplexTypeMergingExpression$class.dataType(Expression.scala:723)
    at org.apache.spark.sql.catalyst.expressions.CaseWhen.dataType(conditionalExpressions.scala:121)
    at org.apache.spark.sql.catalyst.expressions.Alias.toAttribute(namedExpressions.scala:176)
    at org.apache.spark.sql.catalyst.plans.logical.Project$$anonfun$output$1.apply(basicLogicalOperators.scala:52)
    at org.apache.spark.sql.catalyst.plans.logical.Project$$anonfun$output$1.apply(basicLogicalOperators.scala:52)
    at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
    at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
    at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)
    at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48)
    at scala.collection.TraversableLike$class.map(TraversableLike.scala:234)
    at scala.collection.AbstractTraversable.map(Traversable.scala:104)
    at org.apache.spark.sql.catalyst.plans.logical.Project.output(basicLogicalOperators.scala:52)
    at org.apache.spark.sql.catalyst.plans.logical.Join.output(basicLogicalOperators.scala:311)
    at org.apache.spark.sql.catalyst.plans.logical.Join.output(basicLogicalOperators.scala:311)
    at org.apache.spark.sql.catalyst.plans.QueryPlan$$anonfun$inputSet$1.apply(QueryPlan.scala:51)
    at org.apache.spark.sql.catalyst.plans.QueryPlan$$anonfun$inputSet$1.apply(QueryPlan.scala:51)
    at scala.collection.TraversableLike$$anonfun$flatMap$1.apply(TraversableLike.scala:241)
    at scala.collection.TraversableLike$$anonfun$flatMap$1.apply(TraversableLike.scala:241)
    at scala.collection.immutable.List.foreach(List.scala:392)
    at scala.collection.TraversableLike$class.flatMap(TraversableLike.scala:241)
    at scala.collection.immutable.List.flatMap(List.scala:355)
    at org.apache.spark.sql.catalyst.plans.QueryPlan.inputSet(QueryPlan.scala:51)
    at org.apache.spark.sql.catalyst.plans.QueryPlan.missingInput(QueryPlan.scala:63)
    at org.apache.spark.sql.catalyst.plans.QueryPlan.statePrefix(QueryPlan.scala:173)
    at org.apache.spark.sql.catalyst.plans.logical.LogicalPlan.statePrefix(LogicalPlan.scala:65)
    at org.apache.spark.sql.catalyst.plans.QueryPlan.simpleString(QueryPlan.scala:175)
    at org.apache.spark.sql.catalyst.plans.QueryPlan.verboseString(QueryPlan.scala:177)
    at org.apache.spark.sql.catalyst.trees.TreeNode.generateTreeString(TreeNode.scala:551)
    at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$generateTreeString$3.apply(TreeNode.scala:569)
    at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$generateTreeString$3.apply(TreeNode.scala:569)
    at scala.collection.immutable.List.foreach(List.scala:392)
    at org.apache.spark.sql.catalyst.trees.TreeNode.generateTreeString(TreeNode.scala:569)
    at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$generateTreeString$3.apply(TreeNode.scala:569)
    at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$generateTreeString$3.apply(TreeNode.scala:569)
    at scala.collection.immutable.List.foreach(List.scala:392)
    at org.apache.spark.sql.catalyst.trees.TreeNode.generateTreeString(TreeNode.scala:569)
    at org.apache.spark.sql.catalyst.trees.TreeNode.generateTreeString(TreeNode.scala:571)
    at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$generateTreeString$3.apply(TreeNode.scala:569)
    at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$generateTreeString$3.apply(TreeNode.scala:569)
    at scala.collection.immutable.List.foreach(List.scala:392)
    at org.apache.spark.sql.catalyst.trees.TreeNode.generateTreeString(TreeNode.scala:569)
    at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$generateTreeString$3.apply(TreeNode.scala:569)
    at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$generateTreeString$3.apply(TreeNode.scala:569)
    at scala.collection.immutable.List.foreach(List.scala:392)
    at org.apache.spark.sql.catalyst.trees.TreeNode.generateTreeString(TreeNode.scala:569)
    at org.apache.spark.sql.catalyst.trees.TreeNode.generateTreeString(TreeNode.scala:571)
    at org.apache.spark.sql.catalyst.trees.TreeNode.generateTreeString(TreeNode.scala:571)
    at org.apache.spark.sql.catalyst.trees.TreeNode.treeString(TreeNode.scala:475)
    at org.apache.spark.sql.catalyst.trees.TreeNode.treeString(TreeNode.scala:472)
    at org.apache.spark.sql.catalyst.trees.TreeNode.toString(TreeNode.scala:469)
    at java.lang.String.valueOf(String.java:2994)
    at java.lang.StringBuilder.append(StringBuilder.java:131)
    at scala.StringContext.standardInterpolator(StringContext.scala:125)
    at scala.StringContext.s(StringContext.scala:95)
    at org.apache.spark.sql.AnalysisException$$anonfun$1.apply(AnalysisException.scala:46)
    at org.apache.spark.sql.AnalysisException$$anonfun$1.apply(AnalysisException.scala:46)
    at scala.Option.map(Option.scala:146)
    at org.apache.spark.sql.AnalysisException.getMessage(AnalysisException.scala:46)
    at java.lang.Throwable.getLocalizedMessage(Throwable.java:392)
    at java.lang.Throwable.toString(Throwable.java:481)
    at java.lang.String.valueOf(String.java:2994)
    at scala.collection.mutable.StringBuilder.append(StringBuilder.scala:200)
    at scala.collection.TraversableOnce$$anonfun$addString$1.apply(TraversableOnce.scala:359)
    at scala.collection.Iterator$class.foreach(Iterator.scala:891)
    at scala.collection.AbstractIterator.foreach(Iterator.scala:1334)
    at scala.collection.TraversableOnce$class.addString(TraversableOnce.scala:357)
    at scala.collection.AbstractIterator.addString(Iterator.scala:1334)
    at scala.collection.TraversableOnce$class.mkString(TraversableOnce.scala:323)
    at scala.collection.AbstractIterator.mkString(Iterator.scala:1334)
    at scala.runtime.ScalaRunTime$._toString(ScalaRunTime.scala:166)
    at scala.Some.toString(Option.scala:333)
    at java.lang.String.valueOf(String.java:2994)
    at scala.collection.mutable.StringBuilder.append(StringBuilder.scala:200)
    at scala.collection.TraversableOnce$$anonfun$addString$1.apply(TraversableOnce.scala:364)
    at scala.collection.Iterator$class.foreach(Iterator.scala:891)
    at scala.collection.AbstractIterator.foreach(Iterator.scala:1334)
    at scala.collection.TraversableOnce$class.addString(TraversableOnce.scala:357)
    at scala.collection.AbstractIterator.addString(Iterator.scala:1334)
    at scala.collection.TraversableOnce$class.mkString(TraversableOnce.scala:323)
    at scala.collection.AbstractIterator.mkString(Iterator.scala:1334)
    at scala.runtime.ScalaRunTime$._toString(ScalaRunTime.scala:166)
    at org.scalatest.events.TestFailed.toString(Event.scala:472)
    at java.text.MessageFormat.subformat(MessageFormat.java:1280)
    at java.text.MessageFormat.format(MessageFormat.java:865)
    at java.text.Format.format(Format.java:157)
    at org.scalatest.Resources$.makeString(Resources.scala:35)
    at org.scalatest.Resources$.apply(Resources.scala:38)
    at org.scalatest.DispatchReporter$Propagator.run(DispatchReporter.scala:244)
    at java.lang.Thread.run(Thread.java:748)

我的 Maven 依赖项:

    <dependency>
        <groupId>org.scalatest</groupId>
        <artifactId>scalatest_2.11</artifactId>
        <version>2.1.3</version>
        <scope>test</scope>
    </dependency>

    <dependency>
        <groupId>com.holdenkarau</groupId>
        <artifactId>spark-testing-base_2.11</artifactId>
        <version>2.4.4_0.14.0</version>
        <scope>test</scope>
    </dependency>

我只在测试期间遇到此问题。
我得到了最新的 intellij 版本:2020.1.4
我已经关注了这篇文章的答案,但没有任何改变:Why am I getting this error when running Scala 2.13 tests in IntelliJ, but not with Scala 2.12? :选中复选框Delegate IDE build/run actions to Maven

【问题讨论】:

    标签: scala maven apache-spark intellij-idea


    【解决方案1】:

    我的问题来自关于联合 2 数据帧的火花错误,我不能,但消息不明确。

    如果您有同样的问题,可以尝试使用本地 spark 会话进行测试。

    从您的测试类中删除 DataFrameSuiteBase 并改为创建本地 spark 会话:

    之前:

    class A extends FunSuite with BeforeAndAfterAll with DataFrameSuiteBase  {
      
    test("MethodB") {
      val df = spark.read()...
    
      }
    }
    

    之后:

    class A extends FunSuite with BeforeAndAfterAll {
      
    val spark = SparkSession.builder().master("local[*]").appName("test").GetOrCreate()
    
    test("MethodB") {
      val df = spark.read()...
    
      }
    }
    

    【讨论】:

      猜你喜欢
      • 2015-03-02
      • 2021-02-03
      • 2017-05-24
      • 2020-11-01
      • 2013-08-16
      • 2019-10-31
      • 1970-01-01
      • 2021-08-17
      • 1970-01-01
      相关资源
      最近更新 更多