引起：java.lang.ClassNotFoundException: play.api.libs.functional.syntax.package

Question

我在尝试运行我的代码时收到以下错误（由：java.lang.ClassNotFoundException: play.api.libs.functional.syntax.package）我有正确的依赖关系并在我的 spark 中添加了正确的 Jar lib 文件夹仍然无法弄清楚。

play-json_2.11-2.7.4.jar --> /usr/lib/spark/jars/

<dependency>
   <groupId>com.typesafe.play</groupId>
   <artifactId>play-json_2.11</artifactId>
   <version>2.7.0</version>
</dependency>

spark-submit --jars /usr/lib/spark/jars/play-json_2.11-2.7.4.jar \
--class com.MyClass \
${jar_path}

谢谢 斯里

Answer 1

修复了我已经遮蔽了几个运行良好的罐子的错误。

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
         http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>com.blah</groupId>
    <artifactId>blah</artifactId>
    <version>2</version>

    <properties>
        <jacoco.report.ut>${project.build.directory}/jacoco-ut.exec</jacoco.report.ut>
        <jacoco.report.it>${project.build.directory}/jacoco-it.exec</jacoco.report.it>
        <sonar.language>scala</sonar.language>
        <sonar.sources>src/main/scala</sonar.sources>
        <sonar.tests>src/test/scala</sonar.tests>
        <sonar.junit.reportPaths>${project.build.directory}/surefire-reports</sonar.junit.reportPaths>
        <sonar.jacoco.reportPaths>${jacoco.report.ut},${jacoco.report.it}</sonar.jacoco.reportPaths>
        <sonar-maven-plugin.version>3.2</sonar-maven-plugin.version>
        <sonar.host.url>https://sonar.cloud.blah.com </sonar.host.url>
        <sonar.projectKey>blah</sonar.projectKey>
        <sonar.coverage.jacoco.xmlReportPaths>target/jacoco-ut/jacoco.xml</sonar.coverage.jacoco.xmlReportPaths>
        <jacoco.plugin.version>0.8.3</jacoco.plugin.version>
        <java.version>1.8</java.version>
        <encoding>UTF-8</encoding>
        <scala.version>2.11.8</scala.version>
        <jackson.version>2.10.3</jackson.version>
        <sonar.version>3.0</sonar.version>
        <sonar.pluginKey>scala</sonar.pluginKey>
        <stream-poc.version>0.2.4-23d5a6b</stream-poc.version>
        <grpc.version>1.24.0</grpc.version>
        <protobuf.version>3.6.1</protobuf.version>
        <guava.version>26.0-android</guava.version>
        <spark.version>2.4.5</spark.version>

    </properties>


    <distributionManagement>
        <repository>
            <id>central</id>
            <name>blah Artifactory-releases</name>
            <url>https://artifactory.cloud.blah.com:443/artifactory/maven-internalfacing</url>
        </repository>
        <snapshotRepository>
            <id>snapshots</id>
            <name>blah Artifactory-snapshots</name>
            <url>https://artifactory.cloud.blah.com:443/artifactory/maven-internalfacing</url>
        </snapshotRepository>
    </distributionManagement>

    <dependencies>

        <dependency>
            <groupId>org.scala-lang</groupId>
            <artifactId>scala-library</artifactId>
            <version>${scala.version}</version>
        </dependency>

        <dependency>
            <groupId>org.scalatest</groupId>
            <artifactId>scalatest_2.11</artifactId>
            <version>3.0.3</version>
            <scope>test</scope>
        </dependency>

        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-core_2.11</artifactId>
            <version>2.4.5</version>
        </dependency>

        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-sql_2.11</artifactId>
            <version>2.4.5</version>
        </dependency>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-sql-kafka-0-10_2.11</artifactId>
            <version>2.4.5</version>
        </dependency>

        <dependency>
            <groupId>com.holdenkarau</groupId>
            <artifactId>spark-testing-base_2.11</artifactId>
            <version>2.4.5_0.14.0</version>
            <scope>test</scope>
            <exclusions>
                <exclusion>
                    <groupId>org.sl4j</groupId>
                    <artifactId>sl4j-log4j12</artifactId>
                </exclusion>
            </exclusions>
        </dependency>

        <dependency>
            <groupId>com.amazonaws</groupId>
            <artifactId>aws-java-sdk</artifactId>
            <version>1.11.579</version>
        </dependency>

        <dependency>
            <groupId>com.amazonaws</groupId>
            <artifactId>aws-java-sdk-s3</artifactId>
            <version>1.11.579</version>
        </dependency>

        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-aws</artifactId>
            <version>2.8.5</version>
        </dependency>

        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-common</artifactId>
            <version>2.8.5</version>
        </dependency>

        <dependency>
            <groupId>com.typesafe</groupId>
            <artifactId>config</artifactId>
            <version>1.4.0</version>
        </dependency>

        <dependency>
            <groupId>com.fasterxml.jackson.core</groupId>
            <artifactId>jackson-annotations</artifactId>
            <version>2.6.7</version>
        </dependency>

        <dependency>
            <groupId>com.fasterxml.jackson.core</groupId>
            <artifactId>jackson-core</artifactId>
            <version>2.6.7</version>
        </dependency>

        <dependency>
            <groupId>com.fasterxml.jackson.core</groupId>
            <artifactId>jackson-databind</artifactId>
            <version>2.6.7</version>
        </dependency>

        <dependency>
            <groupId>org.scalamock</groupId>
            <artifactId>scalamock_2.11</artifactId>
            <version>4.4.0</version>
            <scope>test</scope>
        </dependency>

        <dependency>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-api</artifactId>
            <version>1.7.16</version>
        </dependency>
        <dependency>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-log4j12</artifactId>
            <version>1.7.9</version>
        </dependency>

        <dependency>
            <groupId>org.codehaus.sonar</groupId>
            <artifactId>sonar-plugin-api</artifactId>
            <version>${sonar.version}</version>
        </dependency>

        <dependency>
            <groupId>org.codehaus.sonar.plugins</groupId>
            <artifactId>sonar-surefire-plugin</artifactId>
            <version>${sonar.version}</version>
            <scope>provided</scope>
        </dependency>

        <dependency>
            <groupId>org.codehaus.sonar</groupId>
            <artifactId>sonar-testing-harness</artifactId>
            <version>${sonar.version}</version>
        </dependency>

        <dependency>
            <groupId>org.codehaus.sonar.plugins</groupId>
            <artifactId>sonar-cobertura-plugin</artifactId>
            <version>${sonar.version}</version>
            <scope>provided</scope>
        </dependency>

        <dependency>
            <groupId>com.softwaremill.macwire</groupId>
            <artifactId>macros_2.11</artifactId>
            <version>2.3.7</version>
        </dependency>

        <dependency>
            <groupId>org.scalaj</groupId>
            <artifactId>scalaj-http_2.11</artifactId>
            <version>2.4.2</version>
        </dependency>

        <dependency>
            <groupId>com.typesafe.play</groupId>
            <artifactId>play-json_2.11</artifactId>
            <version>2.7.4</version>
        </dependency>

        <!-- https://mvnrepository.com/artifact/io.findify/s3mock -->
        <dependency>
            <groupId>io.findify</groupId>
            <artifactId>s3mock_2.11</artifactId>
            <version>0.2.6</version>
            <scope>test</scope>
        </dependency>

        <dependency>
            <groupId>ch.qos.logback</groupId>
            <artifactId>logback-core</artifactId>
            <version>1.2.3</version>
        </dependency>

        <dependency>
            <groupId>ch.qos.logback</groupId>
            <artifactId>logback-classic</artifactId>
            <version>1.2.3</version>
        </dependency>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-streaming-kafka-0-10_2.11</artifactId>
            <version>2.4.5</version>
        </dependency>

        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-streaming_2.11</artifactId>
            <version>2.4.5</version>
        </dependency>

        <dependency>
            <groupId>io.grpc</groupId>
            <artifactId>grpc-netty</artifactId>
            <version>${grpc.version}</version>
        </dependency>
        <dependency>
            <groupId>io.grpc</groupId>
            <artifactId>grpc-protobuf</artifactId>
            <version>${grpc.version}</version>
        </dependency>

        <dependency>
            <groupId>com.google.guava</groupId>
            <artifactId>guava</artifactId>
            <version>${guava.version}</version>
        </dependency>

        <dependency>
            <groupId>com.google.protobuf</groupId>
            <artifactId>protobuf-java</artifactId>
            <version>${protobuf.version}</version>
        </dependency>

        <dependency>
            <groupId>com.lihaoyi</groupId>
            <artifactId>os-lib_2.11</artifactId>
            <version>0.2.9</version>
        </dependency>

        <dependency>
            <groupId>org.wvlet.airframe</groupId>
            <artifactId>airframe-log_2.11</artifactId>
            <version>20.10.3</version>
        </dependency>

        <!--

                <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-yarn_2.11</artifactId>
            <version>2.3.0</version>
        </dependency>

        <dependency>
            <groupId>io.github.embeddedkafka</groupId>
            <artifactId>embedded-kafka_2.11</artifactId>
            <version>2.4.0</version>
        </dependency>

        https://github.com/scallop/scallop
        <dependency>
            <groupId>org.rogach</groupId>
            <artifactId>scallop</artifactId>
            <version>4.0.1</version>
        </dependency>

        https://github.com/ben-manes/caffeine
        https://code.google.com/archive/p/findbugs-tutorials/
        //find bugs

        -->

    </dependencies>

    <build>
        <sourceDirectory>src/main/scala</sourceDirectory>
        <testSourceDirectory>src/test/scala</testSourceDirectory>
        <pluginManagement>
            <plugins>
                <plugin>
                    <groupId>org.sonarsource.scanner.maven</groupId>
                    <artifactId>sonar-maven-plugin</artifactId>
                    <version>3.6.0.1398</version>
                </plugin>
            </plugins>
        </pluginManagement>
        <plugins>
            <!-- Scala -->
            <plugin>
                <groupId>net.alchim31.maven</groupId>
                <artifactId>scala-maven-plugin</artifactId>
                <version>3.2.0</version>
                <executions>
                    <execution>
                        <goals>
                            <goal>compile</goal>
                            <goal>testCompile</goal>
                        </goals>
                    </execution>
                </executions>
            </plugin>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-shade-plugin</artifactId>
                <version>3.2.1</version>
                <executions>
                    <execution>
                        <phase>package</phase>
                        <goals>
                            <goal>shade</goal>
                        </goals>
                        <configuration>
                            <transformers>
                                <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
                                    <resource>reference.conf</resource>
                                </transformer>
                                <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
                                    <resource>META-INF/services/org.apache.spark.sql.sources.DataSourceRegister</resource>
                                </transformer>
                            </transformers>
                            <relocations>
                                <relocation>
                                    <pattern>com</pattern>
                                    <shadedPattern>shaded.com</shadedPattern>
                                    <includes>
                                        <include>com.google.common.**</include>
                                        <include>com.google.protobuf.**</include>
                                    </includes>
                                </relocation>
                            </relocations>
                            <artifactSet>
                                <excludes>
                                    <exclude>classworlds:classworlds</exclude>
                                    <exclude>junit:junit</exclude>
                                    <exclude>jmock:*</exclude>
                                    <exclude>*:xml-apis</exclude>
                                    <exclude>org.apache.maven:lib:tests</exclude>
                                    <exclude>org.scalatest:*</exclude>
                                    <exclude>org.scalactic:*</exclude>
                                </excludes>
                            </artifactSet>
                            <filters>
                                <filter>
                                    <artifact>*:*</artifact>
                                    <excludes>
                                        <exclude>META-INF/*.SF</exclude>
                                        <exclude>META-INF/*.DSA</exclude>
                                        <exclude>META-INF/*.RSA</exclude>
                                    </excludes>
                                </filter>
                            </filters>
                        </configuration>
                    </execution>
                </executions>
            </plugin>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-compiler-plugin</artifactId>
                <version>3.5.1</version>
                <configuration>
                    <source>${java.version}</source>
                    <target>${java.version}</target>
                </configuration>
            </plugin>
            <plugin>
                <groupId>org.codehaus.mojo</groupId>
                <artifactId>cobertura-maven-plugin</artifactId>
                <version>2.5</version>
                <configuration>
                    <formats>
                        <format>html</format>
                        <format>xml</format>
                    </formats>
                </configuration>
            </plugin>
            <plugin>
                <groupId>org.codehaus.sonar-plugins.pdf-report</groupId>
                <artifactId>maven-pdfreport-plugin</artifactId>
                <version>1.2</version>
            </plugin>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-release-plugin</artifactId>
                <dependencies>
                    <dependency>
                        <groupId>org.sonatype.plexus</groupId>
                        <artifactId>plexus-cipher</artifactId>
                        <version>1.4</version>
                    </dependency>
                </dependencies>
            </plugin>

            <plugin>
                <groupId>org.scalatest</groupId>
                <artifactId>scalatest-maven-plugin</artifactId>
                <version>2.0.0</version>
                <configuration>
                    <reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
                    <junitxml>.</junitxml>
                </configuration>
                <executions>
                    <execution>
                        <id>test</id>
                        <goals>
                            <goal>test</goal>
                        </goals>
                    </execution>
                </executions>
            </plugin>


            <!-- Scoverage -->
            <plugin>
                <groupId>org.scoverage</groupId>
                <artifactId>scoverage-maven-plugin</artifactId>
                <version>1.1.0</version>
                <configuration>
                    <scalaVersion>${scala.version}</scalaVersion>
                    <aggregate>true</aggregate>
                    <highlighting>true</highlighting>
                    <failOnMinimumCoverage>false</failOnMinimumCoverage>
                    <excludedPackages>
                        misc
                    </excludedPackages>
                </configuration>
                <executions>
                    <execution>
                        <id>scoverage-report</id>
                        <goals>
                            <!-- Needed for Sonar -->
                            <goal>report-only</goal>
                        </goals>
                        <phase>prepare-package</phase>
                    </execution>
                </executions>
            </plugin>

            <plugin>
                <groupId>org.scala-tools</groupId>
                <artifactId>maven-scala-plugin</artifactId>
                <executions>
                    <execution>
                        <id>compile</id>
                        <goals>
                            <goal>compile</goal>
                        </goals>
                        <phase>compile</phase>
                    </execution>
                    <execution>
                        <id>test-compile</id>
                        <goals>
                            <goal>testCompile</goal>
                        </goals>
                        <phase>test-compile</phase>
                    </execution>
                    <execution>
                        <phase>process-resources</phase>
                        <goals>
                            <goal>compile</goal>
                        </goals>
                    </execution>
                </executions>
            </plugin>


            <!-- For all projects, the nexus-staging-maven-plugin provides the ability
                to deploy the final packaged artifact in Nexus. -->

        </plugins>
        <finalName>
            ${project.artifactId}
            <!-- Don't change this. -->
        </finalName>
    </build>
</project>