java - 无法连接 Spark-Cloudant

Question

我尝试使用 Java 代码从 Cloudant 获取数据并收到错误，

我尝试使用以下 Spark 和 cloudant-spark 版本，

Spark 2.0.0,

Spark 2.0.1,

Spark 2.0.2

所有版本都出现与下面发布的错误相同的错误。

如果我添加 scala 依赖项来解决此错误，则该错误与 Spark 库冲突。

下面是我的java代码，

package spark.cloudant.connecter;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.SQLContext;
import com.cloudant.spark.*;

public class cloudantconnecter {
    public static void main(String[] args) throws Exception {

        try {
            SparkConf sparkConf = new SparkConf().setAppName("spark cloudant connecter").setMaster("local[*]");
            sparkConf.set("spark.streaming.concurrentJobs", "30");

            JavaSparkContext sc = new JavaSparkContext(sparkConf);

            SQLContext sqlContext = new SQLContext(sc);
            System.out.print("initialization successfully");


            Dataset<org.apache.spark.sql.Row> st = sqlContext.read().format("com.cloudant.spark")
                    .option("cloudant.host", "HOSTNAME").option("cloudant.username", "USERNAME")
                    .option("cloudant.password", "PASSWORD").load("DATABASENAME");

            st.printSchema();


        } catch (

        Exception e) {
            e.printStackTrace();
        }
    }
}

Maven 依赖项

<dependencies>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-core_2.10</artifactId>
            <version>2.0.0</version>
        </dependency>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-mllib_2.10</artifactId>
            <version>2.0.0</version>
        </dependency>
        <dependency>
            <groupId>cloudant-labs</groupId>
            <artifactId>spark-cloudant</artifactId>
            <version>2.0.0-s_2.11</version>
        </dependency>
    </dependencies>

获取错误详细信息，

Exception in thread "main" java.lang.NoSuchMethodError: scala/Predef$.ArrowAssoc(Ljava/lang/Object;)Ljava/lang/Object; (loaded from file:/C:/Users/Administrator/.m2/repository/org/scala-lang/scala-library/2.10.6/scala-library-2.10.6.jar by sun.misc.Launcher$AppClassLoader@9f916f97) called from class scalaj.http.HttpConstants$ (loaded from file:/C:/Users/Administrator/.m2/repository/org/scalaj/scalaj-http_2.11/2.3.0/scalaj-http_2.11-2.3.0.jar by sun.misc.Launcher$AppClassLoader@9f916f97).
    at scalaj.http.HttpConstants$.liftedTree1$1(Http.scala:637)
    at scalaj.http.HttpConstants$.<init>(Http.scala:636)
    at scalaj.http.HttpConstants$.<clinit>(Http.scala)
    at scalaj.http.BaseHttp$.$lessinit$greater$default$2(Http.scala:754)
    at scalaj.http.Http$.<init>(Http.scala:738)
    at scalaj.http.Http$.<clinit>(Http.scala)
    at com.cloudant.spark.common.JsonStoreDataAccess.getQueryResult(JsonStoreDataAccess.scala:152)
    at com.cloudant.spark.common.JsonStoreDataAccess.getTotalRows(JsonStoreDataAccess.scala:99)
    at com.cloudant.spark.common.JsonStoreRDD.totalRows$lzycompute(JsonStoreRDD.scala:56)
    at com.cloudant.spark.common.JsonStoreRDD.totalRows(JsonStoreRDD.scala:55)
    at com.cloudant.spark.common.JsonStoreRDD.totalPartition$lzycompute(JsonStoreRDD.scala:59)
    at com.cloudant.spark.common.JsonStoreRDD.totalPartition(JsonStoreRDD.scala:58)
    at com.cloudant.spark.common.JsonStoreRDD.getPartitions(JsonStoreRDD.scala:81)
    at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:248)
    at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:246)
    at scala.Option.getOrElse(Option.scala:120)
    at org.apache.spark.rdd.RDD.partitions(RDD.scala:246)
    at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35)
    at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:248)
    at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:246)
    at scala.Option.getOrElse(Option.scala:120)
    at org.apache.spark.rdd.RDD.partitions(RDD.scala:246)
    at org.apache.spark.SparkContext.runJob(SparkContext.scala:1934)
    at org.apache.spark.rdd.RDD$$anonfun$fold$1.apply(RDD.scala:1046)
    at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
    at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
    at org.apache.spark.rdd.RDD.withScope(RDD.scala:358)
    at org.apache.spark.rdd.RDD.fold(RDD.scala:1040)
    at org.apache.spark.sql.execution.datasources.json.InferSchema$.infer(InferSchema.scala:68)
    at org.apache.spark.sql.DataFrameReader$$anonfun$3.apply(DataFrameReader.scala:317)
    at org.apache.spark.sql.DataFrameReader$$anonfun$3.apply(DataFrameReader.scala:317)
    at scala.Option.getOrElse(Option.scala:120)
    at org.apache.spark.sql.DataFrameReader.json(DataFrameReader.scala:316)
    at com.cloudant.spark.DefaultSource.create(DefaultSource.scala:127)
    at com.cloudant.spark.DefaultSource.createRelation(DefaultSource.scala:105)
    at com.cloudant.spark.DefaultSource.createRelation(DefaultSource.scala:100)
    at org.apache.spark.sql.execution.datasources.DataSource.resolveRelation(DataSource.scala:315)
    at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:149)
    at org.apache.spark.sql.DataFrameReader.load(DataFrameReader.scala:132)
    at spark.cloudant.connecter.cloudantconnecter.main(cloudantconnecter.java:24)

Answer 1

显示错误是因为使用 scala 2.10 提到了相关库，并使用 2.11 提到了包 Spark cloudant 库

因此请将库spark-core_2.10更改为spark-core_2.11

所以现在依赖项是，

<dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-core_2.11</artifactId>
            <version>2.0.1</version>
        </dependency>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-mllib_2.11</artifactId>
            <version>2.0.1</version>
        </dependency>   
        <dependency>
            <groupId>cloudant-labs</groupId>
            <artifactId>spark-cloudant</artifactId>
            <version>2.0.0-s_2.11</version>
        </dependency>