The import org. Apache. Spark. Streaming. _
The import org. Apache. Spark. Streaming. Kafka. _
The import org. Apache. Spark. SparkConf
Import the Java. Util. Properties
The import org. Apache. Spark. Streaming. Kafka. KafkaUtils
The import org. Apache. Kafka
The import org. Apache. Spark. Streaming. StreamingContext. _
The import org. Apache. Spark. Streaming. {Seconds, StreamingContext}
The import org. Apache. Spark. SparkContext
The import org. Apache. Spark. SparkContext. _
The import org. Apache. Spark. Streaming. StreamingContext
The import org. Apache. Spark. Streaming. Dstream. InputDStream
The import org. Apache. Spark. Streaming. Util. _
//import org. Apache. Spark. Examples. Streaming. _
The import org. Apache. Kafka. Serializer. StringDecoder
The import org. Apache. Spark. Streaming. Util. _
Import kafka. Utils. VerifiableProperties//red to add after, and is wrong, cannot find kafka utils package such as
/* *
* @ author ${MuJinXing}
*/
The object App {
Def main (args: Array [String]) {
//if (args. Length<4) {
//System. Err. Println (" the Usage: KafkaWordCount & lt; ZkQuorum>
//System. Exit (1)
//}
System. SetProperty (" hadoop. Home. Dir ", "D: \ \ appbase \ \ hadoop - 2.7.4 \ \ hadoop - 2.7.4")
Val topic=Set (" linuxSysteminfos ")
Val brokers="10.218.7.232:9092
"
Val sparkConf=new sparkConf (). SetMaster (" local [2] "). The setAppName (" App ")
Val sc=new SparkContext (sparkConf)
Val SSC=new StreamingContext (sc, Seconds (10))
//SSC. SparkContext. SetLogLevel (" ERROR ")
Val kafkaParams=Map [String, String] (metadata. Broker. "the list" - & gt; Brokers, "serializer. Class - & gt;" " Kafka. Serializer. StringEnvoder ")
Val kafkaStream=KafkaUtils. CreateDirectStream (SSC, kafkaParams, topic)
KafkaStream. Print ()
SSC. Start ()
SSC. AwaitTermination ()
}
}
After the operation of the mistakes is
17/09/20 22:35:10 ERROR Executor: Exception in task 0.0 in stage 0.0 (dar)
Java. Lang. NoSuchMethodException: scala. Runtime. Nothing $. & lt; init> (kafka. Utils. VerifiableProperties)
The at Java. Lang. Class. GetConstructor0 (Class. Java: 3074)
The at Java. Lang. Class. GetConstructor (Class. Java: 1817)
The at org. Apache. Spark. Streaming. Kafka. KafkaRDD $KafkaRDDIterator. & lt; init> (KafkaRDD. Scala: 150)
At org.apache.spark.streaming.kafka.KafkaRDD.com pute (KafkaRDD. Scala: 136)
At org.apache.spark.rdd.RDD.com puteOrReadCheckpoint (RDD. Scala: 319)
The at org. Apache. Spark. RDD. RDD. Iterator (RDD. Scala: 283)
The at org. Apache. Spark. The scheduler. ResultTask. RunTask (ResultTask. Scala: 70)
At org. Apache. Spark. The scheduler. Task. Run (86) Task. Scala:
The at org. Apache. Spark. Executor. $TaskRunner executor. Run (executor. Scala: 274)
The at Java. Util. Concurrent. ThreadPoolExecutor. RunWorker (ThreadPoolExecutor. Java: 1142)
The at Java. Util. Concurrent. ThreadPoolExecutor $Worker. The run (ThreadPoolExecutor. Java: 617)
The at Java. Lang. Thread. The run (Thread. Java: 745)
17/09/20 22:35:11 WARN TaskSetManager: Lost task in stage 0.0 0.0 (dar 0, localhost) : Java. Lang. NoSuchMethodException: scala. Runtime. Nothing $. & lt; init> (kafka. Utils. VerifiableProperties)
The at Java. Lang. Class. GetConstructor0 (Class. Java: 3074)
The at Java. Lang. Class. GetConstructor (Class. Java: 1817)
The at org. Apache. Spark. Streaming. Kafka. KafkaRDD $KafkaRDDIterator. & lt; init> (KafkaRDD. Scala: 150)
At org.apache.spark.streaming.kafka.KafkaRDD.com pute (KafkaRDD. Scala: 136)
At org.apache.spark.rdd.RDD.com puteOrReadCheckpoint (RDD. Scala: 319)
The at org. Apache. Spark. RDD. RDD. Iterator (RDD. Scala: 283)
The at org. Apache. Spark. The scheduler. ResultTask. RunTask (ResultTask. Scala: 70)
At org. Apache. Spark. The scheduler. Task. Run (86) Task. Scala:
The at org. Apache. Spark. Executor. $TaskRunner executor. Run (executor. Scala: 274)
The at Java. Util. Concurrent. ThreadPoolExecutor. RunWorker (ThreadPoolExecutor. Java: 1142)
The at Java. Util. Concurrent. ThreadPoolExecutor $Worker. The run (ThreadPoolExecutor. Java: 617)
The at Java. Lang. Thread. The run (Thread. Java: 745)
After adding kafka. Utils. VerifiableProperties class found
The object utils is not a member of package org. Apache. Kafka
But sure kafka package has been included in the maven and there are such, forget your great god can help solve
CodePudding user response:
From the top, expectations which the great spirit from the watchCodePudding user response:
Val kafkaStream=KafkaUtils. CreateDirectStream (SSC, kafkaParams, topic)Modified to try below, specify the type of the function on the parameter
Val kafkaStream=KafkaUtils. CreateDirectStream [String, String, StringDecoder, StringDecoder] (SSC, kafkaParams, topic)