[我知道类似的问题已得到简短回答，但由于缺乏最低声誉，我无法在此添加我个人的其他疑问...因此，我在这里问]]
我想使用Apache Spark + Kafka处理Twitter数据。我为此创建了一个模式。但是，当我运行它时，出现以下错误。我在很多地方搜索了此错误，但是我找不到想要的解决方案，或者它没有用。上次我使用较小的内存空间运行Spark时，以为内存不足，但仍然遇到相同的错误。这是我收到此错误的代码：
from kafka import KafkaConsumer from pyspark.streaming import StreamingContext import json import pandas as pd from pyspark import SparkConf,SparkContext from pyspark.streaming.kafka import KafkaUtils #cd /opt/hadoop-3.2.0-7/hadoop/spark $sudo ./bin/spark-submit --packages org.apache.spark:spark-streaming-kafka-0-8_2.11:2.3.0 /opt/twitterConsumer.py conf = SparkConf() conf.setAppName("BDA-Twitter-Spark-Kafka") sc = SparkContext(conf=conf) sc.setLogLevel("ERROR") ssc = StreamingContext(sc,1) KafkaStream = KafkaUtils.createStream(ssc, "localhost:2181",'tks',{"xmas":1}) # directKafkaStream = KafkaUtils.createDirectStream(ssc, [topic], {"metadata.broker.list": brokers}) KafkaStream.pprint() print("HERE1") ssc.start() ssc.awaitTermination()我的错误是：
ERROR Executor: Exception in task 0.0 in stage 0.0 (TID 0) java.lang.AbstractMethodError at org.apache.spark.internal.Logging$class.initializeLogIfNecessary(Logging.scala:99) at org.apache.spark.streaming.kafka.KafkaReceiver.initializeLogIfNecessary(KafkaInputDStream.scala:68) at org.apache.spark.internal.Logging$class.log(Logging.scala:46) at org.apache.spark.streaming.kafka.KafkaReceiver.log(KafkaInputDStream.scala:68) at org.apache.spark.internal.Logging$class.logInfo(Logging.scala:54) at org.apache.spark.streaming.kafka.KafkaReceiver.logInfo(KafkaInputDStream.scala:68) at org.apache.spark.streaming.kafka.KafkaReceiver.onStart(KafkaInputDStream.scala:90) at org.apache.spark.streaming.receiver.ReceiverSupervisor.startReceiver(ReceiverSupervisor.scala:149) at org.apache.spark.streaming.receiver.ReceiverSupervisor.start(ReceiverSupervisor.scala:131) at org.apache.spark.streaming.scheduler.ReceiverTracker$ReceiverTrackerEndpoint$$anonfun$9.apply(ReceiverTracker.scala:601) at org.apache.spark.streaming.scheduler.ReceiverTracker$ReceiverTrackerEndpoint$$anonfun$9.apply(ReceiverTracker.scala:591) at org.apache.spark.SparkContext$$anonfun$37.apply(SparkContext.scala:2212) at org.apache.spark.SparkContext$$anonfun$37.apply(SparkContext.scala:2212) at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90) at org.apache.spark.scheduler.Task.run(Task.scala:121) at org.apache.spark.executor.Executor$TaskRunner$$anonfun$10.apply(Executor.scala:408) at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1360) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:414) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) 19/12/29 09:57:49 ERROR TaskSetManager: Task 0 in stage 0.0 failed 1 times; aborting job 19/12/29 09:57:49 ERROR ReceiverTracker: Receiver has been stopped. Try to restart it. org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 0.0 failed 1 times, most recent failure: Lost task 0.0 in stage 0.0 (TID 0, localhost, executor driver): java.lang.AbstractMethodError at org.apache.spark.internal.Logging$class.initializeLogIfNecessary(Logging.scala:99) at org.apache.spark.streaming.kafka.KafkaReceiver.initializeLogIfNecessary(KafkaInputDStream.scala:68) at org.apache.spark.internal.Logging$class.log(Logging.scala:46) at org.apache.spark.streaming.kafka.KafkaReceiver.log(KafkaInputDStream.scala:68) at org.apache.spark.internal.Logging$class.logInfo(Logging.scala:54) at org.apache.spark.streaming.kafka.KafkaReceiver.logInfo(KafkaInputDStream.scala:68) at org.apache.spark.streaming.kafka.KafkaReceiver.onStart(KafkaInputDStream.scala:90) at org.apache.spark.streaming.receiver.ReceiverSupervisor.startReceiver(ReceiverSupervisor.scala:149) at org.apache.spark.streaming.receiver.ReceiverSupervisor.start(ReceiverSupervisor.scala:131) at org.apache.spark.streaming.scheduler.ReceiverTracker$ReceiverTrackerEndpoint$$anonfun$9.apply(ReceiverTracker.scala:601) at org.apache.spark.streaming.scheduler.ReceiverTracker$ReceiverTrackerEndpoint$$anonfun$9.apply(ReceiverTracker.scala:591) at org.apache.spark.SparkContext$$anonfun$37.apply(SparkContext.scala:2212) at org.apache.spark.SparkContext$$anonfun$37.apply(SparkContext.scala:2212) at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90) at org.apache.spark.scheduler.Task.run(Task.scala:121) at org.apache.spark.executor.Executor$TaskRunner$$anonfun$10.apply(Executor.scala:408) at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1360) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:414) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748) Driver stacktrace: at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1889) at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1877) at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1876) at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1876) at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:926) at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:926) at scala.Option.foreach(Option.scala:257) at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:926) at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:2110) at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2059) at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:2048) at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:49) Caused by: java.lang.AbstractMethodError at org.apache.spark.internal.Logging$class.initializeLogIfNecessary(Logging.scala:99) at org.apache.spark.streaming.kafka.KafkaReceiver.initializeLogIfNecessary(KafkaInputDStream.scala:68) at org.apache.spark.internal.Logging$class.log(Logging.scala:46) at org.apache.spark.streaming.kafka.KafkaReceiver.log(KafkaInputDStream.scala:68) at org.apache.spark.internal.Logging$class.logInfo(Logging.scala:54) at org.apache.spark.streaming.kafka.KafkaReceiver.logInfo(KafkaInputDStream.scala:68) at org.apache.spark.streaming.kafka.KafkaReceiver.onStart(KafkaInputDStream.scala:90) at org.apache.spark.streaming.receiver.ReceiverSupervisor.startReceiver(ReceiverSupervisor.scala:149) at org.apache.spark.streaming.receiver.ReceiverSupervisor.start(ReceiverSupervisor.scala:131) at org.apache.spark.streaming.scheduler.ReceiverTracker$ReceiverTrackerEndpoint$$anonfun$9.apply(ReceiverTracker.scala:601) at org.apache.spark.streaming.scheduler.ReceiverTracker$ReceiverTrackerEndpoint$$anonfun$9.apply(ReceiverTracker.scala:591) at org.apache.spark.SparkContext$$anonfun$37.apply(SparkContext.scala:2212) at org.apache.spark.SparkContext$$anonfun$37.apply(SparkContext.scala:2212) at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90) at org.apache.spark.scheduler.Task.run(Task.scala:121) at org.apache.spark.executor.Executor$TaskRunner$$anonfun$10.apply(Executor.scala:408) at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1360) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:414) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) at java.lang.Thread.run(Thread.java:748)这里如何匹配所有所需工具的版本？
我知道类似的问题已经简短回答，但是由于缺乏最低声誉，我无法在此添加我的个人其他疑问...因此，我在这里询问我要处理Twitter数据...