I am trying to stream the data from Twitter API but unable to execute the code.
Following are the code snippet and error. Please let me know what is the issue if anyone can help here.
Code:
from pyspark.sql import SparkSession
from pyspark.sql.functions import *
from pyspark.sql.types import *
from pyspark.sql import functions as F
if __name__ == "__main__":
# create Spark session
spark = SparkSession.builder.appName("TwitterSentimentAnalysis").getOrCreate()
# read the tweet data from socket
tweet_df = spark \
.readStream \
.format("socket") \
.option("host", "127.0.0.1") \
.option("port", 3333) \
.load()
# type cast the column value
tweet_df_string = tweet_df.selectExpr("CAST(value AS STRING)")
# split words based on space, filter out hashtag values and group them up
tweets_tab = tweet_df_string.withColumn('word', explode(split(F.col('value'), ' '))) \
.groupBy('word') \
.count() \
.sort('count', ascending=False). \
filter(F.col('word').contains('#'))
# write the above data into memory. consider the entire analysis in all iteration (output mode = complete). and let the trigger runs in every 2 secs.
writeTweet = tweets_tab.writeStream\
.outputMode('complete')\
.format('console')\
.start()
print("----- streaming is running -------")
Error:
Error : ERROR StreamMetadata: Error writing stream metadata StreamMetadata(ea206b0f-718b-49f5-bd97-9e7488f643cd) to file:/C:/Users/aakash%2520uppadhaya/AppData/Local/Temp/temporary-41057c59-fcac-4d2e-9a71-a6e53c57c2ec/metadata java.io.FileNotFoundException: File file:/C:/Users/aakash%2520uppadhaya/AppData/Local/Temp/temporary-41057c59-fcac-4d2e-9a71-a6e53c57c2ec does not exist