Accessing Table from remote mysql database, which works fine locally from eclipse but throwing error while trying from spark-submit.
ERROR SparkContext: Error initializing SparkContext.java.lang.IllegalArgumentException: java.net.UnknownHostException: namenode
at org.apache.hadoop.security.SecurityUtil.buildTokenService(SecurityUtil.java:378)
Spark-submit below:
spark-submit \
--jars /home/hduser/.ivy2/cache/com.typesafe/config/bundles/config-1.3.1.jar \
--class "retail.stack" --master local[2] \
--driver-memory 2g --executor-memory 2g --executor-cores 2 \
sample-spark-180417_2.11-1.0.jar \
/home/hduser/spark-training/workspace/demos/output/extra_in_src \
/home/hduser/spark-training/workspace/demos/output/extradest \
/home/hduser/spark-training/workspace/demos/output/bestMatch
Code Below:
val spark = SparkSession.builder().master("local").appName("Database")
.getOrCreate()
val jdbcUsername = "root"
val jdbcPassword = "root"
val url = "jdbc:mysql://192.168.175.35:3306/sample?useSSL=false"
val connectionProperties = new java.util.Properties()
connectionProperties.put("user", jdbcUsername)
connectionProperties.put("password", jdbcPassword)
val querySource = "(select * from source1) as src"
val queryDestination = "(select * from destination1) as dest"
val source = spark.read.jdbc(url, querySource, connectionProperties).rdd.map(_.mkString(","))
val destination = spark.read.jdbc(url, queryDestination, connectionProperties).rdd.map(_.mkString(","))
Thank you
