I am trying to create a simple web application using Spring Boot and Apache Spark by Intellij.
SparkConfiguration.java
package com.example.sparkspringboot4;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaSparkContext;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.stereotype.Component;
@Component
@Configuration
@ConfigurationProperties(prefix = "spark")
@EnableConfigurationProperties
public class SparkConfiguration {
private String sparkHome = ".";
private String appName = "sparkPatrol";
private String master = "local";
@Bean
@ConditionalOnMissingBean(SparkConf.class)
public SparkConf sparkConf() {
SparkConf conf = new SparkConf().setAppName(appName).setMaster(master);
return conf;
}
@Bean
@ConditionalOnMissingBean(JavaSparkContext.class)
public JavaSparkContext javaSparkContext() {
return new JavaSparkContext(sparkConf());
}
public String getSparkHome() {
return sparkHome;
}
public void setSparkHome(String sparkHome) {
this.sparkHome = sparkHome;
}
public String getAppName() {
return appName;
}
public void setAppName(String appName) {
this.appName = appName;
}
public String getMaster() {
return master;
}
public void setMaster(String master) {
this.master = master;
}
}
And the pom.xml
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>3.1.0</version>
<relativePath/> <!-- lookup parent from repository -->
</parent>
<groupId>com.example</groupId>
<artifactId>SparkSpringBoot4</artifactId>
<version>0.0.1-SNAPSHOT</version>
<name>SparkSpringBoot4</name>
<description>SparkSpringBoot4</description>
<properties>
<java.version>17</java.version>
<scala.version>2.12</scala.version>
<spark.version>2.4.4</spark.version>
</properties>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_${scala.version}</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming_${scala.version}</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_${scala.version}</artifactId>
<version>${spark.version}</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>1.5.2</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>
But it got failure during run:
Description:
An attempt was made to call a method that does not exist. The attempt was made from the following location:
org.apache.spark.internal.Logging$.org$apache$spark$internal$Logging$$isLog4j12(Logging.scala:205)
The following method did not exist:
'org.slf4j.impl.StaticLoggerBinder org.slf4j.impl.StaticLoggerBinder.getSingleton()'
The calling method's class, org.apache.spark.internal.Logging$, was loaded from the following location:
jar:file:/C:/Users/XXX/.m2/repository/org/apache/spark/spark-core_2.12/2.4.4/spark-core_2.12-2.4.4.jar!/org/apache/spark/internal/Logging$.class
The called method's class, org.slf4j.impl.StaticLoggerBinder, is available from the following locations:
jar:file:/C:/Users/XXX/.m2/repository/org/slf4j/slf4j-log4j12/1.5.2/slf4j-log4j12-1.5.2.jar!/org/slf4j/impl/StaticLoggerBinder.class
The called method's class hierarchy was loaded from the following locations:
org.slf4j.impl.StaticLoggerBinder: file:/C:/Users/XXX/.m2/repository/org/slf4j/slf4j-log4j12/1.5.2/slf4j-log4j12-1.5.2.jar
Action:
Correct the classpath of your application so that it contains compatible versions of the classes org.apache.spark.internal.Logging$ and org.slf4j.impl.StaticLoggerBinder
Process finished with exit code 1
The reason I added org.slf4j dependency is based on NoClassDefFoundError: org/slf4j/impl/StaticLoggerBinder
Is there any clue for this issue? Thanks!
Edit: The reason I added org.slf4j dependency is because of the failure I faced initially. Below is the error message without this org.slf4j dependency.
org.springframework.beans.factory.BeanCreationException: Error creating bean with name 'javaSparkContext' defined in class path resource [com/example/sparkspringboot4/SparkConfiguration.class]: Failed to instantiate [org.apache.spark.api.java.JavaSparkContext]: Factory method 'javaSparkContext' threw exception with message: org/slf4j/impl/StaticLoggerBinder
at org.springframework.beans.factory.support.ConstructorResolver.instantiate(ConstructorResolver.java:659) ~[spring-beans-6.0.9.jar:6.0.9]
at org.springframework.beans.factory.support.ConstructorResolver.instantiateUsingFactoryMethod(ConstructorResolver.java:493) ~[spring-beans-6.0.9.jar:6.0.9]
at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.instantiateUsingFactoryMethod(AbstractAutowireCapableBeanFactory.java:1332) ~[spring-beans-6.0.9.jar:6.0.9]
at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBeanInstance(AbstractAutowireCapableBeanFactory.java:1162) ~[spring-beans-6.0.9.jar:6.0.9]
at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.doCreateBean(AbstractAutowireCapableBeanFactory.java:560) ~[spring-beans-6.0.9.jar:6.0.9]
at org.springframework.beans.factory.support.AbstractAutowireCapableBeanFactory.createBean(AbstractAutowireCapableBeanFactory.java:520) ~[spring-beans-6.0.9.jar:6.0.9]
at org.springframework.beans.factory.support.AbstractBeanFactory.lambda$doGetBean$0(AbstractBeanFactory.java:326) ~[spring-beans-6.0.9.jar:6.0.9]
at org.springframework.beans.factory.support.DefaultSingletonBeanRegistry.getSingleton(DefaultSingletonBeanRegistry.java:234) ~[spring-beans-6.0.9.jar:6.0.9]
at org.springframework.beans.factory.support.AbstractBeanFactory.doGetBean(AbstractBeanFactory.java:324) ~[spring-beans-6.0.9.jar:6.0.9]
at org.springframework.beans.factory.support.AbstractBeanFactory.getBean(AbstractBeanFactory.java:200) ~[spring-beans-6.0.9.jar:6.0.9]
at org.springframework.beans.factory.support.DefaultListableBeanFactory.preInstantiateSingletons(DefaultListableBeanFactory.java:973) ~[spring-beans-6.0.9.jar:6.0.9]
at org.springframework.context.support.AbstractApplicationContext.finishBeanFactoryInitialization(AbstractApplicationContext.java:941) ~[spring-context-6.0.9.jar:6.0.9]
at org.springframework.context.support.AbstractApplicationContext.refresh(AbstractApplicationContext.java:608) ~[spring-context-6.0.9.jar:6.0.9]
at org.springframework.boot.SpringApplication.refresh(SpringApplication.java:733) ~[spring-boot-3.1.0.jar:3.1.0]
at org.springframework.boot.SpringApplication.refreshContext(SpringApplication.java:435) ~[spring-boot-3.1.0.jar:3.1.0]
at org.springframework.boot.SpringApplication.run(SpringApplication.java:311) ~[spring-boot-3.1.0.jar:3.1.0]
at org.springframework.boot.SpringApplication.run(SpringApplication.java:1305) ~[spring-boot-3.1.0.jar:3.1.0]
at org.springframework.boot.SpringApplication.run(SpringApplication.java:1294) ~[spring-boot-3.1.0.jar:3.1.0]
at com.example.sparkspringboot4.SparkSpringBoot4Application.main(SparkSpringBoot4Application.java:10) ~[classes/:na]
Caused by: org.springframework.beans.BeanInstantiationException: Failed to instantiate [org.apache.spark.api.java.JavaSparkContext]: Factory method 'javaSparkContext' threw exception with message: org/slf4j/impl/StaticLoggerBinder
at org.springframework.beans.factory.support.SimpleInstantiationStrategy.instantiate(SimpleInstantiationStrategy.java:171) ~[spring-beans-6.0.9.jar:6.0.9]
at org.springframework.beans.factory.support.ConstructorResolver.instantiate(ConstructorResolver.java:655) ~[spring-beans-6.0.9.jar:6.0.9]
... 18 common frames omitted
Caused by: java.lang.NoClassDefFoundError: org/slf4j/impl/StaticLoggerBinder
at org.apache.spark.internal.Logging$.org$apache$spark$internal$Logging$$isLog4j12(Logging.scala:205) ~[spark-core_2.12-2.4.4.jar:2.4.4]
at org.apache.spark.internal.Logging.initializeLogging(Logging.scala:119) ~[spark-core_2.12-2.4.4.jar:2.4.4]
at org.apache.spark.internal.Logging.initializeLogIfNecessary(Logging.scala:108) ~[spark-core_2.12-2.4.4.jar:2.4.4]
at org.apache.spark.internal.Logging.initializeLogIfNecessary$(Logging.scala:102) ~[spark-core_2.12-2.4.4.jar:2.4.4]
at org.apache.spark.SparkContext.initializeLogIfNecessary(SparkContext.scala:73) ~[spark-core_2.12-2.4.4.jar:2.4.4]
at org.apache.spark.internal.Logging.initializeLogIfNecessary(Logging.scala:99) ~[spark-core_2.12-2.4.4.jar:2.4.4]
at org.apache.spark.internal.Logging.initializeLogIfNecessary$(Logging.scala:98) ~[spark-core_2.12-2.4.4.jar:2.4.4]
at org.apache.spark.SparkContext.initializeLogIfNecessary(SparkContext.scala:73) ~[spark-core_2.12-2.4.4.jar:2.4.4]
at org.apache.spark.internal.Logging.log(Logging.scala:46) ~[spark-core_2.12-2.4.4.jar:2.4.4]
at org.apache.spark.internal.Logging.log$(Logging.scala:44) ~[spark-core_2.12-2.4.4.jar:2.4.4]
at org.apache.spark.SparkContext.log(SparkContext.scala:73) ~[spark-core_2.12-2.4.4.jar:2.4.4]
at org.apache.spark.internal.Logging.logInfo(Logging.scala:54) ~[spark-core_2.12-2.4.4.jar:2.4.4]
at org.apache.spark.internal.Logging.logInfo$(Logging.scala:53) ~[spark-core_2.12-2.4.4.jar:2.4.4]
at org.apache.spark.SparkContext.logInfo(SparkContext.scala:73) ~[spark-core_2.12-2.4.4.jar:2.4.4]
at org.apache.spark.SparkContext.<init>(SparkContext.scala:183) ~[spark-core_2.12-2.4.4.jar:2.4.4]
at org.apache.spark.api.java.JavaSparkContext.<init>(JavaSparkContext.scala:58) ~[spark-core_2.12-2.4.4.jar:2.4.4]
at com.example.sparkspringboot4.SparkConfiguration.javaSparkContext(SparkConfiguration.java:34) ~[classes/:na]
at com.example.sparkspringboot4.SparkConfiguration$$SpringCGLIB$$0.CGLIB$javaSparkContext$7(<generated>) ~[classes/:na]
at com.example.sparkspringboot4.SparkConfiguration$$SpringCGLIB$$2.invoke(<generated>) ~[classes/:na]
at org.springframework.cglib.proxy.MethodProxy.invokeSuper(MethodProxy.java:258) ~[spring-core-6.0.9.jar:6.0.9]
at org.springframework.context.annotation.ConfigurationClassEnhancer$BeanMethodInterceptor.intercept(ConfigurationClassEnhancer.java:331) ~[spring-context-6.0.9.jar:6.0.9]
at com.example.sparkspringboot4.SparkConfiguration$$SpringCGLIB$$0.javaSparkContext(<generated>) ~[classes/:na]
at java.base/jdk.internal.reflect.DirectMethodHandleAccessor.invoke(DirectMethodHandleAccessor.java:104) ~[na:na]
at java.base/java.lang.reflect.Method.invoke(Method.java:578) ~[na:na]
at org.springframework.beans.factory.support.SimpleInstantiationStrategy.instantiate(SimpleInstantiationStrategy.java:139) ~[spring-beans-6.0.9.jar:6.0.9]
... 19 common frames omitted
Caused by: java.lang.ClassNotFoundException: org.slf4j.impl.StaticLoggerBinder
at java.base/jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:641) ~[na:na]
at java.base/jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188) ~[na:na]
at java.base/java.lang.ClassLoader.loadClass(ClassLoader.java:521) ~[na:na]
... 44 common frames omitted
