In Spark 2.4, there exists class SharedSQLContext
and related APIs have been removed in Spark 3. The equivalent of SharedSQLContext
from Spark 2.4 is the SparkSession
object in Spark 3.
I'm relatively new to Scala/Java, how do I approach converting the class below to work with SparkSession
object?
class KafkaDataConsumerSuite extends SharedSQLContext with PrivateMethodTester {
protected var testUtils: KafkaTestUtils = _
override def beforeAll(): Unit = {
super.beforeAll()
testUtils = new KafkaTestUtils(Map[String, Object]())
testUtils.setup()
}
override def afterAll(): Unit = {
if (testUtils != null) {
testUtils.teardown()
testUtils = null
}
super.afterAll()
}
test("SPARK-19886: Report error cause correctly in reportDataLoss") {
val cause = new Exception("D'oh!")
val reportDataLoss = PrivateMethod[Unit]('reportDataLoss0)
val e = intercept[IllegalStateException] {
BlackbirdInternalKafkaConsumer.invokePrivate(reportDataLoss(true, "message", cause))
}
assert(e.getCause === cause)
}
}