0

In Spark 2.4, there exists class SharedSQLContext and related APIs have been removed in Spark 3. The equivalent of SharedSQLContext from Spark 2.4 is the SparkSession object in Spark 3.

I'm relatively new to Scala/Java, how do I approach converting the class below to work with SparkSession object?

class KafkaDataConsumerSuite extends SharedSQLContext with PrivateMethodTester {

  protected var testUtils: KafkaTestUtils = _

  override def beforeAll(): Unit = {
    super.beforeAll()
    testUtils = new KafkaTestUtils(Map[String, Object]())
    testUtils.setup()
  }

  override def afterAll(): Unit = {
    if (testUtils != null) {
      testUtils.teardown()
      testUtils = null
    }
    super.afterAll()
  }

  test("SPARK-19886: Report error cause correctly in reportDataLoss") {
    val cause = new Exception("D'oh!")
    val reportDataLoss = PrivateMethod[Unit]('reportDataLoss0)
    val e = intercept[IllegalStateException] {
      BlackbirdInternalKafkaConsumer.invokePrivate(reportDataLoss(true, "message", cause))
    }
    assert(e.getCause === cause)
  }
}
sojim2
  • 1,245
  • 2
  • 15
  • 38
  • Just for the record, the previous question https://stackoverflow.com/questions/75648721/migrating-from-spark-2-4-to-spark-3-whats-spark-2-4s-sharedsqlcontext-equival – Dmytro Mitin Mar 06 '23 at 15:53

0 Answers0