Deserialize/Parse JSON to Objects AbstractMethodError in Scala with Spray in Spark Cluster

Question

I'm trying to do some Scala JSON parsing in Spark. It work locally with my tests, but fails on a remote cluster.

Relevant Imports:

import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.scala.DefaultScalaModule
import com.fasterxml.jackson.module.scala.experimental.ScalaObjectMapper
import spray.json._

The classes:

case class Item(name: String,
                      siteType: String,
                      productCategoryType: String,
                      variants: List[VariantRange]) extends Serializable
case class VariantRange(name: String, start: Double, end: Double) extends Serializable

case class ItemList(items: List[Item])

object ItemJsonProtocol extends DefaultJsonProtocol {
  implicit val variantRangeFormat = jsonFormat(VariantRange, "name", "start", "end")
  implicit val itemFormat = jsonFormat(Item, "name", "siteType", "productCategoryType", "variants")

  implicit object itemListJsonFormat extends RootJsonFormat[ItemList] {
    override def write(obj: ItemList): JsValue = ???

    override def read(json: JsValue): ItemList = ItemList(json.convertTo[List[Item]])
  }
}

The parsing code:

def parseItems(json: String) = {
  val mapper = new ObjectMapper() with ScalaObjectMapper
  mapper.registerModule(DefaultScalaModule)
  val jsonAst = json.parseJson

  import ItemJsonProtocol._
  jsonAst.convertTo[ItemList].items
}

def parseItems(): List[Item] = {
  parseItems(Source.fromInputStream(getClass.getResourceAsStream("/data/items.json")).mkString)
}

This test passes locally:

  test("Parses resources file") {
    val items = parseItems()
    items shouldNot be (null)
  }

But in Spark it shows this:

15/12/07 17:59:05 ERROR ApplicationMaster: User class threw exception: java.lang.AbstractMethodError: com.intentmedia.data.AggregationJob$$anon$1.com$fasterxml$jackson$module$scala$experimental$ScalaObjectMapper$_setter_$com$fasterxml$jackson$module$scala$experimental$ScalaObjectMapper$$typeCache_$eq(Lorg/spark-project/guava/cache/LoadingCache;)V
java.lang.AbstractMethodError: com.intentmedia.data.AggregationJob$$anon$1.com$fasterxml$jackson$module$scala$experimental$ScalaObjectMapper$_setter_$com$fasterxml$jackson$module$scala$experimental$ScalaObjectMapper$$typeCache_$eq(Lorg/spark-project/guava/cache/LoadingCache;)V
    at com.fasterxml.jackson.module.scala.experimental.ScalaObjectMapper$class.$init$(ScalaObjectMapper.scala:50)
    at com.intentmedia.data.AggregationJob$$anon$1.<init>(AggregationJob.scala:76)
    at com.intentmedia.data.AggregationJob$.parseExperiments(AggregationJob.scala:76)
    at com.intentmedia.data.AggregationJob$.parseExperiments(AggregationJob.scala:85)
    at com.intentmedia.data.AggregationJob$.runWorkflow(AggregationJob.scala:66)
    at com.intentmedia.spark.SparkJob$class.main(SparkJob.scala:43)
    at com.intentmedia.data.AggregationJob$.main(AggregationJob.scala:49)
    at com.intentmedia.data.AggregationJob.main(AggregationJob.scala)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:606)
    at org.apache.spark.deploy.yarn.ApplicationMaster$$anon$2.run(ApplicationMaster.scala:525)
15/12/07 17:59:05 INFO ApplicationMaster: Final app status: FAILED, exitCode: 15, (reason: User class threw exception: java.lang.AbstractMethodError: com.intentmedia.data.AggregationJob$$anon$1.com$fasterxml$jackson$module$scala$experimental$ScalaObjectMapper$_setter_$com$fasterxml$jackson$module$scala$experimental$ScalaObjectMapper$$typeCache_$eq(Lorg/spark-project/guava/cache/LoadingCache;)V)

Do I need to use some kind of caching when I read the objects in? Am I not handling resource files correctly?