3

I'm trying to run the Sedona Spark Visualization tutorial code. Example link: https://sedona.apache.org/tutorial/viz/

The following dataset is my baseDF:

latitude longitude
-88.331492 32.324142
-88.175933 32.360763
-88.388954 32.357073
-88.221102 32.35078

I'm following the example code in docs, my code is below:

  var sparkSession = SparkSession.builder()
    .master("local[*]") // Delete this if run in cluster mode
    .appName("Sedona Viz") // Change this to a proper name
    .config("spark.serializer", classOf[KryoSerializer].getName) // org.apache.spark.serializer.KryoSerializer
    .config("spark.kryo.registrator", classOf[SedonaVizKryoRegistrator].getName) // org.apache.sedona.viz.core.Serde.SedonaVizKryoRegistrator
    .getOrCreate()

    SedonaSQLRegistrator.registerAll(sparkSession)
    SedonaVizRegistrator.registerAll(sparkSession)
    

    baseDF.createOrReplaceTempView("baseDF")

    var myNewDF = sparkSession.sql(
      """
      SELECT ST_Point(cast(baseDF.latitude as Decimal(24,20)),cast(baseDF.longitude as Decimal(24,20))) as shape
      FROM baseDF
      """.stripMargin)
    myNewDF.createOrReplaceTempView("pointtable")

    var pixelizer = sparkSession.sql(
      """
      SELECT ST_Envelope_Aggr(shape) as bound FROM pointtable
      """.stripMargin)
    pixelizer.createOrReplaceTempView("boundtable")

  var stPixelize = sparkSession.sql(
    """
    SELECT pixel, shape FROM pointtable
    LATERAL VIEW explode(ST_Pixelize(ST_Transform(shape, 'epsg:4326','epsg:3857'), 256, 256, (SELECT ST_Transform(bound, 'epsg:4326','epsg:3857') FROM boundtable))) AS pixel
    """
  )
  stPixelize.createOrReplaceTempView("pixels")


  //Aggregate
  var agregate = sparkSession.sql(
    """
  SELECT pixel, count(*) as weight
  FROM pixels
  GROUP BY pixel
    """
  )
  agregate.createOrReplaceTempView("pixelaggregates")

  //COLORIZE
  var colorize = sparkSession.sql(
    """
    SELECT pixel, ST_Colorize(weight, (SELECT max(weight) FROM pixelaggregates)) as color
    FROM pixelaggregates
    """
    
  )
  colorize.createOrReplaceTempView("pixelaggregates")


  var render = sparkSession.sql(
    """
    SELECT ST_Render(pixel,color) AS image, (SELECT ST_AsText(bound) FROM boundtable) AS boundary
    FROM pixelaggregates
    """
  )
  render.createOrReplaceTempView("images")
  
   
  var image = sparkSession.table("images").take(1)(0)(0).asInstanceOf[ImageSerializableWrapper].getImage
  var imageGenerator = new ImageGenerator
  imageGenerator.SaveRasterImageAsLocalFile(image, System.getProperty("user.dir")+"/target/points", ImageType.PNG)

And then, I got the following error:

21/07/15 16:03:52 ERROR Executor: Exception in task 0.0 in stage 15.0 (TID 433)
java.lang.AssertionError: assertion failed
    at scala.Predef$.assert(Predef.scala:208)
    at org.apache.spark.sql.sedona_viz.expressions.ST_Render.merge(Render.scala:98)
    at org.apache.spark.sql.execution.aggregate.ScalaUDAF.merge(udaf.scala:444)
    at org.apache.spark.sql.execution.aggregate.AggregationIterator$$anonfun$1.$anonfun$applyOrElse$3(AggregationIterator.scala:199)

I looked at the code and, apparently, it's going through an assertion, and there, the resolution is less than or equal to 0, however, I created some prints and tested some values ​​and set the resolution to 256 and still, it still gives the same error. Am I doing something wrong or is it a bug?

PiFace
  • 526
  • 3
  • 19

0 Answers0