Merge branch 'release/0.7.0'

Signed-off-by: Simeon H.K. Fitch <[email protected]>

Signed-off-by: Simeon H.K. Fitch <[email protected]>

Author: metasim

.travis.yml

@@ -13,13 +13,18 @@ scala:
 jdk:
   - oraclejdk8
 
+addons:
+  apt_packages:
+    - pandoc
+
 sbt_args: -no-colors
 
 script:
-  - sbt -Dfile.encoding=UTF8 clean coverage test coverageReport
+  - sbt -Dfile.encoding=UTF8 clean core/test datasource/test
+  #  - sbt -Dfile.encoding=UTF8 clean coverage test coverageReport
   # Tricks to avoid unnecessary cache updates
   - find $HOME/.sbt -name "*.lock" | xargs rm
   - find $HOME/.ivy2 -name "ivydata-*.properties" | xargs rm
 
-after_success:
-  - bash <(curl -s https://codecov.io/bash)
+#after_success:
+#  - bash <(curl -s https://codecov.io/bash)

build.sbt

@@ -8,11 +8,15 @@ lazy val root = project
   .settings(publish / skip := true)
   .settings(releaseSettings)
 
+lazy val deployment = project
+  .dependsOn(root)
+  .disablePlugins(SparkPackagePlugin)
+
 lazy val core = project
   .disablePlugins(SparkPackagePlugin)
 
 lazy val pyrasterframes = project
-  .dependsOn(core, datasource)
+  .dependsOn(core, datasource, experimental)
   .settings(assemblySettings)
 
 lazy val datasource = project

core/src/main/scala/astraea/spark/rasterframes/PairRDDConverter.scala

@@ -106,7 +106,8 @@ object PairRDDConverter {
       val basename = TILE_COLUMN.columnName
 
       val tiles = for(i ← 1 to bands) yield {
-        StructField(s"${basename}_$i" , serializableTileUDT, nullable = false)
+        val name = if(bands <= 1) basename else s"${basename}_$i"
+        StructField(name , serializableTileUDT, nullable = false)
       }
 
       StructType(base.fields.patch(1, tiles, 1))

core/src/main/scala/astraea/spark/rasterframes/RasterFunctions.scala

@@ -24,12 +24,13 @@ import astraea.spark.rasterframes.expressions.ExplodeTileExpression
 import astraea.spark.rasterframes.functions.{CellCountAggregateFunction, CellMeanAggregateFunction}
 import astraea.spark.rasterframes.stats.{CellHistogram, CellStatistics}
 import astraea.spark.rasterframes.{functions ⇒ F}
-import com.vividsolutions.jts.geom.Envelope
+import com.vividsolutions.jts.geom.{Envelope, Geometry}
+import geotrellis.proj4.CRS
 import geotrellis.raster.mapalgebra.local.LocalTileBinaryOp
 import geotrellis.raster.{CellType, Tile}
 import org.apache.spark.annotation.Experimental
 import org.apache.spark.sql._
-import org.apache.spark.sql.functions.udf
+import org.apache.spark.sql.functions.{lit, udf}
 import org.apache.spark.sql.rf._
 
 import scala.reflect.runtime.universe._
@@ -60,7 +61,7 @@ trait RasterFunctions {
   def tileDimensions(col: Column): Column = expressions.DimensionsExpression(col.expr).asColumn
 
   /** Extracts the bounding box of a geometry as a JTS envelope. */
-  def box2D(col: Column): TypedColumn[Any, Envelope] = expressions.Box2DExpression(col.expr).asColumn.as[Envelope]
+  def envelope(col: Column): TypedColumn[Any, Envelope] = expressions.EnvelopeExpression(col.expr).asColumn.as[Envelope]
 
   /** Flattens Tile into an array. A numeric type parameter is required. */
   @Experimental
@@ -88,6 +89,10 @@ trait RasterFunctions {
   def convertCellType(col: Column, cellType: CellType): TypedColumn[Any, Tile] =
     udf[Tile, Tile](F.convertCellType(cellType)).apply(col).as[Tile]
 
+  /** Change the Tile's cell type */
+  def convertCellType(col: Column, cellTypeName: String): TypedColumn[Any, Tile] =
+    udf[Tile, Tile](F.convertCellType(cellTypeName)).apply(col).as[Tile]
+
   /** Assign a `NoData` value to the Tiles. */
   def withNoData(col: Column, nodata: Double) = withAlias("withNoData", col)(
     udf[Tile, Tile](F.withNoData(nodata)).apply(col)
@@ -209,36 +214,214 @@ trait RasterFunctions {
     udf(F.localAdd).apply(left, right)
   ).as[Tile]
 
+  /** Cellwise addition of a scalar to a tile. */
+  def localAddScalar[T: Numeric](tileCol: Column, value: T): TypedColumn[Any, Tile] = {
+    val f = value match {
+      case i: Int => F.localAddScalarInt(_: Tile, i)
+      case d: Double => F.localAddScalar(_: Tile, d)
+    }
+
+    udf(f).apply(tileCol).as(s"localAddScalar($tileCol, $value)").as[Tile]
+  }
+
   /** Cellwise subtraction between two Tiles. */
   def localSubtract(left: Column, right: Column): TypedColumn[Any, Tile] =
   withAlias("localSubtract", left, right)(
     udf(F.localSubtract).apply(left, right)
   ).as[Tile]
 
+  /** Cellwise subtraction of a scalar from a tile. */
+  def localSubtractScalar[T: Numeric](tileCol: Column, value: T): TypedColumn[Any, Tile] = {
+    val f = value match {
+      case i: Int => F.localSubtractScalarInt(_: Tile, i)
+      case d: Double => F.localSubtractScalar(_: Tile, d)
+    }
+
+    udf(f).apply(tileCol).as(s"localSubtractScalar($tileCol, $value)").as[Tile]
+  }
+
   /** Cellwise multiplication between two Tiles. */
   def localMultiply(left: Column, right: Column): TypedColumn[Any, Tile] =
   withAlias("localMultiply", left, right)(
     udf(F.localMultiply).apply(left, right)
   ).as[Tile]
 
+  /** Cellwise multiplication of a tile by a scalar. */
+  def localMultiplyScalar[T: Numeric](tileCol: Column, value: T): TypedColumn[Any, Tile] = {
+    val f = value match {
+      case i: Int => F.localMultiplyScalarInt(_: Tile, i)
+      case d: Double => F.localMultiplyScalar(_: Tile, d)
+    }
+
+    udf(f).apply(tileCol).as(s"localMultiplyScalar($tileCol, $value)").as[Tile]
+  }
+
   /** Cellwise division between two Tiles. */
   def localDivide(left: Column, right: Column): TypedColumn[Any, Tile] =
   withAlias("localDivide", left, right)(
     udf(F.localDivide).apply(left, right)
   ).as[Tile]
 
+  /** Cellwise division of a tile by a scalar. */
+  def localDivideScalar[T: Numeric](tileCol: Column, value: T): TypedColumn[Any, Tile] = {
+    val f = value match {
+      case i: Int => F.localDivideScalarInt(_: Tile, i)
+      case d: Double => F.localDivideScalar(_: Tile, d)
+    }
+
+    udf(f).apply(tileCol).as(s"localDivideScalar($tileCol, $value)").as[Tile]
+  }
+
   /** Perform an arbitrary GeoTrellis `LocalTileBinaryOp` between two Tile columns. */
   def localAlgebra(op: LocalTileBinaryOp, left: Column, right: Column):
   TypedColumn[Any, Tile] =
     withAlias(opName(op), left, right)(
       udf[Tile, Tile, Tile](op.apply).apply(left, right)
     ).as[Tile]
 
+  /** Compute the normalized difference of two tile columns */
+  def normalizedDifference(left: Column, right: Column): TypedColumn[Any, Tile] =
+    withAlias("normalizedDifference", left, right)(
+      udf(F.normalizedDifference).apply(left, right)
+    ).as[Tile]
+
+  /** Constructor for constant tile column */
+  def makeConstantTile(value: Number, cols: Int, rows: Int, cellType: String): TypedColumn[Any, Tile] =
+    udf(() => F.makeConstantTile(value, cols, rows, cellType)).apply().as(s"constant_$cellType").as[Tile]
+
+  /** Alias for column of constant tiles of zero */
+  def tileZeros(cols: Int, rows: Int, cellType: String = "float64"): TypedColumn[Any, Tile] =
+    udf(() => F.tileZeros(cols, rows, cellType)).apply().as(s"zeros_$cellType").as[Tile]
+
+  /** Alias for column of constant tiles of one */
+  def tileOnes(cols: Int, rows: Int, cellType: String = "float64"): TypedColumn[Any, Tile] =
+    udf(() => F.tileOnes(cols, rows, cellType)).apply().as(s"ones_$cellType").as[Tile]
+
+  /** Where the mask tile contains NODATA, replace values in the source tile with NODATA */
+  def mask(sourceTile: Column, maskTile: Column): TypedColumn[Any, Tile] =
+    withAlias("mask", sourceTile, maskTile)(
+      udf(F.mask).apply(sourceTile, maskTile)
+    ).as[Tile]
+
+  /** Where the mask tile equals the mask value, replace values in the source tile with NODATA */
+  def maskByValue(sourceTile: Column, maskTile: Column, maskValue: Column): TypedColumn[Any, Tile] =
+    withAlias("maskByValue", sourceTile, maskTile, maskValue)(
+      udf(F.maskByValue).apply(sourceTile, maskTile, maskValue)
+    ).as[Tile]
+
+  /** Where the mask tile DOES NOT contain NODATA, replace values in the source tile with NODATA */
+  def inverseMask(sourceTile: Column, maskTile: Column): TypedColumn[Any, Tile] =
+    withAlias("inverseMask", sourceTile, maskTile)(
+      udf(F.inverseMask).apply(sourceTile, maskTile)
+    ).as[Tile]
+
+  /** Create a tile where cells in the grid defined by cols, rows, and bounds are filled with the given value. */
+  def rasterize(geometry: Column, bounds: Column, value: Column, cols: Int, rows: Int): TypedColumn[Any, Tile] =
+    withAlias("rasterize", geometry)(
+      udf(F.rasterize(_: Geometry, _: Geometry, _: Int, cols, rows)).apply(geometry, bounds, value)
+    ).as[Tile]
+
+  /** Reproject a column of geometry from one CRS to another. */  
+  def reprojectGeometry(sourceGeom: Column, srcCRS: CRS, dstCRS: CRS): TypedColumn[Any, Geometry] =
+    withAlias("reprojectGeometry", sourceGeom)(
+      udf(F.reprojectGeometry(_: Geometry, srcCRS, dstCRS)).apply(sourceGeom)
+    ).as[Geometry]
+
   /** Render Tile as ASCII string for debugging purposes. */
   @Experimental
   def renderAscii(col: Column): TypedColumn[Any, String] =
   withAlias("renderAscii", col)(
     udf[String, Tile](F.renderAscii).apply(col)
   ).as[String]
 
+  /** Cellwise less than value comparison between two tiles. */
+  def localLess(left: Column, right: Column): TypedColumn[Any, Tile] =
+    withAlias("localLess", left, right)(
+      udf(F.localLess).apply(left, right)
+    ).as[Tile]
+
+
+  /** Cellwise less than value comparison between a tile and a scalar. */
+  def localLessScalar[T: Numeric](tileCol: Column, value: T): TypedColumn[Any, Tile] = {
+    val f = value match{
+      case i: Int ⇒ F.localLessScalarInt(_: Tile, i)
+      case d: Double ⇒ F.localLessScalar(_: Tile, d)
+    }
+    udf(f).apply(tileCol).as(s"localLessScalar($tileCol, $value)").as[Tile]
+  }
+
+  /** Cellwise less than or equal to value comparison between a tile and a scalar. */
+  def localLessEqual(left: Column, right: Column): TypedColumn[Any, Tile]  =
+    withAlias("localLessEqual", left, right)(
+      udf(F.localLess).apply(left, right)
+    ).as[Tile]
+
+  /** Cellwise less than or equal to value comparison between a tile and a scalar. */
+  def localLessEqualScalar[T: Numeric](tileCol: Column, value: T): TypedColumn[Any, Tile] = {
+    val f = value match{
+      case i: Int ⇒ F.localLessEqualScalarInt(_: Tile, i)
+      case d: Double ⇒ F.localLessEqualScalar(_: Tile, d)
+    }
+    udf(f).apply(tileCol).as(s"localLessEqualScalar($tileCol, $value)").as[Tile]
+  }
+
+  /** Cellwise greater than value comparison between two tiles. */
+  def localGreater(left: Column, right: Column): TypedColumn[Any, Tile] =
+    withAlias("localGreater", left, right)(
+      udf(F.localGreater).apply(left, right)
+    ).as[Tile]
+
+
+  /** Cellwise greater than value comparison between a tile and a scalar. */
+  def localGreaterScalar[T: Numeric](tileCol: Column, value: T): TypedColumn[Any, Tile] = {
+    val f = value match{
+      case i: Int ⇒ F.localGreaterScalarInt(_: Tile, i)
+      case d: Double ⇒ F.localGreaterScalar(_: Tile, d)
+    }
+    udf(f).apply(tileCol).as(s"localGreaterScalar($tileCol, $value)").as[Tile]
+  }
+
+  /** Cellwise greater than or equal to value comparison between two tiles. */
+  def localGreaterEqual(left: Column, right: Column): TypedColumn[Any, Tile]  =
+    withAlias("localGreaterEqual", left, right)(
+      udf(F.localGreaterEqual).apply(left, right)
+    ).as[Tile]
+
+  /** Cellwise greater than or equal to value comparison between a tile and a scalar. */
+  def localGreaterEqualScalar[T: Numeric](tileCol: Column, value: T): TypedColumn[Any, Tile] = {
+    val f = value match{
+      case i: Int ⇒ F.localGreaterEqualScalarInt(_: Tile, i)
+      case d: Double ⇒ F.localGreaterEqualScalar(_: Tile, d)
+    }
+    udf(f).apply(tileCol).as(s"localGreaterEqualScalar($tileCol, $value)").as[Tile]
+  }
+
+  /** Cellwise equal to value comparison between two tiles. */
+  def localEqual(left: Column, right: Column): TypedColumn[Any, Tile]  =
+    withAlias("localEqual", left, right)(
+      udf(F.localEqual).apply(left, right)
+    ).as[Tile]
+
+  /** Cellwise equal to value comparison between a tile and a scalar. */
+  def localEqualScalar[T: Numeric](tileCol: Column, value: T): TypedColumn[Any, Tile] = {
+    val f = value match{
+      case i: Int ⇒ F.localEqualScalarInt(_: Tile, i)
+      case d: Double ⇒ F.localEqualScalar(_: Tile, d)
+    }
+    udf(f).apply(tileCol).as(s"localEqualScalar($tileCol, $value)").as[Tile]
+  }
+  /** Cellwise inequality comparison between two tiles. */
+  def localUnequal(left: Column, right: Column): TypedColumn[Any, Tile]  =
+    withAlias("localUnequal", left, right)(
+      udf(F.localUnequal).apply(left, right)
+    ).as[Tile]
+
+  /** Cellwise inequality comparison between a tile and a scalar. */
+  def localUnequalScalar[T: Numeric](tileCol: Column, value: T): TypedColumn[Any, Tile] = {
+    val f = value match{
+      case i: Int ⇒ F.localUnequalScalarInt(_: Tile, i)
+      case d: Double ⇒ F.localUnequalScalar(_: Tile, d)
+    }
+    udf(f).apply(tileCol).as(s"localUnequalScalar($tileCol, $value)").as[Tile]
+  }
 }

core/src/main/scala/astraea/spark/rasterframes/encoders/StandardEncoders.scala

@@ -25,31 +25,30 @@ import geotrellis.spark.tiling.LayoutDefinition
 import geotrellis.spark.{KeyBounds, SpaceTimeKey, SpatialKey, TemporalKey, TileLayerMetadata}
 import geotrellis.vector.Extent
 import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
-import org.apache.spark.sql.{Encoder, Encoders}
 
 import scala.reflect.runtime.universe._
 
 /**
  * Implicit encoder definitions for RasterFrame types.
  */
 trait StandardEncoders {
-  implicit val spatialKeyEncoder = ExpressionEncoder[SpatialKey]
-  implicit val temporalKeyEncoder = ExpressionEncoder[TemporalKey]
-  implicit val spaceTimeKeyEncoder = ExpressionEncoder[SpaceTimeKey]
-  implicit val statsEncoder = ExpressionEncoder[CellStatistics]
-  implicit val histEncoder = ExpressionEncoder[CellHistogram]
-  implicit val layoutDefinitionEncoder = ExpressionEncoder[LayoutDefinition]
-  implicit val stkBoundsEncoder = ExpressionEncoder[KeyBounds[SpaceTimeKey]]
-  implicit val extentEncoder = ExpressionEncoder[Extent]
+  implicit def spatialKeyEncoder = ExpressionEncoder[SpatialKey]
+  implicit def temporalKeyEncoder = ExpressionEncoder[TemporalKey]
+  implicit def spaceTimeKeyEncoder = ExpressionEncoder[SpaceTimeKey]
+  implicit def statsEncoder = ExpressionEncoder[CellStatistics]
+  implicit def histEncoder = ExpressionEncoder[CellHistogram]
+  implicit def layoutDefinitionEncoder = ExpressionEncoder[LayoutDefinition]
+  implicit def stkBoundsEncoder = ExpressionEncoder[KeyBounds[SpaceTimeKey]]
+  implicit def extentEncoder = ExpressionEncoder[Extent]
 
   implicit def singlebandTileEncoder = ExpressionEncoder[Tile]()
-  implicit def tileLayerMetadataEncoder[K: TypeTag]: Encoder[TileLayerMetadata[K]] = TileLayerMetadataEncoder[K]()
-  implicit val crsEncoder = CRSEncoder()
-  implicit val projectedExtentEncoder = ProjectedExtentEncoder()
-  implicit val temporalProjectedExtentEncoder = TemporalProjectedExtentEncoder()
-  implicit val cellTypeEncoder = CellTypeEncoder()
-  implicit val uriEncoder = URIEncoder()
-  implicit val envelopeEncoder = EnvelopeEncoder()
+  implicit def tileLayerMetadataEncoder[K: TypeTag]: ExpressionEncoder[TileLayerMetadata[K]] = TileLayerMetadataEncoder[K]()
+  implicit def crsEncoder = CRSEncoder()
+  implicit def projectedExtentEncoder = ProjectedExtentEncoder()
+  implicit def temporalProjectedExtentEncoder = TemporalProjectedExtentEncoder()
+  implicit def cellTypeEncoder = CellTypeEncoder()
+  implicit def uriEncoder = URIEncoder()
+  implicit def envelopeEncoder = EnvelopeEncoder()
 }
 
 object StandardEncoders extends StandardEncoders

core/src/main/scala/astraea/spark/rasterframes/encoders/TileLayerMetadataEncoder.scala

@@ -20,8 +20,8 @@
 package astraea.spark.rasterframes.encoders
 
 import geotrellis.spark.{KeyBounds, TileLayerMetadata}
-import org.apache.spark.sql.Encoder
 import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
+
 import scala.reflect.runtime.universe._
 
 /**
@@ -40,7 +40,7 @@ object TileLayerMetadataEncoder {
     "crs" -> crsEncoder
   )
 
-  def apply[K: TypeTag](): Encoder[TileLayerMetadata[K]] = {
+  def apply[K: TypeTag](): ExpressionEncoder[TileLayerMetadata[K]] = {
     val boundsEncoder = ExpressionEncoder[KeyBounds[K]]()
     val fEncoders = fieldEncoders :+ ("bounds" -> boundsEncoder)
     DelegatingSubfieldEncoder(fEncoders: _*)

core/src/main/scala/astraea/spark/rasterframes/expressions/Box2DExpression.scala renamed to core/src/main/scala/astraea/spark/rasterframes/expressions/EnvelopeExpression.scala

@@ -11,11 +11,11 @@ import org.apache.spark.sql.types._
  *
  * @since 2/22/18
  */
-case class Box2DExpression(child: Expression) extends UnaryExpression
+case class EnvelopeExpression(child: Expression) extends UnaryExpression
   with CodegenFallback with GeomDeserializerSupport  {
 
-  override def toString: String = s"Box2D($child)"
-  override def nodeName: String = "Box2D"
+  override def toString: String = s"Envelope($child)"
+  override def nodeName: String = "Envelope"
 
   override protected def nullSafeEval(input: Any): Any = {
     val geom = extractGeometry(child, input)