DO NOT MERGE: fast-binary-sort macros scrooge thrift

.gitignore

@@ -33,3 +33,5 @@ tutorial/data/jsonoutput0.tsv
 tutorial/data/avrooutput0.avro
 .scalding_repl
 scalding-hadoop-test/NOTICE
+
+/scalding-commons-macros/NOTICE

.travis.yml

@@ -11,47 +11,64 @@ matrix:
       env: BUILD="base" TEST_TARGET="scalding-args scalding-date"
       script: "scripts/run_test.sh"
 
-    - scala: 2.11.4
+    - scala: 2.11.5
       env: BUILD="base" TEST_TARGET="scalding-args scalding-date"
       script: "scripts/run_test.sh"
 
     - scala: 2.10.4
       env: BUILD="base" TEST_TARGET="scalding-avro scalding-hraven scalding-commons"
       script: "scripts/run_test.sh"
 
-    - scala: 2.11.4
+    - scala: 2.11.5
       env: BUILD="base" TEST_TARGET="scalding-avro scalding-hraven scalding-commons"
       script: "scripts/run_test.sh"
 
     - scala: 2.10.4
       env: BUILD="base" TEST_TARGET="scalding-core"
       script: "scripts/run_test.sh"
 
-    - scala: 2.11.4
+    - scala: 2.11.5
       env: BUILD="base" TEST_TARGET="scalding-core"
       script: "scripts/run_test.sh"
 
     - scala: 2.10.4
       env: BUILD="base" TEST_TARGET="scalding-hadoop-test"
       script: "scripts/run_test.sh"
 
-    - scala: 2.11.4
+    - scala: 2.11.5
       env: BUILD="base" TEST_TARGET="scalding-hadoop-test"
       script: "scripts/run_test.sh"
 
     - scala: 2.10.4
       env: BUILD="base" TEST_TARGET="scalding-jdbc scalding-json"
       script: "scripts/run_test.sh"
 
-    - scala: 2.11.4
+    - scala: 2.11.5
       env: BUILD="base" TEST_TARGET="scalding-jdbc scalding-json"
       script: "scripts/run_test.sh"
 
+    - scala: 2.10.4
+      env: BUILD="base" TEST_TARGET="scalding-macros"
+      script: "scripts/run_test.sh"
+
+    - scala: 2.11.5
+      env: BUILD="base" TEST_TARGET="scalding-macros"
+      script: "scripts/run_test.sh"
+
+# not committed yet
+    # - scala: 2.10.4
+    #   env: BUILD="base" TEST_TARGET="scalding-commons-macros"
+    #   script: "scripts/run_test.sh"
+
+    # - scala: 2.11.5
+    #   env: BUILD="base" TEST_TARGET="scalding-commons-macros"
+    #   script: "scripts/run_test.sh"
+
     - scala: 2.10.4
       env: BUILD="base" TEST_TARGET="scalding-parquet scalding-parquet-scrooge"
       script: "scripts/run_test.sh"
 
-    - scala: 2.11.4
+    - scala: 2.11.5
       env: BUILD="base" TEST_TARGET="scalding-parquet scalding-parquet-scrooge"
       script: "scripts/run_test.sh"
 
@@ -65,7 +82,7 @@ matrix:
       - "scripts/build_assembly_no_test.sh scalding"
       - "scripts/test_tutorials.sh"
 
-    - scala: 2.11.4
+    - scala: 2.11.5
       env: BUILD="test tutorials"
       script:
       - "scripts/build_assembly_no_test.sh scalding"
@@ -77,7 +94,7 @@ matrix:
       - "scripts/build_assembly_no_test.sh scalding"
       - "scripts/test_matrix_tutorials.sh"
 
-    - scala: 2.11.4
+    - scala: 2.11.5
       env: BUILD="test matrix tutorials"
       script:
       - "scripts/build_assembly_no_test.sh scalding"
@@ -91,7 +108,7 @@ matrix:
       - "scripts/build_assembly_no_test.sh scalding-core"
       - "scripts/test_typed_tutorials.sh"
 
-    - scala: 2.11.4
+    - scala: 2.11.5
       env: BUILD="test typed tutorials"
       script:
       - "scripts/build_assembly_no_test.sh scalding-core"

project/Build.scala

@@ -21,7 +21,7 @@ object ScaldingBuild extends Build {
   def isScala210x(scalaVersion: String) = scalaBinaryVersion(scalaVersion) == "2.10"
 
   val scalaTestVersion = "2.2.2"
-  val scalaCheckVersion = "1.11.5"
+  val scalaCheckVersion = "1.12.1"
   val hadoopVersion = "1.2.1"
   val algebirdVersion = "0.9.0"
   val bijectionVersion = "0.7.2"
@@ -39,6 +39,7 @@ object ScaldingBuild extends Build {
   val cascadingAvroVersion = "2.1.2"
   val avroVersion = "1.7.4"
   val json4SVersion = "3.2.11"
+  val scroogeVersion = "3.17.0"
 
   val printDependencyClasspath = taskKey[Unit]("Prints location of the dependencies")
 
@@ -192,6 +193,7 @@ object ScaldingBuild extends Build {
     scaldingDate,
     scaldingCore,
     scaldingCommons,
+    scaldingCommonsMacros,
     scaldingAvro,
     scaldingParquet,
     scaldingParquetScrooge,
@@ -275,10 +277,12 @@ object ScaldingBuild extends Build {
       "com.hadoop.gplcompression" % "hadoop-lzo" % hadoopLzoVersion,
       // TODO: split this out into scalding-thrift
       "org.apache.thrift" % "libthrift" % thriftVersion,
+      // TODO: split this out into a scalding-scrooge
+      "com.twitter" %% "scrooge-serializer" % scroogeVersion % "provided",
       "org.slf4j" % "slf4j-api" % slf4jVersion,
       "org.slf4j" % "slf4j-log4j12" % slf4jVersion % "provided"
     )
-  ).dependsOn(scaldingArgs, scaldingDate, scaldingCore)
+  ).dependsOn(scaldingArgs, scaldingDate, scaldingCore, scaldingHadoopTest % "test")
 
   lazy val scaldingAvro = module("avro").settings(
     libraryDependencies ++= Seq(
@@ -415,6 +419,34 @@ object ScaldingBuild extends Build {
   addCompilerPlugin("org.scalamacros" % "paradise" % "2.0.1" cross CrossVersion.full)
   ).dependsOn(scaldingCore, scaldingHadoopTest)
 
+  lazy val scaldingCommonsMacros = module("commons-macros").settings(
+    libraryDependencies <++= (scalaVersion) { scalaVersion => Seq(
+      "org.scala-lang" % "scala-library" % scalaVersion,
+      "org.scala-lang" % "scala-reflect" % scalaVersion,
+      "com.twitter" %% "bijection-macros" % bijectionVersion,
+      "com.twitter" % "chill-thrift" % chillVersion % "test",
+      "com.twitter" %% "scrooge-serializer" % scroogeVersion % "provided",
+      "org.apache.thrift" % "libthrift" % thriftVersion
+    ) ++ (if(isScala210x(scalaVersion)) Seq("org.scalamacros" %% "quasiquotes" % "2.0.1") else Seq())
+  },
+  addCompilerPlugin("org.scalamacros" % "paradise" % "2.0.1" cross CrossVersion.full)
+  ).dependsOn(scaldingCore, scaldingMacros, scaldingCommons, scaldingHadoopTest % "test", scaldingCommonsMacrosFixtures % "test")
+
+
+  // Only used in local test
+  // not published so not in the aggregate target
+  // the scalding commons macros target will pull it in to be built.
+  lazy val scaldingCommonsMacrosFixtures = module("commons-macros-fixtures").settings(
+    libraryDependencies <++= (scalaVersion) { scalaVersion => Seq(
+      "org.scala-lang" % "scala-library" % scalaVersion,
+      "org.scala-lang" % "scala-reflect" % scalaVersion,
+      "com.twitter" %% "scrooge-serializer" % scroogeVersion % "provided",
+      "org.apache.thrift" % "libthrift" % thriftVersion
+    )
+   }
+  )
+
+
   // This one uses a different naming convention
   lazy val maple = Project(
     id = "maple",

scalding-commons-macros-fixtures/src/main/scala/com/twitter/scalding/commons/macros/scalathrift/TestEnum.scala

@@ -0,0 +1,94 @@
+/**
+ * Generated by Scrooge
+ *   version: ?
+ *   rev: ?
+ *   built at: ?
+ */
+package com.twitter.scalding.commons.macros.scalathrift
+
+import com.twitter.scrooge.ThriftEnum
+
+@javax.annotation.Generated(value = Array("com.twitter.scrooge.Compiler"))
+case object TestEnum {
+
+  case object Zero extends com.twitter.scalding.commons.macros.scalathrift.TestEnum {
+    val value = 0
+    val name = "Zero"
+    val originalName = "ZERO"
+  }
+
+  case object One extends com.twitter.scalding.commons.macros.scalathrift.TestEnum {
+    val value = 1
+    val name = "One"
+    val originalName = "ONE"
+  }
+
+  case object Two extends com.twitter.scalding.commons.macros.scalathrift.TestEnum {
+    val value = 2
+    val name = "Two"
+    val originalName = "TWO"
+  }
+
+  case object Large extends com.twitter.scalding.commons.macros.scalathrift.TestEnum {
+    val value = 70000
+    val name = "Large"
+    val originalName = "LARGE"
+  }
+
+  case object Huge extends com.twitter.scalding.commons.macros.scalathrift.TestEnum {
+    val value = 2147483647
+    val name = "Huge"
+    val originalName = "HUGE"
+  }
+
+  /**
+   * Find the enum by its integer value, as defined in the Thrift IDL.
+   * @throws NoSuchElementException if the value is not found.
+   */
+  def apply(value: Int): com.twitter.scalding.commons.macros.scalathrift.TestEnum = {
+    value match {
+      case 0 => com.twitter.scalding.commons.macros.scalathrift.TestEnum.Zero
+      case 1 => com.twitter.scalding.commons.macros.scalathrift.TestEnum.One
+      case 2 => com.twitter.scalding.commons.macros.scalathrift.TestEnum.Two
+      case 70000 => com.twitter.scalding.commons.macros.scalathrift.TestEnum.Large
+      case 2147483647 => com.twitter.scalding.commons.macros.scalathrift.TestEnum.Huge
+      case _ => throw new NoSuchElementException(value.toString)
+    }
+  }
+
+  /**
+   * Find the enum by its integer value, as defined in the Thrift IDL.
+   * Returns None if the value is not found
+   */
+  def get(value: Int): Option[com.twitter.scalding.commons.macros.scalathrift.TestEnum] = {
+    value match {
+      case 0 => scala.Some(com.twitter.scalding.commons.macros.scalathrift.TestEnum.Zero)
+      case 1 => scala.Some(com.twitter.scalding.commons.macros.scalathrift.TestEnum.One)
+      case 2 => scala.Some(com.twitter.scalding.commons.macros.scalathrift.TestEnum.Two)
+      case 70000 => scala.Some(com.twitter.scalding.commons.macros.scalathrift.TestEnum.Large)
+      case 2147483647 => scala.Some(com.twitter.scalding.commons.macros.scalathrift.TestEnum.Huge)
+      case _ => scala.None
+    }
+  }
+
+  def valueOf(name: String): Option[com.twitter.scalding.commons.macros.scalathrift.TestEnum] = {
+    name.toLowerCase match {
+      case "zero" => scala.Some(com.twitter.scalding.commons.macros.scalathrift.TestEnum.Zero)
+      case "one" => scala.Some(com.twitter.scalding.commons.macros.scalathrift.TestEnum.One)
+      case "two" => scala.Some(com.twitter.scalding.commons.macros.scalathrift.TestEnum.Two)
+      case "large" => scala.Some(com.twitter.scalding.commons.macros.scalathrift.TestEnum.Large)
+      case "huge" => scala.Some(com.twitter.scalding.commons.macros.scalathrift.TestEnum.Huge)
+      case _ => scala.None
+    }
+  }
+
+  lazy val list: List[com.twitter.scalding.commons.macros.scalathrift.TestEnum] = scala.List[com.twitter.scalding.commons.macros.scalathrift.TestEnum](
+    com.twitter.scalding.commons.macros.scalathrift.TestEnum.Zero,
+    com.twitter.scalding.commons.macros.scalathrift.TestEnum.One,
+    com.twitter.scalding.commons.macros.scalathrift.TestEnum.Two,
+    com.twitter.scalding.commons.macros.scalathrift.TestEnum.Large,
+    com.twitter.scalding.commons.macros.scalathrift.TestEnum.Huge)
+}
+
+@javax.annotation.Generated(value = Array("com.twitter.scrooge.Compiler"))
+sealed trait TestEnum extends ThriftEnum with Serializable