Merge pull request #298 from airbnb/tibi-scala_2_12

Build aerosolve using Scala 2.12

airlearner/airlearner-strategy/build.gradle

@@ -1,7 +1,7 @@
 apply plugin: 'com.jfrog.bintray'
 
 // Includes 'scala' plugin
-apply from: "$rootDir/airlearner/configs/scala211.gradle"
+apply from: "$rootDir/airlearner/configs/scala212.gradle"
 
 repositories {
   mavenLocal()
@@ -10,9 +10,9 @@ repositories {
 dependencies {
   compile libraries.typesafe_config
 
-  provided "org.apache.spark:spark-core_2.11:${sparkVersion}"
-  provided "org.apache.spark:spark-hive_2.11:${sparkVersion}"
-  provided "org.apache.spark:spark-mllib_2.11:${sparkVersion}"
+  provided "org.apache.spark:spark-core_2.12:${sparkVersion}"
+  provided "org.apache.spark:spark-hive_2.12:${sparkVersion}"
+  provided "org.apache.spark:spark-mllib_2.12:${sparkVersion}"
 
   compile project(':airlearner:airlearner-utils')
 

airlearner/airlearner-utils/build.gradle

@@ -6,7 +6,7 @@ apply plugin: 'java'
 apply plugin: 'com.jfrog.bintray'
 
 // Includes 'scala' plugin
-apply from: "$rootDir/airlearner/configs/scala211.gradle"
+apply from: "$rootDir/airlearner/configs/scala212.gradle"
 
 repositories {
   mavenLocal()
@@ -20,8 +20,8 @@ dependencies {
   compile libraries.slf4j_simple
   compile libraries.typesafe_config
 
-  provided "org.apache.spark:spark-core_2.11:${sparkVersion}"
-  provided "org.apache.spark:spark-hive_2.11:${sparkVersion}"
+  provided "org.apache.spark:spark-core_2.12:${sparkVersion}"
+  provided "org.apache.spark:spark-hive_2.12:${sparkVersion}"
 }
 
 bintray {

airlearner/airlearner-utils/src/main/scala/com/airbnb/common/ml/util/PipelineUtil.scala

@@ -15,7 +15,7 @@ import org.apache.hadoop.io.compress.{CompressionCodec, GzipCodec}
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql._
 import org.apache.spark.sql.hive.HiveContext
-import org.apache.spark.{Accumulator, SparkContext}
+import org.apache.spark.SparkContext
 import org.joda.time.format.{DateTimeFormat, DateTimeFormatter}
 import org.joda.time.{DateTime, Days}
 
@@ -299,52 +299,6 @@ object PipelineUtil extends ScalaLogging {
     true
   }
 
-  // Create a pair of spark accumulators that track (success, failure) counts.
-  def addStatusAccumulators(sc: SparkContext,
-      accName: String,
-      accumulators: mutable.Map[String, Accumulator[Int]]): Unit = {
-    accumulators.put(accName + ".success", sc.accumulator(0, accName + ".success"))
-    accumulators.put(accName + ".failure", sc.accumulator(0, accName + ".failure"))
-  }
-
-
-  def countAllFailureCounters(accumulators: mutable.Map[String, Accumulator[Int]]): Long = {
-    var failureCount = 0
-    for ( (name, accumulator) <- accumulators) {
-      logger.info("- Accumulator {} : {}", name, accumulator.value.toString )
-      if (name.endsWith(".failure")) {
-        failureCount += accumulator.value
-      }
-    }
-    failureCount
-  }
-
-  def validateSuccessCounters(accumulators: mutable.Map[String, Accumulator[Int]],
-      minSuccess: Int): Boolean = {
-    for ( (name, accumulator) <- accumulators) {
-      if (name.endsWith(".success") && accumulator.value < minSuccess) {
-        logger.error("Failed counter: {} = {} < {}", name, accumulator.value.toString, minSuccess.toString)
-        return false
-      }
-    }
-    true
-  }
-
-  // TODO(kim): cleanup, write to hdfs directly instead of via SparkContext.
-  def saveCountersAsTextFile(accumulators: mutable.Map[String, Accumulator[Int]],
-      sc: SparkContext,
-      hdfsFilePath: String): Unit = {
-    var summary = Array("Summarizing counters:")
-
-    for ( (name, accumulator) <- accumulators) {
-      val logLine = "- %s = %d".format(name, accumulator.value )
-      summary :+= logLine
-      logger.info(logLine)
-    }
-
-    saveAndCommitAsTextFile(sc.parallelize(summary), hdfsFilePath, 1, true)
-  }
-
   def getHDFSBufferedOutputStream(output: String): BufferedOutputStream = {
     new BufferedOutputStream(getHDFSOutputStream(output))
   }

airlearner/airlearner-utils/src/main/scala/com/airbnb/common/ml/util/ScalaLogging.scala

@@ -1,7 +1,7 @@
 package com.airbnb.common.ml.util
 
 // scalastyle:off ban.logger.factory
-import com.typesafe.scalalogging.slf4j.Logger
+import com.typesafe.scalalogging.Logger
 import org.slf4j.LoggerFactory
 
 

airlearner/airlearner-xgboost/build.gradle

@@ -1,7 +1,7 @@
 apply plugin: 'com.jfrog.bintray'
 
 // Includes 'scala' plugin
-apply from: "$rootDir/airlearner/configs/scala211.gradle"
+apply from: "$rootDir/airlearner/configs/scala212.gradle"
 
 repositories {
   mavenLocal()
@@ -10,9 +10,9 @@ repositories {
 dependencies {
   compile libraries.typesafe_config
 
-  provided "org.apache.spark:spark-core_2.11:${sparkVersion}"
-  provided "org.apache.spark:spark-hive_2.11:${sparkVersion}"
-  provided "org.apache.spark:spark-mllib_2.11:${sparkVersion}"
+  provided "org.apache.spark:spark-core_2.12:${sparkVersion}"
+  provided "org.apache.spark:spark-hive_2.12:${sparkVersion}"
+  provided "org.apache.spark:spark-mllib_2.12:${sparkVersion}"
 
   compile project(':airlearner:airlearner-utils')
   compile files('local-lib/xgboost4j-0.7.jar')

airlearner/configs/scala210.gradle

@@ -9,8 +9,6 @@ ext {
 // Language support
 apply plugin: 'java'
 apply plugin: 'scala'
-// Tooling support
-apply plugin: 'scalaStyle'
 
 
 /**
@@ -61,18 +59,6 @@ tasks.withType(ScalaCompile) {
 }
 
 
-/**
- * Automated Scala style checking as part of the build check task
- */
-check.dependsOn << ['scalaStyle']
-scalaStyle {
-  configLocation = "$rootDir/airlearner/configs/scalastyle_config.xml"
-  source = 'src/main/scala'
-  testSource = 'src/test/scala'
-  includeTestSourceDirectory = true
-}
-
-
 /**
  * Task: `repl`
  *

airlearner/configs/scala211.gradle

@@ -9,8 +9,6 @@ ext {
 // Language support
 apply plugin: 'java'
 apply plugin: 'scala'
-// Tooling support
-apply plugin: 'scalaStyle'
 
 
 /**
@@ -58,18 +56,6 @@ tasks.withType(ScalaCompile) {
 }
 
 
-/**
- * Automated Scala style checking as part of the build check task
- */
-check.dependsOn << ['scalaStyle']
-scalaStyle {
-    configLocation = "$rootDir/airlearner/configs/scalastyle_config.xml"
-    source = 'src/main/scala'
-    testSource = 'src/test/scala'
-    includeTestSourceDirectory = true
-}
-
-
 /**
  * Task: `repl`
  *

airlearner/configs/scala212.gradle

@@ -0,0 +1,75 @@
+/**
+ * This Gradle file sets up common build settings for Scala 2.10.x projects
+ */
+
+ext {
+    scala212Version = '2.12.7'
+}
+
+// Language support
+apply plugin: 'java'
+apply plugin: 'scala'
+
+
+/**
+ * Include basic package dependencies that ALL Airbnb Scala projects will need.
+ * This should include no team or project specific code. That will only bloat other
+ * Scala projects.
+ *
+ * Additionally, don't include domain specific libraries such as Spark.
+ */
+dependencies {
+    compile libraries.scala_library_212
+    compile libraries.scala_logging_slf4j_212
+    compile libraries.org_scala_lang_modules_scala_java8_compat_2_12
+}
+
+/**
+ * Set standard Scala compilation options
+ */
+def getScalaCompileAdditionalParameters = {
+    def compileOptions = [
+            // Emit warning and location for usages of features that should be imported explicitly.
+            '-feature',
+            // Output messages about what the compiler is doing.
+            '-verbose',
+            // Enable recommended additional warnings.
+            '-Xlint',
+    ]
+    if (project.hasProperty('SkipWarning')) {
+        compileOptions << '-nowarn'
+    }
+    compileOptions
+}
+tasks.withType(ScalaCompile) {
+    // Target Java 1.8 level compatibility
+    sourceCompatibility = '1.8'
+    targetCompatibility = '1.8'
+
+    scalaCompileOptions.with {
+        deprecation = true
+        unchecked = true
+        optimize = true
+        debugLevel = 'vars'
+        additionalParameters = getScalaCompileAdditionalParameters()
+    }
+}
+
+
+/**
+ * Task: `repl`
+ *
+ * Will initiate a command-line Scala console that you can use
+ * to interact live with your project's code.
+ *
+ * Run with `gradlew repl --console plain --no-daemon`
+ */
+task repl(type: JavaExec) {
+    dependencies {
+        compile group: 'org.scala-lang', name: 'scala-compiler', version: scala212Version
+    }
+    main = 'scala.tools.nsc.MainGenericRunner'
+    classpath = sourceSets.main.runtimeClasspath
+    standardInput System.in
+    args '-usejavacp'
+}

airlearner/libraries.gradle

@@ -1,18 +1,21 @@
 allprojects {
   ext {
-    sparkVersion = '2.3.0'
+    sparkVersion = '3.1.1'
 
     libraries = [
         lombok: 'org.projectlombok:lombok:1.16.8',
         joda_time: 'joda-time:joda-time:2.8.2',
         joda_convert: 'org.joda:joda-convert:1.8',
         scala_library_211: 'org.scala-lang:scala-library:2.11.8',
+        scala_library_212: 'org.scala-lang:scala-library:2.12.7',
         scala_logging_slf4j_211: 'com.typesafe.scala-logging:scala-logging-slf4j_2.11:2.1.2',
+        scala_logging_slf4j_212: 'com.typesafe.scala-logging:scala-logging_2.12:3.9.2',
         slf4j_api: 'org.slf4j:slf4j-api:1.7.9',
         slf4j_simple: 'org.slf4j:slf4j-simple:1.7.9',
         typesafe_config: 'com.typesafe:config:1.3.0',
         org_typelevel_cats_core_2_11_1_0_1: 'org.typelevel:cats-core_2.11:1.0.1',
         org_scala_lang_modules_scala_java8_compat_2_11: 'org.scala-lang.modules:scala-java8-compat_2.11:0.8.0',
+        org_scala_lang_modules_scala_java8_compat_2_12: 'org.scala-lang.modules:scala-java8-compat_2.12:0.8.0',
     ]
   }
 }

build.gradle

@@ -5,6 +5,7 @@ buildscript {
   repositories {
     jcenter()
     maven { url "https://plugins.gradle.org/m2/" }
+    maven { url "https://repo.grails.org/grails/core/" }
   }
 
   dependencies {
@@ -17,9 +18,6 @@ buildscript {
     classpath group: 'gradle.plugin.com.palantir.gradle.gitversion',
         name: 'gradle-git-version',
         version: '0.7.3'
-    classpath group: 'org.github.ngbinh.scalastyle',
-        name: 'gradle-scalastyle-plugin_2.10',
-        version: '0.8.2'
   }
 }
 

core/build.gradle

@@ -39,5 +39,6 @@ dependencies {
   compile 'org.projectlombok:lombok:1.14.8'
   compile 'org.apache.commons:commons-lang3:3.4'
   compile 'org.apache.commons:commons-math3:3.6.1'
+  compile 'org.apache.httpcomponents:httpcore:4.4.16'
   testCompile 'org.slf4j:slf4j-log4j12:1.7.21'
 }

core/src/main/java/com/airbnb/aerosolve/core/models/LinearModel.java

@@ -14,12 +14,12 @@
 import com.google.common.hash.HashCode;
 import lombok.Getter;
 import lombok.Setter;
-import org.apache.http.annotation.NotThreadSafe;
+import org.apache.http.annotation.Contract;
 
 /**
  * A linear model backed by a hash map.
  */
-@NotThreadSafe
+@Contract (threading = org.apache.http.annotation.ThreadingBehavior.UNSAFE)
 public class LinearModel extends AbstractModel {
 
   @Getter @Setter

core/src/main/java/com/airbnb/aerosolve/core/util/Debug.java

@@ -144,8 +144,8 @@ public static Example loadExampleFromResource(String name) {
   // Save example to path
   // If you hit permission error, touch and chmod the file
   public static void saveExample(Example example, String path) {
-    TSerializer serializer = new TSerializer(new TBinaryProtocol.Factory());
     try {
+      TSerializer serializer = new TSerializer(new TBinaryProtocol.Factory());
       byte[] buf = serializer.serialize(example);
       FileOutputStream fos = new FileOutputStream(path);
       fos.write(buf);

core/src/main/java/com/airbnb/aerosolve/core/util/Util.java

@@ -49,8 +49,8 @@ public class Util implements Serializable {
   // manipulate in spark. e.g. if we  wanted to see the 50 weights in a model
   // val top50 = sc.textFile("model.bz2").map(Util.decodeModel).sortBy(x => -x.weight).take(50);
   public static String encode(TBase obj) {
-    TSerializer serializer = new TSerializer();
     try {
+      TSerializer serializer = new TSerializer();
       byte[] bytes = serializer.serialize(obj);
       return new String(Base64.encodeBase64(bytes));
     } catch (Exception e) {

thrift-cli.gradle

@@ -1,7 +1,7 @@
 apply plugin: "java"
 
 dependencies {
-  compile 'org.apache.thrift:libthrift:0.9.1'
+  compile 'org.apache.thrift:libthrift:0.20.0'
 }
 
 def genJavaDir = new File("${->buildDir}/gen-java")

training/build.gradle

@@ -31,7 +31,7 @@ bintray {
 }
 
 dependencies {
-  compile 'org.scala-lang:scala-library:2.11.8'
+  compile 'org.scala-lang:scala-library:2.12.7'
   compile project(':core')
 
   compile ('org.apache.avro:avro:1.7.7') {
@@ -41,18 +41,18 @@ dependencies {
     transitive = false
   }
 
-  compile 'com.fasterxml.jackson.module:jackson-module-scala_2.11:2.7.8'
+  compile 'com.fasterxml.jackson.module:jackson-module-scala_2.12:2.11.4'
   compile 'joda-time:joda-time:2.5'
   compile 'org.apache.hadoop:hadoop-client:2.2.0'
   compile 'com.databricks:spark-csv_2.10:1.4.0'
 
-  provided 'org.apache.spark:spark-core_2.11:2.3.0'
-  provided 'org.apache.spark:spark-hive_2.11:2.3.0'
+  provided 'org.apache.spark:spark-core_2.12:3.1.1'
+  provided 'org.apache.spark:spark-hive_2.12:3.1.1'
 
-  testCompile 'org.apache.spark:spark-core_2.11:2.3.0'
-  testCompile 'org.apache.spark:spark-core_2.11:2.3.0:tests'
+  testCompile 'org.apache.spark:spark-core_2.12:3.1.1'
+  testCompile 'org.apache.spark:spark-core_2.12:3.1.1:tests'
   testCompile 'org.scalatest:scalatest_2.11:3.0.5'
-  testCompile 'org.apache.spark:spark-hive_2.11:2.3.0'
+  testCompile 'org.apache.spark:spark-hive_2.12:3.1.1'
   testCompile 'org.mockito:mockito-all:1.9.5'
   testCompile 'org.slf4j:slf4j-log4j12:1.7.21'
 }