Merge pull request #103 from eed3si9n/wip/source_jars

Resolve source JARs

multiversion/src/main/scala/multiversion/commands/ExportCommand.scala

@@ -108,7 +108,11 @@ case class ExportCommand(
       for {
         withUrls <- lintUrls(thirdparty, allowUrl)
         initialResolutions <- runResolutions(withUrls, None, withUrls.coursierDeps, coursierCache)
-        initialIndex = ResolutionIndex.fromResolutions(withUrls, initialResolutions)
+        initialIndex = ResolutionIndex.fromResolutions(
+          withUrls,
+          initialResolutions,
+          resolveSources = false
+        )
         withSelectedVersions = selectVersionsFromIndex(withUrls, initialIndex)
         withOverriddenTargets = overrideTargets(withSelectedVersions, initialIndex)
         intermediateResolutions <- runResolutions(
@@ -118,14 +122,22 @@ case class ExportCommand(
           coursierCache
         )
         intermediateIndex =
-          ResolutionIndex.fromResolutions(withOverriddenTargets, intermediateResolutions)
+          ResolutionIndex.fromResolutions(
+            withOverriddenTargets,
+            intermediateResolutions,
+            resolveSources = false
+          )
         resolutions <- runResolutions(
           withOverriddenTargets,
           Some(intermediateIndex),
           withOverriddenTargets.coursierDeps,
           coursierCache
         )
-        index = ResolutionIndex.fromResolutions(withOverriddenTargets, resolutions)
+        index = ResolutionIndex.fromResolutions(
+          withOverriddenTargets,
+          resolutions,
+          resolveSources = true
+        )
         _ <- lintEvictedDeclaredDependencies(
           withUrls,
           initialIndex,
@@ -285,15 +297,25 @@ case class ExportCommand(
     val files: List[Task[List[Either[Throwable, ArtifactOutput]]]] =
       resolvedArtifacts.map { r =>
         val logger = progressBar.loggers.newCacheLogger(r.dependency)
-        val url = r.artifact.checksumUrls.getOrElse("SHA-256", r.artifact.url)
         type Fetch[T] = Task[Either[ArtifactError, T]]
         def tryFetch(artifact: Artifact, policy: CachePolicy): Fetch[File] =
           cache
             .withCachePolicies(List(policy))
             .withLogger(logger)
             .file(artifact)
             .run
-        val shaAttempts: List[Fetch[File]] = for {
+        def foldShas(attempts: List[Fetch[File]]): Fetch[File] =
+          if (attempts.isEmpty) Task.point(Left(new ArtifactError.NotFound("")))
+          else
+            attempts.tail.foldLeft(attempts.head) { case (task, nextAttempt) =>
+              task.flatMap {
+                case Left(_) =>
+                  // Fetch failed, try next (Url, CachePolicy) combination
+                  nextAttempt
+                case success => Task.point(success)
+              }
+            }
+        val shas = foldShas(for {
           // Attempt 1: Fetch "*.jar.sha256" URL locally
           // Attempt 2: Fetch "*.jar" URL locally
           // Attempt 3: Fetch "*.jar.sha256" URL remotely
@@ -303,27 +325,34 @@ case class ExportCommand(
             Some(r.artifact.url)
           ).flatten
           policy <- List(CachePolicy.LocalOnly, CachePolicy.Update)
-        } yield tryFetch(r.artifact.withUrl(url), policy)
-        val shas = shaAttempts.tail.foldLeft(shaAttempts.head) { case (task, nextAttempt) =>
-          task.flatMap {
-            case Left(_) =>
-              // Fetch failed, try next (Url, CachePolicy) combination
-              nextAttempt
-            case success => Task.point(success)
-          }
-        }
-        shas.map {
+        } yield tryFetch(r.artifact.withUrl(url), policy))
+        val sourcesShas = foldShas(for {
+          url <- List(
+            r.sourcesArtifact.flatMap(_.checksumUrls.get("SHA-256")),
+            r.sourcesArtifact.map(_.url),
+          ).flatten
+          policy <- List(CachePolicy.LocalOnly, CachePolicy.Update)
+        } yield tryFetch(r.artifact.withUrl(url), policy))
+        shas.flatMap {
           case Right(file) =>
-            List(Try {
-              val output = ArtifactOutput(
-                dependency = r.dependency,
-                artifact = r.artifact,
-                artifactSha256 = Sha256.compute(file)
-              )
-              outputIndex.put(r.dependency.bazelLabel, output)
-              output
-            }.toEither)
-
+            (sourcesShas
+              .map {
+                case Right(sourceFile) => Some(sourceFile)
+                case Left(_)           => None
+              })
+              .map { sourceSha =>
+                List(Try {
+                  val output = ArtifactOutput(
+                    dependency = r.dependency,
+                    artifact = r.artifact,
+                    artifactSha256 = Sha256.compute(file),
+                    sourcesArtifact = r.sourcesArtifact,
+                    sourcesArtifactSha256 = sourceSha.map(Sha256.compute)
+                  )
+                  outputIndex.put(r.dependency.bazelLabel, output)
+                  output
+                }.toEither)
+              }
           case Left(value) =>
             // Ignore download failures. It's common that some dependencies have
             // pom files but no jar files. For example,
@@ -332,7 +361,7 @@ case class ExportCommand(
             // helpful to distinguish these kinds of dependencies but they are
             // true by default so I'm not sure if they're intended to be used
             // for that purpose.
-            Nil
+            Task.point(Nil)
         }
       }
     val all = runParallelTasks(files, progressBar, cache.ec).flatten

multiversion/src/main/scala/multiversion/outputs/ArtifactOutput.scala

@@ -25,6 +25,7 @@ final case class ArtifactOutput(
       s"_${dependency.publication.classifier.value}"
     else ""
   val label: String = dependency.bazelLabel
+  val sourcesLabel: String = label + "_sources"
   val repr: String =
     s"""|Artifact(
         |  dep = "${label}",
@@ -36,13 +37,30 @@ final case class ArtifactOutput(
   val version: String = dependency.version
   val mavenLabel: String = dependency.mavenLabel
 
+  def httpFiles: List[TargetOutput] =
+    List(httpFile) ::: sourcesHttpFile.toList
+
   def httpFile: TargetOutput =
     TargetOutput(
       kind = "http_file",
       "name" -> Docs.literal(label),
       "urls" -> Docs.array(artifact.url),
       "sha256" -> Docs.literal(artifactSha256)
     )
+
+  def sourcesHttpFile: Option[TargetOutput] =
+    (sourcesArtifact, sourcesArtifactSha256) match {
+      case (Some(art), Some(sha)) =>
+        Some(
+          TargetOutput(
+            kind = "http_file",
+            "name" -> Docs.literal(sourcesLabel),
+            "urls" -> Docs.array(art.url),
+            "sha256" -> Docs.literal(sha)
+          )
+        )
+      case _ => None
+    }
 }
 
 object ArtifactOutput {
@@ -59,21 +77,28 @@ object ArtifactOutput {
       val reconciledDependency = index.reconciledDependency(dependency)
       outputIndex
         .get(reconciledDependency.bazelLabel)
-        .map(o => (o.mavenLabel, o.label + cfg.suffix))
+        .map(o =>
+          (
+            o.mavenLabel,
+            o.label + cfg.suffix,
+            o.sourcesArtifactSha256.map { case _ => o.sourcesLabel }
+          )
+        )
     }
 
-    val (jars, depLabels) =
+    val (jars, depLabels, sourceJars) =
       if (jarsAndLabels.nonEmpty) {
-        jarsAndLabels.unzip
+        jarsAndLabels.unzip3
       } else {
         // Some resolutions produce no artifacts because they configure a classifier that
         // doesn't exist. In this case, we return the dependencies that were resolved
         // alongside this non-existent artifact.
         index.unevictedArtifacts.collect {
-          case ResolvedDependency(config, dependency, _, _) if config.targets.contains(target) =>
-            (dependency.mavenLabel, "_" + dependency.bazelLabel)
-        }.unzip
+          case ResolvedDependency(config, dependency, _, _, _) if config.targets.contains(target) =>
+            (dependency.mavenLabel, "_" + dependency.bazelLabel, None: Option[String])
+        }.unzip3
       }
+    val sourceJarOpt = sourceJars.flatten.headOption
 
     val overriddingTargets = for {
       config <- targetConfigs
@@ -82,14 +107,27 @@ object ArtifactOutput {
     } yield dependency
 
     val allLabels = (overriddingTargets ++ depLabels).distinct
-    TargetOutput(
-      kind = "scala_import",
-      "name" -> Docs.literal(name),
-      "jars" -> Docs.array(jars: _*),
-      "deps" -> Docs.array(allLabels: _*),
-      "exports" -> Docs.array(allLabels: _*),
-      "visibility" -> Docs.array("//visibility:public")
-    ).toDoc
+    sourceJarOpt match {
+      case Some(sourceJar) =>
+        TargetOutput(
+          kind = "scala_import",
+          "name" -> Docs.literal(name),
+          "jars" -> Docs.array(jars: _*),
+          "deps" -> Docs.array(allLabels: _*),
+          "exports" -> Docs.array(allLabels: _*),
+          "srcjar" -> Docs.literal(sourceJar),
+          "visibility" -> Docs.array("//visibility:public")
+        ).toDoc
+      case _ =>
+        TargetOutput(
+          kind = "scala_import",
+          "name" -> Docs.literal(name),
+          "jars" -> Docs.array(jars: _*),
+          "deps" -> Docs.array(allLabels: _*),
+          "exports" -> Docs.array(allLabels: _*),
+          "visibility" -> Docs.array("//visibility:public")
+        ).toDoc
+    }
   }
 
   private def buildGenruleAndImportDoc(

multiversion/src/main/scala/multiversion/outputs/DepsOutput.scala

@@ -19,7 +19,7 @@ final case class DepsOutput(
     val allTargets = index.thirdparty.dependencies2.flatMap(_.targets).toSet
 
     val httpFiles = Doc
-      .intercalate(Doc.line, artifacts.map(_.httpFile.toDoc))
+      .intercalate(Doc.line, artifacts.flatMap(_.httpFiles.map(_.toDoc)))
       .nested(4)
       .render(width)
     val thirdPartyImports = Doc

multiversion/src/main/scala/multiversion/outputs/ResolutionIndex.scala

@@ -5,11 +5,14 @@ import java.util.Locale
 import scala.collection.mutable
 
 import coursier.core.ArtifactSource
+import coursier.core.Classifier
 import coursier.core.Dependency
 import coursier.core.Module
 import coursier.core.Project
+import coursier.core.Publication
 import coursier.core.Resolution
 import coursier.core.Version
+import coursier.util.Artifact
 import coursier.version.VersionCompatibility
 import coursier.version.VersionInterval
 import coursier.version.VersionParse
@@ -23,22 +26,49 @@ import multiversion.resolvers.ResolvedDependency
 final case class ResolutionIndex(
     thirdparty: ThirdpartyConfig,
     resolutions: List[DependencyResolution],
-    roots: collection.Map[Dependency, collection.Set[Dependency]]
+    roots: collection.Map[Dependency, collection.Set[Dependency]],
+    resolveSources: Boolean,
 ) {
   import ResolutionIndex._
 
   // list of all artifacts including transitive JARs
   val rawArtifacts: List[ResolvedDependency] = for {
     r <- resolutions
     resolutionModule = r.dep.coursierModule(thirdparty.scala)
-    (d, p, a) <- r.res.dependencyArtifacts() if a.url.endsWith(".jar")
+    DependencyArtifact(d, p, a, sa) <- dependencyArtifactsWithSources(r.res)
+    if a.url.endsWith(".jar")
     dependency = ResolutionIndex.actualDependency(d, r.res.projectCache)
     artifact = r.dep.url match {
       case Some(url) if dependency.module == resolutionModule =>
         a.withUrl(url).withChecksumUrls(Map.empty)
       case _ => a
     }
-  } yield ResolvedDependency(r.dep, dependency, p, artifact)
+  } yield ResolvedDependency(r.dep, dependency, p, artifact, sa)
+
+  /**
+   * Optionally resolve the source JARs.
+   */
+  private def dependencyArtifactsWithSources(
+      res: Resolution
+  ): Seq[DependencyArtifact] = {
+    // grab source JARs and turn them into an immutable Map, so we can look them up
+    val sourceArtifacts =
+      if (resolveSources)
+        Map(res.dependencyArtifacts(Some(List(Classifier.sources))).map { case (d, p, a) =>
+          (d, a)
+        }: _*)
+      else Map.empty[Dependency, Artifact]
+    for {
+      (d, p, a) <- res.dependencyArtifacts()
+    } yield DependencyArtifact(d, p, a, sourceArtifacts.get(d))
+  }
+
+  private case class DependencyArtifact(
+      dependency: Dependency,
+      classfilePublication: Publication,
+      classfileArtifact: Artifact,
+      sourcesArtifact: Option[Artifact],
+  )
 
   val resolvedArtifacts: List[ResolvedDependency] = (rawArtifacts
     .groupBy(_.dependency.bazelLabel)
@@ -78,7 +108,7 @@ final case class ResolutionIndex(
   val allDependencies: collection.Map[Module, collection.Set[(Dependency, VersionConfig)]] = {
     val result =
       mutable.LinkedHashMap.empty[Module, mutable.LinkedHashSet[(Dependency, VersionConfig)]]
-    rawArtifacts.foreach { case ResolvedDependency(config, d, _, _) =>
+    rawArtifacts.foreach { case ResolvedDependency(config, d, _, _, _) =>
       val buf = result.getOrElseUpdate(
         d.module,
         mutable.LinkedHashSet.empty
@@ -150,7 +180,8 @@ final case class ResolutionIndex(
 object ResolutionIndex {
   def fromResolutions(
       thirdparty: ThirdpartyConfig,
-      resolutions: List[DependencyResolution]
+      resolutions: List[DependencyResolution],
+      resolveSources: Boolean,
   ): ResolutionIndex = {
     val roots =
       mutable.LinkedHashMap.empty[Dependency, mutable.LinkedHashSet[Dependency]]
@@ -166,7 +197,8 @@ object ResolutionIndex {
     ResolutionIndex(
       thirdparty,
       resolutions,
-      roots
+      roots,
+      resolveSources,
     )
   }
 

multiversion/src/main/scala/multiversion/resolvers/ResolvedDependency.scala

@@ -9,13 +9,15 @@ final case class ResolvedDependency(
     config: DependencyConfig,
     dependency: Dependency,
     publication: Publication,
-    artifact: Artifact
+    artifact: Artifact,
+    sourcesArtifact: Option[Artifact],
 ) {
   override def toString: String =
     s"""ResolvedDependency(
        |  config = $config,
        |  dependncy = $dependency,
        |  publication = $publication,
-       |  artifact = $artifact
+       |  artifact = $artifact,
+       |  sourcesArtifact = $sourcesArtifact,
        |)""".stripMargin
 }

tests/src/main/scala/tests/BaseSuite.scala

@@ -85,6 +85,8 @@ abstract class BaseSuite extends MopedSuite(MultiVersion.app) {
     List(s"kind(scala_import, allpaths($target, @maven//:all))")
   def allJars(target: String): List[String] =
     List(s"kind(file, allpaths($target, @maven//:all))")
+  def nonEmptySrcjar(target: String): List[String] =
+    List(s"""attr("srcjar", ".+", deps($target))""")
   def allScalaLibDeps(target: String): List[String] =
     List(s"kind(scala_library, deps($target))")
 

tests/src/test/scala/tests/commands/SourceJarTest.scala

@@ -0,0 +1,13 @@
+package tests.commands
+
+class SourceJarTest extends tests.BaseSuite with tests.ConfigSyntax {
+  checkDeps(
+    "scalatest",
+    """|  - dependency: org.scalatest:scalatest_2.12:3.1.2
+       |    targets: [scalatest]
+       |""".stripMargin,
+    queryArgs = nonEmptySrcjar("@maven//:scalatest"),
+    expectedQuery = """@maven//:scalatest
+         |""".stripMargin
+  )
+}