Skip to content

Commit

Permalink
Virtualize packageBin
Browse files Browse the repository at this point in the history 
This caches packageTask
  • Loading branch information
@eed3si9n
eed3si9n committed Dec 3, 2023
1 parent 11920d4 commit 5f558e2
Show file tree
Hide file tree
Showing 16 changed files with 442 additions and 195 deletions.
1 change: 1 addition & 0 deletions build.sbt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -680,6 +680,7 @@ lazy val actionsProj = (project in file("main-actions"))
stdTaskProj,
taskProj,
testingProj,
utilCacheResolver,
utilLogging,
utilRelation,
utilTracking,
Expand Down
6 changes: 3 additions & 3 deletions core-macros/src/main/scala/sbt/internal/util/appmacro/Cont.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,23 +95,23 @@ trait Cont:
given qctx.type = qctx
Expr
.summon[HashWriter[A]]
.getOrElse(sys.error(s"HashWriter[A] not found for ${TypeRepr.of[A].typeSymbol}"))
.getOrElse(sys.error(s"HashWriter[A] not found for ${TypeRepr.of[A].show}"))

def summonJsonFormat[A: Type]: Expr[JsonFormat[A]] =
import conv.qctx
import qctx.reflect.*
given qctx.type = qctx
Expr
.summon[JsonFormat[A]]
.getOrElse(sys.error(s"JsonFormat[A] not found for ${TypeRepr.of[A].typeSymbol}"))
.getOrElse(sys.error(s"JsonFormat[A] not found for ${TypeRepr.of[A].show}"))

def summonClassTag[A: Type]: Expr[ClassTag[A]] =
import conv.qctx
import qctx.reflect.*
given qctx.type = qctx
Expr
.summon[ClassTag[A]]
.getOrElse(sys.error(s"ClassTag[A] not found for ${TypeRepr.of[A].typeSymbol}"))
.getOrElse(sys.error(s"ClassTag[A] not found for ${TypeRepr.of[A].show}"))

/**
* Implementation of a macro that provides a direct syntax for applicative functors and monads.
Expand Down
212 changes: 159 additions & 53 deletions main-actions/src/main/scala/sbt/Pkg.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,17 +13,25 @@ import java.util.jar.{ Attributes, Manifest }
import scala.collection.JavaConverters._
import sbt.io.IO

import sjsonnew.JsonFormat
import sjsonnew.{
:*:,
Builder,
IsoLList,
JsonFormat,
LList,
LNil,
Unbuilder,
deserializationError,
flatUnionFormat4
}

import sbt.util.Logger

import sbt.util.{ CacheStoreFactory, FilesInfo, ModifiedFileInfo, PlainFileInfo }
import sbt.util.FileInfo.{ exists, lastModified }
import sbt.util.CacheImplicits._
import sbt.util.Tracked.{ inputChanged, outputChanged }
import scala.sys.process.Process

sealed trait PackageOption
import xsbti.{ FileConverter, HashedVirtualFileRef, VirtualFile, VirtualFileRef }

/**
* == Package ==
Expand All @@ -33,18 +41,17 @@ sealed trait PackageOption
* @see [[https://docs.oracle.com/javase/tutorial/deployment/jar/index.html]]
*/
object Pkg:
final case class JarManifest(m: Manifest) extends PackageOption {
assert(m != null)
}
final case class MainClass(mainClassName: String) extends PackageOption
final case class ManifestAttributes(attributes: (Attributes.Name, String)*) extends PackageOption
def ManifestAttributes(attributes: (String, String)*): ManifestAttributes = {
def JarManifest(m: Manifest) = PackageOption.JarManifest(m)
def MainClass(mainClassName: String) = PackageOption.MainClass(mainClassName)
def MainfestAttributes(attributes: (Attributes.Name, String)*) =
PackageOption.ManifestAttributes(attributes: _*)
def ManifestAttributes(attributes: (String, String)*) = {
val converted = for ((name, value) <- attributes) yield (new Attributes.Name(name), value)
new ManifestAttributes(converted: _*)
PackageOption.ManifestAttributes(converted: _*)
}
// 2010-01-01
private val default2010Timestamp: Long = 1262304000000L
final case class FixedTimestamp(value: Option[Long]) extends PackageOption
def FixedTimestamp(value: Option[Long]) = PackageOption.FixedTimestamp(value)
val keepTimestamps: Option[Long] = None
val fixed2010Timestamp: Option[Long] = Some(default2010Timestamp)
def gitCommitDateTimestamp: Option[Long] =
Expand Down Expand Up @@ -72,10 +79,9 @@ object Pkg:
.orElse(Some(default2010Timestamp))

def timeFromConfiguration(config: Configuration): Option[Long] =
(config.options.collect { case t: FixedTimestamp => t }).headOption match {
case Some(FixedTimestamp(value)) => value
case _ => defaultTimestamp
}
(config.options.collect { case t: PackageOption.FixedTimestamp => t }).headOption match
case Some(PackageOption.FixedTimestamp(value)) => value
case _ => defaultTimestamp

def mergeAttributes(a1: Attributes, a2: Attributes) = a1.asScala ++= a2.asScala
// merges `mergeManifest` into `manifest` (mutating `manifest` in the process)
Expand All @@ -98,18 +104,34 @@ object Pkg:
* @param options additional package information, e.g. jar manifest, main class or manifest attributes
*/
final class Configuration(
val sources: Seq[(File, String)],
val jar: File,
val sources: Seq[(HashedVirtualFileRef, String)],
val jar: VirtualFileRef,
val options: Seq[PackageOption]
)

object Configuration:
given IsoLList.Aux[
Configuration,
Vector[(HashedVirtualFileRef, String)] :*: VirtualFileRef :*: Seq[PackageOption] :*: LNil
] =
import sbt.util.CacheImplicits.given
import sbt.util.PathHashWriters.given
LList.iso(
(c: Configuration) =>
("sources", c.sources.toVector) :*: ("jar", c.jar) :*: ("options", c.options) :*: LNil,
(in: Vector[(HashedVirtualFileRef, String)] :*: VirtualFileRef :*: Seq[PackageOption] :*:
LNil) => Configuration(in.head, in.tail.head, in.tail.tail.head),
)
given JsonFormat[Configuration] = summon[JsonFormat[Configuration]]
end Configuration

/**
* @param conf the package configuration that should be build
* @param cacheStoreFactory used for jar caching. We try to avoid rebuilds as much as possible
* @param log feedback for the user
*/
def apply(conf: Configuration, cacheStoreFactory: CacheStoreFactory, log: Logger): Unit =
apply(conf, cacheStoreFactory, log, timeFromConfiguration(conf))
def apply(conf: Configuration, converter: FileConverter, log: Logger): VirtualFile =
apply(conf, converter, log, timeFromConfiguration(conf))

/**
* @param conf the package configuration that should be build
Expand All @@ -119,42 +141,31 @@ object Pkg:
*/
def apply(
conf: Configuration,
cacheStoreFactory: CacheStoreFactory,
converter: FileConverter,
log: Logger,
time: Option[Long]
): Unit = {
): VirtualFile =
val manifest = toManifest(conf, log)
val out = converter.toPath(conf.jar).toFile()
val sources = conf.sources.map { case (vf, path) =>
converter.toPath(vf).toFile() -> path
}
makeJar(sources, out, manifest, log, time)
converter.toVirtualFile(out.toPath())

def toManifest(conf: Configuration, log: Logger): Manifest =
val manifest = new Manifest
val main = manifest.getMainAttributes
for (option <- conf.options) {
option match {
case JarManifest(mergeManifest) => mergeManifests(manifest, mergeManifest); ()
case MainClass(mainClassName) => main.put(Attributes.Name.MAIN_CLASS, mainClassName); ()
case ManifestAttributes(attributes @ _*) => main.asScala ++= attributes; ()
case FixedTimestamp(value) => ()
for option <- conf.options do
option match
case PackageOption.JarManifest(mergeManifest) => mergeManifests(manifest, mergeManifest); ()
case PackageOption.MainClass(mainClassName) =>
main.put(Attributes.Name.MAIN_CLASS, mainClassName); ()
case PackageOption.ManifestAttributes(attributes @ _*) => main.asScala ++= attributes; ()
case PackageOption.FixedTimestamp(value) => ()
case _ => log.warn("Ignored unknown package option " + option)
}
}
setVersion(main)

type Inputs = (Seq[(File, String)], FilesInfo[ModifiedFileInfo], Manifest)
val cachedMakeJar = inputChanged(cacheStoreFactory make "inputs") {
(inChanged, inputs: Inputs) =>
import exists.format
val (sources, _, manifest) = inputs
outputChanged(cacheStoreFactory make "output") { (outChanged, jar: PlainFileInfo) =>
if (inChanged || outChanged) {
makeJar(sources, jar.file, manifest, log, time)
jar.file
()
} else log.debug("Jar uptodate: " + jar.file)
}
}

val inputFiles = conf.sources.map(_._1).toSet
val inputs = (conf.sources.distinct, lastModified(inputFiles), manifest)
cachedMakeJar(inputs)(() => exists(conf.jar))
()
}
manifest

/**
* updates the manifest version is there is none present.
Expand All @@ -172,7 +183,7 @@ object Pkg:
import Attributes.Name._
val attribKeys = Seq(SPECIFICATION_TITLE, SPECIFICATION_VERSION, SPECIFICATION_VENDOR)
val attribVals = Seq(name, version, orgName)
ManifestAttributes(attribKeys zip attribVals: _*)
PackageOption.ManifestAttributes(attribKeys.zip(attribVals): _*)
}
def addImplManifestAttributes(
name: String,
Expand All @@ -195,7 +206,7 @@ object Pkg:
IMPLEMENTATION_VENDOR_ID,
)
val attribVals = Seq(name, version, orgName, org)
ManifestAttributes((attribKeys zip attribVals) ++ {
PackageOption.ManifestAttributes(attribKeys.zip(attribVals) ++ {
homepage map (h => (IMPLEMENTATION_URL, h.toString))
}: _*)
}
Expand All @@ -221,7 +232,7 @@ object Pkg:
def sourcesDebugString(sources: Seq[(File, String)]): String =
"Input file mappings:\n\t" + (sources map { case (f, s) => s + "\n\t " + f } mkString ("\n\t"))

implicit def manifestFormat: JsonFormat[Manifest] = projectFormat[Manifest, Array[Byte]](
given manifestFormat: JsonFormat[Manifest] = projectFormat[Manifest, Array[Byte]](
m => {
val bos = new java.io.ByteArrayOutputStream()
m write bos
Expand All @@ -230,3 +241,98 @@ object Pkg:
bs => new Manifest(new java.io.ByteArrayInputStream(bs))
)
end Pkg

enum PackageOption:
case JarManifest(m: Manifest)
case MainClass(mainClassName: String)
case ManifestAttributes(attributes: (Attributes.Name, String)*)
case FixedTimestamp(value: Option[Long])

object PackageOption:
import Pkg.manifestFormat

private given jarManifestFormat: JsonFormat[PackageOption.JarManifest] =
new JsonFormat[PackageOption.JarManifest]:
override def read[J](
jsOpt: Option[J],
unbuilder: Unbuilder[J]
): PackageOption.JarManifest =
jsOpt match
case Some(js) =>
unbuilder.beginObject(js)
val m = unbuilder.readField[Manifest]("m")
unbuilder.endObject()
PackageOption.JarManifest(m)
case None => deserializationError("Expected JsObject but found None")
override def write[J](obj: PackageOption.JarManifest, builder: Builder[J]): Unit =
builder.beginObject()
builder.addField("m", obj.m)
builder.endObject()

private given mainClassFormat: JsonFormat[PackageOption.MainClass] =
new JsonFormat[PackageOption.MainClass]:
override def read[J](
jsOpt: Option[J],
unbuilder: Unbuilder[J]
): PackageOption.MainClass =
jsOpt match
case Some(js) =>
unbuilder.beginObject(js)
val mainClassName = unbuilder.readField[String]("mainClassName")
unbuilder.endObject()
PackageOption.MainClass(mainClassName)
case None => deserializationError("Expected JsObject but found None")
override def write[J](obj: PackageOption.MainClass, builder: Builder[J]): Unit =
builder.beginObject()
builder.addField("mainClassName", obj.mainClassName)
builder.endObject()

private given manifestAttributesFormat: JsonFormat[PackageOption.ManifestAttributes] =
new JsonFormat[PackageOption.ManifestAttributes]:
override def read[J](
jsOpt: Option[J],
unbuilder: Unbuilder[J]
): PackageOption.ManifestAttributes =
jsOpt match
case Some(js) =>
unbuilder.beginObject(js)
val attributes = unbuilder.readField[Vector[(String, String)]]("attributes")
unbuilder.endObject()
PackageOption.ManifestAttributes(attributes.map { case (k, v) =>
Attributes.Name(k) -> v
}: _*)
case None => deserializationError("Expected JsObject but found None")
override def write[J](obj: PackageOption.ManifestAttributes, builder: Builder[J]): Unit =
builder.beginObject()
builder.addField(
"attributes",
obj.attributes.toVector.map { case (k, v) => k.toString -> v }
)
builder.endObject()

private given fixedTimeStampFormat: JsonFormat[PackageOption.FixedTimestamp] =
new JsonFormat[PackageOption.FixedTimestamp]:
override def read[J](
jsOpt: Option[J],
unbuilder: Unbuilder[J]
): PackageOption.FixedTimestamp =
jsOpt match
case Some(js) =>
unbuilder.beginObject(js)
val value = unbuilder.readField[Option[Long]]("value")
unbuilder.endObject()
PackageOption.FixedTimestamp(value)
case None => deserializationError("Expected JsObject but found None")
override def write[J](obj: PackageOption.FixedTimestamp, builder: Builder[J]): Unit =
builder.beginObject()
builder.addField("value", obj.value)
builder.endObject()

given JsonFormat[PackageOption] = flatUnionFormat4[
PackageOption,
PackageOption.JarManifest,
PackageOption.MainClass,
PackageOption.ManifestAttributes,
PackageOption.FixedTimestamp,
]("type")
end PackageOption
9 changes: 8 additions & 1 deletion main-command/src/main/scala/sbt/BasicKeys.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
package sbt

import java.io.File

import java.nio.file.Path
import sbt.internal.inc.classpath.{ ClassLoaderCache => IncClassLoaderCache }
import sbt.internal.classpath.ClassLoaderCache
import sbt.internal.server.ServerHandler
Expand Down Expand Up @@ -113,6 +113,13 @@ object BasicKeys {
10000
)

val rootOutputDirectory =
AttributeKey[Path](
"rootOutputDirectory",
"Build-wide output directory",
10000
)

// Unlike other BasicKeys, this is not used directly as a setting key,
// and severLog / logLevel is used instead.
private[sbt] val serverLogLevel =
Expand Down
8 changes: 4 additions & 4 deletions main/src/main/contraband-scala/sbt/internal/remotecache/CompileRemoteCacheArtifact.scala
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit 5f558e2

Please sign in to comment.