From cff9993e9a4ff3647a0af647bf163e9e8235cbec Mon Sep 17 00:00:00 2001 From: Lorenzo Gabriele Date: Mon, 7 Oct 2024 14:49:56 +0200 Subject: [PATCH 01/47] [BSP] Remove duplicate sources entries in MillBuildRootModule (#3682) `scriptSources` is already part of `sources`. We were returning those files twice. Pull Request: https://github.com/com-lihaoyi/mill/pull/3682 --- bsp/worker/src/mill/bsp/worker/MillBuildServer.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bsp/worker/src/mill/bsp/worker/MillBuildServer.scala b/bsp/worker/src/mill/bsp/worker/MillBuildServer.scala index 0fdaf9de8c2..e00df71cd0a 100644 --- a/bsp/worker/src/mill/bsp/worker/MillBuildServer.scala +++ b/bsp/worker/src/mill/bsp/worker/MillBuildServer.scala @@ -230,8 +230,7 @@ private class MillBuildServer( tasks = { case module: MillBuildRootModule => Task.Anon { - module.scriptSources().map(p => sourceItem(p.path, false)) ++ - module.sources().map(p => sourceItem(p.path, false)) ++ + module.sources().map(p => sourceItem(p.path, false)) ++ module.generatedSources().map(p => sourceItem(p.path, true)) } case module: JavaModule => From ebee7a08c38a1b7d110cedb0144e849c1b34636d Mon Sep 17 00:00:00 2001 From: Nikita Ogorodnikov <4046447+0xnm@users.noreply.github.com> Date: Mon, 7 Oct 2024 18:41:51 +0200 Subject: [PATCH 02/47] Add very basic Kotlin/JS support: ability to compile the binary (#3678) Another part for #3611. This PR adds a very basic foundation for the Kotlin/JS support. The code provided allows to: * Build code with `org.jetbrains` (most likely) dependencies only * Run it with Node (no browser support) * Execute tests (using `kotlinx-test` so far), but without any test results collection/tests selector, etc. However, I think that full Kotlin/JS support **will take a lot of time**, because even if there is a Scala.JS counterpart available, the way these technologies work is very different (distribution, format, etc.) Issues I've encountered: * Kotlin 2+ is not supported, because with Kotlin 2+ jars are not published anymore and there is only `klib` file available (see https://repo1.maven.org/maven2/org/jetbrains/kotlin/kotlin-stdlib-js/2.0.0/). When Gradle is used, it is able to fetch it using attributes declared in `.module` file, but Coursier is not able to recognize and fetch it. * Kotlin versions below 1.8.20 are not supported, because of the different set of compiler arguments. With the certain effort it is possible to go further in supporting older versions, but I'm not sure if it is needed: since Mill is somewhat experimental, probably there is no need for the users to use old Kotlin versions. * Even if some Kotlin/JS library has `jar` file published with Kotlin/JS library inside, it may be rejected by the compiler. For example, this https://repo1.maven.org/maven2/org/jetbrains/kotlinx/kotlinx-html-js/0.8.0/ has the necessary `jar` file with `.meta.js` / `.kjsm` files inside, but it is rejected by the compiler of Kotlin 1.8/1.9. Here https://repo1.maven.org/maven2/org/jetbrains/kotlin/kotlin-stdlib-js/1.9.24/, for example, nothing is rejected, so I suppose there is an issue in the ABI/metadata version. Not sure how it can be solved (maybe by relying only on `klib`? But `klib` cannot be fetched by Coursier). * Gradle Kotlin plugin is utilizing NPM dependencies to generate `package.json` and add the necessary JS test frameworks/runners there if executed in Node environment, or even webpack for Browser environment. This is also a big chunk of work to be done. For now I've added only test binary execution, but there is no test results collection / test selector. * Kotest cannot be used, because with version 5 only `klib` is published, and `jar` of version 4 is not compatible with the 1.8/1.9 IR compiler. * ~~Kotlin/JS IR compiler has different modes: it can output either IR/Klib or can produce final JS (basically IR+linking). Kotlin Gradle plugin is using 2 passes: to generate IR and then produce final JS. I guess it is done for the better performance / better incremental support, but I, for the initial drop, rely on a single pass (IR+linking in a single compiler invocation) => **need to make `compile` task to produce only IR code and add kind of `link` task to produce executable in the future.**~~ => addressed in the 2nd commit. --------- Co-authored-by: 0xnm <0xnm@users.noreply.github.com> Co-authored-by: Li Haoyi --- .../ROOT/pages/kotlinlib/web-examples.adoc | 5 + .../kotlinlib/web/3-hello-kotlinjs/build.mill | 43 ++ .../web/3-hello-kotlinjs/foo/src/foo/Hello.kt | 11 + .../foo/test/src/foo/HelloTests.kt | 13 + .../src/mill/kotlinlib/KotlinModule.scala | 16 +- .../mill/kotlinlib/js/KotlinJSModule.scala | 512 ++++++++++++++++++ .../kotlin-js/bar/src/bar/Provider.kt | 3 + .../resources/kotlin-js/foo/src/foo/Hello.kt | 7 + .../kotlin-js/foo/test/src/foo/HelloTests.kt | 19 + .../kotlinlib/js/KotlinJSCompileTests.scala | 60 ++ .../js/KotlinJSKotlinVersionsTests.scala | 49 ++ .../mill/kotlinlib/js/KotlinJSLinkTests.scala | 69 +++ .../kotlinlib/js/KotlinJSNodeRunTests.scala | 163 ++++++ .../js/KotlinJSTestModuleTests.scala | 47 ++ .../worker/impl/KotlinWorkerImpl.scala | 14 +- .../kotlinlib/worker/api/KotlinWorker.scala | 8 +- 16 files changed, 1019 insertions(+), 20 deletions(-) create mode 100644 example/kotlinlib/web/3-hello-kotlinjs/build.mill create mode 100644 example/kotlinlib/web/3-hello-kotlinjs/foo/src/foo/Hello.kt create mode 100644 example/kotlinlib/web/3-hello-kotlinjs/foo/test/src/foo/HelloTests.kt create mode 100644 kotlinlib/src/mill/kotlinlib/js/KotlinJSModule.scala create mode 100644 kotlinlib/test/resources/kotlin-js/bar/src/bar/Provider.kt create mode 100644 kotlinlib/test/resources/kotlin-js/foo/src/foo/Hello.kt create mode 100644 kotlinlib/test/resources/kotlin-js/foo/test/src/foo/HelloTests.kt create mode 100644 kotlinlib/test/src/mill/kotlinlib/js/KotlinJSCompileTests.scala create mode 100644 kotlinlib/test/src/mill/kotlinlib/js/KotlinJSKotlinVersionsTests.scala create mode 100644 kotlinlib/test/src/mill/kotlinlib/js/KotlinJSLinkTests.scala create mode 100644 kotlinlib/test/src/mill/kotlinlib/js/KotlinJSNodeRunTests.scala create mode 100644 kotlinlib/test/src/mill/kotlinlib/js/KotlinJSTestModuleTests.scala diff --git a/docs/modules/ROOT/pages/kotlinlib/web-examples.adoc b/docs/modules/ROOT/pages/kotlinlib/web-examples.adoc index 9cc89e14164..6c79de38993 100644 --- a/docs/modules/ROOT/pages/kotlinlib/web-examples.adoc +++ b/docs/modules/ROOT/pages/kotlinlib/web-examples.adoc @@ -12,7 +12,12 @@ It covers setting up a basic backend server with a variety of server frameworks == Ktor Hello World App include::partial$example/kotlinlib/web/1-hello-ktor.adoc[] + == Ktor TodoMvc App include::partial$example/kotlinlib/web/2-todo-ktor.adoc[] +== (Work In Progress) Simple KotlinJS Module + +include::partial$example/kotlinlib/web/3-hello-kotlinjs.adoc[] + diff --git a/example/kotlinlib/web/3-hello-kotlinjs/build.mill b/example/kotlinlib/web/3-hello-kotlinjs/build.mill new file mode 100644 index 00000000000..950fec1eb2a --- /dev/null +++ b/example/kotlinlib/web/3-hello-kotlinjs/build.mill @@ -0,0 +1,43 @@ +// KotlinJS support on Mill is still Work In Progress (WIP). As of time of writing it +// does not support third-party dependencies, Kotlin 2.x with KMP KLIB files, Node.js/Webpack +// test runners and reporting, etc. +// +// The example below demonstrates only the minimal compilation, running, and testing of a single KotlinJS +// module. For more details in fully developing KotlinJS support, see the following ticket: +// +// * https://github.com/com-lihaoyi/mill/issues/3611 + +package build +import mill._, kotlinlib._, kotlinlib.js._ + +object foo extends KotlinJSModule { + def moduleKind = ModuleKind.ESModule + def kotlinVersion = "1.9.25" + def kotlinJSRunTarget = Some(RunTarget.Node) + object test extends KotlinJSModule with KotlinJSKotlinXTests +} + + +/** Usage + +> mill foo.run +Compiling 1 Kotlin sources to .../out/foo/compile.dest/classes... +Hello, world +stringifiedJsObject: ["hello","world","!"] + +> mill foo.test # Test is incorrect, `foo.test`` fails +Compiling 1 Kotlin sources to .../out/foo/test/compile.dest/classes... +Linking IR to .../out/foo/test/linkBinary.dest/binaries +produce executable: .../out/foo/test/linkBinary.dest/binaries +... +error: AssertionError: Expected , actual . + +> cat out/foo/test/linkBinary.dest/binaries/test.js # Generated javascript on disk +...assertEquals_0(getString(), 'Not hello, world');... +... + +> sed -i.bak 's/Not hello, world/Hello, world/g' foo/test/src/foo/HelloTests.kt + +> mill foo.test # passes after fixing test + +*/ diff --git a/example/kotlinlib/web/3-hello-kotlinjs/foo/src/foo/Hello.kt b/example/kotlinlib/web/3-hello-kotlinjs/foo/src/foo/Hello.kt new file mode 100644 index 00000000000..09f3ccd16af --- /dev/null +++ b/example/kotlinlib/web/3-hello-kotlinjs/foo/src/foo/Hello.kt @@ -0,0 +1,11 @@ +package foo + +fun getString() = "Hello, world" + +fun main() { + println(getString()) + + val parsedJsonStr: dynamic = JSON.parse("""{"helloworld": ["hello", "world", "!"]}""") + val stringifiedJsObject = JSON.stringify(parsedJsonStr.helloworld) + println("stringifiedJsObject: " + stringifiedJsObject) +} diff --git a/example/kotlinlib/web/3-hello-kotlinjs/foo/test/src/foo/HelloTests.kt b/example/kotlinlib/web/3-hello-kotlinjs/foo/test/src/foo/HelloTests.kt new file mode 100644 index 00000000000..7526f739947 --- /dev/null +++ b/example/kotlinlib/web/3-hello-kotlinjs/foo/test/src/foo/HelloTests.kt @@ -0,0 +1,13 @@ +package foo + +import kotlin.test.Test +import kotlin.test.assertEquals + +class HelloTests { + + @Test + fun failure() { + assertEquals(getString(), "Not hello, world") + } +} + diff --git a/kotlinlib/src/mill/kotlinlib/KotlinModule.scala b/kotlinlib/src/mill/kotlinlib/KotlinModule.scala index 81e74148746..4afb1c81a12 100644 --- a/kotlinlib/src/mill/kotlinlib/KotlinModule.scala +++ b/kotlinlib/src/mill/kotlinlib/KotlinModule.scala @@ -7,7 +7,7 @@ package kotlinlib import mill.api.{Loose, PathRef, Result} import mill.define.{Command, ModuleRef, Task} -import mill.kotlinlib.worker.api.KotlinWorker +import mill.kotlinlib.worker.api.{KotlinWorker, KotlinWorkerTarget} import mill.scalalib.api.{CompilationResult, ZincWorkerApi} import mill.scalalib.{JavaModule, Lib, ZincWorkerModule} import mill.util.Jvm @@ -92,11 +92,6 @@ trait KotlinModule extends JavaModule { outer => */ def kotlinCompilerIvyDeps: T[Agg[Dep]] = Task { Agg(ivy"org.jetbrains.kotlin:kotlin-compiler:${kotlinCompilerVersion()}") ++ -// ( -// if (Seq("1.0.", "1.1.", "1.2").exists(prefix => kotlinVersion().startsWith(prefix))) -// Agg(ivy"org.jetbrains.kotlin:kotlin-runtime:${kotlinCompilerVersion()}") -// else Seq() -// ) ++ ( if ( !Seq("1.0.", "1.1.", "1.2.0", "1.2.1", "1.2.2", "1.2.3", "1.2.4").exists(prefix => @@ -106,15 +101,8 @@ trait KotlinModule extends JavaModule { outer => Agg(ivy"org.jetbrains.kotlin:kotlin-scripting-compiler:${kotlinCompilerVersion()}") else Seq() ) -// ivy"org.jetbrains.kotlin:kotlin-scripting-compiler-impl:${kotlinCompilerVersion()}", -// ivy"org.jetbrains.kotlin:kotlin-scripting-common:${kotlinCompilerVersion()}", } -// @Deprecated("Use kotlinWorkerTask instead, as this does not need to be cached as Worker") -// def kotlinWorker: Worker[KotlinWorker] = Task.Worker { -// kotlinWorkerTask() -// } - def kotlinWorkerTask: Task[KotlinWorker] = Task.Anon { kotlinWorkerRef().kotlinWorkerManager().get(kotlinCompilerClasspath()) } @@ -264,7 +252,7 @@ trait KotlinModule extends JavaModule { outer => (kotlinSourceFiles ++ javaSourceFiles).map(_.toIO.getAbsolutePath()) ).flatten - val workerResult = kotlinWorkerTask().compile(compilerArgs: _*) + val workerResult = kotlinWorkerTask().compile(KotlinWorkerTarget.Jvm, compilerArgs: _*) val analysisFile = dest / "kotlin.analysis.dummy" os.write(target = analysisFile, data = "", createFolders = true) diff --git a/kotlinlib/src/mill/kotlinlib/js/KotlinJSModule.scala b/kotlinlib/src/mill/kotlinlib/js/KotlinJSModule.scala new file mode 100644 index 00000000000..3540c099642 --- /dev/null +++ b/kotlinlib/src/mill/kotlinlib/js/KotlinJSModule.scala @@ -0,0 +1,512 @@ +package mill.kotlinlib.js + +import mainargs.arg +import mill.api.{PathRef, Result} +import mill.define.{Command, Segment, Task} +import mill.kotlinlib.worker.api.{KotlinWorker, KotlinWorkerTarget} +import mill.kotlinlib.{Dep, DepSyntax, KotlinModule} +import mill.scalalib.Lib +import mill.scalalib.api.CompilationResult +import mill.testrunner.TestResult +import mill.util.Jvm +import mill.{Agg, Args, T} +import upickle.default.{macroRW, ReadWriter => RW} + +import java.io.File +import java.util.zip.ZipFile + +/** + * This module is very experimental. Don't use it, it is still under the development, APIs can change. + */ +trait KotlinJSModule extends KotlinModule { outer => + + // region Kotlin/JS configuration + + /** The kind of JS module generated by the compiler */ + def moduleKind: T[ModuleKind] = ModuleKind.PlainModule + + /** Call main function upon execution. */ + def callMain: T[Boolean] = true + + /** Binary type (if any) to produce. If [[BinaryKind.Executable]] is selected, then .js file(s) will be produced. */ + def kotlinJSBinaryKind: T[Option[BinaryKind]] = Some(BinaryKind.Executable) + + /** Whether to emit a source map. */ + def kotlinJSSourceMap: T[Boolean] = true + + /** Whether to embed sources into source map. */ + def kotlinJSSourceMapEmbedSources: T[SourceMapEmbedSourcesKind] = SourceMapEmbedSourcesKind.Never + + /** ES target to use. List of the supported ones depends on the Kotlin version. If not provided, default is used. */ + def kotlinJSESTarget: T[Option[String]] = None + + /** + * Add variable and function names that you declared in Kotlin code into the source map. See + * [[https://kotlinlang.org/docs/compiler-reference.html#source-map-names-policy-simple-names-fully-qualified-names-no Kotlin docs]] for more details + */ + def kotlinJSSourceMapNamesPolicy: T[SourceMapNamesPolicy] = SourceMapNamesPolicy.No + + /** Split generated .js per-module. Effective only if [[BinaryKind.Executable]] is selected. */ + def splitPerModule: T[Boolean] = true + + /** Run target for the executable (if [[BinaryKind.Executable]] is set). */ + def kotlinJSRunTarget: T[Option[RunTarget]] = None + + // endregion + + // region parent overrides + + override def allSourceFiles: T[Seq[PathRef]] = Task { + Lib.findSourceFiles(allSources(), Seq("kt")).map(PathRef(_)) + } + + override def mandatoryIvyDeps: T[Agg[Dep]] = Task { + Agg( + ivy"org.jetbrains.kotlin:kotlin-stdlib-js:${kotlinVersion()}" + ) + } + + override def transitiveCompileClasspath: T[Agg[PathRef]] = Task { + T.traverse(transitiveModuleCompileModuleDeps)(m => + Task.Anon { + val transitiveModuleArtifactPath = + (if (m.isInstanceOf[KotlinJSModule]) { + m.asInstanceOf[KotlinJSModule].createKlib(T.dest, m.compile().classes) + } else m.compile().classes) + m.localCompileClasspath() ++ Agg(transitiveModuleArtifactPath) + } + )().flatten + } + + /** + * Compiles all the sources to the IR representation. + */ + override def compile: T[CompilationResult] = Task { + kotlinJsCompile( + outputMode = OutputMode.KlibDir, + irClasspath = None, + allKotlinSourceFiles = allKotlinSourceFiles(), + librariesClasspath = compileClasspath(), + callMain = callMain(), + moduleKind = moduleKind(), + produceSourceMaps = kotlinJSSourceMap(), + sourceMapEmbedSourcesKind = kotlinJSSourceMapEmbedSources(), + sourceMapNamesPolicy = kotlinJSSourceMapNamesPolicy(), + splitPerModule = splitPerModule(), + esTarget = kotlinJSESTarget(), + kotlinVersion = kotlinVersion(), + destinationRoot = T.dest, + extraKotlinArgs = kotlincOptions(), + worker = kotlinWorkerTask() + ) + } + + override def runLocal(args: Task[Args] = Task.Anon(Args())): Command[Unit] = + Task.Command { run(args)() } + + override def run(args: Task[Args] = Task.Anon(Args())): Command[Unit] = Task.Command { + val binaryKind = kotlinJSBinaryKind() + if (binaryKind.isEmpty || binaryKind.get != BinaryKind.Executable) { + T.log.error("Run action is only allowed for the executable binary") + } + + val moduleKind = this.moduleKind() + + val linkResult = linkBinary().classes + if ( + moduleKind == ModuleKind.NoModule + && linkResult.path.toIO.listFiles().count(_.getName.endsWith(".js")) > 1 + ) { + T.log.info("No module type is selected for the executable, but multiple .js files found in the output folder." + + " This will probably lead to the dependency resolution failure.") + } + + kotlinJSRunTarget() match { + case Some(RunTarget.Node) => Jvm.runSubprocess( + commandArgs = Seq( + "node", + (linkResult.path / s"${moduleName()}.${moduleKind.extension}").toIO.getAbsolutePath + ) ++ args().value, + envArgs = T.env, + workingDir = T.dest + ) + case Some(x) => + T.log.error(s"Run target $x is not supported") + case None => + throw new IllegalArgumentException("Executable binary should have a run target selected.") + } + + } + + override def runMainLocal( + @arg(positional = true) mainClass: String, + args: String* + ): Command[Unit] = Task.Command[Unit] { + mill.api.Result.Failure("runMain is not supported in Kotlin/JS.") + } + + override def runMain(@arg(positional = true) mainClass: String, args: String*): Command[Unit] = + Task.Command[Unit] { + mill.api.Result.Failure("runMain is not supported in Kotlin/JS.") + } + + /** + * The actual Kotlin compile task (used by [[compile]] and [[kotlincHelp]]). + */ + protected override def kotlinCompileTask( + extraKotlinArgs: Seq[String] = Seq.empty[String] + ): Task[CompilationResult] = Task.Anon { + kotlinJsCompile( + outputMode = OutputMode.KlibDir, + allKotlinSourceFiles = allKotlinSourceFiles(), + irClasspath = None, + librariesClasspath = compileClasspath(), + callMain = callMain(), + moduleKind = moduleKind(), + produceSourceMaps = kotlinJSSourceMap(), + sourceMapEmbedSourcesKind = kotlinJSSourceMapEmbedSources(), + sourceMapNamesPolicy = kotlinJSSourceMapNamesPolicy(), + splitPerModule = splitPerModule(), + esTarget = kotlinJSESTarget(), + kotlinVersion = kotlinVersion(), + destinationRoot = T.dest, + extraKotlinArgs = kotlincOptions() ++ extraKotlinArgs, + worker = kotlinWorkerTask() + ) + } + + /** + * Creates final executable. + */ + def linkBinary: T[CompilationResult] = Task { + kotlinJsCompile( + outputMode = binaryKindToOutputMode(kotlinJSBinaryKind()), + irClasspath = Some(compile().classes), + allKotlinSourceFiles = Seq.empty, + librariesClasspath = compileClasspath(), + callMain = callMain(), + moduleKind = moduleKind(), + produceSourceMaps = kotlinJSSourceMap(), + sourceMapEmbedSourcesKind = kotlinJSSourceMapEmbedSources(), + sourceMapNamesPolicy = kotlinJSSourceMapNamesPolicy(), + splitPerModule = splitPerModule(), + esTarget = kotlinJSESTarget(), + kotlinVersion = kotlinVersion(), + destinationRoot = T.dest, + extraKotlinArgs = kotlincOptions(), + worker = kotlinWorkerTask() + ) + } + + // endregion + + // region private + + private def createKlib(destFolder: os.Path, irPathRef: PathRef): PathRef = { + val outputPath = destFolder / s"${moduleName()}.klib" + Jvm.createJar( + outputPath, + Agg(irPathRef.path), + mill.api.JarManifest.MillDefault, + fileFilter = (_, _) => true + ) + PathRef(outputPath) + } + + private[kotlinlib] def kotlinJsCompile( + outputMode: OutputMode, + allKotlinSourceFiles: Seq[PathRef], + irClasspath: Option[PathRef], + librariesClasspath: Agg[PathRef], + callMain: Boolean, + moduleKind: ModuleKind, + produceSourceMaps: Boolean, + sourceMapEmbedSourcesKind: SourceMapEmbedSourcesKind, + sourceMapNamesPolicy: SourceMapNamesPolicy, + splitPerModule: Boolean, + esTarget: Option[String], + kotlinVersion: String, + destinationRoot: os.Path, + extraKotlinArgs: Seq[String], + worker: KotlinWorker + )(implicit ctx: mill.api.Ctx): Result[CompilationResult] = { + val versionAllowed = kotlinVersion.split("\\.").map(_.toInt) match { + case Array(1, 8, z) => z >= 20 + case Array(1, y, _) => y >= 9 + case Array(2, _, _) => false + case _ => false + } + if (!versionAllowed) { + // have to put this restriction, because for older versions some compiler options either didn't exist or + // had different names. It is possible to go to the lower version supported with a certain effort. + ctx.log.error("Minimum supported Kotlin version for JS target is 1.8.20, maximum is 1.9.25") + return Result.Aborted + } + + // compiler options references: + // * https://kotlinlang.org/docs/compiler-reference.html#kotlin-js-compiler-options + // * https://github.com/JetBrains/kotlin/blob/v1.9.25/compiler/cli/cli-common/src/org/jetbrains/kotlin/cli/common/arguments/K2JSCompilerArguments.kt + + val inputFiles = irClasspath match { + case Some(x) => Seq(s"-Xinclude=${x.path.toIO.getAbsolutePath}") + case None => allKotlinSourceFiles.map(_.path.toIO.getAbsolutePath) + } + + // TODO: Cannot support Kotlin 2+, because it doesn't publish .jar anymore, but .klib files only. Coursier is not + // able to work with that (unlike Gradle, which can leverage .module metadata). + // https://repo1.maven.org/maven2/org/jetbrains/kotlin/kotlin-stdlib-js/2.0.20/ + val librariesCp = librariesClasspath.map(_.path) + .filter(os.exists) + .filter(isKotlinJsLibrary) + + val innerCompilerArgs = Seq.newBuilder[String] + // classpath + innerCompilerArgs ++= Seq("-libraries", librariesCp.iterator.mkString(File.pathSeparator)) + innerCompilerArgs ++= Seq("-main", if (callMain) "call" else "noCall") + innerCompilerArgs += "-meta-info" + if (moduleKind != ModuleKind.NoModule) { + innerCompilerArgs ++= Seq( + "-module-kind", + moduleKind match { + case ModuleKind.AMDModule => "amd" + case ModuleKind.UMDModule => "umd" + case ModuleKind.PlainModule => "plain" + case ModuleKind.ESModule => "es" + case ModuleKind.CommonJSModule => "commonjs" + } + ) + } + // what is the better way to find a module simple name, without root path? + innerCompilerArgs ++= Seq("-ir-output-name", moduleName()) + if (produceSourceMaps) { + innerCompilerArgs += "-source-map" + innerCompilerArgs ++= Seq( + "-source-map-embed-sources", + sourceMapEmbedSourcesKind match { + case SourceMapEmbedSourcesKind.Always => "always" + case SourceMapEmbedSourcesKind.Never => "never" + case SourceMapEmbedSourcesKind.Inlining => "inlining" + } + ) + innerCompilerArgs ++= Seq( + "-source-map-names-policy", + sourceMapNamesPolicy match { + case SourceMapNamesPolicy.No => "no" + case SourceMapNamesPolicy.SimpleNames => "simple-names" + case SourceMapNamesPolicy.FullyQualifiedNames => "fully-qualified-names" + } + ) + } + innerCompilerArgs += "-Xir-only" + if (splitPerModule) { + innerCompilerArgs += s"-Xir-per-module" + innerCompilerArgs += s"-Xir-per-module-output-name=${fullModuleName()}" + } + val outputArgs = outputMode match { + case OutputMode.KlibFile => + Seq( + "-Xir-produce-klib-file", + "-ir-output-dir", + (destinationRoot / "libs").toIO.getAbsolutePath + ) + case OutputMode.KlibDir => + Seq( + "-Xir-produce-klib-dir", + "-ir-output-dir", + (destinationRoot / "classes").toIO.getAbsolutePath + ) + case OutputMode.Js => + Seq( + "-Xir-produce-js", + "-ir-output-dir", + (destinationRoot / "binaries").toIO.getAbsolutePath + ) + } + + innerCompilerArgs ++= outputArgs + innerCompilerArgs += s"-Xir-module-name=${moduleName()}" + innerCompilerArgs ++= (esTarget match { + case Some(x) => Seq("-target", x) + case None => Seq.empty + }) + + val compilerArgs: Seq[String] = Seq( + innerCompilerArgs.result(), + extraKotlinArgs, + // parameters + inputFiles + ).flatten + + val compileDestination = os.Path(outputArgs.last) + if (irClasspath.isEmpty) { + T.log.info( + s"Compiling ${allKotlinSourceFiles.size} Kotlin sources to $compileDestination ..." + ) + } else { + T.log.info(s"Linking IR to $compileDestination") + } + val workerResult = worker.compile(KotlinWorkerTarget.Js, compilerArgs: _*) + + val analysisFile = T.dest / "kotlin.analysis.dummy" + if (!os.exists(analysisFile)) { + os.write(target = analysisFile, data = "", createFolders = true) + } + + val artifactLocation = outputMode match { + case OutputMode.KlibFile => compileDestination / s"${moduleName()}.klib" + case OutputMode.KlibDir => compileDestination + case OutputMode.Js => compileDestination + } + + workerResult match { + case Result.Success(_) => + CompilationResult(analysisFile, PathRef(artifactLocation)) + case Result.Failure(reason, _) => + Result.Failure(reason, Some(CompilationResult(analysisFile, PathRef(artifactLocation)))) + case e: Result.Exception => e + case Result.Aborted => Result.Aborted + case Result.Skipped => Result.Skipped + } + } + + private def binaryKindToOutputMode(binaryKind: Option[BinaryKind]): OutputMode = + binaryKind match { + // still produce IR classes, but they won't be yet linked + case None => OutputMode.KlibDir + case Some(BinaryKind.Library) => OutputMode.KlibFile + case Some(BinaryKind.Executable) => OutputMode.Js + } + + // these 2 exist to ignore values added to the display name in case of the cross-modules + // we already have cross-modules in the paths, so we don't need them here + private def moduleName() = millModuleSegments.value + .filter(_.isInstanceOf[Segment.Label]) + .map(_.asInstanceOf[Segment.Label]) + .last + .value + + private def fullModuleName() = millModuleSegments.value + .filter(_.isInstanceOf[Segment.Label]) + .map(_.asInstanceOf[Segment.Label].value) + .mkString("-") + + // **NOTE**: This logic may (and probably is) be incomplete + private def isKotlinJsLibrary(path: os.Path)(implicit ctx: mill.api.Ctx): Boolean = { + if (os.isDir(path)) { + true + } else if (path.ext == "klib") { + true + } else if (path.ext == "jar") { + try { + // TODO cache these lookups. May be a big performance penalty. + val zipFile = new ZipFile(path.toIO) + zipFile.stream() + .anyMatch(entry => entry.getName.endsWith(".meta.js") || entry.getName.endsWith(".kjsm")) + } catch { + case e: Throwable => + T.log.error(s"Couldn't open ${path.toIO.getAbsolutePath} as archive.\n${e.toString}") + false + } + } else { + T.log.debug(s"${path.toIO.getAbsolutePath} is not a Kotlin/JS library, ignoring it.") + false + } + } + + // endregion + + // region Tests module + + trait KotlinJSTests extends KotlinTests with KotlinJSModule { + + override def testFramework = "" + + override def kotlinJSBinaryKind: T[Option[BinaryKind]] = Some(BinaryKind.Executable) + + override def splitPerModule = false + + override def testLocal(args: String*): Command[(String, Seq[TestResult])] = + Task.Command { + this.test(args: _*)() + } + + override protected def testTask( + args: Task[Seq[String]], + globSelectors: Task[Seq[String]] + ): Task[(String, Seq[TestResult])] = Task.Anon { + // This is a terrible hack, but it works + run()() + ("", Seq.empty[TestResult]) + } + } + + trait KotlinJSKotlinXTests extends KotlinJSTests { + override def ivyDeps = Agg( + ivy"org.jetbrains.kotlin:kotlin-test-js:${kotlinVersion()}" + ) + override def kotlinJSRunTarget: T[Option[RunTarget]] = Some(RunTarget.Node) + } + + // endregion +} + +sealed trait ModuleKind { def extension: String } + +object ModuleKind { + object NoModule extends ModuleKind { val extension = "js" } + implicit val rwNoModule: RW[NoModule.type] = macroRW + object UMDModule extends ModuleKind { val extension = "js" } + implicit val rwUMDModule: RW[UMDModule.type] = macroRW + object CommonJSModule extends ModuleKind { val extension = "js" } + implicit val rwCommonJSModule: RW[CommonJSModule.type] = macroRW + object AMDModule extends ModuleKind { val extension = "js" } + implicit val rwAMDModule: RW[AMDModule.type] = macroRW + object ESModule extends ModuleKind { val extension = "mjs" } + implicit val rwESModule: RW[ESModule.type] = macroRW + object PlainModule extends ModuleKind { val extension = "js" } + implicit val rwPlainModule: RW[PlainModule.type] = macroRW +} + +sealed trait SourceMapEmbedSourcesKind +object SourceMapEmbedSourcesKind { + object Always extends SourceMapEmbedSourcesKind + implicit val rwAlways: RW[Always.type] = macroRW + object Never extends SourceMapEmbedSourcesKind + implicit val rwNever: RW[Never.type] = macroRW + object Inlining extends SourceMapEmbedSourcesKind + implicit val rwInlining: RW[Inlining.type] = macroRW +} + +sealed trait SourceMapNamesPolicy +object SourceMapNamesPolicy { + object SimpleNames extends SourceMapNamesPolicy + implicit val rwSimpleNames: RW[SimpleNames.type] = macroRW + object FullyQualifiedNames extends SourceMapNamesPolicy + implicit val rwFullyQualifiedNames: RW[FullyQualifiedNames.type] = macroRW + object No extends SourceMapNamesPolicy + implicit val rwNo: RW[No.type] = macroRW +} + +sealed trait BinaryKind +object BinaryKind { + object Library extends BinaryKind + implicit val rwLibrary: RW[Library.type] = macroRW + object Executable extends BinaryKind + implicit val rwExecutable: RW[Executable.type] = macroRW + implicit val rw: RW[BinaryKind] = macroRW +} + +sealed trait RunTarget +object RunTarget { + // TODO rely on the node version installed in the env or fetch a specific one? + object Node extends RunTarget + implicit val rwNode: RW[Node.type] = macroRW + implicit val rw: RW[RunTarget] = macroRW +} + +private[kotlinlib] sealed trait OutputMode +private[kotlinlib] object OutputMode { + object Js extends OutputMode + object KlibDir extends OutputMode + object KlibFile extends OutputMode +} diff --git a/kotlinlib/test/resources/kotlin-js/bar/src/bar/Provider.kt b/kotlinlib/test/resources/kotlin-js/bar/src/bar/Provider.kt new file mode 100644 index 00000000000..f1f9580e0d7 --- /dev/null +++ b/kotlinlib/test/resources/kotlin-js/bar/src/bar/Provider.kt @@ -0,0 +1,3 @@ +package bar + +fun getString() = "Hello, world" diff --git a/kotlinlib/test/resources/kotlin-js/foo/src/foo/Hello.kt b/kotlinlib/test/resources/kotlin-js/foo/src/foo/Hello.kt new file mode 100644 index 00000000000..b9193c50092 --- /dev/null +++ b/kotlinlib/test/resources/kotlin-js/foo/src/foo/Hello.kt @@ -0,0 +1,7 @@ +package foo + +import bar.getString + +fun main() { + println(getString()) +} diff --git a/kotlinlib/test/resources/kotlin-js/foo/test/src/foo/HelloTests.kt b/kotlinlib/test/resources/kotlin-js/foo/test/src/foo/HelloTests.kt new file mode 100644 index 00000000000..c55f18f2ec6 --- /dev/null +++ b/kotlinlib/test/resources/kotlin-js/foo/test/src/foo/HelloTests.kt @@ -0,0 +1,19 @@ +package foo + +import bar.getString +import kotlin.test.Test +import kotlin.test.assertEquals + +class HelloTests { + + @Test + fun success() { + assertEquals(getString(), "Hello, world") + } + + @Test + fun failure() { + assertEquals(getString(), "Not hello, world") + } +} + diff --git a/kotlinlib/test/src/mill/kotlinlib/js/KotlinJSCompileTests.scala b/kotlinlib/test/src/mill/kotlinlib/js/KotlinJSCompileTests.scala new file mode 100644 index 00000000000..1d54740d0cf --- /dev/null +++ b/kotlinlib/test/src/mill/kotlinlib/js/KotlinJSCompileTests.scala @@ -0,0 +1,60 @@ +package mill +package kotlinlib +package js + +import mill.testkit.{TestBaseModule, UnitTester} +import utest.{TestSuite, Tests, assert, test} + +object KotlinJSCompileTests extends TestSuite { + + private val kotlinVersion = "1.9.25" + + private val resourcePath = os.Path(sys.env("MILL_TEST_RESOURCE_DIR")) / "kotlin-js" + + object module extends TestBaseModule { + + object bar extends KotlinJSModule { + def kotlinVersion = KotlinJSCompileTests.kotlinVersion + } + + object foo extends KotlinJSModule { + override def kotlinVersion = KotlinJSCompileTests.kotlinVersion + override def moduleDeps = Seq(module.bar) + } + } + + private def testEval() = UnitTester(module, resourcePath) + + def tests: Tests = Tests { + test("compile") { + val eval = testEval() + + val Right(result) = eval.apply(module.foo.compile) + + val irDir = result.value.classes.path + assert( + os.isDir(irDir), + os.exists(irDir / "default" / "manifest"), + os.exists(irDir / "default" / "linkdata" / "package_foo"), + !os.walk(irDir).exists(_.ext == "klib") + ) + } + + test("failures") { + val eval = testEval() + + val compilationUnit = module.foo.millSourcePath / "src" / "foo" / "Hello.kt" + + val Right(_) = eval.apply(module.foo.compile) + + os.write.over(compilationUnit, os.read(compilationUnit) + "}") + + val Left(_) = eval.apply(module.foo.compile) + + os.write.over(compilationUnit, os.read(compilationUnit).dropRight(1)) + + val Right(_) = eval.apply(module.foo.compile) + } + } + +} diff --git a/kotlinlib/test/src/mill/kotlinlib/js/KotlinJSKotlinVersionsTests.scala b/kotlinlib/test/src/mill/kotlinlib/js/KotlinJSKotlinVersionsTests.scala new file mode 100644 index 00000000000..1fc6f381c4e --- /dev/null +++ b/kotlinlib/test/src/mill/kotlinlib/js/KotlinJSKotlinVersionsTests.scala @@ -0,0 +1,49 @@ +package mill +package kotlinlib +package js + +import mill.testkit.{TestBaseModule, UnitTester} +import mill.Cross +import utest.{TestSuite, Tests, test} + +object KotlinJSKotlinVersionsTests extends TestSuite { + + private val resourcePath = os.Path(sys.env("MILL_TEST_RESOURCE_DIR")) / "kotlin-js" + private val kotlinLowestVersion = "1.8.20" + // TODO: Cannot support Kotlin 2+, because it doesn't publish .jar anymore, but .klib files only. Coursier is not + // able to work with that (unlike Gradle, which can leverage .module metadata). + // https://repo1.maven.org/maven2/org/jetbrains/kotlin/kotlin-stdlib-js/2.0.20/ + private val kotlinHighestVersion = "1.9.25" + private val kotlinVersions = Seq(kotlinLowestVersion, kotlinHighestVersion) + + trait KotlinJSCrossModule extends KotlinJSModule with Cross.Module[String] { + def kotlinVersion = crossValue + } + + trait KotlinJSFooCrossModule extends KotlinJSCrossModule { + override def moduleDeps = Seq(module.bar(crossValue)) + } + + object module extends TestBaseModule { + + object bar extends Cross[KotlinJSCrossModule](kotlinVersions) + object foo extends Cross[KotlinJSFooCrossModule](kotlinVersions) + } + + private def testEval() = UnitTester(module, resourcePath) + + def tests: Tests = Tests { + test("compile with lowest Kotlin version") { + val eval = testEval() + + val Right(_) = eval.apply(module.foo(kotlinLowestVersion).compile) + } + + test("compile with highest Kotlin version") { + val eval = testEval() + + val Right(_) = eval.apply(module.foo(kotlinHighestVersion).compile) + } + } + +} diff --git a/kotlinlib/test/src/mill/kotlinlib/js/KotlinJSLinkTests.scala b/kotlinlib/test/src/mill/kotlinlib/js/KotlinJSLinkTests.scala new file mode 100644 index 00000000000..928a3fcb49e --- /dev/null +++ b/kotlinlib/test/src/mill/kotlinlib/js/KotlinJSLinkTests.scala @@ -0,0 +1,69 @@ +package mill.kotlinlib.js + +import mill.testkit.{TestBaseModule, UnitTester} +import mill.{Cross, T} +import utest.{TestSuite, Tests, assert, test} + +import scala.util.Random + +object KotlinJSLinkTests extends TestSuite { + + private val kotlinVersion = "1.9.25" + + private val resourcePath = os.Path(sys.env("MILL_TEST_RESOURCE_DIR")) / "kotlin-js" + + trait KotlinJSCrossModule extends KotlinJSModule with Cross.Module[Boolean] { + override def kotlinVersion = KotlinJSLinkTests.kotlinVersion + override def splitPerModule: T[Boolean] = crossValue + override def kotlinJSBinaryKind: T[Option[BinaryKind]] = Some(BinaryKind.Executable) + override def moduleDeps = Seq(module.bar) + } + + object module extends TestBaseModule { + + object bar extends KotlinJSModule { + def kotlinVersion = KotlinJSLinkTests.kotlinVersion + } + + object foo extends Cross[KotlinJSCrossModule](Seq(true, false)) + } + + private def testEval() = UnitTester(module, resourcePath) + + def tests: Tests = Tests { + test("link { per module }") { + val eval = testEval() + + val Right(result) = eval.apply(module.foo(true).linkBinary) + + val binariesDir = result.value.classes.path + assert( + os.isDir(binariesDir), + os.exists(binariesDir / "foo.js"), + os.exists(binariesDir / "foo.js.map"), + os.exists(binariesDir / "bar.js"), + os.exists(binariesDir / "bar.js.map"), + os.exists(binariesDir / "kotlin-kotlin-stdlib.js"), + os.exists(binariesDir / "kotlin-kotlin-stdlib.js.map") + ) + } + + test("link { fat }") { + val eval = testEval() + + val Right(result) = eval.apply(module.foo(false).linkBinary) + + val binariesDir = result.value.classes.path + assert( + os.isDir(binariesDir), + os.exists(binariesDir / "foo.js"), + os.exists(binariesDir / "foo.js.map"), + !os.exists(binariesDir / "bar.js"), + !os.exists(binariesDir / "bar.js.map"), + !os.exists(binariesDir / "kotlin-kotlin-stdlib.js"), + !os.exists(binariesDir / "kotlin-kotlin-stdlib.js.map") + ) + } + } + +} diff --git a/kotlinlib/test/src/mill/kotlinlib/js/KotlinJSNodeRunTests.scala b/kotlinlib/test/src/mill/kotlinlib/js/KotlinJSNodeRunTests.scala new file mode 100644 index 00000000000..dbc3adc31c2 --- /dev/null +++ b/kotlinlib/test/src/mill/kotlinlib/js/KotlinJSNodeRunTests.scala @@ -0,0 +1,163 @@ +package mill +package kotlinlib +package js + +import mill.eval.EvaluatorPaths +import mill.testkit.{TestBaseModule, UnitTester} +import utest.{TestSuite, Tests, test} + +object KotlinJSNodeRunTests extends TestSuite { + + private val resourcePath = os.Path(sys.env("MILL_TEST_RESOURCE_DIR")) / "kotlin-js" + private val kotlinVersion = "1.9.25" + private val expectedSuccessOutput = "Hello, world" + + object module extends TestBaseModule { + + private val matrix = for { + splits <- Seq(true, false) + modules <- Seq("no", "plain", "es", "amd", "commonjs", "umd") + } yield (splits, modules) + + trait KotlinJsModuleKindCross extends KotlinJSModule with Cross.Module2[Boolean, String] { + + def kotlinVersion = KotlinJSNodeRunTests.kotlinVersion + + override def moduleKind = crossValue2 match { + case "no" => ModuleKind.NoModule + case "plain" => ModuleKind.PlainModule + case "es" => ModuleKind.ESModule + case "amd" => ModuleKind.AMDModule + case "commonjs" => ModuleKind.CommonJSModule + case "umd" => ModuleKind.UMDModule + } + + override def moduleDeps = Seq(module.bar) + override def splitPerModule = crossValue + override def kotlinJSRunTarget = Some(RunTarget.Node) + } + + object bar extends KotlinJSModule { + def kotlinVersion = KotlinJSNodeRunTests.kotlinVersion + } + + object foo extends Cross[KotlinJsModuleKindCross](matrix) + } + + private def testEval() = UnitTester(module, resourcePath) + + def tests: Tests = Tests { + // region with split per module + + test("run { split per module / plain module }") { + val eval = testEval() + + // plain modules cannot handle the dependencies, so if there are multiple js files, it will fail + val Left(_) = eval.apply(module.foo(true, "plain").run()) + } + + test("run { split per module / es module }") { + val eval = testEval() + + val command = module.foo(true, "es").run() + val Right(_) = eval.apply(command) + + assertLogContains(eval, command, expectedSuccessOutput) + } + + test("run { split per module / amd module }") { + val eval = testEval() + + // amd modules have "define" method, it is not known by Node.js + val Left(_) = eval.apply(module.foo(true, "amd").run()) + } + + test("run { split per module / commonjs module }") { + val eval = testEval() + + val command = module.foo(true, "commonjs").run() + val Right(_) = eval.apply(command) + + assertLogContains(eval, command, expectedSuccessOutput) + } + + test("run { split per module / umd module }") { + val eval = testEval() + + val command = module.foo(true, "umd").run() + val Right(_) = eval.apply(command) + + assertLogContains(eval, command, expectedSuccessOutput) + } + + test("run { split per module / no module }") { + val eval = testEval() + + val Left(_) = eval.apply(module.foo(true, "no").run()) + } + + // endregion + + // region without split per module + + test("run { no split per module / plain module }") { + val eval = testEval() + + val command = module.foo(false, "plain").run() + val Right(_) = eval.apply(command) + + assertLogContains(eval, command, expectedSuccessOutput) + } + + test("run { no split per module / es module }") { + val eval = testEval() + + val command = module.foo(false, "es").run() + val Right(_) = eval.apply(command) + + assertLogContains(eval, command, expectedSuccessOutput) + } + + test("run { no split per module / amd module }") { + val eval = testEval() + + // amd modules have "define" method, it is not known by Node.js + val Left(_) = eval.apply(module.foo(false, "amd").run()) + } + + test("run { no split per module / commonjs module }") { + val eval = testEval() + + val command = module.foo(false, "commonjs").run() + val Right(_) = eval.apply(command) + + assertLogContains(eval, command, expectedSuccessOutput) + } + + test("run { no split per module / umd module }") { + val eval = testEval() + + val command = module.foo(false, "umd").run() + val Right(_) = eval.apply(command) + + assertLogContains(eval, command, expectedSuccessOutput) + } + + test("run { no split per module / no module }") { + val eval = testEval() + + val command = module.foo(false, "no").run() + val Right(_) = eval.apply(command) + + assertLogContains(eval, command, expectedSuccessOutput) + } + + // endregion + } + + private def assertLogContains(eval: UnitTester, command: Command[Unit], text: String): Unit = { + val log = EvaluatorPaths.resolveDestPaths(eval.outPath, command).log + assert(os.read(log).contains(text)) + } + +} diff --git a/kotlinlib/test/src/mill/kotlinlib/js/KotlinJSTestModuleTests.scala b/kotlinlib/test/src/mill/kotlinlib/js/KotlinJSTestModuleTests.scala new file mode 100644 index 00000000000..37e2dd138ad --- /dev/null +++ b/kotlinlib/test/src/mill/kotlinlib/js/KotlinJSTestModuleTests.scala @@ -0,0 +1,47 @@ +package mill +package kotlinlib +package js + +import mill.eval.EvaluatorPaths +import mill.testkit.{TestBaseModule, UnitTester} +import utest.{TestSuite, Tests, test} + +object KotlinJSTestModuleTests extends TestSuite { + + private val resourcePath = os.Path(sys.env("MILL_TEST_RESOURCE_DIR")) / "kotlin-js" + + private val kotlinVersion = "1.9.25" + + object module extends TestBaseModule { + + object bar extends KotlinJSModule { + def kotlinVersion = KotlinJSTestModuleTests.kotlinVersion + } + + object foo extends KotlinJSModule { + def kotlinVersion = KotlinJSTestModuleTests.kotlinVersion + override def moduleDeps = Seq(module.bar) + + object test extends KotlinJSModule with KotlinJSKotlinXTests + } + } + + private def testEval() = UnitTester(module, resourcePath) + + def tests: Tests = Tests { + + test("run tests") { + val eval = testEval() + + val command = module.foo.test.test() + val Left(_) = eval.apply(command) + + // temporary, because we are running run() task, it won't be test.log, but run.log + val log = EvaluatorPaths.resolveDestPaths(eval.outPath, command).log / ".." / "run.log" + assert( + os.read(log).contains("AssertionError: Expected , actual .") + ) + } + } + +} diff --git a/kotlinlib/worker/impl/src/mill/kotlinlib/worker/impl/KotlinWorkerImpl.scala b/kotlinlib/worker/impl/src/mill/kotlinlib/worker/impl/KotlinWorkerImpl.scala index 99fcb41f6e7..86804b8943b 100644 --- a/kotlinlib/worker/impl/src/mill/kotlinlib/worker/impl/KotlinWorkerImpl.scala +++ b/kotlinlib/worker/impl/src/mill/kotlinlib/worker/impl/KotlinWorkerImpl.scala @@ -5,18 +5,22 @@ package mill.kotlinlib.worker.impl import mill.api.{Ctx, Result} -import mill.kotlinlib.worker.api.KotlinWorker +import mill.kotlinlib.worker.api.{KotlinWorker, KotlinWorkerTarget} +import org.jetbrains.kotlin.cli.js.K2JsIrCompiler import org.jetbrains.kotlin.cli.jvm.K2JVMCompiler class KotlinWorkerImpl extends KotlinWorker { - def compile(args: String*)(implicit ctx: Ctx): Result[Unit] = { + def compile(target: KotlinWorkerTarget, args: String*)(implicit ctx: Ctx): Result[Unit] = { ctx.log.debug("Using kotlin compiler arguments: " + args.map(v => s"'${v}'").mkString(" ")) - val compiler = new K2JVMCompiler() + val compiler = target match { + case KotlinWorkerTarget.Jvm => new K2JVMCompiler() + case KotlinWorkerTarget.Js => new K2JsIrCompiler() + } val exitCode = compiler.exec(ctx.log.errorStream, args: _*) - if (exitCode.getCode() != 0) { - Result.Failure(s"Kotlin compiler failed with exit code ${exitCode.getCode()} (${exitCode})") + if (exitCode.getCode != 0) { + Result.Failure(s"Kotlin compiler failed with exit code ${exitCode.getCode} ($exitCode)") } else { Result.Success(()) } diff --git a/kotlinlib/worker/src/mill/kotlinlib/worker/api/KotlinWorker.scala b/kotlinlib/worker/src/mill/kotlinlib/worker/api/KotlinWorker.scala index 0c323d1e88b..2fa79895f1b 100644 --- a/kotlinlib/worker/src/mill/kotlinlib/worker/api/KotlinWorker.scala +++ b/kotlinlib/worker/src/mill/kotlinlib/worker/api/KotlinWorker.scala @@ -8,6 +8,12 @@ import mill.api.{Ctx, Result} trait KotlinWorker { - def compile(args: String*)(implicit ctx: Ctx): Result[Unit] + def compile(target: KotlinWorkerTarget, args: String*)(implicit ctx: Ctx): Result[Unit] } + +sealed class KotlinWorkerTarget +object KotlinWorkerTarget { + case object Jvm extends KotlinWorkerTarget + case object Js extends KotlinWorkerTarget +} From 04276232378fb4f36ba3a00fc1fb1c5d9c030166 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Tue, 8 Oct 2024 07:17:13 +0200 Subject: [PATCH 03/47] Bump OS-Lib to 0.11.0 (#3684) --- build.mill | 2 +- main/init/src/mill/init/InitModule.scala | 5 ++--- .../worker/src/mill/scalalib/worker/ZincWorkerImpl.scala | 6 +++--- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/build.mill b/build.mill index 412f6180546..0715d9dc3d6 100644 --- a/build.mill +++ b/build.mill @@ -149,7 +149,7 @@ object Deps { val junitInterface = ivy"com.github.sbt:junit-interface:0.13.3" val commonsIO = ivy"commons-io:commons-io:2.16.1" val log4j2Core = ivy"org.apache.logging.log4j:log4j-core:2.23.1" - val osLib = ivy"com.lihaoyi::os-lib:0.10.7" + val osLib = ivy"com.lihaoyi::os-lib:0.11.0" val pprint = ivy"com.lihaoyi::pprint:0.9.0" val mainargs = ivy"com.lihaoyi::mainargs:0.7.4" val millModuledefsVersion = "0.11.0" diff --git a/main/init/src/mill/init/InitModule.scala b/main/init/src/mill/init/InitModule.scala index 88cec0d5d6c..76b05442767 100644 --- a/main/init/src/mill/init/InitModule.scala +++ b/main/init/src/mill/init/InitModule.scala @@ -1,7 +1,6 @@ package mill.init import mainargs.{Flag, arg} -import mill.api.IO import mill.define.{Discover, ExternalModule} import mill.{Command, Module, T} @@ -54,7 +53,7 @@ trait InitModule extends Module { val extractedDirName = zipName.stripSuffix(".zip") val downloaded = os.temp(requests.get(url)) println(s"Unpacking example...") - val unpackPath = IO.unpackZip(downloaded, os.rel) + val unpackPath = os.unzip(downloaded, T.dest) val extractedPath = T.dest / extractedDirName val conflicting = for { p <- os.walk(extractedPath) @@ -81,7 +80,7 @@ trait InitModule extends Module { os.perms.set(T.workspace / "mill", "rwxrwxrwx") ( - Seq(unpackPath.path.toString()), + Seq(unpackPath.toString()), s"Example download and unpacked to [${T.workspace}]; " + "See `build.mill` for an explanation of this example and instructions on how to use it" ) diff --git a/scalalib/worker/src/mill/scalalib/worker/ZincWorkerImpl.scala b/scalalib/worker/src/mill/scalalib/worker/ZincWorkerImpl.scala index 6575b532a9a..5b7c373992a 100644 --- a/scalalib/worker/src/mill/scalalib/worker/ZincWorkerImpl.scala +++ b/scalalib/worker/src/mill/scalalib/worker/ZincWorkerImpl.scala @@ -197,18 +197,18 @@ class ZincWorkerImpl( os.makeDir.all(workingDir) os.makeDir.all(compileDest) - val sourceFolder = mill.api.IO.unpackZip(compilerBridgeSourcesJar)(workingDir) + val sourceFolder = os.unzip(compilerBridgeSourcesJar, workingDir / "unpacked") val classloader = mill.api.ClassLoader.create( compilerClasspath.iterator.map(_.path.toIO.toURI.toURL).toSeq, null )(ctx0) val (sources, resources) = - os.walk(sourceFolder.path).filter(os.isFile) + os.walk(sourceFolder).filter(os.isFile) .partition(a => a.ext == "scala" || a.ext == "java") resources.foreach { res => - val dest = compileDest / res.relativeTo(sourceFolder.path) + val dest = compileDest / res.relativeTo(sourceFolder) os.move(res, dest, replaceExisting = true, createFolders = true) } From b4d4fea2f623603f1ab9466208ae65908ece1780 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 8 Oct 2024 07:17:40 +0200 Subject: [PATCH 04/47] Bump actions/upload-artifact from 4.3.5 to 4.4.1 (#3685) --- .github/workflows/run-mill-action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-mill-action.yml b/.github/workflows/run-mill-action.yml index cbf949255a6..ce791fddfce 100644 --- a/.github/workflows/run-mill-action.yml +++ b/.github/workflows/run-mill-action.yml @@ -82,7 +82,7 @@ jobs: shell: bash continue-on-error: true - - uses: actions/upload-artifact@v4.3.5 + - uses: actions/upload-artifact@v4.4.1 with: path: . name: ${{ inputs.os }}-artifact From 017b86a3c67d310589b89ddacf04bf933ec28d2e Mon Sep 17 00:00:00 2001 From: Himanshu Mahajan <83700343+himanshumahajan138@users.noreply.github.com> Date: Tue, 8 Oct 2024 11:05:35 +0530 Subject: [PATCH 05/47] Fixes:#3550; Added hello-world Android Kotlin example using Mill (#3679) # Pull Request Fixes: #3550 ## Description Added `Kotlin Android "Hello World" Application Example` while reusing `AndroidSdkModule`, `KotlinModule` and creating new Module `AndroidAppModule` with Kotlin Support and Proper `Documentation`. ## Related Issues - Link to related issue #3550. ## Checklist - [x] Android Support for Kotlin Added in the example section for "HelloWorld" Android Application. - [x] Added Kotlin Source files and Customized AndroidAppModule. - [x] Code is clean and follows project conventions. - [x] Documentation has been updated. - [x] Tests have been added or updated. - [x] All tests pass. ## Additional Notes Actually, I was wondering that we reused the `AndroidSdkModule` but created a new Module `AndroidAppModule` for Andorid Workflow but what we can do is to provide `Kotlin support in the Existing AndroidAppModule` and then based on user query they can use the functionality. So for this i need @lihaoyi Sir your Permission and Guidance, Please Review this PR suggest changes... --- docs/modules/ROOT/nav.adoc | 1 + .../pages/kotlinlib/android-examples.adoc | 50 ++++++++++++++++ .../1-hello-world/app/AndroidManifest.xml | 13 +++++ .../kotlin/com/helloworld/app/MainActivity.kt | 34 +++++++++++ .../android/1-hello-world/build.mill | 58 +++++++++++++++++++ example/package.mill | 1 + .../android/AndroidAppKotlinModule.scala | 21 +++++++ 7 files changed, 178 insertions(+) create mode 100644 docs/modules/ROOT/pages/kotlinlib/android-examples.adoc create mode 100644 example/kotlinlib/android/1-hello-world/app/AndroidManifest.xml create mode 100644 example/kotlinlib/android/1-hello-world/app/src/main/kotlin/com/helloworld/app/MainActivity.kt create mode 100644 example/kotlinlib/android/1-hello-world/build.mill create mode 100644 kotlinlib/src/mill/kotlinlib/android/AndroidAppKotlinModule.scala diff --git a/docs/modules/ROOT/nav.adoc b/docs/modules/ROOT/nav.adoc index 2918f145c5a..f0e9e09e4ae 100644 --- a/docs/modules/ROOT/nav.adoc +++ b/docs/modules/ROOT/nav.adoc @@ -36,6 +36,7 @@ * xref:kotlinlib/publishing.adoc[] * xref:kotlinlib/build-examples.adoc[] * xref:kotlinlib/web-examples.adoc[] +* xref:kotlinlib/android-examples.adoc[] .Build Tool Comparisons * xref:comparisons/maven.adoc[] diff --git a/docs/modules/ROOT/pages/kotlinlib/android-examples.adoc b/docs/modules/ROOT/pages/kotlinlib/android-examples.adoc new file mode 100644 index 00000000000..3ca6163e191 --- /dev/null +++ b/docs/modules/ROOT/pages/kotlinlib/android-examples.adoc @@ -0,0 +1,50 @@ += (Experimental) Android Builds +:page-aliases: android_app_kotlin_examples.adoc + +++++ + +++++ + +This page provides an example of using Mill as a build tool for Android applications. +This workflow is still pretty rough and nowhere near production ready, but can serve as +a starting point for further experimentation and development. + +== Relevant Modules + +These are the main Mill Modules that are relevant for building Android apps: + +* {mill-doc-url}/api/latest/mill/scalalib/AndroidSdkModule.html[`mill.scalalib.AndroidSdkModule`]: Handles Android SDK management and tools. +* {mill-doc-url}/api/latest/mill/kotlinlib/android/AndroidAppKotlinModule.html[`mill.kotlinlib.android.AndroidAppKotlinModule`]: Provides a framework for building Android applications. +* {mill-doc-url}/api/latest/mill/kotlinlib/KotlinModule.html[`mill.kotlinlib.KotlinModule`]: General Kotlin build tasks like compiling Kotlin code and creating JAR files. + +== Simple Android Hello World Application + +include::partial$example/kotlinlib/android/1-hello-world.adoc[] + +This example demonstrates how to create a basic "Hello World" Android application +using the Mill build tool. It outlines the minimum setup required to compile Kotlin code, +package it into an APK, and run the app on an Android device. + +== Understanding `AndroidSdkModule` and `AndroidAppKotlinModule` + +The two main modules you need to understand when building Android apps with Mill +are `AndroidSdkModule` and `AndroidAppKotlinModule`. + +`AndroidSdkModule`: + +* This module manages the installation and configuration of the Android SDK, which includes +tools like `aapt`, `d8`, `zipalign`, and `apksigner`. These tools are used +for compiling, packaging, and signing Android applications. + +`AndroidAppKotlinModule`: +This module provides the step-by-step workflow for building an Android app. It handles +everything from compiling the code to generating a signed APK for distribution. + +1. **Compiling Kotlin code**: The module compiles your Kotlin code into `.class` files, which is the first step in creating an Android app. +2. **Packaging into JAR**: It then packages the compiled `.class` files into a JAR file, which is necessary before converting to Android's format. +3. **Converting to DEX format**: The JAR file is converted into DEX format, which is the executable format for Android applications. +4. **Creating an APK**: The DEX files and Android resources (like layouts and strings) are packaged together into an APK file, which is the installable file for Android devices. +5. **Optimizing with zipalign**: The APK is optimized using `zipalign` to ensure better performance on Android devices. +6. **Signing the APK**: Finally, the APK is signed with a digital signature, allowing it to be distributed and installed on Android devices. diff --git a/example/kotlinlib/android/1-hello-world/app/AndroidManifest.xml b/example/kotlinlib/android/1-hello-world/app/AndroidManifest.xml new file mode 100644 index 00000000000..b33d6eb4174 --- /dev/null +++ b/example/kotlinlib/android/1-hello-world/app/AndroidManifest.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/example/kotlinlib/android/1-hello-world/app/src/main/kotlin/com/helloworld/app/MainActivity.kt b/example/kotlinlib/android/1-hello-world/app/src/main/kotlin/com/helloworld/app/MainActivity.kt new file mode 100644 index 00000000000..27d85181731 --- /dev/null +++ b/example/kotlinlib/android/1-hello-world/app/src/main/kotlin/com/helloworld/app/MainActivity.kt @@ -0,0 +1,34 @@ +package com.helloworld.app + +import android.app.Activity +import android.os.Bundle +import android.widget.TextView +import android.view.Gravity +import android.view.ViewGroup.LayoutParams + +class MainActivity : Activity() { + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + + // Create a new TextView + val textView = TextView(this) + + // Set the text to "Hello, World!" + textView.text = "Hello, World!" + + // Set text size + textView.textSize = 32f + + // Center the text within the view + textView.gravity = Gravity.CENTER + + // Set layout parameters (width and height) + textView.layoutParams = LayoutParams( + LayoutParams.MATCH_PARENT, + LayoutParams.MATCH_PARENT + ) + + // Set the content view to display the TextView + setContentView(textView) + } +} diff --git a/example/kotlinlib/android/1-hello-world/build.mill b/example/kotlinlib/android/1-hello-world/build.mill new file mode 100644 index 00000000000..49df51f1e50 --- /dev/null +++ b/example/kotlinlib/android/1-hello-world/build.mill @@ -0,0 +1,58 @@ +// This section sets up a basic Android project using Mill. +// We utilize `AndroidAppKotlinModule` and `AndroidSdkModule` to streamline the process of +// building an Android application with minimal configuration. +// +// By extending `AndroidAppKotlinModule`, we inherit all Android-related tasks such as +// resource generation, APK building, DEX conversion, and APK signing. +// Additionally, `AndroidSdkModule` is embedded, making SDK management seamless. + +//// SNIPPET:BUILD +package build + +import mill._ +import kotlinlib._ +import mill.kotlinlib.android.AndroidAppKotlinModule +import mill.javalib.android.AndroidSdkModule + +// Create and configure an Android SDK module to manage Android SDK paths and tools. +object androidSdkModule0 extends AndroidSdkModule{ + def buildToolsVersion = "35.0.0" +} + +// Actual android application +object app extends AndroidAppKotlinModule { + + def kotlinVersion = "2.0.0" + def androidSdkModule = mill.define.ModuleRef(androidSdkModule0) +} + +////SNIPPET:END + + +/** Usage + +> ./mill show app.androidApk +".../out/app/androidApk.dest/app.apk" + +*/ + +// This command triggers the build process, which installs the Android Setup, compiles the kotlin +// code, generates Android resources, converts kotlin bytecode to DEX format, packages everything +// into an APK, optimizes the APK using `zipalign`, and finally signs it. +// +// This Mill build configuration is designed to build a simple "Hello World" Android application. +// By extending `AndroidAppKotlinModule`, we leverage its predefined Android build tasks, ensuring that +// all necessary steps (resource generation, APK creation, and signing) are executed automatically. +// +// ### Project Structure: +// The project follows the standard Android app layout. Below is a typical project folder structure: +// +// ---- +// . +// ├── build.mill +// ├── AndroidManifest.xml +// └── app/src/main/kotlin +// └── com/helloworld/app +// └── MainActivity.kt +// ---- +// diff --git a/example/package.mill b/example/package.mill index 9028f1b3488..63d1989c25c 100644 --- a/example/package.mill +++ b/example/package.mill @@ -39,6 +39,7 @@ object `package` extends RootModule with Module { object web extends Cross[ExampleCrossModule](build.listIn(millSourcePath / "web")) } object kotlinlib extends Module { + object android extends Cross[ExampleCrossModuleJava](build.listIn(millSourcePath / "android")) object basic extends Cross[ExampleCrossModuleKotlin](build.listIn(millSourcePath / "basic")) object module extends Cross[ExampleCrossModuleKotlin](build.listIn(millSourcePath / "module")) object dependencies extends Cross[ExampleCrossModuleKotlin](build.listIn(millSourcePath / "dependencies")) diff --git a/kotlinlib/src/mill/kotlinlib/android/AndroidAppKotlinModule.scala b/kotlinlib/src/mill/kotlinlib/android/AndroidAppKotlinModule.scala new file mode 100644 index 00000000000..ddfaf985b9c --- /dev/null +++ b/kotlinlib/src/mill/kotlinlib/android/AndroidAppKotlinModule.scala @@ -0,0 +1,21 @@ +package mill.kotlinlib.android + +import mill.kotlinlib.KotlinModule +import mill.javalib.android.AndroidAppModule + +/** + * Trait for building Android applications using the Mill build tool. + * + * This trait defines all the necessary steps for building an Android app from Kotlin sources, + * integrating both Android-specific tasks and generic Kotlin tasks by extending the + * [[KotlinModule]] (for standard Kotlin tasks) + * and [[AndroidAppModule]] (for Android Application Workflow Process). + * + * It provides a structured way to handle various steps in the Android app build process, + * including compiling Kotlin sources, creating DEX files, generating resources, packaging + * APKs, optimizing, and signing APKs. + * + * [[https://developer.android.com/studio Android Studio Documentation]] + */ +@mill.api.experimental +trait AndroidAppKotlinModule extends AndroidAppModule with KotlinModule {} From 2e6a7d0ced542a3f382c78ef1e903ae44370a4a8 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Tue, 8 Oct 2024 08:38:15 +0200 Subject: [PATCH 06/47] Rebootstrap to pull in os-lib 0.11.0 (#3686) This lets us get rid of the ad-hoc unpackZip code in Mill's own build. Also moved from scalaj-http to requests --- .config/mill-version | 2 +- build.mill | 19 +++----- ci/shared.mill | 43 +------------------ ci/upload.mill | 41 +++++++++--------- .../kotlin/com/helloworld/app/MainActivity.kt | 2 +- .../invalidation/resources/build.mill | 1 - mill-build/build.sc | 1 - 7 files changed, 29 insertions(+), 80 deletions(-) diff --git a/.config/mill-version b/.config/mill-version index 570985f0344..a3ca0aa6b15 100644 --- a/.config/mill-version +++ b/.config/mill-version @@ -1 +1 @@ -0.12.0-RC3 +0.12.0-RC3-17-b4d4fe \ No newline at end of file diff --git a/build.mill b/build.mill index 0715d9dc3d6..0e905c8b268 100644 --- a/build.mill +++ b/build.mill @@ -599,8 +599,9 @@ trait BridgeModule extends MillPublishJavaModule with CrossScalaModule { } def generatedSources = Task { + compilerBridgeSourceJars().foreach { jar => - mill.api.IO.unpackZip(jar.path, os.rel) + os.unzip(jar.path, T.dest) } Seq(PathRef(T.dest)) @@ -984,19 +985,11 @@ def uploadToGithub(authKey: String) = T.command { if (releaseTag == label) { // TODO: check if the tag already exists (e.g. because we created it manually) and do not fail - scalaj.http.Http( - s"https://api.github.com/repos/${Settings.githubOrg}/${Settings.githubRepo}/releases" + requests.post( + s"https://api.github.com/repos/${Settings.githubOrg}/${Settings.githubRepo}/releases", + data = ujson.Obj("tag_name" -> releaseTag, "name" -> releaseTag), + headers = Seq("Authorization" -> ("token " + authKey)) ) - .postData( - ujson.write( - ujson.Obj( - "tag_name" -> releaseTag, - "name" -> releaseTag - ) - ) - ) - .header("Authorization", "token " + authKey) - .asString } val examples = exampleZips().map(z => (z.path, z.path.last)) diff --git a/ci/shared.mill b/ci/shared.mill index 2e5298838ea..d28af501855 100644 --- a/ci/shared.mill +++ b/ci/shared.mill @@ -1,49 +1,8 @@ package build.ci -/** - * Utility code that is shared between our SBT build and our Mill build. SBT - * calls this by shelling out to Ammonite in a subprocess, while Mill loads it - * via import $file - */ -import $ivy.`org.scalaj::scalaj-http:2.4.2` -import mainargs.main -def unpackZip(zipDest: os.Path, url: String) = { - println(s"Unpacking zip $url into $zipDest") - os.makeDir.all(zipDest) - - val bytes = - scalaj.http.Http.apply(url).option(scalaj.http.HttpOptions.followRedirects(true)).asBytes - val byteStream = new java.io.ByteArrayInputStream(bytes.body) - val zipStream = new java.util.zip.ZipInputStream(byteStream) - while ({ - zipStream.getNextEntry match { - case null => false - case entry => - if (!entry.isDirectory) { - val dest = zipDest / os.SubPath(entry.getName) - os.makeDir.all(dest / os.up) - val fileOut = new java.io.FileOutputStream(dest.toString) - val buffer = new Array[Byte](4096) - while ({ - zipStream.read(buffer) match { - case -1 => false - case n => - fileOut.write(buffer, 0, n) - true - } - }) () - fileOut.close() - } - zipStream.closeEntry() - true - } - }) () -} - -@main def downloadTestRepo(label: String, commit: String, dest: os.Path) = { - unpackZip(dest, s"https://github.com/$label/archive/$commit.zip") + os.unzip.stream(requests.get.stream(s"https://github.com/$label/archive/$commit.zip"), dest) dest } diff --git a/ci/upload.mill b/ci/upload.mill index e81ba1caf99..0ef7ec3ae4e 100644 --- a/ci/upload.mill +++ b/ci/upload.mill @@ -1,8 +1,5 @@ package build.ci -import scalaj.http._ -import mainargs.main -@main def apply( uploadedFile: os.Path, tagName: String, @@ -12,18 +9,18 @@ def apply( githubRepo: String ): String = { - val response = Http( - s"https://api.github.com/repos/${githubOrg}/${githubRepo}/releases/tags/${tagName}" + val response = requests.get( + s"https://api.github.com/repos/${githubOrg}/${githubRepo}/releases/tags/${tagName}", + headers = Seq( + "Authorization" -> s"token $authKey", + "Accept" -> "application/vnd.github.v3+json" + ) ) - .header("Authorization", "token " + authKey) - .header("Accept", "application/vnd.github.v3+json") - .asString - val body = response.body - val parsed = ujson.read(body) + val parsed = ujson.read(response) - println("Response code: " + response.code) - println(body) + println("Response code: " + response.statusCode) + println(response.text()) val snapshotReleaseId = parsed("id").num.toInt @@ -31,15 +28,17 @@ def apply( s"https://uploads.github.com/repos/${githubOrg}/${githubRepo}/releases/" + s"$snapshotReleaseId/assets?name=$uploadName" - val res = Http(uploadUrl) - .header("Content-Type", "application/octet-stream") - .header("Authorization", "token " + authKey) - .timeout(connTimeoutMs = 5000, readTimeoutMs = 60000) - .postData(os.read.bytes(uploadedFile)) - .asString - - println(res.body) - val longUrl = ujson.read(res.body)("browser_download_url").str + val res = requests.post( + uploadUrl, + headers = Seq( + "Content-Type" -> "application/octet-stream", + "Authorization" -> s"token $authKey" + ), + data = os.read.stream(uploadedFile) + ) + + println(res.text()) + val longUrl = ujson.read(res)("browser_download_url").str println("Long Url " + longUrl) longUrl diff --git a/example/kotlinlib/android/1-hello-world/app/src/main/kotlin/com/helloworld/app/MainActivity.kt b/example/kotlinlib/android/1-hello-world/app/src/main/kotlin/com/helloworld/app/MainActivity.kt index 27d85181731..f7e04a63de9 100644 --- a/example/kotlinlib/android/1-hello-world/app/src/main/kotlin/com/helloworld/app/MainActivity.kt +++ b/example/kotlinlib/android/1-hello-world/app/src/main/kotlin/com/helloworld/app/MainActivity.kt @@ -14,7 +14,7 @@ class MainActivity : Activity() { val textView = TextView(this) // Set the text to "Hello, World!" - textView.text = "Hello, World!" + textView.text = "Hello, World Kotlin!" // Set text size textView.textSize = 32f diff --git a/integration/invalidation/invalidation/resources/build.mill b/integration/invalidation/invalidation/resources/build.mill index a15f4ae0906..051d65d5b47 100644 --- a/integration/invalidation/invalidation/resources/build.mill +++ b/integration/invalidation/invalidation/resources/build.mill @@ -1,7 +1,6 @@ package build import mill._ import $packages._ -import $ivy.`org.scalaj::scalaj-http:2.4.2` def task = Task { build.a.input() diff --git a/mill-build/build.sc b/mill-build/build.sc index 356e9306d2a..23a803a3e44 100644 --- a/mill-build/build.sc +++ b/mill-build/build.sc @@ -4,7 +4,6 @@ import mill.scalalib._ object `package` extends MillBuildRootModule { override def ivyDeps = Agg( - ivy"org.scalaj::scalaj-http:2.4.2", ivy"de.tototec::de.tobiasroeser.mill.vcs.version::0.4.0", ivy"com.github.lolgab::mill-mima::0.1.1", ivy"net.sourceforge.htmlcleaner:htmlcleaner:2.29", From 561a007738846a5d3c0ffa88f4cd5875fce20cd1 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Tue, 8 Oct 2024 09:17:08 +0200 Subject: [PATCH 07/47] Update upload.mill --- ci/upload.mill | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/upload.mill b/ci/upload.mill index 0ef7ec3ae4e..d175e440c1b 100644 --- a/ci/upload.mill +++ b/ci/upload.mill @@ -34,7 +34,7 @@ def apply( "Content-Type" -> "application/octet-stream", "Authorization" -> s"token $authKey" ), - data = os.read.stream(uploadedFile) + data = os.read.bytes(uploadedFile) ) println(res.text()) From 8b063c48d76837870af188ad48500bb739668f2b Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Tue, 8 Oct 2024 10:20:27 +0200 Subject: [PATCH 08/47] Update os-lib 0.11.1 (#3687) Pulls in https://github.com/com-lihaoyi/os-lib/pull/320 which should let us remove the workaround in https://github.com/com-lihaoyi/mill/commit/561a007738846a5d3c0ffa88f4cd5875fce20cd1 --- build.mill | 2 +- ci/upload.mill | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build.mill b/build.mill index 0e905c8b268..72dd2f4ca10 100644 --- a/build.mill +++ b/build.mill @@ -149,7 +149,7 @@ object Deps { val junitInterface = ivy"com.github.sbt:junit-interface:0.13.3" val commonsIO = ivy"commons-io:commons-io:2.16.1" val log4j2Core = ivy"org.apache.logging.log4j:log4j-core:2.23.1" - val osLib = ivy"com.lihaoyi::os-lib:0.11.0" + val osLib = ivy"com.lihaoyi::os-lib:0.11.1" val pprint = ivy"com.lihaoyi::pprint:0.9.0" val mainargs = ivy"com.lihaoyi::mainargs:0.7.4" val millModuledefsVersion = "0.11.0" diff --git a/ci/upload.mill b/ci/upload.mill index d175e440c1b..292fe350160 100644 --- a/ci/upload.mill +++ b/ci/upload.mill @@ -35,7 +35,7 @@ def apply( "Authorization" -> s"token $authKey" ), data = os.read.bytes(uploadedFile) - ) + ) println(res.text()) val longUrl = ujson.read(res)("browser_download_url").str From fe73de83daae3abd03cd9dec3eb534f66a4d9523 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Tue, 8 Oct 2024 10:50:45 +0200 Subject: [PATCH 09/47] Acquire a lock on the out dir in order to run tasks / commands (#3599) First cut at https://github.com/com-lihaoyi/mill/issues/3519 There's no tests yet, but it works fine in manual tests. --- .../output-directory/resources/build.mill | 12 +++ .../src/OutputDirectoryLockTests.scala | 92 +++++++++++++++++++ main/api/src/mill/api/Logger.scala | 17 +++- .../client/src/mill/main/client/OutFiles.java | 4 + .../src/mill/main/client/lock/DummyLock.java | 22 +++++ .../mill/main/client/lock/DummyTryLocked.java | 11 +++ .../src/mill/main/client/lock/Lock.java | 13 +++ runner/src/mill/runner/MillCliConfig.scala | 14 ++- runner/src/mill/runner/MillMain.scala | 19 +++- 9 files changed, 198 insertions(+), 6 deletions(-) create mode 100644 integration/feature/output-directory/src/OutputDirectoryLockTests.scala create mode 100644 main/client/src/mill/main/client/lock/DummyLock.java create mode 100644 main/client/src/mill/main/client/lock/DummyTryLocked.java diff --git a/integration/feature/output-directory/resources/build.mill b/integration/feature/output-directory/resources/build.mill index b5fae4e0f89..e068bbd11cb 100644 --- a/integration/feature/output-directory/resources/build.mill +++ b/integration/feature/output-directory/resources/build.mill @@ -5,4 +5,16 @@ import mill.scalalib._ object `package` extends RootModule with ScalaModule { def scalaVersion = scala.util.Properties.versionNumberString + + def hello = Task { + "Hello from hello task" + } + + def blockWhileExists(path: os.Path) = Task.Command[String] { + if (!os.exists(path)) + os.write(path, Array.emptyByteArray) + while (os.exists(path)) + Thread.sleep(100L) + "Blocking command done" + } } diff --git a/integration/feature/output-directory/src/OutputDirectoryLockTests.scala b/integration/feature/output-directory/src/OutputDirectoryLockTests.scala new file mode 100644 index 00000000000..25b5a15d078 --- /dev/null +++ b/integration/feature/output-directory/src/OutputDirectoryLockTests.scala @@ -0,0 +1,92 @@ +package mill.integration + +import mill.testkit.UtestIntegrationTestSuite +import utest._ + +import java.io.ByteArrayOutputStream +import java.util.concurrent.{CountDownLatch, Executors} + +import scala.concurrent.duration.Duration +import scala.concurrent.{Await, ExecutionContext, Future} + +object OutputDirectoryLockTests extends UtestIntegrationTestSuite { + + private val pool = Executors.newCachedThreadPool() + private val ec = ExecutionContext.fromExecutorService(pool) + + override def utestAfterAll(): Unit = { + pool.shutdown() + } + + def tests: Tests = Tests { + test("basic") - integrationTest { tester => + import tester._ + val signalFile = workspacePath / "do-wait" + System.err.println("Spawning blocking task") + val blocksFuture = + Future(eval(("show", "blockWhileExists", "--path", signalFile), check = true))(ec) + while (!os.exists(signalFile) && !blocksFuture.isCompleted) + Thread.sleep(100L) + if (os.exists(signalFile)) + System.err.println("Blocking task is running") + else { + System.err.println("Failed to run blocking task") + Predef.assert(blocksFuture.isCompleted) + blocksFuture.value.get.get + } + + val testCommand: os.Shellable = ("show", "hello") + val testMessage = "Hello from hello task" + + System.err.println("Evaluating task without lock") + val noLockRes = eval(("--no-build-lock", testCommand), check = true) + assert(noLockRes.out.contains(testMessage)) + + System.err.println("Evaluating task without waiting for lock (should fail)") + val noWaitRes = eval(("--no-wait-for-build-lock", testCommand)) + assert(noWaitRes.err.contains("Cannot proceed, another Mill process is running tasks")) + + System.err.println("Evaluating task waiting for the lock") + + val lock = new CountDownLatch(1) + val stderr = new ByteArrayOutputStream + var success = false + val futureWaitingRes = Future { + eval( + testCommand, + stderr = os.ProcessOutput { + val expectedMessage = + "Another Mill process is running tasks, waiting for it to be done..." + + (bytes, len) => + stderr.write(bytes, 0, len) + val output = new String(stderr.toByteArray) + if (output.contains(expectedMessage)) + lock.countDown() + }, + check = true + ) + }(ec) + try { + lock.await() + success = true + } finally { + if (!success) { + System.err.println("Waiting task output:") + System.err.write(stderr.toByteArray) + } + } + + System.err.println("Task is waiting for the lock, unblocking it") + os.remove(signalFile) + + System.err.println("Blocking task should exit") + val blockingRes = Await.result(blocksFuture, Duration.Inf) + assert(blockingRes.out.contains("Blocking command done")) + + System.err.println("Waiting task should be free to proceed") + val waitingRes = Await.result(futureWaitingRes, Duration.Inf) + assert(waitingRes.out.contains(testMessage)) + } + } +} diff --git a/main/api/src/mill/api/Logger.scala b/main/api/src/mill/api/Logger.scala index 4b93616d14f..add24defd80 100644 --- a/main/api/src/mill/api/Logger.scala +++ b/main/api/src/mill/api/Logger.scala @@ -2,6 +2,8 @@ package mill.api import java.io.{InputStream, PrintStream} +import mill.main.client.lock.{Lock, Locked} + /** * The standard logging interface of the Mill build tool. * @@ -24,7 +26,7 @@ import java.io.{InputStream, PrintStream} * but when `show` is used both are forwarded to stderr and stdout is only * used to display the final `show` output for easy piping. */ -trait Logger { +trait Logger extends AutoCloseable { def colored: Boolean def systemStreams: SystemStreams @@ -79,4 +81,17 @@ trait Logger { try t finally removePromptLine() } + + def waitForLock(lock: Lock, waitingAllowed: Boolean): Locked = { + val tryLocked = lock.tryLock() + if (tryLocked.isLocked()) + tryLocked + else if (waitingAllowed) { + info("Another Mill process is running tasks, waiting for it to be done...") + lock.lock() + } else { + error("Cannot proceed, another Mill process is running tasks") + throw new Exception("Cannot acquire lock on Mill output directory") + } + } } diff --git a/main/client/src/mill/main/client/OutFiles.java b/main/client/src/mill/main/client/OutFiles.java index 04ffeecb4db..0af23e233a1 100644 --- a/main/client/src/mill/main/client/OutFiles.java +++ b/main/client/src/mill/main/client/OutFiles.java @@ -57,5 +57,9 @@ public class OutFiles { */ final public static String millNoServer = "mill-no-server"; + /** + * Lock file used for exclusive access to the Mill output directory + */ + final public static String millLock = "mill-lock"; } diff --git a/main/client/src/mill/main/client/lock/DummyLock.java b/main/client/src/mill/main/client/lock/DummyLock.java new file mode 100644 index 00000000000..ede8224323d --- /dev/null +++ b/main/client/src/mill/main/client/lock/DummyLock.java @@ -0,0 +1,22 @@ +package mill.main.client.lock; + +import java.util.concurrent.locks.ReentrantLock; + +class DummyLock extends Lock { + + public boolean probe() { + return true; + } + + public Locked lock() { + return new DummyTryLocked(); + } + + public TryLocked tryLock() { + return new DummyTryLocked(); + } + + @Override + public void close() throws Exception { + } +} diff --git a/main/client/src/mill/main/client/lock/DummyTryLocked.java b/main/client/src/mill/main/client/lock/DummyTryLocked.java new file mode 100644 index 00000000000..34ad7b5ea15 --- /dev/null +++ b/main/client/src/mill/main/client/lock/DummyTryLocked.java @@ -0,0 +1,11 @@ +package mill.main.client.lock; + +class DummyTryLocked implements TryLocked { + public DummyTryLocked() { + } + + public boolean isLocked(){ return true; } + + public void release() throws Exception { + } +} diff --git a/main/client/src/mill/main/client/lock/Lock.java b/main/client/src/mill/main/client/lock/Lock.java index 6d729c0ebd6..3870bc07a14 100644 --- a/main/client/src/mill/main/client/lock/Lock.java +++ b/main/client/src/mill/main/client/lock/Lock.java @@ -15,4 +15,17 @@ public void await() throws Exception { */ public abstract boolean probe() throws Exception; public void delete() throws Exception {} + + public static Lock file(String path) throws Exception { + return new FileLock(path); + } + + public static Lock memory() { + return new MemoryLock(); + } + + public static Lock dummy() { + return new DummyLock(); + } + } diff --git a/runner/src/mill/runner/MillCliConfig.scala b/runner/src/mill/runner/MillCliConfig.scala index 61d1ef9fac4..429a394f18a 100644 --- a/runner/src/mill/runner/MillCliConfig.scala +++ b/runner/src/mill/runner/MillCliConfig.scala @@ -130,7 +130,19 @@ case class MillCliConfig( status at the command line and falls back to the legacy ticker """ ) - disablePrompt: Flag = Flag() + disablePrompt: Flag = Flag(), + @arg( + hidden = true, + doc = + """Evaluate tasks / commands without acquiring an exclusive lock on the Mill output directory""" + ) + noBuildLock: Flag = Flag(), + @arg( + hidden = true, + doc = + """Do not wait for an exclusive lock on the Mill output directory to evaluate tasks / commands. Fail if waiting for a lock is needed.""" + ) + noWaitForBuildLock: Flag = Flag() ) import mainargs.ParserForClass diff --git a/runner/src/mill/runner/MillMain.scala b/runner/src/mill/runner/MillMain.scala index 3e1550ff026..10f343d1518 100644 --- a/runner/src/mill/runner/MillMain.scala +++ b/runner/src/mill/runner/MillMain.scala @@ -9,10 +9,12 @@ import mill.api.{MillException, SystemStreams, WorkspaceRoot, internal} import mill.bsp.{BspContext, BspServerResult} import mill.main.BuildInfo import mill.main.client.{OutFiles, ServerFiles} +import mill.main.client.lock.Lock import mill.util.{PromptLogger, PrintLogger, Colors} import java.lang.reflect.InvocationTargetException import scala.util.control.NonFatal +import scala.util.Using @internal object MillMain { @@ -209,6 +211,10 @@ object MillMain { .map(_ => Seq(bspCmd)) .getOrElse(config.leftoverArgs.value.toList) + val out = os.Path(OutFiles.out, WorkspaceRoot.workspaceRoot) + val outLock = + if (config.noBuildLock.value || bspContext.isDefined) Lock.dummy() + else Lock.file((out / OutFiles.millLock).toString) var repeatForBsp = true var loopRes: (Boolean, RunnerState) = (false, RunnerState.empty) while (repeatForBsp) { @@ -235,9 +241,16 @@ object MillMain { colored = colored, colors = colors ) - try new MillBuildBootstrap( + Using.resources( + logger, + logger.waitForLock( + outLock, + waitingAllowed = !config.noWaitForBuildLock.value + ) + ) { (_, _) => + new MillBuildBootstrap( projectRoot = WorkspaceRoot.workspaceRoot, - output = os.Path(OutFiles.out, WorkspaceRoot.workspaceRoot), + output = out, home = config.home, keepGoing = config.keepGoing.value, imports = config.imports, @@ -252,8 +265,6 @@ object MillMain { config.allowPositional.value, systemExit = systemExit ).evaluate() - finally { - logger.close() } }, colors = colors From 4db51dbb35e843e69ce7c2d8662d6ec2a64a0b5f Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Tue, 8 Oct 2024 11:14:57 +0200 Subject: [PATCH 10/47] Rebootstrap to pull in os-lib 0.11.1 changes (#3688) This should let us revert the hack in https://github.com/com-lihaoyi/mill/commit/561a007738846a5d3c0ffa88f4cd5875fce20cd1, will need to re-run publishing to make sure it works (and probably re-bootstrap on the published artifacts to really be sure) --- .config/mill-version | 2 +- ci/upload.mill | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.config/mill-version b/.config/mill-version index a3ca0aa6b15..fc57f1a3116 100644 --- a/.config/mill-version +++ b/.config/mill-version @@ -1 +1 @@ -0.12.0-RC3-17-b4d4fe \ No newline at end of file +0.12.0-RC3-21-8b063c \ No newline at end of file diff --git a/ci/upload.mill b/ci/upload.mill index 292fe350160..afd59c359cc 100644 --- a/ci/upload.mill +++ b/ci/upload.mill @@ -34,7 +34,7 @@ def apply( "Content-Type" -> "application/octet-stream", "Authorization" -> s"token $authKey" ), - data = os.read.bytes(uploadedFile) + data = os.read.stream(uploadedFile) ) println(res.text()) From efa473be3477755c56ed0d402cdf9e47d0c87703 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Tue, 8 Oct 2024 11:55:45 +0200 Subject: [PATCH 11/47] Add some BSP integration tests (#3608) This adds some BSP integration tests, ensuring the Mill BSP server answers correctly basic BSP requests. BSP responses are serialized on disk and committed, and we compare the requests we get to those on disk. These fixtures on disk can be automatically generated and updated by changing a variable (currently, `BspServerUtil.updateFixtures`), so that it's quite convenient to generate them, and update them and get a nice diff of the changes at the same time. --- bsp/src/mill/bsp/BSP.scala | 7 +- bsp/src/mill/bsp/Constants.scala | 2 +- .../src/mill/bsp/worker/MillBuildServer.scala | 35 ++- bsp/worker/src/mill/bsp/worker/State.scala | 13 +- .../SyntheticRootBspBuildTargetData.scala | 5 +- .../ide/bsp-install/resources/build.mill | 15 -- .../ide/bsp-install/src/BspInstallTests.scala | 23 -- .../bsp-server/resources/project/build.mill | 10 + .../build-targets-compile-classpaths.json | 21 ++ .../build-targets-dependency-modules.json | 21 ++ .../build-targets-dependency-sources.json | 18 ++ .../build-targets-javac-options.json | 25 ++ .../build-targets-jvm-run-environments.json | 33 +++ .../build-targets-jvm-test-environments.json | 33 +++ .../snapshots/build-targets-output-paths.json | 45 ++++ .../snapshots/build-targets-resources.json | 22 ++ .../build-targets-scalac-options.json | 25 ++ .../snapshots/build-targets-sources.json | 57 +++++ .../snapshots/initialize-build-result.json | 38 ++++ .../snapshots/workspace-build-targets.json | 119 ++++++++++ .../src/BspInstallDebugTests.scala | 2 + .../bsp-server/src/BspServerTestUtil.scala | 214 ++++++++++++++++++ .../ide/bsp-server/src/BspServerTests.scala | 164 ++++++++++++++ integration/package.mill | 11 +- 24 files changed, 894 insertions(+), 64 deletions(-) delete mode 100644 integration/ide/bsp-install/resources/build.mill delete mode 100644 integration/ide/bsp-install/src/BspInstallTests.scala create mode 100644 integration/ide/bsp-server/resources/project/build.mill create mode 100644 integration/ide/bsp-server/resources/snapshots/build-targets-compile-classpaths.json create mode 100644 integration/ide/bsp-server/resources/snapshots/build-targets-dependency-modules.json create mode 100644 integration/ide/bsp-server/resources/snapshots/build-targets-dependency-sources.json create mode 100644 integration/ide/bsp-server/resources/snapshots/build-targets-javac-options.json create mode 100644 integration/ide/bsp-server/resources/snapshots/build-targets-jvm-run-environments.json create mode 100644 integration/ide/bsp-server/resources/snapshots/build-targets-jvm-test-environments.json create mode 100644 integration/ide/bsp-server/resources/snapshots/build-targets-output-paths.json create mode 100644 integration/ide/bsp-server/resources/snapshots/build-targets-resources.json create mode 100644 integration/ide/bsp-server/resources/snapshots/build-targets-scalac-options.json create mode 100644 integration/ide/bsp-server/resources/snapshots/build-targets-sources.json create mode 100644 integration/ide/bsp-server/resources/snapshots/initialize-build-result.json create mode 100644 integration/ide/bsp-server/resources/snapshots/workspace-build-targets.json rename integration/ide/{bsp-install => bsp-server}/src/BspInstallDebugTests.scala (89%) create mode 100644 integration/ide/bsp-server/src/BspServerTestUtil.scala create mode 100644 integration/ide/bsp-server/src/BspServerTests.scala diff --git a/bsp/src/mill/bsp/BSP.scala b/bsp/src/mill/bsp/BSP.scala index 212601d318a..4eea72bc9b3 100644 --- a/bsp/src/mill/bsp/BSP.scala +++ b/bsp/src/mill/bsp/BSP.scala @@ -75,11 +75,10 @@ object BSP extends ExternalModule with CoursierModule { } private def bspConnectionJson(jobs: Int, debug: Boolean): String = { - val props = sys.props - val millPath = props - .get("mill.main.cli") + val millPath = sys.env.get("MILL_MAIN_CLI") + .orElse(sys.props.get("mill.main.cli")) // we assume, the classpath is an executable jar here - .orElse(props.get("java.class.path")) + .orElse(sys.props.get("java.class.path")) .getOrElse(throw new IllegalStateException("System property 'java.class.path' not set")) upickle.default.write( diff --git a/bsp/src/mill/bsp/Constants.scala b/bsp/src/mill/bsp/Constants.scala index 3bc12cc3577..9dd59b05f48 100644 --- a/bsp/src/mill/bsp/Constants.scala +++ b/bsp/src/mill/bsp/Constants.scala @@ -6,6 +6,6 @@ private[mill] object Constants { val bspProtocolVersion = BuildInfo.bsp4jVersion val bspWorkerImplClass = "mill.bsp.worker.BspWorkerImpl" val bspWorkerBuildInfoClass = "mill.bsp.worker.BuildInfo" - val languages: Seq[String] = Seq("scala", "java") + val languages: Seq[String] = Seq("java", "scala") val serverName = "mill-bsp" } diff --git a/bsp/worker/src/mill/bsp/worker/MillBuildServer.scala b/bsp/worker/src/mill/bsp/worker/MillBuildServer.scala index e00df71cd0a..adbc5e44332 100644 --- a/bsp/worker/src/mill/bsp/worker/MillBuildServer.scala +++ b/bsp/worker/src/mill/bsp/worker/MillBuildServer.scala @@ -4,7 +4,7 @@ import ch.epfl.scala.bsp4j import ch.epfl.scala.bsp4j._ import com.google.gson.JsonObject import mill.api.{DummyTestReporter, Result, Strict} -import mill.bsp.BspServerResult +import mill.bsp.{BspServerResult, Constants} import mill.bsp.worker.Utils.{makeBuildTarget, outputPaths, sanitizeUri} import mill.define.Segment.Label import mill.define.{Args, Discover, ExternalModule, Task} @@ -33,6 +33,8 @@ private class MillBuildServer( ) extends ExternalModule with BuildServer { + import MillBuildServer._ + lazy val millDiscover: Discover = Discover[this.type] private[worker] var cancellator: Boolean => Unit = shutdownBefore => () @@ -71,7 +73,7 @@ private class MillBuildServer( // TODO: scan BspModules and infer their capabilities - val supportedLangs = Seq("java", "scala").asJava + val supportedLangs = Constants.languages.asJava val capabilities = new BuildServerCapabilities capabilities.setBuildTargetChangedProvider(false) @@ -154,7 +156,7 @@ private class MillBuildServer( override def workspaceBuildTargets(): CompletableFuture[WorkspaceBuildTargetsResult] = completableTasksWithState( "workspaceBuildTargets", - targetIds = _.bspModulesById.keySet.toSeq, + targetIds = _.bspModulesIdList.map(_._1), tasks = { case m: BspModule => m.bspBuildTargetData } ) { (ev, state, id, m: BspModule, bspBuildTargetData) => val depsIds = m match { @@ -252,7 +254,7 @@ private class MillBuildServer( override def buildTargetInverseSources(p: InverseSourcesParams) : CompletableFuture[InverseSourcesResult] = { completable(s"buildtargetInverseSources ${p}") { state => - val tasksEvaluators = state.bspModulesById.iterator.collect { + val tasksEvaluators = state.bspModulesIdList.iterator.collect { case (id, (m: JavaModule, ev)) => Task.Anon { val src = m.allSourceFiles() @@ -263,11 +265,9 @@ private class MillBuildServer( } -> ev }.toSeq - val ids = tasksEvaluators - .groupMap(_._2)(_._1) + val ids = groupList(tasksEvaluators)(_._2)(_._1) .flatMap { case (ev, ts) => ev.evalOrThrow()(ts) } .flatten - .toSeq new InverseSourcesResult(ids.asJava) } @@ -641,9 +641,8 @@ private class MillBuildServer( tasks.lift.apply(m).map(ts => (ts, (ev, id))) } - val evaluated = tasksSeq - // group by evaluator (different root module) - .groupMap(_._2)(_._1) + // group by evaluator (different root module) + val evaluated = groupList(tasksSeq.toSeq)(_._2)(_._1) .map { case ((ev, id), ts) => val results = ev.evaluate(ts) val failures = results.results.collect { @@ -674,7 +673,7 @@ private class MillBuildServer( } } - agg(evaluated.flatten.toSeq.asJava, state) + agg(evaluated.flatten.asJava, state) } } @@ -772,3 +771,17 @@ private class MillBuildServer( debug("onRunReadStdin is current unsupported") } } + +private object MillBuildServer { + + /** + * Same as Iterable.groupMap, but returns a sequence instead of a map, and preserves + * the order of appearance of the keys from the input sequence + */ + private def groupList[A, K, B](seq: Seq[A])(key: A => K)(f: A => B): Seq[(K, Seq[B])] = { + val keyIndices = seq.map(key).distinct.zipWithIndex.toMap + seq.groupMap(key)(f) + .toSeq + .sortBy { case (k, _) => keyIndices(k) } + } +} diff --git a/bsp/worker/src/mill/bsp/worker/State.scala b/bsp/worker/src/mill/bsp/worker/State.scala index e4e2a8b8b22..9db3d21a936 100644 --- a/bsp/worker/src/mill/bsp/worker/State.scala +++ b/bsp/worker/src/mill/bsp/worker/State.scala @@ -7,13 +7,13 @@ import mill.define.Module import mill.eval.Evaluator private class State(workspaceDir: os.Path, evaluators: Seq[Evaluator], debug: String => Unit) { - lazy val bspModulesById: Map[BuildTargetIdentifier, (BspModule, Evaluator)] = { + lazy val bspModulesIdList: Seq[(BuildTargetIdentifier, (BspModule, Evaluator))] = { val modules: Seq[(Module, Seq[Module], Evaluator)] = evaluators .map(ev => (ev.rootModule, JavaModuleUtils.transitiveModules(ev.rootModule), ev)) - val map = modules - .flatMap { case (rootModule, otherModules, eval) => - (Seq(rootModule) ++ otherModules).collect { + modules + .flatMap { case (rootModule, modules, eval) => + modules.collect { case m: BspModule => val uri = Utils.sanitizeUri( rootModule.millSourcePath / @@ -24,9 +24,10 @@ private class State(workspaceDir: os.Path, evaluators: Seq[Evaluator], debug: St (new BuildTargetIdentifier(uri), (m, eval)) } } - .toMap + } + lazy val bspModulesById: Map[BuildTargetIdentifier, (BspModule, Evaluator)] = { + val map = bspModulesIdList.toMap debug(s"BspModules: ${map.view.mapValues(_._1.bspDisplayName).toMap}") - map } diff --git a/bsp/worker/src/mill/bsp/worker/SyntheticRootBspBuildTargetData.scala b/bsp/worker/src/mill/bsp/worker/SyntheticRootBspBuildTargetData.scala index 45302e0ce97..c1891628f92 100644 --- a/bsp/worker/src/mill/bsp/worker/SyntheticRootBspBuildTargetData.scala +++ b/bsp/worker/src/mill/bsp/worker/SyntheticRootBspBuildTargetData.scala @@ -5,7 +5,6 @@ import mill.bsp.worker.Utils.{makeBuildTarget, sanitizeUri} import mill.scalalib.bsp.{BspBuildTarget, BspModule} import mill.scalalib.bsp.BspModule.Tag -import java.util.UUID import scala.jdk.CollectionConverters._ import ch.epfl.scala.bsp4j.BuildTarget @@ -15,11 +14,11 @@ import ch.epfl.scala.bsp4j.BuildTarget */ class SyntheticRootBspBuildTargetData(topLevelProjectRoot: os.Path) { val id: BuildTargetIdentifier = new BuildTargetIdentifier( - Utils.sanitizeUri(topLevelProjectRoot / s"synth-build-target-${UUID.randomUUID()}") + Utils.sanitizeUri(topLevelProjectRoot / "mill-synthetic-root-target") ) val bt: BspBuildTarget = BspBuildTarget( - displayName = Some(topLevelProjectRoot.last + "-root"), + displayName = Some("mill-synthetic-root"), baseDirectory = Some(topLevelProjectRoot), tags = Seq(Tag.Manual), languageIds = Seq.empty, diff --git a/integration/ide/bsp-install/resources/build.mill b/integration/ide/bsp-install/resources/build.mill deleted file mode 100644 index 9192880c5fc..00000000000 --- a/integration/ide/bsp-install/resources/build.mill +++ /dev/null @@ -1,15 +0,0 @@ -package build -import mill._ -import mill.api.PathRef -import mill.scalalib._ - -trait HelloBspModule extends ScalaModule { - def scalaVersion = sys.props.getOrElse("TEST_SCALA_2_13_VERSION", ???) - object test extends ScalaTests with TestModule.Utest - - override def generatedSources = Task { - Seq(PathRef(Task.ctx().dest / "classes")) - } -} - -object HelloBsp extends HelloBspModule diff --git a/integration/ide/bsp-install/src/BspInstallTests.scala b/integration/ide/bsp-install/src/BspInstallTests.scala deleted file mode 100644 index 40f9547da61..00000000000 --- a/integration/ide/bsp-install/src/BspInstallTests.scala +++ /dev/null @@ -1,23 +0,0 @@ -package mill.integration - -import mill.testkit.UtestIntegrationTestSuite -import mill.bsp.Constants -import utest._ - -object BspInstallTests extends UtestIntegrationTestSuite { - val bsp4jVersion: String = sys.props.getOrElse("BSP4J_VERSION", ???) - - def tests: Tests = Tests { - test("BSP install") - integrationTest { tester => - import tester._ - assert(eval("mill.bsp.BSP/install").isSuccess) - val jsonFile = workspacePath / Constants.bspDir / s"${Constants.serverName}.json" - assert(os.exists(jsonFile)) - val contents = os.read(jsonFile) - assert( - !contents.contains("--debug"), - contents.contains(s""""bspVersion":"${bsp4jVersion}"""") - ) - } - } -} diff --git a/integration/ide/bsp-server/resources/project/build.mill b/integration/ide/bsp-server/resources/project/build.mill new file mode 100644 index 00000000000..b07b6f12038 --- /dev/null +++ b/integration/ide/bsp-server/resources/project/build.mill @@ -0,0 +1,10 @@ +package build + +import mill._ +import mill.scalalib._ + +object `hello-java` extends JavaModule + +object `hello-scala` extends ScalaModule { + def scalaVersion = Option(System.getenv("TEST_SCALA_2_13_VERSION")).getOrElse(???) +} diff --git a/integration/ide/bsp-server/resources/snapshots/build-targets-compile-classpaths.json b/integration/ide/bsp-server/resources/snapshots/build-targets-compile-classpaths.json new file mode 100644 index 00000000000..f67288640f0 --- /dev/null +++ b/integration/ide/bsp-server/resources/snapshots/build-targets-compile-classpaths.json @@ -0,0 +1,21 @@ +{ + "items": [ + { + "target": { + "uri": "file:///workspace/hello-java" + }, + "classpath": [ + "file:///workspace/hello-java/compile-resources" + ] + }, + { + "target": { + "uri": "file:///workspace/hello-scala" + }, + "classpath": [ + "file:///coursier-cache/https/repo1.maven.org/maven2/org/scala-lang/scala-library//scala-library-.jar", + "file:///workspace/hello-scala/compile-resources" + ] + } + ] +} \ No newline at end of file diff --git a/integration/ide/bsp-server/resources/snapshots/build-targets-dependency-modules.json b/integration/ide/bsp-server/resources/snapshots/build-targets-dependency-modules.json new file mode 100644 index 00000000000..2820a7f6c4a --- /dev/null +++ b/integration/ide/bsp-server/resources/snapshots/build-targets-dependency-modules.json @@ -0,0 +1,21 @@ +{ + "items": [ + { + "target": { + "uri": "file:///workspace/hello-java" + }, + "modules": [] + }, + { + "target": { + "uri": "file:///workspace/hello-scala" + }, + "modules": [ + { + "name": "org.scala-lang:scala-library", + "version": "" + } + ] + } + ] +} \ No newline at end of file diff --git a/integration/ide/bsp-server/resources/snapshots/build-targets-dependency-sources.json b/integration/ide/bsp-server/resources/snapshots/build-targets-dependency-sources.json new file mode 100644 index 00000000000..1a0bee2bbee --- /dev/null +++ b/integration/ide/bsp-server/resources/snapshots/build-targets-dependency-sources.json @@ -0,0 +1,18 @@ +{ + "items": [ + { + "target": { + "uri": "file:///workspace/hello-java" + }, + "sources": [] + }, + { + "target": { + "uri": "file:///workspace/hello-scala" + }, + "sources": [ + "file:///coursier-cache/https/repo1.maven.org/maven2/org/scala-lang/scala-library//scala-library--sources.jar" + ] + } + ] +} \ No newline at end of file diff --git a/integration/ide/bsp-server/resources/snapshots/build-targets-javac-options.json b/integration/ide/bsp-server/resources/snapshots/build-targets-javac-options.json new file mode 100644 index 00000000000..043b0582ab6 --- /dev/null +++ b/integration/ide/bsp-server/resources/snapshots/build-targets-javac-options.json @@ -0,0 +1,25 @@ +{ + "items": [ + { + "target": { + "uri": "file:///workspace/hello-java" + }, + "options": [], + "classpath": [ + "file:///workspace/hello-java/compile-resources" + ], + "classDirectory": "file:///workspace/out/hello-java/compile.dest/classes" + }, + { + "target": { + "uri": "file:///workspace/hello-scala" + }, + "options": [], + "classpath": [ + "file:///coursier-cache/https/repo1.maven.org/maven2/org/scala-lang/scala-library//scala-library-.jar", + "file:///workspace/hello-scala/compile-resources" + ], + "classDirectory": "file:///workspace/out/hello-scala/compile.dest/classes" + } + ] +} \ No newline at end of file diff --git a/integration/ide/bsp-server/resources/snapshots/build-targets-jvm-run-environments.json b/integration/ide/bsp-server/resources/snapshots/build-targets-jvm-run-environments.json new file mode 100644 index 00000000000..585df6807fc --- /dev/null +++ b/integration/ide/bsp-server/resources/snapshots/build-targets-jvm-run-environments.json @@ -0,0 +1,33 @@ +{ + "items": [ + { + "target": { + "uri": "file:///workspace/hello-java" + }, + "classpath": [ + "file:///workspace/hello-java/compile-resources", + "file:///workspace/hello-java/resources", + "file:///workspace/out/hello-java/compile.dest/classes" + ], + "jvmOptions": [], + "workingDirectory": "/workspace", + "environmentVariables": {}, + "mainClasses": [] + }, + { + "target": { + "uri": "file:///workspace/hello-scala" + }, + "classpath": [ + "file:///coursier-cache/https/repo1.maven.org/maven2/org/scala-lang/scala-library//scala-library-.jar", + "file:///workspace/hello-scala/compile-resources", + "file:///workspace/hello-scala/resources", + "file:///workspace/out/hello-scala/compile.dest/classes" + ], + "jvmOptions": [], + "workingDirectory": "/workspace", + "environmentVariables": {}, + "mainClasses": [] + } + ] +} \ No newline at end of file diff --git a/integration/ide/bsp-server/resources/snapshots/build-targets-jvm-test-environments.json b/integration/ide/bsp-server/resources/snapshots/build-targets-jvm-test-environments.json new file mode 100644 index 00000000000..585df6807fc --- /dev/null +++ b/integration/ide/bsp-server/resources/snapshots/build-targets-jvm-test-environments.json @@ -0,0 +1,33 @@ +{ + "items": [ + { + "target": { + "uri": "file:///workspace/hello-java" + }, + "classpath": [ + "file:///workspace/hello-java/compile-resources", + "file:///workspace/hello-java/resources", + "file:///workspace/out/hello-java/compile.dest/classes" + ], + "jvmOptions": [], + "workingDirectory": "/workspace", + "environmentVariables": {}, + "mainClasses": [] + }, + { + "target": { + "uri": "file:///workspace/hello-scala" + }, + "classpath": [ + "file:///coursier-cache/https/repo1.maven.org/maven2/org/scala-lang/scala-library//scala-library-.jar", + "file:///workspace/hello-scala/compile-resources", + "file:///workspace/hello-scala/resources", + "file:///workspace/out/hello-scala/compile.dest/classes" + ], + "jvmOptions": [], + "workingDirectory": "/workspace", + "environmentVariables": {}, + "mainClasses": [] + } + ] +} \ No newline at end of file diff --git a/integration/ide/bsp-server/resources/snapshots/build-targets-output-paths.json b/integration/ide/bsp-server/resources/snapshots/build-targets-output-paths.json new file mode 100644 index 00000000000..12bebf136cf --- /dev/null +++ b/integration/ide/bsp-server/resources/snapshots/build-targets-output-paths.json @@ -0,0 +1,45 @@ +{ + "items": [ + { + "target": { + "uri": "file:///workspace/hello-java" + }, + "outputPaths": [] + }, + { + "target": { + "uri": "file:///workspace/hello-scala" + }, + "outputPaths": [] + }, + { + "target": { + "uri": "file:///workspace/mill-build" + }, + "outputPaths": [] + }, + { + "target": { + "uri": "file:///workspace/mill-synthetic-root-target" + }, + "outputPaths": [ + { + "uri": "file:///workspace/.idea/", + "kind": 2 + }, + { + "uri": "file:///workspace/out/", + "kind": 2 + }, + { + "uri": "file:///workspace/.bsp/", + "kind": 2 + }, + { + "uri": "file:///workspace/.bloop/", + "kind": 2 + } + ] + } + ] +} \ No newline at end of file diff --git a/integration/ide/bsp-server/resources/snapshots/build-targets-resources.json b/integration/ide/bsp-server/resources/snapshots/build-targets-resources.json new file mode 100644 index 00000000000..fc0ab5736d1 --- /dev/null +++ b/integration/ide/bsp-server/resources/snapshots/build-targets-resources.json @@ -0,0 +1,22 @@ +{ + "items": [ + { + "target": { + "uri": "file:///workspace/hello-java" + }, + "resources": [] + }, + { + "target": { + "uri": "file:///workspace/hello-scala" + }, + "resources": [] + }, + { + "target": { + "uri": "file:///workspace/mill-build" + }, + "resources": [] + } + ] +} \ No newline at end of file diff --git a/integration/ide/bsp-server/resources/snapshots/build-targets-scalac-options.json b/integration/ide/bsp-server/resources/snapshots/build-targets-scalac-options.json new file mode 100644 index 00000000000..043b0582ab6 --- /dev/null +++ b/integration/ide/bsp-server/resources/snapshots/build-targets-scalac-options.json @@ -0,0 +1,25 @@ +{ + "items": [ + { + "target": { + "uri": "file:///workspace/hello-java" + }, + "options": [], + "classpath": [ + "file:///workspace/hello-java/compile-resources" + ], + "classDirectory": "file:///workspace/out/hello-java/compile.dest/classes" + }, + { + "target": { + "uri": "file:///workspace/hello-scala" + }, + "options": [], + "classpath": [ + "file:///coursier-cache/https/repo1.maven.org/maven2/org/scala-lang/scala-library//scala-library-.jar", + "file:///workspace/hello-scala/compile-resources" + ], + "classDirectory": "file:///workspace/out/hello-scala/compile.dest/classes" + } + ] +} \ No newline at end of file diff --git a/integration/ide/bsp-server/resources/snapshots/build-targets-sources.json b/integration/ide/bsp-server/resources/snapshots/build-targets-sources.json new file mode 100644 index 00000000000..4010ae48e91 --- /dev/null +++ b/integration/ide/bsp-server/resources/snapshots/build-targets-sources.json @@ -0,0 +1,57 @@ +{ + "items": [ + { + "target": { + "uri": "file:///workspace/hello-java" + }, + "sources": [ + { + "uri": "file:///workspace/hello-java/src", + "kind": 2, + "generated": false + } + ] + }, + { + "target": { + "uri": "file:///workspace/hello-scala" + }, + "sources": [ + { + "uri": "file:///workspace/hello-scala/src", + "kind": 2, + "generated": false + } + ] + }, + { + "target": { + "uri": "file:///workspace/mill-build" + }, + "sources": [ + { + "uri": "file:///workspace/build.mill", + "kind": 1, + "generated": false + }, + { + "uri": "file:///workspace/out/mill-build/generateScriptSources.dest", + "kind": 2, + "generated": true + } + ] + }, + { + "target": { + "uri": "file:///workspace/mill-synthetic-root-target" + }, + "sources": [ + { + "uri": "file:///workspace/src", + "kind": 2, + "generated": false + } + ] + } + ] +} \ No newline at end of file diff --git a/integration/ide/bsp-server/resources/snapshots/initialize-build-result.json b/integration/ide/bsp-server/resources/snapshots/initialize-build-result.json new file mode 100644 index 00000000000..207de64635d --- /dev/null +++ b/integration/ide/bsp-server/resources/snapshots/initialize-build-result.json @@ -0,0 +1,38 @@ +{ + "displayName": "mill-bsp", + "version": "", + "bspVersion": "", + "capabilities": { + "compileProvider": { + "languageIds": [ + "java", + "scala" + ] + }, + "testProvider": { + "languageIds": [ + "java", + "scala" + ] + }, + "runProvider": { + "languageIds": [ + "java", + "scala" + ] + }, + "debugProvider": { + "languageIds": [] + }, + "inverseSourcesProvider": true, + "dependencySourcesProvider": true, + "dependencyModulesProvider": true, + "resourcesProvider": true, + "outputPathsProvider": true, + "buildTargetChangedProvider": false, + "jvmRunEnvironmentProvider": true, + "jvmTestEnvironmentProvider": true, + "canReload": true, + "jvmCompileClasspathProvider": false + } +} \ No newline at end of file diff --git a/integration/ide/bsp-server/resources/snapshots/workspace-build-targets.json b/integration/ide/bsp-server/resources/snapshots/workspace-build-targets.json new file mode 100644 index 00000000000..c2cd21ab20a --- /dev/null +++ b/integration/ide/bsp-server/resources/snapshots/workspace-build-targets.json @@ -0,0 +1,119 @@ +{ + "targets": [ + { + "id": { + "uri": "file:///workspace/hello-java" + }, + "displayName": "hello-java", + "baseDirectory": "file:///workspace/hello-java", + "tags": [ + "library", + "application" + ], + "languageIds": [ + "java" + ], + "dependencies": [], + "capabilities": { + "canCompile": true, + "canTest": false, + "canRun": true, + "canDebug": false + }, + "dataKind": "jvm", + "data": { + "javaHome": "file:///java-home/", + "javaVersion": "" + } + }, + { + "id": { + "uri": "file:///workspace/hello-scala" + }, + "displayName": "hello-scala", + "baseDirectory": "file:///workspace/hello-scala", + "tags": [ + "library", + "application" + ], + "languageIds": [ + "java", + "scala" + ], + "dependencies": [], + "capabilities": { + "canCompile": true, + "canTest": false, + "canRun": true, + "canDebug": false + }, + "dataKind": "scala", + "data": { + "scalaOrganization": "org.scala-lang", + "scalaVersion": "", + "scalaBinaryVersion": "2.13", + "platform": 1, + "jars": [ + "file:///coursier-cache/https/repo1.maven.org/maven2/org/scala-lang/scala-compiler//scala-compiler-.jar", + "file:///coursier-cache/https/repo1.maven.org/maven2/org/scala-lang/scala-reflect//scala-reflect-.jar", + "file:///coursier-cache/https/repo1.maven.org/maven2/org/scala-lang/scala-library//scala-library-.jar", + "file:///coursier-cache/https/repo1.maven.org/maven2/io/github/java-diff-utils/java-diff-utils//java-diff-utils-.jar", + "file:///coursier-cache/https/repo1.maven.org/maven2/org/jline/jline//jline-.jar", + "file:///coursier-cache/https/repo1.maven.org/maven2/net/java/dev/jna/jna//jna-.jar" + ] + } + }, + { + "id": { + "uri": "file:///workspace/mill-build" + }, + "displayName": "mill-build/", + "baseDirectory": "file:///workspace/mill-build", + "tags": [ + "library", + "application" + ], + "languageIds": [ + "java", + "scala" + ], + "dependencies": [], + "capabilities": { + "canCompile": true, + "canTest": false, + "canRun": true, + "canDebug": false + }, + "dataKind": "scala", + "data": { + "scalaOrganization": "org.scala-lang", + "scalaVersion": "", + "scalaBinaryVersion": "2.13", + "platform": 1, + "jars": [ + "file:///coursier-cache/https/repo1.maven.org/maven2/org/scala-lang/scala-compiler//scala-compiler-.jar", + "file:///coursier-cache/https/repo1.maven.org/maven2/org/scala-lang/scala-reflect//scala-reflect-.jar", + "file:///coursier-cache/https/repo1.maven.org/maven2/org/scala-lang/scala-library//scala-library-.jar" + ] + } + }, + { + "id": { + "uri": "file:///workspace/mill-synthetic-root-target" + }, + "displayName": "mill-synthetic-root", + "baseDirectory": "file:///workspace", + "tags": [ + "manual" + ], + "languageIds": [], + "dependencies": [], + "capabilities": { + "canCompile": false, + "canTest": false, + "canRun": false, + "canDebug": false + } + } + ] +} \ No newline at end of file diff --git a/integration/ide/bsp-install/src/BspInstallDebugTests.scala b/integration/ide/bsp-server/src/BspInstallDebugTests.scala similarity index 89% rename from integration/ide/bsp-install/src/BspInstallDebugTests.scala rename to integration/ide/bsp-server/src/BspInstallDebugTests.scala index 2294dab3571..7458444e6a2 100644 --- a/integration/ide/bsp-install/src/BspInstallDebugTests.scala +++ b/integration/ide/bsp-server/src/BspInstallDebugTests.scala @@ -5,6 +5,8 @@ import mill.bsp.Constants import utest._ object BspInstallDebugTests extends UtestIntegrationTestSuite { + override protected def workspaceSourcePath: os.Path = + super.workspaceSourcePath / "project" val bsp4jVersion: String = sys.props.getOrElse("BSP4J_VERSION", ???) // we purposely enable debugging in this simulated test env diff --git a/integration/ide/bsp-server/src/BspServerTestUtil.scala b/integration/ide/bsp-server/src/BspServerTestUtil.scala new file mode 100644 index 00000000000..f7a82cb11ad --- /dev/null +++ b/integration/ide/bsp-server/src/BspServerTestUtil.scala @@ -0,0 +1,214 @@ +package mill.integration + +import ch.epfl.scala.{bsp4j => b} +import com.google.gson.{Gson, GsonBuilder} +import coursier.cache.CacheDefaults +import mill.api.BuildInfo +import mill.bsp.Constants +import org.eclipse.{lsp4j => l} + +import java.io.ByteArrayOutputStream +import java.util.concurrent.atomic.AtomicInteger +import java.util.concurrent.{ExecutorService, Executors, ThreadFactory} + +import scala.jdk.CollectionConverters._ +import scala.reflect.ClassTag + +object BspServerTestUtil { + + val updateSnapshots = false + + def bsp4jVersion: String = sys.props.getOrElse("BSP4J_VERSION", ???) + + trait DummyBuildClient extends b.BuildClient { + def onBuildLogMessage(params: b.LogMessageParams): Unit = () + def onBuildPublishDiagnostics(params: b.PublishDiagnosticsParams): Unit = () + def onBuildShowMessage(params: b.ShowMessageParams): Unit = () + def onBuildTargetDidChange(params: b.DidChangeBuildTarget): Unit = () + def onBuildTaskFinish(params: b.TaskFinishParams): Unit = () + def onBuildTaskProgress(params: b.TaskProgressParams): Unit = () + def onBuildTaskStart(params: b.TaskStartParams): Unit = () + def onRunPrintStderr(params: b.PrintParams): Unit = () + def onRunPrintStdout(params: b.PrintParams): Unit = () + } + + object DummyBuildClient extends DummyBuildClient + + val gson: Gson = new GsonBuilder().setPrettyPrinting().create() + def compareWithGsonSnapshot[T: ClassTag]( + value: T, + snapshotPath: os.Path, + normalizedLocalValues: Seq[(String, String)] = Nil + ): Unit = { + + def normalizeLocalValues(input: String, inverse: Boolean = false): String = + normalizedLocalValues.foldLeft(input) { + case (input0, (from0, to0)) => + val (from, to) = if (inverse) (to0, from0) else (from0, to0) + input0.replace(from, to) + } + + // This can be false only when generating test data for the first time. + // In that case, updateSnapshots needs to be true, so that we write test data on disk. + val exists = os.exists(snapshotPath) + val expectedValueOpt = Option.when(exists) { + gson.fromJson( + normalizeLocalValues(os.read(snapshotPath), inverse = true), + implicitly[ClassTag[T]].runtimeClass + ) + } + + if (!expectedValueOpt.contains(value)) { + lazy val jsonStr = gson.toJson( + value, + implicitly[ClassTag[T]].runtimeClass + ) + if (updateSnapshots) { + System.err.println(if (exists) s"Updating $snapshotPath" else s"Writing $snapshotPath") + os.write.over(snapshotPath, normalizeLocalValues(jsonStr), createFolders = true) + } else { + System.err.println("Expected JSON:") + System.err.println(jsonStr) + Predef.assert( + false, + if (exists) + s"""Error: value differs from snapshot in $snapshotPath + | + |You might want to set BspServerTestUtil.updateSnapshots to true, + |run this test again, and commit the updated test data files. + |""".stripMargin + else s"Error: no snapshot found at $snapshotPath" + ) + } + } else if (updateSnapshots) { + // Snapshot on disk might need to be updated anyway, if normalizedLocalValues changed + // and new strings should be replaced + val obtainedJsonStr = normalizeLocalValues( + gson.toJson( + value, + implicitly[ClassTag[T]].runtimeClass + ) + ) + val expectedJsonStr = os.read(snapshotPath) + if (obtainedJsonStr != expectedJsonStr) { + System.err.println(s"Updating $snapshotPath") + os.write.over(snapshotPath, obtainedJsonStr) + } + } + } + + val bspJsonrpcPool: ExecutorService = Executors.newCachedThreadPool( + new ThreadFactory { + val counter = new AtomicInteger + def newThread(runnable: Runnable): Thread = { + val t = new Thread(runnable, s"mill-bsp-integration-${counter.incrementAndGet()}") + t.setDaemon(true) + t + } + } + ) + + trait MillBuildServer extends b.BuildServer with b.JvmBuildServer + with b.JavaBuildServer with b.ScalaBuildServer + + def withBspServer[T]( + workspacePath: os.Path, + millTestSuiteEnv: Map[String, String] + )(f: (MillBuildServer, b.InitializeBuildResult) => T): T = { + + val bspMetadataFile = workspacePath / Constants.bspDir / s"${Constants.serverName}.json" + assert(os.exists(bspMetadataFile)) + val contents = os.read(bspMetadataFile) + assert( + !contents.contains("--debug"), + contents.contains(s""""bspVersion":"$bsp4jVersion"""") + ) + + val outputOnErrorOnly = System.getenv("CI") != null + + val contentsJson = ujson.read(contents) + val bspCommand = contentsJson("argv").arr.map(_.str) + val stderr = new ByteArrayOutputStream + val proc = os.proc(bspCommand).spawn( + cwd = workspacePath, + stderr = + if (outputOnErrorOnly) + os.ProcessOutput { (bytes, len) => + stderr.write(bytes, 0, len) + } + else os.Inherit, + env = millTestSuiteEnv + ) + + val client: b.BuildClient = DummyBuildClient + + var success = false + try { + val launcher = new l.jsonrpc.Launcher.Builder[MillBuildServer] + .setExecutorService(bspJsonrpcPool) + .setInput(proc.stdout.wrapped) + .setOutput(proc.stdin.wrapped) + .setRemoteInterface(classOf[MillBuildServer]) + .setLocalService(client) + .setExceptionHandler { t => + System.err.println(s"Error during LSP processing: $t") + t.printStackTrace(System.err) + l.jsonrpc.RemoteEndpoint.DEFAULT_EXCEPTION_HANDLER.apply(t) + } + .create() + + launcher.startListening() + + val buildServer = launcher.getRemoteProxy() + + val initRes = buildServer.buildInitialize( + new b.InitializeBuildParams( + "Mill Integration", + BuildInfo.millVersion, + b.Bsp4j.PROTOCOL_VERSION, + workspacePath.toNIO.toUri.toASCIIString, + new b.BuildClientCapabilities(List("java", "scala").asJava) + ) + ).get() + + val value = + try f(buildServer, initRes) + finally buildServer.buildShutdown().get() + success = true + value + } finally { + proc.stdin.close() + proc.stdout.close() + + proc.join(30000L) + + if (!success && outputOnErrorOnly) { + System.err.println(" == BSP server output ==") + System.err.write(stderr.toByteArray) + System.err.println(" == end of BSP server output ==") + } + } + } + + lazy val millWorkspace: os.Path = { + val value = Option(System.getenv("MILL_PROJECT_ROOT")).getOrElse(???) + os.Path(value) + } + + def normalizeLocalValuesForTesting( + workspacePath: os.Path, + coursierCache: os.Path = os.Path(CacheDefaults.location), + javaHome: os.Path = os.Path(sys.props("java.home")), + javaVersion: String = sys.props("java.version") + ): Seq[(String, String)] = + Seq( + workspacePath.toNIO.toUri.toASCIIString.stripSuffix("/") -> "file:///workspace", + coursierCache.toNIO.toUri.toASCIIString -> "file:///coursier-cache/", + millWorkspace.toNIO.toUri.toASCIIString -> "file:///mill-workspace/", + javaHome.toNIO.toUri.toASCIIString.stripSuffix("/") -> "file:///java-home", + ("\"" + javaVersion + "\"") -> "\"\"", + workspacePath.toString -> "/workspace", + coursierCache.toString -> "/coursier-cache", + millWorkspace.toString -> "/mill-workspace" + ) +} diff --git a/integration/ide/bsp-server/src/BspServerTests.scala b/integration/ide/bsp-server/src/BspServerTests.scala new file mode 100644 index 00000000000..cf4c67e83c5 --- /dev/null +++ b/integration/ide/bsp-server/src/BspServerTests.scala @@ -0,0 +1,164 @@ +package mill.integration + +import ch.epfl.scala.{bsp4j => b} +import mill.api.BuildInfo +import mill.bsp.Constants +import mill.integration.BspServerTestUtil._ +import mill.testkit.UtestIntegrationTestSuite +import utest._ + +import scala.jdk.CollectionConverters._ + +object BspServerTests extends UtestIntegrationTestSuite { + def snapshotsPath: os.Path = + super.workspaceSourcePath / "snapshots" + override protected def workspaceSourcePath: os.Path = + super.workspaceSourcePath / "project" + + def tests: Tests = Tests { + test("requestSnapshots") - integrationTest { tester => + import tester._ + eval( + "mill.bsp.BSP/install", + stdout = os.Inherit, + stderr = os.Inherit, + check = true, + env = Map("MILL_MAIN_CLI" -> tester.millExecutable.toString) + ) + + withBspServer( + workspacePath, + millTestSuiteEnv + ) { (buildServer, initRes) => + val scalaVersion = sys.props.getOrElse("TEST_SCALA_2_13_VERSION", ???) + val scalaTransitiveSubstitutions = { + val scalaFetchRes = coursierapi.Fetch.create() + .addDependencies(coursierapi.Dependency.of( + "org.scala-lang", + "scala-compiler", + scalaVersion + )) + .fetchResult() + scalaFetchRes.getDependencies.asScala + .filter(dep => dep.getModule.getOrganization != "org.scala-lang") + .map { dep => + val organization = dep.getModule.getOrganization + val name = dep.getModule.getName + val prefix = (organization.split('.') :+ name).mkString("/") + def basePath(version: String): String = + s"$prefix/$version/$name-$version" + basePath(dep.getVersion) -> basePath(s"<$name-version>") + } + } + + val normalizedLocalValues = + normalizeLocalValuesForTesting(workspacePath) ++ scalaTransitiveSubstitutions ++ Seq( + scalaVersion -> "" + ) + + compareWithGsonSnapshot( + initRes, + snapshotsPath / "initialize-build-result.json", + normalizedLocalValues = Seq( + BuildInfo.millVersion -> "", + Constants.bspProtocolVersion -> "" + ) + ) + + val buildTargets = buildServer.workspaceBuildTargets().get() + compareWithGsonSnapshot( + buildTargets, + snapshotsPath / "workspace-build-targets.json", + normalizedLocalValues = normalizedLocalValues + ) + + val targetIds = buildTargets.getTargets.asScala.map(_.getId).asJava + val metaBuildTargetId = new b.BuildTargetIdentifier( + (workspacePath / "mill-build").toNIO.toUri.toASCIIString.stripSuffix("/") + ) + assert(targetIds.contains(metaBuildTargetId)) + val targetIdsSubset = targetIds.asScala.filter(_ != metaBuildTargetId).asJava + + compareWithGsonSnapshot( + buildServer + .buildTargetSources(new b.SourcesParams(targetIds)) + .get(), + snapshotsPath / "build-targets-sources.json", + normalizedLocalValues = normalizedLocalValues + ) + + compareWithGsonSnapshot( + buildServer + .buildTargetDependencySources(new b.DependencySourcesParams(targetIdsSubset)) + .get(), + snapshotsPath / "build-targets-dependency-sources.json", + normalizedLocalValues = normalizedLocalValues + ) + + compareWithGsonSnapshot( + buildServer + .buildTargetDependencyModules(new b.DependencyModulesParams(targetIdsSubset)) + .get(), + snapshotsPath / "build-targets-dependency-modules.json", + normalizedLocalValues = normalizedLocalValues + ) + + compareWithGsonSnapshot( + buildServer + .buildTargetResources(new b.ResourcesParams(targetIds)) + .get(), + snapshotsPath / "build-targets-resources.json", + normalizedLocalValues = normalizedLocalValues + ) + + compareWithGsonSnapshot( + buildServer + .buildTargetOutputPaths(new b.OutputPathsParams(targetIds)) + .get(), + snapshotsPath / "build-targets-output-paths.json", + normalizedLocalValues = normalizedLocalValues + ) + + compareWithGsonSnapshot( + buildServer + .buildTargetJvmRunEnvironment(new b.JvmRunEnvironmentParams(targetIdsSubset)) + .get(), + snapshotsPath / "build-targets-jvm-run-environments.json", + normalizedLocalValues = normalizedLocalValues + ) + + compareWithGsonSnapshot( + buildServer + .buildTargetJvmTestEnvironment(new b.JvmTestEnvironmentParams(targetIdsSubset)) + .get(), + snapshotsPath / "build-targets-jvm-test-environments.json", + normalizedLocalValues = normalizedLocalValues + ) + + compareWithGsonSnapshot( + buildServer + .buildTargetJvmCompileClasspath(new b.JvmCompileClasspathParams(targetIdsSubset)) + .get(), + snapshotsPath / "build-targets-compile-classpaths.json", + normalizedLocalValues = normalizedLocalValues + ) + + compareWithGsonSnapshot( + buildServer + .buildTargetJavacOptions(new b.JavacOptionsParams(targetIdsSubset)) + .get(), + snapshotsPath / "build-targets-javac-options.json", + normalizedLocalValues = normalizedLocalValues + ) + + compareWithGsonSnapshot( + buildServer + .buildTargetScalacOptions(new b.ScalacOptionsParams(targetIdsSubset)) + .get(), + snapshotsPath / "build-targets-scalac-options.json", + normalizedLocalValues = normalizedLocalValues + ) + } + } + } +} diff --git a/integration/package.mill b/integration/package.mill index 151a4883c32..86e823a78b8 100644 --- a/integration/package.mill +++ b/integration/package.mill @@ -71,9 +71,18 @@ object `package` extends RootModule { object feature extends Cross[IntegrationCrossModule](build.listIn(millSourcePath / "feature")) object invalidation extends Cross[IntegrationCrossModule](build.listIn(millSourcePath / "invalidation")) - object ide extends Cross[IntegrationCrossModule](build.listIn(millSourcePath / "ide")) + object ide extends Cross[IdeIntegrationCrossModule](build.listIn(millSourcePath / "ide")) trait IntegrationCrossModule extends build.MillScalaModule with IntegrationTestModule { override def moduleDeps = super[IntegrationTestModule].moduleDeps + def forkEnv = super.forkEnv() ++ Seq( + "MILL_PROJECT_ROOT" -> T.workspace.toString, + "TEST_SCALA_2_13_VERSION" -> build.Deps.testScala213Version + ) + } + trait IdeIntegrationCrossModule extends IntegrationCrossModule { + def ivyDeps = super.ivyDeps() ++ Agg( + build.Deps.bsp4j + ) } /** Deploy freshly build mill for use in tests */ From dc0733a3c7bb852deaba14a6f7ae4368533263c4 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Tue, 8 Oct 2024 15:37:46 +0200 Subject: [PATCH 12/47] Rebootstrap Mill on latest master (#3689) Last follow up to https://github.com/com-lihaoyi/mill/pull/3688, to ensure the final published zips are valid --- .config/mill-version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.config/mill-version b/.config/mill-version index fc57f1a3116..4b1ab6292ae 100644 --- a/.config/mill-version +++ b/.config/mill-version @@ -1 +1 @@ -0.12.0-RC3-21-8b063c \ No newline at end of file +0.12.0-RC3-23-4db51d From d3afbf6ac8fcc6947bbf16047c4200c23c01bb64 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Tue, 8 Oct 2024 20:27:01 +0200 Subject: [PATCH 13/47] Ensure prompt shows inside exclusive command evaluations (#3690) Fixes https://github.com/com-lihaoyi/mill/issues/3677 * We need to introduce a `Logger#withPromptUnpaused` API, called internally within `EvaluatorImpl`, to make sure we turn on the prompt by default even when called within a `exclusive=true` command which turned it off * We cannot use `SystemStreams.original`, because they do not go through Mills input redirection and so may arrive slightly out of order * `def run` in `build.mill` has to be a `Task.Command(exclusive = true)` for testing stuff using `mill dist.run` to work properly * I moved some of the gnarlier `paused`/`pausedNoticed`/`stopped` state from `PromptLogger` to an encapsulated `PromperLogger#RunningState` class where we can control the mutation and ensure it is done correctly * Moved the periodic calling of `readTerminalDims` out of the `synchronized` block, only the subsequent storing of the data in memory needs to be synchronized --- build.mill | 2 +- main/api/src/mill/api/Logger.scala | 1 + main/api/src/mill/api/SystemStreams.scala | 8 + main/eval/src/mill/eval/EvaluatorImpl.scala | 4 +- main/util/src/mill/util/MultiLogger.scala | 3 + main/util/src/mill/util/PrefixLogger.scala | 1 + main/util/src/mill/util/PromptLogger.scala | 191 +++++++++++------- main/util/src/mill/util/ProxyLogger.scala | 1 + .../src/mill/runner/MillBuildBootstrap.scala | 5 +- runner/src/mill/runner/MillMain.scala | 3 +- 10 files changed, 141 insertions(+), 78 deletions(-) diff --git a/build.mill b/build.mill index 72dd2f4ca10..b06caa812ba 100644 --- a/build.mill +++ b/build.mill @@ -883,7 +883,7 @@ object dist extends MillPublishJavaModule { Jvm.createJar(Agg(), JarManifest(manifestEntries)) } - def run(args: Task[Args] = T.task(Args())) = T.command { + def run(args: Task[Args] = T.task(Args())) = Task.Command(exclusive = true) { args().value match { case Nil => mill.api.Result.Failure("Need to pass in cwd as first argument to dev.run") case wd0 +: rest => diff --git a/main/api/src/mill/api/Logger.scala b/main/api/src/mill/api/Logger.scala index add24defd80..bbb29fa7303 100644 --- a/main/api/src/mill/api/Logger.scala +++ b/main/api/src/mill/api/Logger.scala @@ -62,6 +62,7 @@ trait Logger extends AutoCloseable { private[mill] def removePromptLine(key: Seq[String]): Unit = () private[mill] def removePromptLine(): Unit = () private[mill] def withPromptPaused[T](t: => T): T = t + private[mill] def withPromptUnpaused[T](t: => T): T = t /** * @since Mill 0.10.5 diff --git a/main/api/src/mill/api/SystemStreams.scala b/main/api/src/mill/api/SystemStreams.scala index 29052c4c4af..e42b34d80ed 100644 --- a/main/api/src/mill/api/SystemStreams.scala +++ b/main/api/src/mill/api/SystemStreams.scala @@ -19,6 +19,14 @@ class SystemStreams( object SystemStreams { + /** + * The original system streams of this process, before any redirection. + * + * NOTE: you should not use this! They do not get captured properly by Mill's stdout/err + * redirection, and thus only get picked up from the Mill server log files asynchronously. + * That means that the logs may appear out of order, jumbling your logs and screwing up + * your terminal + */ val original = new SystemStreams(System.out, System.err, System.in) /** diff --git a/main/eval/src/mill/eval/EvaluatorImpl.scala b/main/eval/src/mill/eval/EvaluatorImpl.scala index e379d66b238..02e066419a9 100644 --- a/main/eval/src/mill/eval/EvaluatorImpl.scala +++ b/main/eval/src/mill/eval/EvaluatorImpl.scala @@ -56,7 +56,9 @@ private[mill] case class EvaluatorImpl( ): Evaluator.Results = { // TODO: cleanup once we break bin-compat in Mill 0.13 // disambiguate override hierarchy - super.evaluate(goals, reporter, testReporter, logger, serialCommandExec) + logger.withPromptUnpaused { + super.evaluate(goals, reporter, testReporter, logger, serialCommandExec) + } } override def evalOrThrow(exceptionFactory: Evaluator.Results => Throwable) diff --git a/main/util/src/mill/util/MultiLogger.scala b/main/util/src/mill/util/MultiLogger.scala index 0d3d897f77c..d6ad8855686 100644 --- a/main/util/src/mill/util/MultiLogger.scala +++ b/main/util/src/mill/util/MultiLogger.scala @@ -82,6 +82,9 @@ class MultiLogger( private[mill] override def withPromptPaused[T](t: => T): T = { logger1.withPromptPaused(logger2.withPromptPaused(t)) } + private[mill] override def withPromptUnpaused[T](t: => T): T = { + logger1.withPromptUnpaused(logger2.withPromptUnpaused(t)) + } override def enableTicker: Boolean = logger1.enableTicker || logger2.enableTicker diff --git a/main/util/src/mill/util/PrefixLogger.scala b/main/util/src/mill/util/PrefixLogger.scala index 853734f08ef..9920e8fccf6 100644 --- a/main/util/src/mill/util/PrefixLogger.scala +++ b/main/util/src/mill/util/PrefixLogger.scala @@ -114,6 +114,7 @@ class PrefixLogger( ) } private[mill] override def withPromptPaused[T](t: => T): T = logger0.withPromptPaused(t) + private[mill] override def withPromptUnpaused[T](t: => T): T = logger0.withPromptUnpaused(t) } object PrefixLogger { diff --git a/main/util/src/mill/util/PromptLogger.scala b/main/util/src/mill/util/PromptLogger.scala index daa65b0473b..e86e8bc1545 100644 --- a/main/util/src/mill/util/PromptLogger.scala +++ b/main/util/src/mill/util/PromptLogger.scala @@ -33,66 +33,71 @@ private[mill] class PromptLogger( readTerminalDims(terminfoPath).foreach(termDimensions = _) - private val state = new State( - titleText, - systemStreams0, - currentTimeMillis(), - () => termDimensions, - currentTimeMillis, - infoColor - ) + private object promptLineState extends PromptLineState( + titleText, + systemStreams0, + currentTimeMillis(), + () => termDimensions, + currentTimeMillis, + infoColor + ) - private val streamManager = new StreamManager( - enableTicker, - systemStreams0, - () => state.currentPromptBytes, - interactive = () => termDimensions._1.nonEmpty - ) + private object streamManager extends StreamManager( + enableTicker, + systemStreams0, + () => promptLineState.writeCurrentPrompt(), + interactive = () => termDimensions._1.nonEmpty + ) + + private object runningState extends RunningState( + enableTicker, + () => promptUpdaterThread.interrupt(), + clearOnPause = () => { + // Clear the prompt so the code in `t` has a blank terminal to work with + systemStreams0.err.write(AnsiNav.clearScreen(0).getBytes) + systemStreams0.err.flush() + }, + this + ) - @volatile var stopped = false - @volatile var paused = false - @volatile var pauseNoticed = false + val promptUpdaterThread = new Thread( + () => + while (!runningState.stopped) { + val promptUpdateInterval = + if (termDimensions._1.isDefined) promptUpdateIntervalMillis + else nonInteractivePromptUpdateIntervalMillis - val promptUpdaterThread = new Thread(() => - while (!stopped) { - val promptUpdateInterval = - if (termDimensions._1.isDefined) promptUpdateIntervalMillis - else nonInteractivePromptUpdateIntervalMillis + try Thread.sleep(promptUpdateInterval) + catch { case e: InterruptedException => /*do nothing*/ } - try Thread.sleep(promptUpdateInterval) - catch { case e: InterruptedException => /*do nothing*/ } + readTerminalDims(terminfoPath).foreach(termDimensions = _) - if (!paused) { synchronized { - // Double check the lock so if this was closed during the - // `Thread.sleep`, we skip refreshing the prompt this loop - if (!stopped) { - readTerminalDims(terminfoPath).foreach(termDimensions = _) + if (!runningState.paused && !runningState.stopped) { refreshPrompt() } } - } else { - pauseNoticed = true - } - } + }, + "prompt-logger-updater-thread" ) - def refreshPrompt(): Unit = state.refreshPrompt() + def refreshPrompt(): Unit = promptLineState.refreshPrompt() if (enableTicker && autoUpdate) promptUpdaterThread.start() def info(s: String): Unit = synchronized { systemStreams.err.println(s) } def error(s: String): Unit = synchronized { systemStreams.err.println(s) } - override def setPromptLeftHeader(s: String): Unit = synchronized { state.updateGlobal(s) } - override def clearPromptStatuses(): Unit = synchronized { state.clearStatuses() } + override def setPromptLeftHeader(s: String): Unit = + synchronized { promptLineState.updateGlobal(s) } + override def clearPromptStatuses(): Unit = synchronized { promptLineState.clearStatuses() } override def removePromptLine(key: Seq[String]): Unit = synchronized { - state.updateCurrent(key, None) + promptLineState.updateCurrent(key, None) } def ticker(s: String): Unit = () override def setPromptDetail(key: Seq[String], s: String): Unit = synchronized { - state.updateDetail(key, s) + promptLineState.updateDetail(key, s) } override def reportKey(key: Seq[String]): Unit = synchronized { @@ -111,7 +116,7 @@ private[mill] class PromptLogger( private val reportedIdentifiers = collection.mutable.Set.empty[Seq[String]] override def setPromptLine(key: Seq[String], verboseKeySuffix: String, message: String): Unit = synchronized { - state.updateCurrent(key, Some(s"[${key.mkString("-")}] $message")) + promptLineState.updateCurrent(key, Some(s"[${key.mkString("-")}] $message")) seenIdentifiers(key) = (verboseKeySuffix, message) } @@ -121,45 +126,80 @@ private[mill] class PromptLogger( override def close(): Unit = { synchronized { - if (enableTicker) state.refreshPrompt(ending = true) + if (enableTicker) promptLineState.refreshPrompt(ending = true) streamManager.close() - stopped = true + runningState.stop() } + // Needs to be outside the lock so we don't deadlock with `promptUpdaterThread` - // trying to take the lock one last time before exiting - promptUpdaterThread.interrupt() + // trying to take the lock one last time to check running/paused status before exiting promptUpdaterThread.join() } - def systemStreams = streamManager.systemStreams - - private[mill] override def withPromptPaused[T](t: => T): T = { - if (!enableTicker) t - else { - pauseNoticed = false - paused = true - promptUpdaterThread.interrupt() - try { - // After the prompt gets paused, wait until the `promptUpdaterThread` marks - // `pauseNoticed = true`, so we can be sure it's done printing out prompt updates for - // now and we can proceed with running `t` without any last updates slipping through - while (!pauseNoticed) Thread.sleep(2) - // Clear the prompt so the code in `t` has a blank terminal to work with - systemStreams0.err.write(AnsiNav.clearScreen(0).getBytes) - systemStreams0.err.flush() - t - - } finally paused = false - } - } + def systemStreams = streamManager.proxySystemStreams + + private[mill] override def withPromptPaused[T](t: => T): T = + runningState.withPromptPaused0(true, t) + private[mill] override def withPromptUnpaused[T](t: => T): T = + runningState.withPromptPaused0(false, t) } private[mill] object PromptLogger { + /** + * Manages the paused/unpaused/stopped state of the prompt logger. Encapsulate in a separate + * class because it has to maintain some invariants and ensure book-keeping is properly done + * when the paused state change, e.g. interrupting the prompt updater thread, waiting for + * `pauseNoticed` to fire and clearing the screen when the ticker is paused. + */ + class RunningState( + enableTicker: Boolean, + promptUpdaterThreadInterrupt: () => Unit, + clearOnPause: () => Unit, + // Share the same synchronized lock as the parent PromptLogger, to simplify + // reasoning about concurrency since it's not performance critical + synchronizer: PromptLogger + ) { + @volatile private var stopped0 = false + @volatile private var paused0 = false + def stopped = stopped0 + def paused = paused0 + def stop(): Unit = synchronizer.synchronized { + stopped0 = true + promptUpdaterThreadInterrupt() + } + + def setPaused(prevPaused: Boolean, nextPaused: Boolean): Unit = synchronizer.synchronized { + if (prevPaused != nextPaused) { + paused0 = nextPaused + if (nextPaused) { + promptUpdaterThreadInterrupt() + clearOnPause() + } + } + } + + def withPromptPaused0[T](innerPaused: Boolean, t: => T): T = { + if (!enableTicker) t + else { + val outerPaused = paused0 + try { + setPaused(outerPaused, innerPaused) + t + } finally setPaused(innerPaused, outerPaused) + } + } + } + + /** + * Manages the system stream management logic necessary as part of the prompt. Intercepts + * both stdout/stderr streams, ensuring the prompt is cleared before any output is printed, + * and ensuring the prompt is re-printed after the streams have become quiescent. + */ private class StreamManager( enableTicker: Boolean, systemStreams0: SystemStreams, - currentPromptBytes: () => Array[Byte], + writeCurrentPrompt: () => Unit, interactive: () => Boolean ) { @@ -170,7 +210,7 @@ private[mill] object PromptLogger { val pipe = new PipeStreams() val proxyOut = new ProxyStream.Output(pipe.output, ProxyStream.OUT) val proxyErr: ProxyStream.Output = new ProxyStream.Output(pipe.output, ProxyStream.ERR) - val systemStreams = new SystemStreams( + val proxySystemStreams = new SystemStreams( new PrintStream(proxyOut), new PrintStream(proxyErr), systemStreams0.in @@ -190,9 +230,7 @@ private[mill] object PromptLogger { // every small write when most such prompts will get immediately over-written // by subsequent writes if (enableTicker && src.available() == 0) { - if (interactive()) { - systemStreams0.err.write(currentPromptBytes()) - } + if (interactive()) writeCurrentPrompt() pumperState = PumperState.prompt } } @@ -207,7 +245,7 @@ private[mill] object PromptLogger { } } - val pumperThread = new Thread(pumper) + val pumperThread = new Thread(pumper, "prompt-logger-stream-pumper-thread") pumperThread.start() def close(): Unit = { @@ -219,7 +257,13 @@ private[mill] object PromptLogger { pumperThread.join() } } - private class State( + + /** + * Manages the state and logic necessary to render the status lines making up the prompt. + * Needs to maintain state to implement debouncing logic to ensure the prompt changes + * "smoothly" even as the underlying statuses may be rapidly changed during evaluation. + */ + private class PromptLineState( titleText: String, systemStreams0: SystemStreams, startTimeMillis: Long, @@ -250,8 +294,9 @@ private[mill] object PromptLogger { // Pre-compute the prelude and current prompt as byte arrays so that // writing them out is fast, since they get written out very frequently - @volatile var currentPromptBytes: Array[Byte] = Array[Byte]() + @volatile private var currentPromptBytes: Array[Byte] = Array[Byte]() + def writeCurrentPrompt(): Unit = systemStreams0.err.write(currentPromptBytes) private def updatePromptBytes(ending: Boolean = false) = { val now = currentTimeMillis() for (k <- statuses.keySet) { @@ -348,7 +393,7 @@ private[mill] object PromptLogger { if (consoleDims()._1.nonEmpty || statusesHashCode != lastRenderedPromptHash) { lastRenderedPromptHash = statusesHashCode updatePromptBytes(ending) - systemStreams0.err.write(currentPromptBytes) + writeCurrentPrompt() } } } diff --git a/main/util/src/mill/util/ProxyLogger.scala b/main/util/src/mill/util/ProxyLogger.scala index bf4ca5d4978..be7e847ad84 100644 --- a/main/util/src/mill/util/ProxyLogger.scala +++ b/main/util/src/mill/util/ProxyLogger.scala @@ -37,6 +37,7 @@ class ProxyLogger(logger: Logger) extends Logger { private[mill] override def removePromptLine(): Unit = logger.removePromptLine() private[mill] override def setPromptLeftHeader(s: String): Unit = logger.setPromptLeftHeader(s) private[mill] override def withPromptPaused[T](t: => T): T = logger.withPromptPaused(t) + private[mill] override def withPromptUnpaused[T](t: => T): T = logger.withPromptUnpaused(t) override def enableTicker = logger.enableTicker } diff --git a/runner/src/mill/runner/MillBuildBootstrap.scala b/runner/src/mill/runner/MillBuildBootstrap.scala index 288f57b9e8b..8cede3839fc 100644 --- a/runner/src/mill/runner/MillBuildBootstrap.scala +++ b/runner/src/mill/runner/MillBuildBootstrap.scala @@ -43,7 +43,8 @@ class MillBuildBootstrap( needBuildSc: Boolean, requestedMetaLevel: Option[Int], allowPositionalCommandArgs: Boolean, - systemExit: Int => Nothing + systemExit: Int => Nothing, + streams0: SystemStreams ) { import MillBuildBootstrap._ @@ -357,7 +358,7 @@ class MillBuildBootstrap( disableCallgraph = disableCallgraph, allowPositionalCommandArgs = allowPositionalCommandArgs, systemExit = systemExit, - exclusiveSystemStreams = SystemStreams.original + exclusiveSystemStreams = streams0 ) } diff --git a/runner/src/mill/runner/MillMain.scala b/runner/src/mill/runner/MillMain.scala index 10f343d1518..817e126af7c 100644 --- a/runner/src/mill/runner/MillMain.scala +++ b/runner/src/mill/runner/MillMain.scala @@ -263,7 +263,8 @@ object MillMain { needBuildSc = needBuildSc(config), requestedMetaLevel = config.metaLevel, config.allowPositional.value, - systemExit = systemExit + systemExit = systemExit, + streams0 = streams0 ).evaluate() } }, From c5c3264ccaa774751b56d1250d671f7695468c66 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Wed, 9 Oct 2024 07:55:38 +0200 Subject: [PATCH 14/47] Update mill-version --- .config/mill-version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.config/mill-version b/.config/mill-version index 4b1ab6292ae..264ae60564b 100644 --- a/.config/mill-version +++ b/.config/mill-version @@ -1 +1 @@ -0.12.0-RC3-23-4db51d +0.12.0-RC3-26-d3afbf From 8f64d9138497ad449fc8d0840a2eb317845f7fff Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 9 Oct 2024 11:41:32 +0200 Subject: [PATCH 15/47] Add Kotlin BSP integration test (#3643) This PR adds non-regression integration tests for Kotlin support via BSP. Kotlin modules are currently advertized as Java ones, and IntelliJ seems to handle fine Kotlin sources in those (using non-experimental BSP support). --- bsp/src/mill/bsp/Constants.scala | 2 +- build.mill | 3 +- .../bsp-server/resources/project/build.mill | 9 ++- .../build-targets-compile-classpaths.json | 10 +++ .../build-targets-dependency-modules.json | 11 +++ .../build-targets-dependency-sources.json | 9 +++ .../build-targets-javac-options.json | 12 ++++ .../build-targets-jvm-run-environments.json | 16 +++++ .../build-targets-jvm-test-environments.json | 16 +++++ .../snapshots/build-targets-output-paths.json | 6 ++ .../snapshots/build-targets-resources.json | 6 ++ .../build-targets-scalac-options.json | 12 ++++ .../snapshots/build-targets-sources.json | 12 ++++ .../snapshots/initialize-build-result.json | 9 ++- .../snapshots/workspace-build-targets.json | 27 ++++++++ .../bsp-server/src/BspServerTestUtil.scala | 2 +- .../ide/bsp-server/src/BspServerTests.scala | 68 ++++++++++++------- integration/package.mill | 3 +- .../src/mill/kotlinlib/KotlinModule.scala | 10 ++- .../src/mill/scalalib/bsp/BspModule.scala | 1 + 20 files changed, 210 insertions(+), 34 deletions(-) diff --git a/bsp/src/mill/bsp/Constants.scala b/bsp/src/mill/bsp/Constants.scala index 9dd59b05f48..735b27f3d61 100644 --- a/bsp/src/mill/bsp/Constants.scala +++ b/bsp/src/mill/bsp/Constants.scala @@ -6,6 +6,6 @@ private[mill] object Constants { val bspProtocolVersion = BuildInfo.bsp4jVersion val bspWorkerImplClass = "mill.bsp.worker.BspWorkerImpl" val bspWorkerBuildInfoClass = "mill.bsp.worker.BuildInfo" - val languages: Seq[String] = Seq("java", "scala") + val languages: Seq[String] = Seq("java", "scala", "kotlin") val serverName = "mill-bsp" } diff --git a/build.mill b/build.mill index b06caa812ba..f2691ab2b41 100644 --- a/build.mill +++ b/build.mill @@ -449,7 +449,8 @@ trait MillBaseTestsModule extends TestModule { s"-DTEST_SCALATEST_VERSION=${Deps.TestDeps.scalaTest.dep.version}", s"-DTEST_TEST_INTERFACE_VERSION=${Deps.sbtTestInterface.dep.version}", s"-DTEST_ZIOTEST_VERSION=${Deps.TestDeps.zioTest.dep.version}", - s"-DTEST_ZINC_VERSION=${Deps.zinc.dep.version}" + s"-DTEST_ZINC_VERSION=${Deps.zinc.dep.version}", + s"-DTEST_KOTLIN_VERSION=${Deps.kotlinCompiler.dep.version}" ) } diff --git a/integration/ide/bsp-server/resources/project/build.mill b/integration/ide/bsp-server/resources/project/build.mill index b07b6f12038..270a611fdd4 100644 --- a/integration/ide/bsp-server/resources/project/build.mill +++ b/integration/ide/bsp-server/resources/project/build.mill @@ -1,10 +1,13 @@ package build import mill._ -import mill.scalalib._ -object `hello-java` extends JavaModule +object `hello-java` extends scalalib.JavaModule -object `hello-scala` extends ScalaModule { +object `hello-scala` extends scalalib.ScalaModule { def scalaVersion = Option(System.getenv("TEST_SCALA_2_13_VERSION")).getOrElse(???) } + +object `hello-kotlin` extends kotlinlib.KotlinModule { + def kotlinVersion = Option(System.getenv("TEST_KOTLIN_VERSION")).getOrElse(???) +} diff --git a/integration/ide/bsp-server/resources/snapshots/build-targets-compile-classpaths.json b/integration/ide/bsp-server/resources/snapshots/build-targets-compile-classpaths.json index f67288640f0..f39f59fa7eb 100644 --- a/integration/ide/bsp-server/resources/snapshots/build-targets-compile-classpaths.json +++ b/integration/ide/bsp-server/resources/snapshots/build-targets-compile-classpaths.json @@ -8,6 +8,16 @@ "file:///workspace/hello-java/compile-resources" ] }, + { + "target": { + "uri": "file:///workspace/hello-kotlin" + }, + "classpath": [ + "file:///coursier-cache/https/repo1.maven.org/maven2/org/jetbrains/kotlin/kotlin-stdlib//kotlin-stdlib-.jar", + "file:///coursier-cache/https/repo1.maven.org/maven2/org/jetbrains/annotations//annotations-.jar", + "file:///workspace/hello-kotlin/compile-resources" + ] + }, { "target": { "uri": "file:///workspace/hello-scala" diff --git a/integration/ide/bsp-server/resources/snapshots/build-targets-dependency-modules.json b/integration/ide/bsp-server/resources/snapshots/build-targets-dependency-modules.json index 2820a7f6c4a..5dbf41724f1 100644 --- a/integration/ide/bsp-server/resources/snapshots/build-targets-dependency-modules.json +++ b/integration/ide/bsp-server/resources/snapshots/build-targets-dependency-modules.json @@ -6,6 +6,17 @@ }, "modules": [] }, + { + "target": { + "uri": "file:///workspace/hello-kotlin" + }, + "modules": [ + { + "name": "org.jetbrains.kotlin:kotlin-stdlib", + "version": "" + } + ] + }, { "target": { "uri": "file:///workspace/hello-scala" diff --git a/integration/ide/bsp-server/resources/snapshots/build-targets-dependency-sources.json b/integration/ide/bsp-server/resources/snapshots/build-targets-dependency-sources.json index 1a0bee2bbee..3be9d342e20 100644 --- a/integration/ide/bsp-server/resources/snapshots/build-targets-dependency-sources.json +++ b/integration/ide/bsp-server/resources/snapshots/build-targets-dependency-sources.json @@ -6,6 +6,15 @@ }, "sources": [] }, + { + "target": { + "uri": "file:///workspace/hello-kotlin" + }, + "sources": [ + "file:///coursier-cache/https/repo1.maven.org/maven2/org/jetbrains/kotlin/kotlin-stdlib//kotlin-stdlib--sources.jar", + "file:///coursier-cache/https/repo1.maven.org/maven2/org/jetbrains/annotations//annotations--sources.jar" + ] + }, { "target": { "uri": "file:///workspace/hello-scala" diff --git a/integration/ide/bsp-server/resources/snapshots/build-targets-javac-options.json b/integration/ide/bsp-server/resources/snapshots/build-targets-javac-options.json index 043b0582ab6..65e5269db0d 100644 --- a/integration/ide/bsp-server/resources/snapshots/build-targets-javac-options.json +++ b/integration/ide/bsp-server/resources/snapshots/build-targets-javac-options.json @@ -10,6 +10,18 @@ ], "classDirectory": "file:///workspace/out/hello-java/compile.dest/classes" }, + { + "target": { + "uri": "file:///workspace/hello-kotlin" + }, + "options": [], + "classpath": [ + "file:///coursier-cache/https/repo1.maven.org/maven2/org/jetbrains/kotlin/kotlin-stdlib//kotlin-stdlib-.jar", + "file:///coursier-cache/https/repo1.maven.org/maven2/org/jetbrains/annotations//annotations-.jar", + "file:///workspace/hello-kotlin/compile-resources" + ], + "classDirectory": "file:///workspace/out/hello-kotlin/compile.dest/classes" + }, { "target": { "uri": "file:///workspace/hello-scala" diff --git a/integration/ide/bsp-server/resources/snapshots/build-targets-jvm-run-environments.json b/integration/ide/bsp-server/resources/snapshots/build-targets-jvm-run-environments.json index 585df6807fc..40d001f52ff 100644 --- a/integration/ide/bsp-server/resources/snapshots/build-targets-jvm-run-environments.json +++ b/integration/ide/bsp-server/resources/snapshots/build-targets-jvm-run-environments.json @@ -14,6 +14,22 @@ "environmentVariables": {}, "mainClasses": [] }, + { + "target": { + "uri": "file:///workspace/hello-kotlin" + }, + "classpath": [ + "file:///coursier-cache/https/repo1.maven.org/maven2/org/jetbrains/kotlin/kotlin-stdlib//kotlin-stdlib-.jar", + "file:///coursier-cache/https/repo1.maven.org/maven2/org/jetbrains/annotations//annotations-.jar", + "file:///workspace/hello-kotlin/compile-resources", + "file:///workspace/hello-kotlin/resources", + "file:///workspace/out/hello-kotlin/compile.dest/classes" + ], + "jvmOptions": [], + "workingDirectory": "/workspace", + "environmentVariables": {}, + "mainClasses": [] + }, { "target": { "uri": "file:///workspace/hello-scala" diff --git a/integration/ide/bsp-server/resources/snapshots/build-targets-jvm-test-environments.json b/integration/ide/bsp-server/resources/snapshots/build-targets-jvm-test-environments.json index 585df6807fc..40d001f52ff 100644 --- a/integration/ide/bsp-server/resources/snapshots/build-targets-jvm-test-environments.json +++ b/integration/ide/bsp-server/resources/snapshots/build-targets-jvm-test-environments.json @@ -14,6 +14,22 @@ "environmentVariables": {}, "mainClasses": [] }, + { + "target": { + "uri": "file:///workspace/hello-kotlin" + }, + "classpath": [ + "file:///coursier-cache/https/repo1.maven.org/maven2/org/jetbrains/kotlin/kotlin-stdlib//kotlin-stdlib-.jar", + "file:///coursier-cache/https/repo1.maven.org/maven2/org/jetbrains/annotations//annotations-.jar", + "file:///workspace/hello-kotlin/compile-resources", + "file:///workspace/hello-kotlin/resources", + "file:///workspace/out/hello-kotlin/compile.dest/classes" + ], + "jvmOptions": [], + "workingDirectory": "/workspace", + "environmentVariables": {}, + "mainClasses": [] + }, { "target": { "uri": "file:///workspace/hello-scala" diff --git a/integration/ide/bsp-server/resources/snapshots/build-targets-output-paths.json b/integration/ide/bsp-server/resources/snapshots/build-targets-output-paths.json index 12bebf136cf..167fc54a654 100644 --- a/integration/ide/bsp-server/resources/snapshots/build-targets-output-paths.json +++ b/integration/ide/bsp-server/resources/snapshots/build-targets-output-paths.json @@ -6,6 +6,12 @@ }, "outputPaths": [] }, + { + "target": { + "uri": "file:///workspace/hello-kotlin" + }, + "outputPaths": [] + }, { "target": { "uri": "file:///workspace/hello-scala" diff --git a/integration/ide/bsp-server/resources/snapshots/build-targets-resources.json b/integration/ide/bsp-server/resources/snapshots/build-targets-resources.json index fc0ab5736d1..7360d1c92ff 100644 --- a/integration/ide/bsp-server/resources/snapshots/build-targets-resources.json +++ b/integration/ide/bsp-server/resources/snapshots/build-targets-resources.json @@ -6,6 +6,12 @@ }, "resources": [] }, + { + "target": { + "uri": "file:///workspace/hello-kotlin" + }, + "resources": [] + }, { "target": { "uri": "file:///workspace/hello-scala" diff --git a/integration/ide/bsp-server/resources/snapshots/build-targets-scalac-options.json b/integration/ide/bsp-server/resources/snapshots/build-targets-scalac-options.json index 043b0582ab6..65e5269db0d 100644 --- a/integration/ide/bsp-server/resources/snapshots/build-targets-scalac-options.json +++ b/integration/ide/bsp-server/resources/snapshots/build-targets-scalac-options.json @@ -10,6 +10,18 @@ ], "classDirectory": "file:///workspace/out/hello-java/compile.dest/classes" }, + { + "target": { + "uri": "file:///workspace/hello-kotlin" + }, + "options": [], + "classpath": [ + "file:///coursier-cache/https/repo1.maven.org/maven2/org/jetbrains/kotlin/kotlin-stdlib//kotlin-stdlib-.jar", + "file:///coursier-cache/https/repo1.maven.org/maven2/org/jetbrains/annotations//annotations-.jar", + "file:///workspace/hello-kotlin/compile-resources" + ], + "classDirectory": "file:///workspace/out/hello-kotlin/compile.dest/classes" + }, { "target": { "uri": "file:///workspace/hello-scala" diff --git a/integration/ide/bsp-server/resources/snapshots/build-targets-sources.json b/integration/ide/bsp-server/resources/snapshots/build-targets-sources.json index 4010ae48e91..21295e603a6 100644 --- a/integration/ide/bsp-server/resources/snapshots/build-targets-sources.json +++ b/integration/ide/bsp-server/resources/snapshots/build-targets-sources.json @@ -12,6 +12,18 @@ } ] }, + { + "target": { + "uri": "file:///workspace/hello-kotlin" + }, + "sources": [ + { + "uri": "file:///workspace/hello-kotlin/src", + "kind": 2, + "generated": false + } + ] + }, { "target": { "uri": "file:///workspace/hello-scala" diff --git a/integration/ide/bsp-server/resources/snapshots/initialize-build-result.json b/integration/ide/bsp-server/resources/snapshots/initialize-build-result.json index 207de64635d..3e7a009526d 100644 --- a/integration/ide/bsp-server/resources/snapshots/initialize-build-result.json +++ b/integration/ide/bsp-server/resources/snapshots/initialize-build-result.json @@ -6,19 +6,22 @@ "compileProvider": { "languageIds": [ "java", - "scala" + "scala", + "kotlin" ] }, "testProvider": { "languageIds": [ "java", - "scala" + "scala", + "kotlin" ] }, "runProvider": { "languageIds": [ "java", - "scala" + "scala", + "kotlin" ] }, "debugProvider": { diff --git a/integration/ide/bsp-server/resources/snapshots/workspace-build-targets.json b/integration/ide/bsp-server/resources/snapshots/workspace-build-targets.json index c2cd21ab20a..68aa310ab88 100644 --- a/integration/ide/bsp-server/resources/snapshots/workspace-build-targets.json +++ b/integration/ide/bsp-server/resources/snapshots/workspace-build-targets.json @@ -26,6 +26,33 @@ "javaVersion": "" } }, + { + "id": { + "uri": "file:///workspace/hello-kotlin" + }, + "displayName": "hello-kotlin", + "baseDirectory": "file:///workspace/hello-kotlin", + "tags": [ + "library", + "application" + ], + "languageIds": [ + "java", + "kotlin" + ], + "dependencies": [], + "capabilities": { + "canCompile": true, + "canTest": false, + "canRun": true, + "canDebug": false + }, + "dataKind": "jvm", + "data": { + "javaHome": "file:///java-home/", + "javaVersion": "" + } + }, { "id": { "uri": "file:///workspace/hello-scala" diff --git a/integration/ide/bsp-server/src/BspServerTestUtil.scala b/integration/ide/bsp-server/src/BspServerTestUtil.scala index f7a82cb11ad..46166459c71 100644 --- a/integration/ide/bsp-server/src/BspServerTestUtil.scala +++ b/integration/ide/bsp-server/src/BspServerTestUtil.scala @@ -167,7 +167,7 @@ object BspServerTestUtil { BuildInfo.millVersion, b.Bsp4j.PROTOCOL_VERSION, workspacePath.toNIO.toUri.toASCIIString, - new b.BuildClientCapabilities(List("java", "scala").asJava) + new b.BuildClientCapabilities(List("java", "scala", "kotlin").asJava) ) ).get() diff --git a/integration/ide/bsp-server/src/BspServerTests.scala b/integration/ide/bsp-server/src/BspServerTests.scala index cf4c67e83c5..ead165ffecc 100644 --- a/integration/ide/bsp-server/src/BspServerTests.scala +++ b/integration/ide/bsp-server/src/BspServerTests.scala @@ -15,6 +15,26 @@ object BspServerTests extends UtestIntegrationTestSuite { override protected def workspaceSourcePath: os.Path = super.workspaceSourcePath / "project" + def transitiveDependenciesSubstitutions( + dependency: coursierapi.Dependency, + filter: coursierapi.Dependency => Boolean + ): Seq[(String, String)] = { + val fetchRes = coursierapi.Fetch.create() + .addDependencies(dependency) + .fetchResult() + fetchRes.getDependencies.asScala + .filter(filter) + .map { dep => + val organization = dep.getModule.getOrganization + val name = dep.getModule.getName + val prefix = (organization.split('.') :+ name).mkString("/") + def basePath(version: String): String = + s"$prefix/$version/$name-$version" + basePath(dep.getVersion) -> basePath(s"<$name-version>") + } + .toSeq + } + def tests: Tests = Tests { test("requestSnapshots") - integrationTest { tester => import tester._ @@ -31,29 +51,31 @@ object BspServerTests extends UtestIntegrationTestSuite { millTestSuiteEnv ) { (buildServer, initRes) => val scalaVersion = sys.props.getOrElse("TEST_SCALA_2_13_VERSION", ???) - val scalaTransitiveSubstitutions = { - val scalaFetchRes = coursierapi.Fetch.create() - .addDependencies(coursierapi.Dependency.of( - "org.scala-lang", - "scala-compiler", - scalaVersion - )) - .fetchResult() - scalaFetchRes.getDependencies.asScala - .filter(dep => dep.getModule.getOrganization != "org.scala-lang") - .map { dep => - val organization = dep.getModule.getOrganization - val name = dep.getModule.getName - val prefix = (organization.split('.') :+ name).mkString("/") - def basePath(version: String): String = - s"$prefix/$version/$name-$version" - basePath(dep.getVersion) -> basePath(s"<$name-version>") - } - } - - val normalizedLocalValues = - normalizeLocalValuesForTesting(workspacePath) ++ scalaTransitiveSubstitutions ++ Seq( - scalaVersion -> "" + val scalaTransitiveSubstitutions = transitiveDependenciesSubstitutions( + coursierapi.Dependency.of( + "org.scala-lang", + "scala-compiler", + scalaVersion + ), + _.getModule.getOrganization != "org.scala-lang" + ) + + val kotlinVersion = sys.props.getOrElse("TEST_KOTLIN_VERSION", ???) + val kotlinTransitiveSubstitutions = transitiveDependenciesSubstitutions( + coursierapi.Dependency.of( + "org.jetbrains.kotlin", + "kotlin-stdlib", + kotlinVersion + ), + _.getModule.getOrganization != "org.jetbrains.kotlin" + ) + + val normalizedLocalValues = normalizeLocalValuesForTesting(workspacePath) ++ + scalaTransitiveSubstitutions ++ + kotlinTransitiveSubstitutions ++ + Seq( + scalaVersion -> "", + kotlinVersion -> "" ) compareWithGsonSnapshot( diff --git a/integration/package.mill b/integration/package.mill index 86e823a78b8..d4c4ade0529 100644 --- a/integration/package.mill +++ b/integration/package.mill @@ -76,7 +76,8 @@ object `package` extends RootModule { override def moduleDeps = super[IntegrationTestModule].moduleDeps def forkEnv = super.forkEnv() ++ Seq( "MILL_PROJECT_ROOT" -> T.workspace.toString, - "TEST_SCALA_2_13_VERSION" -> build.Deps.testScala213Version + "TEST_SCALA_2_13_VERSION" -> build.Deps.testScala213Version, + "TEST_KOTLIN_VERSION" -> build.Deps.kotlinCompiler.dep.version ) } trait IdeIntegrationCrossModule extends IntegrationCrossModule { diff --git a/kotlinlib/src/mill/kotlinlib/KotlinModule.scala b/kotlinlib/src/mill/kotlinlib/KotlinModule.scala index 4afb1c81a12..71e64ed7e18 100644 --- a/kotlinlib/src/mill/kotlinlib/KotlinModule.scala +++ b/kotlinlib/src/mill/kotlinlib/KotlinModule.scala @@ -5,10 +5,11 @@ package mill package kotlinlib -import mill.api.{Loose, PathRef, Result} +import mill.api.{Loose, PathRef, Result, internal} import mill.define.{Command, ModuleRef, Task} import mill.kotlinlib.worker.api.{KotlinWorker, KotlinWorkerTarget} import mill.scalalib.api.{CompilationResult, ZincWorkerApi} +import mill.scalalib.bsp.{BspBuildTarget, BspModule} import mill.scalalib.{JavaModule, Lib, ZincWorkerModule} import mill.util.Jvm import mill.util.Util.millProjectModule @@ -314,6 +315,13 @@ trait KotlinModule extends JavaModule { outer => private[kotlinlib] def internalReportOldProblems: Task[Boolean] = zincReportCachedProblems + @internal + override def bspBuildTarget: BspBuildTarget = super.bspBuildTarget.copy( + languageIds = Seq(BspModule.LanguageId.Java, BspModule.LanguageId.Kotlin), + canCompile = true, + canRun = true + ) + /** * A test sub-module linked to its parent module best suited for unit-tests. */ diff --git a/scalalib/src/mill/scalalib/bsp/BspModule.scala b/scalalib/src/mill/scalalib/bsp/BspModule.scala index 42c38637987..edd208ca579 100644 --- a/scalalib/src/mill/scalalib/bsp/BspModule.scala +++ b/scalalib/src/mill/scalalib/bsp/BspModule.scala @@ -46,6 +46,7 @@ object BspModule { object LanguageId { val Java = "java" val Scala = "scala" + val Kotlin = "kotlin" } /** Used to define the [[BspBuildTarget.tags]] field. */ From 5db7ed319ded8ed96dd284c74c65afd0065af587 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Wed, 9 Oct 2024 12:15:05 +0200 Subject: [PATCH 16/47] Pass BSP JvmBuildTarget stuff to ScalaBuildTarget (#3681) This adds the JVM we use in the Scala details in BSP build targets. I don't think Metals uses that for now, and I don't know about IntelliJ. But they might in the future. --- .../src/mill/bsp/worker/MillBuildServer.scala | 14 ++++++++----- .../snapshots/workspace-build-targets.json | 12 +++++++++-- scalalib/src/mill/scalalib/JavaModule.scala | 15 ++++++------- scalalib/src/mill/scalalib/ScalaModule.scala | 21 ++----------------- 4 files changed, 29 insertions(+), 33 deletions(-) diff --git a/bsp/worker/src/mill/bsp/worker/MillBuildServer.scala b/bsp/worker/src/mill/bsp/worker/MillBuildServer.scala index adbc5e44332..727be75c932 100644 --- a/bsp/worker/src/mill/bsp/worker/MillBuildServer.scala +++ b/bsp/worker/src/mill/bsp/worker/MillBuildServer.scala @@ -175,14 +175,12 @@ private class MillBuildServer( bsp4j.ScalaPlatform.forValue(d.platform.number), d.jars.asJava ) + for (jvmBuildTarget <- d.jvmBuildTarget) + target.setJvmBuildTarget(MillBuildServer.jvmBuildTarget(jvmBuildTarget)) Some((dataKind, target)) case Some((dataKind, d: JvmBuildTarget)) => - val target = new bsp4j.JvmBuildTarget().tap { it => - d.javaHome.foreach(jh => it.setJavaHome(jh.uri)) - d.javaVersion.foreach(jv => it.setJavaVersion(jv)) - } - Some((dataKind, target)) + Some((dataKind, jvmBuildTarget(d))) case Some((dataKind, d)) => debug(s"Unsupported dataKind=${dataKind} with value=${d}") @@ -784,4 +782,10 @@ private object MillBuildServer { .toSeq .sortBy { case (k, _) => keyIndices(k) } } + + def jvmBuildTarget(d: JvmBuildTarget): bsp4j.JvmBuildTarget = + new bsp4j.JvmBuildTarget().tap { it => + d.javaHome.foreach(jh => it.setJavaHome(jh.uri)) + d.javaVersion.foreach(jv => it.setJavaVersion(jv)) + } } diff --git a/integration/ide/bsp-server/resources/snapshots/workspace-build-targets.json b/integration/ide/bsp-server/resources/snapshots/workspace-build-targets.json index 68aa310ab88..41a59014648 100644 --- a/integration/ide/bsp-server/resources/snapshots/workspace-build-targets.json +++ b/integration/ide/bsp-server/resources/snapshots/workspace-build-targets.json @@ -87,7 +87,11 @@ "file:///coursier-cache/https/repo1.maven.org/maven2/io/github/java-diff-utils/java-diff-utils//java-diff-utils-.jar", "file:///coursier-cache/https/repo1.maven.org/maven2/org/jline/jline//jline-.jar", "file:///coursier-cache/https/repo1.maven.org/maven2/net/java/dev/jna/jna//jna-.jar" - ] + ], + "jvmBuildTarget": { + "javaHome": "file:///java-home/", + "javaVersion": "" + } } }, { @@ -121,7 +125,11 @@ "file:///coursier-cache/https/repo1.maven.org/maven2/org/scala-lang/scala-compiler//scala-compiler-.jar", "file:///coursier-cache/https/repo1.maven.org/maven2/org/scala-lang/scala-reflect//scala-reflect-.jar", "file:///coursier-cache/https/repo1.maven.org/maven2/org/scala-lang/scala-library//scala-library-.jar" - ] + ], + "jvmBuildTarget": { + "javaHome": "file:///java-home/", + "javaVersion": "" + } } }, { diff --git a/scalalib/src/mill/scalalib/JavaModule.scala b/scalalib/src/mill/scalalib/JavaModule.scala index 67aa7dd5dfc..90d72e7b62b 100644 --- a/scalalib/src/mill/scalalib/JavaModule.scala +++ b/scalalib/src/mill/scalalib/JavaModule.scala @@ -1056,14 +1056,15 @@ trait JavaModule canRun = true ) + @internal + def bspJvmBuildTarget: JvmBuildTarget = + JvmBuildTarget( + javaHome = Option(System.getProperty("java.home")).map(p => BspUri(os.Path(p))), + javaVersion = Option(System.getProperty("java.version")) + ) + @internal override def bspBuildTargetData: Task[Option[(String, AnyRef)]] = Task.Anon { - Some(( - JvmBuildTarget.dataKind, - JvmBuildTarget( - javaHome = Option(System.getProperty("java.home")).map(p => BspUri(os.Path(p))), - javaVersion = Option(System.getProperty("java.version")) - ) - )) + Some((JvmBuildTarget.dataKind, bspJvmBuildTarget)) } } diff --git a/scalalib/src/mill/scalalib/ScalaModule.scala b/scalalib/src/mill/scalalib/ScalaModule.scala index 14fc7df2827..b2b839e5584 100644 --- a/scalalib/src/mill/scalalib/ScalaModule.scala +++ b/scalalib/src/mill/scalalib/ScalaModule.scala @@ -8,14 +8,7 @@ import mill.util.Jvm.createJar import mill.api.Loose.Agg import mill.scalalib.api.{CompilationResult, Versions, ZincWorkerUtil} import mainargs.Flag -import mill.scalalib.bsp.{ - BspBuildTarget, - BspModule, - BspUri, - JvmBuildTarget, - ScalaBuildTarget, - ScalaPlatform -} +import mill.scalalib.bsp.{BspBuildTarget, BspModule, ScalaBuildTarget, ScalaPlatform} import mill.scalalib.dependency.versions.{ValidVersion, Version} import scala.reflect.internal.util.ScalaClassLoader @@ -596,17 +589,7 @@ trait ScalaModule extends JavaModule with TestModule.ScalaModuleBase { outer => scalaBinaryVersion = ZincWorkerUtil.scalaBinaryVersion(scalaVersion()), platform = ScalaPlatform.JVM, jars = scalaCompilerClasspath().map(_.path.toNIO.toUri.toString).iterator.toSeq, - // this is what we want to use, but can't due to a resulting binary incompatibility - // jvmBuildTarget = super.bspBuildTargetData().flatMap { - // case (JvmBuildTarget.dataKind, bt: JvmBuildTarget) => Some(bt) - // case _ => None - // } - jvmBuildTarget = Some( - JvmBuildTarget( - javaHome = Option(System.getProperty("java.home")).map(p => BspUri(os.Path(p))), - javaVersion = Option(System.getProperty("java.version")) - ) - ) + jvmBuildTarget = Some(bspJvmBuildTarget) ) )) } From d46b51204dd06106c612cfcfd8a721c31dea7105 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Wed, 9 Oct 2024 12:29:42 +0200 Subject: [PATCH 17/47] Unify `visualize` and doc-site graphviz rendering into Javet subprocess (#3692) * We cannot do Javet/Node/Graphviz rendering in-process because the native node.js binary necessary to run these things doesn't play well with Mill's classloader juggling, meaning if the `build.mill` gets recompiled in a long-running process it currently fails with a nasty LinkageError. This is an existing problem ``` lihaoyi mill$ ./mill visualize dist._ ... [ [55/59] =========================================== visualize dist._ =============================================== 8s "/Users/lihaoyi/Github/mill/out/visualize.dest/out.txt", "/Users/lihaoyi/Github/mill/out/visualize.dest/out.dot", "/Users/lihaoyi/Github/mill/out/visualize.dest/out.json", "/Users/lihaoyi/Github/mill/out/visualize.dest/out.png", "/Users/lihaoyi/Github/mill/out/visualize.dest/out.svg" ] [3/3] =============================================== visualize dist._ ============================================== 11s lihaoyi mill$ echo "def hello = 1" >> build.mill lihaoyi mill$ ./mill visualize dist._ ... [2] Oct 09, 2024 10:55:57 AM com.caoccao.javet.utils.JavetDefaultLogger error [2] SEVERE: Native Library /private/var/folders/rt/f1pd6fz92x3__6jg0cmgdd580000gn/T/javet/27157/libjavet-v8-macos-arm64.v.3.1.6.dylib already loaded in another classloader [2] Oct 09, 2024 10:55:57 AM com.caoccao.javet.utils.JavetDefaultLogger error [2] SEVERE: java.lang.UnsatisfiedLinkError: Native Library /private/var/folders/rt/f1pd6fz92x3__6jg0cmgdd580000gn/T/javet/27157/libjavet-v8-macos-arm64.v.3.1.6.dylib already loaded in another classloader [2] at java.base/jdk.internal.loader.NativeLibraries.loadLibrary(NativeLibraries.java:201) [2] at java.base/jdk.internal.loader.NativeLibraries.loadLibrary(NativeLibraries.java:174) [2] at java.base/java.lang.ClassLoader.loadLibrary(ClassLoader.java:2389) [2] at java.base/java.lang.Runtime.load0(Runtime.java:755) [2] at java.base/java.lang.System.load(System.java:1953) [2] at com.caoccao.javet.interop.loader.JavetLibLoader.load(JavetLibLoader.java:357) [2] at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) [2] at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) [2] at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) [2] at java.base/java.lang.reflect.Method.invoke(Method.java:568) [2] at com.caoccao.javet.interop.JavetClassLoader.load(JavetClassLoader.java:104) [2] at com.caoccao.javet.interop.V8Host.loadLibrary(V8Host.java:464) [2] at com.caoccao.javet.interop.V8Host.(V8Host.java:85) [2] at com.caoccao.javet.interop.V8Host.(V8Host.java:51) [2] at com.caoccao.javet.interop.V8Host$V8InstanceHolder.(V8Host.java:602) [2] at com.caoccao.javet.interop.V8Host.getV8Instance(V8Host.java:134) [2] at mill.main.graphviz.V8JavascriptEngine.(GraphvizTools.scala:89) [2] at mill.main.graphviz.GraphvizTools$$anon$1$$anonfun$$lessinit$greater$1.get(GraphvizTools.scala:70) [2] at mill.main.graphviz.GraphvizTools$$anon$1$$anonfun$$lessinit$greater$1.get(GraphvizTools.scala:70) [2] at guru.nidi.graphviz.engine.AbstractJsGraphvizEngine.engine(AbstractJsGraphvizEngine.java:65) [2] at guru.nidi.graphviz.engine.AbstractJsGraphvizEngine.doInit(AbstractJsGraphvizEngine.java:49) [2] at guru.nidi.graphviz.engine.AbstractGraphvizEngine.initTask(AbstractGraphvizEngine.java:50) [2] at guru.nidi.graphviz.engine.AbstractGraphvizEngine.init(AbstractGraphvizEngine.java:42) [2] at guru.nidi.graphviz.engine.Graphviz.doUseEngine(Graphviz.java:146) [2] at guru.nidi.graphviz.engine.Graphviz.useEngine(Graphviz.java:138) [2] at guru.nidi.graphviz.engine.Graphviz.useEngine(Graphviz.java:119) [2] at mill.main.graphviz.GraphvizTools$.apply(GraphvizTools.scala:70) [2] at mill.main.graphviz.GraphvizTools.apply(GraphvizTools.scala) [2] at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) [2] at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) [2] at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) [2] at java.base/java.lang.reflect.Method.invoke(Method.java:568) [2] at mill.main.VisualizeModule.$anonfun$worker$5(VisualizeModule.scala:54) [2] at java.base/java.lang.Thread.run(Thread.java:833) [2] ``` * We remove the Antora Krokli from the docsite and just manually parse of the graphviz blocks and render them ourselves, to try and mitigate https://github.com/com-lihaoyi/mill/issues/3411 --- .../modules/ROOT/pages/comparisons/maven.adoc | 5 +- .../ROOT/pages/depth/design-principles.adoc | 10 +- .../ROOT/partials/Intro_to_Mill_Header.adoc | 5 +- docs/package.mill | 47 ++++++++- .../fundamentals/cross/1-simple/build.mill | 6 +- .../cross/10-static-blog/build.mill | 5 +- .../cross/3-outside-dependency/build.mill | 5 +- .../cross/4-cross-dependencies/build.mill | 5 +- .../cross/5-multiple-cross-axes/build.mill | 5 +- .../cross/7-inner-cross-module/build.mill | 5 +- .../fundamentals/modules/7-modules/build.mill | 16 ++-- .../modules/8-diy-java-modules/build.mill | 10 +- .../tasks/1-task-graph/build.mill | 15 ++- .../tasks/2-primary-tasks/build.mill | 30 +++--- .../basic/2-custom-build-logic/build.mill | 5 +- .../basic/4-builtin-commands/build.mill | 8 +- .../scalalib/module/2-custom-tasks/build.mill | 6 +- .../mill/main/graphviz/GraphvizTools.scala | 96 +++++-------------- main/package.mill | 4 +- main/src/mill/main/MainModule.scala | 2 +- main/src/mill/main/VisualizeModule.scala | 80 +++++++++++++--- mill-build/build.sc | 3 +- 22 files changed, 195 insertions(+), 178 deletions(-) diff --git a/docs/modules/ROOT/pages/comparisons/maven.adoc b/docs/modules/ROOT/pages/comparisons/maven.adoc index c839b0c0596..73cc54f353b 100644 --- a/docs/modules/ROOT/pages/comparisons/maven.adoc +++ b/docs/modules/ROOT/pages/comparisons/maven.adoc @@ -569,8 +569,7 @@ def make = Task { } ``` -[graphviz] -.... +```graphviz digraph G { rankdir=LR node [shape=box width=0 height=0 style=filled fillcolor=white] @@ -578,7 +577,7 @@ digraph G { cSources -> make cSources -> cHeaders } -.... +``` In Mill, we define the `makefile`, `cSources`, `cHeaders`, and `make` tasks. The bulk of the logic is in `def make`, which prepares the `makefile` and C sources, diff --git a/docs/modules/ROOT/pages/depth/design-principles.adoc b/docs/modules/ROOT/pages/depth/design-principles.adoc index 16fac99f51e..c3235f73169 100644 --- a/docs/modules/ROOT/pages/depth/design-principles.adoc +++ b/docs/modules/ROOT/pages/depth/design-principles.adoc @@ -199,8 +199,7 @@ build-related questions listed above. === The Object Hierarchy -[graphviz] -.... +```graphviz digraph G { node [shape=box width=0 height=0 style=filled fillcolor=white] bgcolor=transparent @@ -213,7 +212,7 @@ digraph G { foo2 -> "foo2.qux" [style=dashed] foo2 -> "foo2.baz" [style=dashed] } -.... +``` The module hierarchy is the graph of objects, starting from the root of the `build.mill` file, that extend `mill.Module`. At the leaves of the hierarchy are @@ -239,8 +238,7 @@ are sure that it will never clash with any other ``Task``s data. === The Call Graph -[graphviz] -.... +```graphviz digraph G { rankdir=LR node [shape=box width=0 height=0 style=filled fillcolor=white] @@ -275,7 +273,7 @@ digraph G { "qux.sources" -> "qux.compile" -> "qux.classPath" -> "qux.assembly" } } -.... +``` The Scala call graph of "which task references which other task" is core to how Mill operates. This graph is reified via the `T {...}` macro to make it diff --git a/docs/modules/ROOT/partials/Intro_to_Mill_Header.adoc b/docs/modules/ROOT/partials/Intro_to_Mill_Header.adoc index 17e589dd4ae..196f99f731b 100644 --- a/docs/modules/ROOT/partials/Intro_to_Mill_Header.adoc +++ b/docs/modules/ROOT/partials/Intro_to_Mill_Header.adoc @@ -1,5 +1,4 @@ -[graphviz] -.... +```graphviz digraph G { rankdir=LR node [shape=box width=0 height=0 style=filled fillcolor=white] @@ -24,7 +23,7 @@ digraph G { "bar.mainClass" -> "bar.assembly" } } -.... +``` {mill-github-url}[Mill] is a fast multi-language JVM build tool that supports {language}, making your common development workflows xref:comparisons/maven.adoc[5-10x faster to Maven], or diff --git a/docs/package.mill b/docs/package.mill index ad683869e13..3db8a792a7b 100644 --- a/docs/package.mill +++ b/docs/package.mill @@ -1,8 +1,9 @@ package build.docs import mill.util.Jvm import mill._, scalalib._ -import build.contrib import de.tobiasroeser.mill.vcs.version.VcsVersion +import guru.nidi.graphviz.engine.AbstractJsGraphvizEngine +import guru.nidi.graphviz.engine.{Format, Graphviz} /** Generates the mill documentation with Antora. */ object `package` extends RootModule { @@ -32,7 +33,6 @@ object `package` extends RootModule { "@antora/site-generator-default@3.1.9", "gitlab:antora/xref-validator", "@antora/lunr-extension@v1.0.0-alpha.6", - "asciidoctor-kroki@0.18.1" ), envArgs = Map(), workingDir = npmDir @@ -90,6 +90,46 @@ object `package` extends RootModule { createFolders = true ) + Graphviz.useEngine(new AbstractJsGraphvizEngine(true, () => new mill.main.graphviz.V8JavascriptEngine()) {}) + + def walkAllFiles(inputs: Map[(os.Path, Int), String]): Map[(os.Path, Int), String] = { + val output = collection.mutable.Map.empty[(os.Path, Int), String] + for (p <- os.walk(T.dest) if p.ext == "adoc"){ + val outputLines = collection.mutable.ArrayDeque.empty[String] + val graphvizLines = collection.mutable.ArrayDeque.empty[String] + var isGraphViz = false + + for((line, i) <- os.read.lines(p).zipWithIndex){ + line match{ + case "```graphviz" => isGraphViz = true + case "```" if isGraphViz => + isGraphViz = false + if (inputs.isEmpty) output((p, i)) = graphvizLines.mkString("\n") + else { + outputLines.append("++++") + outputLines.append(inputs((p, i))) + outputLines.append("++++") + } + + graphvizLines.clear() + case _ => + if (isGraphViz) graphvizLines.append(line) + else outputLines.append(line) + } + } + if (inputs.nonEmpty) os.write.over(p, outputLines.mkString("\n")) + } + output.toMap + } + + val diagrams = walkAllFiles(Map()) + // Batch the rendering so later it can be done in one call to a single subprocess, + // minimizing per-subprocess overhead needed to spawn them over and over + val renderedDiagrams = diagrams + .map{case (k, s) => (k, Graphviz.fromString(s).render(Format.SVG).toString)} + + walkAllFiles(renderedDiagrams) + PathRef(T.dest) } @@ -167,9 +207,6 @@ object `package` extends RootModule { | utest-github-url: https://github.com/com-lihaoyi/utest | upickle-github-url: https://github.com/com-lihaoyi/upickle | mill-scip-version: ${build.Deps.DocDeps.millScip.dep.version} - | kroki-fetch-diagram: true - | extensions: - | - asciidoctor-kroki |antora: | extensions: | - require: '@antora/lunr-extension' diff --git a/example/fundamentals/cross/1-simple/build.mill b/example/fundamentals/cross/1-simple/build.mill index bde30ca175a..a70ddf743e3 100644 --- a/example/fundamentals/cross/1-simple/build.mill +++ b/example/fundamentals/cross/1-simple/build.mill @@ -9,12 +9,10 @@ trait FooModule extends Cross.Module[String] { def sources = Task.Sources(millSourcePath) } -// [graphviz] -// .... +// ```graphviz // digraph G { // rankdir=LR // node [shape=box width=0 height=0 style=filled fillcolor=white] - // subgraph cluster_2 { // label="foo[2.12]" // style=dashed @@ -34,7 +32,7 @@ trait FooModule extends Cross.Module[String] { // "foo[2.10].sources" // } // } -// .... +// ``` // Cross modules defined using the `Cross[T]` class allow you to define // multiple copies of the same module, differing only in some input key. This diff --git a/example/fundamentals/cross/10-static-blog/build.mill b/example/fundamentals/cross/10-static-blog/build.mill index 7c250b621d4..aeaf7dc900b 100644 --- a/example/fundamentals/cross/10-static-blog/build.mill +++ b/example/fundamentals/cross/10-static-blog/build.mill @@ -119,8 +119,7 @@ def dist = Task { // All caching, incremental re-computation, and parallelism is done using the // Mill task graph. For this simple example, the graph is as follows // -// [graphviz] -// .... +// ```graphviz // digraph G { // node [shape=box width=0 height=0] // "1 - Foo.md" -> "post[1]\nrender" @@ -133,7 +132,7 @@ def dist = Task { // "index" -> "dist" // // } -// .... +// ``` // // This example use case is taken from the following blog post, which contains // some extensions and fun exercises to further familiarize yourself with Mill diff --git a/example/fundamentals/cross/3-outside-dependency/build.mill b/example/fundamentals/cross/3-outside-dependency/build.mill index 1b2cd08a143..7da447d9d18 100644 --- a/example/fundamentals/cross/3-outside-dependency/build.mill +++ b/example/fundamentals/cross/3-outside-dependency/build.mill @@ -11,8 +11,7 @@ def bar = Task { s"hello ${foo("2.10").suffix()}" } def qux = Task { s"hello ${foo("2.10").suffix()} world ${foo("2.12").suffix()}" } -// [graphviz] -// .... +// ```graphviz // digraph G { // rankdir=LR // node [shape=box width=0 height=0 style=filled fillcolor=white] @@ -36,7 +35,7 @@ def qux = Task { s"hello ${foo("2.10").suffix()} world ${foo("2.12").suffix()}" // "foo[2.10].suffix" -> "qux" // "foo[2.10].suffix" -> "bar" // } -// .... +// ``` // Here, `def bar` uses `foo("2.10")` to reference the `"2.10"` instance of diff --git a/example/fundamentals/cross/4-cross-dependencies/build.mill b/example/fundamentals/cross/4-cross-dependencies/build.mill index d701bc24f0d..fc68fa04087 100644 --- a/example/fundamentals/cross/4-cross-dependencies/build.mill +++ b/example/fundamentals/cross/4-cross-dependencies/build.mill @@ -13,8 +13,7 @@ trait BarModule extends Cross.Module[String] { def bigSuffix = Task { "[[[" + foo(crossValue).suffix() + "]]]" } } -// [graphviz] -// .... +// ```graphviz // digraph G { // rankdir=LR // node [shape=box width=0 height=0 style=filled fillcolor=white] @@ -52,7 +51,7 @@ trait BarModule extends Cross.Module[String] { // "foo[2.11].suffix" -> "bar[2.11].bigSuffix" // "foo[2.12].suffix" -> "bar[2.12].bigSuffix" // } -// .... +// ``` // Rather than pssing in a literal `"2.10"` to the `foo` cross module, we pass // in the `crossValue` property that is available within every `Cross.Module`. diff --git a/example/fundamentals/cross/5-multiple-cross-axes/build.mill b/example/fundamentals/cross/5-multiple-cross-axes/build.mill index 8b5ca53fd55..dcaeeee6a5d 100644 --- a/example/fundamentals/cross/5-multiple-cross-axes/build.mill +++ b/example/fundamentals/cross/5-multiple-cross-axes/build.mill @@ -17,8 +17,7 @@ trait FooModule extends Cross.Module2[String, String] { def bar = Task { s"hello ${foo("2.10", "jvm").suffix()}" } -// [graphviz] -// .... +// ```graphviz // digraph G { // rankdir=LR // node [shape=box width=0 height=0 style=filled fillcolor=white] @@ -65,7 +64,7 @@ def bar = Task { s"hello ${foo("2.10", "jvm").suffix()}" } // // "foo[2.10,jvm].suffix" -> bar // } -// .... +// ``` // // This example shows off using a for-loop to generate a list of // cross-key-tuples, as a `Seq[(String, String)]` that we then pass it into the diff --git a/example/fundamentals/cross/7-inner-cross-module/build.mill b/example/fundamentals/cross/7-inner-cross-module/build.mill index ffcdb0fc858..f9374f5832f 100644 --- a/example/fundamentals/cross/7-inner-cross-module/build.mill +++ b/example/fundamentals/cross/7-inner-cross-module/build.mill @@ -19,8 +19,7 @@ trait FooModule extends Cross.Module[String] { def baz = Task { s"hello ${foo("a").bar.param()}" } -// [graphviz] -// .... +// ```graphviz // digraph G { // node [shape=box width=0 height=0 style=filled fillcolor=white] // "root-module" [style=dashed] @@ -45,7 +44,7 @@ def baz = Task { s"hello ${foo("a").bar.param()}" } // "..." [color=white] // "foo[a].bar.name" -> "foo[a].bar.param" [constraint=false] // } -// .... +// ``` // // You can use the `CrossValue` trait within any `Cross.Module` to // propagate the `crossValue` defined by an enclosing `Cross.Module` to some diff --git a/example/fundamentals/modules/7-modules/build.mill b/example/fundamentals/modules/7-modules/build.mill index d447b344e8f..407acc21f0d 100644 --- a/example/fundamentals/modules/7-modules/build.mill +++ b/example/fundamentals/modules/7-modules/build.mill @@ -13,8 +13,7 @@ object foo extends Module { } } -// [graphviz] -// .... +// ```graphviz // digraph G { // node [shape=box width=0 height=0 style=filled fillcolor=white] // "root-module" [style=dashed] @@ -24,7 +23,8 @@ object foo extends Module { // "root-module" -> foo -> "foo.qux" -> "foo.qux.baz" [style=dashed] // foo -> "foo.bar" [style=dashed] // } -// .... +// ``` +// // You would be able to run the two tasks via `mill foo.bar` or `mill // foo.qux.baz`. You can use `mill show foo.bar` or `mill show foo.baz.qux` to // make Mill echo out the string value being returned by each Task. The two @@ -76,8 +76,7 @@ object foo2 extends FooModule { // arrows representing the module tree, and the solid boxes and arrows representing // the task graph -// [graphviz] -// .... +// ```graphviz // digraph G { // node [shape=box width=0 height=0 style=filled fillcolor=white] // bgcolor=transparent @@ -93,7 +92,7 @@ object foo2 extends FooModule { // "foo1.bar" -> "foo1.qux.super" -> "foo1.qux" [constraint=false] // "foo2.bar" -> "foo2.qux" -> "foo2.baz" [constraint=false] // } -// .... +// ``` // Note that the `override` keyword is optional in mill, as is `T{...}` wrapper. @@ -139,8 +138,7 @@ object outer extends MyModule { object inner extends MyModule } -// [graphviz] -// .... +// ```graphviz // digraph G { // node [shape=box width=0 height=0 style=filled fillcolor=white] // "root-module" [style=dashed] @@ -155,7 +153,7 @@ object outer extends MyModule { // outer -> "outer.sources" [style=dashed] // outer -> "outer.task" [style=dashed] // } -// .... +// ``` // * The `outer` module has a `millSourcePath` of `outer/`, and thus a // `outer.sources` referencing `outer/sources/` diff --git a/example/fundamentals/modules/8-diy-java-modules/build.mill b/example/fundamentals/modules/8-diy-java-modules/build.mill index 58775661b73..c7ae5752e00 100644 --- a/example/fundamentals/modules/8-diy-java-modules/build.mill +++ b/example/fundamentals/modules/8-diy-java-modules/build.mill @@ -34,8 +34,7 @@ trait DiyJavaModule extends Module{ // edges (dashed) are not connected; that is because `DiyJavaModule` is abstract, and // needs to be inherited by a concrete `object` before it can be used. -// [graphviz] -// .... +// ```graphviz // digraph G { // rankdir=LR // node [shape=box width=0 height=0 style=filled fillcolor=white] @@ -51,7 +50,7 @@ trait DiyJavaModule extends Module{ // "sources" -> "compile" -> "classPath" -> "assembly" // } // } -// .... +// ``` // // Some notable things to call out: // @@ -87,8 +86,7 @@ object qux extends DiyJavaModule { // duplicated three times - once per module - with the tasks wired up between the modules // according to our overrides for `moduleDeps` -// [graphviz] -// .... +// ```graphviz // digraph G { // rankdir=LR // node [shape=box width=0 height=0 style=filled fillcolor=white] @@ -123,7 +121,7 @@ object qux extends DiyJavaModule { // "qux.sources" -> "qux.compile" -> "qux.classPath" -> "qux.assembly" // } // } -// .... +// ``` // // This simple set of `DiyJavaModule` can be used as follows: diff --git a/example/fundamentals/tasks/1-task-graph/build.mill b/example/fundamentals/tasks/1-task-graph/build.mill index 02caff480a5..4735f7c4f8e 100644 --- a/example/fundamentals/tasks/1-task-graph/build.mill +++ b/example/fundamentals/tasks/1-task-graph/build.mill @@ -26,8 +26,7 @@ def assembly = Task { // This code defines the following task graph, with the boxes being the tasks // and the arrows representing the _data-flow_ between them: // -// [graphviz] -// .... +// ```graphviz // digraph G { // rankdir=LR // node [shape=box width=0 height=0 style=filled fillcolor=white] @@ -35,7 +34,7 @@ def assembly = Task { // resources -> assembly // mainClass -> assembly // } -// .... +// ``` // // This example does not use any of Mill's builtin support for building Java or // Scala projects, and instead builds a pipeline "from scratch" using Mill @@ -66,8 +65,7 @@ My Example Text // * If the files in `sources` change, it will re-evaluate // `compile`, and `assembly` (red) // -// [graphviz] -// .... +// ```graphviz // digraph G { // rankdir=LR // node [shape=box width=0 height=0 style=filled fillcolor=white] @@ -80,13 +78,12 @@ My Example Text // resources [fillcolor=lightgreen] // mainClass [fillcolor=lightgreen] // } -// .... +// ``` // // * If the files in `resources` change, it will only re-evaluate `assembly` (red) // and use the cached output of `compile` (green) // -// [graphviz] -// .... +// ```graphviz // digraph G { // rankdir=LR // node [shape=box width=0 height=0 style=filled fillcolor=white] @@ -99,5 +96,5 @@ My Example Text // sources [fillcolor=lightgreen] // mainClass [fillcolor=lightgreen] // } -// .... +// ``` // \ No newline at end of file diff --git a/example/fundamentals/tasks/2-primary-tasks/build.mill b/example/fundamentals/tasks/2-primary-tasks/build.mill index b6b17d51f36..a1b25c02847 100644 --- a/example/fundamentals/tasks/2-primary-tasks/build.mill +++ b/example/fundamentals/tasks/2-primary-tasks/build.mill @@ -38,15 +38,14 @@ def lineCount: T[Int] = Task { .sum } -// [graphviz] -// .... +// ```graphviz // digraph G { // rankdir=LR // node [shape=box width=0 height=0 style=filled fillcolor=white] // sources -> allSources -> lineCount // sources [color=white] // } -// .... +// ``` // // ``Target``s are defined using the `def foo = Task {...}` syntax, and dependencies // on other tasks are defined using `foo()` to extract the value from them. @@ -142,8 +141,7 @@ def jar = Task { PathRef(Task.dest / "foo.jar") } -// [graphviz] -// .... +// ```graphviz // digraph G { // rankdir=LR // node [shape=box width=0 height=0 style=filled fillcolor=white] @@ -152,7 +150,7 @@ def jar = Task { // allSources [color=white] // resources [color=white] // } -// .... +// ``` /** Usage @@ -199,15 +197,14 @@ def hugeFileName = Task { else "" } -// [graphviz] -// .... +// ```graphviz // digraph G { // rankdir=LR // node [shape=box width=0 height=0 style=filled fillcolor=white] // allSources -> largeFile-> hugeFileName // allSources [color=white] // } -// .... +// ``` /** Usage @@ -242,15 +239,14 @@ def summarizeClassFileStats = Task { ) } -// [graphviz] -// .... +// ```graphviz // digraph G { // rankdir=LR // node [shape=box width=0 height=0 style=filled fillcolor=white] // classFiles -> summarizedClassFileStats // classFiles [color=white] // } -// .... +// ``` /** Usage @@ -276,8 +272,7 @@ def run(mainClass: String, args: String*) = Task.Command { .call(stdout = os.Inherit) } -// [graphviz] -// .... +// ```graphviz // digraph G { // rankdir=LR // node [shape=box width=0 height=0 style=filled fillcolor=white] @@ -286,7 +281,7 @@ def run(mainClass: String, args: String*) = Task.Command { // classFiles [color=white] // resources [color=white] // } -// .... +// ``` // Defined using `Task.Command {...}` syntax, ``Command``s can run arbitrary code, with // dependencies declared using the same `foo()` syntax (e.g. `classFiles()` above). @@ -373,15 +368,14 @@ trait Bar extends Foo { object bar extends Bar -// [graphviz] -// .... +// ```graphviz // digraph G { // rankdir=LR // node [shape=box width=0 height=0 style=filled fillcolor=white] // "bar.sourceRoots.super" -> "bar.sourceRoots" -> "bar.sourceContents" // "bar.additionalSources" -> "bar.sourceRoots" // } -// .... +// ``` /** Usage > ./mill show bar.sourceContents # includes both source folders diff --git a/example/scalalib/basic/2-custom-build-logic/build.mill b/example/scalalib/basic/2-custom-build-logic/build.mill index e9df1c305e7..293f97b547b 100644 --- a/example/scalalib/basic/2-custom-build-logic/build.mill +++ b/example/scalalib/basic/2-custom-build-logic/build.mill @@ -30,8 +30,7 @@ object `package` extends RootModule with ScalaModule { // it with the destination folder of the new `resources` task, which is wired // up to `lineCount`: // -// [graphviz] -// .... +// ```graphviz // digraph G { // rankdir=LR // node [shape=box width=0 height=0 style=filled fillcolor=white] @@ -42,7 +41,7 @@ object `package` extends RootModule with ScalaModule { // allSourceFiles [color=white] // run [color=white] // } -// .... +// ``` /** Usage diff --git a/example/scalalib/basic/4-builtin-commands/build.mill b/example/scalalib/basic/4-builtin-commands/build.mill index 668472dea63..d327198ec53 100644 --- a/example/scalalib/basic/4-builtin-commands/build.mill +++ b/example/scalalib/basic/4-builtin-commands/build.mill @@ -287,11 +287,11 @@ foo.compileClasspath /** Usage > mill visualize foo._ [ - ".../out/visualize.dest/out.txt", ".../out/visualize.dest/out.dot", ".../out/visualize.dest/out.json", ".../out/visualize.dest/out.png", - ".../out/visualize.dest/out.svg" + ".../out/visualize.dest/out.svg", + ".../out/visualize.dest/out.txt" ] */ // @@ -312,11 +312,11 @@ foo.compileClasspath /** Usage > mill visualizePlan foo.run [ - ".../out/visualizePlan.dest/out.txt", ".../out/visualizePlan.dest/out.dot", ".../out/visualizePlan.dest/out.json", ".../out/visualizePlan.dest/out.png", - ".../out/visualizePlan.dest/out.svg" + ".../out/visualizePlan.dest/out.svg", + ".../out/visualizePlan.dest/out.txt" ] */ // diff --git a/example/scalalib/module/2-custom-tasks/build.mill b/example/scalalib/module/2-custom-tasks/build.mill index 85e8ba9ab32..37b9f97b543 100644 --- a/example/scalalib/module/2-custom-tasks/build.mill +++ b/example/scalalib/module/2-custom-tasks/build.mill @@ -57,9 +57,7 @@ object `package` extends RootModule with ScalaModule { // with the boxes representing tasks defined or overriden above and the un-boxed // labels representing existing Mill tasks: // -// [graphviz] -// .... -// +// ```graphviz // digraph G { // rankdir=LR // node [shape=box width=0 height=0] @@ -74,7 +72,7 @@ object `package` extends RootModule with ScalaModule { // compile [color=white] // "..." [color=white] // } -// .... +// ``` // // Mill lets you define new cached Tasks using the `Task {...}` syntax, // depending on existing Tasks e.g. `foo.sources` via the `foo.sources()` diff --git a/main/graphviz/src/mill/main/graphviz/GraphvizTools.scala b/main/graphviz/src/mill/main/graphviz/GraphvizTools.scala index 13ad4be079f..cb416eb7fc5 100644 --- a/main/graphviz/src/mill/main/graphviz/GraphvizTools.scala +++ b/main/graphviz/src/mill/main/graphviz/GraphvizTools.scala @@ -3,84 +3,38 @@ package mill.main.graphviz import com.caoccao.javet.annotations.V8Function import com.caoccao.javet.interception.logging.JavetStandardConsoleInterceptor import com.caoccao.javet.interop.{V8Host, V8Runtime} -import guru.nidi.graphviz.attribute.Rank.RankDir -import guru.nidi.graphviz.attribute.{Rank, Shape, Style} import guru.nidi.graphviz.engine.{AbstractJavascriptEngine, AbstractJsGraphvizEngine, ResultHandler} -import mill.api.PathRef -import mill.define.NamedTask -import org.jgrapht.graph.{DefaultEdge, SimpleDirectedGraph} import org.slf4j.LoggerFactory import org.slf4j.Logger +import java.util.concurrent.Executors +import scala.concurrent.{Await, ExecutionContext, Future, duration} + object GraphvizTools { - def apply(targets: Seq[NamedTask[Any]], rs: Seq[NamedTask[Any]], dest: os.Path): Seq[PathRef] = { - val (sortedGroups, transitive) = mill.eval.Plan.plan(rs) - - val goalSet = rs.toSet - import guru.nidi.graphviz.engine.{Format, Graphviz} - import guru.nidi.graphviz.model.Factory._ - - val edgesIterator = - for ((k, vs) <- sortedGroups.items()) - yield ( - k, - for { - v <- vs.items - dest <- v.inputs.collect { case v: NamedTask[Any] => v } - if goalSet.contains(dest) - } yield dest - ) - - val edges = edgesIterator.map { case (k, v) => (k, v.toArray.distinct) }.toArray - - val indexToTask = edges.flatMap { case (k, vs) => Iterator(k.task) ++ vs }.distinct - val taskToIndex = indexToTask.zipWithIndex.toMap - - val jgraph = new SimpleDirectedGraph[Int, DefaultEdge](classOf[DefaultEdge]) - - for (i <- indexToTask.indices) jgraph.addVertex(i) - for ((src, dests) <- edges; dest <- dests) { - jgraph.addEdge(taskToIndex(src.task), taskToIndex(dest)) - } - - org.jgrapht.alg.TransitiveReduction.INSTANCE.reduce(jgraph) - val nodes = indexToTask.map(t => - node(sortedGroups.lookupValue(t).render) - .`with` { - if (targets.contains(t)) Style.SOLID - else Style.DASHED + def main(args: Array[String]): Unit = { + val executor = Executors.newFixedThreadPool(Runtime.getRuntime.availableProcessors()) + + try { + implicit val ec: ExecutionContext = ExecutionContext.fromExecutor(executor) + val futures = + for (arg <- args.toSeq) yield Future { + val Array(src, dest0, commaSepExtensions) = arg.split(";") + val extensions = commaSepExtensions.split(',') + val dest = os.Path(dest0) + import guru.nidi.graphviz.engine.{Format, Graphviz} + + Graphviz.useEngine(new AbstractJsGraphvizEngine(true, () => new V8JavascriptEngine()) {}) + val gv = Graphviz.fromFile(new java.io.File(src)).totalMemory(128 * 1024 * 1024) + + val outputs = extensions + .map(ext => Format.values().find(_.fileExtension == ext).head -> s"out.$ext") + + for ((fmt, name) <- outputs) gv.render(fmt).toFile((dest / name).toIO) } - .`with`(Shape.BOX) - ) - - var g = graph("example1").directed - for (i <- indexToTask.indices) { - for { - e <- edges(i)._2 - j = taskToIndex(e) - if jgraph.containsEdge(i, j) - } { - g = g.`with`(nodes(j).link(nodes(i))) - } - } - - g = g.graphAttr().`with`(Rank.dir(RankDir.LEFT_TO_RIGHT)) - - Graphviz.useEngine(new AbstractJsGraphvizEngine(true, () => new V8JavascriptEngine()) {}) - val gv = Graphviz.fromGraph(g).totalMemory(128 * 1024 * 1024) - val outputs = Seq( - Format.PLAIN -> "out.txt", - Format.XDOT -> "out.dot", - Format.JSON -> "out.json", - Format.PNG -> "out.png", - Format.SVG -> "out.svg" - ) - - for ((fmt, name) <- outputs) { - gv.render(fmt).toFile((dest / name).toIO) - } - outputs.map(x => mill.PathRef(dest / x._2)) + + Await.result(Future.sequence(futures), duration.Duration.Inf) + } finally executor.shutdown() } } diff --git a/main/package.mill b/main/package.mill index 2afb57b74e2..1ef11936329 100644 --- a/main/package.mill +++ b/main/package.mill @@ -14,7 +14,9 @@ object `package` extends RootModule with build.MillStableScalaModule with BuildI build.Deps.coursierInterface, build.Deps.mainargs, build.Deps.requests, - build.Deps.logback + build.Deps.logback, + build.Deps.jgraphtCore, + ivy"guru.nidi:graphviz-java-min-deps:0.18.1" ) def compileIvyDeps = Agg(build.Deps.scalaReflect(scalaVersion())) diff --git a/main/src/mill/main/MainModule.scala b/main/src/mill/main/MainModule.scala index 3027cc4b31b..0218984c82b 100644 --- a/main/src/mill/main/MainModule.scala +++ b/main/src/mill/main/MainModule.scala @@ -517,7 +517,7 @@ trait MainModule extends BaseModule0 { } private type VizWorker = ( - LinkedBlockingQueue[(scala.Seq[_], scala.Seq[_], os.Path)], + LinkedBlockingQueue[(scala.Seq[NamedTask[Any]], scala.Seq[NamedTask[Any]], os.Path)], LinkedBlockingQueue[Result[scala.Seq[PathRef]]] ) diff --git a/main/src/mill/main/VisualizeModule.scala b/main/src/mill/main/VisualizeModule.scala index 44adcbde07e..f1269795815 100644 --- a/main/src/mill/main/VisualizeModule.scala +++ b/main/src/mill/main/VisualizeModule.scala @@ -4,12 +4,13 @@ import java.util.concurrent.LinkedBlockingQueue import coursier.LocalRepositories import coursier.core.Repository import coursier.maven.MavenRepository -import mill.define.{Discover, ExternalModule, Target} -import mill.api.{PathRef, Result} +import mill.define.{Discover, ExternalModule, Target, NamedTask} import mill.util.Util.millProjectModule -import mill.api.Loose +import mill.api.{Loose, Result, PathRef} import mill.define.Worker -import os.Path +import org.jgrapht.graph.{DefaultEdge, SimpleDirectedGraph} +import guru.nidi.graphviz.attribute.Rank.RankDir +import guru.nidi.graphviz.attribute.{Rank, Shape, Style} object VisualizeModule extends ExternalModule with VisualizeModule { def repositories: Seq[Repository] = Seq( @@ -35,24 +36,73 @@ trait VisualizeModule extends mill.define.TaskModule { * can communicate via in/out queues. */ def worker: Worker[( - LinkedBlockingQueue[(Seq[_], Seq[_], Path)], + LinkedBlockingQueue[(Seq[NamedTask[Any]], Seq[NamedTask[Any]], os.Path)], LinkedBlockingQueue[Result[Seq[PathRef]]] )] = Target.worker { - val in = new LinkedBlockingQueue[(Seq[_], Seq[_], os.Path)]() + val in = new LinkedBlockingQueue[(Seq[NamedTask[Any]], Seq[NamedTask[Any]], os.Path)]() val out = new LinkedBlockingQueue[Result[Seq[PathRef]]]() - val cl = mill.api.ClassLoader.create( - classpath().map(_.path.toNIO.toUri.toURL).toVector, - getClass.getClassLoader - ) val visualizeThread = new java.lang.Thread(() => while (true) { val res = Result.Success { - val (targets, tasks, dest) = in.take() - cl.loadClass("mill.main.graphviz.GraphvizTools") - .getMethod("apply", classOf[Seq[_]], classOf[Seq[_]], classOf[os.Path]) - .invoke(null, targets, tasks, dest) - .asInstanceOf[Seq[PathRef]] + val (targets, rs, dest) = in.take() + val (sortedGroups, transitive) = mill.eval.Plan.plan(rs) + + val goalSet = rs.toSet + import guru.nidi.graphviz.model.Factory._ + val edgesIterator = + for ((k, vs) <- sortedGroups.items()) + yield ( + k, + for { + v <- vs.items + dest <- v.inputs.collect { case v: mill.define.NamedTask[Any] => v } + if goalSet.contains(dest) + } yield dest + ) + + val edges = edgesIterator.map { case (k, v) => (k, v.toArray.distinct) }.toArray + + val indexToTask = edges.flatMap { case (k, vs) => Iterator(k.task) ++ vs }.distinct + val taskToIndex = indexToTask.zipWithIndex.toMap + + val jgraph = new SimpleDirectedGraph[Int, DefaultEdge](classOf[DefaultEdge]) + + for (i <- indexToTask.indices) jgraph.addVertex(i) + for ((src, dests) <- edges; dest <- dests) { + jgraph.addEdge(taskToIndex(src.task), taskToIndex(dest)) + } + + org.jgrapht.alg.TransitiveReduction.INSTANCE.reduce(jgraph) + val nodes = indexToTask.map(t => + node(sortedGroups.lookupValue(t).render) + .`with` { + if (targets.contains(t)) Style.SOLID + else Style.DASHED + } + .`with`(Shape.BOX) + ) + + var g = graph("example1").directed + for (i <- indexToTask.indices) { + for { + e <- edges(i)._2 + j = taskToIndex(e) + if jgraph.containsEdge(i, j) + } { + g = g.`with`(nodes(j).link(nodes(i))) + } + } + + g = g.graphAttr().`with`(Rank.dir(RankDir.LEFT_TO_RIGHT)) + + mill.util.Jvm.runSubprocess( + "mill.main.graphviz.GraphvizTools", + classpath().map(_.path), + mainArgs = Seq(s"${os.temp(g.toString)};$dest;txt,dot,json,png,svg") + ) + + os.list(dest).sorted.map(PathRef(_)) } out.put(res) } diff --git a/mill-build/build.sc b/mill-build/build.sc index 23a803a3e44..454349f6657 100644 --- a/mill-build/build.sc +++ b/mill-build/build.sc @@ -9,6 +9,7 @@ object `package` extends MillBuildRootModule { ivy"net.sourceforge.htmlcleaner:htmlcleaner:2.29", // TODO: implement empty version for ivy deps as we do in import parser ivy"com.lihaoyi::mill-contrib-buildinfo:${mill.api.BuildInfo.millVersion}", - ivy"com.goyeau::mill-scalafix::0.4.1" + ivy"com.goyeau::mill-scalafix::0.4.1", + ivy"com.lihaoyi::mill-main-graphviz:${mill.api.BuildInfo.millVersion}" ) } From b4a0bfcd0390e8729dfc94d99689165555157c8c Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Wed, 9 Oct 2024 15:05:58 +0200 Subject: [PATCH 18/47] Rebootstrap on top of master to make use of updated `GraphvizTools` subprocess API (#3694) Also optimize `GraphvizTools` a bunch to re-use the javascript engine when batch processing lots of diagrams --- .config/mill-version | 2 +- .github/workflows/run-tests.yml | 2 +- docs/package.mill | 16 +++++++++++----- .../src/mill/main/graphviz/GraphvizTools.scala | 14 ++++++++++++-- 4 files changed, 25 insertions(+), 9 deletions(-) diff --git a/.config/mill-version b/.config/mill-version index 264ae60564b..02e50870961 100644 --- a/.config/mill-version +++ b/.config/mill-version @@ -1 +1 @@ -0.12.0-RC3-26-d3afbf +0.12.0-RC3-31-9b84ae \ No newline at end of file diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 7ca644d904b..723d0f89354 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -44,7 +44,7 @@ jobs: - uses: actions/checkout@v4 with: { fetch-depth: 0 } - - run: ./mill -i docs.githubPages + - run: ./mill installLocal && ./target/mill-release -i docs.githubPages linux: needs: build-linux diff --git a/docs/package.mill b/docs/package.mill index 3db8a792a7b..710b5e4315f 100644 --- a/docs/package.mill +++ b/docs/package.mill @@ -90,8 +90,8 @@ object `package` extends RootModule { createFolders = true ) - Graphviz.useEngine(new AbstractJsGraphvizEngine(true, () => new mill.main.graphviz.V8JavascriptEngine()) {}) - + // Walk all files al render graphviz templates ourselves because the only Antora graphviz + // plugin (Kroki) relies on an online web service that is super slow and flaky def walkAllFiles(inputs: Map[(os.Path, Int), String]): Map[(os.Path, Int), String] = { val output = collection.mutable.Map.empty[(os.Path, Int), String] for (p <- os.walk(T.dest) if p.ext == "adoc"){ @@ -125,14 +125,20 @@ object `package` extends RootModule { val diagrams = walkAllFiles(Map()) // Batch the rendering so later it can be done in one call to a single subprocess, // minimizing per-subprocess overhead needed to spawn them over and over - val renderedDiagrams = diagrams - .map{case (k, s) => (k, Graphviz.fromString(s).render(Format.SVG).toString)} + val orderedDiagrams = diagrams.toSeq.map{case ((p, i), s) => (p, i, os.temp(s), os.temp.dir())} + + mill.util.Jvm.runSubprocess( + "mill.main.graphviz.GraphvizTools", + mill.main.VisualizeModule.classpath().map(_.path), + mainArgs = orderedDiagrams.map{case (p, i, src, dest) => s"$src;$dest;svg"} + ) - walkAllFiles(renderedDiagrams) + walkAllFiles(orderedDiagrams.map{case (p, i, src, dest) => ((p, i), os.read(dest / "out.svg"))}.toMap) PathRef(T.dest) } + def supplementalFiles = T.source(millSourcePath / "supplemental-ui") /** diff --git a/main/graphviz/src/mill/main/graphviz/GraphvizTools.scala b/main/graphviz/src/mill/main/graphviz/GraphvizTools.scala index cb416eb7fc5..d8b6b26d356 100644 --- a/main/graphviz/src/mill/main/graphviz/GraphvizTools.scala +++ b/main/graphviz/src/mill/main/graphviz/GraphvizTools.scala @@ -8,6 +8,7 @@ import org.slf4j.LoggerFactory import org.slf4j.Logger import java.util.concurrent.Executors +import guru.nidi.graphviz.engine.{Format, Graphviz} import scala.concurrent.{Await, ExecutionContext, Future, duration} object GraphvizTools { @@ -15,6 +16,17 @@ object GraphvizTools { def main(args: Array[String]): Unit = { val executor = Executors.newFixedThreadPool(Runtime.getRuntime.availableProcessors()) + val threadLocalJsEngines = + new java.util.concurrent.ConcurrentHashMap[Thread, V8JavascriptEngine]() + Graphviz.useEngine( + new AbstractJsGraphvizEngine( + true, + () => { + threadLocalJsEngines.putIfAbsent(Thread.currentThread(), new V8JavascriptEngine()) + threadLocalJsEngines.get(Thread.currentThread()) + } + ) {} + ) try { implicit val ec: ExecutionContext = ExecutionContext.fromExecutor(executor) val futures = @@ -22,9 +34,7 @@ object GraphvizTools { val Array(src, dest0, commaSepExtensions) = arg.split(";") val extensions = commaSepExtensions.split(',') val dest = os.Path(dest0) - import guru.nidi.graphviz.engine.{Format, Graphviz} - Graphviz.useEngine(new AbstractJsGraphvizEngine(true, () => new V8JavascriptEngine()) {}) val gv = Graphviz.fromFile(new java.io.File(src)).totalMemory(128 * 1024 * 1024) val outputs = extensions From 45161b572ab93cc19faa1edb4999d50ff3caa5c5 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Wed, 9 Oct 2024 16:15:50 +0200 Subject: [PATCH 19/47] Rebootstrap on #3694 (#3698) This should let us remove the hack from the github actions doc tests --- .config/mill-version | 2 +- .github/workflows/run-tests.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.config/mill-version b/.config/mill-version index 02e50870961..b30ea878bb5 100644 --- a/.config/mill-version +++ b/.config/mill-version @@ -1 +1 @@ -0.12.0-RC3-31-9b84ae \ No newline at end of file +0.12.0-RC3-32-b4a0bf \ No newline at end of file diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 723d0f89354..7ca644d904b 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -44,7 +44,7 @@ jobs: - uses: actions/checkout@v4 with: { fetch-depth: 0 } - - run: ./mill installLocal && ./target/mill-release -i docs.githubPages + - run: ./mill -i docs.githubPages linux: needs: build-linux From bbec010743c807ad486f5858815a61dcf7775a54 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Wed, 9 Oct 2024 17:38:48 +0200 Subject: [PATCH 20/47] Try to fix doc rendering for old versions of docsite (#3700) --- docs/package.mill | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/docs/package.mill b/docs/package.mill index 710b5e4315f..b870a130a85 100644 --- a/docs/package.mill +++ b/docs/package.mill @@ -90,19 +90,31 @@ object `package` extends RootModule { createFolders = true ) - // Walk all files al render graphviz templates ourselves because the only Antora graphviz + expandDiagramsInDirectoryAdocFile(T.dest, mill.main.VisualizeModule.classpath().map(_.path)) + + PathRef(T.dest) + } + + def expandDiagramsInDirectoryAdocFile(dest: os.Path, + visualizeClassPath: Agg[os.Path]) + (implicit ctx: mill.api.Ctx) = { + + // Walk all files to render graphviz templates ourselves because the only Antora graphviz // plugin (Kroki) relies on an online web service that is super slow and flaky def walkAllFiles(inputs: Map[(os.Path, Int), String]): Map[(os.Path, Int), String] = { val output = collection.mutable.Map.empty[(os.Path, Int), String] - for (p <- os.walk(T.dest) if p.ext == "adoc"){ + for (p <- os.walk(dest) if p.ext == "adoc"){ val outputLines = collection.mutable.ArrayDeque.empty[String] val graphvizLines = collection.mutable.ArrayDeque.empty[String] var isGraphViz = false + var isGraphViz0 = false for((line, i) <- os.read.lines(p).zipWithIndex){ line match{ + case "[graphviz]" => isGraphViz0 = true + case "...." if isGraphViz0 => isGraphViz0 = false; isGraphViz = true case "```graphviz" => isGraphViz = true - case "```" if isGraphViz => + case "```" | "...." if isGraphViz => isGraphViz = false if (inputs.isEmpty) output((p, i)) = graphvizLines.mkString("\n") else { @@ -129,16 +141,14 @@ object `package` extends RootModule { mill.util.Jvm.runSubprocess( "mill.main.graphviz.GraphvizTools", - mill.main.VisualizeModule.classpath().map(_.path), + visualizeClassPath, mainArgs = orderedDiagrams.map{case (p, i, src, dest) => s"$src;$dest;svg"} ) walkAllFiles(orderedDiagrams.map{case (p, i, src, dest) => ((p, i), os.read(dest / "out.svg"))}.toMap) - PathRef(T.dest) } - def supplementalFiles = T.source(millSourcePath / "supplemental-ui") /** @@ -232,6 +242,7 @@ object `package` extends RootModule { os.proc("git", "checkout", oldVersion).call(cwd = checkout, stdout = os.Inherit) val outputFolder = checkout / "out" / "docs" / "source.dest" os.proc("./mill", "-i", "docs.source").call(cwd = checkout, stdout = os.Inherit) + expandDiagramsInDirectoryAdocFile(outputFolder, mill.main.VisualizeModule.classpath().map(_.path)) sanitizeAntoraYml(outputFolder, oldVersion, oldVersion, oldVersion) PathRef(outputFolder) } From 3d0bb6ed8177eff27c2a9ed7151b56f699d5e7d6 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Wed, 9 Oct 2024 20:17:13 +0200 Subject: [PATCH 21/47] Re-implement ticker prefixes using `key: Seq[String]` concatenation (#3697) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This re-implements the ticker log prefixes that were left out in https://github.com/com-lihaoyi/mill/pull/3577, both for meta-build prefixes and nested-evaluation prefixes, e.g. the log prefix now looks like `[build.mill-1]` `[mill-build/build.mill-1]` etc. * We no longer need `dynamicTickerPrefix`, because we now perform this context management by wrapping nested `PrefixLogger`s * Required some gross mangling to consolidate `Logger` and `ColorLogger` interfaces in order to make things properly stackable * We needed to move the aggregation of keys to a top-down process, rather than bottom up, since it is not easy to take an already prefixed line `[1] my logs line` and append to the prefix `[1-2] my logs line`. Thus we need to make sure the log line is properly prefixed at the bottom-most logger, and the log line is passed up via `unprefixedSystemStreams` to avoid redundant prefixing Screenshot 2024-10-09 at 7 36 05 PM --- .../src/mill/bsp/worker/MillBspLogger.scala | 4 +- .../imports/1-import-ivy/bar/src/Bar.scala | 8 --- .../imports/2-import-ivy-scala/build.mill | 12 ++-- .../2-import-ivy-scala/foo/src/Foo.java | 12 ---- main/api/src/mill/api/Logger.scala | 7 ++- main/eval/src/mill/eval/EvaluatorCore.scala | 3 +- main/eval/src/mill/eval/GroupEvaluator.scala | 32 +++++----- main/src/mill/main/MainModule.scala | 12 ++-- main/util/src/mill/util/ColorLogger.scala | 4 +- main/util/src/mill/util/MultiLogger.scala | 23 ++++++- main/util/src/mill/util/PrefixLogger.scala | 62 +++++++++++-------- main/util/src/mill/util/PromptLogger.scala | 17 +++-- main/util/src/mill/util/ProxyLogger.scala | 6 +- .../src/mill/runner/MillBuildBootstrap.scala | 8 +-- 14 files changed, 112 insertions(+), 98 deletions(-) delete mode 100644 example/extending/imports/1-import-ivy/bar/src/Bar.scala delete mode 100644 example/extending/imports/2-import-ivy-scala/foo/src/Foo.java diff --git a/bsp/worker/src/mill/bsp/worker/MillBspLogger.scala b/bsp/worker/src/mill/bsp/worker/MillBspLogger.scala index 7f386b4dc37..59c4104621c 100644 --- a/bsp/worker/src/mill/bsp/worker/MillBspLogger.scala +++ b/bsp/worker/src/mill/bsp/worker/MillBspLogger.scala @@ -20,8 +20,8 @@ import mill.util.{ColorLogger, ProxyLogger} private class MillBspLogger(client: BuildClient, taskId: Int, logger: Logger) extends ProxyLogger(logger) with ColorLogger { - def infoColor = fansi.Color.Blue - def errorColor = fansi.Color.Red + override def infoColor = fansi.Color.Blue + override def errorColor = fansi.Color.Red override def ticker(s: String): Unit = { try { diff --git a/example/extending/imports/1-import-ivy/bar/src/Bar.scala b/example/extending/imports/1-import-ivy/bar/src/Bar.scala deleted file mode 100644 index 1a51746c2ef..00000000000 --- a/example/extending/imports/1-import-ivy/bar/src/Bar.scala +++ /dev/null @@ -1,8 +0,0 @@ -package bar - -object Bar { - val value = os.read(os.resource / "snippet.txt") - def main(args: Array[String]): Unit = { - println("generated snippet.txt resource: " + value) - } -} diff --git a/example/extending/imports/2-import-ivy-scala/build.mill b/example/extending/imports/2-import-ivy-scala/build.mill index 1d88bbb182c..98582af6543 100644 --- a/example/extending/imports/2-import-ivy-scala/build.mill +++ b/example/extending/imports/2-import-ivy-scala/build.mill @@ -6,7 +6,7 @@ import mill._, scalalib._ import $ivy.`com.lihaoyi::scalatags:0.12.0`, scalatags.Text.all._ -object foo extends ScalaModule { +object bar extends ScalaModule { def scalaVersion = "2.13.8" def ivyDeps = Agg(ivy"com.lihaoyi::os-lib:0.10.7") @@ -25,17 +25,17 @@ object foo extends ScalaModule { /** Usage -> mill foo.compile +> mill bar.compile compiling 1 Scala source... ... -> mill foo.run +> mill bar.run generated snippet.txt resource:

hello

world

-> mill show foo.assembly -".../out/foo/assembly.dest/out.jar" +> mill show bar.assembly +".../out/bar/assembly.dest/out.jar" -> ./out/foo/assembly.dest/out.jar # mac/linux +> ./out/bar/assembly.dest/out.jar # mac/linux generated snippet.txt resource:

hello

world

*/ diff --git a/example/extending/imports/2-import-ivy-scala/foo/src/Foo.java b/example/extending/imports/2-import-ivy-scala/foo/src/Foo.java deleted file mode 100644 index 1cec754f604..00000000000 --- a/example/extending/imports/2-import-ivy-scala/foo/src/Foo.java +++ /dev/null @@ -1,12 +0,0 @@ -package foo; -import java.io.*; - -public class Foo{ - public static void main(String[] args) throws IOException{ - InputStream res = Foo.class.getResourceAsStream("/snippet.txt"); - - try (BufferedReader br = new BufferedReader(new InputStreamReader(res))) { - System.out.println("generated snippet.txt resource: " + br.readLine()); - } - } -} diff --git a/main/api/src/mill/api/Logger.scala b/main/api/src/mill/api/Logger.scala index bbb29fa7303..f50543dac32 100644 --- a/main/api/src/mill/api/Logger.scala +++ b/main/api/src/mill/api/Logger.scala @@ -1,7 +1,6 @@ package mill.api import java.io.{InputStream, PrintStream} - import mill.main.client.lock.{Lock, Locked} /** @@ -27,8 +26,11 @@ import mill.main.client.lock.{Lock, Locked} * used to display the final `show` output for easy piping. */ trait Logger extends AutoCloseable { + def infoColor: fansi.Attrs = fansi.Attrs.Empty + def errorColor: fansi.Attrs = fansi.Attrs.Empty def colored: Boolean + private[mill] def unprefixedSystemStreams: SystemStreams = systemStreams def systemStreams: SystemStreams def errorStream: PrintStream = systemStreams.err @@ -95,4 +97,7 @@ trait Logger extends AutoCloseable { throw new Exception("Cannot acquire lock on Mill output directory") } } + + def withOutStream(outStream: PrintStream): Logger = this + private[mill] def logPrefixKey: Seq[String] = Nil } diff --git a/main/eval/src/mill/eval/EvaluatorCore.scala b/main/eval/src/mill/eval/EvaluatorCore.scala index dafe01eafa4..bfc25731686 100644 --- a/main/eval/src/mill/eval/EvaluatorCore.scala +++ b/main/eval/src/mill/eval/EvaluatorCore.scala @@ -142,8 +142,7 @@ private[mill] trait EvaluatorCore extends GroupEvaluator { val contextLogger = new PrefixLogger( logger0 = logger, - key = if (!logger.enableTicker) Nil else Seq(countMsg), - tickerContext = GroupEvaluator.dynamicTickerPrefix.value, + key0 = if (!logger.enableTicker) Nil else Seq(countMsg), verboseKeySuffix = verboseKeySuffix, message = tickerPrefix, noPrefix = exclusive diff --git a/main/eval/src/mill/eval/GroupEvaluator.scala b/main/eval/src/mill/eval/GroupEvaluator.scala index cc65235a2e5..676505ecf98 100644 --- a/main/eval/src/mill/eval/GroupEvaluator.scala +++ b/main/eval/src/mill/eval/GroupEvaluator.scala @@ -12,7 +12,6 @@ import scala.collection.mutable import scala.reflect.NameTransformer.encode import scala.util.control.NonFatal import scala.util.hashing.MurmurHash3 -import scala.util.DynamicVariable /** * Logic around evaluating a single group, which is a collection of [[Task]]s @@ -188,22 +187,20 @@ private[mill] trait GroupEvaluator { if (labelled.task.flushDest) os.remove.all(paths.dest) val (newResults, newEvaluated) = - GroupEvaluator.dynamicTickerPrefix.withValue(s"[$countMsg] $targetLabel > ") { - evaluateGroup( - group, - results, - inputsHash, - paths = Some(paths), - maybeTargetLabel = targetLabel, - counterMsg = countMsg, - verboseKeySuffix = verboseKeySuffix, - zincProblemReporter, - testReporter, - logger, - executionContext, - exclusive - ) - } + evaluateGroup( + group, + results, + inputsHash, + paths = Some(paths), + maybeTargetLabel = targetLabel, + counterMsg = countMsg, + verboseKeySuffix = verboseKeySuffix, + zincProblemReporter, + testReporter, + logger, + executionContext, + exclusive + ) newResults(labelled.task) match { case TaskResult(Result.Failure(_, Some((v, _))), _) => @@ -496,7 +493,6 @@ private[mill] trait GroupEvaluator { } private[mill] object GroupEvaluator { - val dynamicTickerPrefix = new DynamicVariable("") case class Results( newResults: Map[Task[_], TaskResult[(Val, Int)]], diff --git a/main/src/mill/main/MainModule.scala b/main/src/mill/main/MainModule.scala index 0218984c82b..df6ddd5a239 100644 --- a/main/src/mill/main/MainModule.scala +++ b/main/src/mill/main/MainModule.scala @@ -51,12 +51,14 @@ object MainModule { )(f: Seq[(Any, Option[(RunScript.TaskName, ujson.Value)])] => ujson.Value) : Result[ujson.Value] = { + // When using `show`, redirect all stdout of the evaluated tasks so the + // printed JSON is the only thing printed to stdout. + val redirectLogger = log + .withOutStream(evaluator.baseLogger.errorStream) + .asInstanceOf[mill.util.ColorLogger] + RunScript.evaluateTasksNamed( - // When using `show`, redirect all stdout of the evaluated tasks so the - // printed JSON is the only thing printed to stdout. - evaluator.withBaseLogger( - evaluator.baseLogger.withOutStream(evaluator.baseLogger.errorStream) - ), + evaluator.withBaseLogger(redirectLogger), targets, Separated ) match { diff --git a/main/util/src/mill/util/ColorLogger.scala b/main/util/src/mill/util/ColorLogger.scala index 98756b935cc..eb424683ca5 100644 --- a/main/util/src/mill/util/ColorLogger.scala +++ b/main/util/src/mill/util/ColorLogger.scala @@ -5,7 +5,5 @@ import mill.api.Logger import java.io.PrintStream trait ColorLogger extends Logger { - def infoColor: fansi.Attrs - def errorColor: fansi.Attrs - def withOutStream(outStream: PrintStream): ColorLogger = this + override def withOutStream(outStream: PrintStream): ColorLogger = this } diff --git a/main/util/src/mill/util/MultiLogger.scala b/main/util/src/mill/util/MultiLogger.scala index d6ad8855686..9dda1ea4e4b 100644 --- a/main/util/src/mill/util/MultiLogger.scala +++ b/main/util/src/mill/util/MultiLogger.scala @@ -1,5 +1,6 @@ package mill.util +import fansi.Attrs import mill.api.{Logger, SystemStreams} import java.io.{InputStream, OutputStream, PrintStream} @@ -10,7 +11,7 @@ class MultiLogger( val logger2: Logger, val inStream0: InputStream, override val debugEnabled: Boolean -) extends Logger { +) extends ColorLogger { override def toString: String = s"MultiLogger($logger1, $logger2)" lazy val systemStreams = new SystemStreams( new MultiStream(logger1.systemStreams.out, logger2.systemStreams.out), @@ -18,6 +19,12 @@ class MultiLogger( inStream0 ) + private[mill] override lazy val unprefixedSystemStreams: SystemStreams = new SystemStreams( + new MultiStream(logger1.unprefixedSystemStreams.out, logger2.unprefixedSystemStreams.out), + new MultiStream(logger1.unprefixedSystemStreams.err, logger2.unprefixedSystemStreams.err), + inStream0 + ) + def info(s: String): Unit = { logger1.info(s) logger2.info(s) @@ -97,6 +104,20 @@ class MultiLogger( debugEnabled ) } + + override def infoColor: Attrs = logger1.infoColor ++ logger2.infoColor + override def errorColor: Attrs = logger1.errorColor ++ logger2.errorColor + private[mill] override def logPrefixKey = logger1.logPrefixKey ++ logger2.logPrefixKey + + override def withOutStream(outStream: PrintStream): ColorLogger = { + new MultiLogger( + colored, + logger1.withOutStream(outStream), + logger2.withOutStream(outStream), + inStream0, + debugEnabled + ) + } } class MultiStream(stream1: OutputStream, stream2: OutputStream) diff --git a/main/util/src/mill/util/PrefixLogger.scala b/main/util/src/mill/util/PrefixLogger.scala index 9920e8fccf6..68cd02e7321 100644 --- a/main/util/src/mill/util/PrefixLogger.scala +++ b/main/util/src/mill/util/PrefixLogger.scala @@ -1,13 +1,12 @@ package mill.util import mill.api.{Logger, SystemStreams} -import pprint.Util.literalize import java.io.PrintStream class PrefixLogger( val logger0: ColorLogger, - key: Seq[String], + key0: Seq[String], tickerContext: String = "", outStream0: Option[PrintStream] = None, errStream0: Option[PrintStream] = None, @@ -18,9 +17,12 @@ class PrefixLogger( // above the output of every command that gets run so we can see who the output belongs to noPrefix: Boolean = false ) extends ColorLogger { - val linePrefix: String = if (noPrefix || key.isEmpty) "" else s"[${key.mkString("-")}] " + private[mill] override val logPrefixKey = logger0.logPrefixKey ++ key0 + assert(key0.forall(_.nonEmpty)) + val linePrefix: String = + if (noPrefix || logPrefixKey.isEmpty) "" else s"[${logPrefixKey.mkString("-")}] " override def toString: String = - s"PrefixLogger($logger0, ${literalize(linePrefix)}, ${literalize(tickerContext)})" + s"PrefixLogger($logger0, $key0)" def this(logger0: ColorLogger, context: String, tickerContext: String) = this(logger0, Seq(context), tickerContext, None, None) def this( @@ -34,67 +36,77 @@ class PrefixLogger( override def colored = logger0.colored - def infoColor = logger0.infoColor - def errorColor = logger0.errorColor + override def infoColor = logger0.infoColor + override def errorColor = logger0.errorColor val systemStreams = new SystemStreams( out = outStream0.getOrElse( new PrintStream(new LinePrefixOutputStream( infoColor(linePrefix).render, - logger0.systemStreams.out, - () => reportKey(key) + logger0.unprefixedSystemStreams.out, + () => reportKey(logPrefixKey) )) ), err = errStream0.getOrElse( new PrintStream(new LinePrefixOutputStream( infoColor(linePrefix).render, - logger0.systemStreams.err, - () => reportKey(key) + logger0.unprefixedSystemStreams.err, + () => reportKey(logPrefixKey) )) ), logger0.systemStreams.in ) + private[mill] override val unprefixedSystemStreams = new SystemStreams( + outStream0.getOrElse(logger0.unprefixedSystemStreams.out), + errStream0.getOrElse(logger0.unprefixedSystemStreams.err), + logger0.unprefixedSystemStreams.in + ) + override def rawOutputStream = logger0.rawOutputStream override def info(s: String): Unit = { - reportKey(key) + reportKey(logPrefixKey) logger0.info(infoColor(linePrefix) + s) } override def error(s: String): Unit = { - reportKey(key) + reportKey(logPrefixKey) logger0.error(infoColor(linePrefix) + s) } - override def ticker(s: String): Unit = setPromptDetail(key, s) + override def ticker(s: String): Unit = setPromptDetail(logPrefixKey, s) override def setPromptDetail(key: Seq[String], s: String): Unit = logger0.setPromptDetail(key, s) private[mill] override def setPromptLine( - key: Seq[String], + callKey: Seq[String], verboseKeySuffix: String, message: String - ): Unit = - logger0.setPromptLine(key, verboseKeySuffix, message) + ): Unit = { + + logger0.setPromptLine(callKey, verboseKeySuffix, message) + } private[mill] override def setPromptLine(): Unit = - setPromptLine(key, verboseKeySuffix, message) + setPromptLine(logPrefixKey, verboseKeySuffix, message) override def debug(s: String): Unit = { - if (debugEnabled) reportKey(key) + if (debugEnabled) reportKey(logPrefixKey) logger0.debug(infoColor(linePrefix) + s) } override def debugEnabled: Boolean = logger0.debugEnabled override def withOutStream(outStream: PrintStream): PrefixLogger = new PrefixLogger( logger0.withOutStream(outStream), - Seq(infoColor(linePrefix).toString()), + logPrefixKey, infoColor(tickerContext).toString(), outStream0 = Some(outStream), errStream0 = Some(systemStreams.err) ) - private[mill] override def reportKey(key: Seq[String]): Unit = logger0.reportKey(key) - private[mill] override def removePromptLine(key: Seq[String]): Unit = - logger0.removePromptLine(key) - private[mill] override def removePromptLine(): Unit = removePromptLine(key) + + private[mill] override def reportKey(callKey: Seq[String]): Unit = + logger0.reportKey(callKey) + private[mill] override def removePromptLine(callKey: Seq[String]): Unit = + logger0.removePromptLine(callKey) + private[mill] override def removePromptLine(): Unit = removePromptLine(logPrefixKey) private[mill] override def setPromptLeftHeader(s: String): Unit = logger0.setPromptLeftHeader(s) override def enableTicker = logger0.enableTicker @@ -104,8 +116,8 @@ class PrefixLogger( message: String ): Logger = { new PrefixLogger( - logger0, - key :+ subKeySuffix, + this, + Seq(subKeySuffix), tickerContext, outStream0, errStream0, diff --git a/main/util/src/mill/util/PromptLogger.scala b/main/util/src/mill/util/PromptLogger.scala index e86e8bc1545..8846355f433 100644 --- a/main/util/src/mill/util/PromptLogger.scala +++ b/main/util/src/mill/util/PromptLogger.scala @@ -275,17 +275,14 @@ private[mill] object PromptLogger { private implicit def seqOrdering = new Ordering[Seq[String]] { def compare(xs: Seq[String], ys: Seq[String]): Int = { - xs.lengthCompare(ys) match { - case 0 => - val iter = xs.iterator.zip(ys) - while (iter.nonEmpty) { - val (x, y) = iter.next() - if (x > y) return 1 - else if (y > x) return -1 - } - return 0 - case n => n + val iter = xs.iterator.zip(ys) + while (iter.nonEmpty) { + val (x, y) = iter.next() + if (x > y) return 1 + else if (y > x) return -1 } + + return xs.lengthCompare(ys) } } private val statuses = collection.mutable.SortedMap.empty[Seq[String], Status] diff --git a/main/util/src/mill/util/ProxyLogger.scala b/main/util/src/mill/util/ProxyLogger.scala index be7e847ad84..240fcabd685 100644 --- a/main/util/src/mill/util/ProxyLogger.scala +++ b/main/util/src/mill/util/ProxyLogger.scala @@ -1,6 +1,6 @@ package mill.util -import mill.api.Logger +import mill.api.{Logger, SystemStreams} import java.io.PrintStream @@ -40,4 +40,8 @@ class ProxyLogger(logger: Logger) extends Logger { private[mill] override def withPromptUnpaused[T](t: => T): T = logger.withPromptUnpaused(t) override def enableTicker = logger.enableTicker + override def infoColor: fansi.Attrs = logger.infoColor + override def errorColor: fansi.Attrs = logger.errorColor + private[mill] override def logPrefixKey: Seq[String] = logger.logPrefixKey + private[mill] override def unprefixedSystemStreams: SystemStreams = logger.unprefixedSystemStreams } diff --git a/runner/src/mill/runner/MillBuildBootstrap.scala b/runner/src/mill/runner/MillBuildBootstrap.scala index 8cede3839fc..eff84c6f879 100644 --- a/runner/src/mill/runner/MillBuildBootstrap.scala +++ b/runner/src/mill/runner/MillBuildBootstrap.scala @@ -337,9 +337,9 @@ class MillBuildBootstrap( depth: Int ): Evaluator = { - val bootLogPrefix = - if (depth == 0) "" - else "[" + (Seq.fill(depth - 1)(millBuild) ++ Seq("build.mill")).mkString("/") + "] " + val bootLogPrefix: Seq[String] = + if (depth == 0) Nil + else Seq((Seq.fill(depth - 1)(millBuild) ++ Seq("build.mill")).mkString("/")) mill.eval.EvaluatorImpl( home, @@ -347,7 +347,7 @@ class MillBuildBootstrap( recOut(output, depth), recOut(output, depth), rootModule, - new PrefixLogger(logger, Nil, tickerContext = bootLogPrefix), + new PrefixLogger(logger, bootLogPrefix), classLoaderSigHash = millClassloaderSigHash, classLoaderIdentityHash = millClassloaderIdentityHash, workerCache = workerCache.to(collection.mutable.Map), From 0cdfefc95d36dc5bab92f823944914d53acf2878 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Wed, 9 Oct 2024 22:05:52 +0200 Subject: [PATCH 22/47] Clean up out folder lock logic (#3704) * The locking logic doesn't belong on `Logger`, moved to `MillMain` * Make it only start the logger once the lock is taken so the ticker doesn't show when Mill is not making progress * Simplify the test, a lot, and kill the dummy locks since they aren't really used * Store the `millActiveCommand` in the `out/` folder and print it out as part of the lock waiting message ``` lihaoyi mill$ /Users/lihaoyi/.cache/mill/download/0.12.0-RC3-35-c2dddd-DIRTYd553ab5d resolve _ Mill version 0.12.0-RC3-35-c2dddd-DIRTYd553ab5d is different than configured for this directory! Configured version is 0.12.0-RC3-32-b4a0bf (/Users/lihaoyi/Github/mill/.config/mill-version) Another Mill process is running '{scalalib,scalajslib,scalanativelib}.__.test', waiting for it to be done... ``` --- .../output-directory/resources/build.mill | 13 +- .../src/OutputDirectoryLockTests.scala | 95 ++++++-------- main/api/src/mill/api/Logger.scala | 14 -- .../client/src/mill/main/client/OutFiles.java | 5 + .../src/mill/main/client/lock/DummyLock.java | 22 ---- .../mill/main/client/lock/DummyTryLocked.java | 11 -- .../src/mill/main/client/lock/Lock.java | 5 - runner/src/mill/runner/MillCliConfig.scala | 2 +- runner/src/mill/runner/MillMain.scala | 122 ++++++++++++------ 9 files changed, 137 insertions(+), 152 deletions(-) delete mode 100644 main/client/src/mill/main/client/lock/DummyLock.java delete mode 100644 main/client/src/mill/main/client/lock/DummyTryLocked.java diff --git a/integration/feature/output-directory/resources/build.mill b/integration/feature/output-directory/resources/build.mill index e068bbd11cb..21d1473b74c 100644 --- a/integration/feature/output-directory/resources/build.mill +++ b/integration/feature/output-directory/resources/build.mill @@ -11,10 +11,15 @@ object `package` extends RootModule with ScalaModule { } def blockWhileExists(path: os.Path) = Task.Command[String] { - if (!os.exists(path)) - os.write(path, Array.emptyByteArray) - while (os.exists(path)) - Thread.sleep(100L) + os.write(path, Array.emptyByteArray) + + while (os.exists(path)) Thread.sleep(100L) "Blocking command done" } + + def writeMarker(path: os.Path) = Task.Command[String] { + os.write(path, Array.emptyByteArray) + + "Write marker done" + } } diff --git a/integration/feature/output-directory/src/OutputDirectoryLockTests.scala b/integration/feature/output-directory/src/OutputDirectoryLockTests.scala index 25b5a15d078..bcfce43b82f 100644 --- a/integration/feature/output-directory/src/OutputDirectoryLockTests.scala +++ b/integration/feature/output-directory/src/OutputDirectoryLockTests.scala @@ -2,91 +2,78 @@ package mill.integration import mill.testkit.UtestIntegrationTestSuite import utest._ +import utest.asserts.{RetryInterval, RetryMax} -import java.io.ByteArrayOutputStream -import java.util.concurrent.{CountDownLatch, Executors} - -import scala.concurrent.duration.Duration +import java.util.concurrent.Executors +import scala.concurrent.duration.{Duration, DurationInt} import scala.concurrent.{Await, ExecutionContext, Future} object OutputDirectoryLockTests extends UtestIntegrationTestSuite { private val pool = Executors.newCachedThreadPool() - private val ec = ExecutionContext.fromExecutorService(pool) + private implicit val ec = ExecutionContext.fromExecutorService(pool) override def utestAfterAll(): Unit = { pool.shutdown() } - + implicit val retryMax: RetryMax = RetryMax(30000.millis) + implicit val retryInterval: RetryInterval = RetryInterval(50.millis) def tests: Tests = Tests { test("basic") - integrationTest { tester => import tester._ val signalFile = workspacePath / "do-wait" - System.err.println("Spawning blocking task") - val blocksFuture = - Future(eval(("show", "blockWhileExists", "--path", signalFile), check = true))(ec) - while (!os.exists(signalFile) && !blocksFuture.isCompleted) - Thread.sleep(100L) - if (os.exists(signalFile)) - System.err.println("Blocking task is running") - else { - System.err.println("Failed to run blocking task") - Predef.assert(blocksFuture.isCompleted) - blocksFuture.value.get.get + // Kick off blocking task in background + Future { + eval(("show", "blockWhileExists", "--path", signalFile), check = true) } + // Wait for blocking task to write signal file, to indicate it has begun + eventually { os.exists(signalFile) } + val testCommand: os.Shellable = ("show", "hello") val testMessage = "Hello from hello task" - System.err.println("Evaluating task without lock") + // --no-build-lock allows commands to complete despite background blocker val noLockRes = eval(("--no-build-lock", testCommand), check = true) assert(noLockRes.out.contains(testMessage)) - System.err.println("Evaluating task without waiting for lock (should fail)") + // --no-wait-for-build-lock causes commands fail due to background blocker val noWaitRes = eval(("--no-wait-for-build-lock", testCommand)) - assert(noWaitRes.err.contains("Cannot proceed, another Mill process is running tasks")) - - System.err.println("Evaluating task waiting for the lock") - - val lock = new CountDownLatch(1) - val stderr = new ByteArrayOutputStream - var success = false + assert( + noWaitRes + .err + .contains( + s"Another Mill process is running 'show blockWhileExists --path $signalFile', failing" + ) + ) + + // By default, we wait until the background blocking task completes + val waitingLogFile = workspacePath / "waitingLogFile" + val waitingCompleteFile = workspacePath / "waitingCompleteFile" val futureWaitingRes = Future { eval( - testCommand, - stderr = os.ProcessOutput { - val expectedMessage = - "Another Mill process is running tasks, waiting for it to be done..." - - (bytes, len) => - stderr.write(bytes, 0, len) - val output = new String(stderr.toByteArray) - if (output.contains(expectedMessage)) - lock.countDown() - }, + ("show", "writeMarker", "--path", waitingCompleteFile), + stderr = waitingLogFile, check = true ) - }(ec) - try { - lock.await() - success = true - } finally { - if (!success) { - System.err.println("Waiting task output:") - System.err.write(stderr.toByteArray) - } } - System.err.println("Task is waiting for the lock, unblocking it") - os.remove(signalFile) - - System.err.println("Blocking task should exit") - val blockingRes = Await.result(blocksFuture, Duration.Inf) - assert(blockingRes.out.contains("Blocking command done")) + // Ensure we see the waiting message + eventually { + os.read(waitingLogFile) + .contains( + s"Another Mill process is running 'show blockWhileExists --path $signalFile', waiting for it to be done..." + ) + } - System.err.println("Waiting task should be free to proceed") + // Even after task starts waiting on blocking task, it is not complete + assert(!futureWaitingRes.isCompleted) + assert(!os.exists(waitingCompleteFile)) + // Terminate blocking task, make sure waiting task now completes + os.remove(signalFile) val waitingRes = Await.result(futureWaitingRes, Duration.Inf) - assert(waitingRes.out.contains(testMessage)) + assert(os.exists(waitingCompleteFile)) + assert(waitingRes.out == "\"Write marker done\"") } } } diff --git a/main/api/src/mill/api/Logger.scala b/main/api/src/mill/api/Logger.scala index f50543dac32..430b6352f99 100644 --- a/main/api/src/mill/api/Logger.scala +++ b/main/api/src/mill/api/Logger.scala @@ -1,7 +1,6 @@ package mill.api import java.io.{InputStream, PrintStream} -import mill.main.client.lock.{Lock, Locked} /** * The standard logging interface of the Mill build tool. @@ -85,19 +84,6 @@ trait Logger extends AutoCloseable { finally removePromptLine() } - def waitForLock(lock: Lock, waitingAllowed: Boolean): Locked = { - val tryLocked = lock.tryLock() - if (tryLocked.isLocked()) - tryLocked - else if (waitingAllowed) { - info("Another Mill process is running tasks, waiting for it to be done...") - lock.lock() - } else { - error("Cannot proceed, another Mill process is running tasks") - throw new Exception("Cannot acquire lock on Mill output directory") - } - } - def withOutStream(outStream: PrintStream): Logger = this private[mill] def logPrefixKey: Seq[String] = Nil } diff --git a/main/client/src/mill/main/client/OutFiles.java b/main/client/src/mill/main/client/OutFiles.java index 0af23e233a1..b69b93b535c 100644 --- a/main/client/src/mill/main/client/OutFiles.java +++ b/main/client/src/mill/main/client/OutFiles.java @@ -62,4 +62,9 @@ public class OutFiles { */ final public static String millLock = "mill-lock"; + /** + * Any active Mill command that is currently run, for debugging purposes + */ + final public static String millActiveCommand = "mill-active-command"; + } diff --git a/main/client/src/mill/main/client/lock/DummyLock.java b/main/client/src/mill/main/client/lock/DummyLock.java deleted file mode 100644 index ede8224323d..00000000000 --- a/main/client/src/mill/main/client/lock/DummyLock.java +++ /dev/null @@ -1,22 +0,0 @@ -package mill.main.client.lock; - -import java.util.concurrent.locks.ReentrantLock; - -class DummyLock extends Lock { - - public boolean probe() { - return true; - } - - public Locked lock() { - return new DummyTryLocked(); - } - - public TryLocked tryLock() { - return new DummyTryLocked(); - } - - @Override - public void close() throws Exception { - } -} diff --git a/main/client/src/mill/main/client/lock/DummyTryLocked.java b/main/client/src/mill/main/client/lock/DummyTryLocked.java deleted file mode 100644 index 34ad7b5ea15..00000000000 --- a/main/client/src/mill/main/client/lock/DummyTryLocked.java +++ /dev/null @@ -1,11 +0,0 @@ -package mill.main.client.lock; - -class DummyTryLocked implements TryLocked { - public DummyTryLocked() { - } - - public boolean isLocked(){ return true; } - - public void release() throws Exception { - } -} diff --git a/main/client/src/mill/main/client/lock/Lock.java b/main/client/src/mill/main/client/lock/Lock.java index 3870bc07a14..d7f65ee2b85 100644 --- a/main/client/src/mill/main/client/lock/Lock.java +++ b/main/client/src/mill/main/client/lock/Lock.java @@ -23,9 +23,4 @@ public static Lock file(String path) throws Exception { public static Lock memory() { return new MemoryLock(); } - - public static Lock dummy() { - return new DummyLock(); - } - } diff --git a/runner/src/mill/runner/MillCliConfig.scala b/runner/src/mill/runner/MillCliConfig.scala index 429a394f18a..afd41f14f95 100644 --- a/runner/src/mill/runner/MillCliConfig.scala +++ b/runner/src/mill/runner/MillCliConfig.scala @@ -140,7 +140,7 @@ case class MillCliConfig( @arg( hidden = true, doc = - """Do not wait for an exclusive lock on the Mill output directory to evaluate tasks / commands. Fail if waiting for a lock is needed.""" + """Do not wait for an exclusive lock on the Mill output directory to evaluate tasks / commands.""" ) noWaitForBuildLock: Flag = Flag() ) diff --git a/runner/src/mill/runner/MillMain.scala b/runner/src/mill/runner/MillMain.scala index 817e126af7c..606c4ca2fd0 100644 --- a/runner/src/mill/runner/MillMain.scala +++ b/runner/src/mill/runner/MillMain.scala @@ -10,7 +10,7 @@ import mill.bsp.{BspContext, BspServerResult} import mill.main.BuildInfo import mill.main.client.{OutFiles, ServerFiles} import mill.main.client.lock.Lock -import mill.util.{PromptLogger, PrintLogger, Colors} +import mill.util.{Colors, PrintLogger, PromptLogger} import java.lang.reflect.InvocationTargetException import scala.util.control.NonFatal @@ -212,9 +212,7 @@ object MillMain { .getOrElse(config.leftoverArgs.value.toList) val out = os.Path(OutFiles.out, WorkspaceRoot.workspaceRoot) - val outLock = - if (config.noBuildLock.value || bspContext.isDefined) Lock.dummy() - else Lock.file((out / OutFiles.millLock).toString) + var repeatForBsp = true var loopRes: (Boolean, RunnerState) = (false, RunnerState.empty) while (repeatForBsp) { @@ -228,44 +226,46 @@ object MillMain { evaluate = (prevState: Option[RunnerState]) => { adjustJvmProperties(userSpecifiedProperties, initialSystemProperties) - val logger = getLogger( - streams, - config, - mainInteractive, - enableTicker = - config.ticker - .orElse(config.enableTicker) - .orElse(Option.when(config.disableTicker.value)(false)), - printLoggerState, - serverDir, - colored = colored, - colors = colors - ) - Using.resources( - logger, - logger.waitForLock( - outLock, - waitingAllowed = !config.noWaitForBuildLock.value + withOutLock( + config.noBuildLock.value || bspContext.isDefined, + config.noWaitForBuildLock.value, + out, + targetsAndParams, + streams + ) { + val logger = getLogger( + streams, + config, + mainInteractive, + enableTicker = + config.ticker + .orElse(config.enableTicker) + .orElse(Option.when(config.disableTicker.value)(false)), + printLoggerState, + serverDir, + colored = colored, + colors = colors ) - ) { (_, _) => - new MillBuildBootstrap( - projectRoot = WorkspaceRoot.workspaceRoot, - output = out, - home = config.home, - keepGoing = config.keepGoing.value, - imports = config.imports, - env = env, - threadCount = threadCount, - targetsAndParams = targetsAndParams, - prevRunnerState = prevState.getOrElse(stateCache), - logger = logger, - disableCallgraph = config.disableCallgraph.value, - needBuildSc = needBuildSc(config), - requestedMetaLevel = config.metaLevel, - config.allowPositional.value, - systemExit = systemExit, - streams0 = streams0 - ).evaluate() + Using.resource(logger) { _ => + try new MillBuildBootstrap( + projectRoot = WorkspaceRoot.workspaceRoot, + output = out, + home = config.home, + keepGoing = config.keepGoing.value, + imports = config.imports, + env = env, + threadCount = threadCount, + targetsAndParams = targetsAndParams, + prevRunnerState = prevState.getOrElse(stateCache), + logger = logger, + disableCallgraph = config.disableCallgraph.value, + needBuildSc = needBuildSc(config), + requestedMetaLevel = config.metaLevel, + config.allowPositional.value, + systemExit = systemExit, + streams0 = streams0 + ).evaluate() + } } }, colors = colors @@ -416,4 +416,44 @@ object MillMain { for (k <- systemPropertiesToUnset) System.clearProperty(k) for ((k, v) <- desiredProps) System.setProperty(k, v) } + + def withOutLock[T]( + noBuildLock: Boolean, + noWaitForBuildLock: Boolean, + out: os.Path, + targetsAndParams: Seq[String], + streams: SystemStreams + )(t: => T): T = { + if (noBuildLock) t + else { + val outLock = Lock.file((out / OutFiles.millLock).toString) + + def activeTaskString = + try { + os.read(out / OutFiles.millActiveCommand) + } catch { + case e => "" + } + + def activeTaskPrefix = s"Another Mill process is running '$activeTaskString'," + Using.resource { + val tryLocked = outLock.tryLock() + if (tryLocked.isLocked()) tryLocked + else if (noWaitForBuildLock) { + throw new Exception(s"$activeTaskPrefix failing") + } else { + + streams.err.println( + s"$activeTaskPrefix waiting for it to be done..." + ) + outLock.lock() + } + } { _ => + os.write.over(out / OutFiles.millActiveCommand, targetsAndParams.mkString(" ")) + try t + finally os.remove.all(out / OutFiles.millActiveCommand) + } + } + } + } From 9a3c110d3cb62f15ea370a30b436135724b4b219 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Thu, 10 Oct 2024 11:43:07 +0200 Subject: [PATCH 23/47] Update coursier to 2.1.14 (#3705) Fixes https://github.com/com-lihaoyi/mill/issues/3695 along with https://github.com/com-lihaoyi/mill/pull/3701 (the two PRs are needed) --- build.mill | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.mill b/build.mill index f2691ab2b41..f4a9b0f2e24 100644 --- a/build.mill +++ b/build.mill @@ -120,7 +120,7 @@ object Deps { val asmTree = ivy"org.ow2.asm:asm-tree:9.7" val bloopConfig = ivy"ch.epfl.scala::bloop-config:1.5.5" - val coursier = ivy"io.get-coursier::coursier:2.1.13" + val coursier = ivy"io.get-coursier::coursier:2.1.14" val coursierInterface = ivy"io.get-coursier:interface:1.0.19" val cask = ivy"com.lihaoyi::cask:0.9.4" From 6b8316948af6c0cf93f8e1c3c653e19aa7070255 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Thu, 10 Oct 2024 11:44:29 +0200 Subject: [PATCH 24/47] Drop unnecessary filter on artifacts (#3701) Artifacts should already be filtered by `coursier.Artifacts` (via the default classifiers and artifact types stuff), no need to filter them further --- main/util/src/mill/util/CoursierSupport.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main/util/src/mill/util/CoursierSupport.scala b/main/util/src/mill/util/CoursierSupport.scala index 2b37781bb81..0ab43fc6b3d 100644 --- a/main/util/src/mill/util/CoursierSupport.scala +++ b/main/util/src/mill/util/CoursierSupport.scala @@ -104,7 +104,7 @@ trait CoursierSupport { Agg.from( res.files .map(os.Path(_)) - .filter(path => path.ext == "jar" && resolveFilter(path)) + .filter(resolveFilter) .map(PathRef(_, quick = true)) ) ++ localTestDeps.flatten ) From c10c2585ca6d0e3109cafa8a6c081bc0f6e2d40a Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Thu, 10 Oct 2024 11:45:21 +0200 Subject: [PATCH 25/47] Make use of new `os.zip.open` operation to create assemblies (#3707) Should be basically the same logic, just now we use the version bundled within `os.zip` --- scalalib/src/mill/scalalib/Assembly.scala | 56 +++++++++++------------ 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/scalalib/src/mill/scalalib/Assembly.scala b/scalalib/src/mill/scalalib/Assembly.scala index 8b389863b2d..13801a86ef9 100644 --- a/scalalib/src/mill/scalalib/Assembly.scala +++ b/scalalib/src/mill/scalalib/Assembly.scala @@ -6,9 +6,8 @@ import mill.api.{Ctx, IO, PathRef} import os.Generator import java.io.{ByteArrayInputStream, InputStream, SequenceInputStream} -import java.net.URI import java.nio.file.attribute.PosixFilePermission -import java.nio.file.{FileSystems, Files, StandardOpenOption} +import java.nio.file.StandardOpenOption import java.util.Collections import java.util.jar.JarFile import java.util.regex.Pattern @@ -222,44 +221,46 @@ object Assembly { os.remove(rawJar) // use the `base` (the upstream assembly) as a start - val baseUri = "jar:" + rawJar.toIO.getCanonicalFile.toURI.toASCIIString - val hm = base.fold(Map("create" -> "true")) { b => - os.copy(b, rawJar) - Map.empty - } + base.foreach(os.copy.over(_, rawJar)) var addedEntryCount = 0 // Add more files by copying files to a JAR file system - Using.resource(FileSystems.newFileSystem(URI.create(baseUri), hm.asJava)) { zipFs => - val manifestPath = zipFs.getPath(JarFile.MANIFEST_NAME) - Files.createDirectories(manifestPath.getParent) - val manifestOut = Files.newOutputStream( + Using.resource(os.zip.open(rawJar)) { zipRoot => + val manifestPath = zipRoot / os.SubPath(JarFile.MANIFEST_NAME) + os.makeDir.all(manifestPath / os.up) + Using.resource(os.write.outputStream( manifestPath, - StandardOpenOption.TRUNCATE_EXISTING, - StandardOpenOption.CREATE - ) - manifest.build.write(manifestOut) - manifestOut.close() + openOptions = Seq( + StandardOpenOption.TRUNCATE_EXISTING, + StandardOpenOption.CREATE + ) + )) { manifestOut => + manifest.build.write(manifestOut) + } val (mappings, resourceCleaner) = Assembly.loadShadedClasspath(inputPaths, assemblyRules) try { Assembly.groupAssemblyEntries(mappings, assemblyRules).foreach { case (mapping, entry) => - val path = zipFs.getPath(mapping).toAbsolutePath + val path = zipRoot / os.SubPath(mapping) entry match { case entry: AppendEntry => val separated = entry.inputStreams .flatMap(inputStream => Seq(new ByteArrayInputStream(entry.separator.getBytes), inputStream()) ) - val cleaned = if (Files.exists(path)) separated else separated.drop(1) - val concatenated = new SequenceInputStream(Collections.enumeration(cleaned.asJava)) - addedEntryCount += 1 - writeEntry(path, concatenated, append = true) + val cleaned = if (os.exists(path)) separated else separated.drop(1) + Using.resource(new SequenceInputStream(Collections.enumeration(cleaned.asJava))) { + concatenated => + addedEntryCount += 1 + writeEntry(path, concatenated, append = true) + } case entry: WriteOnceEntry => addedEntryCount += 1 - writeEntry(path, entry.inputStream(), append = false) + Using.resource(entry.inputStream()) { stream => + writeEntry(path, stream, append = false) + } } } } finally { @@ -297,15 +298,14 @@ object Assembly { Assembly(PathRef(destJar), addedEntryCount) } - private def writeEntry(p: java.nio.file.Path, inputStream: InputStream, append: Boolean): Unit = { - if (p.getParent != null) Files.createDirectories(p.getParent) + private def writeEntry(p: os.Path, inputStream: InputStream, append: Boolean): Unit = { + if (p.segmentCount != 0) os.makeDir.all(p / os.up) val options = if (append) Seq(StandardOpenOption.APPEND, StandardOpenOption.CREATE) else Seq(StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE) - val outputStream = java.nio.file.Files.newOutputStream(p, options: _*) - IO.stream(inputStream, outputStream) - outputStream.close() - inputStream.close() + Using.resource(os.write.outputStream(p, openOptions = options)) { outputStream => + IO.stream(inputStream, outputStream) + } } } From 463547bc8a61eca29f4f21e7d8515b5268d28bda Mon Sep 17 00:00:00 2001 From: Jamie Thompson Date: Thu, 10 Oct 2024 12:07:24 +0200 Subject: [PATCH 26/47] use case object in KotlinJSModule (#3708) These appear to be ADTs - and they need to generate upickle readwriters makes it easier for the Scala 3 upgrade to get this in now --- .../mill/kotlinlib/js/KotlinJSModule.scala | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/kotlinlib/src/mill/kotlinlib/js/KotlinJSModule.scala b/kotlinlib/src/mill/kotlinlib/js/KotlinJSModule.scala index 3540c099642..07f284a4a37 100644 --- a/kotlinlib/src/mill/kotlinlib/js/KotlinJSModule.scala +++ b/kotlinlib/src/mill/kotlinlib/js/KotlinJSModule.scala @@ -453,45 +453,45 @@ trait KotlinJSModule extends KotlinModule { outer => sealed trait ModuleKind { def extension: String } object ModuleKind { - object NoModule extends ModuleKind { val extension = "js" } + case object NoModule extends ModuleKind { val extension = "js" } implicit val rwNoModule: RW[NoModule.type] = macroRW - object UMDModule extends ModuleKind { val extension = "js" } + case object UMDModule extends ModuleKind { val extension = "js" } implicit val rwUMDModule: RW[UMDModule.type] = macroRW - object CommonJSModule extends ModuleKind { val extension = "js" } + case object CommonJSModule extends ModuleKind { val extension = "js" } implicit val rwCommonJSModule: RW[CommonJSModule.type] = macroRW - object AMDModule extends ModuleKind { val extension = "js" } + case object AMDModule extends ModuleKind { val extension = "js" } implicit val rwAMDModule: RW[AMDModule.type] = macroRW - object ESModule extends ModuleKind { val extension = "mjs" } + case object ESModule extends ModuleKind { val extension = "mjs" } implicit val rwESModule: RW[ESModule.type] = macroRW - object PlainModule extends ModuleKind { val extension = "js" } + case object PlainModule extends ModuleKind { val extension = "js" } implicit val rwPlainModule: RW[PlainModule.type] = macroRW } sealed trait SourceMapEmbedSourcesKind object SourceMapEmbedSourcesKind { - object Always extends SourceMapEmbedSourcesKind + case object Always extends SourceMapEmbedSourcesKind implicit val rwAlways: RW[Always.type] = macroRW - object Never extends SourceMapEmbedSourcesKind + case object Never extends SourceMapEmbedSourcesKind implicit val rwNever: RW[Never.type] = macroRW - object Inlining extends SourceMapEmbedSourcesKind + case object Inlining extends SourceMapEmbedSourcesKind implicit val rwInlining: RW[Inlining.type] = macroRW } sealed trait SourceMapNamesPolicy object SourceMapNamesPolicy { - object SimpleNames extends SourceMapNamesPolicy + case object SimpleNames extends SourceMapNamesPolicy implicit val rwSimpleNames: RW[SimpleNames.type] = macroRW - object FullyQualifiedNames extends SourceMapNamesPolicy + case object FullyQualifiedNames extends SourceMapNamesPolicy implicit val rwFullyQualifiedNames: RW[FullyQualifiedNames.type] = macroRW - object No extends SourceMapNamesPolicy + case object No extends SourceMapNamesPolicy implicit val rwNo: RW[No.type] = macroRW } sealed trait BinaryKind object BinaryKind { - object Library extends BinaryKind + case object Library extends BinaryKind implicit val rwLibrary: RW[Library.type] = macroRW - object Executable extends BinaryKind + case object Executable extends BinaryKind implicit val rwExecutable: RW[Executable.type] = macroRW implicit val rw: RW[BinaryKind] = macroRW } @@ -499,14 +499,14 @@ object BinaryKind { sealed trait RunTarget object RunTarget { // TODO rely on the node version installed in the env or fetch a specific one? - object Node extends RunTarget + case object Node extends RunTarget implicit val rwNode: RW[Node.type] = macroRW implicit val rw: RW[RunTarget] = macroRW } private[kotlinlib] sealed trait OutputMode private[kotlinlib] object OutputMode { - object Js extends OutputMode - object KlibDir extends OutputMode - object KlibFile extends OutputMode + case object Js extends OutputMode + case object KlibDir extends OutputMode + case object KlibFile extends OutputMode } From 365635e0129aed933eaadb0d678aaf7c6b480e0e Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Thu, 10 Oct 2024 13:07:03 +0200 Subject: [PATCH 27/47] Add kotlinx-html to `3-hello-kotlinjs` example (#3709) Takes advantage of the new klib support pulled in by https://github.com/com-lihaoyi/mill/pull/3705 We don't support the `module` file indirection, so for now we have to reference the `-js` version of the artifact directly --- .../kotlinlib/web/3-hello-kotlinjs/build.mill | 19 +++++++++++-------- .../web/3-hello-kotlinjs/foo/src/foo/Hello.kt | 9 +++++++-- .../foo/test/src/foo/HelloTests.kt | 6 ++++-- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/example/kotlinlib/web/3-hello-kotlinjs/build.mill b/example/kotlinlib/web/3-hello-kotlinjs/build.mill index 950fec1eb2a..cd69d50d31c 100644 --- a/example/kotlinlib/web/3-hello-kotlinjs/build.mill +++ b/example/kotlinlib/web/3-hello-kotlinjs/build.mill @@ -1,9 +1,9 @@ // KotlinJS support on Mill is still Work In Progress (WIP). As of time of writing it -// does not support third-party dependencies, Kotlin 2.x with KMP KLIB files, Node.js/Webpack -// test runners and reporting, etc. +// Node.js/Webpack test runners and reporting, etc. // -// The example below demonstrates only the minimal compilation, running, and testing of a single KotlinJS -// module. For more details in fully developing KotlinJS support, see the following ticket: +// The example below demonstrates only the minimal compilation, running, and testing of +// a single KotlinJS module using a single third-party dependency. For more details in +// fully developing KotlinJS support, see the following ticket: // // * https://github.com/com-lihaoyi/mill/issues/3611 @@ -14,6 +14,9 @@ object foo extends KotlinJSModule { def moduleKind = ModuleKind.ESModule def kotlinVersion = "1.9.25" def kotlinJSRunTarget = Some(RunTarget.Node) + def ivyDeps = Agg( + ivy"org.jetbrains.kotlinx:kotlinx-html-js:0.11.0", + ) object test extends KotlinJSModule with KotlinJSKotlinXTests } @@ -22,7 +25,7 @@ object foo extends KotlinJSModule { > mill foo.run Compiling 1 Kotlin sources to .../out/foo/compile.dest/classes... -Hello, world +

Hello World

stringifiedJsObject: ["hello","world","!"] > mill foo.test # Test is incorrect, `foo.test`` fails @@ -30,13 +33,13 @@ Compiling 1 Kotlin sources to .../out/foo/test/compile.dest/classes... Linking IR to .../out/foo/test/linkBinary.dest/binaries produce executable: .../out/foo/test/linkBinary.dest/binaries ... -error: AssertionError: Expected , actual . +error: AssertionError: Expected <

Hello World

>, actual <

Hello World Wrong

>. > cat out/foo/test/linkBinary.dest/binaries/test.js # Generated javascript on disk -...assertEquals_0(getString(), 'Not hello, world');... +...assertEquals_0(..., '

Hello World Wrong<\/h1>');... ... -> sed -i.bak 's/Not hello, world/Hello, world/g' foo/test/src/foo/HelloTests.kt +> sed -i.bak 's/Hello World Wrong/Hello World/g' foo/test/src/foo/HelloTests.kt > mill foo.test # passes after fixing test diff --git a/example/kotlinlib/web/3-hello-kotlinjs/foo/src/foo/Hello.kt b/example/kotlinlib/web/3-hello-kotlinjs/foo/src/foo/Hello.kt index 09f3ccd16af..b3348c98139 100644 --- a/example/kotlinlib/web/3-hello-kotlinjs/foo/src/foo/Hello.kt +++ b/example/kotlinlib/web/3-hello-kotlinjs/foo/src/foo/Hello.kt @@ -1,11 +1,16 @@ package foo -fun getString() = "Hello, world" +import kotlinx.html.* +import kotlinx.html.stream.createHTML fun main() { - println(getString()) + println(hello()) val parsedJsonStr: dynamic = JSON.parse("""{"helloworld": ["hello", "world", "!"]}""") val stringifiedJsObject = JSON.stringify(parsedJsonStr.helloworld) println("stringifiedJsObject: " + stringifiedJsObject) } + +fun hello(): String { + return createHTML().h1 { +"Hello World" }.toString() +} \ No newline at end of file diff --git a/example/kotlinlib/web/3-hello-kotlinjs/foo/test/src/foo/HelloTests.kt b/example/kotlinlib/web/3-hello-kotlinjs/foo/test/src/foo/HelloTests.kt index 7526f739947..fc33731c87a 100644 --- a/example/kotlinlib/web/3-hello-kotlinjs/foo/test/src/foo/HelloTests.kt +++ b/example/kotlinlib/web/3-hello-kotlinjs/foo/test/src/foo/HelloTests.kt @@ -6,8 +6,10 @@ import kotlin.test.assertEquals class HelloTests { @Test - fun failure() { - assertEquals(getString(), "Not hello, world") + fun testHello() { + val result = hello() + assertEquals(result.trim(), "

Hello World Wrong

") + result } } From a4d3e944037359cd7869e98068ed1774892e8847 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Thu, 10 Oct 2024 15:31:02 +0200 Subject: [PATCH 28/47] Try to fix deprecated discover warning (#3711) fixes https://github.com/com-lihaoyi/mill/issues/3385 Somehow the `@nowarn` annotations don't seem to work after we moved the `millDiscover` definition into the main module body (???) so instead of relying on those I try to fix the deprecated warning in the `Discover` macro itself by identifying type annotations and explicitly asking not to dereference them Manually tested by running the commands below, observed that no deprecation warning was printed ``` rm -rf /Users/lihaoyi/Github/mill/example/javalib/basic/1-simple/out ./mill -i dist.run example/javalib/basic/1-simple -i run -t hello ``` --- main/define/src/mill/define/Discover.scala | 11 ++++++++++- runner/src/mill/runner/CodeGen.scala | 20 ++++++++++---------- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/main/define/src/mill/define/Discover.scala b/main/define/src/mill/define/Discover.scala index c3136c2a0a0..03316412f80 100644 --- a/main/define/src/mill/define/Discover.scala +++ b/main/define/src/mill/define/Discover.scala @@ -141,11 +141,20 @@ object Discover { } if overridesRoutes._1.nonEmpty || overridesRoutes._2.nonEmpty || overridesRoutes._3.nonEmpty } yield { + val lhs0 = discoveredModuleType match { + // Explicitly do not de-alias type refs, so type aliases to deprecated + // types do not result in spurious deprecation warnings appearing + case tr: TypeRef => tr + // Other types are fine + case _ => discoveredModuleType.typeSymbol.asClass.toType + } + + val lhs = q"classOf[$lhs0]" + // by wrapping the `overridesRoutes` in a lambda function we kind of work around // the problem of generating a *huge* macro method body that finally exceeds the // JVM's maximum allowed method size val overridesLambda = q"(() => $overridesRoutes)()" - val lhs = q"classOf[${discoveredModuleType.typeSymbol.asClass}]" q"$lhs -> $overridesLambda" } diff --git a/runner/src/mill/runner/CodeGen.scala b/runner/src/mill/runner/CodeGen.scala index fa75337cb4c..58d549d69a5 100644 --- a/runner/src/mill/runner/CodeGen.scala +++ b/runner/src/mill/runner/CodeGen.scala @@ -161,11 +161,7 @@ object CodeGen { newScriptCode = objectData.name.applyTo(newScriptCode, wrapperObjectName) newScriptCode = objectData.obj.applyTo(newScriptCode, "abstract class") - val millDiscover = - if (segments.nonEmpty) "" - else - """@_root_.scala.annotation.nowarn - | override lazy val millDiscover: _root_.mill.define.Discover = _root_.mill.define.Discover[this.type]""".stripMargin + val millDiscover = discoverSnippet(segments) s"""$pkgLine |$aliasImports @@ -224,11 +220,7 @@ object CodeGen { s"extends _root_.mill.main.RootModule.Subfolder " } - val millDiscover = - if (segments.nonEmpty) "" - else - """@_root_.scala.annotation.nowarn - | override lazy val millDiscover: _root_.mill.define.Discover = _root_.mill.define.Discover[this.type]""".stripMargin + val millDiscover = discoverSnippet(segments) // User code needs to be put in a separate class for proper submodule // object initialization due to https://github.com/scala/scala3/issues/21444 @@ -240,6 +232,14 @@ object CodeGen { } + def discoverSnippet(segments: Seq[String]): String = { + if (segments.nonEmpty) "" + else + """override lazy val millDiscover: _root_.mill.define.Discover = _root_.mill.define.Discover[this.type] + |""".stripMargin + + } + private case class Snippet(var text: String = null, var start: Int = -1, var end: Int = -1) { def applyTo(s: String, replacement: String): String = s.patch(start, replacement.padTo(end - start, ' '), end - start) From fe6ac679df18410fd953ee11bc58bf847b60e624 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Thu, 10 Oct 2024 16:19:03 +0200 Subject: [PATCH 29/47] Bump OutputDirectoryLockTests wait timeout to try and mitigate flakiness Seems the 30s `retryMax` is timing out sometimes. Not sure if it's actually stuck or not, but if not bumping it to 60s should help --- .../feature/output-directory/src/OutputDirectoryLockTests.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/feature/output-directory/src/OutputDirectoryLockTests.scala b/integration/feature/output-directory/src/OutputDirectoryLockTests.scala index bcfce43b82f..13799e273aa 100644 --- a/integration/feature/output-directory/src/OutputDirectoryLockTests.scala +++ b/integration/feature/output-directory/src/OutputDirectoryLockTests.scala @@ -16,7 +16,7 @@ object OutputDirectoryLockTests extends UtestIntegrationTestSuite { override def utestAfterAll(): Unit = { pool.shutdown() } - implicit val retryMax: RetryMax = RetryMax(30000.millis) + implicit val retryMax: RetryMax = RetryMax(60000.millis) implicit val retryInterval: RetryInterval = RetryInterval(50.millis) def tests: Tests = Tests { test("basic") - integrationTest { tester => From 9a0ffc21acb56f8c7604f4e5c37a495fba863baf Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Thu, 10 Oct 2024 16:19:57 +0200 Subject: [PATCH 30/47] Try and speed up proguard contrib module/tests by re-using global java runtime jar (#3713) Proguard is a long pole in the windows contrib test job, hopefully this makes it a bit faster Seems to cut a few minutes; previously it ran in 12-17min, with this PR it ran in 10+ min --- .../src/mill/contrib/proguard/Proguard.scala | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/contrib/proguard/src/mill/contrib/proguard/Proguard.scala b/contrib/proguard/src/mill/contrib/proguard/Proguard.scala index 7c577ff2549..662ab690b09 100644 --- a/contrib/proguard/src/mill/contrib/proguard/Proguard.scala +++ b/contrib/proguard/src/mill/contrib/proguard/Proguard.scala @@ -65,18 +65,8 @@ trait Proguard extends ScalaModule { * Keep in sync with [[javaHome]]. */ def java9RtJar: T[Seq[PathRef]] = Task { - if (mill.main.client.Util.isJava9OrAbove) { - val rt = T.dest / Export.rtJarName - if (!os.exists(rt)) { - T.log.outputStream.println( - s"Preparing Java runtime JAR; this may take a minute or two ..." - ) - Export.rtTo(rt.toIO, false) - } - Seq(PathRef(rt)) - } else { - Seq() - } + if (mill.main.client.Util.isJava9OrAbove) Seq(PathRef(T.home / Export.rtJarName)) + else Seq() } /** From 01ab379ec98b0aa70711ae5dae7a207f4d9760f3 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Thu, 10 Oct 2024 16:32:15 +0200 Subject: [PATCH 31/47] More cleanups for PromptLogger (#3712) * Move more pure logic into `PromptLoggerUtil` static methods * Make prompt line replacement logic more aggressive, such that even after dead lines finish transitioning they can be replaced, fixing an issue where the prompt height sometimes grew unnecessarily due to blank lines --- main/api/src/mill/api/Logger.scala | 2 +- main/eval/src/mill/eval/EvaluatorCore.scala | 2 +- main/util/src/mill/util/MultiLogger.scala | 6 +- main/util/src/mill/util/PrefixLogger.scala | 3 +- main/util/src/mill/util/PromptLogger.scala | 61 +++++-------------- .../util/src/mill/util/PromptLoggerUtil.scala | 35 +++++++++++ main/util/src/mill/util/ProxyLogger.scala | 3 +- .../src/mill/util/PromptLoggerTests.scala | 8 +-- 8 files changed, 63 insertions(+), 57 deletions(-) diff --git a/main/api/src/mill/api/Logger.scala b/main/api/src/mill/api/Logger.scala index 430b6352f99..b9a5dd64ddf 100644 --- a/main/api/src/mill/api/Logger.scala +++ b/main/api/src/mill/api/Logger.scala @@ -58,7 +58,7 @@ trait Logger extends AutoCloseable { ): Unit = ticker(s"${key.mkString("-")} $message") private[mill] def setPromptLine(): Unit = () - private[mill] def setPromptLeftHeader(s: String): Unit = () + private[mill] def setPromptHeaderPrefix(s: String): Unit = () private[mill] def clearPromptStatuses(): Unit = () private[mill] def removePromptLine(key: Seq[String]): Unit = () private[mill] def removePromptLine(): Unit = () diff --git a/main/eval/src/mill/eval/EvaluatorCore.scala b/main/eval/src/mill/eval/EvaluatorCore.scala index bfc25731686..932d73092bd 100644 --- a/main/eval/src/mill/eval/EvaluatorCore.scala +++ b/main/eval/src/mill/eval/EvaluatorCore.scala @@ -111,7 +111,7 @@ private[mill] trait EvaluatorCore extends GroupEvaluator { ) val verboseKeySuffix = s"/${terminals0.size}" - logger.setPromptLeftHeader(s"$countMsg$verboseKeySuffix") + logger.setPromptHeaderPrefix(s"$countMsg$verboseKeySuffix") if (failed.get()) None else { val upstreamResults = upstreamValues diff --git a/main/util/src/mill/util/MultiLogger.scala b/main/util/src/mill/util/MultiLogger.scala index 9dda1ea4e4b..357763657b3 100644 --- a/main/util/src/mill/util/MultiLogger.scala +++ b/main/util/src/mill/util/MultiLogger.scala @@ -81,9 +81,9 @@ class MultiLogger( logger1.removePromptLine() logger2.removePromptLine() } - private[mill] override def setPromptLeftHeader(s: String): Unit = { - logger1.setPromptLeftHeader(s) - logger2.setPromptLeftHeader(s) + private[mill] override def setPromptHeaderPrefix(s: String): Unit = { + logger1.setPromptHeaderPrefix(s) + logger2.setPromptHeaderPrefix(s) } private[mill] override def withPromptPaused[T](t: => T): T = { diff --git a/main/util/src/mill/util/PrefixLogger.scala b/main/util/src/mill/util/PrefixLogger.scala index 68cd02e7321..7ee393516a9 100644 --- a/main/util/src/mill/util/PrefixLogger.scala +++ b/main/util/src/mill/util/PrefixLogger.scala @@ -107,7 +107,8 @@ class PrefixLogger( private[mill] override def removePromptLine(callKey: Seq[String]): Unit = logger0.removePromptLine(callKey) private[mill] override def removePromptLine(): Unit = removePromptLine(logPrefixKey) - private[mill] override def setPromptLeftHeader(s: String): Unit = logger0.setPromptLeftHeader(s) + private[mill] override def setPromptHeaderPrefix(s: String): Unit = + logger0.setPromptHeaderPrefix(s) override def enableTicker = logger0.enableTicker private[mill] override def subLogger( diff --git a/main/util/src/mill/util/PromptLogger.scala b/main/util/src/mill/util/PromptLogger.scala index 8846355f433..1a08d6cefcc 100644 --- a/main/util/src/mill/util/PromptLogger.scala +++ b/main/util/src/mill/util/PromptLogger.scala @@ -88,16 +88,16 @@ private[mill] class PromptLogger( def error(s: String): Unit = synchronized { systemStreams.err.println(s) } - override def setPromptLeftHeader(s: String): Unit = - synchronized { promptLineState.updateGlobal(s) } + override def setPromptHeaderPrefix(s: String): Unit = + synchronized { promptLineState.setHeaderPrefix(s) } override def clearPromptStatuses(): Unit = synchronized { promptLineState.clearStatuses() } override def removePromptLine(key: Seq[String]): Unit = synchronized { - promptLineState.updateCurrent(key, None) + promptLineState.setCurrent(key, None) } def ticker(s: String): Unit = () override def setPromptDetail(key: Seq[String], s: String): Unit = synchronized { - promptLineState.updateDetail(key, s) + promptLineState.setDetail(key, s) } override def reportKey(key: Seq[String]): Unit = synchronized { @@ -116,7 +116,7 @@ private[mill] class PromptLogger( private val reportedIdentifiers = collection.mutable.Set.empty[Seq[String]] override def setPromptLine(key: Seq[String], verboseKeySuffix: String, message: String): Unit = synchronized { - promptLineState.updateCurrent(key, Some(s"[${key.mkString("-")}] $message")) + promptLineState.setCurrent(key, Some(s"[${key.mkString("-")}] $message")) seenIdentifiers(key) = (verboseKeySuffix, message) } @@ -273,19 +273,8 @@ private[mill] object PromptLogger { ) { private var lastRenderedPromptHash = 0 - private implicit def seqOrdering = new Ordering[Seq[String]] { - def compare(xs: Seq[String], ys: Seq[String]): Int = { - val iter = xs.iterator.zip(ys) - while (iter.nonEmpty) { - val (x, y) = iter.next() - if (x > y) return 1 - else if (y > x) return -1 - } - - return xs.lengthCompare(ys) - } - } - private val statuses = collection.mutable.SortedMap.empty[Seq[String], Status] + private val statuses = collection.mutable.SortedMap + .empty[Seq[String], Status](PromptLoggerUtil.seqStringOrdering) private var headerPrefix = "" // Pre-compute the prelude and current prompt as byte arrays so that @@ -308,6 +297,7 @@ private[mill] object PromptLogger { if (ending) statuses.clear() val (termWidth0, termHeight0) = consoleDims() + val interactive = consoleDims()._1.nonEmpty // don't show prompt for non-interactive terminal val currentPromptLines = renderPrompt( termWidth0.getOrElse(defaultTermWidth), @@ -317,38 +307,23 @@ private[mill] object PromptLogger { s"[$headerPrefix]", titleText, statuses.toSeq.map { case (k, v) => (k.mkString("-"), v) }, - interactive = consoleDims()._1.nonEmpty, + interactive = interactive, infoColor = infoColor, ending = ending ) - val currentPromptStr = - if (termWidth0.isEmpty) currentPromptLines.mkString("\n") + "\n" - else { - // For the ending prompt, leave the cursor at the bottom on a new line rather than - // scrolling back left/up. We do not want further output to overwrite the header as - // it will no longer re-render - val backUp = - if (ending) "\n" - else AnsiNav.left(9999) + AnsiNav.up(currentPromptLines.length - 1) - - AnsiNav.clearScreen(0) + - currentPromptLines.mkString("\n") + - backUp - } - - currentPromptBytes = currentPromptStr.getBytes + currentPromptBytes = renderPromptWrapped(currentPromptLines, interactive, ending).getBytes } def clearStatuses(): Unit = synchronized { statuses.clear() } - def updateGlobal(s: String): Unit = synchronized { headerPrefix = s } + def setHeaderPrefix(s: String): Unit = synchronized { headerPrefix = s } - def updateDetail(key: Seq[String], detail: String): Unit = synchronized { + def setDetail(key: Seq[String], detail: String): Unit = synchronized { statuses.updateWith(key)(_.map(se => se.copy(next = se.next.map(_.copy(detail = detail))))) } - def updateCurrent(key: Seq[String], sOpt: Option[String]): Unit = synchronized { + def setCurrent(key: Seq[String], sOpt: Option[String]): Unit = synchronized { val now = currentTimeMillis() def stillTransitioning(status: Status) = { @@ -357,7 +332,7 @@ private[mill] object PromptLogger { val sOptEntry = sOpt.map(StatusEntry(_, now, "")) statuses.updateWith(key) { case None => - statuses.find { case (k, v) => v.next.isEmpty && stillTransitioning(v) } match { + statuses.find { case (k, v) => v.next.isEmpty } match { case Some((reusableKey, reusableValue)) => statuses.remove(reusableKey) Some(reusableValue.copy(next = sOptEntry)) @@ -369,13 +344,7 @@ private[mill] object PromptLogger { // If still performing a transition, do not update the `prevTransitionTime` // since we do not want to delay the transition that is already in progress if (stillTransitioning(existing)) existing.copy(next = sOptEntry) - else { - existing.copy( - next = sOptEntry, - beginTransitionTime = now, - prev = existing.next - ) - } + else existing.copy(next = sOptEntry, beginTransitionTime = now, prev = existing.next) ) } } diff --git a/main/util/src/mill/util/PromptLoggerUtil.scala b/main/util/src/mill/util/PromptLoggerUtil.scala index fd261ed1b70..128e93e158b 100644 --- a/main/util/src/mill/util/PromptLoggerUtil.scala +++ b/main/util/src/mill/util/PromptLoggerUtil.scala @@ -149,6 +149,28 @@ private object PromptLoggerUtil { header :: body ::: footer } + // Wrap the prompt in the necessary clear-screens/newlines/move-cursors + // according to whether it is interactive or ending + def renderPromptWrapped( + currentPromptLines: Seq[String], + interactive: Boolean, + ending: Boolean + ): String = { + if (!interactive) currentPromptLines.mkString("\n") + "\n" + else { + // For the ending prompt, leave the cursor at the bottom on a new line rather than + // scrolling back left/up. We do not want further output to overwrite the header as + // it will no longer re-render + val backUp = + if (ending) "\n" + else AnsiNav.left(9999) + AnsiNav.up(currentPromptLines.length - 1) + + AnsiNav.clearScreen(0) + + currentPromptLines.mkString("\n") + + backUp + } + } + def renderHeader( headerPrefix0: String, titleText0: String, @@ -204,4 +226,17 @@ private object PromptLoggerUtil { } ??? } + + private[mill] val seqStringOrdering = new Ordering[Seq[String]] { + def compare(xs: Seq[String], ys: Seq[String]): Int = { + val iter = xs.iterator.zip(ys) + while (iter.nonEmpty) { + val (x, y) = iter.next() + if (x > y) return 1 + else if (y > x) return -1 + } + + return xs.lengthCompare(ys) + } + } } diff --git a/main/util/src/mill/util/ProxyLogger.scala b/main/util/src/mill/util/ProxyLogger.scala index 240fcabd685..bb1f1d1d27d 100644 --- a/main/util/src/mill/util/ProxyLogger.scala +++ b/main/util/src/mill/util/ProxyLogger.scala @@ -35,7 +35,8 @@ class ProxyLogger(logger: Logger) extends Logger { override def rawOutputStream: PrintStream = logger.rawOutputStream private[mill] override def removePromptLine(key: Seq[String]): Unit = logger.removePromptLine(key) private[mill] override def removePromptLine(): Unit = logger.removePromptLine() - private[mill] override def setPromptLeftHeader(s: String): Unit = logger.setPromptLeftHeader(s) + private[mill] override def setPromptHeaderPrefix(s: String): Unit = + logger.setPromptHeaderPrefix(s) private[mill] override def withPromptPaused[T](t: => T): T = logger.withPromptPaused(t) private[mill] override def withPromptUnpaused[T](t: => T): T = logger.withPromptUnpaused(t) diff --git a/main/util/test/src/mill/util/PromptLoggerTests.scala b/main/util/test/src/mill/util/PromptLoggerTests.scala index b9f95ef7b4d..044247339f6 100644 --- a/main/util/test/src/mill/util/PromptLoggerTests.scala +++ b/main/util/test/src/mill/util/PromptLoggerTests.scala @@ -59,7 +59,7 @@ object PromptLoggerTests extends TestSuite { val (baos, promptLogger, prefixLogger) = setup(() => now, os.temp()) - promptLogger.setPromptLeftHeader("123/456") + promptLogger.setPromptHeaderPrefix("123/456") promptLogger.setPromptLine(Seq("1"), "/456", "my-task") now += 10000 @@ -108,7 +108,7 @@ object PromptLoggerTests extends TestSuite { var now = 0L val (baos, promptLogger, prefixLogger) = setup(() => now, os.temp("80 40")) - promptLogger.setPromptLeftHeader("123/456") + promptLogger.setPromptHeaderPrefix("123/456") promptLogger.refreshPrompt() check(promptLogger, baos)( " [123/456] ========================== TITLE ==================================" @@ -278,7 +278,7 @@ object PromptLoggerTests extends TestSuite { @volatile var now = 0L val (baos, promptLogger, prefixLogger) = setup(() => now, os.temp("80 40")) - promptLogger.setPromptLeftHeader("123/456") + promptLogger.setPromptHeaderPrefix("123/456") promptLogger.refreshPrompt() check(promptLogger, baos)( " [123/456] ========================== TITLE ==================================" @@ -329,7 +329,7 @@ object PromptLoggerTests extends TestSuite { @volatile var now = 0L val (baos, promptLogger, prefixLogger) = setup(() => now, os.temp("80 40")) - promptLogger.setPromptLeftHeader("123/456") + promptLogger.setPromptHeaderPrefix("123/456") promptLogger.refreshPrompt() promptLogger.setPromptLine(Seq("1"), "/456", "my-task") From 80d1640658dae984888be2eac6b47d830bab93b2 Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Thu, 10 Oct 2024 16:44:11 +0200 Subject: [PATCH 32/47] Add JavaModule.artifactTypes (#3703) Allowing users to ask for more artifact types to be fetched and put in the classpath --- .../src/DocAnnotationsTests.scala | 2 +- main/util/src/mill/util/CoursierSupport.scala | 30 +++++++++++++++++-- .../src/mill/scalalib/CoursierModule.scala | 28 +++++++++++++++-- scalalib/src/mill/scalalib/JavaModule.scala | 13 ++++++-- scalalib/src/mill/scalalib/Lib.scala | 29 ++++++++++++++++-- .../src/mill/scalalib/ZincWorkerModule.scala | 4 +-- 6 files changed, 94 insertions(+), 12 deletions(-) diff --git a/integration/feature/docannotations/src/DocAnnotationsTests.scala b/integration/feature/docannotations/src/DocAnnotationsTests.scala index 77bdb767d1e..02d27311844 100644 --- a/integration/feature/docannotations/src/DocAnnotationsTests.scala +++ b/integration/feature/docannotations/src/DocAnnotationsTests.scala @@ -93,7 +93,7 @@ object DocAnnotationsTests extends UtestIntegrationTestSuite { assert( globMatches( - """core.ivyDepsTree(JavaModule.scala:884) + """core.ivyDepsTree(JavaModule.scala:893) | Command to print the transitive dependency tree to STDOUT. | | --inverse Invert the tree representation, so that the root is on the bottom val diff --git a/main/util/src/mill/util/CoursierSupport.scala b/main/util/src/mill/util/CoursierSupport.scala index 0ab43fc6b3d..ba09b7bfb0b 100644 --- a/main/util/src/mill/util/CoursierSupport.scala +++ b/main/util/src/mill/util/CoursierSupport.scala @@ -6,7 +6,7 @@ import coursier.error.ResolutionError.CantDownloadModule import coursier.params.ResolutionParams import coursier.parse.RepositoryParser import coursier.util.Task -import coursier.{Artifacts, Classifier, Dependency, Repository, Resolution, Resolve} +import coursier.{Artifacts, Classifier, Dependency, Repository, Resolution, Resolve, Type} import mill.api.Loose.Agg import mill.api.{Ctx, PathRef, Result} @@ -44,7 +44,8 @@ trait CoursierSupport { customizer: Option[Resolution => Resolution] = None, ctx: Option[mill.api.Ctx.Log] = None, coursierCacheCustomizer: Option[FileCache[Task] => FileCache[Task]] = None, - resolveFilter: os.Path => Boolean = _ => true + resolveFilter: os.Path => Boolean = _ => true, + artifactTypes: Option[Set[Type]] = None ): Result[Agg[PathRef]] = { def isLocalTestDep(dep: Dependency): Option[Seq[PathRef]] = { val org = dep.module.organization.value @@ -86,6 +87,7 @@ trait CoursierSupport { if (sources) Set(Classifier("sources")) else Set.empty ) + .withArtifactTypesOpt(artifactTypes) .eitherResult() artifactsResultOrError match { @@ -112,6 +114,30 @@ trait CoursierSupport { } } + @deprecated("Use the override accepting artifactTypes", "Mill after 0.12.0-RC3") + def resolveDependencies( + repositories: Seq[Repository], + deps: IterableOnce[Dependency], + force: IterableOnce[Dependency], + sources: Boolean, + mapDependencies: Option[Dependency => Dependency], + customizer: Option[Resolution => Resolution], + ctx: Option[mill.api.Ctx.Log], + coursierCacheCustomizer: Option[FileCache[Task] => FileCache[Task]], + resolveFilter: os.Path => Boolean + ): Result[Agg[PathRef]] = + resolveDependencies( + repositories, + deps, + force, + sources, + mapDependencies, + customizer, + ctx, + coursierCacheCustomizer, + resolveFilter + ) + @deprecated( "Prefer resolveDependenciesMetadataSafe instead, which returns a Result instead of throwing exceptions", "0.12.0" diff --git a/scalalib/src/mill/scalalib/CoursierModule.scala b/scalalib/src/mill/scalalib/CoursierModule.scala index 9ecdfcbfeeb..4594916e4a0 100644 --- a/scalalib/src/mill/scalalib/CoursierModule.scala +++ b/scalalib/src/mill/scalalib/CoursierModule.scala @@ -1,7 +1,7 @@ package mill.scalalib import coursier.cache.FileCache -import coursier.{Dependency, Repository, Resolve} +import coursier.{Dependency, Repository, Resolve, Type} import coursier.core.Resolution import mill.define.Task import mill.api.PathRef @@ -48,14 +48,20 @@ trait CoursierModule extends mill.Module { * * @param deps The dependencies to resolve. * @param sources If `true`, resolve source dependencies instead of binary dependencies (JARs). + * @param artifactTypes If non-empty, pull the passed artifact types rather than the default ones from coursier * @return The [[PathRef]]s to the resolved files. */ - def resolveDeps(deps: Task[Agg[BoundDep]], sources: Boolean = false): Task[Agg[PathRef]] = + def resolveDeps( + deps: Task[Agg[BoundDep]], + sources: Boolean = false, + artifactTypes: Option[Set[Type]] = None + ): Task[Agg[PathRef]] = Task.Anon { Lib.resolveDependencies( repositories = repositoriesTask(), deps = deps(), sources = sources, + artifactTypes = artifactTypes, mapDependencies = Some(mapDependencies()), customizer = resolutionCustomizer(), coursierCacheCustomizer = coursierCacheCustomizer(), @@ -63,6 +69,13 @@ trait CoursierModule extends mill.Module { ) } + @deprecated("Use the override accepting artifactTypes", "Mill after 0.12.0-RC3") + def resolveDeps( + deps: Task[Agg[BoundDep]], + sources: Boolean + ): Task[Agg[PathRef]] = + resolveDeps(deps, sources, None) + /** * Map dependencies before resolving them. * Override this to customize the set of dependencies. @@ -134,18 +147,27 @@ object CoursierModule { def resolveDeps[T: CoursierModule.Resolvable]( deps: IterableOnce[T], - sources: Boolean = false + sources: Boolean = false, + artifactTypes: Option[Set[coursier.Type]] = None ): Agg[PathRef] = { Lib.resolveDependencies( repositories = repositories, deps = deps.map(implicitly[CoursierModule.Resolvable[T]].bind(_, bind)), sources = sources, + artifactTypes = artifactTypes, mapDependencies = mapDependencies, customizer = customizer, coursierCacheCustomizer = coursierCacheCustomizer, ctx = ctx ).getOrThrow } + + @deprecated("Use the override accepting artifactTypes", "Mill after 0.12.0-RC3") + def resolveDeps[T: CoursierModule.Resolvable]( + deps: IterableOnce[T], + sources: Boolean + ): Agg[PathRef] = + resolveDeps(deps, sources, None) } sealed trait Resolvable[T] { diff --git a/scalalib/src/mill/scalalib/JavaModule.scala b/scalalib/src/mill/scalalib/JavaModule.scala index 90d72e7b62b..f72bc67c743 100644 --- a/scalalib/src/mill/scalalib/JavaModule.scala +++ b/scalalib/src/mill/scalalib/JavaModule.scala @@ -1,11 +1,11 @@ package mill package scalalib -import coursier.Repository import coursier.core.Resolution import coursier.parse.JavaOrScalaModule import coursier.parse.ModuleParser import coursier.util.ModuleMatcher +import coursier.{Repository, Type} import mainargs.{Flag, arg} import mill.Agg import mill.api.{Ctx, JarManifest, MillException, PathRef, Result, internal} @@ -155,6 +155,12 @@ trait JavaModule */ def runIvyDeps: T[Agg[Dep]] = Task { Agg.empty[Dep] } + /** + * Default artifact types to fetch and put in the classpath. Add extra types + * here if you'd like fancy artifact extensions to be fetched. + */ + def artifactTypes: T[Set[Type]] = Task { coursier.core.Resolution.defaultTypes } + /** * Options to pass to the java compiler */ @@ -551,7 +557,10 @@ trait JavaModule } def resolvedRunIvyDeps: T[Agg[PathRef]] = Task { - defaultResolver().resolveDeps(runIvyDeps().map(bindDependency()) ++ transitiveIvyDeps()) + defaultResolver().resolveDeps( + runIvyDeps().map(bindDependency()) ++ transitiveIvyDeps(), + artifactTypes = Some(artifactTypes()) + ) } /** diff --git a/scalalib/src/mill/scalalib/Lib.scala b/scalalib/src/mill/scalalib/Lib.scala index c606a519c4a..07a10461524 100644 --- a/scalalib/src/mill/scalalib/Lib.scala +++ b/scalalib/src/mill/scalalib/Lib.scala @@ -2,7 +2,7 @@ package mill package scalalib import coursier.util.Task -import coursier.{Dependency, Repository, Resolution} +import coursier.{Dependency, Repository, Resolution, Type} import mill.api.{Ctx, Loose, PathRef, Result} import mill.main.BuildInfo import mill.main.client.EnvVars @@ -89,7 +89,8 @@ object Lib { ctx: Option[Ctx.Log] = None, coursierCacheCustomizer: Option[ coursier.cache.FileCache[Task] => coursier.cache.FileCache[Task] - ] = None + ] = None, + artifactTypes: Option[Set[Type]] = None ): Result[Agg[PathRef]] = { val depSeq = deps.iterator.toSeq mill.util.Jvm.resolveDependencies( @@ -97,6 +98,7 @@ object Lib { deps = depSeq.map(_.dep), force = depSeq.filter(_.force).map(_.dep), sources = sources, + artifactTypes = artifactTypes, mapDependencies = mapDependencies, customizer = customizer, ctx = ctx, @@ -104,6 +106,29 @@ object Lib { ).map(_.map(_.withRevalidateOnce)) } + @deprecated("Use the override accepting artifactTypes", "Mill after 0.12.0-RC3") + def resolveDependencies( + repositories: Seq[Repository], + deps: IterableOnce[BoundDep], + sources: Boolean, + mapDependencies: Option[Dependency => Dependency], + customizer: Option[coursier.core.Resolution => coursier.core.Resolution], + ctx: Option[Ctx.Log], + coursierCacheCustomizer: Option[ + coursier.cache.FileCache[Task] => coursier.cache.FileCache[Task] + ] + ): Result[Agg[PathRef]] = + resolveDependencies( + repositories, + deps, + sources, + mapDependencies, + customizer, + ctx, + coursierCacheCustomizer, + None + ) + def scalaCompilerIvyDeps(scalaOrganization: String, scalaVersion: String): Loose.Agg[Dep] = if (ZincWorkerUtil.isDotty(scalaVersion)) Agg( diff --git a/scalalib/src/mill/scalalib/ZincWorkerModule.scala b/scalalib/src/mill/scalalib/ZincWorkerModule.scala index 6861fdcacf0..611e2482ef3 100644 --- a/scalalib/src/mill/scalalib/ZincWorkerModule.scala +++ b/scalalib/src/mill/scalalib/ZincWorkerModule.scala @@ -127,8 +127,8 @@ trait ZincWorkerModule extends mill.Module with OfflineSupportModule { self: Cou val bridgeJar = resolveDependencies( repositories, Seq(bridgeDep.bindDep("", "", "")), - useSources, - Some(overrideScalaLibrary(scalaVersion, scalaOrganization)) + sources = useSources, + mapDependencies = Some(overrideScalaLibrary(scalaVersion, scalaOrganization)) ).map(deps => ZincWorkerUtil.grepJar(deps, bridgeName, bridgeVersion, useSources) ) From fba6c262a97ade99fcf9ae64b1da14ffad3b8a6f Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Fri, 11 Oct 2024 11:51:32 +0200 Subject: [PATCH 33/47] Update mill-version (#3718) --- .config/mill-version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.config/mill-version b/.config/mill-version index b30ea878bb5..825bbb0a69e 100644 --- a/.config/mill-version +++ b/.config/mill-version @@ -1 +1 @@ -0.12.0-RC3-32-b4a0bf \ No newline at end of file +0.12.0-RC3-46-80d164 From cc1aa6f1f3dc85da1e3c9ae2ffeb869804664c1c Mon Sep 17 00:00:00 2001 From: Jamie Thompson Date: Fri, 11 Oct 2024 13:04:12 +0200 Subject: [PATCH 34/47] upgrade moduledefs and mainargs (#3720) pre-prep for scala 3 port --- build.mill | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.mill b/build.mill index f4a9b0f2e24..60b3afa4be7 100644 --- a/build.mill +++ b/build.mill @@ -151,8 +151,8 @@ object Deps { val log4j2Core = ivy"org.apache.logging.log4j:log4j-core:2.23.1" val osLib = ivy"com.lihaoyi::os-lib:0.11.1" val pprint = ivy"com.lihaoyi::pprint:0.9.0" - val mainargs = ivy"com.lihaoyi::mainargs:0.7.4" - val millModuledefsVersion = "0.11.0" + val mainargs = ivy"com.lihaoyi::mainargs:0.7.6" + val millModuledefsVersion = "0.11.1" val millModuledefsString = s"com.lihaoyi::mill-moduledefs:${millModuledefsVersion}" val millModuledefs = ivy"${millModuledefsString}" val millModuledefsPlugin = From 7221788fc4f9bb2cfabbbb9e3a3782751d2751ff Mon Sep 17 00:00:00 2001 From: Alexandre Archambault Date: Fri, 11 Oct 2024 14:02:51 +0200 Subject: [PATCH 35/47] Fix JavaModule.artifactTypes (#3719) Follow-up of https://github.com/com-lihaoyi/mill/pull/3703 that added `JavaModule.artifactTypes`. `JavaModule` performs two dependency resolutions: one for `compileClasspath`, the other for `runClasspath`. https://github.com/com-lihaoyi/mill/pull/3703 used `artifactTypes` in the `runClasspath` one, but not in the `compileClasspath` one. The PR here fixes that, and adds unit tests for both paths. This addresses the issues found around https://github.com/com-lihaoyi/mill/pull/3696#issuecomment-2405891608. --- .../src/DocAnnotationsTests.scala | 2 +- scalalib/src/mill/scalalib/JavaModule.scala | 5 +++- .../resources/pomArtifactType/.placeholder | 0 .../src/mill/scalalib/ResolveDepsTests.scala | 29 +++++++++++++++++++ 4 files changed, 34 insertions(+), 2 deletions(-) create mode 100644 scalalib/test/resources/pomArtifactType/.placeholder diff --git a/integration/feature/docannotations/src/DocAnnotationsTests.scala b/integration/feature/docannotations/src/DocAnnotationsTests.scala index 02d27311844..4ef2900e86c 100644 --- a/integration/feature/docannotations/src/DocAnnotationsTests.scala +++ b/integration/feature/docannotations/src/DocAnnotationsTests.scala @@ -93,7 +93,7 @@ object DocAnnotationsTests extends UtestIntegrationTestSuite { assert( globMatches( - """core.ivyDepsTree(JavaModule.scala:893) + """core.ivyDepsTree(JavaModule.scala:896) | Command to print the transitive dependency tree to STDOUT. | | --inverse Invert the tree representation, so that the root is on the bottom val diff --git a/scalalib/src/mill/scalalib/JavaModule.scala b/scalalib/src/mill/scalalib/JavaModule.scala index f72bc67c743..4d92101bcb4 100644 --- a/scalalib/src/mill/scalalib/JavaModule.scala +++ b/scalalib/src/mill/scalalib/JavaModule.scala @@ -545,7 +545,10 @@ trait JavaModule * Resolved dependencies based on [[transitiveIvyDeps]] and [[transitiveCompileIvyDeps]]. */ def resolvedIvyDeps: T[Agg[PathRef]] = Task { - defaultResolver().resolveDeps(transitiveCompileIvyDeps() ++ transitiveIvyDeps()) + defaultResolver().resolveDeps( + transitiveCompileIvyDeps() ++ transitiveIvyDeps(), + artifactTypes = Some(artifactTypes()) + ) } /** diff --git a/scalalib/test/resources/pomArtifactType/.placeholder b/scalalib/test/resources/pomArtifactType/.placeholder new file mode 100644 index 00000000000..e69de29bb2d diff --git a/scalalib/test/src/mill/scalalib/ResolveDepsTests.scala b/scalalib/test/src/mill/scalalib/ResolveDepsTests.scala index cd55d1b14b5..a54a2d023f9 100644 --- a/scalalib/test/src/mill/scalalib/ResolveDepsTests.scala +++ b/scalalib/test/src/mill/scalalib/ResolveDepsTests.scala @@ -4,6 +4,7 @@ import coursier.maven.MavenRepository import mill.api.Result.{Failure, Success} import mill.api.{PathRef, Result} import mill.api.Loose.Agg +import mill.testkit.{UnitTester, TestBaseModule} import utest._ object ResolveDepsTests extends TestSuite { @@ -28,6 +29,21 @@ object ResolveDepsTests extends TestSuite { assert(upickle.default.read[Dep](upickle.default.write(dep)) == dep) } } + + object TestCase extends TestBaseModule { + object pomStuff extends JavaModule { + def ivyDeps = Agg( + // Dependency whose packaging is "pom", as it's meant to be used + // as a "parent dependency" by other dependencies, rather than be pulled + // as we do here. We do it anyway, to check that pulling the "pom" artifact + // type brings that dependency POM file in the class path. We need a dependency + // that has a "pom" packaging for that. + ivy"org.apache.hadoop:hadoop-yarn-server:3.4.0" + ) + def artifactTypes = super.artifactTypes() + coursier.Type("pom") + } + } + val tests = Tests { test("resolveValidDeps") { val deps = Agg(ivy"com.lihaoyi::pprint:0.5.3") @@ -95,5 +111,18 @@ object ResolveDepsTests extends TestSuite { val Failure(errMsg, _) = evalDeps(deps) assert(errMsg.contains("fake")) } + + test("pomArtifactType") { + val sources = os.Path(sys.env("MILL_TEST_RESOURCE_DIR")) / "pomArtifactType" + UnitTester(TestCase, sourceRoot = sources).scoped { eval => + val Right(compileResult) = eval(TestCase.pomStuff.compileClasspath) + val compileCp = compileResult.value.toSeq.map(_.path) + assert(compileCp.exists(_.lastOpt.contains("hadoop-yarn-server-3.4.0.pom"))) + + val Right(runResult) = eval(TestCase.pomStuff.runClasspath) + val runCp = runResult.value.toSeq.map(_.path) + assert(runCp.exists(_.lastOpt.contains("hadoop-yarn-server-3.4.0.pom"))) + } + } } } From 7da02654a3f6631dfb0847f81fac42c80b6c2723 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sat, 12 Oct 2024 08:59:26 +0200 Subject: [PATCH 36/47] Cherry-pick source compatible Scala 3 changes (#3721) Pulls in some changes from https://github.com/com-lihaoyi/mill/pull/3369 to try and reduce the size of that diff and simplify future backports --- .../src/mill/bsp/worker/MillBuildServer.scala | 2 +- bsp/worker/src/mill/bsp/worker/Utils.scala | 2 +- build.mill | 1 + .../mill/contrib/buildinfo/BuildInfo.scala | 2 +- .../src/mill/playlib/RouterModule.scala | 1 + .../playlib/src/mill/playlib/Version.scala | 4 +- .../src/mill/playlib/PlayModuleTests.scala | 3 +- .../api/ScoverageReportWorkerApi.scala | 46 --------- .../api/ScoverageReportWorkerApi2.java | 95 +++++++++++++++++++ .../contrib/scoverage/ScoverageModule.scala | 2 +- .../contrib/scoverage/ScoverageReport.scala | 2 +- .../scoverage/ScoverageReportWorker.scala | 65 +++++++++++-- .../worker/ScoverageReportWorkerImpl.scala | 32 ++++--- .../src/mill/twirllib/TwirlModule.scala | 4 +- .../src/mill/twirllib/HelloWorldTests.scala | 13 ++- example/thirdparty/mockito/build.mill | 14 +++ example/thirdparty/netty/build.mill | 4 +- .../src/DocAnnotationsTests.scala | 2 +- .../mill-build/mill-build/build.mill | 2 +- .../src/MultiLevelBuildTests.scala | 34 +++---- main/api/src/mill/api/AggWrapper.scala | 13 +-- main/api/src/mill/api/ClassLoader.scala | 2 +- main/api/src/mill/api/Ctx.scala | 8 +- main/api/src/mill/api/FixSizedCache.scala | 2 +- main/api/src/mill/api/JarOps.scala | 6 +- main/api/src/mill/api/JsonFormatters.scala | 2 +- main/api/src/mill/api/KeyedLockedCache.scala | 2 +- main/api/src/mill/api/PathRef.scala | 6 +- main/api/src/mill/api/Result.scala | 18 ++-- main/api/src/mill/api/Retry.scala | 4 +- main/api/src/mill/api/SystemStreams.scala | 6 +- main/codesig/package.mill | 3 +- main/codesig/src/JvmModel.scala | 2 +- main/codesig/src/ResolvedCalls.scala | 4 +- main/codesig/src/SpanningForest.scala | 6 +- .../basic/17-scala-lambda/src/Hello.scala | 7 +- .../src/Hello.scala | 5 +- .../6-classes-misc-scala/src/Hello.scala | 2 +- .../8-linked-list-scala/src/Hello.scala | 4 +- .../realistic/5-parser/src/Hello.scala | 10 +- .../test/src/mill/define/BasePathTests.scala | 20 ++-- main/eval/src/mill/eval/EvaluatorCore.scala | 37 ++++---- main/eval/src/mill/eval/GroupEvaluator.scala | 4 +- .../test/src/mill/eval/EvaluationTests.scala | 47 ++++----- main/init/src/mill/init/InitModule.scala | 5 +- main/package.mill | 18 +++- .../src/mill/resolve/ExpandBraces.scala | 2 +- main/resolve/src/mill/resolve/ParseArgs.scala | 2 +- .../mill/resolve/ResolveNotFoundHandler.scala | 6 +- main/server/src/mill/main/server/Server.scala | 3 +- main/src/mill/main/MainModule.scala | 4 +- main/src/mill/main/TokenReaders.scala | 1 + main/util/src/mill/util/PrefixLogger.scala | 6 +- .../src/mill/scalajslib/ScalaJSModule.scala | 4 +- .../mill/scalajslib/CompileLinkTests.scala | 9 +- .../mill/scalajslib/EsModuleRemapTests.scala | 10 +- .../scalajslib/FullOptESModuleTests.scala | 5 +- .../mill/scalajslib/MultiModuleTests.scala | 5 +- .../mill/scalajslib/NodeJSConfigTests.scala | 7 +- .../mill/scalajslib/OutputPatternsTests.scala | 5 +- .../scalajslib/ScalaTestsErrorTests.scala | 5 +- .../scalajslib/SmallModulesForTests.scala | 5 +- .../src/mill/scalajslib/SourceMapTests.scala | 5 +- .../scalajslib/TopLevelExportsTests.scala | 5 +- .../src/mill/scalalib/CrossSbtModule.scala | 1 - scalalib/src/mill/scalalib/Dep.scala | 8 +- scalalib/src/mill/scalalib/JavaModule.scala | 6 +- .../src/mill/scalalib/PublishModule.scala | 6 +- scalalib/src/mill/scalalib/ScalaModule.scala | 5 +- .../src/mill/scalalib/ZincWorkerModule.scala | 1 - .../dependency/updates/UpdatesFinder.scala | 2 +- .../dependency/versions/VersionParser.scala | 2 +- .../mill/scalalib/giter8/Giter8Module.scala | 21 +++- .../mill/scalalib/publish/VersionScheme.scala | 3 +- .../mill/scalanativelib/CompileRunTests.scala | 2 +- .../mill/scalanativelib/ExclusionsTests.scala | 1 + .../mill/scalanativelib/FeaturesTests.scala | 1 + .../scalanativelib/ScalaTestsErrorTests.scala | 5 +- .../src/mill/testkit/IntegrationTester.scala | 2 +- testkit/src/mill/testkit/UnitTester.scala | 2 +- 80 files changed, 477 insertions(+), 253 deletions(-) delete mode 100644 contrib/scoverage/api/src/mill/contrib/scoverage/api/ScoverageReportWorkerApi.scala create mode 100644 contrib/scoverage/api/src/mill/contrib/scoverage/api/ScoverageReportWorkerApi2.java diff --git a/bsp/worker/src/mill/bsp/worker/MillBuildServer.scala b/bsp/worker/src/mill/bsp/worker/MillBuildServer.scala index 727be75c932..23ec8ca7d39 100644 --- a/bsp/worker/src/mill/bsp/worker/MillBuildServer.scala +++ b/bsp/worker/src/mill/bsp/worker/MillBuildServer.scala @@ -632,7 +632,7 @@ private class MillBuildServer( State ) => V): CompletableFuture[V] = { val prefix = hint.split(" ").head - completable(hint) { state: State => + completable(hint) { (state: State) => val ids = state.filterNonSynthetic(targetIds(state).asJava).asScala val tasksSeq = ids.flatMap { id => val (m, ev) = state.bspModulesById(id) diff --git a/bsp/worker/src/mill/bsp/worker/Utils.scala b/bsp/worker/src/mill/bsp/worker/Utils.scala index 5fe3f57b190..7b96c76a88e 100644 --- a/bsp/worker/src/mill/bsp/worker/Utils.scala +++ b/bsp/worker/src/mill/bsp/worker/Utils.scala @@ -34,7 +34,7 @@ private object Utils { originId: String, bspIdsByModule: Map[BspModule, BuildTargetIdentifier], client: BuildClient - ): Int => Option[CompileProblemReporter] = { moduleHashCode: Int => + ): Int => Option[CompileProblemReporter] = { (moduleHashCode: Int) => bspIdsByModule.find(_._1.hashCode == moduleHashCode).map { case (module: JavaModule, targetId) => val buildTarget = module.bspBuildTarget diff --git a/build.mill b/build.mill index 60b3afa4be7..365a2225c13 100644 --- a/build.mill +++ b/build.mill @@ -56,6 +56,7 @@ object Deps { // When updating, run "Publish Bridges" Github Actions for the new version // and then add to it `bridgeScalaVersions` val scalaVersion = "2.13.14" + val scala2Version = "2.13.14" // The Scala 2.12.x version to use for some workers val workerScalaVersion212 = "2.12.19" diff --git a/contrib/buildinfo/src/mill/contrib/buildinfo/BuildInfo.scala b/contrib/buildinfo/src/mill/contrib/buildinfo/BuildInfo.scala index 2f83bc60050..dcd6f0d8bce 100644 --- a/contrib/buildinfo/src/mill/contrib/buildinfo/BuildInfo.scala +++ b/contrib/buildinfo/src/mill/contrib/buildinfo/BuildInfo.scala @@ -173,7 +173,7 @@ object BuildInfo { |package ${buildInfoPackageName} | |object $buildInfoObjectName { - | private[this] val buildInfoProperties: java.util.Properties = new java.util.Properties() + | private val buildInfoProperties: java.util.Properties = new java.util.Properties() | | { | val buildInfoInputStream = getClass diff --git a/contrib/playlib/src/mill/playlib/RouterModule.scala b/contrib/playlib/src/mill/playlib/RouterModule.scala index fdfeaa2605f..11b921257b5 100644 --- a/contrib/playlib/src/mill/playlib/RouterModule.scala +++ b/contrib/playlib/src/mill/playlib/RouterModule.scala @@ -80,6 +80,7 @@ trait RouterModule extends ScalaModule with Version { repositoriesTask(), artifactSuffix = playMinorVersion() match { case "2.6" => "_2.12" + case "2.7" | "2.8" => "_2.13" case _ => "_2.13" } ) diff --git a/contrib/playlib/src/mill/playlib/Version.scala b/contrib/playlib/src/mill/playlib/Version.scala index c129d3820cf..a9640fac8ee 100644 --- a/contrib/playlib/src/mill/playlib/Version.scala +++ b/contrib/playlib/src/mill/playlib/Version.scala @@ -12,11 +12,11 @@ private[playlib] trait Version extends Module { playVersion().split('.').take(2).mkString(".") } - private[playlib] def playOrganization: T[String] = Task.Anon { + private[playlib] def playOrganization: Task[String] = Task.Anon { if (playVersion().startsWith("2.")) "com.typesafe.play" else "org.playframework" } - private[playlib] def component(id: String) = Task.Anon { + private[playlib] def component(id: String): Task[Dep] = Task.Anon { ivy"${playOrganization()}::$id::${playVersion()}" } } diff --git a/contrib/playlib/test/src/mill/playlib/PlayModuleTests.scala b/contrib/playlib/test/src/mill/playlib/PlayModuleTests.scala index afd219fcd7c..a6a97db4d6f 100644 --- a/contrib/playlib/test/src/mill/playlib/PlayModuleTests.scala +++ b/contrib/playlib/test/src/mill/playlib/PlayModuleTests.scala @@ -1,6 +1,7 @@ package mill package playlib +import mill.scalalib.api.ZincWorkerUtil import mill.testkit.{TestBaseModule, UnitTester} import utest.{TestSuite, Tests, assert, _} @@ -102,7 +103,7 @@ object PlayModuleTests extends TestSuite with PlayTestSuite { os.RelPath("controllers/routes$javascript.class"), os.RelPath("controllers/javascript/ReverseHomeController.class"), os.RelPath("controllers/javascript/ReverseAssets.class"), - if (scalaVersion.startsWith("3.")) os.RelPath("router/Routes$$anon$1.class") + if (ZincWorkerUtil.isScala3(scalaVersion)) os.RelPath("router/Routes$$anon$1.class") else os.RelPath("router/Routes$$anonfun$routes$1.class"), os.RelPath("router/Routes.class"), os.RelPath("router/RoutesPrefix$.class"), diff --git a/contrib/scoverage/api/src/mill/contrib/scoverage/api/ScoverageReportWorkerApi.scala b/contrib/scoverage/api/src/mill/contrib/scoverage/api/ScoverageReportWorkerApi.scala deleted file mode 100644 index 55fc01d3dc7..00000000000 --- a/contrib/scoverage/api/src/mill/contrib/scoverage/api/ScoverageReportWorkerApi.scala +++ /dev/null @@ -1,46 +0,0 @@ -package mill.contrib.scoverage.api - -import mill.api.Ctx - -trait ScoverageReportWorkerApi { - import ScoverageReportWorkerApi._ - - @deprecated("Use other overload instead.", "Mill after 0.10.7") - def report( - reportType: ReportType, - sources: Seq[os.Path], - dataDirs: Seq[os.Path] - )(implicit - ctx: Ctx - ): Unit = { - report(reportType, sources, dataDirs, ctx.workspace) - } - - def report( - reportType: ReportType, - sources: Seq[os.Path], - dataDirs: Seq[os.Path], - sourceRoot: os.Path - )(implicit - ctx: Ctx - ): Unit = { - // FIXME: We only call the deprecated version here, to preserve binary compatibility. Remove when appropriate. - ctx.log.error( - "Binary compatibility stub may cause infinite loops with StackOverflowError. You need to implement: def report(ReportType, Seq[Path], Seq[Path], os.Path): Unit" - ) - report(reportType, sources, dataDirs) - } -} - -object ScoverageReportWorkerApi { - sealed trait ReportType - sealed trait FileReportType extends ReportType { def folderName: String } - object ReportType { - final case object Html extends FileReportType { val folderName: String = "htmlReport" } - final case object Xml extends FileReportType { val folderName: String = "xmlReport" } - final case object XmlCobertura extends FileReportType { - val folderName: String = "xmlCoberturaReport" - } - final case object Console extends ReportType - } -} diff --git a/contrib/scoverage/api/src/mill/contrib/scoverage/api/ScoverageReportWorkerApi2.java b/contrib/scoverage/api/src/mill/contrib/scoverage/api/ScoverageReportWorkerApi2.java new file mode 100644 index 00000000000..09868af8b3e --- /dev/null +++ b/contrib/scoverage/api/src/mill/contrib/scoverage/api/ScoverageReportWorkerApi2.java @@ -0,0 +1,95 @@ +package mill.contrib.scoverage.api; + +import java.nio.file.Path; +import java.nio.file.Files; +import java.io.IOException; +import java.io.Serializable; + +public interface ScoverageReportWorkerApi2 { + + interface Logger { + void info(String msg); + void error(String msg); + void debug(String msg); + } + + interface Ctx { + Logger log(); + Path dest(); + } + + public static abstract class ReportType implements Serializable { + private String name; + + /*private[api]*/ + ReportType(String name) {} + + public static final ReportType Console = new ConsoleModule(); + public static final FileReportType Html = new HtmlModule(); + public static final FileReportType Xml = new XmlModule(); + public static final FileReportType XmlCobertura = new XmlCoberturaModule(); + + /* private[api]*/ + static final class ConsoleModule extends ReportType implements Serializable { + /* private[api]*/ + ConsoleModule() { + super("Console"); + } + }; + + /* private[api]*/ + static final class HtmlModule extends FileReportType implements Serializable { + /* private[api]*/ + HtmlModule() { + super("Html", "htmlReport"); + } + }; + + /* private[api]*/ + static final class XmlModule extends FileReportType implements Serializable { + /* private[api]*/ + XmlModule() { + super("Xml", "xmlReport"); + } + } + + /* private[api]*/ + static final class XmlCoberturaModule extends FileReportType implements Serializable { + /* private[api]*/ + XmlCoberturaModule() { + super("XmlCobertura", "xmlCoberturaReport"); + } + } + + @Override + public String toString() { + return name; + } + } + + public static abstract class FileReportType extends ReportType implements Serializable { + private final String folderName; + + /*private[api]*/ + FileReportType(String name, String folderName) { + super(name); + this.folderName = folderName; + } + + public String folderName() { + return folderName; + } + } + + void report(ReportType reportType, Path[] sources, Path[] dataDirs, Path sourceRoot, Ctx ctx); + + static void makeAllDirs(Path path) throws IOException { + // Replicate behavior of `os.makeDir.all(path)` + if (Files.isDirectory(path) && Files.isSymbolicLink(path)) { + // do nothing + } else { + Files.createDirectories(path); + } + } + +} diff --git a/contrib/scoverage/src/mill/contrib/scoverage/ScoverageModule.scala b/contrib/scoverage/src/mill/contrib/scoverage/ScoverageModule.scala index fba21cb2ccd..e7824a266b0 100644 --- a/contrib/scoverage/src/mill/contrib/scoverage/ScoverageModule.scala +++ b/contrib/scoverage/src/mill/contrib/scoverage/ScoverageModule.scala @@ -3,7 +3,7 @@ package mill.contrib.scoverage import coursier.Repository import mill._ import mill.api.{Loose, PathRef, Result} -import mill.contrib.scoverage.api.ScoverageReportWorkerApi.ReportType +import mill.contrib.scoverage.api.ScoverageReportWorkerApi2.ReportType import mill.main.BuildInfo import mill.scalalib.api.ZincWorkerUtil import mill.scalalib.{Dep, DepSyntax, JavaModule, ScalaModule} diff --git a/contrib/scoverage/src/mill/contrib/scoverage/ScoverageReport.scala b/contrib/scoverage/src/mill/contrib/scoverage/ScoverageReport.scala index fd25e64e3da..e7c6ed5670f 100644 --- a/contrib/scoverage/src/mill/contrib/scoverage/ScoverageReport.scala +++ b/contrib/scoverage/src/mill/contrib/scoverage/ScoverageReport.scala @@ -1,6 +1,6 @@ package mill.contrib.scoverage -import mill.contrib.scoverage.api.ScoverageReportWorkerApi.ReportType +import mill.contrib.scoverage.api.ScoverageReportWorkerApi2.ReportType import mill.define.{Command, Module, Task} import mill.eval.Evaluator import mill.resolve.{Resolve, SelectMode} diff --git a/contrib/scoverage/src/mill/contrib/scoverage/ScoverageReportWorker.scala b/contrib/scoverage/src/mill/contrib/scoverage/ScoverageReportWorker.scala index 799ed317178..600c90d3bb9 100644 --- a/contrib/scoverage/src/mill/contrib/scoverage/ScoverageReportWorker.scala +++ b/contrib/scoverage/src/mill/contrib/scoverage/ScoverageReportWorker.scala @@ -2,13 +2,18 @@ package mill.contrib.scoverage import mill.{Agg, Task} import mill.api.{ClassLoader, Ctx, PathRef} -import mill.contrib.scoverage.api.ScoverageReportWorkerApi +import mill.contrib.scoverage.api.ScoverageReportWorkerApi2 import mill.define.{Discover, ExternalModule, Worker} +import ScoverageReportWorker.ScoverageReportWorkerApiBridge +import ScoverageReportWorkerApi2.ReportType +import ScoverageReportWorkerApi2.{Logger => ApiLogger} +import ScoverageReportWorkerApi2.{Ctx => ApiCtx} + class ScoverageReportWorker extends AutoCloseable { - private[this] var scoverageClCache = Option.empty[(Long, ClassLoader)] + private var scoverageClCache = Option.empty[(Long, ClassLoader)] - def bridge(classpath: Agg[PathRef])(implicit ctx: Ctx): ScoverageReportWorkerApi = { + def bridge(classpath: Agg[PathRef])(implicit ctx: Ctx): ScoverageReportWorkerApiBridge = { val classloaderSig = classpath.hashCode val cl = scoverageClCache match { @@ -24,11 +29,43 @@ class ScoverageReportWorker extends AutoCloseable { cl } - cl - .loadClass("mill.contrib.scoverage.worker.ScoverageReportWorkerImpl") - .getDeclaredConstructor() - .newInstance() - .asInstanceOf[api.ScoverageReportWorkerApi] + val worker = + cl + .loadClass("mill.contrib.scoverage.worker.ScoverageReportWorkerImpl") + .getDeclaredConstructor() + .newInstance() + .asInstanceOf[api.ScoverageReportWorkerApi2] + + def ctx0(implicit ctx: Ctx): ApiCtx = { + val logger = new ApiLogger { + def info(msg: String): Unit = ctx.log.info(msg) + def error(msg: String): Unit = ctx.log.error(msg) + def debug(msg: String): Unit = ctx.log.debug(msg) + } + new ApiCtx { + def log() = logger + def dest() = ctx.dest.toNIO + } + } + + new ScoverageReportWorkerApiBridge { + override def report( + reportType: ReportType, + sources: Seq[os.Path], + dataDirs: Seq[os.Path], + sourceRoot: os.Path + )(implicit + ctx: Ctx + ): Unit = { + worker.report( + reportType, + sources.map(_.toNIO).toArray, + dataDirs.map(_.toNIO).toArray, + sourceRoot.toNIO, + ctx0 + ) + } + } } override def close(): Unit = { @@ -37,6 +74,18 @@ class ScoverageReportWorker extends AutoCloseable { } object ScoverageReportWorker extends ExternalModule { + import ScoverageReportWorkerApi2.ReportType + + trait ScoverageReportWorkerApiBridge { + def report( + reportType: ReportType, + sources: Seq[os.Path], + dataDirs: Seq[os.Path], + sourceRoot: os.Path + )(implicit + ctx: Ctx + ): Unit + } def scoverageReportWorker: Worker[ScoverageReportWorker] = Task.Worker { new ScoverageReportWorker() } diff --git a/contrib/scoverage/worker2/src/mill/contrib/scoverage/worker/ScoverageReportWorkerImpl.scala b/contrib/scoverage/worker2/src/mill/contrib/scoverage/worker/ScoverageReportWorkerImpl.scala index a478a91c533..1393af627f1 100644 --- a/contrib/scoverage/worker2/src/mill/contrib/scoverage/worker/ScoverageReportWorkerImpl.scala +++ b/contrib/scoverage/worker2/src/mill/contrib/scoverage/worker/ScoverageReportWorkerImpl.scala @@ -1,42 +1,46 @@ package mill.contrib.scoverage.worker -import mill.contrib.scoverage.api.ScoverageReportWorkerApi import _root_.scoverage.reporter.{ CoberturaXmlWriter, CoverageAggregator, ScoverageHtmlWriter, ScoverageXmlWriter } -import mill.api.Ctx -import mill.contrib.scoverage.api.ScoverageReportWorkerApi.ReportType + +import mill.contrib.scoverage.api.ScoverageReportWorkerApi2 +import ScoverageReportWorkerApi2.ReportType +import ScoverageReportWorkerApi2.Ctx + +import java.nio.file.Path /** * Scoverage Worker for Scoverage 2.x */ -class ScoverageReportWorkerImpl extends ScoverageReportWorkerApi { +class ScoverageReportWorkerImpl extends ScoverageReportWorkerApi2 { override def report( reportType: ReportType, - sources: Seq[os.Path], - dataDirs: Seq[os.Path], - sourceRoot: os.Path - )(implicit ctx: Ctx): Unit = + sources: Array[Path], + dataDirs: Array[Path], + sourceRoot: Path, + ctx: Ctx + ): Unit = try { ctx.log.info(s"Processing coverage data for ${dataDirs.size} data locations") - CoverageAggregator.aggregate(dataDirs.map(_.toIO), sourceRoot.toIO) match { + CoverageAggregator.aggregate(dataDirs.map(_.toFile).toIndexedSeq, sourceRoot.toFile) match { case Some(coverage) => - val sourceFolders = sources.map(_.toIO) + val sourceFolders = sources.map(_.toFile).toIndexedSeq val folder = ctx.dest - os.makeDir.all(folder) + ScoverageReportWorkerApi2.makeAllDirs(folder) reportType match { case ReportType.Html => - new ScoverageHtmlWriter(sourceFolders, folder.toIO, None) + new ScoverageHtmlWriter(sourceFolders, folder.toFile, None) .write(coverage) case ReportType.Xml => - new ScoverageXmlWriter(sourceFolders, folder.toIO, false, None) + new ScoverageXmlWriter(sourceFolders, folder.toFile, false, None) .write(coverage) case ReportType.XmlCobertura => - new CoberturaXmlWriter(sourceFolders, folder.toIO, None) + new CoberturaXmlWriter(sourceFolders, folder.toFile, None) .write(coverage) case ReportType.Console => ctx.log.info(s"Statement coverage.: ${coverage.statementCoverageFormatted}%") diff --git a/contrib/twirllib/src/mill/twirllib/TwirlModule.scala b/contrib/twirllib/src/mill/twirllib/TwirlModule.scala index c2d6fad03b8..0c3269e78c2 100644 --- a/contrib/twirllib/src/mill/twirllib/TwirlModule.scala +++ b/contrib/twirllib/src/mill/twirllib/TwirlModule.scala @@ -33,7 +33,7 @@ trait TwirlModule extends mill.Module { twirlModule => * Replicate the logic from twirl build, * see: https://github.com/playframework/twirl/blob/2.0.1/build.sbt#L12-L17 */ - private def scalaParserCombinatorsVersion: T[String] = twirlScalaVersion.map { + private def scalaParserCombinatorsVersion: Task[String] = twirlScalaVersion.map { case v if v.startsWith("2.") => "1.1.2" case _ => "2.3.0" } @@ -57,7 +57,7 @@ trait TwirlModule extends mill.Module { twirlModule => * @since Mill after 0.10.5 */ trait TwirlResolver extends CoursierModule { - override def resolveCoursierDependency: Task[Dep => Dependency] = Task.Anon { d: Dep => + override def resolveCoursierDependency: Task[Dep => Dependency] = Task.Anon { (d: Dep) => Lib.depToDependency(d, twirlScalaVersion()) } diff --git a/contrib/twirllib/test/src/mill/twirllib/HelloWorldTests.scala b/contrib/twirllib/test/src/mill/twirllib/HelloWorldTests.scala index a8094cc725b..36b0b02c063 100644 --- a/contrib/twirllib/test/src/mill/twirllib/HelloWorldTests.scala +++ b/contrib/twirllib/test/src/mill/twirllib/HelloWorldTests.scala @@ -7,6 +7,7 @@ import utest.{TestSuite, Tests, assert, _} trait HelloWorldTests extends TestSuite { val testTwirlVersion: String + val wildcard: String trait HelloWorldModule extends mill.twirllib.TwirlModule { def twirlVersion = testTwirlVersion @@ -39,8 +40,8 @@ trait HelloWorldTests extends TestSuite { ) def expectedDefaultImports: Seq[String] = Seq( - "import _root_.play.twirl.api.TwirlFeatureImports._", - "import _root_.play.twirl.api.TwirlHelperImports._", + s"import _root_.play.twirl.api.TwirlFeatureImports.$wildcard", + s"import _root_.play.twirl.api.TwirlHelperImports.$wildcard", "import _root_.play.twirl.api.Html", "import _root_.play.twirl.api.JavaScript", "import _root_.play.twirl.api.Txt", @@ -48,8 +49,8 @@ trait HelloWorldTests extends TestSuite { ) def testAdditionalImports: Seq[String] = Seq( - "mill.twirl.test.AdditionalImport1._", - "mill.twirl.test.AdditionalImport2._" + s"mill.twirl.test.AdditionalImport1.$wildcard", + s"mill.twirl.test.AdditionalImport2.$wildcard" ) def testConstructorAnnotations = Seq( @@ -159,13 +160,17 @@ trait HelloWorldTests extends TestSuite { object HelloWorldTests1_3 extends HelloWorldTests { override val testTwirlVersion = "1.3.16" + override val wildcard = "_" } object HelloWorldTests1_5 extends HelloWorldTests { override val testTwirlVersion = "1.5.2" + override val wildcard = "_" } object HelloWorldTests1_6 extends HelloWorldTests { override val testTwirlVersion = "1.6.2" + override val wildcard = "_" } object HelloWorldTests2_0 extends HelloWorldTests { override val testTwirlVersion = "2.0.1" + override val wildcard = "_" } diff --git a/example/thirdparty/mockito/build.mill b/example/thirdparty/mockito/build.mill index 0af38beb6ad..ad0fbf5e3b1 100644 --- a/example/thirdparty/mockito/build.mill +++ b/example/thirdparty/mockito/build.mill @@ -42,11 +42,20 @@ trait MockitoModule extends MavenModule{ def testRuntimeIvyDeps: T[Agg[Dep]] = Agg.empty[Dep] def testFramework = "com.novocode.junit.JUnitFramework" def testForkArgs: T[Seq[String]] = Seq.empty[String] + + def testFilteredSources: T[Seq[PathRef]] = Task { Seq.empty[PathRef] } + object test extends MavenTests{ def moduleDeps = super.moduleDeps ++ MockitoModule.this.testModuleDeps def testFramework = MockitoModule.this.testFramework def runIvyDeps = testRuntimeIvyDeps() def forkArgs = testForkArgs() + def allSourceFiles = Task { + val base = super.allSourceFiles() + val filtered = testFilteredSources().toSet + if (filtered.isEmpty) base + else base.filterNot(filtered.contains) + } def ivyDeps = testIvyDeps() ++ Agg( @@ -85,6 +94,11 @@ object `package` extends RootModule with MockitoModule{ super.resources() ++ Seq(PathRef(Task.dest)) } + def testFilteredSources: T[Seq[PathRef]] = Task { + // test `add_listeners_concurrently_sanity_check` is flaky + Seq(PathRef(millSourcePath / "src/test/java/org/mockitousage/debugging/StubbingLookupListenerCallbackTest.java")) + } + object subprojects extends Module { object android extends MockitoModule { def moduleDeps = Seq(build) diff --git a/example/thirdparty/netty/build.mill b/example/thirdparty/netty/build.mill index f7b10579034..b9a1a7f3099 100644 --- a/example/thirdparty/netty/build.mill +++ b/example/thirdparty/netty/build.mill @@ -253,8 +253,8 @@ object common extends NettyModule{ val shell = new groovy.lang.GroovyShell() val context = new java.util.HashMap[String, Object] - context.put("collection.template.dir", Task.workspace + "/common/src/main/templates") - context.put("collection.template.test.dir", Task.workspace + "/common/src/test/templates") + context.put("collection.template.dir", s"${Task.workspace}/common/src/main/templates") + context.put("collection.template.test.dir", s"${Task.workspace}/common/src/test/templates") context.put("collection.src.dir", (Task.dest / "src").toString) context.put("collection.testsrc.dir", (Task.dest / "testsrc").toString) shell.setProperty("properties", context) diff --git a/integration/feature/docannotations/src/DocAnnotationsTests.scala b/integration/feature/docannotations/src/DocAnnotationsTests.scala index 4ef2900e86c..ee3449a00ea 100644 --- a/integration/feature/docannotations/src/DocAnnotationsTests.scala +++ b/integration/feature/docannotations/src/DocAnnotationsTests.scala @@ -93,7 +93,7 @@ object DocAnnotationsTests extends UtestIntegrationTestSuite { assert( globMatches( - """core.ivyDepsTree(JavaModule.scala:896) + """core.ivyDepsTree(JavaModule.scala:...) | Command to print the transitive dependency tree to STDOUT. | | --inverse Invert the tree representation, so that the root is on the bottom val diff --git a/integration/invalidation/multi-level-editing/resources/mill-build/mill-build/build.mill b/integration/invalidation/multi-level-editing/resources/mill-build/mill-build/build.mill index 7d6f134d94e..4efe9a06fcd 100644 --- a/integration/invalidation/multi-level-editing/resources/mill-build/mill-build/build.mill +++ b/integration/invalidation/multi-level-editing/resources/mill-build/mill-build/build.mill @@ -8,7 +8,7 @@ object `package` extends MillBuildRootModule { Task.dest / "MetaConstant.scala", """package constant |object MetaConstant{ - | def scalatagsVersion = "0.8.2" + | def scalatagsVersion = "0.13.1" |} |""".stripMargin ) diff --git a/integration/invalidation/multi-level-editing/src/MultiLevelBuildTests.scala b/integration/invalidation/multi-level-editing/src/MultiLevelBuildTests.scala index ba9554d6c7a..7c92a4486d4 100644 --- a/integration/invalidation/multi-level-editing/src/MultiLevelBuildTests.scala +++ b/integration/invalidation/multi-level-editing/src/MultiLevelBuildTests.scala @@ -87,7 +87,7 @@ object MultiLevelBuildTests extends UtestIntegrationTestSuite { assert(res.isSuccess == false) // Prepend a "\n" to allow callsites to use "\n" to test for start of // line, even though the first line doesn't have a "\n" at the start - val err = "\n" + res.err + val err = "```\n" + res.err + "\n```" for (expected <- expectedSnippets) { assert(err.contains(expected)) } @@ -130,7 +130,7 @@ object MultiLevelBuildTests extends UtestIntegrationTestSuite { test("validEdits") - integrationTest { tester => import tester._ - runAssertSuccess(tester, "

hello

world

0.8.2

!") + runAssertSuccess(tester, "

hello

world

0.13.1

!") checkWatchedFiles( tester, fooPaths(tester), @@ -143,7 +143,7 @@ object MultiLevelBuildTests extends UtestIntegrationTestSuite { checkChangedClassloaders(tester, null, true, true, true) modifyFile(workspacePath / "foo/src/Example.scala", _.replace("!", "?")) - runAssertSuccess(tester, "

hello

world

0.8.2

?") + runAssertSuccess(tester, "

hello

world

0.13.1

?") checkWatchedFiles( tester, fooPaths(tester), @@ -155,7 +155,7 @@ object MultiLevelBuildTests extends UtestIntegrationTestSuite { checkChangedClassloaders(tester, null, false, false, false) modifyFile(workspacePath / "build.mill", _.replace("hello", "HELLO")) - runAssertSuccess(tester, "

HELLO

world

0.8.2

?") + runAssertSuccess(tester, "

HELLO

world

0.13.1

?") checkWatchedFiles( tester, fooPaths(tester), @@ -169,7 +169,7 @@ object MultiLevelBuildTests extends UtestIntegrationTestSuite { workspacePath / "mill-build/build.mill", _.replace("def scalatagsVersion = ", "def scalatagsVersion = \"changed-\" + ") ) - runAssertSuccess(tester, "

HELLO

world

changed-0.8.2

?") + runAssertSuccess(tester, "

HELLO

world

changed-0.13.1

?") checkWatchedFiles( tester, fooPaths(tester), @@ -181,7 +181,7 @@ object MultiLevelBuildTests extends UtestIntegrationTestSuite { modifyFile( workspacePath / "mill-build/mill-build/build.mill", - _.replace("0.8.2", "0.12.0") + _.replace("0.13.1", "0.12.0") ) runAssertSuccess(tester, "

HELLO

world

changed-0.12.0

?") checkWatchedFiles( @@ -195,9 +195,9 @@ object MultiLevelBuildTests extends UtestIntegrationTestSuite { modifyFile( workspacePath / "mill-build/mill-build/build.mill", - _.replace("0.12.0", "0.8.2") + _.replace("0.12.0", "0.13.1") ) - runAssertSuccess(tester, "

HELLO

world

changed-0.8.2

?") + runAssertSuccess(tester, "

HELLO

world

changed-0.13.1

?") checkWatchedFiles( tester, fooPaths(tester), @@ -211,7 +211,7 @@ object MultiLevelBuildTests extends UtestIntegrationTestSuite { workspacePath / "mill-build/build.mill", _.replace("def scalatagsVersion = \"changed-\" + ", "def scalatagsVersion = ") ) - runAssertSuccess(tester, "

HELLO

world

0.8.2

?") + runAssertSuccess(tester, "

HELLO

world

0.13.1

?") checkWatchedFiles( tester, fooPaths(tester), @@ -222,7 +222,7 @@ object MultiLevelBuildTests extends UtestIntegrationTestSuite { checkChangedClassloaders(tester, null, true, true, false) modifyFile(workspacePath / "build.mill", _.replace("HELLO", "hello")) - runAssertSuccess(tester, "

hello

world

0.8.2

?") + runAssertSuccess(tester, "

hello

world

0.13.1

?") checkWatchedFiles( tester, fooPaths(tester), @@ -233,7 +233,7 @@ object MultiLevelBuildTests extends UtestIntegrationTestSuite { checkChangedClassloaders(tester, null, true, false, false) modifyFile(workspacePath / "foo/src/Example.scala", _.replace("?", "!")) - runAssertSuccess(tester, "

hello

world

0.8.2

!") + runAssertSuccess(tester, "

hello

world

0.13.1

!") checkWatchedFiles( tester, fooPaths(tester), @@ -252,7 +252,7 @@ object MultiLevelBuildTests extends UtestIntegrationTestSuite { def fixParseError(p: os.Path) = modifyFile(p, _.replace("extendx", "extends")) - runAssertSuccess(tester, "

hello

world

0.8.2

!") + runAssertSuccess(tester, "

hello

world

0.13.1

!") checkWatchedFiles( tester, fooPaths(tester), @@ -300,7 +300,7 @@ object MultiLevelBuildTests extends UtestIntegrationTestSuite { checkChangedClassloaders(tester, null, null, null, null) fixParseError(workspacePath / "build.mill") - runAssertSuccess(tester, "

hello

world

0.8.2

!") + runAssertSuccess(tester, "

hello

world

0.13.1

!") checkWatchedFiles( tester, fooPaths(tester), @@ -319,7 +319,7 @@ object MultiLevelBuildTests extends UtestIntegrationTestSuite { def fixCompileError(p: os.Path) = modifyFile(p, _.replace("import doesnt.exist", "")) - runAssertSuccess(tester, "

hello

world

0.8.2

!") + runAssertSuccess(tester, "

hello

world

0.13.1

!") checkWatchedFiles( tester, fooPaths(tester), @@ -382,7 +382,7 @@ object MultiLevelBuildTests extends UtestIntegrationTestSuite { checkChangedClassloaders(tester, null, null, true, false) fixCompileError(workspacePath / "build.mill") - runAssertSuccess(tester, "

hello

world

0.8.2

!") + runAssertSuccess(tester, "

hello

world

0.13.1

!") checkWatchedFiles( tester, fooPaths(tester), @@ -407,7 +407,7 @@ object MultiLevelBuildTests extends UtestIntegrationTestSuite { def fixRuntimeError(p: os.Path) = modifyFile(p, _.replaceFirst(Regex.quote(runErrorSnippet), "\\{")) - runAssertSuccess(tester, "

hello

world

0.8.2

!") + runAssertSuccess(tester, "

hello

world

0.13.1

!") checkWatchedFiles( tester, fooPaths(tester), @@ -475,7 +475,7 @@ object MultiLevelBuildTests extends UtestIntegrationTestSuite { checkChangedClassloaders(tester, null, true, true, false) fixRuntimeError(workspacePath / "build.mill") - runAssertSuccess(tester, "

hello

world

0.8.2

!") + runAssertSuccess(tester, "

hello

world

0.13.1

!") checkWatchedFiles( tester, fooPaths(tester), diff --git a/main/api/src/mill/api/AggWrapper.scala b/main/api/src/mill/api/AggWrapper.scala index 06382846437..9be41fb579b 100644 --- a/main/api/src/mill/api/AggWrapper.scala +++ b/main/api/src/mill/api/AggWrapper.scala @@ -85,16 +85,15 @@ private[mill] sealed class AggWrapper(strictUniqueness: Boolean) { Mutable.newBuilder[V] } - private[this] val set0 = mutable.LinkedHashSet.empty[V] + private val set0 = mutable.LinkedHashSet.empty[V] def contains(v: V): Boolean = set0.contains(v) def coll: Mutable[V] = this - override def toIterable: Iterable[V] = set0.toIterable + override def toIterable: Iterable[V] = set0 def append(v: V): AnyVal = { if (!contains(v)) { - set0.add(v) - + return set0.add(v) } else if (strictUniqueness) { throw new Exception("Duplicated item inserted into OrderedSet: " + v) } @@ -118,7 +117,9 @@ private[mill] sealed class AggWrapper(strictUniqueness: Boolean) { override def filter(f: V => Boolean): Mutable[V] = { val output = new Agg.Mutable[V] - for (i <- items) if (f(i)) output.append(i) + for (i <- items) if (f(i)) { + val _ = output.append(i) + } output } @@ -148,7 +149,7 @@ private[mill] sealed class AggWrapper(strictUniqueness: Boolean) { def iterator: Iterator[V] = items override def hashCode(): Int = items.map(_.hashCode()).sum override def equals(other: Any): Boolean = other match { - case s: Agg[_] => items.sameElements(s.items) + case s: Agg[?] => items.sameElements(s.items) case _ => super.equals(other) } override def toString: String = items.mkString("Agg(", ", ", ")") diff --git a/main/api/src/mill/api/ClassLoader.scala b/main/api/src/mill/api/ClassLoader.scala index 6f2b5ec7f2b..5db97159d63 100644 --- a/main/api/src/mill/api/ClassLoader.scala +++ b/main/api/src/mill/api/ClassLoader.scala @@ -26,7 +26,7 @@ object ClassLoader { makeUrls(urls).toArray, refinePlatformParent(parent) ) { - override def findClass(name: String): Class[_] = { + override def findClass(name: String): Class[?] = { if (sharedPrefixes.exists(name.startsWith)) { logger.foreach( _.debug(s"About to load class [${name}] from shared classloader [${sharedLoader}]") diff --git a/main/api/src/mill/api/Ctx.scala b/main/api/src/mill/api/Ctx.scala index ca08069896b..59ad59d3d67 100644 --- a/main/api/src/mill/api/Ctx.scala +++ b/main/api/src/mill/api/Ctx.scala @@ -73,7 +73,7 @@ object Ctx { } trait Args { - def args: IndexedSeq[_] + def args: IndexedSeq[?] } /** @@ -167,7 +167,7 @@ object Ctx { * implementation of a `Task`. */ class Ctx( - val args: IndexedSeq[_], + val args: IndexedSeq[?], dest0: () => os.Path, val log: Logger, val home: os.Path, @@ -185,7 +185,7 @@ class Ctx( with Ctx.Workspace { def this( - args: IndexedSeq[_], + args: IndexedSeq[?], dest0: () => os.Path, log: Logger, home: os.Path, @@ -194,7 +194,7 @@ class Ctx( testReporter: TestReporter, workspace: os.Path ) = { - this(args, dest0, log, home, env, reporter, testReporter, workspace, i => ???, null) + this(args, dest0, log, home, env, reporter, testReporter, workspace, _ => ???, null) } def dest: os.Path = dest0() def arg[T](index: Int): T = { diff --git a/main/api/src/mill/api/FixSizedCache.scala b/main/api/src/mill/api/FixSizedCache.scala index 5d71874246b..0ba0f2a12e5 100644 --- a/main/api/src/mill/api/FixSizedCache.scala +++ b/main/api/src/mill/api/FixSizedCache.scala @@ -15,7 +15,7 @@ import java.util.concurrent.{ConcurrentHashMap, Semaphore} class FixSizedCache[T](perKeySize: Int) extends KeyedLockedCache[T] { // Cache Key -> (Semaphore, Array of cached elements) - private[this] val keyToCache: ConcurrentHashMap[Long, (Semaphore, Array[(Boolean, Option[T])])] = + private val keyToCache: ConcurrentHashMap[Long, (Semaphore, Array[(Boolean, Option[T])])] = new ConcurrentHashMap override def withCachedValue[V](key: Long)(f: => T)(f2: T => V): V = { diff --git a/main/api/src/mill/api/JarOps.scala b/main/api/src/mill/api/JarOps.scala index 9f2e0d2a737..540c706ca29 100644 --- a/main/api/src/mill/api/JarOps.scala +++ b/main/api/src/mill/api/JarOps.scala @@ -73,7 +73,7 @@ trait JarOps { os.remove.all(jar) val seen = mutable.Set.empty[os.RelPath] - seen.add(os.sub / "META-INF/MANIFEST.MF") + val _ = seen.add(os.sub / "META-INF/MANIFEST.MF") val jarStream = new JarOutputStream( new BufferedOutputStream(new FileOutputStream(jar.toIO)), @@ -84,7 +84,7 @@ trait JarOps { assert(inputPaths.iterator.forall(os.exists(_))) if (includeDirs) { - seen.add(os.sub / "META-INF") + val _ = seen.add(os.sub / "META-INF") val entry = new JarEntry("META-INF/") entry.setTime(curTime) jarStream.putNextEntry(entry) @@ -99,7 +99,7 @@ trait JarOps { else os.walk(p).map(sub => (sub, sub.subRelativeTo(p))).sorted if (includeDirs || os.isFile(file)) && !seen(mapping) && fileFilter(p, mapping) } { - seen.add(mapping) + val _ = seen.add(mapping) val name = mapping.toString() + (if (os.isDir(file)) "/" else "") val entry = new JarEntry(name) entry.setTime(mTime(file)) diff --git a/main/api/src/mill/api/JsonFormatters.scala b/main/api/src/mill/api/JsonFormatters.scala index dda6a1c3cc1..bf1b448955f 100644 --- a/main/api/src/mill/api/JsonFormatters.scala +++ b/main/api/src/mill/api/JsonFormatters.scala @@ -62,7 +62,7 @@ trait JsonFormatters { ) ) - implicit def enumFormat[T <: java.lang.Enum[_]: ClassTag]: RW[T] = + implicit def enumFormat[T <: java.lang.Enum[?]: ClassTag]: RW[T] = upickle.default.readwriter[String].bimap( _.name(), (s: String) => diff --git a/main/api/src/mill/api/KeyedLockedCache.scala b/main/api/src/mill/api/KeyedLockedCache.scala index 7a8fc21a92f..3ee62796f3c 100644 --- a/main/api/src/mill/api/KeyedLockedCache.scala +++ b/main/api/src/mill/api/KeyedLockedCache.scala @@ -11,7 +11,7 @@ trait KeyedLockedCache[T] { object KeyedLockedCache { class RandomBoundedCache[T](hotParallelism: Int, coldCacheSize: Int) extends KeyedLockedCache[T] { - private[this] val random = new scala.util.Random(313373) + private val random = new scala.util.Random(313373) val available = new java.util.concurrent.Semaphore(hotParallelism) // Awful asymptotic complexity, but our caches are tiny n < 10 so it doesn't matter diff --git a/main/api/src/mill/api/PathRef.scala b/main/api/src/mill/api/PathRef.scala index ad1ae5a44eb..483efa569c1 100644 --- a/main/api/src/mill/api/PathRef.scala +++ b/main/api/src/mill/api/PathRef.scala @@ -1,10 +1,13 @@ package mill.api +import scala.language.implicitConversions + import java.nio.{file => jnio} import java.security.{DigestOutputStream, MessageDigest} import java.util.concurrent.ConcurrentHashMap import scala.util.{DynamicVariable, Using} import upickle.default.{ReadWriter => RW} +import scala.annotation.nowarn /** * A wrapper around `os.Path` that calculates it's hashcode based @@ -69,7 +72,7 @@ object PathRef { if (pathRef.sig != changedSig) { throw new PathRefValidationException(pathRef) } - map.put(mapKey(pathRef), pathRef) + val _ = map.put(mapKey(pathRef), pathRef) } } def clear(): Unit = map.clear() @@ -197,6 +200,7 @@ object PathRef { ) // scalafix:off; we want to hide the unapply method + @nowarn("msg=unused") private def unapply(pathRef: PathRef): Option[(os.Path, Boolean, Int, Revalidate)] = { Some((pathRef.path, pathRef.quick, pathRef.sig, pathRef.revalidate)) } diff --git a/main/api/src/mill/api/Result.scala b/main/api/src/mill/api/Result.scala index ad7af1e051b..afe73a5c2d1 100644 --- a/main/api/src/mill/api/Result.scala +++ b/main/api/src/mill/api/Result.scala @@ -13,9 +13,10 @@ sealed trait Result[+T] { def flatMap[V](f: T => Result[V]): Result[V] def asSuccess: Option[Result.Success[T]] = None def asFailing: Option[Result.Failing[T]] = None - def getOrThrow: T = this match { + def getOrThrow: T = (this: @unchecked) match { case Result.Success(v) => v - case f: Result.Failing[_] => throw f + case f: Result.Failing[?] => throw f + // no cases for Skipped or Aborted? } } @@ -95,10 +96,15 @@ object Result { current = current.head.getCause :: current } current.reverse - .flatMap(ex => - Seq(ex.toString) ++ - ex.getStackTrace.dropRight(outerStack.value.length).map(" " + _) - ) + .flatMap { ex => + val elements = ex.getStackTrace.dropRight(outerStack.value.length) + val formatted = + // for some reason .map without the explicit ArrayOps conversion doesn't work, + // and results in `Result[String]` instead of `Array[String]` + new scala.collection.ArrayOps(elements).map(" " + _) + Seq(ex.toString) ++ formatted + + } .mkString("\n") } } diff --git a/main/api/src/mill/api/Retry.scala b/main/api/src/mill/api/Retry.scala index 2398d4e87e7..ca35b0537f7 100644 --- a/main/api/src/mill/api/Retry.scala +++ b/main/api/src/mill/api/Retry.scala @@ -38,9 +38,9 @@ case class Retry( if (timeoutMillis == -1) t(retryCount) else { val result = Promise[T] - val thread = new Thread(() => { + val thread = new Thread({ () => result.complete(scala.util.Try(t(retryCount))) - }) + }: Runnable) thread.start() Await.result(result.future, Duration.apply(timeoutMillis, TimeUnit.MILLISECONDS)) } diff --git a/main/api/src/mill/api/SystemStreams.scala b/main/api/src/mill/api/SystemStreams.scala index e42b34d80ed..3912eb57042 100644 --- a/main/api/src/mill/api/SystemStreams.scala +++ b/main/api/src/mill/api/SystemStreams.scala @@ -132,9 +132,9 @@ object SystemStreams { def setTopLevelSystemStreamProxy(): Unit = { // Make sure to initialize `Console` to cache references to the original // `System.{in,out,err}` streams before we redirect them - Console.out - Console.err - Console.in + val _ = Console.out + val _ = Console.err + val _ = Console.in System.setIn(ThreadLocalStreams.In) System.setOut(ThreadLocalStreams.Out) System.setErr(ThreadLocalStreams.Err) diff --git a/main/codesig/package.mill b/main/codesig/package.mill index 51c622e5054..7e0120def83 100644 --- a/main/codesig/package.mill +++ b/main/codesig/package.mill @@ -47,7 +47,8 @@ object `package` extends RootModule with build.MillPublishScalaModule { build.Deps.mainargs, build.Deps.requests, build.Deps.osLib, - build.Deps.upickle + build.Deps.upickle, + build.Deps.sourcecode ) } } diff --git a/main/codesig/src/JvmModel.scala b/main/codesig/src/JvmModel.scala index 00c36c8fd31..e63eeffa937 100644 --- a/main/codesig/src/JvmModel.scala +++ b/main/codesig/src/JvmModel.scala @@ -139,7 +139,7 @@ object JvmModel { sealed class Prim(val pretty: String) extends JType - object Prim extends { + object Prim { def read(s: String): Prim = all(s(0)) val all: Map[Char, Prim] = Map( diff --git a/main/codesig/src/ResolvedCalls.scala b/main/codesig/src/ResolvedCalls.scala index 7b2fe32a26d..6bb5b5698f7 100644 --- a/main/codesig/src/ResolvedCalls.scala +++ b/main/codesig/src/ResolvedCalls.scala @@ -113,7 +113,9 @@ object ResolvedCalls { val externalSamDefiners = externalSummary .directMethods .map { case (k, v) => (k, v.collect { case (sig, true) => sig }) } - .collect { case (k, Seq(v)) => (k, v) } + .collect { case (k, Seq(v)) => + (k, v) + } // Scala 3.5.0-RC6 - can not infer MethodSig here val allSamDefiners = localSamDefiners ++ externalSamDefiners diff --git a/main/codesig/src/SpanningForest.scala b/main/codesig/src/SpanningForest.scala index 115d60b3554..dd870bd4827 100644 --- a/main/codesig/src/SpanningForest.scala +++ b/main/codesig/src/SpanningForest.scala @@ -36,7 +36,11 @@ object SpanningForest { .groupMap(_._1)(_._2) ResolvedCalls.breadthFirst(rootChangedNodeIndices) { index => - val nextIndices = downstreamGraphEdges.getOrElse(index, Array()) + val nextIndices = + downstreamGraphEdges.getOrElse( + index, + Array[Int]() + ) // needed to add explicit type for Scala 3.5.0-RC6 // We build up the spanningForest during a normal breadth first search, // using the `nodeMapping` to quickly find an vertice's tree node so we // can add children to it. We need to duplicate the `seen.contains` logic diff --git a/main/codesig/test/cases/callgraph/basic/17-scala-lambda/src/Hello.scala b/main/codesig/test/cases/callgraph/basic/17-scala-lambda/src/Hello.scala index 45fdc0c8195..d9c7a6141ff 100644 --- a/main/codesig/test/cases/callgraph/basic/17-scala-lambda/src/Hello.scala +++ b/main/codesig/test/cases/callgraph/basic/17-scala-lambda/src/Hello.scala @@ -1,9 +1,14 @@ package hello object Hello { + + trait MyFunction0[T] { + def apply(): T + } + def main(): Int = { - val foo = () => used() + val foo: MyFunction0[Int] = () => used() foo() } def used(): Int = 2 diff --git a/main/codesig/test/cases/callgraph/complicated/13-iterator-inherit-external-filter-scala/src/Hello.scala b/main/codesig/test/cases/callgraph/complicated/13-iterator-inherit-external-filter-scala/src/Hello.scala index 7096870a717..6861bc5ecf2 100644 --- a/main/codesig/test/cases/callgraph/complicated/13-iterator-inherit-external-filter-scala/src/Hello.scala +++ b/main/codesig/test/cases/callgraph/complicated/13-iterator-inherit-external-filter-scala/src/Hello.scala @@ -26,10 +26,11 @@ object Hello { private[this] var hdDefined: Boolean = false def hasNext: Boolean = hdDefined || { - do { + while ({ if (!parent.hasNext) return false hd = parent.next() - } while (!pred(hd)) + !pred(hd) + }) {} hdDefined = true true } diff --git a/main/codesig/test/cases/callgraph/complicated/6-classes-misc-scala/src/Hello.scala b/main/codesig/test/cases/callgraph/complicated/6-classes-misc-scala/src/Hello.scala index 39896097bae..a724d7d6d3b 100644 --- a/main/codesig/test/cases/callgraph/complicated/6-classes-misc-scala/src/Hello.scala +++ b/main/codesig/test/cases/callgraph/complicated/6-classes-misc-scala/src/Hello.scala @@ -32,7 +32,7 @@ class DoubleDetMatrix(aa: Float, ab: Float, ba: Float, bb: Float) } class LinkedList { - def push(i: Int) { + def push(i: Int): Unit = { val n = new Inner(i, head) head = n } diff --git a/main/codesig/test/cases/callgraph/complicated/8-linked-list-scala/src/Hello.scala b/main/codesig/test/cases/callgraph/complicated/8-linked-list-scala/src/Hello.scala index 061c4c14aeb..5235986b920 100644 --- a/main/codesig/test/cases/callgraph/complicated/8-linked-list-scala/src/Hello.scala +++ b/main/codesig/test/cases/callgraph/complicated/8-linked-list-scala/src/Hello.scala @@ -9,7 +9,7 @@ object Hello { def head: A def tail: TestList[A] - def foreach[U](f: A => U) { + def foreach[U](f: A => U): Unit = { var these = this while (!these.isEmpty) { f(these.head) @@ -21,7 +21,7 @@ object Hello { object TestNil extends TestList[Nothing] { def isEmpty = true def head = throw new Exception() - def tail = throw new Exception() + override def tail: Nothing = throw new Exception() } class TestCons[B](val head: B, val tl: TestList[B]) extends TestList[B] { diff --git a/main/codesig/test/cases/callgraph/realistic/5-parser/src/Hello.scala b/main/codesig/test/cases/callgraph/realistic/5-parser/src/Hello.scala index 1fb9a5a8a10..52df0ce57f6 100644 --- a/main/codesig/test/cases/callgraph/realistic/5-parser/src/Hello.scala +++ b/main/codesig/test/cases/callgraph/realistic/5-parser/src/Hello.scala @@ -6,15 +6,15 @@ class Word(s: String) extends Phrase class Pair(lhs: Phrase, rhs: Phrase) extends Phrase object Parser { - def prefix[_: P] = P("hello" | "goodbye").!.map(new Word(_)) + def prefix[$: P] = P("hello" | "goodbye").!.map(new Word(_)) - def suffix[_: P] = P("world" | "seattle").!.map(new Word(_)) + def suffix[$: P] = P("world" | "seattle").!.map(new Word(_)) - def ws[_: P] = P(" ".rep(1)) + def ws[$: P] = P(" ".rep(1)) - def parened[_: P] = P("(" ~ parser ~ ")") + def parened[$: P] = P("(" ~ parser ~ ")") - def parser[_: P]: P[Phrase] = P((parened | prefix) ~ ws ~ (parened | suffix)).map { + def parser[$: P]: P[Phrase] = P((parened | prefix) ~ ws ~ (parened | suffix)).map { case (lhs, rhs) => new Pair(lhs, rhs) } } diff --git a/main/define/test/src/mill/define/BasePathTests.scala b/main/define/test/src/mill/define/BasePathTests.scala index dce543ca02b..55901cb9778 100644 --- a/main/define/test/src/mill/define/BasePathTests.scala +++ b/main/define/test/src/mill/define/BasePathTests.scala @@ -5,6 +5,17 @@ import mill.testkit.TestBaseModule import utest._ object BasePathTests extends TestSuite { + + object overriddenBasePath extends TestBaseModule { + override def millSourcePath = os.pwd / "overriddenBasePathRootValue" + object nested extends Module { + override def millSourcePath = super.millSourcePath / "overriddenBasePathNested" + object nested extends Module { + override def millSourcePath = super.millSourcePath / "overriddenBasePathDoubleNested" + } + } + } + val testGraphs = new TestGraphs val tests = Tests { def checkMillSourcePath[T <: Module](m: T)(f: T => Module, segments: String*): Unit = { @@ -54,15 +65,6 @@ object BasePathTests extends TestSuite { checkMillSourcePath(TestGraphs.nestedCrosses)(_.cross("210").cross2("js"), "cross", "cross2") } test("overridden") { - object overriddenBasePath extends TestBaseModule { - override def millSourcePath = os.pwd / "overriddenBasePathRootValue" - object nested extends Module { - override def millSourcePath = super.millSourcePath / "overriddenBasePathNested" - object nested extends Module { - override def millSourcePath = super.millSourcePath / "overriddenBasePathDoubleNested" - } - } - } assert( overriddenBasePath.millSourcePath == os.pwd / "overriddenBasePathRootValue", overriddenBasePath.nested.millSourcePath == os.pwd / "overriddenBasePathRootValue/nested/overriddenBasePathNested", diff --git a/main/eval/src/mill/eval/EvaluatorCore.scala b/main/eval/src/mill/eval/EvaluatorCore.scala index 932d73092bd..9e7af3684bd 100644 --- a/main/eval/src/mill/eval/EvaluatorCore.scala +++ b/main/eval/src/mill/eval/EvaluatorCore.scala @@ -265,10 +265,10 @@ private[mill] trait EvaluatorCore extends GroupEvaluator { c.getInterfaces.iterator.flatMap(resolveTransitiveParents) } - val classToTransitiveClasses = sortedGroups + val classToTransitiveClasses: Map[Class[?], IndexedSeq[Class[?]]] = sortedGroups .values() .flatten - .collect { case namedTask: NamedTask[_] => namedTask.ctx.enclosingCls } + .collect { case namedTask: NamedTask[?] => namedTask.ctx.enclosingCls } .map(cls => cls -> resolveTransitiveParents(cls).toVector) .toMap @@ -277,22 +277,23 @@ private[mill] trait EvaluatorCore extends GroupEvaluator { .flatMap(_._2) .toSet - val allTransitiveClassMethods = allTransitiveClasses - .map { cls => - val cMangledName = cls.getName.replace('.', '$') - cls -> cls.getDeclaredMethods - .flatMap { m => - Seq( - m.getName -> m, - // Handle scenarios where private method names get mangled when they are - // not really JVM-private due to being accessed by Scala nested objects - // or classes https://github.com/scala/bug/issues/9306 - m.getName.stripPrefix(cMangledName + "$$") -> m, - m.getName.stripPrefix(cMangledName + "$") -> m - ) - }.toMap - } - .toMap + val allTransitiveClassMethods: Map[Class[?], Map[String, java.lang.reflect.Method]] = + allTransitiveClasses + .map { cls => + val cMangledName = cls.getName.replace('.', '$') + cls -> cls.getDeclaredMethods + .flatMap { m => + Seq( + m.getName -> m, + // Handle scenarios where private method names get mangled when they are + // not really JVM-private due to being accessed by Scala nested objects + // or classes https://github.com/scala/bug/issues/9306 + m.getName.stripPrefix(cMangledName + "$$") -> m, + m.getName.stripPrefix(cMangledName + "$") -> m + ) + }.toMap + } + .toMap (classToTransitiveClasses, allTransitiveClassMethods) } diff --git a/main/eval/src/mill/eval/GroupEvaluator.scala b/main/eval/src/mill/eval/GroupEvaluator.scala index 676505ecf98..a5db8328364 100644 --- a/main/eval/src/mill/eval/GroupEvaluator.scala +++ b/main/eval/src/mill/eval/GroupEvaluator.scala @@ -383,7 +383,7 @@ private[mill] trait GroupEvaluator { .task .writerOpt .map { w => - upickle.default.writeJs(v.value)(w.asInstanceOf[upickle.default.Writer[Any]]) + upickle.default.writeJs(v.value)(using w.asInstanceOf[upickle.default.Writer[Any]]) } .orElse { labelled.task.asWorker.map { w => @@ -438,7 +438,7 @@ private[mill] trait GroupEvaluator { _ <- Option.when(cached.inputsHash == inputsHash)(()) reader <- labelled.task.readWriterOpt parsed <- - try Some(upickle.default.read(cached.value)(reader)) + try Some(upickle.default.read(cached.value)(using reader)) catch { case e: PathRef.PathRefValidationException => logger.debug( diff --git a/main/eval/test/src/mill/eval/EvaluationTests.scala b/main/eval/test/src/mill/eval/EvaluationTests.scala index ea0462c123f..021d5790663 100644 --- a/main/eval/test/src/mill/eval/EvaluationTests.scala +++ b/main/eval/test/src/mill/eval/EvaluationTests.scala @@ -1,6 +1,6 @@ package mill.eval -import mill.util.TestUtil.{Test, test} +import mill.util.TestUtil.Test import mill.define.{TargetImpl, Task} import mill.T import mill.util.{TestGraphs, TestUtil} @@ -61,9 +61,10 @@ class EvaluationTests(threadCount: Option[Int]) extends TestSuite { object graphs extends TestGraphs() import graphs._ import TestGraphs._ - utest.test("evaluateSingle") { + import utest._ + test("evaluateSingle") { - utest.test("singleton") { + test("singleton") { import singleton._ val check = new Checker(singleton) // First time the target is evaluated @@ -73,7 +74,7 @@ class EvaluationTests(threadCount: Option[Int]) extends TestSuite { // After incrementing the counter, it forces re-evaluation check(single, expValue = 1, expEvaled = Agg(single)) } - utest.test("backtickIdentifiers") { + test("backtickIdentifiers") { import graphs.bactickIdentifiers._ val check = new Checker(bactickIdentifiers) @@ -85,7 +86,7 @@ class EvaluationTests(threadCount: Option[Int]) extends TestSuite { `up-target`.counter += 1 check(`a-down-target`, expValue = 2, expEvaled = Agg(`up-target`, `a-down-target`)) } - utest.test("pair") { + test("pair") { import pair._ val check = new Checker(pair) check(down, expValue = 0, expEvaled = Agg(up, down)) @@ -96,7 +97,7 @@ class EvaluationTests(threadCount: Option[Int]) extends TestSuite { up.counter += 1 check(down, expValue = 2, expEvaled = Agg(up, down)) } - utest.test("anonTriple") { + test("anonTriple") { import anonTriple._ val check = new Checker(anonTriple) val middle = down.inputs(0) @@ -112,7 +113,7 @@ class EvaluationTests(threadCount: Option[Int]) extends TestSuite { check(down, expValue = 3, expEvaled = Agg(middle, down)) } - utest.test("diamond") { + test("diamond") { import diamond._ val check = new Checker(diamond) check(down, expValue = 0, expEvaled = Agg(up, left, right, down)) @@ -130,7 +131,7 @@ class EvaluationTests(threadCount: Option[Int]) extends TestSuite { right.counter += 1 check(down, expValue = 5, expEvaled = Agg(right, down)) } - utest.test("anonDiamond") { + test("anonDiamond") { import anonDiamond._ val check = new Checker(anonDiamond) val left = down.inputs(0).asInstanceOf[TestUtil.Test] @@ -151,7 +152,7 @@ class EvaluationTests(threadCount: Option[Int]) extends TestSuite { check(down, expValue = 5, expEvaled = Agg(left, right, down)) } - utest.test("bigSingleTerminal") { + test("bigSingleTerminal") { import bigSingleTerminal._ val check = new Checker(bigSingleTerminal) @@ -170,8 +171,8 @@ class EvaluationTests(threadCount: Option[Int]) extends TestSuite { } } - utest.test("evaluateMixed") { - utest.test("separateGroups") { + test("evaluateMixed") { + test("separateGroups") { // Make sure that `left` and `right` are able to recompute separately, // even though one depends on the other @@ -189,7 +190,7 @@ class EvaluationTests(threadCount: Option[Int]) extends TestSuite { assert(filtered3 == Agg(change, right)) } - utest.test("triangleTask") { + test("triangleTask") { import triangleTask._ val checker = new Checker(triangleTask) @@ -197,7 +198,7 @@ class EvaluationTests(threadCount: Option[Int]) extends TestSuite { checker(left, 1, Agg(), extraEvaled = -1) } - utest.test("multiTerminalGroup") { + test("multiTerminalGroup") { import multiTerminalGroup._ val checker = new Checker(multiTerminalGroup) @@ -205,7 +206,7 @@ class EvaluationTests(threadCount: Option[Int]) extends TestSuite { checker(left, 1, Agg(left), extraEvaled = -1) } - utest.test("multiTerminalBoundary") { + test("multiTerminalBoundary") { import multiTerminalBoundary._ @@ -214,7 +215,7 @@ class EvaluationTests(threadCount: Option[Int]) extends TestSuite { checker(task2, 4, Agg(), extraEvaled = -1, secondRunNoOp = false) } - utest.test("overrideSuperTask") { + test("overrideSuperTask") { // Make sure you can override targets, call their supers, and have the // overridden target be allocated a spot within the overridden/ folder of // the main publicly-available target @@ -234,7 +235,7 @@ class EvaluationTests(threadCount: Option[Int]) extends TestSuite { !overridden.contains("object") ) } - utest.test("overrideSuperCommand") { + test("overrideSuperCommand") { // Make sure you can override commands, call their supers, and have the // overridden command be allocated a spot within the super/ folder of // the main publicly-available command @@ -261,7 +262,7 @@ class EvaluationTests(threadCount: Option[Int]) extends TestSuite { !overridden.contains("object1") ) } - utest.test("nullTasks") { + test("nullTasks") { import nullTasks._ val checker = new Checker(nullTasks) checker(nullTarget1, null, Agg(nullTarget1), extraEvaled = -1) @@ -288,7 +289,7 @@ class EvaluationTests(threadCount: Option[Int]) extends TestSuite { checker(nc4, null, Agg(nc4), extraEvaled = -1, secondRunNoOp = false) } - utest.test("tasksAreUncached") { + test("tasksAreUncached") { // Make sure the tasks `left` and `middle` re-compute every time, while // the target `right` does not // @@ -301,7 +302,7 @@ class EvaluationTests(threadCount: Option[Int]) extends TestSuite { var leftCount = 0 var rightCount = 0 var middleCount = 0 - def up = Task { test.anon() } + def up = Task { TestUtil.test.anon() } def left = Task.Anon { leftCount += 1; up() + 1 } def middle = Task.Anon { middleCount += 1; 100 } def right = Task { rightCount += 1; 10000 } @@ -352,7 +353,7 @@ class EvaluationTests(threadCount: Option[Int]) extends TestSuite { assert(leftCount == 4, middleCount == 4, rightCount == 1) } } - utest.test("stackableOverrides") { + test("stackableOverrides") { // Make sure you can override commands, call their supers, and have the // overridden command be allocated a spot within the super/ folder of // the main publicly-available command @@ -376,7 +377,7 @@ class EvaluationTests(threadCount: Option[Int]) extends TestSuite { ) assert(os.read(checker.evaluator.outPath / "m/f.json").contains(" 6,")) } - utest.test("stackableOverrides2") { + test("stackableOverrides2") { // When the supers have the same name, qualify them until they are distinct import StackableOverrides2._ @@ -398,7 +399,7 @@ class EvaluationTests(threadCount: Option[Int]) extends TestSuite { ) assert(os.read(checker.evaluator.outPath / "m/f.json").contains(" 6,")) } - utest.test("stackableOverrides3") { + test("stackableOverrides3") { // When the supers have the same name, qualify them until they are distinct import StackableOverrides3._ @@ -420,7 +421,7 @@ class EvaluationTests(threadCount: Option[Int]) extends TestSuite { ) assert(os.read(checker.evaluator.outPath / "m/f.json").contains(" 6,")) } - utest.test("privateTasksInMixedTraits") { + test("privateTasksInMixedTraits") { // Make sure we can have private cached targets in different trait with the same name, // and caching still works when these traits are mixed together import PrivateTasksInMixedTraits._ diff --git a/main/init/src/mill/init/InitModule.scala b/main/init/src/mill/init/InitModule.scala index 76b05442767..87bfcd41a04 100644 --- a/main/init/src/mill/init/InitModule.scala +++ b/main/init/src/mill/init/InitModule.scala @@ -57,7 +57,7 @@ trait InitModule extends Module { val extractedPath = T.dest / extractedDirName val conflicting = for { p <- os.walk(extractedPath) - val rel = p.relativeTo(extractedPath) + rel = p.relativeTo(extractedPath) if os.exists(T.workspace / rel) } yield rel @@ -97,7 +97,8 @@ trait InitModule extends Module { private def usingExamples[T](fun: Seq[(ExampleId, ExampleUrl)] => T): Try[T] = Using(getClass.getClassLoader.getResourceAsStream("exampleList.txt")) { exampleList => val reader = upickle.default.reader[Seq[(ExampleId, ExampleUrl)]] - val exampleNames: Seq[(ExampleId, ExampleUrl)] = upickle.default.read(exampleList)(reader) + val exampleNames: Seq[(ExampleId, ExampleUrl)] = + upickle.default.read(exampleList)(using reader) fun(exampleNames) } } diff --git a/main/package.mill b/main/package.mill index 1ef11936329..083d84bce05 100644 --- a/main/package.mill +++ b/main/package.mill @@ -5,6 +5,7 @@ import mill.scalalib._ import mill.contrib.buildinfo.BuildInfo import mill.T import mill.define.Cross +import mill.scalalib.api.ZincWorkerUtil object `package` extends RootModule with build.MillStableScalaModule with BuildInfo { @@ -19,12 +20,19 @@ object `package` extends RootModule with build.MillStableScalaModule with BuildI ivy"guru.nidi:graphviz-java-min-deps:0.18.1" ) - def compileIvyDeps = Agg(build.Deps.scalaReflect(scalaVersion())) + def compileIvyDeps = T { + if (ZincWorkerUtil.isScala3(scalaVersion())) Agg.empty else Agg(build.Deps.scalaReflect(scalaVersion())) + } def buildInfoPackageName = "mill.main" def buildInfoMembers = Seq( BuildInfo.Value("scalaVersion", scalaVersion(), "Scala version used to compile mill core."), + BuildInfo.Value( + "workerScalaVersion213", + build.Deps.scala2Version, + "Scala 2.13 version used by some workers." + ), BuildInfo.Value( "workerScalaVersion212", build.Deps.workerScalaVersion212, @@ -97,9 +105,15 @@ object `package` extends RootModule with build.MillStableScalaModule with BuildI object define extends build.MillStableScalaModule { def moduleDeps = Seq(api, util) - def compileIvyDeps = Agg(build.Deps.scalaReflect(scalaVersion())) + def compileIvyDeps = T { + if (ZincWorkerUtil.isScala3(scalaVersion())) Agg(build.Deps.scalaCompiler(scalaVersion())) + else Agg(build.Deps.scalaReflect(scalaVersion())) + } def ivyDeps = Agg( build.Deps.millModuledefs, + // TODO: somewhere sourcecode is included transitively, + // but we need the latest version to bring the macro improvements. + build.Deps.sourcecode, // Necessary so we can share the JNA classes throughout the build process build.Deps.jna, build.Deps.jnaPlatform, diff --git a/main/resolve/src/mill/resolve/ExpandBraces.scala b/main/resolve/src/mill/resolve/ExpandBraces.scala index fe68892a749..b0d7dc4ecbb 100644 --- a/main/resolve/src/mill/resolve/ExpandBraces.scala +++ b/main/resolve/src/mill/resolve/ExpandBraces.scala @@ -30,7 +30,7 @@ private object ExpandBraces { } def expandBraces(selectorString: String): Either[String, Seq[String]] = { - parse(selectorString, parser(_)) match { + parse(selectorString, parser(using _)) match { case f: Parsed.Failure => Left(s"Parsing exception ${f.msg}") case Parsed.Success(fragmentLists, _) => Right(expandRec(fragmentLists.toList).map(_.mkString)) diff --git a/main/resolve/src/mill/resolve/ParseArgs.scala b/main/resolve/src/mill/resolve/ParseArgs.scala index 36d6a8545b3..ae32b9089bf 100644 --- a/main/resolve/src/mill/resolve/ParseArgs.scala +++ b/main/resolve/src/mill/resolve/ParseArgs.scala @@ -88,7 +88,7 @@ object ParseArgs { def extractSegments(selectorString: String) : Either[String, (Option[Segments], Option[Segments])] = - parse(selectorString, selector(_)) match { + parse(selectorString, selector(using _)) match { case f: Parsed.Failure => Left(s"Parsing exception ${f.msg}") case Parsed.Success(selector, _) => Right(selector) } diff --git a/main/resolve/src/mill/resolve/ResolveNotFoundHandler.scala b/main/resolve/src/mill/resolve/ResolveNotFoundHandler.scala index 792cb920228..7b6551ed1fa 100644 --- a/main/resolve/src/mill/resolve/ResolveNotFoundHandler.scala +++ b/main/resolve/src/mill/resolve/ResolveNotFoundHandler.scala @@ -42,7 +42,7 @@ private object ResolveNotFoundHandler { val search = revSelectorsSoFar.render val lastSearchOpt = for { - Segment.Label(s) <- Option(lastSegment) + case Segment.Label(s) <- Option(lastSegment) if s != "_" && s != "__" possibility <- findMostSimilar(s, allPossibleNames) } yield "__." + possibility @@ -71,13 +71,13 @@ private object ResolveNotFoundHandler { } def errorMsgLabel( - given: String, + `given`: String, possibleMembers: Set[String], prefixSegments: Segments, fullSegments: Segments, allPossibleNames: Set[String] ): String = { - val suggestion = findMostSimilar(given, possibleMembers) match { + val suggestion = findMostSimilar(`given`, possibleMembers) match { case None => hintListLabel(prefixSegments, fullSegments.value.last, allPossibleNames) case Some(similar) => " Did you mean " + diff --git a/main/server/src/mill/main/server/Server.scala b/main/server/src/mill/main/server/Server.scala index 0852c4afef3..9b24dab9d0d 100644 --- a/main/server/src/mill/main/server/Server.scala +++ b/main/server/src/mill/main/server/Server.scala @@ -56,7 +56,8 @@ abstract class Server[T]( serverLog("handling run") try handleRun(sock, initialSystemProperties) catch { - case e: Throwable => serverLog(e + "\n" + e.getStackTrace.mkString("\n")) + case e: Throwable => + serverLog(e.toString + "\n" + e.getStackTrace.mkString("\n")) } finally sock.close(); true } diff --git a/main/src/mill/main/MainModule.scala b/main/src/mill/main/MainModule.scala index df6ddd5a239..1771b96608b 100644 --- a/main/src/mill/main/MainModule.scala +++ b/main/src/mill/main/MainModule.scala @@ -438,7 +438,7 @@ trait MainModule extends BaseModule0 { for { workerSegments <- evaluator.workerCache.keys.toList if allSegments.exists(workerSegments.startsWith) - (_, Val(closeable: AutoCloseable)) <- + case (_, Val(closeable: AutoCloseable)) <- evaluator.mutableWorkerCache.remove(workerSegments) } { closeable.close() @@ -481,7 +481,7 @@ trait MainModule extends BaseModule0 { */ def shutdown(): Command[Unit] = Task.Command(exclusive = true) { Target.log.info("Shutting down Mill server...") - Target.ctx.systemExit(0) + Target.ctx().systemExit(0) () } diff --git a/main/src/mill/main/TokenReaders.scala b/main/src/mill/main/TokenReaders.scala index 60c1b03e5d0..44374818aba 100644 --- a/main/src/mill/main/TokenReaders.scala +++ b/main/src/mill/main/TokenReaders.scala @@ -69,4 +69,5 @@ trait TokenReaders0 { case t: TokensReader.Leftover[_, _] => new LeftoverTaskTokenReader[T](t) } + def given = () // dummy for scala 2/3 compat } diff --git a/main/util/src/mill/util/PrefixLogger.scala b/main/util/src/mill/util/PrefixLogger.scala index 7ee393516a9..2718177b53b 100644 --- a/main/util/src/mill/util/PrefixLogger.scala +++ b/main/util/src/mill/util/PrefixLogger.scala @@ -67,11 +67,11 @@ class PrefixLogger( override def info(s: String): Unit = { reportKey(logPrefixKey) - logger0.info(infoColor(linePrefix) + s) + logger0.info("" + infoColor(linePrefix) + s) } override def error(s: String): Unit = { reportKey(logPrefixKey) - logger0.error(infoColor(linePrefix) + s) + logger0.error("" + infoColor(linePrefix) + s) } override def ticker(s: String): Unit = setPromptDetail(logPrefixKey, s) override def setPromptDetail(key: Seq[String], s: String): Unit = logger0.setPromptDetail(key, s) @@ -90,7 +90,7 @@ class PrefixLogger( override def debug(s: String): Unit = { if (debugEnabled) reportKey(logPrefixKey) - logger0.debug(infoColor(linePrefix) + s) + logger0.debug("" + infoColor(linePrefix) + s) } override def debugEnabled: Boolean = logger0.debugEnabled diff --git a/scalajslib/src/mill/scalajslib/ScalaJSModule.scala b/scalajslib/src/mill/scalajslib/ScalaJSModule.scala index 1d88914ded0..10b412d1741 100644 --- a/scalajslib/src/mill/scalajslib/ScalaJSModule.scala +++ b/scalajslib/src/mill/scalajslib/ScalaJSModule.scala @@ -76,7 +76,7 @@ trait ScalaJSModule extends scalalib.ScalaModule { outer => def scalaJSLinkerClasspath: T[Loose.Agg[PathRef]] = Task { val commonDeps = Seq( - ivy"org.scala-js::scalajs-sbt-test-adapter:${scalaJSVersion()}" + ivy"org.scala-js:scalajs-sbt-test-adapter_2.13:${scalaJSVersion()}" ) val scalajsImportMapDeps = scalaJSVersion() match { case s"1.$n.$_" if n.toIntOption.exists(_ >= 16) && scalaJSImportMap().nonEmpty => @@ -92,7 +92,7 @@ trait ScalaJSModule extends scalalib.ScalaModule { outer => ) case "1" => Seq( - ivy"org.scala-js::scalajs-linker:${scalaJSVersion()}" + ivy"org.scala-js:scalajs-linker_2.13:${scalaJSVersion()}" ) ++ scalaJSJsEnvIvyDeps() } // we need to use the scala-library of the currently running mill diff --git a/scalajslib/test/src/mill/scalajslib/CompileLinkTests.scala b/scalajslib/test/src/mill/scalajslib/CompileLinkTests.scala index efd9e0ac40b..43822fbe870 100644 --- a/scalajslib/test/src/mill/scalajslib/CompileLinkTests.scala +++ b/scalajslib/test/src/mill/scalajslib/CompileLinkTests.scala @@ -44,7 +44,7 @@ object CompileLinkTests extends TestSuite { ) object `test-utest` extends ScalaJSTests with TestModule.Utest { - override def sources = Task.Sources { millSourcePath / "src/utest" } + override def sources = Task.Sources { this.millSourcePath / "src/utest" } val utestVersion = if (ZincWorkerUtil.isScala3(crossScalaVersion)) "0.7.7" else "0.7.5" override def ivyDeps = Agg( ivy"com.lihaoyi::utest::$utestVersion" @@ -52,7 +52,7 @@ object CompileLinkTests extends TestSuite { } object `test-scalatest` extends ScalaJSTests with TestModule.ScalaTest { - override def sources = Task.Sources { millSourcePath / "src/scalatest" } + override def sources = Task.Sources { this.millSourcePath / "src/scalatest" } override def ivyDeps = Agg( ivy"org.scalatest::scalatest::3.1.2" ) @@ -68,7 +68,10 @@ object CompileLinkTests extends TestSuite { object test extends ScalaJSTests with TestModule.Utest } - override lazy val millDiscover = Discover[this.type] + override lazy val millDiscover = { + import mill.main.TokenReaders.given + Discover[this.type] + } } val millSourcePath = os.Path(sys.env("MILL_TEST_RESOURCE_DIR")) / "hello-js-world" diff --git a/scalajslib/test/src/mill/scalajslib/EsModuleRemapTests.scala b/scalajslib/test/src/mill/scalajslib/EsModuleRemapTests.scala index 025fe010679..488c051082e 100644 --- a/scalajslib/test/src/mill/scalajslib/EsModuleRemapTests.scala +++ b/scalajslib/test/src/mill/scalajslib/EsModuleRemapTests.scala @@ -25,7 +25,10 @@ object EsModuleRemapTests extends TestSuite { ESModuleImportMapping.Prefix("@stdlib/linspace", remapTo) ) - override lazy val millDiscover = Discover[this.type] + override lazy val millDiscover = { + import mill.main.TokenReaders.given + Discover[this.type] + } } object OldJsModule extends TestBaseModule with ScalaJSModule { @@ -38,7 +41,10 @@ object EsModuleRemapTests extends TestSuite { ESModuleImportMapping.Prefix("@stdlib/linspace", remapTo) ) - override lazy val millDiscover = Discover[this.type] + override lazy val millDiscover = { + import mill.main.TokenReaders.given + Discover[this.type] + } } val millSourcePath = os.Path(sys.env("MILL_TEST_RESOURCE_DIR")) / "esModuleRemap" diff --git a/scalajslib/test/src/mill/scalajslib/FullOptESModuleTests.scala b/scalajslib/test/src/mill/scalajslib/FullOptESModuleTests.scala index 9da8030876f..417b8a34c34 100644 --- a/scalajslib/test/src/mill/scalajslib/FullOptESModuleTests.scala +++ b/scalajslib/test/src/mill/scalajslib/FullOptESModuleTests.scala @@ -16,7 +16,10 @@ object FullOptESModuleTests extends TestSuite { override def moduleKind = ModuleKind.ESModule } - override lazy val millDiscover = Discover[this.type] + override lazy val millDiscover = { + import mill.main.TokenReaders.given + Discover[this.type] + } } val millSourcePath = os.Path(sys.env("MILL_TEST_RESOURCE_DIR")) / "hello-js-world" diff --git a/scalajslib/test/src/mill/scalajslib/MultiModuleTests.scala b/scalajslib/test/src/mill/scalajslib/MultiModuleTests.scala index 7df52a447d1..4ca8c7d6543 100644 --- a/scalajslib/test/src/mill/scalajslib/MultiModuleTests.scala +++ b/scalajslib/test/src/mill/scalajslib/MultiModuleTests.scala @@ -29,7 +29,10 @@ object MultiModuleTests extends TestSuite { override def millSourcePath = MultiModule.millSourcePath / "shared" } - override lazy val millDiscover = Discover[this.type] + override lazy val millDiscover = { + import mill.main.TokenReaders.given + Discover[this.type] + } } val evaluator = UnitTester(MultiModule, sourcePath) diff --git a/scalajslib/test/src/mill/scalajslib/NodeJSConfigTests.scala b/scalajslib/test/src/mill/scalajslib/NodeJSConfigTests.scala index 730866725fa..3c6b3dca10c 100644 --- a/scalajslib/test/src/mill/scalajslib/NodeJSConfigTests.scala +++ b/scalajslib/test/src/mill/scalajslib/NodeJSConfigTests.scala @@ -43,7 +43,7 @@ object NodeJSConfigTests extends TestSuite { override def jsEnvConfig = Task { JsEnvConfig.NodeJs(args = nodeArgs) } object `test-utest` extends ScalaJSTests with TestModule.Utest { - override def sources = Task.Sources { millSourcePath / "src/utest" } + override def sources = Task.Sources { this.millSourcePath / "src/utest" } override def ivyDeps = Agg( ivy"com.lihaoyi::utest::$utestVersion" ) @@ -51,7 +51,10 @@ object NodeJSConfigTests extends TestSuite { } } - override lazy val millDiscover = Discover[this.type] + override lazy val millDiscover = { + import mill.main.TokenReaders.given + Discover[this.type] + } } val millSourcePath = os.Path(sys.env("MILL_TEST_RESOURCE_DIR")) / "hello-js-world" diff --git a/scalajslib/test/src/mill/scalajslib/OutputPatternsTests.scala b/scalajslib/test/src/mill/scalajslib/OutputPatternsTests.scala index 34dbb7f9f6e..4e4aaca417c 100644 --- a/scalajslib/test/src/mill/scalajslib/OutputPatternsTests.scala +++ b/scalajslib/test/src/mill/scalajslib/OutputPatternsTests.scala @@ -18,7 +18,10 @@ object OutputPatternsTests extends TestSuite { override def scalaJSOutputPatterns = OutputPatterns.fromJSFile("%s.mjs") } - override lazy val millDiscover = Discover[this.type] + override lazy val millDiscover = { + import mill.main.TokenReaders.given + Discover[this.type] + } } val millSourcePath = os.Path(sys.env("MILL_TEST_RESOURCE_DIR")) / "hello-js-world" diff --git a/scalajslib/test/src/mill/scalajslib/ScalaTestsErrorTests.scala b/scalajslib/test/src/mill/scalajslib/ScalaTestsErrorTests.scala index 97d1d3e84e6..6b9f5165af4 100644 --- a/scalajslib/test/src/mill/scalajslib/ScalaTestsErrorTests.scala +++ b/scalajslib/test/src/mill/scalajslib/ScalaTestsErrorTests.scala @@ -16,7 +16,10 @@ object ScalaTestsErrorTests extends TestSuite { override def hierarchyChecks(): Unit = {} } } - override lazy val millDiscover = Discover[this.type] + override lazy val millDiscover = { + import mill.main.TokenReaders.given + Discover[this.type] + } } def tests: Tests = Tests { diff --git a/scalajslib/test/src/mill/scalajslib/SmallModulesForTests.scala b/scalajslib/test/src/mill/scalajslib/SmallModulesForTests.scala index 25c5c7150b2..07a155406f4 100644 --- a/scalajslib/test/src/mill/scalajslib/SmallModulesForTests.scala +++ b/scalajslib/test/src/mill/scalajslib/SmallModulesForTests.scala @@ -15,7 +15,10 @@ object SmallModulesForTests extends TestSuite { override def moduleKind = ModuleKind.ESModule override def moduleSplitStyle = ModuleSplitStyle.SmallModulesFor(List("app")) - override lazy val millDiscover = Discover[this.type] + override lazy val millDiscover = { + import mill.main.TokenReaders.given + Discover[this.type] + } } val millSourcePath = os.Path(sys.env("MILL_TEST_RESOURCE_DIR")) / "small-modules-for" diff --git a/scalajslib/test/src/mill/scalajslib/SourceMapTests.scala b/scalajslib/test/src/mill/scalajslib/SourceMapTests.scala index a52ba7edaa3..b83b11552ba 100644 --- a/scalajslib/test/src/mill/scalajslib/SourceMapTests.scala +++ b/scalajslib/test/src/mill/scalajslib/SourceMapTests.scala @@ -15,7 +15,10 @@ object SourceMapTests extends TestSuite { override def scalaJSSourceMap = false } - override lazy val millDiscover = Discover[this.type] + override lazy val millDiscover = { + import mill.main.TokenReaders.given + Discover[this.type] + } } val millSourcePath = os.Path(sys.env("MILL_TEST_RESOURCE_DIR")) / "hello-js-world" diff --git a/scalajslib/test/src/mill/scalajslib/TopLevelExportsTests.scala b/scalajslib/test/src/mill/scalajslib/TopLevelExportsTests.scala index 6ea7bbaf780..c3eba073ce8 100644 --- a/scalajslib/test/src/mill/scalajslib/TopLevelExportsTests.scala +++ b/scalajslib/test/src/mill/scalajslib/TopLevelExportsTests.scala @@ -13,7 +13,10 @@ object TopLevelExportsTests extends TestSuite { sys.props.getOrElse("TEST_SCALAJS_VERSION", ???) // at least "1.8.0" override def moduleKind = ModuleKind.ESModule - override lazy val millDiscover = Discover[this.type] + override lazy val millDiscover = { + import mill.main.TokenReaders.given + Discover[this.type] + } } val millSourcePath = os.Path(sys.env("MILL_TEST_RESOURCE_DIR")) / "top-level-exports" diff --git a/scalalib/src/mill/scalalib/CrossSbtModule.scala b/scalalib/src/mill/scalalib/CrossSbtModule.scala index 82434dc9d50..e80fce57571 100644 --- a/scalalib/src/mill/scalalib/CrossSbtModule.scala +++ b/scalalib/src/mill/scalalib/CrossSbtModule.scala @@ -2,7 +2,6 @@ package mill.scalalib import mill.api.PathRef import mill.{T, Task} -import mill.scalalib.{CrossModuleBase, SbtModule} import scala.annotation.nowarn diff --git a/scalalib/src/mill/scalalib/Dep.scala b/scalalib/src/mill/scalalib/Dep.scala index b857131bc50..0c1d41bf351 100644 --- a/scalalib/src/mill/scalalib/Dep.scala +++ b/scalalib/src/mill/scalalib/Dep.scala @@ -172,11 +172,11 @@ object Dep { (dep: Dep) => unparse(dep) match { case Some(s) => ujson.Str(s) - case None => upickle.default.writeJs[Dep](dep)(rw0) + case None => upickle.default.writeJs[Dep](dep)(using rw0) }, { case s: ujson.Str => parse(s.value) - case v: ujson.Value => upickle.default.read[Dep](v)(rw0) + case v: ujson.Value => upickle.default.read[Dep](v)(using rw0) } ) @@ -279,7 +279,7 @@ object BoundDep { upickle.default.readwriter[ujson.Value].bimap[BoundDep]( bdep => { Dep.unparse(Dep(bdep.dep, CrossVersion.Constant("", false), bdep.force)) match { - case None => upickle.default.writeJs(bdep)(jsonify0) + case None => upickle.default.writeJs(bdep)(using jsonify0) case Some(s) => ujson.Str(s) } }, @@ -287,7 +287,7 @@ object BoundDep { case ujson.Str(s) => val dep = Dep.parse(s) BoundDep(dep.dep, dep.force) - case v => upickle.default.read[BoundDep](v)(jsonify0) + case v => upickle.default.read[BoundDep](v)(using jsonify0) } ) } diff --git a/scalalib/src/mill/scalalib/JavaModule.scala b/scalalib/src/mill/scalalib/JavaModule.scala index 4d92101bcb4..e7da0fb7760 100644 --- a/scalalib/src/mill/scalalib/JavaModule.scala +++ b/scalalib/src/mill/scalalib/JavaModule.scala @@ -258,8 +258,10 @@ trait JavaModule else compileModuleDepsChecked val deps = (normalDeps ++ compileDeps).distinct val asString = - s"${if (recursive) "Recursive module" - else "Module"} dependencies of ${millModuleSegments.render}:\n\t${deps + s"${ + if (recursive) "Recursive module" + else "Module" + } dependencies of ${millModuleSegments.render}:\n\t${deps .map { dep => dep.millModuleSegments.render ++ (if (compileModuleDepsChecked.contains(dep) || !normalDeps.contains(dep)) " (compile)" diff --git a/scalalib/src/mill/scalalib/PublishModule.scala b/scalalib/src/mill/scalalib/PublishModule.scala index cefe014d11f..eb3a7a22f01 100644 --- a/scalalib/src/mill/scalalib/PublishModule.scala +++ b/scalalib/src/mill/scalalib/PublishModule.scala @@ -62,10 +62,11 @@ trait PublishModule extends JavaModule { outer => } def publishXmlDeps: Task[Agg[Dependency]] = Task.Anon { - val ivyPomDeps = (ivyDeps() ++ mandatoryIvyDeps()).map(resolvePublishDependency().apply(_)) + val ivyPomDeps = + (ivyDeps() ++ mandatoryIvyDeps()).map(resolvePublishDependency.apply().apply(_)) val compileIvyPomDeps = compileIvyDeps() - .map(resolvePublishDependency().apply(_)) + .map(resolvePublishDependency.apply().apply(_)) .filter(!ivyPomDeps.contains(_)) .map(_.copy(scope = Scope.Provided)) @@ -314,6 +315,7 @@ object PublishModule extends ExternalModule with TaskModule { (payload.map { case (p, f) => (p.path, f) }, meta) } object PublishData { + import mill.scalalib.publish.artifactFormat implicit def jsonify: upickle.default.ReadWriter[PublishData] = upickle.default.macroRW } diff --git a/scalalib/src/mill/scalalib/ScalaModule.scala b/scalalib/src/mill/scalalib/ScalaModule.scala index b2b839e5584..c91ef4add7b 100644 --- a/scalalib/src/mill/scalalib/ScalaModule.scala +++ b/scalalib/src/mill/scalalib/ScalaModule.scala @@ -11,7 +11,10 @@ import mainargs.Flag import mill.scalalib.bsp.{BspBuildTarget, BspModule, ScalaBuildTarget, ScalaPlatform} import mill.scalalib.dependency.versions.{ValidVersion, Version} +// this import requires scala-reflect library to be on the classpath +// it was duplicated to scala3-compiler, but is that too powerful to add as a dependency? import scala.reflect.internal.util.ScalaClassLoader + import scala.util.Using /** @@ -56,7 +59,7 @@ trait ScalaModule extends JavaModule with TestModule.ScalaModuleBase { outer => def scalaVersion: T[String] override def mapDependencies: Task[coursier.Dependency => coursier.Dependency] = Task.Anon { - super.mapDependencies().andThen { d: coursier.Dependency => + super.mapDependencies().andThen { (d: coursier.Dependency) => val artifacts = if (ZincWorkerUtil.isDotty(scalaVersion())) Set("dotty-library", "dotty-compiler") diff --git a/scalalib/src/mill/scalalib/ZincWorkerModule.scala b/scalalib/src/mill/scalalib/ZincWorkerModule.scala index 611e2482ef3..eb2ebb6f6b1 100644 --- a/scalalib/src/mill/scalalib/ZincWorkerModule.scala +++ b/scalalib/src/mill/scalalib/ZincWorkerModule.scala @@ -2,7 +2,6 @@ package mill.scalalib import coursier.Repository import mainargs.Flag -import mill.Agg import mill._ import mill.api.{Ctx, FixSizedCache, KeyedLockedCache, PathRef, Result} import mill.define.{ExternalModule, Discover} diff --git a/scalalib/src/mill/scalalib/dependency/updates/UpdatesFinder.scala b/scalalib/src/mill/scalalib/dependency/updates/UpdatesFinder.scala index 863019599d5..3ef00db20d0 100644 --- a/scalalib/src/mill/scalalib/dependency/updates/UpdatesFinder.scala +++ b/scalalib/src/mill/scalalib/dependency/updates/UpdatesFinder.scala @@ -74,5 +74,5 @@ private[dependency] object UpdatesFinder { case (_, _) => false } - private def isUpdate(current: Version) = current < _ + private def isUpdate(current: Version) = current < (_: Version) } diff --git a/scalalib/src/mill/scalalib/dependency/versions/VersionParser.scala b/scalalib/src/mill/scalalib/dependency/versions/VersionParser.scala index d4cb56c816f..ef3757467b8 100644 --- a/scalalib/src/mill/scalalib/dependency/versions/VersionParser.scala +++ b/scalalib/src/mill/scalalib/dependency/versions/VersionParser.scala @@ -27,5 +27,5 @@ private[dependency] object VersionParser { } def parse(text: String): Parsed[(Seq[Long], Seq[String], Seq[String])] = - fastparse.parse(text, versionParser(_)) + fastparse.parse(text, versionParser(using _)) } diff --git a/scalalib/src/mill/scalalib/giter8/Giter8Module.scala b/scalalib/src/mill/scalalib/giter8/Giter8Module.scala index c254512d2c8..e7ab57573e2 100644 --- a/scalalib/src/mill/scalalib/giter8/Giter8Module.scala +++ b/scalalib/src/mill/scalalib/giter8/Giter8Module.scala @@ -16,11 +16,22 @@ trait Giter8Module extends CoursierModule { def init(args: String*): Command[Unit] = Task.Command { T.log.info("Creating a new project...") - val giter8Dependencies = defaultResolver().resolveDeps { - val scalaBinVersion = ZincWorkerUtil.scalaBinaryVersion(BuildInfo.scalaVersion) - Loose.Agg(ivy"org.foundweekends.giter8:giter8_${scalaBinVersion}:0.14.0" - .bindDep("", "", "")) - } + + val giter8Dependencies = + try { + defaultResolver().resolveDeps { + val scalaBinVersion = { + val bv = ZincWorkerUtil.scalaBinaryVersion(BuildInfo.scalaVersion) + if (bv == "3") "2.13" else bv + } + Loose.Agg(ivy"org.foundweekends.giter8:giter8_${scalaBinVersion}:0.14.0" + .bindDep("", "", "")) + } + } catch { + case e: Exception => + T.log.error("Failed to resolve giter8 dependencies\n" + e.getMessage) + throw e + } Jvm.runSubprocess( "giter8.Giter8", diff --git a/scalalib/src/mill/scalalib/publish/VersionScheme.scala b/scalalib/src/mill/scalalib/publish/VersionScheme.scala index 8a53196c37a..6fcba7457b5 100644 --- a/scalalib/src/mill/scalalib/publish/VersionScheme.scala +++ b/scalalib/src/mill/scalalib/publish/VersionScheme.scala @@ -33,5 +33,6 @@ object VersionScheme { case object Strict extends VersionScheme("strict") implicit val rwStrict: ReadWriter[Strict.type] = macroRW - implicit val rwVersionScheme: ReadWriter[VersionScheme.type] = macroRW + // edit @bishabosha: why was it `.type`, I assume it is meant to infer a sum type? + implicit val rwVersionScheme: ReadWriter[VersionScheme /*.type*/ ] = macroRW } diff --git a/scalanativelib/test/src/mill/scalanativelib/CompileRunTests.scala b/scalanativelib/test/src/mill/scalanativelib/CompileRunTests.scala index 5c78f4138b7..121080cb301 100644 --- a/scalanativelib/test/src/mill/scalanativelib/CompileRunTests.scala +++ b/scalanativelib/test/src/mill/scalanativelib/CompileRunTests.scala @@ -60,7 +60,7 @@ object CompileRunTests extends TestSuite { ) object test extends ScalaNativeTests with TestModule.Utest { - override def sources = Task.Sources { millSourcePath / "src/utest" } + override def sources = Task.Sources { this.millSourcePath / "src/utest" } override def ivyDeps = super.ivyDeps() ++ Agg( ivy"com.lihaoyi::utest::$utestVersion" ) diff --git a/scalanativelib/test/src/mill/scalanativelib/ExclusionsTests.scala b/scalanativelib/test/src/mill/scalanativelib/ExclusionsTests.scala index 4947f2962cc..d3698c0c587 100644 --- a/scalanativelib/test/src/mill/scalanativelib/ExclusionsTests.scala +++ b/scalanativelib/test/src/mill/scalanativelib/ExclusionsTests.scala @@ -1,5 +1,6 @@ package mill.scalanativelib +import mill.given import mill.Agg import mill.scalalib._ import mill.define.Discover diff --git a/scalanativelib/test/src/mill/scalanativelib/FeaturesTests.scala b/scalanativelib/test/src/mill/scalanativelib/FeaturesTests.scala index a796c0c9542..9df8455f8fc 100644 --- a/scalanativelib/test/src/mill/scalanativelib/FeaturesTests.scala +++ b/scalanativelib/test/src/mill/scalanativelib/FeaturesTests.scala @@ -1,5 +1,6 @@ package mill.scalanativelib +import mill.given import mill.define.Discover import mill.testkit.UnitTester import mill.testkit.TestBaseModule diff --git a/scalanativelib/test/src/mill/scalanativelib/ScalaTestsErrorTests.scala b/scalanativelib/test/src/mill/scalanativelib/ScalaTestsErrorTests.scala index e8b94860296..48c9d85a270 100644 --- a/scalanativelib/test/src/mill/scalanativelib/ScalaTestsErrorTests.scala +++ b/scalanativelib/test/src/mill/scalanativelib/ScalaTestsErrorTests.scala @@ -17,7 +17,10 @@ object ScalaTestsErrorTests extends TestSuite { } } - override lazy val millDiscover = Discover[this.type] + override lazy val millDiscover = { + import mill.main.TokenReaders.given + Discover[this.type] + } } def tests: Tests = Tests { diff --git a/testkit/src/mill/testkit/IntegrationTester.scala b/testkit/src/mill/testkit/IntegrationTester.scala index 043b530a5c5..982e2ad672f 100644 --- a/testkit/src/mill/testkit/IntegrationTester.scala +++ b/testkit/src/mill/testkit/IntegrationTester.scala @@ -105,7 +105,7 @@ object IntegrationTester { * Returns the raw text of the `.json` metadata file */ def text: String = { - val Seq((List(selector), _)) = + val Seq((Seq(selector), _)) = mill.resolve.ParseArgs.apply(Seq(selector0), SelectMode.Separated).getOrElse(???) val segments = selector._2.getOrElse(Segments()).value.flatMap(_.pathSegments) diff --git a/testkit/src/mill/testkit/UnitTester.scala b/testkit/src/mill/testkit/UnitTester.scala index 90768b2fc13..3740cf3f24f 100644 --- a/testkit/src/mill/testkit/UnitTester.scala +++ b/testkit/src/mill/testkit/UnitTester.scala @@ -189,7 +189,7 @@ class UnitTester( } def close(): Unit = { - for ((_, Val(obsolete: AutoCloseable)) <- evaluator.workerCache.values) { + for (case (_, Val(obsolete: AutoCloseable)) <- evaluator.workerCache.values) { obsolete.close() } } From 2831d97820abe4548960c1c3900abd96865a88da Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Sat, 12 Oct 2024 21:34:52 +0200 Subject: [PATCH 37/47] More Scala 3 compatibility backports (#3724) Turned on `-Xsource:3` and fixed the most obvious syntax issues, silenced the other less-trivial warnings --- build.mill | 7 ++++++- .../output-directory/src/OutputDirectoryLockTests.scala | 2 +- main/eval/test/src/mill/eval/ModuleTests.scala | 2 +- runner/src/mill/runner/MillBuildRootModule.scala | 2 +- scalalib/src/mill/scalalib/CoursierModule.scala | 4 ++-- .../worker/src/mill/scalalib/worker/ZincWorkerImpl.scala | 4 ++-- 6 files changed, 13 insertions(+), 8 deletions(-) diff --git a/build.mill b/build.mill index 365a2225c13..9c2c741026a 100644 --- a/build.mill +++ b/build.mill @@ -411,7 +411,12 @@ trait MillScalaModule extends ScalaModule with MillJavaModule with ScalafixModul "-P:acyclic:force", "-feature", "-Xlint:unused", - "-Xlint:adapted-args" + "-Xlint:adapted-args", + "-Xsource:3", + "-Wconf:msg=inferred type changes:silent", + "-Wconf:msg=case companions no longer extend FunctionN:silent", + "-Wconf:msg=access modifiers for:silent", + "-Wconf:msg=found in a package prefix of the required type:silent" ) def scalacPluginIvyDeps = diff --git a/integration/feature/output-directory/src/OutputDirectoryLockTests.scala b/integration/feature/output-directory/src/OutputDirectoryLockTests.scala index 13799e273aa..a8d6e6532ac 100644 --- a/integration/feature/output-directory/src/OutputDirectoryLockTests.scala +++ b/integration/feature/output-directory/src/OutputDirectoryLockTests.scala @@ -11,7 +11,7 @@ import scala.concurrent.{Await, ExecutionContext, Future} object OutputDirectoryLockTests extends UtestIntegrationTestSuite { private val pool = Executors.newCachedThreadPool() - private implicit val ec = ExecutionContext.fromExecutorService(pool) + private implicit val ec: ExecutionContext = ExecutionContext.fromExecutorService(pool) override def utestAfterAll(): Unit = { pool.shutdown() diff --git a/main/eval/test/src/mill/eval/ModuleTests.scala b/main/eval/test/src/mill/eval/ModuleTests.scala index 72750f5e081..f9160143c0c 100644 --- a/main/eval/test/src/mill/eval/ModuleTests.scala +++ b/main/eval/test/src/mill/eval/ModuleTests.scala @@ -9,7 +9,7 @@ import mill.define.Discover import utest._ object TestExternalModule extends mill.define.ExternalModule with mill.define.TaskModule { - def defaultCommandName = "x" + def defaultCommandName() = "x" def x = Task { 13 } object inner extends mill.Module { def y = Task { 17 } diff --git a/runner/src/mill/runner/MillBuildRootModule.scala b/runner/src/mill/runner/MillBuildRootModule.scala index 10ae284a0af..7e1676849f0 100644 --- a/runner/src/mill/runner/MillBuildRootModule.scala +++ b/runner/src/mill/runner/MillBuildRootModule.scala @@ -226,7 +226,7 @@ abstract class MillBuildRootModule()(implicit ) } - override def bindDependency: Task[Dep => BoundDep] = Task.Anon { dep: Dep => + override def bindDependency: Task[Dep => BoundDep] = Task.Anon { (dep: Dep) => super.bindDependency().apply(dep).exclude(resolveDepsExclusions(): _*) } diff --git a/scalalib/src/mill/scalalib/CoursierModule.scala b/scalalib/src/mill/scalalib/CoursierModule.scala index 4594916e4a0..a952acb62eb 100644 --- a/scalalib/src/mill/scalalib/CoursierModule.scala +++ b/scalalib/src/mill/scalalib/CoursierModule.scala @@ -23,7 +23,7 @@ trait CoursierModule extends mill.Module { * Bind a dependency ([[Dep]]) to the actual module contetxt (e.g. the scala version and the platform suffix) * @return The [[BoundDep]] */ - def bindDependency: Task[Dep => BoundDep] = Task.Anon { dep: Dep => + def bindDependency: Task[Dep => BoundDep] = Task.Anon { (dep: Dep) => BoundDep((resolveCoursierDependency(): @nowarn).apply(dep), dep.force) } @@ -80,7 +80,7 @@ trait CoursierModule extends mill.Module { * Map dependencies before resolving them. * Override this to customize the set of dependencies. */ - def mapDependencies: Task[Dependency => Dependency] = Task.Anon { d: Dependency => d } + def mapDependencies: Task[Dependency => Dependency] = Task.Anon { (d: Dependency) => d } /** * The repositories used to resolved dependencies with [[resolveDeps()]]. diff --git a/scalalib/worker/src/mill/scalalib/worker/ZincWorkerImpl.scala b/scalalib/worker/src/mill/scalalib/worker/ZincWorkerImpl.scala index 5b7c373992a..f8bb3ccdea5 100644 --- a/scalalib/worker/src/mill/scalalib/worker/ZincWorkerImpl.scala +++ b/scalalib/worker/src/mill/scalalib/worker/ZincWorkerImpl.scala @@ -144,7 +144,7 @@ class ZincWorkerImpl( compilerClasspath, scalacPluginClasspath, Seq() - ) { compilers: Compilers => + ) { (compilers: Compilers) => if (ZincWorkerUtil.isDotty(scalaVersion) || ZincWorkerUtil.isScala3Milestone(scalaVersion)) { // dotty 0.x and scala 3 milestones use the dotty-doc tool val dottydocClass = @@ -388,7 +388,7 @@ class ZincWorkerImpl( compilerClasspath = compilerClasspath, scalacPluginClasspath = scalacPluginClasspath, javacOptions = javacOptions - ) { compilers: Compilers => + ) { (compilers: Compilers) => compileInternal( upstreamCompileOutput = upstreamCompileOutput, sources = sources, From 91ff5f8e10fddf33d88ec2233f100f30a76e6844 Mon Sep 17 00:00:00 2001 From: Nikita Ogorodnikov <4046447+0xnm@users.noreply.github.com> Date: Mon, 14 Oct 2024 06:24:22 +0200 Subject: [PATCH 38/47] Add Kotest support for Kotlin/JS (#3723) This PR brings [Kotest](https://github.com/kotest/kotest) test engine support for Kotlin/JS target test execution. Replaces #3710. --------- Co-authored-by: Li Haoyi --- .../kotlinlib/web/3-hello-kotlinjs/build.mill | 35 +++--- .../foo/test/src/foo/HelloTests.kt | 15 --- .../{foo => }/src/foo/Hello.kt | 4 +- .../test/src/foo/HelloTests.kt | 11 ++ .../mill/kotlinlib/js/KotlinJSModule.scala | 109 +++++++++++++++--- .../foo/test/src/foo/HelloKotestTests.kt | 16 +++ ...ests.kt => HelloKotlinTestPackageTests.kt} | 0 .../js/KotlinJSKotestModuleTests.scala | 56 +++++++++ ...KotlinJSKotlinTestPackageModuleTests.scala | 55 +++++++++ .../js/KotlinJSTestModuleTests.scala | 47 -------- 10 files changed, 251 insertions(+), 97 deletions(-) delete mode 100644 example/kotlinlib/web/3-hello-kotlinjs/foo/test/src/foo/HelloTests.kt rename example/kotlinlib/web/3-hello-kotlinjs/{foo => }/src/foo/Hello.kt (81%) create mode 100644 example/kotlinlib/web/3-hello-kotlinjs/test/src/foo/HelloTests.kt create mode 100644 kotlinlib/test/resources/kotlin-js/foo/test/src/foo/HelloKotestTests.kt rename kotlinlib/test/resources/kotlin-js/foo/test/src/foo/{HelloTests.kt => HelloKotlinTestPackageTests.kt} (100%) create mode 100644 kotlinlib/test/src/mill/kotlinlib/js/KotlinJSKotestModuleTests.scala create mode 100644 kotlinlib/test/src/mill/kotlinlib/js/KotlinJSKotlinTestPackageModuleTests.scala delete mode 100644 kotlinlib/test/src/mill/kotlinlib/js/KotlinJSTestModuleTests.scala diff --git a/example/kotlinlib/web/3-hello-kotlinjs/build.mill b/example/kotlinlib/web/3-hello-kotlinjs/build.mill index cd69d50d31c..800fb8db491 100644 --- a/example/kotlinlib/web/3-hello-kotlinjs/build.mill +++ b/example/kotlinlib/web/3-hello-kotlinjs/build.mill @@ -1,46 +1,47 @@ -// KotlinJS support on Mill is still Work In Progress (WIP). As of time of writing it -// Node.js/Webpack test runners and reporting, etc. +// Kotlin/JS support on Mill is still Work In Progress (WIP). As of time of writing it +// supports Node.js, but lacks support of Browser, Webpack, test runners, reporting, etc. // // The example below demonstrates only the minimal compilation, running, and testing of -// a single KotlinJS module using a single third-party dependency. For more details in -// fully developing KotlinJS support, see the following ticket: +// a single Kotlin/JS module using a single third-party dependency. For more details in +// fully developing Kotlin/JS support, see the following ticket: // // * https://github.com/com-lihaoyi/mill/issues/3611 package build import mill._, kotlinlib._, kotlinlib.js._ -object foo extends KotlinJSModule { +object `package` extends RootModule with KotlinJSModule { def moduleKind = ModuleKind.ESModule def kotlinVersion = "1.9.25" def kotlinJSRunTarget = Some(RunTarget.Node) def ivyDeps = Agg( ivy"org.jetbrains.kotlinx:kotlinx-html-js:0.11.0", ) - object test extends KotlinJSModule with KotlinJSKotlinXTests + object test extends KotlinJSModule with KotestTests } /** Usage -> mill foo.run -Compiling 1 Kotlin sources to .../out/foo/compile.dest/classes... +> mill run +Compiling 1 Kotlin sources to .../out/compile.dest/classes...

Hello World

stringifiedJsObject: ["hello","world","!"] -> mill foo.test # Test is incorrect, `foo.test`` fails -Compiling 1 Kotlin sources to .../out/foo/test/compile.dest/classes... -Linking IR to .../out/foo/test/linkBinary.dest/binaries -produce executable: .../out/foo/test/linkBinary.dest/binaries +> mill test # Test is incorrect, `test` fails +Compiling 1 Kotlin sources to .../out/test/compile.dest/classes... +Linking IR to .../out/test/linkBinary.dest/binaries +produce executable: .../out/test/linkBinary.dest/binaries +... +error: ...AssertionFailedError: expected:<"

Hello World Wrong

"> but was:<"

Hello World

... ... -error: AssertionError: Expected <

Hello World

>, actual <

Hello World Wrong

>. -> cat out/foo/test/linkBinary.dest/binaries/test.js # Generated javascript on disk -...assertEquals_0(..., '

Hello World Wrong<\/h1>');... +> cat out/test/linkBinary.dest/binaries/test.js # Generated javascript on disk +...shouldBe(..., '

Hello World Wrong<\/h1>');... ... -> sed -i.bak 's/Hello World Wrong/Hello World/g' foo/test/src/foo/HelloTests.kt +> sed -i.bak 's/Hello World Wrong/Hello World/g' test/src/foo/HelloTests.kt -> mill foo.test # passes after fixing test +> mill test # passes after fixing test */ diff --git a/example/kotlinlib/web/3-hello-kotlinjs/foo/test/src/foo/HelloTests.kt b/example/kotlinlib/web/3-hello-kotlinjs/foo/test/src/foo/HelloTests.kt deleted file mode 100644 index fc33731c87a..00000000000 --- a/example/kotlinlib/web/3-hello-kotlinjs/foo/test/src/foo/HelloTests.kt +++ /dev/null @@ -1,15 +0,0 @@ -package foo - -import kotlin.test.Test -import kotlin.test.assertEquals - -class HelloTests { - - @Test - fun testHello() { - val result = hello() - assertEquals(result.trim(), "

Hello World Wrong

") - result - } -} - diff --git a/example/kotlinlib/web/3-hello-kotlinjs/foo/src/foo/Hello.kt b/example/kotlinlib/web/3-hello-kotlinjs/src/foo/Hello.kt similarity index 81% rename from example/kotlinlib/web/3-hello-kotlinjs/foo/src/foo/Hello.kt rename to example/kotlinlib/web/3-hello-kotlinjs/src/foo/Hello.kt index b3348c98139..47d4a851c36 100644 --- a/example/kotlinlib/web/3-hello-kotlinjs/foo/src/foo/Hello.kt +++ b/example/kotlinlib/web/3-hello-kotlinjs/src/foo/Hello.kt @@ -12,5 +12,5 @@ fun main() { } fun hello(): String { - return createHTML().h1 { +"Hello World" }.toString() -} \ No newline at end of file + return createHTML(prettyPrint = false).h1 { text("Hello World") }.toString() +} diff --git a/example/kotlinlib/web/3-hello-kotlinjs/test/src/foo/HelloTests.kt b/example/kotlinlib/web/3-hello-kotlinjs/test/src/foo/HelloTests.kt new file mode 100644 index 00000000000..bb7cc3c4535 --- /dev/null +++ b/example/kotlinlib/web/3-hello-kotlinjs/test/src/foo/HelloTests.kt @@ -0,0 +1,11 @@ +package foo + +import io.kotest.core.spec.style.FunSpec +import io.kotest.matchers.shouldBe + +class HelloTests: FunSpec({ + test("hello") { + val result = hello() + result shouldBe "

Hello World Wrong

" + } +}) diff --git a/kotlinlib/src/mill/kotlinlib/js/KotlinJSModule.scala b/kotlinlib/src/mill/kotlinlib/js/KotlinJSModule.scala index 07f284a4a37..2acfda58be5 100644 --- a/kotlinlib/src/mill/kotlinlib/js/KotlinJSModule.scala +++ b/kotlinlib/src/mill/kotlinlib/js/KotlinJSModule.scala @@ -122,14 +122,17 @@ trait KotlinJSModule extends KotlinModule { outer => } kotlinJSRunTarget() match { - case Some(RunTarget.Node) => Jvm.runSubprocess( + case Some(RunTarget.Node) => { + val testBinaryPath = (linkResult.path / s"${moduleName()}.${moduleKind.extension}") + .toIO.getAbsolutePath + Jvm.runSubprocess( commandArgs = Seq( - "node", - (linkResult.path / s"${moduleName()}.${moduleKind.extension}").toIO.getAbsolutePath - ) ++ args().value, + "node" + ) ++ args().value ++ Seq(testBinaryPath), envArgs = T.env, workingDir = T.dest ) + } case Some(x) => T.log.error(s"Run target $x is not supported") case None => @@ -379,16 +382,16 @@ trait KotlinJSModule extends KotlinModule { outer => // these 2 exist to ignore values added to the display name in case of the cross-modules // we already have cross-modules in the paths, so we don't need them here - private def moduleName() = millModuleSegments.value - .filter(_.isInstanceOf[Segment.Label]) - .map(_.asInstanceOf[Segment.Label]) - .last - .value + private def fullModuleNameSegments() = { + millModuleSegments.value + .collect { case label: Segment.Label => label.value } match { + case Nil => Seq("root") + case segments => segments + } + } - private def fullModuleName() = millModuleSegments.value - .filter(_.isInstanceOf[Segment.Label]) - .map(_.asInstanceOf[Segment.Label].value) - .mkString("-") + private def moduleName() = fullModuleNameSegments().last + private def fullModuleName() = fullModuleNameSegments().mkString("-") // **NOTE**: This logic may (and probably is) be incomplete private def isKotlinJsLibrary(path: os.Path)(implicit ctx: mill.api.Ctx): Boolean = { @@ -417,8 +420,45 @@ trait KotlinJSModule extends KotlinModule { outer => // region Tests module + /** + * Generic trait to run tests for Kotlin/JS which doesn't specify test + * framework. For the particular implementation see [[KotlinTestPackageTests]] or [[KotestTests]]. + */ trait KotlinJSTests extends KotlinTests with KotlinJSModule { + // region private + + // TODO may be optimized if there is a single folder for all modules + // but may be problematic if modules use different NPM packages versions + private def nodeModulesDir = Task(persistent = true) { + PathRef(T.dest) + } + + // NB: for the packages below it is important to use specific version + // otherwise with random versions there is a possibility to have conflict + // between the versions of the shared transitive deps + private def mochaModule = Task { + val workingDir = nodeModulesDir().path + Jvm.runSubprocess( + commandArgs = Seq("npm", "install", "mocha@10.2.0"), + envArgs = T.env, + workingDir = workingDir + ) + PathRef(workingDir / "node_modules" / "mocha" / "bin" / "mocha.js") + } + + private def sourceMapSupportModule = Task { + val workingDir = nodeModulesDir().path + Jvm.runSubprocess( + commandArgs = Seq("npm", "install", "source-map-support@0.5.21"), + envArgs = T.env, + workingDir = nodeModulesDir().path + ) + PathRef(workingDir / "node_modules" / "source-map-support" / "register.js") + } + + // endregion + override def testFramework = "" override def kotlinJSBinaryKind: T[Option[BinaryKind]] = Some(BinaryKind.Executable) @@ -435,16 +475,53 @@ trait KotlinJSModule extends KotlinModule { outer => globSelectors: Task[Seq[String]] ): Task[(String, Seq[TestResult])] = Task.Anon { // This is a terrible hack, but it works - run()() + run(Task.Anon { + Args(args() ++ Seq( + // TODO this is valid only for the NodeJS target. Once browser support is + // added, need to have different argument handling + "--require", + sourceMapSupportModule().path.toString(), + mochaModule().path.toString() + )) + })() ("", Seq.empty[TestResult]) } + + override def kotlinJSRunTarget: T[Option[RunTarget]] = Some(RunTarget.Node) } - trait KotlinJSKotlinXTests extends KotlinJSTests { + /** + * Run tests for Kotlin/JS target using `kotlin.test` package. + */ + trait KotlinTestPackageTests extends KotlinJSTests { override def ivyDeps = Agg( ivy"org.jetbrains.kotlin:kotlin-test-js:${kotlinVersion()}" ) - override def kotlinJSRunTarget: T[Option[RunTarget]] = Some(RunTarget.Node) + } + + /** + * Run tests for Kotlin/JS target using Kotest framework. + */ + trait KotestTests extends KotlinJSTests { + + def kotestVersion: T[String] = "5.9.1" + + private def kotestProcessor = Task { + defaultResolver().resolveDeps( + Agg( + ivy"io.kotest:kotest-framework-multiplatform-plugin-embeddable-compiler-jvm:${kotestVersion()}" + ) + ).head + } + + override def kotlincOptions = super.kotlincOptions() ++ Seq( + s"-Xplugin=${kotestProcessor().path}" + ) + + override def ivyDeps = Agg( + ivy"io.kotest:kotest-framework-engine-js:${kotestVersion()}", + ivy"io.kotest:kotest-assertions-core-js:${kotestVersion()}" + ) } // endregion diff --git a/kotlinlib/test/resources/kotlin-js/foo/test/src/foo/HelloKotestTests.kt b/kotlinlib/test/resources/kotlin-js/foo/test/src/foo/HelloKotestTests.kt new file mode 100644 index 00000000000..56b3def1857 --- /dev/null +++ b/kotlinlib/test/resources/kotlin-js/foo/test/src/foo/HelloKotestTests.kt @@ -0,0 +1,16 @@ +package foo + +import bar.getString +import io.kotest.core.spec.style.FunSpec +import io.kotest.matchers.shouldBe + +class HelloTests: FunSpec({ + + test("success") { + getString() shouldBe "Hello, world" + } + + test("failure") { + getString() shouldBe "Not hello, world" + } +}) diff --git a/kotlinlib/test/resources/kotlin-js/foo/test/src/foo/HelloTests.kt b/kotlinlib/test/resources/kotlin-js/foo/test/src/foo/HelloKotlinTestPackageTests.kt similarity index 100% rename from kotlinlib/test/resources/kotlin-js/foo/test/src/foo/HelloTests.kt rename to kotlinlib/test/resources/kotlin-js/foo/test/src/foo/HelloKotlinTestPackageTests.kt diff --git a/kotlinlib/test/src/mill/kotlinlib/js/KotlinJSKotestModuleTests.scala b/kotlinlib/test/src/mill/kotlinlib/js/KotlinJSKotestModuleTests.scala new file mode 100644 index 00000000000..b9327e55576 --- /dev/null +++ b/kotlinlib/test/src/mill/kotlinlib/js/KotlinJSKotestModuleTests.scala @@ -0,0 +1,56 @@ +package mill +package kotlinlib.js + +import mill.eval.EvaluatorPaths +import mill.testkit.{TestBaseModule, UnitTester} +import utest.{assert, TestSuite, Tests, test} + +object KotlinJSKotestModuleTests extends TestSuite { + + private val resourcePath = os.Path(sys.env("MILL_TEST_RESOURCE_DIR")) / "kotlin-js" + + private val kotlinVersion = "1.9.25" + + object module extends TestBaseModule { + + object bar extends KotlinJSModule { + def kotlinVersion = KotlinJSKotestModuleTests.kotlinVersion + } + + object foo extends KotlinJSModule { + def kotlinVersion = KotlinJSKotestModuleTests.kotlinVersion + override def moduleDeps = Seq(module.bar) + + object test extends KotlinJSModule with KotestTests { + override def allSourceFiles = super.allSourceFiles() + .filter(!_.path.toString().endsWith("HelloKotlinTestPackageTests.kt")) + } + } + } + + private def testEval() = UnitTester(module, resourcePath) + + def tests: Tests = Tests { + + test("run tests") { + val eval = testEval() + + val command = module.foo.test.test() + val Left(_) = eval.apply(command) + + // temporary, because we are running run() task, it won't be test.log, but run.log + val log = + os.read(EvaluatorPaths.resolveDestPaths(eval.outPath, command).log / ".." / "run.log") + assert( + log.contains( + "AssertionFailedError: expected:<\"Not hello, world\"> but was:<\"Hello, world\">" + ), + log.contains("1 passing"), + log.contains("1 failing"), + // verify that source map is applied, otherwise all stack entries will point to .js + log.contains("HelloKotestTests.kt:") + ) + } + } + +} diff --git a/kotlinlib/test/src/mill/kotlinlib/js/KotlinJSKotlinTestPackageModuleTests.scala b/kotlinlib/test/src/mill/kotlinlib/js/KotlinJSKotlinTestPackageModuleTests.scala new file mode 100644 index 00000000000..fdd9b2039a0 --- /dev/null +++ b/kotlinlib/test/src/mill/kotlinlib/js/KotlinJSKotlinTestPackageModuleTests.scala @@ -0,0 +1,55 @@ +package mill +package kotlinlib +package js + +import mill.eval.EvaluatorPaths +import mill.testkit.{TestBaseModule, UnitTester} +import utest.{assert, TestSuite, Tests, test} + +object KotlinJSKotlinTestPackageModuleTests extends TestSuite { + + private val resourcePath = os.Path(sys.env("MILL_TEST_RESOURCE_DIR")) / "kotlin-js" + + private val kotlinVersion = "1.9.25" + + object module extends TestBaseModule { + + object bar extends KotlinJSModule { + def kotlinVersion = KotlinJSKotlinTestPackageModuleTests.kotlinVersion + } + + object foo extends KotlinJSModule { + def kotlinVersion = KotlinJSKotlinTestPackageModuleTests.kotlinVersion + override def moduleDeps = Seq(module.bar) + + object test extends KotlinJSModule with KotlinTestPackageTests { + override def allSourceFiles = super.allSourceFiles() + .filter(!_.path.toString().endsWith("HelloKotestTests.kt")) + } + } + } + + private def testEval() = UnitTester(module, resourcePath) + + def tests: Tests = Tests { + + test("run tests") { + val eval = testEval() + + val command = module.foo.test.test() + val Left(_) = eval.apply(command) + + // temporary, because we are running run() task, it won't be test.log, but run.log + val log = + os.read(EvaluatorPaths.resolveDestPaths(eval.outPath, command).log / ".." / "run.log") + assert( + log.contains("AssertionError: Expected , actual ."), + log.contains("1 passing"), + log.contains("1 failing"), + // verify that source map is applied, otherwise all stack entries will point to .js + log.contains("HelloKotlinTestPackageTests.kt:") + ) + } + } + +} diff --git a/kotlinlib/test/src/mill/kotlinlib/js/KotlinJSTestModuleTests.scala b/kotlinlib/test/src/mill/kotlinlib/js/KotlinJSTestModuleTests.scala deleted file mode 100644 index 37e2dd138ad..00000000000 --- a/kotlinlib/test/src/mill/kotlinlib/js/KotlinJSTestModuleTests.scala +++ /dev/null @@ -1,47 +0,0 @@ -package mill -package kotlinlib -package js - -import mill.eval.EvaluatorPaths -import mill.testkit.{TestBaseModule, UnitTester} -import utest.{TestSuite, Tests, test} - -object KotlinJSTestModuleTests extends TestSuite { - - private val resourcePath = os.Path(sys.env("MILL_TEST_RESOURCE_DIR")) / "kotlin-js" - - private val kotlinVersion = "1.9.25" - - object module extends TestBaseModule { - - object bar extends KotlinJSModule { - def kotlinVersion = KotlinJSTestModuleTests.kotlinVersion - } - - object foo extends KotlinJSModule { - def kotlinVersion = KotlinJSTestModuleTests.kotlinVersion - override def moduleDeps = Seq(module.bar) - - object test extends KotlinJSModule with KotlinJSKotlinXTests - } - } - - private def testEval() = UnitTester(module, resourcePath) - - def tests: Tests = Tests { - - test("run tests") { - val eval = testEval() - - val command = module.foo.test.test() - val Left(_) = eval.apply(command) - - // temporary, because we are running run() task, it won't be test.log, but run.log - val log = EvaluatorPaths.resolveDestPaths(eval.outPath, command).log / ".." / "run.log" - assert( - os.read(log).contains("AssertionError: Expected , actual .") - ) - } - } - -} From 3542fdd9cb9f3f8391d859b228a26d9d53d0cde1 Mon Sep 17 00:00:00 2001 From: Nikita Ogorodnikov <4046447+0xnm@users.noreply.github.com> Date: Mon, 14 Oct 2024 06:43:15 +0200 Subject: [PATCH 39/47] Add Kotlin/JS webapp example (#3725) Point 4 in #3611: add a sample of Kotlin/JS frontend+ Ktor backend. --------- Co-authored-by: 0xnm <0xnm@users.noreply.github.com> Co-authored-by: Li Haoyi --- .../web/4-webapp-kotlinjs/build.mill | 75 ++++ .../4-webapp-kotlinjs/client/src/ClientApp.kt | 80 ++++ .../resources/webapp/index.css | 393 ++++++++++++++++++ .../web/4-webapp-kotlinjs/src/WebApp.kt | 210 ++++++++++ .../4-webapp-kotlinjs/test/src/WebAppTests.kt | 28 ++ .../client/src/ClientApp.scala | 2 +- 6 files changed, 787 insertions(+), 1 deletion(-) create mode 100644 example/kotlinlib/web/4-webapp-kotlinjs/build.mill create mode 100644 example/kotlinlib/web/4-webapp-kotlinjs/client/src/ClientApp.kt create mode 100644 example/kotlinlib/web/4-webapp-kotlinjs/resources/webapp/index.css create mode 100644 example/kotlinlib/web/4-webapp-kotlinjs/src/WebApp.kt create mode 100644 example/kotlinlib/web/4-webapp-kotlinjs/test/src/WebAppTests.kt diff --git a/example/kotlinlib/web/4-webapp-kotlinjs/build.mill b/example/kotlinlib/web/4-webapp-kotlinjs/build.mill new file mode 100644 index 00000000000..8b091b63b86 --- /dev/null +++ b/example/kotlinlib/web/4-webapp-kotlinjs/build.mill @@ -0,0 +1,75 @@ +package build + +import mill._, kotlinlib._, kotlinlib.js._ + +object `package` extends RootModule with KotlinModule { + + def kotlinVersion = "1.9.24" + def ktorVersion = "2.3.12" + def kotlinHtmlVersion = "0.11.0" + + def mainClass = Some("webapp.WebApp") + + def ivyDeps = Agg( + ivy"io.ktor:ktor-server-core-jvm:$ktorVersion", + ivy"io.ktor:ktor-server-netty-jvm:$ktorVersion", + ivy"io.ktor:ktor-server-html-builder-jvm:$ktorVersion", + ) + + def resources = Task { + os.makeDir(Task.dest / "webapp") + val jsPath = client.linkBinary().classes.path + // Move root.js[.map]into the proper filesystem position + // in the resource folder for the web server code to pick up + os.copy(jsPath / "client.js", Task.dest / "webapp/client.js") + os.copy(jsPath / "client.js.map", Task.dest / "webapp/client.js.map") + super.resources() ++ Seq(PathRef(Task.dest)) + } + + object test extends KotlinTests with TestModule.Junit5 { + def ivyDeps = super.ivyDeps() ++ Agg( + ivy"io.kotest:kotest-runner-junit5-jvm:5.9.1", + ivy"io.ktor:ktor-server-test-host-jvm:$ktorVersion" + ) + } + + object client extends KotlinJSModule { + def kotlinVersion = "1.9.24" + + override def splitPerModule = false + + def ivyDeps = Agg( + ivy"org.jetbrains.kotlinx:kotlinx-html-js:$kotlinHtmlVersion", + ) + } +} + +// A minimal example of a Kotlin backend server wired up with a Kotlin/JS +// front-end. The backend code is identical to the <<_todomvc_web_app>> example, but +// we replace the `main.js` client side code with the Javascript output of +// `ClientApp.kt`. +// +// Note that the client-side Kotlin code is the simplest 1-to-1 translation of +// the original Javascript, using `kotlinx.browser`, as this example is intended to +// demonstrate the `build.mill` config in Mill. A real codebase is likely to use +// Javascript or Kotlin/JS UI frameworks to manage the UI, but those are beyond the +// scope of this example. + +/** Usage + +> ./mill test +...webapp.WebAppTestssimpleRequest ... + +> ./mill runBackground + +> curl http://localhost:8092 +...What needs to be done... +... + +> curl http://localhost:8092/static/client.js +...bindEvent(this, 'todo-all', '/list/all', 'all')... +... + +> ./mill clean runBackground + +*/ diff --git a/example/kotlinlib/web/4-webapp-kotlinjs/client/src/ClientApp.kt b/example/kotlinlib/web/4-webapp-kotlinjs/client/src/ClientApp.kt new file mode 100644 index 00000000000..ee9c57d8458 --- /dev/null +++ b/example/kotlinlib/web/4-webapp-kotlinjs/client/src/ClientApp.kt @@ -0,0 +1,80 @@ +package client + +import kotlinx.browser.document +import kotlinx.browser.window +import org.w3c.dom.Element +import org.w3c.dom.HTMLInputElement +import org.w3c.dom.asList +import org.w3c.dom.events.KeyboardEvent +import org.w3c.dom.get +import org.w3c.fetch.RequestInit + +object ClientApp { + + private var state = "all" + + private val todoApp: Element + get() = checkNotNull(document.getElementsByClassName("todoapp")[0]) + + private fun postFetchUpdate(url: String) { + window + .fetch(url, RequestInit(method = "POST")) + .then { it.text() } + .then { text -> + todoApp.innerHTML = text + initListeners() + } + } + + private fun bindEvent(cls: String, url: String, endState: String? = null) { + document.getElementsByClassName(cls)[0] + ?.addEventListener("click", { + postFetchUpdate(url) + if (endState != null) state = endState + } + ) + } + + private fun bindIndexedEvent(cls: String, block: (String) -> String) { + for (elem in document.getElementsByClassName(cls).asList()) { + elem.addEventListener( + "click", + { postFetchUpdate(block(elem.getAttribute("data-todo-index")!!)) } + ) + } + } + + fun initListeners() { + bindIndexedEvent("destroy") { + "/delete/$state/$it" + } + bindIndexedEvent("toggle") { + "/toggle/$state/$it" + } + bindEvent("toggle-all", "/toggle-all/$state") + bindEvent("todo-all", "/list/all", "all") + bindEvent("todo-active", "/list/active", "active") + bindEvent("todo-completed", "/list/completed", "completed") + bindEvent("clear-completed", "/clear-completed/$state") + + val newTodoInput = document.getElementsByClassName("new-todo")[0] as HTMLInputElement + newTodoInput.addEventListener( + "keydown", + { + check(it is KeyboardEvent) + if (it.keyCode == 13) { + window + .fetch("/add/$state", RequestInit(method = "POST", body = newTodoInput.value)) + .then { it.text() } + .then { text -> + newTodoInput.value = "" + todoApp.innerHTML = text + initListeners() + } + } + } + ) + } +} + +fun main(args: Array) = ClientApp.initListeners() diff --git a/example/kotlinlib/web/4-webapp-kotlinjs/resources/webapp/index.css b/example/kotlinlib/web/4-webapp-kotlinjs/resources/webapp/index.css new file mode 100644 index 00000000000..f731c2205d3 --- /dev/null +++ b/example/kotlinlib/web/4-webapp-kotlinjs/resources/webapp/index.css @@ -0,0 +1,393 @@ +@charset 'utf-8'; + +html, +body { + margin: 0; + padding: 0; +} + +button { + margin: 0; + padding: 0; + border: 0; + background: none; + font-size: 100%; + vertical-align: baseline; + font-family: inherit; + font-weight: inherit; + color: inherit; + -webkit-appearance: none; + appearance: none; + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; +} + +body { + font: 14px 'Helvetica Neue', Helvetica, Arial, sans-serif; + line-height: 1.4em; + background: #f5f5f5; + color: #111111; + min-width: 230px; + max-width: 550px; + margin: 0 auto; + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; + font-weight: 300; +} + +.hidden { + display: none; +} + +.todoapp { + background: #fff; + margin: 130px 0 40px 0; + position: relative; + box-shadow: 0 2px 4px 0 rgba(0, 0, 0, 0.2), + 0 25px 50px 0 rgba(0, 0, 0, 0.1); +} + +.todoapp input::-webkit-input-placeholder { + font-style: italic; + font-weight: 400; + color: rgba(0, 0, 0, 0.4); +} + +.todoapp input::-moz-placeholder { + font-style: italic; + font-weight: 400; + color: rgba(0, 0, 0, 0.4); +} + +.todoapp input::input-placeholder { + font-style: italic; + font-weight: 400; + color: rgba(0, 0, 0, 0.4); +} + +.todoapp h1 { + position: absolute; + top: -140px; + width: 100%; + font-size: 80px; + font-weight: 200; + text-align: center; + color: #b83f45; + -webkit-text-rendering: optimizeLegibility; + -moz-text-rendering: optimizeLegibility; + text-rendering: optimizeLegibility; +} + +.new-todo, +.edit { + position: relative; + margin: 0; + width: 100%; + font-size: 24px; + font-family: inherit; + font-weight: inherit; + line-height: 1.4em; + color: inherit; + padding: 6px; + border: 1px solid #999; + box-shadow: inset 0 -1px 5px 0 rgba(0, 0, 0, 0.2); + box-sizing: border-box; + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; +} + +.new-todo { + padding: 16px 16px 16px 60px; + height: 65px; + border: none; + background: rgba(0, 0, 0, 0.003); + box-shadow: inset 0 -2px 1px rgba(0,0,0,0.03); +} + +.main { + position: relative; + z-index: 2; + border-top: 1px solid #e6e6e6; +} + +.toggle-all { + width: 1px; + height: 1px; + border: none; /* Mobile Safari */ + opacity: 0; + position: absolute; + right: 100%; + bottom: 100%; +} + +.toggle-all + label { + display: flex; + align-items: center; + justify-content: center; + width: 45px; + height: 65px; + font-size: 0; + position: absolute; + top: -65px; + left: -0; +} + +.toggle-all + label:before { + content: '❯'; + display: inline-block; + font-size: 22px; + color: #949494; + padding: 10px 27px 10px 27px; + -webkit-transform: rotate(90deg); + transform: rotate(90deg); +} + +.toggle-all:checked + label:before { + color: #484848; +} + +.todo-list { + margin: 0; + padding: 0; + list-style: none; +} + +.todo-list li { + position: relative; + font-size: 24px; + border-bottom: 1px solid #ededed; +} + +.todo-list li:last-child { + border-bottom: none; +} + +.todo-list li.editing { + border-bottom: none; + padding: 0; +} + +.todo-list li.editing .edit { + display: block; + width: calc(100% - 43px); + padding: 12px 16px; + margin: 0 0 0 43px; +} + +.todo-list li.editing .view { + display: none; +} + +.todo-list li .toggle { + text-align: center; + width: 40px; + /* auto, since non-WebKit browsers doesn't support input styling */ + height: auto; + position: absolute; + top: 0; + bottom: 0; + margin: auto 0; + border: none; /* Mobile Safari */ + -webkit-appearance: none; + appearance: none; +} + +.todo-list li .toggle { + opacity: 0; +} + +.todo-list li .toggle + label { + /* + Firefox requires `#` to be escaped - https://bugzilla.mozilla.org/show_bug.cgi?id=922433 + IE and Edge requires *everything* to be escaped to render, so we do that instead of just the `#` - https://developer.microsoft.com/en-us/microsoft-edge/platform/issues/7157459/ + */ + background-image: url('data:image/svg+xml;utf8,%3Csvg%20xmlns%3D%22http%3A//www.w3.org/2000/svg%22%20width%3D%2240%22%20height%3D%2240%22%20viewBox%3D%22-10%20-18%20100%20135%22%3E%3Ccircle%20cx%3D%2250%22%20cy%3D%2250%22%20r%3D%2250%22%20fill%3D%22none%22%20stroke%3D%22%23949494%22%20stroke-width%3D%223%22/%3E%3C/svg%3E'); + background-repeat: no-repeat; + background-position: center left; +} + +.todo-list li .toggle:checked + label { + background-image: url('data:image/svg+xml;utf8,%3Csvg%20xmlns%3D%22http%3A%2F%2Fwww.w3.org%2F2000%2Fsvg%22%20width%3D%2240%22%20height%3D%2240%22%20viewBox%3D%22-10%20-18%20100%20135%22%3E%3Ccircle%20cx%3D%2250%22%20cy%3D%2250%22%20r%3D%2250%22%20fill%3D%22none%22%20stroke%3D%22%2359A193%22%20stroke-width%3D%223%22%2F%3E%3Cpath%20fill%3D%22%233EA390%22%20d%3D%22M72%2025L42%2071%2027%2056l-4%204%2020%2020%2034-52z%22%2F%3E%3C%2Fsvg%3E'); +} + +.todo-list li label { + overflow-wrap: break-word; + padding: 15px 15px 15px 60px; + display: block; + line-height: 1.2; + transition: color 0.4s; + font-weight: 400; + color: #484848; +} + +.todo-list li.completed label { + color: #949494; + text-decoration: line-through; +} + +.todo-list li .destroy { + display: none; + position: absolute; + top: 0; + right: 10px; + bottom: 0; + width: 40px; + height: 40px; + margin: auto 0; + font-size: 30px; + color: #949494; + transition: color 0.2s ease-out; +} + +.todo-list li .destroy:hover, +.todo-list li .destroy:focus { + color: #C18585; +} + +.todo-list li .destroy:after { + content: '×'; + display: block; + height: 100%; + line-height: 1.1; +} + +.todo-list li:hover .destroy { + display: block; +} + +.todo-list li .edit { + display: none; +} + +.todo-list li.editing:last-child { + margin-bottom: -1px; +} + +.footer { + padding: 10px 15px; + height: 20px; + text-align: center; + font-size: 15px; + border-top: 1px solid #e6e6e6; +} + +.footer:before { + content: ''; + position: absolute; + right: 0; + bottom: 0; + left: 0; + height: 50px; + overflow: hidden; + box-shadow: 0 1px 1px rgba(0, 0, 0, 0.2), + 0 8px 0 -3px #f6f6f6, + 0 9px 1px -3px rgba(0, 0, 0, 0.2), + 0 16px 0 -6px #f6f6f6, + 0 17px 2px -6px rgba(0, 0, 0, 0.2); +} + +.todo-count { + float: left; + text-align: left; +} + +.todo-count strong { + font-weight: 300; +} + +.filters { + margin: 0; + padding: 0; + list-style: none; + position: absolute; + right: 0; + left: 0; +} + +.filters li { + display: inline; +} + +.filters li a { + color: inherit; + margin: 3px; + padding: 3px 7px; + text-decoration: none; + border: 1px solid transparent; + border-radius: 3px; +} + +.filters li a:hover { + border-color: #DB7676; +} + +.filters li a.selected { + border-color: #CE4646; +} + +.clear-completed, +html .clear-completed:active { + float: right; + position: relative; + line-height: 19px; + text-decoration: none; + cursor: pointer; +} + +.clear-completed:hover { + text-decoration: underline; +} + +.info { + margin: 65px auto 0; + color: #4d4d4d; + font-size: 11px; + text-shadow: 0 1px 0 rgba(255, 255, 255, 0.5); + text-align: center; +} + +.info p { + line-height: 1; +} + +.info a { + color: inherit; + text-decoration: none; + font-weight: 400; +} + +.info a:hover { + text-decoration: underline; +} + +/* + Hack to remove background from Mobile Safari. + Can't use it globally since it destroys checkboxes in Firefox +*/ +@media screen and (-webkit-min-device-pixel-ratio:0) { + .toggle-all, + .todo-list li .toggle { + background: none; + } + + .todo-list li .toggle { + height: 40px; + } +} + +@media (max-width: 430px) { + .footer { + height: 50px; + } + + .filters { + bottom: 10px; + } +} + +:focus, +.toggle:focus + label, +.toggle-all:focus + label { + box-shadow: 0 0 2px 2px #CF7D7D; + outline: 0; +} \ No newline at end of file diff --git a/example/kotlinlib/web/4-webapp-kotlinjs/src/WebApp.kt b/example/kotlinlib/web/4-webapp-kotlinjs/src/WebApp.kt new file mode 100644 index 00000000000..b43e12ac068 --- /dev/null +++ b/example/kotlinlib/web/4-webapp-kotlinjs/src/WebApp.kt @@ -0,0 +1,210 @@ +package webapp + +import io.ktor.server.application.* +import io.ktor.server.engine.* +import io.ktor.server.html.* +import io.ktor.server.http.content.* +import io.ktor.server.netty.* +import io.ktor.server.request.* +import io.ktor.server.response.* +import io.ktor.server.routing.* +import io.ktor.server.util.* +import kotlinx.html.* +import kotlinx.html.stream.createHTML + +object WebApp { + data class Todo(val checked: Boolean, val text: String) + + private val todos = mutableListOf( + Todo(true, "Get started with Cask"), + Todo(false, "Profit!") + ) + + private fun FlowContent.list(state: String) = renderBody(state) + + private fun FlowContent.add(state: String, text: String) { + todos.add(Todo(false, text)) + renderBody(state) + } + + private fun FlowContent.delete(state: String, index: Int) { + todos.removeAt(index) + renderBody(state) + } + + private fun FlowContent.toggle(state: String, index: Int) { + todos[index] = todos[index].let { + it.copy(checked = !it.checked) + } + renderBody(state) + } + + private fun FlowContent.clearCompleted(state: String) { + todos.removeAll { it.checked } + renderBody(state) + } + + private fun FlowContent.toggleAll(state: String) { + val next = todos.any { !it.checked } + for (item in todos.withIndex()) { + todos[item.index] = item.value.copy(checked = next) + } + renderBody(state) + } + + private fun FlowContent.renderBody(state: String) { + val filteredTodos = when (state) { + "all" -> todos.withIndex() + "active" -> todos.withIndex().filter { !it.value.checked } + "completed" -> todos.withIndex().filter { it.value.checked } + else -> throw IllegalStateException("Unknown state=$state") + } + div { + header(classes = "header") { + h1{ + +"todos" + } + input(classes = "new-todo") { + placeholder = "What needs to be done?" + } + } + section(classes = "main") { + input( + classes = "toggle-all", + type = InputType.checkBox + ) { + id = "toggle-all" + checked = todos.any { it.checked } + } + label { + htmlFor = "toggle-all" + +"Mark all as complete" + } + ul(classes = "todo-list") { + filteredTodos.forEach { (index, todo) -> + li(classes = if (todo.checked) "completed" else "") { + div(classes = "view") { + form { + input(classes = "toggle", type = InputType.checkBox) { + checked = todo.checked + attributes["data-todo-index"] = index.toString() + } + label { +todo.text } + } + form { + button(classes = "destroy") { + attributes["data-todo-index"] = index.toString() + } + } + } + input(classes = "edit") { + value = todo.text + } + } + } + } + } + footer(classes = "footer") { + span(classes = "todo-count") { + strong { + +todos.filter { !it.checked }.size.toString() + } + +" items left" + } + ul(classes = "filters") { + li(classes = "todo-all") { + a(classes = if (state == "all") "selected" else "") { +"All" } + } + li(classes = "todo-active") { + a(classes = if (state == "active") "selected" else "") { +"Active" } + } + li(classes = "todo-completed") { + a(classes = if (state == "completed") "selected" else "") { +"Completed" } + } + } + button(classes = "clear-completed") { +"Clear completed" } + } + } + } + + private fun HTML.renderIndex() { + head { + meta(charset = "utf-8") + meta(name = "viewport", content = "width=device-width, initial-scale=1") + title("Template • TodoMVC") + link(rel = "stylesheet", href = "/static/index.css") + } + body { + section(classes = "todoapp") { + renderBody("all") + } + footer(classes = "info") { + p { +"Double-click to edit a todo" } + p { + +"Created by " + a(href = "http://todomvc.com") { +"Li Haoyi" } + } + p { + +"Part of " + a(href = "http://todomvc.com") { +"TodoMVC" } + } + } + script(src = "/static/client.js", block = {}) + } + } + + fun configureRoutes(app: Application) { + with(app) { + routing { + get("/") { + call.respondHtml { + renderIndex() + } + } + post("/toggle-all/{state}") { + call.respondText { + createHTML().div { toggleAll(call.parameters.getOrFail("state")) } + } + } + post("/clear-completed/{state}") { + call.respondText { + createHTML().div { clearCompleted(call.parameters.getOrFail("state")) } + } + } + post("/toggle/{state}/{index}") { + call.parameters.run { + call.respondText { + createHTML().div { toggle(getOrFail("state"), getOrFail("index")) } + } + } + } + post("/delete/{state}/{index}") { + call.parameters.run { + call.respondText { + createHTML().div { delete(getOrFail("state"), getOrFail("index")) } + } + } + } + post("/add/{state}") { + val requestText = call.receiveText() + call.respondText { + createHTML().div { add(call.parameters.getOrFail("state"), requestText) } + } + } + post("/list/{state}") { + call.respondText { + createHTML().div { list(call.parameters.getOrFail("state")) } + } + } + staticResources("/static", "webapp") + } + } + } + + @JvmStatic + fun main(args: Array) { + embeddedServer(Netty, port = 8092, host = "0.0.0.0") { + configureRoutes(this) + }.start(wait = true) + } +} diff --git a/example/kotlinlib/web/4-webapp-kotlinjs/test/src/WebAppTests.kt b/example/kotlinlib/web/4-webapp-kotlinjs/test/src/WebAppTests.kt new file mode 100644 index 00000000000..0786f28c639 --- /dev/null +++ b/example/kotlinlib/web/4-webapp-kotlinjs/test/src/WebAppTests.kt @@ -0,0 +1,28 @@ +package webapp + +import io.kotest.core.spec.style.FunSpec +import io.kotest.matchers.shouldBe +import io.kotest.matchers.string.shouldContain +import io.ktor.client.HttpClient +import io.ktor.client.request.get +import io.ktor.client.statement.bodyAsText +import io.ktor.http.HttpStatusCode +import io.ktor.server.testing.testApplication + +class WebAppTests : FunSpec({ + + suspend fun withServer(f: suspend HttpClient.() -> Unit) { + testApplication { + application { WebApp.configureRoutes(this) } + client.use { client -> f(client) } + } + } + + test("simpleRequest") { + withServer { + val response = get("/") + response.status shouldBe HttpStatusCode.OK + response.bodyAsText() shouldContain "What needs to be done?" + } + } +}) diff --git a/example/scalalib/web/4-webapp-scalajs/client/src/ClientApp.scala b/example/scalalib/web/4-webapp-scalajs/client/src/ClientApp.scala index f26a43f3409..77c4b4b6e7f 100644 --- a/example/scalalib/web/4-webapp-scalajs/client/src/ClientApp.scala +++ b/example/scalalib/web/4-webapp-scalajs/client/src/ClientApp.scala @@ -41,7 +41,7 @@ object ClientApp{ bindIndexedEvent("toggle", index => s"/toggle/$state/$index") bindEvent("toggle-all", s"/toggle-all/$state", None) bindEvent("todo-all", s"/list/all", Some("all")) - bindEvent("todo-active", s"/list/all", Some("active")) + bindEvent("todo-active", s"/list/active", Some("active")) bindEvent("todo-completed", s"/list/completed", Some("completed")) bindEvent("clear-completed", s"/clear-completed/$state", None) From c9695a1bee53d55cedf5ed3fdaeb1504da2b9db0 Mon Sep 17 00:00:00 2001 From: Nikita Ogorodnikov <4046447+0xnm@users.noreply.github.com> Date: Mon, 14 Oct 2024 07:10:59 +0200 Subject: [PATCH 40/47] Add shared JVM/JS Kotlin code example (#3728) This PR addresses task 5 from #3611 by adding and example of code sharing between JVM and JS Kotlin targets. I also simply copied `PlatformScalaModule` -> `PlatformKotlinModule`, because type aliasing won't work: doc is different (and need to extend from `KotlinModule` still). --------- Co-authored-by: 0xnm <0xnm@users.noreply.github.com> Co-authored-by: Li Haoyi --- .../web/5-webapp-kotlinjs-shared/build.mill | 118 ++++++ .../client/src/ClientApp.kt | 89 ++++ .../resources/logback.xml | 11 + .../resources/webapp/index.css | 393 ++++++++++++++++++ .../shared/src/Shared.kt | 77 ++++ .../5-webapp-kotlinjs-shared/src/WebApp.kt | 125 ++++++ .../test/src/WebAppTests.kt | 28 ++ .../mill/kotlinlib/PlatformKotlinModule.scala | 37 ++ 8 files changed, 878 insertions(+) create mode 100644 example/kotlinlib/web/5-webapp-kotlinjs-shared/build.mill create mode 100644 example/kotlinlib/web/5-webapp-kotlinjs-shared/client/src/ClientApp.kt create mode 100644 example/kotlinlib/web/5-webapp-kotlinjs-shared/resources/logback.xml create mode 100644 example/kotlinlib/web/5-webapp-kotlinjs-shared/resources/webapp/index.css create mode 100644 example/kotlinlib/web/5-webapp-kotlinjs-shared/shared/src/Shared.kt create mode 100644 example/kotlinlib/web/5-webapp-kotlinjs-shared/src/WebApp.kt create mode 100644 example/kotlinlib/web/5-webapp-kotlinjs-shared/test/src/WebAppTests.kt create mode 100644 kotlinlib/src/mill/kotlinlib/PlatformKotlinModule.scala diff --git a/example/kotlinlib/web/5-webapp-kotlinjs-shared/build.mill b/example/kotlinlib/web/5-webapp-kotlinjs-shared/build.mill new file mode 100644 index 00000000000..22888ec6264 --- /dev/null +++ b/example/kotlinlib/web/5-webapp-kotlinjs-shared/build.mill @@ -0,0 +1,118 @@ +package build +import mill._, kotlinlib._, kotlinlib.js._ + +trait AppKotlinModule extends KotlinModule { + def kotlinVersion = "1.9.25" +} + +trait AppKotlinJSModule extends AppKotlinModule with KotlinJSModule + +object `package` extends RootModule with AppKotlinModule { + + def ktorVersion = "2.3.12" + def kotlinHtmlVersion = "0.11.0" + def kotlinxSerializationVersion = "1.6.3" + + def mainClass = Some("webapp.WebApp") + + def moduleDeps = Seq(shared.jvm) + + def ivyDeps = Agg( + ivy"io.ktor:ktor-server-core-jvm:$ktorVersion", + ivy"io.ktor:ktor-server-netty-jvm:$ktorVersion", + ivy"io.ktor:ktor-server-html-builder-jvm:$ktorVersion", + ivy"io.ktor:ktor-server-content-negotiation-jvm:$ktorVersion", + ivy"io.ktor:ktor-serialization-kotlinx-json-jvm:$ktorVersion", + ivy"ch.qos.logback:logback-classic:1.5.8", + ) + + def resources = Task { + os.makeDir(Task.dest / "webapp") + val jsPath = client.linkBinary().classes.path + os.copy(jsPath / "client.js", Task.dest / "webapp/client.js") + os.copy(jsPath / "client.js.map", Task.dest / "webapp/client.js.map") + super.resources() ++ Seq(PathRef(Task.dest)) + } + + object test extends KotlinTests with TestModule.Junit5 { + def ivyDeps = super.ivyDeps() ++ Agg( + ivy"io.kotest:kotest-runner-junit5-jvm:5.9.1", + ivy"io.ktor:ktor-server-test-host-jvm:$ktorVersion" + ) + } + + object shared extends Module { + + trait SharedModule extends AppKotlinModule with PlatformKotlinModule { + def processors = Task { + defaultResolver().resolveDeps( + Agg( + ivy"org.jetbrains.kotlin:kotlin-serialization-compiler-plugin:${kotlinVersion()}" + ) + ) + } + + def kotlincOptions = super.kotlincOptions() ++ Seq( + s"-Xplugin=${processors().head.path}" + ) + } + + object jvm extends SharedModule { + def ivyDeps = super.ivyDeps() ++ Agg( + ivy"org.jetbrains.kotlinx:kotlinx-html-jvm:$kotlinHtmlVersion", + ivy"org.jetbrains.kotlinx:kotlinx-serialization-json-jvm:$kotlinxSerializationVersion", + ) + } + object js extends SharedModule with AppKotlinJSModule { + def ivyDeps = super.ivyDeps() ++ Agg( + ivy"org.jetbrains.kotlinx:kotlinx-html-js:$kotlinHtmlVersion", + ivy"org.jetbrains.kotlinx:kotlinx-serialization-json-js:$kotlinxSerializationVersion", + ) + } + } + + object client extends AppKotlinJSModule { + def splitPerModule = false + def moduleDeps = Seq(shared.js) + def ivyDeps = Agg( + ivy"org.jetbrains.kotlinx:kotlinx-html-js:$kotlinHtmlVersion", + ivy"org.jetbrains.kotlinx:kotlinx-serialization-json-js:$kotlinxSerializationVersion", + ) + } +} + +// A Kotlin/JVM backend server wired up with a Kotlin/JS front-end, with a +// `shared` module containing code that is used in both client and server. +// Rather than the server sending HTML for the initial page load and HTML for +// page updates, it sends HTML for the initial load and JSON for page updates +// which is then rendered into HTML on the client. +// +// The JSON serialization logic and HTML generation logic in the `shared` module +// is shared between client and server, and uses libraries like `kotlinx-serialization` and +// `kotlinx-html` which work on both Kotlin/JVM and Kotlin/JS. This allows us to freely +// move code between the client and server, without worrying about what +// platform or language the code was originally implemented in. +// +// This is a minimal example of shared code compiled to Kotlin/JVM and Kotlin/JS, +// running on both client and server, meant for illustrating the build +// configuration. A full exploration of client-server code sharing techniques +// is beyond the scope of this example. + +/** Usage + +> ./mill test +...webapp.WebAppTestssimpleRequest ... + +> ./mill runBackground + +> curl http://localhost:8093 +...What needs to be done... +... + +> curl http://localhost:8093/static/client.js +...kotlin.js... +... + +> ./mill clean runBackground + +*/ diff --git a/example/kotlinlib/web/5-webapp-kotlinjs-shared/client/src/ClientApp.kt b/example/kotlinlib/web/5-webapp-kotlinjs-shared/client/src/ClientApp.kt new file mode 100644 index 00000000000..ff590b30e7e --- /dev/null +++ b/example/kotlinlib/web/5-webapp-kotlinjs-shared/client/src/ClientApp.kt @@ -0,0 +1,89 @@ +package client + +import kotlinx.browser.document +import kotlinx.browser.window +import kotlinx.html.div +import kotlinx.html.stream.createHTML +import kotlinx.serialization.* +import kotlinx.serialization.json.* +import org.w3c.dom.Element +import org.w3c.dom.HTMLInputElement +import org.w3c.dom.asList +import org.w3c.dom.events.KeyboardEvent +import org.w3c.dom.get +import org.w3c.fetch.RequestInit +import shared.* + +object ClientApp { + + private var state = "all" + + private val todoApp: Element + get() = checkNotNull(document.getElementsByClassName("todoapp")[0]) + + private fun postFetchUpdate(url: String) { + window + .fetch(url, RequestInit(method = "POST")) + .then { it.text() } + .then { text -> + todoApp.innerHTML = createHTML().div { + renderBody(Json.decodeFromString>(text), state) + } + initListeners() + } + } + + private fun bindEvent(cls: String, url: String, endState: String? = null) { + document.getElementsByClassName(cls)[0] + ?.addEventListener("click", { + postFetchUpdate(url) + if (endState != null) state = endState + } + ) + } + + private fun bindIndexedEvent(cls: String, block: (String) -> String) { + for (elem in document.getElementsByClassName(cls).asList()) { + elem.addEventListener( + "click", + { postFetchUpdate(block(elem.getAttribute("data-todo-index")!!)) } + ) + } + } + + fun initListeners() { + bindIndexedEvent("destroy") { + "/delete/$state/$it" + } + bindIndexedEvent("toggle") { + "/toggle/$state/$it" + } + bindEvent("toggle-all", "/toggle-all/$state") + bindEvent("todo-all", "/list/all", "all") + bindEvent("todo-active", "/list/active", "active") + bindEvent("todo-completed", "/list/completed", "completed") + bindEvent("clear-completed", "/clear-completed/$state") + + val newTodoInput = document.getElementsByClassName("new-todo")[0] as HTMLInputElement + newTodoInput.addEventListener( + "keydown", + { + check(it is KeyboardEvent) + if (it.keyCode == 13) { + window + .fetch("/add/$state", RequestInit(method = "POST", body = newTodoInput.value)) + .then { it.text() } + .then { text -> + newTodoInput.value = "" + todoApp.innerHTML = createHTML().div { + renderBody(Json.decodeFromString>(text), state) + } + initListeners() + } + } + } + ) + } +} + +fun main(args: Array) = ClientApp.initListeners() diff --git a/example/kotlinlib/web/5-webapp-kotlinjs-shared/resources/logback.xml b/example/kotlinlib/web/5-webapp-kotlinjs-shared/resources/logback.xml new file mode 100644 index 00000000000..d330b77b822 --- /dev/null +++ b/example/kotlinlib/web/5-webapp-kotlinjs-shared/resources/logback.xml @@ -0,0 +1,11 @@ + + + + %d{YYYY-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + + + + + + + diff --git a/example/kotlinlib/web/5-webapp-kotlinjs-shared/resources/webapp/index.css b/example/kotlinlib/web/5-webapp-kotlinjs-shared/resources/webapp/index.css new file mode 100644 index 00000000000..f731c2205d3 --- /dev/null +++ b/example/kotlinlib/web/5-webapp-kotlinjs-shared/resources/webapp/index.css @@ -0,0 +1,393 @@ +@charset 'utf-8'; + +html, +body { + margin: 0; + padding: 0; +} + +button { + margin: 0; + padding: 0; + border: 0; + background: none; + font-size: 100%; + vertical-align: baseline; + font-family: inherit; + font-weight: inherit; + color: inherit; + -webkit-appearance: none; + appearance: none; + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; +} + +body { + font: 14px 'Helvetica Neue', Helvetica, Arial, sans-serif; + line-height: 1.4em; + background: #f5f5f5; + color: #111111; + min-width: 230px; + max-width: 550px; + margin: 0 auto; + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; + font-weight: 300; +} + +.hidden { + display: none; +} + +.todoapp { + background: #fff; + margin: 130px 0 40px 0; + position: relative; + box-shadow: 0 2px 4px 0 rgba(0, 0, 0, 0.2), + 0 25px 50px 0 rgba(0, 0, 0, 0.1); +} + +.todoapp input::-webkit-input-placeholder { + font-style: italic; + font-weight: 400; + color: rgba(0, 0, 0, 0.4); +} + +.todoapp input::-moz-placeholder { + font-style: italic; + font-weight: 400; + color: rgba(0, 0, 0, 0.4); +} + +.todoapp input::input-placeholder { + font-style: italic; + font-weight: 400; + color: rgba(0, 0, 0, 0.4); +} + +.todoapp h1 { + position: absolute; + top: -140px; + width: 100%; + font-size: 80px; + font-weight: 200; + text-align: center; + color: #b83f45; + -webkit-text-rendering: optimizeLegibility; + -moz-text-rendering: optimizeLegibility; + text-rendering: optimizeLegibility; +} + +.new-todo, +.edit { + position: relative; + margin: 0; + width: 100%; + font-size: 24px; + font-family: inherit; + font-weight: inherit; + line-height: 1.4em; + color: inherit; + padding: 6px; + border: 1px solid #999; + box-shadow: inset 0 -1px 5px 0 rgba(0, 0, 0, 0.2); + box-sizing: border-box; + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; +} + +.new-todo { + padding: 16px 16px 16px 60px; + height: 65px; + border: none; + background: rgba(0, 0, 0, 0.003); + box-shadow: inset 0 -2px 1px rgba(0,0,0,0.03); +} + +.main { + position: relative; + z-index: 2; + border-top: 1px solid #e6e6e6; +} + +.toggle-all { + width: 1px; + height: 1px; + border: none; /* Mobile Safari */ + opacity: 0; + position: absolute; + right: 100%; + bottom: 100%; +} + +.toggle-all + label { + display: flex; + align-items: center; + justify-content: center; + width: 45px; + height: 65px; + font-size: 0; + position: absolute; + top: -65px; + left: -0; +} + +.toggle-all + label:before { + content: '❯'; + display: inline-block; + font-size: 22px; + color: #949494; + padding: 10px 27px 10px 27px; + -webkit-transform: rotate(90deg); + transform: rotate(90deg); +} + +.toggle-all:checked + label:before { + color: #484848; +} + +.todo-list { + margin: 0; + padding: 0; + list-style: none; +} + +.todo-list li { + position: relative; + font-size: 24px; + border-bottom: 1px solid #ededed; +} + +.todo-list li:last-child { + border-bottom: none; +} + +.todo-list li.editing { + border-bottom: none; + padding: 0; +} + +.todo-list li.editing .edit { + display: block; + width: calc(100% - 43px); + padding: 12px 16px; + margin: 0 0 0 43px; +} + +.todo-list li.editing .view { + display: none; +} + +.todo-list li .toggle { + text-align: center; + width: 40px; + /* auto, since non-WebKit browsers doesn't support input styling */ + height: auto; + position: absolute; + top: 0; + bottom: 0; + margin: auto 0; + border: none; /* Mobile Safari */ + -webkit-appearance: none; + appearance: none; +} + +.todo-list li .toggle { + opacity: 0; +} + +.todo-list li .toggle + label { + /* + Firefox requires `#` to be escaped - https://bugzilla.mozilla.org/show_bug.cgi?id=922433 + IE and Edge requires *everything* to be escaped to render, so we do that instead of just the `#` - https://developer.microsoft.com/en-us/microsoft-edge/platform/issues/7157459/ + */ + background-image: url('data:image/svg+xml;utf8,%3Csvg%20xmlns%3D%22http%3A//www.w3.org/2000/svg%22%20width%3D%2240%22%20height%3D%2240%22%20viewBox%3D%22-10%20-18%20100%20135%22%3E%3Ccircle%20cx%3D%2250%22%20cy%3D%2250%22%20r%3D%2250%22%20fill%3D%22none%22%20stroke%3D%22%23949494%22%20stroke-width%3D%223%22/%3E%3C/svg%3E'); + background-repeat: no-repeat; + background-position: center left; +} + +.todo-list li .toggle:checked + label { + background-image: url('data:image/svg+xml;utf8,%3Csvg%20xmlns%3D%22http%3A%2F%2Fwww.w3.org%2F2000%2Fsvg%22%20width%3D%2240%22%20height%3D%2240%22%20viewBox%3D%22-10%20-18%20100%20135%22%3E%3Ccircle%20cx%3D%2250%22%20cy%3D%2250%22%20r%3D%2250%22%20fill%3D%22none%22%20stroke%3D%22%2359A193%22%20stroke-width%3D%223%22%2F%3E%3Cpath%20fill%3D%22%233EA390%22%20d%3D%22M72%2025L42%2071%2027%2056l-4%204%2020%2020%2034-52z%22%2F%3E%3C%2Fsvg%3E'); +} + +.todo-list li label { + overflow-wrap: break-word; + padding: 15px 15px 15px 60px; + display: block; + line-height: 1.2; + transition: color 0.4s; + font-weight: 400; + color: #484848; +} + +.todo-list li.completed label { + color: #949494; + text-decoration: line-through; +} + +.todo-list li .destroy { + display: none; + position: absolute; + top: 0; + right: 10px; + bottom: 0; + width: 40px; + height: 40px; + margin: auto 0; + font-size: 30px; + color: #949494; + transition: color 0.2s ease-out; +} + +.todo-list li .destroy:hover, +.todo-list li .destroy:focus { + color: #C18585; +} + +.todo-list li .destroy:after { + content: '×'; + display: block; + height: 100%; + line-height: 1.1; +} + +.todo-list li:hover .destroy { + display: block; +} + +.todo-list li .edit { + display: none; +} + +.todo-list li.editing:last-child { + margin-bottom: -1px; +} + +.footer { + padding: 10px 15px; + height: 20px; + text-align: center; + font-size: 15px; + border-top: 1px solid #e6e6e6; +} + +.footer:before { + content: ''; + position: absolute; + right: 0; + bottom: 0; + left: 0; + height: 50px; + overflow: hidden; + box-shadow: 0 1px 1px rgba(0, 0, 0, 0.2), + 0 8px 0 -3px #f6f6f6, + 0 9px 1px -3px rgba(0, 0, 0, 0.2), + 0 16px 0 -6px #f6f6f6, + 0 17px 2px -6px rgba(0, 0, 0, 0.2); +} + +.todo-count { + float: left; + text-align: left; +} + +.todo-count strong { + font-weight: 300; +} + +.filters { + margin: 0; + padding: 0; + list-style: none; + position: absolute; + right: 0; + left: 0; +} + +.filters li { + display: inline; +} + +.filters li a { + color: inherit; + margin: 3px; + padding: 3px 7px; + text-decoration: none; + border: 1px solid transparent; + border-radius: 3px; +} + +.filters li a:hover { + border-color: #DB7676; +} + +.filters li a.selected { + border-color: #CE4646; +} + +.clear-completed, +html .clear-completed:active { + float: right; + position: relative; + line-height: 19px; + text-decoration: none; + cursor: pointer; +} + +.clear-completed:hover { + text-decoration: underline; +} + +.info { + margin: 65px auto 0; + color: #4d4d4d; + font-size: 11px; + text-shadow: 0 1px 0 rgba(255, 255, 255, 0.5); + text-align: center; +} + +.info p { + line-height: 1; +} + +.info a { + color: inherit; + text-decoration: none; + font-weight: 400; +} + +.info a:hover { + text-decoration: underline; +} + +/* + Hack to remove background from Mobile Safari. + Can't use it globally since it destroys checkboxes in Firefox +*/ +@media screen and (-webkit-min-device-pixel-ratio:0) { + .toggle-all, + .todo-list li .toggle { + background: none; + } + + .todo-list li .toggle { + height: 40px; + } +} + +@media (max-width: 430px) { + .footer { + height: 50px; + } + + .filters { + bottom: 10px; + } +} + +:focus, +.toggle:focus + label, +.toggle-all:focus + label { + box-shadow: 0 0 2px 2px #CF7D7D; + outline: 0; +} \ No newline at end of file diff --git a/example/kotlinlib/web/5-webapp-kotlinjs-shared/shared/src/Shared.kt b/example/kotlinlib/web/5-webapp-kotlinjs-shared/shared/src/Shared.kt new file mode 100644 index 00000000000..905453ffb49 --- /dev/null +++ b/example/kotlinlib/web/5-webapp-kotlinjs-shared/shared/src/Shared.kt @@ -0,0 +1,77 @@ +package shared + +import kotlinx.html.* +import kotlinx.html.stream.createHTML +import kotlinx.serialization.Serializable + +@Serializable +data class Todo(val checked: Boolean, val text: String) + +fun FlowContent.renderBody(todos: List, state: String) { + val filteredTodos = when (state) { + "all" -> todos.withIndex() + "active" -> todos.withIndex().filter { !it.value.checked } + "completed" -> todos.withIndex().filter { it.value.checked } + else -> throw IllegalStateException("Unknown state=$state") + } + div { + header(classes = "header") { + h1 { +"todos" } + input(classes = "new-todo") { + placeholder = "What needs to be done?" + } + } + section(classes = "main") { + input( + classes = "toggle-all", + type = InputType.checkBox + ) { + id = "toggle-all" + checked = todos.any { it.checked } + } + label { + htmlFor = "toggle-all" + +"Mark all as complete" + } + ul(classes = "todo-list") { + filteredTodos.forEach { (index, todo) -> + li(classes = if (todo.checked) "completed" else "") { + div(classes = "view") { + input(classes = "toggle", type = InputType.checkBox) { + checked = todo.checked + attributes["data-todo-index"] = index.toString() + } + label { +todo.text } + button(classes = "destroy") { + attributes["data-todo-index"] = index.toString() + } + } + input(classes = "edit") { + value = todo.text + } + } + } + } + } + footer(classes = "footer") { + span(classes = "todo-count") { + strong { + +todos.filter { !it.checked }.size.toString() + } + +" items left" + } + ul(classes = "filters") { + li(classes = "todo-all") { + a(classes = if (state == "all") "selected" else "") { +"All" } + } + li(classes = "todo-active") { + a(classes = if (state == "active") "selected" else "") { +"Active" } + } + li(classes = "todo-completed") { + a(classes = if (state == "completed") "selected" else "") { +"Completed" } + } + } + button(classes = "clear-completed") { +"Clear completed" } + } + } +} diff --git a/example/kotlinlib/web/5-webapp-kotlinjs-shared/src/WebApp.kt b/example/kotlinlib/web/5-webapp-kotlinjs-shared/src/WebApp.kt new file mode 100644 index 00000000000..58e02667e36 --- /dev/null +++ b/example/kotlinlib/web/5-webapp-kotlinjs-shared/src/WebApp.kt @@ -0,0 +1,125 @@ +package webapp + +import io.ktor.serialization.kotlinx.json.* +import io.ktor.server.application.* +import io.ktor.server.engine.* +import io.ktor.server.html.* +import io.ktor.server.http.content.* +import io.ktor.server.netty.* +import io.ktor.server.plugins.contentnegotiation.* +import io.ktor.server.request.* +import io.ktor.server.response.* +import io.ktor.server.routing.* +import io.ktor.server.util.* +import kotlinx.html.* +import shared.* + +object WebApp { + + private val todos = mutableListOf( + Todo(true, "Get started with Cask"), + Todo(false, "Profit!") + ) + + fun add(state: String, text: String) { + todos.add(Todo(false, text)) + } + + fun delete(state: String, index: Int) { + todos.removeAt(index) + } + + fun toggle(state: String, index: Int) { + todos[index] = todos[index].let { + it.copy(checked = !it.checked) + } + } + + fun clearCompleted(state: String) { + todos.removeAll { it.checked } + } + + fun toggleAll(state: String) { + val next = todos.any { !it.checked } + for (item in todos.withIndex()) { + todos[item.index] = item.value.copy(checked = next) + } + } + + private fun HTML.renderIndex() { + head { + meta(charset = "utf-8") + meta(name = "viewport", content = "width=device-width, initial-scale=1") + title("Template • TodoMVC") + link(rel = "stylesheet", href = "/static/index.css") + } + body { + section(classes = "todoapp") { + renderBody(todos, "all") + } + footer(classes = "info") { + p { +"Double-click to edit a todo" } + p { + +"Created by " + a(href = "http://todomvc.com") { +"Li Haoyi" } + } + p { + +"Part of " + a(href = "http://todomvc.com") { +"TodoMVC" } + } + } + script(src = "/static/client.js", block = {}) + } + } + + fun configureRoutes(app: Application) { + with(app) { + routing { + get("/") { + call.respondHtml { + renderIndex() + } + } + post("/toggle-all/{state}") { + toggleAll(call.parameters.getOrFail("state")) + call.respond(todos) + } + post("/clear-completed/{state}") { + clearCompleted(call.parameters.getOrFail("state")) + call.respond(todos) + } + post("/toggle/{state}/{index}") { + call.parameters.run { + toggle(getOrFail("state"), getOrFail("index")) + call.respond(todos) + } + } + post("/delete/{state}/{index}") { + call.parameters.run { + delete(getOrFail("state"), getOrFail("index")) + call.respond(todos) + } + } + post("/add/{state}") { + val requestText = call.receiveText() + add(call.parameters.getOrFail("state"), requestText) + call.respond(todos) + } + post("/list/{state}") { + call.respond(todos) + } + staticResources("/static", "webapp") + } + } + } + + @JvmStatic + fun main(args: Array) { + embeddedServer(Netty, port = 8093, host = "0.0.0.0") { + install(ContentNegotiation) { + json() + } + configureRoutes(this) + }.start(wait = true) + } +} diff --git a/example/kotlinlib/web/5-webapp-kotlinjs-shared/test/src/WebAppTests.kt b/example/kotlinlib/web/5-webapp-kotlinjs-shared/test/src/WebAppTests.kt new file mode 100644 index 00000000000..0786f28c639 --- /dev/null +++ b/example/kotlinlib/web/5-webapp-kotlinjs-shared/test/src/WebAppTests.kt @@ -0,0 +1,28 @@ +package webapp + +import io.kotest.core.spec.style.FunSpec +import io.kotest.matchers.shouldBe +import io.kotest.matchers.string.shouldContain +import io.ktor.client.HttpClient +import io.ktor.client.request.get +import io.ktor.client.statement.bodyAsText +import io.ktor.http.HttpStatusCode +import io.ktor.server.testing.testApplication + +class WebAppTests : FunSpec({ + + suspend fun withServer(f: suspend HttpClient.() -> Unit) { + testApplication { + application { WebApp.configureRoutes(this) } + client.use { client -> f(client) } + } + } + + test("simpleRequest") { + withServer { + val response = get("/") + response.status shouldBe HttpStatusCode.OK + response.bodyAsText() shouldContain "What needs to be done?" + } + } +}) diff --git a/kotlinlib/src/mill/kotlinlib/PlatformKotlinModule.scala b/kotlinlib/src/mill/kotlinlib/PlatformKotlinModule.scala new file mode 100644 index 00000000000..23ff797e7ad --- /dev/null +++ b/kotlinlib/src/mill/kotlinlib/PlatformKotlinModule.scala @@ -0,0 +1,37 @@ +package mill.kotlinlib + +import mill._ +import os.Path + +/** + * A [[KotlinModule]] intended for defining `.jvm`/`.js`/etc. submodules + * It supports additional source directories per platform, e.g. `src-jvm/` or + * `src-js/`. + * + * Adjusts the [[millSourcePath]] and [[artifactNameParts]] to ignore the last + * path segment, which is assumed to be the name of the platform the module is + * built against and not something that should affect the filesystem path or + * artifact name + */ +trait PlatformKotlinModule extends KotlinModule { + override def millSourcePath: Path = super.millSourcePath / os.up + + /** + * The platform suffix of this [[PlatformKotlinModule]]. Useful if you want to + * further customize the source paths or artifact names. + */ + def platformKotlinSuffix: String = millModuleSegments + .value + .collect { case l: mill.define.Segment.Label => l.value } + .last + + override def sources: T[Seq[PathRef]] = Task.Sources { + super.sources().flatMap { source => + val platformPath = + PathRef(source.path / _root_.os.up / s"${source.path.last}-$platformKotlinSuffix") + Seq(source, platformPath) + } + } + + override def artifactNameParts: T[Seq[String]] = super.artifactNameParts().dropRight(1) +} From 43dc23e52917e72e45ff8e222ffc72ddf59e9c0f Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Mon, 14 Oct 2024 14:55:39 +0800 Subject: [PATCH 41/47] Re-enable kotlin IDE tests, move images into subfolders (#3731) --- .../ROOT/images/{ => basic}/ChromeTracing.png | Bin .../ROOT/images/{ => basic}/IntellijApp.png | Bin .../ROOT/images/{ => basic}/IntellijBuild.png | Bin .../{ => basic}/IntellijFileTypeConfig.png | Bin .../images/{ => basic}/IntellijRefresh.png | Bin .../ROOT/images/{ => basic}/VSCodeApp.png | Bin .../ROOT/images/{ => basic}/VSCodeBuild.png | Bin .../ROOT/images/{ => basic}/VSCodeRefresh.png | Bin .../ROOT/images/{ => basic}/VisualizeJava.svg | 0 .../images/{ => basic}/VisualizePlanJava.svg | 0 .../images/{ => basic}/VisualizePlanScala.svg | 0 .../{ => comparisons}/GatlingCompileGraph.svg | 0 .../GatlingCompileProfile.png | Bin .../IntellijGatlingMillPlugin1.png | Bin .../IntellijGatlingMillPlugin2.png | Bin .../IntellijGatlingMillTask1.png | Bin .../IntellijGatlingMillTask2.png | Bin .../IntellijGatlingMillTask3.png | Bin .../IntellijGatlingSbtPlugin1.png | Bin .../IntellijGatlingSbtPlugin2.png | Bin .../IntellijGatlingSbtTask1.png | Bin .../IntellijGatlingSbtTask2.png | Bin .../{ => comparisons}/MockitoCompileGraph.svg | 0 .../MockitoCompileProfile.png | Bin .../{ => comparisons}/NettyCompileGraph.svg | 0 .../{ => comparisons}/NettyCompileProfile.png | Bin .../ROOT/pages/comparisons/gradle.adoc | 4 ++-- .../modules/ROOT/pages/comparisons/maven.adoc | 4 ++-- docs/modules/ROOT/pages/comparisons/sbt.adoc | 22 +++++++++--------- .../pages/kotlinlib/installation-ide.adoc | 6 ++--- .../partials/Installation_IDE_Support.adoc | 14 +++++------ .../ROOT/partials/Intro_to_Mill_Footer.adoc | 2 +- .../javalib/module/1-common-config/build.mill | 2 +- .../module/1-common-config/build.mill | 2 +- .../basic/4-builtin-commands/build.mill | 4 ++-- .../module/1-common-config/build.mill | 2 +- 36 files changed, 31 insertions(+), 31 deletions(-) rename docs/modules/ROOT/images/{ => basic}/ChromeTracing.png (100%) rename docs/modules/ROOT/images/{ => basic}/IntellijApp.png (100%) rename docs/modules/ROOT/images/{ => basic}/IntellijBuild.png (100%) rename docs/modules/ROOT/images/{ => basic}/IntellijFileTypeConfig.png (100%) rename docs/modules/ROOT/images/{ => basic}/IntellijRefresh.png (100%) rename docs/modules/ROOT/images/{ => basic}/VSCodeApp.png (100%) rename docs/modules/ROOT/images/{ => basic}/VSCodeBuild.png (100%) rename docs/modules/ROOT/images/{ => basic}/VSCodeRefresh.png (100%) rename docs/modules/ROOT/images/{ => basic}/VisualizeJava.svg (100%) rename docs/modules/ROOT/images/{ => basic}/VisualizePlanJava.svg (100%) rename docs/modules/ROOT/images/{ => basic}/VisualizePlanScala.svg (100%) rename docs/modules/ROOT/images/{ => comparisons}/GatlingCompileGraph.svg (100%) rename docs/modules/ROOT/images/{ => comparisons}/GatlingCompileProfile.png (100%) rename docs/modules/ROOT/images/{ => comparisons}/IntellijGatlingMillPlugin1.png (100%) rename docs/modules/ROOT/images/{ => comparisons}/IntellijGatlingMillPlugin2.png (100%) rename docs/modules/ROOT/images/{ => comparisons}/IntellijGatlingMillTask1.png (100%) rename docs/modules/ROOT/images/{ => comparisons}/IntellijGatlingMillTask2.png (100%) rename docs/modules/ROOT/images/{ => comparisons}/IntellijGatlingMillTask3.png (100%) rename docs/modules/ROOT/images/{ => comparisons}/IntellijGatlingSbtPlugin1.png (100%) rename docs/modules/ROOT/images/{ => comparisons}/IntellijGatlingSbtPlugin2.png (100%) rename docs/modules/ROOT/images/{ => comparisons}/IntellijGatlingSbtTask1.png (100%) rename docs/modules/ROOT/images/{ => comparisons}/IntellijGatlingSbtTask2.png (100%) rename docs/modules/ROOT/images/{ => comparisons}/MockitoCompileGraph.svg (100%) rename docs/modules/ROOT/images/{ => comparisons}/MockitoCompileProfile.png (100%) rename docs/modules/ROOT/images/{ => comparisons}/NettyCompileGraph.svg (100%) rename docs/modules/ROOT/images/{ => comparisons}/NettyCompileProfile.png (100%) diff --git a/docs/modules/ROOT/images/ChromeTracing.png b/docs/modules/ROOT/images/basic/ChromeTracing.png similarity index 100% rename from docs/modules/ROOT/images/ChromeTracing.png rename to docs/modules/ROOT/images/basic/ChromeTracing.png diff --git a/docs/modules/ROOT/images/IntellijApp.png b/docs/modules/ROOT/images/basic/IntellijApp.png similarity index 100% rename from docs/modules/ROOT/images/IntellijApp.png rename to docs/modules/ROOT/images/basic/IntellijApp.png diff --git a/docs/modules/ROOT/images/IntellijBuild.png b/docs/modules/ROOT/images/basic/IntellijBuild.png similarity index 100% rename from docs/modules/ROOT/images/IntellijBuild.png rename to docs/modules/ROOT/images/basic/IntellijBuild.png diff --git a/docs/modules/ROOT/images/IntellijFileTypeConfig.png b/docs/modules/ROOT/images/basic/IntellijFileTypeConfig.png similarity index 100% rename from docs/modules/ROOT/images/IntellijFileTypeConfig.png rename to docs/modules/ROOT/images/basic/IntellijFileTypeConfig.png diff --git a/docs/modules/ROOT/images/IntellijRefresh.png b/docs/modules/ROOT/images/basic/IntellijRefresh.png similarity index 100% rename from docs/modules/ROOT/images/IntellijRefresh.png rename to docs/modules/ROOT/images/basic/IntellijRefresh.png diff --git a/docs/modules/ROOT/images/VSCodeApp.png b/docs/modules/ROOT/images/basic/VSCodeApp.png similarity index 100% rename from docs/modules/ROOT/images/VSCodeApp.png rename to docs/modules/ROOT/images/basic/VSCodeApp.png diff --git a/docs/modules/ROOT/images/VSCodeBuild.png b/docs/modules/ROOT/images/basic/VSCodeBuild.png similarity index 100% rename from docs/modules/ROOT/images/VSCodeBuild.png rename to docs/modules/ROOT/images/basic/VSCodeBuild.png diff --git a/docs/modules/ROOT/images/VSCodeRefresh.png b/docs/modules/ROOT/images/basic/VSCodeRefresh.png similarity index 100% rename from docs/modules/ROOT/images/VSCodeRefresh.png rename to docs/modules/ROOT/images/basic/VSCodeRefresh.png diff --git a/docs/modules/ROOT/images/VisualizeJava.svg b/docs/modules/ROOT/images/basic/VisualizeJava.svg similarity index 100% rename from docs/modules/ROOT/images/VisualizeJava.svg rename to docs/modules/ROOT/images/basic/VisualizeJava.svg diff --git a/docs/modules/ROOT/images/VisualizePlanJava.svg b/docs/modules/ROOT/images/basic/VisualizePlanJava.svg similarity index 100% rename from docs/modules/ROOT/images/VisualizePlanJava.svg rename to docs/modules/ROOT/images/basic/VisualizePlanJava.svg diff --git a/docs/modules/ROOT/images/VisualizePlanScala.svg b/docs/modules/ROOT/images/basic/VisualizePlanScala.svg similarity index 100% rename from docs/modules/ROOT/images/VisualizePlanScala.svg rename to docs/modules/ROOT/images/basic/VisualizePlanScala.svg diff --git a/docs/modules/ROOT/images/GatlingCompileGraph.svg b/docs/modules/ROOT/images/comparisons/GatlingCompileGraph.svg similarity index 100% rename from docs/modules/ROOT/images/GatlingCompileGraph.svg rename to docs/modules/ROOT/images/comparisons/GatlingCompileGraph.svg diff --git a/docs/modules/ROOT/images/GatlingCompileProfile.png b/docs/modules/ROOT/images/comparisons/GatlingCompileProfile.png similarity index 100% rename from docs/modules/ROOT/images/GatlingCompileProfile.png rename to docs/modules/ROOT/images/comparisons/GatlingCompileProfile.png diff --git a/docs/modules/ROOT/images/IntellijGatlingMillPlugin1.png b/docs/modules/ROOT/images/comparisons/IntellijGatlingMillPlugin1.png similarity index 100% rename from docs/modules/ROOT/images/IntellijGatlingMillPlugin1.png rename to docs/modules/ROOT/images/comparisons/IntellijGatlingMillPlugin1.png diff --git a/docs/modules/ROOT/images/IntellijGatlingMillPlugin2.png b/docs/modules/ROOT/images/comparisons/IntellijGatlingMillPlugin2.png similarity index 100% rename from docs/modules/ROOT/images/IntellijGatlingMillPlugin2.png rename to docs/modules/ROOT/images/comparisons/IntellijGatlingMillPlugin2.png diff --git a/docs/modules/ROOT/images/IntellijGatlingMillTask1.png b/docs/modules/ROOT/images/comparisons/IntellijGatlingMillTask1.png similarity index 100% rename from docs/modules/ROOT/images/IntellijGatlingMillTask1.png rename to docs/modules/ROOT/images/comparisons/IntellijGatlingMillTask1.png diff --git a/docs/modules/ROOT/images/IntellijGatlingMillTask2.png b/docs/modules/ROOT/images/comparisons/IntellijGatlingMillTask2.png similarity index 100% rename from docs/modules/ROOT/images/IntellijGatlingMillTask2.png rename to docs/modules/ROOT/images/comparisons/IntellijGatlingMillTask2.png diff --git a/docs/modules/ROOT/images/IntellijGatlingMillTask3.png b/docs/modules/ROOT/images/comparisons/IntellijGatlingMillTask3.png similarity index 100% rename from docs/modules/ROOT/images/IntellijGatlingMillTask3.png rename to docs/modules/ROOT/images/comparisons/IntellijGatlingMillTask3.png diff --git a/docs/modules/ROOT/images/IntellijGatlingSbtPlugin1.png b/docs/modules/ROOT/images/comparisons/IntellijGatlingSbtPlugin1.png similarity index 100% rename from docs/modules/ROOT/images/IntellijGatlingSbtPlugin1.png rename to docs/modules/ROOT/images/comparisons/IntellijGatlingSbtPlugin1.png diff --git a/docs/modules/ROOT/images/IntellijGatlingSbtPlugin2.png b/docs/modules/ROOT/images/comparisons/IntellijGatlingSbtPlugin2.png similarity index 100% rename from docs/modules/ROOT/images/IntellijGatlingSbtPlugin2.png rename to docs/modules/ROOT/images/comparisons/IntellijGatlingSbtPlugin2.png diff --git a/docs/modules/ROOT/images/IntellijGatlingSbtTask1.png b/docs/modules/ROOT/images/comparisons/IntellijGatlingSbtTask1.png similarity index 100% rename from docs/modules/ROOT/images/IntellijGatlingSbtTask1.png rename to docs/modules/ROOT/images/comparisons/IntellijGatlingSbtTask1.png diff --git a/docs/modules/ROOT/images/IntellijGatlingSbtTask2.png b/docs/modules/ROOT/images/comparisons/IntellijGatlingSbtTask2.png similarity index 100% rename from docs/modules/ROOT/images/IntellijGatlingSbtTask2.png rename to docs/modules/ROOT/images/comparisons/IntellijGatlingSbtTask2.png diff --git a/docs/modules/ROOT/images/MockitoCompileGraph.svg b/docs/modules/ROOT/images/comparisons/MockitoCompileGraph.svg similarity index 100% rename from docs/modules/ROOT/images/MockitoCompileGraph.svg rename to docs/modules/ROOT/images/comparisons/MockitoCompileGraph.svg diff --git a/docs/modules/ROOT/images/MockitoCompileProfile.png b/docs/modules/ROOT/images/comparisons/MockitoCompileProfile.png similarity index 100% rename from docs/modules/ROOT/images/MockitoCompileProfile.png rename to docs/modules/ROOT/images/comparisons/MockitoCompileProfile.png diff --git a/docs/modules/ROOT/images/NettyCompileGraph.svg b/docs/modules/ROOT/images/comparisons/NettyCompileGraph.svg similarity index 100% rename from docs/modules/ROOT/images/NettyCompileGraph.svg rename to docs/modules/ROOT/images/comparisons/NettyCompileGraph.svg diff --git a/docs/modules/ROOT/images/NettyCompileProfile.png b/docs/modules/ROOT/images/comparisons/NettyCompileProfile.png similarity index 100% rename from docs/modules/ROOT/images/NettyCompileProfile.png rename to docs/modules/ROOT/images/comparisons/NettyCompileProfile.png diff --git a/docs/modules/ROOT/pages/comparisons/gradle.adoc b/docs/modules/ROOT/pages/comparisons/gradle.adoc index 93ccf700a96..1c9a3d652b5 100644 --- a/docs/modules/ROOT/pages/comparisons/gradle.adoc +++ b/docs/modules/ROOT/pages/comparisons/gradle.adoc @@ -190,7 +190,7 @@ and associated test suites, but how do these different modules depend on each ot Mill, you can run `./mill visualize __.compile`, and it will show you how the `compile` task of each module depends on the others: -image::MockitoCompileGraph.svg[] +image::comparisons/MockitoCompileGraph.svg[] Apart from the static dependency graph, another thing of interest may be the performance profile and timeline: where the time is spent when you actually compile everything. With @@ -198,7 +198,7 @@ Mill, when you run a compilation using `./mill -j 10 __.compile`, you automatica `out/mill-chrome-profile.json` file that you can load into your `chrome://tracing` page and visualize where your build is spending time and where the performance bottlenecks are: -image::MockitoCompileProfile.png[] +image::comparisons/MockitoCompileProfile.png[] If you want to inspect the tree of third-party dependencies used by any module, the built in `ivyDepsTree` command lets you do that easily: diff --git a/docs/modules/ROOT/pages/comparisons/maven.adoc b/docs/modules/ROOT/pages/comparisons/maven.adoc index 73cc54f353b..7a3a50c6429 100644 --- a/docs/modules/ROOT/pages/comparisons/maven.adoc +++ b/docs/modules/ROOT/pages/comparisons/maven.adoc @@ -599,7 +599,7 @@ and associated test suites, but how do these different modules depend on each ot Mill, you can run `./mill visualize __.compile`, and it will show you how the `compile` task of each module depends on the others: -image::NettyCompileGraph.svg[] +image::comparisons/NettyCompileGraph.svg[] Apart from the static dependency graph, another thing of interest may be the performance profile and timeline: where the time is spent when you actually compile everything. With @@ -607,7 +607,7 @@ Mill, when you run a compilation using `./mill -j 10 __.compile`, you automatica `out/mill-chrome-profile.json` file that you can load into your `chrome://tracing` page and visualize where your build is spending time and where the performance bottlenecks are: -image::NettyCompileProfile.png[] +image::comparisons/NettyCompileProfile.png[] If you want to inspect the tree of third-party dependencies used by any module, the built in `ivyDepsTree` command lets you do that easily: diff --git a/docs/modules/ROOT/pages/comparisons/sbt.adoc b/docs/modules/ROOT/pages/comparisons/sbt.adoc index c87ad5b26f7..a625b93ae07 100644 --- a/docs/modules/ROOT/pages/comparisons/sbt.adoc +++ b/docs/modules/ROOT/pages/comparisons/sbt.adoc @@ -187,13 +187,13 @@ IDEs like IntelliJ are nominally able to parse and analyze your SBT files, the a provide is often not very useful. For example, consider the inspection and jump-to-definition experience of looking into an SBT Task: -image::IntellijGatlingSbtTask1.png[] -image::IntellijGatlingSbtTask2.png[] +image::comparisons/IntellijGatlingSbtTask1.png[] +image::comparisons/IntellijGatlingSbtTask2.png[] Or an SBT plugin: -image::IntellijGatlingSbtPlugin1.png[] -image::IntellijGatlingSbtPlugin2.png[] +image::comparisons/IntellijGatlingSbtPlugin1.png[] +image::comparisons/IntellijGatlingSbtPlugin2.png[] In general, although your IDE can make sure the name of the task exists, and the type is correct, it is unable to pull up any further information about the task: its documentation, its implementation, @@ -205,22 +205,22 @@ at all: what it does, where it is assigned, etc. In comparison, for Mill, IDEs like Intellij are able to provide much more intelligence. e.g. when inspecting a task, it is able to pull up the documentation comment: -image::IntellijGatlingMillTask1.png[] +image::comparisons/IntellijGatlingMillTask1.png[] It is able to pull up any overridden implementations of task, directly in the editor: -image::IntellijGatlingMillTask2.png[] +image::comparisons/IntellijGatlingMillTask2.png[] And you can easily navigate to the overriden implementations to see where they are defined and what you are overriding: -image::IntellijGatlingMillTask3.png[] +image::comparisons/IntellijGatlingMillTask3.png[] Mill's equivalent of SBT plugins are just Scala traits, and again you can easily pull up their documentation in-line in the editor or jump to their full implementation: -image::IntellijGatlingMillPlugin1.png[] -image::IntellijGatlingMillPlugin2.png[] +image::comparisons/IntellijGatlingMillPlugin1.png[] +image::comparisons/IntellijGatlingMillPlugin2.png[] In general, navigating around your build in Mill is much more straightforward than navigating around your build in SBT. All your normal IDE functionality works perfectly: @@ -237,7 +237,7 @@ and associated test suites, but how do these different modules depend on each ot Mill, you can run `./mill visualize __.compile`, and it will show you how the `compile` task of each module depends on the others: -image::GatlingCompileGraph.svg[] +image::comparisons/GatlingCompileGraph.svg[] Apart from the static dependency graph, another thing of interest may be the performance profile and timeline: where the time is spent when you actually compile everything. With @@ -245,7 +245,7 @@ Mill, when you run a compilation using `./mill -j 10 __.compile`, you automatica `out/mill-chrome-profile.json` file that you can load into your `chrome://tracing` page and visualize where your build is spending time and where the performance bottlenecks are: -image::GatlingCompileProfile.png[] +image::comparisons/GatlingCompileProfile.png[] If you want to inspect the tree of third-party dependencies used by any module, the built in `ivyDepsTree` command lets you do that easily: diff --git a/docs/modules/ROOT/pages/kotlinlib/installation-ide.adoc b/docs/modules/ROOT/pages/kotlinlib/installation-ide.adoc index 9f3b2c42ea6..b07b0c74ce6 100644 --- a/docs/modules/ROOT/pages/kotlinlib/installation-ide.adoc +++ b/docs/modules/ROOT/pages/kotlinlib/installation-ide.adoc @@ -1,6 +1,6 @@ = Installation and IDE Support +:language: Kotlin +:language-small: kotlin -Kotlin IDE support is work in progress. For details, see the following issue: - -* https://github.com/com-lihaoyi/mill/issues/3606 \ No newline at end of file +include::partial$Installation_IDE_Support.adoc[] \ No newline at end of file diff --git a/docs/modules/ROOT/partials/Installation_IDE_Support.adoc b/docs/modules/ROOT/partials/Installation_IDE_Support.adoc index 12d9f6d9cd7..193fce7bcd7 100644 --- a/docs/modules/ROOT/partials/Installation_IDE_Support.adoc +++ b/docs/modules/ROOT/partials/Installation_IDE_Support.adoc @@ -95,19 +95,19 @@ containing a Mill `build.mill` file, and IntelliJ will automatically load the Mill build. This will provide support both for your application code, as well as the code in the `build.mill`: -image::IntellijApp.png[] +image::basic/IntellijApp.png[] -image::IntellijBuild.png[] +image::basic/IntellijBuild.png[] If IntelliJ does not highlight the `.mill` files correctly, you can explicitly enable it by adding `*.mill` to the `Scala` file type: -image::IntellijFileTypeConfig.png[] +image::basic/IntellijFileTypeConfig.png[] If you make changes to your Mill `build.mill`, you can ask Intellij to load those updates by opening the "BSP" tab and clicking the "Refresh" button -image::IntellijRefresh.png[] +image::basic/IntellijRefresh.png[] ==== IntelliJ IDEA XML Support @@ -157,14 +157,14 @@ containing a Mill `build.mill` file, and VSCode will ask you to import your Mill build. This will provide support both for your application code, as well as the code in the `build.mill`: -image::VSCodeApp.png[] +image::basic/VSCodeApp.png[] -image::VSCodeBuild.png[] +image::basic/VSCodeBuild.png[] If you make changes to your Mill `build.sc`, you can ask VSCode to load those updates by opening the "BSP" tab and clicking the "Refresh" button -image::VSCodeRefresh.png[] +image::basic/VSCodeRefresh.png[] === Debugging IDE issues diff --git a/docs/modules/ROOT/partials/Intro_to_Mill_Footer.adoc b/docs/modules/ROOT/partials/Intro_to_Mill_Footer.adoc index 55fe01c3228..1b73c54d438 100644 --- a/docs/modules/ROOT/partials/Intro_to_Mill_Footer.adoc +++ b/docs/modules/ROOT/partials/Intro_to_Mill_Footer.adoc @@ -49,7 +49,7 @@ loaded into the Chrome browser's `chrome://tracing` page for visualization. This can make it much easier to analyze your parallel runs to find out what's taking the most time: -image::ChromeTracing.png[ChromeTracing.png] +image::basic/ChromeTracing.png[ChromeTracing.png] Note that the maximal possible parallelism depends both on the number of cores available as well as the task and module structure of your project, as tasks that diff --git a/example/javalib/module/1-common-config/build.mill b/example/javalib/module/1-common-config/build.mill index fc38507ec93..16346910a89 100644 --- a/example/javalib/module/1-common-config/build.mill +++ b/example/javalib/module/1-common-config/build.mill @@ -50,7 +50,7 @@ object `package` extends RootModule with JavaModule { > mill visualizePlan run */ // -// image::VisualizePlanJava.svg[VisualizePlanJava.svg] +// image::basic/VisualizePlanJava.svg[VisualizePlanJava.svg] // // (right-click open in new tab to see full sized) // diff --git a/example/kotlinlib/module/1-common-config/build.mill b/example/kotlinlib/module/1-common-config/build.mill index 5aaccae67a7..d4249bf847e 100644 --- a/example/kotlinlib/module/1-common-config/build.mill +++ b/example/kotlinlib/module/1-common-config/build.mill @@ -53,7 +53,7 @@ object `package` extends RootModule with KotlinModule { > mill visualizePlan run */ // -// image::VisualizePlanJava.svg[VisualizePlanJava.svg] +// image::basic/VisualizePlanJava.svg[VisualizePlanJava.svg] // // (right-click open in new tab to see full sized) // diff --git a/example/scalalib/basic/4-builtin-commands/build.mill b/example/scalalib/basic/4-builtin-commands/build.mill index d327198ec53..4b4dbc74dc9 100644 --- a/example/scalalib/basic/4-builtin-commands/build.mill +++ b/example/scalalib/basic/4-builtin-commands/build.mill @@ -302,7 +302,7 @@ foo.compileClasspath // // The above command generates the following diagram (right-click open in new tab to see full sized): // -// image::VisualizeJava.svg[VisualizeJava.svg] +// image::basic/VisualizeJava.svg[VisualizeJava.svg] // // `visualize` can be very handy for trying to understand the dependency graph of // tasks within your Mill build. @@ -327,7 +327,7 @@ foo.compileClasspath // // The above command generates the following diagram (right-click open in new tab to see full sized): // -// image::VisualizePlanJava.svg[VisualizePlanJava.svg] +// image::basic/VisualizePlanJava.svg[VisualizePlanJava.svg] // // // == init diff --git a/example/scalalib/module/1-common-config/build.mill b/example/scalalib/module/1-common-config/build.mill index 91d0be5f7da..9a6155dfa98 100644 --- a/example/scalalib/module/1-common-config/build.mill +++ b/example/scalalib/module/1-common-config/build.mill @@ -61,7 +61,7 @@ object `package` extends RootModule with ScalaModule { > mill visualizePlan run */ // -// image::VisualizePlanScala.svg[VisualizePlanScala.svg] +// image::basic/VisualizePlanScala.svg[VisualizePlanScala.svg] // // (right-click open in new tab to see full sized) // From fc03373a9242717b9cec239d316b30413786c1c2 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Mon, 14 Oct 2024 20:50:37 +0800 Subject: [PATCH 42/47] Documentation Review for javalib/kotlinlib/scalalib/fundamentals/extending/depth (#3734) Did a once-through reading of most of the documentation site, with the exception of `scalalib/intro.html` and the three in `comparisons/` which will need a deeper review Fixes https://github.com/com-lihaoyi/mill/issues/3493 --- build.mill | 92 +++++++++---------- contrib/scalapblib/readme.adoc | 2 +- .../ROOT/images/basic/VisualizeCompiles.svg | 49 ++++++++++ .../ROOT/images/basic/VisualizeTestDeps.svg | 55 +++++++++++ docs/modules/ROOT/nav.adoc | 3 +- .../ROOT/pages/depth/design-principles.adoc | 4 +- .../ROOT/pages/depth/evaluation-model.adoc | 4 +- .../ROOT/pages/depth/large-builds.adoc | 11 +++ .../ROOT/pages/extending/contrib-plugins.adoc | 10 +- ...g-plugins.adoc => import-ivy-plugins.adoc} | 40 ++++++-- .../ROOT/pages/extending/import-ivy.adoc | 19 ---- .../ROOT/pages/extending/meta-build.adoc | 10 +- .../pages/extending/thirdparty-plugins.adoc | 2 +- .../ROOT/pages/fundamentals/cross-builds.adoc | 2 +- .../ROOT/pages/fundamentals/modules.adoc | 1 + .../ROOT/pages/fundamentals/out-dir.adoc | 4 +- .../ROOT/pages/fundamentals/tasks.adoc | 4 +- .../ROOT/pages/javalib/android-examples.adoc | 4 +- .../ROOT/pages/javalib/dependencies.adoc | 4 +- .../ROOT/pages/javalib/module-config.adoc | 20 ++-- .../pages/kotlinlib/android-examples.adoc | 2 +- .../ROOT/pages/kotlinlib/dependencies.adoc | 4 +- docs/modules/ROOT/pages/kotlinlib/intro.adoc | 7 ++ .../ROOT/pages/kotlinlib/module-config.adoc | 24 +++-- .../ROOT/pages/scalalib/dependencies.adoc | 4 +- docs/modules/ROOT/pages/scalalib/intro.adoc | 2 +- .../ROOT/pages/scalalib/module-config.adoc | 19 ++-- .../Intro_Maven_Gradle_Comparison.adoc | 2 +- .../partials/Intro_to_Mill_BlogVideo.adoc | 2 +- .../ROOT/partials/Intro_to_Mill_Header.adoc | 6 +- docs/package.mill | 2 + .../large/10-multi-file-builds/build.mill | 17 ++-- .../depth/large/11-helper-files/build.mill | 8 +- .../depth/large/12-helper-files-sc/build.sc | 22 +++-- example/depth/sandbox/3-breaking/build.mill | 4 +- .../extending/imports/1-import-ivy/build.mill | 2 +- .../imports/3-contrib-import/build.mill | 10 ++ .../metabuild/3-autoformatting/.scalafmt.conf | 4 + .../metabuild/3-autoformatting/build.mill | 31 +++++++ .../plugins/7-writing-mill-plugins/build.mill | 25 +++-- .../resources/example-test-project/build.mill | 2 +- .../integration-test-project/build.mill | 2 +- .../cross/10-static-blog/build.mill | 4 +- .../cross/11-default-cross-module/build.mill | 9 +- .../cross/3-outside-dependency/build.mill | 4 +- .../cross/4-cross-dependencies/build.mill | 2 +- .../cross/7-inner-cross-module/build.mill | 11 ++- .../dependencies/1-search-updates/build.mill | 1 - .../fundamentals/modules/7-modules/build.mill | 14 +-- .../out-dir/1-custom-out/build.mill | 15 +-- .../tasks/1-task-graph/build.mill | 4 +- .../tasks/2-primary-tasks/build.mill | 36 ++++---- .../tasks/3-anonymous-tasks/build.mill | 2 +- .../fundamentals/tasks/4-inputs/build.mill | 14 +-- .../tasks/5-persistent-tasks/build.mill | 14 ++- example/javalib/basic/1-simple/build.mill | 4 +- .../basic/2-custom-build-logic/build.mill | 2 +- .../build.mill | 0 .../src/foo/Foo.java | 0 .../textfile.txt | 0 .../javalib/linting/1-error-prone/build.mill | 3 +- .../javalib/linting/2-checkstyle/build.mill | 6 +- .../linting/3-palantirformat/build.mill | 8 +- .../4-compilation-execution-flags/build.mill | 2 +- .../basic/2-custom-build-logic/build.mill | 2 +- .../basic/4-builtin-commands/build.mill | 12 +-- .../build.mill | 0 .../src/foo/Foo.kt | 0 .../textfile.txt | 0 example/kotlinlib/linting/1-detekt/build.mill | 7 +- example/kotlinlib/linting/2-ktlint/build.mill | 9 +- example/kotlinlib/linting/3-ktfmt/build.mill | 8 +- example/scalalib/basic/1-simple/build.mill | 2 +- .../basic/2-custom-build-logic/build.mill | 10 +- .../scalalib/basic/3-multi-module/build.mill | 19 +++- .../basic/4-builtin-commands/build.mill | 30 +++++- .../dependencies/1-ivy-deps/build.mill | 3 +- .../2-run-compile-deps/build.mill | 2 +- .../dependencies/3-unmanaged-jars/build.mill | 9 +- .../build.mill | 12 ++- .../src/Foo.scala | 0 .../textfile.txt | 0 .../5-repository-config/build.mill | 12 ++- .../scalalib/module/2-custom-tasks/build.mill | 24 +++-- .../scalalib/testing/2-test-deps/build.mill | 6 +- .../mill/kotlinlib/PlatformKotlinModule.scala | 26 +----- .../mill/kotlinlib/ktfmt/KtfmtModule.scala | 10 +- .../contrib/ktfmt/KtfmtModuleTests.scala | 18 ++-- main/src/mill/main/MainModule.scala | 10 +- main/src/mill/main/VisualizeModule.scala | 24 +++-- .../palantirformat/PalantirFormatModule.scala | 3 +- .../mill/scalalib/PlatformModuleBase.scala | 27 ++++++ .../mill/scalalib/PlatformScalaModule.scala | 9 +- 93 files changed, 644 insertions(+), 346 deletions(-) create mode 100644 docs/modules/ROOT/images/basic/VisualizeCompiles.svg create mode 100644 docs/modules/ROOT/images/basic/VisualizeTestDeps.svg rename docs/modules/ROOT/pages/extending/{using-plugins.adoc => import-ivy-plugins.adoc} (56%) delete mode 100644 docs/modules/ROOT/pages/extending/import-ivy.adoc create mode 100644 example/extending/metabuild/3-autoformatting/.scalafmt.conf create mode 100644 example/extending/metabuild/3-autoformatting/build.mill rename example/javalib/dependencies/{4-downloading-non-maven-jars => 4-downloading-unmanaged-jars}/build.mill (100%) rename example/javalib/dependencies/{4-downloading-non-maven-jars => 4-downloading-unmanaged-jars}/src/foo/Foo.java (100%) rename example/javalib/dependencies/{4-downloading-non-maven-jars => 4-downloading-unmanaged-jars}/textfile.txt (100%) rename example/kotlinlib/dependencies/{4-downloading-non-maven-jars => 4-downloading-unmanaged-jars}/build.mill (100%) rename example/kotlinlib/dependencies/{4-downloading-non-maven-jars => 4-downloading-unmanaged-jars}/src/foo/Foo.kt (100%) rename example/kotlinlib/dependencies/{4-downloading-non-maven-jars => 4-downloading-unmanaged-jars}/textfile.txt (100%) rename example/scalalib/dependencies/{4-downloading-non-maven-jars => 4-downloading-unmanaged-jars}/build.mill (66%) rename example/scalalib/dependencies/{4-downloading-non-maven-jars => 4-downloading-unmanaged-jars}/src/Foo.scala (100%) rename example/scalalib/dependencies/{4-downloading-non-maven-jars => 4-downloading-unmanaged-jars}/textfile.txt (100%) create mode 100644 scalalib/src/mill/scalalib/PlatformModuleBase.scala diff --git a/build.mill b/build.mill index 9c2c741026a..5ff7c20bba0 100644 --- a/build.mill +++ b/build.mill @@ -329,9 +329,9 @@ trait MillJavaModule extends JavaModule { def testDepPaths = Task { upstreamAssemblyClasspath() ++ Seq(compile().classes) ++ resources() } def testTransitiveDeps: T[Map[String, String]] = Task { - val upstream = T.traverse(moduleDeps ++ compileModuleDeps) { + val upstream = Task.traverse(moduleDeps ++ compileModuleDeps) { case m: MillJavaModule => m.testTransitiveDeps.map(Some(_)) - case _ => T.task(None) + case _ => Task.Anon(None) }().flatten.flatten val current = Seq(testDep()) upstream.toMap ++ current @@ -342,28 +342,28 @@ trait MillJavaModule extends JavaModule { if (this == build.main) Seq(build.main) else Seq(this, build.main.test) - def writeLocalTestOverrides = T.task { + def writeLocalTestOverrides = Task.Anon { for ((k, v) <- testTransitiveDeps()) { - os.write(T.dest / "mill" / "local-test-overrides" / k, v, createFolders = true) + os.write(Task.dest / "mill" / "local-test-overrides" / k, v, createFolders = true) } - Seq(PathRef(T.dest)) + Seq(PathRef(Task.dest)) } def runClasspath = super.runClasspath() ++ writeLocalTestOverrides() - def repositoriesTask = T.task { + def repositoriesTask = Task.Anon { super.repositoriesTask() ++ Seq(MavenRepository("https://oss.sonatype.org/content/repositories/releases")) } - def mapDependencies: Task[coursier.Dependency => coursier.Dependency] = T.task { + def mapDependencies: Task[coursier.Dependency => coursier.Dependency] = Task.Anon { super.mapDependencies().andThen { dep => forcedVersions.find(f => f.dep.module.organization.value == dep.module.organization.value && f.dep.module.name.value == dep.module.name.value ).map { forced => val newDep = dep.withVersion(forced.dep.version) - T.log.debug(s"Forcing version of ${dep.module} from ${dep.version} to ${newDep.version}") + Task.log.debug(s"Forcing version of ${dep.module} from ${dep.version} to ${newDep.version}") newDep }.getOrElse(dep) } @@ -589,9 +589,9 @@ trait BridgeModule extends MillPublishJavaModule with CrossScalaModule { ivy"org.scala-lang:scala-compiler:${crossScalaVersion}" ) - def resources = T.sources { - os.copy(generatedSources().head.path / "META-INF", T.dest / "META-INF") - Seq(PathRef(T.dest)) + def resources = Task.Sources { + os.copy(generatedSources().head.path / "META-INF", Task.dest / "META-INF") + Seq(PathRef(Task.dest)) } def compilerBridgeIvyDeps: T[Agg[Dep]] = Agg( @@ -600,7 +600,7 @@ trait BridgeModule extends MillPublishJavaModule with CrossScalaModule { def compilerBridgeSourceJars: T[Agg[PathRef]] = Task { resolveDeps( - T.task { compilerBridgeIvyDeps().map(bindDependency()) }, + Task.Anon { compilerBridgeIvyDeps().map(bindDependency()) }, sources = true )() } @@ -608,10 +608,10 @@ trait BridgeModule extends MillPublishJavaModule with CrossScalaModule { def generatedSources = Task { compilerBridgeSourceJars().foreach { jar => - os.unzip(jar.path, T.dest) + os.unzip(jar.path, Task.dest) } - Seq(PathRef(T.dest)) + Seq(PathRef(Task.dest)) } } @@ -752,7 +752,7 @@ object idea extends MillPublishScalaModule { */ object dist0 extends MillPublishJavaModule { // disable scalafix here because it crashes when a module has no sources - def fix(args: String*): Command[Unit] = T.command {} + def fix(args: String*): Command[Unit] = Task.Command {} def moduleDeps = Seq(build.runner, idea) def testTransitiveDeps = build.runner.testTransitiveDeps() ++ Seq( @@ -779,7 +779,7 @@ object dist extends MillPublishJavaModule { (s"com.lihaoyi-${dist.artifactId()}", dist0.runClasspath().map(_.path).mkString("\n")) ) - def genTask(m: ScalaModule) = T.task { Seq(m.jar(), m.sourceJar()) ++ m.runClasspath() } + def genTask(m: ScalaModule) = Task.Anon { Seq(m.jar(), m.sourceJar()) ++ m.runClasspath() } def forkArgs: T[Seq[String]] = Task { val genIdeaArgs = @@ -801,7 +801,7 @@ object dist extends MillPublishJavaModule { def launcher = Task { val isWin = scala.util.Properties.isWin - val outputPath = T.dest / (if (isWin) "run.bat" else "run") + val outputPath = Task.dest / (if (isWin) "run.bat" else "run") os.write(outputPath, prependShellScript()) if (!isWin) os.perms.set(outputPath, "rwxrwxrwx") @@ -840,22 +840,22 @@ object dist extends MillPublishJavaModule { prependShellScript = launcherScript(shellArgs, cmdArgs, Agg("$0"), Agg("%~dpnx0")), assemblyRules = assemblyRules ).path, - T.dest / filename + Task.dest / filename ) - PathRef(T.dest / filename) + PathRef(Task.dest / filename) } def assembly = Task { - T.traverse(allPublishModules)(m => m.publishLocalCached)() + Task.traverse(allPublishModules)(m => m.publishLocalCached)() val raw = rawAssembly().path - os.copy(raw, T.dest / raw.last) - PathRef(T.dest / raw.last) + os.copy(raw, Task.dest / raw.last) + PathRef(Task.dest / raw.last) } def prependShellScript = Task { val (millArgs, otherArgs) = forkArgs().partition(arg => arg.startsWith("-DMILL") && !arg.startsWith("-DMILL_VERSION")) // Pass Mill options via file, due to small max args limit in Windows - val vmOptionsFile = T.dest / "mill.properties" + val vmOptionsFile = Task.dest / "mill.properties" val millOptionsContent = millArgs.map(_.drop(2).replace("\\", "/")).mkString( "\r\n" @@ -890,11 +890,11 @@ object dist extends MillPublishJavaModule { Jvm.createJar(Agg(), JarManifest(manifestEntries)) } - def run(args: Task[Args] = T.task(Args())) = Task.Command(exclusive = true) { + def run(args: Task[Args] = Task.Anon(Args())) = Task.Command(exclusive = true) { args().value match { case Nil => mill.api.Result.Failure("Need to pass in cwd as first argument to dev.run") case wd0 +: rest => - val wd = os.Path(wd0, T.workspace) + val wd = os.Path(wd0, Task.workspace) os.makeDir.all(wd) try { Jvm.runSubprocess( @@ -918,32 +918,32 @@ object dist extends MillPublishJavaModule { * @param ivyRepo The local Ivy repository where Mill modules should be published to */ def installLocal(binFile: String = DefaultLocalMillReleasePath, ivyRepo: String = null) = - T.command { - PathRef(installLocalTask(T.task(binFile), ivyRepo)()) + Task.Command { + PathRef(installLocalTask(Task.Anon(binFile), ivyRepo)()) } -def installLocalCache() = T.command { +def installLocalCache() = Task.Command { val path = installLocalTask( - T.task((os.home / ".cache" / "mill" / "download" / millVersion()).toString()) + Task.Anon((os.home / ".cache" / "mill" / "download" / millVersion()).toString()) )() - T.log.outputStream.println(path.toString()) + Task.log.outputStream.println(path.toString()) PathRef(path) } -def installLocalTask(binFile: Task[String], ivyRepo: String = null): Task[os.Path] = T.task { +def installLocalTask(binFile: Task[String], ivyRepo: String = null): Task[os.Path] = Task.Anon { val millBin = dist.assembly() - val targetFile = os.Path(binFile(), T.workspace) + val targetFile = os.Path(binFile(), Task.workspace) if (os.exists(targetFile)) - T.log.info(s"Overwriting existing local Mill binary at ${targetFile}") + Task.log.info(s"Overwriting existing local Mill binary at ${targetFile}") os.copy.over(millBin.path, targetFile, createFolders = true) - T.log.info(s"Published ${dist.allPublishModules.size} modules and installed ${targetFile}") + Task.log.info(s"Published ${dist.allPublishModules.size} modules and installed ${targetFile}") targetFile } -def millBootstrap = T.sources(T.workspace / "mill") +def millBootstrap = Task.Sources(Task.workspace / "mill") def bootstrapLauncher = Task { - val outputPath = T.dest / "mill" + val outputPath = Task.dest / "mill" val millBootstrapGrepPrefix = "(\n *DEFAULT_MILL_VERSION=)" val millDownloadUrlPrefix = "(\n *MILL_DOWNLOAD_URL=)" @@ -959,12 +959,12 @@ def bootstrapLauncher = Task { PathRef(outputPath) } -def examplePathsWithArtifactName:Task[Seq[(os.Path,String)]] = T.task{ +def examplePathsWithArtifactName:Task[Seq[(os.Path,String)]] = Task.Anon{ for { exampleMod <- build.example.exampleModules path = exampleMod.millSourcePath } yield { - val example = path.subRelativeTo(T.workspace) + val example = path.subRelativeTo(Task.workspace) val artifactName = millVersion() + "-" + example.segments.mkString("-") (path, artifactName) } @@ -973,16 +973,16 @@ def examplePathsWithArtifactName:Task[Seq[(os.Path,String)]] = T.task{ def exampleZips: T[Seq[PathRef]] = Task { examplePathsWithArtifactName().map{ case (examplePath, exampleStr) => - os.copy(examplePath, T.dest / exampleStr, createFolders = true) - os.write(T.dest / exampleStr / ".mill-version", millLastTag()) - os.copy(bootstrapLauncher().path, T.dest / exampleStr / "mill") - val zip = T.dest / s"$exampleStr.zip" - os.proc("zip", "-r", zip, exampleStr).call(cwd = T.dest) + os.copy(examplePath, Task.dest / exampleStr, createFolders = true) + os.write(Task.dest / exampleStr / ".mill-version", millLastTag()) + os.copy(bootstrapLauncher().path, Task.dest / exampleStr / "mill") + val zip = Task.dest / s"$exampleStr.zip" + os.proc("zip", "-r", zip, exampleStr).call(cwd = Task.dest) PathRef(zip) } } -def uploadToGithub(authKey: String) = T.command { +def uploadToGithub(authKey: String) = Task.Command { val vcsState = VcsVersion.vcsState() val label = vcsState.format() if (label != millVersion()) sys.error("Modified mill version detected, aborting upload") @@ -1030,8 +1030,8 @@ def validate(): Command[Unit] = { val tasks = resolveTasks("__.compile", "__.minaReportBinaryIssues") val sources = resolveTasks("__.sources") - T.command { - T.sequence(tasks)() + Task.Command { + Task.sequence(tasks)() mill.scalalib.scalafmt.ScalafmtModule.checkFormatAll(Tasks(sources))() build.docs.localPages() () diff --git a/contrib/scalapblib/readme.adoc b/contrib/scalapblib/readme.adoc index 6f1b0696e2b..bddf2e3aa5f 100644 --- a/contrib/scalapblib/readme.adoc +++ b/contrib/scalapblib/readme.adoc @@ -78,6 +78,6 @@ object example extends ScalaPBModule { def scalaVersion = "2.12.6" def scalaPBVersion = "0.7.4" override def scalaPBAdditionalArgs = - Seq(s"--zio_out=${T.dest.toIO.getCanonicalPath}") + Seq(s"--zio_out=${Task.dest.toIO.getCanonicalPath}") } ---- diff --git a/docs/modules/ROOT/images/basic/VisualizeCompiles.svg b/docs/modules/ROOT/images/basic/VisualizeCompiles.svg new file mode 100644 index 00000000000..0bcc404a14f --- /dev/null +++ b/docs/modules/ROOT/images/basic/VisualizeCompiles.svg @@ -0,0 +1,49 @@ + + +example1 + + + +bar.compile + +bar.compile + + + +bar.test.compile + +bar.test.compile + + + +bar.compile->bar.test.compile + + + + + +foo.compile + +foo.compile + + + +bar.compile->foo.compile + + + + + +foo.test.compile + +foo.test.compile + + + +foo.compile->foo.test.compile + + + + + diff --git a/docs/modules/ROOT/images/basic/VisualizeTestDeps.svg b/docs/modules/ROOT/images/basic/VisualizeTestDeps.svg new file mode 100644 index 00000000000..5bf2abaecde --- /dev/null +++ b/docs/modules/ROOT/images/basic/VisualizeTestDeps.svg @@ -0,0 +1,55 @@ + + +example1 + + + +baz.compile + +baz.compile + + + +baz.test.compile + +baz.test.compile + + + +baz.compile->baz.test.compile + + + + + +qux.compile + +qux.compile + + + +baz.compile->qux.compile + + + + + +qux.test.compile + +qux.test.compile + + + +baz.test.compile->qux.test.compile + + + + + +qux.compile->qux.test.compile + + + + + diff --git a/docs/modules/ROOT/nav.adoc b/docs/modules/ROOT/nav.adoc index f0e9e09e4ae..6a85f69cef6 100644 --- a/docs/modules/ROOT/nav.adoc +++ b/docs/modules/ROOT/nav.adoc @@ -61,8 +61,7 @@ // either section above, it is probably an important enough topic it is worth // breaking out on its own .Extending Mill -* xref:extending/import-ivy.adoc[] -* xref:extending/using-plugins.adoc[] +* xref:extending/import-ivy-plugins.adoc[] * xref:extending/contrib-plugins.adoc[] // See also the list in Contrib_Plugins.adoc ** xref:contrib/artifactory.adoc[] diff --git a/docs/modules/ROOT/pages/depth/design-principles.adoc b/docs/modules/ROOT/pages/depth/design-principles.adoc index c3235f73169..721d71ae55d 100644 --- a/docs/modules/ROOT/pages/depth/design-principles.adoc +++ b/docs/modules/ROOT/pages/depth/design-principles.adoc @@ -66,9 +66,9 @@ is what allows us to be aggressive about caching and parallelizing the evaluation of build tasks during a build. Many kinds of build steps do require files on disk, and for that Mill provides -the `T.dest` folder. This is a folder on disk dedicated to each build task, +the `Task.dest` folder. This is a folder on disk dedicated to each build task, so that it can read and write things to it without worrying about conflicts -with other tasks that have their own `T.dest` folders. In effect, this makes +with other tasks that have their own `Task.dest` folders. In effect, this makes even file output "pure": we can know precisely where a task's output files live when we need to invalidate them, and it allows multiple tasks all reading and writing to the filesystem to do so safely even when in parallel. diff --git a/docs/modules/ROOT/pages/depth/evaluation-model.adoc b/docs/modules/ROOT/pages/depth/evaluation-model.adoc index aa877407500..fabdd3088aa 100644 --- a/docs/modules/ROOT/pages/depth/evaluation-model.adoc +++ b/docs/modules/ROOT/pages/depth/evaluation-model.adoc @@ -70,10 +70,10 @@ the overall workflow remains fast even for large projects: * ``Task``s are evaluated in dependency order - * xref:fundamentals/tasks.adoc#_targets[Target]s only re-evaluate if their input ``Task``s + * xref:fundamentals/tasks.adoc#_cached_tasks[Cached Task]s only re-evaluate if their input ``Task``s change. - * xref:fundamentals/tasks.adoc#_persistent_tasks[Task.Persistent]s preserve the `T.dest` folder on disk between runs, + * xref:fundamentals/tasks.adoc#_persistent_tasks[Task.Persistent]s preserve the `Task.dest` folder on disk between runs, allowing for finer-grained caching than Mill's default task-by-task caching and invalidation diff --git a/docs/modules/ROOT/pages/depth/large-builds.adoc b/docs/modules/ROOT/pages/depth/large-builds.adoc index 012e2fa0d88..a8d9f82bd69 100644 --- a/docs/modules/ROOT/pages/depth/large-builds.adoc +++ b/docs/modules/ROOT/pages/depth/large-builds.adoc @@ -1,6 +1,17 @@ = Structuring Large Builds :page-aliases: Structuring_Large_Builds.adoc +This section walks through Mill features and techniques used for managing large builds. +While Mill works great for small single-module projects, it is also able to work +effectively with large projects with hundreds of modules. Mill's own build for the +https://github.com/com-lihaoyi/mill[com-lihaoyi/mill] project has ~400 modules, and +other proprietary projects may have many more. + +Mill modules are cheap: having more modules does not significantly impact performance +or resource usage, so you are encouraged to break up your project into modules to manage +the layering of your codebase or benefit from parallelism. + + == Multi-file Builds include::partial$example/depth/large/10-multi-file-builds.adoc[] diff --git a/docs/modules/ROOT/pages/extending/contrib-plugins.adoc b/docs/modules/ROOT/pages/extending/contrib-plugins.adoc index 42ad5927491..ca920f8bb51 100644 --- a/docs/modules/ROOT/pages/extending/contrib-plugins.adoc +++ b/docs/modules/ROOT/pages/extending/contrib-plugins.adoc @@ -4,7 +4,7 @@ The ((plugins)) in this section are hosted in the Mill git tree and developed / maintained by the community. -For details about including plugins in your `build.mill` read xref:extending/using-plugins.adoc[Using Mill Plugins]. +For details about including plugins in your `build.mill` read xref:extending/import-ivy-plugins.adoc[Using Mill Plugins]. [CAUTION] -- @@ -28,6 +28,11 @@ import $ivy.`com.lihaoyi::mill-contrib-bloop:` -- +== Importing Contrib Modules + +include::partial$example/extending/imports/3-contrib-import.adoc[] + + == List of Contrib Plugins // See also the list in nav.adoc @@ -50,6 +55,3 @@ import $ivy.`com.lihaoyi::mill-contrib-bloop:` * xref:contrib/versionfile.adoc[] -== Importing Contrib Modules - -include::partial$example/extending/imports/3-contrib-import.adoc[] diff --git a/docs/modules/ROOT/pages/extending/using-plugins.adoc b/docs/modules/ROOT/pages/extending/import-ivy-plugins.adoc similarity index 56% rename from docs/modules/ROOT/pages/extending/using-plugins.adoc rename to docs/modules/ROOT/pages/extending/import-ivy-plugins.adoc index e335900dcbe..2f3254f5e3f 100644 --- a/docs/modules/ROOT/pages/extending/using-plugins.adoc +++ b/docs/modules/ROOT/pages/extending/import-ivy-plugins.adoc @@ -1,16 +1,40 @@ -= Using Plugins -:page-aliases: Using_Plugins.adoc -Mill plugins are ordinary jars and are loaded as any other external dependency with the xref:extending/import-ivy.adoc[`import $ivy` mechanism]. += Import Libraries and Plugins +:page-aliases: Import_File_And_Import_Ivy.adoc, Using_Plugins.adoc -There exist a large number of Mill plugins, Many of them are available on GitHub and via Maven Central. We also have a list of plugins, which is most likely not complete, but it might be a good start if you are looking for plugins: xref:Thirdparty_Plugins.adoc[]. +This page illustrates usage of `import $ivy`. +`import $ivy` lets you import JVM dependencies into your `build.mill`, so +you can use arbitrary third-party libraries at build-time. This makes +lets you perform computations at build-time rather than run-time, +speeding up your application start up. `import $ivy` can be used on any +JVM library on Maven Central. + + +== Importing Java Libraries + +include::partial$example/extending/imports/1-import-ivy.adoc[] + + +== Importing Scala Libraries + +include::partial$example/extending/imports/2-import-ivy-scala.adoc[] + +== Importing Plugins + +Mill plugins are ordinary JVM libraries jars and are loaded as any other external dependency with +the xref:extending/import-ivy-plugins.adoc[`import $ivy` mechanism]. + +There exist a large number of Mill plugins, Many of them are available on GitHub and via +Maven Central. We also have a list of plugins, which is most likely not complete, but it +might be a good start if you are looking for plugins: xref:Thirdparty_Plugins.adoc[]. Some plugin contributions are also hosted in Mill's own git tree as xref:Contrib_Plugins.adoc[]. Mill plugins are typically bound to a specific version range of Mill. -This is called the binary platform. -To ease the use of the correct versions and avoid runtime issues (caused by binary incompatible plugins, which are hard to debug) you can apply one of the following techniques: +This is called the binary platform. To ease the use of the correct versions and avoid runtime +issues (caused by binary incompatible plugins, which are hard to debug) you can apply one of the +following techniques: -== Use the specific Mill Binary Platform notation +=== Use the specific Mill Binary Platform notation [source,scala] ---- @@ -33,7 +57,7 @@ import $ivy.`:::_mill$MILL_BIN_PLATFORM:` ---- -== Use special placeholders in your `import $ivy` +=== Use special placeholders in your `import $ivy` `$MILL_VERSION` :: + diff --git a/docs/modules/ROOT/pages/extending/import-ivy.adoc b/docs/modules/ROOT/pages/extending/import-ivy.adoc deleted file mode 100644 index 44746c7507f..00000000000 --- a/docs/modules/ROOT/pages/extending/import-ivy.adoc +++ /dev/null @@ -1,19 +0,0 @@ -= import $ivy -:page-aliases: Import_File_And_Import_Ivy.adoc - -// This page illustrates usage of `import $ivy`. -// `import $ivy` lets you import JVM dependencies into your `build.mill`, so -// you can use arbitrary third-party libraries at build-time. This makes -// lets you perform computations at build-time rather than run-time, -// speeding up your application start up. `import $ivy` can be used on any -// JVM library on Maven Central. -// - -== Importing Java Libraries - -include::partial$example/extending/imports/1-import-ivy.adoc[] - - -== Importing Scala Libraries - -include::partial$example/extending/imports/2-import-ivy-scala.adoc[] \ No newline at end of file diff --git a/docs/modules/ROOT/pages/extending/meta-build.adoc b/docs/modules/ROOT/pages/extending/meta-build.adoc index 8cf595d5697..77d26f0c43a 100644 --- a/docs/modules/ROOT/pages/extending/meta-build.adoc +++ b/docs/modules/ROOT/pages/extending/meta-build.adoc @@ -2,15 +2,19 @@ :page-aliases: The_Mill_Meta_Build.adoc The meta-build manages the compilation of the `build.mill`. -If you don't configure it explicitly, a built-in synthetic meta-build is used. +Customizing the meta-build gives you greater control over how exactly your +`build.mill` evaluates. + To customize it, you need to explicitly enable it with `import $meta._`. Once enabled, the meta-build lives in the `mill-build/` directory. It needs to contain a top-level module of type `MillBuildRootModule`. +If you don't configure it explicitly, a built-in synthetic meta-build is used. Meta-builds are recursive, which means, it can itself have a nested meta-builds, and so on. -To run a task on a meta-build, you specifying the `--meta-level` option to select the meta-build level. +To run a task on a meta-build, you specify the `--meta-level` option to select +the meta-build level. == Autoformatting the `build.mill` @@ -20,7 +24,7 @@ You only need a `.scalafmt.conf` config file which at least needs configure the .Run Scalafmt on the `build.mill` (and potentially included files) ---- -$ mill --meta-level 1 mill.scalalib.scalafmt.ScalafmtModule/reformatAll sources +$ mill --meta-level 1 mill.scalalib.scalafmt.ScalafmtModule/ ---- * `--meta-level 1` selects the first meta-build. Without any customization, this is the only built-in meta-build. diff --git a/docs/modules/ROOT/pages/extending/thirdparty-plugins.adoc b/docs/modules/ROOT/pages/extending/thirdparty-plugins.adoc index 1cb7e1c097a..3684e8c70fd 100644 --- a/docs/modules/ROOT/pages/extending/thirdparty-plugins.adoc +++ b/docs/modules/ROOT/pages/extending/thirdparty-plugins.adoc @@ -5,7 +5,7 @@ The Plugins in this section are developed/maintained outside the mill git tree. This list is most likely not complete. If you wrote a Mill plugin or find that one is missing in this list, please open a {mill-github-url}/pulls[pull request] and add that plugin with a short description (in alphabetical order). -For details about including plugins in your `build.mill` read xref:extending/using-plugins.adoc[Using Mill Plugins]. +For details about including plugins in your `build.mill` read xref:extending/import-ivy-plugins.adoc[Using Mill Plugins]. CAUTION: Besides the documentation provided here, we urge you to consult the respective linked plugin documentation pages. The usage examples given here are most probably incomplete and sometimes outdated! diff --git a/docs/modules/ROOT/pages/fundamentals/cross-builds.adoc b/docs/modules/ROOT/pages/fundamentals/cross-builds.adoc index 9779f86bec9..5f4bf60f67c 100644 --- a/docs/modules/ROOT/pages/fundamentals/cross-builds.adoc +++ b/docs/modules/ROOT/pages/fundamentals/cross-builds.adoc @@ -11,7 +11,7 @@ config and building it across a variety of source folders. include::partial$example/fundamentals/cross/1-simple.adoc[] -== Default Cross Modules +== Cross Modules Defaults include::partial$example/fundamentals/cross/11-default-cross-module.adoc[] diff --git a/docs/modules/ROOT/pages/fundamentals/modules.adoc b/docs/modules/ROOT/pages/fundamentals/modules.adoc index 86a811eed4e..3036bcb3b74 100644 --- a/docs/modules/ROOT/pages/fundamentals/modules.adoc +++ b/docs/modules/ROOT/pages/fundamentals/modules.adoc @@ -1,5 +1,6 @@ = Modules :page-aliases: Modules.adoc + `mill.Module` serves two main purposes: 1. As ``object``s, they serve as namespaces that let you group related ``Task``s diff --git a/docs/modules/ROOT/pages/fundamentals/out-dir.adoc b/docs/modules/ROOT/pages/fundamentals/out-dir.adoc index a03d5da57e2..271db12464e 100644 --- a/docs/modules/ROOT/pages/fundamentals/out-dir.adoc +++ b/docs/modules/ROOT/pages/fundamentals/out-dir.adoc @@ -52,7 +52,7 @@ out/ ---- <1> The `main` directory contains all files associated with tasks and submodules of the `main` module. -<2> The `compile` task has tried to access its scratch space via `T.dest`. Here you will find the actual compile results. +<2> The `compile` task has tried to access its scratch space via `Task.dest`. Here you will find the actual compile results. <3> Two tasks printed something out while they ran. You can find these outputs in the `*.log` files. <4> Three tasks are overridden but re-use the result of their `super`-tasks in some way. You can find these result under the `*.super/` path. @@ -75,7 +75,7 @@ by `foo.json` via `PathRef` references. `foo.dest/`:: optional, a path for the `Task` to use either as a scratch space, or to place generated files that are returned using `PathRef` references. -A `Task` should only output files within its own given `foo.dest/` folder (available as `T.dest`) to avoid +A `Task` should only output files within its own given `foo.dest/` folder (available as `Task.dest`) to avoid conflicting with another `Task`, but can name files within `foo.dest/` arbitrarily. `foo.log`:: diff --git a/docs/modules/ROOT/pages/fundamentals/tasks.adoc b/docs/modules/ROOT/pages/fundamentals/tasks.adoc index e2c8702282f..4e9792396aa 100644 --- a/docs/modules/ROOT/pages/fundamentals/tasks.adoc +++ b/docs/modules/ROOT/pages/fundamentals/tasks.adoc @@ -3,7 +3,7 @@ One of Mill's core abstractions is its _Task Graph_: this is how Mill defines, orders and caches work it needs to do, and exists independently of any support -for building Scala. +for building Java, Kotlin, or Scala. Mill task graphs are primarily built using methods and macros defined on `mill.define.Task`, aliased as `T` for conciseness: @@ -17,7 +17,7 @@ different Task types: [cols="<,<,<,<,<,<,<"] |=== -| |Target |Command |Source/Input |Anonymous Task |Persistent Target |Worker +| |Target |Command |Source/Input |Anonymous Task |Persistent Task |Worker |Cached to Disk |X | | | |X | |JSON Writable |X |X |X| |X | diff --git a/docs/modules/ROOT/pages/javalib/android-examples.adoc b/docs/modules/ROOT/pages/javalib/android-examples.adoc index 28a26118678..2b3e71c40a2 100644 --- a/docs/modules/ROOT/pages/javalib/android-examples.adoc +++ b/docs/modules/ROOT/pages/javalib/android-examples.adoc @@ -15,9 +15,9 @@ a starting point for further experimentation and development. These are the main Mill Modules that are relevant for building Android apps: -* {mill-doc-url}/api/latest/mill/scalalib/AndroidSdkModule.html[`mill.scalalib.AndroidSdkModule`]: Handles Android SDK management and tools. +* {mill-doc-url}/api/latest/mill/javalib/android/AndroidSdkModule.html[`mill.javalib.android.AndroidSdkModule`]: Handles Android SDK management and tools. * {mill-doc-url}/api/latest/mill/javalib/android/AndroidAppModule.html[`mill.javalib.android.AndroidAppModule`]: Provides a framework for building Android applications. -* {mill-doc-url}/api/latest/mill/scalalib/JavaModule.html[`mill.scalalib.JavaModule`]: General Java build tasks like compiling Java code and creating JAR files. +* {mill-doc-url}/api/latest/mill/scalalib/JavaModule.html[`mill.javalib.JavaModule`]: General Java build tasks like compiling Java code and creating JAR files. == Simple Android Hello World Application diff --git a/docs/modules/ROOT/pages/javalib/dependencies.adoc b/docs/modules/ROOT/pages/javalib/dependencies.adoc index fbcb34b149c..70f86237e2f 100644 --- a/docs/modules/ROOT/pages/javalib/dependencies.adoc +++ b/docs/modules/ROOT/pages/javalib/dependencies.adoc @@ -22,9 +22,9 @@ include::partial$example/javalib/dependencies/2-run-compile-deps.adoc[] include::partial$example/javalib/dependencies/3-unmanaged-jars.adoc[] -== Downloading Non-Maven Jars +== Downloading Unmanaged Jars -include::partial$example/javalib/dependencies/4-downloading-non-maven-jars.adoc[] +include::partial$example/javalib/dependencies/4-downloading-unmanaged-jars.adoc[] == Repository Config diff --git a/docs/modules/ROOT/pages/javalib/module-config.adoc b/docs/modules/ROOT/pages/javalib/module-config.adoc index fbd934a31d4..d5bf9df68a5 100644 --- a/docs/modules/ROOT/pages/javalib/module-config.adoc +++ b/docs/modules/ROOT/pages/javalib/module-config.adoc @@ -7,6 +7,9 @@ gtag('config', 'AW-16649289906'); ++++ +:language: Java +:language-small: java + This page goes into more detail about the various configuration options for `JavaModule`. @@ -18,15 +21,6 @@ Many of the APIs covered here are listed in the API documentation: include::partial$example/javalib/module/1-common-config.adoc[] -== Custom Tasks - -include::partial$example/javalib/module/2-custom-tasks.adoc[] - -== Overriding Tasks - -include::partial$example/javalib/module/3-override-tasks.adoc[] - - == Compilation & Execution Flags include::partial$example/javalib/module/4-compilation-execution-flags.adoc[] @@ -55,8 +49,14 @@ include::partial$example/javalib/module/11-main-class.adoc[] include::partial$example/javalib/module/13-assembly-config.adoc[] +== Custom Tasks + +include::partial$example/javalib/module/2-custom-tasks.adoc[] + +== Overriding Tasks + +include::partial$example/javalib/module/3-override-tasks.adoc[] == Native C Code with JNI include::partial$example/javalib/module/15-jni.adoc[] - diff --git a/docs/modules/ROOT/pages/kotlinlib/android-examples.adoc b/docs/modules/ROOT/pages/kotlinlib/android-examples.adoc index 3ca6163e191..e45ec649f12 100644 --- a/docs/modules/ROOT/pages/kotlinlib/android-examples.adoc +++ b/docs/modules/ROOT/pages/kotlinlib/android-examples.adoc @@ -15,7 +15,7 @@ a starting point for further experimentation and development. These are the main Mill Modules that are relevant for building Android apps: -* {mill-doc-url}/api/latest/mill/scalalib/AndroidSdkModule.html[`mill.scalalib.AndroidSdkModule`]: Handles Android SDK management and tools. +* {mill-doc-url}/api/latest/mill/javalib/android/AndroidSdkModule.html[`mill.javalib.android.AndroidSdkModule`]: Handles Android SDK management and tools. * {mill-doc-url}/api/latest/mill/kotlinlib/android/AndroidAppKotlinModule.html[`mill.kotlinlib.android.AndroidAppKotlinModule`]: Provides a framework for building Android applications. * {mill-doc-url}/api/latest/mill/kotlinlib/KotlinModule.html[`mill.kotlinlib.KotlinModule`]: General Kotlin build tasks like compiling Kotlin code and creating JAR files. diff --git a/docs/modules/ROOT/pages/kotlinlib/dependencies.adoc b/docs/modules/ROOT/pages/kotlinlib/dependencies.adoc index 7660b5aed8b..fc4440e85fd 100644 --- a/docs/modules/ROOT/pages/kotlinlib/dependencies.adoc +++ b/docs/modules/ROOT/pages/kotlinlib/dependencies.adoc @@ -27,9 +27,9 @@ include::partial$example/kotlinlib/dependencies/2-run-compile-deps.adoc[] include::partial$example/kotlinlib/dependencies/3-unmanaged-jars.adoc[] -== Downloading Non-Maven Jars +== Downloading Unmanaged Jars -include::partial$example/kotlinlib/dependencies/4-downloading-non-maven-jars.adoc[] +include::partial$example/kotlinlib/dependencies/4-downloading-unmanaged-jars.adoc[] == Repository Config diff --git a/docs/modules/ROOT/pages/kotlinlib/intro.adoc b/docs/modules/ROOT/pages/kotlinlib/intro.adoc index 28426f07056..e956a775784 100644 --- a/docs/modules/ROOT/pages/kotlinlib/intro.adoc +++ b/docs/modules/ROOT/pages/kotlinlib/intro.adoc @@ -25,6 +25,13 @@ gtag('config', 'AW-16649289906'); :language-small: kotlin :language-ext: kt +NOTE: *Kotlin support in Mill is experimental*! A lot of stuff works, which is +documented under this section, but a lot of stuff doesn't. In particular, +support for Android, KotlinJS and Kotlin-Multi-Platform is still in its infancy. +The API is not yet stable and may evolve. Try it out but please be aware of its +limitations! + + include::partial$Intro_to_Mill_Header.adoc[] diff --git a/docs/modules/ROOT/pages/kotlinlib/module-config.adoc b/docs/modules/ROOT/pages/kotlinlib/module-config.adoc index 2b10af5a3bf..5c858ec0369 100644 --- a/docs/modules/ROOT/pages/kotlinlib/module-config.adoc +++ b/docs/modules/ROOT/pages/kotlinlib/module-config.adoc @@ -7,6 +7,9 @@ gtag('config', 'AW-16649289906'); ++++ +:language: Kotlin +:language-small: kotlin + This page goes into more detail about the various configuration options for `KotlinModule`. @@ -20,14 +23,6 @@ Many of the APIs covered here are listed in the API documentation: include::partial$example/kotlinlib/module/1-common-config.adoc[] -== Custom Tasks - -include::partial$example/kotlinlib/module/2-custom-tasks.adoc[] - -== Overriding Tasks - -include::partial$example/kotlinlib/module/3-override-tasks.adoc[] - == Compilation & Execution Flags @@ -41,7 +36,7 @@ include::partial$example/kotlinlib/module/7-resources.adoc[] include::partial$example/kotlinlib/module/8-kotlin-compiler-plugins.adoc[] -== Javadoc Config +== Doc-Jar Generation include::partial$example/kotlinlib/module/9-docjar.adoc[] @@ -54,7 +49,16 @@ include::partial$example/kotlinlib/module/11-main-class.adoc[] include::partial$example/kotlinlib/module/13-assembly-config.adoc[] + +== Custom Tasks + +include::partial$example/kotlinlib/module/2-custom-tasks.adoc[] + +== Overriding Tasks + +include::partial$example/kotlinlib/module/3-override-tasks.adoc[] + + == Native C Code with JNI include::partial$example/kotlinlib/module/15-jni.adoc[] - diff --git a/docs/modules/ROOT/pages/scalalib/dependencies.adoc b/docs/modules/ROOT/pages/scalalib/dependencies.adoc index ec754cab751..033e0976ff0 100644 --- a/docs/modules/ROOT/pages/scalalib/dependencies.adoc +++ b/docs/modules/ROOT/pages/scalalib/dependencies.adoc @@ -26,9 +26,9 @@ include::partial$example/scalalib/dependencies/2-run-compile-deps.adoc[] include::partial$example/scalalib/dependencies/3-unmanaged-jars.adoc[] -== Downloading Non-Maven Jars +== Downloading Unmanaged Jars -include::partial$example/scalalib/dependencies/4-downloading-non-maven-jars.adoc[] +include::partial$example/scalalib/dependencies/4-downloading-unmanaged-jars.adoc[] == Repository Config diff --git a/docs/modules/ROOT/pages/scalalib/intro.adoc b/docs/modules/ROOT/pages/scalalib/intro.adoc index d0706a2c0c2..0e807800568 100644 --- a/docs/modules/ROOT/pages/scalalib/intro.adoc +++ b/docs/modules/ROOT/pages/scalalib/intro.adoc @@ -41,7 +41,7 @@ Compared to SBT: * **Mill makes customizing the build yourself much easier**: most of what build tools do work with files and call subprocesses, and Mill makes doing that yourself easy. This means you can always make your Mill build do exactly what you want, and are not - beholden to third-party plugins that may not exist, be well maintained, or interact well + beholden to third-party plugins that may not meet your exact needs or interact well with each other. * **Mill is much more performant**: SBT has enough overhead that even a dozen diff --git a/docs/modules/ROOT/pages/scalalib/module-config.adoc b/docs/modules/ROOT/pages/scalalib/module-config.adoc index 73d1f89c79d..936169c0313 100644 --- a/docs/modules/ROOT/pages/scalalib/module-config.adoc +++ b/docs/modules/ROOT/pages/scalalib/module-config.adoc @@ -7,6 +7,8 @@ gtag('config', 'AW-16649289906'); ++++ +:language: Scala +:language-small: scala This page goes into more detail about the various configuration options for `ScalaModule`. @@ -20,15 +22,6 @@ Many of the APIs covered here are listed in the Scaladoc: include::partial$example/scalalib/module/1-common-config.adoc[] -== Custom Tasks - -include::partial$example/scalalib/module/2-custom-tasks.adoc[] - -== Overriding Tasks - -include::partial$example/scalalib/module/3-override-tasks.adoc[] - - == Compilation & Execution Flags include::partial$example/scalalib/module/4-compilation-execution-flags.adoc[] @@ -57,6 +50,14 @@ include::partial$example/scalalib/module/13-assembly-config.adoc[] include::partial$example/scalalib/module/15-unidoc.adoc[] +== Custom Tasks + +include::partial$example/scalalib/module/2-custom-tasks.adoc[] + +== Overriding Tasks + +include::partial$example/scalalib/module/3-override-tasks.adoc[] + == Using the Ammonite Repl / Scala console diff --git a/docs/modules/ROOT/partials/Intro_Maven_Gradle_Comparison.adoc b/docs/modules/ROOT/partials/Intro_Maven_Gradle_Comparison.adoc index 464f3f43778..88c63934edb 100644 --- a/docs/modules/ROOT/partials/Intro_Maven_Gradle_Comparison.adoc +++ b/docs/modules/ROOT/partials/Intro_Maven_Gradle_Comparison.adoc @@ -19,7 +19,7 @@ xref:comparisons/maven.adoc[Compared to Maven]: grow beyond just compiling a single language: needing custom code generation, linting workflows, tool integrations, output artifacts, or support for additional languages. Mill makes doing this yourself easy, so you are not beholden - to third-party plugins that may not exist or interact well with each other. + to third-party plugins that may not meet your exact needs or interact well with each other. * **Mill automatically caches and parallelizes your build**: Not just the built-in tasks that Mill ships with, but also any custom tasks or modules. diff --git a/docs/modules/ROOT/partials/Intro_to_Mill_BlogVideo.adoc b/docs/modules/ROOT/partials/Intro_to_Mill_BlogVideo.adoc index 796dc4a51b5..511d0faee47 100644 --- a/docs/modules/ROOT/partials/Intro_to_Mill_BlogVideo.adoc +++ b/docs/modules/ROOT/partials/Intro_to_Mill_BlogVideo.adoc @@ -3,7 +3,7 @@ If you're interested in the fundamental ideas behind Mill, rather than the user-facing benefits discussed above, check out the section on Mill Design Principles: -- <> +- xref:depth/design-principles.adoc[Mill Design Principles] The rest of this page contains a quick introduction to getting start with using Mill to build a simple {language} program. The other pages of this doc-site go into diff --git a/docs/modules/ROOT/partials/Intro_to_Mill_Header.adoc b/docs/modules/ROOT/partials/Intro_to_Mill_Header.adoc index 196f99f731b..bc2e131ff22 100644 --- a/docs/modules/ROOT/partials/Intro_to_Mill_Header.adoc +++ b/docs/modules/ROOT/partials/Intro_to_Mill_Header.adoc @@ -50,7 +50,7 @@ even as it grows from a small project to a large codebase or monorepo with hundr * *Flexibility*: Mill's custom tasks and modules allow anything from xref:fundamentals/tasks.adoc#primitive-tasks[adding simple build steps], up to xref:fundamentals/modules.adoc#_use_case_diy_java_modules[entire language toolchains]. - You can xref:extending/import-ivy.adoc[import any JVM library as part of your build], + You can xref:extending/import-ivy-plugins.adoc[import any JVM library as part of your build], use Mill's rich ecosystem of xref:extending/thirdparty-plugins.adoc[Third-Party Mill Plugins], - or xref:extending/writing-plugins.adoc[write plugins yourself] and publish them - on Maven Central for others to use. + or xref:extending/writing-plugins.adoc[write plugins yourself] and + xref:extending/writing-plugins.adoc#_publishing[publish them on Maven Central] for others to use. diff --git a/docs/package.mill b/docs/package.mill index b870a130a85..3768b9788fd 100644 --- a/docs/package.mill +++ b/docs/package.mill @@ -119,7 +119,9 @@ object `package` extends RootModule { if (inputs.isEmpty) output((p, i)) = graphvizLines.mkString("\n") else { outputLines.append("++++") + outputLines.append("
") outputLines.append(inputs((p, i))) + outputLines.append("
") outputLines.append("++++") } diff --git a/example/depth/large/10-multi-file-builds/build.mill b/example/depth/large/10-multi-file-builds/build.mill index a20bf39d3b5..04fb82a1cf0 100644 --- a/example/depth/large/10-multi-file-builds/build.mill +++ b/example/depth/large/10-multi-file-builds/build.mill @@ -1,3 +1,12 @@ +// Mill allows you to break up your `build.mill` file into smaller files by defining the +// build-related logic for any particular subfolder as a `package.mill` file in that subfolder. +// This can be very useful to keep large Mill builds maintainable, as each folder's build logic +// gets co-located with the files that need to be built, and speeds up compilation of the +// build logic since each `build.mill` or `package.mill` file can be compiled independently when +// it is modified without re-compiling all the others. +// +// Usage of sub-folder `package.mill` files is enabled by the magic import `import $packages._` + package build import $packages._ @@ -11,14 +20,6 @@ trait MyModule extends ScalaModule { /** See Also: bar/qux/package.mill */ -// Mill allows you to break up your `build.mill` file into smaller files by defining the -// build-related logic for any particular subfolder as a `package.mill` file in that subfolder. -// This can be very useful to keep large Mill builds maintainable, as each folder's build logic -// gets co-located with the files that need to be built, and speeds up compilation of the -// build logic since each `build.mill` or `package.mill` file can be compiled independently when -// it is modified without re-compiling all the others. -// -// Usage of sub-folder `package.mill` files is enabled by the magic import `import $packages._` // // In this example, the root `build.mill` only contains the `trait MyModule`, but it is // `foo/package.mill` and `bar/qux/package.mill` that define modules using it. The modules diff --git a/example/depth/large/11-helper-files/build.mill b/example/depth/large/11-helper-files/build.mill index 374a8764007..58de0e8eb8b 100644 --- a/example/depth/large/11-helper-files/build.mill +++ b/example/depth/large/11-helper-files/build.mill @@ -1,3 +1,8 @@ + +// Apart from having `package` files in subfolders to define modules, Mill +// also allows you to have helper code in any `*.mill` file in the same folder +// as your `build.mill` or a `package.mill`. + package build import $packages._ import mill._, scalalib._ @@ -16,9 +21,6 @@ object `package` extends RootModule with MyModule{ /** See Also: foo/versions.mill */ -// Apart from having `package` files in subfolders to define modules, Mill -// also allows you to have helper code in any `*.mill` file in the same folder -// as your `build.mill` or a `package.mill`. // // Different helper scripts and ``build.mill``/``package`` files can all refer to // each other using the `build` object, which marks the root object of your build. diff --git a/example/depth/large/12-helper-files-sc/build.sc b/example/depth/large/12-helper-files-sc/build.sc index 836719c7d71..2dc4f1881d4 100644 --- a/example/depth/large/12-helper-files-sc/build.sc +++ b/example/depth/large/12-helper-files-sc/build.sc @@ -1,24 +1,28 @@ +// To ease the migration from Mill 0.11.x, the older `.sc` file extension is also supported +// for Mill build files, and the `package` declaration is optional in such files. Note that +// this means that IDE support using `.sc` files will not be as good as IDE support using the +// current `.mill` extension with `package` declaration, so you should use `.mill` whenever +// possible + + import mill._, scalalib._ import $packages._ import $file.foo.versions import $file.util, util.MyModule object `package` extends RootModule with MyModule{ - def forkEnv = Map( - "MY_SCALA_VERSION" -> build.scalaVersion(), - "MY_PROJECT_VERSION" -> versions.myProjectVersion, - ) + def forkEnv = T{ + Map( + "MY_SCALA_VERSION" -> build.scalaVersion(), + "MY_PROJECT_VERSION" -> versions.myProjectVersion, + ) + } } /** See Also: util.sc */ /** See Also: foo/package.sc */ /** See Also: foo/versions.sc */ -// To ease the migration from Mill 0.11.x, the older `.sc` file extension is also supported -// for Mill build files, and the `package` declaration is optional in such files. Note that -// this means that IDE support using `.sc` files will not be as good as IDE support using the -// current `.mill` extension with `package` declaration, so you should use `.mill` whenever -// possible /** Usage diff --git a/example/depth/sandbox/3-breaking/build.mill b/example/depth/sandbox/3-breaking/build.mill index b1fa4c1b216..3d7c0a2739f 100644 --- a/example/depth/sandbox/3-breaking/build.mill +++ b/example/depth/sandbox/3-breaking/build.mill @@ -12,10 +12,10 @@ package build import mill._, javalib._ -def tWorkspaceTask = Task { println(Task.workspace) } +def myTask = Task { println(Task.workspace) } /** Usage -> ./mill tWorkspaceTask +> ./mill myTask */ // Whereas `MILL_WORKSPACE_ROOT` as well as in tests, which can access the diff --git a/example/extending/imports/1-import-ivy/build.mill b/example/extending/imports/1-import-ivy/build.mill index f93ad19e09b..c9c2095b710 100644 --- a/example/extending/imports/1-import-ivy/build.mill +++ b/example/extending/imports/1-import-ivy/build.mill @@ -27,7 +27,7 @@ object foo extends JavaModule { // This is a toy example: we generate a resource `snippet.txt` containing // `

hello

world

` that the application can read at runtime. // However, it demonstrates how you can easily move logic from application code at runtime -// to build logic at build time, while using the same set of JVM libraries and packages +// to build logic at build time, while using the same set of Java libraries and packages // you are already familiar with. This makes it easy to pre-compute things at build time // to reduce runtime latency or application startup times. // diff --git a/example/extending/imports/3-contrib-import/build.mill b/example/extending/imports/3-contrib-import/build.mill index eeb326c7b77..a847c3d8816 100644 --- a/example/extending/imports/3-contrib-import/build.mill +++ b/example/extending/imports/3-contrib-import/build.mill @@ -24,3 +24,13 @@ foo.BuildInfo.scalaVersion: 2.13.10 */ +// Contrib modules halfway between builtin Mill modules and +// xref:extending/thirdparty-plugins.adoc[Third-party Plugins]: +// +// * Like builtin modules, contrib modules are tested and released as part of +// Mill's own CI process, ensuring there is always a version of the plugin +// compatible with any Mill version +// +// * Like third-party plugins, contrib modules are submitted by third-parties, +// and do now maintain the same binary compatibility guarantees of Mill's +// builtin comdules \ No newline at end of file diff --git a/example/extending/metabuild/3-autoformatting/.scalafmt.conf b/example/extending/metabuild/3-autoformatting/.scalafmt.conf new file mode 100644 index 00000000000..5cfdd6768ce --- /dev/null +++ b/example/extending/metabuild/3-autoformatting/.scalafmt.conf @@ -0,0 +1,4 @@ +version = "3.8.4-RC1" + +runner.dialect = scala213 + diff --git a/example/extending/metabuild/3-autoformatting/build.mill b/example/extending/metabuild/3-autoformatting/build.mill new file mode 100644 index 00000000000..c42a1b0df9c --- /dev/null +++ b/example/extending/metabuild/3-autoformatting/build.mill @@ -0,0 +1,31 @@ +// As an example of running a task on the meta-build, you can format the `build.mill` with Scalafmt. +// Everything is already provided by Mill. +// You only need a `.scalafmt.conf` config file which at least needs configure the Scalafmt version. + +package build +import mill._ + +object foo extends Module {def task=Task{"2.13.4"}} + +/** See Also: .scalafmt.conf */ +/** Usage + +> cat build.mill # build.mill is initially poorly formatted +object foo extends Module {def task=Task{"2.13.4"}} +... + +> mill --meta-level 1 mill.scalalib.scalafmt.ScalafmtModule/ + +> cat build.mill # build.mill is now well formatted +object foo extends Module { def task = Task { "2.13.4" } } +... +*/ + +// +// * `--meta-level 1` selects the first meta-build. Without any customization, this is +// the only built-in meta-build. +// * `mill.scalalib.scalafmt.ScalafmtModule/reformatAll` is a generic task to format scala +// source files with Scalafmt. It requires the tasks that refer to the source files as argument +// * `sources` this selects the `sources` tasks of the meta-build, which at least contains +// the `build.mill`. + diff --git a/example/extending/plugins/7-writing-mill-plugins/build.mill b/example/extending/plugins/7-writing-mill-plugins/build.mill index 451597eb0f0..c18ba92bb62 100644 --- a/example/extending/plugins/7-writing-mill-plugins/build.mill +++ b/example/extending/plugins/7-writing-mill-plugins/build.mill @@ -1,6 +1,6 @@ // This example demonstrates how to write and test Mill plugin, and publish it to // Sonatype's Maven Central so it can be used by other developers over the internet -// via xref:extending/import-ivy.adoc[import $ivy]. +// via xref:extending/import-ivy-plugins.adoc[import $ivy]. // == Project Configuration package build @@ -10,13 +10,18 @@ import mill.main.BuildInfo.millVersion object myplugin extends ScalaModule with PublishModule { def scalaVersion = "2.13.8" + // Set the `platformSuffix` so the name indicates what Mill version it is compiled for + def platformSuffix = "_mill" + mill.main.BuildInfo.millBinPlatform + + // Depend on `mill-dist` so we can compile against Mill APIs def ivyDeps = Agg(ivy"com.lihaoyi:mill-dist:$millVersion") - // Testing Config + // Testing Config, with necessary setup for unit/integration/example tests object test extends ScalaTests with TestModule.Utest{ def ivyDeps = Agg(ivy"com.lihaoyi::mill-testkit:$millVersion") def forkEnv = Map("MILL_EXECUTABLE_PATH" -> millExecutable.assembly().path.toString) + // Create a Mill executable configured for testing our plugin object millExecutable extends JavaModule{ def ivyDeps = Agg(ivy"com.lihaoyi:mill-dist:$millVersion") def mainClass = Some("mill.runner.client.MillClientMain") @@ -51,6 +56,9 @@ object myplugin extends ScalaModule with PublishModule { // and to configure it for publishing to Maven Central via `PublishModule`. // It looks like any other Scala project, except for a few things to take note: // +// * We set the `platformSuffix` to indicate which Mill binary API version +// they are compiled against (to ensure that improper usage raises an easy to understand +// error) // * A dependency on `com.lihaoyi:mill-dist:$millVersion` // * A test dependency on `com.lihaoyi::mill-testkit:$millVersion` // * An `object millExecutable` that adds some resources to the published `mill-dist` @@ -60,6 +68,10 @@ object myplugin extends ScalaModule with PublishModule { // == Plugin Implementation +// Although Mill plugins can contain arbitrary code, the most common +// way that plugins interact with your project is by providing ``trait``s for +// your modules to inherit from. +// // Like any other `trait`, a Mill plugin's traits modules allow you to: // // * Add additional tasks to an existing module @@ -184,12 +196,13 @@ compiling 1 Scala source... > sed -i.bak 's/0.0.1/0.0.2/g' build.mill > ./mill myplugin.publishLocal -Publishing Artifact(com.lihaoyi,myplugin_2.13,0.0.2) to ivy repo... +Publishing Artifact(com.lihaoyi,myplugin_mill0.11_2.13,0.0.2) to ivy repo... */ // Mill plugins are JVM libraries like any other library written in Java or Scala. Thus they // are published the same way: by extending `PublishModule` and defining the module's `publishVersion` -// and `pomSettings`. Once done, you can publish the plugin locally via `publishLocal` below, -// or to Maven Central via `mill.scalalib.public.PublishModule/`for other developers to -// use. +// and `pomSettings`. Once done, you can publish the plugin locally via `publishLocal`, +// or to Maven Central via `mill.scalalib.public.PublishModule/` for other developers to +// use. For more details on publishing Mill projects, see the documentation for +// xref:scalalib/publishing.adoc[Publishing Scala Projects] diff --git a/example/extending/plugins/7-writing-mill-plugins/myplugin/test/resources/example-test-project/build.mill b/example/extending/plugins/7-writing-mill-plugins/myplugin/test/resources/example-test-project/build.mill index 5439fd2a676..0acaea4ee0e 100644 --- a/example/extending/plugins/7-writing-mill-plugins/myplugin/test/resources/example-test-project/build.mill +++ b/example/extending/plugins/7-writing-mill-plugins/myplugin/test/resources/example-test-project/build.mill @@ -1,5 +1,5 @@ package build -import $ivy.`com.lihaoyi::myplugin:0.0.1` +import $ivy.`com.lihaoyi::myplugin::0.0.1` import mill._, myplugin._ object `package` extends RootModule with LineCountJavaModule{ diff --git a/example/extending/plugins/7-writing-mill-plugins/myplugin/test/resources/integration-test-project/build.mill b/example/extending/plugins/7-writing-mill-plugins/myplugin/test/resources/integration-test-project/build.mill index 089665c7193..0c61e9288a2 100644 --- a/example/extending/plugins/7-writing-mill-plugins/myplugin/test/resources/integration-test-project/build.mill +++ b/example/extending/plugins/7-writing-mill-plugins/myplugin/test/resources/integration-test-project/build.mill @@ -1,5 +1,5 @@ package build -import $ivy.`com.lihaoyi::myplugin:0.0.1` +import $ivy.`com.lihaoyi::myplugin::0.0.1` import mill._, myplugin._ object `package` extends RootModule with LineCountJavaModule{ diff --git a/example/fundamentals/cross/10-static-blog/build.mill b/example/fundamentals/cross/10-static-blog/build.mill index aeaf7dc900b..c32ac8a8cf1 100644 --- a/example/fundamentals/cross/10-static-blog/build.mill +++ b/example/fundamentals/cross/10-static-blog/build.mill @@ -1,6 +1,6 @@ // The following example demonstrates a use case: using cross modules to -// turn files on disk into blog posts. To begin with, we import two third-party -// libraries - Commonmark and Scalatags - to deal with Markdown parsing and +// turn files on disk into blog posts. To begin with, we xref:extending/import-ivy-plugins.adoc[import $ivy] +// two third-party libraries - Commonmark and Scalatags - to deal with Markdown parsing and // HTML generation respectively: package build import $ivy.`com.lihaoyi::scalatags:0.12.0`, scalatags.Text.all._ diff --git a/example/fundamentals/cross/11-default-cross-module/build.mill b/example/fundamentals/cross/11-default-cross-module/build.mill index f0d678bfb90..1457be69583 100644 --- a/example/fundamentals/cross/11-default-cross-module/build.mill +++ b/example/fundamentals/cross/11-default-cross-module/build.mill @@ -13,17 +13,18 @@ object bar extends Cross[FooModule]("2.10", "2.11", "2.12") { // For convenience, you can omit the selector for the default cross segment. -// By default, this is the first cross value specified. +// By default, this is the first cross value specified, but you can override +// it by specifying `def defaultCrossSegments` /** Usage -> mill show foo[2.10].suffix +> mill show foo[2.10].suffix # explicit cross value given "_2.10" -> mill show foo[].suffix +> mill show foo[].suffix # no cross value given, defaults to first cross value "_2.10" -> mill show bar[].suffix +> mill show bar[].suffix # no cross value given, defaults to overriden `defaultCrossSegments` "_2.12" */ diff --git a/example/fundamentals/cross/3-outside-dependency/build.mill b/example/fundamentals/cross/3-outside-dependency/build.mill index 7da447d9d18..9ed5d524274 100644 --- a/example/fundamentals/cross/3-outside-dependency/build.mill +++ b/example/fundamentals/cross/3-outside-dependency/build.mill @@ -1,4 +1,6 @@ -// You can refer to tasks defined in cross-modules as follows: +// You can refer to tasks defined in cross-modules using the `foo("2.10")` syntax, +// as given below: + package build import mill._ diff --git a/example/fundamentals/cross/4-cross-dependencies/build.mill b/example/fundamentals/cross/4-cross-dependencies/build.mill index fc68fa04087..16a12a43d2c 100644 --- a/example/fundamentals/cross/4-cross-dependencies/build.mill +++ b/example/fundamentals/cross/4-cross-dependencies/build.mill @@ -53,7 +53,7 @@ trait BarModule extends Cross.Module[String] { // } // ``` -// Rather than pssing in a literal `"2.10"` to the `foo` cross module, we pass +// Rather than passing in a literal `"2.10"` to the `foo` cross module, we pass // in the `crossValue` property that is available within every `Cross.Module`. // This ensures that each version of `bar` depends on the corresponding version // of `foo`: `bar("2.10")` depends on `foo("2.10")`, `bar("2.11")` depends on diff --git a/example/fundamentals/cross/7-inner-cross-module/build.mill b/example/fundamentals/cross/7-inner-cross-module/build.mill index f9374f5832f..ae9afaab96a 100644 --- a/example/fundamentals/cross/7-inner-cross-module/build.mill +++ b/example/fundamentals/cross/7-inner-cross-module/build.mill @@ -49,13 +49,14 @@ def baz = Task { s"hello ${foo("a").bar.param()}" } // You can use the `CrossValue` trait within any `Cross.Module` to // propagate the `crossValue` defined by an enclosing `Cross.Module` to some // nested module. In this case, we use it to bind `crossValue` so it can be -// used in `def param`. This lets you reduce verbosity by defining the `Cross` +// used in `def param`. +// +// This lets you reduce verbosity by defining the `Cross` // once for a group of modules rather than once for every single module in -// that group. There are corresponding `InnerCrossModuleN` traits for cross -// modules that take multiple inputs. +// that group. In the example above, we define the cross module once for +// `object foo extends Cross`, and then the nested modules `bar` and `qux` +// get automatically duplicated once for each `crossValue = "a"` and `crossValue = "b"` // -// You can reference the modules and tasks defined within such a -// `CrossValue` as is done in `def qux` above /** Usage diff --git a/example/fundamentals/dependencies/1-search-updates/build.mill b/example/fundamentals/dependencies/1-search-updates/build.mill index eb248e2d25b..22cb7f31046 100644 --- a/example/fundamentals/dependencies/1-search-updates/build.mill +++ b/example/fundamentals/dependencies/1-search-updates/build.mill @@ -1,4 +1,3 @@ -// == Search for dependency updates // Mill can search for updated versions of your project's dependencies, if // available from your project's configured repositories. Note that it uses diff --git a/example/fundamentals/modules/7-modules/build.mill b/example/fundamentals/modules/7-modules/build.mill index 407acc21f0d..359027f5b18 100644 --- a/example/fundamentals/modules/7-modules/build.mill +++ b/example/fundamentals/modules/7-modules/build.mill @@ -171,17 +171,17 @@ object outer extends MyModule { */ -// You can use `millSourcePath` to automatically set the source folders of your -// modules to match the build structure. You are not forced to rigidly use -// `millSourcePath` to define the source folders of all your code, but it can simplify -// the common case where you probably want your build-layout and on-disk-layout to -// be the same. +// You should use `millSourcePath` to set the source folders of your +// modules to match the build structure. In almost every case, a module's source files +// live at some relative path within the module's folder, and using `millSourcePath` +// ensures that the relative path to the module's source files remains the same +// regardless of where your module lives in the build hierarchy. // // E.g. for `mill.scalalib.ScalaModule`, the Scala source code is assumed by // default to be in `millSourcePath / "src"` while resources are automatically // assumed to be in `millSourcePath / "resources"`. // -// You can override `millSourcePath`: +// You can also override `millSourcePath`: object outer2 extends MyModule { def millSourcePath = super.millSourcePath / "nested" @@ -218,7 +218,7 @@ object outer2 extends MyModule { */ -// *Note that `os.pwd` of the Mill process is set to an empty `sandbox/` folder by default.* +// NOTE: *`os.pwd` of the Mill process is set to an empty `sandbox/` folder by default.* // When defining a module's source files, you should always use `millSourcePath` to ensure the // paths defined are relative to the module's root folder, so the module logic can continue // to work even if moved into a different subfolder. In the rare case where you need the diff --git a/example/fundamentals/out-dir/1-custom-out/build.mill b/example/fundamentals/out-dir/1-custom-out/build.mill index b36b339830b..fe36a067883 100644 --- a/example/fundamentals/out-dir/1-custom-out/build.mill +++ b/example/fundamentals/out-dir/1-custom-out/build.mill @@ -1,6 +1,8 @@ // The default location for Mill's output directory is `out/` under the project workspace. -// A task `printDest` of a module `foo` will have a default scratch space folder -// `out/foo/printDest.dest/`: +// If you'd rather use another location than `out/`, that lives +// in a faster or a writable filesystem for example, you can change the output directory +// via the `MILL_OUTPUT_DIR` environment variable. + package build import mill._ @@ -11,15 +13,6 @@ object foo extends Module { } } -/** Usage -> ./mill foo.printDest -... -.../out/foo/printDest.dest -*/ - -// If you'd rather use another location than `out/`, that lives -// in a faster or a writable filesystem for example, you can change the output directory -// via the `MILL_OUTPUT_DIR` environment variable. /** Usage > MILL_OUTPUT_DIR=build-stuff/working-dir ./mill foo.printDest diff --git a/example/fundamentals/tasks/1-task-graph/build.mill b/example/fundamentals/tasks/1-task-graph/build.mill index 4735f7c4f8e..eab8786442f 100644 --- a/example/fundamentals/tasks/1-task-graph/build.mill +++ b/example/fundamentals/tasks/1-task-graph/build.mill @@ -1,4 +1,6 @@ -// The following is a simple self-contained example using Mill to compile Java: +// The following is a simple self-contained example using Mill to compile Java, +// making use of the `Task.Source` and `Task` types to define a simple build graph +// with some input source files and intermediate build steps: package build import mill._ diff --git a/example/fundamentals/tasks/2-primary-tasks/build.mill b/example/fundamentals/tasks/2-primary-tasks/build.mill index a1b25c02847..46d7ac3ef8b 100644 --- a/example/fundamentals/tasks/2-primary-tasks/build.mill +++ b/example/fundamentals/tasks/2-primary-tasks/build.mill @@ -23,7 +23,7 @@ def resources = Task.Source { millSourcePath / "resources" } // they watch source files and folders and cause downstream tasks to // re-compute if a change is detected. -// === Targets +// === Cached Tasks def allSources = Task { os.walk(sources().path) @@ -47,7 +47,7 @@ def lineCount: T[Int] = Task { // } // ``` // -// ``Target``s are defined using the `def foo = Task {...}` syntax, and dependencies +// ``Cached Tasks``s are defined using the `def foo = Task {...}` syntax, and dependencies // on other tasks are defined using `foo()` to extract the value from them. // Apart from the `foo()` calls, the `Task {...}` block contains arbitrary code that // does some work and returns a result. @@ -59,10 +59,10 @@ def lineCount: T[Int] = Task { // // * https://github.com/com-lihaoyi/os-lib[OS-Lib Library Documentation] -// If a target's inputs change but its output does not, e.g. someone changes a +// If a cached task's inputs change but its output does not, e.g. someone changes a // comment within the source files that doesn't affect the classfiles, then // downstream tasks do not re-evaluate. This is determined using the -// `.hashCode` of the Target's return value. +// `.hashCode` of the cached task's return value. /** Usage @@ -75,7 +75,7 @@ Computing line count */ -// Furthermore, when code changes occur, targets only invalidate if the code change +// Furthermore, when code changes occur, cached tasks only invalidate if the code change // may directly or indirectly affect it. e.g. adding a comment to `lineCount` will // not cause it to recompute: @@ -88,7 +88,7 @@ Computing line count // .sum // ``` // -// But changing the code of the target or any upstream helper method will cause the +// But changing the code of the cached task or any upstream helper method will cause the // old value to be invalidated and a new value re-computed (with a new `println`) // next time it is invoked: // @@ -101,12 +101,12 @@ Computing line count // .sum // ``` // -// For more information on how the bytecode analysis necessary for invalidating targets +// For more information on how the bytecode analysis necessary for invalidating cached tasks // based on code-changes work, see https://github.com/com-lihaoyi/mill/pull/2417[PR#2417] // that implemented it. // -// The return-value of targets has to be JSON-serializable via -// {upickle-github-url}[uPickle]. You can run targets directly from the command +// The return-value of cached tasks has to be JSON-serializable via +// {upickle-github-url}[uPickle]. You can run cached tasks directly from the command // line, or use `show` if you want to see the JSON content or pipe it to // external tools. See the uPickle library documentation for more details: // @@ -163,7 +163,7 @@ Generating jar */ -// *Note that `os.pwd` of the Mill process is set to an empty `sandbox/` folder by default.* +// NOTE: *`os.pwd` of the Mill process is set to an empty `sandbox/` folder by default.* // This is to stop you from accidentally reading and writing files to the base repository root, // which would cause problems with Mill's caches not invalidating properly or files from different // tasks colliding and causing issues. @@ -259,7 +259,9 @@ def summarizeClassFileStats = Task { */ - +// For more details on how to use uPickle, check out the +// https://github.com/com-lihaoyi/upickle[uPickle library documentation] +// // === Commands def run(mainClass: String, args: String*) = Task.Command { @@ -309,11 +311,12 @@ foo.txt resource: My Example Text // arguments not parsed earlier. Default values for command line arguments are also supported. // See the mainargs documentation for more details: // -// * [MainArgs Library Documentation](https://github.com/com-lihaoyi/mainargs[MainArgs]) +// * https://github.com/com-lihaoyi/mainargs[MainArgs Library Documentation] // // By default, all command parameters need to be named, except for variadic parameters -// of type `T*` or `mainargs.Leftover[T]`. You can use the flag `--allow-positional-command-args` -// to allow arbitrary arguments to be passed positionally, as shown below: +// of type `T*` or `mainargs.Leftover[T]`, or those marked as `@arg(positional = true)`. +// You can use also the flag `--allow-positional-command-args` to globally allow +// arguments to be passed positionally, as shown below: /** Usage @@ -333,7 +336,7 @@ foo.txt resource: My Example Text // -// Like <<_targets>>, a command only evaluates after all its upstream +// Like <<_cached_tasks>>, a command only evaluates after all its upstream // dependencies have completed, and will not begin to run if any upstream // dependency has failed. // @@ -347,7 +350,8 @@ foo.txt resource: My Example Text // Tasks can be overriden, with the overriden task callable via `super`. // You can also override a task with a different type of task, e.g. below -// we override `sourceRoots` which is a `Task.Sources` with a `Task{}` target: +// we override `sourceRoots` which is a `Task.Sources` with a cached `Task{}` +// that depends on the original via `super`: // trait Foo extends Module { diff --git a/example/fundamentals/tasks/3-anonymous-tasks/build.mill b/example/fundamentals/tasks/3-anonymous-tasks/build.mill index 5655699e31d..996a22dd164 100644 --- a/example/fundamentals/tasks/3-anonymous-tasks/build.mill +++ b/example/fundamentals/tasks/3-anonymous-tasks/build.mill @@ -18,7 +18,7 @@ def printFileData(fileName: String) = Task.Command { // // Anonymous task's output does not need to be JSON-serializable, their output is // not cached, and they can be defined with or without arguments. -// Unlike <<_targets>> or <<_commands>>, anonymous tasks can be defined +// Unlike <<_cached_tasks>> or <<_commands>>, anonymous tasks can be defined // anywhere and passed around any way you want, until you finally make use of them // within a downstream task or command. // diff --git a/example/fundamentals/tasks/4-inputs/build.mill b/example/fundamentals/tasks/4-inputs/build.mill index 70facdce691..7b7c65c00b8 100644 --- a/example/fundamentals/tasks/4-inputs/build.mill +++ b/example/fundamentals/tasks/4-inputs/build.mill @@ -13,13 +13,13 @@ def myInput = Task.Input { // arbitrary block of code. // // Inputs can be used to force re-evaluation of some external property that may -// affect your build. For example, if I have a <<_targets, Target>> `bar` that +// affect your build. For example, if I have a <<_cached_task, cached task>> `bar` that // calls out to `git` to compute the latest commit hash and message directly, // that target does not have any `Task` inputs and so will never re-compute // even if the external `git` status changes: def gitStatusTask = Task { - "v-" + + "version-" + os.proc("git", "log", "-1", "--pretty=format:%h-%B ") .call(cwd = Task.workspace) .out @@ -33,12 +33,12 @@ def gitStatusTask = Task { > git commit --allow-empty -m "Initial-Commit" > ./mill show gitStatusTask -"v-...-Initial-Commit" +"version-...-Initial-Commit" > git commit --allow-empty -m "Second-Commit" > ./mill show gitStatusTask # Mill didn't pick up the git change! -"v-...-Initial-Commit" +"version-...-Initial-Commit" */ @@ -56,7 +56,7 @@ def gitStatusInput = Task.Input { .text() .trim() } -def gitStatusTask2 = Task { "v-" + gitStatusInput() } +def gitStatusTask2 = Task { "version-" + gitStatusInput() } // This makes `gitStatusInput` to always re-evaluate every build, and only if // the output of `gitStatusInput` changes will `gitStatusTask2` re-compute @@ -66,12 +66,12 @@ def gitStatusTask2 = Task { "v-" + gitStatusInput() } > git commit --allow-empty -m "Initial-Commit" > ./mill show gitStatusTask2 -"v-...-Initial-Commit" +"version-...-Initial-Commit" > git commit --allow-empty -m "Second-Commit" > ./mill show gitStatusTask2 # Mill picked up git change -"v-...-Second-Commit" +"version-...-Second-Commit" */ diff --git a/example/fundamentals/tasks/5-persistent-tasks/build.mill b/example/fundamentals/tasks/5-persistent-tasks/build.mill index c227f8b75ad..509e9ed655e 100644 --- a/example/fundamentals/tasks/5-persistent-tasks/build.mill +++ b/example/fundamentals/tasks/5-persistent-tasks/build.mill @@ -1,9 +1,15 @@ -// Persistent targets defined using `Task(Persistent = True)` are similar to normal +// Persistent targets defined using `Task(persistent = true)` are similar to normal // ``Target``s, except their `Task.dest` folder is not cleared before every // evaluation. This makes them useful for caching things on disk in a more -// fine-grained manner than Mill's own Target-level caching. +// fine-grained manner than Mill's own Task-level caching: the task can +// maintain a cache of one or more files on disk, and decide itself which files +// (or parts of which files!) need to invalidate, rather than having all generated +// files wiped out every time (which is the default behavior for normal Tasks). // -// Below is a semi-realistic example of using a `Task(Persistent = True)` target: +// +// Below is a semi-realistic example of using `Task(persistent = true)` to compress +// files in an input folder, and re-use previously-compressed files if a file in the +// input folder did not change: package build import mill._, scalalib._ import java.util.Arrays @@ -51,7 +57,7 @@ def compressBytes(input: Array[Byte]) = { // Since persistent tasks have long-lived state on disk that lives beyond a // single evaluation, this raises the possibility of the disk contents getting // into a bad state and causing all future evaluations to fail. It is left up -// to the person implementing the `Task(Persistent = True)` to ensure their implementation +// to user of `Task(persistent = true)` to ensure their implementation // is eventually consistent. You can also use `mill clean` to manually purge // the disk contents to start fresh. diff --git a/example/javalib/basic/1-simple/build.mill b/example/javalib/basic/1-simple/build.mill index 3f4e27a8b3e..62b544475fd 100644 --- a/example/javalib/basic/1-simple/build.mill +++ b/example/javalib/basic/1-simple/build.mill @@ -43,8 +43,8 @@ object `package` extends RootModule with JavaModule { // //// SNIPPET:DEPENDENCIES // -// This example project uses two third-party dependencies - ArgParse4J for CLI -// argument parsing, Apache Commons Text for HTML escaping - and uses them to wrap a +// This example project uses two third-party dependencies - https://argparse4j.github.io/[ArgParse4J] for CLI +// argument parsing, https://commons.apache.org/proper/commons-text/[Apache Commons Text] for HTML escaping - and uses them to wrap a // given input string in HTML templates with proper escaping. // // Typical usage of a `JavaModule` is shown below diff --git a/example/javalib/basic/2-custom-build-logic/build.mill b/example/javalib/basic/2-custom-build-logic/build.mill index 9c3326a0d98..cab5488b01e 100644 --- a/example/javalib/basic/2-custom-build-logic/build.mill +++ b/example/javalib/basic/2-custom-build-logic/build.mill @@ -11,6 +11,6 @@ object `package` extends RootModule with JavaModule { /** Generate resources using lineCount of sources */ override def resources = Task { os.write(Task.dest / "line-count.txt", "" + lineCount()) - Seq(PathRef(Task.dest)) + super.resources() ++ Seq(PathRef(Task.dest)) } } diff --git a/example/javalib/dependencies/4-downloading-non-maven-jars/build.mill b/example/javalib/dependencies/4-downloading-unmanaged-jars/build.mill similarity index 100% rename from example/javalib/dependencies/4-downloading-non-maven-jars/build.mill rename to example/javalib/dependencies/4-downloading-unmanaged-jars/build.mill diff --git a/example/javalib/dependencies/4-downloading-non-maven-jars/src/foo/Foo.java b/example/javalib/dependencies/4-downloading-unmanaged-jars/src/foo/Foo.java similarity index 100% rename from example/javalib/dependencies/4-downloading-non-maven-jars/src/foo/Foo.java rename to example/javalib/dependencies/4-downloading-unmanaged-jars/src/foo/Foo.java diff --git a/example/javalib/dependencies/4-downloading-non-maven-jars/textfile.txt b/example/javalib/dependencies/4-downloading-unmanaged-jars/textfile.txt similarity index 100% rename from example/javalib/dependencies/4-downloading-non-maven-jars/textfile.txt rename to example/javalib/dependencies/4-downloading-unmanaged-jars/textfile.txt diff --git a/example/javalib/linting/1-error-prone/build.mill b/example/javalib/linting/1-error-prone/build.mill index dddaf713839..8b9a656368a 100644 --- a/example/javalib/linting/1-error-prone/build.mill +++ b/example/javalib/linting/1-error-prone/build.mill @@ -1,11 +1,10 @@ -// When adding the `ErrorPromeModule` to your `JavaModule`, +// When adding the `ErrorProneModule` to your `JavaModule`, // the `error-prone` compiler plugin automatically detects various kind of programming errors. package build import mill._, javalib._, errorprone._ - object `package` extends RootModule with JavaModule with ErrorProneModule { def errorProneOptions = Seq("-XepAllErrorsAsWarnings") } diff --git a/example/javalib/linting/2-checkstyle/build.mill b/example/javalib/linting/2-checkstyle/build.mill index 6ac56a38fd3..04e6164db39 100644 --- a/example/javalib/linting/2-checkstyle/build.mill +++ b/example/javalib/linting/2-checkstyle/build.mill @@ -107,11 +107,11 @@ Audit done. // - Version `6.3` or above is required for `plain` and `xml` formats. // - Setting `checkstyleOptions` might cause failures with legacy versions. // -// == CheckstyleXsltModule +// === CheckstyleXsltModule // // This plugin extends the `mill.contrib.checkstyle.CheckstyleModule` with the ability to generate reports by applying https://www.w3.org/TR/xslt/[XSL Transformations] on a Checkstyle output report. // -// === Auto detect XSL Transformations +// ==== Auto detect XSL Transformations // // XSLT files are detected automatically provided a prescribed directory structure is followed. // [source,scala] @@ -132,7 +132,7 @@ Audit done. // */ // ---- // -// === Specify XSL Transformations manually +// ==== Specify XSL Transformations manually // // For a custom setup, adapt the following example. // [source,scala] diff --git a/example/javalib/linting/3-palantirformat/build.mill b/example/javalib/linting/3-palantirformat/build.mill index db162e3c846..0205b8d3ff1 100644 --- a/example/javalib/linting/3-palantirformat/build.mill +++ b/example/javalib/linting/3-palantirformat/build.mill @@ -17,17 +17,17 @@ object `package` extends RootModule with PalantirFormatModule /** Usage -> ./mill palantirformat --check # check should fail initially +> ./mill palantirformat --check # check should fail initially ...checking format in java sources ... ...src/A.java error: ...palantirformat aborted due to format error(s) (or invalid plugin settings/palantirformat options) -> ./mill palantirformat # format all Java source files +> ./mill palantirformat # format all Java source files ...formatting java sources ... -> ./mill palantirformat --check # check should succeed now +> ./mill palantirformat --check # check should succeed now ...checking format in java sources ... -> ./mill mill.javalib.palantirformat.PalantirFormatModule/ __.sources # alternatively, use external module to check/format +> ./mill mill.javalib.palantirformat.PalantirFormatModule/ # alternatively, use external module to check/format ...formatting java sources ... */ diff --git a/example/javalib/module/4-compilation-execution-flags/build.mill b/example/javalib/module/4-compilation-execution-flags/build.mill index da3dc450491..751eabf21c2 100644 --- a/example/javalib/module/4-compilation-execution-flags/build.mill +++ b/example/javalib/module/4-compilation-execution-flags/build.mill @@ -8,7 +8,7 @@ object `package` extends RootModule with JavaModule{ def javacOptions = Seq("-deprecation") } -// You can pass flags to the Kotlin compiler via `javacOptions`. +// You can pass flags to the Java compiler via `javacOptions`. /** Usage diff --git a/example/kotlinlib/basic/2-custom-build-logic/build.mill b/example/kotlinlib/basic/2-custom-build-logic/build.mill index f486ee06b3c..4a2d3d031a7 100644 --- a/example/kotlinlib/basic/2-custom-build-logic/build.mill +++ b/example/kotlinlib/basic/2-custom-build-logic/build.mill @@ -16,7 +16,7 @@ object `package` extends RootModule with KotlinModule { /** Generate resources using lineCount of sources */ override def resources = Task { os.write(Task.dest / "line-count.txt", "" + lineCount()) - Seq(PathRef(Task.dest)) + super.resources() ++ Seq(PathRef(Task.dest)) } object test extends KotlinTests with TestModule.Junit5 { diff --git a/example/kotlinlib/basic/4-builtin-commands/build.mill b/example/kotlinlib/basic/4-builtin-commands/build.mill index a0c34745640..522d9e6b221 100644 --- a/example/kotlinlib/basic/4-builtin-commands/build.mill +++ b/example/kotlinlib/basic/4-builtin-commands/build.mill @@ -3,8 +3,12 @@ package build import mill._, kotlinlib._ trait MyModule extends KotlinModule { - def kotlinVersion = "1.9.24" + object test extends KotlinTests with TestModule.Junit5 { + def ivyDeps = super.ivyDeps() ++ Agg( + ivy"io.kotest:kotest-runner-junit5-jvm:5.9.1" + ) + } } object foo extends MyModule { @@ -20,10 +24,4 @@ object bar extends MyModule { def ivyDeps = Agg( ivy"org.jetbrains.kotlinx:kotlinx-html-jvm:0.11.0" ) - - object test extends KotlinTests with TestModule.Junit5 { - def ivyDeps = super.ivyDeps() ++ Agg( - ivy"io.kotest:kotest-runner-junit5-jvm:5.9.1" - ) - } } diff --git a/example/kotlinlib/dependencies/4-downloading-non-maven-jars/build.mill b/example/kotlinlib/dependencies/4-downloading-unmanaged-jars/build.mill similarity index 100% rename from example/kotlinlib/dependencies/4-downloading-non-maven-jars/build.mill rename to example/kotlinlib/dependencies/4-downloading-unmanaged-jars/build.mill diff --git a/example/kotlinlib/dependencies/4-downloading-non-maven-jars/src/foo/Foo.kt b/example/kotlinlib/dependencies/4-downloading-unmanaged-jars/src/foo/Foo.kt similarity index 100% rename from example/kotlinlib/dependencies/4-downloading-non-maven-jars/src/foo/Foo.kt rename to example/kotlinlib/dependencies/4-downloading-unmanaged-jars/src/foo/Foo.kt diff --git a/example/kotlinlib/dependencies/4-downloading-non-maven-jars/textfile.txt b/example/kotlinlib/dependencies/4-downloading-unmanaged-jars/textfile.txt similarity index 100% rename from example/kotlinlib/dependencies/4-downloading-non-maven-jars/textfile.txt rename to example/kotlinlib/dependencies/4-downloading-unmanaged-jars/textfile.txt diff --git a/example/kotlinlib/linting/1-detekt/build.mill b/example/kotlinlib/linting/1-detekt/build.mill index 0cd22e9c6b8..55a2427821e 100644 --- a/example/kotlinlib/linting/1-detekt/build.mill +++ b/example/kotlinlib/linting/1-detekt/build.mill @@ -5,11 +5,14 @@ import kotlinlib.KotlinModule import kotlinlib.detekt.DetektModule object `package` extends RootModule with KotlinModule with DetektModule { - def kotlinVersion = "1.9.24" - } + +// This example shows how to use the https://github.com/detekt/detekt[Detekt] +// static code analyzer for linting a `KotlinModule`, by mixing in the trait +// `DetektModule` and calling the `detekt` task: + /** See Also: src/example/Foo.kt */ /** Usage diff --git a/example/kotlinlib/linting/2-ktlint/build.mill b/example/kotlinlib/linting/2-ktlint/build.mill index 8117f00867a..762c902c4eb 100644 --- a/example/kotlinlib/linting/2-ktlint/build.mill +++ b/example/kotlinlib/linting/2-ktlint/build.mill @@ -7,20 +7,25 @@ import kotlinlib.KotlinModule import kotlinlib.ktlint.KtlintModule object `package` extends RootModule with KotlinModule with KtlintModule { - def kotlinVersion = "1.9.24" def ktlintConfig = Some(PathRef(T.workspace / ".editorconfig")) - } +// This example shows how to use the https://github.com/pinterest/ktlint[KtLint] +// linter on a `KotlinModule`, by mixing in the trait `KtlintModule` and calling the +// `ktlint` task. `ktlint` also supports autoformatting to automatically resolve +// code formatting violations, via the `--format` flag shown below: + /** Usage > ./mill ktlint # run ktlint to produce a report, defaults to warning without error error: ...src/example/FooWrong.kt:6:28: Missing newline before ")" (standard:parameter-list-wrapping)... ...src/example/FooWrong.kt:6:28: Newline expected before closing parenthesis (standard:function-signature)... ...src/example/FooWrong.kt:6:28: Missing trailing comma before ")" (standard:trailing-comma-on-declaration-site)... + > ./mill ktlint --format true > ./mill ktlint # after fixing the violations, ktlint no longer errors */ + diff --git a/example/kotlinlib/linting/3-ktfmt/build.mill b/example/kotlinlib/linting/3-ktfmt/build.mill index b97892fd2f4..43103321a0a 100644 --- a/example/kotlinlib/linting/3-ktfmt/build.mill +++ b/example/kotlinlib/linting/3-ktfmt/build.mill @@ -7,17 +7,21 @@ import kotlinlib.KotlinModule import kotlinlib.ktfmt.KtfmtModule object `package` extends RootModule with KotlinModule with KtfmtModule { - def kotlinVersion = "1.9.24" - } +// This example demonstrates how to use the https://github.com/facebook/ktfmt[KtFmt] +// autoformatter from Facebook both to enforce and apply formatting to your `KotlinModule` +// source files. You can configure a non-default version of KtFmt by overriding `def ktfmtVersion` + /** Usage > ./mill ktfmt --format=false # run ktfmt to produce a list of files which should be formatter ...src/example/FooWrong.kt... + > ./mill ktfmt # running without arguments will format all files Done formatting ...src/example/FooWrong.kt + > ./mill ktfmt # after fixing the violations, ktfmt no longer prints any file > ./mill mill.kotlinlib.ktfmt.KtfmtModule/ __.sources # alternatively, use external module to check/format diff --git a/example/scalalib/basic/1-simple/build.mill b/example/scalalib/basic/1-simple/build.mill index 097fdc011d2..2dee802ce5d 100644 --- a/example/scalalib/basic/1-simple/build.mill +++ b/example/scalalib/basic/1-simple/build.mill @@ -146,7 +146,7 @@ error: Missing argument: --text // `mill inspect compile` to inspect a task's doc-comment documentation or what // it depends on, or `mill show foo.scalaVersion` to show the output of any task. // -// The most common *tasks* that Mill can run are cached *targets*, such as +// The most common *tasks* that Mill can run are cached tasks, such as // `compile`, and un-cached *commands* such as `foo.run`. Targets do not // re-evaluate unless one of their inputs changes, whereas commands re-run every // time. diff --git a/example/scalalib/basic/2-custom-build-logic/build.mill b/example/scalalib/basic/2-custom-build-logic/build.mill index 293f97b547b..e806d0edbdd 100644 --- a/example/scalalib/basic/2-custom-build-logic/build.mill +++ b/example/scalalib/basic/2-custom-build-logic/build.mill @@ -19,7 +19,7 @@ object `package` extends RootModule with ScalaModule { /** Generate resources using lineCount of sources */ override def resources = Task { os.write(Task.dest / "line-count.txt", "" + lineCount()) - Seq(PathRef(Task.dest)) + super.resources() ++ Seq(PathRef(Task.dest)) } } @@ -35,9 +35,9 @@ object `package` extends RootModule with ScalaModule { // rankdir=LR // node [shape=box width=0 height=0 style=filled fillcolor=white] // allSourceFiles -> lineCount -> resources -> "..." -> run -// "resources.super" -> "..." [style=invis] +// "resources.super" -> "resources" // "..." [color=white] -// "resources.super" [style=dashed] +// "resources.super" [color=white] // allSourceFiles [color=white] // run [color=white] // } @@ -64,7 +64,7 @@ Inputs: // `allSourceFiles` (an existing task) and is in-turn used in our override of // `resources` (also an existing task). `os.read.lines` and `os.write` come // from the https://github.com/com-lihaoyi/os-lib[OS-Lib] library, which is -// one of Mill's <>. This generated file can then be +// one of Mill's xref:fundamentals/bundled-libraries.adoc[Bundled Libraries]. This generated file can then be // loaded and used at runtime, as see in the output of `mill run` // // While this is a toy example, it shows how easy it is to customize your Mill @@ -72,7 +72,7 @@ Inputs: // most real-world projects. // // This customization is done in a principled fashion familiar to most -// programmers - object-orienting overrides - rather than ad-hoc +// programmers - `override` and `super` - rather than ad-hoc // monkey-patching or mutation common in other build tools. You never have // "spooky action at a distance" affecting your build / graph definition, and // your IDE can always help you find the final override of any particular build diff --git a/example/scalalib/basic/3-multi-module/build.mill b/example/scalalib/basic/3-multi-module/build.mill index d98dadee143..3919ab9924a 100644 --- a/example/scalalib/basic/3-multi-module/build.mill +++ b/example/scalalib/basic/3-multi-module/build.mill @@ -24,12 +24,16 @@ object bar extends MyModule { // We don't mark either module as top-level using `extends RootModule`, so // running tasks needs to use the module name as the prefix e.g. `foo.run` or // `bar.run`. You can define multiple modules the same way you define a single -// module, using `def moduleDeps` to define the relationship between them. +// module, using `def moduleDeps` to define the relationship between them. Modules +// can also be nested within each other, as `foo.test` and `bar.test` are nested +// within `foo` and `bar` respectively // // Note that we split out the `test` submodule configuration common to both // modules into a separate `trait MyModule`. This lets us avoid the need to // copy-paste common settings, while still letting us define any per-module -// configuration such as `ivyDeps` specific to a particular module. +// configuration such as `ivyDeps` specific to a particular module. This is a +// common pattern within Mill builds to avoid the need to copy-paste common +// configuration // // The above builds expect the following project layout: // @@ -90,6 +94,9 @@ Bar.value:

world

// Mill's evaluator will ensure that the modules are compiled in the right // order, and recompiled as necessary when source code in each module changes. +// The unique path on disk that Mill automatically assigns each task also ensures +// you do not need to worry about choosing a path on disk to cache outputs, or +// filesystem collisions if multiple tasks write to the same path. // // You can use wildcards and brace-expansion to select // multiple tasks at once or to shorten the path to deeply nested tasks. If @@ -106,8 +113,8 @@ Bar.value:

world

// |========================================================== // // -// You can use the `+` symbol to add another task with optional arguments. -// If you need to feed a `+` as argument to your task, you can mask it by +// You can use the + symbol to add another task with optional arguments. +// If you need to feed a + as argument to your task, you can mask it by // preceding it with a backslash (`\`). // @@ -121,4 +128,6 @@ Bar.value:

world

> mill __.compile + foo.__.test # Runs all `compile` tasks and all tests under `foo`. */ -// For more details on the query syntax, check out the documentation for <> +// For more details on the query syntax, check out the query syntax documentation: + +// - xref:fundamentals/query-syntax.adoc[Task Query Syntax] diff --git a/example/scalalib/basic/4-builtin-commands/build.mill b/example/scalalib/basic/4-builtin-commands/build.mill index 4b4dbc74dc9..1df3c95423e 100644 --- a/example/scalalib/basic/4-builtin-commands/build.mill +++ b/example/scalalib/basic/4-builtin-commands/build.mill @@ -15,6 +15,7 @@ import mill._, scalalib._ trait MyModule extends ScalaModule { def scalaVersion = "2.13.11" + object test extends ScalaTests with TestModule.Utest } object foo extends MyModule { @@ -81,7 +82,7 @@ foo.artifactName */ -// See the documentation for <> for more details what you +// See the documentation for xref:fundamentals/query-syntax.adoc[Task Query Syntax] for more details what you // can pass to `resolve` // == inspect @@ -305,7 +306,32 @@ foo.compileClasspath // image::basic/VisualizeJava.svg[VisualizeJava.svg] // // `visualize` can be very handy for trying to understand the dependency graph of -// tasks within your Mill build. +// tasks within your Mill build: who depends on what? Who do I need to override to affect +// a particular task? Which tasks depend on another and need to run sequentially, and which +// do not and can be run in parallel? +// +// The above example shows the outcome of using `visualize` on multiple tasks within a single +// module, but you can also use `visualize` on a single task in multiple modules to see how they are related: +// +/** Usage +> mill visualize __.compile + +> cat out/visualize.dest/out.dot +digraph "example1" { +graph ["rankdir"="LR"] +"bar.compile" ["style"="solid","shape"="box"] +"bar.test.compile" ["style"="solid","shape"="box"] +"foo.compile" ["style"="solid","shape"="box"] +"foo.test.compile" ["style"="solid","shape"="box"] +"bar.compile" -> "foo.compile" +"bar.compile" -> "bar.test.compile" +"foo.compile" -> "foo.test.compile" +} +*/ + +// image::basic/VisualizeCompiles.svg[VisualizeCompiles.svg] +// +// // // == visualizePlan // diff --git a/example/scalalib/dependencies/1-ivy-deps/build.mill b/example/scalalib/dependencies/1-ivy-deps/build.mill index 94313567825..79c07a78feb 100644 --- a/example/scalalib/dependencies/1-ivy-deps/build.mill +++ b/example/scalalib/dependencies/1-ivy-deps/build.mill @@ -23,6 +23,7 @@ object `package` extends RootModule with ScalaModule { // dependencies // // * Triple `:::` syntax (e.g. `ivy"org.scalamacros:::paradise:2.1.1"`) defines +// * Triple `:::` syntax (e.g. `ivy"org.scalamacros:::paradise:2.1.1"`) defines // dependencies cross-published against the full Scala version e.g. `2.12.4` // instead of just `2.12`. These are typically Scala compiler plugins or // similar. @@ -33,7 +34,7 @@ object `package` extends RootModule with ScalaModule { // // * `ivy"org.apache.spark::spark-sql:2.4.0;classifier=tests`. // -// Please consult the <> section for even more details. +// Please consult the xref:fundamentals/library-deps.adoc[Library Dependencies in Mill] section for more details. //// SNIPPET:USAGE diff --git a/example/scalalib/dependencies/2-run-compile-deps/build.mill b/example/scalalib/dependencies/2-run-compile-deps/build.mill index 233b8f51b34..c2fd26eea45 100644 --- a/example/scalalib/dependencies/2-run-compile-deps/build.mill +++ b/example/scalalib/dependencies/2-run-compile-deps/build.mill @@ -20,7 +20,7 @@ object foo extends ScalaModule { //// SNIPPET:END // You can also declare compile-time-only dependencies with `compileIvyDeps`. -// These are present in the compile classpath, but will not propagated to the +// These are present in the compile classpath, but will not propagate to the // transitive dependencies. //// SNIPPET:BUILD2 diff --git a/example/scalalib/dependencies/3-unmanaged-jars/build.mill b/example/scalalib/dependencies/3-unmanaged-jars/build.mill index 9b6bda3d581..8bbc09f3b3a 100644 --- a/example/scalalib/dependencies/3-unmanaged-jars/build.mill +++ b/example/scalalib/dependencies/3-unmanaged-jars/build.mill @@ -22,4 +22,11 @@ object `package` extends RootModule with ScalaModule { Key: name, Value: John Key: age, Value: 30 -*/ \ No newline at end of file +*/ + +// in most scenarios you should rely on `ivyDeps`/`moduleDeps` and let Mill manage +// the compilation/downloading/caching of classpath jars for you, as Mill will +// automatically pull in transitive dependencies which are generally needed for things +// to work. But in the rare case you receive a jar or folder-full-of-classfiles +// from somewhere and need to include it in your project, `unmanagedClasspath` is the +// way to do it. \ No newline at end of file diff --git a/example/scalalib/dependencies/4-downloading-non-maven-jars/build.mill b/example/scalalib/dependencies/4-downloading-unmanaged-jars/build.mill similarity index 66% rename from example/scalalib/dependencies/4-downloading-non-maven-jars/build.mill rename to example/scalalib/dependencies/4-downloading-unmanaged-jars/build.mill index c28ba3b23d1..a79e2089f51 100644 --- a/example/scalalib/dependencies/4-downloading-non-maven-jars/build.mill +++ b/example/scalalib/dependencies/4-downloading-unmanaged-jars/build.mill @@ -17,10 +17,10 @@ object `package` extends RootModule with ScalaModule { //// SNIPPET:END // You can also override `unmanagedClasspath` to point it at jars that you want to -// download from arbitrary URLs. Note that tasks like `unmanagedClasspath` are -// cached, so your jar is downloaded only once and re-used indefinitely after that. +// download from arbitrary URLs. // `requests.get` comes from the https://github.com/com-lihaoyi/requests-scala[Requests-Scala] -// library, one of Mill's <>. +// library, one of Mill's xref:fundamentals/bundled-libraries.adoc[Bundled Libraries]. +// /** Usage @@ -30,3 +30,9 @@ hear me moo I weigh twice as much as you */ + +// Note that tasks like `unmanagedClasspath` are +// cached, so your jar is downloaded only once and re-used indefinitely after that. +// This is usually not a problem, because usually URLs follow the rule that +// https://www.w3.org/Provider/Style/URI[Cool URIs don't change], and so jars +// downloaded from the same URL will always contain the same contents. diff --git a/example/scalalib/dependencies/4-downloading-non-maven-jars/src/Foo.scala b/example/scalalib/dependencies/4-downloading-unmanaged-jars/src/Foo.scala similarity index 100% rename from example/scalalib/dependencies/4-downloading-non-maven-jars/src/Foo.scala rename to example/scalalib/dependencies/4-downloading-unmanaged-jars/src/Foo.scala diff --git a/example/scalalib/dependencies/4-downloading-non-maven-jars/textfile.txt b/example/scalalib/dependencies/4-downloading-unmanaged-jars/textfile.txt similarity index 100% rename from example/scalalib/dependencies/4-downloading-non-maven-jars/textfile.txt rename to example/scalalib/dependencies/4-downloading-unmanaged-jars/textfile.txt diff --git a/example/scalalib/dependencies/5-repository-config/build.mill b/example/scalalib/dependencies/5-repository-config/build.mill index 05249d8fe42..b7ca0d0ba61 100644 --- a/example/scalalib/dependencies/5-repository-config/build.mill +++ b/example/scalalib/dependencies/5-repository-config/build.mill @@ -1,5 +1,6 @@ -// By default, dependencies are resolved from maven central, but you can add -// your own resolvers by overriding the `repositoriesTask` task in the module: +// By default, dependencies are resolved from https://central.sonatype.com/[Maven Central], +// the standard package repository for JVM languages like Java, Kotlin, or Scala. You +// can also add your own resolvers by overriding the `repositoriesTask` task in the module: //// SNIPPET:BUILD1 package build @@ -26,9 +27,12 @@ object foo extends ScalaModule { //// SNIPPET:END -// Mill read https://get-coursier.io/[coursier] config files automatically. +// Mill uses the https://get-coursier.io/[Coursier] dependency resolver, and reads +// Coursier config files automatically. +// +// You can configure Coursier to use an alternate download location for Maven Central +// artifacts via a `mirror.properties` file: // -// It is possible to setup mirror with `mirror.properties` // [source,properties] // ---- // central.from=https://repo1.maven.org/maven2 diff --git a/example/scalalib/module/2-custom-tasks/build.mill b/example/scalalib/module/2-custom-tasks/build.mill index 37b9f97b543..c4fcc9c9c17 100644 --- a/example/scalalib/module/2-custom-tasks/build.mill +++ b/example/scalalib/module/2-custom-tasks/build.mill @@ -78,9 +78,9 @@ object `package` extends RootModule with ScalaModule { // depending on existing Tasks e.g. `foo.sources` via the `foo.sources()` // syntax to extract their current value, as shown in `lineCount` above. The // return-type of a Task has to be JSON-serializable (using -// https://github.com/lihaoyi/upickle[uPickle], one of Mill's <>) +// https://github.com/lihaoyi/upickle[uPickle], one of Mill's xref:fundamentals/bundled-libraries.adoc[Bundled Libraries]) // and the Task is cached when first run until its inputs change (in this case, if -// someone edits the `foo.sources` files which live in `foo/src`. Cached Tasks +// someone edits the `foo.sources` files which live in `foo/src`). Cached Tasks // cannot take parameters. // // Note that depending on a task requires use of parentheses after the task @@ -113,13 +113,25 @@ my.line.count: 14 // to compile some Javascript, generate sources to feed into a compiler, or // create some custom jar/zip assembly with the files you want , all of these // can simply be custom tasks with your code running in the `Task {...}` block. +// You can also import arbitrary Java or Scala libraries from Maven Central via +// xref:extending/import-ivy-plugins.adoc[import $ivy] to use in your build. // // You can create arbitrarily long chains of dependent tasks, and Mill will // handle the re-evaluation and caching of the tasks' output for you. // Mill also provides you a `Task.dest` folder for you to use as scratch space or -// to store files you want to return: all files a task creates should live -// within `Task.dest`, and any files you want to modify should be copied into -// `Task.dest` before being modified. That ensures that the files belonging to a -// particular task all live in one place, avoiding file-name conflicts and +// to store files you want to return: +// +// * Any files a task creates should live +// within `Task.dest` +// +// * Any files a task modifies should be copied into +// `Task.dest` before being modified. +// +// * Any files that a task returns should be returned as a `PathRef` to a path +// within `Task.dest` +// +// That ensures that the files belonging to a +// particular task all live in one place, avoiding file-name conflicts, +// preventing race conditions when tasks evaluate in parallel, and // letting Mill automatically invalidate the files when the task's inputs // change. \ No newline at end of file diff --git a/example/scalalib/testing/2-test-deps/build.mill b/example/scalalib/testing/2-test-deps/build.mill index 72a8e10e31f..217cc30bea0 100644 --- a/example/scalalib/testing/2-test-deps/build.mill +++ b/example/scalalib/testing/2-test-deps/build.mill @@ -37,7 +37,11 @@ object baz extends ScalaModule { //// SNIPPET:END // In this example, not only does `qux` depend on `baz`, but we also make -// `qux.test` depend on `baz.test`. That lets `qux.test` make use of the +// `qux.test` depend on `baz.test`. +// +// image::basic/VisualizeTestDeps.svg[VisualizeTestDeps.svg] +// +// That lets `qux.test` make use of the // `BazTestUtils` class that `baz.test` defines, allowing us to re-use this // test helper throughout multiple modules' test suites diff --git a/kotlinlib/src/mill/kotlinlib/PlatformKotlinModule.scala b/kotlinlib/src/mill/kotlinlib/PlatformKotlinModule.scala index 23ff797e7ad..84229d5b756 100644 --- a/kotlinlib/src/mill/kotlinlib/PlatformKotlinModule.scala +++ b/kotlinlib/src/mill/kotlinlib/PlatformKotlinModule.scala @@ -1,7 +1,6 @@ package mill.kotlinlib -import mill._ -import os.Path +import mill.scalalib.PlatformModuleBase /** * A [[KotlinModule]] intended for defining `.jvm`/`.js`/etc. submodules @@ -13,25 +12,4 @@ import os.Path * built against and not something that should affect the filesystem path or * artifact name */ -trait PlatformKotlinModule extends KotlinModule { - override def millSourcePath: Path = super.millSourcePath / os.up - - /** - * The platform suffix of this [[PlatformKotlinModule]]. Useful if you want to - * further customize the source paths or artifact names. - */ - def platformKotlinSuffix: String = millModuleSegments - .value - .collect { case l: mill.define.Segment.Label => l.value } - .last - - override def sources: T[Seq[PathRef]] = Task.Sources { - super.sources().flatMap { source => - val platformPath = - PathRef(source.path / _root_.os.up / s"${source.path.last}-$platformKotlinSuffix") - Seq(source, platformPath) - } - } - - override def artifactNameParts: T[Seq[String]] = super.artifactNameParts().dropRight(1) -} +trait PlatformKotlinModule extends PlatformModuleBase with KotlinModule diff --git a/kotlinlib/src/mill/kotlinlib/ktfmt/KtfmtModule.scala b/kotlinlib/src/mill/kotlinlib/ktfmt/KtfmtModule.scala index 281f0a9fa47..dcf4f33cb55 100644 --- a/kotlinlib/src/mill/kotlinlib/ktfmt/KtfmtModule.scala +++ b/kotlinlib/src/mill/kotlinlib/ktfmt/KtfmtModule.scala @@ -1,7 +1,6 @@ package mill.kotlinlib.ktfmt import mill._ -import mainargs.Leftover import mill.api.{Loose, PathRef} import mill.define.{Discover, ExternalModule} import mill.kotlinlib.{DepSyntax, Versions} @@ -49,13 +48,12 @@ trait KtfmtModule extends KtfmtBaseModule { */ def ktfmt( @mainargs.arg ktfmtArgs: KtfmtArgs, - @mainargs.arg(positional = true) sources: Leftover[String] + @mainargs.arg(positional = true) sources: Tasks[Seq[PathRef]] = + Tasks.resolveMainDefault("__.sources") ): Command[Unit] = Task.Command { - val _sources = if (sources.value.isEmpty) { + val _sources: Seq[PathRef] = if (sources.value.isEmpty) { this.sources() - } else { - sources.value.iterator.map(rel => PathRef(millSourcePath / os.RelPath(rel))) - } + } else T.sequence(sources.value)().flatten KtfmtModule.ktfmtAction( ktfmtArgs.style, ktfmtArgs.format, diff --git a/kotlinlib/test/src/mill/kotlinlib/contrib/ktfmt/KtfmtModuleTests.scala b/kotlinlib/test/src/mill/kotlinlib/contrib/ktfmt/KtfmtModuleTests.scala index 3b9677a4218..75fbad0890b 100644 --- a/kotlinlib/test/src/mill/kotlinlib/contrib/ktfmt/KtfmtModuleTests.scala +++ b/kotlinlib/test/src/mill/kotlinlib/contrib/ktfmt/KtfmtModuleTests.scala @@ -1,9 +1,7 @@ package mill.kotlinlib.ktfmt -import mainargs.Leftover -import mill.{T, api} +import mill.{PathRef, T, Task, api} import mill.kotlinlib.KotlinModule -import mill.kotlinlib.ktfmt.{KtfmtArgs, KtfmtModule} import mill.main.Tasks import mill.testkit.{TestBaseModule, UnitTester} import utest.{TestSuite, Tests, assert, test} @@ -12,6 +10,10 @@ object KtfmtModuleTests extends TestSuite { val kotlinVersion = "1.9.24" + object module extends TestBaseModule with KotlinModule with KtfmtModule { + override def kotlinVersion: T[String] = KtfmtModuleTests.kotlinVersion + } + def tests: Tests = Tests { val (before, after) = { @@ -62,7 +64,7 @@ object KtfmtModuleTests extends TestSuite { test("ktfmt - explicit files") { checkState( - afterFormat(before, sources = Seq("src/Example.kt")), + afterFormat(before, sources = Seq(module.sources)), after / "style" / "kotlin" ) } @@ -94,13 +96,9 @@ object KtfmtModuleTests extends TestSuite { style: String = "kotlin", format: Boolean = true, removeUnusedImports: Boolean = true, - sources: Seq[String] = Seq.empty + sources: Seq[mill.define.NamedTask[Seq[PathRef]]] = Seq.empty ): Seq[os.Path] = { - object module extends TestBaseModule with KotlinModule with KtfmtModule { - override def kotlinVersion: T[String] = KtfmtModuleTests.kotlinVersion - } - val eval = UnitTester(module, moduleRoot) eval(module.ktfmt( @@ -109,7 +107,7 @@ object KtfmtModuleTests extends TestSuite { format = format, removeUnusedImports = removeUnusedImports ), - sources = Leftover(sources: _*) + sources = Tasks(sources) )).fold( { case api.Result.Exception(cause, _) => throw cause diff --git a/main/src/mill/main/MainModule.scala b/main/src/mill/main/MainModule.scala index 1771b96608b..7b7ff6bbeeb 100644 --- a/main/src/mill/main/MainModule.scala +++ b/main/src/mill/main/MainModule.scala @@ -531,11 +531,11 @@ trait MainModule extends BaseModule0 { planTasks: Option[List[NamedTask[_]]] = None ): Result[Seq[PathRef]] = { def callVisualizeModule( - rs: List[NamedTask[Any]], - allRs: List[NamedTask[Any]] + tasks: List[NamedTask[Any]], + transitiveTasks: List[NamedTask[Any]] ): Result[Seq[PathRef]] = { val (in, out) = vizWorker - in.put((rs, allRs, ctx.dest)) + in.put((tasks, transitiveTasks, ctx.dest)) val res = out.take() res.map { v => println(upickle.default.write(v.map(_.path.toString()), indent = 2)) @@ -550,9 +550,7 @@ trait MainModule extends BaseModule0 { ) match { case Left(err) => Result.Failure(err) case Right(rs) => planTasks match { - case Some(allRs) => { - callVisualizeModule(rs, allRs) - } + case Some(allRs) => callVisualizeModule(rs, allRs) case None => callVisualizeModule(rs, rs) } } diff --git a/main/src/mill/main/VisualizeModule.scala b/main/src/mill/main/VisualizeModule.scala index f1269795815..f73e0a84e5e 100644 --- a/main/src/mill/main/VisualizeModule.scala +++ b/main/src/mill/main/VisualizeModule.scala @@ -4,13 +4,14 @@ import java.util.concurrent.LinkedBlockingQueue import coursier.LocalRepositories import coursier.core.Repository import coursier.maven.MavenRepository -import mill.define.{Discover, ExternalModule, Target, NamedTask} +import mill.define.{Discover, ExternalModule, NamedTask, Target} import mill.util.Util.millProjectModule -import mill.api.{Loose, Result, PathRef} +import mill.api.{Loose, PathRef, Result} import mill.define.Worker import org.jgrapht.graph.{DefaultEdge, SimpleDirectedGraph} import guru.nidi.graphviz.attribute.Rank.RankDir import guru.nidi.graphviz.attribute.{Rank, Shape, Style} +import mill.eval.Graph object VisualizeModule extends ExternalModule with VisualizeModule { def repositories: Seq[Repository] = Seq( @@ -45,10 +46,15 @@ trait VisualizeModule extends mill.define.TaskModule { val visualizeThread = new java.lang.Thread(() => while (true) { val res = Result.Success { - val (targets, rs, dest) = in.take() - val (sortedGroups, transitive) = mill.eval.Plan.plan(rs) + val (tasks, transitiveTasks, dest) = in.take() + val transitive = Graph.transitiveTargets(tasks) + val topoSorted = Graph.topoSorted(transitive) + val sortedGroups = Graph.groupAroundImportantTargets(topoSorted) { + case x: NamedTask[Any] if transitiveTasks.contains(x) => x + } + val (plannedForRender, _) = mill.eval.Plan.plan(transitiveTasks) - val goalSet = rs.toSet + val goalSet = transitiveTasks.toSet import guru.nidi.graphviz.model.Factory._ val edgesIterator = for ((k, vs) <- sortedGroups.items()) @@ -63,21 +69,21 @@ trait VisualizeModule extends mill.define.TaskModule { val edges = edgesIterator.map { case (k, v) => (k, v.toArray.distinct) }.toArray - val indexToTask = edges.flatMap { case (k, vs) => Iterator(k.task) ++ vs }.distinct + val indexToTask = edges.flatMap { case (k, vs) => Iterator(k) ++ vs }.distinct val taskToIndex = indexToTask.zipWithIndex.toMap val jgraph = new SimpleDirectedGraph[Int, DefaultEdge](classOf[DefaultEdge]) for (i <- indexToTask.indices) jgraph.addVertex(i) for ((src, dests) <- edges; dest <- dests) { - jgraph.addEdge(taskToIndex(src.task), taskToIndex(dest)) + jgraph.addEdge(taskToIndex(src), taskToIndex(dest)) } org.jgrapht.alg.TransitiveReduction.INSTANCE.reduce(jgraph) val nodes = indexToTask.map(t => - node(sortedGroups.lookupValue(t).render) + node(plannedForRender.lookupValue(t).render) .`with` { - if (targets.contains(t)) Style.SOLID + if (tasks.contains(t)) Style.SOLID else Style.DASHED } .`with`(Shape.BOX) diff --git a/scalalib/src/mill/javalib/palantirformat/PalantirFormatModule.scala b/scalalib/src/mill/javalib/palantirformat/PalantirFormatModule.scala index cd089c50af4..a5ee4b63380 100644 --- a/scalalib/src/mill/javalib/palantirformat/PalantirFormatModule.scala +++ b/scalalib/src/mill/javalib/palantirformat/PalantirFormatModule.scala @@ -97,7 +97,8 @@ object PalantirFormatModule extends ExternalModule with PalantirFormatBaseModule */ def formatAll( check: mainargs.Flag = mainargs.Flag(value = false), - @mainargs.arg(positional = true) sources: Tasks[Seq[PathRef]] + @mainargs.arg(positional = true) sources: Tasks[Seq[PathRef]] = + Tasks.resolveMainDefault("__.sources") ): Command[Unit] = Task.Command { val _sources = T.sequence(sources.value)().iterator.flatten diff --git a/scalalib/src/mill/scalalib/PlatformModuleBase.scala b/scalalib/src/mill/scalalib/PlatformModuleBase.scala new file mode 100644 index 00000000000..f9c6e4ca111 --- /dev/null +++ b/scalalib/src/mill/scalalib/PlatformModuleBase.scala @@ -0,0 +1,27 @@ +package mill.scalalib + +import mill._ +import os.Path + +trait PlatformModuleBase extends JavaModule { + override def millSourcePath: Path = super.millSourcePath / os.up + + /** + * The platform suffix of this [[PlatformModuleBase]]. Useful if you want to + * further customize the source paths or artifact names. + */ + def platformCrossSuffix: String = millModuleSegments + .value + .collect { case l: mill.define.Segment.Label => l.value } + .last + + override def sources: T[Seq[PathRef]] = Task.Sources { + super.sources().flatMap { source => + val platformPath = + PathRef(source.path / _root_.os.up / s"${source.path.last}-${platformCrossSuffix}") + Seq(source, platformPath) + } + } + + override def artifactNameParts: T[Seq[String]] = super.artifactNameParts().dropRight(1) +} diff --git a/scalalib/src/mill/scalalib/PlatformScalaModule.scala b/scalalib/src/mill/scalalib/PlatformScalaModule.scala index 82b15306ebd..2655d904979 100644 --- a/scalalib/src/mill/scalalib/PlatformScalaModule.scala +++ b/scalalib/src/mill/scalalib/PlatformScalaModule.scala @@ -1,7 +1,6 @@ package mill.scalalib -import mill._ -import os.Path +import mill.{PathRef, T, Task} /** * A [[ScalaModule]] intended for defining `.jvm`/`.js`/`.native` submodules @@ -14,8 +13,10 @@ import os.Path * built against and not something that should affect the filesystem path or * artifact name */ -trait PlatformScalaModule extends ScalaModule { - override def millSourcePath: Path = super.millSourcePath / os.up +trait PlatformScalaModule extends /* PlatformModuleBase with*/ ScalaModule { + // Cannot move stuff to PlatformModuleBase due to bincompat concerns + + override def millSourcePath: os.Path = super.millSourcePath / os.up /** * The platform suffix of this [[PlatformScalaModule]]. Useful if you want to From 5d0c8bfede67e78f69969232022b4d4a19a0bfa4 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Tue, 15 Oct 2024 00:11:31 +0800 Subject: [PATCH 43/47] Add broken internal links checker to docsite tests (#3736) Couldn't find any convenient ones on google that were easy to customize (e.g. whitelisting links, files, etc.) so I just wrote one myself using JSoup Will add a broken external links checker in a follow up --- .github/workflows/run-tests.yml | 2 +- contrib/playlib/readme.adoc | 4 +- .../ROOT/pages/depth/evaluation-model.adoc | 2 +- .../pages/extending/import-ivy-plugins.adoc | 2 +- .../ROOT/pages/fundamentals/query-syntax.adoc | 4 +- .../partials/Installation_IDE_Support.adoc | 4 +- .../ROOT/partials/Intro_to_Mill_Header.adoc | 2 +- docs/package.mill | 67 ++++++++++++++++++- .../tasks/2-primary-tasks/build.mill | 2 +- .../fundamentals/tasks/4-inputs/build.mill | 2 +- mill-build/build.sc | 3 +- 11 files changed, 78 insertions(+), 16 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 7ca644d904b..ce45802ec4b 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -44,7 +44,7 @@ jobs: - uses: actions/checkout@v4 with: { fetch-depth: 0 } - - run: ./mill -i docs.githubPages + - run: ./mill -i docs.githubPages + docs.checkBrokenLinks linux: needs: build-linux diff --git a/contrib/playlib/readme.adoc b/contrib/playlib/readme.adoc index ca36f54f2af..9070e9cd6dc 100644 --- a/contrib/playlib/readme.adoc +++ b/contrib/playlib/readme.adoc @@ -128,8 +128,8 @@ object core extends PlayApiModule { == Play configuration options -The Play modules themselves don't have specific configuration options at this point but the <> and the <<_twirl_configuration_options>> are applicable. +The Play modules themselves don't have specific configuration options at this point but the <<_router_configuration_options,router +module configuration options>> and the <> are applicable. == Additional play libraries diff --git a/docs/modules/ROOT/pages/depth/evaluation-model.adoc b/docs/modules/ROOT/pages/depth/evaluation-model.adoc index fabdd3088aa..2f3cd7b860b 100644 --- a/docs/modules/ROOT/pages/depth/evaluation-model.adoc +++ b/docs/modules/ROOT/pages/depth/evaluation-model.adoc @@ -39,7 +39,7 @@ of. In general, we have found that having "two places" to put code - outside of The hard boundary between these two phases is what lets users easily query and visualize their module hierarchy and task graph without running them: using -xref:scalalib/builtin-commands.adoc#inspect[inspect], xref:scalalib/builtin-commands.adoc#plan[plan], +xref:scalalib/builtin-commands.adoc#_inspect[inspect], xref:scalalib/builtin-commands.adoc#_plan[plan], xref:scalalib/builtin-commands.adoc#_visualize[visualize], etc.. This helps keep your Mill build discoverable even as the `build.mill` codebase grows. diff --git a/docs/modules/ROOT/pages/extending/import-ivy-plugins.adoc b/docs/modules/ROOT/pages/extending/import-ivy-plugins.adoc index 2f3254f5e3f..82bd846eb29 100644 --- a/docs/modules/ROOT/pages/extending/import-ivy-plugins.adoc +++ b/docs/modules/ROOT/pages/extending/import-ivy-plugins.adoc @@ -21,7 +21,7 @@ include::partial$example/extending/imports/2-import-ivy-scala.adoc[] == Importing Plugins Mill plugins are ordinary JVM libraries jars and are loaded as any other external dependency with -the xref:extending/import-ivy-plugins.adoc[`import $ivy` mechanism]. +the `import $ivy` mechanism. There exist a large number of Mill plugins, Many of them are available on GitHub and via Maven Central. We also have a list of plugins, which is most likely not complete, but it diff --git a/docs/modules/ROOT/pages/fundamentals/query-syntax.adoc b/docs/modules/ROOT/pages/fundamentals/query-syntax.adoc index 6f8b461beae..36ac714e7c6 100644 --- a/docs/modules/ROOT/pages/fundamentals/query-syntax.adoc +++ b/docs/modules/ROOT/pages/fundamentals/query-syntax.adoc @@ -28,7 +28,7 @@ There are two kind of segments: _label segments_ and _cross segments_. _Label segments_ are the components of a task path and have the same restriction as Scala identifiers. They must start with a letter and may contain letters, numbers and a limited set of special characters `-` (dash), `_` (underscore). -They are used to denote Mill modules, tasks, but in the case of xref:fundamentals/modules.adoc#external-modules[external modules] their Scala package names. +They are used to denote Mill modules, tasks, but in the case of xref:fundamentals/modules.adoc#_external_modules[external modules] their Scala package names. _Cross segments_ start with a label segment but contain additional square brackets (`[`, `]`]) and are used to denote cross module and their parameters. @@ -133,7 +133,7 @@ There is a subtile difference between the expansion of < mill foo.run hello # <1> diff --git a/docs/modules/ROOT/partials/Installation_IDE_Support.adoc b/docs/modules/ROOT/partials/Installation_IDE_Support.adoc index 193fce7bcd7..9a6c04edf10 100644 --- a/docs/modules/ROOT/partials/Installation_IDE_Support.adoc +++ b/docs/modules/ROOT/partials/Installation_IDE_Support.adoc @@ -389,7 +389,7 @@ automatically open a pull request to update your Mill version (in `.mill-version` or `.config/mill-version` file), whenever there is a newer version available. TIP: Scala Steward can also -xref:scalalib/module-config.adoc#_keeping_up_to_date_with_scala_steward[scan your project dependencies] +xref:scalalib/dependencies.adoc#_keeping_up_to_date_with_scala_steward[scan your project dependencies] and keep them up-to-date. === Development Releases @@ -401,7 +401,7 @@ https://github.com/com-lihaoyi/mill/releases[available] as binaries named `+#.#.#-n-hash+` linked to the latest tag. The easiest way to use a development release is to use one of the -<<_bootstrap_scripts>>, which support <<_overriding_mill_versions>> via an +<<_bootstrap_scripts>>, which support overriding Mill versions via an `MILL_VERSION` environment variable or a `.mill-version` or `.config/mill-version` file. diff --git a/docs/modules/ROOT/partials/Intro_to_Mill_Header.adoc b/docs/modules/ROOT/partials/Intro_to_Mill_Header.adoc index bc2e131ff22..017e3a7e346 100644 --- a/docs/modules/ROOT/partials/Intro_to_Mill_Header.adoc +++ b/docs/modules/ROOT/partials/Intro_to_Mill_Header.adoc @@ -28,7 +28,7 @@ digraph G { {mill-github-url}[Mill] is a fast multi-language JVM build tool that supports {language}, making your common development workflows xref:comparisons/maven.adoc[5-10x faster to Maven], or xref:comparisons/gradle.adoc[2-4x faster than Gradle], and -xref:comparisons/sbt[easier to use than SBT]. +xref:comparisons/sbt.adoc[easier to use than SBT]. Mill aims to make your JVM project's build process performant, maintainable, and flexible even as it grows from a small project to a large codebase or monorepo with hundreds of modules: diff --git a/docs/package.mill b/docs/package.mill index 3768b9788fd..55846d780dd 100644 --- a/docs/package.mill +++ b/docs/package.mill @@ -1,9 +1,9 @@ package build.docs +import org.jsoup._ import mill.util.Jvm import mill._, scalalib._ import de.tobiasroeser.mill.vcs.version.VcsVersion -import guru.nidi.graphviz.engine.AbstractJsGraphvizEngine -import guru.nidi.graphviz.engine.{Format, Graphviz} +import collection.JavaConverters._ /** Generates the mill documentation with Antora. */ object `package` extends RootModule { @@ -205,7 +205,7 @@ object `package` extends RootModule { | sources: | - url: ${if (authorMode) build.baseDir else build.Settings.projectUrl} | branches: [] - | tags: ${build.Settings.legacyDocTags.map("'" + _ + "'").mkString("[", ",", "]")} + | tags: ${build.Settings.legacyDocTags.filter(_ => !authorMode).map("'" + _ + "'").mkString("[", ",", "]")} | start_path: docs/antora | |${taggedSources.mkString("\n\n")} @@ -265,6 +265,7 @@ object `package` extends RootModule { T.log.outputStream.println( s"You can browse the local pages at: ${(pages.path / "index.html").toNIO.toUri()}" ) + pages } def generatePages(authorMode: Boolean) = T.task { extraSources: Seq[os.Path] => @@ -346,4 +347,64 @@ object `package` extends RootModule { } } } + + def allLinksAndAnchors: T[IndexedSeq[(os.Path, Seq[(String, String)], Seq[(String, String)], Set[String])]] = Task { + val base = fastPages().path + val validExtensions = Set("html", "scala") + for (path <- os.walk(base) if validExtensions(path.ext)) + yield { + val parsed = Jsoup.parse(os.read(path)) + val (remoteLinks, localLinks) = parsed + .select("a") + .asScala + .map(e => (e.toString, e.attr("href"))) + .toSeq + .partition{case (e, l) => l.startsWith("http://") || l.startsWith("https://")} + ( + path, + remoteLinks, + localLinks.map{case (e, l) => (e, l.stripPrefix("file:"))}, + parsed.select("*").asScala.map(_.attr("id")).filter(_.nonEmpty).toSet, + ) + } + } + def checkBrokenLinks() = Task.Command{ + if (brokenLinks().nonEmpty){ + throw new Exception("Broken Links: " + upickle.default.write(brokenLinks(), indent = 2)) + } + } + def brokenLinks: T[Map[os.Path, Seq[(String, String)]]] = Task{ + val allLinksAndAnchors0 = allLinksAndAnchors() + val pathsToIds = allLinksAndAnchors() + .map{case (path, remoteLinks, localLinks, ids) => (path, ids)} + .toMap + + val brokenLinksPerPath: Seq[(os.Path, Seq[(String, String)])] = + for ((path, remoteLinks, localLinks, ids) <- allLinksAndAnchors0) yield{ + ( + path, + localLinks.flatMap{case (elementString, url) => + val (baseUrl, anchorOpt) = url match { + case s"#$anchor" => (path.toString, Some(anchor)) + case s"$prefix#$anchor" => (prefix, Some(anchor)) + + case url => (url, None) + } + + val dest0 = os.Path(baseUrl, path / "..") + val possibleDests = Seq(dest0, dest0 / "index.html") + possibleDests.find(os.exists(_)) match{ + case None => Some((elementString, url)) + case Some(dest) => + anchorOpt.collect{case a if !pathsToIds.getOrElse(dest, Set()).contains(a) => (elementString, url)} + } + } + ) + } + + val nonEmptyBrokenLinksPerPath = brokenLinksPerPath + .filter{ case (path, items) => path.last != "404.html" && items.nonEmpty } + + nonEmptyBrokenLinksPerPath.toMap + } } diff --git a/example/fundamentals/tasks/2-primary-tasks/build.mill b/example/fundamentals/tasks/2-primary-tasks/build.mill index 46d7ac3ef8b..914ff7f7c76 100644 --- a/example/fundamentals/tasks/2-primary-tasks/build.mill +++ b/example/fundamentals/tasks/2-primary-tasks/build.mill @@ -1,7 +1,7 @@ // There are three primary kinds of _Tasks_ that you should care about: // // * <<_sources>>, defined using `Task.Sources {...}` -// * <<_tasks>>, defined using `Task {...}` +// * <<_cached_tasks>>, defined using `Task {...}` // * <<_commands>>, defined using `Task.Command {...}` // === Sources diff --git a/example/fundamentals/tasks/4-inputs/build.mill b/example/fundamentals/tasks/4-inputs/build.mill index 7b7c65c00b8..01efc400aa8 100644 --- a/example/fundamentals/tasks/4-inputs/build.mill +++ b/example/fundamentals/tasks/4-inputs/build.mill @@ -13,7 +13,7 @@ def myInput = Task.Input { // arbitrary block of code. // // Inputs can be used to force re-evaluation of some external property that may -// affect your build. For example, if I have a <<_cached_task, cached task>> `bar` that +// affect your build. For example, if I have a xref:#_cached_tasks[cached task] `bar` that // calls out to `git` to compute the latest commit hash and message directly, // that target does not have any `Task` inputs and so will never re-compute // even if the external `git` status changes: diff --git a/mill-build/build.sc b/mill-build/build.sc index 454349f6657..769fca21f6a 100644 --- a/mill-build/build.sc +++ b/mill-build/build.sc @@ -10,6 +10,7 @@ object `package` extends MillBuildRootModule { // TODO: implement empty version for ivy deps as we do in import parser ivy"com.lihaoyi::mill-contrib-buildinfo:${mill.api.BuildInfo.millVersion}", ivy"com.goyeau::mill-scalafix::0.4.1", - ivy"com.lihaoyi::mill-main-graphviz:${mill.api.BuildInfo.millVersion}" + ivy"com.lihaoyi::mill-main-graphviz:${mill.api.BuildInfo.millVersion}", + ivy"org.jsoup:jsoup:1.12.1" ) } From 4a9d469c8b49599dfd1fff9b59030e04facfb17c Mon Sep 17 00:00:00 2001 From: Nikita Ogorodnikov <4046447+0xnm@users.noreply.github.com> Date: Tue, 15 Oct 2024 00:57:53 +0200 Subject: [PATCH 44/47] Lift Kotlin 2+ version usage constraint for Kotlin/JS (#3739) Since `klib` files can be fetched now, it removes the constraint for the Kotlin 2+ version usage (where `stdlib` is now published only as `klib`). I added a `kotlinVersion` to the build info, but it is not actually used anywhere in the module code (because the default version for the module is never specified), only in the test. Co-authored-by: 0xnm <0xnm@users.noreply.github.com> --- build.mill | 3 ++- kotlinlib/package.mill | 1 + .../mill/kotlinlib/js/KotlinJSModule.scala | 12 ++++------ .../qux/src/qux/ExternalDependency.kt | 8 +++++++ .../js/KotlinJSKotlinVersionsTests.scala | 24 +++++++++++++------ 5 files changed, 32 insertions(+), 16 deletions(-) create mode 100644 kotlinlib/test/resources/kotlin-js/qux/src/qux/ExternalDependency.kt diff --git a/build.mill b/build.mill index 5ff7c20bba0..ade4f58256b 100644 --- a/build.mill +++ b/build.mill @@ -187,7 +187,8 @@ object Deps { val requests = ivy"com.lihaoyi::requests:0.9.0" val logback = ivy"ch.qos.logback:logback-classic:1.5.7" val sonatypeCentralClient = ivy"com.lumidion::sonatype-central-client-requests:0.3.0" - val kotlinCompiler = ivy"org.jetbrains.kotlin:kotlin-compiler:1.9.24" + val kotlinVersion = "2.0.21" + val kotlinCompiler = ivy"org.jetbrains.kotlin:kotlin-compiler:$kotlinVersion" object RuntimeDeps { val errorProneCore = ivy"com.google.errorprone:error_prone_core:2.31.0" diff --git a/kotlinlib/package.mill b/kotlinlib/package.mill index 871838829c1..01183c4fd23 100644 --- a/kotlinlib/package.mill +++ b/kotlinlib/package.mill @@ -15,6 +15,7 @@ object `package` extends RootModule with build.MillPublishScalaModule with Build def buildInfoPackageName = "mill.kotlinlib" def buildInfoObjectName = "Versions" def buildInfoMembers = Seq( + BuildInfo.Value("kotlinVersion", build.Deps.kotlinVersion, "Version of Kotlin"), BuildInfo.Value("koverVersion", build.Deps.RuntimeDeps.koverVersion, "Version of Kover."), BuildInfo.Value("ktfmtVersion", build.Deps.RuntimeDeps.ktfmtVersion, "Version of Ktfmt."), BuildInfo.Value("detektVersion", build.Deps.RuntimeDeps.detektVersion, "Version of Detekt."), diff --git a/kotlinlib/src/mill/kotlinlib/js/KotlinJSModule.scala b/kotlinlib/src/mill/kotlinlib/js/KotlinJSModule.scala index 2acfda58be5..e7205b7d4b2 100644 --- a/kotlinlib/src/mill/kotlinlib/js/KotlinJSModule.scala +++ b/kotlinlib/src/mill/kotlinlib/js/KotlinJSModule.scala @@ -114,8 +114,8 @@ trait KotlinJSModule extends KotlinModule { outer => val linkResult = linkBinary().classes if ( - moduleKind == ModuleKind.NoModule - && linkResult.path.toIO.listFiles().count(_.getName.endsWith(".js")) > 1 + moduleKind == ModuleKind.NoModule && + linkResult.path.toIO.listFiles().count(_.getName.endsWith(".js")) > 1 ) { T.log.info("No module type is selected for the executable, but multiple .js files found in the output folder." + " This will probably lead to the dependency resolution failure.") @@ -236,13 +236,12 @@ trait KotlinJSModule extends KotlinModule { outer => val versionAllowed = kotlinVersion.split("\\.").map(_.toInt) match { case Array(1, 8, z) => z >= 20 case Array(1, y, _) => y >= 9 - case Array(2, _, _) => false - case _ => false + case _ => true } if (!versionAllowed) { // have to put this restriction, because for older versions some compiler options either didn't exist or // had different names. It is possible to go to the lower version supported with a certain effort. - ctx.log.error("Minimum supported Kotlin version for JS target is 1.8.20, maximum is 1.9.25") + ctx.log.error("Minimum supported Kotlin version for JS target is 1.8.20.") return Result.Aborted } @@ -255,9 +254,6 @@ trait KotlinJSModule extends KotlinModule { outer => case None => allKotlinSourceFiles.map(_.path.toIO.getAbsolutePath) } - // TODO: Cannot support Kotlin 2+, because it doesn't publish .jar anymore, but .klib files only. Coursier is not - // able to work with that (unlike Gradle, which can leverage .module metadata). - // https://repo1.maven.org/maven2/org/jetbrains/kotlin/kotlin-stdlib-js/2.0.20/ val librariesCp = librariesClasspath.map(_.path) .filter(os.exists) .filter(isKotlinJsLibrary) diff --git a/kotlinlib/test/resources/kotlin-js/qux/src/qux/ExternalDependency.kt b/kotlinlib/test/resources/kotlin-js/qux/src/qux/ExternalDependency.kt new file mode 100644 index 00000000000..861c017c27f --- /dev/null +++ b/kotlinlib/test/resources/kotlin-js/qux/src/qux/ExternalDependency.kt @@ -0,0 +1,8 @@ +package qux + +import kotlinx.html.div +import kotlinx.html.stream.createHTML + +fun doThing() { + println(createHTML().div { +"Hello" }) +} diff --git a/kotlinlib/test/src/mill/kotlinlib/js/KotlinJSKotlinVersionsTests.scala b/kotlinlib/test/src/mill/kotlinlib/js/KotlinJSKotlinVersionsTests.scala index 1fc6f381c4e..1e324361836 100644 --- a/kotlinlib/test/src/mill/kotlinlib/js/KotlinJSKotlinVersionsTests.scala +++ b/kotlinlib/test/src/mill/kotlinlib/js/KotlinJSKotlinVersionsTests.scala @@ -10,10 +10,7 @@ object KotlinJSKotlinVersionsTests extends TestSuite { private val resourcePath = os.Path(sys.env("MILL_TEST_RESOURCE_DIR")) / "kotlin-js" private val kotlinLowestVersion = "1.8.20" - // TODO: Cannot support Kotlin 2+, because it doesn't publish .jar anymore, but .klib files only. Coursier is not - // able to work with that (unlike Gradle, which can leverage .module metadata). - // https://repo1.maven.org/maven2/org/jetbrains/kotlin/kotlin-stdlib-js/2.0.20/ - private val kotlinHighestVersion = "1.9.25" + private val kotlinHighestVersion = mill.kotlinlib.Versions.kotlinVersion private val kotlinVersions = Seq(kotlinLowestVersion, kotlinHighestVersion) trait KotlinJSCrossModule extends KotlinJSModule with Cross.Module[String] { @@ -21,13 +18,26 @@ object KotlinJSKotlinVersionsTests extends TestSuite { } trait KotlinJSFooCrossModule extends KotlinJSCrossModule { - override def moduleDeps = Seq(module.bar(crossValue)) + override def moduleDeps = Seq(module.bar(crossValue), module.qux(crossValue)) } - object module extends TestBaseModule { + trait KotlinJSQuxCrossModule extends KotlinJSCrossModule { + override def ivyDeps = { + // 0.10+ cannot be built with Kotlin 1.8 (it was built with Kotlin 1.9.10 itself). ABI incompatibility? + val kotlinxHtmlVersion = crossValue.split("\\.").map(_.toInt) match { + case Array(1, 8, _) => "0.9.1" + case _ => "0.11.0" + } + super.ivyDeps() ++ Agg( + ivy"org.jetbrains.kotlinx:kotlinx-html-js:$kotlinxHtmlVersion" + ) + } + } - object bar extends Cross[KotlinJSCrossModule](kotlinVersions) + object module extends TestBaseModule { object foo extends Cross[KotlinJSFooCrossModule](kotlinVersions) + object bar extends Cross[KotlinJSCrossModule](kotlinVersions) + object qux extends Cross[KotlinJSQuxCrossModule](kotlinVersions) } private def testEval() = UnitTester(module, resourcePath) From 06da5d1f212b11ed52194c1022179f1616a472a4 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Tue, 15 Oct 2024 09:38:26 +0800 Subject: [PATCH 45/47] Prompt logger fixes (#3741) --- main/util/src/mill/util/PromptLogger.scala | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/main/util/src/mill/util/PromptLogger.scala b/main/util/src/mill/util/PromptLogger.scala index 1a08d6cefcc..06693ec1418 100644 --- a/main/util/src/mill/util/PromptLogger.scala +++ b/main/util/src/mill/util/PromptLogger.scala @@ -46,7 +46,8 @@ private[mill] class PromptLogger( enableTicker, systemStreams0, () => promptLineState.writeCurrentPrompt(), - interactive = () => termDimensions._1.nonEmpty + interactive = () => termDimensions._1.nonEmpty, + paused = () => runningState.paused ) private object runningState extends RunningState( @@ -57,7 +58,7 @@ private[mill] class PromptLogger( systemStreams0.err.write(AnsiNav.clearScreen(0).getBytes) systemStreams0.err.flush() }, - this + synchronizer = this ) val promptUpdaterThread = new Thread( @@ -149,8 +150,8 @@ private[mill] object PromptLogger { /** * Manages the paused/unpaused/stopped state of the prompt logger. Encapsulate in a separate * class because it has to maintain some invariants and ensure book-keeping is properly done - * when the paused state change, e.g. interrupting the prompt updater thread, waiting for - * `pauseNoticed` to fire and clearing the screen when the ticker is paused. + * when the paused state change, e.g. interrupting the prompt updater thread and clearing + * the screen when the ticker is paused. */ class RunningState( enableTicker: Boolean, @@ -158,7 +159,7 @@ private[mill] object PromptLogger { clearOnPause: () => Unit, // Share the same synchronized lock as the parent PromptLogger, to simplify // reasoning about concurrency since it's not performance critical - synchronizer: PromptLogger + synchronizer: AnyRef ) { @volatile private var stopped0 = false @volatile private var paused0 = false @@ -200,7 +201,8 @@ private[mill] object PromptLogger { enableTicker: Boolean, systemStreams0: SystemStreams, writeCurrentPrompt: () => Unit, - interactive: () => Boolean + interactive: () => Boolean, + paused: () => Boolean ) { // We force both stdout and stderr streams into a single `Piped*Stream` pair via @@ -230,7 +232,10 @@ private[mill] object PromptLogger { // every small write when most such prompts will get immediately over-written // by subsequent writes if (enableTicker && src.available() == 0) { - if (interactive()) writeCurrentPrompt() + // Do not print the prompt when it is paused. Ideally stream redirecting would + // prevent any writes from coming to this stream when paused, somehow writes + // sometimes continue to come in, so just handle them gracefully. + if (interactive() && !paused()) writeCurrentPrompt() pumperState = PumperState.prompt } } From fcd853b837ae65ee12f07b54af836d9acc9b28bd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Oct 2024 09:38:48 +0800 Subject: [PATCH 46/47] Bump actions/upload-artifact from 4.4.1 to 4.4.3 (#3740) --- .github/workflows/run-mill-action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run-mill-action.yml b/.github/workflows/run-mill-action.yml index ce791fddfce..84f4cc75b72 100644 --- a/.github/workflows/run-mill-action.yml +++ b/.github/workflows/run-mill-action.yml @@ -82,7 +82,7 @@ jobs: shell: bash continue-on-error: true - - uses: actions/upload-artifact@v4.4.1 + - uses: actions/upload-artifact@v4.4.3 with: path: . name: ${{ inputs.os }}-artifact From 7c0f7a0d463ffc1021366e4aba0fd5d23916f336 Mon Sep 17 00:00:00 2001 From: Li Haoyi Date: Tue, 15 Oct 2024 13:00:13 +0800 Subject: [PATCH 47/47] Add broken remote link checker (#3738) Will probably need to fix a lot of links before this can land --- contrib/bloop/readme.adoc | 3 +- contrib/testng/readme.adoc | 2 +- .../ROOT/pages/depth/design-principles.adoc | 2 +- .../pages/extending/thirdparty-plugins.adoc | 4 +- .../partials/Installation_IDE_Support.adoc | 2 +- docs/package.mill | 66 +++++++++++++++++-- .../javalib/linting/1-error-prone/build.mill | 2 +- 7 files changed, 70 insertions(+), 11 deletions(-) diff --git a/contrib/bloop/readme.adoc b/contrib/bloop/readme.adoc index a60f36ddae5..1eed3e5b27e 100644 --- a/contrib/bloop/readme.adoc +++ b/contrib/bloop/readme.adoc @@ -46,5 +46,6 @@ located inside a project workspace. == Note regarding current mill support in bloop -The mill-bloop integration currently present in the https://github.com/scalacenter/bloop/blob/master/integrations/mill-bloop/src/main/scala/bloop/integrations/mill/MillBloop.scala#L10[bloop codebase] +The mill-bloop integration currently present in the +https://github.com/scalacenter/bloop[bloop codebase] will be deprecated in favour of this implementation. diff --git a/contrib/testng/readme.adoc b/contrib/testng/readme.adoc index f28d6b07b03..baa12bb3a77 100644 --- a/contrib/testng/readme.adoc +++ b/contrib/testng/readme.adoc @@ -2,7 +2,7 @@ :page-aliases: TestNG_TestFramework.adoc -Provides support for https://testng.org/doc/index.html[TestNG]. +Provides support for https://testng.org[TestNG]. To use TestNG as test framework, you need to add it to the `TestModule.testFramework` property. diff --git a/docs/modules/ROOT/pages/depth/design-principles.adoc b/docs/modules/ROOT/pages/depth/design-principles.adoc index 721d71ae55d..a12dcdcfd93 100644 --- a/docs/modules/ROOT/pages/depth/design-principles.adoc +++ b/docs/modules/ROOT/pages/depth/design-principles.adoc @@ -182,7 +182,7 @@ that would make a build tool hard to understand. Before you continue, take a moment to think: how would you answer to each of those questions using an existing build tool you are familiar with? Different tools like http://www.scala-sbt.org/[SBT], -https://fake.build/legacy-index.html[Fake], https://gradle.org/[Gradle] or +https://fake.build[Fake], https://gradle.org/[Gradle] or https://gruntjs.com/[Grunt] have very different answers. Mill aims to provide the answer to these questions using as few, as familiar diff --git a/docs/modules/ROOT/pages/extending/thirdparty-plugins.adoc b/docs/modules/ROOT/pages/extending/thirdparty-plugins.adoc index 3684e8c70fd..593f9fb4cee 100644 --- a/docs/modules/ROOT/pages/extending/thirdparty-plugins.adoc +++ b/docs/modules/ROOT/pages/extending/thirdparty-plugins.adoc @@ -571,8 +571,8 @@ bash> mill site.jbakeServe == JBuildInfo -This is a https://www.lihaoyi.com/mill/[mill] module similar to -https://www.lihaoyi.com/mill/page/contrib-modules.html#buildinfo[BuildInfo] +This is a Mill module similar to +xref:contrib/buildinfo.adoc[BuildInfo] but for Java. It will generate a Java class containing information from your build. diff --git a/docs/modules/ROOT/partials/Installation_IDE_Support.adoc b/docs/modules/ROOT/partials/Installation_IDE_Support.adoc index 9a6c04edf10..7e317415710 100644 --- a/docs/modules/ROOT/partials/Installation_IDE_Support.adoc +++ b/docs/modules/ROOT/partials/Installation_IDE_Support.adoc @@ -274,7 +274,7 @@ CAUTION: Some of the installations via package managers install a fixed version === OS X -Installation via https://github.com/Homebrew/homebrew-core/blob/master/Formula/mill.rb[homebrew]: +Installation via https://github.com/Homebrew/homebrew-core/blob/master/Formula/m/mill.rb[homebrew]: [source,sh] ---- diff --git a/docs/package.mill b/docs/package.mill index 55846d780dd..2f57e591722 100644 --- a/docs/package.mill +++ b/docs/package.mill @@ -368,14 +368,72 @@ object `package` extends RootModule { ) } } + + def brokenRemoteLinks: T[Map[os.Path, Seq[(String, String, Int)]]] = Task{ + val allLinks = allLinksAndAnchors() + .flatMap { case (path, remoteLinks, localLinks, ids) => remoteLinks } + .map(_._2) + .filter{l => + // ignore example links since those are expected to be unresolved until + // a stable version is published and artifacts are uploaded to github + !l.contains("/example/") && + !l.contains("/releases/download/") && + // Ignore internal repo links in the changelog because there are a lot + // of them and they're not very interesting to check and verify. + !l.contains("https://github.com/com-lihaoyi/mill/pull/") && + !l.contains("https://github.com/com-lihaoyi/mill/milestone/") && + !l.contains("https://github.com/com-lihaoyi/mill/compare/") && + // Link meant for API configuration, not for clicking + !l.contains("https://s01.oss.sonatype.org/service/local") && + // SOmehow this server doesn't respond properly to HEAD requests even though GET works + !l.contains("https://marketplace.visualstudio.com/items") + } + .toSet + + // Try to fetch all the links serially. It isn't worth trying to parallelize it + // because if we go too fast the remote websites tend to rate limit us anyway + val linksToStatusCodes = allLinks.toSeq.zipWithIndex + .map{ case (link, i) => + val key = s"$i/${allLinks.size}" + println(s"Checking link $link $key") + val start = System.currentTimeMillis() + val res = requests.head(link, check = false).statusCode + val duration = System.currentTimeMillis() - start + val remaining = 1000 - duration + if (remaining > 0) Thread.sleep(remaining) // try to avoid being rate limited + (link, res) + } + .toMap + + allLinksAndAnchors() + .map{case (path, remoteLinks, localLinks, ids) => + ( + path, + remoteLinks.collect{ + case (e, l) + if allLinks.contains(l) + && !linksToStatusCodes(l).toString.startsWith("2") => + (e, l, linksToStatusCodes(l)) + } + ) + } + .filter(_._2.nonEmpty) + .toMap + } + + def checkBrokenLinks() = Task.Command{ - if (brokenLinks().nonEmpty){ - throw new Exception("Broken Links: " + upickle.default.write(brokenLinks(), indent = 2)) + if (brokenLocalLinks().nonEmpty){ + throw new Exception("Broken Local Links: " + upickle.default.write(brokenLocalLinks(), indent = 2)) + } + if (brokenRemoteLinks().nonEmpty){ + throw new Exception("Broken Rmote Links: " + upickle.default.write(brokenRemoteLinks(), indent = 2)) } } - def brokenLinks: T[Map[os.Path, Seq[(String, String)]]] = Task{ + + def brokenLocalLinks: T[Map[os.Path, Seq[(String, String)]]] = Task{ val allLinksAndAnchors0 = allLinksAndAnchors() - val pathsToIds = allLinksAndAnchors() + val pathsToIds = allLinksAndAnchors0 .map{case (path, remoteLinks, localLinks, ids) => (path, ids)} .toMap diff --git a/example/javalib/linting/1-error-prone/build.mill b/example/javalib/linting/1-error-prone/build.mill index 8b9a656368a..07c5f6f19ed 100644 --- a/example/javalib/linting/1-error-prone/build.mill +++ b/example/javalib/linting/1-error-prone/build.mill @@ -51,7 +51,7 @@ object `package` extends RootModule with JavaModule with ErrorProneModule { // // `def errorProneVersion: T[String]`:: // The `error-prone` version to use. Defaults to [[BuildInfo.errorProneVersion]], the version used to build and test the module. -// Find the latest at https://mvnrepository.com/artifact/com.google.errorprone/error_prone_core[mvnrepository.com] +// Find the list of versions and changlog at https://github.com/google/error-prone/releases // // `def errorProneOptions: T[Seq[String]]`:: // Options directly given to the `error-prone` processor.