Acquire a lock on the out dir in order to run tasks / commands

com-lihaoyi · Sep 24, 2024 · aa2c64a · aa2c64a
1 parent c64ab07
commit aa2c64a
Show file tree

Hide file tree

Showing 12 changed files with 129 additions and 27 deletions.
diff --git a/main/api/src/mill/api/Logger.scala b/main/api/src/mill/api/Logger.scala
@@ -2,6 +2,8 @@ package mill.api
 
 import java.io.{InputStream, PrintStream}
 
+import mill.main.client.lock.{Lock, Locked}
+
 /**
  * The standard logging interface of the Mill build tool.
  *
@@ -53,4 +55,14 @@ trait Logger {
   def debugEnabled: Boolean = false
 
   def close(): Unit = ()
+
+  def waitForLock(lock: Lock): Locked = {
+    val tryLocked = lock.tryLock()
+    if (tryLocked.isLocked())
+      tryLocked
+    else {
+      info("Another Mill process is running tasks, waiting for it to be done...")
+      lock.lock()
+    }
+  }
 }
diff --git a/main/client/src/mill/main/client/OutFiles.java b/main/client/src/mill/main/client/OutFiles.java
@@ -57,4 +57,9 @@ public class OutFiles {
      */
     final public static String millNoServer = "mill-no-server";
 
+    /**
+     * Lock file used for exclusive access to the Mill output directory
+     */
+    final public static String millLock = "mill-lock";
+
 }
diff --git a/main/client/src/mill/main/client/lock/DummyLock.java b/main/client/src/mill/main/client/lock/DummyLock.java
@@ -0,0 +1,22 @@
+package mill.main.client.lock;
+
+import java.util.concurrent.locks.ReentrantLock;
+
+class DummyLock extends Lock {
+
+    public boolean probe() {
+        return true;
+    }
+
+    public Locked lock() {
+        return new DummyTryLocked();
+    }
+
+    public TryLocked tryLock() {
+        return new DummyTryLocked();
+    }
+
+    @Override
+    public void close() throws Exception {
+    }
+}
diff --git a/main/client/src/mill/main/client/lock/DummyTryLocked.java b/main/client/src/mill/main/client/lock/DummyTryLocked.java
@@ -0,0 +1,11 @@
+package mill.main.client.lock;
+
+class DummyTryLocked implements TryLocked {
+    public DummyTryLocked() {
+    }
+
+    public boolean isLocked(){ return true; }
+
+    public void release() throws Exception {
+    }
+}
diff --git a/main/client/src/mill/main/client/lock/Lock.java b/main/client/src/mill/main/client/lock/Lock.java
@@ -15,4 +15,17 @@ public void await() throws Exception {
      */
     public abstract boolean probe() throws Exception;
     public void delete() throws Exception {}
+
+    public static Lock file(String path) throws Exception {
+        return new FileLock(path);
+    }
+
+    public static Lock memory() {
+        return new MemoryLock();
+    }
+
+    public static Lock dummy() {
+        return new DummyLock();
+    }
+
 }
diff --git a/main/eval/src/mill/eval/EvaluatorCore.scala b/main/eval/src/mill/eval/EvaluatorCore.scala
@@ -7,11 +7,13 @@ import mill.define._
 import mill.eval.Evaluator.TaskResult
 import mill.main.client.OutFiles._
 import mill.main.client.EnvVars
+import mill.main.client.lock.Lock
 import mill.util._
 
 import java.util.concurrent.atomic.{AtomicBoolean, AtomicInteger}
 import scala.collection.mutable
 import scala.concurrent._
+import scala.util.Using
 
 /**
  * Core logic of evaluating tasks, without any user-facing helper methods
@@ -32,8 +34,6 @@ private[mill] trait EvaluatorCore extends GroupEvaluator {
       logger: ColorLogger = baseLogger,
       serialCommandExec: Boolean = false
   ): Evaluator.Results = {
-    os.makeDir.all(outPath)
-
     PathRef.validatedPaths.withValue(new PathRef.ValidatedPaths()) {
       val ec =
         if (effectiveThreadCount == 1) ExecutionContexts.RunNow
@@ -43,7 +43,19 @@ private[mill] trait EvaluatorCore extends GroupEvaluator {
         if (effectiveThreadCount == 1) ""
         else s"#${if (effectiveThreadCount > 9) f"$threadId%02d" else threadId} "
 
-      try evaluate0(goals, logger, reporter, testReporter, ec, contextLoggerMsg, serialCommandExec)
+      try
+        Using.resource(logger.waitForLock(outPathLockOpt.getOrElse(Lock.dummy()))) { _ =>
+          os.makeDir.all(outPath)
+          evaluate0(
+            goals,
+            logger,
+            reporter,
+            testReporter,
+            ec,
+            contextLoggerMsg,
+            serialCommandExec
+          )
+        }
       finally ec.close()
     }
   }

diff --git a/main/eval/src/mill/eval/EvaluatorImpl.scala b/main/eval/src/mill/eval/EvaluatorImpl.scala
@@ -3,6 +3,7 @@ package mill.eval
 import mill.api.{CompileProblemReporter, Strict, TestReporter, Val}
 import mill.api.Strict.Agg
 import mill.define._
+import mill.main.client.lock.Lock
 import mill.util._
 
 import scala.collection.mutable
@@ -17,6 +18,7 @@ private[mill] case class EvaluatorImpl(
     home: os.Path,
     workspace: os.Path,
     outPath: os.Path,
+    outPathLockOpt: Option[Lock],
     externalOutPath: os.Path,
     override val rootModule: mill.define.BaseModule,
     baseLogger: ColorLogger,

diff --git a/main/eval/src/mill/eval/GroupEvaluator.scala b/main/eval/src/mill/eval/GroupEvaluator.scala
@@ -5,6 +5,7 @@ import mill.api.Strict.Agg
 import mill.api._
 import mill.define._
 import mill.eval.Evaluator.TaskResult
+import mill.main.client.lock.Lock
 import mill.util._
 
 import java.io.PrintStream
@@ -23,6 +24,7 @@ private[mill] trait GroupEvaluator {
   def home: os.Path
   def workspace: os.Path
   def outPath: os.Path
+  def outPathLockOpt: Option[Lock]
   def externalOutPath: os.Path
   def rootModule: mill.define.BaseModule
   def classLoaderSigHash: Int

diff --git a/runner/src/mill/runner/MillBuildBootstrap.scala b/runner/src/mill/runner/MillBuildBootstrap.scala
@@ -6,12 +6,15 @@ import mill.main.client.CodeGenConstants._
 import mill.api.{PathRef, Val, internal}
 import mill.eval.Evaluator
 import mill.main.RunScript
+import mill.main.client.lock.Lock
 import mill.resolve.SelectMode
 import mill.define.{BaseModule, Discover, Segments}
 import mill.main.client.OutFiles.{millBuild, millRunnerState}
 
 import java.net.URLClassLoader
 
+import scala.util.Using
+
 /**
  * Logic around bootstrapping Mill, creating a [[MillBuildRootModule.BootstrapModule]]
  * and compiling builds/meta-builds and classloading their [[RootModule]]s so we
@@ -31,6 +34,7 @@ import java.net.URLClassLoader
 class MillBuildBootstrap(
     projectRoot: os.Path,
     output: os.Path,
+    outputLockOpt: Option[Lock],
     home: os.Path,
     keepGoing: Boolean,
     imports: Seq[String],
@@ -53,12 +57,14 @@ class MillBuildBootstrap(
   def evaluate(): Watching.Result[RunnerState] = CliImports.withValue(imports) {
     val runnerState = evaluateRec(0)
 
-    for ((frame, depth) <- runnerState.frames.zipWithIndex) {
-      os.write.over(
-        recOut(output, depth) / millRunnerState,
-        upickle.default.write(frame.loggedData, indent = 4),
-        createFolders = true
-      )
+    Using.resource(logger.waitForLock(outputLockOpt.getOrElse(Lock.dummy()))) { _ =>
+      for ((frame, depth) <- runnerState.frames.zipWithIndex) {
+        os.write.over(
+          recOut(output, depth) / millRunnerState,
+          upickle.default.write(frame.loggedData, indent = 4),
+          createFolders = true
+        )
+      }
     }
 
     Watching.Result(
@@ -344,6 +350,7 @@ class MillBuildBootstrap(
       home,
       projectRoot,
       recOut(output, depth),
+      outputLockOpt,
       recOut(output, depth),
       rootModule,
       PrefixLogger(logger, "", tickerContext = bootLogPrefix),

diff --git a/runner/src/mill/runner/MillCliConfig.scala b/runner/src/mill/runner/MillCliConfig.scala
@@ -123,7 +123,12 @@ case class MillCliConfig(
     )
     metaLevel: Option[Int] = None,
     @arg(doc = "Allows command args to be passed positionally without `--arg` by default")
-    allowPositional: Flag = Flag()
+    allowPositional: Flag = Flag(),
+    @arg(
+      hidden = true,
+      doc = """Evaluate tasks / commands without acquiring an exclusive lock on the Mill output directory"""
+    )
+    noBuildLock: Flag = Flag()
 )
 
 import mainargs.ParserForClass

diff --git a/runner/src/mill/runner/MillMain.scala b/runner/src/mill/runner/MillMain.scala
@@ -9,10 +9,12 @@ import mill.api.{MillException, SystemStreams, WorkspaceRoot, internal}
 import mill.bsp.{BspContext, BspServerResult}
 import mill.main.BuildInfo
 import mill.main.client.OutFiles
+import mill.main.client.lock.Lock
 import mill.util.PrintLogger
 
 import java.lang.reflect.InvocationTargetException
 import scala.util.control.NonFatal
+import scala.util.Using
 
 @internal
 object MillMain {
@@ -215,6 +217,11 @@ object MillMain {
                     .map(_ => Seq(bspCmd))
                     .getOrElse(config.leftoverArgs.value.toList)
 
+                val out = os.Path(OutFiles.out, WorkspaceRoot.workspaceRoot)
+                val outLockOpt =
+                  if (config.noBuildLock.value) None
+                  else Some(Lock.file((out / OutFiles.millLock).toString))
+                val useFineGrainedLock = bspContext.nonEmpty
                 var repeatForBsp = true
                 var loopRes: (Boolean, RunnerState) = (false, RunnerState.empty)
                 while (repeatForBsp) {
@@ -229,23 +236,26 @@ object MillMain {
                     evaluate = (prevState: Option[RunnerState]) => {
                       adjustJvmProperties(userSpecifiedProperties, initialSystemProperties)
 
-                      new MillBuildBootstrap(
-                        projectRoot = WorkspaceRoot.workspaceRoot,
-                        output = os.Path(OutFiles.out, WorkspaceRoot.workspaceRoot),
-                        home = config.home,
-                        keepGoing = config.keepGoing.value,
-                        imports = config.imports,
-                        env = env,
-                        threadCount = threadCount,
-                        targetsAndParams = targetsAndParams,
-                        prevRunnerState = prevState.getOrElse(stateCache),
-                        logger = logger,
-                        disableCallgraph = config.disableCallgraph.value,
-                        needBuildSc = needBuildSc(config),
-                        requestedMetaLevel = config.metaLevel,
-                        config.allowPositional.value,
-                        systemExit = systemExit
-                      ).evaluate()
+                      Using.resource(logger.waitForLock(outLockOpt.filter(_ => !useFineGrainedLock).getOrElse(Lock.dummy()))) { _ =>
+                        new MillBuildBootstrap(
+                          projectRoot = WorkspaceRoot.workspaceRoot,
+                          output = out,
+                          outputLockOpt = if (useFineGrainedLock) outLockOpt else None,
+                          home = config.home,
+                          keepGoing = config.keepGoing.value,
+                          imports = config.imports,
+                          env = env,
+                          threadCount = threadCount,
+                          targetsAndParams = targetsAndParams,
+                          prevRunnerState = prevState.getOrElse(stateCache),
+                          logger = logger,
+                          disableCallgraph = config.disableCallgraph.value,
+                          needBuildSc = needBuildSc(config),
+                          requestedMetaLevel = config.metaLevel,
+                          config.allowPositional.value,
+                          systemExit = systemExit
+                        ).evaluate()
+                      }
                     }
                   )
 

diff --git a/testkit/src/mill/testkit/UnitTester.scala b/testkit/src/mill/testkit/UnitTester.scala
@@ -90,6 +90,7 @@ class UnitTester(
     mill.api.Ctx.defaultHome,
     module.millSourcePath,
     outPath,
+    None,
     outPath,
     module,
     logger,