Java TASTy: use new threadsafe writer implementation (#19690)

Also fix bug where Jar entries for -Yjava-tasty-output have backslash on
Windows.

Copies implementation from
`compiler/src/dotty/tools/backend/jvm/ClassfileWriters.scala`, but this
time I don't close the jar archive except from within Pickler (when its
more explicit that we wont write any longer to the early output jar), I
also no longer perform substitution of `.` by `/` in Pickler, instead
leave it to TastyWriter to decide how to process the classname.

fixes #19681

compiler/src/dotty/tools/dotc/transform/Pickler.scala

@@ -9,7 +9,7 @@ import tasty.*
 import config.Printers.{noPrinter, pickling}
 import config.Feature
 import java.io.PrintStream
-import io.ClassfileWriterOps
+import io.FileWriters.TastyWriter
 import StdNames.{str, nme}
 import Periods.*
 import Phases.*
@@ -19,8 +19,9 @@ import reporting.{ThrowingReporter, Profile, Message}
 import collection.mutable
 import util.concurrent.{Executor, Future}
 import compiletime.uninitialized
-import dotty.tools.io.JarArchive
+import dotty.tools.io.{JarArchive, AbstractFile}
 import dotty.tools.dotc.printing.OutlinePrinter
+import scala.annotation.constructorOnly
 
 object Pickler {
   val name: String = "pickler"
@@ -32,8 +33,17 @@ object Pickler {
    */
   inline val ParallelPickling = true
 
-  class EarlyFileWriter(writer: ClassfileWriterOps):
-    export writer.{writeTasty, close}
+  class EarlyFileWriter private (writer: TastyWriter, origin: AbstractFile):
+    def this(dest: AbstractFile)(using @constructorOnly ctx: Context) = this(TastyWriter(dest), dest)
+
+    export writer.writeTasty
+
+    def close(): Unit =
+      writer.close()
+      origin match {
+        case jar: JarArchive => jar.close() // also close the file system
+        case _ =>
+      }
 }
 
 /** This phase pickles trees */
@@ -184,7 +194,7 @@ class Pickler extends Phase {
   override def runOn(units: List[CompilationUnit])(using Context): List[CompilationUnit] = {
     val sigWriter: Option[Pickler.EarlyFileWriter] = ctx.settings.YjavaTastyOutput.value match
       case jar: JarArchive if jar.exists =>
-        Some(Pickler.EarlyFileWriter(ClassfileWriterOps(jar)))
+        Some(Pickler.EarlyFileWriter(jar))
       case _ =>
         None
     val units0 =
@@ -225,9 +235,11 @@ class Pickler extends Phase {
         (cls, pickled) <- unit.pickled
         if cls.isDefinedInCurrentRun
       do
-        val binaryName = cls.binaryClassName.replace('.', java.io.File.separatorChar).nn
-        val binaryClassName = if (cls.is(Module)) binaryName.stripSuffix(str.MODULE_SUFFIX).nn else binaryName
-        writer.writeTasty(binaryClassName, pickled())
+        val binaryClassName = cls.binaryClassName
+        val internalName =
+          if (cls.is(Module)) binaryClassName.stripSuffix(str.MODULE_SUFFIX).nn
+          else binaryClassName
+        val _ = writer.writeTasty(internalName, pickled())
         count += 1
     finally
       writer.close()

compiler/src/dotty/tools/io/FileWriters.scala

@@ -0,0 +1,254 @@
+package dotty.tools.io
+
+import dotty.tools.dotc.core.Contexts.*
+import dotty.tools.dotc.core.Decorators.em
+import dotty.tools.dotc.report
+import dotty.tools.io.AbstractFile
+import dotty.tools.io.JarArchive
+import dotty.tools.io.PlainFile
+
+import java.io.BufferedOutputStream
+import java.io.DataOutputStream
+import java.io.FileOutputStream
+import java.io.IOException
+import java.nio.ByteBuffer
+import java.nio.channels.ClosedByInterruptException
+import java.nio.channels.FileChannel
+import java.nio.file.FileAlreadyExistsException
+import java.nio.file.Files
+import java.nio.file.Path
+import java.nio.file.StandardOpenOption
+import java.nio.file.attribute.FileAttribute
+import java.util
+import java.util.concurrent.ConcurrentHashMap
+import java.util.zip.CRC32
+import java.util.zip.Deflater
+import java.util.zip.ZipEntry
+import java.util.zip.ZipOutputStream
+import scala.language.unsafeNulls
+
+/** Copied from `dotty.tools.backend.jvm.ClassfileWriters` but no `PostProcessorFrontendAccess` needed */
+object FileWriters {
+  type InternalName = String
+  type NullableFile =  AbstractFile | Null
+
+  /**
+   * The interface to writing classfiles. GeneratedClassHandler calls these methods to generate the
+   * directory and files that are created, and eventually calls `close` when the writing is complete.
+   *
+   * The companion object is responsible for constructing a appropriate and optimal implementation for
+   * the supplied settings.
+   *
+   * Operations are threadsafe.
+   */
+  sealed trait TastyWriter {
+    /**
+     * Write a `.tasty` file.
+     *
+     * @param name the internal name of the class, e.g. "scala.Option"
+     */
+    def writeTasty(name: InternalName, bytes: Array[Byte])(using Context): NullableFile
+
+    /**
+     * Close the writer. Behavior is undefined after a call to `close`.
+     */
+    def close(): Unit
+
+    protected def classToRelativePath(className: InternalName): String =
+      className.replace('.', '/').nn + ".tasty"
+  }
+
+  object TastyWriter {
+
+    def apply(output: AbstractFile)(using Context): TastyWriter = {
+
+      // In Scala 2 depenening on cardinality of distinct output dirs MultiClassWriter could have been used
+      // In Dotty we always use single output directory
+      val basicTastyWriter = new SingleTastyWriter(
+        FileWriter(output, None)
+      )
+
+      basicTastyWriter
+    }
+
+    private final class SingleTastyWriter(underlying: FileWriter) extends TastyWriter {
+
+      override def writeTasty(className: InternalName, bytes: Array[Byte])(using Context): NullableFile = {
+        underlying.writeFile(classToRelativePath(className), bytes)
+      }
+
+      override def close(): Unit = underlying.close()
+    }
+
+  }
+
+  sealed trait FileWriter {
+    def writeFile(relativePath: String, bytes: Array[Byte])(using Context): NullableFile
+    def close(): Unit
+  }
+
+  object FileWriter {
+    def apply(file: AbstractFile, jarManifestMainClass: Option[String])(using Context): FileWriter =
+      if (file.isInstanceOf[JarArchive]) {
+        val jarCompressionLevel = ctx.settings.YjarCompressionLevel.value
+        // Writing to non-empty JAR might be an undefined behaviour, e.g. in case if other files where
+        // created using `AbstractFile.bufferedOutputStream`instead of JarWritter
+        val jarFile = file.underlyingSource.getOrElse{
+          throw new IllegalStateException("No underlying source for jar")
+        }
+        assert(file.isEmpty, s"Unsafe writing to non-empty JAR: $jarFile")
+        new JarEntryWriter(jarFile, jarManifestMainClass, jarCompressionLevel)
+      }
+      else if (file.isVirtual) new VirtualFileWriter(file)
+      else if (file.isDirectory) new DirEntryWriter(file.file.toPath.nn)
+      else throw new IllegalStateException(s"don't know how to handle an output of $file [${file.getClass}]")
+  }
+
+  private final class JarEntryWriter(file: AbstractFile, mainClass: Option[String], compressionLevel: Int) extends FileWriter {
+    //keep these imports local - avoid confusion with scala naming
+    import java.util.jar.Attributes.Name.{MANIFEST_VERSION, MAIN_CLASS}
+    import java.util.jar.{JarOutputStream, Manifest}
+
+    val storeOnly = compressionLevel == Deflater.NO_COMPRESSION
+
+    val jarWriter: JarOutputStream = {
+      import scala.util.Properties.*
+      val manifest = new Manifest
+      val attrs = manifest.getMainAttributes.nn
+      attrs.put(MANIFEST_VERSION, "1.0")
+      attrs.put(ScalaCompilerVersion, versionNumberString)
+      mainClass.foreach(c => attrs.put(MAIN_CLASS, c))
+
+      val jar = new JarOutputStream(new BufferedOutputStream(new FileOutputStream(file.file), 64000), manifest)
+      jar.setLevel(compressionLevel)
+      if (storeOnly) jar.setMethod(ZipOutputStream.STORED)
+      jar
+    }
+
+    lazy val crc = new CRC32
+
+    override def writeFile(relativePath: String, bytes: Array[Byte])(using Context): NullableFile = this.synchronized {
+      val entry = new ZipEntry(relativePath)
+      if (storeOnly) {
+        // When using compression method `STORED`, the ZIP spec requires the CRC and compressed/
+        // uncompressed sizes to be written before the data. The JarOutputStream could compute the
+        // values while writing the data, but not patch them into the stream after the fact. So we
+        // need to pre-compute them here. The compressed size is taken from size.
+        // https://stackoverflow.com/questions/1206970/how-to-create-uncompressed-zip-archive-in-java/5868403
+        // With compression method `DEFLATED` JarOutputStream computes and sets the values.
+        entry.setSize(bytes.length)
+        crc.reset()
+        crc.update(bytes)
+        entry.setCrc(crc.getValue)
+      }
+      jarWriter.putNextEntry(entry)
+      try jarWriter.write(bytes, 0, bytes.length)
+      finally jarWriter.flush()
+      null
+    }
+
+    override def close(): Unit = this.synchronized(jarWriter.close())
+  }
+
+  private final class DirEntryWriter(base: Path) extends FileWriter {
+    val builtPaths = new ConcurrentHashMap[Path, java.lang.Boolean]()
+    val noAttributes = Array.empty[FileAttribute[?]]
+    private val isWindows = scala.util.Properties.isWin
+
+    private def checkName(component: Path)(using Context): Unit = if (isWindows) {
+      val specials = raw"(?i)CON|PRN|AUX|NUL|COM[1-9]|LPT[1-9]".r
+      val name = component.toString
+      def warnSpecial(): Unit = report.warning(em"path component is special Windows device: ${name}")
+      specials.findPrefixOf(name).foreach(prefix => if (prefix.length == name.length || name(prefix.length) == '.') warnSpecial())
+    }
+
+    def ensureDirForPath(baseDir: Path, filePath: Path)(using Context): Unit = {
+      import java.lang.Boolean.TRUE
+      val parent = filePath.getParent
+      if (!builtPaths.containsKey(parent)) {
+        parent.iterator.forEachRemaining(checkName)
+        try Files.createDirectories(parent, noAttributes*)
+        catch {
+          case e: FileAlreadyExistsException =>
+            // `createDirectories` reports this exception if `parent` is an existing symlink to a directory
+            // but that's fine for us (and common enough, `scalac -d /tmp` on mac targets symlink).
+            if (!Files.isDirectory(parent))
+              throw new FileConflictException(s"Can't create directory $parent; there is an existing (non-directory) file in its path", e)
+        }
+        builtPaths.put(baseDir, TRUE)
+        var current = parent
+        while ((current ne null) && (null ne builtPaths.put(current, TRUE))) {
+          current = current.getParent
+        }
+      }
+      checkName(filePath.getFileName())
+    }
+
+    // the common case is that we are are creating a new file, and on MS Windows the create and truncate is expensive
+    // because there is not an options in the windows API that corresponds to this so the truncate is applied as a separate call
+    // even if the file is new.
+    // as this is rare, its best to always try to create a new file, and it that fails, then open with truncate if that fails
+
+    private val fastOpenOptions = util.EnumSet.of(StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)
+    private val fallbackOpenOptions = util.EnumSet.of(StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING)
+
+    override def writeFile(relativePath: String, bytes: Array[Byte])(using Context): NullableFile = {
+      val path = base.resolve(relativePath)
+      try {
+        ensureDirForPath(base, path)
+        val os = if (isWindows) {
+          try FileChannel.open(path, fastOpenOptions)
+          catch {
+            case _: FileAlreadyExistsException => FileChannel.open(path, fallbackOpenOptions)
+          }
+        } else FileChannel.open(path, fallbackOpenOptions)
+
+        try os.write(ByteBuffer.wrap(bytes), 0L)
+        catch {
+          case ex: ClosedByInterruptException =>
+            try Files.deleteIfExists(path) // don't leave a empty of half-written classfile around after an interrupt
+            catch { case _: Throwable => () }
+            throw ex
+        }
+        os.close()
+      } catch {
+        case e: FileConflictException =>
+          report.error(em"error writing ${path.toString}: ${e.getMessage}")
+        case e: java.nio.file.FileSystemException =>
+          if (ctx.settings.Ydebug.value) e.printStackTrace()
+          report.error(em"error writing ${path.toString}: ${e.getClass.getName} ${e.getMessage}")
+      }
+      AbstractFile.getFile(path)
+    }
+
+    override def close(): Unit = ()
+  }
+
+  private final class VirtualFileWriter(base: AbstractFile) extends FileWriter {
+    private def getFile(base: AbstractFile, path: String): AbstractFile = {
+      def ensureDirectory(dir: AbstractFile): AbstractFile =
+        if (dir.isDirectory) dir
+        else throw new FileConflictException(s"${base.path}/${path}: ${dir.path} is not a directory")
+      val components = path.split('/')
+      var dir = base
+      for (i <- 0 until components.length - 1) dir = ensureDirectory(dir) subdirectoryNamed components(i).toString
+      ensureDirectory(dir) fileNamed components.last.toString
+    }
+
+    private def writeBytes(outFile: AbstractFile, bytes: Array[Byte]): Unit = {
+      val out = new DataOutputStream(outFile.bufferedOutput)
+      try out.write(bytes, 0, bytes.length)
+      finally out.close()
+    }
+
+    override def writeFile(relativePath: String, bytes: Array[Byte])(using Context):NullableFile = {
+      val outFile = getFile(base, relativePath)
+      writeBytes(outFile, bytes)
+      outFile
+    }
+    override def close(): Unit = ()
+  }
+
+  /** Can't output a file due to the state of the file system. */
+  class FileConflictException(msg: String, cause: Throwable = null) extends IOException(msg, cause)
+}