From 1f05eea7f4069e5980a665a7bfa6b4ac2bfddc31 Mon Sep 17 00:00:00 2001 From: friendseeker <66892505+Friendseeker@users.noreply.github.com> Date: Sat, 5 Oct 2024 13:53:02 -0700 Subject: [PATCH] Content Hashing --- .../xsbti/api/AnalyzedClass.java | 57 ++++++++++++++++--- .../src/main/contraband/other.json | 18 ++++++ .../scala/sbt/internal/inc/Analysis.scala | 9 +++ .../scala/sbt/internal/inc/Incremental.scala | 32 +++++++++-- .../sbt/internal/inc/IncrementalCommon.scala | 16 +++--- .../consistent/ConsistentAnalysisFormat.scala | 8 ++- 6 files changed, 117 insertions(+), 23 deletions(-) diff --git a/internal/compiler-interface/src/main/contraband-java/xsbti/api/AnalyzedClass.java b/internal/compiler-interface/src/main/contraband-java/xsbti/api/AnalyzedClass.java index a0f614bd36..b56fff81f7 100644 --- a/internal/compiler-interface/src/main/contraband-java/xsbti/api/AnalyzedClass.java +++ b/internal/compiler-interface/src/main/contraband-java/xsbti/api/AnalyzedClass.java @@ -24,6 +24,12 @@ public static AnalyzedClass create(long _compilationTimestamp, String _name, xsb public static AnalyzedClass of(long _compilationTimestamp, String _name, xsbti.api.Lazy _api, int _apiHash, NameHash[] _nameHashes, boolean _hasMacro, int _extraHash, String _provenance) { return new AnalyzedClass(_compilationTimestamp, _name, _api, _apiHash, _nameHashes, _hasMacro, _extraHash, _provenance); } + public static AnalyzedClass create(long _compilationTimestamp, String _name, xsbti.api.Lazy _api, int _apiHash, NameHash[] _nameHashes, boolean _hasMacro, int _extraHash, String _provenance, int _bytecodeHash, int _extraBytecodeHash) { + return new AnalyzedClass(_compilationTimestamp, _name, _api, _apiHash, _nameHashes, _hasMacro, _extraHash, _provenance, _bytecodeHash, _extraBytecodeHash); + } + public static AnalyzedClass of(long _compilationTimestamp, String _name, xsbti.api.Lazy _api, int _apiHash, NameHash[] _nameHashes, boolean _hasMacro, int _extraHash, String _provenance, int _bytecodeHash, int _extraBytecodeHash) { + return new AnalyzedClass(_compilationTimestamp, _name, _api, _apiHash, _nameHashes, _hasMacro, _extraHash, _provenance, _bytecodeHash, _extraBytecodeHash); + } private long compilationTimestamp; private String name; private xsbti.api.Lazy api; @@ -32,6 +38,8 @@ public static AnalyzedClass of(long _compilationTimestamp, String _name, xsbti.a private boolean hasMacro; private int extraHash; private String provenance; + private int bytecodeHash; + private int extraBytecodeHash; protected AnalyzedClass(long _compilationTimestamp, String _name, xsbti.api.Lazy _api, int _apiHash, NameHash[] _nameHashes, boolean _hasMacro) { super(); compilationTimestamp = _compilationTimestamp; @@ -42,6 +50,8 @@ protected AnalyzedClass(long _compilationTimestamp, String _name, xsbti.api.Lazy hasMacro = _hasMacro; extraHash = apiHash; provenance = ""; + bytecodeHash = 0; + extraBytecodeHash = 0; } protected AnalyzedClass(long _compilationTimestamp, String _name, xsbti.api.Lazy _api, int _apiHash, NameHash[] _nameHashes, boolean _hasMacro, int _extraHash) { super(); @@ -53,6 +63,8 @@ protected AnalyzedClass(long _compilationTimestamp, String _name, xsbti.api.Lazy hasMacro = _hasMacro; extraHash = _extraHash; provenance = ""; + bytecodeHash = 0; + extraBytecodeHash = 0; } protected AnalyzedClass(long _compilationTimestamp, String _name, xsbti.api.Lazy _api, int _apiHash, NameHash[] _nameHashes, boolean _hasMacro, int _extraHash, String _provenance) { super(); @@ -64,6 +76,21 @@ protected AnalyzedClass(long _compilationTimestamp, String _name, xsbti.api.Lazy hasMacro = _hasMacro; extraHash = _extraHash; provenance = _provenance; + bytecodeHash = 0; + extraBytecodeHash = 0; + } + protected AnalyzedClass(long _compilationTimestamp, String _name, xsbti.api.Lazy _api, int _apiHash, NameHash[] _nameHashes, boolean _hasMacro, int _extraHash, String _provenance, int _bytecodeHash, int _extraBytecodeHash) { + super(); + compilationTimestamp = _compilationTimestamp; + name = _name; + api = _api; + apiHash = _apiHash; + nameHashes = _nameHashes; + hasMacro = _hasMacro; + extraHash = _extraHash; + provenance = _provenance; + bytecodeHash = _bytecodeHash; + extraBytecodeHash = _extraBytecodeHash; } public long compilationTimestamp() { @@ -95,29 +122,43 @@ public int extraHash() { public String provenance() { return this.provenance; } + /** A hash of generated bytecode of source file hosting the class */ + public int bytecodeHash() { + return this.bytecodeHash; + } + /** A hash of generated bytecode of all upstream dependencies */ + public int extraBytecodeHash() { + return this.extraBytecodeHash; + } public AnalyzedClass withCompilationTimestamp(long compilationTimestamp) { - return new AnalyzedClass(compilationTimestamp, name, api, apiHash, nameHashes, hasMacro, extraHash, provenance); + return new AnalyzedClass(compilationTimestamp, name, api, apiHash, nameHashes, hasMacro, extraHash, provenance, bytecodeHash, extraBytecodeHash); } public AnalyzedClass withName(String name) { - return new AnalyzedClass(compilationTimestamp, name, api, apiHash, nameHashes, hasMacro, extraHash, provenance); + return new AnalyzedClass(compilationTimestamp, name, api, apiHash, nameHashes, hasMacro, extraHash, provenance, bytecodeHash, extraBytecodeHash); } public AnalyzedClass withApi(xsbti.api.Lazy api) { - return new AnalyzedClass(compilationTimestamp, name, api, apiHash, nameHashes, hasMacro, extraHash, provenance); + return new AnalyzedClass(compilationTimestamp, name, api, apiHash, nameHashes, hasMacro, extraHash, provenance, bytecodeHash, extraBytecodeHash); } public AnalyzedClass withApiHash(int apiHash) { - return new AnalyzedClass(compilationTimestamp, name, api, apiHash, nameHashes, hasMacro, extraHash, provenance); + return new AnalyzedClass(compilationTimestamp, name, api, apiHash, nameHashes, hasMacro, extraHash, provenance, bytecodeHash, extraBytecodeHash); } public AnalyzedClass withNameHashes(NameHash[] nameHashes) { - return new AnalyzedClass(compilationTimestamp, name, api, apiHash, nameHashes, hasMacro, extraHash, provenance); + return new AnalyzedClass(compilationTimestamp, name, api, apiHash, nameHashes, hasMacro, extraHash, provenance, bytecodeHash, extraBytecodeHash); } public AnalyzedClass withHasMacro(boolean hasMacro) { - return new AnalyzedClass(compilationTimestamp, name, api, apiHash, nameHashes, hasMacro, extraHash, provenance); + return new AnalyzedClass(compilationTimestamp, name, api, apiHash, nameHashes, hasMacro, extraHash, provenance, bytecodeHash, extraBytecodeHash); } public AnalyzedClass withExtraHash(int extraHash) { - return new AnalyzedClass(compilationTimestamp, name, api, apiHash, nameHashes, hasMacro, extraHash, provenance); + return new AnalyzedClass(compilationTimestamp, name, api, apiHash, nameHashes, hasMacro, extraHash, provenance, bytecodeHash, extraBytecodeHash); } public AnalyzedClass withProvenance(String provenance) { - return new AnalyzedClass(compilationTimestamp, name, api, apiHash, nameHashes, hasMacro, extraHash, provenance); + return new AnalyzedClass(compilationTimestamp, name, api, apiHash, nameHashes, hasMacro, extraHash, provenance, bytecodeHash, extraBytecodeHash); + } + public AnalyzedClass withBytecodeHash(int bytecodeHash) { + return new AnalyzedClass(compilationTimestamp, name, api, apiHash, nameHashes, hasMacro, extraHash, provenance, bytecodeHash, extraBytecodeHash); + } + public AnalyzedClass withExtraBytecodeHash(int extraBytecodeHash) { + return new AnalyzedClass(compilationTimestamp, name, api, apiHash, nameHashes, hasMacro, extraHash, provenance, bytecodeHash, extraBytecodeHash); } public boolean equals(Object obj) { return this == obj; // We have lazy members, so use object identity to avoid circularity. diff --git a/internal/compiler-interface/src/main/contraband/other.json b/internal/compiler-interface/src/main/contraband/other.json index d344ddb0ac..c43c05b67f 100644 --- a/internal/compiler-interface/src/main/contraband/other.json +++ b/internal/compiler-interface/src/main/contraband/other.json @@ -28,6 +28,24 @@ "Combined with a way to tell if the provenance has changed,", "it can be used to short-circuit the 'lookupAnalyzedClass' operation." ] + }, + { + "name": "bytecodeHash", + "type": "int", + "default": "0", + "since": "2.0.0", + "doc": [ + "A hash of generated bytecode of source file hosting the class" + ] + }, + { + "name": "extraBytecodeHash", + "type": "int", + "default": "0", + "since": "2.0.0", + "doc": [ + "A hash of generated bytecode of all upstream dependencies" + ] } ] }, diff --git a/internal/zinc-core/src/main/scala/sbt/internal/inc/Analysis.scala b/internal/zinc-core/src/main/scala/sbt/internal/inc/Analysis.scala index 5f28130713..aef15166c9 100644 --- a/internal/zinc-core/src/main/scala/sbt/internal/inc/Analysis.scala +++ b/internal/zinc-core/src/main/scala/sbt/internal/inc/Analysis.scala @@ -99,6 +99,15 @@ object Analysis { case class Sources(java: Set[String], scala: Set[String]) + def computeBytecodeHash( + localProducts: scala.collection.Set[LocalProduct], + nonLocalProduct: scala.collection.Set[NonLocalProduct] + ): Int = { + val hashes = + localProducts.map(_.classFileStamp.getHash) ++ nonLocalProduct.map(_.classFileStamp.getHash) + hashes.hashCode() + } + def sources(a: Analysis): Sources = { def sourceFileForClass(className: String): VirtualFileRef = a.relations.definesClass(className).headOption.getOrElse { diff --git a/internal/zinc-core/src/main/scala/sbt/internal/inc/Incremental.scala b/internal/zinc-core/src/main/scala/sbt/internal/inc/Incremental.scala index 018ffa3b95..b09cd81941 100644 --- a/internal/zinc-core/src/main/scala/sbt/internal/inc/Incremental.scala +++ b/internal/zinc-core/src/main/scala/sbt/internal/inc/Incremental.scala @@ -18,7 +18,7 @@ import java.nio.file.{ Files, Path, Paths } import java.{ util => ju } import ju.{ EnumSet, Optional, UUID } import ju.concurrent.atomic.AtomicBoolean -import sbt.internal.inc.Analysis.{ LocalProduct, NonLocalProduct } +import sbt.internal.inc.Analysis.{ LocalProduct, NonLocalProduct, computeBytecodeHash } import sbt.internal.inc.JavaInterfaceUtil.EnrichOption import sbt.util.{ InterfaceUtil, Level, Logger } import sbt.util.InterfaceUtil.{ jl2l, jo2o, l2jl, t2 } @@ -946,7 +946,7 @@ private final class AnalysisCallback( private def getAnalysis: Analysis = { val analysis0 = addProductsAndDeps(Analysis.empty) - addUsedNames(addCompilation(analysis0)) + addUsedNames(addCompilation(addExtraBytecodeHash(analysis0))) } def getPostJavaAnalysis: Analysis = { @@ -965,6 +965,24 @@ private final class AnalysisCallback( ) } + private def addExtraBytecodeHash(base: Analysis): Analysis = { + import base.{ apis, relations } + val findUpstream = relations.memberRef.internal.forward _ + val internalAPIs = apis.internal.map { case (className, analyzedClass) => + if (!analyzedClass.hasMacro) { + (className, analyzedClass) + } else { + val upstreamClasses = + IncrementalCommon.transitiveDeps(Set(className), log, logging = false)(findUpstream) + val upstreamAnalyzedClasses = upstreamClasses.map(apis.internalAPI) + val upstreamHashes = upstreamAnalyzedClasses.map(_.bytecodeHash()) + (className, analyzedClass.withExtraBytecodeHash(upstreamHashes.hashCode())) + } + } + val APIs = new MAPIs(internalAPIs, apis.external) + base.copy(apis = APIs) + } + private def companionsWithHash(className: String): (Companions, HashAPI.Hash, HashAPI.Hash) = { val emptyHash = -1 val emptyClass = @@ -992,7 +1010,7 @@ private final class AnalysisCallback( } } - private def analyzeClass(name: String): AnalyzedClass = { + private def analyzeClass(name: String, bytecodeHash: Int): AnalyzedClass = { val hasMacro: Boolean = macroClasses.contains(name) val (companions, apiHash, extraHash) = companionsWithHash(name) val nameHashes = nameHashesForCompanions(name) @@ -1005,7 +1023,9 @@ private final class AnalysisCallback( nameHashes, hasMacro, extraHash, - provenance + provenance, + bytecodeHash, + 0, ) } @@ -1018,7 +1038,6 @@ private final class AnalysisCallback( .getOrElse(src, ConcurrentHashMap.newKeySet[(String, String)]()) .asScala .map(_._1) - val analyzedApis = classesInSrc.map(analyzeClass) val info = SourceInfos.makeInfo( getOrNil(reporteds.iterator.map { case (k, v) => k -> v.asScala.toSeq }.toMap, src), getOrNil(unreporteds.iterator.map { case (k, v) => k -> v.asScala.toSeq }.toMap, src), @@ -1053,6 +1072,9 @@ private final class AnalysisCallback( ) val libDeps = libraries.map(d => (d, binaryClassName(d), stampReader.library(d))) + val bytecodeHash = computeBytecodeHash(localProds, nonLocalProds) + val analyzedApis = classesInSrc.map(analyzeClass(_, bytecodeHash)) + a.addSource( src, analyzedApis, diff --git a/internal/zinc-core/src/main/scala/sbt/internal/inc/IncrementalCommon.scala b/internal/zinc-core/src/main/scala/sbt/internal/inc/IncrementalCommon.scala index 097aec51d3..b21bfff449 100644 --- a/internal/zinc-core/src/main/scala/sbt/internal/inc/IncrementalCommon.scala +++ b/internal/zinc-core/src/main/scala/sbt/internal/inc/IncrementalCommon.scala @@ -41,8 +41,6 @@ private[inc] abstract class IncrementalCommon( options: IncOptions, profiler: RunProfiler ) extends InvalidationProfilerUtils { - private final val TIMESTAMP_2020 = 1577854800000L - // Work around bugs in classpath handling such as the "currently" problematic -javabootclasspath private[this] def enableShallowLookup: Boolean = java.lang.Boolean.getBoolean("xsbt.skip.cp.lookup") @@ -308,18 +306,18 @@ private[inc] abstract class IncrementalCommon( oldAPI: String => AnalyzedClass, newAPI: String => AnalyzedClass ): APIChanges = { - // ignore timestamp pre-2020 since that likely means that we have a hardcoded 2010 timestamp - def timeStampIsSame(ts1: Long, ts2: Long): Boolean = { - (ts1 < TIMESTAMP_2020) || (ts2 < TIMESTAMP_2020) || (ts1 == ts2) + def hashesMatch(a: AnalyzedClass, b: AnalyzedClass, hasMacro: Boolean): Boolean = { + (a.bytecodeHash() == b.bytecodeHash()) && + (a.apiHash == b.apiHash) && + (a.extraHash == b.extraHash) && + (!hasMacro || a.extraBytecodeHash() == b.extraBytecodeHash()) } // log.debug(s"[zinc] detectAPIChanges(recompiledClasses = $recompiledClasses)") def classDiff(className: String, a: AnalyzedClass, b: AnalyzedClass): Option[APIChange] = { // log.debug(s"[zinc] classDiff($className, ${a.name}, ${b.name})") - if ( - timeStampIsSame(a.compilationTimestamp(), b.compilationTimestamp()) && (a.apiHash == b.apiHash) - ) None + val hasMacro = a.hasMacro || b.hasMacro + if (hashesMatch(a, b, hasMacro)) None else { - val hasMacro = a.hasMacro || b.hasMacro if (hasMacro && IncOptions.getRecompileOnMacroDef(options)) { Some(APIChangeDueToMacroDefinition(className)) } else if ( diff --git a/internal/zinc-persist/src/main/scala/sbt/internal/inc/consistent/ConsistentAnalysisFormat.scala b/internal/zinc-persist/src/main/scala/sbt/internal/inc/consistent/ConsistentAnalysisFormat.scala index 73866694aa..03d552a5e7 100644 --- a/internal/zinc-persist/src/main/scala/sbt/internal/inc/consistent/ConsistentAnalysisFormat.scala +++ b/internal/zinc-persist/src/main/scala/sbt/internal/inc/consistent/ConsistentAnalysisFormat.scala @@ -145,6 +145,9 @@ class ConsistentAnalysisFormat(val mappers: ReadWriteMappers, sort: Boolean) { out.int(ac.apiHash()) out.bool(ac.hasMacro) out.string(ac.provenance()) + out.int(ac.extraHash()) + out.int(ac.bytecodeHash()) + out.int(ac.extraBytecodeHash()) val nh0 = ac.nameHashes() val nh = if (nh0.length > 1 && sort) { val nh = nh0.clone() @@ -169,6 +172,9 @@ class ConsistentAnalysisFormat(val mappers: ReadWriteMappers, sort: Boolean) { val ah = in.int() val hm = in.bool() val p = in.string() + val eh = in.int() + val bh = in.int() + val ebh = in.int() val nhNames = in.readStringArray() val nhScopes = in.readArray[UseScope]() { UseScope.values()(in.byte().toInt) } val nhHashes = in.readArray[Int]() { in.int() } @@ -181,7 +187,7 @@ class ConsistentAnalysisFormat(val mappers: ReadWriteMappers, sort: Boolean) { val comp = if (storeApis) Companions.of(readClassLike(in), readClassLike(in)) else APIs.emptyCompanions - AnalyzedClass.of(ts, name, SafeLazyProxy.strict(comp), ah, nameHashes, hm, ah, p) + AnalyzedClass.of(ts, name, SafeLazyProxy.strict(comp), ah, nameHashes, hm, eh, p, bh, ebh) } }