From be9203572d31a3b8d8787f9f8a52c1c432670d51 Mon Sep 17 00:00:00 2001 From: WebFreak001 Date: Wed, 27 Sep 2023 00:30:18 +0200 Subject: [PATCH 1/2] remove GeneratorSettings.cache instead the cache is a property of PackageManager This way dub lib users don't have to try to figure out where the real cache path is and can just continue using the generator and describe APIs like in previous versions. Customizing cache path is no longer easily possible and now must be done through creating a custom PackageManager and use that on `new Project()` to pass into the generators. If needed, APIs to modify m_dirs on startup could be added to Dub subclasses in the future. --- source/dub/commandline.d | 1 - source/dub/dub.d | 22 ++----- source/dub/generators/build.d | 6 +- source/dub/generators/generator.d | 62 +++--------------- source/dub/generators/targetdescription.d | 8 ++- source/dub/packagemanager.d | 77 +++++++++++++++++++++++ source/dub/project.d | 2 +- 7 files changed, 101 insertions(+), 77 deletions(-) diff --git a/source/dub/commandline.d b/source/dub/commandline.d index 5bb5db064..2661f31b8 100644 --- a/source/dub/commandline.d +++ b/source/dub/commandline.d @@ -1878,7 +1878,6 @@ class DescribeCommand : PackageBuildCommand { GeneratorSettings settings = this.baseSettings; if (!settings.config.length) settings.config = m_defaultConfig; - settings.cache = dub.cachePathDontUse(); // See function's description // Ignore other options settings.buildSettings.options = this.baseSettings.buildSettings.options & BuildOption.lowmem; diff --git a/source/dub/dub.d b/source/dub/dub.d index b64a38679..efaee5671 100644 --- a/source/dub/dub.d +++ b/source/dub/dub.d @@ -165,7 +165,7 @@ class Dub { m_packageSuppliers = this.computePkgSuppliers(additional_package_suppliers, skip_registry, environment.get("DUB_REGISTRY", null)); - m_packageManager = new PackageManager(m_rootPath, m_dirs.userPackages, m_dirs.systemSettings, false); + m_packageManager = new PackageManager(m_rootPath, m_dirs.userPackages, m_dirs.systemSettings, m_dirs.cache, false); auto ccps = m_config.customCachePaths; if (ccps.length) @@ -199,7 +199,7 @@ class Dub { // to prevent `init` from reading the project's settings. init(); this.m_rootPath = root; - m_packageManager = new PackageManager(pkg_root); + m_packageManager = new PackageManager(pkg_root, m_dirs.cache); } deprecated("Use the overload that takes `(NativePath pkg_root, NativePath root)`") @@ -698,7 +698,6 @@ class Dub { */ void generateProject(string ide, GeneratorSettings settings) { - settings.cache = this.m_dirs.cache; if (settings.overrideToolWorkingDirectory is NativePath.init) settings.overrideToolWorkingDirectory = m_rootPath; // With a requested `unittest` config, switch to the special test runner @@ -719,7 +718,6 @@ class Dub { */ void testProject(GeneratorSettings settings, string config, NativePath custom_main_file) { - settings.cache = this.m_dirs.cache; if (settings.overrideToolWorkingDirectory is NativePath.init) settings.overrideToolWorkingDirectory = m_rootPath; if (!custom_main_file.empty && !custom_main_file.absolute) custom_main_file = m_rootPath ~ custom_main_file; @@ -1343,22 +1341,14 @@ class Dub { } /** - * Compute and returns the path were artifacts are stored + * Compute and returns the path were artifacts are stored for the given + * package. * - * Expose `dub.generator.generator : packageCache` with this instance's - * configured cache. + * See `dub.packagemanager : PackageManager.packageCache` */ protected NativePath packageCache (Package pkg) const { - return .packageCache(this.m_dirs.cache, pkg); - } - - /// Exposed because `commandLine` replicates `generateProject` for `dub describe` - /// instead of treating it like a regular generator... Remove this once the - /// flaw is fixed, and don't add more calls to this function! - package(dub) NativePath cachePathDontUse () const @safe pure nothrow @nogc - { - return this.m_dirs.cache; + return m_packageManager.packageCache(pkg); } /// Make a `GeneratorSettings` suitable to generate tools (DDOC, DScanner, etc...) diff --git a/source/dub/generators/build.d b/source/dub/generators/build.d index 7cfc7cf7e..3e0342e42 100644 --- a/source/dub/generators/build.d +++ b/source/dub/generators/build.d @@ -263,7 +263,7 @@ class BuildGenerator : ProjectGenerator { m_tempTargetExecutablePath = target_path = getTempDir() ~ format(".dub/build/%s-%s/%s/", packageName, pack.version_, build_id); } else - target_path = targetCacheDir(settings.cache, pack, build_id); + target_path = m_packageMan.targetCacheDir(pack, build_id); if (!settings.force && isUpToDate(target_path, buildsettings, settings, pack, packages, additional_dep_files)) { logInfo("Up-to-date", Color.green, "%s %s: target for configuration [%s] is up to date.", @@ -316,7 +316,7 @@ class BuildGenerator : ProjectGenerator { enum jsonFileName = "db.json"; enum lockFileName = "db.lock"; - const pkgCacheDir = packageCache(settings.cache, pack); + const pkgCacheDir = m_packageMan.packageCache(pack); auto lock = lockFile((pkgCacheDir ~ lockFileName).toNativeString(), 3.seconds); const dbPath = pkgCacheDir ~ jsonFileName; @@ -777,7 +777,7 @@ unittest { // issue #1235 - pass no library files to compiler command line when auto desc = parseJsonString(`{"name": "test", "targetType": "library", "sourceFiles": ["foo.d", "`~libfile~`"]}`); auto pack = new Package(desc, NativePath("/tmp/fooproject")); - auto pman = new PackageManager(pack.path, NativePath("/tmp/foo/"), NativePath("/tmp/foo/"), false); + auto pman = new PackageManager(pack.path, NativePath("/tmp/foo/"), NativePath("/tmp/foo/"), NativePath("/tmp/cache/"), false); auto prj = new Project(pman, pack); final static class TestCompiler : GDCCompiler { diff --git a/source/dub/generators/generator.d b/source/dub/generators/generator.d index b602142c1..bcdb08be0 100644 --- a/source/dub/generators/generator.d +++ b/source/dub/generators/generator.d @@ -121,11 +121,13 @@ class ProjectGenerator protected { Project m_project; NativePath m_tempTargetExecutablePath; + PackageManager m_packageMan; } this(Project project) { m_project = project; + m_packageMan = project.packageManager; } /** Performs the full generator process. @@ -331,7 +333,7 @@ class ProjectGenerator if (genSettings.filterVersions) foreach (ref ti; targets.byValue) - inferVersionFilters(ti); + inferVersionFilters(m_packageMan, ti); // mark packages as visited (only used during upwards propagation) void[0][Package] visited; @@ -629,7 +631,7 @@ class ProjectGenerator } // infer applicable version identifiers - private static void inferVersionFilters(ref TargetInfo ti) + private static void inferVersionFilters(PackageManager packageManager, ref TargetInfo ti) { import std.algorithm.searching : any; import std.file : timeLastModified; @@ -656,7 +658,7 @@ class ProjectGenerator auto srcs = chain(bs.sourceFiles, bs.importFiles, bs.stringImportFiles) .filter!(f => dexts.canFind(f.extension)).filter!exists; // try to load cached filters first - const cacheFilePath = packageCache(NativePath(ti.buildSettings.targetPath), ti.pack) + const cacheFilePath = packageManager.packageCache(ti.pack) ~ "metadata_cache.json"; enum silent_fail = true; auto cache = jsonFromFile(cacheFilePath, silent_fail); @@ -683,7 +685,7 @@ class ProjectGenerator catch (JSONException e) { logWarn("Exception during loading invalid package cache %s.\n%s", - ti.pack.path ~ ".dub/metadata_cache.json", e); + cacheFilePath.toNativeString(), e); } // use ctRegex for performance reasons, only small compile time increase @@ -768,55 +770,6 @@ class ProjectGenerator } } -/** - * Compute and returns the path were artifacts are stored for a given package - * - * Artifacts are stored in: - * `$DUB_HOME/cache/$PKG_NAME/$PKG_VERSION[/+$SUB_PKG_NAME]/` - * Note that the leading `+` in the sub-package name is to avoid any ambiguity. - * - * Dub writes in the returned path a Json description file of the available - * artifacts in this cache location. This Json file is read by 3rd party - * software (e.g. Meson). Returned path should therefore not change across - * future Dub versions. - * - * Build artifacts are usually stored in a sub-folder named "build", - * as their names are based on user-supplied values. - * - * Params: - * cachePath = Base path at which the build cache is located, - * e.g. `$HOME/.dub/cache/` - * pkg = The package. Cannot be `null`. - */ -package(dub) NativePath packageCache(NativePath cachePath, in Package pkg) -{ - import std.algorithm.searching : findSplit; - - assert(pkg !is null); - assert(!cachePath.empty); - - // For subpackages - if (const names = pkg.name.findSplit(":")) - return cachePath ~ names[0] ~ pkg.version_.toString() - ~ ("+" ~ names[2]); - // For regular packages - return cachePath ~ pkg.name ~ pkg.version_.toString(); -} - -/** - * Compute and return the directory where a target should be cached. - * - * Params: - * cachePath = Base path at which the build cache is located, - * e.g. `$HOME/.dub/cache/` - * pkg = The package. Cannot be `null`. - * buildId = The build identifier of the target. - */ -package(dub) NativePath targetCacheDir(NativePath cachePath, in Package pkg, string buildId) -{ - return packageCache(cachePath, pkg) ~ "build" ~ buildId; -} - /** * Provides a unique (per build) identifier * @@ -853,7 +806,6 @@ package(dub) string computeBuildID(in BuildSettings buildsettings, string config } struct GeneratorSettings { - NativePath cache; BuildPlatform platform; Compiler compiler; string config; @@ -1234,7 +1186,7 @@ version(Posix) { auto desc = parseJsonString(`{"name": "test", "targetType": "library", "preGenerateCommands": ["touch $PACKAGE_DIR/source/bar.d"]}`); auto pack = new Package(desc, NativePath("dubtest/preGen".absolutePath)); - auto pman = new PackageManager(pack.path, NativePath("/tmp/foo/"), NativePath("/tmp/foo/"), false); + auto pman = new PackageManager(pack.path, NativePath("/tmp/foo/"), NativePath("/tmp/foo/"), NativePath("/tmp/cache/"), false); auto prj = new Project(pman, pack); final static class TestCompiler : GDCCompiler { diff --git a/source/dub/generators/targetdescription.d b/source/dub/generators/targetdescription.d index 077126e2d..b31d79470 100644 --- a/source/dub/generators/targetdescription.d +++ b/source/dub/generators/targetdescription.d @@ -12,15 +12,21 @@ import dub.compilers.compiler; import dub.description; import dub.generators.generator; import dub.internal.vibecompat.inet.path; +import dub.packagemanager; import dub.project; class TargetDescriptionGenerator : ProjectGenerator { TargetDescription[] targetDescriptions; size_t[string] targetDescriptionLookup; + private { + PackageManager m_packageMan; + } + this(Project project) { super(project); + m_packageMan = project.packageManager; } protected override void generateTargets(GeneratorSettings settings, in TargetInfo[string] targets) @@ -47,7 +53,7 @@ class TargetDescriptionGenerator : ProjectGenerator { d.buildSettings = ti.buildSettings.dup; const buildId = computeBuildID(d.buildSettings, ti.config, settings); const filename = settings.compiler.getTargetFileName(d.buildSettings, settings.platform); - d.cacheArtifactPath = (targetCacheDir(settings.cache, ti.pack, buildId) ~ filename).toNativeString(); + d.cacheArtifactPath = (m_packageMan.targetCacheDir(ti.pack, buildId) ~ filename).toNativeString(); d.dependencies = ti.dependencies.dup; d.linkDependencies = ti.linkDependencies.dup; diff --git a/source/dub/packagemanager.d b/source/dub/packagemanager.d index 0300ba16b..73b117044 100644 --- a/source/dub/packagemanager.d +++ b/source/dub/packagemanager.d @@ -95,6 +95,15 @@ class PackageManager { * of available packages. */ bool m_initialized; + /** + * Path where build artifacts are stored, e.g. `~/.dub/cache/` on POSIX. + * + * See dub.d SpecialDirs.cache for more details. + * + * To initialize with default DUB values, instantiate an instance of + * `DUB` and access its packageManager. + */ + NativePath m_cachePath; } /** @@ -107,12 +116,22 @@ class PackageManager { Params: path = Path of the single repository */ + deprecated("Obtain the package manager from `Dub` or instantiate with a cache path where binaries should go") this(NativePath path) { this.m_internal.searchPath = [ path ]; this.refresh(); } + /// ditto + this(NativePath path, NativePath cachePath) + { + this.m_internal.searchPath = [ path ]; + this.m_cachePath = cachePath; + this.refresh(); + } + + deprecated("Obtain the package manager from `Dub` or instantiate with a cache path where binaries should go") this(NativePath package_path, NativePath user_path, NativePath system_path, bool refresh_packages = true) { m_repositories = [ @@ -123,6 +142,19 @@ class PackageManager { if (refresh_packages) refresh(); } + /// ditto + this(NativePath package_path, NativePath user_path, NativePath system_path, NativePath cache_path, bool refresh_packages = true) + { + m_repositories = [ + Location(package_path ~ ".dub/packages/"), + Location(user_path ~ "packages/"), + Location(system_path ~ "packages/")]; + + m_cachePath = cache_path; + + if (refresh_packages) refresh(); + } + /** Gets/sets the list of paths to search for local packages. */ @property void searchPath(NativePath[] paths) @@ -1011,6 +1043,51 @@ symlink_exit: } } } + + /** + * Compute and returns the path were artifacts are stored for a given package + * + * Artifacts are stored in: + * `$DUB_HOME/cache/$PKG_NAME/$PKG_VERSION[/+$SUB_PKG_NAME]/` + * Note that the leading `+` in the sub-package name is to avoid any ambiguity. + * + * Dub writes in the returned path a Json description file of the available + * artifacts in this cache location. This Json file is read by 3rd party + * software (e.g. Meson). Returned path should therefore not change across + * future Dub versions. + * + * Build artifacts are usually stored in a sub-folder named "build", + * as their names are based on user-supplied values. + * + * Params: + * pkg = The package. Cannot be `null`. + */ + package(dub) NativePath packageCache(in Package pkg) const + { + import std.algorithm.searching : findSplit; + + assert(pkg !is null); + assert(!m_cachePath.empty, "cannot use packageCache when no package cache path has been specified in constructor"); + + // For subpackages + if (const names = pkg.name.findSplit(":")) + return m_cachePath ~ names[0] ~ pkg.version_.toString() + ~ ("+" ~ names[2]); + // For regular packages + return m_cachePath ~ pkg.name ~ pkg.version_.toString(); + } + + /** + * Compute and return the directory where a target should be cached. + * + * Params: + * pkg = The package. Cannot be `null`. + * buildId = The build identifier of the target. + */ + package(dub) NativePath targetCacheDir(in Package pkg, string buildId) const + { + return packageCache(pkg) ~ "build" ~ buildId; + } } deprecated(OverrideDepMsg) diff --git a/source/dub/project.d b/source/dub/project.d index 5abfd4571..3ea9511a6 100644 --- a/source/dub/project.d +++ b/source/dub/project.d @@ -321,7 +321,7 @@ class Project { mainfile = getTempFile("dub_test_root", ".d"); else { import dub.generators.build : computeBuildName; - mainfile = packageCache(settings.cache, this.rootPackage) ~ + mainfile = m_packageManager.packageCache(this.rootPackage) ~ format("code/%s/dub_test_root.d", computeBuildName(config, settings, import_modules)); } From 1485c4517f69c96cd3968358411e43bb2a6f17a0 Mon Sep 17 00:00:00 2001 From: WebFreak001 Date: Wed, 27 Sep 2023 00:44:52 +0200 Subject: [PATCH 2/2] readd deprecated `cache` field --- source/dub/generators/generator.d | 3 +++ 1 file changed, 3 insertions(+) diff --git a/source/dub/generators/generator.d b/source/dub/generators/generator.d index bcdb08be0..4c949b50c 100644 --- a/source/dub/generators/generator.d +++ b/source/dub/generators/generator.d @@ -842,6 +842,9 @@ struct GeneratorSettings { ? getWorkingDirectory() : overrideToolWorkingDirectory; } + + deprecated("This is no longer used, the cache is inside PackageManager now. If you use a project instantiated from a `Dub` instance, the cache defaults to the real DUB cache, e.g. ~/.dub/cache") + NativePath cache; }