Merge pull request #2610 from Geod24/migrate-package-location

PackageManager: Store packages hierarchically, by version

changelog/hierarchy.dd

@@ -0,0 +1,16 @@
+The way packages are stored internally has changed
+
+Previous versions of dub stored packages in the following format:
+`$CACHE_PATH/$PACKAGE_NAME-$PACKAGE_VERSION/$PACKAGE_NAME/`
+Starting from this version, the format will be:
+`$CACHE_PATH/$PACKAGE_NAME/$PACKAGE_VERSION/$PACKAGE_NAME`.
+
+Introducing a new level will help users quickly list what packages
+they actually have installed, and reduce visibility of packages that
+might update frequently. It will render various commands (e.g. `du`)
+more useful, pave the way for a package GC function, and make manual
+browsing easier.
+
+More importantly, it will allow future version of dub to infer the
+version from the path to the package, removing the need to read (or edit)
+the recipe file on every dub invocation.

source/dub/packagemanager.d

@@ -1298,29 +1298,8 @@ private struct Location {
 		if (!path.existsDirectory())
 			return;
 
-		logDebug("iterating dir %s", path.toNativeString());
-		try foreach (pdir; iterateDirectory(path)) {
-			logDebug("iterating dir %s entry %s", path.toNativeString(), pdir.name);
-			if (!pdir.isDirectory) continue;
-
-			// Old / flat directory structure, used in non-standard path
-			// Packages are stored in $ROOT/$SOMETHING/`
-			auto pack_path = path ~ (pdir.name ~ "/");
-			auto packageFile = Package.findPackageFile(pack_path);
-
-			// New (since 2015) managed structure:
-			// $ROOT/$NAME-$VERSION/$NAME
-			// This is the most common code path
-			if (mgr.isManagedPath(path) && packageFile.empty) {
-				foreach (subdir; iterateDirectory(path ~ (pdir.name ~ "/")))
-					if (subdir.isDirectory && pdir.name.startsWith(subdir.name)) {
-						pack_path ~= subdir.name ~ "/";
-						packageFile = Package.findPackageFile(pack_path);
-						break;
-					}
-			}
-
-			if (packageFile.empty) continue;
+		void loadInternal (NativePath pack_path, NativePath packageFile)
+		{
 			Package p;
 			try {
 				foreach (pp; existing_packages)
@@ -1338,6 +1317,38 @@ private struct Location {
 				logDiagnostic("Full error: %s", e.toString().sanitize());
 			}
 		}
+
+		logDebug("iterating dir %s", path.toNativeString());
+		try foreach (pdir; iterateDirectory(path)) {
+			logDebug("iterating dir %s entry %s", path.toNativeString(), pdir.name);
+			if (!pdir.isDirectory) continue;
+
+			// Old / flat directory structure, used in non-standard path
+			// Packages are stored in $ROOT/$SOMETHING/`
+			const pack_path = path ~ (pdir.name ~ "/");
+			auto packageFile = Package.findPackageFile(pack_path);
+			if (!packageFile.empty) {
+				// Deprecated unmanaged directory structure
+				logWarn("Package at path '%s' should be under '%s'",
+						pack_path.toNativeString().color(Mode.bold),
+						(pack_path ~ "$VERSION" ~ pdir.name).toNativeString().color(Mode.bold));
+				logWarn("The package will no longer be detected starting from v1.42.0");
+				loadInternal(pack_path, packageFile);
+			}
+
+			// Managed structure: $ROOT/$NAME/$VERSION/$NAME
+			// This is the most common code path
+			else if (mgr.isManagedPath(path)) {
+				// Iterate over versions of a package
+				foreach (versdir; iterateDirectory(pack_path)) {
+					if (!versdir.isDirectory) continue;
+					auto vers_path = pack_path ~ versdir.name ~ (pdir.name ~ "/");
+					if (!vers_path.existsDirectory()) continue;
+					packageFile = Package.findPackageFile(vers_path);
+					loadInternal(vers_path, packageFile);
+				}
+			}
+		}
 		catch (Exception e)
 			logDiagnostic("Failed to enumerate %s packages: %s", path.toNativeString(), e.toString());
 	}
@@ -1417,9 +1428,7 @@ private struct Location {
 	 */
 	private NativePath getPackagePath (string name, string vers)
 	{
-		// + has special meaning for Optlink
-		string clean_vers = vers.chompPrefix("~").replace("+", "_");
-		NativePath result = this.packagePath ~ (name ~ "-" ~ clean_vers);
+		NativePath result = this.packagePath ~ name ~ vers;
 		result.endsWithSlash = true;
 		return result;
 	}

test/dpath-variable.sh

@@ -6,7 +6,7 @@ rm -rf "$DPATH"
 cd "${CURR_DIR}/dpath-variable"
 "${DUB}" upgrade
 
-if [[ ! -f "$DPATH/dub/packages/gitcompatibledubpackage-1.0.1/gitcompatibledubpackage/dub.json" ]]; then
+if [[ ! -f "$DPATH/dub/packages/gitcompatibledubpackage/1.0.1/gitcompatibledubpackage/dub.json" ]]; then
 	die $LINENO 'Did not get dependencies installed into $DPATH.'
 fi
 
@@ -24,6 +24,6 @@ trap cleanup EXIT
 
 "${DUB}" upgrade
 
-if [[ ! -f "$DPATH_ALIAS/dub2/packages/gitcompatibledubpackage-1.0.1/gitcompatibledubpackage/dub.json" ]]; then
+if [[ ! -f "$DPATH_ALIAS/dub2/packages/gitcompatibledubpackage/1.0.1/gitcompatibledubpackage/dub.json" ]]; then
 	die $LINENO 'Did not get dependencies installed into dubHome (set from config).'
 fi

test/interactive-remove.sh

@@ -9,32 +9,32 @@
 # we need to nuke every `dub` version in the user cache...
 $DUB remove dub -n || true
 
-$DUB fetch [email protected] && [ -d $HOME/.dub/packages/dub-1.9.0/dub ]
-$DUB fetch [email protected] && [ -d $HOME/.dub/packages/dub-1.10.0/dub ]
+$DUB fetch [email protected] && [ -d $HOME/.dub/packages/dub/1.9.0/dub ]
+$DUB fetch [email protected] && [ -d $HOME/.dub/packages/dub/1.10.0/dub ]
 
 echo 1 | $DUB remove dub | tr -d '\n' | grep --ignore-case 'select.*1\.9\.0.*1\.10\.0.*'
-if [ -d $HOME/.dub/packages/dub-1.9.0/dub ]; then
+if [ -d $HOME/.dub/packages/dub/1.9.0/dub ]; then
     die $LINENO 'Failed to remove dub-1.9.0'
 fi
 
-$DUB fetch [email protected] && [ -d $HOME/.dub/packages/dub-1.9.0/dub ]
+$DUB fetch [email protected] && [ -d $HOME/.dub/packages/dub/1.9.0/dub ]
 # EOF aborts remove
 echo -xn '' | $DUB remove dub
-if [ ! -d $HOME/.dub/packages/dub-1.9.0/dub ] || [ ! -d $HOME/.dub/packages/dub-1.10.0/dub ]; then
+if [ ! -d $HOME/.dub/packages/dub/1.9.0/dub ] || [ ! -d $HOME/.dub/packages/dub/1.10.0/dub ]; then
     die $LINENO 'Aborted dub still removed a package'
 fi
 
 # validates input
 echo -e 'abc\n4\n-1\n3' | $DUB remove dub
-if [ -d $HOME/.dub/packages/dub-1.9.0/dub ] || [ -d $HOME/.dub/packages/dub-1.10.0/dub ]; then
+if [ -d $HOME/.dub/packages/dub/1.9.0/dub ] || [ -d $HOME/.dub/packages/dub/1.10.0/dub ]; then
     die $LINENO 'Failed to remove all version of dub'
 fi
 
-$DUB fetch [email protected] && [ -d $HOME/.dub/packages/dub-1.9.0/dub ]
-$DUB fetch [email protected] && [ -d $HOME/.dub/packages/dub-1.10.0/dub ]
+$DUB fetch [email protected] && [ -d $HOME/.dub/packages/dub/1.9.0/dub ]
+$DUB fetch [email protected] && [ -d $HOME/.dub/packages/dub/1.10.0/dub ]
 # is non-interactive with a <version-spec>
 $DUB remove [email protected]
 $DUB remove [email protected]
-if [ -d $HOME/.dub/packages/dub-1.9.0/dub ] || [ -d $HOME/.dub/packages/dub-1.10.0/dub ]; then
+if [ -d $HOME/.dub/packages/dub/1.9.0/dub ] || [ -d $HOME/.dub/packages/dub/1.10.0/dub ]; then
     die $LINENO 'Failed to non-interactively remove specified versions'
 fi

test/version-spec.sh

@@ -43,10 +43,10 @@ $DUB add-local "$CURR_DIR/version-spec/oldfoo"
 $DUB remove-local "$CURR_DIR/version-spec/newfoo"
 $DUB remove-local "$CURR_DIR/version-spec/oldfoo"
 
-$DUB fetch [email protected] && [ -d $HOME/.dub/packages/dub-1.9.0/dub ]
-$DUB fetch dub=1.10.0 && [ -d $HOME/.dub/packages/dub-1.10.0/dub ]
+$DUB fetch [email protected] && [ -d $HOME/.dub/packages/dub/1.9.0/dub ]
+$DUB fetch dub=1.10.0 && [ -d $HOME/.dub/packages/dub/1.10.0/dub ]
 $DUB remove [email protected]
 $DUB remove dub=1.10.0
-if [ -d $HOME/.dub/packages/dub-1.9.0/dub ] || [ -d $HOME/.dub/packages/dub-1.10.0/dub ]; then
+if [ -d $HOME/.dub/packages/dub/1.9.0/dub ] || [ -d $HOME/.dub/packages/dub/1.10.0/dub ]; then
     die $LINENO 'Failed to remove specified versions'
 fi