Flatten python requirements 2

docs/reference/options.md

@@ -2713,17 +2713,16 @@ boolean
  - [https://github.com/cachix/devenv/blob/main/src/modules/languages/python.nix](https://github.com/cachix/devenv/blob/main/src/modules/languages/python.nix)
 
 
-
 ## languages.python.venv.requirements
 
-Contents of pip requirements.txt file.
+Path to pip requirements.txt file as a string.  Must be relative to devenv root.
 This is passed to `pip install -r` during `devenv shell` initialisation.
 
 
 
 
 *Type:*
-null or strings concatenated with “\\n” or path
+null or string
 
 
 
@@ -2734,7 +2733,6 @@ null or strings concatenated with “\\n” or path
  - [https://github.com/cachix/devenv/blob/main/src/modules/languages/python.nix](https://github.com/cachix/devenv/blob/main/src/modules/languages/python.nix)
 
 
-
 ## languages.python.version
 
 The Python version to use.

examples/python/.test.sh

@@ -1,4 +1,5 @@
-#!/usr/bin/env bash
+#!/bin/sh
 set -ex
 python --version | grep "3.11.3"
-python -c "import requests;print(requests)"
+python -c "import requests;print(requests)"
+python -c "import arrow;print(arrow.__version__)" | grep "1.2.3"

examples/python/devenv.nix

@@ -6,6 +6,6 @@
     version = "3.11.3";
 
     venv.enable = true;
-    venv.requirements = ./requirements.txt;
+    venv.requirements = "./requirements.txt";
   };
 }

examples/python/requirements.txt

@@ -1 +1,2 @@
 requests
+-r requirements2.txt

examples/python/requirements2.txt

@@ -0,0 +1 @@
+-r subrequirements/requirements3.txt

examples/python/subrequirements/constraints.txt

@@ -0,0 +1 @@
+arrow==1.2.3

examples/python/subrequirements/requirements3.txt

@@ -0,0 +1,2 @@
+arrow
+-c constraints.txt

src/devenv/cli.py

@@ -179,6 +179,9 @@ def cli(ctx, offline, system, debugger, nix_flags, verbose):
     ctx.obj["gc_root"] = DEVENV_HOME_GC
     ctx.obj["gc_project"] = DEVENV_HOME_GC / str(int(time.time() * 1000))
 
+@cli.group()
+def processes():
+    pass
 
 @cli.group()
 def processes():

src/modules/languages/python.nix

@@ -2,6 +2,7 @@
 
 let
   cfg = config.languages.python;
+  flattenreq = pkgs.writers.writePython3 "flattenreq" { flakeIgnore = [ "E501" ]; } (builtins.readFile ../support/flattenreq.py);
   libraries = lib.makeLibraryPath (
     cfg.libraries
     ++ (lib.optional cfg.manylinux.enable pkgs.pythonManylinuxPackages.manylinux2014Package)
@@ -26,12 +27,6 @@ let
     ];
   };
 
-  requirements = pkgs.writeText "requirements.txt" (
-    if lib.isPath cfg.venv.requirements
-    then builtins.readFile cfg.venv.requirements
-    else cfg.venv.requirements
-  );
-
   nixpkgs-python = config.lib.getInput {
     name = "nixpkgs-python";
     url = "github:cachix/nixpkgs-python";
@@ -45,10 +40,16 @@ let
 
     VENV_PATH="${config.env.DEVENV_STATE}/venv"
 
+    function recreate_venv () {
+      ${pkgs.coreutils}/bin/rm -rf "$VENV_PATH"
+      ${package.interpreter} -m venv --upgrade-deps "$VENV_PATH"
+      echo "${package.interpreter}" > "$VENV_PATH/.devenv_interpreter"
+    }
+
     profile_python="$(${readlink} ${package.interpreter})"
     devenv_interpreter_path="$(${pkgs.coreutils}/bin/cat "$VENV_PATH/.devenv_interpreter" 2> /dev/null|| false )"
     venv_python="$(${readlink} "$devenv_interpreter_path")"
-    requirements="${lib.optionalString (cfg.venv.requirements != null) ''${requirements}''}"
+    requirements="${lib.optionalString (cfg.venv.requirements != null) ''$DEVENV_ROOT/"${cfg.venv.requirements}"''}"
 
     # recreate venv if necessary
     if [ -z $venv_python ] || [ $profile_python != $venv_python ]
@@ -58,24 +59,37 @@ let
       ${lib.optionalString cfg.poetry.enable ''
         [ -f "${config.env.DEVENV_STATE}/poetry.lock.checksum" ] && rm ${config.env.DEVENV_STATE}/poetry.lock.checksum
       ''}
-      echo ${package.interpreter} -m venv --upgrade-deps "$VENV_PATH"
-      ${package.interpreter} -m venv --upgrade-deps "$VENV_PATH"
-      echo "${package.interpreter}" > "$VENV_PATH/.devenv_interpreter"
+      recreate_venv
+      venv_recreated=1
     fi
 
     source "$VENV_PATH"/bin/activate
 
     # reinstall requirements if necessary
+    # -n means nonempty
     if [ -n "$requirements" ]
       then
-        devenv_requirements_path="$(${pkgs.coreutils}/bin/cat "$VENV_PATH/.devenv_requirements" 2> /dev/null|| false )"
-        devenv_requirements="$(${readlink} "$devenv_requirements_path")"
-        if [ -z $devenv_requirements ] || [ $devenv_requirements != $requirements ]
+        tmp="$(${pkgs.coreutils}/bin/mktemp -d)"
+        ${flattenreq} "$requirements" "$tmp"
+
+        existing_requirements="$VENV_PATH/.devenv_requirements"
+        [ -f $existing_requirements ] || existing_requirements="/dev/null"
+        existing_constraints="$VENV_PATH/.devenv_constraints"
+        [ -f $existing_constraints ] || existing_constraints="/dev/null"
+
+        if ! ${pkgs.diffutils}/bin/cmp --silent "$tmp/requirements.txt" "$existing_requirements" || ! ${pkgs.diffutils}/bin/cmp --silent "$tmp/constraints.txt" "$existing_constraints";
           then
-            echo "${requirements}" > "$VENV_PATH/.devenv_requirements"
-            echo "Requirements changed, running pip install -r ${requirements}..."
-           "$VENV_PATH"/bin/pip install -r ${requirements}
+            if [ -z "$venv_recreated" ]
+              then
+                echo "Requirements changed, rebuilding Python venv..."
+                recreate_venv
+            fi
+            echo "Installing requirements..."
+            ${pkgs.coreutils}/bin/install "$tmp/requirements.txt" "$VENV_PATH/.devenv_requirements"
+            ${pkgs.coreutils}/bin/install "$tmp/constraints.txt" "$VENV_PATH/.devenv_constraints"
+           "$VENV_PATH"/bin/pip install -r "$VENV_PATH/.devenv_requirements" -c "$VENV_PATH/.devenv_constraints"
        fi
+       ${pkgs.coreutils}/bin/rm -rf "$tmp"
     fi
   '';
 
@@ -176,10 +190,10 @@ in
     venv.enable = lib.mkEnableOption "Python virtual environment";
 
     venv.requirements = lib.mkOption {
-      type = lib.types.nullOr (lib.types.either lib.types.lines lib.types.path);
+      type = lib.types.nullOr lib.types.str;
       default = null;
       description = ''
-        Contents of pip requirements.txt file.
+        Path to pip requirements.txt file as a string.  Must be relative to devenv root.
         This is passed to `pip install -r` during `devenv shell` initialisation.
       '';
     };

src/modules/support/flattenreq.py

@@ -0,0 +1,74 @@
+import os
+import re
+import sys
+
+envvar_pattern = r"\$\{([^\}]+)\}"
+
+
+def replace_envvars(text, file_path):
+    def replace(match):
+        env_var = match.group(1)
+        val = os.environ.get(env_var, None)
+        if val is None:
+            raise ValueError(
+                f"No such environment variable {env_var} in {text} within "
+                f"{file_path}"
+            )
+        return val
+
+    return re.sub(envvar_pattern, replace, text)
+
+
+def flatten_requirements(file_path, outdir):
+    requirements = set()
+    constraints = set()
+
+    def process_file(file_path):
+        if not file_path.startswith(os.path.sep):
+            prefix = os.getcwd()
+            file_path = os.path.join(prefix, file_path)
+        with open(file_path, "r") as file:
+            for line in file:
+                prefix = os.path.dirname(file_path)
+                line = line.strip()
+                if line.startswith("-r"):
+                    line = replace_envvars(line, file_path)
+                    nested_file_path = re.match(r"-r\s+(.+)", line).group(1)
+                    if not nested_file_path.startswith(os.path.sep):
+                        nested_file_path = os.path.join(prefix, nested_file_path)
+                    process_file(nested_file_path)
+                elif line.startswith("-c"):
+                    line = replace_envvars(line, file_path)
+                    constraint_file_path = re.match(r"-c\s+(.+)", line).group(1)
+                    if not constraint_file_path.startswith(os.path.sep):
+                        constraint_file_path = os.path.join(
+                            prefix, constraint_file_path
+                        )
+                    process_constraints(constraint_file_path)
+                elif not line.startswith("#") and line:
+                    requirements.add(line)
+
+    def process_constraints(constraint_file_path):
+        with open(constraint_file_path, "r") as file:
+            for line in file:
+                line = line.strip()
+                if not line.startswith("#") and line:
+                    constraints.add(line)
+
+    process_file(file_path)
+
+    with open(os.path.join(outdir, "requirements.txt"), "w") as output_file:
+        for req in sorted(requirements):
+            output_file.write(req + "\n")
+
+    with open(os.path.join(outdir, "constraints.txt"), "w") as con_output_file:
+        for con in sorted(constraints):
+            con_output_file.write(con + "\n")
+
+
+if __name__ == "__main__":
+    requirements_file_path, outdir = sys.argv[1], sys.argv[2]
+    if os.path.isfile(outdir):
+        raise OSError(f"{outdir} is an existing file")
+    os.makedirs(outdir, exist_ok=True)
+    flatten_requirements(requirements_file_path, outdir)