Checkpoint improvements to deps

honesty/cache.py

@@ -13,7 +13,9 @@
 
 import aiohttp
 import appdirs
+from indexurl import get_index_url
 from keke import kev, ktrace
+from requests.adapters import HTTPAdapter
 from requests.sessions import Session
 
 
@@ -27,7 +29,6 @@ def cache_dir(pkg: str) -> Path:
     appdirs.user_cache_dir("honesty", "python-packaging"),
     "pypi",
 )
-DEFAULT_HONESTY_INDEX_URL = "https://pypi.org/simple/"
 BUFFER_SIZE = 4096 * 1024  # 4M
 
 
@@ -47,7 +48,7 @@ def __init__(
         self.cache_path = Path(cache_dir).expanduser()
 
         if not index_url:
-            index_url = os.environ.get("HONESTY_INDEX_URL", DEFAULT_HONESTY_INDEX_URL)
+            index_url = os.environ.get("HONESTY_INDEX_URL", get_index_url())
         assert isinstance(index_url, str), index_url
         if not index_url.endswith("/"):
             # in a browser, this would be a redirect; we don't know that here.
@@ -72,6 +73,8 @@ def __init__(
         self._cskwargs = cskwargs
         if sync_session is None:
             sync_session = Session()
+            sync_session.mount("http://", HTTPAdapter(pool_maxsize=100))
+            sync_session.mount("https://", HTTPAdapter(pool_maxsize=100))
         self.sync_session = sync_session
 
     @ktrace("pkg", "url")
@@ -117,8 +120,9 @@ def fetch(
                     headers = {"If-None-Match": hdrs["etag"]}
                 elif "last-modified" in hdrs:
                     headers = {"If-Modified-Since": hdrs["last-modified"]}
-                else:
-                    raise Exception(f"Unknown headers {hdrs!r}")
+                # pydepot doesn't provide this yet
+                # else:
+                #    raise Exception(f"Unknown headers {hdrs!r}")
 
         # TODO reconsider timeout
         with kev("get", have_headers=bool(headers)):

honesty/cmdline.py

@@ -25,6 +25,7 @@
 from .checker import guess_license, has_nativemodules, is_pep517, run_checker
 from .deps import DepWalker, is_canonical, print_deps, print_flat_deps
 from .releases import async_parse_index, FileType, Package, parse_index
+from .requirements import _iter_simple_requirements
 from .vcs import CloneAnalyzer, extract2
 
 try:
@@ -432,12 +433,21 @@ def checkcache() -> None:
     help="Just pick the newest version of the package instead of showing deps",
 )
 @click.option(
-    "--python-version", default="3.7.5", help="Python version x.y.z, always 3 numbers"
+    "--python-version",
+    default=".".join(map(str, sys.version_info[:3])),
+    help="Python version x.y.z, always 3 numbers",
+    show_default=True,
 )
 @click.option("--sys-platform", default="linux", help="linux,darwin,win32")
 @click.option("--historical", help="yyyy-mm-dd of a historical date to simulate")
 @click.option("--have", help="pkg==ver to assume already installed", multiple=True)
-@click.option("--use-json", is_flag=True, default=True, show_default=True)
+@click.option("--nouse-json", is_flag=True)
+@click.option(
+    "-r",
+    "--requirement_file",
+    multiple=True,
+    help="Requirements files, specify flag multiple times",
+)
 @click.argument("reqs", nargs=-1)
 def deps(
     include_extras: bool,
@@ -449,9 +459,26 @@ def deps(
     reqs: List[str],
     historical: str,
     have: List[str],
-    use_json: bool,
+    nouse_json: bool,
+    requirement_file: List[str],
 ) -> None:
-    logging.basicConfig(level=logging.DEBUG if verbose else logging.WARNING)
+    new_have = []
+    for h in have:
+        k, _, v = h.partition("==")
+        new_have.append(f"{canonicalize_name(k)}=={v}")
+    have = new_have
+
+    # Command above is called "list" :(
+    reqs = [i for i in reqs]
+
+    if requirement_file:
+        reqs.extend(
+            [
+                str(r)
+                for rf in requirement_file
+                for r in _iter_simple_requirements(Path(rf))
+            ]
+        )
 
     trim_newer: Optional[datetime]
     if historical:
@@ -480,6 +507,7 @@ def current_versions_callback(p: str) -> Optional[str]:
         sys_platform,
         only_first=pick,
         trim_newer=trim_newer,
+        use_json=not nouse_json,
     )
     walker.enqueue(reqs)
     deptree = walker.walk(

honesty/deps.py

@@ -89,7 +89,7 @@ def _all_current_versions_unknown(cn: str) -> Optional[str]:
 
 KeyType = Tuple[str, Version, Optional[Tuple[str, ...]]]
 
-POOL = ThreadPoolExecutor(10)
+POOL = ThreadPoolExecutor(24)
 
 
 class DepWalker:
@@ -585,7 +585,7 @@ def print_deps(
         )
         if key in seen:
             print(
-                f"{prefix}{x.target.name}{dep_extras} (=={x.target.version}) (already listed){' ; ' + str(x.markers) if x.markers else ''}"
+                f"{prefix}{x.target.name}{dep_extras} (=={x.target.version}) (already listed){' ; ' + str(x.markers) if x.markers else ''} via {x.constraints or '*'}"
             )
         else:
             if key[0] in known_conflicts: