From a9de40989ab9b96d5531fa8d055a430dc993f9dc Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Wed, 13 Nov 2024 19:30:57 -0800 Subject: [PATCH 1/5] Parse the PKG-INFO file from sdist archives. --- metadata_please/sdist.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/metadata_please/sdist.py b/metadata_please/sdist.py index 7194c1c..a59d046 100644 --- a/metadata_please/sdist.py +++ b/metadata_please/sdist.py @@ -33,9 +33,14 @@ def basic_metadata_from_zip_sdist(zf: ZipFile) -> BasicMetadata: if not requires: return BasicMetadata((), frozenset(), "-") - data = zf.read(requires[0]) - assert data is not None - return BasicMetadata.from_sdist_pkg_info_and_requires(b"", data) + requires_data = zf.read(requires[0]) + assert requires_data is not None + + pkg_info = next(f for f in zf.namelist() if f.endswith("/PKG-INFO")) + pkg_info_data = zf.read(pkg_info) + assert pkg_info_data is not None + + return BasicMetadata.from_sdist_pkg_info_and_requires(pkg_info_data, requires_data) def from_tar_sdist(tf: TarFile) -> bytes: @@ -58,6 +63,7 @@ def from_tar_sdist(tf: TarFile) -> bytes: buf.append(f"Requires-Dist: {req}\n") for extra in sorted(extras): buf.append(f"Provides-Extra: {extra}\n") + return ("".join(buf)).encode("utf-8") @@ -68,7 +74,14 @@ def basic_metadata_from_tar_sdist(tf: TarFile) -> BasicMetadata: if not requires: return BasicMetadata((), frozenset()) - fo = tf.extractfile(requires[0]) - assert fo is not None + requires_fo = tf.extractfile(requires[0]) + assert requires_fo is not None + + pkg_info = next(f for f in tf.getnames() if f.endswith("/PKG-INFO")) + + pkg_info_fo = tf.extractfile(pkg_info) + assert pkg_info_fo is not None - return BasicMetadata.from_sdist_pkg_info_and_requires(b"", fo.read()) + return BasicMetadata.from_sdist_pkg_info_and_requires( + pkg_info_fo.read(), requires_fo.read() + ) From cb424e2feac57d5254c3e7a1b346d9a2c7e31268 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Wed, 13 Nov 2024 20:03:21 -0800 Subject: [PATCH 2/5] Add tests for sdist tar. --- metadata_please/sdist.py | 22 +++++++++++----------- metadata_please/tests/sdist.py | 22 ++++++++++++++++++++-- 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/metadata_please/sdist.py b/metadata_please/sdist.py index a59d046..a730395 100644 --- a/metadata_please/sdist.py +++ b/metadata_please/sdist.py @@ -30,11 +30,11 @@ def from_zip_sdist(zf: ZipFile) -> bytes: def basic_metadata_from_zip_sdist(zf: ZipFile) -> BasicMetadata: requires = [f for f in zf.namelist() if f.endswith("/requires.txt")] requires.sort(key=len) - if not requires: - return BasicMetadata((), frozenset(), "-") - - requires_data = zf.read(requires[0]) - assert requires_data is not None + if requires: + requires_data = zf.read(requires[0]) + assert requires_data is not None + else: + requires_data = b"" pkg_info = next(f for f in zf.namelist() if f.endswith("/PKG-INFO")) pkg_info_data = zf.read(pkg_info) @@ -71,11 +71,11 @@ def basic_metadata_from_tar_sdist(tf: TarFile) -> BasicMetadata: # XXX Why do ZipFile and TarFile not have a common interface ?! requires = [f for f in tf.getnames() if f.endswith("/requires.txt")] requires.sort(key=len) - if not requires: - return BasicMetadata((), frozenset()) - - requires_fo = tf.extractfile(requires[0]) - assert requires_fo is not None + if requires: + requires_fo = tf.extractfile(requires[0]) + assert requires_fo is not None + else: + requires_fo = b"" pkg_info = next(f for f in tf.getnames() if f.endswith("/PKG-INFO")) @@ -83,5 +83,5 @@ def basic_metadata_from_tar_sdist(tf: TarFile) -> BasicMetadata: assert pkg_info_fo is not None return BasicMetadata.from_sdist_pkg_info_and_requires( - pkg_info_fo.read(), requires_fo.read() + pkg_info_fo.read(), requires_fo and requires_fo.read() ) diff --git a/metadata_please/tests/sdist.py b/metadata_please/tests/sdist.py index b222c95..e0fee06 100644 --- a/metadata_please/tests/sdist.py +++ b/metadata_please/tests/sdist.py @@ -15,6 +15,7 @@ def test_requires_as_expected(self) -> None: z = MemoryZipFile( { "foo/__init__.py": b"", + "foo.egg-info/PKG-INFO": b"\n", "foo.egg-info/requires.txt": b"""\ a [e] @@ -36,6 +37,7 @@ def test_basic_metadata(self) -> None: z = MemoryZipFile( { "foo/__init__.py": b"", + "foo.egg-info/PKG-INFO": b"\n", "foo.egg-info/requires.txt": b"""\ a [e] @@ -68,6 +70,7 @@ def test_basic_metadata_absl_py_09(self) -> None: z = MemoryZipFile( { "foo/__init__.py": b"", + "foo.egg-info/PKG-INFO": b"\n", "foo.egg-info/requires.txt": b"""\ six @@ -94,7 +97,7 @@ def test_basic_metadata_absl_py_09(self) -> None: class TarSdistTest(unittest.TestCase): def test_requires_as_expected(self) -> None: t = MemoryTarFile( - ["foo.egg-info/requires.txt", "foo/__init__.py"], + ["foo.egg-info/PKG-INFO", "foo.egg-info/requires.txt", "foo/__init__.py"], read_value=b"""\ a [e] @@ -113,7 +116,7 @@ def test_requires_as_expected(self) -> None: def test_basic_metadata(self) -> None: t = MemoryTarFile( - ["foo.egg-info/requires.txt", "foo/__init__.py"], + ["foo.egg-info/PKG-INFO", "foo.egg-info/requires.txt", "foo/__init__.py"], read_value=b"""\ a [e] @@ -126,3 +129,18 @@ def test_basic_metadata(self) -> None: bm.reqs, ) self.assertEqual({"e"}, bm.provides_extra) + + def test_metadata_fields_from_tar_sdist(self) -> None: + t = MemoryTarFile( + ["foo.egg-info/PKG-INFO", "foo/__init__.py"], + read_value=b"""Requires-Dist: foo\nVersion: 1.2.58\nSummary: Some Summary\nHome-page: http://example.com\nAuthor: Chicken\nAuthor-email: duck@example.com\nKeywords: farm,animals\nRequires-Python: >=3.6\nDescription-Content-Type: text/markdown\n""", + ) + bm = basic_metadata_from_tar_sdist(t) # type: ignore + self.assertEqual("1.2.58", bm.version) + self.assertEqual("Some Summary", bm.summary) + self.assertEqual("http://example.com", bm.url) + self.assertEqual("Chicken", bm.author) + self.assertEqual("duck@example.com", bm.author_email) + self.assertEqual("farm,animals", bm.keywords) + self.assertEqual("text/markdown", bm.long_description_content_type) + self.assertEqual(None, bm.description) From a40ae7e542cf7a78dd7b80351bffab92c98b6557 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Wed, 13 Nov 2024 20:21:34 -0800 Subject: [PATCH 3/5] Add tests for sdist zip. --- metadata_please/tests/sdist.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/metadata_please/tests/sdist.py b/metadata_please/tests/sdist.py index e0fee06..bb6a78b 100644 --- a/metadata_please/tests/sdist.py +++ b/metadata_please/tests/sdist.py @@ -93,6 +93,24 @@ def test_basic_metadata_absl_py_09(self) -> None: ) self.assertEqual({"test"}, bm.provides_extra) + def test_basic_metadata_fields(self) -> None: + z = MemoryZipFile( + { + "foo/__init__.py": b"", + "foo.egg-info/PKG-INFO": b"Requires-Dist: foo\nVersion: 1.2.58\nSummary: Some Summary\nHome-page: http://example.com\nAuthor: Chicken\nAuthor-email: duck@example.com\nKeywords: farm,animals\nRequires-Python: >=3.6\nDescription-Content-Type: text/markdown", + } + ) + bm = basic_metadata_from_zip_sdist(z) # type: ignore + self.assertEqual(["foo"], bm.reqs) + self.assertEqual("1.2.58", bm.version) + self.assertEqual("Some Summary", bm.summary) + self.assertEqual("http://example.com", bm.url) + self.assertEqual("Chicken", bm.author) + self.assertEqual("duck@example.com", bm.author_email) + self.assertEqual("farm,animals", bm.keywords) + self.assertEqual("text/markdown", bm.long_description_content_type) + self.assertEqual(None, bm.description) + class TarSdistTest(unittest.TestCase): def test_requires_as_expected(self) -> None: From 2121ba3ccddbd37c972ded64a268d1babb2fcf37 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Thu, 14 Nov 2024 13:47:21 -0800 Subject: [PATCH 4/5] Find first PKG-INFO and update tests to abstract the contents. --- metadata_please/sdist.py | 20 ++++++++++++++++---- metadata_please/tests/metadata_contents.py | 15 +++++++++++++++ metadata_please/tests/sdist.py | 14 +++++++++----- metadata_please/tests/wheel.py | 7 ++++--- 4 files changed, 44 insertions(+), 12 deletions(-) create mode 100644 metadata_please/tests/metadata_contents.py diff --git a/metadata_please/sdist.py b/metadata_please/sdist.py index a730395..0c50f5a 100644 --- a/metadata_please/sdist.py +++ b/metadata_please/sdist.py @@ -36,7 +36,13 @@ def basic_metadata_from_zip_sdist(zf: ZipFile) -> BasicMetadata: else: requires_data = b"" - pkg_info = next(f for f in zf.namelist() if f.endswith("/PKG-INFO")) + # Find the PKG-INFO file with the shortest path. This is to avoid picking up + # a PKG-INFO file from a nested test directory. + pkg_info = sorted( + (f for f in zf.namelist() if f == "PKG-INFO" or f.endswith("/PKG-INFO")), + key=len, + )[0] + pkg_info_data = zf.read(pkg_info) assert pkg_info_data is not None @@ -74,14 +80,20 @@ def basic_metadata_from_tar_sdist(tf: TarFile) -> BasicMetadata: if requires: requires_fo = tf.extractfile(requires[0]) assert requires_fo is not None + requires_data = requires_fo.read() else: - requires_fo = b"" + requires_data = b"" - pkg_info = next(f for f in tf.getnames() if f.endswith("/PKG-INFO")) + # Find the PKG-INFO file with the shortest path. This is to avoid picking up + # a PKG-INFO file from a nested test directory. + pkg_info = sorted( + (f for f in tf.getnames() if f == "PKG-INFO" or f.endswith("/PKG-INFO")), + key=len, + )[0] pkg_info_fo = tf.extractfile(pkg_info) assert pkg_info_fo is not None return BasicMetadata.from_sdist_pkg_info_and_requires( - pkg_info_fo.read(), requires_fo and requires_fo.read() + pkg_info_fo.read(), requires_data ) diff --git a/metadata_please/tests/metadata_contents.py b/metadata_please/tests/metadata_contents.py new file mode 100644 index 0000000..e59e228 --- /dev/null +++ b/metadata_please/tests/metadata_contents.py @@ -0,0 +1,15 @@ +METADATA_CONTENTS = b"""\ +Requires-Dist: foo +Version: 1.2.58 +Summary: Some Summary +Home-page: http://example.com +Author: Chicken +Author-email: duck@example.com +Keywords: farm,animals +Requires-Python: >=3.6 +Description-Content-Type: text/markdown + +# Foo + +A very important package. +""" diff --git a/metadata_please/tests/sdist.py b/metadata_please/tests/sdist.py index bb6a78b..9bdbc6d 100644 --- a/metadata_please/tests/sdist.py +++ b/metadata_please/tests/sdist.py @@ -8,6 +8,7 @@ ) from ._tar import MemoryTarFile from ._zip import MemoryZipFile +from .metadata_contents import METADATA_CONTENTS class ZipSdistTest(unittest.TestCase): @@ -94,10 +95,13 @@ def test_basic_metadata_absl_py_09(self) -> None: self.assertEqual({"test"}, bm.provides_extra) def test_basic_metadata_fields(self) -> None: + """ + Modern setuptools will drop a PKG-INFO file in a sdist that is very similar to the METADATA file in a wheel. + """ z = MemoryZipFile( { "foo/__init__.py": b"", - "foo.egg-info/PKG-INFO": b"Requires-Dist: foo\nVersion: 1.2.58\nSummary: Some Summary\nHome-page: http://example.com\nAuthor: Chicken\nAuthor-email: duck@example.com\nKeywords: farm,animals\nRequires-Python: >=3.6\nDescription-Content-Type: text/markdown", + "PKG-INFO": METADATA_CONTENTS, } ) bm = basic_metadata_from_zip_sdist(z) # type: ignore @@ -109,7 +113,7 @@ def test_basic_metadata_fields(self) -> None: self.assertEqual("duck@example.com", bm.author_email) self.assertEqual("farm,animals", bm.keywords) self.assertEqual("text/markdown", bm.long_description_content_type) - self.assertEqual(None, bm.description) + self.assertEqual("# Foo\n\nA very important package.\n", bm.description) class TarSdistTest(unittest.TestCase): @@ -150,8 +154,8 @@ def test_basic_metadata(self) -> None: def test_metadata_fields_from_tar_sdist(self) -> None: t = MemoryTarFile( - ["foo.egg-info/PKG-INFO", "foo/__init__.py"], - read_value=b"""Requires-Dist: foo\nVersion: 1.2.58\nSummary: Some Summary\nHome-page: http://example.com\nAuthor: Chicken\nAuthor-email: duck@example.com\nKeywords: farm,animals\nRequires-Python: >=3.6\nDescription-Content-Type: text/markdown\n""", + ["PKG-INFO", "foo/__init__.py"], + read_value=METADATA_CONTENTS, ) bm = basic_metadata_from_tar_sdist(t) # type: ignore self.assertEqual("1.2.58", bm.version) @@ -161,4 +165,4 @@ def test_metadata_fields_from_tar_sdist(self) -> None: self.assertEqual("duck@example.com", bm.author_email) self.assertEqual("farm,animals", bm.keywords) self.assertEqual("text/markdown", bm.long_description_content_type) - self.assertEqual(None, bm.description) + self.assertEqual("# Foo\n\nA very important package.\n", bm.description) diff --git a/metadata_please/tests/wheel.py b/metadata_please/tests/wheel.py index 7343202..5cce392 100644 --- a/metadata_please/tests/wheel.py +++ b/metadata_please/tests/wheel.py @@ -1,7 +1,8 @@ import unittest -from ..wheel import basic_metadata_from_wheel, from_wheel, InvalidWheel +from ..wheel import InvalidWheel, basic_metadata_from_wheel, from_wheel from ._zip import MemoryZipFile +from .metadata_contents import METADATA_CONTENTS class WheelTest(unittest.TestCase): @@ -58,7 +59,7 @@ def test_basic_metadata(self) -> None: def test_basic_metadata_more_fields(self) -> None: z = MemoryZipFile( { - "foo.dist-info/METADATA": b"Requires-Dist: foo\nVersion: 1.2.58\nSummary: Some Summary\nHome-page: http://example.com\nAuthor: Chicken\nAuthor-email: duck@example.com\nKeywords: farm,animals\nRequires-Python: >=3.6\nDescription-Content-Type: text/markdown", + "foo.dist-info/METADATA": METADATA_CONTENTS, "foo/__init__.py": b"", } ) @@ -71,4 +72,4 @@ def test_basic_metadata_more_fields(self) -> None: self.assertEqual("duck@example.com", bm.author_email) self.assertEqual("farm,animals", bm.keywords) self.assertEqual("text/markdown", bm.long_description_content_type) - self.assertEqual(None, bm.description) + self.assertEqual("# Foo\n\nA very important package.\n", bm.description) From e69ee20ffd0c146fd07dcba5eaeb0474515ae752 Mon Sep 17 00:00:00 2001 From: Amjith Ramanujam Date: Thu, 14 Nov 2024 13:48:34 -0800 Subject: [PATCH 5/5] Fix formatting. --- metadata_please/tests/wheel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metadata_please/tests/wheel.py b/metadata_please/tests/wheel.py index 5cce392..33ec67e 100644 --- a/metadata_please/tests/wheel.py +++ b/metadata_please/tests/wheel.py @@ -1,6 +1,6 @@ import unittest -from ..wheel import InvalidWheel, basic_metadata_from_wheel, from_wheel +from ..wheel import basic_metadata_from_wheel, from_wheel, InvalidWheel from ._zip import MemoryZipFile from .metadata_contents import METADATA_CONTENTS