Skip to content

Commit

Permalink
Trying to remove the embedded version of VTK (#25)
Browse files Browse the repository at this point in the history 
* First try to extract VTK from the wheel

* Working on removing the embedded version of VTK
  • Loading branch information
@jmwright
jmwright authored Mar 20, 2024
1 parent 5639090 commit 25ab547
Show file tree
Hide file tree
Showing 2 changed files with 95 additions and 93 deletions.
43 changes: 22 additions & 21 deletions .github/workflows/build-wheel.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,18 +21,18 @@ jobs:
steps:

# see https://github.com/marketplace/actions/download-workflow-artifact
- name: Download artifact
uses: dawidd6/action-download-artifact@v2
if: ${{ matrix.os == 'ubuntu-22.04' }}
with:
workflow: build-vtk.yml
path: /tmp/vtk-wheels

- name: List downloaded wheels
if: ${{ matrix.os == 'ubuntu-22.04' }}
run: |
find /tmp/vtk-wheels -ls
ls -ld /tmp/vtk-wheels/*-cp${{ matrix.python-version }}/vtk-*.whl
# - name: Download artifact
# uses: dawidd6/action-download-artifact@v2
# if: ${{ matrix.os == 'ubuntu-22.04' }}
# with:
# workflow: build-vtk.yml
# path: /tmp/vtk-wheels

# - name: List downloaded wheels
# if: ${{ matrix.os == 'ubuntu-22.04' }}
# run: |
# find /tmp/vtk-wheels -ls
# ls -ld /tmp/vtk-wheels/*-cp${{ matrix.python-version }}/vtk-*.whl

- name: Checkout project
uses: actions/checkout@v2
Expand All @@ -59,6 +59,7 @@ jobs:
shell: bash -l {0}
run: |
if [ "$RUNNER_OS" == "Linux" ]; then
sudo apt update
sudo apt install -y libegl1 libegl1-mesa-dev
fi
Expand All @@ -72,7 +73,7 @@ jobs:
if: ${{ matrix.os == 'ubuntu-22.04' }}
shell: bash -l {0}
run: |
conda install -c cadquery -n cadquery-ocp -y ocp=7.7.2.* vtk=9.2.* occt=7.7.2=all*
conda install -c cadquery -n cadquery-ocp -y ocp=7.7.2.* vtk=9.2.* occt=7.7.2=all* auditwheel
- name: Pip Deps Setup 1
shell: bash -l {0}
Expand All @@ -86,17 +87,16 @@ jobs:
run: |
pip install auditwheel patchelf
- name: Manylinux Build 1
shell: bash -l {0}
if: ${{ matrix.os == 'ubuntu-22.04' }}
run: |
export VTK_MANYLINUX=/tmp/vtk-manylinux
pip install -t $VTK_MANYLINUX --no-deps /tmp/vtk-wheels/*-cp${{ matrix.python-version }}/vtk-*.whl
python -m build --no-isolation --wheel
# - name: Manylinux Build 1
# shell: bash -l {0}
# if: ${{ matrix.os == 'ubuntu-22.04' }}
# run: |
# export VTK_MANYLINUX=/tmp/vtk-manylinux
# pip install -t $VTK_MANYLINUX --no-deps /tmp/vtk-wheels/*-cp${{ matrix.python-version }}/vtk-*.whl
# python -m build --no-isolation --wheel

- name: Conda-only Build
shell: bash -l {0}
if: ${{ matrix.os != 'ubuntu-22.04' }}
run: |
python -m build --no-isolation --wheel
Expand All @@ -113,6 +113,7 @@ jobs:
conda deactivate
conda create --yes -n cadquerytest python=${{ matrix.python-version }}
conda activate cadquerytest
ls dist/*
pip install dist/*.whl
python -c "import OCP;print('OCP imported successfully')"
Expand Down
145 changes: 73 additions & 72 deletions setup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,54 +55,54 @@
import zipfile


VTK_MANYLINUX = None
if platform.system() == "Linux" and os.getenv("VTK_MANYLINUX"):
VTK_MANYLINUX = pathlib.Path(os.getenv("VTK_MANYLINUX")).resolve()
if not (VTK_MANYLINUX / "vtkmodules" / "__init__.py").exists():
raise Exception(f"invalid VTK_MANYLINUX: {VTK_MANYLINUX}")


def populate_lib_dir(conda_prefix, vtk_manylinux, out_dir):
"""Merge non-vtk conda libs and vtk manylinux libs into a single directory"""

conda_prefix = pathlib.Path(conda_prefix)
vtk_manylinux = pathlib.Path(vtk_manylinux)
out_dir = pathlib.Path(out_dir)

fns = list((conda_prefix / "conda-meta").glob("vtk-9*-qt_*.json"))
if len(fns) != 1:
raise Exception(f"could not find unique vtk meta: {fns}")
vtk_meta = json.loads(fns[0].read_text())
vtk_files = {conda_prefix / f for f in vtk_meta["files"]}

# Do not copy regular VTK files because we will use manylinux
# files instead. Do copy VTK symlinks because they're expected by
# other conda packages which we may bundle into the wheel.
vtk_skipped_files = []
for f in (conda_prefix / "lib").iterdir():
if f.is_dir():
continue
if f in vtk_files and not f.is_symlink():
vtk_skipped_files.append(f)
else:
shutil.copy2(f, out_dir, follow_symlinks=False)

# Copy VTK files. Symlinks in the target will be followed so we
# are fixing broken symlinks.
for f in (vtk_manylinux / "vtkmodules").glob("lib*.so*"):
shutil.copy2(f, out_dir, follow_symlinks=False)

# The VTK manylinux wheel may not provide all the libraries of the
# VTK conda pkg, so there may be missing files or broken symlinks.
# If this is problematic then auditwheel will fail. For now,
# print some helpful information.
for f in vtk_skipped_files:
if not (out_dir / f.name).exists():
print("manylinux whl did not provide:", f)

# Copy the rest of the libraries bundled in the VTK wheel.
for f in (vtk_manylinux / "vtk.libs").glob("lib*.so*"):
shutil.copy2(f, out_dir, follow_symlinks=False)
# VTK_MANYLINUX = None
# if platform.system() == "Linux" and os.getenv("VTK_MANYLINUX"):
# VTK_MANYLINUX = pathlib.Path(os.getenv("VTK_MANYLINUX")).resolve()
# if not (VTK_MANYLINUX / "vtkmodules" / "__init__.py").exists():
# raise Exception(f"invalid VTK_MANYLINUX: {VTK_MANYLINUX}")


# def populate_lib_dir(conda_prefix, vtk_manylinux, out_dir):
# """Merge non-vtk conda libs and vtk manylinux libs into a single directory"""

# conda_prefix = pathlib.Path(conda_prefix)
# vtk_manylinux = pathlib.Path(vtk_manylinux)
# out_dir = pathlib.Path(out_dir)

# fns = list((conda_prefix / "conda-meta").glob("vtk-9*-qt_*.json"))
# if len(fns) != 1:
# raise Exception(f"could not find unique vtk meta: {fns}")
# vtk_meta = json.loads(fns[0].read_text())
# vtk_files = {conda_prefix / f for f in vtk_meta["files"]}

# # Do not copy regular VTK files because we will use manylinux
# # files instead. Do copy VTK symlinks because they're expected by
# # other conda packages which we may bundle into the wheel.
# vtk_skipped_files = []
# for f in (conda_prefix / "lib").iterdir():
# if f.is_dir():
# continue
# if f in vtk_files and not f.is_symlink():
# vtk_skipped_files.append(f)
# else:
# shutil.copy2(f, out_dir, follow_symlinks=False)

# # Copy VTK files. Symlinks in the target will be followed so we
# # are fixing broken symlinks.
# for f in (vtk_manylinux / "vtkmodules").glob("lib*.so*"):
# shutil.copy2(f, out_dir, follow_symlinks=False)

# # The VTK manylinux wheel may not provide all the libraries of the
# # VTK conda pkg, so there may be missing files or broken symlinks.
# # If this is problematic then auditwheel will fail. For now,
# # print some helpful information.
# for f in vtk_skipped_files:
# if not (out_dir / f.name).exists():
# print("manylinux whl did not provide:", f)

# # Copy the rest of the libraries bundled in the VTK wheel.
# for f in (vtk_manylinux / "vtk.libs").glob("lib*.so*"):
# shutil.copy2(f, out_dir, follow_symlinks=False)


class copy_installed(setuptools.command.build_ext.build_ext):
Expand All @@ -116,22 +116,22 @@ def build_extension(self, ext):
# OCP is a single-file extension; just copy it
shutil.copy(OCP.__file__, self.build_lib)
# vtkmodules is a package; copy it while excluding __pycache__
if VTK_MANYLINUX:
# Copy python source files and python extensions, but not
# shared libraries. We will point auditwheel to them
# separately to be bundled in.
shutil.copytree(
os.path.join(VTK_MANYLINUX, "vtkmodules"),
os.path.join(self.build_lib, "vtkmodules"),
ignore=shutil.ignore_patterns("__pycache__", "libvtk*.so*"),
)
else:
assert vtkmodules.__file__.endswith(os.path.join(os.sep, "vtkmodules", "__init__.py"))
shutil.copytree(
os.path.dirname(vtkmodules.__file__),
os.path.join(self.build_lib, "vtkmodules"),
ignore=shutil.ignore_patterns("__pycache__"),
)
# if VTK_MANYLINUX:
# # Copy python source files and python extensions, but not
# # shared libraries. We will point auditwheel to them
# # separately to be bundled in.
# shutil.copytree(
# os.path.join(VTK_MANYLINUX, "vtkmodules"),
# os.path.join(self.build_lib, "vtkmodules"),
# ignore=shutil.ignore_patterns("__pycache__", "libvtk*.so*"),
# )
# else:
# assert vtkmodules.__file__.endswith(os.path.join(os.sep, "vtkmodules", "__init__.py"))
# shutil.copytree(
# os.path.dirname(vtkmodules.__file__),
# os.path.join(self.build_lib, "vtkmodules"),
# ignore=shutil.ignore_patterns("__pycache__"),
# )


class bdist_wheel_repaired(wheel.bdist_wheel.bdist_wheel):
Expand Down Expand Up @@ -163,12 +163,12 @@ def run(self):
out_dir = os.path.join(self.dist_dir, "repaired")
system = platform.system()
if system == "Linux":
if VTK_MANYLINUX:
# Create a lib dir with VTK manylinux files merged in,
# and point auditwheel to it.
merged_lib_dir = tempfile.mkdtemp()
populate_lib_dir(conda_prefix, VTK_MANYLINUX, merged_lib_dir)
lib_path = merged_lib_dir
# if VTK_MANYLINUX:
# # Create a lib dir with VTK manylinux files merged in,
# # and point auditwheel to it.
# merged_lib_dir = tempfile.mkdtemp()
# populate_lib_dir(conda_prefix, VTK_MANYLINUX, merged_lib_dir)
# lib_path = merged_lib_dir
repair_wheel_linux(lib_path, bad_whl, out_dir)
elif system == "Darwin":
repair_wheel_macos(lib_path, bad_whl, out_dir)
Expand Down Expand Up @@ -272,7 +272,7 @@ def add_licenses_bundled(conda_prefix, whl, added_files):
with open(os.path.join(dist_info,"LICENSES_bundled"), "w") as f:
f.write("This wheel distribution bundles a number of libraries that\n")
f.write("are compatibly licensed. We list them here.\n")
write_licenses(conda_prefix, whl, ["ocp", "vtk"], added_files, f)
write_licenses(conda_prefix, whl, ["ocp"], added_files, f) # , "vtk"

subprocess.check_call(["wheel", "pack", "-d", os.path.dirname(whl), os.path.dirname(dist_info)])

Expand Down Expand Up @@ -378,7 +378,7 @@ def try_unmangle(n):
setup(
name="cadquery-ocp",
version=wheel_version,
description="OCP+VTK wheel with shared library dependencies bundled.",
description="OCP wheel with shared library dependencies bundled.",
long_description=open("README.md").read(),
long_description_content_type='text/markdown',
author="adam-urbanczyk, fp473 (wheel generation), roipoussiere (wheel generation)",
Expand All @@ -396,6 +396,7 @@ def try_unmangle(n):
"Topic :: Software Development :: Libraries :: Python Modules",
"Topic :: Scientific/Engineering"
],
install_requires=["vtk"],
# Dummy extension to trigger build_ext
ext_modules=[Extension("__dummy__", sources=[])],
cmdclass={"bdist_wheel": bdist_wheel_repaired, "build_ext": copy_installed},
Expand Down

0 comments on commit 25ab547

Please sign in to comment.