Generate Python wheels in CI (#1317)

This PR adds a new GitHub Actions workflow that generates Python wheels and an sdist that all can be uploaded to PyPI. It generates wheels for Python 3.7, 3.8, 3.9 3.10, and 3.11 for Windows, macOS and Linux. I used https://github.com/scikit-build/scikit-build-core as the build backend. It's a new tool that is quite simple and allows to build Python extensions (C, C++, etc) without setuptools. Since it's a build backend, everything can be built with pip install. or https://github.com/pypa/build (python -m build).
AcademySoftwareFoundation · Aug 22, 2023 · c545829 · c545829
1 parent c5b376c
commit c545829
Show file tree

Hide file tree

Showing 33 changed files with 554 additions and 224 deletions.
diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml
@@ -0,0 +1,128 @@
+name: python
+
+on:
+  push:
+    paths-ignore:
+      - '**.md'
+  pull_request:
+    paths-ignore:
+      - '**.md'
+  workflow_dispatch:
+
+jobs:
+  # Generate the sdist first. We'll use it to create the wheels.
+  # https://packaging.python.org/en/latest/flow#the-source-distribution-sdist
+  sdist:
+    name: Generate Source Distribution
+    runs-on: ubuntu-latest
+    outputs:
+      sdist_filename: ${{ steps.generate.outputs.filename }}
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - uses: actions/setup-python@v4
+      with:
+        python-version: 3.11
+
+    - name: Install Build Command
+      run: python -m pip install build
+
+    - name: Generate Sdist
+      id: generate
+      run: |
+        python -m build -s . --outdir dist
+        echo "filename=$(ls dist)" >> "$GITHUB_OUTPUT"
+
+    - name: Upload Sdist
+      uses: actions/upload-artifact@v3
+      with:
+        name: sdist
+        path: ./dist/*.tar.gz
+
+  # Create the wheels. It'll use the sdist to confirm that we can compile MaterialX from the sdist.
+  # https://packaging.python.org/en/latest/flow#the-built-distributions-wheels
+  wheels:
+    name: Generate Wheel
+    runs-on: ${{ matrix.os }}
+    needs: ['sdist']
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ['37', '38', '39', '310', '311']
+        os: ['ubuntu-latest', 'macos-latest', 'windows-latest']
+
+    steps:
+    - name: Download Sdist
+      uses: actions/download-artifact@v3
+      with:
+        name: sdist
+        path: sdist
+
+    - name: Build Wheel
+      # https://cibuildwheel.readthedocs.io/en/stable/
+      uses: pypa/[email protected]
+      with:
+        # Build from the sdist. We want to make sure it's valid and works as expected.
+        package-dir: ${{ github.workspace }}/sdist/${{ needs.sdist.outputs.sdist_filename }}
+        output-dir: wheels
+      env:
+        CIBW_BUILD: 'cp${{ matrix.python-version }}-*'
+        CIBW_SKIP: '*musllinux*'
+        CIBW_ARCHS: 'auto64'
+        # https://github.com/pypa/manylinux
+        # manylinux2014 is CentOS 7 based. Which means GCC 10 and glibc 2.17.
+        CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
+        CIBW_BEFORE_ALL_LINUX: yum install -y libXt-devel
+        CIBW_BEFORE_ALL_MACOS: sudo xcode-select -switch /Applications/Xcode_13.4.app
+        CIBW_BUILD_VERBOSITY: 1
+        CIBW_ENVIRONMENT: CMAKE_BUILD_PARALLEL_LEVEL=2
+        # CIBW_BUILD_FRONTEND: build  # https://github.com/pypa/build
+        MACOSX_DEPLOYMENT_TARGET: '10.15'
+
+    - name: Upload Wheel
+      uses: actions/upload-artifact@v3
+      with:
+        name: wheels
+        path: ./wheels/*.whl
+
+  test:
+    name: Test Wheel
+    needs: ['wheels']
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
+        os: ['ubuntu-latest', 'macos-latest', 'windows-latest']
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Download Wheels
+      uses: actions/download-artifact@v3
+      with:
+        name: wheels
+        path: wheels
+
+    - name: Install Wheel
+      run: python -m pip install MaterialX --find-links wheels --no-index
+
+    - name: Python Tests
+      shell: bash
+      run: |
+        set -e
+        python python/MaterialXTest/main.py
+        python python/MaterialXTest/genshader.py
+        python python/Scripts/mxformat.py ./resources/Materials/TestSuite/stdlib/upgrade --yes --upgrade
+        python python/Scripts/mxvalidate.py ./resources/Materials/Examples/StandardSurface/standard_surface_marble_solid.mtlx --stdlib --verbose
+        python python/Scripts/mxdoc.py --docType md ./libraries/pbrlib/pbrlib_defs.mtlx
+        python python/Scripts/mxdoc.py --docType html ./libraries/bxdf/standard_surface.mtlx
+        python python/Scripts/generateshader.py ./resources/Materials/Examples/StandardSurface --target glsl
+        python python/Scripts/generateshader.py ./resources/Materials/Examples/StandardSurface --target osl
+        python python/Scripts/generateshader.py ./resources/Materials/Examples/StandardSurface --target mdl
+        python python/Scripts/generateshader.py ./resources/Materials/Examples/StandardSurface --target msl
diff --git a/.gitignore b/.gitignore
@@ -1 +1,2 @@
 build
+dist
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -31,7 +31,7 @@ if (MATERIALX_BUILD_JS)
     endif()
 endif()
 
-project(MaterialX)
+project(MaterialX VERSION ${MATERIALX_LIBRARY_VERSION})
 
 option(MATERIALX_BUILD_PYTHON "Build the MaterialX Python package from C++ bindings. Requires Python 3.6 or greater." OFF)
 option(MATERIALX_BUILD_VIEWER "Build the MaterialX Viewer." OFF)
@@ -101,6 +101,12 @@ set(MATERIALX_OSL_BINARY_OSLC "" CACHE FILEPATH "Full path to the OSL compiler b
 set(MATERIALX_OSL_BINARY_TESTRENDER "" CACHE FILEPATH "Full path to the OSL test render binary.")
 set(MATERIALX_OSL_INCLUDE_PATH "" CACHE PATH "Full path to OSL shader includes (e.g. 'stdosl.h').")
 
+set(MATERIALX_PYTHON_FOLDER_NAME "python/MaterialX" CACHE INTERNAL "Folder name to user for installing the Python library.")
+
+if(SKBUILD)
+    set(MATERIALX_PYTHON_FOLDER_NAME "MaterialX")
+endif()
+
 # Helpers for MDL validation
 if (MATERIALX_BUILD_GEN_MDL)
     set(MATERIALX_MDLC_EXECUTABLE "" CACHE FILEPATH "Full path to the mdlc binary.")
@@ -207,6 +213,12 @@ set(MATERIALX_SAME_DIR_RPATH "${RPATH_RELATIVE_SYMBOL};${CMAKE_INSTALL_PREFIX}/$
 set(MATERIALX_UP_ONE_RPATH "${RPATH_RELATIVE_SYMBOL}/../${MATERIALX_INSTALL_LIB_PATH};${MATERIALX_SAME_DIR_RPATH}")
 # For linking to libraries where source is two directories deep, ie: "MATX/python/MaterialX/../../lib"
 set(MATERIALX_UP_TWO_RPATH "${RPATH_RELATIVE_SYMBOL}/../../${MATERIALX_INSTALL_LIB_PATH};${MATERIALX_SAME_DIR_RPATH}")
+if(SKBUILD)
+    # When building the Python wheels, we don't want to set any RPATH because
+    # we want to wheel to be self-contained. We don't want any interference from
+    # external paths.
+    set(MATERIALX_UP_TWO_RPATH "${RPATH_RELATIVE_SYMBOL}")
+endif()
 
 # Adjust compiler settings
 if(MSVC)
@@ -300,7 +312,7 @@ if(MATERIALX_BUILD_RENDER)
     if(MATERIALX_BUILD_GRAPH_EDITOR)
         add_subdirectory(source/MaterialXGraphEditor)
     endif()
-    if(MATERIALX_INSTALL_RESOURCES)
+    if(MATERIALX_INSTALL_RESOURCES AND NOT SKBUILD)
         add_subdirectory(resources)
     endif()
 endif()
@@ -335,26 +347,28 @@ if(${CMAKE_VERSION} VERSION_GREATER "3.6.2")
 endif()
 
 # Install root-level documents
-install(FILES LICENSE CHANGELOG.md README.md THIRD-PARTY.md
-        DESTINATION .)
-
-set(MATERIALX_GEN_CONFIG_PATH "${MATERIALX_INSTALL_LIB_PATH}/cmake/${CMAKE_PROJECT_NAME}")
-
-include(CMakePackageConfigHelpers)
-configure_package_config_file(cmake/modules/MaterialXConfig.cmake.in
-                              ${CMAKE_BINARY_DIR}/cmake/${CMAKE_PROJECT_NAME}Config.cmake
-                              INSTALL_DESTINATION "${MATERIALX_GEN_CONFIG_PATH}"
-                              PATH_VARS CMAKE_INSTALL_PREFIX CMAKE_PROJECT_NAME)
-write_basic_package_version_file(${CMAKE_BINARY_DIR}/cmake/${CMAKE_PROJECT_NAME}ConfigVersion.cmake
-                                 VERSION ${MATERIALX_LIBRARY_VERSION}
-                                 COMPATIBILITY AnyNewerVersion)
-
-# Install the auto-generated CMake configuration files:
-
-install(EXPORT MaterialX
-        DESTINATION "${MATERIALX_GEN_CONFIG_PATH}"
-        FILE ${CMAKE_PROJECT_NAME}Targets.cmake)
-
-install(FILES "${CMAKE_BINARY_DIR}/cmake/${CMAKE_PROJECT_NAME}ConfigVersion.cmake"
-        "${CMAKE_BINARY_DIR}/cmake/${CMAKE_PROJECT_NAME}Config.cmake"
-        DESTINATION "${MATERIALX_GEN_CONFIG_PATH}")
+if(NOT SKBUILD)
+    install(FILES LICENSE CHANGELOG.md README.md THIRD-PARTY.md
+            DESTINATION .)
+
+    set(MATERIALX_GEN_CONFIG_PATH "${MATERIALX_INSTALL_LIB_PATH}/cmake/${CMAKE_PROJECT_NAME}")
+
+    include(CMakePackageConfigHelpers)
+    configure_package_config_file(cmake/modules/MaterialXConfig.cmake.in
+                                ${CMAKE_BINARY_DIR}/cmake/${CMAKE_PROJECT_NAME}Config.cmake
+                                INSTALL_DESTINATION "${MATERIALX_GEN_CONFIG_PATH}"
+                                PATH_VARS CMAKE_INSTALL_PREFIX CMAKE_PROJECT_NAME)
+    write_basic_package_version_file(${CMAKE_BINARY_DIR}/cmake/${CMAKE_PROJECT_NAME}ConfigVersion.cmake
+                                    VERSION ${MATERIALX_LIBRARY_VERSION}
+                                    COMPATIBILITY AnyNewerVersion)
+
+    # Install the auto-generated CMake configuration files:
+
+    install(EXPORT MaterialX
+            DESTINATION "${MATERIALX_GEN_CONFIG_PATH}"
+            FILE ${CMAKE_PROJECT_NAME}Targets.cmake)
+
+    install(FILES "${CMAKE_BINARY_DIR}/cmake/${CMAKE_PROJECT_NAME}ConfigVersion.cmake"
+            "${CMAKE_BINARY_DIR}/cmake/${CMAKE_PROJECT_NAME}Config.cmake"
+            DESTINATION "${MATERIALX_GEN_CONFIG_PATH}")
+endif()
diff --git a/libraries/CMakeLists.txt b/libraries/CMakeLists.txt
@@ -1,20 +1,26 @@
-install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/
-        DESTINATION "${MATERIALX_INSTALL_STDLIB_PATH}" MESSAGE_NEVER
-        PATTERN "CMakeLists.txt" EXCLUDE
-        PATTERN "pbrlib_genosl_impl.*" EXCLUDE)
-
 if (MATERIALX_OSL_LEGACY_CLOSURES)
     set(PBRLIB_SUFFIX "legacy")
 else()
     set(PBRLIB_SUFFIX "mtlx")
 endif()
 
-install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/pbrlib/genosl/pbrlib_genosl_impl.${PBRLIB_SUFFIX}"
-        DESTINATION "${MATERIALX_INSTALL_STDLIB_PATH}/pbrlib/genosl/" RENAME pbrlib_genosl_impl.mtlx)
+if(NOT SKBUILD)
+    install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/
+            DESTINATION "${MATERIALX_INSTALL_STDLIB_PATH}"
+            PATTERN "CMakeLists.txt" EXCLUDE
+            PATTERN "pbrlib_genosl_impl.*" EXCLUDE)
+    install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/pbrlib/genosl/pbrlib_genosl_impl.${PBRLIB_SUFFIX}"
+            DESTINATION "${MATERIALX_INSTALL_STDLIB_PATH}/pbrlib/genosl/" RENAME pbrlib_genosl_impl.mtlx)
+endif()
+
+set(MATERIALX_PYTHON_LIBRARIES_PATH "${MATERIALX_PYTHON_FOLDER_NAME}/${MATERIALX_INSTALL_STDLIB_PATH}")
+if(SKBUILD)
+    set(MATERIALX_PYTHON_LIBRARIES_PATH "${SKBUILD_PLATLIB_DIR}/MaterialX/libraries")
+endif()
 
 install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/
-        DESTINATION "python/MaterialX/${MATERIALX_INSTALL_STDLIB_PATH}" 
+        DESTINATION "${MATERIALX_PYTHON_LIBRARIES_PATH}"
         PATTERN "CMakeLists.txt" EXCLUDE
         PATTERN "pbrlib_genosl_impl.*" EXCLUDE)
 install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/pbrlib/genosl/pbrlib_genosl_impl.${PBRLIB_SUFFIX}"
-        DESTINATION "python/MaterialX/${MATERIALX_INSTALL_STDLIB_PATH}/pbrlib/genosl/" RENAME pbrlib_genosl_impl.mtlx)
+        DESTINATION "${MATERIALX_PYTHON_LIBRARIES_PATH}/pbrlib/genosl/" RENAME pbrlib_genosl_impl.mtlx)
diff --git a/pyproject.toml b/pyproject.toml
@@ -0,0 +1,78 @@
+[build-system]
+# Use a fixed version because we use an experimental feature
+# (a custom plugin) and for now that functionality has
+# no compatibility promises.
+requires = ["scikit-build-core==0.4.4"]
+build-backend = "scikit_build_core.build"
+
+[project]
+name = "MaterialX"
+dynamic = ["version"]
+
+authors = [
+  { name="Contributors to the MaterialX project", email="[email protected]" },
+]
+readme = "README.md"
+requires-python = ">=3.6"
+
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: Apache Software License",
+    "Operating System :: OS Independent",
+]
+
+[project.urls]
+"Homepage" = "https://materialx.org"
+"Source" = "https://github.com/AcademySoftwareFoundation/MaterialX"
+"Bug Tracker" = "https://github.com/AcademySoftwareFoundation/MaterialX/issues"
+
+[project.scripts]
+baketextures = "MaterialX._scripts.baketextures:main"
+generateshader = "MaterialX._scripts.generateshader:main"
+genmdl = "MaterialX._scripts.genmdl:main"
+mxdoc = "MaterialX._scripts.mxdoc:main"
+mxupdate = "MaterialX._scripts.mxupdate:main"
+mxvalidate = "MaterialX._scripts.mxvalidate:main"
+translateshader = "MaterialX._scripts.translateshader:main"
+writenodegraphs = "MaterialX._scripts.writenodegraphs:main"
+
+[tool.scikit-build]
+cmake.minimum-version = "3.18"
+cmake.verbose = false
+cmake.build-type = "Release"
+
+# Enable experimental features if any are available
+# In this case we need custom local plugin to get
+# the project version from cmake.
+experimental = true
+metadata.version.provider = "mtx_skbuild_plugin"
+metadata.version.provider-path = "./python"
+
+# Uncoment when developing locally to enable inplace builds.
+# build-dir = "build/"
+
+logging.level = "DEBUG"
+
+# Since the python package doesn't live in a standard directory
+# in the source (i.e ./src or ./), we need to manually specify
+# where the package is.
+wheel.packages = ["python/MaterialX"]
+
+sdist.exclude = [
+    "/build",
+    "/dist",
+    "/resources",
+    "/javascript",
+    "/documents",
+    "/.github",
+    "MANIFEST.in"
+]
+
+[tool.scikit-build.cmake.define]
+MATERIALX_BUILD_SHARED_LIBS = 'OFF' # Be explicit
+MATERIALX_BUILD_PYTHON = 'ON'
+MATERIALX_TEST_RENDER = 'OFF'
+MATERIALX_WARNINGS_AS_ERRORS = 'ON'
+MATERIALX_BUILD_TESTS = 'OFF'
+# TODO: How could we harmonize this variable with SKBUILD?
+MATERIALX_INSTALL_PYTHON = 'OFF'
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
@@ -1,18 +1,25 @@
-set(SETUP_PY_IN "${CMAKE_CURRENT_SOURCE_DIR}/setup.py.in")
-set(SETUP_PY "${CMAKE_INSTALL_PREFIX}/python/setup.py")
-
-configure_file(${SETUP_PY_IN} ${SETUP_PY})
+if(NOT SKBUILD)
+  install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/MaterialX" DESTINATION "python" MESSAGE_NEVER)
+  install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/Scripts" DESTINATION "python" MESSAGE_NEVER)
+endif()
 
-install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/MaterialX" DESTINATION "python" MESSAGE_NEVER)
-install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/Scripts" DESTINATION "python" MESSAGE_NEVER)
+if(SKBUILD)
+  install(
+    DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/Scripts/"
+    DESTINATION "${MATERIALX_PYTHON_FOLDER_NAME}/_scripts"
+    PATTERN "README.md" EXCLUDE
+  )
+endif()
 
 if(MATERIALX_PYTHON_OCIO_DIR)
   if(NOT EXISTS "${MATERIALX_PYTHON_OCIO_DIR}/config.ocio")
     message(WARNING "No file named config.ocio was found in the given OCIO directory.")
   endif()
-  install(DIRECTORY "${MATERIALX_PYTHON_OCIO_DIR}/" DESTINATION "python/MaterialX/config/" MESSAGE_NEVER)
+  install(DIRECTORY "${MATERIALX_PYTHON_OCIO_DIR}/" DESTINATION "${MATERIALX_PYTHON_FOLDER_NAME}/config/" MESSAGE_NEVER)
 endif()
 
-if(MATERIALX_INSTALL_PYTHON AND PYTHON_EXECUTABLE)
-  install(CODE "execute_process(COMMAND ${PYTHON_EXECUTABLE} ${SETUP_PY} install clean --all)" MESSAGE_NEVER)
+if(MATERIALX_INSTALL_PYTHON AND PYTHON_EXECUTABLE AND NOT SKBUILD)
+  set(SETUP_PY "${CMAKE_INSTALL_PREFIX}/python/setup.py")
+  configure_file("${CMAKE_CURRENT_SOURCE_DIR}/setup.py.in" "${SETUP_PY}")
+  install(CODE "execute_process(COMMAND ${PYTHON_EXECUTABLE} ${SETUP_PY} install clean --all)")
 endif()
diff --git a/python/MaterialX/_scripts/README.md b/python/MaterialX/_scripts/README.md
@@ -0,0 +1,2 @@
+This directory is empty buit it's used when packaging the Python library.
+the files in ../../Scripts will be copied inside.
diff --git a/python/MaterialX/_scripts/__init__.py b/python/MaterialX/_scripts/__init__.py
@@ -0,0 +1 @@
+# Only required for entry-points.