Dev shell tests with pytest

nix/conftest.py

@@ -0,0 +1,11 @@
+import platform
+import pytest
+
+ALL = set(["aarch64", "x86_64"])
+
+
+def pytest_runtest_setup(item):
+    supported_platforms = ALL.intersection(mark.name for mark in item.iter_markers())
+    plat = platform.machine()
+    if supported_platforms and plat not in supported_platforms:
+        pytest.skip("platform specific test; not running on {}".format(plat))

nix/pyenv.nix

@@ -35,7 +35,7 @@ let
     };
 in pkgs.python310.withPackages (ps:
   [
-    ps.pep8
+    ps.autopep8
     ps.pytest
     ps.pytest-xdist
     ps.pytest-rerunfailures

nix/pytest.ini

@@ -0,0 +1,4 @@
+[pytest]
+markers =
+    aarch64: tests that should pass on aarch64
+    x86_64: tests that should pass on x86_64

nix/shell.sh

@@ -1,6 +1,8 @@
-echo nix/shell.sh: Entering a devShell
+pyfiglet -f starwars "s2n-tls devShell"
+echo nix/shell.sh: Entering a devShell...checking environment.
 export SRC_ROOT=$(pwd)
 export PATH=$SRC_ROOT/build/bin:$PATH
+pytest -rfs -v ./nix/test_devshell.py
 
 banner()
 {
@@ -9,7 +11,6 @@ banner()
     echo "+---------------------------------------------------------+"
 }
 
-
 function clean {
     banner "Cleanup ./build"
     rm -rf ./build
@@ -116,28 +117,6 @@ function do-clang-format {
     echo $src_files | xargs -n 1 -P $(nproc) clang-format -style=file -i)
 }
 
-function test_toolchain_counts {
-    # This is a starting point for a unit test of the devShell.
-    # The choosen S2N_LIBCRYPTO should be 2, and the others should be zero.
-    banner "Checking the CMAKE_INCLUDE_PATH for libcrypto counts"
-    echo $CMAKE_INCLUDE_PATH|gawk 'BEGIN{RS=":"; o10=0; o11=0; o3=0;awslc=0;libre=0}
-      /openssl-3.0/{o3++}
-      /openssl-1.1/{o11++}
-      /openssl-1.0/{o10++}
-      /aws-lc/{awslc++}
-      /libressl/{libre++}
-      END{print "\nOpenssl3:\t",o3,"\nOpenssl1.1:\t",o11,"\nOpenssl1.0.2:\t",o10,"\nAwlc:\t\t",awslc,"\nLibreSSL:\t", libre}'
-    banner "Checking tooling counts (these should all be 1)"
-    echo -e "\nOpenssl integ:\t $(openssl version|grep -c '1.1.1')"
-    echo -e "Corretto 17:\t $(java -version 2>&1|grep -ce 'Runtime.*Corretto-17')"
-    echo -e "gnutls-cli:\t $(gnutls-cli --version |grep -c 'gnutls-cli 3.7')"
-    echo -e "gnutls-serv:\t $(gnutls-serv --version |grep -c 'gnutls-serv 3.7')"
-    echo -e "Nix Python:\t $(which python|grep -c '/nix/store')"
-    echo -e "Nix pytest:\t $(which pytest|grep -c '/nix/store')"
-    echo -e "Nix sslyze:\t $(which sslyze|grep -c '/nix/store')"
-    echo -e "python nassl:\t $(pip freeze|grep -c 'nassl')"
-}
-
 function test_nonstandard_compilation {
     # Any script that needs to compile s2n in a non-standard way can run here
     ./codebuild/bin/test_dynamic_load.sh $(mktemp -d)

nix/test_devshell.py

@@ -0,0 +1,84 @@
+import os
+import shutil
+import subprocess
+import pytest
+
+
+ALL_LCS = ["openssl-3.0", "openssl-1.1.1",
+           "openssl-1.0.2", "libressl", "aws-lc"]
+
+
+def translate_lc(lc: str):
+    # awslc for S2N_LIBCRYPTO is special.
+    if "awslc" in lc:
+        return "aws-lc"
+    else:
+        return lc
+
+
+@pytest.mark.parametrize("lc", [os.getenv("S2N_LIBCRYPTO")])
+def test_s2n_libcrypto(lc):
+    # Validate S2N_LIBCRYPTO is in the CMAKE_INCLUDE_PATH
+    assert lc is not None
+    libcrypto = translate_lc(lc)
+    include_path = os.getenv("CMAKE_INCLUDE_PATH")
+    assert include_path is not None
+    assert libcrypto in ALL_LCS
+    assert libcrypto in include_path
+
+
+@pytest.mark.parametrize("lc", [os.getenv("S2N_LIBCRYPTO")])
+def test_s2n_libcrypto_uniq(lc):
+    # Make certain we only have the preferred libcrypto in CMAKE_INCLUDE_PATH.
+    assert lc is not None
+    libcrypto = translate_lc(lc)
+    include_path = os.getenv("CMAKE_INCLUDE_PATH")
+    assert include_path is not None
+    ALL_LCS.remove(libcrypto)
+    for library in ALL_LCS:
+        assert library not in include_path
+
+
+@pytest.mark.parametrize("cmd,expected,version", [
+    ("gnutls-serv", "gnutls-serv 3.7", "--version"),
+    ("gnutls-cli", "gnutls-cli 3.7", "--version"),
+    ("openssl", "OpenSSL 1.1.1", "version"),
+    ("java", "Corretto-17", "--version")])
+def test_utility_versions(cmd, expected, version):
+    # Valildate utility is in the path and the correct version.
+    abspath = shutil.which(cmd)
+    result = ""
+    with subprocess.Popen([abspath, version], shell=False, stdout=subprocess.PIPE) as p:
+        output = p.stdout.readlines()
+        for line in output:
+            result += line.decode().strip()
+        ' '.join(result)
+        assert expected in result
+
+
+def test_python():
+    # Validate python _from nix_ is in the PATH.
+    assert 'nix' in shutil.which('python')
+
+
+def test_pytest():
+    # Validate pytest _from nix_ is in the PATH.
+    assert 'nix' in shutil.which('pytest')
+
+
+@pytest.mark.x86_64
+def test_sslyze():
+    # On x86, validate sslyze is available
+    assert 'nix' in shutil.which('sslyze')
+
+
+@pytest.mark.x86_64
+def test_nassl():
+    # On x86 validate nassl and version.
+    import nassl
+    assert nassl.__name__ == "nassl"
+    assert nassl.__version__ == "5.0.0"
+
+
+if __name__ == "__main__":
+    print("Use pytest")