chevah_build

#!/usr/bin/env bash
#
# Chevah Build Script for Python.
#
# build
# test
# compat (for the compat repo tests)

# Bash checks
set -o nounset    # always check if variables exist
set -o errexit    # always exit on error
set -o errtrace   # trap errors in functions as well
set -o pipefail   # don't ignore exit codes when piping output

PYTHON_BUILD_VERSION="2.7.18"
LIBFFI_VERSION="3.4.4"
ZLIB_VERSION="1.3"
BZIP2_VERSION="1.0.8"
# We statically build the BSD libedit on selected platforms to get the
# readline module available without linking to the GPL-only readline libs.
LIBEDIT_VERSION="20170329-3.1"
OPENSSL_VERSION="1.1.1w"
SQLITE_VERSION="3.43.1"

# Python modules versions to be used everywhere possible.
PYSQLITE_VERSION="2.8.3"
CFFI_VERSION="1.15.1"
SCANDIR_VERSION="1.10.0"
PSUTIL_VERSION="5.9.5"
SUBPROCESS32_VERSION="3.5.4"

# Versions no longer upgradable because of Python 2 deprecation.
# pyOpenSSL 19.1.0 is used with OpenSSL 1.0.2 libs.
PYOPENSSL_VERSION="21.0.0"
# Backported fix for https://github.com/pypa/pip/issues/9827
# at https://github.com/chevah/pip/tree/20.3.4chevah.
PIP_VERSION="20.3.4chevah1"
# For safety alerts, we need to ignore some vulnerabilities which are either:
#   * not present in the final tarball, e.g. for wheel, safety, etc.,
#   * not at all relevant, e.g. those for cryptography's bundled openssl,
#   * not actually relevant for these old versions, e.g. 53048 for cryptography,
#   * patched by us, e.g. 40291 for pip, 53048 for cryptography,
#   * not patched: 52495 for setuptools.
# pip <21.1, click <8, dparse <0.5.2, wheel <0.38, safety <2.2, pywin32 <301.
SAFETY_IGNORED_OPTS="-i 40291 -i 47833 -i 50571 -i 51499 -i 51358 -i 54687"
# setuptools <65.5.1, requests <2.31.0, certifi <2023.07.22.
SAFETY_IGNORED_OPTS="$SAFETY_IGNORED_OPTS -i 52495 -i 58755 -i 52365 -i 59956"
# These are related to cryptography's bundled OpenSSL libs. We don't use those.
SAFETY_IGNORED_OPTS="$SAFETY_IGNORED_OPTS -i 53306 -i 53298 -i 53305 -i 53301"
SAFETY_IGNORED_OPTS="$SAFETY_IGNORED_OPTS -i 53307 -i 53304 -i 53302 -i 53299"
SAFETY_IGNORED_OPTS="$SAFETY_IGNORED_OPTS -i 53303 -i 59062 -i 60225 -i 60223"
SAFETY_IGNORED_OPTS="$SAFETY_IGNORED_OPTS -i 60224"
# Other cryptography vulnerabilities, see a few lines above for more details.
SAFETY_IGNORED_OPTS="$SAFETY_IGNORED_OPTS -i 53048 -i 59473"
# setuptools 44.x is the last series to support Python 2.7.
# More at https://github.com/pypa/setuptools/pull/1955.
SETUPTOOLS_VERSION="44.1.1"
# Version 3.2.1 (with patches) from python-modules/ is used with OpenSSL 1.0.2.
# Our patched versions fix CVE-2023-23931.
CRYPTOGRAPHY_VERSION="3.3.2chevah"
# bcrypt 3.2.0 requires at least Python 3.6.
BCRYPT_VERSION="3.1.7"
# setproctitle 1.2.x requires at least Python 3.6.
SETPROCTITLE_VERSION="1.1.10"
# pywin32 300 requires at least Python 3.5.
PYWIN32_VERSION="228"

# pycparser is explicitly installed to work around setuptools auto dependencies.
PYCPARSER_VERSION="2.21"

# Current revision for the VC++ 9.0 redistributable version used on Windows.
REDISTRIBUTABLE_VERSION="9.0.30729.9518"

# Git revision to inject into Python's sys.version string through chevahbs.
PYTHON_PACKAGE_VERSION="$(git rev-parse --short=7 HEAD)"

# Export the variables needed by the chevahbs scripts and the test phase.
export PYTHON_BUILD_VERSION PYTHON_PACKAGE_VERSION REDISTRIBUTABLE_VERSION
export BUILD_ZLIB="no"
export BUILD_BZIP2="yes"
export BUILD_LIBEDIT="yes"
export BUILD_LIBFFI="no"
export BUILD_OPENSSL="no"

# Patches for EXTRA_LIBRARIES modules should reside in a sub-directory named
# "python-modules/lib-${LIB_VERSION}-patches", to be applied when needed
# during the build. See the patches for PyCrypto 2.6.1 for an example.
EXTRA_LIBRARIES="\
    python-modules/cffi-${CFFI_VERSION} \
    "
# List of default Python modules installed using pip.
PIP_LIBRARIES="\
    cryptography==${CRYPTOGRAPHY_VERSION} \
    pyOpenSSL==${PYOPENSSL_VERSION} \
    scandir==${SCANDIR_VERSION} \
    subprocess32==${SUBPROCESS32_VERSION} \
    bcrypt==${BCRYPT_VERSION} \
    psutil==${PSUTIL_VERSION} \
    setproctitle==${SETPROCTITLE_VERSION}
    "
# cryptography 3.2.1 was last upstream version to support OpenSSL 1.0.2.
# Don't list cryptography here though. When building on AIX with its compiler,
# it won't link correctly to non-system OpenSSL libs. Use EXTRA_LIBRARIES.
# PyOpenSSL 19.1.0 was last upstream version to support OpenSSL 1.0.2.
PIP_LIBRARIES_OPENSSL_102="\
    pyOpenSSL==19.1.0 \
    scandir==${SCANDIR_VERSION} \
    subprocess32==${SUBPROCESS32_VERSION} \
    bcrypt==${BCRYPT_VERSION} \
    psutil==${PSUTIL_VERSION} \
    setproctitle==${SETPROCTITLE_VERSION}
    "

PYPI_SITE="https://bin.chevah.com:20443/pypi"
# Arguments that are sent when using pip.
PIP_ARGS="\
    --index-url=${PYPI_SITE}/simple \
    --no-cache-dir \
    --no-warn-script-location \
    "

PROG=$0
DIST_FOLDER='dist'
BUILD_FOLDER='build'

# Import shared code.
. ./functions.sh

# Import build environment variables found through "detect_os" command option.
if [ -f ./BUILD_ENV_VARS ]; then
    . ./BUILD_ENV_VARS
else
    (>&2 echo "Missing BUILD_ENV_VARS file!")
    echo "Run './brink.sh detect_os' first, as per the README file..."
    exit 100
fi

# List of OS packages required for building Python/pyOpenSSL/cryptography etc.
# Lately we don't install anything automatically, we just check for the
# presence of required packages. Or at least of the required commands...
# This build of Python / pyOpenSSL / cryptography / etc.  requires:
# a C compiler, make, m4, libs and headers for OpenSSL / zlib / libffi,
# git (for patching Python's version), patch (for applying our own patches).
# To build libedit for the readline module, we need the headers of
# a curses library, automake and libtool.
# On platforms with a choice of C compilers, you may choose among the
# available compilers by setting CC and CXX further down in this script.
COMMON_PKGS="gcc make m4 automake libtool patch"
DEB_PKGS="$COMMON_PKGS git libffi-dev zlib1g-dev libncurses5-dev libssl-dev"
RPM_PKGS="$COMMON_PKGS git libffi-devel zlib-devel ncurses-devel openssl-devel"
# No automated Windows package management, but here's what it's needed.
CHOCO_PKGS="vcpython27 make git StrawberryPerl nasm 7zip"

# $ARCH can be used to force a 32bit build on a 64bit machine, e.g. by exporting
# ARCH in brink.sh as "x86" instead of "x64", or "ppc" instead of "ppc64".
# $ARCH is also used when statically building libffi and for testing.
# $OS is used when patching/configuring/building/testing.
export ARCH
export OS

LOCAL_PYTHON_BINARY_DIST="$PYTHON_VERSION-$OS-$ARCH"
INSTALL_FOLDER=$PWD/${BUILD_FOLDER}/$LOCAL_PYTHON_BINARY_DIST
PYTHON_BIN=$INSTALL_FOLDER/bin/python
PYTHON_BUILD_FOLDER="$PYTHON_VERSION-$OS-$ARCH"

# Used when building cffi.
export CHEVAH_BUILD_PATH=$INSTALL_FOLDER

# CXX is not really needed, we export it so that g++ won't get picked up when
# not using gcc, and thus silence the associated configure warning on stderr.
case $OS in
    win)
        # On Windows, python executable is installed at a different path.
        PYTHON_BIN=$INSTALL_FOLDER/lib/python
        # For Windows we don't build everything from source yet.
        # But what we build is done in a very different way, no CC/MAKE/etc.
        export BUILD_LIBEDIT="no"
        export BUILD_BZIP2="no"
        # MSYS2's Perl is not good enough for building OpenSSL.
        export PATH="/c/Strawberry/perl/bin/:$PATH:/c/Program Files/NASM/"
        export BUILD_OPENSSL="yes"
        # Python modules are installed only using PIP.
        EXTRA_LIBRARIES=""
        PIP_LIBRARIES="$PIP_LIBRARIES \
            pywin32==${PYWIN32_VERSION} \
            "
        ;;
    aix*)
        # Only IBM's XL C compiler is supported.
        export CC="xlc_r"
        export CXX="xlC_r"
        export MAKE="gmake"
        export PATH="/usr/vac/bin:$PATH"
        export CFLAGS="${CFLAGS:-} -O2 -D_LARGE_FILES=1"
        # IBM's OpenSSL libs are mixed 32/64bit binaries in AIX, so we need to
        # be specific about what kind of build we want, because otherwise we
        # might get 64bit libraries.
        if [ "${ARCH%64}" = "$ARCH" ]; then
            export OBJECT_MODE="32"
            export ABI="32"
            export AR="ar -X32"
            if [ "${CC}" != "gcc" ]; then
                export CFLAGS="$CFLAGS -qmaxmem=16384 -q32"
            fi
        else
            export OBJECT_MODE="64"
            export ABI="mode64"
            export AR="ar -X64"
            if [ "${CC}" != "gcc" ]; then
                export CFLAGS="$CFLAGS -qmaxmem=16384 -q64"
            fi
        fi
        export BUILD_LIBFFI="yes"
        # There are zlib system libs, but we build as portable as possible now.
        export BUILD_ZLIB="yes"
        # libedit requires __STDC_ISO_10646__.
        export BUILD_LIBEDIT="no"
        # As of January 2021, OpenSSL 1.0.2u is the latest version from IBM.
        export BUILD_OPENSSL="yes"
        # 1.1.1 tests fail on AIX, use 1.0.2 with patches from CentOS 7.
        OPENSSL_VERSION="1.0.2v-chevah5"
        # Perl's Test::Simple and its deps are required for building OpenSSL.
        execute perl -MTest::Simple -e 1
        # cryptography 3.2.x, last version to support OpenSSL 1.0.2.
        export CRYPTOGRAPHY_ALLOW_OPENSSL_102="yes"
        PIP_LIBRARIES=$PIP_LIBRARIES_OPENSSL_102
        # cryptography 3.2.1 builds fine against non-system OpenSSL 1.0.2 libs,
        # but linking is not done right when building with xlc 12.1 through pip.
        EXTRA_LIBRARIES="$EXTRA_LIBRARIES \
            python-modules/cryptography-3.2.1 \
        "
        add_ignored_safety_ids_for_cryptography32
        # On AIX 7.1, cryptography build fails with Let's Encrypt certificates.
        # This is enough to fix it, no need to redefine PIP_ARGS as well.
        PYPI_SITE="http://bin.chevah.com:20080/pypi"
        ;;
    sol*)
        # Only Sun's Studio compiler is supported.
        export CC="cc"
        export CXX="CC"
        export MAKE="gmake"
        # Needed for the subprocess32 module.
        # More at https://github.com/google/python-subprocess32/issues/40.
        export CFLAGS="$CFLAGS -DHAVE_DIRFD"
        # Arch-specific bits and paths.
        export CFLAGS="$CFLAGS -m64"
        export LDFLAGS="$LDFLAGS -m64 -L/usr/lib/64 -R/usr/lib/64"
        # System includes bzip2 libs by default.
        export BUILD_BZIP2="no"
        # Solaris 11.4 has multiple system libffi libs present.
        export BUILD_LIBFFI="yes"
        # OpenSSL 1.0.2 has extended support: https://tinyurl.com/2ck2sm6s.
        export CRYPTOGRAPHY_ALLOW_OPENSSL_102="yes"
        # Build cryptography against system OpenSSL with our patches.
        # cryptography 3.2.1, last version working with OpenSSL 1.0.2.
        EXTRA_LIBRARIES="$EXTRA_LIBRARIES \
            python-modules/cryptography-3.2.1 \
        "
        # Use the appropriate PIP_LIBRARIES env var.
        PIP_LIBRARIES="$PIP_LIBRARIES_OPENSSL_102"
        add_ignored_safety_ids_for_cryptography32
        ;;
    macos)
        export CC="clang"
        export CXX="clang++"
        export MAKE="make"
        # Build as compatible as it makes sense. See brink.sh for the reason.
        export CFLAGS="${CFLAGS:-} -mmacosx-version-min=10.12"
        # setup.py skips building readline by default, as it sets this to
        # "10.4", and then tries to avoid the broken readline in OS X 10.4.
        export MACOSX_DEPLOYMENT_TARGET=10.12
        # System included bzip2 libs by default up to and including macOS 10.15.
        export BUILD_BZIP2="yes"
        # Apparently, macOS 11 doesn't include zlib libraries either.
        export BUILD_ZLIB="yes"
        # Building readline fails on macOS 11, didn't look into it.
        export BUILD_LIBEDIT="no"
        # 10.13 and newer come with LibreSSL instead of the old OpenSSL libs.
        # But 10.13 has version 2.2.7, while cryptography 2.9 requires 2.7.
        # Therefore, we build OpenSSL for both stdlib and cryptography.
        export BUILD_OPENSSL="yes"
        # Perl's Test::Simple and its deps are required for building OpenSSL.
        execute perl -MTest::Simple -e 1
        ;;
    fbsd*)
        export CC="clang"
        export CXX="clang++"
        export MAKE="make"
        # libffi not available in the base system, only as port/package.
        export BUILD_LIBFFI="yes"
        # System includes bzip2 libs by default.
        export BUILD_BZIP2="no"
        # Build scandir through EXTRA_LIBRARIES, to apply a needed patch.
        PIP_LIBRARIES="\
            cryptography==${CRYPTOGRAPHY_VERSION} \
            pyOpenSSL==${PYOPENSSL_VERSION} \
            subprocess32==${SUBPROCESS32_VERSION} \
            bcrypt==${BCRYPT_VERSION} \
            psutil==${PSUTIL_VERSION} \
            setproctitle==${SETPROCTITLE_VERSION}
            "
        EXTRA_LIBRARIES="$EXTRA_LIBRARIES \
            python-modules/scandir-${SCANDIR_VERSION} \
            "
        ;;
    obsd*)
        export CC="clang"
        export CXX="clang++"
        export MAKE="make"
        # libffi not available in the base system, only as port/package.
        export BUILD_LIBFFI="yes"
        ;;
    lnx*)
        export CC="gcc"
        export CXX="g++"
        export MAKE="make"
        # Build as portable as possible, only glibc 2.x should be needed.
        export BUILD_LIBFFI="yes"
        export BUILD_ZLIB="yes"
        export BUILD_LIBEDIT="no"
        # Generic Linux might be an old distro with OpenSSL 0.9.8 libraries.
        # To avoid linking to local libs, we build our own OpenSSL libs.
        export BUILD_OPENSSL="yes"
        # OpenSSL tests on CentOS 5 require an updated Perl 5.10 in /usr/local.
        export PATH="/usr/local/bin:$PATH"
        # In particular, Perl's Test::Simple and its deps are required.
        execute perl -MTest::Simple -e 1
        ;;
    *)
        # Only supported Linux distributions should be left.
        export CC="gcc"
        export CXX="g++"
        export MAKE="make"
        ;;
esac

# Compiler-dependent flags. At this moment we should know what compiler is used.
if [ "${OS%sol*}" = "" ]; then
    if [ ${ARCH} = "sparc64" ]; then
        # Required for compiling GMP on Solaris for SPARC with Sun Studio.
        export CFLAGS="${CFLAGS:-} -xcode=abs64"
    elif [ ${ARCH} = "x64" -a "${OS%sol11*}" = "" ]; then
        # Not all packages enable PIC, we force it to avoid relocation issues.
        export CFLAGS="${CFLAGS:-} -Kpic"
    fi
elif [ "${OS%fbsd*}" = "" -o "${OS%obsd*}" = "" ]; then
    # Use PIC (Position Independent Code) on FreeBSD and OpenBSD with Clang.
    export CFLAGS="${CFLAGS:-} -fPIC"
elif [ "${CC:-}" = "gcc" -a ${ARCH%%64} != "$ARCH" ]; then
    # Use PIC (Position Independent Code) with GCC on 64-bit arches.
    export CFLAGS="${CFLAGS:-} -fPIC"
fi

# Parallel builds where applicable.
get_number_of_cpus
JOBS=1
case "$ARCH" in
    sparc*)
        # Twice the number of physical CPUs is optimal on SPARC machines.
        let JOBS=2*CPUS
        ;;
    *)
        # On other virtual and physical machines this is close to optimum.
        let JOBS=CPUS
        ;;
esac
# Only add to $MAKE if there is one, so that NMAKE on Windows is not confused.
# But not on AIX, where there are random errors w/ parallel builds on IBM Cloud.
if [ -n "${MAKE-}" -a "${OS%aix*}" != "" ]; then
    export MAKE="$MAKE -j${JOBS}"
fi


#
# Check for OS packages required to build Python.
#
check_dependencies() {

    packages=""
    missing_packages=""
    check_command="command -v"

    case $OS in
        # Debian-derived distros are similar in this regard.
        ubuntu*)
            packages=$DEB_PKGS
            check_command="dpkg --status"
            ;;
        rhel*|amzn*)
            packages=$RPM_PKGS
            check_command="rpm --query"
            ;;
        # On remaining OS'es we just check for some of the needed commands.
        lnx*)
            # Generic Linux builds need Perl 5.10.0+ for building OpenSSL.
            # For testing OpenSSL, Test::More 0.96 or newer is needed.
            packages="$CC make m4 git patch perl"
            ;;
        aix*)
            # /usr/bin/patch is not good enough, GNU patch is needed.
            # seq is from coreutils RPM.
            packages="git gmake patch xlc_r seq sudo"
            ;;
        macos)
            packages="$CC make m4 libtool git patch perl"
            ;;
        win)
            # To not get confused by MSYS2's perl, we check for wperl.
            packages="git patch wperl nasm 7z"
            ;;
        *)
            packages="$CC make m4 git patch"
            ;;
    esac

    if [ -n "$packages" ]; then
        echo "Checking for required packages or commands..."
        for package in $packages ; do
            echo "Checking if $package is available..."
            $check_command $package
            if [ $? -ne 0 ]; then
                echo "Missing required dependency: $package"
                missing_packages="$missing_packages $package"
            fi
        done
    fi

    if [ -n "$missing_packages" ]; then
        (>&2 echo "Missing required dependencies: $missing_packages.")
        exit 101
    fi
    if [ -n "$packages" ]; then
        echo "All required dependencies are present: $packages"
    fi

    # Many systems don't have this installed and we don't really need it.
    command -v makeinfo
    if [ $? -ne 0 ]; then
        (>&2 echo "Missing makeinfo, trying to link it to /bin/true in ~/bin.")
        execute mkdir -p ~/bin
        execute rm -f ~/bin/makeinfo
        execute ln -s /bin/true ~/bin/makeinfo
        export PATH=$PATH:~/bin/
    fi
}


help_text_clean="Clean the build."
command_clean() {
    if [ -e ${BUILD_FOLDER} ]; then
        echo 'Previous build sub-directory found. Removing...'
        execute rm -rf ${BUILD_FOLDER}
    fi
}

help_text_build="Create the Python binaries for current OS."
command_build() {
    check_dependencies

    # Clean the build dir to avoid contamination from previous builds.
    command_clean

    echo "Building version ${PYTHON_BUILD_VERSION}.${PYTHON_PACKAGE_VERSION}..."
    # Build stuff statically on most platforms, install headers and libs in the
    # following locations and make sure they are picked up when building Python.
    # We used to add the new include path to $CPPFLAGS, but it's not as portable
    # as copying the includes (HP-UX's linker fails with -I when not using GCC).
    mkdir -p $INSTALL_FOLDER/{include,lib}
    export LDFLAGS="-L${INSTALL_FOLDER}/lib/ ${LDFLAGS:-}"
    export PKG_CONFIG_PATH="$INSTALL_FOLDER/lib/pkgconfig/:${PKG_CONFIG_PATH:-}"

    if [ "$BUILD_LIBFFI" = "yes" ]; then
        build 'libffi' "libffi-$LIBFFI_VERSION" ${PYTHON_BUILD_FOLDER}
    else
        (>&2 echo "Skip building LIBFFI!")
    fi

    if [ "$BUILD_ZLIB" = "yes" ]; then
        build 'zlib' "zlib-$ZLIB_VERSION" ${PYTHON_BUILD_FOLDER}
    else
        (>&2 echo "Skip building ZLIB!")
    fi

    if [ "$BUILD_BZIP2" = "yes" ]; then
        build 'bzip2' "bzip2-$BZIP2_VERSION" ${PYTHON_BUILD_FOLDER}
    else
        (>&2 echo "Skip building BZIP2!")
    fi

    if [ "$BUILD_LIBEDIT" = "yes" ]; then
        build 'libedit' "libedit-$LIBEDIT_VERSION" ${PYTHON_BUILD_FOLDER}
    else
        (>&2 echo "Skip building LIBEDIT!")
    fi

    if [ "$BUILD_OPENSSL" = "yes" ]; then
        # The Windows build needs it.
        export OPENSSL_VERSION
        build 'openssl' "openssl-$OPENSSL_VERSION" ${PYTHON_BUILD_FOLDER}
        if [ "${OS%aix*}" = "" ]; then
            export LDFLAGS="-L${INSTALL_FOLDER}/lib/ ${LDFLAGS}"
            export PKG_CONFIG_PATH="\
                ${INSTALL_FOLDER}/lib/pkgconfig/:${PKG_CONFIG_PATH}"
        elif [ "${OS%lnx*}" = "" ]; then
            # Passing -rpath to ldd is needed for building cryptography w/ pip.
            export LDFLAGS="-Wl,-rpath,${INSTALL_FOLDER}/lib/ ${LDFLAGS}"
            export PKG_CONFIG_PATH="\
                ${INSTALL_FOLDER}/lib64/pkgconfig/:${PKG_CONFIG_PATH}"
        fi
        # On Windows, the BATs from pyca/infra do the install part.
        if [ "${OS%win*}" != "" ]; then
            # To make sure they're found, we're back to using CPPFLAGS.
            export CPPFLAGS="${CPPFLAGS:-} -I${INSTALL_FOLDER}/include"
        fi
    else
        (>&2 echo "Skip building OpenSSL!")
    fi

    case $OS in
        win)
            build 'python' "Python-$PYTHON_BUILD_VERSION-windows" \
                ${PYTHON_BUILD_FOLDER}
            ;;
        macos)
            # The headers for zlib are missing on macOS 10.14+.
            # Required CPPFLAGS are already set after building OpenSSL.
            echo "Copying zlib $ZLIB_VERSION headers..."
            execute cp src/zlib/zlib-${ZLIB_VERSION}/*.h \
                $INSTALL_FOLDER/include/
            build 'python' "Python-$PYTHON_BUILD_VERSION" ${PYTHON_BUILD_FOLDER}
            ;;
        *)
            build 'python' "Python-$PYTHON_BUILD_VERSION" ${PYTHON_BUILD_FOLDER}
            ;;
    esac

    aix_ld_hack init
    command_build_sqlite
    command_build_python_extra_libraries
    aix_ld_hack cleanup
    cleanup_install_dir

    make_dist 'python' ${PYTHON_BUILD_FOLDER}

    # Generate a SFTP batch for uploading the package through GitHub actions.
    build_publish_dist_sftp_batch
}


#
# Build pysqlite with static linked SQLite.
#
command_build_sqlite() {

    if [ $OS = 'win' ]; then
        echo "Updating SQLite DLL on Windows..."
        execute cp src/sqlite/sqlite3-${SQLITE_VERSION}-${ARCH}.dll \
            ${INSTALL_FOLDER}/lib/DLLs/sqlite3.dll
        return
    fi

    target_folder=${BUILD_FOLDER}/pysqlite

    initialize_python_module \
        "python-modules/pysqlite-$PYSQLITE_VERSION" $target_folder

    # Bring SQLite headers and source handy.
    execute cp src/sqlite/SQLite-${SQLITE_VERSION}/*.h \
        ${INSTALL_FOLDER}/include/
    execute cp src/sqlite/SQLite-${SQLITE_VERSION}/sqlite3.c ${target_folder}/

    # Build and install.
    execute pushd $target_folder
        execute $PYTHON_BIN setup.py build_static
        execute $PYTHON_BIN setup.py install
    execute popd

}


#
# Compile and install all Python extra libraries.
#
command_build_python_extra_libraries() {

    # Install the latest PIP and setuptools.
    # But first patch it to get the exact version.
    # get-pip will always try to get the latest version, so we ask it to not
    # install things and then we manually pass what we want to install.
    execute $PYTHON_BIN python-modules/get-pip.py $PIP_ARGS \
        pip==$PIP_VERSION --no-setuptools setuptools==$SETUPTOOLS_VERSION

    # pycparser is installed first as setup_requires is ugly.
    # https://pip.pypa.io/en/stable/reference/pip_install/#controlling-setup-requires
    execute $PYTHON_BIN -m pip \
        install $PIP_ARGS -U pycparser==$PYCPARSER_VERSION

    if [ $OS = 'win' ]; then
        echo "Skip makefile updating on Windows"
    else
        # Update Python config Makefile to use the python that we have just
        # created.
        makefile=$INSTALL_FOLDER/lib/$PYTHON_VERSION/config/Makefile
        makefile_orig=$INSTALL_FOLDER/lib/$PYTHON_VERSION/config/Makefile.orig

        execute cp $makefile $makefile_orig
        execute sed "s#^prefix=.*#prefix= $INSTALL_FOLDER#" $makefile_orig \
            > $makefile
    fi

    for library in $EXTRA_LIBRARIES ; do
        # Library is in the form pyopenssl/PyOpenssl-2.4.5
        version_folder=${library#*/}
        target_folder=${BUILD_FOLDER}/$version_folder

        # Patch the sources if there's a patches sub-dir with the right name.
        if [ -d "$library"-patches ]; then
            for patch_file in "$library"-patches/*; do
                echo "Applying patch: $patch_file"
                execute pushd "$library"
                    execute patch -p1 < ../../"$patch_file"
                execute popd
            done
        fi
        initialize_python_module $library $target_folder

        execute pushd $target_folder
            execute $PYTHON_BIN setup.py install
        execute popd

    done

    for library in $PIP_LIBRARIES ; do
        execute $PYTHON_BIN -m pip install $PIP_ARGS $library
    done

    # When done, uninstall wheel.
    execute $PYTHON_BIN -m pip uninstall --yes wheel

    if [ "$OS" = "win" ]; then
        echo "Patching pyWin32 manifests to use our redistributable version"
        wipe_manifest $INSTALL_FOLDER/lib/Lib/site-packages/win32/pythonservice.exe
        wipe_manifest $INSTALL_FOLDER/lib/Lib/site-packages/win32/perfmondata.dll
        wipe_manifest $INSTALL_FOLDER/lib/Lib/site-packages/pywin32_system32/pythoncomloader27.dll

        echo "Copy Python runtime to pyWin32 package"
        execute cp $INSTALL_FOLDER/lib/*CRT.manifest \
            $INSTALL_FOLDER/lib/Lib/site-packages/win32/
        execute cp $INSTALL_FOLDER/lib/python27.dll.manifest \
            $INSTALL_FOLDER/lib/Lib/site-packages/win32/
        execute cp $INSTALL_FOLDER/lib/python27.dll \
            $INSTALL_FOLDER/lib/Lib/site-packages/win32/
        execute cp $INSTALL_FOLDER/lib/msvc?90.dll \
            $INSTALL_FOLDER/lib/Lib/site-packages/win32/
        execute cp $INSTALL_FOLDER/lib/Lib/site-packages/pywin32_system32/*.dll \
            $INSTALL_FOLDER/lib/Lib/site-packages/win32/
    fi

}


#
# Initialize Python module for build.
#
initialize_python_module(){
    source_folder=$1
    target_folder=$2

    execute rm -rf $target_folder
    execute cp -r $source_folder $target_folder
    execute pushd $target_folder
        if [ -f setup.cfg ] ; then
            echo "[build_ext]" >> setup.cfg
            echo "include_dirs=$INSTALL_FOLDER/include" >> setup.cfg
            echo "library_dirs=$INSTALL_FOLDER/lib" >> setup.cfg

            # We want to download dependencies from our PyPI and block
            # the official.
            echo "[easy_install]" >> setup.cfg
            echo "find_links=${PYPI_SITE}/" >> setup.cfg
            echo "allow_hosts=*bin.chevah.com*" >> setup.cfg
        fi
    execute popd
}


#
# Test the newly created Python binary dist.
#
help_text_test=\
"Run a quick test for the Python from build."
command_test() {
    local test_file='test_python_binary_dist.py'
    local errors_log='stderr.log'
    local errors_unexpected='errors_unexpected.log'
    # Regexp to match messages allowed on stderr during our Python tests.
    local errors_allowed='enable-meta-key'

    execute mkdir -p build/
    execute cp python-modules/chevah-python-test/${test_file} build/
    execute cp python-modules/chevah-python-test/get_binaries_deps.sh build/
    execute pushd build

    echo '##### Executing Chevah tests (with redirected stderr)... #####'
    # Avoid 'No entry for terminal type "unknown"' warnings on stderr.
    export TERM="dumb"
    # To see the error logged on stderr, allow the following command to fail.
    $PYTHON_BIN ${test_file} 2>${errors_log}
    if [ -s "$errors_log" ]; then
        egrep -v "$errors_allowed" $errors_log > $errors_unexpected
        if [ -s $errors_unexpected ]; then
            (>&2 echo "There were unexpected errors logged to stderr:")
            execute cat $errors_unexpected
            exit 102
        fi
    fi

    echo '##### Testing for outdated packages and security issues... #####'
    execute $PYTHON_BIN -m pip list --outdated --format=columns
    # Install wheel back for better collection of needed dependencies.
    execute $PYTHON_BIN -m pip install $PIP_ARGS wheel
    # Move include/ back for building some deps, like Cython on ARM64.
    execute mv $INSTALL_FOLDER/lib/include $INSTALL_FOLDER/
    # Safety needs PyYAML, which needs Cython, which needs to be built on AIX.
    aix_ld_hack init
    # This is the newest version that still works with Python 2.7.x.
    execute $PYTHON_BIN -m pip install $PIP_ARGS safety==1.8.7
    execute $PYTHON_BIN -m safety check --full-report \
        ${SAFETY_FALSE_POSITIVES_OPTS-} ${SAFETY_IGNORED_OPTS:-}
    if [ -n "${SAFETY_FALSE_POSITIVES_OPTS-}" ]; then
        SAFETY_FALSE_POSITIVES_IDS="\
            $(echo $SAFETY_FALSE_POSITIVES_OPTS | sed s/\-i\ //g)"
        echo "Following Safety DB IDs were excepted as false positives:"
        echo "${SAFETY_FALSE_POSITIVES_IDS}"
    fi
    if [ -n "${SAFETY_IGNORED_OPTS-}" ]; then
        SAFETY_IGNORED_IDS="\
            $(echo $SAFETY_IGNORED_OPTS | sed s/\-i\ //g)"
        (>&2 echo "Following Safety DB IDs were excepted from checks:")
        (>&2 echo "${SAFETY_IGNORED_IDS}")
    fi
    aix_ld_hack cleanup
    # Avoid having output messages sometimes mangled for some reason.
    sleep 1

    echo '##### Executing tests for included Python modules... #####'

    echo 'Testing scandir...'
    # There are a series of issues with the scandir tests in Linux/Unix.
    # Reported upstream at https://github.com/benhoyt/scandir/issues/78.
    SCANDIR_FOLDER="../python-modules/scandir-${SCANDIR_VERSION}"
    case "$OS-$ARCH" in
        macos*|win*)
            execute $PYTHON_BIN ${SCANDIR_FOLDER}/test/run_tests.py
            ;;
        aix*)
            (>&2 echo -e "\tSkipping because of upstream issues.")
            ;;
        lnx*)
            if [ -f /.dockerenv ]; then
                (>&2 echo -e "\tSkipping as it fails under Docker.")
            else
                execute $PYTHON_BIN ${SCANDIR_FOLDER}/test/run_tests.py
            fi
            ;;
        *)
            # UTF-8 locale is needed for the tests to pass on remaining OS'es.
            export LC_ALL=C.UTF-8
            execute $PYTHON_BIN ${SCANDIR_FOLDER}/test/run_tests.py
            export LC_ALL=C
            ;;
    esac

    execute popd
}


#
# Test the newly created Python binary dist.
#
help_text_compat=\
"Run a full test for the chevah/compat master."
command_compat() {
    # To have a better test coverage, we run compat's tests.
    # Parts of this hackishly force compat to use the local Python version.
    execute pushd build
    echo '##### Running chevah.compat tests... #####'
    execute rm -rf compat
    execute git clone https://github.com/chevah/compat.git --depth=1 -b py2-support
    execute pushd compat
    # Copy over current brink stuff, as some changes might require it.
    execute cp ../../brink.{conf,sh} ./
    # We patch the Python version to match the one that we have just built
    # and copy it in cache so that it will be picked up by the new build.
    execute echo -e "\nPYTHON_CONFIGURATION=default@${PYTHON_BUILD_VERSION}.${PYTHON_PACKAGE_VERSION}" >> brink.conf
    # Populate the cache with the latest version so that we don't have to
    # download it.
    execute mkdir cache
    execute cp -r ../$LOCAL_PYTHON_BINARY_DIST cache/
    # Make sure everything is done from scratch in the current dir.
    unset CHEVAH_CACHE CHEVAH_BUILD
    # Some tests might fail due to causes which are not related to python.
    execute ./brink.sh deps
    if [ "${CHEVAH_CONTAINER:-}" = "yes" ]; then
        execute ./brink.sh test_ci2
    else
        execute ./brink.sh test_ci
    fi
    execute popd

    execute popd
}


help_text_publish=\
"Upload Python binaries for current OS."
command_publish() {
    execute rsync -qrlDvcz \
        ${DIST_FOLDER}/$PYTHON_VERSION/ \
        ${BINARY_DIST_PUBLISH_URI}/$PYTHON_VERSION/
}


# Launch the whole thing.
select_command $@