automate pypi releases

.github/workflows/python_build.yml

@@ -33,9 +33,6 @@ jobs:
         restore-keys: |
           python-${{ runner.OS }}-target-incremental-
 
-    - uses: actions/setup-python@v2
-      with:
-        python-version: 3.7
     - uses: actions/setup-python@v2
       with:
         python-version: 3.8
@@ -48,11 +45,11 @@ jobs:
     - name: 'Install Python devel headers'
       run: sudo apt install -y python3-dev
 
-    - name: Install minimal nightly with clippy and rustfmt
+    - name: Install minimal stable with clippy and rustfmt
       uses: actions-rs/toolchain@v1
       with:
         profile: default
-        toolchain: nightly
+        toolchain: stable
         override: true
     - name: Check
       run: cargo clippy
@@ -61,8 +58,9 @@ jobs:
     # - name: Run Rust tests
     #   run: cargo test
 
-    - name: 'Install Python build Dependencies'
-      run: pip install maturin
+    - name: Install maturin
+      run: |
+        cargo install --git https://github.com/PyO3/maturin.git --rev 98636cea89c328b3eba4ebb548124f75c8018200 maturin
 
     - name: Run Python tests
       run: |
@@ -71,5 +69,5 @@ jobs:
         maturin build --manylinux off
         ls -lh ../target/wheels
         PY_MINOR_VER=$(python -V | cut -d. -f 2)
-        pip install $(printf ../target/wheels/deltalake_python-*-cp3${PY_MINOR_VER}-*.whl)'[devel,pandas]'
+        pip install $(printf ../target/wheels/deltalake-*-cp3${PY_MINOR_VER}-*.whl)'[devel,pandas]'
         py.test tests

.github/workflows/python_release.yml

@@ -0,0 +1,115 @@
+name: Release to PyPI
+
+on:
+  push:
+    tags: [ 'python-v*' ]
+
+defaults:
+  run:
+    working-directory: ./python
+
+jobs:
+  validate-release-tag:
+    name: Validate git tag
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+      - run: |
+          PUSHED_TAG=${GITHUB_REF##*/}
+          CURR_VER=$( grep version Cargo.toml | head -n 1 | awk '{print $3}' | tr -d '"' )
+          if [[ "${PUSHED_TAG}" != "python-v${CURR_VER}" ]]; then
+            echo "Cargo metadata has version set to ${CURR_VER}, but got pushed tag ${PUSHED_TAG}."
+            exit 1
+          fi
+
+  release-github-pypi:
+    needs: validate-release-tag
+    name: PyPI release
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - macOS-10.15
+          - macOS-11.0
+          - windows-2019
+        include:
+          - target: x86_64-apple-darwin
+            os: macOS-10.15
+          - target: x86_64-apple-darwin
+            os: macOS-11.0
+          - target: x86_64-pc-windows-msvc
+            os: windows-2019
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+          target: ${{ matrix.target }}
+
+      - name: Install matruin
+        run: |
+          cargo install --git https://github.com/PyO3/maturin.git --rev 98636cea89c328b3eba4ebb548124f75c8018200 maturin
+
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.6
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+
+      - name: Publish to pypi (without sdist)
+        env:
+          MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
+        run: maturin publish -b pyo3 --target ${{ matrix.target }} --no-sdist
+
+  release-github-pypi-manylinux:
+    needs: validate-release-tag
+    name: PyPI release manylinux
+    runs-on: ubuntu-20.04
+    # we use arrow manylinux image instead of quay.io/pypa/manylinux2014_x86_64:2020-12-25-6634688
+    # for the openssl support.
+    # for unkonwn reason, upstream manylinux container deletes openssl
+    # installations at the end of the build.
+    container: apache/arrow-dev:amd64-centos-6.10-python-manylinux2010
+    steps:
+      # actions/checkout@v2 is a node action, which runs on a fairly new
+      # version of node. however, manylinux environment's glibc is too old for
+      # that version of the node. so we will have to use v1 instead, which is a
+      # docker based action.
+      - uses: actions/checkout@v1
+
+      # actions-rs/toolchain@v1 is a node action, which runs on a fairly new
+      # version of node. however, manylinux environment's glibc is too old for
+      # that version of the node. so we will have to install rust manually here.
+      - name: Install Rust
+        run: |
+          curl https://sh.rustup.rs -sSf | sh -s -- -y
+          $HOME/.cargo/bin/rustup default stable
+          echo "$HOME/.cargo/bin" >> $GITHUB_PATH
+
+      - name: Install matruin
+        run: |
+          cargo install --git https://github.com/PyO3/maturin.git --rev 98636cea89c328b3eba4ebb548124f75c8018200 maturin
+
+      - name: Enable manylinux Python targets
+        run: |
+            echo "/opt/python/cp36-cp36m/bin" >> $GITHUB_PATH
+            echo "/opt/python/cp37-cp37m/bin" >> $GITHUB_PATH
+            echo "/opt/python/cp38-cp38/bin" >> $GITHUB_PATH
+            echo "/opt/python/cp39-cp39/bin" >> $GITHUB_PATH
+      - name: Publish manylinux to pypi (without sdist)
+        env:
+          # manually set pkg config path for openssl link
+          PKG_CONFIG_PATH: /usr/local/lib64/pkgconfig
+          MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
+        run: maturin publish -b pyo3 --target x86_64-unknown-linux-gnu --no-sdist

.github/workflows/ruby_build.yml

@@ -28,11 +28,11 @@ jobs:
         key: ruby-${{ runner.OS }}-target-incremental-${{ hashFiles('**/Cargo.lock') }}
         restore-keys: |
           ruby-${{ runner.OS }}-target-incremental-
-    - name: Install minimal nightly with clippy and rustfmt
+    - name: Install minimal stable with clippy and rustfmt
       uses: actions-rs/toolchain@v1
       with:
         profile: default
-        toolchain: nightly
+        toolchain: stable
         override: true
     - name: 'Set up Ruby'
       uses: actions/setup-ruby@v1

python/Cargo.toml

@@ -24,12 +24,13 @@ version = "0.*"
 features = ["s3", "azure"]
 
 [package.metadata.maturin]
+name = "deltalake"
 classifier = [
     "Development Status :: 3 - Alpha",
     "License :: OSI Approved :: Apache Software License",
     "Programming Language :: Python :: 3 :: Only"
 ]
-project-url = ["Repo, https://github.com/delta-io/delta.rs"]
+project-url = { Repo = "https://github.com/delta-io/delta-rs" }
 requires-dist = [
     "pyarrow>=2",
     "pandas; extra =='pandas'",

python/README.md

@@ -65,3 +65,17 @@ To install development version of the package into your current Python environme
 ```bash
 $ maturin develop
 ```
+
+Build manylinux wheels
+----------------------
+
+```bash
+docker run -e PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig -it -v `pwd`:/io apache/arrow-dev:amd64-centos-6.10-python-manylinux2010 bash
+curl https://sh.rustup.rs -sSf | sh -s -- -y
+source $HOME/.cargo/env
+rustup default stable
+cargo install --git https://github.com/PyO3/maturin.git --rev 98636cea89c328b3eba4ebb548124f75c8018200 maturin
+cd /io/python
+export PATH=/opt/python/cp37-cp37m/bin:/opt/python/cp38-cp38/bin:$PATH
+maturin publish -b pyo3 --target x86_64-unknown-linux-gnu --no-sdist
+```

rust/Cargo.toml

@@ -30,9 +30,10 @@ azure_storage = { git = "https://github.com/Azure/azure-sdk-for-rust", optional
 rusoto_core = { version = "0.43", optional = true }
 rusoto_s3 = { version = "0.43", optional = true }
 
-arrow  = { git = "https://github.com/apache/arrow.git", branch = "master" }
-datafusion = { git = "https://github.com/apache/arrow.git", branch = "master", optional = true }
-parquet = { git = "https://github.com/apache/arrow.git", branch = "master" }
+# Cannot use branch constraint due to https://github.com/PyO3/maturin/issues/389
+arrow  = { git = "https://github.com/apache/arrow.git", rev = "6c3547347e9d95f7d0c77d5949cb8fcf6983ca9b" }
+datafusion = { git = "https://github.com/apache/arrow.git", rev = "6c3547347e9d95f7d0c77d5949cb8fcf6983ca9b", optional = true }
+parquet = { git = "https://github.com/apache/arrow.git", rev = "6c3547347e9d95f7d0c77d5949cb8fcf6983ca9b" }
 crossbeam = { version = "0", optional = true }
 cfg-if = "1"
 async-trait = "0.1"