From 3fd538a587184a87e2b41a526e089247007aa526 Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Tue, 8 Mar 2022 16:46:18 -0800 Subject: [PATCH] fix: Roll back depot_tools, bypass vpython (#1045) Using the latest depot_tools no longer works. depot_tools also wants to auto-update itself, which must now be disabled. We also need to disable the copy of python (vpython) included in depot_tools, since for some distros, it has dependencies on system libraries that no longer exist. Finally, we need to force some distros to use python 2, because our build system is ancient and needs to be ripped out and replaced some day soon. This fixes build issues in our CI, our Dockerfiles, and in general on certain platforms or distros. Closes #1023 --- .github/workflows/build_and_test.yaml | 3 ++- .../custom-actions/build-packager/action.yaml | 20 +++++++++++++++- .github/workflows/github_release.yaml | 3 ++- Dockerfile | 14 +++++++---- docs/source/build_instructions.md | 23 ++++++++----------- packager/testing/dockers/Alpine_Dockerfile | 12 ++++++---- packager/testing/dockers/ArchLinux_Dockerfile | 9 +++++++- packager/testing/dockers/CentOS_Dockerfile | 16 ++++++++++--- packager/testing/dockers/Debian_Dockerfile | 9 +++++++- packager/testing/dockers/Fedora_Dockerfile | 10 +++++--- packager/testing/dockers/OpenSUSE_Dockerfile | 9 +++++++- packager/testing/dockers/Ubuntu_Dockerfile | 9 +++++++- 12 files changed, 101 insertions(+), 36 deletions(-) diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index 4bc4d8a4d92..208ba0b3555 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -121,7 +121,8 @@ jobs: - name: Install depot tools shell: bash run: | - git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git + git clone -b chrome/4147 https://chromium.googlesource.com/chromium/tools/depot_tools.git + touch depot_tools/.disable_auto_update echo "${GITHUB_WORKSPACE}/depot_tools" >> $GITHUB_PATH - name: Setup gclient diff --git a/.github/workflows/custom-actions/build-packager/action.yaml b/.github/workflows/custom-actions/build-packager/action.yaml index 6c20a74344b..0556ae86b13 100644 --- a/.github/workflows/custom-actions/build-packager/action.yaml +++ b/.github/workflows/custom-actions/build-packager/action.yaml @@ -52,12 +52,30 @@ runs: echo "::endgroup::" fi + - name: Force Python 2 to support ancient build system (non-Linux only) + if: runner.os != 'Linux' + uses: actions/setup-python@v2 + with: + python-version: '2.x' + + - name: Force Python 2 to support ancient build system (Linux only) + if: runner.os == 'Linux' + shell: bash + run: | + echo "::group::Install python2" + sudo apt install -y python2 + sudo ln -sf python2 /usr/bin/python + echo "::endgroup::" + - name: Install depot tools shell: bash run: | echo "::group::Install depot_tools" - git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git + git clone -b chrome/4147 https://chromium.googlesource.com/chromium/tools/depot_tools.git + touch depot_tools/.disable_auto_update echo "${GITHUB_WORKSPACE}/depot_tools" >> $GITHUB_PATH + # Bypass VPYTHON included by depot_tools. Prefer the system installation. + echo "VPYTHON_BYPASS=manually managed python not supported by chrome operations" >> $GITHUB_ENV echo "::endgroup::" - name: Build ninja (arm only) diff --git a/.github/workflows/github_release.yaml b/.github/workflows/github_release.yaml index 05faa4267ef..0d018670c6c 100644 --- a/.github/workflows/github_release.yaml +++ b/.github/workflows/github_release.yaml @@ -200,7 +200,8 @@ jobs: - name: Install depot tools shell: bash run: | - git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git + git clone -b chrome/4147 https://chromium.googlesource.com/chromium/tools/depot_tools.git + touch depot_tools/.disable_auto_update echo "${GITHUB_WORKSPACE}/depot_tools" >> $GITHUB_PATH - name: Setup gclient diff --git a/Dockerfile b/Dockerfile index 59be6b0e5ab..7a395a5030e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,14 +3,21 @@ FROM alpine:3.11 as builder # Install utilities, libraries, and dev tools. RUN apk add --no-cache \ bash curl \ - bsd-compat-headers linux-headers \ + bsd-compat-headers c-ares-dev linux-headers \ build-base git ninja python2 python3 +# Default to python2 because our build system is ancient. +RUN ln -sf python2 /usr/bin/python + # Install depot_tools. WORKDIR / -RUN git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git +RUN git clone -b chrome/4147 https://chromium.googlesource.com/chromium/tools/depot_tools.git +RUN touch depot_tools/.disable_auto_update ENV PATH $PATH:/depot_tools +# Bypass VPYTHON included by depot_tools. Prefer the system installation. +ENV VPYTHON_BYPASS="manually managed python not supported by chrome operations" + # Alpine uses musl which does not have mallinfo defined in malloc.h. Define the # structure to workaround a Chromium base bug. RUN sed -i \ @@ -18,9 +25,6 @@ RUN sed -i \ /usr/include/malloc.h ENV GYP_DEFINES='musl=1' -# Bypass VPYTHON included by depot_tools, which no longer works in Alpine. -ENV VPYTHON_BYPASS="manually managed python not supported by chrome operations" - # Build shaka-packager from the current directory, rather than what has been # merged. WORKDIR shaka_packager diff --git a/docs/source/build_instructions.md b/docs/source/build_instructions.md index 340a668a05e..416c0184963 100644 --- a/docs/source/build_instructions.md +++ b/docs/source/build_instructions.md @@ -56,14 +56,18 @@ GYP_MSVS_OVERRIDE_PATH="C:/Program Files (x86)/Microsoft Visual Studio/2019/Comm ## Install `depot_tools` -### Linux and Mac - -Clone the `depot_tools` repository from Chromium: +Clone a particular branch of the `depot_tools` repository from Chromium: ```shell -git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git +git clone -b chrome/4147 https://chromium.googlesource.com/chromium/tools/depot_tools.git +touch depot_tools/.disable_auto_update ``` +The latest version of depot_tools will not work, so please use that branch! + + +### Linux and Mac + Add `depot_tools` to the end of your PATH (you will probably want to put this in your `~/.bashrc` or `~/.zshrc`). Assuming you cloned `depot_tools` to `/path/to/depot_tools`: @@ -74,17 +78,8 @@ export PATH="$PATH:/path/to/depot_tools" ### Windows -Download the -[depot_tools bundle](https://storage.googleapis.com/chrome-infra/depot_tools.zip) -and extract it somewhere. - -**WARNING: DO NOT** use drag-n-drop or copy-n-paste extract from Explorer, -this will not extract the hidden “.git” folder which is necessary for -depot_tools to autoupdate itself. You can use “Extract all…” from the context -menu though. - Add depot_tools to the start of your PATH (must be ahead of any installs of -Python). Assuming you unzipped the bundle to C:\src\depot_tools, open: +Python). Assuming you cloned the repo to C:\src\depot_tools, open: Control Panel → System and Security → System → Advanced system settings diff --git a/packager/testing/dockers/Alpine_Dockerfile b/packager/testing/dockers/Alpine_Dockerfile index 2ce7850a499..df32f525cfd 100644 --- a/packager/testing/dockers/Alpine_Dockerfile +++ b/packager/testing/dockers/Alpine_Dockerfile @@ -6,11 +6,18 @@ RUN apk add --no-cache \ bsd-compat-headers c-ares-dev linux-headers \ build-base git ninja python2 python3 +# Default to python2 because our build system is ancient. +RUN ln -sf python2 /usr/bin/python + # Install depot_tools. WORKDIR / -RUN git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git +RUN git clone -b chrome/4147 https://chromium.googlesource.com/chromium/tools/depot_tools.git +RUN touch depot_tools/.disable_auto_update ENV PATH $PATH:/depot_tools +# Bypass VPYTHON included by depot_tools. Prefer the system installation. +ENV VPYTHON_BYPASS="manually managed python not supported by chrome operations" + # Alpine uses musl which does not have mallinfo defined in malloc.h. Define the # structure to workaround a Chromium base bug. RUN sed -i \ @@ -18,8 +25,5 @@ RUN sed -i \ /usr/include/malloc.h ENV GYP_DEFINES='musl=1' -# Bypass VPYTHON included by depot_tools, which no longer works in Alpine. -ENV VPYTHON_BYPASS="manually managed python not supported by chrome operations" - # Build and run this docker by mapping shaka-packager with # -v "shaka-packager:/shaka-packager". diff --git a/packager/testing/dockers/ArchLinux_Dockerfile b/packager/testing/dockers/ArchLinux_Dockerfile index e5065e9b73f..1b0435687e8 100644 --- a/packager/testing/dockers/ArchLinux_Dockerfile +++ b/packager/testing/dockers/ArchLinux_Dockerfile @@ -6,10 +6,17 @@ RUN pacman -Sy --needed --noconfirm \ c-ares \ gcc git python2 python3 +# Default to python2 because our build system is ancient. +RUN ln -sf python2 /usr/bin/python + # Install depot_tools. WORKDIR / -RUN git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git +RUN git clone -b chrome/4147 https://chromium.googlesource.com/chromium/tools/depot_tools.git +RUN touch depot_tools/.disable_auto_update ENV PATH /depot_tools:$PATH +# Bypass VPYTHON included by depot_tools. Prefer the system installation. +ENV VPYTHON_BYPASS="manually managed python not supported by chrome operations" + # Build and run this docker by mapping shaka-packager with # -v "shaka-packager:/shaka-packager". diff --git a/packager/testing/dockers/CentOS_Dockerfile b/packager/testing/dockers/CentOS_Dockerfile index 327581d3ecb..e2c604b04bd 100644 --- a/packager/testing/dockers/CentOS_Dockerfile +++ b/packager/testing/dockers/CentOS_Dockerfile @@ -1,18 +1,28 @@ FROM centos:8 +# Fix up CentOS repo info, which is outdated and not maintained in DockerHub. +# See https://stackoverflow.com/a/71309215 +RUN cd /etc/yum.repos.d/ +RUN sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* +RUN sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* + # Install utilities, libraries, and dev tools. RUN yum install -y \ which \ c-ares-devel libatomic \ gcc-c++ git python2 python3 -# Default to python3. -RUN alternatives --set python /usr/bin/python3 +# Default to python2 because our build system is ancient. +RUN ln -sf python2 /usr/bin/python # Install depot_tools. WORKDIR / -RUN git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git +RUN git clone -b chrome/4147 https://chromium.googlesource.com/chromium/tools/depot_tools.git +RUN touch depot_tools/.disable_auto_update ENV PATH /depot_tools:$PATH +# Bypass VPYTHON included by depot_tools. Prefer the system installation. +ENV VPYTHON_BYPASS="manually managed python not supported by chrome operations" + # Build and run this docker by mapping shaka-packager with # -v "shaka-packager:/shaka-packager". diff --git a/packager/testing/dockers/Debian_Dockerfile b/packager/testing/dockers/Debian_Dockerfile index cba5b5e31d2..12a271740e1 100644 --- a/packager/testing/dockers/Debian_Dockerfile +++ b/packager/testing/dockers/Debian_Dockerfile @@ -7,9 +7,16 @@ RUN apt-get install -y \ libc-ares-dev \ build-essential git python python3 +# Default to python2 because our build system is ancient. +RUN ln -sf python2 /usr/bin/python + # Install depot_tools. -RUN git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git +RUN git clone -b chrome/4147 https://chromium.googlesource.com/chromium/tools/depot_tools.git +RUN touch depot_tools/.disable_auto_update ENV PATH /depot_tools:$PATH +# Bypass VPYTHON included by depot_tools. Prefer the system installation. +ENV VPYTHON_BYPASS="manually managed python not supported by chrome operations" + # Build and run this docker by mapping shaka-packager with # -v "shaka-packager:/shaka-packager". diff --git a/packager/testing/dockers/Fedora_Dockerfile b/packager/testing/dockers/Fedora_Dockerfile index 63319f71c5c..f4edfb531e5 100644 --- a/packager/testing/dockers/Fedora_Dockerfile +++ b/packager/testing/dockers/Fedora_Dockerfile @@ -6,13 +6,17 @@ RUN yum install -y \ c-ares-devel libatomic \ gcc-c++ git python2 -# Default to python3. -RUN alternatives --install /usr/bin/python python /usr/bin/python3 3 +# Default to python2 because our build system is ancient. +RUN ln -sf python2 /usr/bin/python # Install depot_tools. WORKDIR / -RUN git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git +RUN git clone -b chrome/4147 https://chromium.googlesource.com/chromium/tools/depot_tools.git +RUN touch depot_tools/.disable_auto_update ENV PATH /depot_tools:$PATH +# Bypass VPYTHON included by depot_tools. Prefer the system installation. +ENV VPYTHON_BYPASS="manually managed python not supported by chrome operations" + # Build and run this docker by mapping shaka-packager with # -v "shaka-packager:/shaka-packager". diff --git a/packager/testing/dockers/OpenSUSE_Dockerfile b/packager/testing/dockers/OpenSUSE_Dockerfile index 8059a1f920c..99cb654df3e 100644 --- a/packager/testing/dockers/OpenSUSE_Dockerfile +++ b/packager/testing/dockers/OpenSUSE_Dockerfile @@ -6,10 +6,17 @@ RUN zypper in -y \ c-ares-devel \ gcc-c++ git python python3 +# Default to python2 because our build system is ancient. +RUN ln -sf python2 /usr/bin/python + # Install depot_tools. WORKDIR / -RUN git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git +RUN git clone -b chrome/4147 https://chromium.googlesource.com/chromium/tools/depot_tools.git +RUN touch depot_tools/.disable_auto_update ENV PATH /depot_tools:$PATH +# Bypass VPYTHON included by depot_tools. Prefer the system installation. +ENV VPYTHON_BYPASS="manually managed python not supported by chrome operations" + # Build and run this docker by mapping shaka-packager with # -v "shaka-packager:/shaka-packager". diff --git a/packager/testing/dockers/Ubuntu_Dockerfile b/packager/testing/dockers/Ubuntu_Dockerfile index 03b5be72cd6..3e3a14ba8e1 100644 --- a/packager/testing/dockers/Ubuntu_Dockerfile +++ b/packager/testing/dockers/Ubuntu_Dockerfile @@ -7,9 +7,16 @@ RUN apt-get install -y \ libc-ares-dev \ build-essential git python python3 +# Default to python2 because our build system is ancient. +RUN ln -sf python2 /usr/bin/python + # Install depot_tools. -RUN git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git +RUN git clone -b chrome/4147 https://chromium.googlesource.com/chromium/tools/depot_tools.git +RUN touch depot_tools/.disable_auto_update ENV PATH /depot_tools:$PATH +# Bypass VPYTHON included by depot_tools. Prefer the system installation. +ENV VPYTHON_BYPASS="manually managed python not supported by chrome operations" + # Build and run this docker by mapping shaka-packager with # -v "shaka-packager:/shaka-packager".