From cab264b13328b5cf188bb083a675ebb2e71b5177 Mon Sep 17 00:00:00 2001 From: Peter Squicciarini <stripedpajamas273@gmail.com> Date: Mon, 5 Oct 2020 10:12:28 -0700 Subject: [PATCH 01/10] (experiment) use containers for Linux builds --- .github/workflows/linux.yml | 58 +++++-------------------------------- build.sh | 10 +++---- check_tags.sh | 4 +-- update_version.sh | 4 +-- 4 files changed, 16 insertions(+), 60 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 0f03d7b..76785b6 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -10,19 +10,17 @@ on: jobs: linux: runs-on: ubuntu-latest - continue-on-error: ${{ matrix.probably_fails }} + container: ${{ matrix.image }} strategy: fail-fast: false matrix: - arch: [x64, arm64] - probably_fails: [false] - include: - - arch: arm - probably_fails: true + vscode_arch: [x64, arm64, armhf] + npm_arch: [x64, arm64, armv7l] + image: ['vscodehub.azurecr.io/vscode-linux-build-agent:x64', 'vscodehub.azurecr.io/vscode-linux-build-agent:buster-arm64', 'vscodehub.azurecr.io/vscode-linux-build-agent:buster-armhf'] env: OS_NAME: 'linux' - BUILDARCH: ${{ matrix.arch }} + VSCODE_ARCH: ${{ matrix.vscode_arch }} steps: - uses: actions/checkout@v2 @@ -44,57 +42,17 @@ jobs: . check_tags.sh echo "::set-env name=SHOULD_BUILD::$SHOULD_BUILD" - - name: Install build deps - run: | - triplet= - case $BUILDARCH in - arm) - arch=armhf - triplet=arm-linux-gnueabihf - ;; - - arm64) - arch=arm64 - triplet=aarch64-linux-gnu - ;; - esac - - if [[ -n "$triplet" ]]; then - sudo sed 's/^deb /deb [arch=amd64] '/g -i /etc/apt/sources.list - echo "deb [arch=$arch] http://ports.ubuntu.com/ubuntu-ports/ trusty main" | sudo tee -a /etc/apt/sources.list.d/$arch.list >/dev/null - sudo dpkg --add-architecture $arch - sudo apt-get update - sudo apt-get install libc6-dev-$arch-cross gcc-$triplet g++-$triplet `apt-cache search x11proto | grep ^x11proto | cut -f 1 -d ' '` xz-utils pkg-config - mkdir -p dl - pushd dl - apt-get download libx11-dev:$arch libx11-6:$arch libxkbfile-dev:$arch libxkbfile1:$arch libxau-dev:$arch libxdmcp-dev:$arch libxcb1-dev:$arch libsecret-1-dev:$arch libsecret-1-0:$arch libpthread-stubs0-dev:$arch libglib2.0-dev:$arch libglib2.0-0:$arch libffi-dev:$arch libffi6:$arch zlib1g:$arch libpcre3-dev:$arch libpcre3:$arch - for i in *.deb; do ar x $i; sudo tar -C / -xf data.tar.*; rm -f data.tar.*; done - popd - echo "::set-env name=CC::/usr/bin/$triplet-gcc" - echo "::set-env name=CXX::/usr/bin/$triplet-g++" - echo "::set-env name=CC_host::/usr/bin/gcc" - echo "::set-env name=CXX_host::/usr/bin/g++" - echo "::set-env name=PKG_CONFIG_LIBDIR::/usr/lib/$triplet/pkgconfig:/usr/lib/pkgconfig:/usr/share/pkgconfig" - export CC=/usr/bin/$triplet-gcc - export CXX=/usr/bin/$triplet-g++ - export CC_host=/usr/bin/gcc - export CXX_host=/usr/bin/g++ - export PKG_CONFIG_LIBDIR=/usr/lib/$triplet/pkgconfig:/usr/lib/pkgconfig:/usr/share/pkgconfig - fi - if: env.SHOULD_BUILD == 'yes' - - name: Build env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - npm_config_arch: ${{ matrix.arch }} - npm_config_target_arch: ${{ matrix.arch }} + npm_config_arch: ${{ matrix.npm_arch }} run: ./build.sh if: env.SHOULD_BUILD == 'yes' - name: Zip release run: | - cd VSCode-linux-${BUILDARCH} - tar czf ../VSCodium-linux-${BUILDARCH}-${LATEST_MS_TAG}.tar.gz . + cd VSCode-linux-${VSCODE_ARCH} + tar czf ../VSCodium-linux-${VSCODE_ARCH}-${LATEST_MS_TAG}.tar.gz . if: env.SHOULD_BUILD == 'yes' - name: Generate shasums diff --git a/build.sh b/build.sh index add4ed0..a03f326 100755 --- a/build.sh +++ b/build.sh @@ -9,7 +9,7 @@ if [[ "$SHOULD_BUILD" == "yes" ]]; then echo "LATEST_MS_COMMIT: ${LATEST_MS_COMMIT}" echo "BUILD_SOURCEVERSION: ${BUILD_SOURCEVERSION}" - if [[ "$OS_NAME" != "osx" ]]; then + if [[ "$CI_WINDOWS" == "True" ]]; then export npm_config_arch="$BUILDARCH" export npm_config_target_arch="$BUILDARCH" fi @@ -36,12 +36,10 @@ if [[ "$SHOULD_BUILD" == "yes" ]]; then yarn gulp "vscode-win32-${BUILDARCH}-system-setup" yarn gulp "vscode-win32-${BUILDARCH}-user-setup" else # linux - yarn gulp vscode-linux-${BUILDARCH}-min-ci + yarn gulp vscode-linux-${VSCODE_ARCH}-min-ci - yarn gulp "vscode-linux-${BUILDARCH}-build-deb" - if [[ "$BUILDARCH" == "x64" ]]; then - yarn gulp "vscode-linux-${BUILDARCH}-build-rpm" - fi + yarn gulp "vscode-linux-${VSCODE_ARCH}-build-deb" + yarn gulp "vscode-linux-${VSCODE_ARCH}-build-rpm" . ../create_appimage.sh fi diff --git a/check_tags.sh b/check_tags.sh index 7a91127..65b6438 100755 --- a/check_tags.sh +++ b/check_tags.sh @@ -15,7 +15,7 @@ if [ "$GITHUB_TOKEN" != "" ]; then echo "Building on Mac because we have no ZIP" export SHOULD_BUILD="yes" fi - elif [[ $BUILDARCH == "arm64" ]]; then + elif [[ $VSCODE_ARCH == "arm64" ]]; then HAVE_ARM64_DEB=$(echo $VSCODIUM_ASSETS | jq 'map(.name) | contains(["arm64.deb"])') HAVE_ARM64_TAR=$(echo $VSCODIUM_ASSETS | jq --arg suffix "arm64-$LATEST_MS_TAG.tar.gz" 'map(.name) | contains([$suffix])') if [[ "$HAVE_ARM64_DEB" != "true" ]]; then @@ -29,7 +29,7 @@ if [ "$GITHUB_TOKEN" != "" ]; then if [[ "$SHOULD_BUILD" != "yes" ]]; then echo "Already have all the Linux arm64 builds" fi - elif [[ $BUILDARCH == "arm" ]]; then + elif [[ $VSCODE_ARCH == "armhf" ]]; then HAVE_ARM_DEB=$(echo $VSCODIUM_ASSETS | jq 'map(.name) | contains(["armhf.deb"])') HAVE_ARM_TAR=$(echo $VSCODIUM_ASSETS | jq --arg suffix "arm-$LATEST_MS_TAG.tar.gz" 'map(.name) | contains([$suffix])') if [[ "$HAVE_ARM_DEB" != "true" ]]; then diff --git a/update_version.sh b/update_version.sh index db25bb3..224c702 100755 --- a/update_version.sh +++ b/update_version.sh @@ -125,8 +125,8 @@ else # linux # update service links to tar.gz file # see https://update.code.visualstudio.com/api/update/linux-x64/stable/VERSION # as examples - ASSET_NAME=VSCodium-linux-${BUILDARCH}-${LATEST_MS_TAG}.tar.gz - VERSION_PATH="linux/${BUILDARCH}" + ASSET_NAME=VSCodium-linux-${VSCODE_ARCH}-${LATEST_MS_TAG}.tar.gz + VERSION_PATH="linux/${VSCODE_ARCH}" JSON="$(generateJson ${ASSET_NAME})" updateLatestVersion "$VERSION_PATH" "$JSON" fi From 6d33f26e3ba51f2942ac62f7ccb5c7968802f9a5 Mon Sep 17 00:00:00 2001 From: Peter Squicciarini <stripedpajamas273@gmail.com> Date: Mon, 5 Oct 2020 10:21:49 -0700 Subject: [PATCH 02/10] (experiment) fix matrix setup --- .github/workflows/linux.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 76785b6..ddf5e97 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -15,12 +15,21 @@ jobs: fail-fast: false matrix: vscode_arch: [x64, arm64, armhf] - npm_arch: [x64, arm64, armv7l] - image: ['vscodehub.azurecr.io/vscode-linux-build-agent:x64', 'vscodehub.azurecr.io/vscode-linux-build-agent:buster-arm64', 'vscodehub.azurecr.io/vscode-linux-build-agent:buster-armhf'] + include: + - vscode_arch: x64 + npm_arch: x64 + image: vscodehub.azurecr.io/vscode-linux-build-agent:x64 + - vscode_arch: arm64 + npm_arch: arm64 + image: vscodehub.azurecr.io/vscode-linux-build-agent:buster-arm64 + - vscode_arch: armhf + npm_arch: armv7l + image: vscodehub.azurecr.io/vscode-linux-build-agent:buster-armhf env: OS_NAME: 'linux' VSCODE_ARCH: ${{ matrix.vscode_arch }} + steps: - uses: actions/checkout@v2 From 661608ed6f6a0564ca8a3d61a6a463bbf7d3f649 Mon Sep 17 00:00:00 2001 From: Peter Squicciarini <stripedpajamas273@gmail.com> Date: Mon, 5 Oct 2020 15:30:05 -0700 Subject: [PATCH 03/10] (experiment) use public Docker images --- .github/workflows/linux.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index ddf5e97..00847ad 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -18,13 +18,13 @@ jobs: include: - vscode_arch: x64 npm_arch: x64 - image: vscodehub.azurecr.io/vscode-linux-build-agent:x64 + image: vscodium/vscodium-linux-build-agent:x64 - vscode_arch: arm64 npm_arch: arm64 - image: vscodehub.azurecr.io/vscode-linux-build-agent:buster-arm64 + image: vscodium/vscodium-linux-build-agent:buster-arm64 - vscode_arch: armhf npm_arch: armv7l - image: vscodehub.azurecr.io/vscode-linux-build-agent:buster-armhf + image: vscodium/vscodium-linux-build-agent:buster-armhf env: OS_NAME: 'linux' From 138269d3cfe3595df97c74c700c568b051c5ff9b Mon Sep 17 00:00:00 2001 From: Peter Squicciarini <stripedpajamas273@gmail.com> Date: Mon, 5 Oct 2020 16:14:45 -0700 Subject: [PATCH 04/10] (experiment) add debug lines --- .github/workflows/linux.yml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 00847ad..e5c45c7 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -10,7 +10,12 @@ on: jobs: linux: runs-on: ubuntu-latest - container: ${{ matrix.image }} + container: + image: ${{ matrix.image }} + env: + OS_NAME: 'linux' + VSCODE_ARCH: ${{ matrix.vscode_arch }} + strategy: fail-fast: false matrix: @@ -26,10 +31,6 @@ jobs: npm_arch: armv7l image: vscodium/vscodium-linux-build-agent:buster-armhf - env: - OS_NAME: 'linux' - VSCODE_ARCH: ${{ matrix.vscode_arch }} - steps: - uses: actions/checkout@v2 @@ -38,6 +39,11 @@ jobs: with: node-version: 12.14.1 + - name: Debug + run: | + ls -al + ls /__w + - name: Clone VSCode repo run: | . get_repo.sh From 4c7c82fe067868f4964ba4adafe75b2512920598 Mon Sep 17 00:00:00 2001 From: Peter Squicciarini <stripedpajamas273@gmail.com> Date: Mon, 5 Oct 2020 16:31:17 -0700 Subject: [PATCH 05/10] (experiment) use and try another way of executing scripts --- .github/workflows/linux.yml | 14 ++------------ check_tags.sh | 1 + get_repo.sh | 3 +++ 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index e5c45c7..a5a7f1d 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -39,23 +39,13 @@ jobs: with: node-version: 12.14.1 - - name: Debug - run: | - ls -al - ls /__w - - name: Clone VSCode repo - run: | - . get_repo.sh - echo "::set-env name=LATEST_MS_TAG::$LATEST_MS_TAG" - echo "::set-env name=LATEST_MS_COMMIT::$LATEST_MS_COMMIT" + run: ./get_repo.sh - name: Check existing VSCodium tags/releases env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - . check_tags.sh - echo "::set-env name=SHOULD_BUILD::$SHOULD_BUILD" + run: ./check_tags.sh - name: Build env: diff --git a/check_tags.sh b/check_tags.sh index 65b6438..5c9b4d2 100755 --- a/check_tags.sh +++ b/check_tags.sh @@ -69,3 +69,4 @@ if [ "$GITHUB_TOKEN" != "" ]; then fi fi +echo "SHOULD_BUILD=$SHOULD_BUILD" >> $GITHUB_ENV diff --git a/get_repo.sh b/get_repo.sh index 047cf33..9883aac 100755 --- a/get_repo.sh +++ b/get_repo.sh @@ -8,3 +8,6 @@ echo "Got the latest MS tag: ${LATEST_MS_TAG}" git clone https://github.com/Microsoft/vscode.git --branch $LATEST_MS_TAG --depth 1 +echo "LATEST_MS_COMMIT=$LATEST_MS_COMMIT" >> $GITHUB_ENV +echo "LATEST_MS_TAG=$LATEST_MS_TAG" >> $GITHUB_ENV + From 1082261042697c7a0c3ef1690879aff4ebd369ed Mon Sep 17 00:00:00 2001 From: Peter Squicciarini <stripedpajamas273@gmail.com> Date: Mon, 5 Oct 2020 16:46:22 -0700 Subject: [PATCH 06/10] (experiment) Install Yarn manually --- .github/workflows/linux.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index a5a7f1d..4089b48 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -39,6 +39,9 @@ jobs: with: node-version: 12.14.1 + - name: Install Yarn + run: npm install -g yarn + - name: Clone VSCode repo run: ./get_repo.sh From 8c04967b7ef89858f8d701bc5f8786ba2ab78d76 Mon Sep 17 00:00:00 2001 From: Peter Squicciarini <stripedpajamas273@gmail.com> Date: Mon, 5 Oct 2020 17:54:42 -0700 Subject: [PATCH 07/10] (experiment) remove further BUILDARCH branches and s/armhf/arm/ --- .github/workflows/linux.yml | 2 +- create_appimage.sh | 2 +- prepare_vscode.sh | 9 +-------- 3 files changed, 3 insertions(+), 10 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 4089b48..b9a83ff 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -27,7 +27,7 @@ jobs: - vscode_arch: arm64 npm_arch: arm64 image: vscodium/vscodium-linux-build-agent:buster-arm64 - - vscode_arch: armhf + - vscode_arch: arm npm_arch: armv7l image: vscodium/vscodium-linux-build-agent:buster-armhf diff --git a/create_appimage.sh b/create_appimage.sh index 9b80ac3..089f0e9 100755 --- a/create_appimage.sh +++ b/create_appimage.sh @@ -1,5 +1,5 @@ #!/bin/bash -if [[ "$BUILDARCH" == "x64" ]]; then +if [[ "$VSCODE_ARCH" == "x64" ]]; then # install a dep needed for this process sudo apt-get install desktop-file-utils diff --git a/prepare_vscode.sh b/prepare_vscode.sh index 48a4f66..11cda1f 100755 --- a/prepare_vscode.sh +++ b/prepare_vscode.sh @@ -2,7 +2,7 @@ set -e -if [[ "$OS_NAME" != "osx" ]]; then +if [[ "$CI_WINDOWS" == "True" ]]; then export npm_config_arch="$BUILDARCH" export npm_config_target_arch="$BUILDARCH" fi @@ -23,13 +23,6 @@ else yarn postinstall fi -if [[ "$BUILDARCH" == *"arm"* ]]; then - sed -i -z 's/,\n[^\n]*arm[^\n]*//' node_modules/vscode-sqlite3/binding.gyp - sed -i "s/Release\/sqlite'/Release\/sqlite.node'/" node_modules/vscode-sqlite3/lib/sqlite3.js - yarn add -D electron-rebuild - npx electron-rebuild -f -w vscode-sqlite3 -fi - mv product.json product.json.bak # set fields in product.json From 07ec36ff6924144e6263aaf834407ebbc27f1d74 Mon Sep 17 00:00:00 2001 From: Peter Squicciarini <stripedpajamas273@gmail.com> Date: Mon, 5 Oct 2020 17:56:54 -0700 Subject: [PATCH 08/10] (experiment) bugfix matrix --- .github/workflows/linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index b9a83ff..227fe5f 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -19,7 +19,7 @@ jobs: strategy: fail-fast: false matrix: - vscode_arch: [x64, arm64, armhf] + vscode_arch: [x64, arm64, arm] include: - vscode_arch: x64 npm_arch: x64 From 9f3136fd35bc928c3665cea3b621810cb10d5ba1 Mon Sep 17 00:00:00 2001 From: Peter Squicciarini <stripedpajamas273@gmail.com> Date: Mon, 5 Oct 2020 19:48:04 -0700 Subject: [PATCH 09/10] (experiment) bugfix arm check and skip RPM for arms --- build.sh | 5 ++++- check_tags.sh | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/build.sh b/build.sh index a03f326..375a9f2 100755 --- a/build.sh +++ b/build.sh @@ -39,7 +39,10 @@ if [[ "$SHOULD_BUILD" == "yes" ]]; then yarn gulp vscode-linux-${VSCODE_ARCH}-min-ci yarn gulp "vscode-linux-${VSCODE_ARCH}-build-deb" - yarn gulp "vscode-linux-${VSCODE_ARCH}-build-rpm" + + if [[ "$VSCODE_ARCH" == "x64" ]]; then + yarn gulp "vscode-linux-${VSCODE_ARCH}-build-rpm" + fi . ../create_appimage.sh fi diff --git a/check_tags.sh b/check_tags.sh index 5c9b4d2..7efa3c7 100755 --- a/check_tags.sh +++ b/check_tags.sh @@ -29,7 +29,7 @@ if [ "$GITHUB_TOKEN" != "" ]; then if [[ "$SHOULD_BUILD" != "yes" ]]; then echo "Already have all the Linux arm64 builds" fi - elif [[ $VSCODE_ARCH == "armhf" ]]; then + elif [[ $VSCODE_ARCH == "arm" ]]; then HAVE_ARM_DEB=$(echo $VSCODIUM_ASSETS | jq 'map(.name) | contains(["armhf.deb"])') HAVE_ARM_TAR=$(echo $VSCODIUM_ASSETS | jq --arg suffix "arm-$LATEST_MS_TAG.tar.gz" 'map(.name) | contains([$suffix])') if [[ "$HAVE_ARM_DEB" != "true" ]]; then From 45af9e614bf3d795396b4f6d03b9f3178b209115 Mon Sep 17 00:00:00 2001 From: Peter Squicciarini <stripedpajamas273@gmail.com> Date: Mon, 5 Oct 2020 21:30:07 -0700 Subject: [PATCH 10/10] (experiment) skip GH action env writes in Windows --- get_repo.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/get_repo.sh b/get_repo.sh index 9883aac..fc45f2d 100755 --- a/get_repo.sh +++ b/get_repo.sh @@ -8,6 +8,9 @@ echo "Got the latest MS tag: ${LATEST_MS_TAG}" git clone https://github.com/Microsoft/vscode.git --branch $LATEST_MS_TAG --depth 1 -echo "LATEST_MS_COMMIT=$LATEST_MS_COMMIT" >> $GITHUB_ENV -echo "LATEST_MS_TAG=$LATEST_MS_TAG" >> $GITHUB_ENV +# for GH actions +if [[ "$CI_WINDOWS" != "True" ]]; then + echo "LATEST_MS_COMMIT=$LATEST_MS_COMMIT" >> $GITHUB_ENV + echo "LATEST_MS_TAG=$LATEST_MS_TAG" >> $GITHUB_ENV +fi