refactor end-to-end.yml workflow to use a top-level end-to-end.sh

.forgejo/prepare-end-to-end/action.yml

@@ -0,0 +1,19 @@
+runs:
+  using: "composite"
+  steps:
+      - uses: https://code.forgejo.org/actions/setup-forgejo@v2
+        with:
+          install-only: true
+      - run: forgejo-binary.sh ensure_user forgejo
+      - uses: actions/download-artifact@v3
+        with:
+          name: forgejo-dev
+          path: /srv/forgejo-binaries
+      - name: chown/chmod /srv/forgejo-binaries
+        run: |
+          chmod +x /srv/forgejo-binaries/*
+          chown -R forgejo /srv/forgejo-binaries
+      - run: |
+          script=$(pwd)/end-to-end.sh
+          $script run dependencies
+          $script clobber

.forgejo/workflows/end-to-end.yml

@@ -0,0 +1,70 @@
+on:
+  pull_request:
+  push:
+    branches:
+      - 'main'
+
+jobs:
+  build:
+    runs-on: docker
+    container:
+      image: 'docker.io/node:20-bookworm'
+    steps:
+      - uses: https://code.forgejo.org/actions/checkout@v4
+      - uses: https://code.forgejo.org/actions/setup-go@v4
+        with:
+          go-version: "1.21"
+      - name: lib/build.sh
+        run: |
+          set -x
+
+          #
+          # SQLite needs gcc to be compiled
+          #
+          export DEBIAN_FRONTEND=noninteractive
+          apt-get update -qq
+          apt-get -q install -y -qq build-essential
+
+          d=/tmp/forgejo-binaries
+          mkdir $d /tmp/forgejo-upload
+
+          for version in $(ls forgejo/sources) ; do
+            forgejo=$d/forgejo-$version-dev
+            lib/build.sh $version $d
+            $forgejo --version
+            mv $forgejo /tmp/forgejo-upload
+          done
+      - uses: actions/upload-artifact@v3
+        with:
+          name: forgejo-dev
+          path: /tmp/forgejo-upload
+
+  packages:
+    needs: [build]
+    runs-on: lxc-bookworm
+    steps:
+      - uses: https://code.forgejo.org/actions/checkout@v4
+      - uses: ./.forgejo/prepare-end-to-end
+      - run: su forgejo -c "./end-to-end.sh test_packages"
+
+  upgrade:
+    needs: [build]
+    runs-on: lxc-bookworm
+    steps:
+      - name: cache S3 binaries
+        id: S3
+        uses: https://code.forgejo.org/actions/cache@v3
+        with:
+          path: |
+            /usr/local/bin/minio
+            /usr/local/bin/mc
+            /usr/local/bin/garage
+          key: S3
+
+      - name: skip if S3 cache hit
+        if: steps.S3.outputs.cache-hit != 'true'
+        run: echo no hit
+
+      - uses: https://code.forgejo.org/actions/checkout@v4
+      - uses: ./.forgejo/prepare-end-to-end
+      - run: su forgejo -c "./end-to-end.sh test_upgrades"

.forgejo/workflows/upgrade.yml

@@ -1,42 +0,0 @@
-name: upgrade
-
-on:
-  pull_request:
-  push:
-    branches:
-      - 'main'
-
-jobs:
-  upgrade:
-    runs-on: lxc-bookworm
-    steps:
-      - name: cache S3 binaries
-        id: S3
-        uses: https://code.forgejo.org/actions/cache@v3
-        with:
-          path: |
-            /usr/local/bin/minio
-            /usr/local/bin/mc
-            /usr/local/bin/garage
-          key: S3
-
-      - name: skip if S3 cache hit
-        if: steps.S3.outputs.cache-hit != 'true'
-        run: echo no hit
-
-      - uses: https://code.forgejo.org/actions/checkout@v4
-      - uses: https://code.forgejo.org/actions/setup-forgejo@v2
-        with:
-          install-only: true
-      - uses: https://code.forgejo.org/actions/setup-go@v4
-        with:
-          go-version: "1.21"
-      - run: |
-          set -x
-          forgejo-binary.sh ensure_user forgejo
-          apt-get -q install -y -qq build-essential
-      - run: |
-          script=$(pwd)/forgejo/upgrades/test-upgrade.sh
-          $script run dependencies
-          $script clobber
-          su forgejo -c "$script test_upgrades"

README.md

@@ -37,8 +37,8 @@ git clone https://code.forgejo.org/actions/setup-forgejo
 export PATH=$(pwd)/setup-forgejo:$PATH
 git clone https://code.forgejo.org/forgejo/end-to-end
 cd end-to-end
-export DIR=/tmp/end-to-end
+export DIR=/tmp/forgejo-end-to-end
-rm -fr /tmp/end-to-end ; sudo rm -fr /srv/example ; sudo mkdir /srv/example ; sudo chown -R $USER /srv/example
+rm -fr /tmp/forgejo-end-to-end ; sudo rm -fr /srv/example ; sudo mkdir /srv/example ; sudo chown -R $USER /srv/example
 ```
 
 Run using Forgejo built from source.

end-to-end.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+# SPDX-License-Identifier: MIT
+
+#
+# Debug loop from the source tree:
+#
+# ./end-to-end.sh dependencies
+# ./end-to-end.sh build_all
+# VERBOSE=true ./end-to-end.sh test_downgrade_1.20.2_fails
+#
+# Everything happens in /tmp/forgejo-end-to-end
+#
+
+SELF_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+SELF="${BASH_SOURCE[0]}"
+source $SELF_DIR/lib/lib.sh
+
+source $SELF_DIR/forgejo/upgrades/test-upgrade.sh
+source $SELF_DIR/packages/packages.sh
+
+"$@"

forgejo/upgrades/test-upgrade-1.20-storage.sh

@@ -12,7 +12,7 @@ function test_downgrade_1.20.2_fails() {
     start 1.20.3-0
     stop
     download 1.20.2-0
-    timeout 60 $DIR/forgejo-1.20.2-0 --config $work_path/app.ini --work-path $work_path || true
+    timeout 60 $DIR_BINARIES/forgejo-1.20.2-0 --config $work_path/app.ini --work-path $work_path || true
     if ! grep --fixed-strings --quiet 'use the newer database' $work_path/log/forgejo.log ; then
         cat $work_path/log/forgejo.log
         return 1
@@ -41,7 +41,7 @@ function test_bug_storage_merged() {
 
     log_info "upgrading from 1.20.2-0 with [storage].PATH fails"
     download 1.20.3-0
-    timeout 60 $DIR/forgejo-1.20.3-0 --config $work_path/app.ini --work-path $work_path || true
+    timeout 60 $DIR_BINARIES/forgejo-1.20.3-0 --config $work_path/app.ini --work-path $work_path || true
     if ! grep --fixed-strings --quiet '[storage].PATH is set and may create storage issues' $work_path/log/forgejo.log ; then
         cat $work_path/log/forgejo.log
         return 1
@@ -206,7 +206,7 @@ function test_bug_storage_misplace() {
 
     log_info "upgrading from 1.20.2-0 with conflicting sections fails"
     download 1.20.3-0
-    timeout 60 $DIR/forgejo-1.20.3-0 --config $work_path/app.ini --work-path $work_path || true
+    timeout 60 $DIR_BINARIES/forgejo-1.20.3-0 --config $work_path/app.ini --work-path $work_path || true
     for path in ${STORAGE_PATHS} ; do
         if ! grep --fixed-strings --quiet "[storage.$path] may conflict" $work_path/log/forgejo.log ; then
             cat $work_path/log/forgejo.log

forgejo/upgrades/test-upgrade.sh

@@ -11,19 +11,17 @@
 # Everything happens in /tmp/forgejo-upgrades
 #
 
-SELF_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+UPGRADE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-SELF="${BASH_SOURCE[0]}"
-source $SELF_DIR/../../lib/lib.sh
 
 STORAGE_PATHS="attachments avatars lfs packages repo-archive repo-avatars"
 STORAGE_FUN="attachments avatars lfs packages repo_archive repo_avatars"
 : ${FORGEJO_REPO:=fixture}
 
-source $SELF_DIR/fixtures.sh
+source $UPGRADE_DIR/fixtures.sh
 
 function reset() {
     local config=$1
-    reset_forgejo $SELF_DIR/$config-app.ini
+    reset_forgejo $UPGRADE_DIR/$config-app.ini
     reset_minio
     reset_garage
 }
@@ -86,17 +84,14 @@ function test_forgejo_database_version() {
     test "$expected_version" = "$actual_version"
 }
 
-source $SELF_DIR/test-upgrade-1.20-storage.sh
+source $UPGRADE_DIR/test-upgrade-1.20-storage.sh
-source $SELF_DIR/test-upgrade-forgejo-database-v3.sh
+source $UPGRADE_DIR/test-upgrade-forgejo-database-v3.sh
-source $SELF_DIR/../../packages/packages.sh
 
 function test_upgrades() {
     run stop
     run dependencies
     run build_all
 
-    test_packages
-
     run test_successful_upgrades
     run test_storage_stable_s3 minio
     run test_storage_stable_s3 garage
@@ -104,5 +99,3 @@ function test_upgrades() {
     test_upgrade_1_20_storage
     run test_forgejo_database_v3_upgrades
 }
-
-"$@"

lib/build.sh

@@ -7,18 +7,19 @@ set -ex
 SELF_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
 version=$1
-DIR=$2
+dir_binaries=$2
 
 v=$(echo $version | sed -E -e 's/^([0-9]+\.[0-9]+).*/\1/')
+src=$dir_binaries/src-$v
 read url ref semver < $SELF_DIR/../forgejo/sources/$v
 
-if ! test -d $DIR/src ; then
+if ! test -d $src ; then
-    mkdir -p $DIR/src
+    mkdir -p $src
-    cd $DIR/src
+    cd $src
     git init
     git remote add origin $url
 else
-    cd $DIR/src
+    cd $src
 fi
 
 if ! [[ "$ref" =~ ^refs/ ]] ; then
@@ -30,4 +31,4 @@ git switch --force-create $v $ref
 export TAGS="bindata sqlite sqlite_unlock_notify" FORGEJO_VERSION=$semver
 make deps-backend backend
 make generate forgejo
-cp -a forgejo $DIR/forgejo-$v-dev
+cp -a forgejo $dir_binaries/forgejo-$v-dev

lib/lib.sh

@@ -29,6 +29,7 @@ RELEASE_NUMBERS="7.0"
 PREFIX===============
 HOST_PORT=$IP:3000
 DIR=/tmp/forgejo-end-to-end
+DIR_BINARIES=/srv/forgejo-binaries
 export DOT_FORGEJO_CURL=$DIR/forgejo-curl
 export DOT=$DOT_FORGEJO_CURL # for backward compatibility with forgejo-curl.sh 1.0.0
 : ${FORGEJO_USER:=root}
@@ -70,8 +71,13 @@ function dependencies() {
 
 function build_all() {
     for dev in $RELEASE_NUMBERS ; do
-	log_info $dev-dev
+	local forgejo=$DIR_BINARIES/forgejo-$dev-dev
-	$LIB_DIR/build.sh $dev $DIR
+	if test -f $forgejo ; then
+	    log_info $dev already exists
+	else
+	    $LIB_DIR/build.sh $dev $DIR_BINARIES
+	    log_info $dev built from sources
+	fi
     done
 }
 
@@ -108,18 +114,18 @@ function full_version() {
 function download() {
     local version=$1
 
-    if ! test -f $DIR/forgejo-$version ; then
+    if ! test -f $DIR_BINARIES/forgejo-$version ; then
-        mkdir -p $DIR
+        mkdir -p $DIR_BINARIES
 	for owner in forgejo forgejo-experimental forgejo-integration ; do
 	    full_version=$(full_version $version $owner)
 	    if test "$full_version" = "" ; then
 		continue
 	    fi
-            if wget -O $DIR/forgejo-$version --quiet https://codeberg.org/$owner/forgejo/releases/download/v$full_version/forgejo-$full_version-linux-amd64 ; then
+            if wget -O $DIR_BINARIES/forgejo-$version --quiet https://codeberg.org/$owner/forgejo/releases/download/v$full_version/forgejo-$full_version-linux-amd64 ; then
 		break
 	    fi
 	done
-	if test -s $DIR/forgejo-$version ; then
+	if test -s $DIR_BINARIES/forgejo-$version ; then
 	    if test "$version" != "$full_version" ; then
 		log_info "downloaded $full_version for $version"
 	    fi
@@ -127,7 +133,7 @@ function download() {
 	    echo unable to download Forgejo $version
 	    return 1
 	fi
-        chmod +x $DIR/forgejo-$version
+        chmod +x $DIR_BINARIES/forgejo-$version
     fi
 }
 
@@ -147,7 +153,7 @@ function start_forgejo() {
 
     download $version
     local work_path=$DIR/forgejo-work-path
-    daemon --chdir=$DIR --unsafe --env="TERM=$TERM" --env="HOME=$HOME" --env="PATH=$PATH" --pidfile=$DIR/forgejo-pid --errlog=$DIR/forgejo-err.log --output=$DIR/forgejo-out.log -- $DIR/forgejo-$version --config $work_path/app.ini --work-path $work_path
+    daemon --chdir=$DIR --unsafe --env="TERM=$TERM" --env="HOME=$HOME" --env="PATH=$PATH" --pidfile=$DIR/forgejo-pid --errlog=$DIR/forgejo-err.log --output=$DIR/forgejo-out.log -- $DIR_BINARIES/forgejo-$version --config $work_path/app.ini --work-path $work_path
     if ! retry grep 'Starting server on' $work_path/log/forgejo.log ; then
         grep '' $DIR/*.log
         grep '' $work_path/log/*.log 2> /dev/null
@@ -247,7 +253,7 @@ function create_user_and_login() {
 
     local work_path=$DIR/forgejo-work-path
     local email="$FORGEJO_USER@example.com"
-    local cli="$DIR/forgejo-$version --config $work_path/app.ini --work-path $work_path"
+    local cli="$DIR_BINARIES/forgejo-$version --config $work_path/app.ini --work-path $work_path"
     if ! $cli admin user list | grep --quiet "$email" ; then
 	$cli admin user create --admin --username "$FORGEJO_USER" --password "$FORGEJO_PASSWORD" --email $email
     fi