Merge pull request 'federation: stub that checks the nodeinfo of the two instances' (#195) from earl-warren/end-to-end:wip-federation into main

Reviewed-on: https://code.forgejo.org/forgejo/end-to-end/pulls/195

.forgejo/workflows/end-to-end.yml

@@ -76,6 +76,17 @@ jobs:
         if: always()
         run: su forgejo -c "./end-to-end.sh show_logs"
 
+  federation:
+    needs: [build]
+    runs-on: lxc-bookworm
+    steps:
+      - uses: https://code.forgejo.org/actions/checkout@v4
+      - uses: ./.forgejo/prepare-end-to-end
+      - run: su forgejo -c "./end-to-end.sh test_federation"
+      - name: full logs
+        if: always()
+        run: su forgejo -c "./end-to-end.sh show_logs"
+
   actions-docs:
     needs: [build]
     runs-on: lxc-bookworm

actions/actions.sh

@@ -74,7 +74,7 @@ function actions_setup() {
 function actions_teardown() {
     forgejo-curl.sh logout
     forgejo-runner.sh teardown
-    stop_daemon forgejo
+    stop_forgejo
 }
 
 function actions_runner_version() {

actions/default-app.ini

@@ -1,5 +1,5 @@
 RUN_MODE = prod
-WORK_PATH = ${WORK_PATH}
+WORK_PATH = forgejo-work-path
 
 [server]
 APP_DATA_PATH = ${WORK_PATH}/data

end-to-end.sh

@@ -15,6 +15,7 @@ SELF_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 SELF="${BASH_SOURCE[0]}"
 source $SELF_DIR/lib/lib.sh
 
+source $SELF_DIR/federation/federation.sh
 source $SELF_DIR/actions/actions.sh
 source $SELF_DIR/forgejo/fixtures.sh
 source $SELF_DIR/storage/storage.sh

federation/ONE-app.ini

@@ -0,0 +1,33 @@
+RUN_MODE = prod
+WORK_PATH = forgejo-ONE
+
+[server]
+APP_DATA_PATH = ${WORK_PATH}/data
+DOMAIN = ${IP}
+HTTP_PORT = 3001
+SSH_LISTEN_PORT = 2201
+
+[queue]
+TYPE = immediate
+
+[database]
+DB_TYPE = sqlite3
+PATH = ${WORK_PATH}/forgejo.db
+
+[log]
+MODE = file
+LEVEL = trace
+ROUTER = file
+
+[log.file]
+FILE_NAME = forgejo.log
+
+[security]
+INSTALL_LOCK = true
+
+[repository]
+ENABLE_PUSH_CREATE_USER = true
+DEFAULT_PUSH_CREATE_PRIVATE = false
+
+[federation]
+ENABLED = true

federation/TWO-app.ini

@@ -0,0 +1,33 @@
+RUN_MODE = prod
+WORK_PATH = forgejo-TWO
+
+[server]
+APP_DATA_PATH = ${WORK_PATH}/data
+DOMAIN = ${IP}
+HTTP_PORT = 3002
+SSH_LISTEN_PORT = 2202
+
+[queue]
+TYPE = immediate
+
+[database]
+DB_TYPE = sqlite3
+PATH = ${WORK_PATH}/forgejo.db
+
+[log]
+MODE = file
+LEVEL = trace
+ROUTER = file
+
+[log.file]
+FILE_NAME = forgejo.log
+
+[security]
+INSTALL_LOCK = true
+
+[repository]
+ENABLE_PUSH_CREATE_USER = true
+DEFAULT_PUSH_CREATE_PRIVATE = false
+
+[federation]
+ENABLED = true

federation/federation.sh

@@ -0,0 +1,85 @@
+# Copyright 2024 The Forgejo Authors
+# SPDX-License-Identifier: MIT
+
+FEDERATION_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+export FEDERATION_INSTANCES="ONE TWO"
+export FEDERATION_CONFIGS
+
+function federation_setup_variables() {
+    if test "$FEDERATION_CONFIGS" ; then
+	return
+    fi
+    for instance in $FEDERATION_INSTANCES ; do
+	local config=$FEDERATION_DIR/$instance-app.ini
+	FEDERATION_CONFIGS="$FEDERATION_CONFIGS $config"
+	local base=$(work_path_base $config)
+	local work_path=$DIR/$base
+	local host_port=$(get_host_port $config)
+
+	eval export ${instance}_CONFIG=$config
+	eval export ${instance}_CURL=$work_path/forgejo-curl.sh
+	eval export ${instance}_HOST_PORT=$host_port
+    done
+}
+
+function federation_verify_scenario() {
+  local scenario=$1
+
+  export scenario
+  export SCENARIO_DIR=$FEDERATION_DIR/scenario-$scenario
+
+  if test -f $SCENARIO_DIR/setup.sh ; then
+    echo "============================ SETUP scenario-$scenario ==================="
+    bash -ex $SCENARIO_DIR/setup.sh || return 1
+  fi
+
+  echo "============================ RUN scenario-$scenario ==================="
+  bash -ex $SCENARIO_DIR/run.sh || return 1
+
+  if test -f $SCENARIO_DIR/teardown.sh ; then
+    echo "============================ TEARDOWN scenario-$scenario ==================="
+    bash -ex $SCENARIO_DIR/teardown.sh || return 1
+  fi
+}
+
+function federation_setup() {
+    federation_setup_variables
+
+    local version=$1
+    federation_teardown
+
+    local config
+    for config in $FEDERATION_CONFIGS ; do
+	reset_forgejo $config
+	start_forgejo $version $config
+    done
+}
+
+function federation_teardown() {
+    federation_setup_variables
+
+    local config
+    for config in $FEDERATION_CONFIGS ; do
+	stop_forgejo $config
+    done
+}
+
+function test_federation() {
+    federation_setup_variables
+
+    local versions="${1:-$RELEASE_NUMBERS_AND_DEV}"
+
+    for version in $versions ; do
+
+        if dpkg --compare-versions $version lt 7.1 ; then
+	    continue
+	fi
+
+	federation_setup $version
+
+	for scenario in star ; do
+	    run federation_verify_scenario $scenario
+	done
+    done
+}

federation/scenario-star/run.sh

@@ -0,0 +1,2 @@
+test forgejo = $($ONE_CURL api_json $ONE_HOST_PORT/api/v1/nodeinfo | jq -r .software.name)
+test forgejo = $($TWO_CURL api_json $TWO_HOST_PORT/api/v1/nodeinfo | jq -r .software.name)

forgejo/fixtures/doctor.sh

@@ -1,7 +1,9 @@
 # SPDX-License-Identifier: MIT
 
 function doctor_run() {
-    local version=$1
+    local config=$1
+    local base=$(work_path_base $config)
+    local work_path=$DIR/$base
 
-    $DIR/forgejocli doctor check --all # --log-file -
+    $work_path/forgejocli doctor check --all # --log-file -
 }

lib/lib.sh

@@ -29,14 +29,13 @@ IP=$(hostname -I | cut -f1 -d' ')
 RELEASE_NUMBERS="7.0 8.0"
 
 PREFIX===============
-HOST_PORT=$IP:3000
 export DIR=/tmp/forgejo-end-to-end
 DIR_BINARIES=/srv/forgejo-binaries
 export DOT_FORGEJO_CURL=$DIR/forgejo-curl
 export DOT=$DOT_FORGEJO_CURL # for backward compatibility with forgejo-curl.sh 1.0.0
 : ${FORGEJO_USER:=root}
 : ${FORGEJO_PASSWORD:=admin1234}
-RELEASE_NUMBERS_AND_DEV="$(for r in $RELEASE_NUMBERS ; do echo -n "$r $r-dev " ; done)"
+: ${RELEASE_NUMBERS_AND_DEV:="$(for r in $RELEASE_NUMBERS ; do echo -n "$r $r-dev " ; done)"}
 ORGANIZATIONS=$(cat $LIB_DIR/ORGANIZATIONS)
 
 function log_info() {
@@ -52,7 +51,7 @@ function dependencies() {
 
     if ! which make curl daemon git-lfs jq sqlite3 > /dev/null ; then
         $SUDO apt-get update -qq
-        $SUDO apt-get install -y -qq make curl daemon git-lfs jq sqlite3 gettext-base
+        $SUDO apt-get install -y -qq make curl daemon git-lfs jq sqlite3
     fi
 
     if ! test -f /usr/local/bin/mc || ! test -f /usr/local/bin/minio ; then
@@ -141,9 +140,12 @@ function download() {
 }
 
 function cleanup_logs() {
-    local work_path=$DIR/forgejo-work-path
+    local config=$1
 
-    rm -f $DIR/*.log
+    local base=$(work_path_base $config)
+    local work_path=$DIR/$base
+
+    rm -f $DIR/$base*.log
     rm -f $work_path/log/*.log
 }
 
@@ -151,27 +153,37 @@ function clobber() {
     rm -fr /tmp/forgejo-end-to-end
 }
 
+function stop_forgejo() {
+    local config=$1
+
+    stop_daemon $(work_path_base $config)
+}
+
 function start_forgejo() {
     local version=$1
+    local config=$2
 
     download $version
-    start_forgejo_daemon $version $DIR_BINARIES/forgejo-$version
+    start_forgejo_daemon $version $DIR_BINARIES/forgejo-$version $config
 }
 
 function start_forgejo_daemon() {
     local version=$1
     local binary=$2
+    local config=$3
 
-    local work_path=$DIR/forgejo-work-path
+    local base=$(work_path_base $config)
-    daemon --chdir=$DIR --unsafe --env="TERM=$TERM" --env="HOME=$HOME" --env="PATH=$PATH" --pidfile=$DIR/forgejo-pid --errlog=$DIR/forgejo-err.log --output=$DIR/forgejo-out.log -- $binary --config $work_path/app.ini --work-path $work_path
+    local work_path=$DIR/$base
+    daemon --chdir=$DIR --unsafe --env="TERM=$TERM" --env="HOME=$HOME" --env="PATH=$PATH" --pidfile=$DIR/$base-pid --errlog=$DIR/$base-err.log --output=$DIR/$base-out.log -- $binary --config $work_path/app.ini --work-path $work_path
     if ! retry grep --no-messages --quiet 'Starting server on' $work_path/log/forgejo.log ; then
-        grep '' $DIR/*.log
+        grep '' $DIR/$base*.log
         grep '' $work_path/log/*.log 2> /dev/null
         return 1
     fi
-    echo "$binary --config $work_path/app.ini --work-path $work_path" '"$@"' > $DIR/forgejocli
+    echo "$binary --config $work_path/app.ini --work-path $work_path" '"$@"' > $work_path/forgejocli
-    chmod +x $DIR/forgejocli
+    chmod +x $work_path/forgejocli
-    create_user_and_login $version
+    cp -a $work_path/forgejocli $DIR/forgejocli # because setup-forgejo/forgejo-runner.sh expects it here
+    create_user_and_login $version $config
 }
 
 function start_minio() {
@@ -245,12 +257,35 @@ function start() {
     start_forgejo $version
 }
 
+function get_host_port() {
+    local config="$1"
+    if test -z "$config" ; then
+	echo "$IP:3000"
+    else
+	echo "$IP:$(sed -n -e 's/^HTTP_PORT *= *\(.*\)/\1/p' < $config)"
+    fi
+}
+
+HOST_PORT=$(get_host_port)
+
+function work_path_base() {
+    local config="$1"
+    if test -z "$config" ; then
+	echo forgejo-work-path
+    else
+	sed -n -e 's/^WORK_PATH *= *\(.*\)/\1/p' < $config
+    fi
+}
+
 function reset_forgejo() {
     local config=$1
-    local work_path=$DIR/forgejo-work-path
+    local work_path=$DIR/$(work_path_base $config)
     rm -fr $work_path
     mkdir -p $work_path
-    IP=$IP WORK_PATH=$work_path envsubst < $config > $work_path/app.ini
+    sed -e "s/\${IP}/$IP/g" \
+	-e "s|\${WORK_PATH}|$work_path|g" \
+	-e "s|^WORK_PATH = .*|WORK_PATH = $work_path|" \
+        < $config > $work_path/app.ini
 }
 
 function reset_minio() {
@@ -263,10 +298,13 @@ function reset_garage() {
 
 function create_user_and_login() {
     local version=$1
+    local config=$2
+
+    local work_path=$DIR/$(work_path_base $config)
 
     local email="$FORGEJO_USER@example.com"
-    if ! $DIR/forgejocli admin user list | grep --quiet "$email" ; then
+    if ! $work_path/forgejocli admin user list | grep --quiet "$email" ; then
-	$DIR/forgejocli admin user create --admin --username "$FORGEJO_USER" --password "$FORGEJO_PASSWORD" --email $email
+	$work_path/forgejocli admin user create --admin --username "$FORGEJO_USER" --password "$FORGEJO_PASSWORD" --email $email
     fi
 
     forgejo-curl.sh logout
@@ -274,7 +312,17 @@ function create_user_and_login() {
     if echo $version | grep --quiet 1.18 ; then
         scopes=""
     fi
-    forgejo-curl.sh --user  "$FORGEJO_USER"  --password "$FORGEJO_PASSWORD" $scopes login http://${HOST_PORT}
+    forgejo-curl.sh --user "$FORGEJO_USER" --password "$FORGEJO_PASSWORD" $scopes login http://$(get_host_port $config)
+
+    local forgejo_curl=$work_path/forgejo-curl.sh
+    cat > $forgejo_curl <<EOF
+#!/bin/bash
+export DOT_FORGEJO_CURL=$work_path/forgejo-curl
+export DOT=$work_path/forgejo-curl # for backward compatibility with forgejo-curl.sh 1.0.0
+forgejo-curl.sh "\$@"
+EOF
+    chmod +x $forgejo_curl
+    cp -a $DOT_FORGEJO_CURL $work_path/forgejo-curl
 }
 
 function stop_daemon() {
@@ -295,11 +343,13 @@ function stop_daemon() {
 }
 
 function stop() {
-    stop_daemon forgejo
+    local config="$1"
+
+    stop_forgejo $config
     stop_daemon minio
     stop_daemon garage
 
-    cleanup_logs
+    cleanup_logs $config
 }
 
 function show_logs() {
@@ -307,7 +357,7 @@ function show_logs() {
 	cd $DIR
 	set +e
         grep --with-filename --text '' *.log
-        grep --with-filename --text '' forgejo-work-path/log/*.log
+        grep --with-filename --text '' */log/*.log
 	grep --with-filename --text '' *.out
     )
 }

packages/alpine-app.ini

@@ -1,5 +1,5 @@
 RUN_MODE = prod
-WORK_PATH = ${WORK_PATH}
+WORK_PATH = forgejo-work-path
 
 [server]
 APP_DATA_PATH = ${WORK_PATH}/data

packages/alpine.sh

@@ -3,7 +3,7 @@
 
 function test_packages_alpine_version() {
     local alpine_version=$1 forgejo_version=$2
-    stop_daemon forgejo
+    stop_forgejo
     reset_forgejo $PACKAGES_DIR/alpine-app.ini
     start_forgejo $forgejo_version
 

storage/default-app.ini

@@ -1,5 +1,5 @@
 RUN_MODE = prod
-WORK_PATH = ${WORK_PATH}
+WORK_PATH = forgejo-work-path
 
 [server]
 APP_DATA_PATH = ${WORK_PATH}/data

storage/specific-app.ini

@@ -1,5 +1,5 @@
 RUN_MODE = prod
-WORK_PATH = ${WORK_PATH}
+WORK_PATH = forgejo-work-path
 
 [server]
 APP_DATA_PATH = ${WORK_PATH}/elsewhere

storage/stable-s3-app.ini

@@ -1,5 +1,5 @@
 RUN_MODE = prod
-WORK_PATH = ${WORK_PATH}
+WORK_PATH = forgejo-work-path
 
 [server]
 APP_DATA_PATH = ${WORK_PATH}/elsewhere

upgrade/default-app.ini

@@ -1,5 +1,5 @@
 RUN_MODE = prod
-WORK_PATH = ${WORK_PATH}
+WORK_PATH = forgejo-work-path
 
 [server]
 APP_DATA_PATH = ${WORK_PATH}/data

upgrade/upgrade.sh

@@ -4,7 +4,7 @@ UPGRADE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
 function upgrade_reset() {
     local config=$1
-    reset_forgejo $UPGRADE_DIR/$config-app.ini
+    reset_forgejo $config
     reset_minio
 }
 
@@ -26,8 +26,8 @@ function cleanup_storage() {
 
 function test_successful_upgrades() {
     stop
-    for config in default ; do
+    for config in $UPGRADE_DIR/default-app.ini ; do
-        log_info "using $config app.ini"
+        log_info "using $config"
         upgrade_reset $config
 
 	version=1.21
@@ -36,7 +36,7 @@ function test_successful_upgrades() {
 	start $version
 	fixture_create
 	fixture_assert
-	doctor_run $version
+	doctor_run $config
 
         for version in $RELEASE_NUMBERS_AND_DEV ; do
             stop
@@ -44,7 +44,7 @@ function test_successful_upgrades() {
             start $version
             verify_storage
 	    fixture_assert
-	    doctor_run $version
+	    doctor_run $config
         done
     done
 }
@@ -68,8 +68,8 @@ function start_gitea() {
 
 function test_gitea_upgrades() {
     stop
-    for config in default ; do
+    for config in $UPGRADE_DIR/default-app.ini ; do
-        log_info "using $config app.ini"
+        log_info "using $config"
         upgrade_reset $config
 
 	version=1.21.10
@@ -79,7 +79,7 @@ function test_gitea_upgrades() {
 	start_gitea $version
 	fixture_create
 	fixture_assert
-	doctor_run $version
+	doctor_run $config
 
         for version in 7.0-test ; do
             stop
@@ -87,7 +87,7 @@ function test_gitea_upgrades() {
             start $version
             verify_storage
 	    fixture_assert
-	    doctor_run $version
+	    doctor_run $config
         done
     done
 }