From 4a19e9c90b077f78321df59f9ebca29e68cb3317 Mon Sep 17 00:00:00 2001 From: Earl Warren Date: Tue, 4 Jun 2024 12:00:42 +0200 Subject: [PATCH 1/6] lib: get the base of WORK_PATH from app.ini instead of hardcoding forgejo-work-path to be the base of WORK_PATH relative to DIR, read it from the app.ini file. It will allow multiple Forgejo instances to run simultaneously, each using a different directory. --- actions/default-app.ini | 2 +- lib/lib.sh | 14 +++++++++++--- packages/alpine-app.ini | 2 +- storage/default-app.ini | 2 +- storage/specific-app.ini | 2 +- storage/stable-s3-app.ini | 2 +- upgrade/default-app.ini | 2 +- 7 files changed, 17 insertions(+), 9 deletions(-) diff --git a/actions/default-app.ini b/actions/default-app.ini index 7ef8df1..4f30f27 100644 --- a/actions/default-app.ini +++ b/actions/default-app.ini @@ -1,5 +1,5 @@ RUN_MODE = prod -WORK_PATH = ${WORK_PATH} +WORK_PATH = forgejo-work-path [server] APP_DATA_PATH = ${WORK_PATH}/data diff --git a/lib/lib.sh b/lib/lib.sh index dc1a319..8930e44 100644 --- a/lib/lib.sh +++ b/lib/lib.sh @@ -52,7 +52,7 @@ function dependencies() { if ! which make curl daemon git-lfs jq sqlite3 > /dev/null ; then $SUDO apt-get update -qq - $SUDO apt-get install -y -qq make curl daemon git-lfs jq sqlite3 gettext-base + $SUDO apt-get install -y -qq make curl daemon git-lfs jq sqlite3 fi if ! test -f /usr/local/bin/mc || ! test -f /usr/local/bin/minio ; then @@ -247,10 +247,18 @@ function start() { function reset_forgejo() { local config=$1 - local work_path=$DIR/forgejo-work-path + local work_path_base=$(sed -n -e 's/^WORK_PATH *= *\(.*\)/\1/p' < $config) + if test -z "$work_path_base" ; then + echo "no line found starting with WORK_PATH = in the file $config" + return 1 + fi + local work_path=$DIR/$work_path_base rm -fr $work_path mkdir -p $work_path - IP=$IP WORK_PATH=$work_path envsubst < $config > $work_path/app.ini + sed -e "s/\${IP}/$IP/g" \ + -e "s|\${WORK_PATH}|$work_path|g" \ + -e "s|^WORK_PATH = .*|WORK_PATH = $work_path|" \ + < $config > $work_path/app.ini } function reset_minio() { diff --git a/packages/alpine-app.ini b/packages/alpine-app.ini index 113f242..ee378c5 100644 --- a/packages/alpine-app.ini +++ b/packages/alpine-app.ini @@ -1,5 +1,5 @@ RUN_MODE = prod -WORK_PATH = ${WORK_PATH} +WORK_PATH = forgejo-work-path [server] APP_DATA_PATH = ${WORK_PATH}/data diff --git a/storage/default-app.ini b/storage/default-app.ini index 113f242..ee378c5 100644 --- a/storage/default-app.ini +++ b/storage/default-app.ini @@ -1,5 +1,5 @@ RUN_MODE = prod -WORK_PATH = ${WORK_PATH} +WORK_PATH = forgejo-work-path [server] APP_DATA_PATH = ${WORK_PATH}/data diff --git a/storage/specific-app.ini b/storage/specific-app.ini index 980a3b0..c30911e 100644 --- a/storage/specific-app.ini +++ b/storage/specific-app.ini @@ -1,5 +1,5 @@ RUN_MODE = prod -WORK_PATH = ${WORK_PATH} +WORK_PATH = forgejo-work-path [server] APP_DATA_PATH = ${WORK_PATH}/elsewhere diff --git a/storage/stable-s3-app.ini b/storage/stable-s3-app.ini index 667bfba..cedbfa5 100644 --- a/storage/stable-s3-app.ini +++ b/storage/stable-s3-app.ini @@ -1,5 +1,5 @@ RUN_MODE = prod -WORK_PATH = ${WORK_PATH} +WORK_PATH = forgejo-work-path [server] APP_DATA_PATH = ${WORK_PATH}/elsewhere diff --git a/upgrade/default-app.ini b/upgrade/default-app.ini index e8134aa..a39c034 100644 --- a/upgrade/default-app.ini +++ b/upgrade/default-app.ini @@ -1,5 +1,5 @@ RUN_MODE = prod -WORK_PATH = ${WORK_PATH} +WORK_PATH = forgejo-work-path [server] APP_DATA_PATH = ${WORK_PATH}/data From 2461666aca3716d22194596504e8e321eef9c131 Mon Sep 17 00:00:00 2001 From: Earl Warren Date: Tue, 4 Jun 2024 13:04:05 +0200 Subject: [PATCH 2/6] lib: use the WORK_PATH base instead of forgejo for the daemon Instead of hardcoding "forgejo" as the base for the daemon to store the PID and the logs, use the base of the WORK_PATH so that a given work path can run a dedicated forgejo instance by the same name. --- actions/actions.sh | 2 +- lib/lib.sh | 48 +++++++++++++++++++++++++++++++--------------- packages/alpine.sh | 2 +- 3 files changed, 35 insertions(+), 17 deletions(-) diff --git a/actions/actions.sh b/actions/actions.sh index dd38510..ac114de 100755 --- a/actions/actions.sh +++ b/actions/actions.sh @@ -74,7 +74,7 @@ function actions_setup() { function actions_teardown() { forgejo-curl.sh logout forgejo-runner.sh teardown - stop_daemon forgejo + stop_forgejo } function actions_runner_version() { diff --git a/lib/lib.sh b/lib/lib.sh index 8930e44..0c4c843 100644 --- a/lib/lib.sh +++ b/lib/lib.sh @@ -141,9 +141,12 @@ function download() { } function cleanup_logs() { - local work_path=$DIR/forgejo-work-path + local config=$1 - rm -f $DIR/*.log + local base=$(work_path_base $config) + local work_path=$DIR/$base + + rm -f $DIR/$base*.log rm -f $work_path/log/*.log } @@ -151,21 +154,30 @@ function clobber() { rm -fr /tmp/forgejo-end-to-end } +function stop_forgejo() { + local config=$1 + + stop_daemon $(work_path_base $config) +} + function start_forgejo() { local version=$1 + local config=$2 download $version - start_forgejo_daemon $version $DIR_BINARIES/forgejo-$version + start_forgejo_daemon $version $DIR_BINARIES/forgejo-$version $config } function start_forgejo_daemon() { local version=$1 local binary=$2 + local config=$3 - local work_path=$DIR/forgejo-work-path - daemon --chdir=$DIR --unsafe --env="TERM=$TERM" --env="HOME=$HOME" --env="PATH=$PATH" --pidfile=$DIR/forgejo-pid --errlog=$DIR/forgejo-err.log --output=$DIR/forgejo-out.log -- $binary --config $work_path/app.ini --work-path $work_path + local base=$(work_path_base $config) + local work_path=$DIR/$base + daemon --chdir=$DIR --unsafe --env="TERM=$TERM" --env="HOME=$HOME" --env="PATH=$PATH" --pidfile=$DIR/$base-pid --errlog=$DIR/$base-err.log --output=$DIR/$base-out.log -- $binary --config $work_path/app.ini --work-path $work_path if ! retry grep --no-messages --quiet 'Starting server on' $work_path/log/forgejo.log ; then - grep '' $DIR/*.log + grep '' $DIR/$base*.log grep '' $work_path/log/*.log 2> /dev/null return 1 fi @@ -245,14 +257,18 @@ function start() { start_forgejo $version } +function work_path_base() { + local config="$1" + if test -z "$config" ; then + echo forgejo-work-path + else + sed -n -e 's/^WORK_PATH *= *\(.*\)/\1/p' < $config + fi +} + function reset_forgejo() { local config=$1 - local work_path_base=$(sed -n -e 's/^WORK_PATH *= *\(.*\)/\1/p' < $config) - if test -z "$work_path_base" ; then - echo "no line found starting with WORK_PATH = in the file $config" - return 1 - fi - local work_path=$DIR/$work_path_base + local work_path=$DIR/$(work_path_base $config) rm -fr $work_path mkdir -p $work_path sed -e "s/\${IP}/$IP/g" \ @@ -303,11 +319,13 @@ function stop_daemon() { } function stop() { - stop_daemon forgejo + local config="$1" + + stop_forgejo $config stop_daemon minio stop_daemon garage - cleanup_logs + cleanup_logs $config } function show_logs() { @@ -315,7 +333,7 @@ function show_logs() { cd $DIR set +e grep --with-filename --text '' *.log - grep --with-filename --text '' forgejo-work-path/log/*.log + grep --with-filename --text '' */log/*.log grep --with-filename --text '' *.out ) } diff --git a/packages/alpine.sh b/packages/alpine.sh index b3e5b5d..ae82913 100644 --- a/packages/alpine.sh +++ b/packages/alpine.sh @@ -3,7 +3,7 @@ function test_packages_alpine_version() { local alpine_version=$1 forgejo_version=$2 - stop_daemon forgejo + stop_forgejo reset_forgejo $PACKAGES_DIR/alpine-app.ini start_forgejo $forgejo_version From b62866f3aea6581ef40d019ed069438224beb1bb Mon Sep 17 00:00:00 2001 From: Earl Warren Date: Tue, 4 Jun 2024 14:41:56 +0200 Subject: [PATCH 3/6] lib: move forgejocli to the forgejo instance WORK_PATH Instead of having it in $DIR so that there can be one forgejocli per forgejo instance. --- forgejo/fixtures/doctor.sh | 6 ++++-- lib/lib.sh | 14 +++++++++----- upgrade/upgrade.sh | 18 +++++++++--------- 3 files changed, 22 insertions(+), 16 deletions(-) diff --git a/forgejo/fixtures/doctor.sh b/forgejo/fixtures/doctor.sh index 04e5b99..c0d8109 100644 --- a/forgejo/fixtures/doctor.sh +++ b/forgejo/fixtures/doctor.sh @@ -1,7 +1,9 @@ # SPDX-License-Identifier: MIT function doctor_run() { - local version=$1 + local config=$1 + local base=$(work_path_base $config) + local work_path=$DIR/$base - $DIR/forgejocli doctor check --all # --log-file - + $work_path/forgejocli doctor check --all # --log-file - } diff --git a/lib/lib.sh b/lib/lib.sh index 0c4c843..7ed14a1 100644 --- a/lib/lib.sh +++ b/lib/lib.sh @@ -181,9 +181,10 @@ function start_forgejo_daemon() { grep '' $work_path/log/*.log 2> /dev/null return 1 fi - echo "$binary --config $work_path/app.ini --work-path $work_path" '"$@"' > $DIR/forgejocli - chmod +x $DIR/forgejocli - create_user_and_login $version + echo "$binary --config $work_path/app.ini --work-path $work_path" '"$@"' > $work_path/forgejocli + chmod +x $work_path/forgejocli + cp -a $work_path/forgejocli $DIR/forgejocli # because setup-forgejo/forgejo-runner.sh expects it here + create_user_and_login $version $config } function start_minio() { @@ -287,10 +288,13 @@ function reset_garage() { function create_user_and_login() { local version=$1 + local config=$2 + + local work_path=$DIR/$(work_path_base $config) local email="$FORGEJO_USER@example.com" - if ! $DIR/forgejocli admin user list | grep --quiet "$email" ; then - $DIR/forgejocli admin user create --admin --username "$FORGEJO_USER" --password "$FORGEJO_PASSWORD" --email $email + if ! $work_path/forgejocli admin user list | grep --quiet "$email" ; then + $work_path/forgejocli admin user create --admin --username "$FORGEJO_USER" --password "$FORGEJO_PASSWORD" --email $email fi forgejo-curl.sh logout diff --git a/upgrade/upgrade.sh b/upgrade/upgrade.sh index 3b89393..8d9ccb4 100755 --- a/upgrade/upgrade.sh +++ b/upgrade/upgrade.sh @@ -4,7 +4,7 @@ UPGRADE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" function upgrade_reset() { local config=$1 - reset_forgejo $UPGRADE_DIR/$config-app.ini + reset_forgejo $config reset_minio } @@ -26,8 +26,8 @@ function cleanup_storage() { function test_successful_upgrades() { stop - for config in default ; do - log_info "using $config app.ini" + for config in $UPGRADE_DIR/default-app.ini ; do + log_info "using $config" upgrade_reset $config version=1.21 @@ -36,7 +36,7 @@ function test_successful_upgrades() { start $version fixture_create fixture_assert - doctor_run $version + doctor_run $config for version in $RELEASE_NUMBERS_AND_DEV ; do stop @@ -44,7 +44,7 @@ function test_successful_upgrades() { start $version verify_storage fixture_assert - doctor_run $version + doctor_run $config done done } @@ -68,8 +68,8 @@ function start_gitea() { function test_gitea_upgrades() { stop - for config in default ; do - log_info "using $config app.ini" + for config in $UPGRADE_DIR/default-app.ini ; do + log_info "using $config" upgrade_reset $config version=1.21.10 @@ -79,7 +79,7 @@ function test_gitea_upgrades() { start_gitea $version fixture_create fixture_assert - doctor_run $version + doctor_run $config for version in 7.0-test ; do stop @@ -87,7 +87,7 @@ function test_gitea_upgrades() { start $version verify_storage fixture_assert - doctor_run $version + doctor_run $config done done } From 078107026b8e76af75b414a654e0d7ce8e29a5b0 Mon Sep 17 00:00:00 2001 From: Earl Warren Date: Tue, 4 Jun 2024 14:57:36 +0200 Subject: [PATCH 4/6] lib: get_host_port is $IP:3000 or read from app.ini --- lib/lib.sh | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/lib/lib.sh b/lib/lib.sh index 7ed14a1..d900dd9 100644 --- a/lib/lib.sh +++ b/lib/lib.sh @@ -29,7 +29,6 @@ IP=$(hostname -I | cut -f1 -d' ') RELEASE_NUMBERS="7.0 8.0" PREFIX=============== -HOST_PORT=$IP:3000 export DIR=/tmp/forgejo-end-to-end DIR_BINARIES=/srv/forgejo-binaries export DOT_FORGEJO_CURL=$DIR/forgejo-curl @@ -258,6 +257,17 @@ function start() { start_forgejo $version } +function get_host_port() { + local config="$1" + if test -z "$config" ; then + echo "$IP:3000" + else + echo "$IP:$(sed -n -e 's/^HTTP_PORT *= *\(.*\)/\1/p' < $config)" + fi +} + +HOST_PORT=$(get_host_port) + function work_path_base() { local config="$1" if test -z "$config" ; then From 19ce0fe6b1a99fbc0d92a4596ee185056a6281f3 Mon Sep 17 00:00:00 2001 From: Earl Warren Date: Tue, 4 Jun 2024 15:29:15 +0200 Subject: [PATCH 5/6] lib: add $work_path/forgejo-curl.sh for each Forgejo instance The $work_path/forgejo-curl.sh for each Forgejo instance is logged in with admin permissions and ready to use. The default forgejo-curl.sh remains logged in the last Forgejo instance that was launched. --- lib/lib.sh | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/lib/lib.sh b/lib/lib.sh index d900dd9..7567a08 100644 --- a/lib/lib.sh +++ b/lib/lib.sh @@ -35,7 +35,7 @@ export DOT_FORGEJO_CURL=$DIR/forgejo-curl export DOT=$DOT_FORGEJO_CURL # for backward compatibility with forgejo-curl.sh 1.0.0 : ${FORGEJO_USER:=root} : ${FORGEJO_PASSWORD:=admin1234} -RELEASE_NUMBERS_AND_DEV="$(for r in $RELEASE_NUMBERS ; do echo -n "$r $r-dev " ; done)" +: ${RELEASE_NUMBERS_AND_DEV:="$(for r in $RELEASE_NUMBERS ; do echo -n "$r $r-dev " ; done)"} ORGANIZATIONS=$(cat $LIB_DIR/ORGANIZATIONS) function log_info() { @@ -312,7 +312,17 @@ function create_user_and_login() { if echo $version | grep --quiet 1.18 ; then scopes="" fi - forgejo-curl.sh --user "$FORGEJO_USER" --password "$FORGEJO_PASSWORD" $scopes login http://${HOST_PORT} + forgejo-curl.sh --user "$FORGEJO_USER" --password "$FORGEJO_PASSWORD" $scopes login http://$(get_host_port $config) + + local forgejo_curl=$work_path/forgejo-curl.sh + cat > $forgejo_curl < Date: Tue, 4 Jun 2024 16:30:48 +0200 Subject: [PATCH 6/6] federation: stub that checks the nodeinfo of the two instances --- .forgejo/workflows/end-to-end.yml | 11 ++++ end-to-end.sh | 1 + federation/ONE-app.ini | 33 +++++++++++ federation/TWO-app.ini | 33 +++++++++++ federation/federation.sh | 85 ++++++++++++++++++++++++++++ federation/scenario-star/run.sh | 2 + federation/scenario-star/setup.sh | 0 federation/scenario-star/teardown.sh | 0 8 files changed, 165 insertions(+) create mode 100644 federation/ONE-app.ini create mode 100644 federation/TWO-app.ini create mode 100755 federation/federation.sh create mode 100644 federation/scenario-star/run.sh create mode 100644 federation/scenario-star/setup.sh create mode 100644 federation/scenario-star/teardown.sh diff --git a/.forgejo/workflows/end-to-end.yml b/.forgejo/workflows/end-to-end.yml index cb8fa30..c1b9171 100644 --- a/.forgejo/workflows/end-to-end.yml +++ b/.forgejo/workflows/end-to-end.yml @@ -76,6 +76,17 @@ jobs: if: always() run: su forgejo -c "./end-to-end.sh show_logs" + federation: + needs: [build] + runs-on: lxc-bookworm + steps: + - uses: https://code.forgejo.org/actions/checkout@v4 + - uses: ./.forgejo/prepare-end-to-end + - run: su forgejo -c "./end-to-end.sh test_federation" + - name: full logs + if: always() + run: su forgejo -c "./end-to-end.sh show_logs" + actions-docs: needs: [build] runs-on: lxc-bookworm diff --git a/end-to-end.sh b/end-to-end.sh index 04478c3..ab9454b 100755 --- a/end-to-end.sh +++ b/end-to-end.sh @@ -15,6 +15,7 @@ SELF_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" SELF="${BASH_SOURCE[0]}" source $SELF_DIR/lib/lib.sh +source $SELF_DIR/federation/federation.sh source $SELF_DIR/actions/actions.sh source $SELF_DIR/forgejo/fixtures.sh source $SELF_DIR/storage/storage.sh diff --git a/federation/ONE-app.ini b/federation/ONE-app.ini new file mode 100644 index 0000000..4dcdacf --- /dev/null +++ b/federation/ONE-app.ini @@ -0,0 +1,33 @@ +RUN_MODE = prod +WORK_PATH = forgejo-ONE + +[server] +APP_DATA_PATH = ${WORK_PATH}/data +DOMAIN = ${IP} +HTTP_PORT = 3001 +SSH_LISTEN_PORT = 2201 + +[queue] +TYPE = immediate + +[database] +DB_TYPE = sqlite3 +PATH = ${WORK_PATH}/forgejo.db + +[log] +MODE = file +LEVEL = trace +ROUTER = file + +[log.file] +FILE_NAME = forgejo.log + +[security] +INSTALL_LOCK = true + +[repository] +ENABLE_PUSH_CREATE_USER = true +DEFAULT_PUSH_CREATE_PRIVATE = false + +[federation] +ENABLED = true diff --git a/federation/TWO-app.ini b/federation/TWO-app.ini new file mode 100644 index 0000000..656588a --- /dev/null +++ b/federation/TWO-app.ini @@ -0,0 +1,33 @@ +RUN_MODE = prod +WORK_PATH = forgejo-TWO + +[server] +APP_DATA_PATH = ${WORK_PATH}/data +DOMAIN = ${IP} +HTTP_PORT = 3002 +SSH_LISTEN_PORT = 2202 + +[queue] +TYPE = immediate + +[database] +DB_TYPE = sqlite3 +PATH = ${WORK_PATH}/forgejo.db + +[log] +MODE = file +LEVEL = trace +ROUTER = file + +[log.file] +FILE_NAME = forgejo.log + +[security] +INSTALL_LOCK = true + +[repository] +ENABLE_PUSH_CREATE_USER = true +DEFAULT_PUSH_CREATE_PRIVATE = false + +[federation] +ENABLED = true diff --git a/federation/federation.sh b/federation/federation.sh new file mode 100755 index 0000000..de1e515 --- /dev/null +++ b/federation/federation.sh @@ -0,0 +1,85 @@ +# Copyright 2024 The Forgejo Authors +# SPDX-License-Identifier: MIT + +FEDERATION_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +export FEDERATION_INSTANCES="ONE TWO" +export FEDERATION_CONFIGS + +function federation_setup_variables() { + if test "$FEDERATION_CONFIGS" ; then + return + fi + for instance in $FEDERATION_INSTANCES ; do + local config=$FEDERATION_DIR/$instance-app.ini + FEDERATION_CONFIGS="$FEDERATION_CONFIGS $config" + local base=$(work_path_base $config) + local work_path=$DIR/$base + local host_port=$(get_host_port $config) + + eval export ${instance}_CONFIG=$config + eval export ${instance}_CURL=$work_path/forgejo-curl.sh + eval export ${instance}_HOST_PORT=$host_port + done +} + +function federation_verify_scenario() { + local scenario=$1 + + export scenario + export SCENARIO_DIR=$FEDERATION_DIR/scenario-$scenario + + if test -f $SCENARIO_DIR/setup.sh ; then + echo "============================ SETUP scenario-$scenario ===================" + bash -ex $SCENARIO_DIR/setup.sh || return 1 + fi + + echo "============================ RUN scenario-$scenario ===================" + bash -ex $SCENARIO_DIR/run.sh || return 1 + + if test -f $SCENARIO_DIR/teardown.sh ; then + echo "============================ TEARDOWN scenario-$scenario ===================" + bash -ex $SCENARIO_DIR/teardown.sh || return 1 + fi +} + +function federation_setup() { + federation_setup_variables + + local version=$1 + federation_teardown + + local config + for config in $FEDERATION_CONFIGS ; do + reset_forgejo $config + start_forgejo $version $config + done +} + +function federation_teardown() { + federation_setup_variables + + local config + for config in $FEDERATION_CONFIGS ; do + stop_forgejo $config + done +} + +function test_federation() { + federation_setup_variables + + local versions="${1:-$RELEASE_NUMBERS_AND_DEV}" + + for version in $versions ; do + + if dpkg --compare-versions $version lt 7.1 ; then + continue + fi + + federation_setup $version + + for scenario in star ; do + run federation_verify_scenario $scenario + done + done +} diff --git a/federation/scenario-star/run.sh b/federation/scenario-star/run.sh new file mode 100644 index 0000000..8d10fd2 --- /dev/null +++ b/federation/scenario-star/run.sh @@ -0,0 +1,2 @@ +test forgejo = $($ONE_CURL api_json $ONE_HOST_PORT/api/v1/nodeinfo | jq -r .software.name) +test forgejo = $($TWO_CURL api_json $TWO_HOST_PORT/api/v1/nodeinfo | jq -r .software.name) diff --git a/federation/scenario-star/setup.sh b/federation/scenario-star/setup.sh new file mode 100644 index 0000000..e69de29 diff --git a/federation/scenario-star/teardown.sh b/federation/scenario-star/teardown.sh new file mode 100644 index 0000000..e69de29