end-to-end/lib/lib.sh

#!/bin/bash
# Copyright 2024 The Forgejo Authors
# SPDX-License-Identifier: MIT

LIB_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

source $LIB_DIR/api.sh

if ${VERBOSE:-false}; then
    set -ex
    PS4='${BASH_SOURCE[0]}:$LINENO: ${FUNCNAME[0]}:  '
else
    set -e
fi

set -o pipefail

export DEBIAN_FRONTEND=noninteractive

if test $(id -u) != 0; then
    SUDO=sudo
fi

IP=$(hostname -I | cut -f1 -d' ')

#
# Forgejo releases for which a branch exists (7.0/forgejo etc.)
#
RELEASE_NUMBERS="7.0 8.0 9.0"

PREFIX===============
export DIR=/tmp/forgejo-end-to-end
DIR_BINARIES=/srv/forgejo-binaries
export DOT_FORGEJO_CURL=$DIR/forgejo-curl
export DOT=$DOT_FORGEJO_CURL # for backward compatibility with forgejo-curl.sh 1.0.0
: ${FORGEJO_USER:=root}
: ${FORGEJO_PASSWORD:=admin1234}
ORGANIZATIONS=$(cat $LIB_DIR/ORGANIZATIONS)

function log_info() {
    echo "$PREFIX $@"
}

function dependencies() {

    if ! test -f /usr/local/bin/forgejo-curl.sh; then
        $SUDO curl --fail -sS https://code.forgejo.org/forgejo/forgejo-curl/raw/branch/main/forgejo-curl.sh -o /usr/local/bin/forgejo-curl.sh
        $SUDO chmod +x /usr/local/bin/forgejo-curl.sh
    fi

    if ! which make curl daemon git-lfs jq sqlite3 skopeo >/dev/null; then
        $SUDO apt-get update -qq
        $SUDO apt-get install -y -qq make curl daemon git-lfs jq sqlite3 skopeo
    fi

    if ! test -f /usr/local/bin/mc || ! test -f /usr/local/bin/minio; then
        $SUDO curl --fail -sS https://dl.min.io/client/mc/release/linux-amd64/mc -o /usr/local/bin/mc
        $SUDO curl --fail -sS https://dl.min.io/server/minio/release/linux-amd64/minio -o /usr/local/bin/minio
    fi
    if ! test -x /usr/local/bin/mc || ! test -x /usr/local/bin/minio; then
        $SUDO chmod +x /usr/local/bin/mc
        $SUDO chmod +x /usr/local/bin/minio
    fi

    if ! test -f /usr/local/bin/garage >/dev/null; then
        $SUDO curl --fail -sS https://garagehq.deuxfleurs.fr/_releases/v0.8.2/x86_64-unknown-linux-musl/garage -o /usr/local/bin/garage
    fi
    if ! test -x /usr/local/bin/garage >/dev/null; then
        $SUDO chmod +x /usr/local/bin/garage
    fi
}

function build_all() {
    for dev in $RELEASE_NUMBERS; do
        local forgejo=$DIR_BINARIES/forgejo-$dev-dev
        if test -f $forgejo; then
            log_info $dev already exists
        else
            $LIB_DIR/build.sh $dev $DIR_BINARIES
            log_info $dev built from sources
        fi
    done
}

function retry() {
    rm -f $DIR/wait-for.out
    success=false
    for delay in 1 1 5 5 15; do
        if "$@" >>$DIR/wait-for.out 2>&1; then
            success=true
            break
        fi
        cat $DIR/wait-for.out
        echo waiting $delay
        sleep $delay
    done
    if test $success = false; then
        cat $DIR/wait-for.out
        return 1
    fi
}

function full_version() {
    local version=$1
    local owner=$2

    if [[ $version =~ ^[0-9]+\.[0-9]+$ ]]; then
        full_version=$(curl -sS "https://codeberg.org/api/v1/repos/$owner/forgejo/releases?limit=50" | jq -r '.[] | .tag_name | select(startswith("v'$version'"))' | sort --reverse --version-sort | head -1)
        echo ${full_version#v}
    else
        echo $version
    fi
}

function download_forgejo() {
    local version=$1

    if ! test -f $DIR_BINARIES/forgejo-$version; then
        mkdir -p $DIR_BINARIES
        for owner in $ORGANIZATIONS; do
            full_version=$(full_version $version $owner)
            if test "$full_version" = ""; then
                continue
            fi
            if wget -O $DIR_BINARIES/forgejo-$version --quiet https://codeberg.org/$owner/forgejo/releases/download/v$full_version/forgejo-$full_version-linux-amd64; then
                break
            fi
        done
        if test -s $DIR_BINARIES/forgejo-$version; then
            if test "$version" != "$full_version"; then
                log_info "downloaded Forgejo $full_version for $version"
            fi
        else
            echo unable to download Forgejo $version
            return 1
        fi
        chmod +x $DIR_BINARIES/forgejo-$version
    fi
}

function download_gitea() {
    local version=$1

    if ! test -f $DIR_BINARIES/gitea-$version; then
        mkdir -p $DIR_BINARIES
        if [[ $version =~ ^[0-9]+\.[0-9]+$ ]]; then
            full_version=$(curl -sS "https://api.github.com/repos/go-gitea/gitea/releases?per_page=100" | jq -r '.[] | .tag_name | select(startswith("v'$version'"))' | grep -v -e '-rc' | sort --reverse --version-sort | head -1)
            full_version=${full_version#v}
        else
            full_version=$version
        fi
        wget -O $DIR_BINARIES/gitea-$version --quiet https://dl.gitea.com/gitea/$full_version/gitea-$full_version-linux-amd64

        if test -s $DIR_BINARIES/gitea-$version; then
            if test "$version" != "$full_version"; then
                log_info "downloaded Gitea $full_version for $version"
            fi
        else
            echo unable to download Gitea $version
            return 1
        fi
        chmod +x $DIR_BINARIES/gitea-$version
    fi
}

function cleanup_logs() {
    local config=$1

    local base=$(work_path_base $config)
    local work_path=$DIR/$base

    rm -f $DIR/$base*.log
    rm -f $work_path/log/*.log
}

function clobber() {
    rm -fr /tmp/forgejo-end-to-end
}

: ${GITLAB_USER:=root}
: ${GITLAB_PASSWORD:=Wrobyak4}
: ${GITLAB_PORT:=8181}

function start_gitlab_cache_load() {
    local image=$1
    local d=$DIR_BINARIES/gitlab
    if test -d $d; then
        log_info "loading $image from $d"
        skopeo copy dir:$d docker-daemon:$image
    fi
}

function start_gitlab_cache_save() {
    local image=$1
    local d=$DIR_BINARIES/gitlab
    if ! test -d $d; then
        log_info "saving $image to $d"
        skopeo copy docker-daemon:$image dir:$d
    fi
}

function start_gitlab() {
    local image=$1
    local config=$2

    start_gitlab_cache_load $image

    local GITLAB_OMNIBUS_CONFIG="nginx['listen_https'] = false ; nginx['listen_port'] = 8181 ; external_url 'http://$IP:$GITLAB_PORT'; gitlab_rails['gitlab_shell_ssh_port'] = 2221; $config"
    docker run --name="test-gitlab" --shm-size=128M -d \
        -e GITLAB_OMNIBUS_CONFIG="$GITLAB_OMNIBUS_CONFIG" \
        -p 2221:22 -p $GITLAB_PORT:8181 \
        $image >&/dev/null </dev/null

    start_gitlab_cache_save $image

    for i in $(seq 10); do
        if test $(curl --silent http://$IP:$GITLAB_PORT -o /dev/null -w "%{http_code}") = 302; then
            docker exec test-gitlab gitlab-rails runner "user = User.find_by_username 'root'; user.password = '$GITLAB_PASSWORD'; user.password_confirmation = '$GITLAB_PASSWORD'; user.password_automatically_set = false ; user.save!"
            docker exec test-gitlab$serial gitlab-rails runner "Gitlab::CurrentSettings.current_application_settings.update(default_vcs_type: 'git')"
            log_info "GitLab is ready"
            return
        fi
        log_info "waiting for GitLab to come up $i"
        sleep 30
    done
    log_info "GitLab did not come up"
    docker logs test-gitlab
    false
}

function stop_gitlab() {
    docker rm -f test-gitlab
}

function stop_forgejo() {
    local config=$1

    stop_daemon $(work_path_base $config)
}

function start_gitea() {
    local version=$1
    local config=$2

    download_gitea $version
    start_forgejo_daemon $version $DIR_BINARIES/gitea-$version $config
}

function start_forgejo() {
    local version=$1
    local config=$2

    download_forgejo $version
    start_forgejo_daemon $version $DIR_BINARIES/forgejo-$version $config
}

function start_forgejo_daemon() {
    local version=$1
    local binary=$2
    local config=$3

    local base=$(work_path_base $config)
    local work_path=$DIR/$base
    daemon --chdir=$DIR --unsafe --env="TERM=$TERM" --env="HOME=$HOME" --env="PATH=$PATH" --pidfile=$DIR/$base-pid --errlog=$DIR/$base-err.log --output=$DIR/$base-out.log -- $binary --config $work_path/app.ini --work-path $work_path
    if ! retry grep --no-messages --quiet 'Starting server on' $work_path/log/forgejo.log; then
        grep '' $DIR/$base*.log
        grep '' $work_path/log/*.log 2>/dev/null
        return 1
    fi
    echo "$binary --config $work_path/app.ini --work-path $work_path" '"$@"' >$work_path/forgejocli
    chmod +x $work_path/forgejocli
    cp -a $work_path/forgejocli $DIR/forgejocli # because setup-forgejo/forgejo-runner.sh expects it here
    create_user_and_login $version $config
}

function start_minio() {
    mkdir -p $DIR/minio
    daemon --chdir=$DIR --unsafe \
        --env="PATH=$PATH" \
        --env=MINIO_ROOT_USER=123456 \
        --env=MINIO_ROOT_PASSWORD=12345678 \
        --env=MINIO_VOLUMES=$DIR/minio \
        --pidfile=$DIR/minio-pid --errlog=$DIR/minio-err.log --output=$DIR/minio-out.log -- /usr/local/bin/minio server
    retry mc alias set testS3 http://127.0.0.1:9000 123456 12345678 >&/dev/null
    mc alias set testS3 http://127.0.0.1:9000 123456 12345678
}

function start_garage() {
    mkdir -p $DIR/garage/{data,meta}
    cat >$DIR/garage/garage.toml <<EOF
metadata_dir = "$DIR/garage/meta"
data_dir = "$DIR/garage/data"
db_engine = "lmdb"

replication_mode = "none"

rpc_bind_addr = "127.0.0.1:3901"
rpc_public_addr = "127.0.0.1:3901"
rpc_secret = "$(openssl rand -hex 32)"

[s3_api]
s3_region = "us-east-1"
api_bind_addr = "127.0.0.1:9000"
root_domain = ".s3.garage.localhost"

[s3_web]
bind_addr = "127.0.0.1:3902"
root_domain = ".web.garage.localhost"
index = "index.html"

[k2v_api]
api_bind_addr = "127.0.0.1:3904"

[admin]
api_bind_addr = "127.0.0.1:3903"
admin_token = "$(openssl rand -base64 32)"
EOF

    daemon --chdir=$DIR --unsafe \
        --env="PATH=$PATH" \
        --env=RUST_LOG=garage_api=debug \
        --pidfile=$DIR/garage-pid --errlog=$DIR/garage-err.log --output=$DIR/garage-out.log -- /usr/local/bin/garage -c $DIR/garage/garage.toml server

    retry garage -c $DIR/garage/garage.toml status
    garage -c $DIR/garage/garage.toml layout assign -z dc1 -c 1 $(garage -c $DIR/garage/garage.toml status | tail -1 | grep -o '[0-9a-z]*' | head -1)
    ver=$(garage -c $DIR/garage/garage.toml layout show | grep -oP '(?<=Current cluster layout version: )\d+')
    garage -c $DIR/garage/garage.toml layout apply --version $((ver + 1))
    garage -c $DIR/garage/garage.toml key info test || garage -c $DIR/garage/garage.toml key import -n test 123456 12345678
    garage -c $DIR/garage/garage.toml key allow --create-bucket test
    retry mc alias set testS3 http://127.0.0.1:9000 123456 12345678
}

function start_s3() {
    local s3_backend=$1

    start_$s3_backend
}

function start() {
    local version=$1
    local s3_backend=${2:-minio}

    start_s3 $s3_backend
    start_forgejo $version
}

function get_host_port() {
    local config="$1"
    if test -z "$config"; then
        echo "$IP:3000"
    else
        echo "$IP:$(sed -n -e 's/^HTTP_PORT *= *\(.*\)/\1/p' <$config)"
    fi
}

HOST_PORT=$(get_host_port)

function work_path_base() {
    local config="$1"
    if test -z "$config"; then
        echo forgejo-work-path
    else
        sed -n -e 's/^WORK_PATH *= *\(.*\)/\1/p' <$config
    fi
}

function reset_forgejo() {
    local config=$1
    local work_path=$DIR/$(work_path_base $config)
    rm -fr $work_path
    mkdir -p $work_path
    sed -e "s/\${IP}/$IP/g" \
        -e "s|\${WORK_PATH}|$work_path|g" \
        -e "s|^WORK_PATH = .*|WORK_PATH = $work_path|" \
        <$config >$work_path/app.ini
}

function reset_minio() {
    rm -fr $DIR/minio
}

function reset_garage() {
    rm -fr $DIR/garage
}

function create_user_and_login() {
    local version=$1
    local config=$2

    local work_path=$DIR/$(work_path_base $config)

    local email="$FORGEJO_USER@example.com"
    if ! $work_path/forgejocli admin user list | grep --quiet "$email"; then
        $work_path/forgejocli admin user create --admin --username "$FORGEJO_USER" --password "$FORGEJO_PASSWORD" --email $email
    fi

    forgejo-curl.sh logout
    local scopes='--scopes ["all"]'
    if echo $version | grep --quiet 1.18; then
        scopes=""
    fi
    forgejo-curl.sh --user "$FORGEJO_USER" --password "$FORGEJO_PASSWORD" $scopes login http://$(get_host_port $config)

    local forgejo_curl=$work_path/forgejo-curl.sh
    cat >$forgejo_curl <<EOF
#!/bin/bash
export DOT_FORGEJO_CURL=$work_path/forgejo-curl
export DOT=$work_path/forgejo-curl # for backward compatibility with forgejo-curl.sh 1.0.0
forgejo-curl.sh "\$@"
EOF
    chmod +x $forgejo_curl
    cp -a $DOT_FORGEJO_CURL $work_path/forgejo-curl
}

function stop_daemon() {
    local daemon=$1

    if test -f $DIR/$daemon-pid; then
        local pid=$(cat $DIR/$daemon-pid)
        kill -TERM $pid
        pidwait $pid || true
        for delay in 1 1 2 2 5 5; do
            if ! test -f $DIR/$daemon-pid; then
                break
            fi
            sleep $delay
        done
        ! test -f $DIR/$daemon-pid
    fi
}

function stop() {
    local config="$1"

    stop_forgejo $config
    stop_daemon minio
    stop_daemon garage

    cleanup_logs $config
}

function show_logs() {
    if ! test -d $DIR; then
        log_info "$DIR does not exist, no logs to display"
        return
    fi
    (
        cd $DIR
        set +e
        grep --with-filename --text '' *.log
        grep --with-filename --text '' */log/*.log
        grep --with-filename --text '' *.out
    )
}

function run() {
    local fun=$1
    shift

    echo Start running $fun "$@"
    mkdir -p $DIR
    >$DIR/$fun.out
    tail --follow $DIR/$fun.out |& sed --unbuffered -n -e "/^$PREFIX/s/^$PREFIX //p" &
    local pid=$!
    if ! VERBOSE=true $SELF $fun "$@" >&$DIR/$fun.out; then
        kill $pid
        cat $DIR/$fun.out
        echo Failure running $fun
        return 1
    fi
    kill $pid
    echo Success running $fun
}