feat(federation): add scenario-mastodon setup and teardown (#1274)

2/5 taken out of #1269

This patch adds the setup and teardown of Mastodon for use in the end-to-end tests

Co-Authored-By: zam <mirco.zachmann@meissa.de>
Co-Authored-By: erik <erik.seiert@meissa-gmbh.de>
Co-Authored-By: Michael Jerger <michael.jerger@meissa-gmbh.de>

Reviewed-on: https://code.forgejo.org/forgejo/end-to-end/pulls/1274
Reviewed-by: earl-warren <earl-warren@noreply.code.forgejo.org>
Reviewed-by: Beowulf <beowulf@noreply.code.forgejo.org>
Co-authored-by: famfo <famfo@famfo.xyz>
Co-committed-by: famfo <famfo@famfo.xyz>

federation/federation.sh

@@ -78,6 +78,7 @@ function test_federation() {
         # NOTE: newline seperated, not comma :>
         scenarios=(
             "star 7.1"
+            "mastodon 14.0"
         )
 
         for scenario_version_str in "${scenarios[@]}"; do

federation/scenario-mastodon/.gitignore

@@ -0,0 +1,3 @@
+/resources/certs/forgejo*
+/forgejo/certs/*
+!/forgejo/certs/.gitkeep

federation/scenario-mastodon/TEST_INSTRUCTION.md

@@ -0,0 +1,38 @@
+# Manual testing
+
+1. compile forgejo binary to test on a debian/ubuntu system
+2. start applications
+    * local forgejo:
+        ```sh
+        cd federation/scenario-mastodon
+        # create cert & startup & create test accounts
+        SCENARIO_DIR="." ./setup.sh
+        # Mastodon password in "/tmp/forgejo-end-to-end/federation_scenario-mastodon-env"
+        # Bind forgejo to localhost:3003 and set the domain to `forgejo`
+        ```
+    * containerized forgejo:
+        ```sh
+        cd federation/scenario-mastodon
+        # Generate self-signed certs on first run
+        source ./functions.sh
+        generate_certs forgejo "./resources/certs"
+        mv "./resources/certs/forgejo"* "forgejo/certs"
+
+        # Setup container
+        export FORGEJO_PATH="/path/to/forgejo/binary"
+        export COMPOSE_PROFILES="forgejo_container"
+        export MASTODON_HOST="mastodon-app"
+
+        podman-compose up -d
+        podman-compose exec forgejo-app bash -c "/init/create_test_account.sh"
+        # Mastodon password is the last line of the output
+        podman-compose exec mastodon-app bash -c "/init/create_test_account.sh"
+        ```
+3. login to mastodon: http://localhost:4000
+   1. test@localhost - password from start app.
+4. search for forgejo user: `https://forgejo/api/v1/activitypub/user-id/1`
+5. Press follow
+6. FYI:
+   1. login to forgejo: http://localhost:3003/
+   2. me - me
+   3. swagger-uri: http://localhost:3003/api/swagger#/activitypub

federation/scenario-mastodon/compose.yaml

@@ -0,0 +1,95 @@
+networks:
+    external_network:
+    internal_network:
+        internal: true
+
+services:
+    forgejo:
+        image: code.forgejo.org/oci/alpine:latest
+        volumes:
+            - ./forgejo/certs/:/usr/local/share/ca-certificates/
+            - ./forgejo/init/:/init/
+            - ./forgejo/etc/lighttpd.conf:/etc/lighttpd.conf
+        restart: unless-stopped
+        entrypoint: "sh /init/init.sh"
+        networks:
+            - external_network
+            - internal_network
+
+    forgejo-app:
+        profiles:
+            - forgejo_container
+        # built from https://code.forgejo.org/federation/build-mastodon/src/branch/main/debian-containerfile
+        image: code.forgejo.org/federation/debian:trixie-cacerts
+        tmpfs:
+            - /data
+        volumes:
+            - ./forgejo-app/init/:/init/
+            - ./forgejo-app/config/:/config/
+            - ./resources/certs/:/usr/local/share/ca-certificates/
+            - "${FORGEJO_PATH}:/usr/local/bin/forgejo"
+        ports:
+            - 3003:3003
+        entrypoint: "sh /init/init.sh"
+        networks:
+            - external_network
+            - internal_network
+
+    postgres:
+        image: code.forgejo.org/oci/postgres:14
+        tmpfs:
+            - /var/lib/postgresql/data
+        environment:
+            POSTGRES_USER: postgres
+            POSTGRES_DB: postgres
+            POSTGRES_PASSWORD: postgres
+            POSTGRES_HOST_AUTH_METHOD: trust
+        networks:
+            - internal_network
+
+    redis:
+        image: code.forgejo.org/oci/redis:7.2
+        tmpfs:
+            - /var/lib/redis/
+        networks:
+            - internal_network
+
+    mastodon-app:
+        # built from https://code.forgejo.org/federation/build-mastodon/src/branch/main/mastodon-containerfile
+        image: code.forgejo.org/federation/mastodon:v4.5-test
+        volumes:
+            - ./resources/certs:/usr/local/share/ca-certificates/
+            - ./mastodon-app/init/:/init/
+        environment: &mastodon_env
+            DB_HOST: postgres
+            DB_USER: postgres
+            DB_PASS: postgres
+            REDIS_HOST: redis
+            PORT: 4000
+            LOCAL_DOMAIN: ${MASTODON_HOST}:4000
+            ALTERNATE_DOMAINS: ${MASTODON_HOST},localhost
+            EMAIL_DOMAIN_ALLOWLIST: localhost
+            AUTHORIZED_FETCH: "true"
+            ALLOWED_PRIVATE_ADDRESSES: 0.0.0.0/0,::/0
+            SECRET_KEY_BASE: bc1bdb4d3d57a2c292a8f145d5d3c921
+            ACTIVE_RECORD_ENCRYPTION_DETERMINISTIC_KEY: fkSxKD2bF396kdQbrP1EJ7WbU7ZgNokR
+            ACTIVE_RECORD_ENCRYPTION_KEY_DERIVATION_SALT: r0hvVmzBVsjxC7AMlwhOzmtc36ZCOS1E
+            ACTIVE_RECORD_ENCRYPTION_PRIMARY_KEY: PhdFyyfy5xJ7WVd2lWBpcPScRQHzRTNr
+        ports:
+            - "4000:4000"
+        entrypoint: "sh /init/init.sh"
+        networks:
+            - external_network
+            - internal_network
+
+    mastodon-sidekiq:
+        # built from https://code.forgejo.org/federation/build-mastodon/src/branch/main/mastodon-containerfile
+        image: code.forgejo.org/federation/mastodon:v4.5-test
+        volumes:
+            - ./resources/certs:/usr/local/share/ca-certificates/
+            - ./mastodon-sidekiq/init/:/init/
+        restart: unless-stopped
+        environment: *mastodon_env
+        entrypoint: "sh /init/init.sh"
+        networks:
+            - internal_network

federation/scenario-mastodon/forgejo-app/config/app.ini

@@ -0,0 +1,32 @@
+RUN_MODE = prod
+
+[server]
+APP_DATA_PATH = /data
+DOMAIN = forgejo
+ROOT_URL = https://forgejo/
+HTTP_PORT = 3003
+PROTOCOL = http
+
+[queue]
+TYPE = immediate
+
+[database]
+DB_TYPE = sqlite3
+PATH = /data/forgejo.db
+
+[log]
+MODE = console
+LEVEL = debug
+
+[security]
+INSTALL_LOCK = true
+
+[repository]
+ENABLE_PUSH_CREATE_USER = true
+DEFAULT_PUSH_CREATE_PRIVATE = false
+
+[federation]
+ENABLED = true
+
+[session]
+COOKIE_SECURE = false

federation/scenario-mastodon/forgejo-app/init/create_test_account.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+sleep 10
+
+/usr/local/bin/forgejo -c /etc/forgejo/app.ini admin user create --username me --password me --email "me@example.com" --admin --must-change-password=false
+/usr/local/bin/forgejo -c /etc/forgejo/app.ini admin user create --username to-be-followd --password to-be-followd --email "to-be-followd@example.com" --admin --must-change-password=false
+/usr/local/bin/forgejo -c /etc/forgejo/app.ini admin user generate-access-token -u me -t token --scopes write:activitypub,write:repository,write:user

federation/scenario-mastodon/forgejo-app/init/init.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+sudo update-ca-certificates
+
+sudo mkdir -p /data/forgejo
+sudo chown forgejo:forgejo /data/forgejo
+sudo install -D -o forgejo -g forgejo /config/app.ini /etc/forgejo/app.ini
+
+/usr/local/bin/forgejo -c /etc/forgejo/app.ini
+

federation/scenario-mastodon/forgejo/etc/lighttpd.conf

@@ -0,0 +1,22 @@
+server.document-root = "/var/www"
+
+server.modules = (
+    "mod_openssl",
+    "mod_proxy"
+)
+
+$SERVER["socket"] == ":443" {
+    ssl.engine = "enable"
+    ssl.pemfile = "/usr/local/share/ca-certificates/forgejo-snakeoil.crt"
+    ssl.privkey = "/usr/local/share/ca-certificates/forgejo-snakeoil.key"
+}
+
+proxy.server = (
+    "" => (
+        "forgejo" => (
+            "host" => "host.containers.internal",
+            "port" => 3003,
+        )
+    )
+)
+

federation/scenario-mastodon/forgejo/init/init.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env sh
+
+apk add lighttpd
+
+lighttpd -D -f /etc/lighttpd.conf
+

federation/scenario-mastodon/functions.sh

@@ -0,0 +1,32 @@
+function generate_certs() {
+
+    host=${1}
+    cert_location=${2}
+
+    rootCertificate="${cert_location}/custom-snakeoil-rootCA.crt"
+    rootCertKey="${cert_location}/custom-snakeoil-rootCA.key"
+    extensionFile="${cert_location}/${host}-snakeoil.ext"
+    keyFile="${cert_location}/${host}-snakeoil.key"
+    csrFile="${cert_location}/${host}-snakeoil.csr"
+    crtAltSubFile="${cert_location}/${host}-snakeoil.crt"
+
+    cat << EOF > ${extensionFile}
+authorityKeyIdentifier=keyid,issuer
+basicConstraints=CA:FALSE
+subjectAltName = @alt_names
+[alt_names]
+DNS.1 = ${host}
+DNS.2 = localhost
+EOF
+
+    # Create CSR
+    openssl req -newkey rsa:2048 -nodes -keyout ${keyFile} \
+        -out ${csrFile} -subj "/CN=${host}" -addext "subjectAltName=DNS:localhost"
+
+    # Sign Our CSR with the root CA cert
+    openssl x509 -req -CA ${rootCertificate} \
+        -CAkey ${rootCertKey} \
+        -in ${csrFile} \
+        -out ${crtAltSubFile} \
+        -days 3650 -CAcreateserial -extfile ${extensionFile}
+}

federation/scenario-mastodon/mastodon-app/init/create_test_account.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+account_create="$(tootctl accounts create test --email test@localhost --role Owner --confirmed)"
+tootctl account modify test --approve
+
+echo "${account_create}"
+
+password=$(echo "${account_create}" |
+        tail -n 1 |
+        sed 's/New password: //' |
+        sed 's/\r//'
+)
+
+echo "$password"

federation/scenario-mastodon/mastodon-app/init/init.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+sudo update-ca-certificates
+sudo touch /opt/mastodon/log/prod.log
+sudo chown mastodon: /opt/mastodon/log/prod.log
+rails db:setup
+bundle exec puma -C config/puma.rb
+#sleep 2h

federation/scenario-mastodon/mastodon-sidekiq/init/init.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+sudo update-ca-certificates
+sudo touch /opt/mastodon/log/prod.log
+sudo chown mastodon: /opt/mastodon/log/prod.log
+bundle exec sidekiq
+#sleep 2h

federation/scenario-mastodon/resources/certs/custom-snakeoil-rootCA.crt

@@ -0,0 +1,19 @@
+-----BEGIN CERTIFICATE-----
+MIIDGTCCAgGgAwIBAgIUU+o53CfULAHTgPuCq/Ua8JRTWLIwDQYJKoZIhvcNAQEL
+BQAwGzEZMBcGA1UEAwwQY29kZS5mb3JnZWpvLm9yZzAgFw0yNTEwMjMwODI2NTVa
+GA8yMTI1MDkyOTA4MjY1NVowGzEZMBcGA1UEAwwQY29kZS5mb3JnZWpvLm9yZzCC
+ASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKJl1YjGLoYK5J7wCbgL7WCZ
+DGznwmp2SxpxJx/9Y6Pldt1QXTp0/VrR/H8iu3bNh13jmupXTKyGHU94MTNHER1T
+/jbGw8LRPWCIYNjpYFZU17glTGs/DOieh7acuuvu+imoamk1oRLVaaHDewz09Tc2
+wkklCH+2ME2TC6mHIqpHOLO//ESEU0Glo+/mVMEcTJf9zdDhSGhZPKhXEaWgrLMo
+EV3r0hpxHCE21OeaRrOKjMtOfp6/v0yKAsJ0QbLHXw1JDauiMWHUxp1H3jIHGgtc
+0ALdiS1fhYi8zX3bkbMfctjREVtbhO0Aqps5pvDjhjTNyTmSQ3dkkU37W7Mz+WkC
+AwEAAaNTMFEwHQYDVR0OBBYEFKXOSikliKVL6VYbbcaOR3k3khKyMB8GA1UdIwQY
+MBaAFKXOSikliKVL6VYbbcaOR3k3khKyMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI
+hvcNAQELBQADggEBADE4RGzIXT3LsLGxqRiuOPrXnOTy6izD0sXraJGPWOTEm0MB
+H+MnV1YmqRiafwCTbKyiTfzF50JX0zwa6NnkK7k7tyht7O2B0/1VfsrIaXGBP05B
+pZMC1bMSaDEu+zvRUDFvNiE5Oxkw0LGy44o36e3SNCEXMCYU3fiTX/5IxfB/a1Bk
++5tNfpK4CKwyk2/pb8ClgBldYGxfp/hyzTVh7y4c5bSRzoawGxq2ipfmJbSBYEme
+vyySFXJI1W9ih8utE2sQKbRS7YrwxSNS9Uj6qwixTlHB5a/MzlFmD630VkmkI1qq
+5VkQxq7dJSTlVHX8qiQqvGoPPIp5ucbBDmyj2A8=
+-----END CERTIFICATE-----

federation/scenario-mastodon/resources/certs/custom-snakeoil-rootCA.key

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQCiZdWIxi6GCuSe
+8Am4C+1gmQxs58JqdksacScf/WOj5XbdUF06dP1a0fx/Irt2zYdd45rqV0yshh1P
+eDEzRxEdU/42xsPC0T1giGDY6WBWVNe4JUxrPwzonoe2nLrr7vopqGppNaES1Wmh
+w3sM9PU3NsJJJQh/tjBNkwuphyKqRzizv/xEhFNBpaPv5lTBHEyX/c3Q4UhoWTyo
+VxGloKyzKBFd69IacRwhNtTnmkaziozLTn6ev79MigLCdEGyx18NSQ2rojFh1Mad
+R94yBxoLXNAC3YktX4WIvM1925GzH3LY0RFbW4TtAKqbOabw44Y0zck5kkN3ZJFN
++1uzM/lpAgMBAAECggEAAdGmR0j0k/ISyfhYnFJfFAfBN+x0a1wl7rOjDP/Tg7r9
+Ln21yzYTJJcfnu5TaOfLH84KrRwrT6JhhfaYn64PC8PmH/rXDftPsFSOt/DZo2+B
+vaSgGyWcMVqdnNOOep6IXq36rr3krwQra14Rmbbm36AYihh+iuzbB4w0vPvpwDwv
+G9DITTLrIwDRbXReZ13FjqphP5dqT6jG7BnRLe7vyQ7CJwYdUDLkg2YLykwbHvco
+DISS3IoYgF5R7KaGpwH2iFmeHYwXaVH9Y0RyjEvxp87iK8Awvm4yn7hWBx9NsMm2
+of7107TlFBU81JbG09KQZ+mS3S4C9XfQScqBoP66tQKBgQDYS4QhwkYFNusN8z+U
+63IGMxJXMF0bvF1zqjhrfkdX16jDD7v8cMbZAjk80obmRdhFpNiuaCT7dbnXpt7L
+Ppfu6IkcJsAJc32lbWGc56XzfzDnNecvQbCbMnRYI5GDzcTTY+ObgyhJv4SwuD3E
+kdHhxOFPuxlLFey9eg9vC6ADnQKBgQDANX5oEqvVty8gn+cOU3fEiqQ+ZqvcgMcL
+CUQYIAUDvXjZuXTbU0cSzgZnZi3JwtEVwj0u+0eXXlju6AVgKC/yd4aIpQ2f6hXO
+UlODWIqiNq3lVgjatO6zQ2CXuBeV7crX6odNrhkvSwSPNlC2Ra7QZ8Uk0PpX0sTm
+JyuMA/WBPQKBgQCkqulPYj44nhTZrAUN9Sn7+knOQy2/feqPsln9zEe4YqFCz+nI
+SHu6nuzAl27IRQhgDR5BuVvebUQtIAeiKGc3JaWs3vt4topDtUCJWfqHpJ+whuMY
+oSQ5I3Jb38ha1f8xCG0x6ep0KvB0MfAkhPeKsH7wWnrpJSn1HsY9PlZ2KQKBgQCS
+/xZKb6UdEDipocDqkukw1bsgwhLD03TmOtLqBGrxXlFzacM2DW14sznwkBOKj0Sq
+eF+kc6Zf3Fb1d1rNHE73B3RLQre1yiedIBcgh3GW92xszSx+XwuC64+O2Mqo7jBI
+iuOpg/Fc2umEwUxe6dH1Lrd2HaCn09ikD+bc8RYsHQKBgQDXfAiAf3MdzG8+4zgD
+kzfZsUXRUm3ACqrkhlUOzPsh2y5yGO0gxkrUR4ps2+wn0aapwYi+JdbEHpDqI4Ze
+vE3d8ZgLF4ER+uuiayuItMIFa8T0AiH1oJMptUY4kgj1AaG0b3+ZxU/uGXjpciyA
+a7/psxJ+in3AQ005JGir8rx+gA==
+-----END PRIVATE KEY-----

federation/scenario-mastodon/run.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env bash
+
+source "$DIR/federation_scenario-$scenario-env"
+
+if [[ -z "$password" ]]; then
+    exit 1
+fi
+
+if [[ -z "$port" ]]; then
+    exit 1
+fi
+
+status_code="$(curl -o /dev/null -w '%{http_code}' "http://localhost:$port/")"
+if [[ "$status_code" != "200" ]]; then
+    exit 1
+fi
+

federation/scenario-mastodon/setup.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+source "$SCENARIO_DIR/../../lib/lib.sh"
+source "$SCENARIO_DIR/functions.sh"
+
+echo "Setting up Forgejo x Mastodon"
+
+if [[ ! -d "$DIR" ]]; then
+    mkdir -p "$DIR"
+fi
+
+echo "Generating forgejo & mastodon self-signed certificate"
+
+generate_certs forgejo "$SCENARIO_DIR/resources/certs"
+
+mv "$SCENARIO_DIR/resources/certs/forgejo"* "$SCENARIO_DIR/forgejo/certs/"
+
+export MASTODON_HOST="localhost"
+
+podman-compose \
+    -f "$SCENARIO_DIR/compose.yaml" \
+    up -d
+
+function wait_up() {
+    command="$(podman-compose \
+        -f "$SCENARIO_DIR/compose.yaml" \
+        exec mastodon-app \
+        bin/tootctl accounts
+    )"
+
+    first="$(head -n 1 <<< "$command")"
+    [[ "$first" == *"Commands:"* ]] && echo "ready"
+}
+
+retry wait_up
+
+password="$(podman-compose \
+    -f "$SCENARIO_DIR/compose.yaml" \
+    exec mastodon-app \
+    bash -c "/init/create_test_account.sh" | \
+    tail -n 1 | \
+    sed 's/\r//'
+)"
+
+cat << EOF > "$DIR/federation_scenario-mastodon-env"
+password="$password"
+port="4000"
+EOF

federation/scenario-mastodon/teardown.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+podman-compose \
+    -f "$SCENARIO_DIR/compose.yaml" \
+    down
+
+rm "$DIR/federation_scenario-mastodon-env"
+