Operational notes

Postgres upgrade

1. Stop services that use postgres

systemctl stop matrix-synapse grafana

2. Login as postgres user

sudo -su postgres
old=16
cd /var/lib/postgresql/
pg_old=$(nix-build --no-out-link -A postgresql_${old:?} '<nixpkgs>')
pg_new=$(nix-build --no-out-link -A postgresql_$((old+1)) '<nixpkgs>')

3. Initialize new data directory

$pg_new/bin/initdb --encoding=UTF8 --locale=C $((old+1))

4. Run check

$pg_new/bin/pg_upgrade \
	--old-bindir=$pg_old/bin \
	--new-bindir=$pg_new/bin \
	--old-datadir=/var/lib/postgresql/${old:?} \
	--new-datadir=/var/lib/postgresql/$((old+1)) \
	-j16 \
	--clone \
	--check

5. Stop the old Postgres

systemctl stop postgresql

6. Run the migration

$pg_new/bin/pg_upgrade \
	--old-bindir=$pg_old/bin \
	--new-bindir=$pg_new/bin \
	--old-datadir=/var/lib/postgresql/${old:?} \
	--new-datadir=/var/lib/postgresql/$((old+1)) \
	-j16 \
	--clone

7. Start the new Postgres

services.postgres.packages = pkgs.postgresql_17;
nixos-rebuild switch

Cleanup (after a few days):

sudo -su postgres
vacuumdb --all --analyze-in-stages
cd /var/lib/postgresql/
./delete_old_cluster.sh
rm -v delete_old_cluster.sh

Tarball

nix build .#nixosConfigurations.<hostname>.config.system.build.image

Matrix

HTTP

Configure berlin.ccc.de web server to send federation traffic to the matrix server:

server {
  hostname berlin.ccc.de;
  location = /.well-known/matrix/server {
    default_type application/json;
    add_header Access-Control-Allow-Origin "*";
    return 200 '{"m.server":"matrix.berlin.ccc.de:443"}';
  }
  location = /.well-known/matrix/client {
    default_type application/json;
    add_header Access-Control-Allow-Origin "*";
    return 200 '{"m.homeserver": {"base_url": "https://matrix.berlin.ccc.de"}}';
  }
}

DNS

_matrix-fed._tcp.berlin.ccc.de.  IN SRV    10 0 443 matrix.berlin.ccc.de.
_matrix-fed._tcp.berlin.ccc.de.  IN SRV    10 0 443 matrix.berlin.ccc.de.
matrix.berlin.ccc.de.            IN A      195.160.173.25
matrix.berlin.ccc.de.            IN AAAA   2001:678:760:cccb::25
matrix.berlin.ccc.de.            IN CAA    0 issue "letsencrypt.org;validationmethods=http-01"
matrix.berlin.ccc.de.            IN CAA    0 iodef "mailto:caa@xengi.de"
matrix.berlin.ccc.de.            IN SSHFP  1 1 f40e117b002957939a454070adbbafe42d6f5842
matrix.berlin.ccc.de.            IN SSHFP  1 2 8ba0c605a365ef5369e91c531dd86fabfe4ce6dbd5e8280093ec2672d67c329b
matrix.berlin.ccc.de.            IN SSHFP  4 1 62d10fa57f8a1aa7469cd9b00621e4ce89261d91
matrix.berlin.ccc.de.            IN SSHFP  4 2 ca80a6685984da140ac850e4951fa31e70b616e87f62f46437af3bfd215af887

Bots

register_new_matrix_user \
    -c homeserver.yaml https://matrix.berlin.ccc.de \
    --user_domain berlin.ccc.de \
    --user <YOUR_BOT> \
    --password <YOUR_PASSWORD>

Draupnir

Remove rate limit for account:

curl -X DELETE https://matrix.berlin.ccc.de/_synapse/admin/v1/users/@admin:berlin.ccc.de/override_ratelimit

Set rate limit for account:

curl -X POST -d '{"messages_per_second":0,"burst_count":0}' https://matrix.berlin.ccc.de/_synapse/admin/v1/users/@admin:berlin.ccc.de/override_ratelimit