Compare commits

..
25 changed files with 350 additions and 319 deletions

View file

@ -1,5 +1,11 @@
# Nix based CCCB infra
## Folder structure
- `./services` holds generic service configuration that is shared between hosts
- `./hosts` holds host specific configuration
- `./secrets` holds age encrypted secrets using [agenix](https://github.com/ryantm/agenix)
## Admin handbook
### Update a container
@ -7,7 +13,7 @@
```shell
ssh <container>
cd /etc/nixos
nix run .#apps.nixos-diff # Show what changes would be applied
nix run .#apps.nixos-diff # git pull + build + diff wth running config
nixos-rebuild switch # Apply changes
```
@ -15,12 +21,13 @@ nixos-rebuild switch # Apply changes
Production:
- [Matrix](./README.matrix.md)
- [Matrix (matrix.berlin.ccc.de)](./README.matrix.md)
Testing:
- [Hedgedoc](./README.hedgedoc.md)
- [Postgres](./README.postgres.md)
- [Hedgedoc (md.berlin.ccc.de)](./README.hedgedoc.md)
- [Postgres (sql.berlin.ccc.de)](./README.postgres.md)
- [Grafana/Prometheus (monitoring.berlin.ccc.de)](./README.monitoring.md)
---

10
README.monitoring.md Normal file
View file

@ -0,0 +1,10 @@
# Monitoring
## Grafana
## Prometheus
---
Build with ❤️ and ❄️.

View file

@ -100,6 +100,16 @@
modules = [
agenix.nixosModules.default
{ environment.systemPackages = [ (agenix.packages.${system}.default) ]; }
{
age.secrets = {
hedgedoc-env = {
file = ./secrets/hedgedoc-env.age;
mode = "440";
owner = "hedgedoc";
group = "hedgedoc";
};
};
}
./hosts/md
];
};
@ -124,11 +134,53 @@
group = "postgres";
mode = "0400";
};
postgres-grafana = {
file = ./secrets/postgres-grafana.age;
owner = "postgres";
group = "postgres";
mode = "0400";
};
};
}
./hosts/sql
];
};
nixosConfigurations."monitoring" = nixpkgs.lib.nixosSystem {
#system = "x86_64-linux";
#pkgs = import nixpkgs { inherit system; };
inherit system;
modules = [
agenix.nixosModules.default
{ environment.systemPackages = [ (agenix.packages.${system}.default) ]; }
{
age.secrets = {
postgres-grafana = {
file = ./secrets/postgres-grafana.age;
owner = "postgres";
group = "postgres";
mode = "0400";
};
};
}
./hosts/monitoring
];
};
nixosConfigurations."www" = nixpkgs.lib.nixosSystem {
#system = "x86_64-linux";
#pkgs = import nixpkgs { inherit system; };
inherit system;
modules = [
./hosts/www
];
};
nixosConfigurations."git-run" = nixpkgs.lib.nixosSystem {
#system = "x86_64-linux";
#pkgs = import nixpkgs { inherit system; };
inherit system;
modules = [
./hosts/git-run
];
};
};
#);
}

View file

@ -8,6 +8,7 @@
{
imports = [
(modulesPath + "/virtualisation/proxmox-lxc.nix")
../services/node-exporter.nix
];
systemd.suppressedSystemUnits = [

View file

@ -5,11 +5,9 @@
../common.nix
../../services/openssh.nix
../../services/nginx.nix
../../services/postgres.nix
../../services/synapse.nix
../../services/draupnir.nix
../../services/prometheus.nix
../../services/grafana.nix
./nginx.nix
./synapse.nix
./draupnir.nix
];
networking = {

16
hosts/matrix/draupnir.nix Normal file
View file

@ -0,0 +1,16 @@
{ config, ... }:
{
services.draupnir = {
enable = true;
settings = {
homeserverUrl = "https://matrix.berlin.ccc.de";
managementRoom = "!ZYWNuaQBkkenNklCSm:matrix.org"; # #cccb-moderators:berlin.ccc.de
autojoinOnlyIfManager = true;
recordIgnoredInvites = true;
roomStateBackingStore.enabled = true;
displayReports = true;
};
secrets.accessToken = config.age.secrets.draupnir_access_token.path;
};
}

24
hosts/matrix/nginx.nix Normal file
View file

@ -0,0 +1,24 @@
{ config, pkgs, ... }:
{
services.nginx.virtualHosts."matrix.berlin.ccc.de" = {
default = true;
quic = true;
kTLS = true;
forceSSL = true;
enableACME = true;
locations = {
#"/.well-known/acme-challenge".root = config.security.acme.defaults.webroot;
"/".return = "418 \"🫖\"";
"~ ^(/_matrix|/_synapse/client)" = {
recommendedProxySettings = true;
proxyPass = "http://[::1]:8008";
extraConfig = ''
client_max_body_size 64M;
proxy_set_header X-Request-ID $request_id;
proxy_http_version 1.1;
'';
};
};
};
}

74
hosts/matrix/synapse.nix Normal file
View file

@ -0,0 +1,74 @@
{ config, ... }:
{
networking.firewall.extraInputRules = ''
ip saddr 195.160.173.14 tcp dport 9009 accept
ip6 saddr 2001:678:760:cccb::14 tcp dport 9009 accept
'';
services = {
matrix-synapse = {
enable = true;
settings = {
server_name = config.networking.domain;
public_baseurl = "https://${config.networking.hostName}.${config.networking.domain}:443/";
# Creates "/var/lib/matrix-synapse/homeserver.signing.key" on first launch
signing_key_path = config.age.secrets.matrix_signing_key.path;
registration_shared_secret_path = config.age.secrets.matrix_registration_shared_secret.path;
database = {
name = "psycopg2";
args = {
user = "matrix-synapse";
database = "matrix-synapse";
};
};
listeners = [
{
type = "http";
tls = false;
x_forwarded = true;
port = 8008;
bind_addresses = [ "::1" ];
#request_id_header = "X-Request-ID";
resources = [
{
compress = false;
names = [
"client"
"federation"
];
}
];
}
{
type = "metrics";
tls = false;
port = 9009;
bind_addresses = ["::" "0.0.0.0"];
resources = [
{
compress = false;
names = [ "metrics" ];
}
];
}
];
enable_metrics = true;
dynamic_thumbnails = true;
max_upload_size = "128M";
max_image_pixels = "64M";
retention = {
enabled = true;
default_policy = {
min_lifetime = "1d";
max_lifetime = "1y";
};
allowed_lifetime_min = "1d";
allowed_lifetime_max = "1y";
};
};
enableRegistrationScript = true;
};
};
}

View file

@ -4,7 +4,7 @@
imports = [
../common.nix
../../services/openssh.nix
../../services/hedgedoc.nix
./hedgedoc.nix
];
networking = {

24
hosts/md/hedgedoc.nix Normal file
View file

@ -0,0 +1,24 @@
{ config, pkgs, ... }:
let
db = {
host = "sql.berlin.ccc.de";
port = 5432;
username = "hedgedoc";
database = "hedgedoc";
};
in
{
services.hedgedoc = {
enable = true;
settings = {
domain = "${config.networking.hostName}.${config.networking.domain}";
dbURL = "postgres://${db.username}:\${DB_PASSWORD}@${db.host}:${toString db.port}/${db.name}";
# sync with config.age.secrets.postgres-hedgedoc.path
environmentFile = config.age.secrets.hedgedoc-env.path;
protocolUseSSL = true;
enableStatsApi = true;
};
};
}

44
hosts/md/nginx.nix Normal file
View file

@ -0,0 +1,44 @@
{ config, pkgs, ... }:
let
cfg = config.services.hedgedoc.settings;
in
{
services.nginx.virtualHosts."${config.networking.hostName}.${config.networking.domain}" = {
default = true;
quic = true;
kTLS = true;
forceSSL = true;
enableACME = true;
locations = {
"/" = {
recommendedProxySettings = true;
proxyPass = "http://${cfg.host}:${toString cfg.port}";
};
"/socket.io/" = {
recommendedProxySettings = true;
proxyWebsockets = true;
proxyPass = "http://${cfg.host}:${toString cfg.port}";
};
"/metrics" = {
recommendedProxySettings = true;
proxyPass = "http://${cfg.host}:${toString cfg.port}";
extraConfig = ''
allow 195.160.173.14;
allow 2001:678:760:cccb::14;
deny all;
'';
};
"/status" = {
recommendedProxySettings = true;
proxyPass = "http://${cfg.host}:${toString cfg.port}";
extraConfig = ''
allow 195.160.173.14;
allow 2001:678:760:cccb::14;
deny all;
'';
};
};
};
}

View file

@ -0,0 +1,41 @@
{ ... }:
{
imports = [
../common.nix
../../services/openssh.nix
../../services/nginx.nix
./nginx.nix
./prometheus.nix
./grafana.nix
];
networking = {
hostName = "monitoring";
firewall = {
allowedTCPPorts = [
80 # HTTP/1
443 # HTTP/2
];
allowedUDPPorts = [
443 # HTTP/3
];
};
};
services = {
openssh.banner = ''
__
__/\ \__ __
___ ___ ___ ___ /\_\ \ ,_\ ___ _ __ /\_\ ___ __
/' __` __`\ / __`\ /' _ `\/\ \ \ \/ / __`\/\`'__\/\ \ /' _ `\ /'_ `\
/\ \/\ \/\ \/\ \L\ \/\ \/\ \ \ \ \ \_/\ \L\ \ \ \/ \ \ \/\ \/\ \/\ \L\ \
\ \_\ \_\ \_\ \____/\ \_\ \_\ \_\ \__\ \____/\ \_\ \ \_\ \_\ \_\ \____ \
\/_/\/_/\/_/\/___/ \/_/\/_/\/_/\/__/\/___/ \/_/ \/_/\/_/\/_/\/___L\ \
/\____/
\_/__/
'';
};
system.stateVersion = "25.11";
}

View file

@ -0,0 +1,47 @@
{ config, ... }:
# Dashboards:
# - Synapse: https://github.com/element-hq/synapse/tree/master/contrib/grafana
{
services = {
grafana = {
enable = true;
settings = {
server.http_addr = "::1";
database = {
type = "postgres";
host = "sql.berlin.ccc.de";
name = "grafana";
user = "grafana";
password = "$__file{${config.age.secrets.postgres_grafana.path}}";
};
security = {
secret_key = "$__file{${config.age.secrets.grafana_secret_key.path}}";
admin_user = "xengi";
admin_password = "$__file{${config.age.secrets.grafana_admin_password.path}}";
admin_email = "grafana@xengi.de";
};
analytics = {
reporting_enabled = false;
feedback_links_enabled = false;
};
};
provision = {
enable = true;
datasources.settings.datasources = [
{
name = "Prometheus";
type = "prometheus";
url = "http://${config.services.prometheus.listenAddress}:${toString config.services.prometheus.port}";
jsonData = {
httpMethod = "GET";
prometheusType = "Prometheus";
cacheLevel = "High";
};
}
];
};
};
};
}

View file

@ -0,0 +1,29 @@
{ config, pkgs, ... }:
{
services.nginx = {
upstreams.grafana.servers."localhost:3000" = {};
virtualHosts."${config.networking.hostName}.${config.networking.domain}" = {
default = true;
quic = true;
kTLS = true;
forceSSL = true;
enableACME = true;
#auth_basic "Administrator’s Area";
#auth_basic_user_file ${config.age.secrets.grafana-basic-auth.path};
locations = {
#"/.well-known/acme-challenge".root = config.security.acme.defaults.webroot;
"/" = {
recommendedProxySettings = true;
proxyPass = "http://grafana";
};
"/api/live/" = {
recommendedProxySettings = true;
proxyWebsockets = true;
proxyPass = "http://grafana";
};
};
};
};
}

View file

@ -0,0 +1,88 @@
{ config, pkgs, lib, ... }:
{
services.prometheus = {
enable = true;
retentionTime = "14d";
listenAddress = "[::1]";
scrapeConfigs = [
{
job_name = "hedgedoc";
scrape_interval = "15s";
scheme = "https";
static_configs = [{ targets = ["md.berlin.ccc.de:443"]; }];
}
{
job_name = "synapse";
scrape_interval = "15s";
static_configs = [{ targets = ["matrix.berlin.ccc.de:9009"]; }];
}
{
job_name = "node";
scrape_interval = "15s";
static_configs = [
{
targets = [
"matrix.${config.networking.domain}:${toString config.services.prometheus.exporters.node.port}"
"md.${config.networking.domain}:${toString config.services.prometheus.exporters.node.port}"
"postgres.${config.networking.domain}:${toString config.services.prometheus.exporters.node.port}"
"monitoring:${toString config.services.prometheus.exporters.node.port}"
];
}
];
}
{
job_name = "nginx";
scrape_interval = "15s";
static_configs = [
{
targets = [
"monitoring:${toString config.services.prometheus.exporters.nginx.port}"
"matrix:${toString config.services.prometheus.exporters.nginx.port}"
];
}
];
}
];
ruleFiles = [
# https://github.com/element-hq/synapse/tree/master/contrib/prometheus
(pkgs.writeText "prom-synapse-rules.yaml" ''
groups:
- name: synapse
rules:
- record: 'synapse_federation_client_sent'
labels:
type: "EDU"
expr: 'synapse_federation_client_sent_edus_total + 0'
- record: 'synapse_federation_client_sent'
labels:
type: "PDU"
expr: 'synapse_federation_client_sent_pdu_destinations_count_total + 0'
- record: 'synapse_federation_client_sent'
labels:
type: "Query"
expr: 'sum(synapse_federation_client_sent_queries) by (job)'
- record: 'synapse_federation_server_received'
labels:
type: "EDU"
expr: 'synapse_federation_server_received_edus_total + 0'
- record: 'synapse_federation_server_received'
labels:
type: "PDU"
expr: 'synapse_federation_server_received_pdus_total + 0'
- record: 'synapse_federation_server_received'
labels:
type: "Query"
expr: 'sum(synapse_federation_server_received_queries) by (job)'
- record: 'synapse_federation_transaction_queue_pending'
labels:
type: "EDU"
expr: 'synapse_federation_transaction_queue_pending_edus + 0'
- record: 'synapse_federation_transaction_queue_pending'
labels:
type: "PDU"
expr: 'synapse_federation_transaction_queue_pending_pdus + 0'
'')
];
};
}

View file

@ -1,35 +0,0 @@
{ ... }:
{
imports = [
../common.nix
../../services/openssh.nix
../../services/powerdns.nix
];
networking = {
hostName = "powerdns";
firewall = {
allowedTCPPorts = [
53 # DNS
];
allowedUDPPorts = [
53 # DNS
];
};
};
services = {
openssh.banner = ''
__ __
/\ \__ /\ \
___ ____ ___ ____\ \ ,_\ ___ ___ ___\ \ \____
/' _ `\ /',__\ / __`\ /',__\\ \ \/ /'___\ /'___\ /'___\ \ '__`\
/\ \/\ \/\__, `\__/\ \L\ \/\__, `\\ \ \_ __/\ \__//\ \__//\ \__/\ \ \L\ \
\ \_\ \_\/\____/\_\ \____/\/\____/ \ \__\/\_\ \____\ \____\ \____\\ \_,__/
\/_/\/_/\/___/\/_/\/___/ \/___/ \/__/\/_/\/____/\/____/\/____/ \/___/
'';
};
system.stateVersion = "25.11";
}

88
hosts/sql/postgresql.nix Normal file
View file

@ -0,0 +1,88 @@
{ config, pkgs, ... }:
let
fqdn = "sql.${config.networking.domain}";
# Create postgres-<username> entry in agenix
# mkEntry <username> <last IP address octet>
entries = [
(mkEntry "matrix-synapse" 25) # matrix.berlin.ccc.de
(mkEntry "hedgedoc" 26) # md.berlin.ccc.de
(mkEntry "grafana" 255) # mon.berlin.ccc.de
];
mkEntry = name: octet: {
user = {
name = name;
ensureDBOwnership = true;
};
database = name;
# TYPE DATABASE USER ADDRESS METHOD
auth = ''
#hostssl ${name} ${name} 195.160.173.${toString octet}/32 scram-sha-256
#hostssl ${name} ${name} 2001:678:760:cccb::${toString octet}/128 scram-sha-256
host ${name} ${name} 195.160.173.${toString octet}/32 scram-sha-256
host ${name} ${name} 2001:678:760:cccb::${toString octet}/128 scram-sha-256
'';
};
mkPasswordSQL = e: ''
DO $do$
BEGIN
IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = '${e.user.name}') THEN
EXECUTE format(
'ALTER ROLE %I WITH PASSWORD %L',
'${e.user.name}',
trim(both E'\n' from pg_read_file('${config.age.secrets."postgres-${e.user.name}".path}'))
);
END IF;
END
$do$;
'';
passwordScript = pkgs.writeText "postgres-passwords.sql" (builtins.concatStringsSep "\n" (map mkPasswordSQL entries));
in
{
services = {
#nginx = {
# enable = true;
# virtualHosts."${fqdn}" = {
# enableACME = true;
# locations."/".return = "418";
# };
#};
postgresql = {
enable = true;
package = pkgs.postgresql_18;
enableJIT = true;
initdbArgs = [
"--locale=C"
"--encoding=UTF8"
];
settings.listen_addresses = "*";
enableTCPIP = true;
#settings = {
# ssl = "on";
# ssl_cert_file = "${config.security.acme.certs."${fqdn}".directory}/server.crt";
# ssl_key_file = "${config.security.acme.certs."${fqdn}".directory}/server.key";
# ssl_ca_file = "${config.security.acme.certs."${fqdn}".directory}/ca.crt";
#};
ensureUsers = map (e: e.user) entries;
ensureDatabases = map (e: e.database) entries;
authentication = "${builtins.concatStringsSep "\n" (map (e: e.auth) entries)}";
};
postgresqlBackup = {
enable = true;
startAt = "@daily";
compression = "zstd";
};
prometheus.exporters.postgres = {
enable = true;
openFirewall = true;
firewallRules = services.prometheus.exporters.node.firewallRules;
}:
};
systemd.services.postgresql.postStart = ''
${config.services.postgresql.package}/bin/psql \
--dbname=postgres \
--no-password \
--file=${passwordScript}
'';
}

19
secrets/hedgedoc-env.age Normal file
View file

@ -0,0 +1,19 @@
age-encryption.org/v1
-> ssh-ed25519 uH+n1w Hl5GuG/K5bKRhYLwG3z1pUfRPPs/O0T9ELWTAMT/DG8
9nX8jNHmCQXa5Wv7huCTEk5kz3Me9QVSxJYPdrH5udw
-> ssh-ed25519 EvLbWw T+0rljpwVOktHR21v5JvBPe1nog0TVN1erGtIyRMbQ4
tpNz5eRC8vFFbdtXA6Vp+7X1VDq5doJi4hM1K/FOyVE
-> ssh-ed25519 dM+fLQ R9t+cS4ye0jOaubNcMaqu8/APLkzAopkZh76tM0jQgM
DL4DykdFXXQONPqDGv5LKTrlg9+4BPHdXNPMGwPF7a4
-> ssh-ed25519 jxWM2Q PTEewNsybqn/4gejSGy5BNucQ5izKtqUGp6mGroYizg
HEaBhzmp+0ymUUbzCgb4KpyZJ2lKYKNlaI9zMY58CJg
-> ssh-ed25519 /yCUCg 6wdCIPRGgPnPxzCdUDnDOl5lI2Fsl9DoA4QmM3DWfEs
ZdQ1sHtAsYrlaWNDZmf2+Gu9vIIp7adD6MI1oJyPPdU
-> ssh-ed25519 FGp51g 1Lvo3hKE72UUKJaN4U1XlXoP8j7EAHN0UPIP11FurCI
csB1x/PYsQgQ0gPHJAD8EcHHVo1JJ8NCtx45KpreqaE
-> ssh-ed25519 fEJY/A LlmksK8HR5YNpMwcqJUN5sgAM8jXCuanTNU+A53UMhY
kntfp+IUE1OLq03WLyuynyqSeUlrhy5piYKcqg9/DAg
--- NxH5yKP69Pq4DQx2Ziad7ECw6BdlbSfo7+vm9V3YWm8
•–
ЧµiŽmßÈñ¬ð²åè<>®v0Šêà_d§’$Z#[.vÿÒØ£Xæä×<10>•jý±³ZH's€Òy4Q§ƒ·<C692>³²mÊ”<>
—ZÂö]^ÂŒ¥ nEÊ>w ¥f7ˆ<37><CB86>%~¿äD¼ÞV²

Binary file not shown.

View file

@ -17,17 +17,22 @@ let
_matrix = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIApAkkhHLj918co/wUGuyW8WCPYHxsNM4uo32XDEu7VV root@matrix";
_md = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFdFkdEEDXo8+k5YZpI1O2GqZlxcpCDtxqVun35duITm root@md";
_sql = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPcSXjDSyVVVdJbpheOhT0fIuOGFk+jsHhjrAVnBNLQV root@sql";
_mon = "";
in
{
"pushover_app_token.age".publicKeys = users ++ [ _matrix ];
"pushover_user_key.age".publicKeys = users ++ [ _matrix ];
"matrix_admin_password.age".publicKeys = users;
"draupnir_access_token.age".publicKeys = users ++ [ _matrix ];
"matrix_signing_key.age".publicKeys = users ++ [ _matrix ];
"matrix_registration_shared_secret.age".publicKeys = users ++ [ _matrix ];
"pushover_app_token.age".publicKeys = users ++ [ _matrix ];
"pushover_user_key.age".publicKeys = users ++ [ _matrix ];
"grafana_admin_password.age".publicKeys = users ++ [ _matrix ];
"grafana_secret_key.age".publicKeys = users ++ [ _matrix ];
"hedgedoc-env.age".publicKeys = users ++ [ _md ];
"postgres-matrix-synapse.age".publicKeys = users ++ [ _sql _matrix ];
"postgres-hedgedoc.age".publicKeys = users ++ [ _sql _md ];
"postgres-grafana.age".publicKeys = users ++ [ _sql _mon ];
}

Some files were not shown because too many files have changed in this diff Show more