Compare commits

..

No commits in common. "cd2b61f1fb73c109acb025f1ec01f1e1f5f20f42" and "854dace74e992bd86d8e9575581e47982809d5e9" have entirely different histories.

25 changed files with 319 additions and 350 deletions

View file

@ -1,11 +1,5 @@
# Nix based CCCB infra
## Folder structure
- `./services` holds generic service configuration that is shared between hosts
- `./hosts` holds host specific configuration
- `./secrets` holds age encrypted secrets using [agenix](https://github.com/ryantm/agenix)
## Admin handbook
### Update a container
@ -13,7 +7,7 @@
```shell
ssh <container>
cd /etc/nixos
nix run .#apps.nixos-diff # git pull + build + diff wth running config
nix run .#apps.nixos-diff # Show what changes would be applied
nixos-rebuild switch # Apply changes
```
@ -21,13 +15,12 @@ nixos-rebuild switch # Apply changes
Production:
- [Matrix (matrix.berlin.ccc.de)](./README.matrix.md)
- [Matrix](./README.matrix.md)
Testing:
- [Hedgedoc (md.berlin.ccc.de)](./README.hedgedoc.md)
- [Postgres (sql.berlin.ccc.de)](./README.postgres.md)
- [Grafana/Prometheus (monitoring.berlin.ccc.de)](./README.monitoring.md)
- [Hedgedoc](./README.hedgedoc.md)
- [Postgres](./README.postgres.md)
---

View file

@ -1,10 +0,0 @@
# Monitoring
## Grafana
## Prometheus
---
Build with ❤️ and ❄️.

View file

@ -100,16 +100,6 @@
modules = [
agenix.nixosModules.default
{ environment.systemPackages = [ (agenix.packages.${system}.default) ]; }
{
age.secrets = {
hedgedoc-env = {
file = ./secrets/hedgedoc-env.age;
mode = "440";
owner = "hedgedoc";
group = "hedgedoc";
};
};
}
./hosts/md
];
};
@ -134,53 +124,11 @@
group = "postgres";
mode = "0400";
};
postgres-grafana = {
file = ./secrets/postgres-grafana.age;
owner = "postgres";
group = "postgres";
mode = "0400";
};
};
}
./hosts/sql
];
};
nixosConfigurations."monitoring" = nixpkgs.lib.nixosSystem {
#system = "x86_64-linux";
#pkgs = import nixpkgs { inherit system; };
inherit system;
modules = [
agenix.nixosModules.default
{ environment.systemPackages = [ (agenix.packages.${system}.default) ]; }
{
age.secrets = {
postgres-grafana = {
file = ./secrets/postgres-grafana.age;
owner = "postgres";
group = "postgres";
mode = "0400";
};
};
}
./hosts/monitoring
];
};
nixosConfigurations."www" = nixpkgs.lib.nixosSystem {
#system = "x86_64-linux";
#pkgs = import nixpkgs { inherit system; };
inherit system;
modules = [
./hosts/www
];
};
nixosConfigurations."git-run" = nixpkgs.lib.nixosSystem {
#system = "x86_64-linux";
#pkgs = import nixpkgs { inherit system; };
inherit system;
modules = [
./hosts/git-run
];
};
};
#);
}

View file

@ -8,7 +8,6 @@
{
imports = [
(modulesPath + "/virtualisation/proxmox-lxc.nix")
../services/node-exporter.nix
];
systemd.suppressedSystemUnits = [

View file

@ -5,9 +5,11 @@
../common.nix
../../services/openssh.nix
../../services/nginx.nix
./nginx.nix
./synapse.nix
./draupnir.nix
../../services/postgres.nix
../../services/synapse.nix
../../services/draupnir.nix
../../services/prometheus.nix
../../services/grafana.nix
];
networking = {

View file

@ -1,24 +0,0 @@
{ config, pkgs, ... }:
{
services.nginx.virtualHosts."matrix.berlin.ccc.de" = {
default = true;
quic = true;
kTLS = true;
forceSSL = true;
enableACME = true;
locations = {
#"/.well-known/acme-challenge".root = config.security.acme.defaults.webroot;
"/".return = "418 \"🫖\"";
"~ ^(/_matrix|/_synapse/client)" = {
recommendedProxySettings = true;
proxyPass = "http://[::1]:8008";
extraConfig = ''
client_max_body_size 64M;
proxy_set_header X-Request-ID $request_id;
proxy_http_version 1.1;
'';
};
};
};
}

View file

@ -4,7 +4,7 @@
imports = [
../common.nix
../../services/openssh.nix
./hedgedoc.nix
../../services/hedgedoc.nix
];
networking = {

View file

@ -1,24 +0,0 @@
{ config, pkgs, ... }:
let
db = {
host = "sql.berlin.ccc.de";
port = 5432;
username = "hedgedoc";
database = "hedgedoc";
};
in
{
services.hedgedoc = {
enable = true;
settings = {
domain = "${config.networking.hostName}.${config.networking.domain}";
dbURL = "postgres://${db.username}:\${DB_PASSWORD}@${db.host}:${toString db.port}/${db.name}";
# sync with config.age.secrets.postgres-hedgedoc.path
environmentFile = config.age.secrets.hedgedoc-env.path;
protocolUseSSL = true;
enableStatsApi = true;
};
};
}

View file

@ -1,44 +0,0 @@
{ config, pkgs, ... }:
let
cfg = config.services.hedgedoc.settings;
in
{
services.nginx.virtualHosts."${config.networking.hostName}.${config.networking.domain}" = {
default = true;
quic = true;
kTLS = true;
forceSSL = true;
enableACME = true;
locations = {
"/" = {
recommendedProxySettings = true;
proxyPass = "http://${cfg.host}:${toString cfg.port}";
};
"/socket.io/" = {
recommendedProxySettings = true;
proxyWebsockets = true;
proxyPass = "http://${cfg.host}:${toString cfg.port}";
};
"/metrics" = {
recommendedProxySettings = true;
proxyPass = "http://${cfg.host}:${toString cfg.port}";
extraConfig = ''
allow 195.160.173.14;
allow 2001:678:760:cccb::14;
deny all;
'';
};
"/status" = {
recommendedProxySettings = true;
proxyPass = "http://${cfg.host}:${toString cfg.port}";
extraConfig = ''
allow 195.160.173.14;
allow 2001:678:760:cccb::14;
deny all;
'';
};
};
};
}

View file

@ -1,41 +0,0 @@
{ ... }:
{
imports = [
../common.nix
../../services/openssh.nix
../../services/nginx.nix
./nginx.nix
./prometheus.nix
./grafana.nix
];
networking = {
hostName = "monitoring";
firewall = {
allowedTCPPorts = [
80 # HTTP/1
443 # HTTP/2
];
allowedUDPPorts = [
443 # HTTP/3
];
};
};
services = {
openssh.banner = ''
__
__/\ \__ __
___ ___ ___ ___ /\_\ \ ,_\ ___ _ __ /\_\ ___ __
/' __` __`\ / __`\ /' _ `\/\ \ \ \/ / __`\/\`'__\/\ \ /' _ `\ /'_ `\
/\ \/\ \/\ \/\ \L\ \/\ \/\ \ \ \ \ \_/\ \L\ \ \ \/ \ \ \/\ \/\ \/\ \L\ \
\ \_\ \_\ \_\ \____/\ \_\ \_\ \_\ \__\ \____/\ \_\ \ \_\ \_\ \_\ \____ \
\/_/\/_/\/_/\/___/ \/_/\/_/\/_/\/__/\/___/ \/_/ \/_/\/_/\/_/\/___L\ \
/\____/
\_/__/
'';
};
system.stateVersion = "25.11";
}

View file

@ -1,29 +0,0 @@
{ config, pkgs, ... }:
{
services.nginx = {
upstreams.grafana.servers."localhost:3000" = {};
virtualHosts."${config.networking.hostName}.${config.networking.domain}" = {
default = true;
quic = true;
kTLS = true;
forceSSL = true;
enableACME = true;
#auth_basic "Administrators Area";
#auth_basic_user_file ${config.age.secrets.grafana-basic-auth.path};
locations = {
#"/.well-known/acme-challenge".root = config.security.acme.defaults.webroot;
"/" = {
recommendedProxySettings = true;
proxyPass = "http://grafana";
};
"/api/live/" = {
recommendedProxySettings = true;
proxyWebsockets = true;
proxyPass = "http://grafana";
};
};
};
};
}

View file

@ -0,0 +1,35 @@
{ ... }:
{
imports = [
../common.nix
../../services/openssh.nix
../../services/powerdns.nix
];
networking = {
hostName = "powerdns";
firewall = {
allowedTCPPorts = [
53 # DNS
];
allowedUDPPorts = [
53 # DNS
];
};
};
services = {
openssh.banner = ''
__ __
/\ \__ /\ \
___ ____ ___ ____\ \ ,_\ ___ ___ ___\ \ \____
/' _ `\ /',__\ / __`\ /',__\\ \ \/ /'___\ /'___\ /'___\ \ '__`\
/\ \/\ \/\__, `\__/\ \L\ \/\__, `\\ \ \_ __/\ \__//\ \__//\ \__/\ \ \L\ \
\ \_\ \_\/\____/\_\ \____/\/\____/ \ \__\/\_\ \____\ \____\ \____\\ \_,__/
\/_/\/_/\/___/\/_/\/___/ \/___/ \/__/\/_/\/____/\/____/\/____/ \/___/
'';
};
system.stateVersion = "25.11";
}

View file

@ -7,7 +7,6 @@ let
entries = [
(mkEntry "matrix-synapse" 25) # matrix.berlin.ccc.de
(mkEntry "hedgedoc" 26) # md.berlin.ccc.de
(mkEntry "grafana" 255) # mon.berlin.ccc.de
];
mkEntry = name: octet: {
user = {
@ -48,14 +47,6 @@ in
# };
#};
postgresql = {
enable = true;
package = pkgs.postgresql_18;
enableJIT = true;
initdbArgs = [
"--locale=C"
"--encoding=UTF8"
];
settings.listen_addresses = "*";
enableTCPIP = true;
#settings = {
# ssl = "on";
@ -67,16 +58,6 @@ in
ensureDatabases = map (e: e.database) entries;
authentication = "${builtins.concatStringsSep "\n" (map (e: e.auth) entries)}";
};
postgresqlBackup = {
enable = true;
startAt = "@daily";
compression = "zstd";
};
prometheus.exporters.postgres = {
enable = true;
openFirewall = true;
firewallRules = services.prometheus.exporters.node.firewallRules;
}:
};
systemd.services.postgresql.postStart = ''
${config.services.postgresql.package}/bin/psql \

View file

@ -1,19 +0,0 @@
age-encryption.org/v1
-> ssh-ed25519 uH+n1w Hl5GuG/K5bKRhYLwG3z1pUfRPPs/O0T9ELWTAMT/DG8
9nX8jNHmCQXa5Wv7huCTEk5kz3Me9QVSxJYPdrH5udw
-> ssh-ed25519 EvLbWw T+0rljpwVOktHR21v5JvBPe1nog0TVN1erGtIyRMbQ4
tpNz5eRC8vFFbdtXA6Vp+7X1VDq5doJi4hM1K/FOyVE
-> ssh-ed25519 dM+fLQ R9t+cS4ye0jOaubNcMaqu8/APLkzAopkZh76tM0jQgM
DL4DykdFXXQONPqDGv5LKTrlg9+4BPHdXNPMGwPF7a4
-> ssh-ed25519 jxWM2Q PTEewNsybqn/4gejSGy5BNucQ5izKtqUGp6mGroYizg
HEaBhzmp+0ymUUbzCgb4KpyZJ2lKYKNlaI9zMY58CJg
-> ssh-ed25519 /yCUCg 6wdCIPRGgPnPxzCdUDnDOl5lI2Fsl9DoA4QmM3DWfEs
ZdQ1sHtAsYrlaWNDZmf2+Gu9vIIp7adD6MI1oJyPPdU
-> ssh-ed25519 FGp51g 1Lvo3hKE72UUKJaN4U1XlXoP8j7EAHN0UPIP11FurCI
csB1x/PYsQgQ0gPHJAD8EcHHVo1JJ8NCtx45KpreqaE
-> ssh-ed25519 fEJY/A LlmksK8HR5YNpMwcqJUN5sgAM8jXCuanTNU+A53UMhY
kntfp+IUE1OLq03WLyuynyqSeUlrhy5piYKcqg9/DAg
--- NxH5yKP69Pq4DQx2Ziad7ECw6BdlbSfo7+vm9V3YWm8
•–
ЧµiŽmßÈñ¬ð²åè<>®v0Šêà_d§’$Z#[.vÿÒØ£Xæä×<10>•jý±³ZH's€Òy4Q§ƒ·<C692>³²<>
—ZÂö]^ÂŒ¥ nEÊ>w ¥f7ˆ<37><CB86>%~¿äD¼Þ

Binary file not shown.

View file

@ -17,22 +17,17 @@ let
_matrix = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIApAkkhHLj918co/wUGuyW8WCPYHxsNM4uo32XDEu7VV root@matrix";
_md = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFdFkdEEDXo8+k5YZpI1O2GqZlxcpCDtxqVun35duITm root@md";
_sql = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPcSXjDSyVVVdJbpheOhT0fIuOGFk+jsHhjrAVnBNLQV root@sql";
_mon = "";
in
{
"pushover_app_token.age".publicKeys = users ++ [ _matrix ];
"pushover_user_key.age".publicKeys = users ++ [ _matrix ];
"matrix_admin_password.age".publicKeys = users;
"draupnir_access_token.age".publicKeys = users ++ [ _matrix ];
"matrix_signing_key.age".publicKeys = users ++ [ _matrix ];
"matrix_registration_shared_secret.age".publicKeys = users ++ [ _matrix ];
"pushover_app_token.age".publicKeys = users ++ [ _matrix ];
"pushover_user_key.age".publicKeys = users ++ [ _matrix ];
"grafana_admin_password.age".publicKeys = users ++ [ _matrix ];
"grafana_secret_key.age".publicKeys = users ++ [ _matrix ];
"hedgedoc-env.age".publicKeys = users ++ [ _md ];
"postgres-matrix-synapse.age".publicKeys = users ++ [ _sql _matrix ];
"postgres-hedgedoc.age".publicKeys = users ++ [ _sql _md ];
"postgres-grafana.age".publicKeys = users ++ [ _sql _mon ];
}

View file

@ -11,10 +11,9 @@
server.http_addr = "::1";
database = {
type = "postgres";
host = "sql.berlin.ccc.de";
name = "grafana";
user = "grafana";
password = "$__file{${config.age.secrets.postgres_grafana.path}}";
host = "/run/postgresql";
};
security = {
secret_key = "$__file{${config.age.secrets.grafana_secret_key.path}}";
@ -43,5 +42,17 @@
];
};
};
postgresql = {
ensureUsers = [
{
name = config.services.grafana.settings.database.user;
ensureDBOwnership = true;
}
];
ensureDatabases = [
config.services.grafana.settings.database.name
];
};
};
}

89
services/hedgedoc.nix Normal file
View file

@ -0,0 +1,89 @@
{ config, pkgs, ... }:
let
fqdn = "hedgedoc.berlin.ccc.de";
cfg = config.services.hedgedoc.settings;
in
{
services = {
hedgedoc = {
enable = true;
settings = {
domain = fqdn;
#environmentFile = config.age.secrets.hedgedoc_settings.path;
protocolUseSSL = true;
db = {
dialect = "postgresql";
host = "/run/postgresql";
username = "hedgedoc";
database = "hedgedoc";
};
enableStatsApi = true;
};
};
nginx = {
enable = true;
resolver.addresses = [
"[2606:4700:4700::1111]"
"[2620:fe::fe]"
"1.1.1.1"
"9.9.9.9"
];
statusPage = true; # http://127.0.0.1/nginx_status
sslProtocols = "TLSv1.3";
recommendedTlsSettings = true;
recommendedOptimisation = true;
recommendedGzipSettings = true;
recommendedBrotliSettings = true;
virtualHosts."${fqdn}" = {
default = true;
quic = true;
kTLS = true;
forceSSL = true;
enableACME = true;
locations = {
"/" = {
proxyPass = "http://${cfg.host}:${toString cfg.port}";
recommendedProxySettings = true;
};
"/socket.io/" = {
proxyPass = "http://${cfg.host}:${toString cfg.port}";
proxyWebsockets = true;
recommendedProxySettings = true;
};
"/metrics" = {
proxyPass = "http://${cfg.host}:${toString cfg.port}";
recommendedProxySettings = true;
#allow 195.160.173.255;
#allow 2001:678:760:cccb::ffff;
#deny all;
};
"/status" = {
proxyPass = "http://${cfg.host}:${toString cfg.port}";
recommendedProxySettings = true;
#allow 195.160.173.255;
#allow 2001:678:760:cccb::ffff;
#deny all;
};
};
};
};
postgresql = {
enable = true;
package = pkgs.postgresql_18;
enableJIT = true;
initdbArgs = [
"--locale=C"
"--encoding=UTF8"
];
ensureUsers = [{ name = cfg.db.username; ensureDBOwnership = true; }];
ensureDatabases = [ cfg.db.database ];
};
postgresqlBackup = {
enable = true;
startAt = "*-*-* 09:00:00";
compression = "zstd";
};
};
}

View file

@ -1,10 +1,12 @@
{ config, pkgs, ... }:
let
fqdn = "matrix.berlin.ccc.de";
in
{
users.users.nginx.extraGroups = [ "acme" ];
services = {
nginx = {
services.nginx = {
enable = true;
resolver.addresses = [
"[2606:4700:4700::1111]"
@ -18,11 +20,25 @@
recommendedOptimisation = true;
recommendedGzipSettings = true;
recommendedBrotliSettings = true;
virtualHosts."${fqdn}" = {
default = true;
quic = true;
kTLS = true;
forceSSL = true;
enableACME = true;
locations = {
#"/.well-known/acme-challenge".root = config.security.acme.defaults.webroot;
"/".return = "418 \"🫖\"";
"~ ^(/_matrix|/_synapse/client)" = {
recommendedProxySettings = true;
proxyPass = "http://[::1]:8008";
extraConfig = ''
client_max_body_size 64M;
proxy_set_header X-Request-ID $request_id;
proxy_http_version 1.1;
'';
};
};
prometheus.exporters.nginx = {
enable = true;
firewallRules = config.services.prometheus.exporters.node.firewallRules;
openFirewall = true;
};
};
}

View file

@ -1,13 +0,0 @@
{ ... }:
{
services.prometheus.exporters.node = {
enable = true;
openFirewall = true;
firewallRules = ''
ip saddr 195.160.173.14 tcp dport 9187 counter accept
ip6 saddr 2001:678:760:cccb::14 tcp dport 9187 counter accept
'';
};
}

21
services/postgres.nix Normal file
View file

@ -0,0 +1,21 @@
{ pkgs, ... }:
{
services = {
postgresql = {
enable = true;
package = pkgs.postgresql_18;
enableJIT = true;
initdbArgs = [
"--locale=C"
"--encoding=UTF8"
];
settings.listen_addresses = "*";
};
postgresqlBackup = {
enable = true;
startAt = "@daily";
compression = "zstd";
};
};
}

72
services/powerdns.nix Normal file
View file

@ -0,0 +1,72 @@
{ config, ... }:
{
# exposes prometheus metrics at http://127.0.0.1:8081/metrics
services = {
powerdns = {
enable = true;
secretFile = config.age.secrets.powerdns.path;
# API_KEY=supersecret123!
# WEBSERVER_PASSWORD=supersecre123!
extraConfig = ''
api=yes
api-key=$API_KEY
local-address=0.0.0.0, ::
local-port=53
log-timestamp=no # journald already does this
resolver=127.0.0.54:5300 # Used for ALIAS lookup
secondary=yes
version-string=anonymous
webserver-password=$WEBSERVER_PASSWORD
webserver-port=8081
launch=bind
'';
};
powerdns-admin = {
enable = true;
secretKeyFile = config.age.secrets.powerdns-admin-cookie-secret.path;
saltFile = config.age.secrets.powerdns-admin-salt.path;
extraArgs = [];
config = ''
# PDA
SIGNUP_ENABLED = True
LOCAL_DB_ENABLED = True
# Flask
BIND_ADDRESS = '127.0.0.1'
PORT = 8000
#SESSION_COOKIE_SECURE = True
# Flask-Session
import cachelib
SESSION_TYPE = 'cachelib'
SESSION_CACHELIB = cachelib.simple.SimpleCache()
# Flask-SQLAlchemy
SQLALCHEMY_DATABASE_URI = 'postgresql://powerdnsadmin@/powerdnsadmin?host=/run/postgresql'
SQLALCHEMY_TRACK_MODIFICATIONS = True
# FLask-SeaSurf
#CSRF_COOKIE_SECURE = True
'';
};
postgresql = {
enable = true;
package = pkgs.postgresql_18;
ensureUsers = [
{
name = "pda";
ensureDBOwnership = true;
}
];
ensureDatabases = [ "pda" ];
};
postgresqlBackup = {
enable = true;
compression = "zstd";
startAt = "@midnight";
};
};
}

View file

@ -5,42 +5,43 @@
enable = true;
retentionTime = "14d";
listenAddress = "[::1]";
exporters = {
node = {
enable = true;
listenAddress = config.services.prometheus.listenAddress;
};
nginx = {
enable = true;
listenAddress = config.services.prometheus.listenAddress;
};
#postgres = {};
};
scrapeConfigs = [
{
job_name = "hedgedoc";
scrape_interval = "15s";
scheme = "https";
static_configs = [{ targets = ["md.berlin.ccc.de:443"]; }];
}
{
job_name = "synapse";
scrape_interval = "15s";
static_configs = [{ targets = ["matrix.berlin.ccc.de:9009"]; }];
static_configs = [
{
targets = lib.pipe config.services.matrix-synapse.settings.listeners [
(lib.filter (l: l.type == "metrics"))
builtins.head
(l: [ "[${builtins.head l.bind_addresses}]:${toString l.port}" ])
];
}
];
}
{
job_name = "node";
scrape_interval = "15s";
static_configs = [
{
targets = [
"matrix.${config.networking.domain}:${toString config.services.prometheus.exporters.node.port}"
"md.${config.networking.domain}:${toString config.services.prometheus.exporters.node.port}"
"postgres.${config.networking.domain}:${toString config.services.prometheus.exporters.node.port}"
"monitoring:${toString config.services.prometheus.exporters.node.port}"
];
}
{ targets = [ "${config.services.prometheus.exporters.node.listenAddress}:${toString config.services.prometheus.exporters.node.port}" ]; }
];
}
{
job_name = "nginx";
scrape_interval = "15s";
static_configs = [
{
targets = [
"monitoring:${toString config.services.prometheus.exporters.nginx.port}"
"matrix:${toString config.services.prometheus.exporters.nginx.port}"
];
}
{ targets = [ "${config.services.prometheus.exporters.nginx.listenAddress}:${toString config.services.prometheus.exporters.nginx.port}" ]; }
];
}
];

View file

@ -1,18 +1,16 @@
{ config, ... }:
let
domain = "berlin.ccc.de";
in
{
networking.firewall.extraInputRules = ''
ip saddr 195.160.173.14 tcp dport 9009 accept
ip6 saddr 2001:678:760:cccb::14 tcp dport 9009 accept
'';
services = {
matrix-synapse = {
enable = true;
settings = {
server_name = config.networking.domain;
public_baseurl = "https://${config.networking.hostName}.${config.networking.domain}:443/";
# Creates "/var/lib/matrix-synapse/homeserver.signing.key" on first launch
server_name = domain;
public_baseurl = "https://matrix.${domain}:443/";
# "/var/lib/matrix-synapse/homeserver.signing.key"
signing_key_path = config.age.secrets.matrix_signing_key.path;
registration_shared_secret_path = config.age.secrets.matrix_registration_shared_secret.path;
database = {
@ -44,7 +42,7 @@
type = "metrics";
tls = false;
port = 9009;
bind_addresses = ["::" "0.0.0.0"];
bind_addresses = [ "::1" ];
resources = [
{
compress = false;
@ -70,5 +68,17 @@
};
enableRegistrationScript = true;
};
postgresql = {
ensureUsers = [
{
name = config.services.matrix-synapse.settings.database.args.user;
ensureDBOwnership = true;
}
];
ensureDatabases = [
config.services.matrix-synapse.settings.database.args.database
];
};
};
}