improve all the things
This commit is contained in:
parent
7cbd49fe42
commit
0357003655
10 changed files with 171 additions and 41 deletions
15
README.md
15
README.md
|
|
@ -1,5 +1,11 @@
|
||||||
# Nix based CCCB infra
|
# Nix based CCCB infra
|
||||||
|
|
||||||
|
## Folder structure
|
||||||
|
|
||||||
|
- `./services` holds generic service configuration that is shared between hosts
|
||||||
|
- `./hosts` holds host specific configuration
|
||||||
|
- `./secrets` holds age encrypted secrets using [agenix](https://github.com/ryantm/agenix)
|
||||||
|
|
||||||
## Admin handbook
|
## Admin handbook
|
||||||
|
|
||||||
### Update a container
|
### Update a container
|
||||||
|
|
@ -7,7 +13,7 @@
|
||||||
```shell
|
```shell
|
||||||
ssh <container>
|
ssh <container>
|
||||||
cd /etc/nixos
|
cd /etc/nixos
|
||||||
nix run .#apps.nixos-diff # Show what changes would be applied
|
nix run .#apps.nixos-diff # git pull + build + diff wth running config
|
||||||
nixos-rebuild switch # Apply changes
|
nixos-rebuild switch # Apply changes
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
@ -15,12 +21,13 @@ nixos-rebuild switch # Apply changes
|
||||||
|
|
||||||
Production:
|
Production:
|
||||||
|
|
||||||
- [Matrix](./README.matrix.md)
|
- [Matrix (matrix.berlin.ccc.de)](./README.matrix.md)
|
||||||
|
|
||||||
Testing:
|
Testing:
|
||||||
|
|
||||||
- [Hedgedoc](./README.hedgedoc.md)
|
- [Hedgedoc (md.berlin.ccc.de)](./README.hedgedoc.md)
|
||||||
- [Postgres](./README.postgres.md)
|
- [Postgres (sql.berlin.ccc.de)](./README.postgres.md)
|
||||||
|
- [Grafana/Prometheus (monitoring.berlin.ccc.de)](./README.monitoring.md)
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|
|
||||||
10
README.monitoring.md
Normal file
10
README.monitoring.md
Normal file
|
|
@ -0,0 +1,10 @@
|
||||||
|
# Monitoring
|
||||||
|
|
||||||
|
## Grafana
|
||||||
|
|
||||||
|
## Prometheus
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
Build with ❤️ and ❄️.
|
||||||
|
|
||||||
36
flake.nix
36
flake.nix
|
|
@ -100,6 +100,16 @@
|
||||||
modules = [
|
modules = [
|
||||||
agenix.nixosModules.default
|
agenix.nixosModules.default
|
||||||
{ environment.systemPackages = [ (agenix.packages.${system}.default) ]; }
|
{ environment.systemPackages = [ (agenix.packages.${system}.default) ]; }
|
||||||
|
{
|
||||||
|
age.secrets = {
|
||||||
|
hedgedoc-env = {
|
||||||
|
file = ./secrets/hedgedoc-env.age;
|
||||||
|
mode = "440";
|
||||||
|
owner = "hedgedoc";
|
||||||
|
group = "hedgedoc";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
}
|
||||||
./hosts/md
|
./hosts/md
|
||||||
];
|
];
|
||||||
};
|
};
|
||||||
|
|
@ -124,11 +134,37 @@
|
||||||
group = "postgres";
|
group = "postgres";
|
||||||
mode = "0400";
|
mode = "0400";
|
||||||
};
|
};
|
||||||
|
postgres-grafana = {
|
||||||
|
file = ./secrets/postgres-grafana.age;
|
||||||
|
owner = "postgres";
|
||||||
|
group = "postgres";
|
||||||
|
mode = "0400";
|
||||||
|
};
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
./hosts/sql
|
./hosts/sql
|
||||||
];
|
];
|
||||||
};
|
};
|
||||||
|
nixosConfigurations."monitoring" = nixpkgs.lib.nixosSystem {
|
||||||
|
#system = "x86_64-linux";
|
||||||
|
#pkgs = import nixpkgs { inherit system; };
|
||||||
|
inherit system;
|
||||||
|
modules = [
|
||||||
|
agenix.nixosModules.default
|
||||||
|
{ environment.systemPackages = [ (agenix.packages.${system}.default) ]; }
|
||||||
|
{
|
||||||
|
age.secrets = {
|
||||||
|
postgres-grafana = {
|
||||||
|
file = ./secrets/postgres-grafana.age;
|
||||||
|
owner = "postgres";
|
||||||
|
group = "postgres";
|
||||||
|
mode = "0400";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
}
|
||||||
|
./hosts/monitoring
|
||||||
|
];
|
||||||
|
};
|
||||||
};
|
};
|
||||||
#);
|
#);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -8,6 +8,7 @@
|
||||||
{
|
{
|
||||||
imports = [
|
imports = [
|
||||||
(modulesPath + "/virtualisation/proxmox-lxc.nix")
|
(modulesPath + "/virtualisation/proxmox-lxc.nix")
|
||||||
|
../../services/node-exporter.nix
|
||||||
];
|
];
|
||||||
|
|
||||||
systemd.suppressedSystemUnits = [
|
systemd.suppressedSystemUnits = [
|
||||||
|
|
|
||||||
41
hosts/monitoring/default.nix
Normal file
41
hosts/monitoring/default.nix
Normal file
|
|
@ -0,0 +1,41 @@
|
||||||
|
{ ... }:
|
||||||
|
|
||||||
|
{
|
||||||
|
imports = [
|
||||||
|
../common.nix
|
||||||
|
../../services/openssh.nix
|
||||||
|
../../services/nginx.nix
|
||||||
|
./nginx.nix
|
||||||
|
./prometheus.nix
|
||||||
|
./grafana.nix
|
||||||
|
];
|
||||||
|
|
||||||
|
networking = {
|
||||||
|
hostName = "monitoring";
|
||||||
|
firewall = {
|
||||||
|
allowedTCPPorts = [
|
||||||
|
80 # HTTP/1
|
||||||
|
443 # HTTP/2
|
||||||
|
];
|
||||||
|
allowedUDPPorts = [
|
||||||
|
443 # HTTP/3
|
||||||
|
];
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
services = {
|
||||||
|
openssh.banner = ''
|
||||||
|
__
|
||||||
|
__/\ \__ __
|
||||||
|
___ ___ ___ ___ /\_\ \ ,_\ ___ _ __ /\_\ ___ __
|
||||||
|
/' __` __`\ / __`\ /' _ `\/\ \ \ \/ / __`\/\`'__\/\ \ /' _ `\ /'_ `\
|
||||||
|
/\ \/\ \/\ \/\ \L\ \/\ \/\ \ \ \ \ \_/\ \L\ \ \ \/ \ \ \/\ \/\ \/\ \L\ \
|
||||||
|
\ \_\ \_\ \_\ \____/\ \_\ \_\ \_\ \__\ \____/\ \_\ \ \_\ \_\ \_\ \____ \
|
||||||
|
\/_/\/_/\/_/\/___/ \/_/\/_/\/_/\/__/\/___/ \/_/ \/_/\/_/\/_/\/___L\ \
|
||||||
|
/\____/
|
||||||
|
\_/__/
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
|
||||||
|
system.stateVersion = "25.11";
|
||||||
|
}
|
||||||
47
hosts/monitoring/grafana.nix
Normal file
47
hosts/monitoring/grafana.nix
Normal file
|
|
@ -0,0 +1,47 @@
|
||||||
|
{ config, ... }:
|
||||||
|
|
||||||
|
# Dashboards:
|
||||||
|
# - Synapse: https://github.com/element-hq/synapse/tree/master/contrib/grafana
|
||||||
|
|
||||||
|
{
|
||||||
|
services = {
|
||||||
|
grafana = {
|
||||||
|
enable = true;
|
||||||
|
settings = {
|
||||||
|
server.http_addr = "::1";
|
||||||
|
database = {
|
||||||
|
type = "postgres";
|
||||||
|
host = "sql.berlin.ccc.de";
|
||||||
|
name = "grafana";
|
||||||
|
user = "grafana";
|
||||||
|
password = "$__file{${config.age.secrets.postgres_grafana.path}}";
|
||||||
|
};
|
||||||
|
security = {
|
||||||
|
secret_key = "$__file{${config.age.secrets.grafana_secret_key.path}}";
|
||||||
|
admin_user = "xengi";
|
||||||
|
admin_password = "$__file{${config.age.secrets.grafana_admin_password.path}}";
|
||||||
|
admin_email = "grafana@xengi.de";
|
||||||
|
};
|
||||||
|
analytics = {
|
||||||
|
reporting_enabled = false;
|
||||||
|
feedback_links_enabled = false;
|
||||||
|
};
|
||||||
|
};
|
||||||
|
provision = {
|
||||||
|
enable = true;
|
||||||
|
datasources.settings.datasources = [
|
||||||
|
{
|
||||||
|
name = "Prometheus";
|
||||||
|
type = "prometheus";
|
||||||
|
url = "http://${config.services.prometheus.listenAddress}:${toString config.services.prometheus.port}";
|
||||||
|
jsonData = {
|
||||||
|
httpMethod = "GET";
|
||||||
|
prometheusType = "Prometheus";
|
||||||
|
cacheLevel = "High";
|
||||||
|
};
|
||||||
|
}
|
||||||
|
];
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
}
|
||||||
29
hosts/monitoring/nginx.nix
Normal file
29
hosts/monitoring/nginx.nix
Normal file
|
|
@ -0,0 +1,29 @@
|
||||||
|
{ config, pkgs, ... }:
|
||||||
|
|
||||||
|
{
|
||||||
|
services.nginx = {
|
||||||
|
upstreams.grafana.servers."localhost:3000" = {};
|
||||||
|
virtualHosts."${config.networking.hostName}.${config.networking.domain}" = {
|
||||||
|
default = true;
|
||||||
|
quic = true;
|
||||||
|
kTLS = true;
|
||||||
|
forceSSL = true;
|
||||||
|
enableACME = true;
|
||||||
|
#auth_basic "Administrator’s Area";
|
||||||
|
#auth_basic_user_file ${config.age.secrets.grafana-basic-auth.path};
|
||||||
|
locations = {
|
||||||
|
#"/.well-known/acme-challenge".root = config.security.acme.defaults.webroot;
|
||||||
|
"/" = {
|
||||||
|
recommendedProxySettings = true;
|
||||||
|
proxyPass = "http://grafana";
|
||||||
|
};
|
||||||
|
"/api/live/" = {
|
||||||
|
recommendedProxySettings = true;
|
||||||
|
proxyWebsockets = true;
|
||||||
|
proxyPass = "http://grafana";
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
88
hosts/monitoring/prometheus.nix
Normal file
88
hosts/monitoring/prometheus.nix
Normal file
|
|
@ -0,0 +1,88 @@
|
||||||
|
{ config, pkgs, lib, ... }:
|
||||||
|
|
||||||
|
{
|
||||||
|
services.prometheus = {
|
||||||
|
enable = true;
|
||||||
|
retentionTime = "14d";
|
||||||
|
listenAddress = "[::1]";
|
||||||
|
scrapeConfigs = [
|
||||||
|
{
|
||||||
|
job_name = "hedgedoc";
|
||||||
|
scrape_interval = "15s";
|
||||||
|
scheme = "https";
|
||||||
|
static_configs = [{ targets = ["md.berlin.ccc.de:443"]; }];
|
||||||
|
}
|
||||||
|
{
|
||||||
|
job_name = "synapse";
|
||||||
|
scrape_interval = "15s";
|
||||||
|
static_configs = [{ targets = ["matrix.berlin.ccc.de:9009"]; }];
|
||||||
|
}
|
||||||
|
{
|
||||||
|
job_name = "node";
|
||||||
|
scrape_interval = "15s";
|
||||||
|
static_configs = [
|
||||||
|
{
|
||||||
|
targets = [
|
||||||
|
"matrix.${config.networking.domain}:${toString config.services.prometheus.exporters.node.port}"
|
||||||
|
"md.${config.networking.domain}:${toString config.services.prometheus.exporters.node.port}"
|
||||||
|
"postgres.${config.networking.domain}:${toString config.services.prometheus.exporters.node.port}"
|
||||||
|
"monitoring:${toString config.services.prometheus.exporters.node.port}"
|
||||||
|
];
|
||||||
|
}
|
||||||
|
];
|
||||||
|
}
|
||||||
|
{
|
||||||
|
job_name = "nginx";
|
||||||
|
scrape_interval = "15s";
|
||||||
|
static_configs = [
|
||||||
|
{
|
||||||
|
targets = [
|
||||||
|
"monitoring:${toString config.services.prometheus.exporters.nginx.port}"
|
||||||
|
"matrix:${toString config.services.prometheus.exporters.nginx.port}"
|
||||||
|
];
|
||||||
|
}
|
||||||
|
];
|
||||||
|
}
|
||||||
|
];
|
||||||
|
ruleFiles = [
|
||||||
|
# https://github.com/element-hq/synapse/tree/master/contrib/prometheus
|
||||||
|
(pkgs.writeText "prom-synapse-rules.yaml" ''
|
||||||
|
groups:
|
||||||
|
- name: synapse
|
||||||
|
rules:
|
||||||
|
- record: 'synapse_federation_client_sent'
|
||||||
|
labels:
|
||||||
|
type: "EDU"
|
||||||
|
expr: 'synapse_federation_client_sent_edus_total + 0'
|
||||||
|
- record: 'synapse_federation_client_sent'
|
||||||
|
labels:
|
||||||
|
type: "PDU"
|
||||||
|
expr: 'synapse_federation_client_sent_pdu_destinations_count_total + 0'
|
||||||
|
- record: 'synapse_federation_client_sent'
|
||||||
|
labels:
|
||||||
|
type: "Query"
|
||||||
|
expr: 'sum(synapse_federation_client_sent_queries) by (job)'
|
||||||
|
- record: 'synapse_federation_server_received'
|
||||||
|
labels:
|
||||||
|
type: "EDU"
|
||||||
|
expr: 'synapse_federation_server_received_edus_total + 0'
|
||||||
|
- record: 'synapse_federation_server_received'
|
||||||
|
labels:
|
||||||
|
type: "PDU"
|
||||||
|
expr: 'synapse_federation_server_received_pdus_total + 0'
|
||||||
|
- record: 'synapse_federation_server_received'
|
||||||
|
labels:
|
||||||
|
type: "Query"
|
||||||
|
expr: 'sum(synapse_federation_server_received_queries) by (job)'
|
||||||
|
- record: 'synapse_federation_transaction_queue_pending'
|
||||||
|
labels:
|
||||||
|
type: "EDU"
|
||||||
|
expr: 'synapse_federation_transaction_queue_pending_edus + 0'
|
||||||
|
- record: 'synapse_federation_transaction_queue_pending'
|
||||||
|
labels:
|
||||||
|
type: "PDU"
|
||||||
|
expr: 'synapse_federation_transaction_queue_pending_pdus + 0'
|
||||||
|
'')
|
||||||
|
];
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
@ -17,17 +17,22 @@ let
|
||||||
_matrix = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIApAkkhHLj918co/wUGuyW8WCPYHxsNM4uo32XDEu7VV root@matrix";
|
_matrix = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIApAkkhHLj918co/wUGuyW8WCPYHxsNM4uo32XDEu7VV root@matrix";
|
||||||
_md = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFdFkdEEDXo8+k5YZpI1O2GqZlxcpCDtxqVun35duITm root@md";
|
_md = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFdFkdEEDXo8+k5YZpI1O2GqZlxcpCDtxqVun35duITm root@md";
|
||||||
_sql = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPcSXjDSyVVVdJbpheOhT0fIuOGFk+jsHhjrAVnBNLQV root@sql";
|
_sql = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPcSXjDSyVVVdJbpheOhT0fIuOGFk+jsHhjrAVnBNLQV root@sql";
|
||||||
|
_mon = "";
|
||||||
in
|
in
|
||||||
{
|
{
|
||||||
|
"pushover_app_token.age".publicKeys = users ++ [ _matrix ];
|
||||||
|
"pushover_user_key.age".publicKeys = users ++ [ _matrix ];
|
||||||
"matrix_admin_password.age".publicKeys = users;
|
"matrix_admin_password.age".publicKeys = users;
|
||||||
"draupnir_access_token.age".publicKeys = users ++ [ _matrix ];
|
"draupnir_access_token.age".publicKeys = users ++ [ _matrix ];
|
||||||
"matrix_signing_key.age".publicKeys = users ++ [ _matrix ];
|
"matrix_signing_key.age".publicKeys = users ++ [ _matrix ];
|
||||||
"matrix_registration_shared_secret.age".publicKeys = users ++ [ _matrix ];
|
"matrix_registration_shared_secret.age".publicKeys = users ++ [ _matrix ];
|
||||||
"pushover_app_token.age".publicKeys = users ++ [ _matrix ];
|
|
||||||
"pushover_user_key.age".publicKeys = users ++ [ _matrix ];
|
|
||||||
"grafana_admin_password.age".publicKeys = users ++ [ _matrix ];
|
"grafana_admin_password.age".publicKeys = users ++ [ _matrix ];
|
||||||
"grafana_secret_key.age".publicKeys = users ++ [ _matrix ];
|
"grafana_secret_key.age".publicKeys = users ++ [ _matrix ];
|
||||||
|
|
||||||
|
"hedgedoc-env.age".publicKeys = users ++ [ _md ];
|
||||||
|
|
||||||
"postgres-matrix-synapse.age".publicKeys = users ++ [ _sql _matrix ];
|
"postgres-matrix-synapse.age".publicKeys = users ++ [ _sql _matrix ];
|
||||||
"postgres-hedgedoc.age".publicKeys = users ++ [ _sql _md ];
|
"postgres-hedgedoc.age".publicKeys = users ++ [ _sql _md ];
|
||||||
|
"postgres-grafana.age".publicKeys = users ++ [ _sql _mon ];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
13
services/node-exporter.nix
Normal file
13
services/node-exporter.nix
Normal file
|
|
@ -0,0 +1,13 @@
|
||||||
|
{ ... }:
|
||||||
|
|
||||||
|
{
|
||||||
|
services.prometheus.exporters.node = {
|
||||||
|
enable = true;
|
||||||
|
openFirewall = true;
|
||||||
|
firewallRules = ''
|
||||||
|
ip saddr 195.160.173.14 tcp dport 9187 counter accept
|
||||||
|
ip6 saddr 2001:678:760:cccb::14 tcp dport 9187 counter accept
|
||||||
|
'';
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
Loading…
Reference in a new issue