less scrapers

.gitignore

@@ -1,3 +1,5 @@
+result
+
 # Created by https://www.toptal.com/developers/gitignore/api/linux,windows,macos,direnv,pycharm+all,vim
 # Edit at https://www.toptal.com/developers/gitignore?templates=linux,windows,macos,direnv,pycharm+all,vim
 

README.md

@@ -12,11 +12,16 @@
 
 ### Update a container
 
+Build and check for changes:
+
 ```shell
-ssh <container>
+nix run .#remote-diff-git
-cd /etc/nixos
+```
-nix run .#apps.nixos-diff   # Show what changes would be applied
+
-nixos-rebuild switch        # Apply changes
+Build and apply:
+
+```shell
+nix run .#remote-apply-git
 ```
 
 ### Special things

flake.lock

@@ -86,11 +86,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1789233862,
+        "lastModified": 1790328731,
-        "narHash": "sha256-mka8SFIzRsHoxBh6LGCFizdp8N1B81clSSe3wzid6Yc=",
+        "narHash": "sha256-Rvx1gJOPmsXVvislwhD16OSbqYjYD/jnLEdY4w77few=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "6eb0f007b69f39ababe79d7025d8d0484553d163",
+        "rev": "392b320636af0527be6f300a8f57e5ab5f43ba38",
         "type": "github"
       },
       "original": {

flake.nix

@@ -15,48 +15,7 @@
       flake-utils,
       agenix,
     }:
-    flake-utils.lib.eachDefaultSystem (
+    let
-      system:
-      let
-        pkgs = import nixpkgs { inherit system; };
-      in
-      {
-        formatter = pkgs.nixfmt-tree;
-        apps = {
-          nixos-diff = {
-            meta.description = "Compare the current system with the configuration from the updated flake";
-            type = "app";
-            program = "${pkgs.writeShellScript "nixos-diff.sh" ''
-              ${pkgs.git}/bin/git checkout flake.lock
-              ${pkgs.git}/bin/git pull --ff-only
-              ${pkgs.nix}/bin/nix flake update
-              ${pkgs.nixos-rebuild}/bin/nixos-rebuild build --log-format internal-json -v |& ${pkgs.nix-output-monitor}/bin/nom --json
-              ${pkgs.nvd}/bin/nvd diff /run/current-system ./result
-            ''}";
-          };
-          deploy = {
-            meta.description = "Deploy a NixOS configuration to a remote host";
-            type = "app";
-            program = "${pkgs.writeShellScript "deploy.sh" ''
-              set -e
-              host="''${1:?usage: nix run .#deploy -- <host>}"
-              ${pkgs.openssh}/bin/ssh -tt "$host" 'bash -s' <<'EOF'
-              set -e
-              cd /etc/nixos
-              nix run .#nixos-diff && nixos-rebuild switch
-              EOF
-            ''}";
-          };
-        };
-        devShells.default = pkgs.mkShell {
-          packages = [
-            (agenix.packages.${system}.default)
-            pkgs.age
-          ];
-        };
-      }
-    )
-    // {
       nixosConfigurations = {
         matrix = nixpkgs.lib.nixosSystem {
           system = "x86_64-linux";
@@ -309,5 +268,75 @@
           ];
         };
       };
+    in
+    (flake-utils.lib.eachDefaultSystem (
+      system:
+      let
+        pkgs = import nixpkgs { inherit system; };
+        remoteDiffApps = nixpkgs.lib.mapAttrs' (hostname: _: {
+          name = "remote-diff-${hostname}";
+          value = {
+            meta.description = "Build ${hostname} locally and compare remotely to current system";
+            type = "app";
+            program = "${pkgs.writeShellScript "remote-diff-${hostname}.sh" ''
+              set -euo pipefail
+              result=$(NIX_SSHOPTS="-o VisualHostKey=no" \
+                ${pkgs.nixos-rebuild}/bin/nixos-rebuild build \
+                --flake ${self}#${hostname} \
+                --target-host ${hostname} \
+                --use-substitutes \
+                --log-format internal-json -v 2> >(${pkgs.nix-output-monitor}/bin/nom --json))
+              ${pkgs.openssh}/bin/ssh -t ${hostname} \
+                nix run nixpkgs#nvd -- diff /run/current-system "$result"
+            ''}";
+          };
+        }) nixosConfigurations;
+        remoteApplyApps = nixpkgs.lib.mapAttrs' (hostname: _: {
+          name = "remote-apply-${hostname}";
+          value = {
+            meta.description = "Build ${hostname} locally and apply remotely";
+            type = "app";
+            program = "${pkgs.writeShellScript "remote-apply-${hostname}.sh" ''
+              set -euo pipefail
+              NIX_SSHOPTS="-o VisualHostKey=no" \
+                ${pkgs.nixos-rebuild}/bin/nixos-rebuild switch \
+                --flake ${self}#${hostname} \
+                --target-host ${hostname} \
+                --use-substitutes \
+                --log-format internal-json -v \
+              |& ${pkgs.nix-output-monitor}/bin/nom --json
+            ''}";
+          };
+        }) nixosConfigurations;
+      in
+      {
+        formatter = pkgs.nixfmt-tree;
+        apps = {
+          deploy = {
+            meta.description = "Deploy a NixOS configuration to a remote host";
+            type = "app";
+            program = "${pkgs.writeShellScript "deploy.sh" ''
+              set -e
+              host="''${1:?usage: nix run .#deploy -- <host>}"
+              ${pkgs.openssh}/bin/ssh -tt "$host" 'bash -s' <<'EOF'
+              set -e
+              cd /etc/nixos
+              nix run .#nixos-diff && nixos-rebuild switch
+              EOF
+            ''}";
+          };
+        }
+        // remoteDiffApps
+        // remoteApplyApps;
+        devShells.default = pkgs.mkShell {
+          packages = [
+            agenix.packages.${system}.default
+            pkgs.age
+          ];
+        };
+      }
+    ))
+    // {
+      inherit nixosConfigurations;
     };
 }

hosts/git/forgejo.nix

@@ -75,7 +75,8 @@ in
 
     nginx = {
       appendHttpConfig = ''
-        limit_req_zone $binary_remote_addr zone=forgejo_archive:10m rate=2r/s;
+        limit_req_zone $binary_remote_addr zone=forgejo_archive:10m rate=10r/m;
+        limit_conn_zone $binary_remote_addr zone=forgejo_archive_conn:10m;
       '';
       virtualHosts."${fqdn}" = {
         quic = true;
@@ -106,6 +107,8 @@ in
           };
           "~ ^/[^/]+/[^/]+/archive/".extraConfig = ''
             limit_req zone=forgejo_archive burst=10 nodelay;
+            limit_conn forgejo_archive_conn 2;
+            limit_req_status 429;
           '';
           "= /robots.txt".extraConfig = ''
             default_type text/plain;