cf7d357f17
The docs, option descriptions and comments that named the swarm root, the swarm CA or the hive CA as an existing trust anchor now name the secret store's pki root or the trust bundle that holds it: - docs/swarm/README.md, secrets.md, bao.md - deploy.bao.clientCaFile's and gateway.verifiedProxyTo's descriptions - comments in hive-c0re/src/meta.rs, hive-agent/src/swarm_queue.rs, swarm-logs/src/auth.rs, swarm-controller/src/vcs_metrics.rs, hive-forge, swarm-authelia, swarm-ci-runner, glue-bao-tls, swarm-bao, swarm-grafana, swarm-otel, otel, hive-c0re, lib/hive-ca-trust.nix, hive-forge/Cargo.toml and the flake-update workflow Nothing behaves differently: every edit is a nix, Rust or TOML comment, an option description, or a test assertion message. The Rust edits change the workspace source, which this PR already rebuilds for hivectl. Refs #4588
514 lines
21 KiB
Nix
514 lines
21 KiB
Nix
# The swarm's Forgejo Actions runner, in a `swarm-ci-runner` container on the
|
|
# host that sets `deploy.ci-runner.enable`. It reaches the forge by name and
|
|
# reads its registration token from the secret store under a certificate of
|
|
# its own, so nothing here asks whether this host runs the forge or a hive.
|
|
#
|
|
# ⚠️ Two properties here are load-bearing and neither is visible from the
|
|
# config that implements it:
|
|
#
|
|
# - The container gets its OWN network namespace, not the host's, so a PR's
|
|
# nix build cannot reach host-loopback services. That is a
|
|
# prompt-injection boundary; moving this to host netns for convenience
|
|
# removes it. `docs/networking/network.md::Container isolation`.
|
|
# - The forge ADMIN token never enters this container. swarm-controller holds
|
|
# it and publishes only the registration token, which the runner's own store
|
|
# identity reads. `docs/scheduler/ci.md::Registration`.
|
|
#
|
|
# Non-ephemeral on purpose: the runner writes its registered credentials
|
|
# (`.runner`) to its state dir and reuses them, so an ephemeral container
|
|
# would register a new runner on every restart.
|
|
{
|
|
pkgs,
|
|
lib,
|
|
config,
|
|
...
|
|
}:
|
|
let
|
|
cfg = config.services.hyperhive.deploy.ci-runner;
|
|
forgeCfg = config.services.hyperhive.swarm.forge;
|
|
baoCfg = config.services.hyperhive.swarm.bao;
|
|
baoDeploy = config.services.hyperhive.deploy.bao;
|
|
gatewayCfg = config.services.hyperhive.gateway;
|
|
networkCfg = config.services.hyperhive.network;
|
|
tlsCfg = config.services.hyperhive.deploy.hive-controller.tls;
|
|
|
|
machine = "swarm-ci-runner";
|
|
# The upstream module names its unit `gitea-runner-<instance>`.
|
|
instance = "swarm";
|
|
runnerUnit = "gitea-runner-${instance}";
|
|
# Holds the registration token and registers; see its definition.
|
|
registerUnit = "swarm-ci-runner-register";
|
|
|
|
# What the runner registers against and records in `.runner`. The precond
|
|
# below compares the two, so both come from here.
|
|
forgeUrl = "https://${forgeCfg.domain}";
|
|
|
|
# The forge's gateway serves a leaf from the store's `pki` mount in
|
|
# self-signed mode.
|
|
# The runner daemon is Go, whose `SSL_CERT_FILE` replaces the system store,
|
|
# so it gets the assembled bundle; the Node actions it runs get
|
|
# `NODE_EXTRA_CA_CERTS`, which is additive, further down.
|
|
caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; };
|
|
caBundleModule = caTrust.trustBundle {
|
|
inherit pkgs;
|
|
name = machine;
|
|
consumers = [
|
|
runnerUnit
|
|
registerUnit
|
|
];
|
|
};
|
|
|
|
# The upstream unit is `DynamicUser`, so its state dir is
|
|
# `/var/lib/private/gitea-runner`; `/var/lib/gitea-runner` is a symlink to
|
|
# it, which is the spelling a step run as root (`+`) can follow.
|
|
runnerFile = "/var/lib/gitea-runner/${instance}/.runner";
|
|
|
|
# Inside the container, root-only. The token file holds the bare token and
|
|
# is a `LoadCredential=` of the register unit alone; the host unit below
|
|
# writes it through the container's root, at `hostTokenFile`.
|
|
stateDir = "/var/lib/swarm-ci-runner";
|
|
tokenFile = "${stateDir}/registration-token";
|
|
tokenCredential = "registration-token";
|
|
hostTokenFile = "/var/lib/nixos-containers/${machine}${tokenFile}";
|
|
# The epoch second the daemon last started. The precond reads the daemon's
|
|
# journal from there.
|
|
daemonStartedFile = "${stateDir}/daemon-started";
|
|
|
|
# The nix half of `swarm_secret_client::ci::registration_token_path`, in the
|
|
# form `bao kv get` takes (no KV v2 `data/`).
|
|
storePath = "secret/swarm/services/ci-runner/registration-token";
|
|
|
|
# What Forgejo answers a runner it has no record of — deleted in the admin
|
|
# panel, or a forge whose database was replaced — on every call but
|
|
# `Register` (`routers/api/actions/runner/interceptor.go`). The daemon logs
|
|
# it and exits 1 when its startup `Declare` gets it. ⚠️ Nothing this module
|
|
# prints may contain it: the precond would read its own line as a rejection.
|
|
rejectedMessage = "unregistered runner";
|
|
|
|
haveClientIdentity =
|
|
baoDeploy.ciRunnerClientCertFile != null && baoDeploy.ciRunnerClientKeyFile != null;
|
|
|
|
atomicWriteSecret = import ./lib/atomic-write-secret.nix { };
|
|
refreshConsumer = import ./lib/refresh-consumer.nix { };
|
|
storeRetry = import ./lib/store-retry.nix { };
|
|
|
|
privateNetwork = true;
|
|
in
|
|
{
|
|
options.services.hyperhive.deploy.ci-runner = {
|
|
# `enable` is in ./deploy.nix.
|
|
|
|
name = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "swarm-ci-runner";
|
|
description = ''
|
|
Runner name as shown in the Forgejo admin panel.
|
|
'';
|
|
};
|
|
|
|
concurrency = lib.mkOption {
|
|
type = lib.types.ints.positive;
|
|
default = 1;
|
|
example = 4;
|
|
description = ''
|
|
Maximum number of workflow jobs the runner executes in parallel.
|
|
Each job gets its own temporary working directory; parallel jobs
|
|
share the host's nix store and the container's cargo registry
|
|
cache. Higher values trade memory and CPU headroom for throughput.
|
|
'';
|
|
};
|
|
|
|
labels = lib.mkOption {
|
|
type = lib.types.listOf lib.types.str;
|
|
default = [ "hive-ci:host" ];
|
|
example = [
|
|
"hive-ci:host"
|
|
"nix:host"
|
|
];
|
|
description = ''
|
|
Runner labels in `<name>:<scheme>` format. The `host` scheme runs
|
|
commands directly in the container (no docker/podman). This repo's
|
|
workflows target the default with `runs-on: hive-ci`.
|
|
'';
|
|
};
|
|
|
|
package = lib.mkOption {
|
|
type = lib.types.package;
|
|
default = pkgs.gitea-actions-runner;
|
|
defaultText = lib.literalExpression "pkgs.gitea-actions-runner";
|
|
description = "gitea-actions-runner package.";
|
|
};
|
|
|
|
jobTimeout = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "1h";
|
|
example = "3h";
|
|
description = ''
|
|
Per-job wall-clock timeout the runner enforces (act_runner's
|
|
`runner.timeout`). A job that exceeds it is killed, so a hung build
|
|
does not hold a runner slot indefinitely. Accepts a Go duration
|
|
string (`30m`, `1h`, `2h30m`). Enforced by the runner process, so
|
|
it only fires while that process is healthy.
|
|
'';
|
|
};
|
|
};
|
|
|
|
config = lib.mkIf cfg.enable {
|
|
assertions = [
|
|
{
|
|
assertion = haveClientIdentity;
|
|
message = ''
|
|
services.hyperhive.deploy.ci-runner.enable requires the runner's
|
|
swarm-secret-store client identity: set both
|
|
|
|
services.hyperhive.deploy.bao.ciRunnerClientCertFile
|
|
services.hyperhive.deploy.bao.ciRunnerClientKeyFile
|
|
|
|
swarm-bao-ci-runner.service reads the forge's runner registration
|
|
token with it, and the runner cannot register without that token.
|
|
|
|
On the host that runs the store, glue-bao-tls.nix supplies both as
|
|
defaults. Elsewhere, copy ci-runner.pem and ci-runner-key.pem from
|
|
the store host's /var/lib/swarm-bao-pki/ and name them here.
|
|
'';
|
|
}
|
|
];
|
|
|
|
# The container hangs off the bridge, takes its address from the
|
|
# gateway's DHCP and resolves through its dnsmasq. The gateway also runs
|
|
# `hive-tls-ca`, which writes the trust bundle bound in below.
|
|
services.hyperhive.network.enable = lib.mkDefault true;
|
|
services.hyperhive.gateway.enable = lib.mkDefault true;
|
|
services.hyperhive.gateway.dns.enable = lib.mkDefault true;
|
|
|
|
# Shaped after ./swarm-grafana.nix's `swarm-bao-grafana-oidc`, except that
|
|
# a missing token fails too: swarm-controller publishes it at its start and
|
|
# every five minutes, so a retry is what turns "no value there" into one.
|
|
systemd.services.swarm-bao-ci-runner = {
|
|
description = "fetch the CI runner's registration token from the swarm secret store";
|
|
after = lib.optionals baoDeploy.enable [
|
|
"swarm-bao-pki.service"
|
|
"container@${baoCfg.machine}.service"
|
|
];
|
|
wants = lib.optionals baoDeploy.enable [ "container@${baoCfg.machine}.service" ];
|
|
requires = lib.optionals baoDeploy.enable [ "swarm-bao-pki.service" ];
|
|
before = [ "container@${machine}.service" ];
|
|
wantedBy = [
|
|
"multi-user.target"
|
|
"container@${machine}.service"
|
|
];
|
|
path = [
|
|
baoDeploy.package
|
|
pkgs.coreutils
|
|
pkgs.systemd
|
|
];
|
|
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
|
|
serviceConfig = storeRetry.serviceConfig // {
|
|
Type = "oneshot";
|
|
RemainAfterExit = true;
|
|
SyslogIdentifier = "swarm-bao-ci-runner";
|
|
TimeoutStartSec = 30;
|
|
};
|
|
environment = {
|
|
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
|
BAO_CLIENT_CERT = baoDeploy.ciRunnerClientCertFile;
|
|
BAO_CLIENT_KEY = baoDeploy.ciRunnerClientKeyFile;
|
|
}
|
|
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
|
|
BAO_CACERT = baoDeploy.serverCaFile;
|
|
};
|
|
script = ''
|
|
set -euo pipefail
|
|
|
|
${atomicWriteSecret}
|
|
${refreshConsumer}
|
|
|
|
err="$(mktemp)"
|
|
trap 'rm -f "$err"' EXIT
|
|
|
|
# `-token-only` answers on stdout and needs no token helper.
|
|
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
|
|
echo "could not log in to swarm-bao with the CI runner's certificate; leaving its registration token as it is." >&2
|
|
if [ -s "$err" ]; then
|
|
cat "$err" >&2
|
|
else
|
|
echo "bao failed without writing a diagnostic." >&2
|
|
fi
|
|
exit 1
|
|
fi
|
|
export BAO_TOKEN
|
|
|
|
if ! token="$(bao kv get -field=value ${lib.escapeShellArg storePath} 2>"$err")"; then
|
|
echo "swarm-bao did not return ${storePath}; retrying until swarm-controller publishes it." >&2
|
|
if [ -s "$err" ]; then
|
|
cat "$err" >&2
|
|
else
|
|
echo "bao failed without writing a diagnostic." >&2
|
|
fi
|
|
exit 1
|
|
fi
|
|
|
|
if [ -z "$token" ]; then
|
|
echo "swarm-bao returned an empty ${storePath}; retrying." >&2
|
|
exit 1
|
|
fi
|
|
|
|
# Root-owned: systemd reads a `LoadCredential=` source as root, before
|
|
# the runner drops to its dynamic user.
|
|
install -d -m 0700 ${lib.escapeShellArg (builtins.dirOf hostTokenFile)}
|
|
if secret_differs ${lib.escapeShellArg hostTokenFile} "$token"; then
|
|
atomic_write_secret 0600 "" ${lib.escapeShellArg hostTokenFile} "$token"
|
|
fi
|
|
|
|
# `LoadCredential=` copies the file at start, so a runner still waiting
|
|
# to register sees a new token only once it restarts.
|
|
refresh_consumer ${lib.escapeShellArg machine} ${runnerUnit}.service ${lib.escapeShellArg hostTokenFile}
|
|
'';
|
|
};
|
|
|
|
systemd.services."container@${machine}" = caTrust.containerOrdering;
|
|
|
|
containers.${machine} = {
|
|
autoStart = true;
|
|
ephemeral = false;
|
|
# Journal files on the host, not inside the container: nixpkgs hardcodes
|
|
# --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it.
|
|
extraFlags = [ "--link-journal=host" ];
|
|
inherit privateNetwork;
|
|
hostBridge = networkCfg.bridgeName;
|
|
|
|
bindMounts = {
|
|
# Builds go through the HOST nix-daemon, so they inherit its
|
|
# buildMachines and max-jobs. The directory rather than the socket
|
|
# file, so the mount survives a daemon restart replacing the inode.
|
|
"/nix/var/nix/daemon-socket" = {
|
|
hostPath = "/nix/var/nix/daemon-socket";
|
|
isReadOnly = false;
|
|
};
|
|
}
|
|
// caTrust.bindMount;
|
|
|
|
config =
|
|
{
|
|
pkgs,
|
|
lib,
|
|
config,
|
|
...
|
|
}:
|
|
let
|
|
runnerCfg = config.services.gitea-actions-runner;
|
|
instanceCfg = runnerCfg.instances.${instance};
|
|
# The same file upstream generates and hands the daemon.
|
|
runnerConfigFile = (pkgs.formats.yaml { }).generate "config.yaml" instanceCfg.settings;
|
|
# In the form the daemon rewrites `.runner`'s labels to on start
|
|
# (act_runner's `labels.Parse` then `ToStrings`): `<name>:<scheme>`
|
|
# with `host` for a missing scheme, then `:<arg>` if there is one.
|
|
normaliseLabel =
|
|
label:
|
|
let
|
|
parts = lib.splitString ":" label;
|
|
scheme = if lib.length parts >= 2 then lib.elemAt parts 1 else "host";
|
|
arg = lib.concatStringsSep ":" (lib.drop 2 parts);
|
|
in
|
|
"${lib.head parts}:${scheme}" + lib.optionalString (arg != "") ":${arg}";
|
|
registeredLabels = lib.concatStringsSep "," (
|
|
lib.sort lib.lessThan (map normaliseLabel instanceCfg.labels)
|
|
);
|
|
in
|
|
{
|
|
imports = [
|
|
../container-modules/swarm-container.nix
|
|
caBundleModule
|
|
];
|
|
|
|
# Its own netns, so it keeps its own firewall; resolvconf stays on
|
|
# and serves `networking.nameservers` below.
|
|
services.hyperhive.swarmContainer = {
|
|
inherit privateNetwork;
|
|
writesOwnResolvConf = false;
|
|
};
|
|
|
|
# The gateway's dnsmasq on the bridge answers the forge's name with
|
|
# the bridge IP where this host runs the forge, and forwards it
|
|
# upstream everywhere else.
|
|
networking.nameservers = [ networkCfg.bridgeIp ];
|
|
networking.interfaces.eth0.useDHCP = true;
|
|
|
|
# nspawn containers can't create user namespaces, so nix sandboxing
|
|
# fails; see docs/process/gotchas.md. `fallback` is a client-side
|
|
# option sent per daemon connection, so the host's value does not
|
|
# reach builds started here.
|
|
nix.settings.sandbox-fallback = lib.mkForce true;
|
|
nix.settings.fallback = true;
|
|
nix.settings.experimental-features = [
|
|
"nix-command"
|
|
"flakes"
|
|
];
|
|
|
|
# Both the service and its socket are off: the socket unit owns
|
|
# `/nix/var/nix/daemon-socket/socket` and would race the bind mount.
|
|
systemd.globalEnvironment.NIX_REMOTE = "daemon";
|
|
systemd.services.nix-daemon.enable = false;
|
|
systemd.sockets.nix-daemon.enable = false;
|
|
|
|
systemd.tmpfiles.rules = [
|
|
"d ${stateDir} 0700 root root -"
|
|
# `LoadCredential=` fails the unit when the file is missing; an
|
|
# empty one lets the register unit report the missing token.
|
|
"f ${tokenFile} 0600 root root -"
|
|
# Unused: nothing in this module writes or reads them. The first
|
|
# holds a token; the others are upstream's re-registration markers.
|
|
"r ${stateDir}/registration-token.env"
|
|
"r /var/lib/private/gitea-runner/${instance}/.token-hash"
|
|
"r /var/lib/private/gitea-runner/${instance}/.labels"
|
|
];
|
|
|
|
services.gitea-actions-runner.package = cfg.package;
|
|
|
|
services.gitea-actions-runner.instances.${instance} = {
|
|
enable = true;
|
|
inherit (cfg) name labels;
|
|
url = forgeUrl;
|
|
# Upstream refuses an instance without a token source, and turns
|
|
# this one into the `EnvironmentFile=` the daemon unit drops.
|
|
inherit tokenFile;
|
|
settings = {
|
|
runner.capacity = cfg.concurrency;
|
|
runner.timeout = cfg.jobTimeout;
|
|
};
|
|
};
|
|
|
|
environment.systemPackages = [
|
|
pkgs.git
|
|
pkgs.nix
|
|
];
|
|
|
|
# The only unit that can read the registration token: systemd hands
|
|
# a credential to every process of the unit that loads it, and the
|
|
# daemon's processes include every host job. Leaves a usable
|
|
# `.runner` behind or fails.
|
|
#
|
|
# The daemon's dynamic user and state dir: units naming the same
|
|
# `DynamicUser` share its uid, so `.runner` is the daemon's own.
|
|
systemd.services.${registerUnit} = {
|
|
description = "register the CI runner with the forge when it has no .runner";
|
|
wants = [ "network-online.target" ];
|
|
after = [ "network-online.target" ];
|
|
environment = {
|
|
HOME = "/var/lib/gitea-runner/${instance}";
|
|
# In the form the daemon rewrites `.runner`'s labels to.
|
|
SWARM_CI_RUNNER_LABELS = registeredLabels;
|
|
};
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
DynamicUser = true;
|
|
User = "gitea-runner";
|
|
StateDirectory = "gitea-runner";
|
|
LoadCredential = [ "${tokenCredential}:${tokenFile}" ];
|
|
# `register` retries an unreachable forge without end; under the
|
|
# daemon's own start timeout, so the failure is this unit's.
|
|
TimeoutStartSec = 60;
|
|
# Removes `.runner` in the cases where reusing it would fail, so
|
|
# the register step registers again.
|
|
#
|
|
# `+`: root, because the system journal is out of the dynamic
|
|
# user's reach.
|
|
ExecStartPre = "+${pkgs.writeShellScript "swarm-ci-runner-precond" ''
|
|
set -eu
|
|
runner=${runnerFile}
|
|
started=${daemonStartedFile}
|
|
|
|
# The forge refused the registration the daemon last started
|
|
# with. Read on the next start rather than when the daemon
|
|
# exits, so journald has its last lines by then.
|
|
if [ -f "$started" ]; then
|
|
since="$(< "$started")"
|
|
${pkgs.coreutils}/bin/rm -f "$started"
|
|
if ${pkgs.systemd}/bin/journalctl -u ${runnerUnit}.service --since "@$since" -o cat --no-pager \
|
|
| ${pkgs.gnugrep}/bin/grep -qF ${lib.escapeShellArg rejectedMessage}; then
|
|
echo "the forge has no record of this runner's registration; registering again" >&2
|
|
${pkgs.coreutils}/bin/rm -f "$runner"
|
|
fi
|
|
fi
|
|
|
|
# `.runner` records the address it registered at, and the
|
|
# daemon dials that one, not the configured one.
|
|
if [ -f "$runner" ] \
|
|
&& [ "$(${pkgs.jq}/bin/jq -r .address "$runner")" != ${lib.escapeShellArg forgeUrl} ]; then
|
|
echo "$runner names another forge address; registering again at ${forgeUrl}" >&2
|
|
${pkgs.coreutils}/bin/rm -f "$runner"
|
|
fi
|
|
|
|
# The daemon declares the labels `.runner` records, not the
|
|
# configured ones.
|
|
if [ -f "$runner" ] \
|
|
&& [ "$(${pkgs.jq}/bin/jq -r '.labels | sort | join(",")' "$runner")" != "$SWARM_CI_RUNNER_LABELS" ]; then
|
|
echo "$runner names other labels; registering again with $SWARM_CI_RUNNER_LABELS" >&2
|
|
${pkgs.coreutils}/bin/rm -f "$runner"
|
|
fi
|
|
''}";
|
|
};
|
|
script = ''
|
|
set -u
|
|
dir="$STATE_DIRECTORY/${instance}"
|
|
mkdir -p "$dir"
|
|
cd "$dir"
|
|
if [ -e .runner ]; then
|
|
exit 0
|
|
fi
|
|
|
|
token="$(< "$CREDENTIALS_DIRECTORY/${tokenCredential}")"
|
|
if [ -z "$token" ]; then
|
|
echo "no registration token yet; swarm-bao-ci-runner on the host delivers it" >&2
|
|
exit 1
|
|
fi
|
|
exec ${lib.getExe runnerCfg.package} register --no-interactive \
|
|
--instance ${lib.escapeShellArg instanceCfg.url} \
|
|
--token "$token" \
|
|
--name ${lib.escapeShellArg instanceCfg.name} \
|
|
--labels ${lib.escapeShellArg (lib.concatStringsSep "," instanceCfg.labels)} \
|
|
--config ${runnerConfigFile}
|
|
'';
|
|
};
|
|
|
|
systemd.services.${runnerUnit} = {
|
|
# `path` adds to the PATH the upstream module builds instead of
|
|
# replacing it.
|
|
path = [ pkgs.nix ];
|
|
# The upstream unit is `DynamicUser`, whose state dir systemd mounts
|
|
# `noexec`, so a job could not execute what it builds in its
|
|
# workspace. `ExecPaths=` resolves against the host root, where
|
|
# `/var/lib/gitea-runner` is a symlink into `private/`; `-` makes
|
|
# whichever spelling is absent a no-op.
|
|
serviceConfig.ExecPaths = [
|
|
"-/var/lib/gitea-runner"
|
|
"-/var/lib/private/gitea-runner"
|
|
];
|
|
environment = lib.mkIf caTrust.useSelfSigned {
|
|
NODE_EXTRA_CA_CERTS = caTrust.caContainerPath;
|
|
};
|
|
# A host job inherits the daemon's whole environment and can read
|
|
# its credentials, so the daemon carries the token in neither.
|
|
serviceConfig.EnvironmentFile = lib.mkForce [ ];
|
|
# Replaces upstream's pre-start step, which registers from
|
|
# `$TOKEN`, so the register unit is the one registration path.
|
|
#
|
|
# Started from here rather than through `Requires=`: a failed
|
|
# requirement fails the start job, which `Restart=` does not retry,
|
|
# while a failed pre-start step does.
|
|
serviceConfig.ExecStartPre = lib.mkForce [
|
|
"+${pkgs.systemd}/bin/systemctl start ${registerUnit}.service"
|
|
"+${pkgs.writeShellScript "swarm-ci-runner-mark-start" ''
|
|
${pkgs.coreutils}/bin/date +%s > ${daemonStartedFile}
|
|
''}"
|
|
];
|
|
# Restarted until the token arrives, however long the store and
|
|
# the controller take. `mkForce`: upstream sets both.
|
|
serviceConfig.Restart = lib.mkForce "on-failure";
|
|
serviceConfig.RestartSec = lib.mkForce 15;
|
|
startLimitIntervalSec = 0;
|
|
};
|
|
};
|
|
};
|
|
};
|
|
}
|