9407db69b5
Each store reader (forge's OIDC fetch and token publisher, matrix's OIDC fetch, the hive collector's fetch, Grafana's OIDC fetch and admin password, the queue's callout-xkey unit, the CI runner's token fetch, the swarm collector's OIDC fetch) used to read `deploy.bao.enable` in its own module to add `after`/`wants` on `container@<store machine>` and `after`/`requires` on `swarm-bao-pki.service`. Per the #3742 ruling, that cross-service read moves into glue-bao-readers-store-order.nix, which sets the same edges when the store and the reader are both on this host and restates each reader's own gate so no stub unit is rendered. Rendered `after`/`wants`/`requires` are identical as sets to main on 16 fixtures, store here and elsewhere. New module-eval cases in existing fixtures pin both arms. Part of #3742.
516 lines
22 KiB
Nix
516 lines
22 KiB
Nix
# The swarm's Forgejo Actions runner, in a `swarm-ci-runner` container on the
|
|
# host that sets `deploy.ci-runner.enable`. It reaches the forge by name and
|
|
# reads its registration token from the secret store under a certificate of
|
|
# its own, so nothing here asks whether this host runs the forge or a hive.
|
|
#
|
|
# ⚠️ Two properties here are load-bearing and neither is visible from the
|
|
# config that implements it:
|
|
#
|
|
# - The container gets its OWN network namespace, not the host's, so a PR's
|
|
# nix build cannot reach host-loopback services. That is a
|
|
# prompt-injection boundary; moving this to host netns for convenience
|
|
# removes it. `docs/networking/network.md::Container isolation`.
|
|
# - The forge ADMIN token never enters this container. swarm-controller holds
|
|
# it and publishes only the registration token, which the runner's own store
|
|
# identity reads. `docs/scheduler/ci.md::Registration`.
|
|
#
|
|
# Non-ephemeral on purpose: the runner writes its registered credentials
|
|
# (`.runner`) to its state dir and reuses them, so an ephemeral container
|
|
# would register a new runner on every restart.
|
|
{
|
|
pkgs,
|
|
lib,
|
|
config,
|
|
...
|
|
}:
|
|
let
|
|
cfg = config.services.hyperhive.deploy.ci-runner;
|
|
forgeCfg = config.services.hyperhive.swarm.forge;
|
|
baoCfg = config.services.hyperhive.swarm.bao;
|
|
baoDeploy = config.services.hyperhive.deploy.bao;
|
|
gatewayCfg = config.services.hyperhive.gateway;
|
|
networkCfg = config.services.hyperhive.network;
|
|
tlsCfg = config.services.hyperhive.deploy.hive-controller.tls;
|
|
|
|
machine = "swarm-ci-runner";
|
|
# The upstream module names its unit `gitea-runner-<instance>`.
|
|
instance = "swarm";
|
|
runnerUnit = "gitea-runner-${instance}";
|
|
# Holds the registration token and registers; see its definition.
|
|
registerUnit = "swarm-ci-runner-register";
|
|
|
|
# What the runner registers against and records in `.runner`. The precond
|
|
# below compares the two, so both come from here.
|
|
forgeUrl = "https://${forgeCfg.domain}";
|
|
|
|
# The forge's gateway serves a leaf from the store's `pki` mount in
|
|
# self-signed mode.
|
|
# The runner daemon is Go, whose `SSL_CERT_FILE` replaces the system store,
|
|
# so it gets the assembled bundle; the Node actions it runs get
|
|
# `NODE_EXTRA_CA_CERTS`, which is additive, further down.
|
|
caTrust = import ./lib/hive-ca-trust.nix { inherit lib tlsCfg gatewayCfg; };
|
|
caBundleModule = caTrust.trustBundle {
|
|
inherit pkgs;
|
|
name = machine;
|
|
consumers = [
|
|
runnerUnit
|
|
registerUnit
|
|
];
|
|
};
|
|
|
|
# The upstream unit is `DynamicUser`, so its state dir is
|
|
# `/var/lib/private/gitea-runner`; `/var/lib/gitea-runner` is a symlink to
|
|
# it, which is the spelling a step run as root (`+`) can follow.
|
|
runnerFile = "/var/lib/gitea-runner/${instance}/.runner";
|
|
|
|
# Inside the container, root-only. The token file holds the bare token and
|
|
# is a `LoadCredential=` of the register unit alone; the host unit below
|
|
# writes it through the container's root, at `hostTokenFile`.
|
|
stateDir = "/var/lib/swarm-ci-runner";
|
|
tokenFile = "${stateDir}/registration-token";
|
|
tokenCredential = "registration-token";
|
|
hostTokenFile = "/var/lib/nixos-containers/${machine}${tokenFile}";
|
|
# The epoch second the daemon last started. The precond reads the daemon's
|
|
# journal from there.
|
|
daemonStartedFile = "${stateDir}/daemon-started";
|
|
|
|
# The nix half of `swarm_secret_client::ci::registration_token_path`, in the
|
|
# form `bao kv get` takes (no KV v2 `data/`).
|
|
storePath = "secret/swarm/services/ci-runner/registration-token";
|
|
|
|
# What Forgejo answers a runner it has no record of — deleted in the admin
|
|
# panel, or a forge whose database was replaced — on every call but
|
|
# `Register` (`routers/api/actions/runner/interceptor.go`). The daemon logs
|
|
# it and exits 1 when its startup `Declare` gets it. ⚠️ Nothing this module
|
|
# prints may contain it: the precond would read its own line as a rejection.
|
|
rejectedMessage = "unregistered runner";
|
|
|
|
haveClientIdentity =
|
|
baoDeploy.ciRunnerClientCertFile != null && baoDeploy.ciRunnerClientKeyFile != null;
|
|
|
|
atomicWriteSecret = import ./lib/atomic-write-secret.nix { };
|
|
refreshConsumer = import ./lib/refresh-consumer.nix { };
|
|
storeRetry = import ./lib/store-retry.nix { };
|
|
storeRefreshTimer = import ./lib/store-refresh-timer.nix { };
|
|
|
|
privateNetwork = true;
|
|
in
|
|
{
|
|
options.services.hyperhive.deploy.ci-runner = {
|
|
# `enable` is in ./deploy.nix.
|
|
|
|
name = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "swarm-ci-runner";
|
|
description = ''
|
|
Runner name as shown in the Forgejo admin panel.
|
|
'';
|
|
};
|
|
|
|
concurrency = lib.mkOption {
|
|
type = lib.types.ints.positive;
|
|
default = 1;
|
|
example = 4;
|
|
description = ''
|
|
Maximum number of workflow jobs the runner executes in parallel.
|
|
Each job gets its own temporary working directory; parallel jobs
|
|
share the host's nix store and the container's cargo registry
|
|
cache. Higher values trade memory and CPU headroom for throughput.
|
|
'';
|
|
};
|
|
|
|
labels = lib.mkOption {
|
|
type = lib.types.listOf lib.types.str;
|
|
default = [ "hive-ci:host" ];
|
|
example = [
|
|
"hive-ci:host"
|
|
"nix:host"
|
|
];
|
|
description = ''
|
|
Runner labels in `<name>:<scheme>` format. The `host` scheme runs
|
|
commands directly in the container (no docker/podman). This repo's
|
|
workflows target the default with `runs-on: hive-ci`.
|
|
'';
|
|
};
|
|
|
|
package = lib.mkOption {
|
|
type = lib.types.package;
|
|
default = pkgs.gitea-actions-runner;
|
|
defaultText = lib.literalExpression "pkgs.gitea-actions-runner";
|
|
description = "gitea-actions-runner package.";
|
|
};
|
|
|
|
jobTimeout = lib.mkOption {
|
|
type = lib.types.str;
|
|
default = "1h";
|
|
example = "3h";
|
|
description = ''
|
|
Per-job wall-clock timeout the runner enforces (act_runner's
|
|
`runner.timeout`). A job that exceeds it is killed, so a hung build
|
|
does not hold a runner slot indefinitely. Accepts a Go duration
|
|
string (`30m`, `1h`, `2h30m`). Enforced by the runner process, so
|
|
it only fires while that process is healthy.
|
|
'';
|
|
};
|
|
};
|
|
|
|
config = lib.mkIf cfg.enable {
|
|
assertions = [
|
|
{
|
|
assertion = haveClientIdentity;
|
|
message = ''
|
|
services.hyperhive.deploy.ci-runner.enable requires the runner's
|
|
swarm-secret-store client identity: set both
|
|
|
|
services.hyperhive.deploy.bao.ciRunnerClientCertFile
|
|
services.hyperhive.deploy.bao.ciRunnerClientKeyFile
|
|
|
|
swarm-bao-ci-runner.service reads the forge's runner registration
|
|
token with it, and the runner cannot register without that token.
|
|
|
|
Both default to the pair glue-bao-tls.nix mints when
|
|
services.hyperhive.deploy.bao.enable is true (the host that runs
|
|
the store). Otherwise copy ci-runner.pem and ci-runner-key.pem from
|
|
the store host's /var/lib/swarm-bao-pki/ and name them in those two
|
|
options.
|
|
'';
|
|
}
|
|
];
|
|
|
|
# The container hangs off the bridge, takes its address from the
|
|
# gateway's DHCP and resolves through its dnsmasq. The gateway also runs
|
|
# `hive-tls-ca`, which writes the trust bundle bound in below.
|
|
services.hyperhive.network.enable = lib.mkDefault true;
|
|
services.hyperhive.gateway.enable = lib.mkDefault true;
|
|
services.hyperhive.gateway.dns.enable = lib.mkDefault true;
|
|
|
|
# Shaped after ./swarm-grafana.nix's `swarm-bao-grafana-oidc`, except that
|
|
# a missing token fails too: swarm-controller publishes it at its start and
|
|
# every five minutes, so a retry is what turns "no value there" into one.
|
|
systemd.services.swarm-bao-ci-runner = {
|
|
description = "fetch the CI runner's registration token from the swarm secret store";
|
|
# Ordered after the store where it runs here:
|
|
# ./glue-bao-readers-store-order.nix.
|
|
before = [ "container@${machine}.service" ];
|
|
wantedBy = [
|
|
"multi-user.target"
|
|
"container@${machine}.service"
|
|
];
|
|
path = [
|
|
baoDeploy.package
|
|
pkgs.coreutils
|
|
pkgs.systemd
|
|
];
|
|
inherit (storeRetry) startLimitBurst startLimitIntervalSec;
|
|
serviceConfig = storeRetry.serviceConfig // {
|
|
# No `RemainAfterExit`: the timer below starts this again, and
|
|
# starting an active unit is a no-op.
|
|
Type = "oneshot";
|
|
SyslogIdentifier = "swarm-bao-ci-runner";
|
|
TimeoutStartSec = 30;
|
|
};
|
|
environment = {
|
|
BAO_ADDR = "https://${baoCfg.domain}:${toString baoCfg.port}";
|
|
BAO_CLIENT_CERT = baoDeploy.ciRunnerClientCertFile;
|
|
BAO_CLIENT_KEY = baoDeploy.ciRunnerClientKeyFile;
|
|
}
|
|
// lib.optionalAttrs (baoDeploy.serverCaFile != null) {
|
|
BAO_CACERT = baoDeploy.serverCaFile;
|
|
};
|
|
script = ''
|
|
set -euo pipefail
|
|
|
|
${atomicWriteSecret}
|
|
${refreshConsumer}
|
|
|
|
err="$(mktemp)"
|
|
trap 'rm -f "$err"' EXIT
|
|
|
|
# `-token-only` answers on stdout and needs no token helper.
|
|
if ! BAO_TOKEN="$(bao login -method=cert -token-only 2>"$err")"; then
|
|
echo "could not log in to swarm-bao with the CI runner's certificate; leaving its registration token as it is." >&2
|
|
if [ -s "$err" ]; then
|
|
cat "$err" >&2
|
|
else
|
|
echo "bao failed without writing a diagnostic." >&2
|
|
fi
|
|
exit 1
|
|
fi
|
|
export BAO_TOKEN
|
|
|
|
if ! token="$(bao kv get -field=value ${lib.escapeShellArg storePath} 2>"$err")"; then
|
|
echo "swarm-bao did not return ${storePath}; retrying until swarm-controller publishes it." >&2
|
|
if [ -s "$err" ]; then
|
|
cat "$err" >&2
|
|
else
|
|
echo "bao failed without writing a diagnostic." >&2
|
|
fi
|
|
exit 1
|
|
fi
|
|
|
|
if [ -z "$token" ]; then
|
|
echo "swarm-bao returned an empty ${storePath}; retrying." >&2
|
|
exit 1
|
|
fi
|
|
|
|
# Root-owned: systemd reads a `LoadCredential=` source as root, before
|
|
# the runner drops to its dynamic user.
|
|
install -d -m 0700 ${lib.escapeShellArg (builtins.dirOf hostTokenFile)}
|
|
if secret_differs ${lib.escapeShellArg hostTokenFile} "$token"; then
|
|
atomic_write_secret 0600 "" ${lib.escapeShellArg hostTokenFile} "$token"
|
|
fi
|
|
|
|
# `LoadCredential=` copies the file at start, so a runner still waiting
|
|
# to register sees a new token only once it restarts.
|
|
refresh_consumer ${lib.escapeShellArg machine} ${runnerUnit}.service ${lib.escapeShellArg hostTokenFile}
|
|
'';
|
|
};
|
|
|
|
systemd.timers.swarm-bao-ci-runner = storeRefreshTimer "re-read the CI runner's registration token from the swarm secret store";
|
|
|
|
systemd.services."container@${machine}" = caTrust.containerOrdering;
|
|
|
|
containers.${machine} = {
|
|
autoStart = true;
|
|
ephemeral = false;
|
|
# Journal files on the host, not inside the container: nixpkgs hardcodes
|
|
# --link-journal=try-guest, and EXTRA_NSPAWN_FLAGS expands after it.
|
|
extraFlags = [ "--link-journal=host" ];
|
|
inherit privateNetwork;
|
|
hostBridge = networkCfg.bridgeName;
|
|
|
|
bindMounts = {
|
|
# Builds go through the HOST nix-daemon, so they inherit its
|
|
# buildMachines and max-jobs. The directory rather than the socket
|
|
# file, so the mount survives a daemon restart replacing the inode.
|
|
"/nix/var/nix/daemon-socket" = {
|
|
hostPath = "/nix/var/nix/daemon-socket";
|
|
isReadOnly = false;
|
|
};
|
|
}
|
|
// caTrust.bindMount;
|
|
|
|
config =
|
|
{
|
|
pkgs,
|
|
lib,
|
|
config,
|
|
...
|
|
}:
|
|
let
|
|
runnerCfg = config.services.gitea-actions-runner;
|
|
instanceCfg = runnerCfg.instances.${instance};
|
|
# The same file upstream generates and hands the daemon.
|
|
runnerConfigFile = (pkgs.formats.yaml { }).generate "config.yaml" instanceCfg.settings;
|
|
# In the form the daemon rewrites `.runner`'s labels to on start
|
|
# (act_runner's `labels.Parse` then `ToStrings`): `<name>:<scheme>`
|
|
# with `host` for a missing scheme, then `:<arg>` if there is one.
|
|
normaliseLabel =
|
|
label:
|
|
let
|
|
parts = lib.splitString ":" label;
|
|
scheme = if lib.length parts >= 2 then lib.elemAt parts 1 else "host";
|
|
arg = lib.concatStringsSep ":" (lib.drop 2 parts);
|
|
in
|
|
"${lib.head parts}:${scheme}" + lib.optionalString (arg != "") ":${arg}";
|
|
registeredLabels = lib.concatStringsSep "," (
|
|
lib.sort lib.lessThan (map normaliseLabel instanceCfg.labels)
|
|
);
|
|
in
|
|
{
|
|
imports = [
|
|
../container-modules/swarm-container.nix
|
|
caBundleModule
|
|
];
|
|
|
|
# Its own netns, so it keeps its own firewall; resolvconf stays on
|
|
# and serves `networking.nameservers` below.
|
|
services.hyperhive.swarmContainer = {
|
|
inherit privateNetwork;
|
|
writesOwnResolvConf = false;
|
|
};
|
|
|
|
# The gateway's dnsmasq on the bridge answers the forge's name with
|
|
# the bridge IP where this host runs the forge, and forwards it
|
|
# upstream everywhere else.
|
|
networking.nameservers = [ networkCfg.bridgeIp ];
|
|
networking.interfaces.eth0.useDHCP = true;
|
|
|
|
# nspawn containers can't create user namespaces, so nix sandboxing
|
|
# fails; see docs/process/gotchas.md. `fallback` is a client-side
|
|
# option sent per daemon connection, so the host's value does not
|
|
# reach builds started here.
|
|
nix.settings.sandbox-fallback = lib.mkForce true;
|
|
nix.settings.fallback = true;
|
|
nix.settings.experimental-features = [
|
|
"nix-command"
|
|
"flakes"
|
|
];
|
|
|
|
# Both the service and its socket are off: the socket unit owns
|
|
# `/nix/var/nix/daemon-socket/socket` and would race the bind mount.
|
|
systemd.globalEnvironment.NIX_REMOTE = "daemon";
|
|
systemd.services.nix-daemon.enable = false;
|
|
systemd.sockets.nix-daemon.enable = false;
|
|
|
|
systemd.tmpfiles.rules = [
|
|
"d ${stateDir} 0700 root root -"
|
|
# `LoadCredential=` fails the unit when the file is missing; an
|
|
# empty one lets the register unit report the missing token.
|
|
"f ${tokenFile} 0600 root root -"
|
|
# Unused: nothing in this module writes or reads them. The first
|
|
# holds a token; the others are upstream's re-registration markers.
|
|
"r ${stateDir}/registration-token.env"
|
|
"r /var/lib/private/gitea-runner/${instance}/.token-hash"
|
|
"r /var/lib/private/gitea-runner/${instance}/.labels"
|
|
];
|
|
|
|
services.gitea-actions-runner.package = cfg.package;
|
|
|
|
services.gitea-actions-runner.instances.${instance} = {
|
|
enable = true;
|
|
inherit (cfg) name labels;
|
|
url = forgeUrl;
|
|
# Upstream refuses an instance without a token source, and turns
|
|
# this one into the `EnvironmentFile=` the daemon unit drops.
|
|
inherit tokenFile;
|
|
settings = {
|
|
runner.capacity = cfg.concurrency;
|
|
runner.timeout = cfg.jobTimeout;
|
|
};
|
|
};
|
|
|
|
environment.systemPackages = [
|
|
pkgs.git
|
|
pkgs.nix
|
|
];
|
|
|
|
# The only unit that can read the registration token: systemd hands
|
|
# a credential to every process of the unit that loads it, and the
|
|
# daemon's processes include every host job. Leaves a usable
|
|
# `.runner` behind or fails.
|
|
#
|
|
# The daemon's dynamic user and state dir: units naming the same
|
|
# `DynamicUser` share its uid, so `.runner` is the daemon's own.
|
|
systemd.services.${registerUnit} = {
|
|
description = "register the CI runner with the forge when it has no .runner";
|
|
wants = [ "network-online.target" ];
|
|
after = [ "network-online.target" ];
|
|
environment = {
|
|
HOME = "/var/lib/gitea-runner/${instance}";
|
|
# In the form the daemon rewrites `.runner`'s labels to.
|
|
SWARM_CI_RUNNER_LABELS = registeredLabels;
|
|
};
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
DynamicUser = true;
|
|
User = "gitea-runner";
|
|
StateDirectory = "gitea-runner";
|
|
LoadCredential = [ "${tokenCredential}:${tokenFile}" ];
|
|
# `register` retries an unreachable forge without end; under the
|
|
# daemon's own start timeout, so the failure is this unit's.
|
|
TimeoutStartSec = 60;
|
|
# Removes `.runner` in the cases where reusing it would fail, so
|
|
# the register step registers again.
|
|
#
|
|
# `+`: root, because the system journal is out of the dynamic
|
|
# user's reach.
|
|
ExecStartPre = "+${pkgs.writeShellScript "swarm-ci-runner-precond" ''
|
|
set -eu
|
|
runner=${runnerFile}
|
|
started=${daemonStartedFile}
|
|
|
|
# The forge refused the registration the daemon last started
|
|
# with. Read on the next start rather than when the daemon
|
|
# exits, so journald has its last lines by then.
|
|
if [ -f "$started" ]; then
|
|
since="$(< "$started")"
|
|
${pkgs.coreutils}/bin/rm -f "$started"
|
|
if ${pkgs.systemd}/bin/journalctl -u ${runnerUnit}.service --since "@$since" -o cat --no-pager \
|
|
| ${pkgs.gnugrep}/bin/grep -qF ${lib.escapeShellArg rejectedMessage}; then
|
|
echo "the forge has no record of this runner's registration; registering again" >&2
|
|
${pkgs.coreutils}/bin/rm -f "$runner"
|
|
fi
|
|
fi
|
|
|
|
# `.runner` records the address it registered at, and the
|
|
# daemon dials that one, not the configured one.
|
|
if [ -f "$runner" ] \
|
|
&& [ "$(${pkgs.jq}/bin/jq -r .address "$runner")" != ${lib.escapeShellArg forgeUrl} ]; then
|
|
echo "$runner names another forge address; registering again at ${forgeUrl}" >&2
|
|
${pkgs.coreutils}/bin/rm -f "$runner"
|
|
fi
|
|
|
|
# The daemon declares the labels `.runner` records, not the
|
|
# configured ones.
|
|
if [ -f "$runner" ] \
|
|
&& [ "$(${pkgs.jq}/bin/jq -r '.labels | sort | join(",")' "$runner")" != "$SWARM_CI_RUNNER_LABELS" ]; then
|
|
echo "$runner names other labels; registering again with $SWARM_CI_RUNNER_LABELS" >&2
|
|
${pkgs.coreutils}/bin/rm -f "$runner"
|
|
fi
|
|
''}";
|
|
};
|
|
script = ''
|
|
set -u
|
|
dir="$STATE_DIRECTORY/${instance}"
|
|
mkdir -p "$dir"
|
|
cd "$dir"
|
|
if [ -e .runner ]; then
|
|
exit 0
|
|
fi
|
|
|
|
token="$(< "$CREDENTIALS_DIRECTORY/${tokenCredential}")"
|
|
if [ -z "$token" ]; then
|
|
echo "no registration token yet; swarm-bao-ci-runner on the host delivers it" >&2
|
|
exit 1
|
|
fi
|
|
exec ${lib.getExe runnerCfg.package} register --no-interactive \
|
|
--instance ${lib.escapeShellArg instanceCfg.url} \
|
|
--token "$token" \
|
|
--name ${lib.escapeShellArg instanceCfg.name} \
|
|
--labels ${lib.escapeShellArg (lib.concatStringsSep "," instanceCfg.labels)} \
|
|
--config ${runnerConfigFile}
|
|
'';
|
|
};
|
|
|
|
systemd.services.${runnerUnit} = {
|
|
# `path` adds to the PATH the upstream module builds instead of
|
|
# replacing it.
|
|
path = [ pkgs.nix ];
|
|
# The upstream unit is `DynamicUser`, whose state dir systemd mounts
|
|
# `noexec`, so a job could not execute what it builds in its
|
|
# workspace. `ExecPaths=` resolves against the host root, where
|
|
# `/var/lib/gitea-runner` is a symlink into `private/`; `-` makes
|
|
# whichever spelling is absent a no-op.
|
|
serviceConfig.ExecPaths = [
|
|
"-/var/lib/gitea-runner"
|
|
"-/var/lib/private/gitea-runner"
|
|
];
|
|
environment = lib.mkIf caTrust.useSelfSigned {
|
|
NODE_EXTRA_CA_CERTS = caTrust.caContainerPath;
|
|
};
|
|
# A host job inherits the daemon's whole environment and can read
|
|
# its credentials, so the daemon carries the token in neither.
|
|
serviceConfig.EnvironmentFile = lib.mkForce [ ];
|
|
# Replaces upstream's pre-start step, which registers from
|
|
# `$TOKEN`, so the register unit is the one registration path.
|
|
#
|
|
# Started from here rather than through `Requires=`: a failed
|
|
# requirement fails the start job, which `Restart=` does not retry,
|
|
# while a failed pre-start step does.
|
|
serviceConfig.ExecStartPre = lib.mkForce [
|
|
"+${pkgs.systemd}/bin/systemctl start ${registerUnit}.service"
|
|
"+${pkgs.writeShellScript "swarm-ci-runner-mark-start" ''
|
|
${pkgs.coreutils}/bin/date +%s > ${daemonStartedFile}
|
|
''}"
|
|
];
|
|
# Restarted until the token arrives, however long the store and
|
|
# the controller take. `mkForce`: upstream sets both.
|
|
serviceConfig.Restart = lib.mkForce "on-failure";
|
|
serviceConfig.RestartSec = lib.mkForce 15;
|
|
startLimitIntervalSec = 0;
|
|
};
|
|
};
|
|
};
|
|
};
|
|
}
|