From d6be5fefeaa7ed361d858771b4561be2ed83ffd5 Mon Sep 17 00:00:00 2001 From: eyjhb Date: Fri, 14 Mar 2025 16:40:19 +0100 Subject: [PATCH 1/9] nginx: block all `/metrics` endpoints --- shared/applications/server/nginx.nix | 89 ++++++++++++++++------------ 1 file changed, 50 insertions(+), 39 deletions(-) diff --git a/shared/applications/server/nginx.nix b/shared/applications/server/nginx.nix index 0d837d5..9de4208 100644 --- a/shared/applications/server/nginx.nix +++ b/shared/applications/server/nginx.nix @@ -10,47 +10,58 @@ let -out "$out/ca.pem" -keyout "$out/ca.key" ''; in { - services.nginx = { - enable = true; - - recommendedOptimisation = true; - recommendedTlsSettings = true; - recommendedGzipSettings = true; - # recommendedBrotliSettings = true; - recommendedProxySettings = true; - - # only allow PFS-enabled ciphers with AES256 - sslCiphers = "AES256+EECDH:AES256+EDH:!aNULL"; - - # disable access logs - commonHttpConfig= '' - access_log off; - ''; - - # setup a default site - virtualHosts.default = { - default = lib.mkDefault true; - addSSL = true; - - sslCertificateKey = "${snakeOilCa}/ca.key"; - sslCertificate = "${snakeOilCa}/ca.pem"; - - root = pkgs.writeTextDir "index.html" '' - - - Nothing to see - - -

Like I said, nothing to see here

- - - ''; - }; + # block all /metrics endpoints + options.services.nginx.virtualHosts = lib.mkOption { + type = lib.types.attrsOf (lib.types.submodule { + config.locations."/metrics" = lib.mkDefault { + extraConfig = "deny all;"; + }; + }); }; - networking.firewall = { - allowedTCPPorts = [80 443]; - allowedUDPPorts = [443]; + config = { + services.nginx = { + enable = true; + + recommendedOptimisation = true; + recommendedTlsSettings = true; + recommendedGzipSettings = true; + # recommendedBrotliSettings = true; + recommendedProxySettings = true; + + # only allow PFS-enabled ciphers with AES256 + sslCiphers = "AES256+EECDH:AES256+EDH:!aNULL"; + + # disable access logs + commonHttpConfig= '' + access_log off; + ''; + + # setup a default site + virtualHosts.default = { + default = lib.mkDefault true; + addSSL = true; + + sslCertificateKey = "${snakeOilCa}/ca.key"; + sslCertificate = "${snakeOilCa}/ca.pem"; + + root = pkgs.writeTextDir "index.html" '' + + + Nothing to see + + +

Like I said, nothing to see here

+ + + ''; + }; + }; + + networking.firewall = { + allowedTCPPorts = [80 443]; + allowedUDPPorts = [443]; + }; }; } From fc37d7a802fbea2b2f0f1cdb5c5ad232e0eacc13 Mon Sep 17 00:00:00 2001 From: eyjhb Date: Fri, 14 Mar 2025 16:40:23 +0100 Subject: [PATCH 2/9] easy-zfs-mounts: do not rely on fileSystems, so it can be used with impermanence --- shared/modules/easy-zfs-mounts.nix | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/shared/modules/easy-zfs-mounts.nix b/shared/modules/easy-zfs-mounts.nix index ded1b7a..34c3bce 100644 --- a/shared/modules/easy-zfs-mounts.nix +++ b/shared/modules/easy-zfs-mounts.nix @@ -10,9 +10,15 @@ in { default = {}; }; + # config = { + # mine.zfsMounts = let + # zfsFilesystems = lib.filterAttrs (_: v: v.fsType == "zfs") config.fileSystems; + # in lib.mapAttrs' (_: v: lib.nameValuePair v.device v.mountPoint) zfsFilesystems; + # }; + + # TODO: fix this better. We just do this, so we do not rely on fileSystems, otherwise we cannot + # use this with impermanence config = { - mine.zfsMounts = let - zfsFilesystems = lib.filterAttrs (_: v: v.fsType == "zfs") config.fileSystems; - in lib.mapAttrs' (_: v: lib.nameValuePair v.device v.mountPoint) zfsFilesystems; + mine.zfsMounts = lib.mapAttrs' (n: v: lib.nameValuePair ("rpool/" + n) v.mountpoint) config.mine.disks.pools.rpool.datasets; }; } From 4e58a128d84e7b2d31881009d0c039912239ade0 Mon Sep 17 00:00:00 2001 From: eyjhb Date: Fri, 14 Mar 2025 16:40:51 +0100 Subject: [PATCH 3/9] uptime-kuma: rename patch --- machines/gerd/services/uptime-kuma.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/machines/gerd/services/uptime-kuma.nix b/machines/gerd/services/uptime-kuma.nix index 3a86ad5..db2b2bb 100644 --- a/machines/gerd/services/uptime-kuma.nix +++ b/machines/gerd/services/uptime-kuma.nix @@ -23,7 +23,7 @@ in { hash = npmDepsHash; }; patches = [ - (pkgs.writeText "authelia.patch" '' + (pkgs.writeText "uptime-kuma-database-writeable.patch" '' diff --git a/server/database.js b/server/database.js index 3374aff9..9e890d28 100644 --- a/server/database.js From a10111a7910c7b9e26ed98199789c00ec7292606 Mon Sep 17 00:00:00 2001 From: eyjhb Date: Fri, 14 Mar 2025 16:41:44 +0100 Subject: [PATCH 4/9] nextcloud: moved admin into own ldap group --- machines/gerd/services/lldap/default.nix | 2 +- machines/gerd/services/lldap/module/default.nix | 1 + machines/gerd/services/lldap/provision.nix | 1 + machines/gerd/services/nextcloud.nix | 4 ++-- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/machines/gerd/services/lldap/default.nix b/machines/gerd/services/lldap/default.nix index 9115f69..7a599b9 100644 --- a/machines/gerd/services/lldap/default.nix +++ b/machines/gerd/services/lldap/default.nix @@ -275,7 +275,7 @@ in { user_id = name; display_name = name; # required for nextcloud membermail = mkProvisionEmail name; - groups = [ lconfig.groups.admin lconfig.groups.member ]; + groups = with lconfig.groups; [ admin nextcloud_admin member ]; membermaildiskquota = 100*1024*1024; # mb nextcloudquota = 100*1024*1024; # mb }); diff --git a/machines/gerd/services/lldap/module/default.nix b/machines/gerd/services/lldap/module/default.nix index 04032c4..34206d7 100644 --- a/machines/gerd/services/lldap/module/default.nix +++ b/machines/gerd/services/lldap/module/default.nix @@ -162,5 +162,6 @@ in { ${pythonEnv}/bin/python -m bootstrap.main ${configFile} ''; }; + systemd.services.lldap.restartTriggers = [ configFile ]; }; } diff --git a/machines/gerd/services/lldap/provision.nix b/machines/gerd/services/lldap/provision.nix index e5a3af9..fc48c32 100644 --- a/machines/gerd/services/lldap/provision.nix +++ b/machines/gerd/services/lldap/provision.nix @@ -36,6 +36,7 @@ "base_member" = {}; "system_service" = {}; "system_mail" = {}; + "nextcloud_admin" = {}; }; # attributes diff --git a/machines/gerd/services/nextcloud.nix b/machines/gerd/services/nextcloud.nix index a3387aa..81bf6b2 100644 --- a/machines/gerd/services/nextcloud.nix +++ b/machines/gerd/services/nextcloud.nix @@ -49,7 +49,7 @@ let ldapGroupFilter = config.mine.shared.lib.ldap.mkFilter (lconfig: llib: llib.mkAnd [ (llib.mkOC lconfig.oc.groupOfUniqueNames) - (llib.mkOr [ "cn=${lconfig.groups.admin}" "cn=${lconfig.groups.member}"]) + (llib.mkOr [ "cn=${lconfig.groups.nextcloud_admin}" "cn=${lconfig.groups.member}"]) ] ); ldapGroupFilterGroups = "admin;user"; @@ -86,7 +86,7 @@ let done # promote ldap admin group to admins - ${occ} ldap:promote-group ${config.mine.shared.settings.ldap.groups.admin} --yes -n + ${occ} ldap:promote-group ${config.mine.shared.settings.ldap.groups.nextcloud_admin} --yes -n ''; # script for resetting nextcloud admin password on each startup From efb17ea7fabb34dc721117a0a745510452882dec Mon Sep 17 00:00:00 2001 From: eyjhb Date: Fri, 14 Mar 2025 16:43:56 +0100 Subject: [PATCH 5/9] grafana+prometheus: initial setup --- machines/gerd.nix | 4 + machines/gerd/services/lldap/default.nix | 2 +- machines/gerd/services/lldap/provision.nix | 1 + machines/gerd/services/monitoring/default.nix | 6 ++ machines/gerd/services/monitoring/grafana.nix | 91 +++++++++++++++++++ .../gerd/services/monitoring/prometheus.nix | 27 ++++++ secrets/default.nix | 3 + secrets/grafana/authelia-secret.age | 11 +++ secrets/secrets.nix | 3 + 9 files changed, 147 insertions(+), 1 deletion(-) create mode 100644 machines/gerd/services/monitoring/default.nix create mode 100644 machines/gerd/services/monitoring/grafana.nix create mode 100644 machines/gerd/services/monitoring/prometheus.nix create mode 100644 secrets/grafana/authelia-secret.age diff --git a/machines/gerd.nix b/machines/gerd.nix index cee0c9d..3436fba 100644 --- a/machines/gerd.nix +++ b/machines/gerd.nix @@ -32,6 +32,8 @@ ./gerd/services/rallly ./gerd/services/notify + + ./gerd/services/monitoring ]; networking.hostName = "gerd"; @@ -49,6 +51,8 @@ "safe/svcs/stalwart" = { mountpoint = "/srv/stalwart"; extra.options.quota = "5G"; }; "safe/svcs/synapse" = { mountpoint = "/srv/synapse"; extra.options.quota = "5G"; }; "safe/svcs/wger" = { mountpoint = "/srv/wger"; extra.options.quota = "5G"; }; + "safe/svcs/prometheus" = { mountpoint = "/srv/prometheus"; extra.options.quota = "5G"; }; + "safe/svcs/postgresql" = { mountpoint = "/srv/postgresql"; extra.options.quota = "5G"; }; "backup/postgresql" = { mountpoint = "/media/backup/postgresqlbackup"; extra.options.quota = "5G"; }; }; diff --git a/machines/gerd/services/lldap/default.nix b/machines/gerd/services/lldap/default.nix index 7a599b9..bf25375 100644 --- a/machines/gerd/services/lldap/default.nix +++ b/machines/gerd/services/lldap/default.nix @@ -275,7 +275,7 @@ in { user_id = name; display_name = name; # required for nextcloud membermail = mkProvisionEmail name; - groups = with lconfig.groups; [ admin nextcloud_admin member ]; + groups = with lconfig.groups; [ admin nextcloud_admin grafana_admin member ]; membermaildiskquota = 100*1024*1024; # mb nextcloudquota = 100*1024*1024; # mb }); diff --git a/machines/gerd/services/lldap/provision.nix b/machines/gerd/services/lldap/provision.nix index fc48c32..b5f46ff 100644 --- a/machines/gerd/services/lldap/provision.nix +++ b/machines/gerd/services/lldap/provision.nix @@ -37,6 +37,7 @@ "system_service" = {}; "system_mail" = {}; "nextcloud_admin" = {}; + "grafana_admin" = {}; }; # attributes diff --git a/machines/gerd/services/monitoring/default.nix b/machines/gerd/services/monitoring/default.nix new file mode 100644 index 0000000..aea5039 --- /dev/null +++ b/machines/gerd/services/monitoring/default.nix @@ -0,0 +1,6 @@ +{ + imports = [ + ./grafana.nix + ./prometheus.nix + ]; +} diff --git a/machines/gerd/services/monitoring/grafana.nix b/machines/gerd/services/monitoring/grafana.nix new file mode 100644 index 0000000..c19967a --- /dev/null +++ b/machines/gerd/services/monitoring/grafana.nix @@ -0,0 +1,91 @@ +{ config, ... }: + +let + svc_domain = "grafana.${config.mine.shared.settings.domain}"; + + auth_domain = config.mine.shared.settings.authelia.domain; + + grafana_user = config.systemd.services.grafana.serviceConfig.User; +in { + services.grafana = { + enable = true; + settings = { + server = { + http_addr = "127.0.0.1"; + http_port = 3010; + root_url = "https://${svc_domain}"; + }; + + # only allow signun with oauth + auth.disable_login_form = true; + + "auth.generic_oauth" = { + enabled = true; + name = "Authelia"; + icon = "signin"; + client_id = "grafana"; + client_secret = "$__file{${config.age.secrets.grafana-authelia-secret.path}}"; + scopes = "openid profile email groups"; + empty_scopes = false; + auth_url = "https://${auth_domain}/api/oidc/authorization"; + token_url = "https://${auth_domain}/api/oidc/token"; + api_url = "https://${auth_domain}/api/oidc/userinfo"; + login_attribute_path = "preferred_username"; + groups_attribute_path = "groups"; + name_attribute_path = "name"; + use_pkce = true; + + role_attribute_path = config.mine.shared.lib.ldap.mkScope (lconfig: llib: + "contains(groups, '${lconfig.groups.grafana_admin}') && 'Admin' || contains(groups, 'editor') && 'Editor' || 'Viewer'" + ); + }; + }; + + provision = { + enable = true; + + # dashboards.settings.providers = [{ + # name = "my dashboards"; + # options.path = "/etc/grafana-dashboards"; + # }]; + + datasources.settings.datasources = [ + { + name = "Prometheus"; + type = "prometheus"; + url = "http://${config.services.prometheus.listenAddress}:${toString config.services.prometheus.port}"; + } + ]; + }; + }; + + # authelia + services.authelia.instances.main.settings.identity_providers.oidc.clients = [{ + client_id = "grafana"; + client_name = "Grafana"; + client_secret = "$pbkdf2-sha512$310000$81MV1.67njuS/5H2UvVsnA$vaNO3/tzVA76Jho4ngS.xFjDuYn1sDn/9qo7cD0ueMnVvzaoJj00ND5wCGzVSUnvLuxNE/enC1K5r7xKAe/Hrg"; + redirect_uris = [ "https://${svc_domain}/login/generic_oauth" ]; + scopes = [ + "openid" + "email" + "profile" + "groups" + ]; + }]; + + environment.persistence.root.directories = [ + config.services.grafana.dataDir + ]; + + systemd.tmpfiles.rules = [ + "Z ${config.services.grafana.dataDir} 0770 ${grafana_user} ${grafana_user} -" + ]; + + age.secrets.grafana-authelia-secret.owner = grafana_user; + + services.nginx.virtualHosts."${svc_domain}" = { + forceSSL = true; + enableACME = true; + locations."/".proxyPass = "http://localhost:${builtins.toString config.services.grafana.settings.server.http_port}"; + }; +} diff --git a/machines/gerd/services/monitoring/prometheus.nix b/machines/gerd/services/monitoring/prometheus.nix new file mode 100644 index 0000000..e9e9176 --- /dev/null +++ b/machines/gerd/services/monitoring/prometheus.nix @@ -0,0 +1,27 @@ +{ config, ... }: + +let + prometheus_user = config.systemd.services.prometheus.serviceConfig.User; + + fullDataDirPath = "/var/lib/${config.services.prometheus.stateDir}"; + + filesetPath = config.mine.zfsMounts."rpool/safe/svcs/prometheus"; +in { + services.prometheus = { + enable = true; + globalConfig.scrape_interval = "10s"; + listenAddress = "localhost"; + + # default is 15 days, we just set it to 14 to be explicit + retentionTime = "14d"; + }; + + fileSystems."${filesetPath}".neededForBoot = true; + environment.persistence."${filesetPath}".directories = [ + fullDataDirPath + ]; + + systemd.tmpfiles.rules = [ + "Z ${fullDataDirPath} 0770 ${prometheus_user} ${prometheus_user} -" + ]; +} diff --git a/secrets/default.nix b/secrets/default.nix index a17b16d..795e8d4 100644 --- a/secrets/default.nix +++ b/secrets/default.nix @@ -62,6 +62,9 @@ # notify notify-ldap-pass.file = ./notify/ldap-pass.age; notify-env.file = ./notify/env.age; + + # grafana + grafana-authelia-secret.file = ./grafana/authelia-secret.age; }; users.groups.secrets-lldap-bind-user-pass = {}; diff --git a/secrets/grafana/authelia-secret.age b/secrets/grafana/authelia-secret.age new file mode 100644 index 0000000..388e554 --- /dev/null +++ b/secrets/grafana/authelia-secret.age @@ -0,0 +1,11 @@ +age-encryption.org/v1 +-> ssh-ed25519 QSDXqg mcA7aWulfqHTARfxzs9ECZaJRMZKLxZgl4uYXrsL6Tk +IOKrdtTiG/Wc8qQb5zip1F3B4BHAGkEw8hjz22UY80k +-> X25519 kqD2VC9Vw/2rrd/C1TR5He/78anx3UYXNbjs0vNXCz4 +ZYenf1LK+YAlil/oiZIfGGyaK9S6pt8LLpCbmlaKn9s +-> ssh-ed25519 n8n9DQ PlW/1TA71RhclXIC2RlKUUOnqOq3qWy8yshqgM3Nu10 +2j6c3UjFc/RJJrqeWIezHx53DcPHFPi5a8WXnyqkXhU +-> ssh-ed25519 BTp6UA n2idpPd9RFDbzvD2svo3A0NU7kx1nUEYzwFs0gpxn3Q +/4F5l1dXBvF0nWXvT8nxPPCAxB4heeUMSBrGMY3gfng +--- 7xw3+Ket2jYmH8wsoG2ivWUYLkyoR0et5FELrn+zzMo + 9XzvèäJºEŠó«‘y⺈†è}\šÙ©‰ï\xÓºeè”11ûõ¯ƒô7XÒÑb%„á Õ˜.…ïj‰!‹Ä6œšBÃ[/ÆÀx!8Àâ‹ÕÔÿÿÍ´¤'2ŠvRúž§4W:]k \ No newline at end of file diff --git a/secrets/secrets.nix b/secrets/secrets.nix index e6c5e26..e18b798 100644 --- a/secrets/secrets.nix +++ b/secrets/secrets.nix @@ -72,4 +72,7 @@ in # notify "notify/ldap-pass.age".publicKeys = defaultAccess; "notify/env.age".publicKeys = defaultAccess; + + # grafana + "grafana/authelia-secret.age".publicKeys = defaultAccess; } From cb121c5369195e59d3107342a3e2628030c9dcf6 Mon Sep 17 00:00:00 2001 From: eyjhb Date: Fri, 14 Mar 2025 16:45:42 +0100 Subject: [PATCH 6/9] monitoring: added services --- machines/gerd/services/monitoring/default.nix | 11 ++++++ .../gerd/services/monitoring/mon-authelia.nix | 23 +++++++++++++ .../gerd/services/monitoring/mon-forgejo.nix | 14 ++++++++ .../gerd/services/monitoring/mon-hedgedoc.nix | 18 ++++++++++ .../monitoring/mon-matrix-synapse.nix | 27 +++++++++++++++ .../gerd/services/monitoring/mon-miniflux.nix | 16 +++++++++ .../gerd/services/monitoring/mon-postgres.nix | 34 +++++++++++++++++++ .../gerd/services/monitoring/mon-searx.nix | 16 +++++++++ .../gerd/services/monitoring/mon-stalwart.nix | 22 ++++++++++++ .../services/monitoring/mon-uptime-kuma.nix | 12 +++++++ machines/gerd/services/monitoring/mon-zfs.nix | 19 +++++++++++ machines/gerd/services/uptime-kuma.nix | 16 +++++++++ 12 files changed, 228 insertions(+) create mode 100644 machines/gerd/services/monitoring/mon-authelia.nix create mode 100644 machines/gerd/services/monitoring/mon-forgejo.nix create mode 100644 machines/gerd/services/monitoring/mon-hedgedoc.nix create mode 100644 machines/gerd/services/monitoring/mon-matrix-synapse.nix create mode 100644 machines/gerd/services/monitoring/mon-miniflux.nix create mode 100644 machines/gerd/services/monitoring/mon-postgres.nix create mode 100644 machines/gerd/services/monitoring/mon-searx.nix create mode 100644 machines/gerd/services/monitoring/mon-stalwart.nix create mode 100644 machines/gerd/services/monitoring/mon-uptime-kuma.nix create mode 100644 machines/gerd/services/monitoring/mon-zfs.nix diff --git a/machines/gerd/services/monitoring/default.nix b/machines/gerd/services/monitoring/default.nix index aea5039..bfa3f65 100644 --- a/machines/gerd/services/monitoring/default.nix +++ b/machines/gerd/services/monitoring/default.nix @@ -2,5 +2,16 @@ imports = [ ./grafana.nix ./prometheus.nix + + ./mon-postgres.nix + ./mon-stalwart.nix + ./mon-authelia.nix + ./mon-matrix-synapse.nix + ./mon-zfs.nix + ./mon-miniflux.nix + ./mon-hedgedoc.nix + ./mon-forgejo.nix + ./mon-uptime-kuma.nix + ./mon-searx.nix ]; } diff --git a/machines/gerd/services/monitoring/mon-authelia.nix b/machines/gerd/services/monitoring/mon-authelia.nix new file mode 100644 index 0000000..fa1bf3d --- /dev/null +++ b/machines/gerd/services/monitoring/mon-authelia.nix @@ -0,0 +1,23 @@ +{ config, lib, ... }: + +{ + services.authelia.instances.main.settings = { + telemetry.metrics = { + enabled = true; + }; + }; + + services.prometheus.scrapeConfigs = [ + { + job_name = "authelia"; + static_configs = [{ + targets = [ (lib.removePrefix "tcp://" config.services.authelia.instances.main.settings.telemetry.metrics.address) ]; + }]; + metric_relabel_configs = [{ + source_labels = [ "__name__" ]; + target_label = "__name__"; + replacement = "authelia_$1"; + }]; + } + ]; +} diff --git a/machines/gerd/services/monitoring/mon-forgejo.nix b/machines/gerd/services/monitoring/mon-forgejo.nix new file mode 100644 index 0000000..abd8214 --- /dev/null +++ b/machines/gerd/services/monitoring/mon-forgejo.nix @@ -0,0 +1,14 @@ +{ config, ... }: + +{ + services.forgejo.settings.metrics.ENABLED = true; + + services.prometheus.scrapeConfigs = [ + { + job_name = "forgejo"; + static_configs = [{ + targets = [ "localhost:${builtins.toString config.services.forgejo.settings.server.HTTPPORT}" ]; + }]; + } + ]; +} diff --git a/machines/gerd/services/monitoring/mon-hedgedoc.nix b/machines/gerd/services/monitoring/mon-hedgedoc.nix new file mode 100644 index 0000000..d9de0f7 --- /dev/null +++ b/machines/gerd/services/monitoring/mon-hedgedoc.nix @@ -0,0 +1,18 @@ +{ config, ... }: + +{ + services.hedgedoc.settings = { + # enabled by default anyways + # TODO(eyJhb): disable exposing this to the WORLD + enableStatsApi = true; + }; + + services.prometheus.scrapeConfigs = [ + { + job_name = "hedgedoc"; + static_configs = [{ + targets = [ "localhost:${builtins.toString config.services.hedgedoc.settings.port}"]; + }]; + } + ]; +} diff --git a/machines/gerd/services/monitoring/mon-matrix-synapse.nix b/machines/gerd/services/monitoring/mon-matrix-synapse.nix new file mode 100644 index 0000000..d569554 --- /dev/null +++ b/machines/gerd/services/monitoring/mon-matrix-synapse.nix @@ -0,0 +1,27 @@ +let + metrics_port = 9734; +in { + services.matrix-synapse = { + settings = { + enable_metrics = true; + listeners = [ + { + port = metrics_port; + type = "metrics"; + bind_addresses = [ "localhost" ]; + tls = false; + resources = []; + } + ]; + }; + }; + + services.prometheus.scrapeConfigs = [ + { + job_name = "matrix-synapse"; + static_configs = [{ + targets = [ "localhost:${builtins.toString metrics_port}"]; + }]; + } + ]; +} diff --git a/machines/gerd/services/monitoring/mon-miniflux.nix b/machines/gerd/services/monitoring/mon-miniflux.nix new file mode 100644 index 0000000..03642a4 --- /dev/null +++ b/machines/gerd/services/monitoring/mon-miniflux.nix @@ -0,0 +1,16 @@ +{ config, ... }: + +{ + services.miniflux.config = { + METRICS_COLLECTOR = 1; + }; + + services.prometheus.scrapeConfigs = [ + { + job_name = "miniflux"; + static_configs = [{ + targets = [ config.services.miniflux.config.LISTEN_ADDR ]; + }]; + } + ]; +} diff --git a/machines/gerd/services/monitoring/mon-postgres.nix b/machines/gerd/services/monitoring/mon-postgres.nix new file mode 100644 index 0000000..8f307d5 --- /dev/null +++ b/machines/gerd/services/monitoring/mon-postgres.nix @@ -0,0 +1,34 @@ +{ config, pkgs, ... }: + +{ + services.prometheus.exporters.postgres = { + enable = true; + listenAddress = "localhost"; + runAsLocalSuperUser = true; + + extraFlags = let + extraQuery = pkgs.writeText "prometehus-postgres-query.yaml" '' + pg_database: + query: "SELECT pg_database.datname, pg_database_size(pg_database.datname) as size FROM pg_database" + metrics: + - datname: + usage: "LABEL" + description: "Name of the database" + - size: + usage: "GAUGE" + description: "Disk space used by the database" + ''; + in [ + "--extend.query-path=${extraQuery}" + ]; + }; + + services.prometheus.scrapeConfigs = [ + { + job_name = "postgres"; + static_configs = [{ + targets = [ "localhost:${toString config.services.prometheus.exporters.postgres.port}" ]; + }]; + } + ]; +} diff --git a/machines/gerd/services/monitoring/mon-searx.nix b/machines/gerd/services/monitoring/mon-searx.nix new file mode 100644 index 0000000..5e8e0df --- /dev/null +++ b/machines/gerd/services/monitoring/mon-searx.nix @@ -0,0 +1,16 @@ +{ config, ... }: + +{ + services.searx.settings.general.open_metrics = "thisreallydoesnotmatterasitisnotaccessiblefromoutsideofthisserver"; + + services.prometheus.scrapeConfigs = [ + { + job_name = "searx"; + basic_auth.username = "canbeanything"; + basic_auth.password = config.services.searx.settings.general.open_metrics; + static_configs = [{ + targets = [ config.services.searx.uwsgiConfig.http ]; + }]; + } + ]; +} diff --git a/machines/gerd/services/monitoring/mon-stalwart.nix b/machines/gerd/services/monitoring/mon-stalwart.nix new file mode 100644 index 0000000..fcf704d --- /dev/null +++ b/machines/gerd/services/monitoring/mon-stalwart.nix @@ -0,0 +1,22 @@ +{ config, ... }: + +{ + services.stalwart-mail.settings = { + metrics.prometheus.enable = true; + }; + + services.prometheus.scrapeConfigs = [ + { + job_name = "stalwart"; + metrics_path = "/metrics/prometheus"; + static_configs = [{ + targets = [ "localhost:${toString config.mine.shared.settings.mail.ports.http_management}" ]; + }]; + metric_relabel_configs = [{ + source_labels = [ "__name__" ]; + target_label = "__name__"; + replacement = "stalwart_$1"; + }]; + } + ]; +} diff --git a/machines/gerd/services/monitoring/mon-uptime-kuma.nix b/machines/gerd/services/monitoring/mon-uptime-kuma.nix new file mode 100644 index 0000000..c6471e4 --- /dev/null +++ b/machines/gerd/services/monitoring/mon-uptime-kuma.nix @@ -0,0 +1,12 @@ +{ config, ... }: + +{ + services.prometheus.scrapeConfigs = [ + { + job_name = "uptime-kuma"; + static_configs = [{ + targets = [ "localhost:${builtins.toString config.services.uptime-kuma.settings.PORT}" ]; + }]; + } + ]; +} diff --git a/machines/gerd/services/monitoring/mon-zfs.nix b/machines/gerd/services/monitoring/mon-zfs.nix new file mode 100644 index 0000000..244d179 --- /dev/null +++ b/machines/gerd/services/monitoring/mon-zfs.nix @@ -0,0 +1,19 @@ +{ config, pkgs, ... }: + +{ + services.prometheus.exporters.zfs = { + enable = true; + listenAddress = "localhost"; + + extraFlags = [ "--collector.dataset-snapshot" ]; + }; + + services.prometheus.scrapeConfigs = [ + { + job_name = "zfs"; + static_configs = [{ + targets = [ "localhost:${toString config.services.prometheus.exporters.zfs.port}" ]; + }]; + } + ]; +} diff --git a/machines/gerd/services/uptime-kuma.nix b/machines/gerd/services/uptime-kuma.nix index db2b2bb..654bdc0 100644 --- a/machines/gerd/services/uptime-kuma.nix +++ b/machines/gerd/services/uptime-kuma.nix @@ -37,6 +37,22 @@ in { const Dialect = require("knex/lib/dialects/sqlite3/index.js"); '') + # TODO(eyJhb): do we really want this? + (pkgs.writeText "uptime-kuma-disable-metrics-auth.patch" '' + diff --git a/server/server.js b/server/server.js + index db58ae82..d650a42a 100644 + --- a/server/server.js + +++ b/server/server.js + @@ -292,7 +292,7 @@ let needSetup = false; + + // Prometheus API metrics /metrics + // With Basic Auth using the first user's username/password + - app.get("/metrics", apiAuth, prometheusAPIMetrics()); + + app.use("/metrics", prometheusAPIMetrics()); + + app.use("/", expressStaticGzip("dist", { + enableBrotli: true, + '') ]; }); }; From f0346a3c3818f75bd90ff66ce3a881a1872bee55 Mon Sep 17 00:00:00 2001 From: eyjhb Date: Fri, 14 Mar 2025 17:09:21 +0100 Subject: [PATCH 7/9] monitoring: added nextcloud as well --- machines/gerd/services/monitoring/default.nix | 1 + .../services/monitoring/mon-nextcloud.nix | 45 ++++++++++++++++++ secrets/default.nix | 1 + secrets/nextcloud/serverinfo-token.age | Bin 0 -> 595 bytes secrets/secrets.nix | 1 + 5 files changed, 48 insertions(+) create mode 100644 machines/gerd/services/monitoring/mon-nextcloud.nix create mode 100644 secrets/nextcloud/serverinfo-token.age diff --git a/machines/gerd/services/monitoring/default.nix b/machines/gerd/services/monitoring/default.nix index bfa3f65..9704826 100644 --- a/machines/gerd/services/monitoring/default.nix +++ b/machines/gerd/services/monitoring/default.nix @@ -13,5 +13,6 @@ ./mon-forgejo.nix ./mon-uptime-kuma.nix ./mon-searx.nix + ./mon-nextcloud.nix ]; } diff --git a/machines/gerd/services/monitoring/mon-nextcloud.nix b/machines/gerd/services/monitoring/mon-nextcloud.nix new file mode 100644 index 0000000..823f4fb --- /dev/null +++ b/machines/gerd/services/monitoring/mon-nextcloud.nix @@ -0,0 +1,45 @@ +{ config, lib, pkgs, ... }: + +let + # occ bin + occ = config.services.nextcloud.occ + "/bin/nextcloud-occ"; + + nextcloudSetupServerinfoToken = pkgs.writeShellScript "nextcloud-setup-serverinfo-token.sh" '' + # set serverinfo_token + SERVERINFO_TOKEN="$(cat $CREDENTIALS_DIRECTORY/nextcloud-serverinfo-token)" + ${occ} config:app:set serverinfo token --value "$SERVERINFO_TOKEN" > /dev/null 2>&1 + ''; +in { + systemd.services.nextcloud-setup = { + # runs this after all the main nextcloud-setup stuff + script = lib.mkAfter '' + ${nextcloudSetupServerinfoToken} + ''; + + # setup credentials for service + serviceConfig.LoadCredential = [ + "nextcloud-serverinfo-token:${config.age.secrets.nextcloud-serverinfo-token.path}" + ]; + }; + + services.prometheus.exporters.nextcloud = { + enable = true; + listenAddress = "localhost"; + tokenFile = config.age.secrets.nextcloud-serverinfo-token.path; + url = let + scheme = if config.services.nextcloud.https then "https" else "http"; + in "${scheme}://${config.services.nextcloud.hostName}"; + }; + + # setup permissions + age.secrets.nextcloud-serverinfo-token.owner = config.services.prometheus.exporters.nextcloud.user; + + services.prometheus.scrapeConfigs = [ + { + job_name = "nextcloud"; + static_configs = [{ + targets = [ "localhost:${builtins.toString config.services.prometheus.exporters.nextcloud.port}" ]; + }]; + } + ]; +} diff --git a/secrets/default.nix b/secrets/default.nix index 795e8d4..23e56f8 100644 --- a/secrets/default.nix +++ b/secrets/default.nix @@ -34,6 +34,7 @@ nextcloud-admin-pass.file = ./nextcloud/admin-pass.age; nextcloud-secrets.file = ./nextcloud/secrets.age; nextcloud-smtp-pass.file = ./nextcloud/smtp-pass.age; + nextcloud-serverinfo-token.file = ./nextcloud/serverinfo-token.age; # stalwart stalwart-admin-fallback-password.file = ./stalwart/admin-fallback-password.age; diff --git a/secrets/nextcloud/serverinfo-token.age b/secrets/nextcloud/serverinfo-token.age new file mode 100644 index 0000000000000000000000000000000000000000..c1da5c49c05f6a3f303e7cb8d06a85a3174ce70a GIT binary patch literal 595 zcmYdHPt{G$OD?J`D9Oyv)5|YP*Do{V(zR14F3!+RO))YxHMCR+40eeqOjmHX4D~2Z zD$Xj?4-0qo)Aur~C@8i}s&J~RC{AEp%uu|WzqL?ei#VE2O*(cS}E63C@D$hU5t2jF&In6zz z(mf*3yUNuoML)f~G_A_QBOhb~vVZa{@+@5f6(T(fN}VILLkmI*jFJM)DhiU5DwCrE zeWN@)@(mL$-BOcNvkW5*DwFd}xU%xy6N|hf!u*`|BT@^hyp7X6f&;P)0;__Yt8$CO zOd>5K+}%?3Q-Vq((QR`IDKHClR4A>83<}W>4vNgq@XZQKj7UlkGEK}VFN$>WG!6=} ztjaVGH7yD=baKxMiz<(dtjefJboVKV z@T%|* Date: Fri, 14 Mar 2025 17:09:52 +0100 Subject: [PATCH 8/9] prometheus: increased scrape_timeout --- machines/gerd/services/monitoring/prometheus.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/machines/gerd/services/monitoring/prometheus.nix b/machines/gerd/services/monitoring/prometheus.nix index e9e9176..894c7ea 100644 --- a/machines/gerd/services/monitoring/prometheus.nix +++ b/machines/gerd/services/monitoring/prometheus.nix @@ -10,6 +10,7 @@ in { services.prometheus = { enable = true; globalConfig.scrape_interval = "10s"; + globalConfig.scrape_timeout = "10s"; listenAddress = "localhost"; # default is 15 days, we just set it to 14 to be explicit From e18bdf5e3bb52656609a20bb5481028936d367b0 Mon Sep 17 00:00:00 2001 From: eyjhb Date: Fri, 14 Mar 2025 17:10:18 +0100 Subject: [PATCH 9/9] gerd.nix cleanup, maybe? --- machines/gerd.nix | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/machines/gerd.nix b/machines/gerd.nix index 3436fba..2c9f463 100644 --- a/machines/gerd.nix +++ b/machines/gerd.nix @@ -19,18 +19,14 @@ ./gerd/services/murmur.nix ./gerd/services/hedgedoc.nix ./gerd/services/cyberchef.nix - ./gerd/services/nextcloud.nix + ./gerd/services/nextcloud.nix ./gerd/services/stalwart ./gerd/services/wger ./gerd/services/searx.nix ./gerd/services/miniflux.nix - ./gerd/services/matrix - ./gerd/services/uptime-kuma.nix - ./gerd/services/rallly - ./gerd/services/notify ./gerd/services/monitoring