grafana+prometheus: initial setup

machines/gerd.nix

@@ -32,6 +32,8 @@
     ./gerd/services/rallly
 
     ./gerd/services/notify
+
+    ./gerd/services/monitoring
   ];
 
   networking.hostName = "gerd";
@@ -49,6 +51,8 @@
         "safe/svcs/stalwart" = { mountpoint = "/srv/stalwart"; extra.options.quota = "5G"; };
         "safe/svcs/synapse" = { mountpoint = "/srv/synapse"; extra.options.quota = "5G"; };
         "safe/svcs/wger" = { mountpoint = "/srv/wger"; extra.options.quota = "5G"; };
+        "safe/svcs/prometheus" = { mountpoint = "/srv/prometheus"; extra.options.quota = "5G"; };
+        
         "safe/svcs/postgresql" = { mountpoint = "/srv/postgresql"; extra.options.quota = "5G"; };
         "backup/postgresql" = { mountpoint = "/media/backup/postgresqlbackup"; extra.options.quota = "5G"; };
       };

machines/gerd/services/lldap/default.nix

@@ -275,7 +275,7 @@ in {
       user_id = name;
       display_name = name; # required for nextcloud
       membermail = mkProvisionEmail name;
-      groups = with lconfig.groups; [ admin nextcloud_admin member ];
+      groups = with lconfig.groups; [ admin nextcloud_admin grafana_admin member ];
       membermaildiskquota = 100*1024*1024; # mb
       nextcloudquota = 100*1024*1024; # mb
     });

machines/gerd/services/lldap/provision.nix

@@ -37,6 +37,7 @@
         "system_service" = {};
         "system_mail" = {};
         "nextcloud_admin" = {};
+        "grafana_admin" = {};
       };
 
       # attributes

machines/gerd/services/monitoring/default.nix

@@ -0,0 +1,6 @@
+{
+  imports = [
+    ./grafana.nix
+    ./prometheus.nix
+  ];
+}

machines/gerd/services/monitoring/grafana.nix

@@ -0,0 +1,91 @@
+{ config, ... }:
+
+let
+  svc_domain = "grafana.${config.mine.shared.settings.domain}";
+
+  auth_domain = config.mine.shared.settings.authelia.domain;
+
+  grafana_user = config.systemd.services.grafana.serviceConfig.User;
+in {
+  services.grafana = {
+    enable = true;
+    settings = {
+      server = {
+        http_addr = "127.0.0.1";
+        http_port = 3010;
+        root_url = "https://${svc_domain}";
+      };
+
+      # only allow signun with oauth
+      auth.disable_login_form = true;
+
+      "auth.generic_oauth" = {
+        enabled = true;
+        name = "Authelia";
+        icon = "signin";
+        client_id = "grafana";
+        client_secret = "$__file{${config.age.secrets.grafana-authelia-secret.path}}";
+        scopes = "openid profile email groups";
+        empty_scopes = false;
+        auth_url = "https://${auth_domain}/api/oidc/authorization";
+        token_url = "https://${auth_domain}/api/oidc/token";
+        api_url = "https://${auth_domain}/api/oidc/userinfo";
+        login_attribute_path = "preferred_username";
+        groups_attribute_path = "groups";
+        name_attribute_path = "name";
+        use_pkce = true;
+
+        role_attribute_path = config.mine.shared.lib.ldap.mkScope (lconfig: llib:
+          "contains(groups, '${lconfig.groups.grafana_admin}') && 'Admin' || contains(groups, 'editor') && 'Editor' || 'Viewer'"
+        );
+      };
+    };
+
+    provision = {
+      enable = true;
+      
+      # dashboards.settings.providers = [{
+      # name = "my dashboards";
+      # options.path = "/etc/grafana-dashboards";
+      # }];
+      
+      datasources.settings.datasources = [
+        {
+          name = "Prometheus";
+          type = "prometheus";
+          url = "http://${config.services.prometheus.listenAddress}:${toString config.services.prometheus.port}";
+        }
+      ];
+    };
+  };
+
+  # authelia
+  services.authelia.instances.main.settings.identity_providers.oidc.clients = [{
+    client_id = "grafana";
+    client_name = "Grafana";
+    client_secret = "$pbkdf2-sha512$310000$81MV1.67njuS/5H2UvVsnA$vaNO3/tzVA76Jho4ngS.xFjDuYn1sDn/9qo7cD0ueMnVvzaoJj00ND5wCGzVSUnvLuxNE/enC1K5r7xKAe/Hrg";
+    redirect_uris = [ "https://${svc_domain}/login/generic_oauth" ];
+    scopes = [
+      "openid"
+      "email"
+      "profile"
+      "groups"
+    ];
+  }];
+
+  environment.persistence.root.directories = [
+    config.services.grafana.dataDir
+  ];
+
+  systemd.tmpfiles.rules = [
+    "Z ${config.services.grafana.dataDir} 0770 ${grafana_user} ${grafana_user} -"
+  ];
+
+  age.secrets.grafana-authelia-secret.owner = grafana_user;
+
+  services.nginx.virtualHosts."${svc_domain}" = {
+    forceSSL = true;
+    enableACME = true;
+    locations."/".proxyPass = "http://localhost:${builtins.toString config.services.grafana.settings.server.http_port}";
+  };
+}

machines/gerd/services/monitoring/prometheus.nix

@@ -0,0 +1,27 @@
+{ config, ... }:
+
+let
+  prometheus_user = config.systemd.services.prometheus.serviceConfig.User;
+
+  fullDataDirPath = "/var/lib/${config.services.prometheus.stateDir}";
+
+  filesetPath = config.mine.zfsMounts."rpool/safe/svcs/prometheus";
+in {
+  services.prometheus = {
+    enable = true;
+    globalConfig.scrape_interval = "10s";
+    listenAddress = "localhost";
+
+    # default is 15 days, we just set it to 14 to be explicit
+    retentionTime = "14d";
+  };
+
+  fileSystems."${filesetPath}".neededForBoot = true;
+  environment.persistence."${filesetPath}".directories = [
+    fullDataDirPath
+  ];
+
+  systemd.tmpfiles.rules = [
+    "Z ${fullDataDirPath} 0770 ${prometheus_user} ${prometheus_user} -"
+  ];
+}

secrets/default.nix

@@ -62,6 +62,9 @@
     # notify
     notify-ldap-pass.file = ./notify/ldap-pass.age;
     notify-env.file = ./notify/env.age;
+
+    # grafana
+    grafana-authelia-secret.file = ./grafana/authelia-secret.age;
   };
 
   users.groups.secrets-lldap-bind-user-pass = {};

secrets/grafana/authelia-secret.age

@@ -0,0 +1,11 @@
+age-encryption.org/v1
+-> ssh-ed25519 QSDXqg mcA7aWulfqHTARfxzs9ECZaJRMZKLxZgl4uYXrsL6Tk
+IOKrdtTiG/Wc8qQb5zip1F3B4BHAGkEw8hjz22UY80k
+-> X25519 kqD2VC9Vw/2rrd/C1TR5He/78anx3UYXNbjs0vNXCz4
+ZYenf1LK+YAlil/oiZIfGGyaK9S6pt8LLpCbmlaKn9s
+-> ssh-ed25519 n8n9DQ PlW/1TA71RhclXIC2RlKUUOnqOq3qWy8yshqgM3Nu10
+2j6c3UjFc/RJJrqeWIezHx53DcPHFPi5a8WXnyqkXhU
+-> ssh-ed25519 BTp6UA n2idpPd9RFDbzvD2svo3A0NU7kx1nUEYzwFs0gpxn3Q
+/4F5l1dXBvF0nWXvT8nxPPCAxB4heeUMSBrGMY3gfng
+--- 7xw3+Ket2jYmH8wsoG2ivWUYLkyoR0et5FELrn+zzMo
+	9XzvèäJºEŠó«‘y⺈†è}\šÙ©‰ï\xÓºeè”11ûõ¯ƒô7XÒÑb%„á Õ˜.…ïj‰!‹Ä6œšBÃ[/ÆÀx!8Àâ‹ÕÔÿÿÍ´¤'2ŠvRúž§4W:]k

secrets/secrets.nix

@@ -72,4 +72,7 @@ in
   # notify
   "notify/ldap-pass.age".publicKeys = defaultAccess;
   "notify/env.age".publicKeys = defaultAccess;
+
+  # grafana
+  "grafana/authelia-secret.age".publicKeys = defaultAccess;
 }