nginx update with grafana, loki, and promtail

.gitignore

@@ -295,3 +295,4 @@ venv.bak/
 .mypy_cache/
 
 # End of https://www.gitignore.io/api/python
+switchtypes.tsv

nix/nixos-configurations/monitor/configuration.nix

@@ -0,0 +1,190 @@
+{
+  config,
+  pkgs,
+  inputs,
+  ...
+}:
+let
+  hostname = "monitoring.scale.lan";
+  dashboard = pkgs.copyPathToStore ../../../monitoring/openwrt_dashboard.json;
+in
+{
+  boot.kernelParams = [
+    "console=ttyS0"
+    "boot.shell_on_fail"
+  ];
+
+  systemd.network = {
+    enable = true;
+    networks = {
+      "10-lan" = {
+        # to match enp0 or eth0
+        name = "e*0*";
+        enable = true;
+        address = [
+          "10.0.3.6/24"
+          "2001:470:f026:103::6"
+        ];
+        routes = [
+          { routeConfig.Gateway = "10.0.3.1"; }
+          { routeConfig.Gateway = "2001:470:f026:103::1"; }
+        ];
+      };
+    };
+  };
+  networking.hostName = "monitor";
+  networking.firewall.allowedTCPPorts = [
+    80
+    443
+  ];
+
+  # TODO: How to handle sudo esculation
+  security.sudo.wheelNeedsPassword = false;
+
+  environment.systemPackages = with pkgs; [
+    vim
+    git
+    bintools
+  ];
+
+  services = {
+    prometheus = {
+      enable = true;
+      enableReload = true;
+      scrapeConfigs = [
+        {
+          job_name = "prometheus";
+          static_configs = [
+            {
+              targets = [ "localhost:${toString config.services.prometheus.exporters.node.port}" ];
+              labels = {
+                instance = "localhost";
+              };
+            }
+          ];
+        }
+        {
+          job_name = "ap";
+          static_configs = builtins.fromJSON (
+            builtins.readFile "${pkgs.scale-network.scaleInventory}/config/prom.json"
+          );
+        }
+      ];
+    };
+
+    grafana = {
+      enable = true;
+      settings = {
+        server = {
+          http_addr = "127.0.0.1";
+          http_port = 3000;
+          domain = "${hostname}";
+        };
+        analytics.reporting_enabled = false;
+      };
+      provision = {
+        # Can use just datasources anymore
+        # https://github.com/NixOS/nixpkgs/blob/41de143fda10e33be0f47eab2bfe08a50f234267/nixos/modules/services/monitoring/grafana.nix#L101-L104
+        datasources.settings.datasources = [
+          {
+            name = "prometheus";
+            uid = "P1809F7CD0C75ACF3";
+            type = "prometheus";
+            access = "proxy";
+            url = "http://127.0.0.1:${toString config.services.prometheus.port}";
+          }
+        ];
+        dashboards.settings.providers = [
+          {
+            name = "openwrt";
+            options.path = dashboard;
+          }
+        ];
+      };
+    };
+
+    services.promtail = {
+      enable = true;
+      configuration = {
+        server = {
+          http_listen_port = 3300;
+          grpc_listen_port = 0;
+        };
+        positions = {
+          filename = "/tmp/positions.yaml";
+        };
+        clients = [{
+          url = "http://127.0.0.1:3100/loki/api/v1/push";
+        }];
+        scrape_configs = [{
+          job_name = "journal";
+          journal = {
+            max_age = "12h";
+            labels = {
+              job = "systemd-journal";
+              host = "pihole";
+            };
+          };
+          relabel_configs = [{
+            source_labels = [ "__journal__systemd_unit" ];
+            target_label = "unit";
+          }];
+        }];
+      };
+    };
+
+    nginx = {
+      enable = true;
+      recommendedProxySettings = true;
+      recommendedOptimisation = true;
+      recommendedGzipSettings = true;
+      # recommendedTlsSettings = true;
+
+      upstreams = {
+        "grafana" = {
+          servers = {
+            "127.0.0.1:3100" = {};
+          };
+        };
+        "loki" = {
+          servers = {
+            "127.0.0.1:3200" = {};
+          };
+        };
+        "promtail" = {
+          servers = {
+            "127.0.0.1:3300" = {};
+          };
+        };
+      };
+      virtualHosts.grafana = {
+        locations."/" = {
+          proxyPass = "http://grafana";
+          proxyWebsockets = true;
+        };
+        listen = [{
+          addr = "192.168.1.147";
+          port = 8010;
+        }];
+      };
+
+      # confirm with http://192.168.1.10:8030/loki/api/v1/status/buildinfo
+      #     (or)     /config /metrics /ready
+      virtualHosts.loki = {
+        locations."/".proxyPass = "http://loki";
+        listen = [{
+          addr = "192.168.1.147";
+          port = 8020;
+        }];
+      };
+
+      virtualHosts.promtail = {
+        locations."/".proxyPass = "http://promtail";
+        listen = [{
+          addr = "192.168.1.147";
+          port = 8030;
+        }];
+      };
+    };
+  };
+}