From f86111c840d7ee1cbeb34f52e8860e64008c3cec Mon Sep 17 00:00:00 2001 From: jeffreytjung <80990772+jeffreytjung@users.noreply.github.com> Date: Wed, 19 Feb 2025 19:36:19 -0800 Subject: [PATCH 1/5] Update configuration.nix with promtail Promtail is a part of the monitoring stack because it is the processor for logs. Promtail will send logs to a Loki service which is what injests the logs. Both services are needed, but promtail goes in front of Loki. The Grafana dashboard will then be able to have Loki presets loaded into modules. This push is just a build that puts the promtail servicee on the local NixOS host. --- .../monitor/configuration.nix | 32 ++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/nix/nixos-configurations/monitor/configuration.nix b/nix/nixos-configurations/monitor/configuration.nix index 0616e8be..b6c1e9d8 100644 --- a/nix/nixos-configurations/monitor/configuration.nix +++ b/nix/nixos-configurations/monitor/configuration.nix @@ -102,7 +102,37 @@ in ]; }; }; - + + services.promtail = { + enable = true; + configuration = { + server = { + http_listen_port = 3300; + grpc_listen_port = 0; + }; + positions = { + filename = "/tmp/positions.yaml"; + }; + clients = [{ + url = "http://127.0.0.1:3200/loki/api/v1/push"; + }]; + scrape_configs = [{ + job_name = "journal"; + journal = { + max_age = "12h"; + labels = { + job = "systemd-journal"; + host = "pihole"; + }; + }; + relabel_configs = [{ + source_labels = [ "__journal__systemd_unit" ]; + target_label = "unit"; + }]; + }]; + }; + }; + nginx = { enable = false; # TODO: TLS enabled From 486a30e44da6fb7d7f4619f7fa0e1218a2a522ee Mon Sep 17 00:00:00 2001 From: Jeffrey Jung Date: Fri, 28 Feb 2025 13:08:19 -0800 Subject: [PATCH 2/5] change to the port for promtail to refer to the correct Loki port... removed switchtype.tsv --- nix/nixos-configurations/monitor/configuration.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix/nixos-configurations/monitor/configuration.nix b/nix/nixos-configurations/monitor/configuration.nix index b6c1e9d8..e8ea62c3 100644 --- a/nix/nixos-configurations/monitor/configuration.nix +++ b/nix/nixos-configurations/monitor/configuration.nix @@ -114,7 +114,7 @@ in filename = "/tmp/positions.yaml"; }; clients = [{ - url = "http://127.0.0.1:3200/loki/api/v1/push"; + url = "http://127.0.0.1:3100/loki/api/v1/push"; }]; scrape_configs = [{ job_name = "journal"; From c7ec3618dda4262f8ae36015b6b62af55e2c22ba Mon Sep 17 00:00:00 2001 From: Jeffrey Jung Date: Fri, 28 Feb 2025 13:18:01 -0800 Subject: [PATCH 3/5] I have excluded the switchtypes.tsv file --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 96b340e8..cc2082b9 100644 --- a/.gitignore +++ b/.gitignore @@ -295,3 +295,4 @@ venv.bak/ .mypy_cache/ # End of https://www.gitignore.io/api/python +switchtypes.tsv From 4edb94babea47827fe54466ce3d489acc17c30cc Mon Sep 17 00:00:00 2001 From: Jeffrey Jung Date: Fri, 28 Feb 2025 17:50:49 -0800 Subject: [PATCH 4/5] correct port on the loopback in promtail in configuration.nix --- .../monitor/configuration.nix | 152 ++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100644 nix/nixos-configurations/monitor/configuration.nix diff --git a/nix/nixos-configurations/monitor/configuration.nix b/nix/nixos-configurations/monitor/configuration.nix new file mode 100644 index 00000000..e8ea62c3 --- /dev/null +++ b/nix/nixos-configurations/monitor/configuration.nix @@ -0,0 +1,152 @@ +{ + config, + pkgs, + inputs, + ... +}: +let + hostname = "monitoring.scale.lan"; + dashboard = pkgs.copyPathToStore ../../../monitoring/openwrt_dashboard.json; +in +{ + boot.kernelParams = [ + "console=ttyS0" + "boot.shell_on_fail" + ]; + + systemd.network = { + enable = true; + networks = { + "10-lan" = { + # to match enp0 or eth0 + name = "e*0*"; + enable = true; + address = [ + "10.0.3.6/24" + "2001:470:f026:103::6" + ]; + routes = [ + { routeConfig.Gateway = "10.0.3.1"; } + { routeConfig.Gateway = "2001:470:f026:103::1"; } + ]; + }; + }; + }; + networking.hostName = "monitor"; + networking.firewall.allowedTCPPorts = [ + 80 + 443 + ]; + + # TODO: How to handle sudo esculation + security.sudo.wheelNeedsPassword = false; + + environment.systemPackages = with pkgs; [ + vim + git + bintools + ]; + + services = { + prometheus = { + enable = true; + enableReload = true; + scrapeConfigs = [ + { + job_name = "prometheus"; + static_configs = [ + { + targets = [ "localhost:${toString config.services.prometheus.exporters.node.port}" ]; + labels = { + instance = "localhost"; + }; + } + ]; + } + { + job_name = "ap"; + static_configs = builtins.fromJSON ( + builtins.readFile "${pkgs.scale-network.scaleInventory}/config/prom.json" + ); + } + ]; + }; + + grafana = { + enable = true; + settings = { + server = { + http_addr = "127.0.0.1"; + http_port = 3000; + domain = "${hostname}"; + }; + analytics.reporting_enabled = false; + }; + provision = { + # Can use just datasources anymore + # https://github.com/NixOS/nixpkgs/blob/41de143fda10e33be0f47eab2bfe08a50f234267/nixos/modules/services/monitoring/grafana.nix#L101-L104 + datasources.settings.datasources = [ + { + name = "prometheus"; + uid = "P1809F7CD0C75ACF3"; + type = "prometheus"; + access = "proxy"; + url = "http://127.0.0.1:${toString config.services.prometheus.port}"; + } + ]; + dashboards.settings.providers = [ + { + name = "openwrt"; + options.path = dashboard; + } + ]; + }; + }; + + services.promtail = { + enable = true; + configuration = { + server = { + http_listen_port = 3300; + grpc_listen_port = 0; + }; + positions = { + filename = "/tmp/positions.yaml"; + }; + clients = [{ + url = "http://127.0.0.1:3100/loki/api/v1/push"; + }]; + scrape_configs = [{ + job_name = "journal"; + journal = { + max_age = "12h"; + labels = { + job = "systemd-journal"; + host = "pihole"; + }; + }; + relabel_configs = [{ + source_labels = [ "__journal__systemd_unit" ]; + target_label = "unit"; + }]; + }]; + }; + }; + + nginx = { + enable = false; + # TODO: TLS enabled + # Good example enable TLS, but would like to keep it out of the /nix/store + # ref: https://github.com/NixOS/nixpkgs/blob/c6fd903606866634312e40cceb2caee8c0c9243f/nixos/tests/custom-ca.nix#L80 + virtualHosts."${hostname}" = { + default = true; + # ACME wont work for us on the private network + enableACME = false; + locations."/" = { + proxyPass = "http://${toString config.services.grafana.settings.server.http_addr}:${toString config.services.grafana.settings.server.http_port}/"; + proxyWebsockets = true; + }; + }; + }; + }; +} From ba2a088e73dc8101190f20fbbac688e7f03bb2ef Mon Sep 17 00:00:00 2001 From: Jeffrey Jung Date: Sat, 1 Mar 2025 18:09:15 -0800 Subject: [PATCH 5/5] updated nginx with parts for grafana, promtail, and loki --- .../monitor/configuration.nix | 56 ++++++++++++++++--- 1 file changed, 47 insertions(+), 9 deletions(-) diff --git a/nix/nixos-configurations/monitor/configuration.nix b/nix/nixos-configurations/monitor/configuration.nix index e8ea62c3..d33f3008 100644 --- a/nix/nixos-configurations/monitor/configuration.nix +++ b/nix/nixos-configurations/monitor/configuration.nix @@ -134,18 +134,56 @@ in }; nginx = { - enable = false; - # TODO: TLS enabled - # Good example enable TLS, but would like to keep it out of the /nix/store - # ref: https://github.com/NixOS/nixpkgs/blob/c6fd903606866634312e40cceb2caee8c0c9243f/nixos/tests/custom-ca.nix#L80 - virtualHosts."${hostname}" = { - default = true; - # ACME wont work for us on the private network - enableACME = false; + enable = true; + recommendedProxySettings = true; + recommendedOptimisation = true; + recommendedGzipSettings = true; + # recommendedTlsSettings = true; + + upstreams = { + "grafana" = { + servers = { + "127.0.0.1:3100" = {}; + }; + }; + "loki" = { + servers = { + "127.0.0.1:3200" = {}; + }; + }; + "promtail" = { + servers = { + "127.0.0.1:3300" = {}; + }; + }; + }; + virtualHosts.grafana = { locations."/" = { - proxyPass = "http://${toString config.services.grafana.settings.server.http_addr}:${toString config.services.grafana.settings.server.http_port}/"; + proxyPass = "http://grafana"; proxyWebsockets = true; }; + listen = [{ + addr = "192.168.1.147"; + port = 8010; + }]; + }; + + # confirm with http://192.168.1.10:8030/loki/api/v1/status/buildinfo + # (or) /config /metrics /ready + virtualHosts.loki = { + locations."/".proxyPass = "http://loki"; + listen = [{ + addr = "192.168.1.147"; + port = 8020; + }]; + }; + + virtualHosts.promtail = { + locations."/".proxyPass = "http://promtail"; + listen = [{ + addr = "192.168.1.147"; + port = 8030; + }]; }; }; };