First attempt at sending logs to grafana

entrypoint.sh

@@ -203,5 +203,8 @@ fi
 echo -e "\e[34m[Info] Running database migration...\e[0m"
 yarn workspace @sourcebot/db prisma:migrate:prod
 
+# Create the log directory
+mkdir -p /var/log/sourcebot
+
 # Run supervisord
 exec supervisord -c /etc/supervisor/conf.d/supervisord.conf

grafana.alloy

@@ -1,3 +1,28 @@
+
+local.file_match "local_files" {
+  path_targets = [
+    {"__path__" = "/var/log/sourcebot/*.log"}
+  ]
+  sync_period = "5s"
+}
+
+loki.source.file "log_scrape" {
+  targets    = local.file_match.local_files.targets
+  forward_to = [loki.write.grafana_loki.receiver]
+  tail_from_end = true
+}
+
+loki.write "grafana_loki" {
+  endpoint {
+    url = sys.env("GRAFANA_ENDPOINT")
+
+    basic_auth {
+      username = sys.env("GRAFANA_USERNAME")
+      password = sys.env("GRAFANA_PASSWORD")
+    }
+  }
+}
+
 prometheus.scrape "local_app" {
   targets = [
     {

supervisord.conf

@@ -1,6 +1,7 @@
 [supervisord]
 nodaemon=true
-logfile=/dev/null
+logfile=/var/log/sourcebot/supervisord.log
+pidfile=/var/run/supervisord.pid
 logfile_maxbytes=0
 
 [program:zoekt]
@@ -8,8 +9,8 @@ command=./prefix-output.sh zoekt-webserver -index %(ENV_DATA_CACHE_DIR)s/index -
 autostart=true
 autorestart=true
 startretries=3
-stdout_logfile=/dev/fd/1
-stdout_logfile_maxbytes=0
+stdout_logfile=/var/log/sourcebot/zoekt.log
+stderr_logfile=/var/log/sourcebot/zoekt-error.log
 redirect_stderr=true
 
 [program:web]
@@ -17,8 +18,12 @@ command=./prefix-output.sh node packages/web/server.js
 autostart=true
 autorestart=true
 startretries=3
-stdout_logfile=/dev/fd/1
+stdout_logfile=/var/log/sourcebot/web.log
+stderr_logfile=/var/log/sourcebot/web-error.log
 stdout_logfile_maxbytes=0
+stderr_logfile_maxbytes=0
+stdout_logfile_backups=0
+stderr_logfile_backups=0
 redirect_stderr=true
 
 [program:backend]
@@ -26,8 +31,12 @@ command=./prefix-output.sh node packages/backend/dist/index.js --cacheDir %(ENV_
 autostart=true
 autorestart=true
 startretries=3
-stdout_logfile=/dev/fd/1
+stdout_logfile=/var/log/sourcebot/backend.log
+stderr_logfile=/var/log/sourcebot/backend-error.log
 stdout_logfile_maxbytes=0
+stderr_logfile_maxbytes=0
+stdout_logfile_backups=0
+stderr_logfile_backups=0
 redirect_stderr=true
 
 [program:redis]
@@ -35,8 +44,12 @@ command=redis-server
 autostart=true
 autorestart=true
 startretries=3
-stdout_logfile=/dev/fd/1
+stdout_logfile=/var/log/sourcebot/redis.log
+stderr_logfile=/var/log/sourcebot/redis-error.log
 stdout_logfile_maxbytes=0
+stderr_logfile_maxbytes=0
+stdout_logfile_backups=0
+stderr_logfile_backups=0
 redirect_stderr=true
 
 [program:alloy]
@@ -44,6 +57,10 @@ command=alloy run grafana.alloy
 autostart=true
 autorestart=true
 startretries=3
-stdout_logfile=/dev/fd/1
+stdout_logfile=/var/log/sourcebot/alloy.log
+stderr_logfile=/var/log/sourcebot/alloy-error.log
 stdout_logfile_maxbytes=0
+stderr_logfile_maxbytes=0
+stdout_logfile_backups=0
+stderr_logfile_backups=0
 redirect_stderr=true