add observability: LiveDashboard in prod, error tracking, JSON logging

- Move LiveDashboard to /admin/dashboard behind session auth (all envs)
- Add ErrorTracker at /admin/errors for auto-captured exceptions
- Add Oban job and LiveView metrics to telemetry module
- Add logger_json for structured JSON logs in production
- Enable os_mon for CPU/disk/memory in LiveDashboard OS Data tab
- Extend logger metadata with oban_worker and oban_queue fields

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

config/config.exs

@@ -75,7 +75,7 @@ config :tailwind,
 # Configures Elixir's Logger
 config :logger, :default_formatter,
   format: "$time $metadata[$level] $message\n",
-  metadata: [:request_id]
+  metadata: [:request_id, :oban_worker, :oban_queue]
 
 # Use Jason for JSON parsing in Phoenix
 config :phoenix, :json_library, Jason
@@ -83,6 +83,12 @@ config :phoenix, :json_library, Jason
 # ex_money configuration for currency handling
 config :ex_money, default_cldr_backend: SimpleshopTheme.Cldr
 
+# Error tracking (stored in DB, auto-captures Phoenix/LiveView/Oban errors)
+config :error_tracker,
+  repo: SimpleshopTheme.Repo,
+  otp_app: :simpleshop_theme,
+  plugins: [ErrorTracker.Plugins.Pruner]
+
 # Stripe configuration
 config :stripity_stripe, api_version: "2024-12-18.acacia"
 

config/prod.exs

@@ -17,5 +17,8 @@ config :swoosh, local: false
 # Do not print debug messages in production
 config :logger, level: :info
 
+# Structured JSON logs for production (machine-parseable by fly logs, journalctl, Loki, etc.)
+config :logger, :default_handler, formatter: {LoggerJSON.Formatters.Basic, []}
+
 # Runtime production configuration, including reading
 # of environment variables, is done on config/runtime.exs.

lib/simpleshop_theme_web/router.ex

@@ -2,6 +2,8 @@ defmodule SimpleshopThemeWeb.Router do
   use SimpleshopThemeWeb, :router
 
   import SimpleshopThemeWeb.UserAuth
+  import Phoenix.LiveDashboard.Router
+  import ErrorTracker.Web.Router
 
   pipeline :browser do
     plug :accepts, ["html"]
@@ -89,19 +91,19 @@ defmodule SimpleshopThemeWeb.Router do
     post "/stripe", StripeWebhookController, :handle
   end
 
-  # Enable LiveDashboard and Swoosh mailbox preview in development
+  # LiveDashboard and ErrorTracker behind admin auth (available in all environments)
-  if Application.compile_env(:simpleshop_theme, :dev_routes) do
+  scope "/admin" do
-    # If you want to use the LiveDashboard in production, you should put
+    pipe_through [:browser, :require_authenticated_user]
-    # it behind authentication and allow only admins to access it.
-    # If your application does not have an admins-only section yet,
-    # you can use Plug.BasicAuth to set up some basic authentication
-    # as long as you are also using SSL (which you should anyway).
-    import Phoenix.LiveDashboard.Router
 
+    live_dashboard "/dashboard", metrics: SimpleshopThemeWeb.Telemetry
+    error_tracker_dashboard("/errors")
+  end
+
+  # Dev-only routes (mailbox preview, error previews)
+  if Application.compile_env(:simpleshop_theme, :dev_routes) do
     scope "/dev" do
       pipe_through :browser
 
-      live_dashboard "/dashboard", metrics: SimpleshopThemeWeb.Telemetry
       forward "/mailbox", Plug.Swoosh.MailboxPreview
 
       # Preview error pages

lib/simpleshop_theme_web/telemetry.ex

@@ -75,6 +75,24 @@ defmodule SimpleshopThemeWeb.Telemetry do
           "The time the connection spent waiting before being checked out for the query"
       ),
 
+      # Oban job metrics
+      summary("oban.job.stop.duration",
+        tags: [:worker],
+        unit: {:native, :millisecond}
+      ),
+      counter("oban.job.stop.duration", tags: [:worker]),
+      counter("oban.job.exception.duration", tags: [:worker]),
+
+      # LiveView metrics
+      summary("phoenix.live_view.mount.stop.duration",
+        tags: [:view],
+        unit: {:native, :millisecond}
+      ),
+      summary("phoenix.live_view.handle_event.stop.duration",
+        tags: [:event],
+        unit: {:native, :millisecond}
+      ),
+
       # VM Metrics
       summary("vm.memory.total", unit: {:byte, :kilobyte}),
       summary("vm.total_run_queue_lengths.total"),

mix.exs

@@ -23,7 +23,7 @@ defmodule SimpleshopTheme.MixProject do
   def application do
     [
       mod: {SimpleshopTheme.Application, []},
-      extra_applications: [:logger, :runtime_tools]
+      extra_applications: [:logger, :runtime_tools, :os_mon]
     ]
   end
 
@@ -76,6 +76,8 @@ defmodule SimpleshopTheme.MixProject do
       {:ex_money, "~> 5.0"},
       {:ex_money_sql, "~> 1.0"},
       {:stripity_stripe, "~> 3.2"},
+      {:error_tracker, "~> 0.7"},
+      {:logger_json, "~> 7.0", only: :prod},
       {:credo, "~> 1.7", only: [:dev, :test], runtime: false},
       {:dialyxir, "~> 1.4", only: [:dev, :test], runtime: false}
     ]

mix.lock

@@ -18,6 +18,7 @@
   "ecto_sqlite3": {:hex, :ecto_sqlite3, "0.22.0", "edab2d0f701b7dd05dcf7e2d97769c106aff62b5cfddc000d1dd6f46b9cbd8c3", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:ecto, "~> 3.13.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "~> 3.13.0", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:exqlite, "~> 0.22", [hex: :exqlite, repo: "hexpm", optional: false]}], "hexpm", "5af9e031bffcc5da0b7bca90c271a7b1e7c04a93fecf7f6cd35bc1b1921a64bd"},
   "elixir_make": {:hex, :elixir_make, "0.9.0", "6484b3cd8c0cee58f09f05ecaf1a140a8c97670671a6a0e7ab4dc326c3109726", [:mix], [], "hexpm", "db23d4fd8b757462ad02f8aa73431a426fe6671c80b200d9710caf3d1dd0ffdb"},
   "erlex": {:hex, :erlex, "0.2.8", "cd8116f20f3c0afe376d1e8d1f0ae2452337729f68be016ea544a72f767d9c12", [:mix], [], "hexpm", "9d66ff9fedf69e49dc3fd12831e12a8a37b76f8651dd21cd45fcf5561a8a7590"},
+  "error_tracker": {:hex, :error_tracker, "0.7.1", "074f10bfb01961bd0d51eb010c6ddbe13bf40eb4ddfbb25538680ddb520ab1aa", [:mix], [{:ecto, "~> 3.13", [hex: :ecto, repo: "hexpm", optional: false]}, {:ecto_sql, "~> 3.13", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:ecto_sqlite3, ">= 0.0.0", [hex: :ecto_sqlite3, repo: "hexpm", optional: true]}, {:igniter, "~> 0.5", [hex: :igniter, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:myxql, ">= 0.0.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:phoenix_ecto, "~> 4.6", [hex: :phoenix_ecto, repo: "hexpm", optional: false]}, {:phoenix_live_view, "~> 1.0", [hex: :phoenix_live_view, repo: "hexpm", optional: false]}, {:plug, "~> 1.10", [hex: :plug, repo: "hexpm", optional: false]}, {:postgrex, ">= 0.0.0", [hex: :postgrex, repo: "hexpm", optional: true]}], "hexpm", "cc77434c084d100e17dcc5ccea6bed05b15748186df5207e21fb761651bf4e0d"},
   "esbuild": {:hex, :esbuild, "0.10.0", "b0aa3388a1c23e727c5a3e7427c932d89ee791746b0081bbe56103e9ef3d291f", [:mix], [{:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "468489cda427b974a7cc9f03ace55368a83e1a7be12fba7e30969af78e5f8c70"},
   "ex_cldr": {:hex, :ex_cldr, "2.46.0", "29b5bb638932ca4fc4339595145e327b797f59963a398c12a6aee1efe5a35b1b", [:mix], [{:cldr_utils, "~> 2.28", [hex: :cldr_utils, repo: "hexpm", optional: false]}, {:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:gettext, "~> 0.19 or ~> 1.0", [hex: :gettext, repo: "hexpm", optional: true]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:nimble_parsec, "~> 0.5 or ~> 1.0", [hex: :nimble_parsec, repo: "hexpm", optional: true]}], "hexpm", "14157ac16694e99c1339ac25a4f10d3df0e0d15cc1a35073b37e195487c1b6cb"},
   "ex_cldr_currencies": {:hex, :ex_cldr_currencies, "2.17.0", "c38d76339dbee413f7dd1aba4cdf05758bd4c0bbfe9c3b1c8602f96082c2890a", [:mix], [{:ex_cldr, "~> 2.38", [hex: :ex_cldr, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm", "9af59bd29407dcca59fa39ded8c1649ae1cf6ec29fd0611576dcad0279bce0db"},
@@ -37,6 +38,7 @@
   "image": {:hex, :image, "0.62.1", "1dd3d8d0d29d6562aa2141b5ef08c0f6a60e2a9f843fe475499b2f4f1ef60406", [:mix], [{:bumblebee, "~> 0.6", [hex: :bumblebee, repo: "hexpm", optional: true]}, {:evision, "~> 0.1.33 or ~> 0.2", [hex: :evision, repo: "hexpm", optional: true]}, {:exla, "~> 0.9", [hex: :exla, repo: "hexpm", optional: true]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: true]}, {:kino, "~> 0.13", [hex: :kino, repo: "hexpm", optional: true]}, {:nx, "~> 0.9", [hex: :nx, repo: "hexpm", optional: true]}, {:nx_image, "~> 0.1", [hex: :nx_image, repo: "hexpm", optional: true]}, {:phoenix_html, "~> 2.1 or ~> 3.2 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: false]}, {:plug, "~> 1.13", [hex: :plug, repo: "hexpm", optional: true]}, {:req, "~> 0.4", [hex: :req, repo: "hexpm", optional: true]}, {:rustler, "> 0.0.0", [hex: :rustler, repo: "hexpm", optional: true]}, {:scholar, "~> 0.3", [hex: :scholar, repo: "hexpm", optional: true]}, {:sweet_xml, "~> 0.7", [hex: :sweet_xml, repo: "hexpm", optional: false]}, {:vix, "~> 0.33", [hex: :vix, repo: "hexpm", optional: false]}], "hexpm", "5a5a7acaf68cfaed8932d478b95152cd7d84071442cac558c59f2d31427e91ab"},
   "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"},
   "lazy_html": {:hex, :lazy_html, "0.1.8", "677a8642e644eef8de98f3040e2520d42d0f0f8bd6c5cd49db36504e34dffe91", [:make, :mix], [{:cc_precompiler, "~> 0.1", [hex: :cc_precompiler, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.9.0", [hex: :elixir_make, repo: "hexpm", optional: false]}, {:fine, "~> 0.1.0", [hex: :fine, repo: "hexpm", optional: false]}], "hexpm", "0d8167d930b704feb94b41414ca7f5779dff9bca7fcf619fcef18de138f08736"},
+  "logger_json": {:hex, :logger_json, "7.0.4", "e315f2b9a755504658a745f3eab90d88d2cd7ac2ecfd08c8da94d8893965ab5c", [:mix], [{:decimal, ">= 0.0.0", [hex: :decimal, repo: "hexpm", optional: true]}, {:ecto, "~> 3.11", [hex: :ecto, repo: "hexpm", optional: true]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: true]}, {:plug, "~> 1.15", [hex: :plug, repo: "hexpm", optional: true]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: true]}], "hexpm", "d1369f8094e372db45d50672c3b91e8888bcd695fdc444a37a0734e96717c45c"},
   "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"},
   "mime": {:hex, :mime, "2.0.7", "b8d739037be7cd402aee1ba0306edfdef982687ee7e9859bee6198c1e7e2f128", [:mix], [], "hexpm", "6171188e399ee16023ffc5b76ce445eb6d9672e2e241d2df6050f3c771e80ccd"},
   "mimerl": {:hex, :mimerl, "1.4.0", "3882a5ca67fbbe7117ba8947f27643557adec38fa2307490c4c4207624cb213b", [:rebar3], [], "hexpm", "13af15f9f68c65884ecca3a3891d50a7b57d82152792f3e19d88650aa126b144"},

priv/repo/migrations/20260208165931_add_error_tracker.exs

@@ -0,0 +1,6 @@
+defmodule SimpleshopTheme.Repo.Migrations.AddErrorTracker do
+  use Ecto.Migration
+
+  def up, do: ErrorTracker.Migration.up(version: 5)
+  def down, do: ErrorTracker.Migration.down(version: 1)
+end