From 2f3b7e7b2175c1d04331f65a92bccf617a1f8347 Mon Sep 17 00:00:00 2001 From: jamey Date: Mon, 2 Mar 2026 17:29:15 +0000 Subject: [PATCH] optimise product queries: skip image blobs, limit listing preloads, add composite index The listing preload (images: :image) was loading the full images table row including the data BLOB column (~3MB per page). Now only loads :id and :source_width. Listing preloads also limited to first 2 images (primary + hover) since product cards don't use the rest. Added composite indexes on (visible, status, inserted_at) and (visible, status, category) to eliminate the TEMP B-TREE sort SQLite was doing. Co-Authored-By: Claude Opus 4.6 --- lib/berrypod/products.ex | 67 ++++++++++++++----- lib/berrypod/search.ex | 14 +++- lib/berrypod/theme/preview_data.ex | 2 +- lib/berrypod_web/live/admin/product_show.ex | 2 +- lib/berrypod_web/live/admin/products.ex | 2 +- ...302171616_add_products_listing_indexes.exs | 14 ++++ 6 files changed, 79 insertions(+), 22 deletions(-) create mode 100644 priv/repo/migrations/20260302171616_add_products_listing_indexes.exs diff --git a/lib/berrypod/products.ex b/lib/berrypod/products.ex index 30666f4..3cee8e9 100644 --- a/lib/berrypod/products.ex +++ b/lib/berrypod/products.ex @@ -132,10 +132,19 @@ defmodule Berrypod.Products do # Storefront queries # ============================================================================= - # Listing pages only need images (price/stock are denormalized on product) - @listing_preloads [images: :image] - # Detail page also needs variants for the variant selector - @detail_preloads [images: :image, variants: []] + # Image preload that skips the data blob — only loads :id and :source_width + # from the images table. The data column holds the raw image binary (up to 5MB + # per row) which is never used in rendering. + defp image_preload_query do + from(i in Berrypod.Media.Image, select: struct(i, [:id, :source_width])) + end + + # Listing pages only need the first 2 images (primary + hover) and don't + # need variants (price/stock are denormalized on the product row). + defp listing_preloads do + pi_query = from(pi in ProductImage, where: pi.position <= 1, order_by: pi.position) + [images: {pi_query, image: image_preload_query()}] + end @doc """ Gets a single visible, active product by slug with full preloads (for detail page). @@ -143,8 +152,11 @@ defmodule Berrypod.Products do def get_visible_product(slug) do Product |> where([p], p.slug == ^slug and p.visible == true and p.status == "active") - |> preload(^@detail_preloads) |> Repo.one() + |> case do + nil -> nil + product -> Repo.preload(product, images: [image: image_preload_query()], variants: []) + end end @doc """ @@ -167,8 +179,8 @@ defmodule Berrypod.Products do |> apply_sort(opts[:sort]) |> maybe_limit(opts[:limit]) |> maybe_exclude(opts[:exclude]) - |> preload(^@listing_preloads) |> Repo.all() + |> Repo.preload(listing_preloads()) end @doc """ @@ -177,13 +189,15 @@ defmodule Berrypod.Products do Accepts the same filter/sort options plus `:page` and `:per_page`. """ def list_visible_products_paginated(opts \\ []) do - Product - |> where([p], p.visible == true and p.status == "active") - |> apply_visible_filters(opts) - |> apply_sort(opts[:sort]) - |> maybe_exclude(opts[:exclude]) - |> preload(^@listing_preloads) - |> Berrypod.Pagination.paginate(page: opts[:page], per_page: opts[:per_page] || 24) + pagination = + Product + |> where([p], p.visible == true and p.status == "active") + |> apply_visible_filters(opts) + |> apply_sort(opts[:sort]) + |> maybe_exclude(opts[:exclude]) + |> Berrypod.Pagination.paginate(page: opts[:page], per_page: opts[:per_page] || 24) + + %{pagination | items: Repo.preload(pagination.items, listing_preloads())} end @doc """ @@ -357,7 +371,7 @@ defmodule Berrypod.Products do |> apply_product_filters(opts) |> maybe_filter_in_stock(opts[:in_stock]) |> apply_sort(opts[:sort]) - |> preload([:provider_connection, images: :image, variants: []]) + |> preload([:provider_connection, images: [image: ^image_preload_query()], variants: []]) |> Repo.all() end @@ -371,7 +385,7 @@ defmodule Berrypod.Products do |> apply_product_filters(opts) |> maybe_filter_in_stock(opts[:in_stock]) |> apply_sort(opts[:sort]) - |> preload([:provider_connection, images: :image, variants: []]) + |> preload([:provider_connection, images: [image: ^image_preload_query()], variants: []]) |> Berrypod.Pagination.paginate(page: opts[:page], per_page: opts[:per_page] || 25) end @@ -397,6 +411,27 @@ defmodule Berrypod.Products do |> Repo.get(id) end + @doc """ + Gets a product by ID with admin preloads (provider, images, variants). + + Excludes image blob data. + """ + def get_product_with_preloads(id) do + Product + |> Repo.get(id) + |> case do + nil -> + nil + + product -> + Repo.preload(product, [ + :provider_connection, + images: [image: image_preload_query()], + variants: [] + ]) + end + end + @doc """ Gets a single product by slug. """ @@ -817,7 +852,7 @@ defmodule Berrypod.Products do def get_variants_with_products(variant_ids) when is_list(variant_ids) do from(v in ProductVariant, where: v.id in ^variant_ids, - preload: [product: [images: :image]] + preload: [product: [images: [image: ^image_preload_query()]]] ) |> Repo.all() |> Map.new(&{&1.id, &1}) diff --git a/lib/berrypod/search.ex b/lib/berrypod/search.ex index 81a734b..377f38e 100644 --- a/lib/berrypod/search.ex +++ b/lib/berrypod/search.ex @@ -11,7 +11,15 @@ defmodule Berrypod.Search do alias Berrypod.Products.Product alias Berrypod.Repo - @listing_preloads [images: :image] + # Search results only need the first 2 images (card display) and only + # source_width from the images table — no blob data. + defp listing_preloads do + alias Berrypod.Products.ProductImage + + image_query = from(i in Berrypod.Media.Image, select: struct(i, [:id, :source_width])) + pi_query = from(pi in ProductImage, where: pi.position <= 1, order_by: pi.position) + [images: {pi_query, image: image_query}] + end # BM25 column weights: title(10), category(5), variant_info(3), description(1) @bm25_weights "10.0, 5.0, 3.0, 1.0" @@ -126,8 +134,8 @@ defmodule Berrypod.Search do Product |> where([p], p.id in ^product_ids) |> where([p], p.visible == true and p.status == "active") - |> preload(^@listing_preloads) |> Repo.all() + |> Repo.preload(listing_preloads()) |> Map.new(&{&1.id, &1}) Enum.flat_map(product_ids, fn id -> @@ -148,8 +156,8 @@ defmodule Berrypod.Search do |> where([p], like(p.title, ^pattern) or like(p.category, ^pattern)) |> order_by([p], p.title) |> limit(20) - |> preload(^@listing_preloads) |> Repo.all() + |> Repo.preload(listing_preloads()) end defp sanitize_like(input) do diff --git a/lib/berrypod/theme/preview_data.ex b/lib/berrypod/theme/preview_data.ex index 8f2bbf0..e9eb7d1 100644 --- a/lib/berrypod/theme/preview_data.ex +++ b/lib/berrypod/theme/preview_data.ex @@ -361,7 +361,7 @@ defmodule Berrypod.Theme.PreviewData do Products.list_products( visible: true, status: "active", - preload: [images: :image, variants: []] + preload: [:images, :variants] ) end diff --git a/lib/berrypod_web/live/admin/product_show.ex b/lib/berrypod_web/live/admin/product_show.ex index e5b886f..079981e 100644 --- a/lib/berrypod_web/live/admin/product_show.ex +++ b/lib/berrypod_web/live/admin/product_show.ex @@ -7,7 +7,7 @@ defmodule BerrypodWeb.Admin.ProductShow do @impl true def mount(%{"id" => id}, _session, socket) do - case Products.get_product(id, preload: [:provider_connection, images: :image, variants: []]) do + case Products.get_product_with_preloads(id) do nil -> socket = socket diff --git a/lib/berrypod_web/live/admin/products.ex b/lib/berrypod_web/live/admin/products.ex index 356420c..53ab2bb 100644 --- a/lib/berrypod_web/live/admin/products.ex +++ b/lib/berrypod_web/live/admin/products.ex @@ -64,7 +64,7 @@ defmodule BerrypodWeb.Admin.Products do @impl true def handle_event("toggle_visibility", %{"id" => id}, socket) do product = - Products.get_product(id, preload: [:provider_connection, images: :image, variants: []]) + Products.get_product_with_preloads(id) case Products.toggle_visibility(product) do {:ok, updated} -> diff --git a/priv/repo/migrations/20260302171616_add_products_listing_indexes.exs b/priv/repo/migrations/20260302171616_add_products_listing_indexes.exs new file mode 100644 index 0000000..7798f1a --- /dev/null +++ b/priv/repo/migrations/20260302171616_add_products_listing_indexes.exs @@ -0,0 +1,14 @@ +defmodule Berrypod.Repo.Migrations.AddProductsListingIndexes do + use Ecto.Migration + + def change do + # Composite covering index for the main listing query: + # WHERE visible = 1 AND status = 'active' ORDER BY inserted_at DESC + # Replaces the individual visible + status indexes for this pattern + # and eliminates the TEMP B-TREE sort SQLite was doing. + create index(:products, [:visible, :status, :inserted_at]) + + # Filtered listing by category (second most common query pattern) + create index(:products, [:visible, :status, :category]) + end +end