add canonical URLs, robots.txt, and sitemap.xml
Canonical: all shop pages now assign og_url (reusing the existing og:url
assign), which the layout renders as <link rel="canonical">. Collection
pages strip the sort param so ?sort=price_asc doesn't create a duplicate
canonical.
robots.txt: dynamic controller disallows /admin/, /api/, /users/,
/webhooks/, /checkout/. Removed robots.txt from static_paths so it
goes through the router instead of Plug.Static.
sitemap.xml: auto-generated from all visible products + categories +
static pages, served as application/xml. 8 tests.
Also updates PROGRESS.md: marks tasks 55, 58, 59, 61, 62 as done.
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-02-23 21:47:35 +00:00
|
|
|
defmodule BerrypodWeb.SeoControllerTest do
|
|
|
|
|
use BerrypodWeb.ConnCase, async: false
|
|
|
|
|
|
|
|
|
|
import Berrypod.AccountsFixtures
|
|
|
|
|
import Berrypod.ProductsFixtures
|
|
|
|
|
|
2026-02-28 11:37:16 +00:00
|
|
|
alias Berrypod.Pages
|
|
|
|
|
alias Berrypod.Pages.PageCache
|
|
|
|
|
|
add canonical URLs, robots.txt, and sitemap.xml
Canonical: all shop pages now assign og_url (reusing the existing og:url
assign), which the layout renders as <link rel="canonical">. Collection
pages strip the sort param so ?sort=price_asc doesn't create a duplicate
canonical.
robots.txt: dynamic controller disallows /admin/, /api/, /users/,
/webhooks/, /checkout/. Removed robots.txt from static_paths so it
goes through the router instead of Plug.Static.
sitemap.xml: auto-generated from all visible products + categories +
static pages, served as application/xml. 8 tests.
Also updates PROGRESS.md: marks tasks 55, 58, 59, 61, 62 as done.
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-02-23 21:47:35 +00:00
|
|
|
setup do
|
2026-02-28 11:37:16 +00:00
|
|
|
PageCache.invalidate_all()
|
add canonical URLs, robots.txt, and sitemap.xml
Canonical: all shop pages now assign og_url (reusing the existing og:url
assign), which the layout renders as <link rel="canonical">. Collection
pages strip the sort param so ?sort=price_asc doesn't create a duplicate
canonical.
robots.txt: dynamic controller disallows /admin/, /api/, /users/,
/webhooks/, /checkout/. Removed robots.txt from static_paths so it
goes through the router instead of Plug.Static.
sitemap.xml: auto-generated from all visible products + categories +
static pages, served as application/xml. 8 tests.
Also updates PROGRESS.md: marks tasks 55, 58, 59, 61, 62 as done.
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-02-23 21:47:35 +00:00
|
|
|
user_fixture()
|
|
|
|
|
{:ok, _} = Berrypod.Settings.set_site_live(true)
|
|
|
|
|
:ok
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
describe "GET /robots.txt" do
|
|
|
|
|
test "returns 200 with text/plain content type", %{conn: conn} do
|
|
|
|
|
conn = get(conn, "/robots.txt")
|
|
|
|
|
assert response_content_type(conn, :text) =~ "text/plain"
|
|
|
|
|
assert response(conn, 200)
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
test "allows crawling of shop pages", %{conn: conn} do
|
|
|
|
|
body = get(conn, "/robots.txt") |> response(200)
|
|
|
|
|
assert body =~ "User-agent: *"
|
|
|
|
|
assert body =~ "Allow: /"
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
test "disallows admin and sensitive paths", %{conn: conn} do
|
|
|
|
|
body = get(conn, "/robots.txt") |> response(200)
|
|
|
|
|
assert body =~ "Disallow: /admin/"
|
|
|
|
|
assert body =~ "Disallow: /api/"
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
test "includes sitemap URL", %{conn: conn} do
|
|
|
|
|
body = get(conn, "/robots.txt") |> response(200)
|
|
|
|
|
assert body =~ "Sitemap:"
|
|
|
|
|
assert body =~ "/sitemap.xml"
|
|
|
|
|
end
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
describe "GET /sitemap.xml" do
|
|
|
|
|
test "returns 200 with application/xml content type", %{conn: conn} do
|
|
|
|
|
conn = get(conn, "/sitemap.xml")
|
|
|
|
|
assert response_content_type(conn, :xml) =~ "application/xml"
|
|
|
|
|
assert response(conn, 200)
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
test "includes static shop pages", %{conn: conn} do
|
|
|
|
|
body = get(conn, "/sitemap.xml") |> response(200)
|
|
|
|
|
assert body =~ "<loc>"
|
|
|
|
|
assert body =~ "/collections/all"
|
|
|
|
|
assert body =~ "/about"
|
|
|
|
|
assert body =~ "/contact"
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
test "includes visible product URLs", %{conn: conn} do
|
|
|
|
|
product = product_fixture(%{slug: "test-sitemap-tee", visible: true, status: "active"})
|
|
|
|
|
|
|
|
|
|
body = get(conn, "/sitemap.xml") |> response(200)
|
|
|
|
|
assert body =~ "/products/#{product.slug}"
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
test "is valid XML with urlset root element", %{conn: conn} do
|
|
|
|
|
body = get(conn, "/sitemap.xml") |> response(200)
|
|
|
|
|
assert body =~ ~s(xmlns="http://www.sitemaps.org/schemas/sitemap/0.9")
|
|
|
|
|
assert body =~ "<urlset"
|
|
|
|
|
assert body =~ "</urlset>"
|
|
|
|
|
end
|
2026-02-28 11:37:16 +00:00
|
|
|
|
|
|
|
|
test "includes published custom pages", %{conn: conn} do
|
|
|
|
|
{:ok, _} = Pages.create_custom_page(%{slug: "faq", title: "FAQ", published: true})
|
|
|
|
|
|
|
|
|
|
body = get(conn, "/sitemap.xml") |> response(200)
|
|
|
|
|
assert body =~ "/faq"
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
test "excludes unpublished custom pages", %{conn: conn} do
|
|
|
|
|
{:ok, _} = Pages.create_custom_page(%{slug: "draft", title: "Draft", published: false})
|
|
|
|
|
|
|
|
|
|
body = get(conn, "/sitemap.xml") |> response(200)
|
|
|
|
|
refute body =~ "/draft"
|
|
|
|
|
end
|
add canonical URLs, robots.txt, and sitemap.xml
Canonical: all shop pages now assign og_url (reusing the existing og:url
assign), which the layout renders as <link rel="canonical">. Collection
pages strip the sort param so ?sort=price_asc doesn't create a duplicate
canonical.
robots.txt: dynamic controller disallows /admin/, /api/, /users/,
/webhooks/, /checkout/. Removed robots.txt from static_paths so it
goes through the router instead of Plug.Static.
sitemap.xml: auto-generated from all visible products + categories +
static pages, served as application/xml. 8 tests.
Also updates PROGRESS.md: marks tasks 55, 58, 59, 61, 62 as done.
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-02-23 21:47:35 +00:00
|
|
|
end
|
|
|
|
|
end
|