Commit 205fc1bfaf3a2db62a8212155e93403e64a828c1

Thomas de Grivel 2022-01-24T12:32:20

robots.txt, _sitemap.txt

diff --git a/README.md b/README.md
index 02dafdb..526461f 100644
--- a/README.md
+++ b/README.md
@@ -123,6 +123,9 @@ location ~ ^(.*/info/refs|.*/git-upload-pack)$ {
      - DONE git remote add kmx git@git.kmx.io:user/repo.git
      - DONE git push -u kmx branch
      - DONE git pull
+ - SEO
+   - /_sitemap.txt
+   - /robots.txt
  - Display markdown files as HTML
  - Releases
    - infos
diff --git a/lib/kmxgit/organisation_manager.ex b/lib/kmxgit/organisation_manager.ex
index 7435a98..9dbdca4 100644
--- a/lib/kmxgit/organisation_manager.ex
+++ b/lib/kmxgit/organisation_manager.ex
@@ -11,7 +11,8 @@ defmodule Kmxgit.OrganisationManager do
     Repo.all from org in Organisation,
       join: s in Slug,
       on: s.organisation_id == org.id,
-      preload: :slug,
+      preload: [:owned_repositories,
+                :slug],
       order_by: s.slug
   end
 
diff --git a/lib/kmxgit/user_manager.ex b/lib/kmxgit/user_manager.ex
index 539ec2c..59666f6 100644
--- a/lib/kmxgit/user_manager.ex
+++ b/lib/kmxgit/user_manager.ex
@@ -10,7 +10,9 @@ defmodule Kmxgit.UserManager do
   alias Kmxgit.UserManager.{Avatar, User, UserToken, UserNotifier}
 
   def list_users do
-    Repo.all from user in User, preload: :slug
+    Repo.all from user in User,
+      preload: [:owned_repositories,
+                :slug]
   end
 
   def count_users do
diff --git a/lib/kmxgit_web/controllers/page_controller.ex b/lib/kmxgit_web/controllers/page_controller.ex
index 3302ef5..0e2a080 100644
--- a/lib/kmxgit_web/controllers/page_controller.ex
+++ b/lib/kmxgit_web/controllers/page_controller.ex
@@ -99,6 +99,20 @@ defmodule KmxgitWeb.PageController do
     |> render(:privacy)
   end
 
+  def robots(conn, _params) do
+    conn
+    |> put_resp_content_type("text/plain")
+    |> render("robots.txt")
+  end
+
+  def sitemap(conn, _params) do
+    conn
+    |> assign(:orgs, OrganisationManager.list_organisations())
+    |> assign(:users, UserManager.list_users())
+    |> put_resp_content_type("text/plain")
+    |> render("sitemap.txt")
+  end
+
   def user_agreement(conn, _params) do
     conn
     |> render(:user_agreement)
diff --git a/lib/kmxgit_web/router.ex b/lib/kmxgit_web/router.ex
index 976fe9e..a04fa74 100644
--- a/lib/kmxgit_web/router.ex
+++ b/lib/kmxgit_web/router.ex
@@ -36,6 +36,8 @@ defmodule KmxgitWeb.Router do
     get  "/_new_admin",               PageController, :new_admin
     post "/_new_admin",               PageController, :new_admin_post
     get  "/_privacy",                 PageController, :privacy
+    get  "/robots.txt",               PageController, :robots
+    get  "/_sitemap.txt",             PageController, :sitemap
     get  "/_user_agreement",          PageController, :user_agreement
 
     delete "/_log_out",        UserSessionController, :delete
diff --git a/lib/kmxgit_web/templates/page/robots.txt.eex b/lib/kmxgit_web/templates/page/robots.txt.eex
new file mode 100644
index 0000000..1ed67e2
--- /dev/null
+++ b/lib/kmxgit_web/templates/page/robots.txt.eex
@@ -0,0 +1,4 @@
+User-agent: *
+Allow: /
+
+Sitemap: <%= Routes.page_url(@conn, :sitemap) %>
diff --git a/lib/kmxgit_web/templates/page/sitemap.txt.eex b/lib/kmxgit_web/templates/page/sitemap.txt.eex
new file mode 100644
index 0000000..e2908ad
--- /dev/null
+++ b/lib/kmxgit_web/templates/page/sitemap.txt.eex
@@ -0,0 +1,6 @@
+<%= Routes.page_url(@conn, :index) %>
+<%= for user <- @users do %><%= Routes.slug_url(@conn, :show, user.slug.slug) %>
+<%= for repo <- user.owned_repositories do %><%= if repo.public_access do %><%= Routes.repository_url(@conn, :show, user.slug.slug, Repository.splat(repo)) %>
+<% end %><% end %><% end %><%= for org <- @orgs do %><%= Routes.slug_url(@conn, :show, org.slug.slug) %>
+<%= for repo <- org.owned_repositories do %><%= if repo.public_access do %><%= Routes.repository_url(@conn, :show, org.slug.slug, Repository.splat(repo)) %>
+<% end %><% end %><% end %>
diff --git a/lib/kmxgit_web/views/page_view.ex b/lib/kmxgit_web/views/page_view.ex
index 3e13757..c6fc170 100644
--- a/lib/kmxgit_web/views/page_view.ex
+++ b/lib/kmxgit_web/views/page_view.ex
@@ -1,3 +1,5 @@
 defmodule KmxgitWeb.PageView do
   use KmxgitWeb, :view
+
+  alias Kmxgit.RepositoryManager.Repository
 end
diff --git a/priv/static/robots.txt b/priv/static/robots.txt
deleted file mode 100644
index 3c9c7c0..0000000
--- a/priv/static/robots.txt
+++ /dev/null
@@ -1,5 +0,0 @@
-# See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file
-#
-# To ban all spiders from the entire site uncomment the next two lines:
-# User-agent: *
-# Disallow: /