From 1486f8b93bd9c1ddf66cf14cd0687733c7e162c1 Mon Sep 17 00:00:00 2001 From: Dan Date: Wed, 31 Dec 2025 13:03:14 +0000 Subject: [PATCH] Footer and robots.txt update Trying to discourage AI bots from viewing my site. --- layouts/partials/site-footer.html | 9 ++- layouts/robots.txt | 98 ++++++++++++++++++++++++++++++- 2 files changed, 104 insertions(+), 3 deletions(-) diff --git a/layouts/partials/site-footer.html b/layouts/partials/site-footer.html index 5a47f6f..e2abd88 100755 --- a/layouts/partials/site-footer.html +++ b/layouts/partials/site-footer.html @@ -1,12 +1,17 @@ diff --git a/layouts/robots.txt b/layouts/robots.txt index 55af02b..5cdce7d 100644 --- a/layouts/robots.txt +++ b/layouts/robots.txt @@ -1,7 +1,103 @@ +# Block AI crawlers and scrapers +User-agent: GPTBot +Disallow: / + +User-agent: ChatGPT-User +Disallow: / + +User-agent: CCBot +Disallow: / + +User-agent: anthropic-ai +Disallow: / + +User-agent: Claude-Web +Disallow: / + +User-agent: Google-Extended +Disallow: / + +User-agent: GoogleOther +Disallow: / + +User-agent: PerplexityBot +Disallow: / + +User-agent: Omgilibot +Disallow: / + +User-agent: FacebookBot +Disallow: / + +User-agent: Diffbot +Disallow: / + +User-agent: Bytespider +Disallow: / + +User-agent: ImagesiftBot +Disallow: / + +User-agent: cohere-ai +Disallow: / + +User-agent: Applebot-Extended +Disallow: / + +User-agent: YouBot +Disallow: / + +User-agent: Meta-ExternalAgent +Disallow: / + +User-agent: Meta-ExternalFetcher +Disallow: / + +User-agent: OAI-SearchBot +Disallow: / + +User-agent: PetalBot +Disallow: / + +User-agent: Amazonbot +Disallow: / + +User-agent: ClaudeBot +Disallow: / + +User-agent: Timpibot +Disallow: / + +User-agent: Kangaroo Bot +Disallow: / + +User-agent: img2dataset +Disallow: / + +User-agent: ISSCyberRiskCrawler +Disallow: / + +User-agent: iaskspider/2.0 +Disallow: / + +User-agent: Scrapy +Disallow: / + +User-agent: Zealbot +Disallow: / + +User-agent: newspaper +Disallow: / + +# Allow other bots User-agent: * -# robotstxt.org - if ENV production variable is false robots will be disallowed. {{ if eq (getenv "HUGO_ENV") "production" | or (eq .Site.Params.env "production") }} Disallow: {{ else }} Disallow: / {{ end }} + +# Sitemap location (update with your actual sitemap URL) +{{ if eq (getenv "HUGO_ENV") "production" | or (eq .Site.Params.env "production") }} +Sitemap: {{ .Site.BaseURL }}sitemap.xml +{{ end }}