Footer and robots.txt update

Trying to discourage AI bots from viewing my site.
This commit is contained in:
Dan 2025-12-31 13:03:14 +00:00
parent 931fa141dc
commit 1486f8b93b
2 changed files with 104 additions and 3 deletions

View file

@ -1,12 +1,17 @@
<footer role="contentinfo">
<div class="crt">
<span class="warning"> This site is very much under construction! </span>
<span class="warning"> Under construction! </span>
&nbsp;&bull;&nbsp;
<a href="https://neocities.org/site/ritualsh" target="_blank"
>Follow on Neocities</a
>
<div>{{ partial "social-follow.html" . }}</div>
&nbsp;&bull;&nbsp;
<a href="https://github.com/unbolt/ritual.sh/" target="_blank">
Open Source
</a>
</div>
</footer>

View file

@ -1,7 +1,103 @@
# Block AI crawlers and scrapers
User-agent: GPTBot
Disallow: /
User-agent: ChatGPT-User
Disallow: /
User-agent: CCBot
Disallow: /
User-agent: anthropic-ai
Disallow: /
User-agent: Claude-Web
Disallow: /
User-agent: Google-Extended
Disallow: /
User-agent: GoogleOther
Disallow: /
User-agent: PerplexityBot
Disallow: /
User-agent: Omgilibot
Disallow: /
User-agent: FacebookBot
Disallow: /
User-agent: Diffbot
Disallow: /
User-agent: Bytespider
Disallow: /
User-agent: ImagesiftBot
Disallow: /
User-agent: cohere-ai
Disallow: /
User-agent: Applebot-Extended
Disallow: /
User-agent: YouBot
Disallow: /
User-agent: Meta-ExternalAgent
Disallow: /
User-agent: Meta-ExternalFetcher
Disallow: /
User-agent: OAI-SearchBot
Disallow: /
User-agent: PetalBot
Disallow: /
User-agent: Amazonbot
Disallow: /
User-agent: ClaudeBot
Disallow: /
User-agent: Timpibot
Disallow: /
User-agent: Kangaroo Bot
Disallow: /
User-agent: img2dataset
Disallow: /
User-agent: ISSCyberRiskCrawler
Disallow: /
User-agent: iaskspider/2.0
Disallow: /
User-agent: Scrapy
Disallow: /
User-agent: Zealbot
Disallow: /
User-agent: newspaper
Disallow: /
# Allow other bots
User-agent: *
# robotstxt.org - if ENV production variable is false robots will be disallowed.
{{ if eq (getenv "HUGO_ENV") "production" | or (eq .Site.Params.env "production") }}
Disallow:
{{ else }}
Disallow: /
{{ end }}
# Sitemap location (update with your actual sitemap URL)
{{ if eq (getenv "HUGO_ENV") "production" | or (eq .Site.Params.env "production") }}
Sitemap: {{ .Site.BaseURL }}sitemap.xml
{{ end }}