Footer and robots.txt update

Trying to discourage AI bots from viewing my site.
This commit is contained in:
Dan 2025-12-31 13:03:14 +00:00
parent 931fa141dc
commit 1486f8b93b
2 changed files with 104 additions and 3 deletions

View file

@ -1,12 +1,17 @@
<footer role="contentinfo"> <footer role="contentinfo">
<div class="crt"> <div class="crt">
<span class="warning"> This site is very much under construction! </span> <span class="warning"> Under construction! </span>
&nbsp;&bull;&nbsp; &nbsp;&bull;&nbsp;
<a href="https://neocities.org/site/ritualsh" target="_blank" <a href="https://neocities.org/site/ritualsh" target="_blank"
>Follow on Neocities</a >Follow on Neocities</a
> >
<div>{{ partial "social-follow.html" . }}</div>
&nbsp;&bull;&nbsp;
<a href="https://github.com/unbolt/ritual.sh/" target="_blank">
Open Source
</a>
</div> </div>
</footer> </footer>

View file

@ -1,7 +1,103 @@
# Block AI crawlers and scrapers
User-agent: GPTBot
Disallow: /
User-agent: ChatGPT-User
Disallow: /
User-agent: CCBot
Disallow: /
User-agent: anthropic-ai
Disallow: /
User-agent: Claude-Web
Disallow: /
User-agent: Google-Extended
Disallow: /
User-agent: GoogleOther
Disallow: /
User-agent: PerplexityBot
Disallow: /
User-agent: Omgilibot
Disallow: /
User-agent: FacebookBot
Disallow: /
User-agent: Diffbot
Disallow: /
User-agent: Bytespider
Disallow: /
User-agent: ImagesiftBot
Disallow: /
User-agent: cohere-ai
Disallow: /
User-agent: Applebot-Extended
Disallow: /
User-agent: YouBot
Disallow: /
User-agent: Meta-ExternalAgent
Disallow: /
User-agent: Meta-ExternalFetcher
Disallow: /
User-agent: OAI-SearchBot
Disallow: /
User-agent: PetalBot
Disallow: /
User-agent: Amazonbot
Disallow: /
User-agent: ClaudeBot
Disallow: /
User-agent: Timpibot
Disallow: /
User-agent: Kangaroo Bot
Disallow: /
User-agent: img2dataset
Disallow: /
User-agent: ISSCyberRiskCrawler
Disallow: /
User-agent: iaskspider/2.0
Disallow: /
User-agent: Scrapy
Disallow: /
User-agent: Zealbot
Disallow: /
User-agent: newspaper
Disallow: /
# Allow other bots
User-agent: * User-agent: *
# robotstxt.org - if ENV production variable is false robots will be disallowed.
{{ if eq (getenv "HUGO_ENV") "production" | or (eq .Site.Params.env "production") }} {{ if eq (getenv "HUGO_ENV") "production" | or (eq .Site.Params.env "production") }}
Disallow: Disallow:
{{ else }} {{ else }}
Disallow: / Disallow: /
{{ end }} {{ end }}
# Sitemap location (update with your actual sitemap URL)
{{ if eq (getenv "HUGO_ENV") "production" | or (eq .Site.Params.env "production") }}
Sitemap: {{ .Site.BaseURL }}sitemap.xml
{{ end }}