Footer and robots.txt update
Trying to discourage AI bots from viewing my site.
This commit is contained in:
parent
931fa141dc
commit
1486f8b93b
2 changed files with 104 additions and 3 deletions
|
|
@ -1,12 +1,17 @@
|
||||||
<footer role="contentinfo">
|
<footer role="contentinfo">
|
||||||
<div class="crt">
|
<div class="crt">
|
||||||
<span class="warning"> This site is very much under construction! </span>
|
<span class="warning"> Under construction! </span>
|
||||||
|
|
||||||
•
|
•
|
||||||
|
|
||||||
<a href="https://neocities.org/site/ritualsh" target="_blank"
|
<a href="https://neocities.org/site/ritualsh" target="_blank"
|
||||||
>Follow on Neocities</a
|
>Follow on Neocities</a
|
||||||
>
|
>
|
||||||
<div>{{ partial "social-follow.html" . }}</div>
|
|
||||||
|
•
|
||||||
|
|
||||||
|
<a href="https://github.com/unbolt/ritual.sh/" target="_blank">
|
||||||
|
Open Source
|
||||||
|
</a>
|
||||||
</div>
|
</div>
|
||||||
</footer>
|
</footer>
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,103 @@
|
||||||
|
# Block AI crawlers and scrapers
|
||||||
|
User-agent: GPTBot
|
||||||
|
Disallow: /
|
||||||
|
|
||||||
|
User-agent: ChatGPT-User
|
||||||
|
Disallow: /
|
||||||
|
|
||||||
|
User-agent: CCBot
|
||||||
|
Disallow: /
|
||||||
|
|
||||||
|
User-agent: anthropic-ai
|
||||||
|
Disallow: /
|
||||||
|
|
||||||
|
User-agent: Claude-Web
|
||||||
|
Disallow: /
|
||||||
|
|
||||||
|
User-agent: Google-Extended
|
||||||
|
Disallow: /
|
||||||
|
|
||||||
|
User-agent: GoogleOther
|
||||||
|
Disallow: /
|
||||||
|
|
||||||
|
User-agent: PerplexityBot
|
||||||
|
Disallow: /
|
||||||
|
|
||||||
|
User-agent: Omgilibot
|
||||||
|
Disallow: /
|
||||||
|
|
||||||
|
User-agent: FacebookBot
|
||||||
|
Disallow: /
|
||||||
|
|
||||||
|
User-agent: Diffbot
|
||||||
|
Disallow: /
|
||||||
|
|
||||||
|
User-agent: Bytespider
|
||||||
|
Disallow: /
|
||||||
|
|
||||||
|
User-agent: ImagesiftBot
|
||||||
|
Disallow: /
|
||||||
|
|
||||||
|
User-agent: cohere-ai
|
||||||
|
Disallow: /
|
||||||
|
|
||||||
|
User-agent: Applebot-Extended
|
||||||
|
Disallow: /
|
||||||
|
|
||||||
|
User-agent: YouBot
|
||||||
|
Disallow: /
|
||||||
|
|
||||||
|
User-agent: Meta-ExternalAgent
|
||||||
|
Disallow: /
|
||||||
|
|
||||||
|
User-agent: Meta-ExternalFetcher
|
||||||
|
Disallow: /
|
||||||
|
|
||||||
|
User-agent: OAI-SearchBot
|
||||||
|
Disallow: /
|
||||||
|
|
||||||
|
User-agent: PetalBot
|
||||||
|
Disallow: /
|
||||||
|
|
||||||
|
User-agent: Amazonbot
|
||||||
|
Disallow: /
|
||||||
|
|
||||||
|
User-agent: ClaudeBot
|
||||||
|
Disallow: /
|
||||||
|
|
||||||
|
User-agent: Timpibot
|
||||||
|
Disallow: /
|
||||||
|
|
||||||
|
User-agent: Kangaroo Bot
|
||||||
|
Disallow: /
|
||||||
|
|
||||||
|
User-agent: img2dataset
|
||||||
|
Disallow: /
|
||||||
|
|
||||||
|
User-agent: ISSCyberRiskCrawler
|
||||||
|
Disallow: /
|
||||||
|
|
||||||
|
User-agent: iaskspider/2.0
|
||||||
|
Disallow: /
|
||||||
|
|
||||||
|
User-agent: Scrapy
|
||||||
|
Disallow: /
|
||||||
|
|
||||||
|
User-agent: Zealbot
|
||||||
|
Disallow: /
|
||||||
|
|
||||||
|
User-agent: newspaper
|
||||||
|
Disallow: /
|
||||||
|
|
||||||
|
# Allow other bots
|
||||||
User-agent: *
|
User-agent: *
|
||||||
# robotstxt.org - if ENV production variable is false robots will be disallowed.
|
|
||||||
{{ if eq (getenv "HUGO_ENV") "production" | or (eq .Site.Params.env "production") }}
|
{{ if eq (getenv "HUGO_ENV") "production" | or (eq .Site.Params.env "production") }}
|
||||||
Disallow:
|
Disallow:
|
||||||
{{ else }}
|
{{ else }}
|
||||||
Disallow: /
|
Disallow: /
|
||||||
{{ end }}
|
{{ end }}
|
||||||
|
|
||||||
|
# Sitemap location (update with your actual sitemap URL)
|
||||||
|
{{ if eq (getenv "HUGO_ENV") "production" | or (eq .Site.Params.env "production") }}
|
||||||
|
Sitemap: {{ .Site.BaseURL }}sitemap.xml
|
||||||
|
{{ end }}
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue