Footer and robots.txt update
Trying to discourage AI bots from viewing my site.
This commit is contained in:
parent
931fa141dc
commit
1486f8b93b
2 changed files with 104 additions and 3 deletions
|
|
@ -1,7 +1,103 @@
|
|||
# Block AI crawlers and scrapers
|
||||
User-agent: GPTBot
|
||||
Disallow: /
|
||||
|
||||
User-agent: ChatGPT-User
|
||||
Disallow: /
|
||||
|
||||
User-agent: CCBot
|
||||
Disallow: /
|
||||
|
||||
User-agent: anthropic-ai
|
||||
Disallow: /
|
||||
|
||||
User-agent: Claude-Web
|
||||
Disallow: /
|
||||
|
||||
User-agent: Google-Extended
|
||||
Disallow: /
|
||||
|
||||
User-agent: GoogleOther
|
||||
Disallow: /
|
||||
|
||||
User-agent: PerplexityBot
|
||||
Disallow: /
|
||||
|
||||
User-agent: Omgilibot
|
||||
Disallow: /
|
||||
|
||||
User-agent: FacebookBot
|
||||
Disallow: /
|
||||
|
||||
User-agent: Diffbot
|
||||
Disallow: /
|
||||
|
||||
User-agent: Bytespider
|
||||
Disallow: /
|
||||
|
||||
User-agent: ImagesiftBot
|
||||
Disallow: /
|
||||
|
||||
User-agent: cohere-ai
|
||||
Disallow: /
|
||||
|
||||
User-agent: Applebot-Extended
|
||||
Disallow: /
|
||||
|
||||
User-agent: YouBot
|
||||
Disallow: /
|
||||
|
||||
User-agent: Meta-ExternalAgent
|
||||
Disallow: /
|
||||
|
||||
User-agent: Meta-ExternalFetcher
|
||||
Disallow: /
|
||||
|
||||
User-agent: OAI-SearchBot
|
||||
Disallow: /
|
||||
|
||||
User-agent: PetalBot
|
||||
Disallow: /
|
||||
|
||||
User-agent: Amazonbot
|
||||
Disallow: /
|
||||
|
||||
User-agent: ClaudeBot
|
||||
Disallow: /
|
||||
|
||||
User-agent: Timpibot
|
||||
Disallow: /
|
||||
|
||||
User-agent: Kangaroo Bot
|
||||
Disallow: /
|
||||
|
||||
User-agent: img2dataset
|
||||
Disallow: /
|
||||
|
||||
User-agent: ISSCyberRiskCrawler
|
||||
Disallow: /
|
||||
|
||||
User-agent: iaskspider/2.0
|
||||
Disallow: /
|
||||
|
||||
User-agent: Scrapy
|
||||
Disallow: /
|
||||
|
||||
User-agent: Zealbot
|
||||
Disallow: /
|
||||
|
||||
User-agent: newspaper
|
||||
Disallow: /
|
||||
|
||||
# Allow other bots
|
||||
User-agent: *
|
||||
# robotstxt.org - if ENV production variable is false robots will be disallowed.
|
||||
{{ if eq (getenv "HUGO_ENV") "production" | or (eq .Site.Params.env "production") }}
|
||||
Disallow:
|
||||
{{ else }}
|
||||
Disallow: /
|
||||
{{ end }}
|
||||
|
||||
# Sitemap location (update with your actual sitemap URL)
|
||||
{{ if eq (getenv "HUGO_ENV") "production" | or (eq .Site.Params.env "production") }}
|
||||
Sitemap: {{ .Site.BaseURL }}sitemap.xml
|
||||
{{ end }}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue