# /home/*/public_html/robots.txt
# https://aspiegel.com/petalbot	 - search engine?
# http://www.bing.com/bingbot.htm - search engine
# http://mj12bot.com/ (free search engine?) User-Agent: MJ12bot
#
#User-agent: Google
#Disallow:
#
#User-agent: *
#Disallow: /

User-agent: Adsbot
Disallow: / 

# Ahrefs web crawler
User-agent: AhrefsBot
Disallow: / 

# Semrush web crawler
User-agent: SemrushBot
Disallow: /

User-agent: ZoominfoBot
Disallow: /

User-agent: coccocbot-web
Disallow: /

# DotBot is OpenSiteExplorer's web crawler
User-agent: DotBot
Disallow: /

# Huawei Assistant and AI Search services, both services are powered by Petal Search enigine
User-agent: PetalBot
Disallow: / 

# <AI bots>

# GPTBot is OpenAI's web crawler
User-agent: GPTBot
Disallow: /

# Google Bard & Gemini
User-agent: Google-Extended
Disallow: /

# ChatGPT-User is OpenAI's web crawler
User-agent: ChatGPT-User
Disallow: /

# Common Crawl bot
User-agent: CCBot
Disallow: /

# PiplBot is PiplBot's web crawler
User-agent: PiplBot
Disallow: /

# anthropic-ai is Anthropic's web crawler
User-agent: anthropic-ai
Disallow: /

# Claude-Web is Claude's web crawler
User-agent: Claude-Web
Disallow: /

# TurnitinBot is Turnitin's web crawler
User-agent: TurnitinBot
Disallow: /

# PetalBot is Petal's web crawler
User-agent: PetalBot
Disallow: /

# MoodleBot is Moodl's web crawler
User-agent: MoodleBot
Disallow: /

# magpie-crawler is Brandwatch.com's web crawler
User-agent: magpie-crawler
Disallow: /

# Meta 
User-agent: meta-externalagent
Disallow: /

User-agent: ImagesiftBot
Disallow: /

# </AI bots>