User-agent: *
Sitemap: https://blog.pragmaticengineer.com/sitemap.xml

User-agent: *
Disallow: /ghost

User-agent: *
Disallow: /engguidebook-bonus-1

User-agent: *
Disallow: /engguidebook-bonus-2

User-agent: *
Disallow: /engguidebook-bonus-3

User-agent: *
Disallow: /engguidebook-bonus-4

User-agent: *
Disallow: /engguidebook-bonus-5

User-agent: *
Disallow: /engguidebook-bonus-all

User-agent: *
Disallow:/tech-industry-researcher

User-agent: *
Disallow:/newsletter-advice

# <AI bots>

# GPTBot is OpenAI's web crawler
User-agent: GPTBot
Disallow: /

# Google Bard & Gemini
User-agent: Google-Extended
Disallow: /

# ChatGPT-User is OpenAI's web crawler
User-agent: ChatGPT-User
Disallow: /

# Common Crawl bot
User-agent: CCBot
Disallow: /

# PiplBot is PiplBot's web crawler
User-agent: PiplBot
Disallow: /

# anthropic-ai is Anthropic's web crawler
User-agent: anthropic-ai
Disallow: /

# Claude-Web is Claude's web crawler
User-agent: Claude-Web
Disallow: /

# TurnitinBot is Turnitin's web crawler
User-agent: TurnitinBot
Disallow: /

# PetalBot is Petal's web crawler
User-agent: PetalBot
Disallow: /

# MoodleBot is Moodl's web crawler
User-agent: MoodleBot
Disallow: /

# magpie-crawler is Brandwatch.com's web crawler
User-agent: magpie-crawler
Disallow: /

# </AI bots>

User-agent: ia_archiver
Disallow: /data-structures-and-algorithms-i-actually-used-day-to-day/

User-agent: ia_archiver
Disallow: /devternity-fake-speakers/