# Safetly.se - Robots.txt # Optimized for AI crawlers, LLMs and search engines # Last updated: January 2025 # ===== LLMs.txt Standard - AI Knowledge Files ===== # Primary AI knowledge file (Swedish) # https://safetly.se/llms.txt # English version for international AI systems # https://safetly.se/llms-en.txt # Primary sitemap Sitemap: https://safetly.se/sitemap.xml # ===== Default rules for all crawlers ===== User-agent: * Allow: / Allow: /llms.txt Allow: /llms-en.txt Allow: /ai-knowledge Allow: /nyheter/ Allow: /ordningsvakt Allow: /vaktare Allow: /entrevard Allow: /civil-butikskontrollant Allow: /safetly Allow: /priser Allow: /om-oss Allow: /kontakt Allow: /demo Disallow: /admin/ Disallow: /api/ Disallow: /_* Disallow: /setup/ Crawl-delay: 0 # Block tracking parameters Disallow: /*?utm_* Disallow: /*?fbclid=* Disallow: /*?gclid=* Disallow: /*?ref=* Disallow: /*?debug=* # ===== AI & LLM Crawlers - Priority Access ===== # OpenAI GPTBot (ChatGPT, GPT-4, GPT-4o) User-agent: GPTBot Allow: / Allow: /llms.txt Allow: /llms-en.txt Allow: /ai-knowledge Crawl-delay: 0 # OpenAI ChatGPT User Browsing User-agent: ChatGPT-User Allow: / Allow: /llms.txt Allow: /llms-en.txt Allow: /ai-knowledge Crawl-delay: 0 # OpenAI Plugins User-agent: OAI-SearchBot Allow: / Allow: /llms.txt Allow: /llms-en.txt Allow: /ai-knowledge Crawl-delay: 0 # Anthropic Claude User-agent: anthropic-ai Allow: / Allow: /llms.txt Allow: /llms-en.txt Allow: /ai-knowledge Crawl-delay: 0 User-agent: Claude-Web Allow: / Allow: /llms.txt Allow: /llms-en.txt Allow: /ai-knowledge Crawl-delay: 0 User-agent: ClaudeBot Allow: / Allow: /llms.txt Allow: /llms-en.txt Allow: /ai-knowledge Crawl-delay: 0 # Google AI (Gemini, Bard, AI Overviews) User-agent: Google-Extended Allow: / Allow: /llms.txt Allow: /llms-en.txt Allow: /ai-knowledge Crawl-delay: 0 User-agent: Googlebot Allow: / Allow: /llms.txt Allow: /llms-en.txt Allow: /ai-knowledge Crawl-delay: 0 # Perplexity AI User-agent: PerplexityBot Allow: / Allow: /llms.txt Allow: /llms-en.txt Allow: /ai-knowledge Crawl-delay: 0 # Microsoft Copilot / Bing AI User-agent: bingbot Allow: / Allow: /llms.txt Allow: /llms-en.txt Allow: /ai-knowledge Crawl-delay: 0 # Apple Intelligence / Siri User-agent: Applebot Allow: / Allow: /llms.txt Allow: /llms-en.txt Allow: /ai-knowledge Crawl-delay: 0 User-agent: Applebot-Extended Allow: / Allow: /llms.txt Allow: /llms-en.txt Allow: /ai-knowledge Crawl-delay: 0 # Meta AI (Llama, WhatsApp AI) User-agent: meta-externalagent Allow: / Allow: /llms.txt Allow: /llms-en.txt Allow: /ai-knowledge Crawl-delay: 0 User-agent: FacebookBot Allow: / Allow: /llms.txt Allow: /llms-en.txt Allow: /ai-knowledge Crawl-delay: 0 # Cohere AI User-agent: cohere-ai Allow: / Allow: /llms.txt Allow: /llms-en.txt Allow: /ai-knowledge Crawl-delay: 0 # You.com AI Search User-agent: YouBot Allow: / Allow: /llms.txt Allow: /llms-en.txt Allow: /ai-knowledge Crawl-delay: 0 # Brave Search AI User-agent: Brave Allow: / Allow: /llms.txt Allow: /llms-en.txt Allow: /ai-knowledge Crawl-delay: 0 # DuckDuckGo AI User-agent: DuckDuckBot Allow: / Allow: /llms.txt Allow: /llms-en.txt Allow: /ai-knowledge Crawl-delay: 0 # Common Crawl (training data for many AI models) User-agent: CCBot Allow: / Allow: /llms.txt Allow: /llms-en.txt Allow: /ai-knowledge Crawl-delay: 1 # Diffbot (AI content extraction) User-agent: Diffbot Allow: / Allow: /llms.txt Allow: /llms-en.txt Allow: /ai-knowledge Crawl-delay: 0 # Neeva AI Search User-agent: NeevaBot Allow: / Allow: /llms.txt Allow: /llms-en.txt Allow: /ai-knowledge Crawl-delay: 0 # Amazon Alexa / AWS AI User-agent: ia_archiver Allow: / Allow: /llms.txt Allow: /llms-en.txt Allow: /ai-knowledge Crawl-delay: 0 # Yandex AI User-agent: YandexBot Allow: / Allow: /llms.txt Allow: /llms-en.txt Allow: /ai-knowledge Crawl-delay: 0 # Baidu AI (for Asian markets) User-agent: Baiduspider Allow: / Allow: /llms.txt Allow: /llms-en.txt Allow: /ai-knowledge Crawl-delay: 0 # ===== Google Specialized Crawlers ===== User-agent: Googlebot-Image Allow: /assets/ Allow: /images/ Allow: /lovable-uploads/ Allow: *.jpg Allow: *.jpeg Allow: *.png Allow: *.webp Allow: *.svg User-agent: Googlebot-News Allow: /nyheter/ Crawl-delay: 0 User-agent: Googlebot-Mobile Allow: / User-agent: Googlebot-Video Allow: / # ===== Social Media Crawlers ===== User-agent: facebookexternalhit Allow: / User-agent: Twitterbot Allow: / User-agent: LinkedInBot Allow: / User-agent: Slackbot Allow: / User-agent: TelegramBot Allow: / User-agent: WhatsApp Allow: / User-agent: Pinterest Allow: / # ===== SEO Tool Bots - Rate Limited ===== User-agent: AhrefsBot Crawl-delay: 10 User-agent: SemrushBot Crawl-delay: 10 User-agent: MozBot Crawl-delay: 10 # ===== Block Malicious/Spam Bots ===== User-agent: DotBot Disallow: / User-agent: MJ12bot Disallow: / User-agent: BLEXBot Disallow: / User-agent: DataForSeoBot Disallow: / User-agent: PetalBot Disallow: / User-agent: Bytespider Disallow: / # ===== Clean URLs - Block Query Strings ===== # Note: This should be at the end as it's a catch-all # Moved to per-bot rules above to avoid conflicts