User-agent: * Allow: / Allow: /blog/ Allow: /flight-training/ Allow: /services/ Allow: /about-us/ Allow: /*.llms.txt$ # Allow major search engines User-agent: Googlebot Allow: / User-agent: Bingbot Allow: / User-agent: Slurp Allow: / # Yahoo Slurp User-agent: DuckDuckBot Allow: / User-agent: Baiduspider Allow: / User-agent: YandexBot Allow: / # Block unwanted crawlers User-agent: AhrefsBot Disallow: / User-agent: SemrushBot Disallow: / User-agent: MJ12bot Disallow: / User-agent: DotBot Disallow: / User-agent: BLEXBot Disallow: / User-agent: SiteAuditBot Disallow: / # Disallow sensitive or duplicate paths Disallow: /api/ Disallow: /404 Disallow: /500 Disallow: /_astro/ Disallow: /contact-us-confirmation Disallow: /discovery-session-confirmation Disallow: /enroll-confirmation Disallow: /discovery-session # Crawl-delay for aggressive bots (optional) # Crawl-delay: 10 # Sitemap reference Sitemap: https://www.flysolidground.com/sitemap-index.xml # Host reference (for Yandex & others) Host: www.flysolidground.com