# Robots.txt for https://www.creativeauthor.online # Optimized for SEO and indexing # Allow all web crawlers to access the site User-agent: * Allow: / # Allow crawling of important directories and pages Allow: /p/ Allow: /public/ Allow: /feeds/posts/default?alt=rss # Disallow specific directories to improve SEO Disallow: /cgi-bin/ Disallow: /junk/ Disallow: /ads/ Disallow: /404 Disallow: /error404 # Sitemap locations for better indexing Sitemap: https://www.creativeauthor.online/sitemap.xml Sitemap: https://www.creativeauthor.online/news-sitemap.xml Sitemap: https://www.creativeauthor.online/atom.xml?redirect=false&start-index=1&max-results=500