User-agent: * Disallow: /admin/ Disallow: /private/ # Allow social bots to preview pages while search engines are restricted User-agent: facebookexternalhit Allow: / User-agent: Twitterbot Allow: / # Specific rules for search engines (optional, if needed) User-agent: Googlebot Disallow: /temp/