# Allow all search engines to crawl the entire site (by default)
User-agent: *
Disallow: /cgi-bin/  # Disallow scripts and server-side code
Disallow: /wp-admin/  # WordPress admin area
Disallow: /wp-login.php  # WordPress login page
Disallow: /cart/  # Block the cart page (if e-commerce)
Disallow: /checkout/  # Block checkout page (if e-commerce)
Disallow: /my-account/  # Block personal account pages (if e-commerce)
Disallow: /order/  # Block order pages (if e-commerce)
Disallow: /search/  # Block search result pages
Disallow: /tag/  # Block tag pages (if not useful)
Disallow: /category/  # Block category archive pages (if not useful)

# Allow Googlebot (or other bots) to crawl and index all content except the restricted areas
User-agent: Googlebot
Allow: /  # Allow all pages to be indexed

# Disallow crawling of specific URL patterns (adjust to match your site structure)
Disallow: /*?*  # Block URLs with query parameters (e.g., ?utm_source)
Disallow: /track/  # Block tracking URLs

# Sitemap location (ensure the path is correct and points to your sitemap)
Sitemap: https://holidayworldwide.in/sitemap.xml

# Block specific crawlers that might cause issues
User-agent: Baiduspider
Disallow: /  # Block Baidu crawler (or other non-essential bots)

# Block duplicate content or low-value pages
Disallow: /duplicate-content/  # Example: a page with duplicate content
Disallow: /private/  # Private or unimportant content