# robots.txt for Redefine Web
# Reference: https://developers.google.com/search/docs/crawling-indexing/robots/create-robots-txt

# ---------- All crawlers ----------
User-agent: *
Allow: /

# WordPress admin (block, but keep admin-ajax.php crawlable — used by front-end
# form submits, carts, and lazy-load handlers that a rendering crawler triggers)
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

# Attack-surface endpoints — never content
Disallow: /xmlrpc.php
Disallow: /wp-login.php
Disallow: /wp-register.php
Disallow: /wp-signup.php
Disallow: /wp-activate.php
Disallow: /wp-mail.php
Disallow: /wp-trackback.php

# WP internals
Disallow: /wp-includes/
Disallow: /wp-json/wp/v2/users/

# On-site search + query-string variants that add no value
Disallow: /?s=
Disallow: /*?replytocom=
Disallow: /*?attachment_id=
Disallow: /*?preview=true
Disallow: /*&preview=

# Trackbacks + comment pagination — duplicate/thin content
Disallow: /trackback/
Disallow: /*/trackback/
Disallow: /*/comment-page-
Disallow: /comments/feed/

# Duplicator + backup dirs (should not exist on production but block anyway)
Disallow: /wp-content/backups-dup-pro/
Disallow: /wp-content/backup-db/
Disallow: /wp-content/backups/
Disallow: /wp-content/upgrade/

# Author archive enumeration path
Disallow: /author/

# Explicitly allow assets — some crawlers block CSS/JS otherwise and misrender
Allow: /wp-content/uploads/
Allow: /wp-content/themes/
Allow: /wp-content/plugins/
Allow: /*.css$
Allow: /*.js$
Allow: /*.woff2$
Allow: /*.svg$
Allow: /*.png$
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.webp$
Allow: /*.avif$

# ---------- Bad-actor SEO scrapers (bandwidth savers) ----------
User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: SEMrushBot-BA
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: SeznamBot
Disallow: /

User-agent: SerpstatBot
Disallow: /

# ---------- Sitemap ----------
Sitemap: http://localhost/newsite/sitemap_index.xml