User-agent: *
Allow: /

# Block admin, login, and sensitive pages
Disallow: /login
Disallow: /cron
Disallow: /admin
Disallow: /admin-new
Disallow: /admin2
Disallow: /blog/wp-admin
Disallow: /blog/wp-login.php
Disallow: /blog/wp-register.php

# Prevent crawling of WordPress JSON API endpoints
Disallow: /blog/wp-json/
Disallow: /blog/?rest_route=

# Block search URLs
Disallow: /search/
Disallow: /?s=
Disallow: *?s=*

# Block post/page parameters
Disallow: *&preview=*

# Block feeds
Disallow: /blog/feed/
Disallow: /feed$
Disallow: /blog/comments/feed
Disallow: /blog/*/feed
Disallow: */feed$
Disallow: /?feed=
Disallow: /wp-feed

# Block spammy directories
Disallow: /trackback/
Disallow: */comments$
Disallow: */trackback
Disallow: */trackback$
Disallow: /wp-comments
Disallow: /wp-trackback
Disallow: */replytocom=

# Block login page (duplicate safe)
Disallow: /login/

# Block known scrapper bots
User-agent: DotBot
Disallow: /
User-agent: GiftGhostBot
Disallow: /
User-agent: Seznam
Disallow: /
User-agent: PaperLiBot
Disallow: /
User-agent: Genieo
Disallow: /
User-agent: Dataprovider/6.101
Disallow: /
User-agent: DataproviderSiteExplorer
Disallow: /
User-agent: Dazoobot/1.0
Disallow: /
User-agent: Diffbot
Disallow: /
User-agent: DomainStatsBot/1.0
Disallow: /
User-agent: DubaiIndex
Disallow: /
User-agent: eCommerceBot
Disallow: /
User-agent: ExpertSearchSpider
Disallow: /
User-agent: Feedbin
Disallow: /
User-agent: Fetch/2.0a
Disallow: /
User-agent: FFbot/1.0
Disallow: /
User-agent: Focusbot/1.1
Disallow: /
User-agent: HuaweiSymantecSpider
Disallow: /
User-agent: JobdiggerSpider
Disallow: /
User-agent: LemurWebCrawler
Disallow: /
User-agent: LipperheyLinkExplorer
Disallow: /
User-agent: LSSRocketCrawler/1.0
Disallow: /
User-agent: LYT.SRv1.5
Disallow: /
User-agent: MiaDev/0.0.1
Disallow: /
User-agent: Najdi.si/3.1
Disallow: /
User-agent: BountiiBot
Disallow: /
User-agent: Experibot_v1
Disallow: /
User-agent: Bixocrawler
Disallow: /
User-agent: Crawler4j
Disallow: /
User-agent: Crowsnest/0.5
Disallow: /
User-agent: CukBot
Disallow: /
User-agent: Drupact/0.7
Disallow: /
User-agent: Ezooms/1.0
Disallow: /
User-agent: Fastbot
Disallow: /
User-agent: Feedly
Disallow: /
User-agent: Feedspot
Disallow: /
User-agent: Clickagy
Disallow: /
User-agent: Classbot
Disallow: /
User-agent: CISPA Vulnerability Notification
Disallow: /
User-agent: CirrusExplorer/1.1
Disallow: /
User-agent: BUbiNG
Disallow: /
User-agent: Findlinks
Disallow: /
User-agent: Bixo
Disallow: /
User-agent: Crawlera/1.10.2
Disallow: /

# Sitemap locations
# Primary indexes only (image + video are children of the shop index)
Sitemap: https://www.standbanner.co.uk/sitemap.xml
Sitemap: https://www.standbanner.co.uk/blog/sitemap_index.xml