# Termly scanner
User-agent: TermlyBot
Allow: /

User-agent: *
Disallow: /archive/
Disallow: /wp-admin/
Disallow: /wp-content/plugins/
Disallow: /wp-content/themes/
Allow: /wp-content/uploads/
Disallow: /*?amp=1
Disallow: /*/?amp=1
Disallow: /*/*/?amp=1
Disallow: /*?noamp=mobile
Disallow: /*/?noamp=mobile
Disallow: /*?random-post=*

Allow: /wp-admin/admin-ajax.php

Sitemap: https://urdu24.com/sitemap_index.xml
Sitemap: https://urdu24.com/sitemap-news.xml

# Block feeds for all bots by default
User-agent: *
Disallow: /feed/
Disallow: /feed$
Disallow: /tag/


# Allow feeds for Google News bot
User-agent: Googlebot-News
Allow: /feed/
Allow: /feed/google-news/

# Allow feeds for Opera News bot
User-agent: OperaNewsBot
Allow: /feed/
Allow: /feed/google-news/

# Allow feeds for Bing News bot (optional)
User-agent: MSNBot-News
Allow: /feed/

# Allow feeds for Googlebot (optional, if you want full crawl)
User-agent: Googlebot
Allow: /feed/
Allow: /feed/google-news/