# robots.txt for forgeup.in
# Last updated: March 2026
# -----------------------------------------------
# RULES FOR ALL CRAWLERS
# -----------------------------------------------
User-agent: *

# == MAIN SITE — HIGH-VALUE INDEXABLE PAGES ==
Allow: /$
Allow: /gmp/
Allow: /ipo/
Allow: /about/
Allow: /disclaimer/
Allow: /privacy/
Allow: /terms/
Allow: /editorial-policy/
Allow: /contact/
Allow: /filings/

# == WORDPRESS NEWS SECTION — ALL CRAWLABLE PAGES ==
Allow: /news/$
Allow: /news/ipo-news/
Allow: /news/startup-news/
Allow: /news/latest-news/
Allow: /news/personal-finance/
Allow: /news/trending/
Allow: /news/blog/
Allow: /news/blog/markets/
Allow: /news/ipo/
Allow: /news/wp-json/oembed/

# Individual news articles
Allow: /news/*.html
Allow: /news/*/

# == DISALLOW — NO SEO VALUE ==

# Block filings sub-pages (only /filings/ itself should be indexed)
Disallow: /filings/*/

# Internal/utility files
Disallow: /contentful-cache.json
Disallow: /main.js
Disallow: /css/
Disallow: /gmp.php

# WordPress system paths
Disallow: /news/wp-admin/
Disallow: /news/wp-includes/
Disallow: /news/wp-content/plugins/
Disallow: /news/wp-content/themes/
Disallow: /news/xmlrpc.php
Disallow: /news/wp-login.php
Disallow: /news/wp-json/
Disallow: /news/?s=
Disallow: /news/page/
Disallow: /news/tag/
Disallow: /news/author/
Disallow: /news/feed/
Disallow: /news/comments/
Disallow: /news/?p=
Disallow: /news/?replytocom=

# -----------------------------------------------
# GOOGLE-SPECIFIC RULES
# -----------------------------------------------
User-agent: Googlebot

# Allow Google to crawl images (needed for Google Images & rich results)
Allow: /assets/
Allow: /news/wp-content/uploads/

# -----------------------------------------------
# BLOCK AGGRESSIVE / SCRAPER BOTS
# -----------------------------------------------
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

# -----------------------------------------------
# SITEMAPS
# -----------------------------------------------
Sitemap: https://forgeup.in/sitemap.xml
Sitemap: https://forgeup.in/news/sitemap_index.xml
Sitemap: https://forgeup.in/news/sitemap-news.xml