User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

Sitemap: https://www.odni.gov/wp-sitemap.xml

# WHAT: SearchGov (usasearch) federal search crawler configuration
# HOW: Add dedicated user-agent directive with optimized crawl delay
# WHY: Enable faster indexing in federal Search.gov results as recommended by digital.gov
# SEE: https://digital.gov/resources/introduction-robots-txt-files
# SEE: https://www.search.gov/manage-search/searchgov-crawler
User-agent: usasearch
Crawl-delay: 2
Allow: /

# WHAT: Default crawl delay for all other bots
# HOW: Set crawl-delay: 10 seconds for non-SearchGov crawlers
# WHY: Balance search engine access with server resource management per digital.gov recommendations
User-agent: *
Crawl-delay: 10
Disallow: /dlm_uploads/