User-agent: sitecheck.internetseer.com
Disallow: /

User-agent: TurnitinBot 
Disallow: /

User-agent: NPBot 
Disallow: /

#
#User-agent: * 
#Disallow: /news/page/
#User-agent: * 
#Disallow: /resources/jails1999.html
#User-agent: * 
#Disallow: /resources/prisons2000.html

Sitemap: http://www.prisonersofthecensus.org/sitemap.xml

User-agent: *
Disallow: /Census_Report_Feb_06.pdf
Disallow: /nycounties/nycounties.pdf
Disallow: /cgi-bin/
Disallow: /404error.html 
Disallow: /ct/report_2013.pdf

User-agent: *
Crawl-delay: 10