# /robots.txt file for https://www.fhwa.dot.gov
# comments? mail web.master@fhwa.dot.gov, FHWAWEB2@dot.gov

# rules for all robots
User-agent: *

# 1 second crawl delay (note - ignored by googlebot)
Crawl-delay: 1

# files/directories excluded from crawl
Disallow: /404.cfm
Disallow: /environment/
Disallow: /exit.cfm
Disallow: /hep/
Disallow: /highwayhistory/gallery.cfm
Disallow: /livability/
Disallow: /pavement/falcon/
Disallow: /pgc/
Disallow: /planning/
Disallow: /property/
Disallow: /research/tfhrc/expertise/
Disallow: /research/tfhrc/projects/projectsdb/
Disallow: /resourcecenter/staff/bios.cfm
Disallow: /real_estate/
Disallow: /temp/
Disallow: /tempBad/
Disallow: /tempcrt/
Disallow: /wyklebio.htm
Disallow: receivingdata.cfm


# exception files from excluded list above
Allow: /environment/index.cfm
Allow: /hep/index.cfm
Allow: /livability/index.cfm
Allow: /pgc/index.cfm
Allow: /planning/index.cfm
Allow: /planning/community_connections/index.cfm
Allow: /real_estate/index.cfm
Allow: /research/tfhrc/expertise/index.cfm
Allow: /research/tfhrc/expertise/rdtorgchart.cfm
Allow: /research/tfhrc/expertise/role.cfm
Allow: /research/tfhrc/projects/projectsdb/index.cfm