# this is the code that will eliminate robots/spiders/search engines
# from searching the FED spatial data archive page.

User-agent: *
# * denotes that all robots will be affected
Disallow: /gis_data/
/fedmac/
/pub/
# any url starting with gis_data will not be searched