User-agent: * Request-rate: 1/5 # maximum rate is one page every 5 seconds Crawl-delay: 30 # Ukranian hackers created this directory - disallow it Disallow: /wp-content/backup # This disallows odd URLs that we didn't create, and which "loop" Disallow: /www.* Disallow: /www.eventsatcommonsensemedia.org/ Disallow: /www.myspace.com/ Disallow: /www.vinebergcommunications.com/ Disallow: /www.bayareagirlsrockcamp.org/ Disallow: /www.forbes.com/ Disallow: /www.globalkids.org/ Disallow: /adventuredad.com/ Disallow: /www.lorieanngrover.com/ Disallow: /www.bizandbuzz.blogspot.com/ Disallow: /newsimg.bbc.co.uk/ Disallow: /www.genv.net/ Disallow: /www.natgeotv.com/ Disallow: /pulse.sxsw.com/ Disallow: /www.pinklockersociety.org/ Disallow: /www.justinachenheadley.com/ Disallow: /newsimg.bbc.co.uk/ Disallow: /www.tzonefoundation.org/ Disallow: /www.besafe.net/ Disallow: /www.youtube.com/ Disallow: /www.chicagotribune.com/ Disallow: /groups.myspace.com/ Disallow: /www.tzonefoundation.org/ Disallow: /www.practicalmoneyskills.com/