# All robots will spider this snazzy website 

User-agent: *

Disallow: /pdfs/

Disallow: /database/

Disallow: /cgi-bin/

Disallow: /tests/

Disallow: /images/

Disallow: /scripts/

Disallow: /gav/

Disallow: /google/

Disallow: /flash/