# Disallow all crawlers access to certain pages.
User-agent: *
Disallow: /OEFront/
Disallow: /universaldegrees/degreeprograms/
Disallow: /aff/
Disallow: /me/
Disallow: /au/
Disallow: /uk/


# Disallow Following crawlers access to the site

User-agent: ia_archiver-web.archive.org
Disallow: /

User-agent: ia_archiver
Disallow: /

User-agent: ia_archiver-web.archive.org
Disallow: robots.txt

User-agent: ia_archiver
Disallow: robots.txt

User-agent: AboutUsBot 
Disallow: /

User-agent: robtexbot
Disallow: /