# see http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file
#
# to ban all spiders from the entire site uncomment the next two lines:
# user-agent: *
# disallow: /

user-agent: *
user-agent: Googlebot
disallow: /dist/robots.*.js
disallow: /assets/robots-*.js
disallow: /collection/browse_results.php
disallow: /collection/object.php
disallow: /collection/artist.php
disallow: /collection/theme.php
disallow: /collection_lb/browse_results.php
disallow: /collection_ge/browse_results.php
disallow: /collection/search.php
disallow: /visit/calendar/search
disallow: /visit/calendar/filters
disallow: /visit/calendar/films?day*
disallow: /media/*
disallow: /screens/*
disallow: /visit/wifi/
disallow: /cdn-cgi/

User-agent: Twitterbot
allow: /media/*