#UVic robots.txt User-agent: * Crawl-delay: 30 #Googlebot specs https://developers.google.com/webmasters/control-crawl-index/docs/robots_txt #More specific rules (the allows here) trump more general rules Allow: */wp-includes/*/*.css$ Allow: */wp-includes/js/*.js$ Allow: */wp-includes/js/*/*.js$ #WORDPRESS-1268 Discourage bots from indexing search pages in robots.txt Disallow: */search/ Disallow: */globalsearch/ #Folders Disallow: */cgi-bin/ Disallow: */et_temp/ Disallow: */exports/ Disallow: */localextras/ Disallow: */related-tags* Disallow: */server-status/ Disallow: */TOSPages/ Disallow: */trackback/ Disallow: */wp-admin/ Disallow: */wp-config* Disallow: */wp-content/cache/ Disallow: */wp-includes/ Disallow: /cart/ Disallow: /checkout/ Disallow: /my-account/ #Queries Disallow: /*?replytocom=* Disallow: /*add_to_wishlist=* Disallow: /*add-to-cart=* Disallow: /*filter_shape=* Disallow: /*product_count=* Disallow: /*product_order=* Disallow: /*product_orderby=* Disallow: /*products-per-page=* Disallow: /*vote=* Disallow: /*woocommerce_waitlist=* #Search, sharing, sorting not needed for bots Disallow: /*?sort=* Disallow: /*share=* Disallow: /*shared=* Disallow: /search?q=* #File patterns Disallow: */wp-content/themes/*.php Disallow: */xmlrpc.php Disallow: /readme.html #Timely Events Calendar Disallow: /*action~* #ecwd Calendar Disallow: /*calendar/?date=* #Tribe Events Calendar Disallow: /*tribe-bar-date=* #Comment Mail Plugin Disallow: /*comment_mail* #Extensions Disallow: /*.7z Disallow: /*.7zip Disallow: /*.asp* Disallow: /*.bak Disallow: /*.bz Disallow: /*.bzip Disallow: /*.cfm Disallow: /*.cgi Disallow: /*.gz Disallow: /*.gzip Disallow: /*.inc Disallow: /*.ini Disallow: /*.json Disallow: /*.|jsp* Disallow: /*.old Disallow: /*.log Disallow: /*.phar Disallow: /*.php4 Disallow: /*.php5 Disallow: /*.php~ Disallow: /*.pl Disallow: /*.py Disallow: /*.rar Disallow: /*.readme Disallow: /*.sh Disallow: /*.sql Disallow: /*.tar Disallow: /*.tgz Disallow: /*.txt Disallow: /*.xml Disallow: /*.zip Disallow: /*.|jsp* Allow: /*.css$ Allow: /*.js$ #Dicourage bots I do not care to support ==== User-agent: msnbot Crawl-delay: 1 # digg mirror User-agent: duggmirror Disallow: / # MJ12bot http://mj12bot.com/ User-agent: MJ12bot Disallow: / User-agent: Sogou web spider Disallow: /* User-agent: Sogou inst spider Disallow: /* User-agent: Sogou spider2 Disallow: /* User-agent: Experibot* Disallow: /* #End of UVic robots.txt