User-agent: CazoodleBot Disallow: / # Block MJ12bot as it is just noise User-agent: MJ12bot Disallow: / # Block dotbot as it cannot parse base urls properly User-agent: dotbot/1.0 Disallow: / # Block Gigabot User-agent: Gigabot Disallow: / # Block trendkite-akashic-crawler User-agent: trendkite-akashic-crawler Disallow: / User-agent: Amazonbot Disallow: /