CODE EXAMPLE FOR PHP

php detect crawler

function is_web_crawler($user_agent) {
  return preg_match('/archiver|bot|crawl|slurp|spider|mediapartners/i',$user_agent);
}
$is_crawler = is_web_crawler($_SERVER['HTTP_USER_AGENT']);