# robots.txt for not crawling work directories User-agent: * Disallow: /work Disallow: /cgi-bin Disallow: /NJIT/601