# www.robotstxt.org/ # http://code.google.com/web/controlcrawlindex/ User-agent: * Disallow: /.git/* Disallow: /app/* Disallow: /bootstrap/* Disallow: /node_modules/* Disallow: /vendor/* Disallow: /admin/* Disallow: /auth/*