User-agent: Scrapy
Allow: /

User-agent: *  
  
# On empêche l'indexation des dossiers sensibles  
Disallow: /wp-admin  
Disallow: /wp-includes  
Disallow: /wp-content/plugins  
Disallow: /wp-content/cache  
Disallow: /wp-content/themes  
Disallow: /trackback  
Disallow: /feed  
Disallow: /comments  
Disallow: /category/*/*  
Disallow: */trackback  
Disallow: */feed  
Disallow: */comments
Disallow: /*.pdf$
# On désindexe tous les URL ayant des paramètres (duplication de contenu) 
Disallow: /*?*  
Disallow: /*?
# On désindexe la page de connexion (contenu inutile)
Disallow: /wp-login.php
 
# disable duggmirror by Digg
User-agent: duggmirror
Disallow: /
 
# allow google image bot to search all images
User-agent: Googlebot-Image
Disallow: /wp-includes/
Allow: /*
# On autorise l'indexation des images  
Allow: /wp-content/uploads
  
# Autoriser Google Image  
User-agent: Googlebot-Image  
Disallow:  
Allow: /*  
  
# Autoriser Google AdSense  
User-agent: Mediapartners-Google*  
Disallow:  
Allow: /*  
  
# On indique au spider le lien vers notre sitemap  
Sitemap: http://allolemonde.fr/sitemapindex.xml