Update robots.txt, add ROBOTS_DISALLOWED and extra-robots.txt

_sources/configs/robots.php

@@ -25,6 +25,7 @@
 	$siteMapUrl = "$server$script/sitemap$subdir/sitemap-index-$identifier.xml";
 
 	echo "Sitemap: $siteMapUrl\n";
+    echo "\n# Content of the robots.txt file:\n";
 }
 
 readfile( 'robots-main.txt' );