Arkadaşlar ben bir wp destek sitesindeki robots.txt yi wp sitem için ftp ye attım ancak şimdi google site araçlarından googlebot hareketlerini inceleğimde robots.txt tarafından kısıtlanan sayfalar gördüm yaklaşık 84 sayfayı kısıtlamış ve bu sayfalar wp arşiv sayfaları ne yapmalıyım elinde wp için uygun robots.txt olan varmı
bendeki robots.txt

# This rule means it applies to all user-agents
User-agent: *
 
# Disallow all directories and files within
Disallow: /cgi-bin/
Disallow: /stats/
Disallow: /dh_
Disallow: /about/legal-notice/
Disallow: /about/copyright-policy/
Disallow: /about/terms-and-conditions/
Disallow: /contact/
Disallow: /tag/
Disallow: /wp-admin/
Disallow: /wp-includes/
 
# Disallow all monthly archive pages
Disallow: /2006/0
Disallow: /2006/1
Disallow: /2007/0
Disallow: /2007/1
 
# The Googlebot is the main search bot for google
User-agent: Googlebot
 
# Disallow all files ending with these extensions
Disallow: /*.php$
Disallow: /*.js$
Disallow: /*.inc$
Disallow: /*.css$
Disallow: /*.gz$
Disallow: /*.wmv$
Disallow: /*.tar$
Disallow: /*.tgz$
Disallow: /*.cgi$
Disallow: /*.xhtml$
 
# Disallow Google from parsing indididual post feeds and trackbacks..
Disallow: */feed/
Disallow: */trackback/
 
# Disallow all files with ? in url
Disallow: /*?*
Disallow: /*?
 
# Disallow all archived monthlies
Disallow: /2006/0*
Disallow: /2007/0*
Disallow: /2006/1*
Disallow: /2007/1*
 
# The Googlebot-Image is the image bot for google
User-agent: Googlebot-Image
 
# Allow Everything
Allow: /*
 
# This is the ad bot for google
User-agent: Mediapartners-Google*
 
# Allow Everything
# BEGIN XML-SITEMAP-PLUGIN
Sitemap: http://www.sinema24.net/sitemap.xml
# END XML-SITEMAP-PLUGIN