搜索引擎蜘蛛及Robots详解

Disallow:/third
Disallow:/beta
Disallow:/java
Disallow:/shockwave
Disallow:/JOBS
Disallow:/pr
Disallow:/Interactive
Disallow:/alt_index.html
Disallow:/webmaster_logs
Disallow:/newscenter
Disallow:/virtual
Disallow:/DIGEST
Disallow:/QUICKNEWS
Disallow:/SEARCH
User-agent:Scooter
Disallow:/cgi-bin
Disallow:/TRANscriptS
Disallow:/development
Disallow:/third
Disallow:/beta
Disallow:/java
Disallow:/shockwave
Disallow:/JOBS
Disallow:/pr
Disallow:/Interactive
Disallow:/alt_index.html
Disallow:/webmaster_logs
Disallow:/newscenter
Disallow:/virtual
Disallow:/DIGEST
Disallow:/QUICKNEWS
Disallow:/SEARCH

User-agent:Ultraseek
Disallow:/cgi-bin
#Disallow:/TRANscriptS
Disallow:/development
Disallow:/third
Disallow:/beta
Disallow:/java
Disallow:/shockwave
Disallow:/JOBS
Disallow:/pr
Disallow:/Interactive
Disallow:/alt_index.html
Disallow:/webmaster_logs
Disallow:/newscenter
Disallow:/virtual
Disallow:/DIGEST
Disallow:/QUICKNEWS
Disallow:/SEARCH


User-agent:smallbear
Disallow:/cgi-bin
Disallow:/java
Disallow:/images
Disallow:/development
Disallow:/third
Disallow:/beta
Disallow:/webmaster_logs
Disallow:/virtual
Disallow:/shockwave
Disallow:/TRANscriptS
Disallow:/newscenter
Disallow:/virtual
Disallow:/DIGEST
Disallow:/QUICKNEWS
Disallow:/SEARCH
Disallow:/alt_index.html
User-agent:GoogleBot
Disallow:/cgi-bin
Disallow:/java
Disallow:/images
Disallow:/development
Disallow:/third
Disallow:/beta
Disallow:/webmaster_logs
Disallow:/virtual
Disallow:/shockwave
Disallow:/TRANscriptS
Disallow:/newscenter
Disallow:/virtual
Disallow:/DIGEST
Disallow:/QUICKNEWS
Disallow:/SEARCH
Disallow:/alt_index.html