nutch爬虫系统分析.doc

nutch爬虫系统分析Nutch分析TOC\o"1-3"\h\z\uHYPERLINK\l"_Toc230174360"1Nutch简介PAGEREF_Toc230174360\h2HYPERLINK\l"_Toc230174361"1.1nutch体系结构PAGEREF_Toc230174361\h2HYPERLINK\l"_Toc230174362"2抓取部分PAGEREF_Toc230174362\h3HYPERLINK\l"_Toc230174363"2.1爬虫的数据结构及含义PAGEREF_Toc230174363\h3HYPERLINK\l"_Toc230174364"2.2抓取目录分析PAGEREF_Toc230174364\h4HYPERLINK\l"_Toc230174365"2.3抓取过程概述PAGEREF_Toc230174365\h4HYPERLINK\l"_Toc230174366"2.4抓取过程分析PAGEREF_Toc230174366\h5HYPERLINK\l"_Toc230174367"2.4.1inject方法PAGEREF_Toc230174367\h6HYPERLINK\l"_Toc230174368"2.4.2generate方法PAGEREF_Toc230174368\h12HYPERLINK\l"_Toc230174369"2.4.3fetch方法PAGEREF_Toc230174369\h14HYPERLINK\l"_Toc230174370"2.4.4parse方法PAGEREF_Toc230174370\h16HYPERLINK\l"_Toc230174371"2.4.5update方法PAGEREF_Toc230174371\h16HYPERLINK\l"_Toc230174372"2.4.6invert方法PAGEREF_Toc230174372\h19HYPERLINK\l"_Toc230174373"2.4.7index方法PAGEREF_Toc230174373\h23HYPERLINK\l"_Toc230174374"2.4.8dedup方法PAGEREF_Toc230174374\h26HYPERLINK\l"_Toc230174375"2.4.9merge方法PAGEREF_Toc230174375\h30HYPERLINK\l"_Toc230174376"3配置文件分析PAGEREF_Toc230174376\h31HYPERLINK\l"_Toc230174377"3.1nutch-default.xml分析PAGEREF_Toc230174377\h31HYPERLINK\l"_Toc230174378"3.1.1<!--fileproperties-->PAGEREF_Toc230174378\h31HYPERLINK\l"_Toc230174379"3.1.2<!--HTTPproperties-->PAGEREF_Toc230174379\h32HYPERLINK\l"_Toc230174380"3.1.3<!--FTPproperties-->PAGEREF_Toc230174380\h35HYPERLINK\l"_Toc230174381"3.1.4<!--webdbproperties-->PAGEREF_Toc230174381\h37HYPERLINK\l"_Toc230174382"3.1.5<!--generateproperties-->PAGEREF_Toc230174382\h41HYPERLINK\l"_Toc230174383"3.1.6<!--fetcherproperties-->PAGEREF_Toc230174383\h42HYPERLINK\l"_Toc230174384"3.1.7<!--indexerproperties-->PAGEREF_Toc230174384\h43HYPERLINK\l"_Toc230174385"3.1.8<!--indexingfilterpluginproperties-->PAGEREF_Toc230174385\h45HYPERLINK\l"_Toc230174386"3.1.9<!--analysisproperties-->PAGEREF_Toc2301