B u9a(ã@søddlZddlZddlZddlZddlZddlmZddlm Z m Z Gdd„dƒZ Gdd„de ej ƒZ Gdd „d e ej ƒZGd d „d e ej ƒZGd d „d e ƒZGdd„deej ƒZGdd„deej ƒZGdd„deƒZGdd„de ej ƒZGdd„de ej ƒZGdd„de ej ƒZGdd„de ej ƒZGdd„deƒZGdd„de ej ƒZGd d!„d!e ej ƒZGd"d#„d#e ej ƒZGd$d%„d%e ej ƒZGd&d'„d'eej ƒZGd(d)„d)e ej ƒZGd*d+„d+e ƒZGd,d-„d-ej ƒZ Gd.d/„d/ej ƒZ!e"d0krôe #¡dS)1éN)Úsupport)ÚBaseHTTPRequestHandlerÚ HTTPServerc@s<eZdZdZdZgZgZdd„Zdd„Zdd„Z d d „Z d S) Ú BaseRobotTestÚZtest_robotparsercCs,t |j¡ ¡}tj ¡|_|j |¡dS)N) ÚioÚStringIOÚ robots_txtÚ readlinesÚurllibÚ robotparserÚRobotFileParserÚparserÚparse)ÚselfÚlines©rú&/usr/lib/python3.7/test_robotparser.pyÚsetUps zBaseRobotTest.setUpcCs$t|tƒr|\}}||fS|j|fS)N)Ú isinstanceÚtupleÚagent)rÚurlrrrrÚget_agent_and_urls zBaseRobotTest.get_agent_and_urlc CsPxJ|jD]@}| |¡\}}|j||d| |j ||¡¡WdQRXqWdS)N)rr)ÚgoodrÚsubTestÚ assertTruerÚ can_fetch)rrrrrrÚtest_good_urlss zBaseRobotTest.test_good_urlsc CsPxJ|jD]@}| |¡\}}|j||d| |j ||¡¡WdQRXqWdS)N)rr)ÚbadrrÚ assertFalserr)rrrrrrÚ test_bad_urls!s zBaseRobotTest.test_bad_urlsN) Ú__name__Ú __module__Ú __qualname__r rrrrrrr!rrrrr src@s"eZdZdZddgZdddgZdS)ÚUserAgentWildcardTestz•User-agent: * Disallow: /cyberworld/map/ # This is an infinite virtual URL space Disallow: /tmp/ # these will soon disappear Disallow: /foo.html ú/z /test.htmlz/cyberworld/map/index.htmlz/tmp/xxxz /foo.htmlN)r"r#r$r rrrrrrr%(sr%c@s eZdZdZdddgZdgZdS)ÚCrawlDelayAndCustomAgentTestzå# robots.txt for http://www.example.com/ User-agent: * Crawl-delay: 1 Request-rate: 3/15 Disallow: /cyberworld/map/ # This is an infinite virtual URL space # Cybermapper knows where to go. User-agent: cybermapper Disallow: r&z /test.html)Z cybermapperz/cyberworld/map/index.htmlz/cyberworld/map/index.htmlN)r"r#r$r rrrrrrr'3s  r'c@seZdZdZgZdddgZdS)ÚRejectAllRobotsTestz(# go away User-agent: * Disallow: / z/cyberworld/map/index.htmlr&z/tmp/N)r"r#r$r rrrrrrr(Dsr(c@seZdZdZdZdd„ZdS)ÚBaseRequestRateTestNc Cs²|j}x¦|j|jD]–}| |¡\}}|j||dp| | |¡|j¡| |¡}| ||j¡|jdk r | |t j j ¡| |j |jj ¡| |j |jj ¡WdQRXqWdS)N)rr)rrrrrÚ assertEqualÚ crawl_delayÚ request_rateZassertIsInstancer r Ú RequestRateZrequestsZseconds)rrrrZparsed_request_raterrrÚtest_request_rateRs"    z%BaseRequestRateTest.test_request_rate)r"r#r$r,r+r.rrrrr)Nsr)c@seZdZdZdgZdS)Ú EmptyFileTestrz/fooN)r"r#r$r rrrrrr/jsr/c@s>eZdZdZdZej dd¡ZdZ dgZ ddd d d d d gZ dS)ÚCrawlDelayAndRequestRateTestz’User-agent: figtree Crawl-delay: 3 Request-rate: 9/30 Disallow: /tmp Disallow: /a%3cd.html Disallow: /a%2fb.html Disallow: /%7ejoe/index.html Úfigtreeé éé)r1z /foo.htmlz/tmpz /tmp.htmlz /tmp/a.htmlz /a%3cd.htmlz /a%3Cd.htmlz /a%2fb.htmlz/~joe/index.htmlN) r"r#r$r rr r r-r,r+rrrrrrr0os  r0c@seZdZdZdS)ÚDifferentAgentTestzFigTree Robot libwww-perl/5.04N)r"r#r$rrrrrr5sr5c@s*eZdZdZdgZddddddgZd Zd S) ÚInvalidRequestRateTestzUser-agent: * Disallow: /tmp/ Disallow: /a%3Cd.html Disallow: /a/b.html Disallow: /%7ejoe/index.html Crawl-delay: 3 Request-rate: 9/banana z/tmpz/tmp/z /tmp/a.htmlz /a%3cd.htmlz /a%3Cd.htmlz /a/b.htmlz/%7Ejoe/index.htmlr4N)r"r#r$r rrr+rrrrr6…s   r6c@seZdZdZdgZgZdS)ÚInvalidCrawlDelayTestz2User-Agent: * Disallow: /. Crawl-delay: pears z /foo.htmlN)r"r#r$r rrrrrrr7•sr7c@s eZdZdZdZdgZdgZdS)ÚAnotherInvalidRequestRateTestzeUser-agent: Googlebot Allow: /folder1/myfile.html Disallow: /folder1/ Request-rate: whale/banana Ú Googlebotz/folder1/myfile.htmlz/folder1/anotherfile.htmlN)r"r#r$r rrrrrrrr8¡sr8c@seZdZdZdZdgZdS)ÚUserAgentOrderingTestzMUser-agent: Googlebot Disallow: / User-agent: Googlebot-Mobile Allow: / r9z/something.jpgN)r"r#r$r rrrrrrr:®s r:c@seZdZdZdS)ÚUserAgentGoogleMobileTestzGooglebot-MobileN)r"r#r$rrrrrr;½sr;c@s eZdZdZdZdgZdgZdS)ÚGoogleURLOrderingTestzJUser-agent: Googlebot Allow: /folder1/myfile.html Disallow: /folder1/ Z googlebotz/folder1/myfile.htmlz/folder1/anotherfile.htmlN)r"r#r$r rrrrrrrr<Ásr<c@seZdZdZdgZdgZdS)ÚDisallowQueryStringTestz2User-agent: * Disallow: /some/path?name=value z /some/pathz/some/path?name=valueN)r"r#r$r rrrrrrr=Îsr=c@seZdZdZdgZdgZdS)ÚUseFirstUserAgentWildcardTestzNUser-agent: * Disallow: /some/path User-agent: * Disallow: /another/path z /another/pathz /some/pathN)r"r#r$r rrrrrrr>Øsr>c@seZdZdZdgZdgZdS)ÚEmptyQueryStringTestz>User-agent: * Allow: /some/path? Disallow: /another/path? z /some/path?z/another/path?N)r"r#r$r rrrrrrr?åsr?c@s0eZdZdZej dd¡ZdZddgZ dgZ dS) ÚDefaultEntryTestzOUser-agent: * Crawl-delay: 1 Request-rate: 3/15 Disallow: /cyberworld/map/ r4éér&z /test.htmlz/cyberworld/map/index.htmlN) r"r#r$r r r r-r,r+rrrrrrr@ðs r@c@seZdZdZdZdd„ZdS)ÚStringFormattingTestzÆUser-agent: * Crawl-delay: 1 Request-rate: 3/15 Disallow: /cyberworld/map/ # This is an infinite virtual URL space # Cybermapper knows where to go. User-agent: cybermapper Disallow: /some/path zzUser-agent: cybermapper Disallow: /some/path User-agent: * Crawl-delay: 1 Request-rate: 3/15 Disallow: /cyberworld/map/ cCs| t|jƒ|j¡dS)N)r*ÚstrrÚexpected_output)rrrrÚtest_string_formattingsz+StringFormattingTest.test_string_formattingN)r"r#r$r rErFrrrrrCýs  rCc@seZdZdd„Zdd„ZdS)Ú RobotHandlercCs| dd¡dS)Ni“zForbidden access)Z send_error)rrrrÚdo_GETszRobotHandler.do_GETcGsdS)Nr)rÚformatÚargsrrrÚ log_messageszRobotHandler.log_messageN)r"r#r$rHrKrrrrrGsrGc@s*eZdZdd„Zdd„Zejdd„ƒZdS)ÚPasswordProtectedSiteTestCasecCsBttjdftƒ|_tjd|jjddid|_d|j_ |j  ¡dS)NrzHTTPServer servingZ poll_intervalg{®Gáz„?)ÚnameÚtargetÚkwargsT) rrÚHOSTrGÚserverÚ threadingZThreadZ serve_foreverÚtZdaemonÚstart)rrrrr#sz#PasswordProtectedSiteTestCase.setUpcCs"|j ¡|j ¡|j ¡dS)N)rQZshutdownrSÚjoinZ server_close)rrrrÚtearDown0s  z&PasswordProtectedSiteTestCase.tearDowncCs\|jj}dtjdt|dƒ}|d}tj ¡}| |¡|  ¡|  |  d|¡¡dS)Nzhttp://ú:rBz /robots.txtÚ*) rQZserver_addressrrPrDr r r Zset_urlÚreadr r)rZaddrrZ robots_urlrrrrÚtestPasswordProtectedSite5s  z7PasswordProtectedSiteTestCase.testPasswordProtectedSiteN)r"r#r$rrVrZ reap_threadsrZrrrrrL!s rLc@sFeZdZdZd e¡Zedd„ƒZdd„Zdd„Z d d „Z d d „Z d S)ÚNetworkTestCasezhttp://www.pythontest.net/z{}elsewhere/robots.txtc Cs@t d¡t |j¡ tj |j¡|_|j  ¡WdQRXdS)NZnetwork) rZrequiresZtransient_internetÚbase_urlr r r r rrY)ÚclsrrrÚ setUpClassEs zNetworkTestCase.setUpClasscCs$d |j|tj |¡dsdnd¡S)Nz{}{}{}rBr&r)rIr\ÚosÚpathÚsplitext)rr`rrrrLszNetworkTestCase.urlcCsV| |jj¡| |jj¡| |j ¡d¡| |j d¡¡| |j d¡¡dS)NrrX)r rÚ disallow_allÚ allow_allZ assertGreaterÚmtimer+r,)rrrrÚ test_basicQs zNetworkTestCase.test_basiccCs˜| |j d| d¡¡¡| |j d|j¡¡| |j d| d¡¡¡| |j d| d¡¡¡| |j d| d¡¡¡| |j d|j¡¡dS)NrXZ elsewhereZNutchZbrianZwebstats)rrrrr r\)rrrrÚtest_can_fetchXs zNetworkTestCase.test_can_fetchcCsftj | d¡¡}| ¡| |j¡| |j¡|  |  ¡d¡|  |  d¡¡|  |  d¡¡dS)Nz i-robot.txtrrX)r r r rrYrrcr rbr*rdZ assertIsNoner+r,)rrrrrÚ test_read_404`s  zNetworkTestCase.test_read_404N) r"r#r$r\rIr Ú classmethodr^rrerfrgrrrrr[@s  r[Ú__main__)$rr_rRZunittestZurllib.robotparserr ZtestrZ http.serverrrrZTestCaser%r'r(r)r/r0r5r6r7r8r:r;r<r=r>r?r@rCrGrLr[r"ÚmainrrrrÚs<           )