ó ‹wÛjc@s"ddlZddlZddlZddlmZddlmZmZddlZyddl Z Wne k rdZ nXdd#d„ƒYZ de ej fd„ƒYZde ej fd „ƒYZd e ej fd „ƒYZd efd „ƒYZde ej fd„ƒYZde ej fd„ƒYZde ej fd„ƒYZde ej fd„ƒYZde ej fd„ƒYZde ej fd„ƒYZdefd„ƒYZeje dƒdej fd„ƒYƒZdej fd „ƒYZd!„Zed"kreƒndS($iÿÿÿÿN(tsupport(tBaseHTTPRequestHandlert HTTPServert BaseRobotTestcBsDeZdZdZgZgZd„Zd„Zd„Zd„Z RS(tttest_robotparsercCs;tj|jƒjƒ}tjƒ|_|jj|ƒdS(N(tStringIOt robots_txtt readlinest robotparsertRobotFileParsertparsertparse(tselftlines((s‹/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_robotparser.pytsetUpscCs2t|tƒr%|\}}||fS|j|fS(N(t isinstancettupletagent(R turlR((s‹/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_robotparser.pytget_agent_and_urls  cCsIxB|jD]7}|j|ƒ\}}|j|jj||ƒƒq WdS(N(tgoodRt assertTrueR t can_fetch(R RR((s‹/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_robotparser.pyttest_good_urlsscCsIxB|jD]7}|j|ƒ\}}|j|jj||ƒƒq WdS(N(tbadRt assertFalseR R(R RR((s‹/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_robotparser.pyt test_bad_urls#s( t__name__t __module__RRRRRRRR(((s‹/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_robotparser.pyR s   tUserAgentWildcardTestcBs)eZdZddgZdddgZRS(s•User-agent: * Disallow: /cyberworld/map/ # This is an infinite virtual URL space Disallow: /tmp/ # these will soon disappear Disallow: /foo.html t/s /test.htmls/cyberworld/map/index.htmls/tmp/xxxs /foo.html(RRRRR(((s‹/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_robotparser.pyR)s tRejectAllRobotsTestcBs#eZdZgZdddgZRS(s(# go away User-agent: * Disallow: / s/cyberworld/map/index.htmlRs/tmp/(RRRRR(((s‹/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_robotparser.pyR 4stUserAgentOrderingTestcBseZdZdZdgZRS(sMUser-agent: Googlebot Disallow: / User-agent: Googlebot-Mobile Allow: / t Googlebots/something.jpg(RRRRR(((s‹/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_robotparser.pyR!>s tUserAgentGoogleMobileTestcBseZdZRS(sGooglebot-Mobile(RRR(((s‹/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_robotparser.pyR#MstGoogleURLOrderingTestcBs&eZdZdZdgZdgZRS(sJUser-agent: Googlebot Allow: /folder1/myfile.html Disallow: /folder1/ t googlebots/folder1/myfile.htmls/folder1/anotherfile.html(RRRRRR(((s‹/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_robotparser.pyR$Qs tDisallowQueryStringTestcBs eZdZdgZdgZRS(s2User-agent: * Disallow: /some/path?name=value s /some/paths/some/path?name=value(RRRRR(((s‹/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_robotparser.pyR&^s tUseFirstUserAgentWildcardTestcBs eZdZdgZdgZRS(sNUser-agent: * Disallow: /some/path User-agent: * Disallow: /another/path s /another/paths /some/path(RRRRR(((s‹/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_robotparser.pyR'hs tEmptyQueryStringTestcBs eZdZdgZdgZRS(s>User-agent: * Allow: /some/path? Disallow: /another/path? s /some/path?s/another/path?(RRRRR(((s‹/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_robotparser.pyR(us tDefaultEntryTestcBs#eZdZddgZdgZRS(sOUser-agent: * Crawl-delay: 1 Request-rate: 3/15 Disallow: /cyberworld/map/ Rs /test.htmls/cyberworld/map/index.html(RRRRR(((s‹/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_robotparser.pyR)€s tStringFormattingTestcBseZdZdZd„ZRS(sÆUser-agent: * Crawl-delay: 1 Request-rate: 3/15 Disallow: /cyberworld/map/ # This is an infinite virtual URL space # Cybermapper knows where to go. User-agent: cybermapper Disallow: /some/path sXUser-agent: cybermapper Disallow: /some/path User-agent: * Disallow: /cyberworld/map/ cCs |jt|jƒ|jƒdS(N(t assertEqualtstrR texpected_output(R ((s‹/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_robotparser.pyttest_string_formatting s(RRRR-R.(((s‹/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_robotparser.pyR*‹s  t RobotHandlercBseZd„Zd„ZRS(cCs|jddƒdS(Ni“sForbidden access(t send_error(R ((s‹/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_robotparser.pytdo_GET¦scGsdS(N((R tformattargs((s‹/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_robotparser.pyt log_message©s(RRR1R4(((s‹/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_robotparser.pyR/¤s s threading required for this testtPasswordProtectedSiteTestCasecBs,eZd„Zd„Zejd„ƒZRS(c Csfttjdftƒ|_tjddd|jjdidd6ƒ|_t |j_ |jj ƒdS(NitnamesHTTPServer servingttargettkwargsg{®Gáz„?t poll_interval( RRtHOSTR/tservert threadingtThreadt serve_forevertttTruetdaemontstart(R ((s‹/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_robotparser.pyR°s   cCs+|jjƒ|jjƒ|jjƒdS(N(R;tshutdownR?tjoint server_close(R ((s‹/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_robotparser.pyttearDown½s  cCsu|jj}dtjdt|dƒ}|d}tjƒ}|j|ƒ|jƒ|j |j d|ƒƒdS(Nshttp://t:is /robots.txtt*( R;tserver_addressRR:R,R R tset_urltreadRR(R taddrRt robots_urlR ((s‹/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_robotparser.pyttestPasswordProtectedSiteÂs     (RRRRFRt reap_threadsRN(((s‹/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_robotparser.pyR5­s tNetworkTestCasecBsPeZdZdjeƒZed„ƒZd„Zd„Zd„Z d„Z RS(shttp://www.pythontest.net/s{}elsewhere/robots.txtcCsLtjdƒtj|jƒ'tj|jƒ|_|jjƒWdQXdS(Ntnetwork( Rtrequiresttransient_internettbase_urlR R RR RK(tcls((s‹/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_robotparser.pyt setUpClassÒs cCs2dj|j|tjj|ƒds+dndƒS(Ns{}{}{}iRR(R2RTtostpathtsplitext(R RX((s‹/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_robotparser.pyRÙscCsC|j|jjƒ|j|jjƒ|j|jjƒdƒdS(Ni(RR t disallow_allt allow_allt assertGreatertmtime(R ((s‹/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_robotparser.pyt test_basicÞscCsÖ|j|jjd|jdƒƒƒ|j|jjd|jƒƒ|j|jjd|jdƒƒƒ|j|jjd|jdƒƒƒ|j|jjd|jdƒƒƒ|j|jjd|jƒƒdS(NRHt elsewheretNutchtbriantwebstats(RR RRRRT(R ((s‹/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_robotparser.pyttest_can_fetchãs %%%%cCs\tj|jdƒƒ}|jƒ|j|jƒ|j|jƒ|j|j ƒdƒdS(Ns i-robot.txti( R R RRKRR[RRZR+R](R R ((s‹/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_robotparser.pyt test_read_404ës  ( RRRTR2Rt classmethodRVRR^RcRd(((s‹/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_robotparser.pyRPÍs   c Cs2tjtttttttt t t t t ƒ dS(N(Rt run_unittestRR R!R#R$R&R'R(R)R*R5RP(((s‹/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_robotparser.pyt test_mainóst__main__((RWR tunittestttestRtBaseHTTPServerRRRR<t ImportErrortNoneRtTestCaseRR R!R#R$R&R'R(R)R*R/t skipUnlessR5RPRgR(((s‹/home/sim/yocto/scarthgap/build/tmp/work/x86_64-linux/python-native/2.7.18/recipe-sysroot-native/usr/lib/python2.7/test/test_robotparser.pyts6              &