ó
|
R\¬Qc @ sl d d l Z d d l Z d d l Z d d l m Z d d l m Z m Z d e j f d YZ e j
|
Z d d Z d Z d d
|
g Z d d d g Z e d e e e d Z d d
|
dB g Z d g Z e d e e e d Z g Z d d d g Z e d e e e d Z g Z d d d d d d d g Z e d e e e d e d e e e d d! Z d g Z d d d d d" d# g Z e d$ e e e d% Z d g Z g Z e d& e e e d' Z d( g Z d) g Z e d* e e e d+ d, d- Z g Z d. g Z e d/ e e e d+ d, g Z d. g Z e d0 e e e d+ d1 d2 Z g Z d. g Z e d3 e e e d+ d, d. g Z g Z e d4 e e e d+ d1 d' Z d( g Z d) g Z e d5 e e e d+ d6 d7 Z d8 g Z d9 g Z e d: e e e d; Z d< g Z d8 g Z e d= e e e d> e j f d? YZ d@ Z e dA k rhd e _ e n d S(C iÿÿÿÿN( t test_support( t urlopent HTTPErrort RobotTestCasec B s# e Z d Z d Z d Z RS( c C sg t j j | | r, d | | f | _ n d | | f | _ | | _ | | _ | | _ | | _ d S( Ns RobotTest(%d, good, %s)s RobotTest(%d, bad, %s)( t unittestt TestCaset __init__t strt parsert urlt goodt agent( t selft indexR R R
|
R ( ( s] /tmp/ndk-User/buildhost/install/prebuilt/darwin-x86_64/lib/python2.7/test/test_robotparser.pyR s c C s~ t | j t r$ | j \ } } n | j } | j } | j r^ | j | j j | | n | j | j j | | d S( N( t
|
isinstanceR t tupleR R
|
t
|
assertTrueR t can_fetcht assertFalse( R R R ( ( s] /tmp/ndk-User/buildhost/install/prebuilt/darwin-x86_64/lib/python2.7/test/test_robotparser.pyt runTest s c C s | j S( N( R ( R ( ( s] /tmp/ndk-User/buildhost/install/prebuilt/darwin-x86_64/lib/python2.7/test/test_robotparser.pyt __str__ s ( t __name__t
|
__module__R R R ( ( ( s] /tmp/ndk-User/buildhost/install/prebuilt/darwin-x86_64/lib/python2.7/test/test_robotparser.pyR s t test_robotparserc C s t j | j } t j } | j | x- | D]% } t j t | | | d | q5 Wx- | D]% } t j t | | | d | qe Wd S( Ni i ( t StringIOt readlinest robotparsert RobotFileParsert parset testst addTestR ( R t
|
robots_txtt good_urlst bad_urlsR t linesR R ( ( s] /tmp/ndk-User/buildhost/install/prebuilt/darwin-x86_64/lib/python2.7/test/test_robotparser.pyt RobotTest! s # s
|
User-agent: *
|
Disallow: /cyberworld/map/ # This is an infinite virtual URL space
|
Disallow: /tmp/ # these will soon disappear
|
Disallow: /foo.html
|
t /s
|
/test.htmls /cyberworld/map/index.htmls /tmp/xxxs /foo.htmli sÁ
|
# robots.txt for http://www.example.com/
|
|
User-agent: *
|
Disallow: /cyberworld/map/ # This is an infinite virtual URL space
|
|
# Cybermapper knows where to go.
|
User-agent: cybermapper
|
Disallow:
|
|
t cybermapperi s%
|
# go away
|
User-agent: *
|
Disallow: /
|
s /tmp/i sm
|
User-agent: figtree
|
Disallow: /tmp
|
Disallow: /a%3cd.html
|
Disallow: /a%2fb.html
|
Disallow: /%7ejoe/index.html
|
s /tmps /tmp.htmls /tmp/a.htmls /a%3cd.htmls /a%3Cd.htmls /a%2fb.htmls /~joe/index.htmli t figtreei s FigTree Robot libwww-perl/5.04sf
|
User-agent: *
|
Disallow: /tmp/
|
Disallow: /a%3Cd.html
|
Disallow: /a/b.html
|
Disallow: /%7ejoe/index.html
|
s /a/b.htmls /%7Ejoe/index.htmli s
|
User-Agent: *
|
Disallow: /.
|
i sG
|
User-agent: Googlebot
|
Allow: /folder1/myfile.html
|
Disallow: /folder1/
|
s /folder1/myfile.htmls /folder1/anotherfile.htmli R t GooglebotsJ
|
User-agent: Googlebot
|
Disallow: /
|
|
User-agent: Googlebot-Mobile
|
Allow: /
|
s /something.jpgi i
|
s Googlebot-MobilesJ
|
User-agent: Googlebot-Mobile
|
Allow: /
|
|
User-agent: Googlebot
|
Disallow: /
|
i i i t googlebots/
|
User-agent: *
|
Disallow: /some/path?name=value
|
s
|
/some/paths /some/path?name=valuei sK
|
User-agent: *
|
Disallow: /some/path
|
|
User-agent: *
|
Disallow: /another/path
|
s /another/pathi t NetworkTestCasec B s e Z d Z d Z RS( c C s t j d t j d ä d } | d } y t | WnE t k
|
r } | j d d h k r | j d | | j f q n X| j d | t j } | j | y | j
|
Wn" t k
|
rß | j d | n X| j | j d
|
| t Wd QXd S( Nt networks mueblesmoraleda.coms http://mueblesmoraleda.coms /robots.txti i s0 %r should return a 401 or 403 HTTP error, not %rs5 %r should return a 401 or 403 HTTP error, not succeeds %s is unavailablet *( R t requirest transient_internetR R t codet skipTestR R t set_urlt readt IOErrort assertEqualR t False( R R t
|
robots_urlt eR ( ( s] /tmp/ndk-User/buildhost/install/prebuilt/darwin-x86_64/lib/python2.7/test/test_robotparser.pyt testPasswordProtectedSiteê s*
|
c C sY t j d t j d 7 t j d } | j | j | j d d Wd QXd S( NR* s www.python.orgs http://www.python.org/robots.txtR+ ( R R, R- R R R1 R R ( R R ( ( s] /tmp/ndk-User/buildhost/install/prebuilt/darwin-x86_64/lib/python2.7/test/test_robotparser.pyt testPythonOrg s
|
( R R R7 R8 ( ( ( s] /tmp/ndk-User/buildhost/install/prebuilt/darwin-x86_64/lib/python2.7/test/test_robotparser.pyR) è s c C s t j t t j t d S( N( R t run_unittestR R) ( ( ( s] /tmp/ndk-User/buildhost/install/prebuilt/darwin-x86_64/lib/python2.7/test/test_robotparser.pyt test_main s t __main__( R% s /cyberworld/map/index.html( R R R t testR t urllib2R R R R t TestSuiteR R# t docR
|
t badR) R: R t verbose( ( ( s] /tmp/ndk-User/buildhost/install/prebuilt/darwin-x86_64/lib/python2.7/test/test_robotparser.pyt <module> s $
|
&
|
