From 9d1a5325741c6377ec1943ee8dab9169a5a24796 Mon Sep 17 00:00:00 2001 From: Padraic Fanning Date: Sat, 17 Apr 2021 15:49:36 -0400 Subject: [PATCH 1/3] Add test_robotparser from CPython 3.8 --- Lib/test/test_robotparser.py | 382 +++++++++++++++++++++++++++++++++++ 1 file changed, 382 insertions(+) create mode 100644 Lib/test/test_robotparser.py diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py new file mode 100644 index 000000000..77cd7c4d2 --- /dev/null +++ b/Lib/test/test_robotparser.py @@ -0,0 +1,382 @@ +import io +import os +import threading +import unittest +import urllib.robotparser +from test import support +from http.server import BaseHTTPRequestHandler, HTTPServer + + +class BaseRobotTest: + robots_txt = '' + agent = 'test_robotparser' + good = [] + bad = [] + site_maps = None + + def setUp(self): + lines = io.StringIO(self.robots_txt).readlines() + self.parser = urllib.robotparser.RobotFileParser() + self.parser.parse(lines) + + def get_agent_and_url(self, url): + if isinstance(url, tuple): + agent, url = url + return agent, url + return self.agent, url + + def test_good_urls(self): + for url in self.good: + agent, url = self.get_agent_and_url(url) + with self.subTest(url=url, agent=agent): + self.assertTrue(self.parser.can_fetch(agent, url)) + + def test_bad_urls(self): + for url in self.bad: + agent, url = self.get_agent_and_url(url) + with self.subTest(url=url, agent=agent): + self.assertFalse(self.parser.can_fetch(agent, url)) + + def test_site_maps(self): + self.assertEqual(self.parser.site_maps(), self.site_maps) + + +class UserAgentWildcardTest(BaseRobotTest, unittest.TestCase): + robots_txt = """\ +User-agent: * +Disallow: /cyberworld/map/ # This is an infinite virtual URL space +Disallow: /tmp/ # these will soon disappear +Disallow: /foo.html + """ + good = ['/', '/test.html'] + bad = ['/cyberworld/map/index.html', '/tmp/xxx', '/foo.html'] + + +class CrawlDelayAndCustomAgentTest(BaseRobotTest, unittest.TestCase): + robots_txt = """\ +# robots.txt for http://www.example.com/ + +User-agent: * +Crawl-delay: 1 +Request-rate: 3/15 +Disallow: /cyberworld/map/ # This is an infinite virtual URL space + +# Cybermapper knows where to go. +User-agent: cybermapper +Disallow: + """ + good = ['/', '/test.html', ('cybermapper', '/cyberworld/map/index.html')] + bad = ['/cyberworld/map/index.html'] + + +class SitemapTest(BaseRobotTest, unittest.TestCase): + robots_txt = """\ +# robots.txt for http://www.example.com/ + +User-agent: * +Sitemap: http://www.gstatic.com/s2/sitemaps/profiles-sitemap.xml +Sitemap: http://www.google.com/hostednews/sitemap_index.xml +Request-rate: 3/15 +Disallow: /cyberworld/map/ # This is an infinite virtual URL space + + """ + good = ['/', '/test.html'] + bad = ['/cyberworld/map/index.html'] + site_maps = ['http://www.gstatic.com/s2/sitemaps/profiles-sitemap.xml', + 'http://www.google.com/hostednews/sitemap_index.xml'] + + +class RejectAllRobotsTest(BaseRobotTest, unittest.TestCase): + robots_txt = """\ +# go away +User-agent: * +Disallow: / + """ + good = [] + bad = ['/cyberworld/map/index.html', '/', '/tmp/'] + + +class BaseRequestRateTest(BaseRobotTest): + request_rate = None + crawl_delay = None + + def test_request_rate(self): + parser = self.parser + for url in self.good + self.bad: + agent, url = self.get_agent_and_url(url) + with self.subTest(url=url, agent=agent): + self.assertEqual(parser.crawl_delay(agent), self.crawl_delay) + + parsed_request_rate = parser.request_rate(agent) + self.assertEqual(parsed_request_rate, self.request_rate) + if self.request_rate is not None: + self.assertIsInstance( + parsed_request_rate, + urllib.robotparser.RequestRate + ) + self.assertEqual( + parsed_request_rate.requests, + self.request_rate.requests + ) + self.assertEqual( + parsed_request_rate.seconds, + self.request_rate.seconds + ) + + +class EmptyFileTest(BaseRequestRateTest, unittest.TestCase): + robots_txt = '' + good = ['/foo'] + + +class CrawlDelayAndRequestRateTest(BaseRequestRateTest, unittest.TestCase): + robots_txt = """\ +User-agent: figtree +Crawl-delay: 3 +Request-rate: 9/30 +Disallow: /tmp +Disallow: /a%3cd.html +Disallow: /a%2fb.html +Disallow: /%7ejoe/index.html + """ + agent = 'figtree' + request_rate = urllib.robotparser.RequestRate(9, 30) + crawl_delay = 3 + good = [('figtree', '/foo.html')] + bad = ['/tmp', '/tmp.html', '/tmp/a.html', '/a%3cd.html', '/a%3Cd.html', + '/a%2fb.html', '/~joe/index.html'] + + +class DifferentAgentTest(CrawlDelayAndRequestRateTest): + agent = 'FigTree Robot libwww-perl/5.04' + + +class InvalidRequestRateTest(BaseRobotTest, unittest.TestCase): + robots_txt = """\ +User-agent: * +Disallow: /tmp/ +Disallow: /a%3Cd.html +Disallow: /a/b.html +Disallow: /%7ejoe/index.html +Crawl-delay: 3 +Request-rate: 9/banana + """ + good = ['/tmp'] + bad = ['/tmp/', '/tmp/a.html', '/a%3cd.html', '/a%3Cd.html', '/a/b.html', + '/%7Ejoe/index.html'] + crawl_delay = 3 + + +class InvalidCrawlDelayTest(BaseRobotTest, unittest.TestCase): + # From bug report #523041 + robots_txt = """\ +User-Agent: * +Disallow: /. +Crawl-delay: pears + """ + good = ['/foo.html'] + # bug report says "/" should be denied, but that is not in the RFC + bad = [] + + +class AnotherInvalidRequestRateTest(BaseRobotTest, unittest.TestCase): + # also test that Allow and Diasallow works well with each other + robots_txt = """\ +User-agent: Googlebot +Allow: /folder1/myfile.html +Disallow: /folder1/ +Request-rate: whale/banana + """ + agent = 'Googlebot' + good = ['/folder1/myfile.html'] + bad = ['/folder1/anotherfile.html'] + + +class UserAgentOrderingTest(BaseRobotTest, unittest.TestCase): + # the order of User-agent should be correct. note + # that this file is incorrect because "Googlebot" is a + # substring of "Googlebot-Mobile" + robots_txt = """\ +User-agent: Googlebot +Disallow: / + +User-agent: Googlebot-Mobile +Allow: / + """ + agent = 'Googlebot' + bad = ['/something.jpg'] + + +class UserAgentGoogleMobileTest(UserAgentOrderingTest): + agent = 'Googlebot-Mobile' + + +class GoogleURLOrderingTest(BaseRobotTest, unittest.TestCase): + # Google also got the order wrong. You need + # to specify the URLs from more specific to more general + robots_txt = """\ +User-agent: Googlebot +Allow: /folder1/myfile.html +Disallow: /folder1/ + """ + agent = 'googlebot' + good = ['/folder1/myfile.html'] + bad = ['/folder1/anotherfile.html'] + + +class DisallowQueryStringTest(BaseRobotTest, unittest.TestCase): + # see issue #6325 for details + robots_txt = """\ +User-agent: * +Disallow: /some/path?name=value + """ + good = ['/some/path'] + bad = ['/some/path?name=value'] + + +class UseFirstUserAgentWildcardTest(BaseRobotTest, unittest.TestCase): + # obey first * entry (#4108) + robots_txt = """\ +User-agent: * +Disallow: /some/path + +User-agent: * +Disallow: /another/path + """ + good = ['/another/path'] + bad = ['/some/path'] + + +class EmptyQueryStringTest(BaseRobotTest, unittest.TestCase): + # normalize the URL first (#17403) + robots_txt = """\ +User-agent: * +Allow: /some/path? +Disallow: /another/path? + """ + good = ['/some/path?'] + bad = ['/another/path?'] + + +class DefaultEntryTest(BaseRequestRateTest, unittest.TestCase): + robots_txt = """\ +User-agent: * +Crawl-delay: 1 +Request-rate: 3/15 +Disallow: /cyberworld/map/ + """ + request_rate = urllib.robotparser.RequestRate(3, 15) + crawl_delay = 1 + good = ['/', '/test.html'] + bad = ['/cyberworld/map/index.html'] + + +class StringFormattingTest(BaseRobotTest, unittest.TestCase): + robots_txt = """\ +User-agent: * +Crawl-delay: 1 +Request-rate: 3/15 +Disallow: /cyberworld/map/ # This is an infinite virtual URL space + +# Cybermapper knows where to go. +User-agent: cybermapper +Disallow: /some/path + """ + + expected_output = """\ +User-agent: cybermapper +Disallow: /some/path + +User-agent: * +Crawl-delay: 1 +Request-rate: 3/15 +Disallow: /cyberworld/map/\ +""" + + def test_string_formatting(self): + self.assertEqual(str(self.parser), self.expected_output) + + +class RobotHandler(BaseHTTPRequestHandler): + + def do_GET(self): + self.send_error(403, "Forbidden access") + + def log_message(self, format, *args): + pass + + +class PasswordProtectedSiteTestCase(unittest.TestCase): + + def setUp(self): + self.server = HTTPServer((support.HOST, 0), RobotHandler) + + self.t = threading.Thread( + name='HTTPServer serving', + target=self.server.serve_forever, + # Short poll interval to make the test finish quickly. + # Time between requests is short enough that we won't wake + # up spuriously too many times. + kwargs={'poll_interval':0.01}) + self.t.daemon = True # In case this function raises. + self.t.start() + + def tearDown(self): + self.server.shutdown() + self.t.join() + self.server.server_close() + + @support.reap_threads + def testPasswordProtectedSite(self): + addr = self.server.server_address + url = 'http://' + support.HOST + ':' + str(addr[1]) + robots_url = url + "/robots.txt" + parser = urllib.robotparser.RobotFileParser() + parser.set_url(url) + parser.read() + self.assertFalse(parser.can_fetch("*", robots_url)) + + +class NetworkTestCase(unittest.TestCase): + + base_url = 'http://www.pythontest.net/' + robots_txt = '{}elsewhere/robots.txt'.format(base_url) + + @classmethod + def setUpClass(cls): + support.requires('network') + with support.transient_internet(cls.base_url): + cls.parser = urllib.robotparser.RobotFileParser(cls.robots_txt) + cls.parser.read() + + def url(self, path): + return '{}{}{}'.format( + self.base_url, path, '/' if not os.path.splitext(path)[1] else '' + ) + + def test_basic(self): + self.assertFalse(self.parser.disallow_all) + self.assertFalse(self.parser.allow_all) + self.assertGreater(self.parser.mtime(), 0) + self.assertFalse(self.parser.crawl_delay('*')) + self.assertFalse(self.parser.request_rate('*')) + + def test_can_fetch(self): + self.assertTrue(self.parser.can_fetch('*', self.url('elsewhere'))) + self.assertFalse(self.parser.can_fetch('Nutch', self.base_url)) + self.assertFalse(self.parser.can_fetch('Nutch', self.url('brian'))) + self.assertFalse(self.parser.can_fetch('Nutch', self.url('webstats'))) + self.assertFalse(self.parser.can_fetch('*', self.url('webstats'))) + self.assertTrue(self.parser.can_fetch('*', self.base_url)) + + def test_read_404(self): + parser = urllib.robotparser.RobotFileParser(self.url('i-robot.txt')) + parser.read() + self.assertTrue(parser.allow_all) + self.assertFalse(parser.disallow_all) + self.assertEqual(parser.mtime(), 0) + self.assertIsNone(parser.crawl_delay('*')) + self.assertIsNone(parser.request_rate('*')) + +if __name__=='__main__': + unittest.main() From fa80d67e7a7bea6d26354c71fda26f16408fbcfd Mon Sep 17 00:00:00 2001 From: Padraic Fanning Date: Sat, 17 Apr 2021 15:52:41 -0400 Subject: [PATCH 2/3] Skip erroring test class --- Lib/test/test_robotparser.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py index 77cd7c4d2..5989d46d9 100644 --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -337,6 +337,7 @@ class PasswordProtectedSiteTestCase(unittest.TestCase): self.assertFalse(parser.can_fetch("*", robots_url)) +@unittest.skip("TODO: RUSTPYTHON, AttributeError: module 'socket' has no attribute 'setdefaulttimeout'") class NetworkTestCase(unittest.TestCase): base_url = 'http://www.pythontest.net/' From 9f7181b6a63f8547ddedd1324b3f075d3fa8772a Mon Sep 17 00:00:00 2001 From: Padraic Fanning Date: Sat, 1 May 2021 14:59:49 -0400 Subject: [PATCH 3/3] Exclude test from macOS test matrix --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 6b5ce0a75..de407ec2b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -95,7 +95,7 @@ jobs: name: run cpython tests (macOS lightweight) run: target/release/rustpython -m test -v -x - test_http_cookiejar + test_http_cookiejar test_robotparser - if: runner.os == 'Windows' name: run cpython tests (windows partial - fixme) run: