Support blank lines between group directives

Author: Martin Skovvang Petersen

robotstxtparser/robotstxtparser.py

@@ -556,12 +556,7 @@ def parse(self, s):
                 line = line.strip()
 
                 if not line:
-                    # An empty line indicates the end of a ruleset.
-                    if current_ruleset and current_ruleset.is_not_empty():
-                        self.__rulesets.append(current_ruleset)
-
-                    current_ruleset = None
-                    previous_line_was_a_user_agent = False
+                    pass
                 else:
                     # Each non-empty line falls into one of six categories:
                     # 1) User-agent: blah blah blah

tests/test_robots.py

@@ -77,7 +77,7 @@ def test_skip_malformed_line(self):
         self.assertTrue(rp.is_allowed('agent', 'http://example.org/no/colon/in/this/line'))
 
     def test_utf8_bom(self):
-        """If there's a utf-8 BOM, we should parse it as such"""
+        """If there's a UTF-8 BOM, we should parse it as such"""
         rp = robotstxtparser.RobotExclusionRulesParser()
         rp.parse(codecs.BOM_UTF8 + b'''
             User-Agent: agent
@@ -163,3 +163,21 @@ def test_rfc_example(self):
         self.assertFalse(rp.is_allowed('anything', 'http://example.org/%7Ejim/jim.html'))
         self.assertTrue(rp.is_allowed('anything', 'http://example.org/%7Emak/mak.html'))
 
+    def test_support_grouping_blank_lines(self):
+        """Make sure blank lines are ignored"""
+        rp = robotstxtparser.RobotExclusionRulesParser()
+        rp.parse('''
+            User-Agent: agent
+            
+            Allow: /path
+            
+            Disallow: /tmp
+            
+            User-Agent: other
+            
+            Disallow: /path
+        ''')
+
+        self.assertTrue(rp.is_allowed('agent', 'http://example.org/path'))
+        self.assertFalse(rp.is_allowed('agent', 'http://example.org/tmp'))
+        self.assertFalse(rp.is_allowed('other', 'http://example.org/path'))