Fixed Spiders labic#24

Govgo govmg
ligiaiv · Oct 3, 2017 · 84e8ee7 · 84e8ee7
1 parent 8b1e291
commit 84e8ee7
Show file tree

Hide file tree

Showing 3 changed files with 19 additions and 4 deletions.
diff --git a/ze/processors/html.py b/ze/processors/html.py
@@ -813,6 +813,16 @@ def __call__(self, value, loader_context):
                 logger.error('Failed to replace "%s" selector from %s:\n%s',
                     selector, spider_name, e)
 
+        if spider_name is 'govmg':
+            try:
+                selector = 'a'
+                for el in html.select(selector):
+                    el.replace_with(el.get_text())
+
+            except Exception as e:
+                logger.error('Failed to replace "%s" selector from %s:\n%s',
+                    selector, spider_name, e)                
+
         # if spider_name is 'govpa':
         #     try:
         #         selector = '.texto'
@@ -1165,6 +1175,7 @@ def __call__(self, value, loader_context):
                     '#noticia_vinculadas',
                     # 'meta',
                     'figure meta',
+                    '.full-sharing',
                     'noframes',
                     'noscript',
                     'object',

diff --git a/ze/spiders/govgo.py b/ze/spiders/govgo.py
@@ -27,7 +27,8 @@ class GovernoGoiasSpider(ZeSpider):
                         "meta[name=description]::attr(content)",
                         'meta[property="og:image"]::attr(content)',
                         '[itemprop="image"]::attr(src)',
-                        '#texto_content img::attr("src")'
+                        '#texto_content img::attr("src")',
+                        '.content figure img::attr("src")'
                     ]
                 }
             },

diff --git a/ze/spiders/govmg.py b/ze/spiders/govmg.py
@@ -16,7 +16,8 @@ class GovernoMinasGeraisSpider(ZeSpider):
                         "meta[name=title]::attr(content)",
                         '[itemprop=headline]::text',
                         '.title-post::text',
-                        '.entry-title::text'
+                        '.entry-title::text',
+                        '.title::text'
                     ]
                 }
             },
@@ -57,7 +58,8 @@ class GovernoMinasGeraisSpider(ZeSpider):
                     "css": [
                         '[itemprop=datePublished]::attr(content)',
                         '.entry-date::text',
-                        '.text-date::attr(datetime)'
+                        '.text-date::attr(datetime)',
+                        '.date::text'
                     ]
                 }
             },
@@ -74,7 +76,8 @@ class GovernoMinasGeraisSpider(ZeSpider):
                         '[itemprop=articleBody]',
                         '.noticia',
                         'div.clear',
-                        'article main'
+                        'article main',
+                        '.content'
                     ]
                 }
             },