From 90e6f022ad27a70182aafe841e9d552ca6a8b19d Mon Sep 17 00:00:00 2001 From: DUSHYANT SINGH Date: Fri, 24 Feb 2017 13:04:02 +0530 Subject: [PATCH 1/4] comments added to explain the logic --- Jansatta/jansatta1.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Jansatta/jansatta1.py b/Jansatta/jansatta1.py index 51c598a..b414789 100644 --- a/Jansatta/jansatta1.py +++ b/Jansatta/jansatta1.py @@ -2,6 +2,7 @@ from lxml import html import time +# List to store all article-urls from each page urls = [] for i in xrange(315) : @@ -10,8 +11,10 @@ tree = html.fromstring(page.content) article_urls = tree.xpath('//div[@class="newslistbx"]/h3/a/@href') for j in article_urls: + # Each article url is appended in a global list urls[] urls.append(j) - + +#Now request is being send to each article page and required data is scraped for k in urls: page = requests.get(k) tree = html.fromstring(page.content) @@ -20,8 +23,9 @@ page_url = k date_time = tree.xpath('//meta [@itemprop="dateModified"]/@content') keyword = tree.xpath('//meta [@name="keywords"]/@content ') - - #collection.insert({"url":page_url, "title":title}) + + # Data will be inserted in mongodb + # collection.insert({"url":page_url, "title":title}) print(title) time.delay(5) From d793a0f92285f29662f1ec115855a2305b135964 Mon Sep 17 00:00:00 2001 From: DUSHYANT SINGH Date: Fri, 24 Feb 2017 13:06:54 +0530 Subject: [PATCH 2/4] iteration bug fixed article urls were starting from 0 page which does not existed, made it to start from 1 --- Jansatta/jansatta1.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jansatta/jansatta1.py b/Jansatta/jansatta1.py index b414789..692159b 100644 --- a/Jansatta/jansatta1.py +++ b/Jansatta/jansatta1.py @@ -5,7 +5,7 @@ # List to store all article-urls from each page urls = [] -for i in xrange(315) : +for i in xrange(1,315) : page = requests.get("http://www.jansatta.com/latest-news/page/"+str(i)+"/?s=news") time.delay(5) tree = html.fromstring(page.content) From 3a051c79e913ec22e84870b1d4eeac4f95d4d040 Mon Sep 17 00:00:00 2001 From: DUSHYANT SINGH Date: Sat, 4 Mar 2017 15:48:29 +0530 Subject: [PATCH 3/4] date format corrected --- Livehindustan/HNC/spiders/HNC_crawler.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Livehindustan/HNC/spiders/HNC_crawler.py b/Livehindustan/HNC/spiders/HNC_crawler.py index 079e5d4..f4ebf89 100644 --- a/Livehindustan/HNC/spiders/HNC_crawler.py +++ b/Livehindustan/HNC/spiders/HNC_crawler.py @@ -32,8 +32,7 @@ def parse_item(self, response): item['summary'] = Selector(response).xpath('//meta[@name="description"]/@content').extract() item['description'] = Selector(response).xpath('//meta[@name="twitter:description"]/@content').extract() item['keywords'] = Selector(response).xpath('//meta[@name="keywords"]/@content').extract() - dt = Selector(response).xpath('//span[@class="story_float_rigth"]/font/text()').extract() - item['date_time'] = dt[1] + item['date_time'] = Selector(response).xpath('//meta[@name="article:published_time"]/@content').extract() item['image_url'] = Selector(response).xpath('//meta[@property="og:image"]/@content').extract() From 68f544c7a948ddda0f094b365291c19f1a185061 Mon Sep 17 00:00:00 2001 From: DUSHYANT SINGH Date: Wed, 8 Mar 2017 14:42:22 +0530 Subject: [PATCH 4/4] image error corrected --- Amarujala/scraper/spiders/amarUjala.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Amarujala/scraper/spiders/amarUjala.py b/Amarujala/scraper/spiders/amarUjala.py index 8745302..528c8e3 100644 --- a/Amarujala/scraper/spiders/amarUjala.py +++ b/Amarujala/scraper/spiders/amarUjala.py @@ -20,6 +20,6 @@ def parse_item(self,response): item['title']=Selector(response).xpath('//meta[@name="twitter:title"]/@content').extract() item['des']=Selector(response).xpath('//meta[@name="twitter:description"]/@content').extract() item['key']=Selector(response).xpath('//meta[@name="keywords"]/@content').extract() - item['imageUrl']=Selector(response).xpath('//meta[@property="og:image"]/@content').extract() + item['imageUrl']=Selector(response).xpath('//meta[@name="twitter:image"]/@content').extract() yield item