diff --git a/Amarujala/scraper/spiders/amarUjala.py b/Amarujala/scraper/spiders/amarUjala.py index 9a58a65..d75b8ae 100644 --- a/Amarujala/scraper/spiders/amarUjala.py +++ b/Amarujala/scraper/spiders/amarUjala.py @@ -20,7 +20,9 @@ def parse_item(self,response): item['title']=Selector(response).xpath('//meta[@name="twitter:title"]/@content').extract() item['des']=Selector(response).xpath('//meta[@name="twitter:description"]/@content').extract() item['key']=Selector(response).xpath('//meta[@name="keywords"]/@content').extract() + item['imageUrl']=Selector(response).xpath('//meta[@name="twitter:image"]/@content').extract() item['imageUrl']=Selector(response).xpath('//meta[@property="og:image"]/@content').extract() date_info=Selector(response).xpath('//span/@datetime').extract() item['date_published']=datetime.strptime(date_info[0],"%Y-%m-%d %H:%M:%S") - yield item + + yield item diff --git a/Jansatta/jansatta1.py b/Jansatta/jansatta1.py index 13e8b2e..5bd58dc 100644 --- a/Jansatta/jansatta1.py +++ b/Jansatta/jansatta1.py @@ -2,16 +2,19 @@ from lxml import html import time +# List to store all article-urls from each page urls = [] -for i in xrange(315) : +for i in xrange(1,315) : page = requests.get("http://www.jansatta.com/latest-news/page/"+str(i)+"/?s=news") time.sleep(5) tree = html.fromstring(page.content) article_urls = tree.xpath('//div[@class="newslistbx"]/h3/a/@href') for j in article_urls: + # Each article url is appended in a global list urls[] urls.append(j) - + +#Now request is being send to each article page and required data is scraped for k in urls: page = requests.get(k) tree = html.fromstring(page.content) @@ -20,8 +23,9 @@ page_url = k date_time = tree.xpath('//meta [@itemprop="dateModified"]/@content') keyword = tree.xpath('//meta [@name="keywords"]/@content ') - - #collection.insert({"url":page_url, "title":title}) + + # Data will be inserted in mongodb + # collection.insert({"url":page_url, "title":title}) print(title) time.sleep(5) diff --git a/Livehindustan/HNC/spiders/HNC_crawler.py b/Livehindustan/HNC/spiders/HNC_crawler.py index 079e5d4..f4ebf89 100644 --- a/Livehindustan/HNC/spiders/HNC_crawler.py +++ b/Livehindustan/HNC/spiders/HNC_crawler.py @@ -32,8 +32,7 @@ def parse_item(self, response): item['summary'] = Selector(response).xpath('//meta[@name="description"]/@content').extract() item['description'] = Selector(response).xpath('//meta[@name="twitter:description"]/@content').extract() item['keywords'] = Selector(response).xpath('//meta[@name="keywords"]/@content').extract() - dt = Selector(response).xpath('//span[@class="story_float_rigth"]/font/text()').extract() - item['date_time'] = dt[1] + item['date_time'] = Selector(response).xpath('//meta[@name="article:published_time"]/@content').extract() item['image_url'] = Selector(response).xpath('//meta[@property="og:image"]/@content').extract()