python 2.7 - Unable to get the text from a span using splash-scrapy -
i trying few examples scrape following name, description,price , image:
http://www.lucy.com/shop/shop-by-activity-running/extra-mile-top-0a2s96?variationid=005
i facing 2 problems here:
a) able name , description correctly price not correctly getting scrapped. values receive price following:
'selling_price': [u'\xa0'],
'selling_price': [u'$35.00'],
b) don't know how can extract image there since canvas element shows image when inspect in firebug. maybe need use javascript struggling with.
following spider have created:
# -*- coding: utf-8 -*- import scrapy import urllib2 import datetime import simplejson json scrapy.spiders import crawlspider dashcrawler.items import dashcrawleritem dashcrawler import constant class lucyspider(crawlspider): name = "lucy" custom_settings = { "dupefilter_class": 'dashcrawler.middlewares.customfilter', "downloader_clientcontextfactory": 'dashcrawler.context.customcontextfactory', # "user_agent": "mozilla/5.0 (x11; linux x86_64) applewebkit/537.36 (khtml, gecko) chrome/49.0.2623.75 safari/537.36", "retry_times": 4, "retry_http_codes": [504, 400], "concurrent_requests": 1, 'downloader_middlewares': { 'scrapy.downloadermiddlewares.useragent.useragentmiddleware': none, 'scrapy.downloadermiddlewares.httpproxy.httpproxymiddleware': 110, # 'scrapy_fake_useragent.middleware.randomuseragentmiddleware': 400, 'scrapy_splash.splashcookiesmiddleware': 723, 'scrapy_splash.splashmiddleware': 725, 'scrapy.downloadermiddlewares.httpcompression.httpcompressionmiddleware': 810, 'scrapy.downloadermiddlewares.httpauth.httpauthmiddleware': 725, } } constant = constant.constants_by_spider(name) http_user = constant['http_user'] http_pass = constant['http_pass'] proxy_ip = constant['proxy_ip'] proxy_auth = constant['proxy_auth'] gender_types = constant['gender_types'] no_result_found = "nenhum resultado encontrado para consulta" availability = "continuar com compra" scrapy_details = constant['scrapy_details'] allowed_domains = constant['allowed_domains'] api_url_call = constant['api_url_call'] count = constant['login_types'] zipcodes_count = 0 start_urls = [] accounts = [] splash_source = """function main(splash) local url = splash.args.url assert(splash:go(url)) assert(splash:wait(1.0)) return splash:html() end""" def start_requests(self): request = urllib2.request(self.api_url_call, headers=self.constant['headers']) retailer_product_configurations = json.load(urllib2.urlopen(request)) self.scrapy_details = retailer_product_configurations login_type = self.scrapy_details["shopper_type"] is_register = "registered" in login_type self.accounts = self.scrapy_details["accounts"] retailer_product_configurations = self.scrapy_details['retailerproductconfigurationurls'] anonymous_user_types = set(["anonymous", "both"]) & set(login_type) #import pudb; pudb.set_trace() if (len(anonymous_user_types) >= 0 , is_register == false) or (len(self.accounts) == 0): yield scrapy.request( self.api_url_call, callback=self.main_url_parse, headers=self.constant['headers'], meta={ "shopper_type": "anonymous", "retailer_product_configurations": retailer_product_configurations, "index": 0, }, errback=self.error ) def main_url_parse(self, response): product_store = [] index = response.meta['index'] retailer_product_configurations = response.meta['retailer_product_configurations'] shopper_type = response.meta['shopper_type'] self.request_count = len(retailer_product_configurations) # index = 0 retailer_product_configuration in retailer_product_configurations: url = retailer_product_configuration['url'] page_type = retailer_product_configuration['urltype'] retailer_id = retailer_product_configuration['retailerid'] max_number_of_records = retailer_product_configuration['maxnumberofrecords'] retailer_product_configuration_id = retailer_product_configuration['id'] product_url = "self.color_url" if page_type == "pdp" else "self.parse" product_store.append(url) index += 1 colorvariant = retailer_product_configuration.get('colorvariant', true) colorvariant = true if colorvariant == none else colorvariant yield scrapy.request( url, callback= self.parse, meta={"product_store": product_store, "start_url": url, "shopper_type": shopper_type, "retailer_id": retailer_id, 'id': retailer_product_configuration_id, 'max_number_of_records': max_number_of_records, "index": index, "colorvariant": colorvariant, 'splash':{ 'args': { 'lua_source': self.splash_source}, 'endpoint': 'execute', } }, errback=self.error, dont_filter=true, ) # error handling def error(self, response): pass def parse(self, response): retailer_id = response.meta['retailer_id'] shopper_type = response.meta['shopper_type'] name = response.xpath('//h1[contains(@class,"product-content-info-name product-info-js")]/text()').extract() img = response.xpath('//div[@class=\'product-primary-image\']//img/@src').extract() # sku_code = name item = dashcrawleritem() item['name'] = response.xpath('//h1[contains(@class,"product-content-info-name product-info-js")]/text()').extract() item['selling_price'] = response.xpath('//span[contains(@class,"product-content-info-offer-price offer-price offer-price-js product-price-amount-js")]/text()').extract() item['description'] = response.xpath('//div[contains(@class,"desc-container pdp-details-desc-container")]/text()').extract() item['retailer_product_configuration_id'] = response.meta['id'] item['retailer_id'] = retailer_id item['crawl_date'] = datetime.datetime.utcnow().strftime("%m/%d/%y %h:%m:%s") item['shopper_type'] = shopper_type item['product_url'] = response.url item['sku_code'] = name item['reference_code'] = "cent-%s" % (name) item['preview_image_url'] = img yield item i new python, scrapy, splash , lua well.
so spider gets urls scrolled api , performs scraping. above mentioned lucy store pdp returned api.
Comments
Post a Comment