-- coding: utf-8 --

import scrapy
from tianmao.items import TianmaoItem

class TopgoodSpider(scrapy.Spider):

name = 'topgood'
allowed_domains = ['list.tmall.com', 'detail.tmall.com']
'''meta = {
    'dont_redirect': True,  # 禁止网页重定向
    'handle_httpstatus_list': [301, 302]  # 对哪些异常返回进行处理
}'''
start_urls = ['http://list.tmall.com/search_product.htm?spm=a220m.1000858.1000724.4.4b3df937tMXU1S&cat=50024399&sort=d&style=g&active=1&industryCatId=50024399&theme=663']

def parse(self, response):
    divs = response.xpath("//div[@id='J_ItemList']/div[@class='product ']/div")
    
    if len(divs) == 0:
        self.log("List Page error -- %s"%response.url)
    
    for div in divs:
        item = TianmaoItem()
        # 价格
        item['GOODS_PRICE'] = div.xpath("p[@class='productPrice']/em/@title")[0].extract()
        # 名称
        item['GOODS_NAME'] = div.xpath("P[@class='productTitle']/a/@title")[0].extract()
        
        pre_Product_Url = div.xpath("p[@class='productTitle']/a/@title").extract_first()
        
        if 'http' not in pre_Product_Url:
            pre_Product_Url = response.urljoin(pre_Product_Url)
        
        item['GOODS_URL'] = pre_Product_Url
        
        yield scrapy.Request(url=pre_Product_Url, meta={'item': item}, callback=self.parse_detail,dont_redirect=True)

def parse_detail(self, response):
    div = response.xpath("//div[@class='extend']/ul")
    
    if len(div) == 0:
        self.log("Detail Page error --%s"%response.url)
    
    div = div[0]
    item = response.meta['item']
    item['SHOP_NAME'] = div.xpath("div[@class='right']/a/text()")[0].extract().strip()
    item['SHOP_URL'] = div.xpath("div[@class='right']/a/@href")[0].extract()
    
    yield item

我在网上找了很多解决方，但是就是不是，用requests可以轻易的抓取到。