爬取千万淘宝商品的python脚本,淘宝商品python脚本,分享之前爬虫爬到的100
爬取千万淘宝商品的python脚本,淘宝商品python脚本,分享之前爬虫爬到的100
分享之前爬虫爬到的1000万条淘宝商品信息。平衡涵盖各种分类,包括商品名、卖家id、地区、价格等信息,json格式。
作者微博: 支持向量姬
import timeimport leveldbfrom urllib.parse import quote_plus import reimport jsonimport itertoolsimport sysimport requestsfrom queue import Queuefrom threading import ThreadURL_BASE = 'http://s.m.taobao.com/search?q={}&n=200&m=api4h5&style=list&page={}'def url_get(url): # print('GET ' + url) header = dict() header['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' header['Accept-Encoding'] = 'gzip,deflate,sdch' header['Accept-Language'] = 'en-US,en;q=0.8' header['Connection'] = 'keep-alive' header['DNT'] = '1' #header['User-Agent'] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.71 Safari/537.36' header['User-Agent'] = 'Mozilla/12.0 (compatible; MSIE 8.0; Windows NT)' return requests.get(url, timeout = 5, headers = header).textdef item_thread(cate_queue, db_cate, db_item): while True: try: cate = cate_queue.get() post_exist = True try: state = db_cate.Get(cate.encode('utf-8')) if state != b'OK': post_exist = False except: post_exist = False if post_exist == True: print('cate-{}: {} already exists ... Ignore'.format(cate, title)) continue db_cate.Put(cate.encode('utf-8'), b'crawling') for item_page in itertools.count(1): url = URL_BASE.format(quote_plus(cate), item_page) for tr in range(5): try: items_obj = json.loads(url_get(url)) break except KeyboardInterrupt: quit() except Exception as e: if tr == 4: raise e if len(items_obj['listItem']) == 0: break for item in items_obj['listItem']: item_obj = dict( _id = int(item['itemNumId']), name = item['name'], price = float(item['price']), query = cate, category = int(item['category']) if item['category'] != '' else 0, nick = item['nick'], area = item['area']) db_item.Put(str(item_obj['_id']).encode('utf-8'), json.dumps(item_obj, ensure_ascii = False).encode('utf-8')) print('Get {} items from {}: {}'.format(len(items_obj['listItem']), cate, item_page)) if 'nav' in items_obj: for na in items_obj['nav']['navCatList']: try: db_cate.Get(na['name'].encode('utf-8')) except: db_cate.Put(na['name'].encode('utf-8'), b'waiting') db_cate.Put(cate.encode('utf-8'), b'OK') print(cate, 'OK') except KeyboardInterrupt: break except Exception as e: print('An {} exception occured'.format(e))def cate_thread(cate_queue, db_cate): while True: try: for key, value in db_cate.RangeIter(): if value != b'OK': print('CateThread: put {} into queue'.format(key.decode('utf-8'))) cate_queue.put(key.decode('utf-8')) time.sleep(10) except KeyboardInterrupt: break except Exception as e: print('CateThread: {}'.format(e))if __name__ == '__main__': db_cate = leveldb.LevelDB('./taobao-cate') db_item = leveldb.LevelDB('./taobao-item') orig_cate = '正装' try: db_cate.Get(orig_cate.encode('utf-8')) except: db_cate.Put(orig_cate.encode('utf-8'), b'waiting') cate_queue = Queue(maxsize = 1000) cate_th = Thread(target = cate_thread, args = (cate_queue, db_cate)) cate_th.start() item_th = [Thread(target = item_thread, args = (cate_queue, db_cate, db_item)) for _ in range(5)] for item_t in item_th: item_t.start() cate_th.join()
相关内容
- python打开url下载url内容,python打开url内容,import urlli
- python定义类的操作符__plus__示例,python__plus_,class adder
- python的类和对象,python对象,# Demonstrat
- __getattr__ demo,__getattr_demo,class empty:
- python实现连接两个url,python实现url,import urlpa
- dns反向解析,根据ip获得域名方法,dns域名,import sys,
- Virtual function in Python,virtualpython,class Super:
- python的iterator object,pythoniterator,class Square
- python获得系统中所有的挂载点,python获得系统挂载,#
- Class method: __init__ and __call__,__init___call_,class Prod:
评论关闭