批量抓取豆瓣电影TOP250数据，抓取豆瓣top250,#!/usr/bin/e

批量抓取豆瓣电影TOP250数据，抓取豆瓣top250,#!/usr/bin/e

#!/usr/bin/env pythonimport urllib.requestfrom bs4 import BeautifulSoupmylist = []print(u'豆瓣电影TOP250:\n 序号 \t 影片名\t 评分\t 评价人数\t 评价')def crawl(url):    headers = {'User-Agent':'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6'}    req = urllib.request.Request(url, headers=headers)    page = urllib.request.urlopen(req, timeout=60)    contents = page.read()    soup = BeautifulSoup(contents)    for tag in soup.find_all('div', class_='item'):        try:            m_order = int(tag.find('em', class_='').get_text())            m_name = tag.span.get_text()            m_rating_score = float(tag.find('div', class_='star').em.get_text())            m_rating_num = tag.find('div', class_='star').span.next_sibling.next_sibling.get_text()            m_comments = tag.find("span", class_="inq").get_text()        except AttributeError:            print("%s %s %s %s %s" % (m_order, m_name, m_rating_score, m_rating_num, "NO COMMENTS"))            mylist.append((m_order, m_name, m_rating_score, m_rating_num, "NO COMMENTS"))        else:            print("%s %s %s %s %s" % (m_order, m_name, m_rating_score, m_rating_num, m_comments))            mylist.append((m_order, m_name, m_rating_score, m_rating_num, m_comments))pagenumber = []for i in range(10):    page_number = 25*i    pagenumber.append(page_number)pagelist = list(map(str, pagenumber))BASE_URL = 'http://movie.douban.com/top250?start='LAST_URL = '&filter=&type='for url in [ BASE_URL + MID_URL + LAST_URL for MID_URL in pagelist ]:    crawl(url)import tablibheaders = ('m_order', 'm_name', 'm_rating_score', 'm_rating_num', 'm_comments')mylist = tablib.Dataset(*mylist, headers=headers)print(mylist.csv)with open('D:\doubanmovielist.xlsx', 'wb') as f:    f.write(mylist.xlsx)