Python抓取电影来了（dianying.fm)
Python抓取电影来了（dianying.fm)
/*title:Python抓取电影来了（dianying.fm)
blog:http://yxmhero1989.blog.163.com/blog/static/11215795620132441055127/
author:insun
*/
小虾的电影FM readself.com
2月中旬前面还是老版本的布局 当时写的代码是这样的：
#!/usr/bin/env python
#-*- coding=utf-8 -*-
import urllib2,urllib,re
import pymongo
import sys,os
import json
import chardet
reload(sys)
sys.setdefaultencoding('utf-8')
db = pymongo.Connection().test
if(os.path.exists('Allsmall')==False):
os.mkdir('Allsmall')
if(os.path.exists('Allbig')==False):
os.mkdir('Allbig')
#http://dianying.fm/genre/%E7%88%B1%E6%83%85 调整后的分类是http://movie.readself.com/category/filter_play-genre_%E5%96%9C%E5%89%A7
#下拉加载和点击加载是JS附加了一段json 要json {"result": -1, "error": "unknown error"}
#一个json加载28个 29*28 = 812 > 500
for i in range(1,49):
#http://movie.readself.com/
print "-----------第"+str(i)+"个链接开始抓取--------------"
url = 'http://movie.readself.com/reflect/category/eyJzb3J0IjogIiIsICJmaWx0ZXIiOiAiIiwgImNhdGFsbCI6ICIiLCAia2V5IjogIiIsICJ5ZWFyIjogIiIsICJnZW5yZSI6ICIiLCAicmVnaW9uIjogIiIsICJzaG93IjogIiIsICJjbGFzcyI6ICIifQ==/'+str(i)
try:
proxies = {'http':'http://219.239.26.23:80'}#或者proxies = {'':'211.167.112.14:80'}
opener = urllib.FancyURLopener(proxies)
f = opener.open(url)
html = f.read()
print html
sys.exit(-1)
#print chardet.detect(html)#{'confidence': 1.0, 'encoding': 'ascii'}
#html = html.replace('\\n','') #这个导致了28个变成了5个 说明后来的正则已经不适合下面的了
html = json.loads(html,encoding='utf-8')
html = str(html)
reg = re.compile(r'<div class="x-movie-entry">.+?<a href="(.*?)".+?title="(.*?)">.+?<img.+?src="(.*?)">.+?</a>.+?<div class="x-movie-caption">.+?</div>',re.S)
#reg = re.compile(r'<div class=\\\"x-movie-entry\\\">.+?<a href=\\\"(.*?)\\\".+?title=\\\"(.*?)\\\">.+?<img.+?src=\\\"(.*?)\\\">.+?</a>.+?<div class=\\\"x-movie-caption\\\">.+?</div>',re.S)
groups = re.findall(reg,html)
print "*****----本链接抓取了"+str(len(groups))+"个影片信息****----------"
for detail in groups:
shortTitle = detail[1]
#三傻大闹宝莱坞 \u4e09\u50bb\u5927\u95f9\u5b9d\u83b1\u575e
shortTitle = shortTitle.decode('unicode_escape').encode('utf-8')
print shortTitle + ' 下载中'
#\u8ba9\u5b50\u5f39\u98de \u后边的四位十六进制数是汉字的UNICODE编码
#\xe8\xae\xa9\xe5\xad\x90\xe5\xbc\xb9\xe9\xa3\x9e gb2312 编码
#b'\xDE\xD5\xB4\xF8' 二进制
smallImage = detail[2]
detailUrl = 'http://movie.readself.com'+detail[0]
text = opener.open(detailUrl).read()
IMDB_stars_need = '.+?<td class="span2"><span class="x-m-label">IMDB评分</span></td>.+?<b>(.*?)</b>'
IMDB_link_need = '.+?<a rel="nofollow" href="(.*?)" target="_blank">IMDB链接</a>.+?'
alias_need = '.+?<td class="span2"><span class="x-m-label">别名</span></td>.+?<td>(.*?)</td>'
if text.count('别名') == 0:
alias_need = '(.*?)'
if text.count('IMDB评分') == 0:
IMDB_stars_need = '(.*?)'
if text.count('IMDB链接') == 0 :
IMDB_link_need = '(.*?)'
detailReg = re.compile('<div class="x-m-title">.+?<h3>(.*?)<span class="x-m-year">(.*?)</span>.+?</h3>.+?</div>'+
'.+?<div class="x-m-poster span3".+?<img.+?src="(.*?)">.+?</div>'+
'.+?<td class="span2"><span class="x-m-label">导演</span></td>.+?<td>(.*?)</td>'+
'.+?<td class="span2"><span class="x-m-label">主演</span></td>.+?<td>(.*?)</td>'+
'.+?<td class="span2"><span class="x-m-label">类型</span></td>.+?<td>(.*?)</td>'+
'.+?<td class="span2"><span class="x-m-label">地区</span></td>.+?<td>(.*?)</td>'+
'.+?<td class="span2"><span class="x-m-label">上映时间</span></td>.+?<td>(.*?)</td>'+
'.+?<td class="span2"><span class="x-m-label">片长</span></td>.+?<td>(.*?)</td>'+
alias_need +
IMDB_stars_need +
'.+?<td class="span2"><span class="x-m-label">豆瓣评分</span></td>.+?<b>(.*?)</b>'+
'.+?<a rel="nofollow" href="(.*?)" target="_blank">豆瓣链接</a>.+?'+
IMDB_link_need +
'.+?<legend>剧情介绍</legend>.+?<div class="">(.*?)</div>'
,re.S)
contents = re.findall(detailReg,text)
magnetReg = re.compile('.+?<tr class="resources">.+?<td>(.*?)</td>'+
'.+?<a rel="nofollow" href="(.*?)" alt="" title="(.*?)">.+?</a>'+
'.+?<a rel="nofollow" href="(.*?)" target="_blank" class="btn btn-mini btn-info">试看</a>'
'.+?<td style="text-align: left;">(.*?)</td>.+?<td style="text-align: left;">(.*?)</td>'+
'.+?<input type="hidden" name="seeders" value="(.*?)" />',re.S)
magnetAll = re.findall(magnetReg,text)
for content in contents:
bigImage = content[2]
values = dict(
shortTitle = shortTitle,
smallImage = smallImage,
title = content[0],
year = content[1],
bigImage = bigImage,
director = content[3],
actors = content[4],
genre = content[5],
area = content[6],
film_date = content[7],
duration = content[8],
alias = content[9],
imdb_rate = content[10],
douban_rate = content[11],
douban_link = content[12],
imdb_link = content[13],
summary = content[14],
magnetAll = magnetAll,
)
db.all.save(values)
#small:http://img2.static.dianying.fm/poster/m/5071cc6d90d7a90a416a596d
#big:http://img1.static.dianying.fm/poster/l/5071cc6d90d7a90a416a596d
smallSave = smallImage[smallImage.rindex('/')+1:]+".jpg"#after
#before = smallImage[:smallImage.rindex('/')+1] #before
bigSave = bigImage[bigImage.rindex('/')+1:]+".jpg"
urllib.urlretrieve(smallImage,'Allsmall/'+smallSave)
urllib.urlretrieve(bigImage,'Allbig/'+bigSave)
print shortTitle+'Save Success!'
except:
'''
print 'Bad Gateway or Bad NetWork or Site Crash!'
sys.exit(-1)
'''
raise
后来xiaoxia改了新版布局 我重写了正则表达式
抓取下来 存进mongodb
废话不说了。
#!/usr/bin/env python
#-*- coding=utf-8 -*-
import urllib2,urllib,re
import pymongo
import sys,os
import json
import chardet
reload(sys)
sys.setdefaultencoding('utf-8')
db = pymongo.Connection().test
if(os.path.exists('Allsmall')==False):
os.mkdir('Allsmall')
if(os.path.exists('Allbig')==False):
os.mkdir('Allbig')
#http://dianying.fm/genre/%E7%88%B1%E6%83%85 调整后的分类是http://movie.readself.com/category/filter_play-genre_%E5%96%9C%E5%89%A7
#下拉加载和点击加载是JS附加了一段json 要json {"result": -1, "error": "unknown error"}
#一个json加载28个 29*28 = 812 > 500
#2013/2/27 貌似做了IP屏蔽 于是加上代理 页面调整了 于是改了2个正则
for i in range(1,49):
#http://movie.readself.com/
print "-----------第"+str(i)+"个链接开始抓取--------------"
url = 'http://movie.readself.com/reflect/category/eyJzb3J0IjogIiIsICJmaWx0ZXIiOiAiIiwgImNhdGFsbCI6ICIiLCAia2V5IjogIiIsICJ5ZWFyIjogIiIsICJnZW5yZSI6ICIiLCAicmVnaW9uIjogIiIsICJzaG93IjogIiIsICJjbGFzcyI6ICIifQ==/'+str(i)
try:
proxies = {'http':'http://219.239.26.23:80'}#或者proxies = {'':'211.167.112.14:80'}
opener = urllib.FancyURLopener(proxies)
f = opener.open(url)
html = f.read()
#print chardet.detect(html)#{'confidence': 1.0, 'encoding': 'ascii'}
#html = html.replace('\\n','') #这个导致了28个变成了5个 说明后来的正则已经不适合下面的了
html = json.loads(html,encoding='utf-8')
html = str(html)
reg = re.compile(r'<div class="x-movie-detail">.+?<a class="z-movie-playmask" .+? href="(.*?)">.+?<img alt="(.*?)".+?src="(.*?)">.+?</a>',re.S)
#reg = re.compile(r'<div class=\\\"x-movie-entry\\\">.+?<a href=\\\"(.*?)\\\".+?title=\\\"(.*?)\\\">.+?<img.+?src=\\\"(.*?)\\\">.+?</a>.+?<div class=\\\"x-movie-caption\\\">.+?</div>',re.S)
groups = re.findall(reg,html)
print "*****----本链接抓取了"+str(len(groups))+"个影片信息****----------"
for detail in groups:
shortTitle = detail[1]
#三傻大闹宝莱坞 \u4e09\u50bb\u5927\u95f9\u5b9d\u83b1\u575e
shortTitle = shortTitle.decode('unicode_escape').encode('utf-8')
print shortTitle + ' 下载中'
#\u8ba9\u5b50\u5f39\u98de \u后边的四位十六进制数是汉字的UNICODE编码
#\xe8\xae\xa9\xe5\xad\x90\xe5\xbc\xb9\xe9\xa3\x9e gb2312 编码
#b'\xDE\xD5\xB4\xF8' 二进制
smallImage = detail[2]
detailUrl = 'http://movie.readself.com'+detail[0]
text = opener.open(detailUrl).read()
IMDB_stars_need = '.+?<span class="badge" style="color: orange; font-weight: bold;">(.*?)</span>'
alias_need = '.+?<td class="span2"><span class="x-m-label">别名</span></td>.+?<td>(.*?)</td>'
IMDB_link_need = '.+?<a rel="nofollow" href="(.*?)" target="_blank">IMDB链接</a>'
if text.count('别名') == 0:
alias_need = '(.*?)'
if text.count('IMDB：') == 0:
IMDB_stars_need = '(.*?)'
if text.count('IMDB链接') == 0 :
IMDB_link_need = '(.*?)'
detailReg = re.compile('<div class="x-m-poster">.+?<img .+? src="(.*?)">.+?</div>'+
'.+?<a rel="nofollow" href="(.*?)" target="_blank">豆瓣链接</a>'+
IMDB_link_need+
'.+?<div class="x-m-title">(.*?)<span class="muted">(.*?)</span>.+?</div>'+
'.+?<td class="span2"><span class="x-m-label">导演</span></td>.+?<td>(.*?)</td>'+
'.+?<td class="span2"><span class="x-m-label">主演</span></td>.+?<td>(.*?)</td>'+
'.+?<td class="span2"><span class="x-m-label">类型</span></td>.+?<td>(.*?)</td>'+
'.+?<td class="span2"><span class="x-m-label">地区</span></td>.+?<td>(.*?)</td>'+
'.+?<td class="span2"><span class="x-m-label">上映时间</span></td>.+?<td>(.*?)</td>'+
'.+?<td class="span2"><span class="x-m-label">片长</span></td>.+?<td>(.*?)</td>'+
alias_need +
'.+?<td class="span2"><span class="x-m-label">评分</span></td><td>'+
'.+?<span class="badge" style="color: green; font-weight: bold;">(.*?)</span>'+
IMDB_stars_need +
'.+?<div class="x-m-summary">(.*?)</div>',re.S)
contents = re.findall(detailReg,text)
magnetReg = re.compile('.+?<tr class="resources".+?>.+?<td style="word-break: break-all;">(.*?).+?<span class="muted">(.*?)</span>.+?</td>'+
'.+?<a .+? rel="nofollow" href="(.*?)" .+? title="(.*?)">.+?</a>'+
'.+?<a rel="nofollow".+?href="(.*?)" target="_blank" .+?>.+?云点播.+?</a>'
'.+?<a class="btn" style="padding:0px;" tclass=" ".+?>(.*?)</a>',re.S)
magnetAll = re.findall(magnetReg,text)
for content in contents:
bigImage = content[0]
values = dict(
shortTitle = shortTitle,
smallImage = smallImage,
bigImage = bigImage,
douban_link = content[1],
imdb_link = content[2],
title = content[3],
year = content[4],
director = content[5],
actors = content[6],
genre = content[7],
area = content[8],
film_date = content[9],
duration = content[10],
alias = content[11],
douban_rate = content[12],
imdb_rate = content[13],
summary = content[14],
magnetAll = magnetAll,
)
db.all.save(values)
#small:http://img2.static.dianying.fm/poster/m/5071cc6d90d7a90a416a596d
#big:http://img1.static.dianying.fm/poster/l/5071cc6d90d7a90a416a596d
smallSave = smallImage[smallImage.rindex('/')+1:]+".jpg"#after
#before = smallImage[:smallImage.rindex('/')+1] #before
bigSave = bigImage[bigImage.rindex('/')+1:]+".jpg"
urllib.urlretrieve(smallImage,'Allsmall/'+smallSave)
urllib.urlretrieve(bigImage,'Allbig/'+bigSave)
print shortTitle+'Save Success!'
except:
'''
print 'Bad Gateway or Bad NetWork or Site Crash!'
sys.exit(-1)
'''
raise
相关内容
- 暂无相关文章
评论关闭