python抓取图片示例,,#!/usr/bin/p
文章由Byrx.net分享于2019-03-23 06:03:30
python抓取图片示例,,#!/usr/bin/p
#!/usr/bin/python# -*- coding:utf-8 -*-import reimport osimport urllib, urllib2, cookielibimport shutilfrom BeautifulSoup import BeautifulSoup # ---- utils ----def normalize_url(url): return "http://" + url if cmp(url[0:7],"http://") != 0 else urldef safeDir(dir): return dir.replace('/', '')# ---- variable ----homepagePrefix = "http://60dxw.comww1.baisex.me/forum-47-"homepageSuffix = ".html"threadPrefix = "http://60dxw.comww1.baisex.me/"homedir = "baixingge"# ---- login ----cookie = urllib2.HTTPCookieProcessor(cookielib.CookieJar())opener = urllib2.build_opener(cookie)# ---- file ----if (os.path.exists(homedir) == False): os.mkdir(homedir)os.chdir(homedir)# ---- crawl ----for page in range(1, 25): pageUrl = '{0}{1}{2}'.format(homepagePrefix,page,homepageSuffix) # ---- mkdir ---- if (os.path.exists(str(page)) == False): os.mkdir(str(page)) os.chdir(str(page)) print pageUrl # ---- download ---- html_body = urllib.urlopen(pageUrl).read() soup = BeautifulSoup(html_body) # ---- extract ---- threaddUrls = [] urlRaws = soup.findAll('th', attrs = {'class' : ['new', 'common']}) urlPattern = re.compile(r'href="([^"]*)"') titlePattern = re.compile(r'>([^<]*)</a>') for urlRaw in urlRaws: h = urlPattern.search(str(urlRaw)) t = titlePattern.search(str(urlRaw)) threadUrl = h.group(1) threadTitle = t.group(1) if (os.path.exists(threadTitle) == False): os.mkdir(safeDir(threadTitle)) else: continue os.chdir(safeDir(threadTitle)) page_url = threadPrefix + threadUrl print "---->{0}".format(page_url) print "---->{0}".format(safeDir(threadTitle)) page_body = urllib.urlopen(page_url).read() page_soup = BeautifulSoup(page_body) imgPattern = re.compile(r'img src="([^"]*)" onload') i = imgPattern.findall(str(page_soup)) index = 0 for img in i: print "-------->{0}".format(img) imgSuffix = img[img.rindex('.'):] imgName = "{0}{1}".format(str(index), imgSuffix) urllib.urlretrieve(img, imgName, None) index += 1 os.chdir("../") os.chdir("../")
相关内容
- 文件结构扁平化,结构扁平化,def flatten(
- Python计算-对列表元素分类统计,python计算列表元素,l
- python模拟Get请求保存网易歌曲的url,pythonget,python模拟
- python包含中文字符串长度,python中文字符串,一个中文
- 俄罗斯方块,,#!/usr/bin/p
- 用番号搜索磁力连接,番号搜索磁力连接,#!/usr/bin/e
- python随机生成手机号码,python手机号码,random.choic
- python获取PC机器名和IP地址,python获取pcip,利用socket包中
- tornado ajax json,tornadojson,#! /usr/bin/
- python分析列表文件的内容是否支持JSON,,#!/usr/bin/p
评论关闭