python之中国大学爬虫，, 1 #!/usr/

python之中国大学爬虫，, 1 #!/usr/

 1 #!/usr/bin/env python3 2 #-*- coding:utf-8 -*- 3 ############################ 4 #File Name: zuihaodaxuepaiming.py 5 #Author: frank 6 #Mail: [email protected] 7 #Created Time:2018-05-15 21:45:55 8 ############################ 9 10 import requests11 from bs4 import BeautifulSoup12 13 14 #1. 获取html页面的内容15 def getHTMLText(url):16     try:17         r = requests.get(url)18         r.raise_for_status()19         #r.encoding = r.apparent_encoding20         r.encoding = ‘utf-8‘21         return r.text22 23     except:24         print("爬取异常")25         return ""26     27 #2. 从html页面内容中爬取大学排名信息, (排名，学校名称，总分)28 def getUnivInfo(uLst, html):29     soup = BeautifulSoup(html, "lxml")30     #获取排名中的大学的数量31     print(len(soup.tbody.find_all("tr", recursive=False)))32     #列出tbody的子节点33     #for tag in soup.tbody.find_all(‘tr‘, recursive=False):34     #    #print("name:{},type:{}".format(tag.name, type(tag)))35     #    print(tag.td.string)36     for tr in soup.tbody.find_all(‘tr‘, recursive=False,limit=1):37         tds = tr(‘td‘)38         #print(tr.prettify())39         #print(tds)40         #print(tds[0].string,tds[1].string, tds[3].string)41         uLst.append([tds[0].string,tds[1].string, tds[3].string])42 43 #3. 显示排名信息44 #def printUnivInfo(uLst, num):45 #    print("{0:^10}\t{1:{3}^10}\t{2:^10}".format("排名","学校名称","总分",chr(12288)))46 #    for uni in uLst[0:num]:47 #        print("{0:^10}\t{1:{3}^10}\t{2:^10}".format(uni[0],uni[1],uni[2],chr(12288)))48 49 def printUnivInfo(uLst, num):50     ptstr = "{0:^10}\t{1:{3}^10}\t{2:^10}"51     print(ptstr.format("排名","学校名称","总分",chr(12288)))52     for uni in uLst[0:num]:53         print(ptstr.format(uni[0],uni[1],uni[2],chr(12288))) #中文字符的空格填充 chr(12288)54 55 def main():56     url = "http://www.zuihaodaxue.cn/zuihaodaxuepaiming2016.html"57     uLst = []58     html = getHTMLText(url)59     #print(html)60     print("getHTMLText")61     getUnivInfo(uLst, html)62     print("getUnivInfo")63     printUnivInfo(uLst, 10)64 65 main()

该实例主要是针对 requests 和 BeautifulSoup的应用。

python之中国大学爬虫