python 多线程下载

python 多线程下载

#-------------------------------------------------------------------------------

# Name:        down

# Purpose:    

#

# Author:      lqefn

#

# Created:     15-09-2010

# Copyright:   (c) lqefn 2010

# Licence:    

#-------------------------------------------------------------------------------

 

from urllib.request import *

from threading import Thread

import os

import time

from datetime import datetime

 

class DownTask(Thread):

    def __init__(self, url, file, start, end):

        Thread.__init__(self)

        self.url = url

        self.startpos = start + file.tell()

        self.endpos = end

        self.file = file

        self.length = self.endpos - self.startpos + 1

        self.nread = 0

        r = Request(url = self.url)

        r.add_header("Range", "bytes=%d-%d"%(self.startpos, self.endpos))

        self.con = urlopen(r)

        print(self.file.name, "start")

        print("need read %d bytes"%self.length)

        print(self.con.info().as_string())

    def run(self):

        if self.startpos >= self.endpos:

            print(self.file.name, "is already completed")

            return

        while self.nread < self.length:

            nleft = self.length - self.nread

            if nleft > 1024:

                d = self.con.read(1024)

            else:

                d = self.con.read(nleft)

            if d:

                self.nread += len(d)

                self.file.write(d)

                self.file.flush()

            else:

                break

        self.file.flush()

        self.con.close()

        print(self.file.name, "complete len/size:%d/%d, read %d byte"%(self.file.tell(), self.endpos-self.startpos+1, self.nread))

 

 

def download(url, nthread = 4, filename = None):

    if filename is None:

        filename = os.path.split(url)[1]

    con = urlopen(url)

    print(con.info().as_string())

    supp = con.headers.get("Accept-Ranges")

    length = int(con.headers.get("Content-length"))

    con.close()

 

    if not supp:

        nthread = 1

    ranges = [0]

    isize = length // nthread

    for i in range(nthread - 1):

        ranges.append((i + 1) * isize)

    ranges.append(length)

 

    print("ranges", ranges)

    files = []

    tasks = []

    for i in range(len(ranges) - 1):

        file = open("%s_part%d"%(filename, i), "ab+")

        files.append(file)

        task = DownTask(url, file, ranges[i], ranges[i + 1] - 1)

        tasks.append(task)

    for t in tasks:

        t.start()

 

    while any(map(lambda t:t.isAlive(), tasks)):

        nread = sum(map(lambda t:t.nread, tasks))

        print("/rcomplete(%%%04.1f) len/size:%d/%d"%(nread * 100.0 / length, nread, length), datetime.now(), end = "", file=sys.stdout)

        sys.stdout.flush()

        time.sleep(0.5)

    for t in tasks:

        t.join()

    file = open(filename, "wb")

    for f in files:

        f.seek(0)

        file.write(f.read())

        f.close()

        os.remove(f.name)

    file.flush()

    print(file.name, "complete")

    file.close()

 

 

 

 

 

if __name__ == "__main__":

    url = "http://nchc.dl.sourceforge.net/project/xlslib/xlslib-1.6.0.zip"

    download(url, 4)

 

 摘自 lqefn的专栏