| #!/usr/bin/python -O |
| # Copyright 1999-2004 Gentoo Foundation |
| # Distributed under the terms of the GNU General Public License v2 |
| # $Id: /var/cvsroot/gentoo-src/portage/bin/mirror.py,v 1.3 2004/10/10 10:07:20 carpaski Exp $ |
| |
| # Defines the number of threads carrying out the downloading. |
| maxsems=5 |
| |
| import os,sys,string |
| os.environ["PORTAGE_CALLER"]="mirror" |
| os.environ["FEATURES"]="mirror cvs" |
| sys.path = ["/usr/lib/portage/pym"]+sys.path |
| |
| import portage |
| from threading import * |
| from output import red,green,blue,bold |
| from random import shuffle |
| from time import sleep |
| |
| |
| def cstrip(mystr,mychars): |
| newstr = "" |
| for x in mystr: |
| if x not in mychars: |
| newstr += x |
| return newstr |
| |
| class fetcher(Thread): |
| def __init__(self, filename, urilist, dest, md5sum): |
| self.filename = filename |
| self.myurilist = urilist |
| self.myuri = None |
| self.mydest = dest |
| self.destpath = self.mydest+"/"+self.filename |
| self.md5sum = md5sum |
| self.result = None |
| Thread.__init__(self) |
| |
| def fetch(self): |
| #print "Started",self.filename |
| sys.stderr.write(".") |
| sys.stderr.flush() |
| portage.spawn("wget -q -P "+str(self.mydest)+" "+self.myuri, free=1) |
| |
| def finished(self): |
| if os.path.exists(self.destpath) and self.md5sum: |
| ok,reason = portage_checksum.verify_all(self.destpath, md5sum) |
| if not ok: |
| portage_util.writemsg("Failed verification:" + reason + "\n") |
| return 1 |
| return 0 |
| |
| def delete(self): |
| if os.path.exists(self.destpath): |
| #print "Unlink:",self.destpath |
| os.unlink(self.destpath) |
| |
| def run(self): |
| if not self.finished(): |
| self.delete() |
| |
| while not self.finished(): |
| if self.myurilist: |
| self.myuri = self.myurilist.pop(0)+"/"+self.filename |
| self.fetch() |
| else: |
| self.delete() |
| self.result = 0 |
| #print "Failed:",self.filename |
| return 1 |
| |
| #print "Finished:",self.filename |
| self.result = 1 |
| return 0 |
| |
| |
| uri_list = {} |
| fetchers = [] |
| fetcher_sem = BoundedSemaphore(value=maxsems) |
| failures = 0 |
| successes = 0 |
| |
| def clean_fetchers(): |
| global fetcher_sem,fetchers,uri_list,failures,successes,maxsems |
| while len(fetchers) == maxsems: |
| for x in fetchers: |
| if not x.isAlive(): |
| failures += (x.result == 0) |
| successes += (x.result == 1) |
| if x.filename in uri_list.keys(): |
| del uri_list[x.filename] |
| del fetchers[fetchers.index(x)] |
| fetcher_sem.release() |
| if len(fetchers) == maxsems: |
| sleep(1) |
| |
| |
| def start_fetcher(fname, urilist, dest, md5sum): |
| global fetcher_sem,fetchers,uri_list,failures,successes |
| fetcher_sem.acquire() |
| fetchers.append(fetcher(fname, urilist, dest, md5sum)) |
| fetchers[-1].start() |
| |
| |
| tpm = portage.thirdpartymirrors |
| destdir = portage.settings["DISTDIR"][:] |
| |
| hugelist = [] |
| for mycp in portage.db["/"]["porttree"].dbapi.cp_all(): |
| hugelist += portage.db["/"]["porttree"].dbapi.cp_list(mycp) |
| shuffle(hugelist) |
| |
| mycount = -1 |
| for mycpv in hugelist: |
| pv = string.split(mycpv, "/")[-1] |
| |
| clean_fetchers() |
| |
| mycount += 1 |
| if ((mycount % 20) == 0): |
| sys.stdout.write("\nCompleted: %s\n" % mycount) |
| sys.stdout.flush() |
| newuri = portage.db["/"]["porttree"].dbapi.aux_get(mycpv,["SRC_URI"])[0] |
| newuri = string.split(newuri) |
| |
| digestpath = portage.db["/"]["porttree"].dbapi.findname(mycpv) |
| digestpath = os.path.dirname(digestpath)+"/files/digest-"+pv |
| md5sums = portage.digestParseFile(digestpath) |
| |
| for x in newuri: |
| clean_fetchers() |
| if not x: |
| continue |
| if (x in [")","(",":","||"]) or (x[-1] == "?"): |
| # ignore it. :) |
| continue |
| x = cstrip(x,"()|?") |
| if not x: |
| continue |
| mybn = os.path.basename(x) |
| mydn = os.path.dirname(x) |
| if mybn not in uri_list.keys(): |
| if (len(mybn) > len("mirror://")) and (mybn[:len("mirror://")] == "mirror://"): |
| mysite = string.split(x[len("mirror://"):], "/")[0] |
| shuffle(tpm[mysite]) |
| uri_list[mybn] = tpm[mysite][:] |
| else: |
| uri_list[mybn] = [os.path.dirname(x)] |
| clean_fetchers() |
| if (not md5sums) or (mybn not in md5sums.keys()): |
| start_fetcher(mybn, uri_list[mybn], destdir, None) |
| else: |
| start_fetcher(mybn, uri_list[mybn], destdir, md5sums[mybn]) |
| else: |
| break |
| |
| sys.stderr.write("\n\nWaiting last set\n") |
| sys.stderr.flush() |
| while fetchers: |
| if fetchers[0].isAlive(): |
| fetchers[0].join() |
| clean_fetchers() |
| |
| print |
| print |
| print "Successes:",successes |
| print "Failures: ",failures |