#!/bin/env python # vim: set fileencoding=utf-8 : import re, socket, os, sys from ftplib import FTP from httplib import HTTPConnection from HTMLParser import HTMLParser from datetime import datetime from time import strptime, strftime, localtime import makenvr fedoramirrors = [] #http://download.fedora.redhat.com/pub/fedora/linux/development/rawhide/source/SRPMS/ #fedoramirrors.append(('download.fedora.redhat.com', '/pub/fedora/linux/development/rawhide/source/SRPMS/')) fedoramirrors.append(('ftp.fi.muni.cz', '/pub/linux/fedora/linux/development/rawhide/source/SRPMS/')) fedoramirrors.append(('sunsite.mff.cuni.cz', 'MIRRORS/fedora.redhat.com/linux/development/rawhide/source/SRPMS/')) fedoramirrors.append(('ftp.heanet.ie', '/pub/fedora/linux/development/rawhide/source/SRPMS/')) debug = 0 class MyHTMLParser(HTMLParser): def __init__(self): HTMLParser.__init__(self) self.data = '' def handle_data(self, data): self.data = self.data + data def isreleaseid(number): bits = number.split('.') try: for bit in bits: int(bit) except: return False return True def getminor(key): bits = key.split('.') minor = int(bits[1]) return minor def sortbyurl(a, b): if a.host == b.host: return a.dir < b.dir return a.host < b.host def sortkeys(a, b): return makenvr.compare_vr(a[0].lower(), b[0].lower(), a[1].lower(), b[1].lower()) class package: def __init__(self, style, srpmname, upstreamprjname='', upstreampkgname='', host='', dir='', pattern='', monthsdiff=0, datestring='', ispost=-1, fixrule=None): self.style = style self.name = srpmname if upstreamprjname == '': self.upstreamprjname = srpmname else: self.upstreamprjname = upstreamprjname if upstreampkgname == '': self.upstreampkgname = self.upstreamprjname else: self.upstreampkgname = upstreampkgname self.host = host self.dir = dir self.pattern = pattern self.monthsdiff = monthsdiff self.datestring = datestring self.ispost = ispost self.fixrule = fixrule droppedlist = [] firstcol = 37 secondcol = 29 def alignit(name, count): for i in range(len(name), count): name = name + ' ' print name, ':', ftplist = [] def collector(line = ''): global ftplist if debug: print 'ftpline is', line ftplist.append(line) class packagemanager: def __init__(self, packagelist): self.keys = {} for package in packagelist: if not self.keys.has_key(package.style): self.keys[package.style] = [] self.keys[package.style].append(package) for key in self.keys: packages = self.keys[key] packages.sort(sortbyurl) self.keys[key] = packages for mirror in fedoramirrors: try: self.ftp = FTP(mirror[0]) self.ftp.login() self.ftp.cwd(mirror[1]) self.srpmlist = self.ftp.nlst() print "Fedora Mirror:", mirror[0] break except: print "Skipping:", mirror[0], "because", sys.exc_info()[1] pass def latestsrpm(self, pack): releases = [] for entry in self.srpmlist: nameprefix = pack.name + '-' prefix = nameprefix if entry[:len(prefix)] == prefix: rpmname, rpmversion, rpmrelease = makenvr.get_nvr(entry, pack.ispost) if rpmname == pack.name: rpmrelease = makenvr.scrub_disttag(rpmrelease) rpmrelease = makenvr.undo_prerelease(rpmrelease) releases.append((rpmversion, rpmrelease)) keys = self.cleanandsort(releases) if len(keys): return keys[0] return None def fetch(self): for key in self.keys: try: if key == 'gnome': # self.ftp = FTP('ftp.acc.umu.se') self.ftp = FTP('ftp.heanet.ie') self.ftp.login() for package in self.keys[key]: try: latest = self.latestsrpm(package) if latest: alignit(package.name+'-'+latest[0]+'-'+latest[1], firstcol) self.getgnomelatest(package, latest) else: droppedlist.append(package.name); except: print 'ERROR fetching:', package.name, 'from' print ' ftp://ftp.heanet.ie'+'/mirrors/ftp.gnome.org/sources/' +\ pack.upstreampkgname print ' error is:', sys.exc_info()[0], sys.exc_info()[1] elif key == 'sourceforge': # self.ftp = FTP('ftp.heanet.ie') self.ftp = FTP('www.mirrorservice.org') self.ftp.login() for package in self.keys[key]: try: latest = self.latestsrpm(package) if latest: alignit(package.name+'-'+latest[0]+'-'+latest[1], firstcol) self.getsourceforgelatest(package, latest) else: droppedlist.append(package.name); except: print 'ERROR fetching:', package.name, 'from' print ' ftp://www.mirrorservice.org'+self.getsourceforgedir(package) print ' error is:', sys.exc_info()[0], sys.exc_info()[1] elif key == 'ftp': lasthost = '' for package in self.keys[key]: try: if package.host != lasthost: self.ftp = FTP(package.host) self.ftp.login() lasthost = package.host self.ftp.cwd(package.dir) dirlist = self.ftp.nlst() dirlist = self.dofixrule(dirlist, package) #print dirlist latest = self.latestsrpm(package) if latest: alignit(package.name+'-'+latest[0]+'-'+latest[1], firstcol) self.getsimplelatest(package.upstreampkgname, latest, dirlist, package.ispost, False) else: droppedlist.append(package.name); except: print 'ERROR fetching:', package.name, 'from' print ' ftp://'+package.host+package.dir print ' error is:', sys.exc_info()[0], sys.exc_info()[1] elif key == 'http-regexp': lasthost = '' lastdir = '' conn = None for package in self.keys[key]: try: if debug: print package.host, package.dir if package.host != lasthost: conn = HTTPConnection(package.host) if package.host != lasthost or package.dir != lastdir: headers = {"User-Agent": "latestpackages/1.0.0"} conn.request("GET", package.dir, headers=headers) response = conn.getresponse() data = response.read().splitlines() lasthost = package.host lastdir = package.dir if debug: print data p = re.compile(package.pattern) dirlist = [] for line in data: try: for m in re.finditer(p, line): if debug: print 'MATCH', line result = m.group(1) if len(package.datestring): dttm = datetime(*strptime(result, package.datestring)[0:6]) if dttm.year == 1900: dttm = datetime(datetime.today().year, \ dttm.month, dttm.day) result = dttm.strftime("%Y%m%d") dirlist.append(package.upstreampkgname+'-'+result+'.tar.gz') except: pass dirlist = self.dofixrule(dirlist, package) if debug: print dirlist latest = self.latestsrpm(package) if latest: alignit(package.name+'-'+latest[0]+'-'+latest[1], firstcol) self.getsimplelatest(package.upstreampkgname, latest, dirlist, \ package.ispost, False, package.monthsdiff) else: droppedlist.append(package.name); except: print 'ERROR fetching:', package.name, 'from' print ' http://'+package.host+package.dir print ' error is:', sys.exc_info()[0], sys.exc_info()[1] elif key == 'http-last-modified': # lasthost = '' conn = None for package in self.keys[key]: try: if debug: print package.dir # if package.host != lasthost: conn = HTTPConnection(package.host) # lasthost = package.host headers = {"User-Agent": "latestpackages/1.0.0"} conn.request("GET", package.dir, headers=headers) response = conn.getresponse() s = response.getheader('last-modified') if debug: print s dirlist = [] result = datetime(*strptime(s, "%a, %d %b %Y %H:%M:%S %Z")[0:6]) datestring = result.strftime("%Y%m%d") dirlist.append(package.upstreampkgname+'-'+datestring+'.tar.gz') latest = self.latestsrpm(package) if latest: alignit(package.name+'-'+latest[0]+'-'+latest[1], firstcol) self.getsimplelatest(package.upstreampkgname, latest, dirlist, \ package.ispost, False, package.monthsdiff) else: droppedlist.append(package.name); except: print 'ERROR fetching:', package.name, 'from' print ' http://'+package.host+package.dir print ' error is:', sys.exc_info()[0], sys.exc_info()[1] elif key == 'ftp-last-modified': global ftplist lasthost = '' for package in self.keys[key]: try: if package.host != lasthost: self.ftp = FTP(package.host) self.ftp.login() lasthost = package.host dirlist = [] pattern = r'.*?(... .\d \d\d\d\d)' p = re.compile(pattern) pattern2 = r'.*? (... .\d )' p2 = re.compile(pattern2) ftplist = [] self.ftp.dir(package.dir, collector) for line in ftplist: m = p.match(line) if m: result = datetime(*strptime(m.group(1), "%b %d %Y")[0:6]) datestring = result.strftime("%Y%m%d") dirlist.append(package.upstreampkgname+'-'+datestring+'.tar.gz') else: m = p2.match(line) if m: try: result = datetime(*strptime(m.group(1), "%b %d ")[0:6]) result = datetime(datetime.today().year, \ result.month, result.day) datestring = result.strftime("%Y%m%d") dirlist.append(package.upstreampkgname+'-'+datestring+'.tar.gz') except: pass if debug: print 'dirlist is now', dirlist latest = self.latestsrpm(package) if latest: alignit(package.name+'-'+latest[0]+'-'+latest[1], firstcol) self.getsimplelatest(package.upstreampkgname, latest, dirlist, package.ispost, False) else: droppedlist.append(package.name); except: print 'ERROR fetching:', package.name, 'from' print ' ftp://'+package.host+package.dir print ' error is:', sys.exc_info()[0], sys.exc_info()[1] elif key == 'svn-propget-date': for package in self.keys[key]: try: #print package.dir if package.dir[-1] == '/': cmd = 'svn propget --revprop -rHEAD svn:date https://' + package.host + \ package.dir fd = os.popen(cmd, "r") s = fd.readline()[:-1] else: cmd = 'svn ls --xml --depth empty https://' + package.host + \ package.dir + '|grep date|sed -e "s///g"|sed -e "s/<\/date>//g"' fd = os.popen(cmd, "r") s = fd.readline()[:-1] dirlist = [] s = s.split('.')[0] result = datetime(*strptime(s, "%Y-%m-%dT%H:%M:%S")[0:6]) datestring = result.strftime("%Y%m%d") if package.dir[-1] == '/': #temp hack dirlist.append(package.upstreampkgname+'-'+datestring+'.tar.gz') else: dirlist.append(package.upstreampkgname+'-'+datestring+'svn.tar.gz') latest = self.latestsrpm(package) if latest: alignit(package.name+'-'+latest[0]+'-'+latest[1], firstcol) self.getsimplelatest(package.upstreampkgname, latest, dirlist, \ package.ispost, False, package.monthsdiff) else: droppedlist.append(package.name); except: print 'ERROR fetching:', package.name, 'from' print ' http://'+package.host+package.dir print ' error is:', sys.exc_info()[0], sys.exc_info()[1] except socket.error: for package in self.keys[key]: print 'TIMEOUT fetching', package.name except: print 'Oops!', sys.exc_info()[0] for package in self.keys[key]: print 'ERROR fetching', package.name def cleanandsort(self, keys): if debug: print 'candidate keys are', keys keys.sort(sortkeys) keys.reverse() return keys def getgnomelatest(self, pack, latest): self.ftp.cwd('/mirrors/ftp.gnome.org/sources/' + pack.upstreampkgname) # self.ftp.cwd('/pub/GNOME/sources/' + pack.upstreampkgname) dirlist = self.ftp.nlst() vrs = [] for entry in dirlist: vrs.append((entry,'1')) keys = self.cleanandsort(vrs) result = 'Upstream ' for key in keys: minor = getminor(key[0]) self.ftp.cwd(key[0]) dirlist = self.ftp.nlst() dirlist = self.dofixrule(dirlist, pack) ret = "NONE!" for file in dirlist: if len(file) > 10 and file[0:10] == 'LATEST-IS-': ret = file[10:] break if not minor % 2: result = result + str(ret)+'-1' alignit(result, secondcol) if sortkeys(latest, (ret,'1')) < 0: print 'NOT UPTODATE' else: print 'UpToDate' break self.ftp.cwd('..') def latestkey(self, keys, latestsrpm, monthsdiff=0): if len(keys) == 0: alignit('MISSING', secondcol) print else: result = 'Upstream ' for key in keys: result = result + key[0]+'-'+key[1] old = False alignit(result, secondcol) if sortkeys(latestsrpm, key) < 0: old = True if monthsdiff != 0: srpmvr = latestsrpm[0]+'-'+latestsrpm[1] upstreamvr = key[0]+'-'+key[1] pattern = r'.*?(\d\d\d\d\d\d\d\d)' p = re.compile(pattern) m = p.match(srpmvr) if m: srpmdate = m.group(1) m = p.match(upstreamvr) if m: upstreamdate = m.group(1) latestyear = int(srpmdate[-8:-4]) latestmonth = int(srpmdate[-4:-2]) for i in range(0, monthsdiff): if latestmonth == 12: latestyear = latestyear + 1 latestmonth = 1 else: latestmonth = latestmonth + 1 theday = int(srpmdate[-2:]) while (1): try: srpmvaliduntil = datetime(latestyear, latestmonth, \ theday) break except ValueError: theday = theday - 1 pass upstreamdt = datetime(int(upstreamdate[-8:-4]), int(upstreamdate[-4:-2]), \ int(upstreamdate[-2:])) if debug: print 'allowing difference of', monthsdiff, 'month between', \ srpmvaliduntil, 'and', upstreamdt if upstreamdt < srpmvaliduntil: old = False if old: print 'NOT UPTODATE' else: print 'UpToDate' break def getsimplelatest(self, name, latest, dirlist, ispost, strict = True, monthsdiff=0): blacklistwords = ['win32', 'MinGW', 'Darwin', 'MSVC', 'hunspell_UNO', 'cs.net', 'or.newer', 'or_newer'] releases = [] for sep in ['-', '_', '.']: for entry in dirlist: nameprefix = name + sep prefix = nameprefix if debug: print 'entry is', entry if debug: print 'prefix is', prefix if entry[:len(prefix)] == prefix: skip = False for word in blacklistwords: if entry.find(word) != -1: skip = True break if skip: continue try: dummyname, version, release = makenvr.make_nvr(entry, '', ispost) releases.append((version, release)) except makenvr.NVRException: pass keys = self.cleanandsort(releases) self.latestkey(keys, latest, monthsdiff) def getsourceforgedir(self, pack): return '/sites/download.sourceforge.net/pub/sourceforge/' + pack.upstreamprjname[0] + '/project/' + \ pack.upstreamprjname[0:2] + '/' + pack.upstreamprjname + '/' + pack.upstreampkgname + '/' def dofixrule(self, dirlist, pack): if not pack.fixrule: return dirlist newdirlist = [] for e in dirlist: newdirlist.append(re.sub(pack.fixrule[0], pack.fixrule[1], e)) return newdirlist def getsourceforgelatest(self, pack, latest): self.ftp.cwd(self.getsourceforgedir(pack)) dirlist = self.ftp.nlst() dirlist = self.dofixrule(dirlist, pack) vrs = [] for entry in dirlist: for sep in ['-', '_', '.']: prefix = pack.upstreampkgname + sep if entry[:len(prefix)] == prefix: entry = entry[len(prefix):] break dummyname, version, release = makenvr.make_nvr(prefix+'-'+entry+'.tar.gz', '', pack.ispost) vrs.append((version, release)) keys = self.cleanandsort(vrs) result = 'Upstream ' self.latestkey(keys, latest) # for key in keys: # self.ftp.cwd(key[0]) # dirlist = self.ftp.nlst() # dirlist = self.dofixrule(dirlist, pack) # ret = "NONE!" # result = result + str(ret)+'-1' # alignit(result, secondcol) # if sortkeys(latest, (ret,'1')) < 0: # print 'NOT UPTODATE' # else: # print 'UpToDate' # break # self.ftp.cwd('..') # # # dirlist = self.ftp.nlst() # if debug: print 'dirlist is', dirlist # dirlist = self.dofixrule(dirlist, pack) # self.getsimplelatest(pack.upstreampkgname, latest, dirlist, pack.ispost, False) packagelist = [] #http regexp packagelist.append(package('http-regexp', 'hunspell-wa', 'aspell-wa', \ 'aspell-wa', 'chanae.walon.org', '/walon/aspell.php', \ r'.*?href="aspell-wa-(.*?).tar.bz2">')) packagelist.append(package('http-regexp', 'hunspell-uz', 'uzbek-wordlist', \ 'uzbek-wordlist', 'www-user.uni-bremen.de', '/~kmashrab/uzbek-word-list/', \ r'.*?href="uzbek-wordlist-(.*?).tar.bz2">')) packagelist.append(package('http-regexp', 'hunspell-sq', 'myspell-sq_AL', \ 'myspell-sq_AL', 'www.shkenca.org', '/k6i/albanian_dictionary_for_myspell_en.html', \ r'.*?href="http://www.shkenca.org/shkarkime/myspell-sq_AL-(.*?).zip">')) packagelist.append(package('http-regexp', 'hunspell-mi', 'hunspell-mi', \ 'hunspell-mi', 'packages.papakupu.maori.nz', '/hunspell/', \ r'.*?href="hunspell-mi-(.*?).tar.gz">')) packagelist.append(package('http-regexp', 'hyphen-mi', 'hyphen-mi', \ 'hyphen-mi', 'packages.papakupu.maori.nz', '/hunspell-hyphen/', \ r'.*?href="hunspell-hyphen-mi-(.*?).tar.gz">')) packagelist.append(package('http-regexp', 'mythes-mi', 'mythes-mi', \ 'mythes-mi', 'packages.papakupu.maori.nz', '/mythes/', \ r'.*?href="mythes-mi-(.*?).tar.gz">')) packagelist.append(package('http-regexp', 'mythes-ne', 'mythes-ne', \ 'mythes-ne', 'hg.services.openoffice.org', '/hg/DEV300/raw-file/tip/dictionaries/ne_NP/README_th_ne_NP_v2.txt', \ r'.*?Release (.*?) ')) packagelist.append(package('http-regexp', 'hunspell-fur', 'myspell-fur', \ 'myspell-fur', 'digilander.libero.it', '/paganf/coretors/dizionaris.html', \ r'.*?href="myspell-fur-(.*?).zip">')) packagelist.append(package('http-regexp', 'fribidi', 'fribidi', \ 'fribidi', 'fribidi.org', '/download/', \ r'.*?href="fribidi-(.*?).tar.gz">')) packagelist.append(package('http-regexp', 'libtextcat', 'libtextcat', \ 'libtextcat', 'software.wise-guys.nl', '/libtextcat/', \ r'.*?href="/download/libtextcat-(.*?).tar.gz">')) packagelist.append(package('http-regexp', 'sac', 'sac', \ 'sac', 'www.w3.org', '/Style/CSS/SAC/', \ r'.*?sacjava-(.*?).zip')) packagelist.append(package('http-regexp', 'agg', 'agg', \ 'agg', 'www.antigrain.com', '/download/', \ r'.*?http://www.antigrain.com/agg-(.*?).tar.gz')) packagelist.append(package('http-regexp', 'icu', 'icu', \ 'icu', 'site.icu-project.org', '/download/', \ r'.*?#ICU4C".*?>(.*?)<')) packagelist.append(package('http-regexp', 'hunspell-ga', 'ispell-gaeilge', \ 'ispell-gaeilge', 'borel.slu.edu', '/ispell/sios.html', \ r'.*?ispell-gaeilge-(.*?).tar.gz')) packagelist.append(package('http-regexp', 'hunspell-sk', 'hunspell-sk', \ 'hunspell-sk', 'www.sk-spell.sk.cx', '/hunspell-sk', \ r'.*?hunspell-sk-(\d\d\d\d\d\d\d\d).zip')) packagelist.append(package('http-regexp', 'hunspell-da', 'myspell-da', \ 'myspell-da', 'da.speling.org', '/filer/', \ r'.*?a href="myspell-da-(.*?).tar.gz">')) packagelist.append(package('http-regexp', 'hunspell-fo', 'myspell-fo', \ 'myspell-fo', 'fo.speling.org', '/filer/', \ r'.*?a href="myspell-fo-(.*?).tar.gz">')) packagelist.append(package('http-regexp', 'hyphen-hu', 'huhyphn', \ 'huhyphn', 'www.tipogral.hu', '/', \ r'.*?hungarian hyphenation patterns v(.*?)')) packagelist.append(package('http-regexp', 'hunspell-ur', 'UrduDictionary', \ 'UrduDictionary', 'urdudictionary.codeplex.com', '/', \ r'.*?>Release (.*?)<')) packagelist.append(package('http-regexp', 'hunspell-af', 'dict_af', \ 'dict_af', 'extensions.services.openoffice.org', '/project/dict_af', \ r'.*?Download extension.*?(.*?)')) packagelist.append(package('http-regexp', 'hunspell-nr', 'dict-nr', \ 'dict-nr', 'extensions.services.openoffice.org', '/project/dict-nr', \ r'.*?Download extension.*?(.*?)')) packagelist.append(package('http-regexp', 'hunspell-ss', 'dict-ss', \ 'dict-ss', 'extensions.services.openoffice.org', '/project/dict-ss', \ r'.*?Download extension.*?(.*?)')) packagelist.append(package('http-regexp', 'hunspell-st', 'dict-st', \ 'dict-st', 'extensions.services.openoffice.org', '/project/dict-st', \ r'.*?Download extension.*?(.*?)')) packagelist.append(package('http-regexp', 'hunspell-ts', 'dict-ts', \ 'dict-ts', 'extensions.services.openoffice.org', '/project/dict-ts', \ r'.*?Download extension.*?(.*?)')) packagelist.append(package('http-regexp', 'hunspell-tn', 'dict-tn', \ 'dict-tn', 'extensions.services.openoffice.org', '/project/dict-tn', \ r'.*?Download extension.*?(.*?)')) packagelist.append(package('http-regexp', 'hunspell-ve', 'dict-ve', \ 'dict-ve', 'extensions.services.openoffice.org', '/project/dict-ve', \ r'.*?Download extension.*?(.*?)')) packagelist.append(package('http-regexp', 'hunspell-xh', 'dict-xh', \ 'dict-xh', 'extensions.services.openoffice.org', '/project/dict-xh', \ r'.*?Download extension.*?(.*?)')) packagelist.append(package('http-regexp', 'hunspell-nso', 'dict-ns_ZA', \ 'dict-ns_ZA', 'extensions.services.openoffice.org', '/project/dict-nso', \ r'.*?Download extension.*?(.*?)')) packagelist.append(package('http-regexp', 'hunspell-zu', 'dict-zu', \ 'dict-zu', 'extensions.services.openoffice.org', '/project/dict-zu', \ r'.*?Download extension.*?(.*?)')) packagelist.append(package('http-regexp', 'hunspell-de', 'igerman98', \ 'igerman98', 'www.j3e.de', '/ispell/igerman98/dict/', \ r'.*?(.*?) packagelist.append(package('http-regexp', 'hunspell-pt', 'pt_BR', \ 'pt_BR', 'www.broffice.org', '/verortografico/baixar/', \ r'.*?files/pt_BR-(\d\d\d\d-\d\d-\d\d)')) packagelist.append(package('http-regexp', 'hunspell-pt', 'pt_PT', \ 'pt_PT', 'natura.di.uminho.pt', '/download/sources/Dictionaries/hunspell/', \ r'.*?hunspell-pt_PT-(\d\d\d\d\d\d\d\d)')) packagelist.append(package('http-regexp', 'hunspell-vi', 'hunspell-spellcheck-vi', \ 'hunspell-spellcheck-vi', 'code.google.com', \ '/p/hunspell-spellcheck-vi/downloads/detail?name=vi_VN.zip', \ r'.*?(.*?)', False, "%b %d, %Y")) packagelist.append(package('http-regexp', 'hunspell-en', 'en_GB', \ 'en_GB', 'wiki.services.openoffice.org', '/wiki/Dictionaries', \ r'.*?en_GB.zip" rel="nofollow">English \(United Kingdom\) (\d\d\d\d-\d\d-\d\d)')) packagelist.append(package('http-regexp', 'hunspell-rw', 'rw_RW', \ 'rw_RW', 'wiki.services.openoffice.org', '/wiki/Dictionaries', \ r'.*?rw_RW.zip" rel="nofollow">Spelling (\d\d\d\d-\d\d-\d\d)')) packagelist.append(package('http-regexp', 'hunspell-mg', 'mg_MG', \ 'mg_MG', 'wiki.services.openoffice.org', '/wiki/Dictionaries', \ r'.*?mg_MG.zip" rel="nofollow">Spelling (\d\d\d\d-\d\d-\d\d)')) packagelist.append(package('http-regexp', 'hunspell-id', 'id_ID', \ 'id_ID', 'wiki.services.openoffice.org', '/wiki/Dictionaries', \ r'.*?id_ID.zip" rel="nofollow">Spelling (\d\d\d\d-\d\d-\d\d)')) packagelist.append(package('http-regexp', 'hunspell-th', 'th_TH', \ 'th_TH', 'wiki.services.openoffice.org', '/wiki/Dictionaries', \ r'.*?th_TH.zip" rel="nofollow">Spelling (\d\d\d\d-\d\d-\d\d)')) packagelist.append(package('http-regexp', 'hunspell-sl', 'sl_SI', \ 'sl_SI', 'wiki.services.openoffice.org', '/wiki/Dictionaries', \ r'.*?sl_SI.zip" rel="nofollow">Spelling (\d\d\d\d-\d\d-\d\d)')) packagelist.append(package('http-regexp', 'hunspell-cy', 'cy_GB', \ 'cy_GB', 'wiki.services.openoffice.org', '/wiki/Dictionaries', \ r'.*?cy_GB.zip" rel="nofollow">Spelling (\d\d\d\d-\d\d-\d\d)')) packagelist.append(package('http-regexp', 'hunspell-ms', 'ms_MY', \ 'ms_MY', 'wiki.services.openoffice.org', '/wiki/Dictionaries', \ r'.*?ms_MY.zip" rel="nofollow">Spelling (\d\d\d\d-\d\d-\d\d)')) packagelist.append(package('http-regexp', 'hunspell-no', 'nb_NO', \ 'spell-norwegian', 'alioth.debian.org', '/frs/?group_id=30577', \ r'.*?spell-norwegian-(.*?).tar.gz')) packagelist.append(package('http-regexp', 'hunspell-mk', 'mk_MK', \ 'mk_MK', 'wiki.services.openoffice.org', '/wiki/Dictionaries', \ r'.*?mk_MK.zip" rel="nofollow">Spelling (\d\d\d\d-\d\d-\d\d)')) packagelist.append(package('http-regexp', 'hunspell-tl', 'tl_PH', \ 'tl_PH', 'wiki.services.openoffice.org', '/wiki/Dictionaries', \ r'.*?tl_PH.zip" rel="nofollow">Spelling (\d\d\d\d-\d\d-\d\d)')) packagelist.append(package('http-regexp', 'mythes-cs', 'thes_cs_CZ_v2', \ 'thes_cs_CZ_v2', 'wiki.services.openoffice.org', '/wiki/Dictionaries', \ r'.*?thes_cs_CZ_v2.zip" rel="nofollow">Thesaurus .*? (\d\d\d\d-\d\d-\d\d)')) packagelist.append(package('http-regexp', 'hyphen-sv', 'hyph_sv_SE', \ 'hyph_sv_SE', 'extensions.services.openoffice.org', '/node/1968', \ r'.*?>Swedish hyphenation (.*?)')) packagelist.append(package('http-regexp', 'mythes-sv', 'SweThes', \ 'SweThes', 'extensions.services.openoffice.org', '/project/SweThes', \ r'.*?Download extension.*?(.*?)')) packagelist.append(package('http-regexp', 'hyphen-ga', 'hyph_ga_IE', \ 'hyph_ga_IE', 'wiki.services.openoffice.org', '/wiki/Dictionaries', \ r'.*?hyph_ga_IE.zip" rel="nofollow">Hyphenation \(version .*?\) (\d\d\d\d-\d\d-\d\d)')) packagelist.append(package('http-regexp', 'mythes-ga', 'thes_ga_IE', \ 'thes_ga_IE', 'wiki.services.openoffice.org', '/wiki/Dictionaries', \ r'.*?thes_ga_IE_v2.zip" rel="nofollow">Thesaurus \(version .*?\) (\d\d\d\d-\d\d-\d\d)')) packagelist.append(package('http-regexp', 'mythes-el', 'thes_el_GR', \ 'thes_el_GR', 'wiki.services.openoffice.org', '/wiki/Dictionaries', \ r'.*?th_el.zip" rel="nofollow">Thesaurus (\d\d\d\d-\d\d-\d\d)')) packagelist.append(package('http-regexp', 'mythes-ru', 'thes_ru_RU', \ 'thes_ru_RU', 'wiki.services.openoffice.org', '/wiki/Dictionaries', \ r'.*?thes_ru_RU_v2.zip" rel="nofollow">Thesaurus (\d\d\d\d-\d\d-\d\d)')) packagelist.append(package('http-regexp', 'hyphen-ru', 'hyph_ru_RU', \ 'hyph_ru_RU', 'wiki.services.openoffice.org', '/wiki/Dictionaries', \ r'.*?hyph_ru_RU.zip" rel="nofollow">Hyphenation (\d\d\d\d-\d\d-\d\d)')) packagelist.append(package('http-regexp', 'hyphen-pl', 'hyph_pl_PL', \ 'hyph_pl_PL', 'wiki.services.openoffice.org', '/wiki/Dictionaries', \ r'.*?hyph_pl_PL.zip" rel="nofollow">Hyphenation (\d\d\d\d-\d\d-\d\d)')) packagelist.append(package('http-regexp', 'hyphen-nl', 'hyph_nl_NL', \ 'hyph_nl_NL', 'wiki.services.openoffice.org', '/wiki/Dictionaries', \ r'.*?hyph_nl_NL.zip" rel="nofollow">Hyphenation (\d\d\d\d-\d\d-\d\d)')) packagelist.append(package('http-regexp', 'hyphen-de', 'hyph_de_DE', \ 'hyph_de_DE', 'wiki.services.openoffice.org', '/wiki/Dictionaries', \ r'.*?hyph_de_DE.zip" rel="nofollow">German \(Germany\) (\d\d\d\d-\d\d-\d\d)')) packagelist.append(package('http-regexp', 'hyphen-da', 'hyph_da_DK', \ 'hyph_da_DK', 'wiki.services.openoffice.org', '/wiki/Dictionaries', \ r'.*?hyph_da_DK.zip" rel="nofollow">Hyphenation (\d\d\d\d-\d\d-\d\d)')) packagelist.append(package('http-regexp', 'hyphen-it', 'hyph_it_IT', \ 'hyph_it_IT', 'wiki.services.openoffice.org', '/wiki/Dictionaries', \ r'.*?hyph_it_IT.zip" rel="nofollow">Hyphenation (\d\d\d\d-\d\d-\d\d)')) packagelist.append(package('http-regexp', 'hyphen-sl', 'hyph_sl_SI', \ 'hyph_sl_SI', 'wiki.services.openoffice.org', '/wiki/Dictionaries', \ r'.*?hyph_sl_SI.zip" rel="nofollow">Hyphenation (\d\d\d\d-\d\d-\d\d)')) packagelist.append(package('http-regexp', 'hyphen-sk', 'hyph_sk_SK', \ 'hyph_sk_SK', 'wiki.services.openoffice.org', '/wiki/Dictionaries', \ r'.*?hyph_sk_SK.zip" rel="nofollow">Hyphenation (\d\d\d\d-\d\d-\d\d)')) packagelist.append(package('http-regexp', 'hyphen-el', 'hyph_el_GR', \ 'hyph_el_GR', 'wiki.services.openoffice.org', '/wiki/Dictionaries', \ r'.*?hyph_el_GR.zip" rel="nofollow">Hyphenation (\d\d\d\d-\d\d-\d\d)')) packagelist.append(package('http-regexp', 'hyphen-es', 'hyph_es_ES', \ 'hyph_es_ES', 'wiki.services.openoffice.org', '/wiki/Dictionaries', \ r'.*?hyph_es_ES.zip" rel="nofollow">Hyphenation (\d\d\d\d-\d\d-\d\d)')) packagelist.append(package('http-regexp', 'hyphen-is', 'hyph_is_IS', \ 'hyph_is_IS', 'wiki.services.openoffice.org', '/wiki/Dictionaries', \ r'.*?hyph_is_IS.zip" rel="nofollow">Hyphenation (\d\d\d\d-\d\d-\d\d)')) packagelist.append(package('http-regexp', 'hyphen-id', 'hyph_id_ID', \ 'hyph_id_ID', 'wiki.services.openoffice.org', '/wiki/Dictionaries', \ r'.*?hyph_id_ID.zip" rel="nofollow">Hyphenation (\d\d\d\d-\d\d-\d\d)')) packagelist.append(package('http-regexp', 'hyphen-uk', 'hyph_uk_UA', \ 'hyph_uk_UA', 'wiki.services.openoffice.org', '/wiki/Dictionaries', \ r'.*?hyph_uk_UA.zip" rel="nofollow">Hyphenation (\d\d\d\d-\d\d-\d\d)')) packagelist.append(package('http-regexp', 'hyphen-pt', 'hyph_pt_PT', \ 'hyph_pt_PT', 'wiki.services.openoffice.org', '/wiki/Dictionaries', \ r'.*?hyph_pt_PT.zip" rel="nofollow">.*? (\d\d\d\d-\d\d-\d\d)')) packagelist.append(package('http-regexp', 'hunspell-ber', 'hunspell-am', \ 'hunspell-am', 'ayaspell.sourceforge.net', '/am.html', \ r'.*?(\d\d\d\d-\d\d-\d\d) (.*?)')) packagelist.append(package('http-regexp', 'hunspell-la', 'dict-la', \ 'dict-la', 'extensions.services.openoffice.org', '/project/dict-la', \ r'.*?Download.*?a href=.*?>(.*?)')) packagelist.append(package('http-regexp', 'hunspell-sw', 'sw_TZ', \ 'sw_TZ', 'wiki.services.openoffice.org', '/wiki/Dictionaries', \ r'.*?sw_TZ.*?Spelling .*? (\d\d\d\d-\d\d-\d\d)')) packagelist.append(package('http-regexp', 'hunspell-cop', 'copt_dict', \ 'copt_dict', 'extensions.services.openoffice.org', '/project/copt_dict', \ r'.*?/node/4579">(.*?)')) packagelist.append(package('http-regexp', 'mythes-ca', 'thesaurus-ca', \ 'thesaurus-ca', 'extensions.services.openoffice.org', '/project/thesaurus-ca', \ r'.*?Download extension.*?(.*?)')) packagelist.append(package('http-regexp', 'mythes-da', 'danske_synonymer', \ 'danske_synonymer', 'extensions.services.openoffice.org', '/project/danske_synonymer', \ r'.*?Download extension.*?(.*?)')) packagelist.append(package('http-regexp', 'hunspell-kk', 'dict-kk', \ 'dict-kk', 'extensions.services.openoffice.org', '/node/1172/release', \ r'.*?This (.*?) release')) packagelist.append(package('http-regexp', 'hunspell-tet', 'tet_ID', \ 'tet_ID', 'wiki.services.openoffice.org', '/wiki/Dictionaries', \ r'.*?tet_ID.*?Spelling (\d\d\d\d-\d\d-\d\d)')) packagelist.append(package('http-regexp', 'hunspell-so', 'Alif-Higgaad-saxe', \ 'Alif-Higgaad-saxe', 'extensions.services.openoffice.org', '/project/Alif-Higgaad-saxe', \ r'.*?Download extension.*?(.*?)')) packagelist.append(package('http-regexp', 'hunspell-sr', 'dict-sr', \ 'dict-sr', 'extensions.services.openoffice.org', '/project/dict-sr', \ r'.*?Download extension.*?(.*?)')) packagelist.append(package('http-regexp', 'hyphen-ku', 'kitandin', \ 'kitandin', 'extensions.services.openoffice.org', '/project/kitandin', \ r'.*?Download extension.*?(.*?)')) packagelist.append(package('http-regexp', 'hunspell-be', 'dict-be-official', \ 'dict-be-official', 'extensions.services.openoffice.org', '/project/dict-be-official', \ r'.*?Download extension.*?(.*?)')) packagelist.append(package('http-regexp', 'hyphen-gl', 'hyphenation-gl', \ # 'hyphenation-gl', 'extensions.services.openoffice.org', '/project/Extension_Guionizador_galego_OOo', \ 'hyphenation-gl', 'extensions.services.openoffice.org', '/en/node/2014', \ # r'.*?Download extension.*?(.*?)')) r'.*?(.*?) ')) packagelist.append(package('http-regexp', 'hunspell-gd', 'gd_GB', \ 'gd_GB', 'extensions.services.openoffice.org', '/project/faclair-afb', \ r'.*?Download extension.*?(.*?)')) packagelist.append(package('http-regexp', 'hunspell-br', 'Dict-br_Drouizig', \ 'Dict-br_Drouizig', 'extensions.services.openoffice.org', '/project/Dict-br_Drouizig', \ r'.*?Download extension.*?(.*?)')) packagelist.append(package('http-regexp', 'hunspell-is', 'dict-is', \ 'dict-is', 'extensions.services.openoffice.org', '/project/dict-is', \ r'.*?Download extension.*?(.*?)')) packagelist.append(package('http-regexp', 'hyphen-ca', 'ca_hyph', \ 'ca_hyph', 'extensions.services.openoffice.org', '/project/ca_hyph', \ r'.*?Download extension.*?(.*?)')) packagelist.append(package('http-regexp', 'mythes-hu', 'hu_dicts', \ 'hu_dicts', 'extensions.services.openoffice.org', '/project/hu_dicts', \ r'.*?Download extension.*?(.*?)')) packagelist.append(package('http-regexp', 'hunspell-grc', 'grc', \ 'grc', 'extensions.services.openoffice.org', '/project/ancientgreekspellchecker', \ r'.*?Download extension.*?(.*?)')) packagelist.append(package('http-regexp', 'hunspell-dsb', 'lower_sorbian_spelling_dictionary', \ 'lower_sorbian_spelling_dictionary', 'extensions.services.openoffice.org', '/project/HunSpellDic_dsb_DE', \ r'.*?Download extension.*?(.*?)')) packagelist.append(package('http-regexp', 'hunspell-qu', 'qu_EC', \ 'qu_EC', 'extensions.services.openoffice.org', '/project/KichwaSpellchecker', \ r'.*?Download extension.*?(.*?)')) packagelist.append(package('http-regexp', 'hunspell-ht', 'kok', \ 'kok', 'extensions.services.openoffice.org', '/project/kok', \ r'.*?Download extension.*?(.*?)')) packagelist.append(package('http-regexp', 'hunspell-eo', 'literumilo', \ 'literumilo', 'extensions.services.openoffice.org', '/project/literumilo', \ r'.*?Download extension.*?(.*?)')) packagelist.append(package('http-regexp', 'hunspell-ast', 'asturianu', \ 'asturianu', 'extensions.services.openoffice.org', '/project/asturianu', \ r'.*?Download extension.*?(.*?)')) packagelist.append(package('http-regexp', 'hunspell-ny', 'chicspell', \ 'chicspell', 'extensions.services.openoffice.org', '/project/chicspell', \ r'.*?Download extension.*?(.*?)')) packagelist.append(package('http-regexp', 'writer2latex', 'writer2latex', \ 'writer2latex', 'extensions.services.openoffice.org', '/en/project/writer2latex', \ r'.*?Download extension.*?(.*?)')) packagelist.append(package('http-regexp', 'hunspell-tpi', 'tok-pisin-spell-checker', \ 'tok-pisin-spell-checker', 'extensions.services.openoffice.org', '/en/project/tok-pisin-spell-checker', \ r'.*?Download extension.*?(.*?)')) packagelist.append(package('http-regexp', 'hunspell-cv', 'hunspell-cv', \ 'hunspell-cv', 'hunspell.chv.su', '/download.shtml', \ r'.*?>dict-cv-(.*?).oxt', fixrule=('-u',''))) packagelist.append(package('http-regexp', 'mythes-pt', 'thes_pt_PT_v2', \ 'thes_pt_PT_v2', 'openthesaurus.caixamagica.pt', '/', \ r'.*?(\d\d\d\d-\d\d-\d\d)\)', 1)) packagelist.append(package('http-regexp', 'hyphen-tk', 'hyph-tk', \ 'hyph-tk', 'tug.org', '/svn/texhyphen/trunk/hyph-utf8/tex/generic/hyph-utf8/patterns/tex/hyph-tk.tex?view=log', \ r'.*?(.*) UTC', datestring="%a %b %d %H:%M:%S %Y")) packagelist.append(package('http-regexp', 'hyphen-mn', 'hyph-mn-cyrl', \ 'hyph-mn-cyrl', 'tug.org', '/svn/texhyphen/trunk/hyph-utf8/tex/generic/hyph-utf8/patterns/tex/hyph-mn-cyrl.tex?view=log', \ r'.*?(.*) UTC', datestring="%a %b %d %H:%M:%S %Y")) packagelist.append(package('http-regexp', 'hyphen-hsb', 'hyph-hsb', \ 'hyph-hsb', 'tug.org', '/svn/texhyphen/trunk/hyph-utf8/tex/generic/hyph-utf8/patterns/tex/hyph-hsb.tex?view=log', \ r'.*?(.*) UTC', datestring="%a %b %d %H:%M:%S %Y")) packagelist.append(package('http-regexp', 'hyphen-sa', 'hyph-sa', \ 'hyph-sa', 'tug.org', '/svn/texhyphen/trunk/hyph-utf8/tex/generic/hyph-utf8/patterns/tex/hyph-sa.tex?view=log', \ r'.*?(.*) UTC', datestring="%a %b %d %H:%M:%S %Y")) packagelist.append(package('http-regexp', 'hyphen-eu', 'hyph-eu', \ 'hyph-eu', 'tug.org', '/svn/texhyphen/trunk/hyph-utf8/tex/generic/hyph-utf8/patterns/tex/hyph-eu.tex?view=log', \ r'.*?(.*) UTC', datestring="%a %b %d %H:%M:%S %Y")) packagelist.append(package('http-regexp', 'hyphen-cy', 'hyph-cy', \ 'hyph-cy', 'tug.org', '/svn/texhyphen/trunk/hyph-utf8/tex/generic/hyph-utf8/patterns/tex/hyph-cy.tex?view=log', \ r'.*?(.*) UTC', datestring="%a %b %d %H:%M:%S %Y")) packagelist.append(package('http-regexp', 'hyphen-grc', 'hyph-grc', \ 'hyph-grc', 'tug.org', '/svn/texhyphen/trunk/hyph-utf8/tex/generic/hyph-utf8/patterns/tex/hyph-grc.tex?view=log', \ r'.*?(.*) UTC', datestring="%a %b %d %H:%M:%S %Y")) packagelist.append(package('http-regexp', 'hyphen-lt', 'hyph-lt', \ 'hyph-lt', 'tug.org', '/svn/texhyphen/trunk/hyph-utf8/tex/generic/hyph-utf8/patterns/tex/hyph-lt.tex?view=log', \ r'.*?(.*) UTC', datestring="%a %b %d %H:%M:%S %Y")) packagelist.append(package('http-regexp', 'hyphen-fa', 'fahyph', \ 'fahyph', 'ftp.heanet.ie', '/pub/CTAN/tex/language/hyphenation/', \ r'.*?fahyph.zip.*?(\d\d-...-\d\d\d\d)', datestring="%d-%b-%Y")) packagelist.append(package('http-regexp', 'hyphen-ia', 'iahyphen', \ 'iahyphen', 'ftp.heanet.ie', '/pub/CTAN/tex/language/hyphenation/', \ r'.*?iahyphen.tex.*?(\d\d-...-\d\d\d\d)', datestring="%d-%b-%Y")) packagelist.append(package('http-regexp', 'hunspell-shs', 'hunspell-shs', \ 'hunspell', 'secpewt.sd73.bc.ca', '/hunspell/', \ r'.*?hunspell-shs-ca.tar.gz.*?(\d\d-...-\d\d\d\d)', datestring="%d-%b-%Y")) packagelist.append(package('http-regexp', 'hunspell-fr', 'hunspell_fr', \ 'hunspell_fr', 'www.dicollecte.org', '/download.php?prj=fr', \ r'.*?hunspell.fr.classique.reforme1990-v(.*?).zip')) packagelist.append(package('http-regexp', 'openoffice-lv', 'dict_lv_LV', \ 'dict_lv_LV', 'dict.dv.lv', '/download.php?prj=lv', \ r'.*?lv_LV(.*?).oxt')) packagelist.append(package('http-regexp', 'hyphen-fr', 'hyph_fr_FR', \ 'hyph_fr_FR', 'www.dicollecte.org', '/download.php?prj=fr', \ r'.*?hyph_fr_(.*?).zip')) packagelist.append(package('http-regexp', 'mythes-fr', 'thesaurus', \ 'thesaurus', 'www.dicollecte.org', '/download.php?prj=fr', \ r'.*?thesaurus-v(.*?).zip')) packagelist.append(package('http-regexp', 'hunspell-se', 'hunspell-se', \ 'hunspell-se', 'www.divvun.no', '/index.html', \ r'.*\s+(.*?):.*?libbase-(.*?).zip<')) packagelist.append(package('http-regexp', 'libserializer', 'jfreereport', \ 'libserializer', 'www.mirrorservice.org', '/sites/download.sourceforge.net/pub/sourceforge/j/project/jf/jfreereport/02.%20Libraries/1.1.2-stable/', \ r'.*?>libserializer-(.*?).zip<')) packagelist.append(package('http-regexp', 'libloader', 'jfreereport', \ 'libloader', 'www.mirrorservice.org', '/sites/download.sourceforge.net/pub/sourceforge/j/project/jf/jfreereport/02.%20Libraries/1.1.2-stable/', \ r'.*?>libloader-(.*?).zip<')) packagelist.append(package('http-regexp', 'pentaho-libxml', 'jfreereport', \ 'libxml', 'www.mirrorservice.org', '/sites/download.sourceforge.net/pub/sourceforge/j/project/jf/jfreereport/02.%20Libraries/1.1.2-stable/', \ r'.*?>libxml-(.*?).zip<')) packagelist.append(package('http-regexp', 'libfonts', 'jfreereport', \ 'libfonts', 'www.mirrorservice.org', '/sites/download.sourceforge.net/pub/sourceforge/j/project/jf/jfreereport/02.%20Libraries/1.1.2-stable/', \ r'.*?>libfonts-(.*?).zip<')) packagelist.append(package('http-regexp', 'libformula', 'jfreereport', \ 'libformula', 'www.mirrorservice.org', '/sites/download.sourceforge.net/pub/sourceforge/j/project/jf/jfreereport/02.%20Libraries/1.1.2-stable/', \ r'.*?>libformula-(.*?).zip<')) packagelist.append(package('http-regexp', 'librepository', 'jfreereport', \ 'librepository', 'www.mirrorservice.org', '/sites/download.sourceforge.net/pub/sourceforge/j/project/jf/jfreereport/02.%20Libraries/1.1.2-stable/', \ r'.*?>librepository-(.*?).zip<')) packagelist.append(package('http-regexp', 'flute', 'jfreereport', \ 'flute', 'www.mirrorservice.org', '/sites/download.sourceforge.net/pub/sourceforge/j/project/jf/jfreereport/30.%20OpenOffice%20Branches/OOo31/', \ r'.*?>flute-(.*?).zip<', ispost=1)) packagelist.append(package('http-regexp', 'liblayout', 'jfreereport', \ 'liblayout', 'www.mirrorservice.org', '/sites/download.sourceforge.net/pub/sourceforge/j/project/jf/jfreereport/30.%20OpenOffice%20Branches/OOo31/', \ r'.*?>liblayout-(.*?).zip<', ispost=1)) packagelist.append(package('http-regexp', 'pentaho-reporting-flow-engine', 'jfreereport', \ 'flow-engine', 'www.mirrorservice.org', '/sites/download.sourceforge.net/pub/sourceforge/j/project/jf/jfreereport/30.%20OpenOffice%20Branches/OOo31/', \ r'.*?>flow-engine-(.*?).zip<', ispost=1)) packagelist.append(package('http-regexp', 'jcommon-serializer', 'jfreereport', \ 'jcommon-serializer', 'www.mirrorservice.org', '/sites/download.sourceforge.net/pub/sourceforge/j/project/jf/jfreereport/99.%20Archive/07.%20JCommon-Serializer/', \ r'.*/sites/download.*?">(.*)', fixrule=('-GA',''))) packagelist.append(package('http-regexp', 'hunspell-lb', 'SpellcheckerLu', \ 'SpellcheckerLu', 'spellchecker.lu', '/download/openoffice/', \ r'.*?Leschten Update: (.*)')) #packagelist = [] #packagelist.append(package('sourceforge', 'hunspell', 'hunspell', 'Hunspell')) socket.setdefaulttimeout(120) # timeout in seconds task = packagemanager(packagelist) task.fetch() if len(droppedlist): print "No SRPMS seen for", for dropped in droppedlist: print dropped, print