mtcontrib
/
minetest-u_skinsdb
mirror of https://github.com/SmallJoker/minetest-u_skinsdb.git
1
0
Fork 2
Code Issues Releases Wiki Activity

fixed pipelining 

cleaning everything up, working around python's stupidity for capturing loop variables (it was saving every image as ...preview_584.png), testing it on 1-9 to make sure they come in as a single block of drains. It works, yay!
This commit is contained in:
user 2015-09-24 20:41:32 +00:00
parent 7778dc5560
commit 057b4cce3c
1 changed files with 20 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
update_from_db.py
View File

@ -6,8 +6,8 @@ from contextlib import closing
import sys,os,shutil,time import sys,os,shutil,time
def die(message,code=23): def die(message,code=23):
print(message,file=sys.stderr) print(message,file=sys.stderr)
raise SystemExit(code) raise SystemExit(code)
server = "minetest.fensta.bplaced.net" server = "minetest.fensta.bplaced.net"
skinsdir = "u_skins/textures/" skinsdir = "u_skins/textures/"
@ -65,6 +65,7 @@ class Pipeline(list):
self.drain() self.drain()
def trydrain(self): def trydrain(self):
for penguin in self: for penguin in self:
print('drain',penguin.url)
try: try:
penguin.response.begin() penguin.response.begin()
penguin.recv(penguin.response) penguin.recv(penguin.response)
@ -73,10 +74,10 @@ class Pipeline(list):
return False return False
except HTTPException as e: except HTTPException as e:
die(penguin.diemessage+' '+repr(e)+' (url='+penguin.url+')') die(penguin.diemessage+' '+repr(e)+' (url='+penguin.url+')')
self.clear() self.clear()
return True return True
def drain(self): def drain(self):
print('draining pipeline...') print('draining pipeline...',len(self))
assert self.sent, "Can't drain without sending the requests!" assert self.sent, "Can't drain without sending the requests!"
self.sent = False self.sent = False
while self.trydrain() is not True: while self.trydrain() is not True:
@ -86,6 +87,7 @@ class Pipeline(list):
self.reopen() self.reopen()
def trysend(self): def trysend(self):
for penguin in pipeline: for penguin in pipeline:
print('fill',penguin.url)
try: try:
self.c.request("GET", penguin.url) self.c.request("GET", penguin.url)
self.c._HTTPConnection__state = _CS_IDLE self.c._HTTPConnection__state = _CS_IDLE
@ -99,7 +101,7 @@ class Pipeline(list):
return True return True
def send(self): def send(self):
if self.sent: return if self.sent: return
print('filling pipeline...') print('filling pipeline...',len(self))
while self.trysend() is not True: while self.trysend() is not True:
self.c.close() self.c.close()
print('derped resending') print('derped resending')
@ -152,15 +154,19 @@ with Pipeline() as pipeline:
f.write(str(s["author"]) + '\n') f.write(str(s["author"]) + '\n')
f.write(str(s["license"])) f.write(str(s["license"]))
url = "/skins/1/" + str(s["id"]) + ".png" url = "/skins/1/" + str(s["id"]) + ".png"
def tryget(r): def closure(skinsdir,previewbase,preview,s):
if r.status != 200: "explanation: python sucks"
print("Error", r.status) def tryget(r):
return print('replacing',s["id"])
@replace(skinsdir,previewbase,path=preview) if r.status != 200:
def go(f): print("Error", r.status)
shutil.copyfileobj(r,f) return
@replace(skinsdir,previewbase,path=preview)
def go(f):
shutil.copyfileobj(r,f)
return tryget
pipeline.append(url,tryget, pipeline.append(url,closure(skinsdir,previewbase,preview,s),
"Couldn't get {} because of a".format( "Couldn't get {} because of a".format(
s["id"])) s["id"]))
if not foundOne: if not foundOne: