mtcontrib/minetest-u_skinsdb
1
0
Fork 2
mirror of https://github.com/SmallJoker/minetest-u_skinsdb.git synced 2024-09-27 06:40:17 +02:00
Code Issues Releases Wiki Activity

fixed pipelining

Browse Source 
cleaning everything up, working around python's stupidity for capturing loop variables (it was saving every image as ...preview_584.png), testing it on 1-9 to make sure they come in as a single block of drains. It works, yay!
This commit is contained in:
user 2015-09-24 20:41:32 +00:00
parent 7778dc5560
commit 057b4cce3c
1 changed files with 20 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
update_from_db.py
View File

@ -65,6 +65,7 @@ class Pipeline(list):
self.drain() self.drain()
def trydrain(self): def trydrain(self):
for penguin in self: for penguin in self:
print('drain',penguin.url)
try: try:
penguin.response.begin() penguin.response.begin()
penguin.recv(penguin.response) penguin.recv(penguin.response)
@ -76,7 +77,7 @@ class Pipeline(list):
self.clear() self.clear()
return True return True
def drain(self): def drain(self):
print('draining pipeline...') print('draining pipeline...',len(self))
assert self.sent, "Can't drain without sending the requests!" assert self.sent, "Can't drain without sending the requests!"
self.sent = False self.sent = False
while self.trydrain() is not True: while self.trydrain() is not True:
@ -86,6 +87,7 @@ class Pipeline(list):
self.reopen() self.reopen()
def trysend(self): def trysend(self):
for penguin in pipeline: for penguin in pipeline:
print('fill',penguin.url)
try: try:
self.c.request("GET", penguin.url) self.c.request("GET", penguin.url)
self.c._HTTPConnection__state = _CS_IDLE self.c._HTTPConnection__state = _CS_IDLE
@ -99,7 +101,7 @@ class Pipeline(list):
return True return True
def send(self): def send(self):
if self.sent: return if self.sent: return
print('filling pipeline...') print('filling pipeline...',len(self))
while self.trysend() is not True: while self.trysend() is not True:
self.c.close() self.c.close()
print('derped resending') print('derped resending')
@ -152,15 +154,19 @@ with Pipeline() as pipeline:
f.write(str(s["author"]) + '\n') f.write(str(s["author"]) + '\n')
f.write(str(s["license"])) f.write(str(s["license"]))
url = "/skins/1/" + str(s["id"]) + ".png" url = "/skins/1/" + str(s["id"]) + ".png"
def closure(skinsdir,previewbase,preview,s):
"explanation: python sucks"
def tryget(r): def tryget(r):
print('replacing',s["id"])
if r.status != 200: if r.status != 200:
print("Error", r.status) print("Error", r.status)
return return
@replace(skinsdir,previewbase,path=preview) @replace(skinsdir,previewbase,path=preview)
def go(f): def go(f):
shutil.copyfileobj(r,f) shutil.copyfileobj(r,f)
return tryget
pipeline.append(url,tryget, pipeline.append(url,closure(skinsdir,previewbase,preview,s),
"Couldn't get {} because of a".format( "Couldn't get {} because of a".format(
s["id"])) s["id"]))
if not foundOne: if not foundOne: