skinsdb/updater/update_from_db.py

181 lines
4.9 KiB
Python
Raw Normal View History

2014-03-17 06:25:18 +01:00
#!/usr/bin/python3
from http.client import HTTPConnection,HTTPException,BadStatusLine,_CS_IDLE
2014-03-17 06:25:18 +01:00
import json
import base64
from contextlib import closing
import sys,os,shutil,time
def die(message,code=23):
print(message,file=sys.stderr)
raise SystemExit(code)
2014-03-17 06:25:18 +01:00
server = "minetest.fensta.bplaced.net"
skinsdir = "../textures/"
metadir = "../meta/"
curskin = 0
curpage = 1
pages = None
2014-03-17 06:25:18 +01:00
def replace(location,base,encoding=None,path=None):
if path is None:
path = os.path.join(location,base)
mode = "wt" if encoding else "wb"
# an unpredictable temp name only needed for a+rwxt directories
tmp = os.path.join(location,'.'+base+'-tmp')
def deco(handle):
with open(tmp,mode,encoding=encoding) as out:
handle(out)
os.rename(tmp,path)
return deco
def maybeReplace(location,base,encoding=None):
def deco(handle):
path = os.path.join(location,base)
if os.path.exists(path): return
return replace(location,base,encoding=encoding,path=path)(handle)
return deco
class Penguin:
"idk"
def __init__(self, url, recv, diemessage):
self.url = url
self.recv = recv
self.diemessage = diemessage
class Pipeline(list):
"Gawd why am I being so elaborate?"
def __init__(self, threshold=10):
"threshold is how many requests in parallel to pipeline"
self.threshold = threshold
self.sent = True
def __enter__(self):
self.reopen()
return self
def __exit__(self,typ,exn,trace):
self.send()
self.drain()
def reopen(self):
self.c = HTTPConnection(server)
self.send()
def append(self,url,recv,diemessage):
self.sent = False
super().append(Penguin(url,recv,diemessage))
if len(self) > self.threshold:
self.send()
self.drain()
def trydrain(self):
for penguin in self:
print('drain',penguin.url)
try:
penguin.response.begin()
penguin.recv(penguin.response)
except BadStatusLine as e:
print('derped requesting',penguin.url)
return False
except HTTPException as e:
die(penguin.diemessage+' '+repr(e)+' (url='+penguin.url+')')
self.clear()
return True
def drain(self):
print('draining pipeline...',len(self))
assert self.sent, "Can't drain without sending the requests!"
self.sent = False
while self.trydrain() is not True:
self.c.close()
print('drain failed, trying again')
time.sleep(1)
self.reopen()
def trysend(self):
for penguin in pipeline:
print('fill',penguin.url)
try:
self.c.request("GET", penguin.url)
self.c._HTTPConnection__state = _CS_IDLE
penguin.response = self.c.response_class(self.c.sock,
method="GET")
# begin LATER so we can send multiple requests w/out response headers
except BadStatusLine:
return False
except HTTPException as e:
die(diemessage+' because of a '+repr(e))
return True
def send(self):
if self.sent: return
print('filling pipeline...',len(self))
while self.trysend() is not True:
self.c.close()
print('derped resending')
time.sleep(1)
self.reopen()
self.sent = True
with Pipeline() as pipeline:
# two connections is okay, right? one for json, one for preview images
c = HTTPConnection(server)
def addpage(page):
global curskin, pages
print("Page: " + str(page))
r = 0
2014-07-23 13:46:42 +02:00
try:
c.request("GET", "/api/get.json.php?getlist&page=" + str(page) + "&outformat=base64")
2014-07-23 13:46:42 +02:00
r = c.getresponse()
except Exception:
2014-08-29 22:02:35 +02:00
if r != 0:
if r.status != 200:
die("Error", r.status)
return
data = r.read().decode()
l = json.loads(data)
if not l["success"]:
die("Success != True")
r = 0
pages = int(l["pages"])
foundOne = False
for s in l["skins"]:
# make sure to increment this, even if the preview exists!
curskin = curskin + 1
previewbase = "character_" + str(curskin) + "_preview.png"
preview = os.path.join(skinsdir, previewbase)
if os.path.exists(preview):
print('skin',curskin,'already retrieved')
continue
print('updating skin',curskin,'id',s["id"])
foundOne = True
@maybeReplace(skinsdir, "character_" + str(curskin) + ".png")
def go(f):
f.write(base64.b64decode(bytes(s["img"], 'utf-8')))
f.close()
@maybeReplace(metadir, "character_" + str(curskin) + ".txt",
encoding='utf-8')
def go(f):
f.write(str(s["name"]) + '\n')
f.write(str(s["author"]) + '\n')
f.write(str(s["license"]))
url = "/skins/1/" + str(s["id"]) + ".png"
def closure(skinsdir,previewbase,preview,s):
"explanation: python sucks"
def tryget(r):
print('replacing',s["id"])
if r.status != 200:
print("Error", r.status)
return
@replace(skinsdir,previewbase,path=preview)
def go(f):
shutil.copyfileobj(r,f)
return tryget
pipeline.append(url,closure(skinsdir,previewbase,preview,s),
"Couldn't get {} because of a".format(
s["id"]))
if not foundOne:
print("No skins updated on this page. Seems we're done?")
#raise SystemExit
addpage(curpage)
while pages > curpage:
curpage = curpage + 1
addpage(curpage)
print("Skins have been updated!")