Merge aa175d8e1e
into e762283dec
This commit is contained in:
commit
05ee1b2152
|
@ -0,0 +1,2 @@
|
|||
character_*.png
|
||||
character_*.txt
|
|
@ -1,61 +1,182 @@
|
|||
#!/usr/bin/python3
|
||||
from http.client import HTTPConnection
|
||||
from http.client import HTTPConnection,HTTPException,BadStatusLine,_CS_IDLE
|
||||
import json
|
||||
import base64
|
||||
from contextlib import closing
|
||||
import sys,os,shutil,time
|
||||
|
||||
def die(message,code=23):
|
||||
print(message,file=sys.stderr)
|
||||
raise SystemExit(code)
|
||||
|
||||
server = "minetest.fensta.bplaced.net"
|
||||
skinsdir = "u_skins/textures/"
|
||||
metadir = "u_skins/meta/"
|
||||
i = 1
|
||||
pages = 1
|
||||
#curskin = 580
|
||||
#curpage = 59
|
||||
curskin = 0
|
||||
curpage = 1
|
||||
pages = None
|
||||
|
||||
c = HTTPConnection(server)
|
||||
def addpage(page):
|
||||
global i, pages
|
||||
print("Page: " + str(page))
|
||||
r = 0
|
||||
try:
|
||||
c.request("GET", "/api/get.json.php?getlist&page=" + str(page) + "&outformat=base64")
|
||||
r = c.getresponse()
|
||||
except Exception:
|
||||
if r != 0:
|
||||
if r.status != 200:
|
||||
print("Error", r.status)
|
||||
exit(r.status)
|
||||
return
|
||||
|
||||
data = r.read().decode()
|
||||
l = json.loads(data)
|
||||
if not l["success"]:
|
||||
print("Success != True")
|
||||
exit(1)
|
||||
r = 0
|
||||
pages = int(l["pages"])
|
||||
for s in l["skins"]:
|
||||
f = open(skinsdir + "character_" + str(i) + ".png", "wb")
|
||||
f.write(base64.b64decode(bytes(s["img"], 'utf-8')))
|
||||
f.close()
|
||||
f = open(metadir + "character_" + str(i) + ".txt", "w")
|
||||
f.write(str(s["name"]) + '\n')
|
||||
f.write(str(s["author"]) + '\n')
|
||||
f.write(str(s["license"]))
|
||||
f.close()
|
||||
try:
|
||||
c.request("GET", "/skins/1/" + str(s["id"]) + ".png")
|
||||
r = c.getresponse()
|
||||
except Exception:
|
||||
if r != 0:
|
||||
if r.status != 200:
|
||||
print("Error", r.status)
|
||||
continue
|
||||
|
||||
data = r.read()
|
||||
f = open(skinsdir + "character_" + str(i) + "_preview.png", "wb")
|
||||
f.write(data)
|
||||
f.close()
|
||||
i = i + 1
|
||||
addpage(1)
|
||||
if pages > 1:
|
||||
for p in range(pages-1):
|
||||
addpage(p+2)
|
||||
print("Skins have been updated!")
|
||||
def replace(location,base,encoding=None,path=None):
|
||||
if path is None:
|
||||
path = os.path.join(location,base)
|
||||
mode = "wt" if encoding else "wb"
|
||||
# an unpredictable temp name only needed for a+rwxt directories
|
||||
tmp = os.path.join(location,'.'+base+'-tmp')
|
||||
def deco(handle):
|
||||
with open(tmp,mode,encoding=encoding) as out:
|
||||
handle(out)
|
||||
os.rename(tmp,path)
|
||||
return deco
|
||||
|
||||
def maybeReplace(location,base,encoding=None):
|
||||
def deco(handle):
|
||||
path = os.path.join(location,base)
|
||||
if os.path.exists(path): return
|
||||
return replace(location,base,encoding=encoding,path=path)(handle)
|
||||
return deco
|
||||
|
||||
class Penguin:
|
||||
"idk"
|
||||
def __init__(self, url, recv, diemessage):
|
||||
self.url = url
|
||||
self.recv = recv
|
||||
self.diemessage = diemessage
|
||||
|
||||
class Pipeline(list):
|
||||
"Gawd why am I being so elaborate?"
|
||||
def __init__(self, threshold=10):
|
||||
"threshold is how many requests in parallel to pipeline"
|
||||
self.threshold = threshold
|
||||
self.sent = True
|
||||
def __enter__(self):
|
||||
self.reopen()
|
||||
return self
|
||||
def __exit__(self,typ,exn,trace):
|
||||
self.send()
|
||||
self.drain()
|
||||
def reopen(self):
|
||||
self.c = HTTPConnection(server)
|
||||
self.send()
|
||||
def append(self,url,recv,diemessage):
|
||||
self.sent = False
|
||||
super().append(Penguin(url,recv,diemessage))
|
||||
if len(self) > self.threshold:
|
||||
self.send()
|
||||
self.drain()
|
||||
def trydrain(self):
|
||||
for penguin in self:
|
||||
print('drain',penguin.url)
|
||||
try:
|
||||
penguin.response.begin()
|
||||
penguin.recv(penguin.response)
|
||||
except BadStatusLine as e:
|
||||
print('derped requesting',penguin.url)
|
||||
return False
|
||||
except HTTPException as e:
|
||||
die(penguin.diemessage+' '+repr(e)+' (url='+penguin.url+')')
|
||||
self.clear()
|
||||
return True
|
||||
def drain(self):
|
||||
print('draining pipeline...',len(self))
|
||||
assert self.sent, "Can't drain without sending the requests!"
|
||||
self.sent = False
|
||||
while self.trydrain() is not True:
|
||||
self.c.close()
|
||||
print('drain failed, trying again')
|
||||
time.sleep(1)
|
||||
self.reopen()
|
||||
def trysend(self):
|
||||
for penguin in pipeline:
|
||||
print('fill',penguin.url)
|
||||
try:
|
||||
self.c.request("GET", penguin.url)
|
||||
self.c._HTTPConnection__state = _CS_IDLE
|
||||
penguin.response = self.c.response_class(self.c.sock,
|
||||
method="GET")
|
||||
# begin LATER so we can send multiple requests w/out response headers
|
||||
except BadStatusLine:
|
||||
return False
|
||||
except HTTPException as e:
|
||||
die(diemessage+' because of a '+repr(e))
|
||||
return True
|
||||
def send(self):
|
||||
if self.sent: return
|
||||
print('filling pipeline...',len(self))
|
||||
while self.trysend() is not True:
|
||||
self.c.close()
|
||||
print('derped resending')
|
||||
time.sleep(1)
|
||||
self.reopen()
|
||||
self.sent = True
|
||||
|
||||
with Pipeline() as pipeline:
|
||||
# two connections is okay, right? one for json, one for preview images
|
||||
c = HTTPConnection(server)
|
||||
def addpage(page):
|
||||
global curskin, pages
|
||||
print("Page: " + str(page))
|
||||
r = 0
|
||||
try:
|
||||
c.request("GET", "/api/get.json.php?getlist&page=" + str(page) + "&outformat=base64")
|
||||
r = c.getresponse()
|
||||
except Exception:
|
||||
if r != 0:
|
||||
if r.status != 200:
|
||||
die("Error", r.status)
|
||||
return
|
||||
|
||||
data = r.read().decode()
|
||||
l = json.loads(data)
|
||||
if not l["success"]:
|
||||
die("Success != True")
|
||||
r = 0
|
||||
pages = int(l["pages"])
|
||||
foundOne = False
|
||||
for s in l["skins"]:
|
||||
# make sure to increment this, even if the preview exists!
|
||||
curskin = curskin + 1
|
||||
previewbase = "character_" + str(curskin) + "_preview.png"
|
||||
preview = os.path.join(skinsdir, previewbase)
|
||||
if os.path.exists(preview):
|
||||
print('skin',curskin,'already retrieved')
|
||||
continue
|
||||
print('updating skin',curskin,'id',s["id"])
|
||||
foundOne = True
|
||||
@maybeReplace(skinsdir, "character_" + str(curskin) + ".png")
|
||||
def go(f):
|
||||
f.write(base64.b64decode(bytes(s["img"], 'utf-8')))
|
||||
f.close()
|
||||
|
||||
@maybeReplace(metadir, "character_" + str(curskin) + ".txt",
|
||||
encoding='utf-8')
|
||||
def go(f):
|
||||
f.write(str(s["name"]) + '\n')
|
||||
f.write(str(s["author"]) + '\n')
|
||||
f.write(str(s["license"]))
|
||||
url = "/skins/1/" + str(s["id"]) + ".png"
|
||||
def closure(skinsdir,previewbase,preview,s):
|
||||
"explanation: python sucks"
|
||||
def tryget(r):
|
||||
print('replacing',s["id"])
|
||||
if r.status != 200:
|
||||
print("Error", r.status)
|
||||
return
|
||||
@replace(skinsdir,previewbase,path=preview)
|
||||
def go(f):
|
||||
shutil.copyfileobj(r,f)
|
||||
return tryget
|
||||
|
||||
pipeline.append(url,closure(skinsdir,previewbase,preview,s),
|
||||
"Couldn't get {} because of a".format(
|
||||
s["id"]))
|
||||
if not foundOne:
|
||||
print("No skins updated on this page. Seems we're done?")
|
||||
#raise SystemExit
|
||||
addpage(curpage)
|
||||
while pages > curpage:
|
||||
curpage = curpage + 1
|
||||
addpage(curpage)
|
||||
print("Skins have been updated!")
|
||||
|
||||
|
|
Loading…
Reference in New Issue