remove all updater to solve CC-BY-NC-SA and Newtonsoft license issues
Fix for #19
This commit is contained in:
Alexander Weber
parent
fa5b3029fd
commit
169721840e
20
README.md
20
README.md
|
|
@ -4,7 +4,6 @@ This Minetest mod offers changeable player skins with a graphical interface for
|
|||
|
||||
## Features
|
||||
|
||||
- Download scripts included for the [Minetest skin database](http://minetest.fensta.bplaced.net)
|
||||
- Flexible skins API to manage the database
|
||||
- [character_creator](https://github.com/minetest-mods/character_creator) support for custom skins
|
||||
- Skin change menu for sfinv (in minetest_game) and [unified_inventory](https://forum.minetest.net/viewtopic.php?t=12767)
|
||||
|
|
@ -16,23 +15,8 @@ This Minetest mod offers changeable player skins with a graphical interface for
|
|||
- Full [3d_armor](https://forum.minetest.net/viewtopic.php?t=4654) support
|
||||
- Compatible to 1.0 and 1.8 Minecraft skins format
|
||||
|
||||
## Update tools
|
||||
|
||||
In order to download the skins from the skin database,
|
||||
you may use one of the listed update tools below.
|
||||
They are located in the `updater/` directory.
|
||||
|
||||
- `update_skins_db.sh` bash and jq required
|
||||
- `update_from_db.py` python3 required
|
||||
- `MT_skins_updater.*` windows or mono (?) required
|
||||
|
||||
|
||||
## License
|
||||
|
||||
If nothing else is specified, it is licensed as GPLv3.
|
||||
|
||||
Fritigern:
|
||||
- update_skins_db.sh (CC-BY-NC-SA 4.0)
|
||||
## License:
|
||||
- GPLv3
|
||||
|
||||
### Credits
|
||||
|
||||
|
|
|
|||
|
|
@ -1,200 +0,0 @@
|
|||
using System;
|
||||
//Json.NET library (http://json.codeplex.com/)
|
||||
using Newtonsoft.Json;
|
||||
using System.Collections.Generic;
|
||||
using System.Net;
|
||||
using System.IO;
|
||||
|
||||
// MT skins updater for the skins mod
|
||||
// Creator: Krock
|
||||
// License: zlib (http://www.zlib.net/zlib_license.html)
|
||||
namespace MT_skins_updater {
|
||||
class Program {
|
||||
static void Main(string[] args) {
|
||||
Console.WriteLine("Welcome to the MT skins updater!");
|
||||
Console.WriteLine("# Created by: Krock (2014-07-10)");
|
||||
Engine e = new Engine();
|
||||
Console.WriteLine(@"Path to the skins mod: (ex. 'E:\Minetest\mods\skinsdb\skins\')");
|
||||
string path = Console.ReadLine();
|
||||
Console.WriteLine("Start updating at page: ('0' to update everything)");
|
||||
int page = getInt(Console.ReadLine());
|
||||
e.Start(path, page);
|
||||
Console.WriteLine("Press any key to exit.");
|
||||
Console.ReadKey(false);
|
||||
}
|
||||
public static int getInt(string i) {
|
||||
int ret = 0;
|
||||
int.TryParse(i, out ret);
|
||||
return (ret > 0)? ret : 0;
|
||||
}
|
||||
}
|
||||
class Engine {
|
||||
string root = "http://minetest.fensta.bplaced.net";
|
||||
bool alternate = true; //should it use the special version of medadata saving?
|
||||
|
||||
public void Start(string path, int page) {
|
||||
if (path.Length < 5) {
|
||||
Console.WriteLine("Too short path. STOP.");
|
||||
return;
|
||||
}
|
||||
if (path[path.Length - 1] != '\\') {
|
||||
path += '\\';
|
||||
}
|
||||
if(!Directory.Exists(path + "meta")){
|
||||
Console.WriteLine("Folder 'meta' not found. STOP.");
|
||||
return;
|
||||
}
|
||||
if(!Directory.Exists(path + "textures")){
|
||||
Console.WriteLine("Folder 'textures' not found. STOP.");
|
||||
return;
|
||||
}
|
||||
WebClient cli = new WebClient();
|
||||
//add useragent to identify
|
||||
cli.Headers.Add("User-Agent", "MT_skin_grabber 1.1");
|
||||
|
||||
bool firstSkin = true;
|
||||
List<string> skin_local = new List<string>();
|
||||
int pages = page,
|
||||
updated = 0;
|
||||
|
||||
for (; page <= pages; page++) {
|
||||
string contents = "";
|
||||
try {
|
||||
contents = cli.DownloadString(root + "/api/get.json.php?getlist&page=" + page);
|
||||
} catch(WebException e) {
|
||||
Console.WriteLine("Whoops! Error at page ID: " + page + ". WebClient sais: " + e.Message);
|
||||
Console.WriteLine("Press any key to skip this page.");
|
||||
Console.ReadKey(false);
|
||||
continue;
|
||||
}
|
||||
Data o = JsonConvert.DeserializeObject<Data>(contents);
|
||||
if (o.pages != pages) {
|
||||
pages = o.pages;
|
||||
}
|
||||
|
||||
Console.WriteLine("# Page " + page + " (" + o.per_page + " skins)");
|
||||
for (int i = 0; i < o.skins.Length; i++) {
|
||||
int id = o.skins[i].id;
|
||||
if(o.skins[i].type != "image/png"){
|
||||
Console.WriteLine("Image type '" + o.skins[i].type + "' not supported at skin ID: " + id);
|
||||
Console.WriteLine("Press any key to continue.");
|
||||
Console.ReadKey(false);
|
||||
continue;
|
||||
}
|
||||
//eliminate special chars!
|
||||
o.skins[i].name = WebUtility.HtmlDecode(o.skins[i].name);
|
||||
o.skins[i].author = WebUtility.HtmlDecode(o.skins[i].author);
|
||||
|
||||
//to delete old, removed skins
|
||||
if (firstSkin) {
|
||||
firstSkin = false;
|
||||
|
||||
string[] files = Directory.GetFiles(path + "textures\\");
|
||||
for (int f = 0; f < files.Length; f++) {
|
||||
string[] filePath = stringSplitLast(files[f], '\\'),
|
||||
fileName = stringSplitLast(filePath[1], '.'),
|
||||
fileVer = stringSplitLast(fileName[0], '_');
|
||||
if (fileVer[1] == "" || fileVer[0] != "character") continue;
|
||||
|
||||
int skinNr = Program.getInt(fileVer[1]);
|
||||
if (skinNr <= id) continue;
|
||||
skin_local.Add(fileName[0]);
|
||||
}
|
||||
} else skin_local.Remove("character_" + id);
|
||||
|
||||
//get file size, only override changed
|
||||
FileInfo localImg = new FileInfo(path + "textures\\character_" + id + ".png");
|
||||
byte[] imageData = Convert.FromBase64String(o.skins[i].img);
|
||||
bool isDif = true;
|
||||
if (localImg.Exists) isDif = (Math.Abs(imageData.Length - localImg.Length) >= 3);
|
||||
|
||||
if (isDif) {
|
||||
File.WriteAllBytes(localImg.FullName, imageData);
|
||||
imageData = null;
|
||||
//previews
|
||||
try {
|
||||
cli.DownloadFile(root + "/skins/1/" + id + ".png", path + "textures\\character_" + id + "_preview.png");
|
||||
} catch (WebException e) {
|
||||
Console.WriteLine("Whoops! Error at skin ID: " + id + ". WebClient sais: " + e.Message);
|
||||
Console.WriteLine("Press any key to continue.");
|
||||
Console.ReadKey(false);
|
||||
}
|
||||
} else {
|
||||
Console.WriteLine("[SKIP] character_" + id);
|
||||
continue;
|
||||
}
|
||||
|
||||
string meta = "";
|
||||
if (!alternate) {
|
||||
meta = "name = \"" + o.skins[i].name + "\",\n";
|
||||
meta += "author = \"" + o.skins[i].author + "\",\n";
|
||||
meta += "comment = \"" + o.skins[i].license + '"';
|
||||
} else {
|
||||
meta = o.skins[i].name + '\n' + o.skins[i].author + '\n' + o.skins[i].license;
|
||||
}
|
||||
File.WriteAllText(path + "meta\\character_" + id + ".txt", meta);
|
||||
updated++;
|
||||
Console.WriteLine("[" + id + "] " + shorten(o.skins[i].name, 20) + "\t by: " + o.skins[i].author + "\t (" + o.skins[i].license + ")");
|
||||
}
|
||||
}
|
||||
foreach (string fileName in skin_local) {
|
||||
if(File.Exists(path + "textures\\" + fileName + ".png")) {
|
||||
File.Delete(path + "textures\\" + fileName + ".png");
|
||||
}
|
||||
if(File.Exists(path + "textures\\" + fileName + "_preview.png")) {
|
||||
File.Delete(path + "textures\\" + fileName + "_preview.png");
|
||||
}
|
||||
if(File.Exists(path + "meta\\" + fileName + ".txt")) {
|
||||
File.Delete(path + "meta\\" + fileName + ".txt");
|
||||
}
|
||||
Console.WriteLine("[DEL] " + fileName + " (deleted skin)");
|
||||
}
|
||||
Console.WriteLine("Done. Updated " + updated + " skins!");
|
||||
}
|
||||
string shorten(string inp, int len) {
|
||||
char[] shr = new char[len];
|
||||
for (int i = 0; i < len; i++) {
|
||||
if (i < inp.Length) {
|
||||
shr[i] = inp[i];
|
||||
} else shr[i] = ' ';
|
||||
}
|
||||
return new string(shr);
|
||||
}
|
||||
|
||||
string[] stringSplitLast(string path, char limiter) {
|
||||
int found = 0;
|
||||
int totalLen = path.Length - 1;
|
||||
for (int i = totalLen; i >= 0; i--) {
|
||||
if (path[i] == limiter) {
|
||||
found = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (found == 0) {
|
||||
return new string[] { "", "" };
|
||||
}
|
||||
|
||||
int len = totalLen - found;
|
||||
char[] str_1 = new char[found],
|
||||
str_2 = new char[len];
|
||||
|
||||
for (int i = 0; i < path.Length; i++) {
|
||||
if (i == found) continue;
|
||||
if (i < found) {
|
||||
str_1[i] = path[i];
|
||||
} else {
|
||||
str_2[i - found - 1] = path[i];
|
||||
}
|
||||
}
|
||||
return new string[] { new string(str_1), new string(str_2) };
|
||||
}
|
||||
}
|
||||
class Data {
|
||||
public Skins_data[] skins;
|
||||
public int page, pages, per_page;
|
||||
}
|
||||
class Skins_data {
|
||||
public string name, author, uploaded, type, license, img;
|
||||
public int id, license_id;
|
||||
}
|
||||
}
|
||||
Binary file not shown.
Binary file not shown.
|
|
@ -1,180 +0,0 @@
|
|||
#!/usr/bin/python3
|
||||
from http.client import HTTPConnection,HTTPException,BadStatusLine,_CS_IDLE
|
||||
import json
|
||||
import base64
|
||||
from contextlib import closing
|
||||
import sys,os,shutil,time
|
||||
|
||||
def die(message,code=23):
|
||||
print(message,file=sys.stderr)
|
||||
raise SystemExit(code)
|
||||
|
||||
server = "minetest.fensta.bplaced.net"
|
||||
skinsdir = "../textures/"
|
||||
metadir = "../meta/"
|
||||
curskin = 0
|
||||
curpage = 1
|
||||
pages = None
|
||||
|
||||
def replace(location,base,encoding=None,path=None):
|
||||
if path is None:
|
||||
path = os.path.join(location,base)
|
||||
mode = "wt" if encoding else "wb"
|
||||
# an unpredictable temp name only needed for a+rwxt directories
|
||||
tmp = os.path.join(location,'.'+base+'-tmp')
|
||||
def deco(handle):
|
||||
with open(tmp,mode,encoding=encoding) as out:
|
||||
handle(out)
|
||||
os.rename(tmp,path)
|
||||
return deco
|
||||
|
||||
def maybeReplace(location,base,encoding=None):
|
||||
def deco(handle):
|
||||
path = os.path.join(location,base)
|
||||
if os.path.exists(path): return
|
||||
return replace(location,base,encoding=encoding,path=path)(handle)
|
||||
return deco
|
||||
|
||||
class Penguin:
|
||||
"idk"
|
||||
def __init__(self, url, recv, diemessage):
|
||||
self.url = url
|
||||
self.recv = recv
|
||||
self.diemessage = diemessage
|
||||
|
||||
class Pipeline(list):
|
||||
"Gawd why am I being so elaborate?"
|
||||
def __init__(self, threshold=10):
|
||||
"threshold is how many requests in parallel to pipeline"
|
||||
self.threshold = threshold
|
||||
self.sent = True
|
||||
def __enter__(self):
|
||||
self.reopen()
|
||||
return self
|
||||
def __exit__(self,typ,exn,trace):
|
||||
self.send()
|
||||
self.drain()
|
||||
def reopen(self):
|
||||
self.c = HTTPConnection(server)
|
||||
self.send()
|
||||
def append(self,url,recv,diemessage):
|
||||
self.sent = False
|
||||
super().append(Penguin(url,recv,diemessage))
|
||||
if len(self) > self.threshold:
|
||||
self.send()
|
||||
self.drain()
|
||||
def trydrain(self):
|
||||
for penguin in self:
|
||||
print('drain',penguin.url)
|
||||
try:
|
||||
penguin.response.begin()
|
||||
penguin.recv(penguin.response)
|
||||
except BadStatusLine as e:
|
||||
print('derped requesting',penguin.url)
|
||||
return False
|
||||
except HTTPException as e:
|
||||
die(penguin.diemessage+' '+repr(e)+' (url='+penguin.url+')')
|
||||
self.clear()
|
||||
return True
|
||||
def drain(self):
|
||||
print('draining pipeline...',len(self))
|
||||
assert self.sent, "Can't drain without sending the requests!"
|
||||
self.sent = False
|
||||
while self.trydrain() is not True:
|
||||
self.c.close()
|
||||
print('drain failed, trying again')
|
||||
time.sleep(1)
|
||||
self.reopen()
|
||||
def trysend(self):
|
||||
for penguin in pipeline:
|
||||
print('fill',penguin.url)
|
||||
try:
|
||||
self.c.request("GET", penguin.url)
|
||||
self.c._HTTPConnection__state = _CS_IDLE
|
||||
penguin.response = self.c.response_class(self.c.sock,
|
||||
method="GET")
|
||||
# begin LATER so we can send multiple requests w/out response headers
|
||||
except BadStatusLine:
|
||||
return False
|
||||
except HTTPException as e:
|
||||
die(diemessage+' because of a '+repr(e))
|
||||
return True
|
||||
def send(self):
|
||||
if self.sent: return
|
||||
print('filling pipeline...',len(self))
|
||||
while self.trysend() is not True:
|
||||
self.c.close()
|
||||
print('derped resending')
|
||||
time.sleep(1)
|
||||
self.reopen()
|
||||
self.sent = True
|
||||
|
||||
with Pipeline() as pipeline:
|
||||
# two connections is okay, right? one for json, one for preview images
|
||||
c = HTTPConnection(server)
|
||||
def addpage(page):
|
||||
global curskin, pages
|
||||
print("Page: " + str(page))
|
||||
r = 0
|
||||
try:
|
||||
c.request("GET", "/api/get.json.php?getlist&page=" + str(page) + "&outformat=base64")
|
||||
r = c.getresponse()
|
||||
except Exception:
|
||||
if r != 0:
|
||||
if r.status != 200:
|
||||
die("Error", r.status)
|
||||
return
|
||||
|
||||
data = r.read().decode()
|
||||
l = json.loads(data)
|
||||
if not l["success"]:
|
||||
die("Success != True")
|
||||
r = 0
|
||||
pages = int(l["pages"])
|
||||
foundOne = False
|
||||
for s in l["skins"]:
|
||||
# make sure to increment this, even if the preview exists!
|
||||
curskin = curskin + 1
|
||||
previewbase = "character_" + str(curskin) + "_preview.png"
|
||||
preview = os.path.join(skinsdir, previewbase)
|
||||
if os.path.exists(preview):
|
||||
print('skin',curskin,'already retrieved')
|
||||
continue
|
||||
print('updating skin',curskin,'id',s["id"])
|
||||
foundOne = True
|
||||
@maybeReplace(skinsdir, "character_" + str(curskin) + ".png")
|
||||
def go(f):
|
||||
f.write(base64.b64decode(bytes(s["img"], 'utf-8')))
|
||||
f.close()
|
||||
|
||||
@maybeReplace(metadir, "character_" + str(curskin) + ".txt",
|
||||
encoding='utf-8')
|
||||
def go(f):
|
||||
f.write(str(s["name"]) + '\n')
|
||||
f.write(str(s["author"]) + '\n')
|
||||
f.write(str(s["license"]))
|
||||
url = "/skins/1/" + str(s["id"]) + ".png"
|
||||
def closure(skinsdir,previewbase,preview,s):
|
||||
"explanation: python sucks"
|
||||
def tryget(r):
|
||||
print('replacing',s["id"])
|
||||
if r.status != 200:
|
||||
print("Error", r.status)
|
||||
return
|
||||
@replace(skinsdir,previewbase,path=preview)
|
||||
def go(f):
|
||||
shutil.copyfileobj(r,f)
|
||||
return tryget
|
||||
|
||||
pipeline.append(url,closure(skinsdir,previewbase,preview,s),
|
||||
"Couldn't get {} because of a".format(
|
||||
s["id"]))
|
||||
if not foundOne:
|
||||
print("No skins updated on this page. Seems we're done?")
|
||||
#raise SystemExit
|
||||
addpage(curpage)
|
||||
while pages > curpage:
|
||||
curpage = curpage + 1
|
||||
addpage(curpage)
|
||||
print("Skins have been updated!")
|
||||
|
||||
|
|
@ -1,80 +0,0 @@
|
|||
#!/bin/bash
|
||||
####
|
||||
# Licenced under Attribution-NonCommercial-ShareAlike 4.0 International
|
||||
# http://creativecommons.org/licenses/by-nc-sa/4.0/
|
||||
#### ATTENTION ####
|
||||
## This script requires that jq and coreutils are installed on your system ##
|
||||
## In Debian-based distros, open a terminal and run
|
||||
## sudo apt-get install jq coreutils
|
||||
###################
|
||||
|
||||
# == Set variables ===
|
||||
# ====================
|
||||
NUMPAGES="1" # Number of pages. Default is 1 page
|
||||
PERPAGE="2000" # Number of items per page. Default is 2000.
|
||||
JSONURL="http://minetest.fensta.bplaced.net/api/get.json.php?getlist&page=$NUMPAGES&outformat=base64&per_page=$PERPAGE" # The URL to the database
|
||||
PREVIEWURL="http://minetest.fensta.bplaced.net/skins/1/" # The url to the location of the previews.
|
||||
curpath="$(dirname $0)" # all path are relative to this script place
|
||||
temp="$curpath"/tmp # Where the temp folder will be. Default is $PWD/tmp, which means that the tmp folder will be put in the current folder
|
||||
METADEST="$curpath"/../meta # This is the folder where the meta data will be saved
|
||||
TEXTUREDEST="$curpath"/../textures # This is the folder where the skins and the previews will be saved
|
||||
|
||||
# === Make a bunch of folders and download the db ===
|
||||
# ===================================================
|
||||
if [ -d "$temp" ]; then
|
||||
rm -r $temp # If the temp dir exists we will remove it and its contents.
|
||||
fi
|
||||
mkdir "$temp" # Make a new temp dir. Redundant? No. We will get rid of it later.
|
||||
|
||||
if [ ! -d "$METADEST" ]; then # Check to see if the meta dir exists, and if not, create it
|
||||
mkdir "$METADEST"
|
||||
fi
|
||||
|
||||
if [ ! -d "$TEXTUREDEST" ]; then # Check to see if the textures dir exists, and if not, create it
|
||||
mkdir "$TEXTUREDEST"
|
||||
fi
|
||||
|
||||
wget "$JSONURL" -O "$temp"/rawdb.txt # Download the entire database
|
||||
|
||||
|
||||
# === Do the JSON thing ===
|
||||
# =========================
|
||||
i="0" # This will be the counter.
|
||||
while true; do
|
||||
ID=$(cat "$temp"/rawdb.txt | jq ".skins[$i].id")
|
||||
if [ "$ID" == "null" ]; then
|
||||
break
|
||||
fi
|
||||
|
||||
if [ ! -f "$METADEST"/character_$ID.txt ] || [ "$1" == "all" ]; then
|
||||
# The next lines are kinda complex. sed is being used to strip the quotes from the variables. I had help...
|
||||
meta_name="$(jq ".skins[$i].name" < "$temp"/rawdb.txt | sed 's/^"//;s/"$//')"
|
||||
meta_author="$(jq ".skins[$i].author" <"$temp"/rawdb.txt | sed 's/^"//;s/"$//')"
|
||||
meta_license="$(jq ".skins[$i].license" <"$temp"/rawdb.txt | sed 's/^"//;s/"$//')"
|
||||
|
||||
echo "# $ID name: $meta_name author: $meta_author license: $meta_license" # Verbosity to show that the script is working.
|
||||
|
||||
echo "$meta_name" > "$METADEST"/character_$ID.txt # Save the meta data to files, this line overwrites the data inside the file
|
||||
echo "$meta_author" >> "$METADEST"/character_$ID.txt # Save the meta data to files, this line is added to the file
|
||||
echo "$meta_license" >> "$METADEST"/character_$ID.txt # Save the meta data to files, and this line is added to the file as well.
|
||||
|
||||
|
||||
# === Extract and save the image from the JSON file ===
|
||||
# ======================================================
|
||||
skin=$(jq ".skins[$i].img" < "$temp"/rawdb.txt | sed 's/^"//;s/"$//') # Strip the quotes from the base64 encoded string
|
||||
echo "$skin" | base64 --decode > "$TEXTUREDEST"/character_"$ID".png # Decode the string, and save it as a .png file
|
||||
|
||||
# === Download a preview image whilst we're at it ===
|
||||
# ====================================================
|
||||
wget -nv "$PREVIEWURL/$ID".png -O "$TEXTUREDEST"/character_"$ID"_preview.png # Downloads a preview of the skin that we just saved.
|
||||
else
|
||||
echo -n "."
|
||||
fi
|
||||
i=$[$i+1] # Increase the counter by one.
|
||||
done
|
||||
|
||||
# === Now we'll clean up the mess ===
|
||||
# ===================================
|
||||
rm -r "$temp" # Remove the temp dir and its contents.
|
||||
|
||||
exit # Not strictly needed, but i like to use it to wrap things up.
|
||||
Loading…
Reference in New Issue