minetest
/
minetest
mirror of https://github.com/minetest/minetest.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Cache textures by checksum

This commit is contained in:
Jonathan Neuschäfer 2012-02-08 11:49:24 +01:00 committed by Perttu Ahola
parent 04085cad3c
commit 4bf5065a9c
6 changed files with 296 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/CMakeLists.txt
View File

@ -238,6 +238,7 @@ set(minetest_SRCS
guiCreateWorld.cpp
guiConfirmMenu.cpp
client.cpp
filecache.cpp
tile.cpp
game.cpp
main.cpp

118
src/client.cpp
View File

@ -38,6 +38,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
#include "sha1.h"
#include "base64.h"
#include "clientmap.h"
#include "filecache.h"
#include "sound.h"
static std::string getTextureCacheDir()
@ -255,6 +256,7 @@ Client::Client(
m_map_seed(0),
m_password(password),
m_access_denied(false),
m_texture_cache(getTextureCacheDir()),
m_texture_receive_progress(0),
m_textures_received(false),
m_itemdef_received(false),
@ -1412,7 +1414,7 @@ void Client::ProcessData(u8 *data, u32 datasize, u16 sender_peer_id)
//read texture from cache
std::string name = deSerializeString(is);
std::string sha1_texture = deSerializeString(is);
// if name contains illegal characters, ignore the texture
if(!string_allowed(name, TEXTURENAME_ALLOWED_CHARS)){
errorstream<<"Client: ignoring illegal texture name "
@ -1420,74 +1422,50 @@ void Client::ProcessData(u8 *data, u32 datasize, u16 sender_peer_id)
continue;
}
std::string tpath = getTextureCacheDir() + DIR_DELIM + name;
// Read data
std::ifstream fis(tpath.c_str(), std::ios_base::binary);
std::string sha1_decoded = base64_decode(sha1_texture);
std::ostringstream tmp_os(std::ios_base::binary);
bool tex_in_cache = m_texture_cache.loadByChecksum(name,
tmp_os, sha1_decoded);
m_texture_name_sha1_map.set(name, sha1_decoded);
if(tex_in_cache) {
if(fis.good() == false){
infostream<<"Client::Texture not found in cache: "
<<name << " expected it at: "<<tpath<<std::endl;
}
else
{
std::ostringstream tmp_os(std::ios_base::binary);
bool bad = false;
for(;;){
char buf[1024];
fis.read(buf, 1024);
std::streamsize len = fis.gcount();
tmp_os.write(buf, len);
if(fis.eof())
break;
if(!fis.good()){
bad = true;
break;
}
}
if(bad){
infostream<<"Client: Failed to read texture from cache\""
<<name<<"\""<<std::endl;
}
else {
SHA1 sha1;
sha1.addBytes(tmp_os.str().c_str(), tmp_os.str().length());
SHA1 sha1;
sha1.addBytes(tmp_os.str().c_str(), tmp_os.str().length());
unsigned char *digest = sha1.getDigest();
unsigned char *digest = sha1.getDigest();
std::string digest_string = base64_encode(digest, 20);
std::string digest_string = base64_encode(digest, 20);
if (digest_string == sha1_texture) {
// Silly irrlicht's const-incorrectness
Buffer<char> data_rw(tmp_os.str().c_str(), tmp_os.str().size());
if (digest_string == sha1_texture) {
// Silly irrlicht's const-incorrectness
Buffer<char> data_rw(tmp_os.str().c_str(), tmp_os.str().size());
// Create an irrlicht memory file
io::IReadFile *rfile = irrfs->createMemoryReadFile(
*data_rw, tmp_os.str().size(), "_tempreadfile");
assert(rfile);
// Read image
video::IImage *img = vdrv->createImageFromFile(rfile);
if(!img){
infostream<<"Client: Cannot create image from data of "
<<"received texture \""<<name<<"\""<<std::endl;
rfile->drop();
}
else {
m_tsrc->insertSourceImage(name, img);
img->drop();
rfile->drop();
texture_found = true;
}
// Create an irrlicht memory file
io::IReadFile *rfile = irrfs->createMemoryReadFile(
*data_rw, tmp_os.str().size(), "_tempreadfile");
assert(rfile);
// Read image
video::IImage *img = vdrv->createImageFromFile(rfile);
if(!img){
infostream<<"Client: Cannot create image from data of "
<<"received texture \""<<name<<"\""<<std::endl;
rfile->drop();
}
else {
infostream<<"Client::Texture cached sha1 hash not matching server hash: "
<<name << ": server ->"<<sha1_texture <<" client -> "<<digest_string<<std::endl;
}
m_tsrc->insertSourceImage(name, img);
img->drop();
rfile->drop();
free(digest);
texture_found = true;
}
}
else {
infostream<<"Client::Texture cached sha1 hash not matching server hash: "
<<name << ": server ->"<<sha1_texture <<" client -> "<<digest_string<<std::endl;
}
free(digest);
}
//add texture request
@ -1598,15 +1576,15 @@ void Client::ProcessData(u8 *data, u32 datasize, u16 sender_peer_id)
fs::CreateAllDirs(getTextureCacheDir());
std::string filename = getTextureCacheDir() + DIR_DELIM + name;
std::ofstream outfile(filename.c_str(), std::ios_base::binary | std::ios_base::trunc);
if (outfile.good()) {
outfile.write(data.c_str(),data.length());
outfile.close();
}
else {
errorstream<<"Client: Unable to open cached texture file "<< filename <<std::endl;
{
core::map<std::string, std::string>::Node *n;
n = m_texture_name_sha1_map.find(name);
if(n == NULL)
errorstream<<"The server sent a texture that has not been announced."
<<std::endl;
else
m_texture_cache.updateByChecksum(name,
data, n->getValue());
}
m_tsrc->insertSourceImage(name, img);
@ -2382,6 +2360,10 @@ void Client::afterContentReceived()
assert(m_nodedef_received);
assert(m_textures_received);
// remove the information about which checksum each texture
// ought to have
m_texture_name_sha1_map.clear();
// Rebuild inherited images and recreate textures
m_tsrc->rebuildImagesAndTextures();

5
src/client.h
View File

@ -34,6 +34,7 @@ with this program; if not, write to the Free Software Foundation, Inc.,
#include "gamedef.h"
#include "inventorymanager.h"
#include "filesys.h"
#include "filecache.h"
struct MeshMakeData;
class MapBlockMesh;
@ -366,6 +367,10 @@ private:
bool m_access_denied;
std::wstring m_access_denied_reason;
Queue<ClientEvent> m_client_event_queue;
FileCache m_texture_cache;
// a map of the name and SHA1 checksum of each texture;
// cleared after content has been recieved
core::map<std::string, std::string> m_texture_name_sha1_map;
float m_texture_receive_progress;
bool m_textures_received;
bool m_itemdef_received;

117
src/filecache.cpp Normal file
View File

@ -0,0 +1,117 @@
/*
Minetest-c55
Copyright (C) 2010 celeron55, Perttu Ahola <celeron55@gmail.com>
Copyright (C) 2012 Jonathan Neuschäfer <j.neuschaefer@gmx.net>
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License along
with this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
#include "filecache.h"
#include "clientserver.h"
#include "log.h"
#include "filesys.h"
#include "utility.h"
#include "hex.h"
#include <string>
#include <iostream>
bool FileCache::loadByPath(const std::string &name, std::ostream &os,
const std::string &path)
{
std::ifstream fis(path.c_str(), std::ios_base::binary);
if(!fis.good()){
infostream<<"FileCache: File not found in cache: "
<<name << " expected it at: "<<path<<std::endl;
return false;
}
bool bad = false;
for(;;){
char buf[1024];
fis.read(buf, 1024);
std::streamsize len = fis.gcount();
os.write(buf, len);
if(fis.eof())
break;
if(!fis.good()){
bad = true;
break;
}
}
if(bad){
infostream<<"FileCache: Failed to read file from cache: \""
<<path<<"\""<<std::endl;
}
return !bad;
}
bool FileCache::updateByPath(const std::string &name, const std::string &data,
const std::string &path)
{
std::ofstream file(path.c_str(), std::ios_base::binary |
std::ios_base::trunc);
if(!file.good())
{
errorstream<<"FileCache: Can't write to file at "
<<path<<std::endl;
return false;
}
file.write(data.c_str(), data.length());
file.close();
return !file.fail();
}
bool FileCache::loadByName(const std::string &name, std::ostream &os)
{
std::string path = m_dir + DIR_DELIM + name;
return loadByPath(name, os, path);
}
bool FileCache::updateByName(const std::string &name, const std::string &data)
{
std::string path = m_dir + DIR_DELIM + name;
return updateByPath(name, data, path);
}
std::string FileCache::getPathFromChecksum(const std::string &name,
const std::string &checksum)
{
std::string checksum_hex = hex_encode(checksum.c_str(), checksum.length());
size_t dot = name.find_last_of('.');;
std::string ext = (dot == std::string::npos)? "" :
name.substr(dot, std::string::npos);
return m_dir + DIR_DELIM + checksum_hex + ext;
}
bool FileCache::loadByChecksum(const std::string &name, std::ostream &os,
const std::string &checksum)
{
std::string path = getPathFromChecksum(name, checksum);
return loadByPath(name, os, path);
}
bool FileCache::updateByChecksum(const std::string &name,
const std::string &data, const std::string &checksum)
{
std::string path = getPathFromChecksum(name, checksum);
return updateByPath(name, data, path);
}

79
src/filecache.h Normal file
View File

@ -0,0 +1,79 @@
/*
Minetest-c55
Copyright (C) 2010 celeron55, Perttu Ahola <celeron55@gmail.com>
Copyright (C) 2012 Jonathan Neuschäfer <j.neuschaefer@gmx.net>
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License along
with this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
#ifndef FILECACHE_HEADER
#define FILECACHE_HEADER
#include <string>
#include <iostream>
class FileCache
{
public:
/*
'dir' is the file cache directory to use.
*/
FileCache(std::string dir):
m_dir(dir)
{
}
/*
Searches the cache for a file with a given name.
If the file is found, lookup copies it into 'os' and
returns true. Otherwise false is returned.
*/
bool loadByName(const std::string &name, std::ostream &os);
/*
Stores a file in the cache based on its name.
Returns true on success, false otherwise.
*/
bool updateByName(const std::string &name, const std::string &data);
/*
Loads a file based on a check sum, which may be any kind of
rather unique byte sequence. Returns true, if the file could
be written into os, false otherwise.
A file name is required to give the disk file a name that
has the right file name extension (e.g. ".png").
*/
bool loadByChecksum(const std::string &name, std::ostream &os,
const std::string &checksum);
/*
Stores a file in the cache based on its checksum.
Returns true on success, false otherwise.
*/
bool updateByChecksum(const std::string &name, const std::string &data,
const std::string &checksum);
private:
std::string m_dir;
bool loadByPath(const std::string &name, std::ostream &os,
const std::string &path);
bool updateByPath(const std::string &name, const std::string &data,
const std::string &path);
std::string getPathFromChecksum(const std::string &name,
const std::string &checksum);
};
#endif

44
src/hex.h Normal file
View File

@ -0,0 +1,44 @@
/*
Minetest-c55
Copyright (C) 2012 Jonathan Neuschäfer <j.neuschaefer@gmx.net>
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License along
with this program; if not, write to the Free Software Foundation, Inc.,
51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
*/
#ifndef HEX_HEADER
#define HEX_HEADER
#include <string>
static const char hex_chars[] = "0123456789abcdef";
static std::string hex_encode(const char *data, unsigned int data_size)
{
std::string ret;
char buf2[3];
buf2[2] = '\0';
for(unsigned int i = 0; i < data_size; i++)
{
unsigned char c = (unsigned char) data[i];
buf2[0] = hex_chars[(c & 0xf0) >> 4];
buf2[1] = hex_chars[c & 0x0f];
ret.append(buf2);
}
return ret;
}
#endif