hedera/db/db-file-loader.c

/*
 * Copyright (C) 2012 - Juan Ferrer Toribio
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
 */

#include "db-file-loader.h"
#include "gio/gio.h"
#include <stdlib.h>
#include <glib/gstdio.h>
#include <errno.h>

#define CACHE_CLEAN_PERIOD		600
#define CACHE_DEFAULT_SIZE		100000000
#define CACHE_CLEAN_MARGIN		(CACHE_DEFAULT_SIZE*20)/100

/**
 * SECTION: db-file-loader
 * @Short_description: utility to download/upload resources from an URL
 * @Title: DbFileLoader
 *
 * #DbFileLoader can download and upload files from a remote or local location by
 * its URL. Both operations will allways be made asynchronously, to retrieve the
 * results you must pass a #DbFileLoaderCallbackFunc.
 * 
 * By default, #DbFileLoader uses a local cache on disk to store the downloaded
 * files. To avoid the use of this feature, create the #DbFileLoader using 
 * db_file_loader_new_simple(). The default cache directory is 'hedera', under
 * g_get_user_cache_dir(). To use another directory as cache or set its size,
 * use g_object_set().
 */

struct _DbFileLoaderPrivate
{
	gchar * dir;
	gchar * cache;
	goffset size;
	gchar * host;
	gchar * path;
	GInetSocketAddress * addr;
	GThreadPool * pool;
	GMutex * mutex;
	GMutex * cache_mutex;
	GHashTable * downloading;
	guint src;
};

G_DEFINE_TYPE (DbFileLoader, db_file_loader, G_TYPE_OBJECT)

typedef struct
{
	DbFileLoader * obj;
	gchar * name;
	GBytes * data;
	GError * error;
	DbFileLoaderCallbackFunc func;
	gpointer user_data;
	GCancellable * cancel;
}
File;

typedef struct
{
	guint64 atime;
	goffset size;
	gchar * path;
}
CacheFile;

static const gchar * WDAY[] = {"Err",
	"Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"};

static const gchar * MONTH[] = {"Err",
	"Jan", "Feb", "Mar", "Apr", "May", "Jun",
	"Jul", "Aug", "Sep", "Oct", "Nov", "Dec"};

static const gchar * format_request = {
	"GET /%s/%s HTTP/1.1\r\n"	// Path to the file
	"Host: %s\r\n"				// Hostname
	"%s"						// If-modified-since: date\r\n
	"\r\n"
};

/**
 * db_file_loader_new:
 * @host: the host where the files are stored
 * @path: the base directory to find the loaded resources
 * 
 * Creates a new #DbFileLoader pointing to @path in @host. The cache is set
 * to default.
 * 
 * Return value: a #DbFileLoader
 **/
DbFileLoader * db_file_loader_new (const gchar * host, const gchar * path)
{
	g_return_val_if_fail (host, NULL);
	g_return_val_if_fail (path, NULL);

	return g_object_new
		(DB_TYPE_FILE_LOADER, "host", host, "path", path, "cache", NULL, NULL);
}

/**
 * db_file_loader_new_simple:
 * @host: the host where the files are stored
 * @path: the base directory to find the loaded resources
 * 
 * Creates a new #DbFileLoader pointing to @path in @host. To use a cache,
 * call db_file_loader_new(). 
 * 
 * Return value: a #DbFileLoader
 **/
DbFileLoader * db_file_loader_new_simple (const gchar * host, const gchar * path)
{
	DbFileLoader * fl;

	g_return_val_if_fail (host, NULL);
	g_return_val_if_fail (path, NULL);

	fl = g_object_new (DB_TYPE_FILE_LOADER, "host", host, "path", path, NULL);

	if (fl->priv->cache)
	{
		g_free (fl->priv->cache);
		fl->priv->cache = NULL;
	}

	return fl;
}

/*
 * db_file_loader_new_full:
 * @host: the host where the files are stored
 * @path: the base directory to find the loaded resources
 * @cache: (allow-none): the base directory for the local cache or %NULL
 * @size: maximal size for the cache
 * 
 * Creates a new #DbFileLoader pointing to @path in @host. Note that the user
 * needs permission to read and write in @cache. If @cache is %NULL, it will be
 * set to default, if @max_cache is 0, the cache has no maximal size.
 * 
 * Return value: a #DbFileLoader or %NULL if @cache is an invalid directory
 **/
DbFileLoader * db_file_loader_new_full
	(const gchar * host, const gchar * path, const gchar * cache, goffset size)
{
	g_return_val_if_fail (host, NULL);
	g_return_val_if_fail (path, NULL);	

//	Comprobar que cache existe y se puede escribir, comprobar tamaño disponible
	return g_object_new (DB_TYPE_FILE_LOADER,
		"host", host, "path", path,
		"cache", cache, "size", size, NULL);
}

//+++++++++++++++++++++++++++++++++++++++++++++++++++ Methods

static File * file_new (DbFileLoader * obj, const gchar * path, const GBytes * data,
	DbFileLoaderCallbackFunc func, gpointer user_data, gboolean async)
{
	File * file = g_new (File, 1);

	file->obj = g_object_ref (obj);
	file->name = g_strdup (path);
	file->data = data ? g_boxed_copy (G_TYPE_BYTES, data) : NULL;
	file->error = NULL;
	file->func = func;
	file-> user_data = user_data;
	file->cancel = async ? g_cancellable_new () : NULL;

	return file;
}

static void file_free (File * file)
{
	if (file)
	{
		g_free (file->name);

		if (file->error)
			g_error_free (file->error);

		if (file->data)
			g_bytes_unref (file->data);

		g_object_unref (file->obj);
		g_free (file);
	}
}

/*
 * Connection management and usage
 */
static GFile * db_file_loader_get_cache_file (DbFileLoader * obj, const gchar * subpath)
{
	gchar * path = g_strconcat (obj->priv->cache, "/", subpath, NULL);
	GFile * file = g_file_new_for_path (path);
	g_free (path);

	return file;
}

static gboolean db_file_loader_callback (File * file)
{
	if (file->cancel && !g_cancellable_is_cancelled (file->cancel))
		file->func (file->obj, file->data, file->error, file->user_data);

	g_hash_table_remove (file->obj->priv->downloading, file);

	return G_SOURCE_REMOVE;
}

static gboolean db_file_loader_resolve_host (DbFileLoader * obj,
	GCancellable * cancel, GError ** error)
{
	gboolean ret = FALSE;
	GResolver * r = g_resolver_get_default ();
	GList * ads = g_resolver_lookup_by_name (r, obj->priv->host, cancel, error);

	if (ads)
	{
		GInetAddress * a = g_list_nth_data (ads, 0);// TODO? Use the entire list		
//		TODO Add the option to set the connection port!!
		obj->priv->addr = G_INET_SOCKET_ADDRESS (g_inet_socket_address_new (a, 80));
		g_resolver_free_addresses (ads);
		ret = TRUE;
	}

	g_object_unref (r);
	return ret;
}

static GIOStream * db_file_loader_connect (DbFileLoader * obj, GCancellable * cancel, GError ** error)
{
	GSocketClient * client = g_socket_client_new ();
	GIOStream * connection = G_IO_STREAM (g_socket_client_connect
		(client, G_SOCKET_CONNECTABLE (obj->priv->addr), cancel, error));
	g_object_unref (client);
	return connection;
}

static gchar * db_file_loader_create_request (DbFileLoader * obj, const gchar * subpath)
{
	gchar * ifmod;
	gchar * request;
	gchar * date = NULL;

	if (obj->priv->cache)
	{
		GFileInfo * info;
		GFile * file = db_file_loader_get_cache_file (obj, subpath);

		info = g_file_query_info
			(file
			,G_FILE_ATTRIBUTE_TIME_CHANGED
			,G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS
			,NULL, NULL);

		g_object_unref (file);

		if (info)
		{
			guint64 date_attr = g_file_info_get_attribute_uint64
				(info, G_FILE_ATTRIBUTE_TIME_CHANGED);
			GDateTime * dt = g_date_time_new_from_unix_utc (date_attr);
			gchar * f_year_time =  g_date_time_format (dt, "%Y %T GMT");
			gchar * f_month_day = g_date_time_format (dt, "%d");
			date = g_strdup_printf ("%s, %s %s %s"
				,WDAY[g_date_time_get_day_of_week (dt)]
				,f_month_day
				,MONTH[g_date_time_get_month (dt)]
				,f_year_time
			);

			g_free (f_month_day);
			g_free (f_year_time);
			g_date_time_unref (dt);
			g_object_unref (info);
		}
	}

	ifmod = date ? g_strconcat ("If-modified-since: ", date, "\r\n", NULL) : "";
	request = g_strdup_printf (format_request,
		obj->priv->path, subpath, obj->priv->host, ifmod);

	if (date)
	{
		g_free (date);
		g_free (ifmod);
	}

	return request;
}

static gboolean db_file_loader_store (DbFileLoader * obj,
	const gchar * subpath, const gchar * data, gsize len, GError ** error)
{
	gsize dir_len;
	gboolean ret = FALSE;
	gchar * path = g_strconcat (obj->priv->cache, "/",subpath, NULL);
	GFile * file = g_file_new_for_path (path);
	gchar * name = g_file_get_basename (file);

	dir_len = strlen (path) - strlen (name);
	gchar dir[dir_len];
	g_strlcpy (dir, path, dir_len);
	g_free (name);

	g_mutex_lock (obj->priv->cache_mutex);

	if (g_mkdir_with_parents (dir, 00700) >= 0)
	{
		if (!g_file_query_exists (file, NULL)
		&& g_file_set_contents (path, data, len, error))
			ret = TRUE;
	}
	else
		g_set_error (error, DB_FILE_LOADER_LOG_DOMAIN,
			g_file_error_from_errno (errno), _("%s not cached"), subpath);

	g_mutex_unlock (obj->priv->cache_mutex);

	g_free (path);
	g_object_unref (file);

	return ret;
}

static gchar * db_file_loader_load_from_cache (DbFileLoader * obj,
	const gchar * subpath, gsize * len, GError ** error)
{
	gchar * data;
	GFile * file = db_file_loader_get_cache_file (obj, subpath);

	g_file_load_contents (file, NULL, &data, len, NULL, error);
	g_object_unref (file);

	return data;
}

static void db_file_loader_thread_download (File * file, DbFileLoader * obj)
{
	gsize len;
	gchar ** split;
	gchar * status_line = NULL;
	gchar * request = NULL;
	gchar * data = NULL;
	GError * error = NULL;
	GIOStream * connection = NULL;
	GDataInputStream * receive_stream = NULL;
	GOutputStream * send_stream;
	GCancellable * cancel = file->cancel;

	g_mutex_lock (obj->priv->mutex);

	if (!obj->priv->addr && !db_file_loader_resolve_host (obj, cancel, &error))
	{
		g_mutex_unlock (obj->priv->mutex);
		goto exit;
	}

	g_mutex_unlock (obj->priv->mutex);

	if (!(connection = db_file_loader_connect (obj, cancel, &error)))
		goto exit;

	request = db_file_loader_create_request (obj, file->name);
	send_stream = g_io_stream_get_output_stream (connection);

	if (0 > g_output_stream_write (send_stream, request, strlen (request), cancel, &error)
	|| !g_output_stream_close (send_stream, cancel, &error))
		goto exit;

	receive_stream = g_data_input_stream_new (g_io_stream_get_input_stream (connection));
	g_data_input_stream_set_newline_type (receive_stream, G_DATA_STREAM_NEWLINE_TYPE_CR_LF);
	status_line = g_data_input_stream_read_line (receive_stream, &len, cancel, &error);

	if (!status_line)
		goto exit;

	split = g_strsplit (status_line, " ", -1);
	guint status = g_strv_length (split) >= 2 ?  atoi (split[1]) : 0;
	g_strfreev (split);

	switch (status)
	{
		gchar * line;

		case 200:
		{
			gboolean content_known;
			guint i, nbytes = 0;
			line = NULL;

			do
			{// Read header fields
				g_free (line);
				line = g_data_input_stream_read_line
					(receive_stream, &len, cancel, &error);

				if (g_str_has_prefix (line, "Content-Length: "))
				{
					nbytes = atoi (line + 16);
					content_known = TRUE;
				}
			}
			while (line && len && !error);

			g_free (line);

			if (!error)
			{
				if (!content_known)
					g_set_error (&error, DB_FILE_LOADER_LOG_DOMAIN, status,
						_("Unknown content length of file %s"), file->name);
				else
				{
					len = nbytes;
					data = g_new (gchar, len);

					for (i = 0; !error && i < len; i++)
						data[i] = g_data_input_stream_read_byte
							(receive_stream, cancel, &error);

					if (!error && data)
						db_file_loader_store (obj, file->name, data, len, &error);
				}
			}

			break;
		}
		case 304:
		{
			data = db_file_loader_load_from_cache (obj, file->name, &len, &error);
			break;
		}
		default:
			if (status_line && !error)
				g_set_error (&error, DB_FILE_LOADER_LOG_DOMAIN, status,
					"%s: '%s'", status_line, file->name);
	}

	if (data)
		file->data = g_bytes_new_take (data, len);

exit:
	g_free (request);
	g_free (status_line);

	if (receive_stream)
	{
		g_input_stream_close (G_INPUT_STREAM (receive_stream), NULL,
			error ? NULL : &error);
		g_object_unref (receive_stream);
	}

	if (connection)
	{
		g_io_stream_close (connection, NULL, error ? NULL : &error);
		g_object_unref (connection);
	}

	file->error = error;

	g_idle_add_full (G_PRIORITY_DEFAULT_IDLE
		,(GSourceFunc) db_file_loader_callback, file
		,(GDestroyNotify) file_free
	);
}

/**
 * db_file_loader_download:
 * @obj: a #DbFileLoader
 * @path: the path to the file from @obj's path
 * @func: (scope async): the #DbFileLoaderCallbackFunc to call after downloading
 * or in case of error
 * @user_data: (closure): data to pass to @func
 * 
 * Downloads a file from @file, which is a relative path to the file from the
 * base URL of the #DbFileLoader. The result will be availble in @func. 
 **/
void db_file_loader_download (DbFileLoader * obj,
	const gchar * path, DbFileLoaderCallbackFunc func, gpointer user_data)
{
	File * file;

	g_return_if_fail (DB_IS_FILE_LOADER (obj));
	g_return_if_fail (path);

	if (!obj->priv->pool)
		obj->priv->pool = g_thread_pool_new ((GFunc) db_file_loader_thread_download,
			obj, 2, FALSE, NULL);

	file = file_new (obj, path, NULL, func, user_data, TRUE);
	g_hash_table_add (obj->priv->downloading, file);

	g_thread_pool_push (obj->priv->pool, file, NULL);
}

static void db_file_loader_thread_upload (DbFileLoader * obj, File * file)
{
	g_idle_add_full (G_PRIORITY_DEFAULT_IDLE,
		(GSourceFunc) db_file_loader_callback, file,
		(GDestroyNotify) file_free);
}

/**
 * db_file_loader_upload:
 * @obj: a #DbFileLoader
 * @data: a #GBytes with the data to upload
 * @path: a string with the path to the file
 * @func: (scope async): the #DbFileLoaderCallbackFunc to call after uploading
 * or in case of error
 * @user_data: (closure): data to pass to @func
 *
 * Uploads @data to @file, which is a relative path to the file from the base
 * URL of the #DbFileLoader. To retrieve the result, connect to the
 * "downloaded" signal and to the "error" signal to manage errors.
 **/
void db_file_loader_upload (DbFileLoader * obj,
	GBytes * data, const gchar * path,
	DbFileLoaderCallbackFunc func, gpointer user_data)
{
	File * file;

	g_return_if_fail (DB_IS_FILE_LOADER (obj));
	g_return_if_fail (data);
	g_return_if_fail (path);
//TODO? create a pool only for uploads
	if (!obj->priv->pool)
		obj->priv->pool = g_thread_pool_new ((GFunc) db_file_loader_thread_upload,
			obj, 2, TRUE, NULL);

	file = file_new (obj, path, data, func, user_data, TRUE);
	
	g_thread_pool_push (obj->priv->pool, file, NULL);
}

static void db_file_loader_cancel (DbFileLoader * obj, const gchar * name)
{
	GHashTableIter iter;
	File * file;

	g_return_if_fail (DB_IS_FILE_LOADER (obj));
	if (!obj->priv->downloading) return;

	g_hash_table_iter_init (&iter, obj->priv->downloading);

	while (g_hash_table_iter_next (&iter, (gpointer *) &file, NULL))
	if (!name || !g_strcmp0 (file->name, name))
	{
		g_cancellable_cancel (file->cancel);
		g_hash_table_iter_steal (&iter);
	}
}

/**
 * db_file_loader_cancel_all:
 * @obj: a #DbFileLoader
 * 
 * Cancels all downloads pending or in progress. Note that after cancelling a
 * task it may delay until it returns to the main loop. For the cause of this,
 * see g_cancellable_cancel(). 
 **/
void db_file_loader_cancel_all (DbFileLoader * obj)
{
	db_file_loader_cancel (obj, NULL);
}

/**
 * db_file_loader_cancel_by_name:
 * @obj: a #DbFileLoader
 * @name: an string with a filename
 * 
 * Cancels the download of the file with name @name, that may be pending or in
 * progress. See db_file_loader_cancel_all() for more information.
 **/
void db_file_loader_cancel_by_name (DbFileLoader * obj, const gchar * name)
{
	if (name)
		db_file_loader_cancel (obj, name);
}

//+++++++++++++++++++++++++++++++++++++++++++++++++++ Properties

enum
{
	 HOST_PROP = 1
	,PATH_PROP
	,CACHE_PROP
	,SIZE_PROP
};

static void db_file_loader_set_property (DbFileLoader * obj, guint id,
	const GValue * value, GParamSpec * pspec)
{
	switch (id)
	{
		case HOST_PROP:
		{
			g_free (obj->priv->host);
			obj->priv->host = g_value_dup_string (value);
			break;
		}
		case PATH_PROP:
		{
			g_free (obj->priv->path);
			obj->priv->path = g_value_dup_string (value);
			break;
		}
		case CACHE_PROP:
		{
			const gchar * cache = g_value_get_string (value); 
			g_free (obj->priv->cache);
			obj->priv->cache =  g_build_filename (
				(cache ? cache : g_get_user_cache_dir ())
				,"hedera"
				,obj->priv->host
				,obj->priv->path
				,NULL);
			break;
		}
		case SIZE_PROP:
		{
			obj->priv->size = g_value_get_int64 (value);
			obj->priv->size = obj->priv->size <= 0 ? 0 : obj->priv->size; 
			break;
		}
		default:
			G_OBJECT_WARN_INVALID_PROPERTY_ID (obj, id, pspec);
	}
}

static void db_file_loader_get_property (DbFileLoader * obj, guint id,
	GValue * value, GParamSpec * pspec)
{
	switch (id)
	{
		case HOST_PROP:
			g_value_set_string (value, obj->priv->host);
			break;
		case PATH_PROP:
			g_value_set_string (value, obj->priv->path);
			break;
		case CACHE_PROP:
			g_value_set_string (value, obj->priv->cache);
		case SIZE_PROP:
			g_value_set_int64 (value, obj->priv->size);
		default:		
			G_OBJECT_WARN_INVALID_PROPERTY_ID (obj, id, pspec);
	}
}

//+++++++++++++++++++++++++++++++++++++++++++++++++++ Class

static void db_file_loader_init (DbFileLoader * obj)
{
	obj->priv = G_TYPE_INSTANCE_GET_PRIVATE
		(obj, DB_TYPE_FILE_LOADER, DbFileLoaderPrivate);
	obj->priv->host = NULL;
	obj->priv->path = NULL;
	obj->priv->cache = NULL;
	obj->priv->size = CACHE_DEFAULT_SIZE;
	obj->priv->addr = NULL;
	obj->priv->pool = NULL;
	obj->priv->mutex = g_new (GMutex, 1);
	g_mutex_init (obj->priv->mutex);
	obj->priv->cache_mutex = g_new (GMutex, 1);
	g_mutex_init (obj->priv->cache_mutex);
	obj->priv->downloading = g_hash_table_new_full
		((GHashFunc) g_direct_hash, (GEqualFunc) g_direct_equal, NULL, NULL);
}

static void db_file_loader_finalize (DbFileLoader * obj)
{
	G_OBJECT_CLASS (db_file_loader_parent_class)->finalize (G_OBJECT (obj));

	g_free (obj->priv->host);
	g_free (obj->priv->path);
	g_free (obj->priv->cache);

	g_clear_object (&obj->priv->addr);

	if (obj->priv->pool)
		g_thread_pool_free (obj->priv->pool, TRUE, TRUE);

	g_mutex_clear (obj->priv->cache_mutex);
	g_free (obj->priv->cache_mutex);
	g_mutex_clear (obj->priv->mutex);
	g_free (obj->priv->mutex);

	g_hash_table_destroy (obj->priv->downloading);
}

static void db_file_loader_class_init (DbFileLoaderClass * klass)
{
	GObjectClass * k = G_OBJECT_CLASS (klass);
	k->finalize = (GObjectFinalizeFunc) db_file_loader_finalize;
	k->set_property = (GObjectSetPropertyFunc) db_file_loader_set_property;
	k->get_property = (GObjectGetPropertyFunc) db_file_loader_get_property;
	g_type_class_add_private (klass, sizeof (DbFileLoaderPrivate));

	g_object_class_install_property (k, HOST_PROP,
		g_param_spec_string ("host"
			,_("Host")
			,_("The host web server name to get the images")
			,NULL
			,G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY
	));

	g_object_class_install_property (k, PATH_PROP,
		g_param_spec_string ("path"
			,_("Path")
			,_("The path of the directory to interact with")
			,NULL
			,G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY
	));

	g_object_class_install_property (k, CACHE_PROP,
		g_param_spec_string ("cache"
			,_("Cache directory")
			,_("The local directory where the downloaded files will be stored. "
			"The default cache directory is 'hedera', under g_get_user_cache_dir().")
			,NULL
			,G_PARAM_READWRITE | G_PARAM_CONSTRUCT_ONLY
	));

	g_object_class_install_property (k, SIZE_PROP,
		g_param_spec_int64 ("size"
			,_("Maximal cache size")
			,_("The maximal size for the contents of the cache directory")
			,0
			,G_MAXOFFSET
			,CACHE_DEFAULT_SIZE
			,G_PARAM_READWRITE
	));
}