kfsmodulepy.cc

nandflash文件系统源代码

//---------------------------------------------------------- -*- Mode: C++ -*-
// $Id: KfsModulePy.cc 227 2008-12-19 21:24:45Z sriramsrao $ 
//
// Created 2006/08/01
// Authors: Blake Lewis and Sriram Rao
//
// Copyright 2008 Quantcast Corp.
// Copyright 2006-2008 Kosmix Corp.
//
// This file is part of Kosmos File System (KFS).
//
// Licensed under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance with
// the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
// 
// \brief Glue code for Python apps to access KFS.
//
//----------------------------------------------------------------------------

#include "Python.h"
#include "structmember.h"
#include "libkfsClient/KfsClient.h"
#include <cstring>
#include <cerrno>
#include <string>
#include <vector>

extern "C" {
#include <sys/stat.h>
#include <sys/types.h>
#include <fcntl.h>
}

// XXX This is part of newer versions of Python; take out after
// we upgrade.
#if !defined(Py_RETURN_NONE)
/* Macro for returning Py_None from a function */
#define Py_RETURN_NONE return Py_INCREF(Py_None), Py_None
#endif // !defined(Py_RETURN_NONE)

using std::string;
using std::vector;
using namespace KFS;

struct kfs_Client {
	PyObject_HEAD
	PyObject *propfile;		// Properties file
	PyObject *cwd;			// Current directory
	KfsClientPtr client;		// The client itself
};

static void Client_dealloc(PyObject *pself);
static PyObject *Client_new(
		PyTypeObject *type, PyObject *args, PyObject *kwds);
static int Client_init(PyObject *pself, PyObject *args, PyObject *kwds);
static PyObject *Client_new(
		PyTypeObject *type, PyObject *args, PyObject *kwds);
static int Client_print(PyObject *pself, FILE *fp, int flags);
static PyObject *Client_repr(PyObject *pself);
static PyObject *kfs_isdir(PyObject *pself, PyObject *args);
static PyObject *kfs_isfile(PyObject *pself, PyObject *args);
static PyObject *kfs_mkdir(PyObject *pself, PyObject *args);
static PyObject *kfs_mkdirs(PyObject *pself, PyObject *args);
static PyObject *kfs_rmdir(PyObject *pself, PyObject *args);
static PyObject *kfs_rmdirs(PyObject *pself, PyObject *args);
static PyObject *kfs_readdir(PyObject *pself, PyObject *args);
static PyObject *kfs_readdirplus(PyObject *pself, PyObject *args);
static PyObject *kfs_create(PyObject *pself, PyObject *args);
static PyObject *kfs_stat(PyObject *pself, PyObject *args);
static PyObject *kfs_remove(PyObject *pself, PyObject *args);
static PyObject *kfs_rename(PyObject *pself, PyObject *args);
static PyObject *kfs_open(PyObject *pself, PyObject *args);
static PyObject *kfs_cd(PyObject *pself, PyObject *args);
static PyObject *kfs_log_level(PyObject *pself, PyObject *args);

static PyMemberDef Client_members[] = {
	{ "properties", T_OBJECT,
		offsetof(kfs_Client, propfile), RO, "properties file" },
	{ "cwd", T_OBJECT,
		offsetof(kfs_Client, cwd), RO, "current directory" },
	{ NULL }
};

static PyMethodDef Client_methods[] = {
	{ "mkdir", kfs_mkdir, METH_VARARGS, "Create directory."},
	{ "mkdirs", kfs_mkdirs, METH_VARARGS, "Create directory tree."},
        { "isdir", kfs_isdir, METH_VARARGS, "Check if a path is a directory."},
	{ "rmdir", kfs_rmdir, METH_VARARGS, "Remove directory."},
	{ "rmdirs", kfs_rmdirs, METH_VARARGS, "Remove directory tree."},
	{ "readdir", kfs_readdir, METH_VARARGS, "Read directory." },
	{ "readdirplus", kfs_readdirplus, METH_VARARGS,
		"Read directory with attributes." },
	{ "stat", kfs_stat, METH_VARARGS, "Stat file." },
	{ "create", kfs_create, METH_VARARGS, "Create file." },
	{ "remove", kfs_remove, METH_VARARGS, "Remove file." },
	{ "rename", kfs_rename, METH_VARARGS, "Rename file or directory." },
	{ "open", kfs_open, METH_VARARGS, "Open file." },
        { "isfile", kfs_isfile, METH_VARARGS, "Check if a path is a file."},
	{ "cd", kfs_cd, METH_VARARGS, "Change directory." },
	{ "log_level", kfs_log_level, METH_VARARGS, "Set log4cpp log level." },
	{ NULL }
};

PyDoc_STRVAR(Client_doc,
"A kfs.client object is an instance of the KFS client library\n"
"that sends RPC's to the KFS metadata and chunk servers in order\n"
"to perform file system operations.  In addition, its 'open' method\n"
"creates kfs.file objects that represent files in KFS.\n\n"
"To create a client, you must supply a 'properties file' which\n"
"defines the hostname and port number for the metaserver, e.g.\n\n"
"\tmy_client = kfs.client('KfsTester.properties')\n\n"
"Methods:\n"
"\tmkdir(path) -- create a directory\n"
"\tmkdirs(path) -- create a directory tree\n"
"\trmdir(path) -- remove a directory (path should be an empty dir)\n"
"\trmdirs(path) -- remove a directory tree\n"
"\treaddir(path) -- return a tuple of directory contents\n"
"\treaddirplus(path) -- directory entries plus attributes\n"
"\tisdir(path) -- return TRUE if path is a directory\n"
"\tisfile(path) -- return TRUE if path is a file\n"
"\tstat(path)   --  file attributes, compatible with os.stat\n"
"\tcreate(path) -- create a file and return a kfs.file object for it\n"
"\tremove(path) -- remove a file\n"
"\topen(path[, mode]) -- open a file and return an object for it\n"
"\tcd(path)     -- change current directory\n"
"\tlog_level(level)     -- change the message log level\n"
"\n\nData:\n"
"\tproperties   -- the name of the properties file\n"
"\tcwd          -- the current directory (for relative paths)\n");

static PyTypeObject kfs_ClientType = {
	PyObject_HEAD_INIT(NULL)
	0,				// ob_size
	"kfs.client",			// tp_name
	sizeof (kfs_Client),		// tp_basicsize
	0,				// tp_itemsize
	Client_dealloc,			// tp_dealloc
	Client_print,			// tp_print
	0,				// tp_getattr
	0,				// tp_setattr
	0,				// tp_compare
	Client_repr,			// tp_repr
	0,				// tp_as_number
	0,				// tp_as_sequence
	0,				// tp_as_mapping
	0,				// tp_hash
	0,				// tp_call
	0,				// tp_str
	0,				// tp_getattro
	0,				// tp_setattro
	0,				// tp_as_buffer
	Py_TPFLAGS_DEFAULT,		// tp_flags
	Client_doc,			// tp_doc
	0,				// tp_traverse
	0,				// tp_clear
	0,				// tp_richcompare
	0,				// tp_weaklistoffest
	0,				// tp_iter
	0,				// tp_iternext
	Client_methods,			// tp_methods
	Client_members,			// tp_members
	0,				// tp_getset
	0,				// tp_base
	0,				// tp_dict
	0,				// tp_descr_get
	0,				// tp_descr_set
	0,				// tp_dictoffset
	Client_init,			// tp_init
	0,				// tp_alloc
	Client_new			// tp_new
};


struct kfs_File {
	PyObject_HEAD
	PyObject *name;			// File name
	PyObject *mode;			// Access mode
	PyObject *pclient;		// Python object for KFS client
	int fd;				// File descriptor
};

static PyObject *
File_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
{
	static PyObject *noname = NULL;
	if (noname == NULL) {
		noname = PyString_FromString("<uninitialized file>");
		if (noname == NULL)
			return NULL;
	}
	kfs_File *self = (kfs_File *)type->tp_alloc(type, 0);
	if (self != NULL) {
		Py_INCREF(noname);
		self->name = noname;
		Py_INCREF(noname);
		self->mode = noname;
		Py_INCREF(noname);
		self->pclient = noname;
		self->fd = -1;
	}
	return (PyObject *)self;
}

static void
File_dealloc(PyObject *pself)
{
	kfs_File *self = (kfs_File *)pself;
	kfs_Client *cl = (kfs_Client *)self->pclient;
	if (self->fd != -1)
		cl->client->Close(self->fd);
	Py_DECREF(self->name);
	Py_DECREF(self->mode);
	Py_DECREF(self->pclient);
	self->ob_type->tp_free((PyObject *)self);
}

static int
modeflag(const char *modestr)
{
	// convert mode string to flag
	int mode = -1;

	if (strcmp(modestr, "r") == 0)
		mode = O_RDWR;
	else if (strcmp(modestr, "w") == 0)
		mode = O_WRONLY;
	else if (strcmp(modestr, "r+") == 0 || strcmp(modestr, "w+") == 0)
		mode = O_RDWR;
	else if (strcmp(modestr, "a") == 0)
		mode = O_WRONLY | O_APPEND;

	return mode;
}

static int
set_file_members(
		kfs_File *self, const char *path,
		const char *modestr, kfs_Client *client, int fd)
{
	int mode;

	// convert mode string to flag
	mode = modeflag(modestr);
	if (mode == -1)
		return -1;

	// open the file if necessary
	if (fd < 0)
		fd = client->client->Open(path, mode);

	if (fd < 0) {
		PyErr_SetString(PyExc_IOError, strerror(-fd));
		return -1;
	}

	// set all of the fields in the kfs_File structure
	Py_DECREF(self->name);
	self->name = PyString_FromString(path);
	Py_DECREF(self->mode);
	self->mode = PyString_FromString(modestr);
	PyObject *pclient = (PyObject *)client;
	Py_INCREF(pclient);
	Py_DECREF(self->pclient);
	self->pclient = pclient;
	self->fd = fd;

	return 0;
}

static int
File_init(PyObject *pself, PyObject *args, PyObject *kwds)
{
	kfs_File *self = (kfs_File *)pself;
	char *nm, *md = "r";
	PyObject *cl;
	static char *kwlist[] = { "client", "name", "mode", NULL };

	int ok = !PyArg_ParseTupleAndKeywords(
			args, kwds, "O!s|s", kwlist, &kfs_ClientType,
			&cl, &nm, &md);
	if (!ok)
		return -1;

	return set_file_members(self, nm, md, (kfs_Client *)cl, -1);
}

static PyObject *
File_repr(PyObject *pself)
{
	kfs_File *self = (kfs_File *)pself;
	return PyString_FromFormat("kfs.file<%s, %s, %d>",
			PyString_AsString(self->name),
			PyString_AsString(self->mode),
			self->fd);
}

static int
File_print(PyObject *pself, FILE *fp, int flags)
{
	kfs_File *self = (kfs_File *)pself;
	fprintf(fp, "kfs.file<%s, %s, %d>\n",
			PyString_AsString(self->name),
			PyString_AsString(self->mode),
			self->fd);
	return 0;
}

static PyObject *
kfs_reopen(PyObject *pself, PyObject *args)
{
	kfs_File *self = (kfs_File *)pself;
	kfs_Client *cl = (kfs_Client *)self->pclient;
	char *modestr = PyString_AsString(self->mode);

	if (!PyArg_ParseTuple(args, "|s", &modestr))
		return NULL;

	int mode = modeflag(modestr);
	if (mode == -1)
		return NULL;

	int fd = cl->client->Open(PyString_AsString(self->name), mode);
	if (fd == -1)
		return NULL;

	self->fd = fd;
	self->mode = PyString_FromString(modestr);
	Py_RETURN_NONE;
}

static PyObject *
kfs_close(PyObject *pself, PyObject *args)
{
	kfs_File *self = (kfs_File *)pself;
	kfs_Client *cl = (kfs_Client *)self->pclient;
	if (self->fd != -1) {
		cl->client->Close(self->fd);
		self->fd = -1;
	}
	Py_RETURN_NONE;
}

static PyObject *
kfs_read(PyObject *pself, PyObject *args)
{
	kfs_File *self = (kfs_File *)pself;
	kfs_Client *cl = (kfs_Client *)self->pclient;
	ssize_t rsize = -1l;

	if (!PyArg_ParseTuple(args, "l", &rsize))
		return NULL;

	if (self->fd == -1) {
		PyErr_SetString(PyExc_IOError, strerror(EBADF));
		return NULL;
	}

#if 0
	if (rsize > PY_SSIZE_T_MAX) {
		PyErr_SetString(PyExc_OverflowError,
"requested number of bytes is more than a Python string can hold");
		return NULL;
	}
#endif // 0

	PyObject *v = PyString_FromStringAndSize((char *)NULL, rsize);
	if (v == NULL)
		return NULL;

	char *buf = PyString_AsString(v);
	ssize_t nr = cl->client->Read(self->fd, buf, rsize);
	if (nr < 0) {
		Py_DECREF(v);
		PyErr_SetString(PyExc_IOError, strerror(-nr));
		return NULL;
	}
	if (nr != rsize)
		_PyString_Resize(&v, nr);
	return v;
}

static PyObject *
kfs_write(PyObject *pself, PyObject *args)
{
	kfs_File *self = (kfs_File *)pself;
	kfs_Client *cl = (kfs_Client *)self->pclient;
	int wsize = -1;
	char *buf = NULL;

	if (!PyArg_ParseTuple(args, "s#", &buf, &wsize))
		return NULL;

	if (self->fd == -1) {
		PyErr_SetString(PyExc_IOError, strerror(EBADF));
		return NULL;
	}

	ssize_t nw = cl->client->Write(self->fd, buf, (ssize_t)wsize);
	if (nw < 0) {
		PyErr_SetString(PyExc_IOError, strerror(-nw));
		return NULL;
	}
	if (nw != wsize) {
		PyObject *msg = PyString_FromFormat(
	"requested write of %d bytes but %ld were written", wsize, nw);
		return msg;
	}
	Py_RETURN_NONE;
}

static PyObject *
kfs_chunkLocations(PyObject *pself, PyObject *args)
{
	kfs_File *self = (kfs_File *)pself;
	kfs_Client *cl = (kfs_Client *)self->pclient;
	int off, len;

	if (!PyArg_ParseTuple(args, "i|i", &off, &len))
		return NULL;

	if (self->fd == -1) {
		PyErr_SetString(PyExc_IOError, strerror(EBADF));
		return NULL;
	}

	vector<vector <string> > results;

	int s = cl->client->GetDataLocation(self->fd, (off_t)off, len, results);
	if (s < 0) {
		PyErr_SetString(PyExc_IOError, strerror(-s));
		return NULL;
	}
	int n = results.size();
	PyObject *outer = PyTuple_New(n);
	for (int i = 0; i < n; i++) {
		int nlocs = results[i].size();
		vector<string> locs = results[i];
		PyObject *inner = PyTuple_New(nlocs);
		for (int j = 0; j < nlocs; j++) {
			PyTuple_SetItem(inner, j, PyString_FromString(locs[j].c_str()));
		}
		PyTuple_SetItem(outer, i, inner);
	}
	return outer;
}

static PyObject *
kfs_dataVerify(PyObject *pself, PyObject *args)
{
	kfs_File *self = (kfs_File *)pself;
	kfs_Client *cl = (kfs_Client *)self->pclient;
	int wsize = -1;
	char *buf = NULL;

	if (!PyArg_ParseTuple(args, "s#", &buf, &wsize))
		return NULL;

	if (self->fd == -1) {
		PyErr_SetString(PyExc_IOError, strerror(EBADF));
		return NULL;
	}

        bool res = cl->client->VerifyDataChecksums(self->fd, 0, buf, (ssize_t)wsize);
        return Py_BuildValue("b", res);
}

static PyObject *
kfs_truncate(PyObject *pself, PyObject *args)
{
	kfs_File *self = (kfs_File *)pself;
	kfs_Client *cl = (kfs_Client *)self->pclient;
	int off;

	if (!PyArg_ParseTuple(args, "i", &off))
		return NULL;

	if (self->fd == -1) {
		PyErr_SetString(PyExc_IOError, strerror(EBADF));
		return NULL;
	}

	int s = cl->client->Truncate(self->fd, (off_t)off);
	if (s < 0) {
		PyErr_SetString(PyExc_IOError, strerror(-s));
		return NULL;
	}
	Py_RETURN_NONE;
}


static PyObject *
kfs_sync(PyObject *pself, PyObject *args)
{
	kfs_File *self = (kfs_File *)pself;
	kfs_Client *cl = (kfs_Client *)self->pclient;
	int s = cl->client->Sync(self->fd);
	if (s < 0) {
		PyErr_SetString(PyExc_IOError, strerror(-s));
		return NULL;
	}
	Py_RETURN_NONE;
}

static PyObject *
kfs_seek(PyObject *pself, PyObject *args)
{
	kfs_File *self = (kfs_File *)pself;
	kfs_Client *cl = (kfs_Client *)self->pclient;
	off_t off;
	int whence = SEEK_SET;

	if (!PyArg_ParseTuple(args, "L|i", &off, &whence))
		return NULL;

	if (self->fd == -1) {
		PyErr_SetString(PyExc_IOError, strerror(EBADF));
		return NULL;
	}

	off_t s = cl->client->Seek(self->fd, (off_t)off, whence);
	if (s < 0) {
		PyErr_SetString(PyExc_IOError, strerror(-s));
		return NULL;
	}
	Py_RETURN_NONE;
}

static PyObject *
kfs_tell(PyObject *pself, PyObject *args)
{
	kfs_File *self = (kfs_File *)pself;
	kfs_Client *cl = (kfs_Client *)self->pclient;

	if (self->fd == -1) {
		PyErr_SetString(PyExc_IOError, strerror(EBADF));
		return NULL;
	}

	int pos = cl->client->Tell(self->fd);
	if (pos < 0) {
		PyErr_SetString(PyExc_IOError, strerror(-pos));
		return NULL;
	}
	return Py_BuildValue("i", pos);
}

static PyMethodDef File_methods[] = {
	{ "open", kfs_reopen, METH_VARARGS, "Open a closed file." },
	{ "close", kfs_close, METH_NOARGS, "Close file." },
	{ "read", kfs_read, METH_VARARGS, "Read from file." },
	{ "write", kfs_write, METH_VARARGS, "Write to file." },
	{ "truncate", kfs_truncate, METH_VARARGS, "Truncate a file." },
	{ "chunk_locations", kfs_chunkLocations, METH_VARARGS, "Get location(s) of a chunk." },
	{ "seek", kfs_seek, METH_VARARGS, "Seek to file offset." },
	{ "tell", kfs_tell, METH_NOARGS, "Return current offset." },