lib_kernel.cpp

orange源码 数据挖掘技术

}
  

PyObject *stdRandomGenerator()
{ return WrapOrange(globalRandom); }

PYCONSTANTFUNC(globalRandom, stdRandomGenerator)


/* ************ EXAMPLE GENERATOR ************ */

TFiletypeDefinition::TFiletypeDefinition(const char *an, PyObject *al, PyObject *as)
: name(an),
  loader(al),
  saver(as)
{
  if (loader == Py_None)
    loader = PYNULL;
  else
    Py_INCREF(loader);

  if (saver == Py_None)
    saver = PYNULL;
  else
    Py_INCREF(saver);
}


TFiletypeDefinition::TFiletypeDefinition(const TFiletypeDefinition &other)
: name(other.name),
  extensions(other.extensions),
  loader(other.loader),
  saver(other.saver)
{
  Py_XINCREF(loader);
  Py_XINCREF(saver);
}


TFiletypeDefinition::~TFiletypeDefinition()
{
  Py_XDECREF(loader);
  Py_XDECREF(saver);
}


vector<TFiletypeDefinition> filetypeDefinitions;

/* lower case to avoid any ambiguity problems (don't know how various compilers can react when
   registerFiletype is cast by the code produced by pyxtract */
ORANGE_API void registerFiletype(const char *name, const vector<string> &extensions, PyObject *loader, PyObject *saver)
{
  TFiletypeDefinition ftd(name, loader, saver);
  ftd.extensions = extensions;
  filetypeDefinitions.push_back(ftd);
}

PStringList PStringList_FromArguments(PyObject *arg);
bool fileExists(const string &s);
const char *getExtension(const char *name);


vector<TFiletypeDefinition>::iterator findFiletypeByExtension(const char *name, bool needLoader, bool needSaver, bool exhaustive)
{
  const char *extension = getExtension(name);

  if (extension) {
    ITERATE(vector<TFiletypeDefinition>, fi, filetypeDefinitions)
      if ((!needLoader || (*fi).loader) && (!needSaver || (*fi).saver))
        ITERATE(TStringList, ei, (*fi).extensions)
          if (*ei == extension)
            return fi;
  }

  else if (exhaustive) {
    ITERATE(vector<TFiletypeDefinition>, fi, filetypeDefinitions)
      if ((!needLoader || (*fi).loader) && (!needSaver || (*fi).saver))
        ITERATE(TStringList, ei, (*fi).extensions)
          if (fileExists(name + *ei))
            return fi;
  }

  return filetypeDefinitions.end();
}


PyObject *registerFileType(PyObject *, PyObject *args) PYARGS(METH_VARARGS, "(name, extensions, loader, saver) -> None")
{
  char *name;
  PyObject *pyextensions, *loader, *saver;
  if (!PyArg_ParseTuple(args, "sOOO:registerFiletype", &name, &loader, &saver, &pyextensions))
    return PYNULL;

  TFiletypeDefinition ftd(name, loader, saver);

  if (PyString_Check(pyextensions))
    ftd.extensions.push_back(PyString_AsString(pyextensions));
  else {
    PStringList extensions = PStringList_FromArguments(pyextensions);
    if (!extensions)
      return PYNULL;
    ftd.extensions = extensions.getReference();
  }
 
  vector<TFiletypeDefinition>::iterator fi(filetypeDefinitions.begin()), fe(filetypeDefinitions.begin());
  for(; (fi != fe) && ((*fi).name != name); fi++);

  if (fi==fe)
    filetypeDefinitions.push_back(ftd);
  else
    *fi = ftd;

  RETURN_NONE;
}


extern char *fileTypes[][2];

PyObject *getRegisteredFileTypes(PyObject *, PyObject *, PyObject *) PYARGS(METH_NOARGS, "() -> ((extension, description, loader, saver), ...)")
{
  char *(*t)[2] = fileTypes;
  while(**t)
    t++;

  int builtIns = t-fileTypes;
  int i = 0;
  PyObject *types = PyTuple_New(builtIns + filetypeDefinitions.size());
  for(t = fileTypes; **t; t++)
    PyTuple_SetItem(types, i++, Py_BuildValue("ss", (*t)[0], (*t)[1]));

  ITERATE(vector<TFiletypeDefinition>, fi, filetypeDefinitions) {
    string exts;
    ITERATE(TStringList, ei, (*fi).extensions)
      exts += (exts.size() ? " *" : "*") + *ei;

    PyObject *ploader = (*fi).loader, *psaver = (*fi).saver;
    if (!ploader) {
      ploader = Py_None;
      Py_INCREF(Py_None);
    }
    if (!psaver) {
      psaver = Py_None;
      Py_INCREF(Py_None);
    }
    PyTuple_SetItem(types, i++, Py_BuildValue("ssOO", (*fi).name.c_str(), exts.c_str(), ploader, psaver));
  }

  return types;
}

#include "examplegen.hpp"
#include "table.hpp"
#include "filter.hpp"


PyObject *loadDataByPython(PyTypeObject *type, char *filename, PyObject *argstuple, PyObject *keywords, bool exhaustiveFilesearch, bool &fileFound)
{
  vector<TFiletypeDefinition>::iterator fi = findFiletypeByExtension(filename, true, false, exhaustiveFilesearch);
  fileFound = fi!=filetypeDefinitions.end();

  if (fileFound) {
    PyObject *res = PyObject_Call((*fi).loader, argstuple, keywords);
    if (!res)
      throw pyexception();
    if (res == Py_None)
      return res;

    if (PyOrExampleTable_Check(res))
      return res;

    PExampleGenerator gen;
    if (!exampleGenFromParsedArgs(res, gen))
      return PYNULL;

    TExampleTable *table = gen.AS(TExampleTable);
    if (!table)
      return PYNULL;

    return WrapNewOrange(table, type);
  }

  return PYNULL;
}

bool readUndefinedSpecs(PyObject *keyws, char *&DK, char *&DC);


bool readBoolFlag(PyObject *keywords, char *flag)
{
  PyObject *pyflag = keywords ? PyDict_GetItemString(keywords, flag) : PYNULL;
  return pyflag && PyObject_IsTrue(pyflag);
}

bool hasFlag(PyObject *keywords, char *flag)
{
  return keywords && (PyDict_GetItemString(keywords, flag) != PYNULL);
}


TExampleTable         *readTable(char *filename, PVarList knownVars, TMetaVector *knownMetas, PDomain knownDomain, bool dontCheckStored, bool dontStore, const char *DK, const char *DC, bool noExcOnUnknown = false, bool noCodedDiscrete = false, bool noClass = false);
TExampleGenerator *readGenerator(char *filename, PVarList knownVars, TMetaVector *knownMetas, PDomain knownDomain, bool dontCheckStored, bool dontStore, const char *DK, const char *DC, bool noExcOnUnknown = false, bool noCodedDiscrete = false, bool noClass = false);

PyObject *loadDataFromFile(PyTypeObject *type, char *filename, PyObject *argstuple, PyObject *keywords, bool generatorOnly = false)
{
  PyObject *res;
  
  bool pythonFileFound;
  res = loadDataByPython(type, filename, argstuple, keywords, false, pythonFileFound);
  if (res)
    if (res != Py_None)
      return res;
    else
      Py_DECREF(Py_None);

  PyErr_Clear();

  bool dontCheckStored = hasFlag(keywords, "dontCheckStored") ? readBoolFlag(keywords, "dontCheckStored") : hasFlag(keywords, "use");
  char *DK = NULL, *DC = NULL;
  if (!readUndefinedSpecs(keywords, DK, DC))
    return PYNULL;

  char *errs = NULL;
  try {
    TExampleGenerator *generator = 
      generatorOnly ? readGenerator(filename, knownVars(keywords), knownMetas(keywords), knownDomain(keywords), dontCheckStored, readBoolFlag(keywords, "dontStore"), DK, DC, false, readBoolFlag(keywords, "noCodedDiscrete"), readBoolFlag(keywords, "noClass"))
                    : readTable(filename, knownVars(keywords), knownMetas(keywords), knownDomain(keywords), dontCheckStored, readBoolFlag(keywords, "dontStore"), DK, DC, false, readBoolFlag(keywords, "noCodedDiscrete"), readBoolFlag(keywords, "noClass"));
    if (generator)
      return WrapNewOrange(generator, type);
  }
  catch (mlexception err) { 
    errs = strdup(err.what());
  }

  res = loadDataByPython(type, filename, argstuple, keywords, true, pythonFileFound);
  if (res)
    return res;

  if (pythonFileFound) {
    PYERROR(PyExc_SystemError, "cannot load the file", PYNULL);
  }
  else {
    PyErr_SetString(PyExc_SystemError, errs);
    free(errs);
    return PYNULL;
  }
}


int pt_ExampleGenerator(PyObject *args, void *egen)
{ 
  *(PExampleGenerator *)(egen) = PyOrExampleGenerator_Check(args) ? PyOrange_AsExampleGenerator(args)
                                                                  : PExampleGenerator(readListOfExamples(args));

  if (!*(PExampleGenerator *)(egen))
    PYERROR(PyExc_TypeError, "invalid example generator", 0)
  else
    return 1;
}


static PDomain ptd_domain;

int ptdf_ExampleGenerator(PyObject *args, void *egen)
{ 
  egen = NULL;

  try {
    if (PyOrExampleGenerator_Check(args)) {
      PExampleGenerator gen = PyOrange_AsExampleGenerator(args);
      if (gen->domain == ptd_domain)
        *(PExampleGenerator *)(egen) = gen;
      else
        *(PExampleGenerator *)(egen) = mlnew TExampleTable(ptd_domain, gen);
    }
    else
      *(PExampleGenerator *)(egen) = PExampleGenerator(readListOfExamples(args, ptd_domain));

    ptd_domain = PDomain();

    if (!*(PExampleGenerator *)(egen))
      PYERROR(PyExc_TypeError, "invalid example generator", 0)
    else
      return 1;
  }

  catch (...) {
    ptd_domain = PDomain();
    throw;
  }
}


converter ptd_ExampleGenerator(PDomain domain)
{ 
  ptd_domain = domain;
  return ptdf_ExampleGenerator;
}


CONSTRUCTOR_KEYWORDS(ExampleGenerator, "domain use useMetas dontCheckStored dontStore filterMetas DC DK NA noClass noCodedDiscrete")

// Class ExampleGenerator is abstract in C++; this constructor returns the derived classes

NO_PICKLE(ExampleGenerator)

PyObject *ExampleGenerator_new(PyTypeObject *type, PyObject *argstuple, PyObject *keywords) BASED_ON(Orange, "(filename)")
{  
  PyTRY
    char *filename = NULL;
    if (PyArg_ParseTuple(argstuple, "s", &filename))
      return loadDataFromFile(type, filename, argstuple, keywords, true);
    else
      return PYNULL;
  PyCATCH;
}


PExampleGenerator exampleGenFromParsedArgs(PyObject *args)
{
 if (PyOrOrange_Check(args)) {
   if (PyOrExampleGenerator_Check(args))
      return PyOrange_AsExampleGenerator(args);
    else 
      PYERROR(PyExc_TypeError, "example generator expected", NULL);
  }
  return PExampleGenerator(readListOfExamples(args));
}


PExampleGenerator exampleGenFromArgs(PyObject *args, int &weightID)
{ 
  PyObject *examples, *pyweight = NULL;
  if (!PyArg_UnpackTuple(args, "exampleGenFromArgs", 1, 2, &examples, &pyweight))
    return PExampleGenerator();

  PExampleGenerator egen = exampleGenFromParsedArgs(examples);
  if (!egen || !weightFromArg_byDomain(pyweight, egen->domain, weightID))
    return PExampleGenerator();

  return egen;
}


PExampleGenerator exampleGenFromArgs(PyObject *args)
{ 
  if (PyTuple_GET_SIZE(args) != 1)
    PYERROR(PyExc_TypeError, "exampleGenFromArgs: examples expected", PExampleGenerator())

  return exampleGenFromParsedArgs(PyTuple_GET_ITEM(args, 0));
}


PyObject *ExampleGenerator_native(PyObject *self, PyObject *args, PyObject *keyws) PYARGS(METH_VARARGS | METH_KEYWORDS, "([nativity, tuple=]) -> examples")
{ PyTRY
    bool tuples = false;
    PyObject *forDC = NULL;
    PyObject *forDK = NULL;
    PyObject *forSpecial = NULL;
    if (keyws) {
      PyObject *pytuples = PyDict_GetItemString(keyws, "tuple");
      tuples = pytuples && (PyObject_IsTrue(pytuples) != 0);

      forDC = PyDict_GetItemString(keyws, "substituteDC");
      forDK = PyDict_GetItemString(keyws, "substituteDK");
      forSpecial = PyDict_GetItemString(keyws, "substituteOther");
    }

    int natvt=2;
    if (args && !PyArg_ParseTuple(args, "|i", &natvt) || ((natvt>=2)))
      PYERROR(PyExc_TypeError, "invalid arguments", PYNULL);
    CAST_TO(TExampleGenerator, eg);

    PyObject *list=PyList_New(0);
    EITERATE(ei, *eg)
      if (natvt<=1) {
        PyObject *obj=convertToPythonNative(*ei, natvt, tuples, forDK, forDC, forSpecial);
        PyList_Append(list, obj);
        Py_DECREF(obj);
      }
        // What happens with: convertToPythonNative((*ei, natvt, tuples))? Funny.
      else {
        PyObject *example=