table.h

Go to the documentation of this file.

// File: $Id$
// Author: John Wu <John.Wu at ACM.org>
// Copyright 2007-2012 the Regents of the University of California
#ifndef IBIS_TABLE_H
#define IBIS_TABLE_H

#include <iostream>     // std::ostream
#include <vector>       // std::vector
#include <map>          // std::map
#include <string>       // std::string
#include "const.h"      // intxx_t, uintxx_t, ... from stdint.h

namespace ibis {

    enum TYPE_T {
        UNKNOWN_TYPE=0,
        OID,
        BYTE,   
        UBYTE,  
        SHORT,  
        USHORT, 
        INT,    
        UINT,   
        LONG,   
        ULONG,  
        FLOAT,  
        DOUBLE, 


        CATEGORY,
        TEXT,
        BLOB
    };
    FASTBIT_CXX_DLLSPEC extern const char** TYPESTRING;
    FASTBIT_CXX_DLLSPEC extern const char* TYPECODE;

    class table;
    class tablex;
    class tableList;
} // namespace ibis

class FASTBIT_CXX_DLLSPEC ibis::table {
public:
    static ibis::table* create(ibis::part&);
    static ibis::table* create(const ibis::partList&);
    static ibis::table* create(const char* dir);
    static ibis::table* create(const char* dir1, const char* dir2);

    virtual ~table() {};

    virtual const char* name() const {return name_.c_str();}
    virtual const char* description() const {return desc_.c_str();}
    virtual uint64_t nRows() const =0;
    virtual uint32_t nColumns() const =0;

    typedef ibis::array_t<const char*> stringList;
    typedef ibis::array_t<ibis::TYPE_T> typeList;
    typedef ibis::array_t<void *> bufferList;
    typedef std::map<const char*, ibis::TYPE_T, ibis::lessi> namesTypes;

    virtual stringList columnNames() const =0; 
    virtual typeList columnTypes() const =0; 

    virtual void describe(std::ostream&) const =0;
    virtual void dumpNames(std::ostream& out, const char* del=", ") const =0;
    virtual int dump(std::ostream& out, const char* del=", ") const =0;
    virtual int dump(std::ostream& out, uint64_t nr,
                     const char* del=", ") const =0;
    virtual int dump(std::ostream& out, uint64_t offset, uint64_t nr,
                     const char* del=", ") const =0;
    virtual int backup(const char* dir, const char* tname=0,
                       const char* tdesc=0) const =0;

    virtual void estimate(const char* cond,
                          uint64_t& nmin, uint64_t& nmax) const =0;
    virtual void estimate(const ibis::qExpr* cond,
                          uint64_t& nmin, uint64_t& nmax) const =0;
    virtual table* select(const char* sel, const char* cond) const =0;
    virtual table* select(const char* sel, const ibis::qExpr* cond) const;

    static table* select(const ibis::constPartList& parts,
                         const char* sel, const char* cond);
    static table* select(const ibis::constPartList& parts,
                         const char* sel, const ibis::qExpr* cond);
    static int64_t computeHits(const ibis::constPartList& parts,
                               const char* cond);
    static int64_t computeHits(const ibis::constPartList& parts,
                               const ibis::qExpr* cond);

    virtual table* groupby(const stringList&) const =0;
    virtual table* groupby(const char*) const;
    virtual void orderby(const stringList&)=0;
    virtual void orderby(const stringList&, const std::vector<bool>&)=0;
    virtual void orderby(const char*);
    virtual void reverseRows()=0;

    virtual int addPartition(const char*) {return -1;}
    virtual int getPartitions(ibis::constPartList&) const {
        return -1;}

    virtual int buildIndex(const char* colname, const char* option=0) =0;
    virtual int buildIndexes(const char* options=0) =0;
    virtual const char* indexSpec(const char* colname=0) const =0;
    virtual void indexSpec(const char* opt, const char* colname=0) =0;
    virtual int combineCategories(const stringList&) {return 0;}

    virtual int64_t
        getColumnAsBytes(const char* cname, char* vals,
                         uint64_t begin=0, uint64_t end=0) const =0;
    virtual int64_t
        getColumnAsUBytes(const char* cname, unsigned char* vals,
                          uint64_t begin=0, uint64_t end=0) const =0;
    virtual int64_t
        getColumnAsShorts(const char* cname, int16_t* vals,
                          uint64_t begin=0, uint64_t end=0) const =0;
    virtual int64_t
        getColumnAsUShorts(const char* cname, uint16_t* vals,
                           uint64_t begin=0, uint64_t end=0) const =0;
    virtual int64_t
        getColumnAsInts(const char* cname, int32_t* vals,
                        uint64_t begin=0, uint64_t end=0) const =0;
    virtual int64_t
        getColumnAsUInts(const char* cname, uint32_t* vals,
                         uint64_t begin=0, uint64_t end=0) const =0;
    virtual int64_t
        getColumnAsLongs(const char* cname, int64_t* vals,
                         uint64_t begin=0, uint64_t end=0) const =0;
    virtual int64_t
        getColumnAsULongs(const char* cname, uint64_t* vals,
                          uint64_t begin=0, uint64_t end=0) const =0;
    virtual int64_t
        getColumnAsFloats(const char* cname, float* vals,
                          uint64_t begin=0, uint64_t end=0) const =0;
    virtual int64_t
        getColumnAsDoubles(const char* cname, double* vals,
                           uint64_t begin=0, uint64_t end=0) const =0;
    virtual int64_t
        getColumnAsDoubles(const char* cname, std::vector<double>& vals,
                           uint64_t begin=0, uint64_t end=0) const =0;
    virtual int64_t
        getColumnAsStrings(const char* cname, std::vector<std::string>& vals,
                           uint64_t begin=0, uint64_t end=0) const =0;

    virtual double getColumnMin(const char* cname) const =0;
    virtual double getColumnMax(const char* cname) const =0;

    virtual long getHistogram(const char* constraints,
                              const char* cname,
                              double begin, double end, double stride,
                              std::vector<uint32_t>& counts) const =0;
    virtual long getHistogram2D(const char* constraints,
                                const char* cname1,
                                double begin1, double end1, double stride1,
                                const char* cname2,
                                double begin2, double end2, double stride2,
                                std::vector<uint32_t>& counts) const =0;
    virtual long getHistogram3D(const char* constraints,
                                const char* cname1,
                                double begin1, double end1, double stride1,
                                const char* cname2,
                                double begin2, double end2, double stride2,
                                const char* cname3,
                                double begin3, double end3, double stride3,
                                std::vector<uint32_t>& counts) const =0;

    struct row {
        std::vector<std::string>   bytesnames; 
        std::vector<signed char>   bytesvalues;
        std::vector<std::string>   ubytesnames; 
        std::vector<unsigned char> ubytesvalues;
        std::vector<std::string>   shortsnames; 
        std::vector<int16_t>       shortsvalues;
        std::vector<std::string>   ushortsnames; 
        std::vector<uint16_t>      ushortsvalues;
        std::vector<std::string>   intsnames; 
        std::vector<int32_t>       intsvalues;
        std::vector<std::string>   uintsnames; 
        std::vector<uint32_t>      uintsvalues;
        std::vector<std::string>   longsnames; 
        std::vector<int64_t>       longsvalues;
        std::vector<std::string>   ulongsnames; 
        std::vector<uint64_t>      ulongsvalues;
        std::vector<std::string>   floatsnames; 
        std::vector<float>         floatsvalues;
        std::vector<std::string>   doublesnames; 
        std::vector<double>        doublesvalues;
        std::vector<std::string>   catsnames; 
        std::vector<std::string>   catsvalues;
        std::vector<std::string>   textsnames; 
        std::vector<std::string>   textsvalues;
        std::vector<std::string>   blobsnames; 
        std::vector<std::string>   blobsvalues;

        void clear();
        void clearValues();
        uint32_t nColumns() const {
            return bytesvalues.size() + ubytesvalues.size() +
                shortsvalues.size() + ushortsvalues.size() +
                intsvalues.size() + uintsvalues.size() +
                longsvalues.size() + ulongsvalues.size() +
                floatsvalues.size() + doublesvalues.size() +
                catsvalues.size() + textsvalues.size() + blobsvalues.size();}
    }; // struct row

    // Cursor class for row-wise data accesses.
    class cursor;
    virtual cursor* createCursor() const =0;

    static void parseNames(char* in, stringList& out);
    static void parseOrderby(char* in, stringList& out,
                             std::vector<bool>& direc);

    static void* allocateBuffer(ibis::TYPE_T, size_t);
    static void freeBuffer(void* buffer, ibis::TYPE_T type);
    static void freeBuffers(bufferList&, typeList&);

protected:

    std::string name_;  
    std::string desc_;  

    table() {};
    table(const char* na, const char* de)
        : name_(na?na:""), desc_(de?de:na?na:"") {};

private:
    // re-enforce the prohibitions on copying and assignment.
    table(const table&);
    table& operator=(const table&);
}; // class ibis::table

class FASTBIT_CXX_DLLSPEC ibis::tablex {
public:
    static ibis::tablex* create();
//     /// Make the incoming table expandable.  Not yet implemented
//     static ibis::tablex* makeExtensible(ibis::table* t);

    virtual ~tablex() {}; // nothing to do.

    virtual int addColumn(const char* cname, ibis::TYPE_T ctype,
                          const char* cdesc=0, const char* idx=0) =0;

    virtual int append(const char* cname, uint64_t begin, uint64_t end,
                       void* values) =0;

    virtual int appendRow(const ibis::table::row&) =0;
    virtual int appendRow(const char* line, const char* delimiters=0) = 0;
    virtual int appendRows(const std::vector<ibis::table::row>&) =0;

    virtual int readCSV(const char* inputfile, int maxrows=0,
                        const char* outputdir=0, const char* delimiters=0) =0;
    virtual int readSQLDump(const char* inputfile, std::string& tname,
                            int maxrows=0, const char* outputdir=0) =0;

    virtual int readNamesAndTypes(const char* filename);
    virtual int parseNamesAndTypes(const char* txt);

    virtual int write(const char* dir, const char* tname=0,
                      const char* tdesc=0, const char* idx=0,
                      const char* nvpairs=0) const =0;
    virtual int writeMetaData(const char* dir, const char* tname=0,
                              const char* tdesc=0, const char* idx=0,
                              const char* nvpairs=0) const =0;

    virtual void clearData() =0;
    virtual int32_t reserveSpace(uint32_t) {return 0;}
    virtual uint32_t capacity() const {return 0;}

    virtual uint32_t mRows() const =0;
    virtual uint32_t mColumns() const =0;
    virtual void describe(std::ostream&) const =0;

    virtual table* toTable(const char* nm=0, const char* de=0) =0;

protected:
    tablex() {}; // Derived classes need this.

private:
    tablex(const tablex&); // no copying
    tablex& operator=(const tablex&); // no assignment
}; // class ibis::tablex

class FASTBIT_CXX_DLLSPEC ibis::tableList {
public:
    typedef std::map< const char*, ibis::table*, ibis::lessi > tableSet;
    typedef tableSet::const_iterator iterator;

    bool empty() const {return tables.empty();}
    uint32_t size() const {return tables.size();}
    iterator begin() const {return tables.begin();}
    iterator end() const {return tables.end();}

    const ibis::table* operator[](const char* tname) const {
        tableSet::const_iterator it = tables.find(tname);
        if (it != tables.end())
            return (*it).second;
        else
            return 0;
    }

    void add(ibis::table*& tb) {
        tableSet::iterator it = tables.find(tb->name());
        if (it == tables.end()) {
            tables[tb->name()] = tb;
            tb=0;
        }
        else {
            ibis::table* tmp = (*it).second;
            tables[tb->name()] = tb;
            tb = tmp;
        }
    }

    void remove(const char* tname) {
        tableSet::iterator it = tables.find(tname);
        if (it != tables.end()) {
            ibis::table* tmp = (*it).second;
            tables.erase(it);
            delete tmp;
        }
    }

    tableList() {};

    ~tableList() {
        while (! tables.empty()) {
            tableSet::iterator it = tables.begin();
            ibis::table* tmp = (*it).second;
            tables.erase(it);
            delete tmp;
        }
    }

private:
    tableSet tables;

    // Can not copy or assign.
    tableList(const tableList&);
    tableList& operator=(const tableList&);
}; // ibis::tableList

class FASTBIT_CXX_DLLSPEC ibis::table::cursor {
public:
    virtual ~cursor() {};
    virtual uint64_t nRows() const =0;
    virtual uint32_t nColumns() const =0;
    virtual ibis::table::typeList columnTypes() const =0;
    virtual ibis::table::stringList columnNames() const =0;
    virtual int fetch() =0;
    virtual int fetch(uint64_t rownum) =0;
    virtual uint64_t getCurrentRowNumber() const =0;

    virtual int fetch(ibis::table::row&) =0;
    virtual int fetch(uint64_t rownum, ibis::table::row&) =0;

    virtual int dump(std::ostream& out, const char* del=", ") const =0;

    virtual int getColumnAsByte(const char* cname, char&) const =0;
    virtual int getColumnAsUByte(const char* cname, unsigned char&) const =0;
    virtual int getColumnAsShort(const char* cname, int16_t&) const =0;
    virtual int getColumnAsUShort(const char* cname, uint16_t&) const =0;
    virtual int getColumnAsInt(const char* cname, int32_t&) const =0;
    virtual int getColumnAsUInt(const char* cname, uint32_t&) const =0;
    virtual int getColumnAsLong(const char* cname, int64_t&) const =0;
    virtual int getColumnAsULong(const char* cname, uint64_t&) const =0;
    virtual int getColumnAsFloat(const char* cname, float&) const =0;
    virtual int getColumnAsDouble(const char* cname, double&) const =0;
    virtual int getColumnAsString(const char* cname, std::string&) const =0;

    virtual int getColumnAsByte(uint32_t cnum, char& val) const =0;
    virtual int getColumnAsUByte(uint32_t cnum, unsigned char& val) const =0;
    virtual int getColumnAsShort(uint32_t cnum, int16_t& val) const =0;
    virtual int getColumnAsUShort(uint32_t cnum, uint16_t& val) const =0;
    virtual int getColumnAsInt(uint32_t cnum, int32_t& val) const =0;
    virtual int getColumnAsUInt(uint32_t cnum, uint32_t& val) const =0;
    virtual int getColumnAsLong(uint32_t cnum, int64_t& val) const =0;
    virtual int getColumnAsULong(uint32_t cnum, uint64_t& val) const =0;
    virtual int getColumnAsFloat(uint32_t cnum, float& val) const =0;
    virtual int getColumnAsDouble(uint32_t cnum, double& val) const =0;
    virtual int getColumnAsString(uint32_t cnum, std::string& val) const =0;

protected:
    cursor() {};
    cursor(const cursor&); // not implemented
    cursor& operator=(const cursor&) ; // not implemented
}; // ibis::table::cursor

inline void ibis::table::row::clear() {
    bytesnames.clear();
    bytesvalues.clear();
    ubytesnames.clear();
    ubytesvalues.clear();
    shortsnames.clear();
    shortsvalues.clear();
    ushortsnames.clear();
    ushortsvalues.clear();
    intsnames.clear();
    intsvalues.clear();
    uintsnames.clear();
    uintsvalues.clear();
    longsnames.clear();
    longsvalues.clear();
    ulongsnames.clear();
    ulongsvalues.clear();
    floatsnames.clear();
    floatsvalues.clear();
    doublesnames.clear();
    doublesvalues.clear();
    catsnames.clear();
    catsvalues.clear();
    textsnames.clear();
    textsvalues.clear();
    blobsnames.clear();
    blobsvalues.clear();
} // ibis::table::row::clear

inline void ibis::table::row::clearValues() {
    bytesvalues.clear();
    ubytesvalues.clear();
    shortsvalues.clear();
    ushortsvalues.clear();
    intsvalues.clear();
    uintsvalues.clear();
    longsvalues.clear();
    ulongsvalues.clear();
    floatsvalues.clear();
    doublesvalues.clear();
    catsvalues.clear();
    textsvalues.clear();
    blobsvalues.clear();
} // ibis::table::row::clearValues
#endif // IBIS_TABLE_H