query.h

Go to the documentation of this file.

// File: $Id$
// Author: John Wu <John.Wu at ACM.org>
//         Lawrence Berkeley National Laboratory
// Copyright 2000-2012 the Regents of the University of California
#ifndef IBIS_QUERY_H
#define IBIS_QUERY_H



#include "part.h"       // ibis::part
#include "whereClause.h"        // ibis::whereClause
#include "selectClause.h"       // ibis::selectClause

class FASTBIT_CXX_DLLSPEC ibis::query {
public:
    enum QUERY_STATE {
        UNINITIALIZED,  //< The query object is currently empty.
        SET_COMPONENTS, //< The query object has a select clause.
        SET_RIDS,       //< The query object contains a list of RIDs.
        SET_PREDICATE,  //< The query object has a where clause.
        SPECIFIED,      //< SET_COMPONENTS & (SET_RIDS | SET_PREDICATE).
        QUICK_ESTIMATE, //< A upper and a lower bound are computed.
        FULL_EVALUATE,  //< The exact hits are computed.
        BUNDLES_TRUNCATED,      //< Only top-K results are stored.
        HITS_TRUNCATED  //< The hit vector has been updated to match bundles.
    };

    virtual ~query();
    query(const char* dir, const ibis::partList& tl);
    query(const char* uid=0, const part* et=0, const char* pref=0);

    const char* id() const {return myID;};      
    const char* dir() const {return myDir;}     
    const char* userName() const {return user;} 

    time_t timestamp() const {return dstime;}
    const part* partition() const {return mypart;}
    const selectClause& components() const {return comps;};

    int setRIDs(const RIDSet& set);
    int setWhereClause(const char *str);
    int setWhereClause(const std::vector<const char*>& names,
                       const std::vector<double>& lbounds,
                       const std::vector<double>& rbounds);
    int setWhereClause(const ibis::qExpr* qexp);
    virtual int setSelectClause(const char *str);
    int setPartition(const ibis::part* tbl);
    int setTable(const ibis::part* tbl) {return setPartition(tbl);}
    virtual const char* getWhereClause() const {return conds.getString();}
    virtual const char* getSelectClause() const {return *comps;}

    void expandQuery();
    void contractQuery();
    std::string removeComplexConditions();

    const RIDSet* getUserRIDs() const {return rids_in;}

    // Functions to perform estimation.

    int estimate();
    long getMinNumHits() const;
    long getMaxNumHits() const;

    // Functions related to full evaluation.

    int evaluate(const bool evalSelect=false);
    const ibis::bitvector* getHitVector() const {return hits;}
    long getNumHits() const;
    long getHitRows(std::vector<uint32_t> &rids) const;
    long countHits() const;

    int  orderby(const char *names) const;
    long limit(const char *names, uint32_t keep,
               bool updateHits = true);

    array_t<signed char>*   getQualifiedBytes(const char* column_name);
    array_t<unsigned char>* getQualifiedUBytes(const char* column_name);
    array_t<int16_t>* getQualifiedShorts(const char* column_name);
    array_t<uint16_t>* getQualifiedUShorts(const char* column_name);
    array_t<int32_t>* getQualifiedInts(const char* column_name);
    array_t<uint32_t>* getQualifiedUInts(const char* column_name);
    array_t<int64_t>* getQualifiedLongs(const char* column_name);
    array_t<uint64_t>* getQualifiedULongs(const char* column_name);
    array_t<float>* getQualifiedFloats(const char* column_name);
    array_t<double>* getQualifiedDoubles(const char* column_name);
    std::vector<std::string>* getQualifiedStrings(const char* column_name);
    RIDSet* getRIDs() const;
    RIDSet* getRIDs(const ibis::bitvector& mask) const;
    const RIDSet* getRIDsInBundle(const uint32_t bid) const;

    void printSelected(std::ostream& out) const;
    void printSelectedWithRID(std::ostream& out) const;

    long sequentialScan(ibis::bitvector& bv) const;

    long getExpandedHits(ibis::bitvector&) const;

    // used by ibis::bundle
    RIDSet* readRIDs() const;
    void writeRIDs(const RIDSet* rids) const;

    void logMessage(const char* event, const char* fmt, ...) const;

    // Functions for cleaning up, retrieving query states
    // and error messages.

    void clear();
    QUERY_STATE getState() const;
    const char* getLastError() const {return lastError;}
    void clearErrorMessage() const {*lastError=0;}

    static bool isValidToken(const char* tok);
    // *** the value 16 is hard coded in functions newToken and ***
    // *** isValidToken ***
    static unsigned tokenLength() {return 16;}

    static void removeQueryRecords()
    {ibis::gParameters().add("query.purgeTempFiles", "true");}
    static void keepQueryRecords()
    {ibis::gParameters().add("query.purgeTempFiles", "false");}

    class result; // Forward declaration only
    class weight;
    class readLock;
    class writeLock;
    friend class readLock;
    friend class writeLock;

protected:
    char* user;         
    whereClause conds;  
    selectClause comps; 
    QUERY_STATE state;  
    ibis::bitvector* hits;
    ibis::bitvector* sup;
    mutable ibis::part::readLock* dslock;       
    mutable char lastError[MAX_LINE+PATH_MAX];  

    void logError(const char* event, const char* fmt, ...) const;
    void logWarning(const char* event, const char* fmt, ...) const;

    void reorderExpr(); // reorder query expression

    bool hasBundles() const;
    int  computeHits();   // generate the hit vector for range queries
    void getBounds();     // get the upper and lower bounds for range queries
    // use index only to come up with a upper bound and a lower bound
    void doEstimate(const qExpr* term, ibis::bitvector& low,
                    ibis::bitvector& high) const;
    int doScan(const qExpr* term, const ibis::bitvector& mask,
               ibis::bitvector& hits) const;
    int doScan(const qExpr* term, ibis::bitvector& hits) const;
    int doEvaluate(const qExpr* term, ibis::bitvector& hits) const;
    int doEvaluate(const qExpr* term, const ibis::bitvector& mask,
                   ibis::bitvector& hits) const;

    int64_t processJoin();

    virtual void writeQuery();
    void readQuery(const ibis::partList& tl);
    void removeFiles();

    void readHits();
    void writeHits() const;
    void printRIDs(const RIDSet& ridset) const;
    uint32_t countPages(unsigned wordsize) const;

    int doExpand(ibis::qExpr* exp0) const;
    int doContract(ibis::qExpr* exp0) const;

    // A group of functions to count the number of pairs
    // satisfying the join conditions.
    int64_t sortJoin(const std::vector<const ibis::deprecatedJoin*>& terms,
                     const ibis::bitvector& mask) const;
    int64_t sortJoin(const ibis::deprecatedJoin& cmp,
                     const ibis::bitvector& mask) const;
    int64_t sortEquiJoin(const ibis::deprecatedJoin& cmp,
                         const ibis::bitvector& mask) const;
    int64_t sortRangeJoin(const ibis::deprecatedJoin& cmp,
                          const ibis::bitvector& mask) const;
    int64_t sortEquiJoin(const ibis::deprecatedJoin& cmp,
                         const ibis::bitvector& mask,
                         const char* pairfile) const;
    int64_t sortRangeJoin(const ibis::deprecatedJoin& cmp,
                          const ibis::bitvector& mask,
                          const char* pairfile) const;
    void orderPairs(const char* pairfile) const;
    int64_t mergePairs(const char* pairfile) const;

    template <typename T1, typename T2>
    int64_t countEqualPairs(const array_t<T1>& val1,
                            const array_t<T2>& val2) const;
    template <typename T1, typename T2>
    int64_t countDeltaPairs(const array_t<T1>& val1,
                            const array_t<T2>& val2, const T1& delta) const;
    template <typename T1, typename T2>
    int64_t recordEqualPairs(const array_t<T1>& val1,
                             const array_t<T2>& val2,
                             const array_t<uint32_t>& ind1,
                             const array_t<uint32_t>& ind2,
                             const char* pairfile) const;
    template <typename T1, typename T2>
    int64_t recordDeltaPairs(const array_t<T1>& val1,
                             const array_t<T2>& val2,
                             const array_t<uint32_t>& ind1,
                             const array_t<uint32_t>& ind2,
                             const T1& delta, const char* pairfile) const;

    // functions for access control
    void gainReadAccess(const char* mesg) const {
        if (ibis::gVerbose > 10)
            logMessage("gainReadAccess", "acquiring a read lock for %s",
                       mesg);
        if (0 != pthread_rwlock_rdlock(&lock))
            logMessage("gainReadAccess",
                       "unable to gain read access to rwlock for %s", mesg);
    }
    void gainWriteAccess(const char* mesg) const {
        if (ibis::gVerbose > 10)
            logMessage("gainWriteAccess", "acquiring a write lock for %s",
                       mesg);
        if (0 != pthread_rwlock_wrlock(&lock))
            logMessage("gainWriteAccess",
                       "unable to gain write access to rwlock for %s", mesg);
    }
    void releaseAccess(const char* mesg) const {
        if (ibis::gVerbose > 10)
            logMessage("releaseAccess", "releasing rwlock for %s", mesg);
        if (0 != pthread_rwlock_unlock(&lock))
            logMessage("releaseAccess", "unable to unlock the rwlock for %s",
                       mesg);
    }

private:
    char* myID;         // The unique ID of this query object
    char* myDir;        // Name of the directory containing the query record
    RIDSet* rids_in;    // Rid list specified in an RID query
    const part* mypart; // Data partition used to process the query
    time_t dstime;              // When query evaluation started
    mutable pthread_rwlock_t lock; // Rwlock for access control

    // private functions
    static char* newToken(const char*); 

    void setMyDir(const char *pref);

    query(const query&);
    query& operator=(const query&);
}; // class ibis::query

namespace ibis {
    template <>
    int64_t query::countEqualPairs(const array_t<int32_t>& val1,
                                   const array_t<uint32_t>& val2) const;
    template <>
    int64_t query::countEqualPairs(const array_t<uint32_t>& val1,
                                   const array_t<int32_t>& val2) const;
    template <>
    int64_t query::countDeltaPairs(const array_t<int32_t>& val1,
                                   const array_t<uint32_t>& val2,
                                   const int32_t& delta) const;
    template <>
    int64_t query::countDeltaPairs(const array_t<uint32_t>& val1,
                                   const array_t<int32_t>& val2,
                                   const uint32_t& delta) const;
    template <>
    int64_t query::recordEqualPairs(const array_t<int32_t>& val1,
                                    const array_t<uint32_t>& val2,
                                    const array_t<uint32_t>& ind1,
                                    const array_t<uint32_t>& ind2,
                                    const char *pairfile) const;
    template <>
    int64_t query::recordEqualPairs(const array_t<uint32_t>& val1,
                                    const array_t<int32_t>& val2,
                                    const array_t<uint32_t>& ind1,
                                    const array_t<uint32_t>& ind2,
                                    const char *pairfile) const;
    template <>
    int64_t query::recordDeltaPairs(const array_t<int32_t>& val1,
                                    const array_t<uint32_t>& val2,
                                    const array_t<uint32_t>& ind1,
                                    const array_t<uint32_t>& ind2,
                                    const int32_t& delta,
                                    const char *pairfile) const;
    template <>
    int64_t query::recordDeltaPairs(const array_t<uint32_t>& val1,
                                    const array_t<int32_t>& val2,
                                    const array_t<uint32_t>& ind1,
                                    const array_t<uint32_t>& ind2,
                                    const uint32_t& delta,
                                    const char *pairfile) const;
}

class ibis::query::weight : public ibis::qExpr::weight {
public:
    virtual double operator()(const ibis::qExpr* ex) const;
    weight(const ibis::part* ds) : dataset(ds) {}

private:
    const ibis::part* dataset;
};

class ibis::query::readLock {
public:
    readLock(const query* q, const char* m) : theQuery(q), mesg(m) {
        theQuery->gainReadAccess(m);
    };
    ~readLock() {theQuery->releaseAccess(mesg);}
private:
    const query* theQuery;
    const char* mesg;

    readLock() {}; // no default constructor
    readLock(const readLock&) {}; // can not copy
}; // class ibis::query::readLock

class ibis::query::writeLock {
public:
    writeLock(const query* q, const char* m) : theQuery(q), mesg(m) {
        theQuery->gainWriteAccess(m);
    };
    ~writeLock() {theQuery->releaseAccess(mesg);}
private:
    const query* theQuery;
    const char* mesg;

    writeLock() {}; // no default constructor
    writeLock(const writeLock&) {}; // can not copy
}; // ibis::query::writeLock
#endif // IBIS_QUERY_H